From 622feeb68517de33719a40ed7be0848aa3eccb48 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 10 Jul 2026 12:34:58 +0800 Subject: [PATCH 01/23] Strengthen refactor regression coverage Replace brittle source-scanning assertions with behavioral unit coverage, add invalid persisted-cron integration coverage, and document intentional error and lock boundaries. Signed-off-by: Lu Zhang --- auth/lib.js | 6 +- control/handlers/deploy.js | 3 + control/routing.js | 2 + control/shared.js | 2 + rust/redis-proxy/src/kv.rs | 463 ++++++++++++++---------- rust/redis-proxy/src/kv/cursor.rs | 43 ++- rust/redis-proxy/src/secrets.rs | 19 +- rust/scheduler/src/cron/dispatch.rs | 105 ++++-- rust/scheduler/src/queue/registry.rs | 3 + tests/integration/cron-triggers.test.js | 34 ++ 10 files changed, 424 insertions(+), 256 deletions(-) diff --git a/auth/lib.js b/auth/lib.js index c00d31e..5e48a7e 100644 --- a/auth/lib.js +++ b/auth/lib.js @@ -53,9 +53,9 @@ export { randomHex }; * }} TokenRecord */ -// Mirrors control/routing.js#RoutingError shape — index.js maps thrown -// instances to HTTP code so policy rejections stay distinct from Redis / -// code exceptions (which 503 fail-closed). +// Mirrors the control/routing status + machine-code shape on purpose, but stays +// auth-local: enhanced_error_serialization preserves {status, reason}, and +// control maps that distinct policy contract without importing auth internals. export class AuthPolicyError extends Error { /** * @param {number} status diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index b8bee81..03640a6 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -58,6 +58,9 @@ const MAX_COMMIT_ATTEMPTS = 5; const DEPLOY_JSON_BODY_MAX_BYTES = 32 * 1024 * 1024; const DEPLOY_ASSET_UPLOAD_CONCURRENCY = 8; +// Deploy keeps thin local wrappers even though they mirror ControlAbort-like +// fields: commit aborts need cleanup/log handling, while request errors are +// pre-commit shape rejections. class DeployAbort extends ControlAbort {} /** diff --git a/control/routing.js b/control/routing.js index 89b144c..f45bb35 100644 --- a/control/routing.js +++ b/control/routing.js @@ -72,6 +72,8 @@ function hostDeclarationsKey(host) { * @typedef {{ hGet: (key: string, field: string) => Promise, incr: (key: string) => Promise, session: (fn: (iso: RedisIso) => Promise) => Promise }} RedisClient */ +// Routing owns promotion/route-specific machine codes. Keep this separate from +// ControlAbort so pure routing helpers do not inherit handler abort semantics. export class RoutingError extends Error { /** @param {number} status @param {string} code @param {string} message @param {Record} [details] */ constructor(status, code, message, details = {}) { diff --git a/control/shared.js b/control/shared.js index 4e5e247..3181bdb 100644 --- a/control/shared.js +++ b/control/shared.js @@ -216,6 +216,8 @@ function onRedisCommand(event) { recordRedisCommand({ metrics: null, log: state.log, service: state.service, event }); } +// Intentionally local, despite looking like routing/deploy/auth errors. Each +// layer owns a slightly different machine-readable wire shape and catch boundary. export class ControlAbort extends Error { /** * @param {number} status diff --git a/rust/redis-proxy/src/kv.rs b/rust/redis-proxy/src/kv.rs index 69b5d70..072d866 100644 --- a/rust/redis-proxy/src/kv.rs +++ b/rust/redis-proxy/src/kv.rs @@ -166,12 +166,14 @@ fn meta_field(key: &str) -> String { format!("{META_FIELD_PREFIX}{key}") } +type GroupedHmgetCommand = (String, Vec, Vec); + fn grouped_hmget_commands( ns: &str, id: &str, keys: &[String], field_prefix: &str, -) -> Vec<(String, Vec, Vec)> { +) -> Vec { grouped_hmget_commands_for_indices(ns, id, keys, 0..keys.len(), field_prefix) } @@ -181,7 +183,7 @@ fn grouped_hmget_commands_for_indices( keys: &[String], indices: impl IntoIterator, field_prefix: &str, -) -> Vec<(String, Vec, Vec)> { +) -> Vec { let mut by_hash: HashMap, Vec)> = HashMap::new(); for index in indices { let key = &keys[index]; @@ -255,22 +257,11 @@ fn parse_hmget_budget_reply( Ok((bytes, out)) } -async fn hmget_with_raw_byte_budget( - conn: &mut ConnectionManager, - total: &mut usize, +fn hmget_with_raw_byte_budget_command( redis_key: &str, + remaining: usize, fields: &[String], -) -> AppResult>>> { - if fields.is_empty() { - return Ok(Vec::new()); - } - // The Lua preflight is the Redis-side atomic budget gate: HSTRLEN and HMGET - // run in one script, so over-budget batches fail before payload bytes leave - // Redis. Callers keep a separate actual-byte counter over returned values as - // defense-in-depth without double-counting stable data against the budget. - let remaining = KV_BATCH_RAW_BYTES_MAX - .checked_sub(*total) - .ok_or_else(raw_bytes_too_large)?; +) -> redis::Cmd { let mut cmd = redis::cmd("EVAL"); cmd.arg(HMGET_WITH_RAW_BYTE_BUDGET_SCRIPT) .arg(1) @@ -279,9 +270,128 @@ async fn hmget_with_raw_byte_budget( for field in fields { cmd.arg(field); } - let (bytes, values) = parse_hmget_budget_reply(cmd.query_async(conn).await?, fields.len())?; - add_batch_raw_bytes(total, bytes)?; - Ok(values) + cmd +} + +async fn query_hmget_with_raw_byte_budget( + conn: &mut ConnectionManager, + redis_key: &str, + remaining: usize, + fields: &[String], +) -> AppResult<(usize, Vec>>)> { + if fields.is_empty() { + return Ok((0, Vec::new())); + } + // HSTRLEN and HMGET run in one Redis-side script, so an over-budget group + // is rejected before its payload bytes leave Redis. + let cmd = hmget_with_raw_byte_budget_command(redis_key, remaining, fields); + parse_hmget_budget_reply(cmd.query_async(conn).await?, fields.len()) +} + +#[derive(Default)] +struct RawByteBudget { + preflight_bytes: usize, + actual_bytes: usize, +} + +impl RawByteBudget { + fn remaining(&self) -> AppResult { + KV_BATCH_RAW_BYTES_MAX + .checked_sub(self.preflight_bytes) + .ok_or_else(raw_bytes_too_large) + } + + fn record_preflight(&mut self, bytes: usize) -> AppResult<()> { + add_batch_raw_bytes(&mut self.preflight_bytes, bytes) + } + + fn record_actual(&mut self, value: Option<&[u8]>) -> AppResult<()> { + if let Some(bytes) = value { + add_batch_raw_bytes(&mut self.actual_bytes, bytes.len())?; + } + Ok(()) + } +} + +struct GroupedHmgetRequest { + redis_key: String, + indices: Vec, + fields: Vec, + remaining: usize, +} + +struct GroupedFieldRead { + pending: std::vec::IntoIter, + output: Vec>>, +} + +impl GroupedFieldRead { + fn new(commands: Vec, output_len: usize) -> Self { + Self { + pending: commands.into_iter(), + output: vec![None; output_len], + } + } + + fn next_request(&mut self, budget: &RawByteBudget) -> AppResult> { + let Some((redis_key, indices, fields)) = self.pending.next() else { + return Ok(None); + }; + if indices.len() != fields.len() || indices.iter().any(|index| *index >= self.output.len()) + { + return Err(AppError::internal_error("invalid grouped KV HMGET plan")); + } + Ok(Some(GroupedHmgetRequest { + redis_key, + indices, + fields, + remaining: budget.remaining()?, + })) + } + + fn apply_response( + &mut self, + budget: &mut RawByteBudget, + request: GroupedHmgetRequest, + preflight_bytes: usize, + values: Vec>>, + ) -> AppResult<()> { + if values.len() != request.indices.len() { + return Err(AppError::internal_error("invalid grouped KV HMGET reply")); + } + budget.record_preflight(preflight_bytes)?; + for (index, value) in request.indices.into_iter().zip(values) { + budget.record_actual(value.as_deref())?; + self.output[index] = value; + } + Ok(()) + } + + fn finish(self) -> Vec>> { + self.output + } +} + +async fn load_grouped_fields_with_raw_byte_budget( + conn: &mut ConnectionManager, + commands: Vec, + output_len: usize, + budget: &mut RawByteBudget, +) -> AppResult>>> { + // The preflight total limits transfer between groups; the separate actual + // total rechecks returned payloads before callers encode a response. + let mut read = GroupedFieldRead::new(commands, output_len); + while let Some(request) = read.next_request(budget)? { + let (preflight_bytes, values) = query_hmget_with_raw_byte_budget( + conn, + &request.redis_key, + request.remaining, + &request.fields, + ) + .await?; + read.apply_response(budget, request, preflight_bytes, values)?; + } + Ok(read.finish()) } pub(crate) fn kv_put_pipeline( @@ -360,6 +470,12 @@ fn decode_metadata(bytes: Option>) -> AppResult> { .transpose() } +fn normalize_list_prefix(prefix: Option) -> AppResult { + let prefix = prefix.unwrap_or_default(); + validate_kv_key(&prefix)?; + Ok(prefix) +} + pub(crate) fn escape_glob_literal(input: &str) -> String { let mut out = String::with_capacity(input.len()); for ch in input.chars() { @@ -396,6 +512,7 @@ pub(crate) async fn kv_get( ); response.headers_mut().insert( CONTENT_LENGTH, + // `usize::to_string()` is decimal ASCII, which is always a valid header value. HeaderValue::from_str(&len.to_string()).unwrap(), ); Ok(response) @@ -452,55 +569,48 @@ pub(crate) async fn kv_get_batch( let include_metadata = body.metadata.unwrap_or(false); let keys = body.keys; let mut conn = state.redis(); - let mut preflight_raw_bytes = 0_usize; - let mut actual_raw_bytes = 0_usize; - let mut values: Vec>> = vec![None; keys.len()]; - for (redis_key, indices, fields) in - grouped_hmget_commands(&q.ns, &q.id, &keys, VALUE_FIELD_PREFIX) - { - let batch = - hmget_with_raw_byte_budget(&mut conn, &mut preflight_raw_bytes, &redis_key, &fields) - .await?; - for (offset, value) in batch.into_iter().enumerate() { - if let Some(bytes) = value.as_ref() { - add_batch_raw_bytes(&mut actual_raw_bytes, bytes.len())?; - } - values[indices[offset]] = value; - } - } + let mut budget = RawByteBudget::default(); + let value_commands = grouped_hmget_commands(&q.ns, &q.id, &keys, VALUE_FIELD_PREFIX); + let mut values = load_grouped_fields_with_raw_byte_budget( + &mut conn, + value_commands, + keys.len(), + &mut budget, + ) + .await?; let mut metadata: Vec> = vec![None; keys.len()]; if include_metadata { let present_value_indices = values .iter() .enumerate() - .filter_map(|(index, value)| value.as_ref().map(|_| index)); - for (redis_key, indices, fields) in grouped_hmget_commands_for_indices( + .filter_map(|(index, value)| value.as_ref().map(|_| index)) + .collect::>(); + let metadata_commands = grouped_hmget_commands_for_indices( &q.ns, &q.id, &keys, present_value_indices, META_FIELD_PREFIX, - ) { - let batch = hmget_with_raw_byte_budget( - &mut conn, - &mut preflight_raw_bytes, - &redis_key, - &fields, - ) - .await?; - for (offset, raw) in batch.into_iter().enumerate() { - let index = indices[offset]; - if values[index].is_none() { - continue; - } - if let Some(bytes) = raw.as_ref() { - add_batch_raw_bytes(&mut actual_raw_bytes, bytes.len())?; - } + ); + let raw_metadata = load_grouped_fields_with_raw_byte_budget( + &mut conn, + metadata_commands, + keys.len(), + &mut budget, + ) + .await?; + for (index, raw) in raw_metadata.into_iter().enumerate() { + if values[index].is_some() { metadata[index] = decode_metadata(raw)?; } } } - record_kv_value_bytes(state.metrics(), "get_batch", "raw_batch", actual_raw_bytes); + record_kv_value_bytes( + state.metrics(), + "get_batch", + "raw_batch", + budget.actual_bytes, + ); let mut entries = Vec::with_capacity(keys.len()); for (index, key) in keys.into_iter().enumerate() { let value = values[index].take(); @@ -586,8 +696,7 @@ pub(crate) async fn kv_list( q.validate_scope()?; let limit = normalize_list_limit(q.limit) as usize; let (mut bucket, mut scan_cursor, mut overflow) = decode_list_cursor(q.cursor)?; - let prefix = q.prefix.unwrap_or_default(); - validate_kv_key(&prefix)?; + let prefix = normalize_list_prefix(q.prefix)?; let include_metadata = q.metadata.unwrap_or(false); let pattern = format!("{VALUE_FIELD_PREFIX}{}*", escape_glob_literal(&prefix)); let mut raw_fields = Vec::new(); @@ -648,27 +757,25 @@ pub(crate) async fn kv_list( .collect::>>()?; let mut metadata = if include_metadata { let mut conn = state.redis(); - let mut preflight_raw_bytes = 0_usize; - let mut actual_raw_bytes = 0_usize; - let mut values: Vec> = vec![None; user_keys.len()]; - for (redis_key, indices, fields) in - grouped_hmget_commands(&q.ns, &q.id, &user_keys, META_FIELD_PREFIX) - { - let batch = hmget_with_raw_byte_budget( - &mut conn, - &mut preflight_raw_bytes, - &redis_key, - &fields, - ) - .await?; - for (offset, raw) in batch.into_iter().enumerate() { - if let Some(bytes) = raw.as_ref() { - add_batch_raw_bytes(&mut actual_raw_bytes, bytes.len())?; - } - values[indices[offset]] = decode_metadata(raw)?; - } - } - record_kv_value_bytes(state.metrics(), "list", "metadata_batch", actual_raw_bytes); + let mut budget = RawByteBudget::default(); + let metadata_commands = grouped_hmget_commands(&q.ns, &q.id, &user_keys, META_FIELD_PREFIX); + let raw_metadata = load_grouped_fields_with_raw_byte_budget( + &mut conn, + metadata_commands, + user_keys.len(), + &mut budget, + ) + .await?; + let values = raw_metadata + .into_iter() + .map(decode_metadata) + .collect::>>()?; + record_kv_value_bytes( + state.metrics(), + "list", + "metadata_batch", + budget.actual_bytes, + ); values } else { Vec::new() @@ -778,57 +885,96 @@ mod tests { } #[test] - fn kv_batch_get_checks_budget_between_value_reads() { - let source = include_str!("kv.rs"); - let start = source - .find("pub(crate) async fn kv_get_batch") - .expect("kv_get_batch route must exist"); - let end = source[start..] - .find("pub(crate) async fn kv_put") - .expect("kv_put should follow kv_get_batch") - + start; - let helper = &source[start..end]; - assert!( - helper.matches("hmget_with_raw_byte_budget").count() >= 2, - "kv_get_batch must atomically budget-check value and metadata HMGET payloads" - ); - assert!( - helper.matches("add_batch_raw_bytes").count() >= 2, - "kv_get_batch must also re-check actual HMGET payload bytes after reading" - ); - assert!( - !helper.contains("redis::pipe"), - "kv_get_batch must not pipeline every bucket before checking the aggregate byte budget" + fn hmget_budget_command_packs_script_key_budget_and_fields() { + let fields = vec!["v:first".to_string(), "v:second".to_string()]; + let commands = parse_packed_commands( + &hmget_with_raw_byte_budget_command("kvh:tenant:store:b:1", 1024, &fields) + .get_packed_command(), ); - assert!( - source.contains("redis.call(\"HMGET\""), - "the atomic KV budget helper should batch reads with Redis HMGET" + + assert_eq!(commands.len(), 1); + let command = &commands[0]; + assert_eq!(command[0], "EVAL"); + assert_eq!(command[1], HMGET_WITH_RAW_BYTE_BUDGET_SCRIPT); + assert_eq!( + &command[2..], + ["1", "kvh:tenant:store:b:1", "1024", "v:first", "v:second"] ); } #[test] - fn kv_list_metadata_batches_metadata_reads() { - let source = include_str!("kv.rs"); - let start = source - .find("pub(crate) async fn kv_list") - .expect("kv_list route must exist"); - let end = source[start..] - .find("#[cfg(test)]") - .expect("tests should follow kv_list") - + start; - let helper = &source[start..end]; - assert!( - helper.contains("grouped_hmget_commands"), - "kv_list metadata reads should group metadata fields by hash bucket" - ); - assert!( - helper.contains("hmget_with_raw_byte_budget"), - "kv_list metadata reads must atomically budget-check raw metadata bytes before returning payloads" + fn grouped_hmget_commands_batches_metadata_fields_by_hash_bucket() { + let keys = vec![ + "alpha".to_string(), + "bravo".to_string(), + "charlie".to_string(), + ]; + let commands = grouped_hmget_commands("tenant", "store", &keys, META_FIELD_PREFIX); + assert_eq!( + commands + .iter() + .map(|(_, indices, fields)| { + assert_eq!(indices.len(), fields.len()); + fields.len() + }) + .sum::(), + keys.len() ); - assert!( - helper.contains("add_batch_raw_bytes"), - "kv_list metadata reads must also re-check actual HMGET payload bytes" + + for (redis_key, indices, fields) in commands { + for (offset, index) in indices.iter().enumerate() { + assert_eq!( + redis_key, + hash_key_for_user_key("tenant", "store", &keys[*index]) + ); + assert_eq!(fields[offset], meta_field(&keys[*index])); + } + } + } + + #[test] + fn grouped_budgeted_reads_decrement_budget_and_restore_input_order() { + let commands = vec![ + ( + "hash-a".to_string(), + vec![2, 0], + vec!["v:charlie".to_string(), "v:alpha".to_string()], + ), + ("hash-b".to_string(), vec![1], vec!["v:bravo".to_string()]), + ]; + let mut read = GroupedFieldRead::new(commands, 3); + let mut budget = RawByteBudget::default(); + + let first = read.next_request(&budget).unwrap().unwrap(); + assert_eq!(first.redis_key, "hash-a"); + assert_eq!(first.remaining, KV_BATCH_RAW_BYTES_MAX); + assert_eq!(first.fields, ["v:charlie", "v:alpha"]); + read.apply_response( + &mut budget, + first, + 1, + vec![Some(b"ccc".to_vec()), Some(b"aa".to_vec())], + ) + .unwrap(); + + let second = read.next_request(&budget).unwrap().unwrap(); + assert_eq!(second.redis_key, "hash-b"); + assert_eq!(second.remaining, KV_BATCH_RAW_BYTES_MAX - 1); + assert_eq!(second.fields, ["v:bravo"]); + read.apply_response(&mut budget, second, 2, vec![Some(b"bbbb".to_vec())]) + .unwrap(); + + assert!(read.next_request(&budget).unwrap().is_none()); + assert_eq!( + read.finish(), + vec![ + Some(b"aa".to_vec()), + Some(b"bbbb".to_vec()), + Some(b"ccc".to_vec()), + ] ); + assert_eq!(budget.preflight_bytes, 3); + assert_eq!(budget.actual_bytes, 9); } #[test] @@ -889,33 +1035,14 @@ mod tests { #[test] fn kv_list_prefix_uses_kv_key_byte_limit() { - let source = include_str!("kv.rs"); - let start = source - .find("pub(crate) async fn kv_list") - .expect("kv_list route must exist"); - let end = source[start..] - .find("let include_metadata") - .expect("kv_list prefix validation should happen before HSCAN setup") - + start; - assert!( - source[start..end].contains("validate_kv_key(&prefix)?"), - "kv_list must reject oversized prefixes before composing the HSCAN pattern" + assert_eq!(normalize_list_prefix(None).unwrap(), ""); + assert_eq!( + normalize_list_prefix(Some("photos/".to_string())).unwrap(), + "photos/" ); let oversized = "x".repeat(KV_KEY_MAX_BYTES + 1); - let q = KvParams { - ns: "tenant".to_string(), - id: "store".to_string(), - key: None, - ttl: None, - exat: None, - prefix: Some(oversized), - limit: None, - metadata: None, - cursor: None, - }; - let prefix = q.prefix.unwrap_or_default(); - let err = validate_kv_key(&prefix).unwrap_err(); + let err = normalize_list_prefix(Some(oversized)).unwrap_err(); assert_eq!(err.status, StatusCode::BAD_REQUEST); assert!(err.message.contains("KV key exceeds 512 byte limit")); } @@ -972,52 +1099,6 @@ mod tests { assert_eq!(err.code, "invalid_request"); } - #[test] - fn kv_list_cursor_overflow_existence_probe_does_not_fetch_values() { - let source = include_str!("kv/cursor.rs"); - let start = source - .find("async fn existing_cursor_overflow_fields") - .expect("existing_cursor_overflow_fields helper must exist"); - let helper = &source[start..]; - assert!( - !helper.contains("candidates.clone()"), - "cursor overflow validation should not clone the overflow candidate vector" - ); - assert!( - !helper.contains("\"MGET\""), - "cursor overflow validation must not fetch KV values" - ); - assert!( - helper.contains("\"HEXISTS\""), - "cursor overflow validation should only probe field existence" - ); - } - - #[test] - fn kv_list_metadata_checks_budget_between_metadata_reads() { - let source = include_str!("kv.rs"); - let start = source - .find("pub(crate) async fn kv_list") - .expect("kv_list route must exist"); - let end = source[start..] - .find("#[cfg(test)]") - .expect("test module should follow kv_list") - + start; - let helper = &source[start..end]; - assert!( - helper.contains("hmget_with_raw_byte_budget"), - "kv_list metadata must enforce the aggregate raw metadata byte budget inside the HMGET helper" - ); - assert!( - helper.contains("add_batch_raw_bytes"), - "kv_list metadata must re-check actual HMGET payload bytes after reading" - ); - assert!( - !helper.contains("redis::pipe"), - "kv_list metadata must not pipeline all metadata before checking the aggregate byte budget" - ); - } - #[test] fn kv_put_pipeline_preserves_value_and_metadata_ttl_contract() { let commands = parse_packed_commands( diff --git a/rust/redis-proxy/src/kv/cursor.rs b/rust/redis-proxy/src/kv/cursor.rs index df19e95..a89c471 100644 --- a/rust/redis-proxy/src/kv/cursor.rs +++ b/rust/redis-proxy/src/kv/cursor.rs @@ -75,14 +75,9 @@ pub(crate) async fn existing_cursor_overflow_fields( if candidates.is_empty() { return Ok(Vec::new()); } + let pipe = existing_cursor_overflow_fields_pipeline(&hash_key, &candidates); let exists: Vec = state - .with_redis(async |mut conn| { - let mut pipe = redis::pipe(); - for field in &candidates { - pipe.cmd("HEXISTS").arg(&hash_key).arg(field); - } - pipe.query_async(&mut conn).await - }) + .with_redis(async |mut conn| pipe.query_async(&mut conn).await) .await?; Ok(candidates .into_iter() @@ -90,3 +85,37 @@ pub(crate) async fn existing_cursor_overflow_fields( .filter_map(|(key, exists)| (exists > 0).then_some(key)) .collect()) } + +fn existing_cursor_overflow_fields_pipeline( + hash_key: &str, + candidates: &[String], +) -> redis::Pipeline { + let mut pipe = redis::pipe(); + for field in candidates { + pipe.cmd("HEXISTS").arg(hash_key).arg(field); + } + pipe +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::parse_packed_commands; + + #[test] + fn existing_cursor_overflow_fields_pipeline_only_probes_existence() { + let candidates = vec!["v:first".to_string(), "v:second".to_string()]; + let commands = parse_packed_commands( + &existing_cursor_overflow_fields_pipeline("kvh:tenant:store:b:1", &candidates) + .get_packed_pipeline(), + ); + + assert_eq!( + commands, + [ + ["HEXISTS", "kvh:tenant:store:b:1", "v:first"], + ["HEXISTS", "kvh:tenant:store:b:1", "v:second"], + ] + ); + } +} diff --git a/rust/redis-proxy/src/secrets.rs b/rust/redis-proxy/src/secrets.rs index 056e344..3e1ce9e 100644 --- a/rust/redis-proxy/src/secrets.rs +++ b/rust/redis-proxy/src/secrets.rs @@ -321,6 +321,8 @@ fn aes_gcm_decrypt(key: &[u8], iv: &[u8], ct: &[u8], tag: &[u8], aad: &[u8]) -> .map_err(|_| secret_decrypt_error("secret envelope has invalid AES-GCM material"))?; let tag = Tag::try_from(tag) .map_err(|_| secret_decrypt_error("secret envelope has invalid AES-GCM material"))?; + // Decrypt into the ciphertext clone and pass the tag separately so large + // secrets do not require a second ct||tag message allocation. cipher .decrypt_inout_detached(&nonce, aad, plaintext.as_mut_slice().into(), &tag) .map_err(|_| secret_decrypt_error("secret envelope authentication failed"))?; @@ -533,23 +535,6 @@ mod tests { )); } - #[test] - fn aes_gcm_decrypt_uses_detached_tag_without_concat_message() { - let source = include_str!("secrets.rs"); - let helper = source - .split("fn aes_gcm_decrypt") - .nth(1) - .expect("aes_gcm_decrypt helper should exist"); - assert!( - helper.contains("decrypt_inout_detached"), - "AES-GCM decrypt should pass the tag separately instead of concatenating ct||tag" - ); - assert!( - !helper.contains("extend_from_slice(tag)"), - "AES-GCM decrypt should not allocate a ct||tag message buffer" - ); - } - #[test] fn decrypt_records_expired_dek_cache_evictions() { let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); diff --git a/rust/scheduler/src/cron/dispatch.rs b/rust/scheduler/src/cron/dispatch.rs index 24ce4e1..10b1728 100644 --- a/rust/scheduler/src/cron/dispatch.rs +++ b/rust/scheduler/src/cron/dispatch.rs @@ -7,7 +7,7 @@ use crate::{ redis_fields_with_error, scheduler_fields_with_error, }; -use super::reference::{RefVerdict, classify_ref, parse_ref}; +use super::reference::{CronEntry, RefVerdict, classify_ref, parse_ref}; use super::slot::{lease_key, next_fire_ms, slot_key, slot_ms_for}; const CRON_CLAIM_ADVANCE_SCRIPT: &str = r#" @@ -70,6 +70,30 @@ async fn remove_ref_from_slot( Ok(()) } +#[derive(Debug, PartialEq, Eq)] +enum CronDispatchDecision { + RemoveInvalidRef { error_message: String }, + AdvanceOnly { next_slot: i64 }, + Fire { next_slot: i64 }, +} + +fn cron_dispatch_decision(entry: &CronEntry, slot_ms: i64, tick_now: i64) -> CronDispatchDecision { + let next_fire = match next_fire_ms(&entry.cron, &entry.timezone, tick_now) { + Ok(ms) => ms, + Err(err) => { + return CronDispatchDecision::RemoveInvalidRef { + error_message: err.message, + }; + } + }; + let next_slot = slot_ms_for(next_fire); + if slot_ms < slot_ms_for(tick_now) { + CronDispatchDecision::AdvanceOnly { next_slot } + } else { + CronDispatchDecision::Fire { next_slot } + } +} + async fn process_ref( state: AppState, reference: String, @@ -139,10 +163,9 @@ async fn process_ref( // moved to next_slot, giving at-most-once per slot. // (c) stranded refs (slot_ms < current_slot) advance but do NOT fire // — preserves CF's "skip missed after outage" semantic. - let current_slot = slot_ms_for(tick_now); - let next_fire = match next_fire_ms(&entry.cron, &entry.timezone, tick_now) { - Ok(ms) => ms, - Err(err) => { + let decision = cron_dispatch_decision(&entry, slot_ms, tick_now); + let next_slot = match &decision { + CronDispatchDecision::RemoveInvalidRef { error_message } => { state .metrics .increment("cron_stale_refs_cleaned", &[("service", SERVICE)], 1.0); @@ -157,14 +180,15 @@ async fn process_ref( "gen": parts.r#gen, "slot": slot_ms, "reason": "next_fire_failed", - "error_message": err.message, + "error_message": error_message, }), ); remove_ref_from_slot(&state, slot_ms, &reference).await?; return Ok(()); } + CronDispatchDecision::AdvanceOnly { next_slot } + | CronDispatchDecision::Fire { next_slot } => *next_slot, }; - let next_slot = slot_ms_for(next_fire); if !claim_and_advance_ref(&state, &reference, slot_ms, next_slot).await? { state.metrics.increment( "cron_fires", @@ -185,7 +209,7 @@ async fn process_ref( return Ok(()); } - if slot_ms < current_slot { + if matches!(decision, CronDispatchDecision::AdvanceOnly { .. }) { state.metrics.increment( "cron_fires", &[("service", SERVICE), ("outcome", "skipped_stale")], @@ -201,7 +225,7 @@ async fn process_ref( "cron_id": parts.cron_id, "from_slot": slot_ms, "to_slot": next_slot, - "lag_ms": current_slot - slot_ms, + "lag_ms": slot_ms_for(tick_now) - slot_ms, }), ); return Ok(()); @@ -314,40 +338,45 @@ pub(crate) async fn tick(state: AppState) -> SchedulerResult<()> { #[cfg(test)] mod tests { - #[test] - fn invalid_next_fire_refs_are_removed_instead_of_bubbling_error() { - let source = include_str!("dispatch.rs"); - let production_source = source - .split("#[cfg(test)]") - .next() - .expect("dispatch source must include production section"); + use super::*; - assert!(production_source.contains( - "let next_fire = match next_fire_ms(&entry.cron, &entry.timezone, tick_now)" - )); - assert!( - !production_source.contains("next_fire_ms(&entry.cron, &entry.timezone, tick_now)?") - ); - assert!(production_source.contains(r#""reason": "next_fire_failed""#)); - assert!( - production_source.contains("remove_ref_from_slot(&state, slot_ms, &reference).await?") + fn cron_entry(cron: &str) -> CronEntry { + CronEntry { + cron: cron.to_string(), + timezone: "UTC".to_string(), + r#gen: 1, + } + } + + #[test] + fn invalid_next_fire_returns_remove_ref_decision() { + let decision = cron_dispatch_decision( + &cron_entry("not a cron"), + 1_700_000_000_000, + 1_700_000_000_000, ); + + match decision { + CronDispatchDecision::RemoveInvalidRef { error_message } => { + assert!(!error_message.is_empty()); + } + other => panic!("expected invalid next-fire drop, got {other:?}"), + } } #[test] - fn previous_slot_refs_advance_without_runtime_fire() { - let source = include_str!("dispatch.rs"); - let claim_pos = source - .find("claim_and_advance_ref(&state, &reference, slot_ms, next_slot)") - .expect("cron dispatch must claim and advance refs before firing"); - let stale_skip_pos = source - .find("if slot_ms < current_slot") - .expect("cron dispatch must skip firing stale lookback slots"); - let runtime_post_pos = source - .find("post_runtime(") - .expect("cron dispatch must post runtime after stale-slot checks"); + fn previous_slot_returns_advance_only_decision() { + let tick_now = 1_700_000_000_000; + let current_slot = slot_ms_for(tick_now); + let entry = cron_entry("*/5 * * * *"); - assert!(claim_pos < stale_skip_pos); - assert!(stale_skip_pos < runtime_post_pos); + assert!(matches!( + cron_dispatch_decision(&entry, current_slot - 60_000, tick_now), + CronDispatchDecision::AdvanceOnly { .. } + )); + assert!(matches!( + cron_dispatch_decision(&entry, current_slot, tick_now), + CronDispatchDecision::Fire { .. } + )); } } diff --git a/rust/scheduler/src/queue/registry.rs b/rust/scheduler/src/queue/registry.rs index a74a6b5..a83cb89 100644 --- a/rust/scheduler/src/queue/registry.rs +++ b/rust/scheduler/src/queue/registry.rs @@ -176,6 +176,9 @@ pub(crate) async fn refresh_consumer_streams(state: &AppState) { } async fn refresh_consumer_streams_for(queues: &QueueState) { + // Lock order is registry -> consumer_streams. Readers clone consumer_streams + // before consulting registry, so no path holds consumer_streams while waiting + // for registry. let registry = queues.registry.read().await; let snapshot = sorted_consumer_streams(®istry); *queues.consumer_streams.write().await = snapshot; diff --git a/tests/integration/cron-triggers.test.js b/tests/integration/cron-triggers.test.js index e652d49..4f73e48 100644 --- a/tests/integration/cron-triggers.test.js +++ b/tests/integration/cron-triggers.test.js @@ -394,6 +394,40 @@ test("scheduler: stranded ref (slotMs < currentSlot) is advanced without firing assert.equal(res.body, "null", "stranded ref must NOT invoke scheduled()"); }); +test("scheduler: invalid persisted cron ref is removed without firing", async () => { + const ns = uniqueNs("cronbadnext"); + const cron = "0 0 1 1 *"; + const d = await adminPost(`/ns/${ns}/worker/w/deploy`, { + code: SCHEDULED_RECORDER, + crons: [{ cron, timezone: "UTC" }], + }); + assertStatus(d, 201, "invalid persisted cron fixture deploy"); + const p = await adminPost(`/ns/${ns}/worker/w/promote`, { version: d.json.version }); + assertStatus(p, 200, "invalid persisted cron fixture promote"); + + const id = cronId(cron, "UTC"); + const entry = cronHashJsonField(ns, id); + const ref = `${ns}:w:${id}:${entry.gen}`; + redisHSet(`crons:${ns}:w`, { + [id]: JSON.stringify({ ...entry, timezone: "Mars/Olympus" }), + }); + + await waitForCurrentSlotFixtureWindow(); + const currentSlot = Math.floor(Date.now() / 60_000) * 60_000; + redisSAdd(`cron-slot:${currentSlot}`, ref); + composeRestart("scheduler"); + + await waitUntil("scheduler to remove invalid persisted cron ref", async () => { + return !redisSMembers(`cron-slot:${currentSlot}`).includes(ref); + }, { timeoutMs: 15_000, intervalMs: 500 }); + + const res = runtimeInternalGetWithHeaders("/", { + "x-worker-id": gatewayWorkerId(ns, "w", d.json.version), + }); + assert.equal(res.status, 200); + assert.equal(res.body, "null", "invalid persisted cron must NOT invoke scheduled()"); +}); + test("scheduler: ref with mismatched gen is SREM'd from bucket without firing", async () => { const ns = uniqueNs("schstale"); await deployAndPromoteWithCrons(ns, "w", [ From c74dec5778fa164b494f08f3b90365130702ba8e Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Sun, 12 Jul 2026 16:01:01 +0800 Subject: [PATCH 02/23] Consolidate duplicated platform primitives Centralize repeated JS and Rust contracts for request handling, retries, Redis keys, observability, queue dispatch, ownership, and secret envelopes; align runtime and deployment wiring; and replace drift-prone fixtures and tests with shared behavioral coverage. Signed-off-by: Lu Zhang --- auth/index.js | 50 +- control/bindings.js | 6 +- control/bundle.js | 17 +- control/d1-migrations.js | 44 +- control/d1-model.js | 3 +- control/env-budget.js | 304 ++++------- control/errors.js | 55 ++ control/handlers/delete-plan.js | 6 +- control/handlers/delete.js | 131 +++-- control/handlers/deploy.js | 49 +- control/handlers/hosts.js | 8 +- control/handlers/ns-secrets.js | 29 +- control/handlers/promote.js | 5 +- control/handlers/versions.js | 44 +- control/handlers/worker-secrets.js | 71 ++- control/handlers/workflows.js | 158 +++--- control/json-body.js | 47 ++ control/lib.js | 104 +++- control/lifecycle-indexes.js | 50 +- control/optimistic.js | 28 + control/r2.js | 89 ++-- control/routing.js | 155 +++--- control/s3.js | 3 +- control/shared.js | 325 +++--------- control/workflows-client.js | 108 ++++ deploy/kubernetes/README.md | 7 +- deploy/kubernetes/README.zh.md | 5 +- deploy/kubernetes/base/network-policy.yaml | 4 - .../metadata-guard/kustomization.yaml | 4 + .../metadata-egress-policy.yaml | 0 .../kustomization.yaml | 3 +- .../local-metadata-guard/kustomization.yaml | 3 +- .../metadata-egress-policy.yaml | 20 - do-runtime/actor.js | 7 +- do-runtime/config.capnp | 3 + do-runtime/owner-client.js | 13 +- do-runtime/owner-registry.js | 23 +- do-runtime/protocol.js | 15 + docker-compose.images.yml | 56 +- docker-compose.yml | 83 +-- docs/modules/cli.md | 3 +- docs/modules/cli.zh.md | 2 +- docs/modules/control-auth.md | 58 +- docs/modules/control-auth.zh.md | 21 +- docs/modules/durable-objects.md | 12 +- docs/modules/durable-objects.zh.md | 2 + docs/modules/gateway.md | 8 +- docs/modules/gateway.zh.md | 4 +- docs/modules/infra.md | 3 + docs/modules/infra.zh.md | 1 + docs/modules/log-tail-observability.md | 23 +- docs/modules/log-tail-observability.zh.md | 5 +- docs/modules/queues-cron.md | 5 + docs/modules/queues-cron.zh.md | 3 + docs/modules/runtime.md | 10 +- docs/modules/runtime.zh.md | 3 +- docs/modules/workflows.md | 12 +- docs/modules/workflows.zh.md | 3 +- docs/redis-key-layout.md | 9 + docs/redis-key-layout.zh.md | 2 + docs/rust-sidecar-standards.md | 4 + docs/rust-sidecar-standards.zh.md | 2 + docs/security.zh.md | 2 +- docs/source-map.md | 17 +- docs/source-map.zh.md | 17 +- docs/testing.md | 12 +- docs/testing.zh.md | 6 +- gateway/index.js | 25 +- gateway/lib.js | 14 + gateway/runtime.js | 5 +- gateway/websocket.js | 11 +- knip.jsonc | 12 +- package-lock.json | 3 +- package.json | 5 +- runtime/_wdl-do-transport.js | 77 ++- runtime/bindings/do.js | 39 +- runtime/bindings/kv.js | 45 +- runtime/bindings/r2.js | 45 +- runtime/bindings/r2/metadata.js | 36 +- runtime/bindings/service.js | 53 +- runtime/config-system.capnp | 16 + runtime/config-user.capnp | 4 + runtime/dispatch.js | 40 +- runtime/dispatch/workflow-json.js | 9 +- runtime/do-client.js | 52 +- runtime/index.js | 22 +- runtime/internal.js | 24 +- runtime/lib.js | 21 +- runtime/load.js | 56 +- runtime/load/env-build.js | 13 +- runtime/load/wrapper-generate.js | 38 +- runtime/r2-client.js | 45 +- runtime/r2-utils.js | 22 + runtime/runtime.js | 2 +- runtime/tail-forwarder.js | 49 ++ rust/common/Cargo.toml | 1 + rust/common/src/internal_auth.rs | 76 ++- rust/common/src/lib.rs | 14 +- rust/common/src/log.rs | 117 ++++- rust/common/src/metrics.rs | 8 +- rust/common/src/queue_keys.rs | 65 +-- rust/common/src/request_id.rs | 8 +- rust/common/src/test_env.rs | 73 +++ rust/redis-proxy/src/kv.rs | 35 +- rust/redis-proxy/src/lib.rs | 14 +- rust/redis-proxy/src/queue.rs | 14 +- rust/redis-proxy/src/secrets.rs | 227 ++++---- rust/scheduler/src/cron/dispatch.rs | 8 +- rust/scheduler/src/cron/reference.rs | 54 ++ rust/scheduler/src/cron/slot.rs | 12 +- rust/scheduler/src/cron/sweep.rs | 33 +- rust/scheduler/src/lib.rs | 2 + rust/scheduler/src/queue/delayed.rs | 4 +- rust/scheduler/src/queue/keys.rs | 55 -- rust/scheduler/src/queue/registry.rs | 19 + rust/scheduler/src/test_fixtures.rs | 57 ++ rust/supervisor/Cargo.toml | 3 + rust/supervisor/src/config.rs | 50 +- rust/supervisor/src/drain.rs | 31 +- rust/supervisor/src/renew.rs | 43 +- rust/workflows/Cargo.toml | 3 + rust/workflows/src/api.rs | 14 +- rust/workflows/src/api/do_alarms/model.rs | 43 +- rust/workflows/src/api/limits.rs | 36 ++ rust/workflows/src/api/progress.rs | 16 +- rust/workflows/src/api/tick/dispatch.rs | 23 +- rust/workflows/src/server.rs | 16 +- rust/workflows/src/tests.rs | 37 +- scripts/_integration-pool.js | 58 +- scripts/_stream-prefix.js | 21 + scripts/integration-environment.js | 13 + scripts/integration-test-plan.js | 21 +- scripts/run-parallel.js | 18 +- shared/base64.js | 25 +- shared/bounded-body.js | 54 +- shared/d1-transport.js | 1 - shared/internal-auth.js | 6 +- shared/ns-pattern.js | 22 + shared/observability.js | 3 +- shared/owner-lease.js | 53 +- shared/queue-keys.js | 34 +- shared/redis-lock.js | 69 +++ shared/s3-retry.js | 46 ++ shared/secret-envelope.js | 15 +- shared/version.js | 34 ++ system-workers/s3-cleanup/src/index.js | 49 +- terraform/modules/compute/security_groups.tf | 4 + tests/fixtures/cross-language-identity.json | 14 + tests/fixtures/internal-auth-contract.json | 18 + tests/fixtures/observability-contract.json | 12 +- tests/fixtures/queue-key-parse.json | 34 ++ tests/fixtures/request-id-sanitizer.json | 6 + .../scheduler-projection-contract.json | 48 ++ tests/fixtures/secret-envelope-parity.json | 74 +++ tests/fixtures/workflow-limits.json | 5 + tests/helpers/control-handler-harness.js | 2 - tests/helpers/control-shared-stub.js | 176 +++---- tests/helpers/load-auth-index.js | 11 +- tests/helpers/load-control-lib.js | 2 + tests/helpers/load-control-routing.js | 5 +- tests/helpers/load-d1-owner-registry.js | 17 +- tests/helpers/load-do-host-actor.js | 3 + tests/helpers/load-do-owner-registry.js | 15 +- tests/helpers/load-runtime-dispatch.js | 1 + tests/helpers/load-runtime-r2-binding.js | 2 + tests/helpers/load-shared-module.js | 2 + tests/helpers/mocks/fake-redis.js | 27 + tests/helpers/mocks/observability.js | 33 +- tests/helpers/style-contract-scanner.js | 114 +++- tests/integration/helpers/env.js | 19 +- .../integration/queue-native-dispatch.test.js | 17 + .../integration/service-bindings-rpc.test.js | 2 - tests/unit/bounded-body.test.js | 40 +- tests/unit/control-d1-migrations.test.js | 26 +- tests/unit/control-d1-model.test.js | 2 + tests/unit/control-d1-store.test.js | 8 +- tests/unit/control-delete-handler.test.js | 267 ++++++++-- tests/unit/control-deploy-watch.test.js | 102 ++-- tests/unit/control-env-budget.test.js | 83 ++- tests/unit/control-handlers-workers.test.js | 8 +- tests/unit/control-handlers-workflows.test.js | 59 ++- tests/unit/control-index.test.js | 18 +- tests/unit/control-lib.test.js | 134 ++++- tests/unit/control-lifecycle-indexes.test.js | 107 ++-- tests/unit/control-logs-tail.test.js | 17 +- tests/unit/control-r2.test.js | 42 ++ tests/unit/control-routing.test.js | 84 +++ tests/unit/control-s3.test.js | 1 + .../control-secret-envelope-handlers.test.js | 205 +++++++- tests/unit/control-shared-stub.test.js | 40 ++ tests/unit/control-shared.test.js | 109 +++- tests/unit/cross-language-identity.test.js | 2 + tests/unit/do-runtime-protocol.test.js | 83 +++ tests/unit/fake-redis.test.js | 16 +- tests/unit/gateway-lib.test.js | 19 + tests/unit/gateway-websocket.test.js | 1 + tests/unit/integration-pool.test.js | 14 +- tests/unit/integration-test-plan.test.js | 42 ++ tests/unit/internal-auth.test.js | 30 +- tests/unit/observability.test.js | 55 ++ tests/unit/queue-keys.test.js | 32 ++ tests/unit/r2-utils.test.js | 26 + tests/unit/redis-lock.test.js | 95 ++++ tests/unit/runtime-binding-surface.test.js | 14 +- tests/unit/runtime-dispatch-handlers.test.js | 35 +- tests/unit/runtime-dispatch-workflows.test.js | 24 + tests/unit/runtime-do-client.test.js | 36 ++ tests/unit/runtime-internal-auth.test.js | 4 +- tests/unit/runtime-kv-binding.test.js | 4 + tests/unit/runtime-lib.test.js | 22 + tests/unit/runtime-load.test.js | 108 +++- .../runtime-loaded-facade-surface.test.js | 11 +- tests/unit/runtime-pool-boundary.test.js | 5 +- tests/unit/runtime-r2-client.test.js | 55 ++ tests/unit/runtime-r2-host.test.js | 27 + tests/unit/runtime-service-binding.test.js | 20 +- tests/unit/runtime-tail-worker.test.js | 3 + tests/unit/runtime-wrapper-generate.test.js | 48 ++ tests/unit/s3-retry.test.js | 36 ++ tests/unit/secret-envelope.test.js | 122 ++--- tests/unit/stream-prefix.test.js | 25 + tests/unit/style-contracts.test.js | 497 +++++++++++++++--- .../unit/test-helper-style-contracts.test.js | 60 +-- tests/unit/version.test.js | 31 +- 224 files changed, 5748 insertions(+), 2789 deletions(-) create mode 100644 control/errors.js create mode 100644 control/json-body.js create mode 100644 control/optimistic.js create mode 100644 control/workflows-client.js create mode 100644 deploy/kubernetes/components/metadata-guard/kustomization.yaml rename deploy/kubernetes/{overlays/local-ingress-metadata-guard => components/metadata-guard}/metadata-egress-policy.yaml (100%) delete mode 100644 deploy/kubernetes/overlays/local-metadata-guard/metadata-egress-policy.yaml create mode 100644 rust/common/src/test_env.rs create mode 100644 rust/scheduler/src/test_fixtures.rs create mode 100644 scripts/_stream-prefix.js create mode 100644 shared/redis-lock.js create mode 100644 shared/s3-retry.js create mode 100644 tests/fixtures/internal-auth-contract.json create mode 100644 tests/fixtures/queue-key-parse.json create mode 100644 tests/fixtures/scheduler-projection-contract.json create mode 100644 tests/fixtures/secret-envelope-parity.json create mode 100644 tests/fixtures/workflow-limits.json create mode 100644 tests/unit/control-shared-stub.test.js create mode 100644 tests/unit/queue-keys.test.js create mode 100644 tests/unit/redis-lock.test.js create mode 100644 tests/unit/runtime-wrapper-generate.test.js create mode 100644 tests/unit/s3-retry.test.js create mode 100644 tests/unit/stream-prefix.test.js diff --git a/auth/index.js b/auth/index.js index 5d73e55..f9cde88 100644 --- a/auth/index.js +++ b/auth/index.js @@ -37,6 +37,12 @@ import { tokenKey, } from "auth-runtime"; import { ROLES } from "shared-auth-roles"; +import { + acquireTokenLock, + createTokenLock, + releaseTokenLock, +} from "shared-redis-lock"; +import { NAMESPACES_KEY } from "shared-version"; const utf8Decoder = new TextDecoder(); @@ -128,33 +134,16 @@ async function scanTokenRecords(runtime, session) { /** * @param {import("shared-redis").RedisSession} session - * @param {string} key - * @param {string} token - */ -async function acquireDelegatedIssueLock(session, key, token) { - const reply = await session.multi() - .set(key, token, { nx: true, ttl: DELEGATED_ISSUE_LOCK_TTL_SECONDS }) - .exec(); - return Array.isArray(reply) && reply[0] === "OK"; -} - -/** - * @param {import("shared-redis").RedisSession} session - * @param {string} key - * @param {string} token + * @param {{ key: string, token: string }} lock * @param {ReturnType} runtime * @param {string | null} requestId */ -async function releaseDelegatedIssueLock(session, key, token, runtime, requestId) { - try { - await session.delIfEq(key, token); - } catch (err) { - runtime.logAuth("warn", "auth_delegated_issue_lock_release_failed", requestId, { +async function releaseDelegatedIssueLock(session, lock, runtime, requestId) { + await releaseTokenLock(session, lock, { + onError: (err) => runtime.logAuth("warn", "auth_delegated_issue_lock_release_failed", requestId, { ...formatError(err), - }); - // Do not turn a completed issue into an unknown/retryable outcome; the - // token-scoped lock has a short TTL and will self-clear. - } + }), + }); } /** @@ -225,7 +214,7 @@ async function chooseDelegatedNamespace(session, records, template) { for (let attempt = 0; attempt < DELEGATED_ISSUE_NAMESPACE_RETRIES; attempt += 1) { const ns = template.nsGenerator.prefix + randomHex(template.nsGenerator.randomHexBytes); if (!isValidTenantNs(ns)) continue; - if (await session.sIsMember("namespaces", ns)) continue; + if (await session.sIsMember(NAMESPACES_KEY, ns)) continue; if (tokenNamespaceClaimExists(records, ns)) continue; const label = renderDelegatedIssueLabel(template, ns); return { ns, label }; @@ -486,16 +475,19 @@ export default class Auth extends WorkerEntrypoint { } const lockKey = delegatedIssueLockKey(issuerTokenId, templateId); - const lockToken = randomHex(16); + const issueLock = createTokenLock(lockKey); const lockAttemptStartedAtMs = Date.now(); try { - const locked = await acquireDelegatedIssueLock(session, lockKey, lockToken); + const locked = await acquireTokenLock(session, issueLock, { + ttlSeconds: DELEGATED_ISSUE_LOCK_TTL_SECONDS, + transactional: true, + }); if (!locked) { throw new AuthPolicyError(409, "delegated_issue_busy", "delegated issue is already in progress"); } } catch (err) { - await releaseDelegatedIssueLock(session, lockKey, lockToken, runtime, requestId); + await releaseDelegatedIssueLock(session, issueLock, runtime, requestId); return runtime.rethrowPolicy(requestId, "delegated_issue", err); } @@ -513,7 +505,7 @@ export default class Auth extends WorkerEntrypoint { } const namespaceChoice = await chooseDelegatedNamespace(session, tokenRecords, template); assertDelegatedIssueLockLocalBudget(lockAttemptStartedAtMs); - await watchDelegatedIssueLockHeld(session, lockKey, lockToken); + await watchDelegatedIssueLockHeld(session, issueLock.key, issueLock.token); const tokenId = generateTokenId(); const { ns, label } = namespaceChoice; const expiresAt = new Date(nowMs + template.ttlSeconds * 1000).toISOString(); @@ -565,7 +557,7 @@ export default class Auth extends WorkerEntrypoint { } catch (err) { return runtime.rethrowPolicy(requestId, "delegated_issue", err); } finally { - await releaseDelegatedIssueLock(session, lockKey, lockToken, runtime, requestId); + await releaseDelegatedIssueLock(session, issueLock, runtime, requestId); } }); } diff --git a/control/bindings.js b/control/bindings.js index 9f3416d..493e568 100644 --- a/control/bindings.js +++ b/control/bindings.js @@ -7,6 +7,7 @@ import { QUEUE_NAME_RE, BINDING_NAME_RE, KV_ID_RE, + D1_DATABASE_ID_RE, R2_BUCKET_NAME_RE, WDL_RESERVED_BINDING_RE, RESERVED_OBJECT_KEYS, @@ -26,8 +27,6 @@ import { MAX_QUEUE_DELAY_SECONDS, } from "control-lib"; -const D1_DATABASE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; - // Narrower than SECRET_KEY_RE (no lowercase) so platform slot names read // as registered identifiers rather than local vars. export const PLATFORM_KEY_RE = /^[A-Z_][A-Z0-9_]*$/; @@ -457,6 +456,9 @@ export async function linkServiceBinding({ targetMeta = await lookupTargetMeta(targetNs, service, targetVersion); if (!targetMeta) targetMeta = {}; } catch (err) { + // The injected reader may classify persisted target metadata separately + // from transport failures; preserve that domain error at the link boundary. + if (err instanceof LinkError) throw err; const message = errorMessage(err); throw new LinkError(502, "service_binding_target_meta_unavailable", `bindings.${bindingName}: failed to read target meta: ${message}`); diff --git a/control/bundle.js b/control/bundle.js index ed3ab19..d2d19b0 100644 --- a/control/bundle.js +++ b/control/bundle.js @@ -14,6 +14,7 @@ import { } from "shared-ns-pattern"; import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; import { normalizeBindings, validateBindings } from "control-bindings"; +import { parseWorkerdDependencyVersion } from "control-lib"; import PACKAGE_JSON_SOURCE from "wdl-package-json-source"; export class BundleConfigError extends Error { @@ -26,22 +27,12 @@ export class BundleConfigError extends Error { } const COMPATIBILITY_DATE_RE = /^(\d{4})-(\d{2})-(\d{2})$/; -const WORKERD_VERSION_RE = /^1\.(\d{4})(\d{2})(\d{2})\.\d+$/; /** @param {string} source */ export function maxWorkerCompatibilityDateFromPackageJson(source) { - let parsed; - try { - parsed = JSON.parse(source); - } catch { - return null; - } - const raw = parsed?.dependencies?.workerd; - if (typeof raw !== "string") return null; - const version = raw.replace(/^[~^]/, ""); - const match = WORKERD_VERSION_RE.exec(version); - if (!match) return null; - const max = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]) + 7)); + const version = parseWorkerdDependencyVersion(source); + if (!version) return null; + const max = new Date(Date.UTC(version.year, version.month - 1, version.day + 7)); return utcDateString(max); } diff --git a/control/d1-migrations.js b/control/d1-migrations.js index 144cc47..958c88f 100644 --- a/control/d1-migrations.js +++ b/control/d1-migrations.js @@ -3,10 +3,16 @@ import { jsonError, readJsonBody, errMessage, - randomHex, + formatError, requireControlLog, requireControlRedis, } from "control-shared"; +import { + acquireTokenLock, + createTokenLock, + releaseTokenLock, + renewTokenLock, +} from "shared-redis-lock"; import { migrationStatus, MIGRATIONS_TABLE_SQL, @@ -60,28 +66,35 @@ function migrationProgress(applied, skipped) { async function acquireMigrationLock(ns, databaseId) { const redis = requireControlRedis(); const key = migrationLockKey(ns, databaseId); - const token = randomHex(); - const ok = await redis.set(key, token, { ttl: MIGRATION_LOCK_TTL_SECONDS, nx: true }); - if (!ok) return null; - return { key, token }; + const lock = createTokenLock(key); + return await acquireTokenLock(redis, lock, { ttlSeconds: MIGRATION_LOCK_TTL_SECONDS }) + ? lock + : null; } /** @param {MigrationLock | null} lock */ export async function renewMigrationLock(lock) { if (!lock) return { ok: false, reason: "lost" }; const redis = requireControlRedis(); - const renewed = await redis.set(lock.key, lock.token, { - ttl: MIGRATION_LOCK_TTL_SECONDS, - ifeq: lock.token, - }); + const renewed = await renewTokenLock(redis, lock, MIGRATION_LOCK_TTL_SECONDS); return renewed ? { ok: true } : { ok: false, reason: "lost" }; } -/** @param {MigrationLock | null} lock */ -async function releaseMigrationLock(lock) { +/** + * @param {MigrationLock | null} lock + * @param {{ log: import("control-shared").ControlLogger, ns: string, databaseId: string, requestId: string }} context + */ +async function releaseMigrationLock(lock, { log, ns, databaseId, requestId }) { if (!lock) return; const redis = requireControlRedis(); - await redis.delIfEq(lock.key, lock.token); + await releaseTokenLock(redis, lock, { + onError: (err) => log("warn", "d1_migration_lock_release_failed", { + request_id: requestId, + namespace: ns, + database_id: databaseId, + ...formatError(err), + }), + }); } /** @@ -320,7 +333,12 @@ export async function applyMigrations({ request, env, ns, databaseId, requestId appliedById.set(migration.id, record); } } finally { - await releaseMigrationLock(lock); + await releaseMigrationLock(lock, { + log, + ns, + databaseId: database.databaseId, + requestId, + }); } log("info", "d1_migrations_applied", { diff --git a/control/d1-model.js b/control/d1-model.js index 1c83c61..e570c10 100644 --- a/control/d1-model.js +++ b/control/d1-model.js @@ -1,7 +1,8 @@ import { bytesToHex } from "shared-hex"; +import { D1_DATABASE_ID_RE } from "shared-ns-pattern"; export { splitSqlStatements } from "shared-sql-splitter"; -export const D1_DATABASE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +export { D1_DATABASE_ID_RE }; export const D1_DATABASE_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; export const D1_MIGRATION_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,191}$/; export const EXECUTE_MODES = new Set(["all", "raw", "run", "exec"]); diff --git a/control/env-budget.js b/control/env-budget.js index 4ef3819..5f11e13 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -1,19 +1,37 @@ import { decryptSecretValue } from "shared-secret-envelope"; -import { errorMessage } from "shared-errors"; +import { BundleMetaError, parseBundleMeta } from "control-lib"; +import { buildWorkerEnv } from "runtime-load-env-build"; import { bundleKey } from "shared-version"; import { WatchError } from "shared-redis"; -const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; -const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; +export { BundleMetaError }; + const DO_ALARMS_BINDING = "__WDL_DO_ALARMS__"; -const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; const ESTIMATED_ASSETS_CDN_BASE = "https://assets.invalid"; // Pessimistic placeholder for version strings that will be allocated after a // secret mutation. Redis INCR results are parsed as JS numbers today, so this // uses the longest safe integer-shaped `v` tag. export const WORKER_LOADER_ENV_VERSION_PLACEHOLDER = "v9007199254740991"; -const ESTIMATED_DO_STORAGE_ID = "do_00000000000000000000000000000000"; -const ESTIMATED_WORKFLOW_KEY = "wf_00000000000000000000000000000000"; +const ESTIMATED_DO_BACKEND = Object.freeze({ __wdlBinding: "internal", name: "DO_BACKEND" }); +const ESTIMATED_DO_OWNER_NETWORK = Object.freeze({ + __wdlBinding: "internal", + name: "DO_OWNER_NETWORK", +}); +const ESTIMATED_WORKFLOWS_BACKEND = Object.freeze({ + __wdlBinding: "internal", + name: "WORKFLOWS_BACKEND", +}); +/** @type {Parameters[7]} */ +const ESTIMATED_RUNTIME_CONTEXT = Object.freeze({ + exports: Object.freeze({ + KV: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "kv", props }), + Assets: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "assets", props }), + QueueProducer: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "queue", props }), + D1Database: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "d1", props }), + R2Bucket: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "r2", props }), + ServiceBinding: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "service", props }), + }), +}); // Mirrors workerd v1.20260701.1 // src/workerd/api/worker-loader.c++ MAX_DYNAMIC_WORKER_ENV_SIZE. @@ -37,6 +55,17 @@ export class WorkerEnvBudgetError extends Error { } } +/** @param {string} ns @param {string} worker @param {string} version @param {unknown} cause */ +function bundleMaterializationError(ns, worker, version, cause) { + return new BundleMetaError({ + namespace: ns, + worker, + version, + message: `Corrupt __meta__ for ${ns}/${worker}/${version}`, + cause, + }); +} + /** @param {Record | null | undefined} source */ function stringRecord(source) { /** @type {Record} */ @@ -54,126 +83,21 @@ function objectRecord(value) { : null; } -/** @param {unknown} value */ -function stringOrFallback(value, fallback = "") { - return typeof value === "string" ? value : fallback; -} - -/** - * @param {{ - * requiredCallerSecrets?: unknown, - * nsSecrets: Record, - * workerSecrets: Record, - * }} args - */ -function callerSecretsForBinding({ requiredCallerSecrets, nsSecrets, workerSecrets }) { - if (!Array.isArray(requiredCallerSecrets) || requiredCallerSecrets.length === 0) return undefined; - /** @type {Record} */ - const callerSecrets = Object.create(null); - for (const key of requiredCallerSecrets) { - if (typeof key !== "string") continue; - if (Object.hasOwn(workerSecrets, key)) { - callerSecrets[key] = workerSecrets[key]; - } else if (Object.hasOwn(nsSecrets, key)) { - callerSecrets[key] = nsSecrets[key]; - } - } - return callerSecrets; -} - -/** - * @param {{ - * name: string, - * spec: Record, - * meta: Record, - * ns: string, - * worker: string, - * version: string, - * assetsCdnBase: string, - * nsSecrets: Record, - * workerSecrets: Record, - * }} args - */ -function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, assetsCdnBase, nsSecrets, workerSecrets }) { - switch (spec.type) { - case "kv": - return { - __wdlBinding: "kv", - props: { ns, id: stringOrFallback(spec.id) }, - }; - case "assets": - return { - __wdlBinding: "assets", - props: { - cdnBase: assetsCdnBase, - prefix: stringOrFallback(objectRecord(meta.assets)?.prefix), - }, - }; - case "queue": - return { - __wdlBinding: "queue", - props: { - ns, - id: stringOrFallback(spec.id), - deliveryDelaySeconds: spec.deliveryDelaySeconds ?? 0, - }, - }; - case "d1": - return { - __wdlBinding: "d1", - props: { - ns, - databaseId: stringOrFallback(spec.databaseId), - binding: name, - }, - }; - case "r2": - return { - __wdlBinding: "r2", - props: { - ns, - bucketName: stringOrFallback(spec.bucketName), - binding: name, - }, - }; - case "do": { - const props = { - ns, - worker, - version, - doStorageId: stringOrFallback(spec.doStorageId, ESTIMATED_DO_STORAGE_ID), - binding: name, - className: stringOrFallback(spec.className), - }; - // DO bindings intentionally mirror runtime's default/factory output shape: - // props are top-level fields, while hostProxy carries the JSRPC namespace. - return { - __wdlBinding: "do", - ...props, - hostProxy: { __wdlBinding: "do-host-proxy", props }, - }; - } - case "service": { - const callerSecrets = callerSecretsForBinding({ - requiredCallerSecrets: spec.requiredCallerSecrets, - nsSecrets, - workerSecrets, - }); - return { - __wdlBinding: "service", - props: { - targetNs: stringOrFallback(spec.ns, ns), - targetWorker: stringOrFallback(spec.service), - targetVersion: stringOrFallback(spec.version), - targetEntrypoint: typeof spec.entrypoint === "string" ? spec.entrypoint : null, - callerNs: ns, - ...(callerSecrets ? { callerSecrets } : {}), - }, - }; - } - default: - return { __wdlBinding: stringOrFallback(spec.type, "unknown") }; - } +/** @param {{ name: string, spec: Record, ns: string, worker: string, version: string }} input */ +function estimatedDoBinding({ name, spec, ns, worker, version }) { + const props = { + ns, + worker, + version, + doStorageId: spec.doStorageId, + binding: name, + className: spec.className, + }; + return { + __wdlBinding: "do", + ...props, + hostProxy: { __wdlBinding: "do-host-proxy", props }, + }; } /** @@ -209,63 +133,37 @@ export function estimatedWorkerLoaderEnv({ const metaRecord = objectRecord(meta); if (!metaRecord || !worker) return env; - let hasDoBinding = false; - let doAlarmStorageId = ESTIMATED_DO_STORAGE_ID; - let hasWorkflowBinding = false; - const workflows = Array.isArray(metaRecord.workflows) ? metaRecord.workflows : []; - for (const workflow of workflows) { - const record = objectRecord(workflow); - if (!record) continue; - const binding = stringOrFallback(record.binding); - if (!binding) continue; - hasWorkflowBinding = true; - env[binding] = { - ns, - worker, - version, - name: stringOrFallback(record.name), - binding, - className: stringOrFallback(record.className), - workflowKey: stringOrFallback(record.workflowKey, ESTIMATED_WORKFLOW_KEY), - }; - } - - const bindings = objectRecord(metaRecord.bindings); - if (bindings) { - for (const [name, rawSpec] of Object.entries(bindings)) { - const spec = objectRecord(rawSpec); - if (!spec) continue; - env[name] = estimatedBindingEnvValue({ - name, - spec, - meta: metaRecord, - ns, - worker, - version, - assetsCdnBase: typeof assetsCdnBase === "string" && assetsCdnBase - ? assetsCdnBase - : ESTIMATED_ASSETS_CDN_BASE, - nsSecrets: nsSecretStrings, - workerSecrets: workerSecretStrings, - }); - if (spec.type === "do") { - hasDoBinding = true; - doAlarmStorageId = stringOrFallback(spec.doStorageId, ESTIMATED_DO_STORAGE_ID); - } + const estimated = buildWorkerEnv( + { ...metaRecord, vars: stringRecord(vars) }, + nsSecretStrings, + workerSecretStrings, + ns, + worker, + version, + typeof assetsCdnBase === "string" && assetsCdnBase + ? assetsCdnBase + : ESTIMATED_ASSETS_CDN_BASE, + ESTIMATED_RUNTIME_CONTEXT, + ESTIMATED_DO_BACKEND, + { + doOwnerNetwork: ESTIMATED_DO_OWNER_NETWORK, + doBindingFactory: estimatedDoBinding, + workflowsBackend: ESTIMATED_WORKFLOWS_BACKEND, } + ); + + let doAlarmStorageId = null; + for (const rawSpec of Object.values(objectRecord(metaRecord.bindings) || {})) { + const spec = objectRecord(rawSpec); + if (spec?.type === "do") doAlarmStorageId = spec.doStorageId; } - if (hasDoBinding) { - env[DO_BACKEND_BINDING] = { __wdlBinding: "internal", name: "DO_BACKEND" }; - env[DO_OWNER_NETWORK_BINDING] = { __wdlBinding: "internal", name: "DO_OWNER_NETWORK" }; - env[DO_ALARMS_BINDING] = { + if (typeof doAlarmStorageId === "string" && doAlarmStorageId) { + estimated[DO_ALARMS_BINDING] = { __wdlBinding: "do-alarms", props: { ns, worker, version, doStorageId: doAlarmStorageId }, }; } - if (hasWorkflowBinding) { - env[WORKFLOWS_BACKEND_BINDING] = { __wdlBinding: "internal", name: "WORKFLOWS_BACKEND" }; - } - return env; + return estimated; } /** @param {string} value */ @@ -471,39 +369,29 @@ export async function assertWorkerVersionsUserEnvBudget({ // whose command protocol is single-flight even though secret decryption is not. for (const { sourceVersion, estimatedVersion } of uniqueChecks) { const rawMeta = await redis.hGet(bundleKey(ns, worker, sourceVersion), "__meta__"); - if (typeof rawMeta !== "string") { - if (retryMissingVersions) throw new WatchError(); - throw new Error(`bundle metadata missing for ${ns}/${worker}@${sourceVersion}`); - } - /** @type {unknown} */ - let parsed; - try { - parsed = JSON.parse(rawMeta); - } catch (err) { - throw new Error( - `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ${errorMessage(err)}`, - { cause: err } - ); - } - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - throw new Error( - `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ` + - "__meta__ must be a JSON object" - ); - } - const meta = /** @type {Record} */ (parsed); - assertWorkerLoaderUserEnvBudget({ + if (typeof rawMeta !== "string" && retryMissingVersions) throw new WatchError(); + const meta = parseBundleMeta(rawMeta, { ns, worker, - version: estimatedVersion, - sourceVersion, - vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) - ? /** @type {Record} */ (meta.vars) - : null, - nsSecrets, - workerSecrets, - meta, - assetsCdnBase, + version: sourceVersion, }); + try { + assertWorkerLoaderUserEnvBudget({ + ns, + worker, + version: estimatedVersion, + sourceVersion, + vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) + ? /** @type {Record} */ (meta.vars) + : null, + nsSecrets, + workerSecrets, + meta, + assetsCdnBase, + }); + } catch (err) { + if (err instanceof WorkerEnvBudgetError || err instanceof BundleMetaError) throw err; + throw bundleMaterializationError(ns, worker, sourceVersion, err); + } } } diff --git a/control/errors.js b/control/errors.js new file mode 100644 index 0000000..ab1a1d8 --- /dev/null +++ b/control/errors.js @@ -0,0 +1,55 @@ +import { jsonError } from "shared-respond"; + +// Control owns this base abort contract. Routing and auth remain separate; +// deploy subclasses it where commit cleanup needs its own catch boundary. + +/** @param {unknown} value @returns {value is Record} */ +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +export class ControlAbort extends Error { + /** + * @param {number} status + * @param {string} code + * @param {{ message?: string, [key: string]: unknown }} [details] + */ + constructor(status, code, details = {}) { + super(details?.message || code); + this.status = status; + this.code = code; + this.details = details; + } +} + +// Domain errors flow through jsonError() so details cannot shadow the +// top-level wire contract. +/** + * @param {unknown} err + * @param {string} [fallbackCode] + * @param {Record} [extraDetails] + */ +export function codedErrorResponse(err, fallbackCode = "internal_error", extraDetails = {}) { + const record = isRecord(err) ? err : {}; + const details = isRecord(record.details) ? record.details : {}; + const status = typeof record.status === "number" ? record.status : 500; + const code = typeof record.code === "string" && record.code ? record.code : fallbackCode; + const recordMessage = typeof record.message === "string" && record.message ? record.message : undefined; + const message = err instanceof ControlAbort + ? (typeof err.details.message === "string" && err.details.message) || err.message || code + : recordMessage || (typeof details.message === "string" && details.message) || code; + return jsonError( + status, + code, + message, + { ...details, ...extraDetails }, + ); +} + +/** + * @param {ControlAbort} err + * @param {Record} [extraDetails] + */ +export function controlAbortResponse(err, extraDetails = {}) { + return codedErrorResponse(err, err.code, extraDetails); +} diff --git a/control/handlers/delete-plan.js b/control/handlers/delete-plan.js index bcb970f..eb1583f 100644 --- a/control/handlers/delete-plan.js +++ b/control/handlers/delete-plan.js @@ -12,7 +12,7 @@ import { stageWorkerHidden, stageWorkerVersionIndexRemove, } from "control-lifecycle-indexes"; -import { bundleKey, patternsKey, routesKey } from "shared-version"; +import { NAMESPACES_KEY, bundleKey, nsHostsKey, patternsKey, routesKey } from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; /** @@ -88,10 +88,10 @@ export function stageWorkerDelete(multi, { collected, channels }) { multi.hDel(routesKey(ns), name); } if (collected.hostsLosingNsOwnership.length) { - multi.sRem(`ns-hosts:${ns}`, collected.hostsLosingNsOwnership); + multi.sRem(nsHostsKey(ns), collected.hostsLosingNsOwnership); } if (!collected.namespaceStillActive) { - multi.sRem("namespaces", ns); + multi.sRem(NAMESPACES_KEY, ns); } multi.del(cronWorkerKey(ns, name)); diff --git a/control/handlers/delete.js b/control/handlers/delete.js index dceb9e0..bb9b550 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -11,6 +11,7 @@ import { assertWorkflowDeleteAllowed, cleanupDoAlarmsForWorker, buildS3CleanupTaskId, recordCleanupIntentOrWarn, ControlAbort, controlAbortResponse, + withOptimisticRetries, ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL, PATTERNS_CHANNEL, } from "control-shared"; import { @@ -21,8 +22,10 @@ import { doStorageIdKey, extractD1Refs, extractOutgoingRefs, formatReferrerBlocker, + bundleAssetPrefix, + parseBundleMeta, } from "control-lib"; -import { bundleKey, patternsKey, routesKey } from "shared-version"; +import { bundleKey, hostsKey, patternsKey, routesKey } from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; import { decodeBulk, WatchError } from "shared-redis"; import { queueConsumerScanPrefix } from "shared-queue-keys"; @@ -80,8 +83,16 @@ const DELETE_COLLECT_READ_CONCURRENCY = 16; class WholeDeleteError extends ControlAbort {} -// Raised when an under-WATCH drift check observes state that disagrees -// with the pre-session snapshot — caller loops to re-collect. +/** @param {string} ns @param {string} name @returns {never} */ +function throwWholeDeleteContention(ns, name) { + throw new WholeDeleteError(503, "whole_delete_contention", { + namespace: ns, name, + message: `exhausted ${MAX_DELETE_ATTEMPTS} attempts; retry later`, + }); +} + +// Raised when collection or an under-WATCH check observes state that +// changed across reads; callers retry from a fresh snapshot. class DriftSignal extends Error { /** @param {string} reason */ constructor(reason) { super(reason); this.name = "DriftSignal"; } @@ -204,7 +215,14 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { let collected = null; let workflowBlocker = null; try { - collected = await collectDeleteInputs({ redis, ns, name }); + collected = await withOptimisticRetries( + async () => await collectDeleteInputs({ redis, ns, name }), + { + attempts: MAX_DELETE_ATTEMPTS, + isRetryableError: (err) => err instanceof DriftSignal, + onExhausted: () => throwWholeDeleteContention(ns, name), + } + ); await assertWorkflowDeleteAllowed({ ns, worker: name }); } catch (err) { if (err instanceof WholeDeleteError) { @@ -284,7 +302,7 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { * @returns {Promise} */ async function executeWholeDelete({ redis, ns, name, principal, requestId, log }) { - for (let attempt = 0; attempt < MAX_DELETE_ATTEMPTS; attempt++) { + return await withOptimisticRetries(async () => { const collected = await collectDeleteInputs({ redis, ns, name }); let workflowBlocker = null; try { @@ -338,24 +356,18 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log } }); } - try { - const result = await runSessionEXEC({ - redis, ns, name, principal, requestId, collected, - }); - await cleanupDoAlarmsOrWarn({ - ns, worker: name, doStorageId: collected.doStorageId, requestId, log, - }); - result.doStorageRetention = describeDoStorageRetention(collected); - return result; - } catch (err) { - if (err instanceof DriftSignal) continue; - if (err instanceof WatchError) continue; - throw err; - } - } - throw new WholeDeleteError(503, "whole_delete_contention", { - namespace: ns, name, - message: `exhausted ${MAX_DELETE_ATTEMPTS} attempts; retry later`, + const result = await runSessionEXEC({ + redis, ns, name, principal, requestId, collected, + }); + await cleanupDoAlarmsOrWarn({ + ns, worker: name, doStorageId: collected.doStorageId, requestId, log, + }); + result.doStorageRetention = describeDoStorageRetention(collected); + return result; + }, { + attempts: MAX_DELETE_ATTEMPTS, + isRetryableError: (err) => err instanceof DriftSignal || err instanceof WatchError, + onExhausted: () => throwWholeDeleteContention(ns, name), }); } @@ -455,8 +467,8 @@ async function collectDeleteInputs({ redis, ns, name }) { const queueConsumerKeys = await scanQueueConsumerKeysForWorker(redis, ns, name); - // Corrupt meta is fail-closed: swallowing would leave S3 orphans and - // stranded reverse referrers after a half-delete. + // Metadata behind retained/active indexes is required: proceeding without + // it would omit S3 and reverse-ref cleanup from a successful delete. const retainedVersions = await redis.zRange(workerVersionsKey(ns, name), 0, -1); /** @type {Record} */ const prefixByVersion = {}; @@ -474,23 +486,26 @@ async function collectDeleteInputs({ redis, ns, name }) { return { ver, rawMeta, referrers }; }); for (const { ver, rawMeta, referrers } of retainedReads) { - if (rawMeta) { - let meta; - try { - meta = JSON.parse(rawMeta); - } catch (err) { - throw new WholeDeleteError(500, "corrupt_meta", { - namespace: ns, name, version: ver, - stage: "retained_meta_parse", - detail: errMessage(err), - }); + if (rawMeta == null) { + const currentVersions = await redis.zRange(workerVersionsKey(ns, name), 0, -1); + if (!currentVersions.includes(ver)) { + throw new DriftSignal("retained versions changed during collection"); } - if (meta && meta.assets && typeof meta.assets.prefix === "string") { - prefixByVersion[ver] = meta.assets.prefix; - } - d1RefsByVersion[ver] = extractD1Refs(meta && meta.bindings); - outgoingRefsByVersion[ver] = extractOutgoingRefs(meta && meta.bindings, ns); } + const meta = parseBundleMeta(rawMeta, { + ns, + worker: name, + version: ver, + makeError: ({ reason }) => new WholeDeleteError(500, "corrupt_meta", { + namespace: ns, name, version: ver, + stage: "retained_meta_parse", + detail: reason, + }), + }); + const assetPrefix = bundleAssetPrefix(meta); + if (assetPrefix !== null) prefixByVersion[ver] = assetPrefix; + d1RefsByVersion[ver] = extractD1Refs(meta.bindings); + outgoingRefsByVersion[ver] = extractOutgoingRefs(meta.bindings, ns); referrersByVersion[ver] = referrers; } @@ -501,23 +516,27 @@ async function collectDeleteInputs({ redis, ns, name }) { let activeRoutes = []; if (activeVersion) { const rawMeta = await redis.hGet(bundleKey(ns, name, activeVersion), "__meta__"); - if (rawMeta) { - let meta; - try { - meta = JSON.parse(rawMeta); - } catch (err) { - throw new WholeDeleteError(500, "corrupt_meta", { - namespace: ns, name, version: activeVersion, - stage: "active_meta_parse", - detail: errMessage(err), - }); + if (rawMeta == null) { + const currentActive = await redis.hGet(routesKey(ns), name); + if (currentActive !== activeVersion) { + throw new DriftSignal("active version changed during collection"); } - activeRoutes = normalizeActiveRoutes(meta, { - namespace: ns, - name, - version: activeVersion, - }); } + const meta = parseBundleMeta(rawMeta, { + ns, + worker: name, + version: activeVersion, + makeError: ({ reason }) => new WholeDeleteError(500, "corrupt_meta", { + namespace: ns, name, version: activeVersion, + stage: "active_meta_parse", + detail: reason, + }), + }); + activeRoutes = normalizeActiveRoutes(meta, { + namespace: ns, + name, + version: activeVersion, + }); } // A host stays in ns-hosts if any OTHER worker in this ns still holds a @@ -664,7 +683,7 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected return await redis.session(async (iso) => { const watchKeys = [ routesKey(ns), - `hosts:${ns}`, + hostsKey(ns), deleteLockKey(ns, name), doStorageIdKey(ns, name), workerVersionsKey(ns, name), diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 03640a6..5b6d883 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -15,7 +15,9 @@ import { extractOutgoingRefs, deleteLockKey, doStorageIdKey, + parseBundleMeta, workflowDefsKey, + platformDomainFromEnv, } from "control-lib"; import { stageD1ReferrerAdds, @@ -37,7 +39,7 @@ import { parseCronList, parseQueueConsumers, } from "control-topology"; -import { formatVersion, parseVersion, bundleKey, routesKey } from "shared-version"; +import { formatVersion, parseVersion, bundleKey, nextVersionKey, routesKey } from "shared-version"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; import { isReservedNs, isValidRouteNs, ROUTES_ALLOWED_RESERVED_NS } from "shared-ns-pattern"; import { putAsset, inferContentType } from "control-s3"; @@ -150,6 +152,16 @@ function isPlatformExportMeta(value) { ); } +/** @param {string} ns @param {string} worker @param {string} version @param {unknown} raw */ +function linkableBundleMeta(ns, worker, version, raw) { + return parseBundleMeta(raw, { + ns, + worker, + version, + makeError: ({ message }) => new LinkError(500, "corrupt_meta", message), + }); +} + /** @param {unknown} err @returns {DeployRequestError} */ function deployRequestErrorFromUnknown(err) { const record = err && typeof err === "object" ? /** @type {Record} */ (err) : {}; @@ -335,7 +347,9 @@ async function validateServiceBindingsPreflight({ redis, ns, name, bindings }) { if (cached) return await cached; const load = (async () => { const rawMeta = await redis.hGet(bundleKey(targetNs, worker, version), "__meta__"); - return rawMeta ? JSON.parse(rawMeta) : null; + return rawMeta == null + ? null + : linkableBundleMeta(targetNs, worker, version, rawMeta); })(); metaCache.set(key, load); return await load; @@ -367,15 +381,9 @@ async function collectPlatformExports(redis) { }))).flat(); const exportsByWorker = await Promise.all(routeEntries.map(async ({ ns, worker, version }) => { - /** @type {{ exports?: unknown }} */ - let meta; - try { - const rawMeta = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); - if (!rawMeta) return []; - meta = JSON.parse(rawMeta); - } catch { - return []; - } + const rawMeta = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); + if (rawMeta == null) return []; + const meta = linkableBundleMeta(ns, worker, version, rawMeta); if (!Array.isArray(meta.exports)) return []; const exportsList = /** @type {unknown[]} */ (meta.exports); return exportsList @@ -731,9 +739,7 @@ export async function handle({ request, env, ns, name, requestId }) { const redis = requireControlRedis(); const s3 = getControlS3(); const log = requireControlLog(); - const platformDomain = typeof env.PLATFORM_DOMAIN === "string" && env.PLATFORM_DOMAIN - ? env.PLATFORM_DOMAIN - : "workers.local"; + const platformDomain = platformDomainFromEnv(env); const parsed = await parseDeployRequestForHandler({ request, ns, platformDomain }); if (parsed.response) return parsed.response; @@ -747,7 +753,16 @@ export async function handle({ request, env, ns, name, requestId }) { deployRequest: parsed.deployRequest, }); } catch (err) { - if (err instanceof LinkError) return codedErrorResponse(err, "link_error"); + if (err instanceof LinkError) { + log(err.status >= 500 ? "error" : "warn", "deploy_rejected", { + request_id: requestId, + namespace: ns, + worker: name, + status: err.status, + reason: err.code, + }); + return codedErrorResponse(err, "link_error"); + } throw err; } const { bindings: mergedBindings, warnings } = platformResult; @@ -785,7 +800,7 @@ export async function handle({ request, env, ns, name, requestId }) { return deployAssetsS3NotConfiguredResponse(warningDetails); } - const num = await redis.incr(`worker:${ns}:${name}:next_version`); + const num = await redis.incr(nextVersionKey(ns, name)); const version = formatVersion(num); const uploadResult = await uploadDeployAssetsBeforeCommit({ @@ -1064,7 +1079,7 @@ async function validateOutgoingRefsForCommit(iso, { outgoingRefs }) { }); } const targetMeta = targetMetas[index]; - if (!targetMeta) { + if (typeof targetMeta !== "string") { throw new DeployAbort(409, "target_drift", { target: { ns: ref.targetNs, diff --git a/control/handlers/hosts.js b/control/handlers/hosts.js index 5bdad0a..84d38f1 100644 --- a/control/handlers/hosts.js +++ b/control/handlers/hosts.js @@ -7,6 +7,8 @@ import { requireControlRedis, } from "control-shared"; import { reconcileHosts, RoutingError } from "control-routing"; +import { platformDomainFromEnv } from "control-lib"; +import { hostsKey } from "shared-version"; /** * @param {{ request: Request, env: Record, method: string, nsName: string, requestId: string }} args @@ -16,7 +18,7 @@ export async function handle({ request, env, method, nsName, requestId }) { const log = requireControlLog(); if (method === "GET") { - const hosts = await redis.sMembers(`hosts:${nsName}`); + const hosts = await redis.sMembers(hostsKey(nsName)); return jsonResponse(200, { namespace: nsName, hosts: [...hosts].toSorted(), @@ -27,9 +29,7 @@ export async function handle({ request, env, method, nsName, requestId }) { if (parsed.response) return parsed.response; const body = /** @type {Record} */ (parsed.body); try { - const platformDomain = typeof env.PLATFORM_DOMAIN === "string" && env.PLATFORM_DOMAIN - ? env.PLATFORM_DOMAIN - : "workers.local"; + const platformDomain = platformDomainFromEnv(env); const hosts = await reconcileHosts(redis, nsName, body, platformDomain); log("info", "hosts_reconciled", { request_id: requestId, diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index f7f1aac..3ab6310 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -1,6 +1,7 @@ import { jsonResponse, jsonError, + errMessage, requireControlLog, requireControlRedis, stringEnv, @@ -17,6 +18,7 @@ import { routesKey, workerVersionsKey } from "shared-version"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; import { workersIndexKey } from "control-lib"; import { + BundleMetaError, WorkerEnvBudgetError, assertWorkerLoaderUserEnvBudget, assertWorkerVersionsUserEnvBudget, @@ -29,9 +31,24 @@ const MAX_NS_SECRET_ATTEMPTS = 5; class NamespaceSecretAbort extends ControlAbort {} -/** @param {unknown} err */ -function namespaceSecretMutationErrorResponse(err) { +/** + * @param {unknown} err + * @param {{ log: import("control-shared").ControlLogger, requestId: string, nsName: string, secretKey: string, method: string }} context + */ +function namespaceSecretMutationErrorResponse(err, { log, requestId, nsName, secretKey, method }) { if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); + if (err instanceof BundleMetaError) { + log("error", "ns_secret_mutation_rejected", { + request_id: requestId, + namespace: nsName, + key: secretKey, + method, + status: err.status, + reason: err.code, + error_detail: errMessage(err.cause), + }); + return codedErrorResponse(err, err.code); + } if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); return null; @@ -200,7 +217,9 @@ export async function handle({ request, env, method, nsName, secretKey, requestI plaintext: put.plaintext, }); } catch (err) { - const response = namespaceSecretMutationErrorResponse(err); + const response = namespaceSecretMutationErrorResponse(err, { + log, requestId, nsName, secretKey, method, + }); if (response) return response; throw err; } @@ -225,7 +244,9 @@ export async function handle({ request, env, method, nsName, secretKey, requestI method: "DELETE", }); } catch (err) { - const response = namespaceSecretMutationErrorResponse(err); + const response = namespaceSecretMutationErrorResponse(err, { + log, requestId, nsName, secretKey, method, + }); if (response) return response; throw err; } diff --git a/control/handlers/promote.js b/control/handlers/promote.js index 4966246..1177437 100644 --- a/control/handlers/promote.js +++ b/control/handlers/promote.js @@ -9,6 +9,7 @@ import { } from "control-shared"; import { parseVersion } from "shared-version"; import { promoteWithRoutes, RoutingError } from "control-routing"; +import { platformDomainFromEnv } from "control-lib"; /** * @param {{ request: Request, env: Record, ns: string, name: string, requestId: string }} args @@ -27,9 +28,7 @@ export async function handle({ request, env, ns, name, requestId }) { } try { - const platformDomain = typeof env.PLATFORM_DOMAIN === "string" && env.PLATFORM_DOMAIN - ? env.PLATFORM_DOMAIN - : "workers.local"; + const platformDomain = platformDomainFromEnv(env); const result = await promoteWithRoutes(redis, ns, name, body.version, { log, requestId }); log("info", "worker_promoted", { request_id: requestId, diff --git a/control/handlers/versions.js b/control/handlers/versions.js index b82e855..eb0eded 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -4,7 +4,6 @@ import { assertWorkflowDeleteAllowed, buildS3CleanupTaskId, recordCleanupIntentOrWarn, ControlAbort, controlAbortResponse, - errMessage, requireControlLog, requireControlRedis, runOptimistic, @@ -13,6 +12,8 @@ import { workerVersionsKey, referrersKey, extractD1Refs, extractOutgoingRefs, formatReferrerBlocker, + bundleAssetPrefix, + parseBundleMeta, } from "control-lib"; import { stageD1ReferrerRemovals, @@ -179,19 +180,14 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque } const bundleMetaRaw = await iso.hGet(bundleKey(ns, name, version), "__meta__"); - if (!bundleMetaRaw) { - throw new VersionDeleteError(404, "version_not_found", { - namespace: ns, name, version, - }); - } - let bundleMeta; - try { - bundleMeta = JSON.parse(bundleMetaRaw); - } catch { - throw new VersionDeleteError(500, "corrupt_meta", { + const bundleMeta = parseBundleMeta(bundleMetaRaw, { + ns, + worker: name, + version, + makeError: () => new VersionDeleteError(500, "corrupt_meta", { namespace: ns, name, version, - }); - } + }), + }); const referrerMembers = await iso.sMembers(referrersKey(ns, name, version)); if (referrerMembers.length > 0) { @@ -203,8 +199,7 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque const outgoingRefs = extractOutgoingRefs(bundleMeta.bindings, ns); const d1Refs = extractD1Refs(bundleMeta.bindings); - const prefix = bundleMeta.assets && bundleMeta.assets.prefix - ? bundleMeta.assets.prefix : null; + const prefix = bundleAssetPrefix(bundleMeta); // Secret-bump COPY can make siblings share the same assets prefix; // skip the cleanup task if any other retained version still points @@ -215,18 +210,17 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque for (const otherV of currentVersions) { if (otherV === version) continue; const otherRaw = await iso.hGet(bundleKey(ns, name, otherV), "__meta__"); - if (!otherRaw) continue; - let otherMeta; - try { - otherMeta = JSON.parse(otherRaw); - } catch (err) { - throw new VersionDeleteError(500, "corrupt_meta", { + const otherMeta = parseBundleMeta(otherRaw, { + ns, + worker: name, + version: otherV, + makeError: ({ reason }) => new VersionDeleteError(500, "corrupt_meta", { namespace: ns, name, version: otherV, stage: "sibling_meta_parse", - detail: errMessage(err), - }); - } - if (otherMeta && otherMeta.assets && otherMeta.assets.prefix === prefix) { + detail: reason, + }), + }); + if (bundleAssetPrefix(otherMeta) === prefix) { skipS3Cleanup = true; break; } diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 4824f77..b074877 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -3,6 +3,7 @@ import { jsonError, ControlAbort, controlAbortResponse, + errMessage, requireControlLog, requireControlRedis, runOptimistic, @@ -21,6 +22,7 @@ import { import { stageWorkerHidden, stageWorkerVisible } from "control-lifecycle-indexes"; import { bumpActiveAndPromote, RoutingError } from "control-routing"; import { + BundleMetaError, WorkerEnvBudgetError, assertWorkerVersionsUserEnvBudget, decryptMutatedSecretHashForBudget, @@ -48,6 +50,36 @@ const MAX_SECRET_ATTEMPTS = 5; class SecretAbort extends ControlAbort {} class SecretNoop extends Error {} +function secretMutationContentionAbort() { + return new SecretAbort(503, "secret_mutation_contention", { + message: "active version changed during secret mutation; retry later", + }); +} + +/** + * @param {{ + * abort: ControlAbort, + * requestId: string, + * ns: string, + * name: string, + * key: string, + * method: string, + * log: (level: string, event: string, fields: Record) => void, + * }} args + */ +function rejectSecretMutation({ abort, requestId, ns, name, key, method, log }) { + log("warn", "secret_mutation_rejected", { + request_id: requestId, + namespace: ns, + worker: name, + key, + method, + status: abort.status, + reason: abort.code, + }); + return controlAbortResponse(abort); +} + /** * @param {{ * request: Request, @@ -173,12 +205,13 @@ export async function handle({ request, env, method, ns, name, subPath, requestI else payload.deleted = true; return jsonResponse(200, payload); } - throw new SecretAbort(503, "secret_mutation_contention", { - message: `active version changed during secret mutation; retry later`, - }); + throw secretMutationContentionAbort(); } catch (err) { if (err instanceof SecretAbort) { - log("warn", "secret_mutation_rejected", { + return rejectSecretMutation({ abort: err, requestId, ns, name, key, method, log }); + } + if (err instanceof BundleMetaError) { + log("error", "secret_mutation_rejected", { request_id: requestId, namespace: ns, worker: name, @@ -186,40 +219,28 @@ export async function handle({ request, env, method, ns, name, subPath, requestI method, status: err.status, reason: err.code, + error_detail: errMessage(err.cause), }); - return controlAbortResponse(err); + return codedErrorResponse(err, err.code); } if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); if (err instanceof RoutingError) { if (err.code === "bump_contention") { - const abort = new SecretAbort(503, "secret_mutation_contention", { - message: `active version changed during secret mutation; retry later`, - }); - log("warn", "secret_mutation_rejected", { - request_id: requestId, - namespace: ns, - worker: name, + return rejectSecretMutation({ + abort: secretMutationContentionAbort(), + requestId, + ns, + name, key, method, - status: abort.status, - reason: abort.code, + log, }); - return controlAbortResponse(abort); } if (err.code === "caller_deleting") { const abort = new SecretAbort(409, "deleting", { namespace: ns, worker: name, }); - log("warn", "secret_mutation_rejected", { - request_id: requestId, - namespace: ns, - worker: name, - key, - method, - status: abort.status, - reason: abort.code, - }); - return controlAbortResponse(abort); + return rejectSecretMutation({ abort, requestId, ns, name, key, method, log }); } return codedErrorResponse(err, err.code); } diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index fd190ee..5d98dfb 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -3,15 +3,16 @@ import { WORKFLOW_NAME_RE, isValidWorkerName, isValidWorkflowName, + parseBundleMeta, workflowDefsKey, } from "control-lib"; import { - codedErrorResponse, - controlInternalJsonHeaders, + ControlAbort, + controlAbortResponse, errMessage, - getControlWorkflows, jsonError, jsonResponse, + postWorkflowsInternal, requireControlLog, requireControlRedis, } from "control-shared"; @@ -21,10 +22,8 @@ const LIFECYCLE_ACTIONS = new Set(["pause", "resume", "restart", "terminate"]); /** * @typedef {import("shared-redis").RedisClient} RedisClient - * @typedef {{ fetch: typeof fetch }} WorkflowBackend * @typedef {{ method: string, url: URL, ns: string, subPath: string[], requestId: string }} WorkflowsHandlerArgs * @typedef {{ redis: RedisClient }} RedisDeps - * @typedef {{ redis: RedisClient, workflows: WorkflowBackend | null }} WorkflowDeps * @typedef {{ name: string, binding: string, className: string, workflowKey: string }} ActiveWorkflowMeta * @typedef {{ name: string, binding: string | null, className: string, workflowKey: string, retired?: boolean }} WorkflowEntry * @typedef {WorkflowEntry & { namespace: string, worker: string, activeVersion: string }} ListedWorkflowEntry @@ -50,8 +49,8 @@ export async function handle({ method, url, ns, subPath, requestId }) { try { return await handleInner({ method, url, ns, subPath, requestId }); } catch (err) { - if (err instanceof WorkflowControlError) { - return codedErrorResponse(err); + if (err instanceof ControlAbort) { + return controlAbortResponse(err); } throw err; } @@ -60,9 +59,8 @@ export async function handle({ method, url, ns, subPath, requestId }) { /** @param {WorkflowsHandlerArgs} args */ async function handleInner({ method, url, ns, subPath, requestId }) { const redis = requireControlRedis(); - const workflows = getControlWorkflows(); const log = requireControlLog(); - const deps = { redis, workflows }; + const deps = { redis }; if (method === "GET" && subPath.length === 0) { const body = await listWorkflowDefinitions(deps, ns); log("info", "workflows_listed", { @@ -78,7 +76,7 @@ async function handleInner({ method, url, ns, subPath, requestId }) { const workflow = await resolveWorkflow(deps, ns, worker, workflowName); if (method === "GET" && subPath.length === 3) { - const body = await callWorkflowsRust(deps, "instances", { + const body = await callWorkflowsRust("instances", { ...workflow.request, options: listOptions(url), requestId, @@ -96,7 +94,7 @@ async function handleInner({ method, url, ns, subPath, requestId }) { if (subPath.length >= 4) { const instanceId = decodePathSegment(subPath[3], "workflow instance id"); if (method === "GET" && subPath.length === 4) { - const body = await callWorkflowsRust(deps, "status", { + const body = await callWorkflowsRust("status", { ...workflow.request, instanceId, options: statusOptions(url), @@ -116,9 +114,11 @@ async function handleInner({ method, url, ns, subPath, requestId }) { if (method === "POST" && subPath.length === 5 && LIFECYCLE_ACTIONS.has(subPath[4])) { const action = subPath[4]; if (workflow.workflow?.retired && action === "restart") { - throw new WorkflowControlError(409, "workflow_not_exported", `Workflow ${ns}/${worker}/${workflowName} is not exported by the active worker version`); + throw new ControlAbort(409, "workflow_not_exported", { + message: `Workflow ${ns}/${worker}/${workflowName} is not exported by the active worker version`, + }); } - const body = await callWorkflowsRust(deps, action, { + const body = await callWorkflowsRust(action, { ...workflow.request, instanceId, requestId, @@ -157,7 +157,7 @@ async function listWorkflowDefinitions({ redis }, ns) { const workflows = []; for (let i = 0; i < routeEntries.length; i += 1) { const [worker, activeVersion] = routeEntries[i]; - const meta = parseBundleMetaRaw(ns, worker, activeVersion, metaRaws[i]); + const meta = workflowBundleMeta(ns, worker, activeVersion, metaRaws[i]); const activeByName = new Map(); for (const workflow of workflowsFromMeta(meta)) { activeByName.set(workflow.name, workflow); @@ -202,14 +202,20 @@ async function listWorkflowDefinitions({ redis }, ns) { */ async function resolveWorkflow({ redis }, ns, worker, workflowName) { if (!isValidWorkerName(worker)) { - throw new WorkflowControlError(400, "invalid_worker_name", `Invalid worker name ${JSON.stringify(worker)}. Must match ${WORKER_NAME_RE}.`); + throw new ControlAbort(400, "invalid_worker_name", { + message: `Invalid worker name ${JSON.stringify(worker)}. Must match ${WORKER_NAME_RE}.`, + }); } if (!isValidWorkflowName(workflowName)) { - throw new WorkflowControlError(400, "invalid_workflow_name", `Invalid workflow name ${JSON.stringify(workflowName)}. Must match ${WORKFLOW_NAME_RE}.`); + throw new ControlAbort(400, "invalid_workflow_name", { + message: `Invalid workflow name ${JSON.stringify(workflowName)}. Must match ${WORKFLOW_NAME_RE}.`, + }); } const activeVersion = await redis.hGet(routesKey(ns), worker); if (!activeVersion) { - throw new WorkflowControlError(404, "worker_not_found", `Worker ${ns}/${worker} is not active`); + throw new ControlAbort(404, "worker_not_found", { + message: `Worker ${ns}/${worker} is not active`, + }); } const meta = await readBundleMeta({ redis }, ns, worker, activeVersion); const workflow = workflowsFromMeta(meta).find((entry) => entry.name === workflowName); @@ -232,7 +238,9 @@ async function resolveWorkflow({ redis }, ns, worker, workflowName) { ? defs[workflowName] : undefined; if (!def) { - throw new WorkflowControlError(404, "workflow_not_found", `Workflow ${ns}/${worker}/${workflowName} is not exported`); + throw new ControlAbort(404, "workflow_not_found", { + message: `Workflow ${ns}/${worker}/${workflowName} is not exported`, + }); } const retiredWorkflow = { name: workflowName, @@ -297,16 +305,16 @@ function parseWorkflowDefs(raw) { * @param {string} ns * @param {string} worker * @param {string} version - * @param {string | Uint8Array | null | undefined} raw - * @returns {unknown} + * @param {unknown} raw + * @returns {Record} */ -function parseBundleMetaRaw(ns, worker, version, raw) { - if (!raw) return null; - try { - return JSON.parse(String(raw)); - } catch { - throw new WorkflowControlError(500, "corrupt_meta", `Corrupt __meta__ for ${ns}/${worker}/${version}`); - } +function workflowBundleMeta(ns, worker, version, raw) { + return parseBundleMeta(raw, { + ns, + worker, + version, + makeError: ({ message }) => new ControlAbort(500, "corrupt_meta", { message }), + }); } /** @@ -329,7 +337,8 @@ async function readWorkflowListRaws({ redis }, ns, routeEntries) { worker_count: routeEntries.length, error_message: errMessage(err), }); - throw new WorkflowControlError(500, "workflow_metadata_unavailable", "Workflow metadata is unavailable", { + throw new ControlAbort(500, "workflow_metadata_unavailable", { + message: "Workflow metadata is unavailable", namespace: ns, worker_count: routeEntries.length, }); @@ -341,7 +350,7 @@ async function readWorkflowListRaws({ redis }, ns, routeEntries) { * @param {string} ns * @param {string} worker * @param {string} version - * @returns {Promise} + * @returns {Promise>} */ async function readBundleMeta({ redis }, ns, worker, version) { let raw; @@ -354,13 +363,14 @@ async function readBundleMeta({ redis }, ns, worker, version) { version, error_message: errMessage(err), }); - throw new WorkflowControlError(500, "workflow_metadata_unavailable", "Workflow metadata is unavailable", { + throw new ControlAbort(500, "workflow_metadata_unavailable", { + message: "Workflow metadata is unavailable", namespace: ns, worker, version, }); } - return parseBundleMetaRaw(ns, worker, version, raw); + return workflowBundleMeta(ns, worker, version, raw); } /** @@ -393,32 +403,21 @@ function isActiveWorkflowMeta(entry) { } /** - * @param {WorkflowDeps} deps * @param {string} endpoint * @param {WorkflowRequest} body * @returns {Promise>} */ -async function callWorkflowsRust({ workflows }, endpoint, body) { - if (!workflows || typeof workflows.fetch !== "function") { - throw new WorkflowControlError(503, "workflow_internal_dispatch_failed", "Workflow backend is unavailable"); - } - let response; - let parsed; - try { - response = await workflows.fetch(`http://workflows/internal/workflows/${endpoint}`, { - method: "POST", - headers: controlInternalJsonHeaders(), - body: JSON.stringify(body), - }); - parsed = await response.json().catch(() => null); - } catch (err) { - requireControlLog()("error", "workflow_backend_request_failed", { +async function callWorkflowsRust(endpoint, body) { + const { response, body: parsed } = await postWorkflowsInternal({ + endpoint: `workflows/${endpoint}`, + body, + logEvent: "workflow_backend_request_failed", + logFields: { request_id: body.requestId || null, endpoint, - error_message: errMessage(err), - }); - throw new WorkflowControlError(503, "workflow_internal_dispatch_failed", "Workflow backend request failed"); - } + }, + timeoutMs: null, + }); if (!response.ok) { if (isUpstreamErrorBody(parsed)) { if (response.status >= 500) { @@ -432,17 +431,21 @@ async function callWorkflowsRust({ workflows }, endpoint, body) { } return throwUpstreamError(response.status, parsed); } - throw new WorkflowControlError( + throw new ControlAbort( response.status >= 400 ? response.status : 502, "workflow_internal_dispatch_failed", - "Workflow backend request failed", - { upstream_status: response.status }, + { + message: "Workflow backend request failed", + upstream_status: response.status, + }, ); } if (!parsed || typeof parsed !== "object") { - throw new WorkflowControlError(502, "workflow_internal_dispatch_failed", "Workflow backend returned an invalid response"); + throw new ControlAbort(502, "workflow_internal_dispatch_failed", { + message: "Workflow backend returned an invalid response", + }); } - return parsed; + return /** @type {Record} */ (parsed); } /** @@ -452,18 +455,22 @@ async function callWorkflowsRust({ workflows }, endpoint, body) { */ function throwUpstreamError(status, body) { if (status >= 500) { - throw new WorkflowControlError( + throw new ControlAbort( status, body.error, - "Workflow backend request failed", - { upstream_status: status }, + { + message: "Workflow backend request failed", + upstream_status: status, + }, ); } - throw new WorkflowControlError( + throw new ControlAbort( status, body.error, - typeof body.message === "string" ? body.message : body.error, - filterDetails(body), + { + ...filterDetails(body), + message: typeof body.message === "string" ? body.message : body.error, + }, ); } @@ -508,7 +515,9 @@ function statusOptions(url) { /** @type {Record} */ const options = {}; if (url.searchParams.has("include_steps") || url.searchParams.has("step_limit")) { - throw new WorkflowControlError(400, "invalid_request", "workflow status query options use camelCase"); + throw new ControlAbort(400, "invalid_request", { + message: "workflow status query options use camelCase", + }); } const includeSteps = url.searchParams.get("includeSteps"); if (includeSteps != null) options.includeSteps = parseBooleanOption(includeSteps, "includeSteps"); @@ -523,7 +532,7 @@ function statusOptions(url) { */ function parseIntegerOption(raw, label) { if (!/^(0|[1-9][0-9]*)$/.test(raw)) { - throw new WorkflowControlError(400, "invalid_request", `${label} must be an integer`); + throw new ControlAbort(400, "invalid_request", { message: `${label} must be an integer` }); } return Number(raw); } @@ -537,7 +546,7 @@ function parseBooleanOption(raw, label) { // numeric options stay strict because an empty number has no useful meaning. if (raw === "" || raw === "1" || raw === "true") return true; if (raw === "0" || raw === "false") return false; - throw new WorkflowControlError(400, "invalid_request", `${label} must be true or false`); + throw new ControlAbort(400, "invalid_request", { message: `${label} must be true or false` }); } /** @@ -548,21 +557,8 @@ function decodePathSegment(value, label) { try { return decodeURIComponent(value); } catch { - throw new WorkflowControlError(400, "invalid_request", `invalid percent-encoding in ${label}`); - } -} - -class WorkflowControlError extends Error { - /** - * @param {number} status - * @param {string} code - * @param {string} message - * @param {Record} [details] - */ - constructor(status, code, message, details = {}) { - super(message); - this.status = status; - this.code = code; - this.details = details; + throw new ControlAbort(400, "invalid_request", { + message: `invalid percent-encoding in ${label}`, + }); } } diff --git a/control/json-body.js b/control/json-body.js new file mode 100644 index 0000000..76c3795 --- /dev/null +++ b/control/json-body.js @@ -0,0 +1,47 @@ +import { BodyTooLargeError, readBoundedText } from "shared-bounded-body"; +import { jsonError } from "shared-respond"; + +export const DEFAULT_JSON_BODY_MAX_BYTES = 1024 * 1024; + +/** + * @param {Request} request + * @param {{ requireObject?: boolean, allowEmpty?: boolean, maxBytes?: number }} [opts] + */ +export async function readJsonBody( + request, + { requireObject = false, allowEmpty = false, maxBytes = DEFAULT_JSON_BODY_MAX_BYTES } = {}, +) { + let body; + try { + const text = await readBoundedText(request, maxBytes); + if (text === "") { + if (!allowEmpty) { + return { + response: jsonError(400, "invalid_json", "Body must be valid JSON"), + }; + } + body = {}; + } else { + body = JSON.parse(text); + } + } catch (err) { + if (err instanceof BodyTooLargeError) { + return { + response: jsonError( + 413, + "request_body_too_large", + `Body must be at most ${maxBytes} bytes` + ), + }; + } + return { + response: jsonError(400, "invalid_json", "Body must be valid JSON"), + }; + } + if (requireObject && (!body || typeof body !== "object" || Array.isArray(body))) { + return { + response: jsonError(400, "invalid_json_object", "Body must be a JSON object"), + }; + } + return { body }; +} diff --git a/control/lib.js b/control/lib.js index 71fa8d9..e5ef8c2 100644 --- a/control/lib.js +++ b/control/lib.js @@ -1,6 +1,6 @@ -// Name grammar predicates, lifecycle Redis key helpers, referrer-index -// encode/decode, URL → {gate, ns} classifier, and secret-key validator. -// Pure data-shaping only. +// Name grammar predicates, lifecycle Redis key helpers, bundle metadata +// parsing, referrer-index encode/decode, URL → {gate, ns} classifier, +// and secret-key validation. Pure data-shaping only. import { NS_PATTERN, @@ -11,24 +11,87 @@ import { QUEUE_NAME_RE, WDL_RESERVED_BINDING_RE, KV_ID_RE, + configuredHostname, + platformDomainFromEnv, validateModulePath, } from "shared-ns-pattern"; import { PLATFORM_TIER_RESERVED_NS, ROLES } from "shared-auth-roles"; +import { errorMessage } from "shared-errors"; import { doStorageIdKey, routesKey, workerVersionsKey } from "shared-version"; export { doStorageIdKey, routesKey, workerVersionsKey }; export { validateModulePath }; export { WORKER_NAME_RE }; export { WORKFLOW_NAME_RE }; +export { configuredHostname, platformDomainFromEnv }; export const NS_RE = new RegExp(`^${NS_PATTERN}$`); export const MAX_QUEUE_DELAY_SECONDS = 86_400; +const WORKERD_DEPENDENCY_VERSION_RE = /^1\.(\d{4})(\d{2})(\d{2})\.(\d+)$/; /** * @typedef {{ callerNs: string, callerWorker: string, callerVersion: string, binding: string }} ReferrerMember * @typedef {{ kind?: string, ns?: string }} AccessPrincipal * @typedef {{ referrers: ReferrerMember[], crossNamespaceReferrerCount?: number }} ReferrerBlocker + * @typedef {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown }} BundleMetaFailure */ +export class BundleMetaError extends Error { + /** @param {{ namespace: string, worker: string, version: string, message: string, cause: unknown }} failure */ + constructor({ namespace, worker, version, message, cause }) { + super(message, { cause }); + this.name = "BundleMetaError"; + this.status = 500; + this.code = "corrupt_meta"; + this.details = { namespace, worker, version }; + } +} + +/** + * @param {unknown} raw + * @param {{ ns: string, worker: string, version: string, makeError?: (failure: BundleMetaFailure) => Error }} options + * @returns {Record} + */ +export function parseBundleMeta(raw, { ns, worker, version, makeError }) { + /** @param {unknown} cause @returns {never} */ + const fail = (cause) => { + const failure = { + namespace: ns, + worker, + version, + message: `Corrupt __meta__ for ${ns}/${worker}/${version}`, + reason: errorMessage(cause), + cause, + }; + throw makeError ? makeError(failure) : new BundleMetaError(failure); + }; + + if (typeof raw !== "string") { + fail(new TypeError("__meta__ must be a JSON string")); + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch (err) { + fail(err); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + fail(new TypeError("__meta__ must be a JSON object")); + } + return /** @type {Record} */ (parsed); +} + +/** + * @param {Record} meta + * @returns {string | null} + */ +export function bundleAssetPrefix(meta) { + const assets = meta.assets; + if (!assets || typeof assets !== "object" || Array.isArray(assets)) return null; + const prefix = /** @type {Record} */ (assets).prefix; + return typeof prefix === "string" ? prefix : null; +} + /** @param {unknown} name */ export function isValidWorkerName(name) { return typeof name === "string" && WORKER_NAME_RE.test(name); @@ -76,14 +139,6 @@ export function projectAccessPrincipal(principal) { return null; } -/** @param {unknown} value */ -export function configuredHostname(value) { - if (typeof value !== "string" || !value.trim()) return null; - const host = value.trim().replace(/\.+$/, "").toLowerCase(); - if (!host || host.includes("/") || host.includes(":") || /\s/.test(host)) return null; - return host; -} - /** @param {unknown} value */ export function configuredPublicUrl(value) { if (typeof value !== "string" || !value.trim()) return null; @@ -100,18 +155,35 @@ export function configuredPublicUrl(value) { return parsed.href.replace(/\/+$/, ""); } -/** @param {string} source */ -export function platformVersionFromPackageJson(source) { +/** + * @param {string} source + * @returns {{ version: string, year: number, month: number, day: number, patch: number } | null} + */ +export function parseWorkerdDependencyVersion(source) { let parsed; try { parsed = JSON.parse(source); } catch { - return "wdl.unknown"; + return null; } const raw = parsed?.dependencies?.workerd; - if (typeof raw !== "string") return "wdl.unknown"; + if (typeof raw !== "string") return null; const version = raw.replace(/^[~^]/, ""); - return /^1\.\d{8}\.\d+$/.test(version) ? `wdl.${version.slice(2)}` : "wdl.unknown"; + const match = WORKERD_DEPENDENCY_VERSION_RE.exec(version); + if (!match) return null; + return { + version, + year: Number(match[1]), + month: Number(match[2]), + day: Number(match[3]), + patch: Number(match[4]), + }; +} + +/** @param {string} source */ +export function platformVersionFromPackageJson(source) { + const parsed = parseWorkerdDependencyVersion(source); + return parsed ? `wdl.${parsed.version.slice(2)}` : "wdl.unknown"; } // ─── Referrer index ──────────────────────────────────────────────── diff --git a/control/lifecycle-indexes.js b/control/lifecycle-indexes.js index 02f80f8..1610493 100644 --- a/control/lifecycle-indexes.js +++ b/control/lifecycle-indexes.js @@ -15,6 +15,8 @@ export const CRON_WORKER_INDEX_KEY = "cron:index:workers"; * @typedef {{ targetNs: string, targetWorker: string, targetVersion: string, binding: string }} OutgoingRef * @typedef {{ binding: string, databaseId?: string, resolvedDatabaseId?: string }} D1Ref * @typedef {{ id: string, gen: number, slot: number }} CronIndexEntry + * @typedef {{ cron: string, timezone: string }} CronSpec + * @typedef {{ cronSeq: number, addedWithPlacement: Array, removed: Array<{ id: string }> }} CronPlan * @typedef {{ queue: string, maxBatchSize: number, maxBatchTimeoutMs: number, maxRetries: number, retryDelaySeconds?: number, deadLetterQueue?: string }} QueueConsumer */ @@ -64,18 +66,33 @@ export function cronSlotKey(slotMs) { return `cron-slot:${slotMs}`; } +/** @param {number} slotMs */ +export function cronSlotExpireAt(slotMs) { + return Math.floor(slotMs / 1000) + 600; +} + /** @param {string} ns @param {string} worker @param {string} cronId @param {number} gen */ export function cronRefMember(ns, worker, cronId, gen) { return `${ns}:${worker}:${cronId}:${gen}`; } +/** @param {string} version @param {number} seq */ +export function cronMetaJson(version, seq) { + return JSON.stringify({ version, seq }); +} + +/** @param {CronSpec & { gen: number }} entry */ +export function cronEntryJson(entry) { + return JSON.stringify({ cron: entry.cron, timezone: entry.timezone, gen: entry.gen }); +} + /** @param {RedisMulti} multi @param {string} ns @param {string} worker @param {CronIndexEntry} entry */ export function stageCronSlotRef(multi, ns, worker, entry) { const key = cronSlotKey(entry.slot); multi.sAdd(key, cronRefMember(ns, worker, entry.id, entry.gen)); // Bound orphan refs (a crashed advance_ref leaves a member behind); // tick ignores slots >60s old, so 10min past slot wall-time is safe. - multi.expireAt(key, Math.floor(entry.slot / 1000) + 600); + multi.expireAt(key, cronSlotExpireAt(entry.slot)); } /** @param {RedisMulti} multi @param {string} ns @param {string} worker */ @@ -88,6 +105,35 @@ export function stageCronWorkerRemoved(multi, ns, worker) { multi.sRem(CRON_WORKER_INDEX_KEY, cronWorkerKey(ns, worker)); } +/** @param {RedisMulti} multi @param {{ ns: string, worker: string, version: string, cronKey: string, seq: number }} args */ +export function stageCronProjectionMeta(multi, { ns, worker, version, cronKey, seq }) { + stageCronWorkerIndexed(multi, ns, worker); + multi.hSet(cronKey, "__meta__", cronMetaJson(version, seq)); +} + +/** + * @param {RedisMulti} multi + * @param {{ ns: string, worker: string, version: string, cronKey: string, existingHash: Record, crons: CronSpec[], plan: CronPlan }} args + */ +export function stageCronProjection( + multi, + { ns, worker, version, cronKey, existingHash, crons, plan } +) { + // __meta__.version lets scheduler build x-worker-id without a second HGET; + // delete the hash when no crons remain so stale refs decay. + if (crons.length === 0) { + if (Object.keys(existingHash).length) multi.del(cronKey); + stageCronWorkerRemoved(multi, ns, worker); + return; + } + stageCronProjectionMeta(multi, { ns, worker, version, cronKey, seq: plan.cronSeq }); + for (const entry of plan.addedWithPlacement) { + multi.hSet(cronKey, entry.id, cronEntryJson(entry)); + stageCronSlotRef(multi, ns, worker, entry); + } + for (const removed of plan.removed) multi.hDel(cronKey, removed.id); +} + /** @param {RedisMulti} multi @param {{ ns: string, worker: string, version: string, refs: OutgoingRef[] }} args */ export function stageOutgoingReferrerAdds(multi, { ns, worker, version, refs }) { for (const ref of refs) { @@ -154,7 +200,7 @@ export function stageD1ReferrerRemovals(multi, { ns, worker, version, refs, data * @param {QueueConsumer} consumer * @returns {{ worker: string, version: string, max_batch_size: string, max_batch_timeout_ms: string, max_retries: string, dead_letter_queue?: string, retry_delay_secs?: string }} */ -function queueConsumerFields(worker, version, consumer) { +export function queueConsumerFields(worker, version, consumer) { /** @type {{ worker: string, version: string, max_batch_size: string, max_batch_timeout_ms: string, max_retries: string, dead_letter_queue?: string, retry_delay_secs?: string }} */ const fields = { worker, diff --git a/control/optimistic.js b/control/optimistic.js new file mode 100644 index 0000000..14d21b5 --- /dev/null +++ b/control/optimistic.js @@ -0,0 +1,28 @@ +import { WatchError } from "shared-redis"; +import { withOptimisticRetries } from "shared-owner-lease"; + +export { withOptimisticRetries }; + +/** + * @template T, S + * @param {{ session: (fn: (session: S) => Promise) => Promise }} redis + * @param {{ attempts?: number, onExhausted: () => unknown | Promise, onWatchError?: (err: unknown, attempt: number) => void, shouldRetryResult?: (result: T, attempt: number) => boolean }} options + * @param {(session: S, attempt: number) => Promise} fn + * @returns {Promise} + */ +export async function runOptimistic( + redis, + { attempts = 5, onExhausted, onWatchError, shouldRetryResult }, + fn +) { + return await withOptimisticRetries( + async (attempt) => await redis.session((session) => fn(session, attempt)), + { + attempts, + isRetryableError: (err) => err instanceof WatchError, + onRetryableError: onWatchError, + shouldRetryResult, + onExhausted: async () => /** @type {T} */ (await onExhausted()), + } + ); +} diff --git a/control/r2.js b/control/r2.js index 40bf00a..454dc8a 100644 --- a/control/r2.js +++ b/control/r2.js @@ -1,5 +1,6 @@ import { SigV4Client } from "@wdl-dev/aws-sigv4"; import { discardResponseBody } from "shared-respond"; +import { S3_TRANSIENT_RETRIES } from "shared-s3-retry"; import { encodeS3KeyPath, encodeS3Query, @@ -13,7 +14,6 @@ import { import { collectXmlFields, listXmlTagValues, xmlTagValueIsTrue } from "shared-s3-xml"; const DEFAULT_LIST_LIMIT = 1000; -const S3_TRANSIENT_RETRIES = 10; /** * @typedef {{ client: SigV4Client, endpoint: string, bucket: string }} R2Admin @@ -60,6 +60,37 @@ function requestHeaders(requestId) { return headers; } +/** + * @param {{ r2: R2Admin, ns: string, bucketName: string, key: string, requestId?: string, method: "GET" | "HEAD" | "DELETE", notFound?: "error" | "null" | "ok" }} args + * @returns {Promise} + */ +async function fetchR2AdminObject({ + r2, + ns, + bucketName, + key, + requestId, + method, + notFound = "error", +}) { + const res = await r2.client.fetch(r2ObjectUrl(r2, { ns, bucketName }, key), { + method, + headers: requestHeaders(requestId), + }); + if (res.status === 404 && notFound !== "error") { + if (notFound === "null") { + await discardResponseBody(res); + return null; + } + return res; + } + if (!res.ok) { + const detail = await res.text().catch(() => ""); + throw new Error(`R2 admin ${method} failed with ${res.status}: ${detail.slice(0, 200)}`); + } + return res; +} + /** @param {unknown} value */ function limitFrom(value) { if (value == null || value === "") return DEFAULT_LIST_LIMIT; @@ -185,54 +216,42 @@ export async function listR2Objects({ /** @param {{ r2: R2Admin, ns: string, bucketName: string, key: string, requestId?: string }} args */ export async function getR2Object({ r2, ns, bucketName, key, requestId }) { - validateR2BucketName(bucketName); - normalizeR2ObjectKey(key); - const res = await r2.client.fetch(r2ObjectUrl(r2, { ns, bucketName }, key), { + return fetchR2AdminObject({ + r2, + ns, + bucketName, + key, + requestId, method: "GET", - headers: requestHeaders(requestId), + notFound: "null", }); - if (res.status === 404) { - await discardResponseBody(res); - return null; - } - if (!res.ok) { - const detail = await res.text().catch(() => ""); - throw new Error(`R2 admin GET failed with ${res.status}: ${detail.slice(0, 200)}`); - } - return res; } /** @param {{ r2: R2Admin, ns: string, bucketName: string, key: string, requestId?: string }} args */ export async function headR2Object({ r2, ns, bucketName, key, requestId }) { - validateR2BucketName(bucketName); - normalizeR2ObjectKey(key); - const res = await r2.client.fetch(r2ObjectUrl(r2, { ns, bucketName }, key), { + return fetchR2AdminObject({ + r2, + ns, + bucketName, + key, + requestId, method: "HEAD", - headers: requestHeaders(requestId), + notFound: "null", }); - if (res.status === 404) { - await discardResponseBody(res); - return null; - } - if (!res.ok) { - const detail = await res.text().catch(() => ""); - throw new Error(`R2 admin HEAD failed with ${res.status}: ${detail.slice(0, 200)}`); - } - return res; } /** @param {{ r2: R2Admin, ns: string, bucketName: string, key: string, requestId?: string }} args */ export async function deleteR2Object({ r2, ns, bucketName, key, requestId }) { - validateR2BucketName(bucketName); - normalizeR2ObjectKey(key); - const res = await r2.client.fetch(r2ObjectUrl(r2, { ns, bucketName }, key), { + const res = await fetchR2AdminObject({ + r2, + ns, + bucketName, + key, + requestId, method: "DELETE", - headers: requestHeaders(requestId), + notFound: "ok", }); - if (!res.ok && res.status !== 404) { - const detail = await res.text().catch(() => ""); - throw new Error(`R2 admin DELETE failed with ${res.status}: ${detail.slice(0, 200)}`); - } + if (!res) throw new Error("R2 admin DELETE unexpectedly returned no response"); await discardResponseBody(res); return { namespace: ns, bucket: bucketName, key, status: "ok" }; } diff --git a/control/routing.js b/control/routing.js index f45bb35..99f5c1e 100644 --- a/control/routing.js +++ b/control/routing.js @@ -2,21 +2,17 @@ // stays connection-local; a nil EXEC raises WatchError and runOptimistic() // retries with fresh reads. -import { - DECLARED_HOSTS_KEY, - HOST_DECLARATIONS_PREFIX, - runOptimistic, -} from "control-shared"; +import { runOptimistic } from "control-shared"; import { d1DatabaseKey, deleteLockKey, extractD1Refs, extractOutgoingRefs, + parseBundleMeta, } from "control-lib"; import { cronWorkerKey, - stageCronSlotRef, - stageCronWorkerIndexed, - stageCronWorkerRemoved, + stageCronProjection, + stageCronProjectionMeta, stageD1ReferrerAdds, stageOutgoingReferrerAdds, stageQueueConsumerProjection, @@ -25,7 +21,19 @@ import { } from "control-lifecycle-indexes"; import { parseHostList } from "control-topology"; import { decodePatternProjection, encodePatternProjection } from "shared-route-projection"; -import { bundleKey, formatVersion, parseVersion, patternsKey, routesKey } from "shared-version"; +import { + DECLARED_HOSTS_KEY, + NAMESPACES_KEY, + bundleKey, + formatVersion, + hostDeclarationsKey, + hostsKey, + nextVersionKey, + nsHostsKey, + parseVersion, + patternsKey, + routesKey, +} from "shared-version"; import { errorMessage } from "shared-errors"; import { diffCrons, nextFireMs, slotMsFor } from "control-cron-index"; import { isReservedNs, ROUTES_ALLOWED_RESERVED_NS } from "shared-ns-pattern"; @@ -43,11 +51,6 @@ const MAX_ATTEMPTS = 5; const PATTERNS_CHANNEL = "patterns:invalidate"; const ROUTES_CHANNEL = "routes:invalidate"; -/** @param {string} host */ -function hostDeclarationsKey(host) { - return `${HOST_DECLARATIONS_PREFIX}${host}`; -} - /** * @typedef {import("shared-route-projection").PatternProjection} PatternProjection * @typedef {Pick & { host: string, slot: string }} RoutePattern @@ -152,27 +155,23 @@ function parseCronMeta(raw, logContext) { async function readMeta(iso, ns, workerName, version) { if (!version) return null; const raw = await iso.hGet(bundleKey(ns, workerName, version), "__meta__"); - if (!raw) return null; - return parseBundleMeta(ns, workerName, version, raw); + return routingBundleMeta(ns, workerName, version, raw); } /** * @param {string} ns * @param {string} workerName * @param {string} version - * @param {string} raw + * @param {unknown} raw */ -function parseBundleMeta(ns, workerName, version, raw) { - try { - return /** @type {BundleMeta} */ (JSON.parse(raw)); - } catch { - // Swallowing would hide old routes/consumers from the diff and leak them. - throw new RoutingError( - 500, - "corrupt_meta", - `Corrupt __meta__ for ${ns}/${workerName}/${version}` - ); - } +function routingBundleMeta(ns, workerName, version, raw) { + // Swallowing would hide old routes/consumers from the diff and leak them. + return /** @type {BundleMeta} */ (parseBundleMeta(raw, { + ns, + worker: workerName, + version, + makeError: ({ message }) => new RoutingError(500, "corrupt_meta", message), + })); } /** @param {RedisIso} iso @param {string} ns @param {string} workerName @param {string | null | undefined} version */ @@ -236,12 +235,12 @@ function computeCronPlan(cronHash, newCrons, now, logContext) { function stageDeclaredHostChanges(multi, ns, toAdd, removals) { const changedHosts = new Set([...toAdd, ...removals.map((entry) => entry.host)]); for (const host of toAdd) { - multi.sAdd(`hosts:${ns}`, host); + multi.sAdd(hostsKey(ns), host); multi.sAdd(hostDeclarationsKey(host), ns); multi.sAdd(DECLARED_HOSTS_KEY, host); } for (const { host, declaredNs } of removals) { - multi.sRem(`hosts:${ns}`, host); + multi.sRem(hostsKey(ns), host); multi.sRem(hostDeclarationsKey(host), ns); if (declaredNs.filter((declared) => declared !== ns).length === 0) { multi.del(hostDeclarationsKey(host)); @@ -251,30 +250,6 @@ function stageDeclaredHostChanges(multi, ns, toAdd, removals) { for (const host of changedHosts) multi.publish(PATTERNS_CHANNEL, host); } -/** @param {RedisMulti} multi @param {string} ns @param {string} workerName @param {string} newVersion @param {string} cronKey @param {Record} cronHash @param {CronSpec[]} newCrons @param {CronPlan} cronPlan */ -function stageCronPlan(multi, ns, workerName, newVersion, cronKey, cronHash, newCrons, cronPlan) { - // __meta__.version lets scheduler build x-worker-id without a - // second HGET; DEL when no crons remain so stale refs decay. - if (newCrons.length === 0) { - if (Object.keys(cronHash).length) multi.del(cronKey); - stageCronWorkerRemoved(multi, ns, workerName); - return; - } - stageCronWorkerIndexed(multi, ns, workerName); - multi.hSet(cronKey, "__meta__", JSON.stringify({ - version: newVersion, seq: cronPlan.cronSeq, - })); - for (const e of cronPlan.addedWithPlacement) { - multi.hSet(cronKey, e.id, JSON.stringify({ - cron: e.cron, timezone: e.timezone, gen: e.gen, - })); - stageCronSlotRef(multi, ns, workerName, e); - } - for (const r of cronPlan.removed) { - multi.hDel(cronKey, r.id); - } -} - /** @param {QueueConsumer[]} oldQueueConsumers @param {QueueConsumer[]} newQueueConsumers */ function computeQueueConsumerPlan(oldQueueConsumers, newQueueConsumers) { const newQueueSet = new Set(newQueueConsumers.map((c) => c.queue)); @@ -322,7 +297,7 @@ async function assertOutgoingDependenciesPresent(iso, targetLabel, outgoingRefs) bundleKey(ref.targetNs, ref.targetWorker, ref.targetVersion), "__meta__" ); - if (!targetMetaRaw) { + if (targetMetaRaw == null) { throw new RoutingError( 409, "service_binding_dependency_missing", @@ -369,7 +344,7 @@ async function assertQueueConsumerOwnership( async function assertDeclaredHosts(iso, ns, newRoutes) { const newHosts = [...new Set(newRoutes.map((r) => r.host))]; if (!newHosts.length) return; - const memberFlags = await iso.sMIsMember(`hosts:${ns}`, ...newHosts); + const memberFlags = await iso.sMIsMember(hostsKey(ns), ...newHosts); const missingIdx = memberFlags.findIndex((flag) => !flag); if (missingIdx !== -1) { const host = newHosts[missingIdx]; @@ -454,10 +429,13 @@ async function assertPlatformAsAvailable(iso, ns, workerName, newExports) { for (let i = 0; i < routeEntries.length; i += 1) { const [otherWorker, otherVersion] = routeEntries[i]; const rawMeta = rawMetas[i]; - const otherMeta = typeof rawMeta === "string" - ? parseBundleMeta(otherNs, otherWorker, /** @type {string} */ (otherVersion), rawMeta) - : null; - if (!otherMeta || !Array.isArray(otherMeta.exports)) continue; + const otherMeta = routingBundleMeta( + otherNs, + otherWorker, + /** @type {string} */ (otherVersion), + rawMeta + ); + if (!Array.isArray(otherMeta.exports)) continue; const heldAs = new Set(); for (const e of otherMeta.exports) if (e.as) heldAs.add(e.as); for (const e of newExports) { @@ -488,10 +466,10 @@ async function readPromoteBundleInputs(iso, ns, workerName, newVersion) { // Every retry re-reads under WATCH: a concurrent single-version // delete (DEL bundle) surfaces here as a 404. const metaRaw = await iso.hGet(bundleKey(ns, workerName, newVersion), "__meta__"); - if (!metaRaw) { + if (metaRaw == null) { throw new RoutingError(404, "version_not_found", `Version ${newVersion} not found for ${ns}/${workerName}`); } - const meta = parseBundleMeta(ns, workerName, newVersion, metaRaw); + const meta = routingBundleMeta(ns, workerName, newVersion, metaRaw); const newRoutes = Array.isArray(meta.routes) ? meta.routes : []; const newCrons = Array.isArray(meta.crons) ? meta.crons : []; @@ -614,23 +592,22 @@ function stagePromoteWithRoutes(multi, ns, workerName, newVersion, inputs, obser stageVersionFlip(multi, ns, workerName, newVersion, inputs.newRoutes, observed.affectedHosts); // In the same MULTI as the route flip — closes the half-state where // routes: is set but the gateway's knownNs gate still 404s. - multi.sAdd("namespaces", ns); - if (plan.nsHostsAdd.length) multi.sAdd(`ns-hosts:${ns}`, plan.nsHostsAdd); - if (plan.nsHostsRem.length) multi.sRem(`ns-hosts:${ns}`, plan.nsHostsRem); + multi.sAdd(NAMESPACES_KEY, ns); + if (plan.nsHostsAdd.length) multi.sAdd(nsHostsKey(ns), plan.nsHostsAdd); + if (plan.nsHostsRem.length) multi.sRem(nsHostsKey(ns), plan.nsHostsRem); stageD1ReferrerAdds(multi, { ns, worker: workerName, version: newVersion, refs: inputs.d1Refs, databaseIdFor: (ref) => String(ref.databaseId), }); - stageCronPlan( - multi, + stageCronProjection(multi, { ns, - workerName, - newVersion, - plan.cronKey, - plan.cronHash, - inputs.newCrons, - plan.cronPlan - ); + worker: workerName, + version: newVersion, + cronKey: plan.cronKey, + existingHash: plan.cronHash, + crons: inputs.newCrons, + plan: plan.cronPlan, + }); stageQueueConsumerPlan(multi, ns, workerName, newVersion, plan.queuePlan); } @@ -655,7 +632,7 @@ export async function promoteWithRoutes(redis, ns, workerName, newVersion, optio }, async (iso) => { await iso.watch( routesKey(ns), - `hosts:${ns}`, + hostsKey(ns), cronKey, deleteLockKey(ns, workerName), bundleKey(ns, workerName, newVersion), @@ -697,7 +674,7 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) ); } - const newNum = await redis.incr(`worker:${ns}:${workerName}:next_version`); + const newNum = await redis.incr(nextVersionKey(ns, workerName)); const newVersion = formatVersion(newNum); return await runOptimistic(redis, { @@ -713,7 +690,7 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) }, async (iso) => { await iso.watch( routesKey(ns), - `hosts:${ns}`, + hostsKey(ns), deleteLockKey(ns, workerName), ); @@ -739,14 +716,15 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) const dstKey = bundleKey(ns, workerName, newVersion); await iso.watch(srcKey, dstKey); - const srcMeta = await readMeta(iso, ns, workerName, currentVersion); - if (!srcMeta) { + const srcMetaRaw = await iso.hGet(srcKey, "__meta__"); + if (srcMetaRaw == null) { throw new RoutingError( 500, "bundle_copy_failed", `COPY ${srcKey} → ${dstKey} failed (source missing?)` ); } + const srcMeta = routingBundleMeta(ns, workerName, currentVersion, srcMetaRaw); const routes = srcMeta && Array.isArray(srcMeta.routes) ? srcMeta.routes : []; const queueConsumers = srcMeta && Array.isArray(srcMeta.queueConsumers) ? srcMeta.queueConsumers : []; @@ -789,7 +767,7 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) multi.copy(srcKey, dstKey, { REPLACE: true }); stageVersionFlip(multi, ns, workerName, newVersion, routes, affectedHosts); // Idempotent — also heals namespaces drift (manual SREM, recovery scripts). - multi.sAdd("namespaces", ns); + multi.sAdd(NAMESPACES_KEY, ns); // Maintain the indexes so a later hard-delete can collect this // bumped version's referrers. @@ -803,10 +781,13 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) }); if (cronMetaParsed) { - stageCronWorkerIndexed(multi, ns, workerName); - multi.hSet(cronKey, "__meta__", JSON.stringify({ - version: newVersion, seq: cronSeqFromMeta(cronMetaParsed, logContext), - })); + stageCronProjectionMeta(multi, { + ns, + worker: workerName, + version: newVersion, + cronKey, + seq: cronSeqFromMeta(cronMetaParsed, logContext), + }); } for (const c of queueConsumers) { stageQueueConsumerProjection(multi, ns, workerName, newVersion, c); @@ -853,8 +834,8 @@ export async function reconcileHosts(redis, ns, body, platformDomain) { ); }, }, async (iso) => { - await iso.watch(`hosts:${ns}`); - const current = await iso.sMembers(`hosts:${ns}`); + await iso.watch(hostsKey(ns)); + const current = await iso.sMembers(hostsKey(ns)); const currentSet = new Set(current); const toAdd = [...bodySet.difference(currentSet)]; const toRemove = [...currentSet.difference(bodySet)]; @@ -869,7 +850,7 @@ export async function reconcileHosts(redis, ns, body, platformDomain) { // Reverse-index fast path; HGETALL only to enrich the 409 message. const reverseFlags = toRemove.length - ? await iso.sMIsMember(`ns-hosts:${ns}`, ...toRemove) + ? await iso.sMIsMember(nsHostsKey(ns), ...toRemove) : []; for (let i = 0; i < toRemove.length; i++) { const h = toRemove[i]; diff --git a/control/s3.js b/control/s3.js index bed6fdb..1e8822f 100644 --- a/control/s3.js +++ b/control/s3.js @@ -3,6 +3,7 @@ import { SigV4Client } from "@wdl-dev/aws-sigv4"; import { encodeS3KeyPath } from "runtime-r2-utils"; +import { S3_TRANSIENT_RETRIES } from "shared-s3-retry"; const TYPE_BY_EXT = { ".html": "text/html; charset=utf-8", @@ -27,8 +28,6 @@ const TYPE_BY_EXT = { ".wasm": "application/wasm", ".map": "application/json", }; -const S3_TRANSIENT_RETRIES = 10; - /** * @typedef {{ client: SigV4Client, endpoint: string, bucket: string }} S3Client */ diff --git a/control/shared.js b/control/shared.js index 3181bdb..9f0ccde 100644 --- a/control/shared.js +++ b/control/shared.js @@ -1,18 +1,24 @@ // Shared state + helpers for control. Handlers import from here; the // dispatcher in index.js calls ensureInit(env) before route dispatch. -import { RedisClient, WatchError, redisDbFromEnv } from "shared-redis"; +import { RedisClient, redisDbFromEnv } from "shared-redis"; import { makeS3Client } from "control-s3"; import { makeR2AdminClient } from "control-r2"; import { extractToken } from "shared-auth-token"; import { validatePrincipalShape } from "shared-auth-roles"; -import { BodyTooLargeError, readBoundedText } from "shared-bounded-body"; import { queueStreamKey } from "shared-queue-keys"; import { internalErrorResponse, jsonError, jsonResponse, sanitizeJsonErrorDetails } from "shared-respond"; import { withInternalAuth } from "shared-internal-auth"; import { errorMessage } from "shared-errors"; import { randomHex } from "shared-random-id"; import { deleteLockKey } from "control-lib"; +import { + DECLARED_HOSTS_KEY, + HOST_DECLARATIONS_SCAN_PATTERN, + HOSTS_SCAN_PATTERN, + hostDeclarationsKey, + namespaceFromHostsKey, +} from "shared-version"; import { S3_CLEANUP_QUEUE_NAME, S3_CLEANUP_TASK_ID_PREFIX, @@ -24,15 +30,29 @@ import { formatError, recordRedisCommand, } from "shared-observability"; +import { + WORKFLOWS_INTERNAL_TIMEOUT_MS, + createPostWorkflowsInternal, +} from "control-workflows-client"; +import { + ControlAbort, + codedErrorResponse, + controlAbortResponse, +} from "control-errors"; +import { runOptimistic, withOptimisticRetries } from "control-optimistic"; +import { + DEFAULT_JSON_BODY_MAX_BYTES, + readJsonBody, +} from "control-json-body"; +import { + acquireTokenLock, + createTokenLock, + releaseTokenLock, +} from "shared-redis-lock"; export const ROUTES_CHANNEL = "routes:invalidate"; export const ROUTES_FLUSH_CHANNEL = "routes:flush"; export const PATTERNS_CHANNEL = "patterns:invalidate"; -export const DEFAULT_JSON_BODY_MAX_BYTES = 1024 * 1024; -export const DECLARED_HOSTS_KEY = "declared-hosts"; -export const HOST_DECLARATIONS_PREFIX = "host-declarations:"; -const HOSTS_PREFIX = "hosts:"; -const DO_ALARM_CLEANUP_TIMEOUT_MS = 5_000; /** * @typedef {import("shared-observability").RedisCommandEvent} RedisCommandEvent @@ -69,6 +89,9 @@ let s3Initialized = false; let r2Initialized = false; export { formatError, internalErrorResponse, jsonError, jsonResponse }; +export { ControlAbort, codedErrorResponse, controlAbortResponse }; +export { runOptimistic, withOptimisticRetries }; +export { DEFAULT_JSON_BODY_MAX_BYTES, readJsonBody }; /** @param {unknown} value @returns {value is Record} */ function isRecord(value) { @@ -101,30 +124,6 @@ export function prefixedId(prefix, bytes = 16) { return `${prefix}${randomHex(bytes)}`; } -/** - * @template T, S - * @param {{ session: (fn: (session: S) => Promise) => Promise }} redis - * @param {{ attempts?: number, onExhausted: () => unknown | Promise, onWatchError?: (err: unknown, attempt: number) => void, shouldRetryResult?: (result: T, attempt: number) => boolean }} options - * @param {(session: S, attempt: number) => Promise} fn - * @returns {Promise} - */ -export async function runOptimistic(redis, { attempts = 5, onExhausted, onWatchError, shouldRetryResult }, fn) { - for (let attempt = 0; attempt < attempts; attempt += 1) { - try { - const result = await redis.session((session) => fn(session, attempt)); - if (shouldRetryResult?.(result, attempt)) continue; - return result; - } catch (err) { - if (err instanceof WatchError) { - onWatchError?.(err, attempt); - continue; - } - throw err; - } - } - return /** @type {T} */ (await onExhausted()); -} - /** @param {Record} env */ export function ensureInit(env) { state.env = env; @@ -199,11 +198,6 @@ export function getControlR2() { return state.r2; } -/** @returns {WorkflowBackend | null} */ -export function getControlWorkflows() { - return state.workflows; -} - export function controlInternalJsonHeaders() { if (!state.env) { throw new Error("control shared state has not been initialized"); @@ -216,107 +210,11 @@ function onRedisCommand(event) { recordRedisCommand({ metrics: null, log: state.log, service: state.service, event }); } -// Intentionally local, despite looking like routing/deploy/auth errors. Each -// layer owns a slightly different machine-readable wire shape and catch boundary. -export class ControlAbort extends Error { - /** - * @param {number} status - * @param {string} code - * @param {{ message?: string, [key: string]: unknown }} [details] - */ - constructor(status, code, details = {}) { - super(details?.message || code); - this.status = status; - this.code = code; - this.details = details; - } -} - -// Domain errors still flow through jsonError() so details cannot shadow the -// top-level wire contract. -/** - * @param {unknown} err - * @param {string} [fallbackCode] - * @param {Record} [extraDetails] - */ -export function codedErrorResponse(err, fallbackCode = "internal_error", extraDetails = {}) { - const record = isRecord(err) ? err : {}; - const details = isRecord(record.details) ? record.details : {}; - const status = typeof record.status === "number" ? record.status : 500; - const code = typeof record.code === "string" && record.code ? record.code : fallbackCode; - const recordMessage = typeof record.message === "string" && record.message ? record.message : undefined; - const message = err instanceof ControlAbort - ? (typeof err.details.message === "string" && err.details.message) || err.message || code - : recordMessage || (typeof details.message === "string" && details.message) || code; - return jsonError( - status, - code, - message, - { ...details, ...extraDetails }, - ); -} - -// ControlAbort always carries code; the separate wrapper keeps abort call sites -// explicit while ordinary coded errors still get a fallback code. -/** - * @param {ControlAbort} err - * @param {Record} [extraDetails] - */ -export function controlAbortResponse(err, extraDetails = {}) { - return codedErrorResponse(err, err.code, extraDetails); -} - -/** - * @param {Request} request - * @param {{ requireObject?: boolean, allowEmpty?: boolean, maxBytes?: number }} [opts] - */ -export async function readJsonBody( - request, - { requireObject = false, allowEmpty = false, maxBytes = DEFAULT_JSON_BODY_MAX_BYTES } = {}, -) { - let body; - try { - const limited = await readTextBody(request, maxBytes); - if ("response" in limited) return { response: limited.response }; - const text = limited.text; - if (text === "") { - if (!allowEmpty) { - return { - response: jsonError(400, "invalid_json", "Body must be valid JSON"), - }; - } - body = {}; - } else { - body = JSON.parse(text); - } - } catch { - return { - response: jsonError(400, "invalid_json", "Body must be valid JSON"), - }; - } - if (requireObject && (!body || typeof body !== "object" || Array.isArray(body))) { - return { - response: jsonError(400, "invalid_json_object", "Body must be a JSON object"), - }; - } - return { body }; -} - -/** - * @param {Request} request - * @param {number} maxBytes - * @returns {Promise<{ text: string } | { response: Response }>} - */ -async function readTextBody(request, maxBytes) { - try { - return { text: await readBoundedText(request, maxBytes) }; - } catch (err) { - if (err instanceof BodyTooLargeError) { - return { response: jsonError(413, "request_body_too_large", `Body must be at most ${maxBytes} bytes`) }; - } - throw err; - } -} +export const postWorkflowsInternal = createPostWorkflowsInternal({ + getWorkflows: () => state.workflows, + headers: controlInternalJsonHeaders, + getLog: () => state.log, +}); // AuthPolicyError → HTTP body shape. 4xx messages are user-actionable; // 5xx messages stay generic so internal auth diagnostics do not leak. @@ -426,11 +324,6 @@ async function scanKeys(redis, pattern) { return keys; } -/** @param {string} key */ -function namespaceFromHostsKey(key) { - return key.startsWith(HOSTS_PREFIX) ? key.slice(HOSTS_PREFIX.length) : ""; -} - /** * Rebuild the global declared-host gate from namespace-owned `hosts:` sets. * This is an ops reload repair path, not the ordinary writer; `/hosts` reconcile @@ -439,8 +332,8 @@ function namespaceFromHostsKey(key) { * @param {RedisClient} [redis] */ export async function rebuildDeclaredHostIndexes(redis = requireControlRedis()) { - const hostKeys = await scanKeys(redis, `${HOSTS_PREFIX}*`); - const oldDeclarationKeys = await scanKeys(redis, `${HOST_DECLARATIONS_PREFIX}*`); + const hostKeys = await scanKeys(redis, HOSTS_SCAN_PATTERN); + const oldDeclarationKeys = await scanKeys(redis, HOST_DECLARATIONS_SCAN_PATTERN); /** @type {Map>} */ const declarationsByHost = new Map(); for (const key of hostKeys) { @@ -459,7 +352,7 @@ export async function rebuildDeclaredHostIndexes(redis = requireControlRedis()) multi.del(DECLARED_HOSTS_KEY, ...oldDeclarationKeys); for (const [host, namespaces] of declarationsByHost) { multi.sAdd(DECLARED_HOSTS_KEY, host); - multi.sAdd(`${HOST_DECLARATIONS_PREFIX}${host}`, [...namespaces]); + multi.sAdd(hostDeclarationsKey(host), [...namespaces]); } await multi.exec(); }); @@ -530,12 +423,8 @@ export function buildS3CleanupTaskId() { return `${S3_CLEANUP_TASK_ID_PREFIX}${crypto.randomUUID()}`; } -export function generateLockToken() { - return crypto.randomUUID(); -} - -// TTL only matters when a handler crashes mid-flow (normal path releases -// in a finally block); 30s covers the Redis critical section and no more. +// Normal paths release this advisory lock in finally; the TTL only bounds +// leaks after a crash. The token fences release, not the protected operation. const DELETE_LOCK_TTL_SECONDS = 30; /** * @param {RedisClient} redis @@ -543,10 +432,11 @@ const DELETE_LOCK_TTL_SECONDS = 30; * @param {string} worker */ export async function acquireDeleteLock(redis, ns, worker) { - const token = generateLockToken(); const key = deleteLockKey(ns, worker); - const reply = await redis.set(key, token, { nx: true, ttl: DELETE_LOCK_TTL_SECONDS }); - return reply === "OK" ? token : null; + const lock = createTokenLock(key); + return await acquireTokenLock(redis, lock, { ttlSeconds: DELETE_LOCK_TTL_SECONDS }) + ? lock.token + : null; } // Compare-and-delete so we don't free a token a TTL-expired-then-reacquired @@ -561,15 +451,13 @@ export async function acquireDeleteLock(redis, ns, worker) { export async function releaseDeleteLock(redis, ns, worker, token, requestId) { if (!token) return; const key = deleteLockKey(ns, worker); - try { - await redis.delIfEq(key, token); - } catch (err) { - requireControlLog()("warn", "delete_lock_release_failed", { + await releaseTokenLock(redis, { key, token }, { + onError: (err) => requireControlLog()("warn", "delete_lock_release_failed", { request_id: requestId, namespace: ns, worker, ...formatError(err), - }); - } + }), + }); } // Standalone intent write for deploy aborts, where the cleanup follows a @@ -628,51 +516,33 @@ export async function recordCleanupIntentOrWarn({ * @param {{ ns: string, worker: string, version?: string, allowCleanup?: boolean }} args */ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefined, allowCleanup = false }) { - if (!state.workflows || typeof state.workflows.fetch !== "function") { - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow lifecycle check is unavailable", - namespace: ns, - worker, - ...(version ? { version } : {}), - }); - } - let response; - let body; - try { - response = await state.workflows.fetch( - "http://workflows/internal/workflows/lifecycle/check-delete", - { - method: "POST", - headers: controlInternalJsonHeaders(), - body: JSON.stringify({ - ns, - worker, - ...(version ? { version } : {}), - ...(allowCleanup ? { allowCleanup: true } : {}), - }), - }, - ); - body = await response.json().catch(() => null); - } catch (err) { - state.log?.("error", "workflow_lifecycle_check_failed", { - namespace: ns, - worker, - ...(version ? { version } : {}), - error_message: errMessage(err), - }); - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow lifecycle check failed", - namespace: ns, + const context = { + namespace: ns, + worker, + ...(version ? { version } : {}), + }; + const { response, body } = await postWorkflowsInternal({ + endpoint: "workflows/lifecycle/check-delete", + body: { + ns, worker, ...(version ? { version } : {}), - }); - } + ...(allowCleanup ? { allowCleanup: true } : {}), + }, + logEvent: "workflow_lifecycle_check_failed", + logFields: context, + errorDetails: context, + unavailableMessage: "Workflow lifecycle check is unavailable", + requestFailedMessage: "Workflow lifecycle check failed", + // The lifecycle scan is unbounded by namespace size. Preserve its + // pre-consolidation behavior instead of imposing the ordinary short + // control-to-workflows request timeout. + timeoutMs: null, + }); if (!response.ok) { throw new ControlAbort(503, "workflow_internal_dispatch_failed", { message: "Workflow lifecycle check failed", - namespace: ns, - worker, - ...(version ? { version } : {}), + ...context, upstream_status: response.status, upstream_error: isRecord(body) && typeof body.error === "string" ? body.error : null, }); @@ -680,9 +550,7 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi if (!isRecord(body) || typeof body.allowed !== "boolean") { throw new ControlAbort(503, "workflow_internal_dispatch_failed", { message: "Workflow lifecycle check returned an invalid response", - namespace: ns, - worker, - ...(version ? { version } : {}), + ...context, }); } if (body.allowed !== true) { @@ -690,9 +558,7 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi message: version ? `${ns}/${worker}/${version} has active workflow instances` : `${ns}/${worker} has active workflow instances`, - namespace: ns, - worker, - ...(version ? { version } : {}), + ...context, count: Number.isFinite(body.count) ? body.count : 0, blockers: Array.isArray(body.blockers) ? body.blockers : [], }); @@ -703,43 +569,22 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi * @param {{ ns: string, worker: string, doStorageId: string }} args */ export async function cleanupDoAlarmsForWorker({ ns, worker, doStorageId }) { - if (!state.workflows || typeof state.workflows.fetch !== "function") { - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow DO alarm cleanup is unavailable", - namespace: ns, - worker, - }); - } - let response; - let body; - try { - response = await state.workflows.fetch( - "http://workflows/internal/workflows/do-alarms/cleanup-worker", - { - method: "POST", - headers: controlInternalJsonHeaders(), - body: JSON.stringify({ ns, worker, doStorageId }), - signal: AbortSignal.timeout(DO_ALARM_CLEANUP_TIMEOUT_MS), - }, - ); - body = await response.json().catch(() => null); - } catch (err) { - state.log?.("error", "workflow_do_alarm_cleanup_failed", { - namespace: ns, - worker, - error_message: errMessage(err), - }); - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow DO alarm cleanup failed", - namespace: ns, - worker, - }); - } + const context = { namespace: ns, worker }; + const { response, body } = await postWorkflowsInternal({ + endpoint: "workflows/do-alarms/cleanup-worker", + body: { ns, worker, doStorageId }, + logEvent: "workflow_do_alarm_cleanup_failed", + logFields: context, + errorDetails: context, + unavailableMessage: "Workflow DO alarm cleanup is unavailable", + requestFailedMessage: "Workflow DO alarm cleanup failed", + // This endpoint was already bounded before transport consolidation. + timeoutMs: WORKFLOWS_INTERNAL_TIMEOUT_MS, + }); if (!response.ok || !isRecord(body) || body.ok !== true) { throw new ControlAbort(503, "workflow_internal_dispatch_failed", { message: "Workflow DO alarm cleanup failed", - namespace: ns, - worker, + ...context, upstream_status: response.status, upstream_error: isRecord(body) && typeof body.error === "string" ? body.error : null, }); diff --git a/control/workflows-client.js b/control/workflows-client.js new file mode 100644 index 0000000..b4dcaf8 --- /dev/null +++ b/control/workflows-client.js @@ -0,0 +1,108 @@ +import { errorMessage } from "shared-errors"; +import { ControlAbort } from "control-errors"; + +export const WORKFLOWS_INTERNAL_TIMEOUT_MS = 5_000; + +/** + * @typedef {{ fetch: typeof fetch }} WorkflowBackend + * @typedef {(level: string, event: string, fields?: Record) => void} WorkflowClientLogger + * @typedef {"unavailable" | "request_failed"} WorkflowTransportFailure + */ + +/** + * @param {{ + * workflows: WorkflowBackend | null | undefined, + * headers: () => HeadersInit, + * endpoint: string, + * body: unknown, + * log?: WorkflowClientLogger | null, + * logEvent: string, + * logFields?: Record, + * timeoutMs: number | null, + * makeError: (failure: WorkflowTransportFailure) => Error, + * }} args + * @returns {Promise<{ response: Response, body: unknown }>} + */ +export async function postWorkflowsInternalRequest({ + workflows, + headers, + endpoint, + body, + log = null, + logEvent, + logFields = {}, + timeoutMs, + makeError, +}) { + if (!workflows || typeof workflows.fetch !== "function") { + throw makeError("unavailable"); + } + + try { + const response = await workflows.fetch(`http://workflows/internal/${endpoint}`, { + method: "POST", + headers: headers(), + body: JSON.stringify(body), + ...(timeoutMs === null ? {} : { signal: AbortSignal.timeout(timeoutMs) }), + }); + return { + response, + body: await response.json().catch(() => null), + }; + } catch (err) { + log?.("error", logEvent, { + ...logFields, + error_message: errorMessage(err), + }); + throw makeError("request_failed"); + } +} + +/** + * @param {{ + * getWorkflows: () => WorkflowBackend | null | undefined, + * headers: () => HeadersInit, + * getLog?: () => WorkflowClientLogger | null | undefined, + * }} dependencies + */ +export function createPostWorkflowsInternal({ getWorkflows, headers, getLog = () => null }) { + /** + * @param {{ + * endpoint: string, + * body: unknown, + * logEvent: string, + * logFields?: Record, + * errorDetails?: Record, + * timeoutMs: number | null, + * unavailableMessage?: string, + * requestFailedMessage?: string, + * }} args + */ + return async function postWorkflowsInternal({ + endpoint, + body, + logEvent, + logFields = {}, + errorDetails = {}, + timeoutMs, + unavailableMessage = "Workflow backend is unavailable", + requestFailedMessage = "Workflow backend request failed", + }) { + return await postWorkflowsInternalRequest({ + workflows: getWorkflows(), + headers, + endpoint, + body, + log: getLog(), + logEvent, + logFields, + timeoutMs, + makeError: (failure) => new ControlAbort(503, "workflow_internal_dispatch_failed", { + ...errorDetails, + message: failure === "unavailable" + ? unavailableMessage + : requestFailedMessage, + }), + }); + }; +} diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md index 1c68276..121bbe0 100644 --- a/deploy/kubernetes/README.md +++ b/deploy/kubernetes/README.md @@ -13,13 +13,14 @@ provider's native tooling. ## Layout - `base/` defines reusable Deployments, StatefulSets, Services, and PVCs. +- `components/metadata-guard/` owns the shared cloud-metadata egress policy. - `overlays/local/` pins local images, local development secrets, a namespace, and local storage defaults. - `overlays/local-ingress/` adds nginx Ingress resources for local hostnames. -- `overlays/local-metadata-guard/` applies the local overlay plus a minimal - metadata egress guard. +- `overlays/local-metadata-guard/` applies the local overlay plus the shared + metadata-guard component. - `overlays/local-ingress-metadata-guard/` applies the local ingress overlay - plus the same metadata egress guard. + plus the same component. - `storage/local-nfs/` contains Helm values for a local ReadWriteMany provisioner that can simulate provider RWX storage on a development cluster. diff --git a/deploy/kubernetes/README.zh.md b/deploy/kubernetes/README.zh.md index e836580..e1ff896 100644 --- a/deploy/kubernetes/README.zh.md +++ b/deploy/kubernetes/README.zh.md @@ -7,10 +7,11 @@ ## 目录结构 - `base/` 定义可复用的 Deployment、StatefulSet、Service 和 PVC。 +- `components/metadata-guard/` 统一维护云 metadata egress policy。 - `overlays/local/` 固定本地镜像、本地开发 secret、namespace 和本地存储默认值。 - `overlays/local-ingress/` 为本地主机名增加 nginx Ingress 资源。 -- `overlays/local-metadata-guard/` 应用本地 overlay，并额外加入一个最小 metadata egress guard。 -- `overlays/local-ingress-metadata-guard/` 应用本地 ingress overlay，并额外加入同一条 metadata egress guard。 +- `overlays/local-metadata-guard/` 应用本地 overlay，并额外加入共享的 metadata-guard component。 +- `overlays/local-ingress-metadata-guard/` 应用本地 ingress overlay，并额外加入同一个 component。 - `storage/local-nfs/` 包含一个本地 ReadWriteMany provisioner 的 Helm values，可在开发集群中模拟云厂商 RWX 存储。 base 使用 production-style 镜像契约：workerd 服务默认使用 `docker.io/getwdl/wdl-workerd:latest`，Rust sidecar 和 Rust service 默认使用 `docker.io/getwdl/wdl-rust:latest`。本地 overlay 会把这些 public image name 映射到 Compose 集成测试本地构建的 `wdl-workerd:dev` 和 `wdl-rust:dev` tag。 diff --git a/deploy/kubernetes/base/network-policy.yaml b/deploy/kubernetes/base/network-policy.yaml index 8cf4981..ea002c9 100644 --- a/deploy/kubernetes/base/network-policy.yaml +++ b/deploy/kubernetes/base/network-policy.yaml @@ -50,10 +50,6 @@ spec: ports: - port: 8081 - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: wdl - app.kubernetes.io/component: system-runtime - podSelector: matchLabels: app.kubernetes.io/name: wdl diff --git a/deploy/kubernetes/components/metadata-guard/kustomization.yaml b/deploy/kubernetes/components/metadata-guard/kustomization.yaml new file mode 100644 index 0000000..7fff9b7 --- /dev/null +++ b/deploy/kubernetes/components/metadata-guard/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - metadata-egress-policy.yaml diff --git a/deploy/kubernetes/overlays/local-ingress-metadata-guard/metadata-egress-policy.yaml b/deploy/kubernetes/components/metadata-guard/metadata-egress-policy.yaml similarity index 100% rename from deploy/kubernetes/overlays/local-ingress-metadata-guard/metadata-egress-policy.yaml rename to deploy/kubernetes/components/metadata-guard/metadata-egress-policy.yaml diff --git a/deploy/kubernetes/overlays/local-ingress-metadata-guard/kustomization.yaml b/deploy/kubernetes/overlays/local-ingress-metadata-guard/kustomization.yaml index 7efb7da..5830040 100644 --- a/deploy/kubernetes/overlays/local-ingress-metadata-guard/kustomization.yaml +++ b/deploy/kubernetes/overlays/local-ingress-metadata-guard/kustomization.yaml @@ -3,4 +3,5 @@ kind: Kustomization namespace: wdl-local resources: - ../local-ingress - - metadata-egress-policy.yaml +components: + - ../../components/metadata-guard diff --git a/deploy/kubernetes/overlays/local-metadata-guard/kustomization.yaml b/deploy/kubernetes/overlays/local-metadata-guard/kustomization.yaml index ab73d19..a8ee06e 100644 --- a/deploy/kubernetes/overlays/local-metadata-guard/kustomization.yaml +++ b/deploy/kubernetes/overlays/local-metadata-guard/kustomization.yaml @@ -3,4 +3,5 @@ kind: Kustomization namespace: wdl-local resources: - ../local - - metadata-egress-policy.yaml +components: + - ../../components/metadata-guard diff --git a/deploy/kubernetes/overlays/local-metadata-guard/metadata-egress-policy.yaml b/deploy/kubernetes/overlays/local-metadata-guard/metadata-egress-policy.yaml deleted file mode 100644 index e28b744..0000000 --- a/deploy/kubernetes/overlays/local-metadata-guard/metadata-egress-policy.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: block-cloud-metadata-egress - labels: - app.kubernetes.io/name: wdl - app.kubernetes.io/component: network-policy -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: wdl - policyTypes: - - Egress - egress: - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 169.254.169.254/32 - - 169.254.170.2/32 diff --git a/do-runtime/actor.js b/do-runtime/actor.js index 9a9f8a3..f00762e 100644 --- a/do-runtime/actor.js +++ b/do-runtime/actor.js @@ -5,6 +5,7 @@ import { buildAlarmRequest, buildFacetName, buildForwardRequest, + DO_OWNERSHIP_CODE, doErrorResponse, DoRuntimeError, normalizeDoConnectRequest, @@ -176,7 +177,7 @@ export class WdlDoHostActor extends DurableObject { */ async dispatchWithFence(invoke, run) { if (!beginInFlightDispatch()) { - throw new DoRuntimeError(503, "task_draining", "DO task is draining"); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); } try { const { owner, leaseRemainingMs } = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); @@ -243,9 +244,9 @@ export class WdlDoHostActor extends DurableObject { if (!schedule(leaseRemainingMs)) { if (scheduleFailureReason === "lease_guard") { - throw new DoRuntimeError(503, "owner_lease_too_short", `DO scope ${owner.ownerKey} owner lease has insufficient remaining budget`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_LEASE_TOO_SHORT, `DO scope ${owner.ownerKey} owner lease has insufficient remaining budget`); } - throw new DoRuntimeError(503, "owner_lease_expired", `DO scope ${owner.ownerKey} owner lease has expired`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_LEASE_EXPIRED, `DO scope ${owner.ownerKey} owner lease has expired`); } try { return await run(); diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index c91cac1..d8e749c 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -108,7 +108,10 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "hex.js", esModule = embed "../shared/hex.js"), (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), + (name = "shared-base64", esModule = embed "../shared/base64.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), + (name = "shared-s3-retry", esModule = embed "../shared/s3-retry.js"), + (name = "respond.js", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-task-identity", esModule = embed "../shared/task-identity.js"), diff --git a/do-runtime/owner-client.js b/do-runtime/owner-client.js index 9dba006..b000b28 100644 --- a/do-runtime/owner-client.js +++ b/do-runtime/owner-client.js @@ -1,5 +1,6 @@ import { DO_INVOKE_CONTENT_TYPE, + DO_OWNERSHIP_CODE, DoRuntimeError, encodeDoInvokeRequest, } from "do-runtime-protocol"; @@ -68,14 +69,14 @@ export async function forwardToOwner(invoke, env, owner, requestId = null, hopCo path: pathname, }), missingEndpointError: () => - new DoRuntimeError(503, "owner_endpoint_missing", `DO scope ${owner.ownerKey} owner has no endpoint`), + new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_ENDPOINT_MISSING, `DO scope ${owner.ownerKey} owner has no endpoint`), hopExhaustedError: () => new DoRuntimeError( 503, - "forward_hop_exhausted", + DO_OWNERSHIP_CODE.FORWARD_HOP_EXHAUSTED, `DO scope ${owner.ownerKey} exceeded the maximum forward depth for ${pathname}` ), - unavailableError: () => new DoRuntimeError(503, "owner_unavailable", "DO owner is unavailable"), + unavailableError: () => new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner is unavailable"), }); } @@ -118,13 +119,13 @@ export async function forwardConnectToOwner(request, invoke, env, owner, request owner_endpoint: owner.endpoint, }), missingEndpointError: () => - new DoRuntimeError(503, "owner_endpoint_missing", `DO scope ${owner.ownerKey} owner has no endpoint`), + new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_ENDPOINT_MISSING, `DO scope ${owner.ownerKey} owner has no endpoint`), hopExhaustedError: () => new DoRuntimeError( 503, - "forward_hop_exhausted", + DO_OWNERSHIP_CODE.FORWARD_HOP_EXHAUSTED, `DO scope ${owner.ownerKey} exceeded the maximum forward depth` ), - unavailableError: () => new DoRuntimeError(503, "owner_unavailable", "DO owner is unavailable"), + unavailableError: () => new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner is unavailable"), }); } diff --git a/do-runtime/owner-registry.js b/do-runtime/owner-registry.js index 64fbd5c..bab7aee 100644 --- a/do-runtime/owner-registry.js +++ b/do-runtime/owner-registry.js @@ -4,6 +4,7 @@ import { } from "shared-redis"; import { buildOwnerKey, + DO_OWNERSHIP_CODE, DoRuntimeError, } from "do-runtime-protocol"; import { @@ -283,7 +284,7 @@ export async function resolveDoOwner(env, invoke) { if (!await ownerStoragePointerCurrent(session, current)) { await session.unwatch(); forgetOwnedScope(ownerKey); - throw new DoRuntimeError(503, "stale_owner_storage", `DO scope ${ownerKey} no longer matches active worker storage`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, `DO scope ${ownerKey} no longer matches active worker storage`); } if (current.taskId !== localTask.taskId) { await session.unwatch(); @@ -292,7 +293,7 @@ export async function resolveDoOwner(env, invoke) { return current; } if (isDraining()) { - throw new DoRuntimeError(503, "task_draining", "DO task is draining"); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); } const alreadyLocal = ownedScopes.has(ownerKey); const counter = alreadyLocal ? current.generation : await currentOwnerGenerationCounter(session, generationKey); @@ -315,7 +316,7 @@ export async function resolveDoOwner(env, invoke) { } if (isDraining()) { - throw new DoRuntimeError(503, "task_draining", "DO task is draining"); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); } const generation = await nextOwnerGeneration(session, generationKey, current?.generation); const owner = ownerRecordFor(env, invoke, localTask, generation, nowMs); @@ -331,7 +332,7 @@ export async function resolveDoOwner(env, invoke) { generation, }); return owner; - }), "owner_claim_raced", `failed to resolve DO owner for ${ownerKey}`); + }), DO_OWNERSHIP_CODE.OWNER_CLAIM_RACED, `failed to resolve DO owner for ${ownerKey}`); } /** @@ -342,21 +343,21 @@ export async function resolveDoOwner(env, invoke) { */ export async function assertCurrentOwnerWithLeaseBudget(env, owner, options = {}) { if (!owner?.ownerKey || !owner.taskId || owner.generation == null) { - throw new DoRuntimeError(503, "owner_fence_missing", "DO host request is missing an owner generation fence"); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, "DO host request is missing an owner generation fence"); } const client = redisClient(env); const { owner: current, nowMs } = await readOwnerWithTimeFromClient(client, owner.ownerKey); if (!current || !ownerFenceMatches(current, owner)) { forgetOwnedScope(owner.ownerKey); - throw new DoRuntimeError(503, "stale_owner_generation", `DO scope ${owner.ownerKey} owner generation is stale`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.STALE_OWNER_GENERATION, `DO scope ${owner.ownerKey} owner generation is stale`); } if (ownerLeaseExpired(current, nowMs)) { forgetOwnedScope(owner.ownerKey); - throw new DoRuntimeError(503, "owner_lease_expired", `DO scope ${owner.ownerKey} owner lease has expired`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_LEASE_EXPIRED, `DO scope ${owner.ownerKey} owner lease has expired`); } if (!await ownerStoragePointerCurrent(client, current)) { forgetOwnedScope(owner.ownerKey); - throw new DoRuntimeError(503, "stale_owner_storage", `DO scope ${owner.ownerKey} no longer matches active worker storage`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, `DO scope ${owner.ownerKey} no longer matches active worker storage`); } const leaseRemainingMs = Number(current.leaseExpiresAt ?? 0) - nowMs; if (leaseRemainingMs < ownerLeaseGuardMs(env)) { @@ -376,7 +377,7 @@ export async function assertCurrentOwnerWithLeaseBudget(env, owner, options = {} } } forgetOwnedScope(owner.ownerKey); - throw new DoRuntimeError(503, "owner_lease_too_short", `DO scope ${owner.ownerKey} owner lease has insufficient remaining budget`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_LEASE_TOO_SHORT, `DO scope ${owner.ownerKey} owner lease has insufficient remaining budget`); } return { owner: current, @@ -432,7 +433,7 @@ export async function renewOwner(env, owner) { await stageOwnerRenew(session.multi(), key, renewed, ownerTtlSeconds(env)).exec(); rememberOwner(renewed); return { renewed: true, owner: renewed, nowMs }; - }), "owner_renew_raced", `failed to renew DO owner for ${owner.ownerKey}`); + }), DO_OWNERSHIP_CODE.OWNER_RENEW_RACED, `failed to renew DO owner for ${owner.ownerKey}`); } /** @param {DoEnv} env */ @@ -510,7 +511,7 @@ export async function releaseOwner(env, owner) { await stageOwnerRelease(session.multi(), key).exec(); forgetOwnedScope(owner.ownerKey); return { released: true, owner: null }; - }), "owner_release_raced", `failed to release DO owner for ${owner.ownerKey}`); + }), DO_OWNERSHIP_CODE.OWNER_RELEASE_RACED, `failed to release DO owner for ${owner.ownerKey}`); } /** @param {DoEnv} env */ diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index a2bba28..2771d70 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -25,6 +25,21 @@ export { DO_HOST_SHARD_COUNT } from "do-runtime-protocol-wire-grammar"; export { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; export { hostIdForObject, hostIdForShard, shardForObjectName } from "do-runtime-protocol-identity"; +export const DO_OWNERSHIP_CODE = Object.freeze({ + OWNER_CLAIM_RACED: "owner_claim_raced", + OWNER_FENCE_MISSING: "owner_fence_missing", + STALE_OWNER_GENERATION: "stale_owner_generation", + OWNER_LEASE_EXPIRED: "owner_lease_expired", + STALE_OWNER_STORAGE: "stale_owner_storage", + OWNER_LEASE_TOO_SHORT: "owner_lease_too_short", + OWNER_RENEW_RACED: "owner_renew_raced", + OWNER_RELEASE_RACED: "owner_release_raced", + OWNER_UNAVAILABLE: "owner_unavailable", + OWNER_ENDPOINT_MISSING: "owner_endpoint_missing", + FORWARD_HOP_EXHAUSTED: "forward_hop_exhausted", + TASK_DRAINING: "task_draining", +}); + const MAX_MODULE_COUNT = 128; const MAX_MODULE_SOURCE_BYTES = 1024 * 1024; const MAX_REQUEST_BODY_BYTES = 1024 * 1024; diff --git a/docker-compose.images.yml b/docker-compose.images.yml index 698ce00..0bda468 100644 --- a/docker-compose.images.yml +++ b/docker-compose.images.yml @@ -1,49 +1,41 @@ +x-rust-published: &rust-published + image: docker.io/getwdl/wdl-rust:latest + pull_policy: always + +x-workerd-published: &workerd-published + image: docker.io/getwdl/wdl-workerd:latest + pull_policy: always + services: redis-proxy-user: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published redis-proxy-system: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published redis-proxy-do: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published scheduler: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published workflows: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published user-runtime: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published system-runtime: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published gateway: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published d1-runtime: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published d1-runtime-a: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published d1-runtime-b: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published d1-runtime-c: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published do-runtime: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published do-runtime-a: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published do-runtime-b: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published do-runtime-c: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published diff --git a/docker-compose.yml b/docker-compose.yml index 97595da..aafd9bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,28 @@ x-internal-auth-env: &internal-auth-env WDL_INTERNAL_AUTH_TOKEN: ${WDL_INTERNAL_AUTH_TOKEN:-local-internal-auth-token} WDL_INTERNAL_AUTH_PREVIOUS_TOKEN: ${WDL_INTERNAL_AUTH_PREVIOUS_TOKEN:-} +x-secret-envelope-env: &secret-envelope-env + SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} + SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} + +x-redis-proxy-service: &redis-proxy-service + <<: *rust-sidecar-image + command: ["/redis-proxy"] + environment: + <<: [*internal-auth-env, *secret-envelope-env] + REDIS_URL: redis://redis:6379 + DATA_REDIS_URL: redis://redis:6379/1 + REDIS_PROXY_PORT: "7070" + healthcheck: + test: ["CMD", "/redis-proxy", "healthcheck"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 10s + depends_on: + redis: + condition: service_healthy + # D1 runtime variants share local storage, health, entrypoint, and image wiring; # each service block below should only declare its task identity. x-d1-runtime-env: &d1-runtime-env @@ -113,64 +135,13 @@ services: COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS: wdl-assets,wdl-r2 redis-proxy-user: - <<: *rust-sidecar-image - command: ["/redis-proxy"] - environment: - <<: *internal-auth-env - REDIS_URL: redis://redis:6379 - DATA_REDIS_URL: redis://redis:6379/1 - REDIS_PROXY_PORT: "7070" - SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} - SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} - healthcheck: - test: ["CMD", "/redis-proxy", "healthcheck"] - interval: 5s - timeout: 5s - retries: 12 - start_period: 10s - depends_on: - redis: - condition: service_healthy + <<: *redis-proxy-service redis-proxy-system: - <<: *rust-sidecar-image - command: ["/redis-proxy"] - environment: - <<: *internal-auth-env - REDIS_URL: redis://redis:6379 - DATA_REDIS_URL: redis://redis:6379/1 - REDIS_PROXY_PORT: "7070" - SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} - SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} - healthcheck: - test: ["CMD", "/redis-proxy", "healthcheck"] - interval: 5s - timeout: 5s - retries: 12 - start_period: 10s - depends_on: - redis: - condition: service_healthy + <<: *redis-proxy-service redis-proxy-do: - <<: *rust-sidecar-image - command: ["/redis-proxy"] - environment: - <<: *internal-auth-env - REDIS_URL: redis://redis:6379 - DATA_REDIS_URL: redis://redis:6379/1 - REDIS_PROXY_PORT: "7070" - SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} - SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} - healthcheck: - test: ["CMD", "/redis-proxy", "healthcheck"] - interval: 5s - timeout: 5s - retries: 12 - start_period: 10s - depends_on: - redis: - condition: service_healthy + <<: *redis-proxy-service d1-runtime: <<: *d1-runtime-service @@ -301,7 +272,7 @@ services: command: ["serve", "-b", "/app/dist/workerd-configs/system-runtime-local.bin", "--experimental"] environment: - <<: *internal-auth-env + <<: [*internal-auth-env, *secret-envelope-env] REDIS_ADDR: redis:6379 DATA_REDIS_ADDR: redis:6379 DATA_REDIS_DB: "1" @@ -313,8 +284,6 @@ services: # Server-side env is BOOTSTRAP_TOKEN (auth's infra-managed ops # token); CLI keeps the ADMIN_TOKEN env name as "value to send". BOOTSTRAP_TOKEN: local-dev-token - SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} - SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} S3_ENDPOINT: http://s3mock:9090 S3_BUCKET: wdl-assets S3_ACCESS_KEY_ID: test diff --git a/docs/modules/cli.md b/docs/modules/cli.md index 90abcd6..63a4dd9 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -82,7 +82,8 @@ CLI output may display: the value without trying to reconstruct it from package metadata. - `minCliVersion`: the minimum downstream CLI version supported by this platform build. - `urls.control`: the control origin that the request actually reached. -- `urls.namespace`: the tenant namespace origin, returned only for namespace tokens. +- `urls.namespace`: the tenant namespace origin, returned only for namespace tokens when + the platform explicitly configures a public `PLATFORM_DOMAIN`. - `urls.assets`: the configured public assets base URL, returned only when the control plane has a safe absolute `http`/`https` `ASSETS_CDN_BASE`; query and fragment are stripped before returning the hint. diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index 9a78364..5d9257f 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -51,7 +51,7 @@ CLI 可以展示： - `platformVersion`：control 返回的 WDL platform version。Canonical derivation 记录在 `control-auth.zh.md` 的 `/whoami` 段；CLI 应直接展示这个值，不应从 package metadata 自行重建。 - `minCliVersion`：当前 platform build 支持的最低下游 CLI 版本。 - `urls.control`：请求实际到达的 control origin。 -- `urls.namespace`：tenant namespace origin，只对 namespace token 返回。 +- `urls.namespace`：tenant namespace origin，只在 caller 是 namespace token 且 platform 显式配置 public `PLATFORM_DOMAIN` 时返回。 - `urls.assets`：配置的 public assets base URL，只在 control plane 有安全的绝对 `http`/`https` `ASSETS_CDN_BASE` 时返回；返回前会去掉 query 和 fragment。 CLI 应把这些字段当作 diagnostics 和 user-facing guidance 的默认值，而不是替代用户显式配置。如果 `minCliVersion` 大于当前 CLI 版本，CLI 应在执行 mutating command 前 warning 或 fail。可选 URL hint 缺失时，应展示为 unavailable，不应自行猜测。 diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index aeaa0b3..299033a 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -14,8 +14,15 @@ admin-host branch. It is not dynamically loaded by workerLoader. Main files: - `control/index.js`: request dispatcher. -- `control/lib.js`: route parser and shared route utilities. -- `control/shared.js`: auth wrapper, JSON responses, Redis singletons, publish helpers. +- `control/lib.js`: pure route/key utilities, bundle metadata parsing, and referrer + shaping. +- `control/shared.js`: auth wrapper, Redis singletons, publish helpers, state-bound + workflow transport wiring, and lifecycle helpers. +- `control/errors.js`, `control/json-body.js`, `control/optimistic.js`: pure Control + response and request-body contracts plus the Control adapter over the shared + optimistic retry loop. +- `control/workflows-client.js`: canonical internal Workflows POST transport with + explicit caller-owned timeout selection. - `control/handlers/*`: endpoint handlers. - `control/routing.js`: promote/reconcile WATCH/MULTI logic. - `auth/index.js`, `auth/lib.js`, `shared/auth-roles.js`: token store, role table, @@ -96,8 +103,11 @@ Control lifecycle operations are split so each critical transition has one autho hash write; only secret-only workers with no retained versions defer their first load-time budget check to deploy. Secret PUT validates the plaintext size and shape, encrypts it into a `WDL-ENC:` envelope before the Redis mutation/WATCH retry loop, - and reuses the same envelope across retries. Runtime - therefore sees a new immutable version id instead of mutable in-place secret changes. + and reuses the same envelope across retries. Envelope JSON uses the fixed + `v,alg,kid,edek,iv,ct,tag` field order and each base64 field is canonical; Control and + redis-proxy both reject non-canonical persisted forms so malformed direct Redis writes + fail closed. Runtime therefore sees a new immutable + version id instead of mutable in-place secret changes. Secret DELETE removes the target field from the env estimate before decrypting the remaining secret hashes, so deleting the corrupt target can still succeed. Any corrupt remaining namespace or worker secret fails closed; direct Redis repair is not a @@ -111,7 +121,10 @@ Control lifecycle operations are split so each critical transition has one autho S3 object cleanup is enqueued only after Redis commit succeeds. - Worker delete lock values and `s3cleanup:` task ids are server-generated random values. `x-request-id` is diagnostic only and may be reused by clients or retries; it - must not become a lock token or cleanup-task id. + must not become a lock token or cleanup-task id. Delete, D1 migration, and delegated + issue locks use the token-fenced primitive in `shared/redis-lock.js`; release is + token-scoped and best-effort because TTL expiry bounds a leaked advisory lock and a + release error must not replace the operation's real result. - Auth is not a middleware convention. `parseControlRoute()` assigns an action, control sends that action and namespace to auth, and auth evaluates the stored token record against `shared/auth-roles.js`. Dispatcher code should not infer permissions from URL @@ -131,9 +144,11 @@ Control lifecycle operations are split so each critical transition has one autho single `x-forwarded-proto` value of `http` or `https`, `/whoami` uses that protocol for `urls.control` and `urls.namespace`; otherwise it falls back to the request URL protocol seen by control. `urls.namespace` is returned only for tenant namespace - tokens; `urls.assets` is returned only when `ASSETS_CDN_BASE` is a safe absolute - `http`/`https` URL, with query and fragment stripped. The endpoint must not grow - into token list, token lookup, or secret-bearing diagnostics. + tokens when `PLATFORM_DOMAIN` is explicitly configured as a public hostname; Control + does not advertise the internal `workers.local` fallback as a public URL. + `urls.assets` is returned only when `ASSETS_CDN_BASE` is a safe absolute `http`/`https` + URL, with query and fragment stripped. The endpoint must not grow into token list, + token lookup, or secret-bearing diagnostics. - A route shape has an `action` only when method, length, and verb exactly match an authorized shape. Missing action is deliberate: non-ops hit the auth unknown-action red line, while ops can still reach dispatcher/handler path and method validation @@ -258,6 +273,20 @@ Auth-specific contract: - Details may add fields but must not override `error`, `message`, or legacy `reason`. Auth reject reason is the `error` machine code; logs may carry `reason` as diagnostic context. +- `control/errors.js::ControlAbort` is the base in-Control coded abort contract. + Routing and Auth retain their boundary-specific error classes; Deploy may subclass + `ControlAbort` where commit cleanup requires a distinct catch boundary. +- `control/json-body.js` owns bounded Control JSON parsing and its `400`/`413` wire + mapping. `control/optimistic.js` binds strict `WatchError` recognition and Redis + sessions to the retry loop owned by `shared/owner-lease.js`. +- `control/lib.js::parseBundleMeta()` is the single parser for persisted bundle + `__meta__`. It requires a JSON object and accepts an error factory so routing, + workflows, delete, deploy, and env-budget paths retain their own catch boundaries. + Absence remains use-site-specific. Paths that need metadata to compute a correct + projection change, uniqueness proof, lifecycle cleanup, workflow view, or environment + budget fail closed while their authoritative route or index still names the bundle. + Deploy discovery/link preflight does not classify absence as `corrupt_meta`; the + watched commit rejects a missing pinned service target as `target_drift`. - Delegated issue 409 reasons have distinct retry meaning: `delegated_issue_busy` is retryable after the issuer/template lock clears, `active_quota_exceeded` is not retryable until existing delegated credentials expire @@ -293,8 +322,13 @@ Auth-specific contract: beginning after a scheduler timeout. Each run processes one page, so very large prefixes drain across multiple cron or queue dispatches instead of holding one scheduler dispatch open for minutes. -- Workflow lifecycle blockers are checked through workflows and fail closed on service - errors. +- All Control-to-Workflows internal POSTs use the canonical transport in + `control/workflows-client.js`. Callers retain endpoint-specific timeout, non-2xx, and + response-body interpretation. Workflow management calls and the + unbounded-by-namespace lifecycle delete scan intentionally have no client-side + timeout, matching their pre-consolidation behavior, while DO-alarm cleanup explicitly + retains its existing five-second bound. Workflow lifecycle blockers fail closed on + service errors. - AUTH JSRPC errors or Redis explosions are control-plane failures and map to 503 fail-closed behavior, not tenant-visible authorization fallbacks. @@ -331,6 +365,10 @@ Verify outcomes are logged as success, reject, or error; 5xx outcomes are error - `tests/unit/control-delete-handler.test.js` - `tests/unit/control-deploy-watch.test.js` - `tests/unit/control-lifecycle-indexes.test.js` +- `tests/unit/control-shared-stub.test.js` +- `tests/unit/control-handlers-workflows.test.js` +- `tests/unit/control-d1-migrations.test.js` +- `tests/unit/redis-lock.test.js` - `tests/unit/auth-lib.test.js` - `tests/unit/auth-index.test.js` - `tests/integration/auth-worker.test.js` diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index c23db0a..a0853d2 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -11,8 +11,10 @@ Control 跑在 system-runtime 的 `:8082`，通过 gateway admin-host 分支进 主要文件： - `control/index.js`：request dispatcher。 -- `control/lib.js`：route parser 和共享 route 工具。 -- `control/shared.js`：auth wrapper、JSON response、Redis singleton、publish helper。 +- `control/lib.js`：纯 route/key 工具、bundle metadata parsing 和 referrer shaping。 +- `control/shared.js`：auth wrapper、Redis singleton、publish helper、state-bound workflow transport wiring 和 lifecycle helper。 +- `control/errors.js`、`control/json-body.js`、`control/optimistic.js`：纯 Control response 与 request-body contract，以及 shared optimistic retry loop 的 Control adapter。 +- `control/workflows-client.js`：timeout 由 caller 显式选择的 canonical internal Workflows POST transport。 - `control/handlers/*`：endpoint handlers。 - `control/routing.js`：promote/reconcile WATCH/MULTI 逻辑。 - `auth/index.js`、`auth/lib.js`、`shared/auth-roles.js`：token store、role table、authorization evaluation。 @@ -66,12 +68,12 @@ Control lifecycle 操作会拆开处理，确保每个关键 transition 只有 - Deploy 解析支持的 Wrangler/JSONC 形状，校验 bindings 和 routes，通过 `worker:::next_version` 分配下一个 immutable version，写入 bundle metadata/modules/assets，然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前，deploy 会按 workerd 64 MiB 上限估算最终 WorkerCode，包含 runtime/do-runtime 注入的 wrapper/client modules 和 workflow import rewrite。真正的 Redis WATCH commit 会在分配真实 version 并完成 metadata materialization（例如 D1 database id 和 workflow keys 解析）之后用 code-budget 和 headroomed `workerLoader` env-budget 做权威检查，然后才写入 version。 - Promote 是唯一 active-route flip。它会 WATCH 本次候选需要的 delete lock、bundle metadata、D1 ref、service-binding target ref、queue consumer key、host declaration 和 pattern key。EXEC 在一个受审计的 transition 中更新 active route、host 反向索引、cron/queue projection、lifecycle index 和 invalidation publication。 -- Secret update/delete 在 worker 有 active route 时，会把 secret-store mutation stage 到 `bumpActiveAndPromote()` 内部，让 budget check、secret hash write、bundle copy 和 route flip 共享同一个 WATCH/MULTI transaction。如果没有 active route，则会在直接写 secret hash 前检查 retained versions 的预算；只有没有 retained versions 的 secret-only worker 会把第一次 load-time budget check 交给 deploy。Secret PUT 会先校验 plaintext 大小和形状，在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope，并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id，而不是原地可变的 secret。Secret DELETE 会先从 env 估算输入里移除目标字段，再解密剩余 secret hash，所以删除损坏的目标字段仍可成功；但剩余 namespace 或 worker secret 只要损坏就 fail closed，direct Redis repair 不是受支持的一致性路径。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata；如果并发 metadata 变化持续使视图失效，control 返回 `namespace_secret_mutation_contention`。 +- Secret update/delete 在 worker 有 active route 时，会把 secret-store mutation stage 到 `bumpActiveAndPromote()` 内部，让 budget check、secret hash write、bundle copy 和 route flip 共享同一个 WATCH/MULTI transaction。如果没有 active route，则会在直接写 secret hash 前检查 retained versions 的预算；只有没有 retained versions 的 secret-only worker 会把第一次 load-time budget check 交给 deploy。Secret PUT 会先校验 plaintext 大小和形状，在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope，并在重试间复用同一个 envelope。Envelope JSON 使用固定的 `v,alg,kid,edek,iv,ct,tag` 字段顺序，各 base64 字段也必须是 canonical shape；Control 与 redis-proxy 都拒绝 non-canonical persisted forms，使 malformed direct Redis writes fail closed。Runtime 因此看到新的 immutable version id，而不是原地可变的 secret。Secret DELETE 会先从 env 估算输入里移除目标字段，再解密剩余 secret hash，所以删除损坏的目标字段仍可成功；但剩余 namespace 或 worker secret 只要损坏就 fail closed，direct Redis repair 不是受支持的一致性路径。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata；如果并发 metadata 变化持续使视图失效，control 返回 `namespace_secret_mutation_contention`。 - Version delete 和 whole-worker delete 都 fail-closed。提交 Redis lifecycle deletion 前，会先收集 active route、retained version、service ref、D1 ref、workflow lifecycle check、queue/cron projection 和 delete lock blocker。S3 object cleanup 只在 Redis commit 成功后 enqueue。 -- Worker delete lock value 和 `s3cleanup:` task id 必须由服务端生成随机值。`x-request-id` 只用于诊断，客户端或重试可能复用它；不能把它用作 lock token 或 cleanup-task id。 +- Worker delete lock value 和 `s3cleanup:` task id 必须由服务端生成随机值。`x-request-id` 只用于诊断，客户端或重试可能复用它；不能把它用作 lock token 或 cleanup-task id。Delete、D1 migration 和 delegated issue lock 使用 `shared/redis-lock.js` 的 token-fenced primitive；release 按 token 限定且是 best-effort，因为 TTL expiry 会限制 advisory lock 泄漏时间，release error 不能覆盖操作的真实结果。 - Auth 不是一层约定俗成的 middleware。`parseControlRoute()` 分配 action，control 把 action 和 namespace 发给 auth，auth 用存储的 token record 对照 `shared/auth-roles.js` 评估权限。Dispatcher 代码不应自己从 URL prefix 推断权限。 - Delegated token issue 刻意不放进 direct `auth.token.*` lifecycle。`POST /auth/delegated-tokens` 使用 action `auth.delegated_token.issue`，因此 `/auth/tokens` issue/list/revoke 仍保持 strict ops-only；`token-issuer` credential 只能请求 Auth 按一个已配置 template materialize token。 -- `/whoami` 是唯一 namespace-less non-ops action。它对任何有效 token 开放，因为只报告当前 token 自己的 principal、token id、request id 和 public diagnostics。`platformVersion` 是由 bundled workerd date version 派生的 WDL version，使用 `wdl.` namespace，例如 `workerd` `1.20260531.1` 会变成 `wdl.20260531.1`；它不是 project release tag，后者的末尾 counter 可以在同一个 workerd date 上为 WDL-only patch 递增。raw workerd version 不返回。`minCliVersion` 是最低支持的下游 CLI 版本。`urls.control` 是到达 control 的 public origin；当 ingress 提供单个 `x-forwarded-proto` 且值为 `http` 或 `https` 时，`/whoami` 会用该协议生成 `urls.control` 和 `urls.namespace`，否则回退到 control 看到的 request URL 协议。`urls.namespace` 只对 tenant namespace token 返回；`urls.assets` 只在 `ASSETS_CDN_BASE` 是安全的绝对 `http`/`https` URL 时返回，并且会去掉 query 和 fragment。它不能扩展成 token list、token lookup 或携带 secret 的 diagnostics。 +- `/whoami` 是唯一 namespace-less non-ops action。它对任何有效 token 开放，因为只报告当前 token 自己的 principal、token id、request id 和 public diagnostics。`platformVersion` 是由 bundled workerd date version 派生的 WDL version，使用 `wdl.` namespace，例如 `workerd` `1.20260531.1` 会变成 `wdl.20260531.1`；它不是 project release tag，后者的末尾 counter 可以在同一个 workerd date 上为 WDL-only patch 递增。raw workerd version 不返回。`minCliVersion` 是最低支持的下游 CLI 版本。`urls.control` 是到达 control 的 public origin；当 ingress 提供单个 `x-forwarded-proto` 且值为 `http` 或 `https` 时，`/whoami` 会用该协议生成 `urls.control` 和 `urls.namespace`，否则回退到 control 看到的 request URL 协议。`urls.namespace` 只在 caller 是 tenant namespace token 且显式配置了 public hostname 形态的 `PLATFORM_DOMAIN` 时返回；Control 不会把内部 `workers.local` fallback 伪装成 public URL。`urls.assets` 只在 `ASSETS_CDN_BASE` 是安全的绝对 `http`/`https` URL 时返回，并且会去掉 query 和 fragment。它不能扩展成 token list、token lookup 或携带 secret 的 diagnostics。 - 只有 method、path length 和 verb 精确匹配 authorized shape 时，route shape 才带 `action`。缺 action 是有意行为：非 ops 会触发 auth unknown-action red line，ops 仍可到 dispatcher/handler 的 path 和 method validation，而不是被 auth 拦下。`/reload` 这类已知 top-level route 的 wrong-method case 会在 ops auth 通过后返回 `405 method_not_allowed`。 ## Redis / Storage 合同 @@ -133,6 +135,9 @@ Auth 子合同： - 如果 route shape 没有 action，非 ops 在 auth 处 fail closed；ops 可以到 dispatcher/handler 的 path 和 method validation；已知 top-level wrong-method route 返回 `405 method_not_allowed`。 - Control JSON error 形状是 `{ error, message }`；details 只能 additive。 - Details 可以增加字段，但不能覆盖 `error`、`message` 或 legacy `reason`。Auth reject reason 是 `error` machine code；日志可以把 `reason` 作为诊断上下文。 +- `control/errors.js::ControlAbort` 是 Control tier 内的基础 coded abort contract。Routing 和 Auth 保留各自 boundary-specific error class；Deploy 可以在 commit cleanup 需要独立 catch boundary 时 subclass `ControlAbort`。 +- `control/json-body.js` 持有 bounded Control JSON parsing 及其 `400`/`413` wire mapping；`control/optimistic.js` 把 strict `WatchError` recognition 和 Redis session 绑定到 `shared/owner-lease.js` 持有的 retry loop。 +- `control/lib.js::parseBundleMeta()` 是 persisted bundle `__meta__` 的唯一 parser。它要求值是 JSON object，并通过 error factory 让 routing、workflows、delete、deploy 和 env-budget 保留各自的 catch boundary。Bundle 缺失的语义仍由具体 use site 持有：当 projection 变更、唯一性证明、lifecycle cleanup、workflow view 或 environment budget 必须消费 metadata 才能产生正确结果时，只要权威 route 或 index 仍指向该 bundle，缺失就 fail closed。Deploy discovery/link preflight 不把缺失归类为 `corrupt_meta`；watched commit 会以 `target_drift` 拒绝缺失的 pinned service target。 - Delegated issue 的 409 reason 有不同 retry 语义：`delegated_issue_busy` 表示 issuer/template lock 清除后可重试；`active_quota_exceeded` 在已有 delegated credential 过期或 revoke 前不应重试；`namespace_collision` 表示 Auth 已耗尽 configured candidate retry budget。 - Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志；除非 endpoint 明确拥有某个 diagnostic response field，否则不进入客户端 body。 - Deploy 在最终 WorkerCode 会碰撞 WDL 注入的 runtime/do-runtime 保留模块名或缺少必要 bundle metadata 时返回 `worker_code_invalid`，在最终 WorkerCode（包含 runtime/do-runtime 注入模块和生成的 workflow keys）超过 workerd 64 MiB dynamic code limit 时返回 `worker_code_too_large`。Deploy 和 secret mutation 在估算的 `workerLoader` env 超过 WDL headroomed 1 MiB budget 时返回 `worker_env_too_large`。`worker_env_too_large` details 包含 `namespace`、可选 `worker`、`env_bytes`、`max_env_bytes`、`upstream_max_env_bytes` 和 `headroom_bytes`。Deploy 阶段的逐版本检查还会包含 `version`。Secret mutation 重新估算既有 version 时，details 还会包含 `source_version` 和 `estimated_version`。`source_version` 是 operator 应检查、删除或 redeploy 的已存版本；`estimated_version` 是本次预算估算使用的 tag，在 worker-secret bump 路径中就是已分配的精确 bump version。 @@ -140,7 +145,7 @@ Auth 子合同： - Control 在进入 Redis mutation loop 前加密 secret PUT value。加密/provider 失败会返回 control error，不写 plaintext fallback。 - Worker delete 先 commit Redis lifecycle state；异步 S3 cleanup enqueue 是 best-effort，失败时返回 warning。 - `s3-cleanup` system worker 会把 cleanup task 持久化在 D1；row 存在后由 cron replay 负责重试。S3 失败使用分钟级 exponential backoff，最高 30 分钟；大前缀 cleanup 每完成一个 S3 List/Delete page 就 checkpoint continuation token，因此正常分页进度不会消耗 failure attempts，也不会在 scheduler timeout 后从头开始。每次 run 只处理一页，所以超大 prefix 会跨多个 cron 或 queue dispatch 排空，而不是让单次 scheduler dispatch 持续数分钟。 -- Workflow lifecycle blocker 通过 workflows 检查；服务错误时 fail closed。 +- 所有 Control-to-Workflows internal POST 都使用 `control/workflows-client.js` 的 canonical transport。Caller 继续持有 endpoint-specific timeout、non-2xx 和 response-body interpretation。Workflow 管理调用和按 namespace 大小无上界的 lifecycle delete scan 都刻意不设 client-side timeout，保持 transport 收敛前的语义；DO-alarm cleanup 则显式保留原有五秒边界。Workflow lifecycle blocker 在服务错误时 fail closed。 - AUTH JSRPC 错误或 Redis 爆炸属于控制面失败，映射为 503 fail closed，而不是 tenant-visible authorization fallback。 ## 安全边界 @@ -170,6 +175,10 @@ Verify outcome 记录为 success、reject 或 error；5xx outcome 是 error log - `tests/unit/control-delete-handler.test.js` - `tests/unit/control-deploy-watch.test.js` - `tests/unit/control-lifecycle-indexes.test.js` +- `tests/unit/control-shared-stub.test.js` +- `tests/unit/control-handlers-workflows.test.js` +- `tests/unit/control-d1-migrations.test.js` +- `tests/unit/redis-lock.test.js` - `tests/unit/auth-lib.test.js` - `tests/unit/auth-index.test.js` - `tests/integration/auth-worker.test.js` diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 27c48d1..b20e689 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -110,6 +110,9 @@ when the logical worker is gone or now points at a different `doStorageId`. - One task owns a class shard at a time. - Generation fencing prevents stale owners from committing after ownership moves. +- `do-runtime/protocol.js` owns the DO ownership error vocabulary and the subset that + proves a request was fence-rejected and is safe to retry through the router. The + injected runtime transport mirrors both sets under a direct parity test. - Facet identity is `className:objectName` inside stable `doStorageId`, so worker promotion preserves object state. - Existing native facets keep the constructed class version until host actor restart or @@ -136,6 +139,10 @@ when the logical worker is gone or now points at a different `doStorageId`. through the router to rediscover the owner; non-idempotent methods and RPC return `owner_unavailable` without replay because the owner may already have applied the request. +- The shared runtime transport owns owner-hint cache wiring, invoke race retry, and + response-header stripping for both the host binding and injected facade. Its connect + wrapper deliberately omits the invoke-only router fallback required to preserve + owner-established WebSocket upgrades. - `WEBSOCKET_RECONNECT_DELAYS_MS` and `WEBSOCKET_MAX_BUFFERED_MESSAGES` tune gateway backend reconnect budget and client-message buffering without a code rebuild. - Alarm delivery is at-least-once. Scheduler wakes Workflows; Workflows promotes due @@ -203,8 +210,9 @@ interrupt. ## Security Boundaries -- do-runtime internal endpoints are private-mesh only and are not - application-authenticated. +- do-runtime internal endpoints are private-mesh only and require the shared + `WDL_INTERNAL_AUTH_TOKEN` through `x-wdl-internal-auth`; health and metrics are the + only unauthenticated endpoints. - Tenant code reaches DOs only through runtime-generated facades and frozen metadata. - Tenant-visible DO metadata and errors must not include owner task ids, backend endpoints, or raw transport error text. diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 7f1e9f7..57601aa 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -67,12 +67,14 @@ workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespa - 同一时间只有一个 task 拥有一个 class shard。 - Generation fence 防止 stale owner 在 ownership 移动后继续 commit。 +- `do-runtime/protocol.js` 持有 DO ownership error vocabulary，以及能够证明 request 已被 fence 拒绝、可安全通过 router 重试的子集。Injected runtime transport 在直接 parity test 下 mirror 这两个 Set。 - Facet identity 是 stable `doStorageId` 内的 `className:objectName`，因此 worker promotion 保留 object state。 - 已构造的 native facet 会保留构造时的 class version，直到 host actor restart 或 facet deletion。Promotion 改变未来 load 和 routing metadata，不会替换当前 host actor 中已经构造的 facet。 - Whole-worker delete 后 redeploy 会分配新的 `doStorageId`；旧 native storage tombstone 给后续 cleanup，而不是立即物理删除。 - WebSocket upgrade 必须在 owner endpoint 上完成。Owner-hinted WebSocket direct retry 不能 fall back 到 router-established 101。 - WDL 会尽量让 client-facing WebSocket 由 gateway 持有并保持连接，包括 user-runtime 或 do-runtime restart 后的 backend reconnect。这个连接连续性目标强于 Cloudflare shutdown 行为；Cloudflare 可以终止 WebSocket 让新 Durable Object instance 接管。当前 backend facet 仍属于 owner scope：初始 `101` 之后，WebSocket message / close event 不会在每一帧重新校验 Redis owner generation。未来增强应在尽量保持 client 连接的同时 rebinding 或拒绝 stale backend owner facet，而不是把 client disconnect 作为主要安全机制。Gateway 重置 backend reconnect epoch 时，旧 epoch 下排队的 client message 可能被丢弃，且没有逐帧 ack/nack。 - Ordinary fetch/RPC 只有在明确的 stale-owner 或 owner-race response 后才会回退到 router。Direct owner transport failure，或 direct owner hop 返回 502/503/504 且没有 fresh owner-hint header 时，会清除 cached hint。安全的 `GET`/`HEAD` request 可以通过 router 重放以重新发现 owner；非幂等 method 和 RPC 会返回 `owner_unavailable`，不会重放，因为 owner 可能已经应用了该请求。 +- Shared runtime transport 统一持有 host binding 与 injected facade 的 owner-hint cache wiring、invoke race retry 和 response-header stripping。Connect wrapper 刻意不包含 invoke-only router fallback，以保留 owner-established WebSocket upgrade 语义。 - `WEBSOCKET_RECONNECT_DELAYS_MS` 和 `WEBSOCKET_MAX_BUFFERED_MESSAGES` 可以在不改代码的情况下调整 gateway backend reconnect budget 和 client-message buffer cap。 - Alarm delivery 是 at-least-once。Scheduler 唤醒 Workflows；Workflows 把到期 internal alarm job promote 到 ready，在 DB 2 run token 下 claim，然后调用 do-runtime `/internal/do/alarms/dispatch`。do-runtime 仍然构造原来的 `DoInvoke{kind:"alarm"}` 请求，并走正常 owner router/fence 路径。 - Alarm due time 是传给 `setAlarm()` 的 Unix millisecond timestamp。Workflows 和 do-runtime 都用各自本地 wall clock 判断这些 timestamp；如果 backend ready hint 在 SQLite alarm row 对 do-runtime 来说尚未到期时抵达，do-runtime 会 ignore 这次 dispatch，但不清 row，让 backend due-index repair 路径之后继续投递。这是 alarm compatibility 边界，不属于 Redis-time owner lease fence。 diff --git a/docs/modules/gateway.md b/docs/modules/gateway.md index 983d976..f9f977a 100644 --- a/docs/modules/gateway.md +++ b/docs/modules/gateway.md @@ -28,11 +28,15 @@ Gateway has three dispatch branches: The resolved `{ ns, worker, version }` becomes `x-worker-id: ::` and `x-worker-prefix` on the runtime request. Literal `__system__` routes go to `RUNTIME_SYSTEM`; all ordinary tenant namespaces go to `RUNTIME_USER`. +Before forwarding, gateway removes client-supplied `x-worker-id`, `x-worker-prefix`, +and every `x-wdl-*` header. The same internal-header policy filters WebSocket upgrade +responses before they cross back onto the public socket. The `ADMIN_HOST` branch is infrastructure traffic, not a loaded-worker request. It does not set `x-worker-id` or `x-worker-prefix`. `PLATFORM_DOMAIN` and -`ADMIN_HOST` are environment-configurable; the code defaults are `workers.local` -and unset admin-host short-circuiting. +`ADMIN_HOST` are environment-configurable. All tiers normalize `PLATFORM_DOMAIN` as a +plain lowercase hostname without trailing dots and default it to `workers.local`; +admin-host short-circuiting remains unset by default. ## Interfaces diff --git a/docs/modules/gateway.zh.md b/docs/modules/gateway.zh.md index 096bf51..8d581df 100644 --- a/docs/modules/gateway.zh.md +++ b/docs/modules/gateway.zh.md @@ -16,7 +16,9 @@ Gateway 有三条 dispatch 分支： 解析出的 `{ ns, worker, version }` 会转成 runtime 请求头 `x-worker-id: ::` 和 `x-worker-prefix`。字面量 `__system__` route 进入 `RUNTIME_SYSTEM`；普通 tenant namespace 进入 `RUNTIME_USER`。 -`ADMIN_HOST` 分支是 infrastructure traffic，不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置；代码默认是 `workers.local`，且未设置 admin-host short-circuit。 +转发前，gateway 会清除客户端提供的 `x-worker-id`、`x-worker-prefix` 和所有 `x-wdl-*` header。同一套内部 header 策略也会在 WebSocket upgrade response 回到公开 socket 前进行过滤。 + +`ADMIN_HOST` 分支是 infrastructure traffic，不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置。所有 tier 都会把 `PLATFORM_DOMAIN` normalize 成不带尾点的纯小写 hostname，并默认使用 `workers.local`；admin-host short-circuit 默认仍未设置。 ## 接口 diff --git a/docs/modules/infra.md b/docs/modules/infra.md index 1f3fa4d..60280e0 100644 --- a/docs/modules/infra.md +++ b/docs/modules/infra.md @@ -158,6 +158,9 @@ Stateful storage: - Runtime internal `:8088`, d1-runtime `:8787`, do-runtime `:8788`, workflows `:9120`, and Redis are private mesh services. Private service calls also require `x-wdl-internal-auth` with the shared `WDL_INTERNAL_AUTH_TOKEN`. +- Kubernetes NetworkPolicies use per-component caller sets. ECS intentionally groups + user-runtime, system-runtime, D1, and DO tasks in one runtime security group, so its + network rules are coarser and are not required to match the Kubernetes caller matrix. - Tenant-running runtime tasks use least-privilege ECS task roles, public-only workerd outbound bindings, and private mesh security groups as their cloud credential and network boundary. ECS Exec should be enabled only where operator access is intended. diff --git a/docs/modules/infra.zh.md b/docs/modules/infra.zh.md index df452c8..144f222 100644 --- a/docs/modules/infra.zh.md +++ b/docs/modules/infra.zh.md @@ -92,6 +92,7 @@ Stateful storage： - user-runtime loaded worker outbound 是 public-only。 - system-runtime 是刻意放宽的特权环境。 - Runtime internal `:8088`、d1-runtime `:8787`、do-runtime `:8788`、workflows `:9120` 和 Redis 都是 private mesh services。Private service call 还必须携带带有共享 `WDL_INTERNAL_AUTH_TOKEN` 的 `x-wdl-internal-auth`。 +- Kubernetes NetworkPolicy 使用按组件区分的 caller 集合。ECS 刻意让 user-runtime、system-runtime、D1 和 DO task 共用一组 runtime security group，因此其网络规则粒度更粗，不要求与 Kubernetes caller 矩阵相同。 - Tenant-running runtime task 使用 least-privilege ECS task role、public-only workerd outbound binding 和 private mesh security group 作为 cloud credential 与 network 边界。ECS Exec 只应在需要 operator access 的地方启用。 ## 可观测性 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index 86484e1..81020f1 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -11,6 +11,9 @@ Shared JS primitives live in `shared/observability.js`. Runtime tailing uses: - `runtime/tail-worker.js` for console/exception capture. - `runtime/tail-forwarder.js` for active-set checks and append POSTs. +- The same runtime forwarder owns invocation start/finish envelope timing and + thrown-value normalization, so gateway dispatch and service-binding fetches emit one + `worker_fetch` shape. - `redis-proxy` logs endpoints for active tail checks and stream appends. - `control/handlers/logs-tail.js` for SSE sessions and heartbeat activation. - CLI `wdl tail` for user consumption. @@ -136,11 +139,12 @@ Common rules: - `x-request-id` propagates across gateway, control/runtime, loaded workers, and D1 where possible. Missing inbound ids are minted at ingress; dirty ids such as - multi-valued, control-character, or overly long ids are treated as absent. JS - entrypoints using `shared/request-scope.js` echo the sanitized id on responses and - log it as `request_id` on `request_complete`; Rust sidecars sanitize inbound ids and - log them where request middleware owns completion. Control's Redis `PUBLISH` path logs - the id locally but does not put it in the pub/sub payload. + multi-valued ids, ids containing Unicode whitespace or control characters, or ids + over 128 UTF-8 bytes are treated as absent. JS entrypoints using + `shared/request-scope.js` echo the sanitized id on responses and log it as + `request_id` on `request_complete`; Rust sidecars sanitize inbound ids and log them + where request middleware owns completion. Control's Redis `PUBLISH` path logs the id + locally but does not put it in the pub/sub payload. - Logs use snake_case fields. Only `level=error` is emitted on stderr; debug/info/warn JSON log lines go to stdout so log routing stays identical across JS, Rust, and embedded workerd shims. @@ -160,10 +164,11 @@ Common rules: `request_complete` logs as `error_code` / `error_message`. - JS and Rust observability implementations intentionally share metric prefix `wdl`, request metric families `requests` / `request_duration_ms` / `request_errors`, - cardinality warning threshold `100`, and Prometheus content type - `text/plain; version=0.0.4; charset=utf-8`. The shared fixture - `tests/fixtures/observability-contract.json` pins those values without introducing - a runtime metrics owner. + cardinality warning threshold `100`, Prometheus content type + `text/plain; version=0.0.4; charset=utf-8`, and the structured-log key order, level + priorities, stream routing, and timestamp shape. The shared fixture + `tests/fixtures/observability-contract.json` pins those values without introducing a + runtime observability owner. - redis-proxy records KV payload sizes in the `kv_value_bytes` summary with only bounded `service`/`operation`/`kind` labels. It records value, metadata, and raw batch byte counts so operators can decide whether large-value offload is needed; diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index 1892098..309da3a 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -10,6 +10,7 @@ Observability 提供有界 metrics、结构化日志、request-id 传播和 live - `runtime/tail-worker.js` 捕获 console/exception。 - `runtime/tail-forwarder.js` 做 active-set check 和 append POST。 +- 同一个 runtime forwarder 统一持有 invocation start/finish envelope timing 和 thrown-value normalization，因此 gateway dispatch 与 service-binding fetch 会输出同一种 `worker_fetch` shape。 - `redis-proxy` logs endpoints 做 active tail check 和 stream append。 - `control/handlers/logs-tail.js` 管理 SSE session 和 heartbeat activation。 - CLI `wdl tail` 给用户消费。 @@ -90,13 +91,13 @@ WDL 在 JS workerd tier 和 Rust 服务中使用同一套可观测性策略： 通用规则： -- `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker 和 D1 传播。缺失的 inbound id 会在入口生成；multi-valued、包含 control char 或过长的脏 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 +- `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker 和 D1 传播。缺失的 inbound id 会在入口生成；multi-valued、包含 Unicode whitespace/control char 或超过 128 UTF-8 bytes 的脏 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 - 日志字段使用 snake_case。只有 `level=error` 写入 stderr；debug/info/warn JSON log line 都写入 stdout，这样 JS、Rust 和 embedded workerd shim 的日志路由保持一致。 - 内部运维日志，包括 system-worker cleanup 日志和防御性的 Redis callback warning，也使用同一套单行 JSON envelope：`ts`、`service`、`level`、`event`，再加 snake_case 字段。JS 服务使用 `shared/observability.js`；Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`；不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 - 产品 API response body 默认使用 camelCase，除非 endpoint 明确记录不同 wire contract。 - Metrics 只使用有界枚举 label。 - 暴露 request metrics 的 Rust HTTP sidecar 使用同一组 `requests`、`request_duration_ms` 和 `request_errors` metric family，以及有界的 `service`/`route`/`status` labels；per-route error context 只进入 `request_complete` 日志中的 `error_code` / `error_message`。 -- JS 和 Rust observability 实现刻意共享 metric prefix `wdl`、request metric families `requests` / `request_duration_ms` / `request_errors`、cardinality warning threshold `100`、Prometheus content type `text/plain; version=0.0.4; charset=utf-8`；共享 fixture `tests/fixtures/observability-contract.json` pin 住这些值，但不引入 runtime metrics owner。 +- JS 和 Rust observability 实现刻意共享 metric prefix `wdl`、request metric families `requests` / `request_duration_ms` / `request_errors`、cardinality warning threshold `100`、Prometheus content type `text/plain; version=0.0.4; charset=utf-8`，以及结构化日志的 key 顺序、level priority、stream routing 和 timestamp shape；共享 fixture `tests/fixtures/observability-contract.json` pin 住这些值，但不引入 runtime observability owner。 - redis-proxy 用 `kv_value_bytes` summary 记录 KV payload size，label 只有有界的 `service`/`operation`/`kind`。它记录 value、metadata 和 raw batch byte count，用来判断是否需要 large-value offload；namespace、key 和 object identity 不进入 metric label。 - Rust `request_complete` log 输出整数 `duration_ms`，让各服务日志字段保持稳定；Prometheus duration summary 仍保留浮点值。 - JS `MetricsRegistry` 在单个 metric name 达到 100 个 series 时输出一次结构化 `metric_cardinality_warning` 日志，之后会丢弃该 metric 的全新 series，但继续更新已有 series。Rust `MetricStore` 当前仍保持同一 warning-only tripwire。该 warning 携带 metric name、观测到的 series 数和配置 limit；tenant-specific 细节本来就不应进入 label。因为这条 warning 由 metrics registry 输出，所以不会被 `LOG_LEVEL` 抑制。 diff --git a/docs/modules/queues-cron.md b/docs/modules/queues-cron.md index cdeb70e..bdac1a6 100644 --- a/docs/modules/queues-cron.md +++ b/docs/modules/queues-cron.md @@ -177,6 +177,9 @@ queue-orphaned:: Stream, messages whose consumer disappeared queue-delayed-wake Stream, wake signal for delayed ZSET dispatcher ``` +`wdl-rust-common::queue_keys` owns the wake stream and its `delayed_key` / `visible_at` +entry fields so the redis-proxy producer and scheduler consumer cannot drift. + Indexes are not authority. Writers add index entries, and scheduler reconcile owns stale cleanup after proving the referenced key is absent. @@ -294,6 +297,8 @@ Representative test anchors: consumers. - `tests/unit/control-lifecycle-indexes.test.js`: JS cron/queue key helpers and projection staging. +- `tests/fixtures/queue-key-parse.json`: shared JS/Rust queue discovery-key parser + contract. - `tests/unit/runtime-lib.test.js`: internal dispatch body normalization. - `tests/unit/runtime-dispatch-handlers.test.js`: scheduled and queue dispatch behavior, tail event envelope behavior. diff --git a/docs/modules/queues-cron.zh.md b/docs/modules/queues-cron.zh.md index 00576a0..20058a4 100644 --- a/docs/modules/queues-cron.zh.md +++ b/docs/modules/queues-cron.zh.md @@ -109,6 +109,8 @@ queue-orphaned:: Stream, consumer 消失后的 message 降落 queue-delayed-wake Stream, delayed ZSET dispatcher 的 wake signal ``` +`wdl-rust-common::queue_keys` 持有 wake stream 及其 `delayed_key` / `visible_at` entry fields，避免 redis-proxy producer 与 scheduler consumer 漂移。 + Index 不是权威状态。Writer 只负责添加 index member；scheduler reconcile 在确认被引用 key 不存在后负责 stale cleanup。 ## Ownership / 并发 / 失败语义 @@ -185,6 +187,7 @@ Runtime tail logs 也会为 loaded worker 执行输出 `worker_scheduled` 和 `w - `tests/unit/control-lib.test.js`：cron 和 queue manifest 解析。 - `tests/unit/control-routing.test.js`：cron 和 queue consumer 的 promote projection。 - `tests/unit/control-lifecycle-indexes.test.js`：JS cron/queue key helper 和 projection staging。 +- `tests/fixtures/queue-key-parse.json`：JS/Rust 共用的 queue discovery-key parser 合同。 - `tests/unit/runtime-lib.test.js`：internal dispatch body normalization。 - `tests/unit/runtime-dispatch-handlers.test.js`：scheduled / queue dispatch 行为，以及 tail event envelope 行为。 - `tests/unit/style-contracts.test.js`：跨 tier Redis key/layout drift 检查。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 43ee18d..c8b4830 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -21,6 +21,9 @@ are: Workers are loaded by immutable id `::`. Promotion creates a new id, so active-version changes naturally cold-load a fresh isolate. +`runtime/load.js` also owns the shared `workerLoader.get()` wrapper: every cache miss is +recorded for later eviction, while only active-version dispatch paths request sibling +eviction. Service bindings load their pinned version without evicting siblings. The two runtime pools use the same image and source but different capnp configs: @@ -84,9 +87,10 @@ services. Runtime therefore treats bindings as adapters: worker secrets. A worker-level secret with the same name wins over a namespace-level secret, which wins over a var. Control enforces a headroomed estimate of workerd's `workerLoader` serialized env budget during deploys and secret mutations. That - estimate includes user vars/secrets plus runtime-injected binding/workflow env values, - including required caller secret copies in platform/service binding props, so an - over-large env fails in the control plane instead of during runtime cold-load. + estimate calls the same `buildWorkerEnv()` materializer as cold-load with shape-only + factories, then measures user vars/secrets plus runtime-injected binding/workflow env + values, including required caller secret copies in platform/service binding props, so + an over-large env fails in the control plane instead of during runtime cold-load. - Stateful bindings such as D1, Durable Objects, and Workflows call dedicated backend services. The hidden backend Fetchers stay in runtime and are removed before tenant code observes `env`. diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 1249d29..c1963f7 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -15,6 +15,7 @@ Runtime 从 Redis 加载不可变 worker version，构建 tenant-facing `env` bi - `runtime/dispatch.js` 和 `runtime/dispatch/*`：fetch/scheduled/queue/workflow dispatch helper。 Worker 通过不可变 id `::` 加载。Promote 会生成新的 id，因此 active-version 变化自然触发 fresh isolate cold-load。 +`runtime/load.js` 也统一持有 `workerLoader.get()` wrapper：每次 cache miss 都会登记供后续 eviction 使用，只有 active-version dispatch path 会请求 sibling eviction；service binding 加载 pinned version 时不会驱逐 sibling。 两个 runtime pool 使用同一个 image 和源码，但 capnp config 不同： @@ -40,7 +41,7 @@ Tenant 可见 binding 包括 KV、R2、D1、Durable Objects、Queues、ASSETS、 workerd 提供 isolate、module evaluation、named entrypoint 和 JSRPC 机制。Cloudflare 生产平台通常在外部服务里实现的 binding 后端，则由 WDL 自己补齐。因此 runtime 把 binding 当作 adapter： - KV、queue producer 这类纯数据 binding 调 colocated redis-proxy sidecar。Loaded worker 看到 Cloudflare-shaped object，但 method call 会先通过 workerd JSRPC 回到 runtime，再经 HTTP 调 redis-proxy。 -- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后，runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`；tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级：bundle vars，然后 namespace secrets，然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret，namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 过程中用留有 headroom 的预算估算完整 workerLoader env，包括用户 vars/secrets、runtime 注入的 binding/workflow env value，以及 platform/service binding props 中复制的 required caller secret，避免超限配置拖到 runtime cold-load 才失败。 +- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后，runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`；tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级：bundle vars，然后 namespace secrets，然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret，namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 过程中用 shape-only factory 调用 cold-load 同一个 `buildWorkerEnv()` materializer，再用留有 headroom 的预算估算完整 workerLoader env，包括用户 vars/secrets、runtime 注入的 binding/workflow env value，以及 platform/service binding props 中复制的 required caller secret，避免超限配置拖到 runtime cold-load 才失败。 - D1、Durable Objects、Workflows 这类 stateful binding 调专门 backend service。Hidden backend Fetcher 留在 runtime 内部，并在 tenant code 观察 `env` 前被删除。 - R2 是 S3-compatible object-storage adapter：runtime 使用平台 credential 签名请求，并发送到配置的 endpoint。 - ASSETS 是 deploy artifact URL helper：control 在 deploy 时把 assets 上传到 S3-compatible storage；runtime 读取 `__meta__.assets` 和 `ASSETS_CDN_BASE`，只暴露 `env.ASSETS.url(path)` 用来生成 tokenized CDN URL。 diff --git a/docs/modules/workflows.md b/docs/modules/workflows.md index a4ea74f..7e0a85f 100644 --- a/docs/modules/workflows.md +++ b/docs/modules/workflows.md @@ -133,6 +133,10 @@ Key families: - Scheduler also wakes Workflows-owned internal DO alarm jobs through the same `/internal/workflows/tick` endpoint; scheduler never reads or writes DO alarm state directly. +- Workflows revalidates persisted DO alarm namespace, worker, and version identity with + the `wdl-rust-common` owners before dispatch. Runtime run dispatch and progress + callbacks share one system-vs-user runtime endpoint selector inside the workflows + crate. - 32 scheduling shards partition ready/due work. - Ready tokens are deduplicated hints; instance hash state is authority. - Execution commits are fenced by `generation`, `runToken`, active instance status, and @@ -154,9 +158,11 @@ Key families: (`step.sleep`, `step.sleepUntil`, `step.waitForEvent`) remain exclusive and must not overlap another in-flight step because they suspend the whole workflow run. - Termination is an explicit non-success terminal outcome and uses error retention. -- Per-instance aggregate payload cap is 16 MiB. Step/event over-cap writes fail the - request; over-cap runtime terminal results transition the instance to failed in the - same transaction. +- A single workflow result is capped at 1 MiB and a runtime-to-workflows backend JSON + request at 2 MiB. Runtime prevalidation and the Rust backend share the pinned + `workflow_payload_too_large` contract. The per-instance aggregate payload cap is + 16 MiB. Step/event over-cap writes fail the request; over-cap runtime terminal + results transition the instance to failed in the same transaction. - Workflows semantic request caps use `request_too_large`; this is distinct from HTTP-body parser `request_body_too_large` in control/runtime protocols. Workflow errors otherwise use the platform `{ error, message }` envelope on HTTP boundaries. diff --git a/docs/modules/workflows.zh.md b/docs/modules/workflows.zh.md index 1e57d35..f7c5887 100644 --- a/docs/modules/workflows.zh.md +++ b/docs/modules/workflows.zh.md @@ -83,13 +83,14 @@ Key families： - Instance 冻结创建时的 worker version/class identity。 - Scheduler 只负责唤醒 workflows；admission、fairness、shard tick、ready/due movement 和 runtime dispatch 都由 workflows 负责。 - Scheduler 也通过同一个 `/internal/workflows/tick` endpoint 唤醒 Workflows-owned internal DO alarm jobs；scheduler 不直接读写 DO alarm state。 +- Workflows 在 dispatch 前用 `wdl-rust-common` owner 重新校验持久化 DO alarm 的 namespace、worker 和 version identity。Runtime run dispatch 与 progress callback 在 workflows crate 内共用同一个 system-vs-user runtime endpoint selector。 - 32 个 scheduling shards 划分 ready/due work。 - Ready token 是去重 hint；instance hash state 是权威状态。 - Execution commit 同时用 `generation`、`runToken`、active instance status 和未过期 run lease fence。Step commit/register 接受同一 run 的 `running` 或 `waiting` 状态，因此一个并行 sibling 进入 retry/wait 后，另一个 sibling 仍可完成；completed runtime terminal 要求 `running`，failed runtime terminal 在 run lease 仍有效时也可以关闭由非法未 await suspending step 造成的同一 run `waiting` 状态。如果 lease 已过期，workflows 只恢复 ready hint，让下一次 claim 在新 lease 下 replay。Lifecycle commit 只用 generation fence，并在同一个 Lua commit 内 rotate `generation`。 - Runtime replay cache 只是 advisory。DB 2 step state 是权威。 - Runtime 可以并发发起多个 `step.do`，常见形式是 `Promise.all`；每次调用按用户代码调用顺序分配 deterministic ordinal，从当前已完成 step frontier 记录 DAG dependencies，并在 run fence 下独立 commit。`step.do` callback 不能启动另一个 workflow step，即使在 callback 的 `await` 之后也不允许；并行 sibling promise 应在 run body 中、callback 代码进入 in-flight 之前创建。如果 run 在已启动 step settle 前返回，会按 invalid run 失败，所以用户代码必须 await 并发 step promise。Suspending operation（`step.sleep`、`step.sleepUntil`、`step.waitForEvent`）仍保持互斥，不能和其它 in-flight step 重叠，因为它们会 suspend 整个 workflow run。 - Termination 是显式 non-success terminal outcome，使用 error retention。 -- 每个 instance 的 aggregate payload cap 是 16 MiB。Step/event 超 cap 写入会让请求失败；runtime terminal result 超 cap 会在同一事务内把 instance 转成 failed。 +- 单个 workflow result 的上限是 1 MiB，runtime-to-workflows backend JSON request 的上限是 2 MiB。Runtime prevalidation 和 Rust backend 共享 pinned `workflow_payload_too_large` contract。每个 instance 的 aggregate payload cap 是 16 MiB。Step/event 超 cap 写入会让请求失败；runtime terminal result 超 cap 会在同一事务内把 instance 转成 failed。 - Workflows 语义 request cap 使用 `request_too_large`；它不同于 control/runtime 协议中的 HTTP-body parser `request_body_too_large`。除此之外，HTTP 边界上的 workflow error 使用平台 `{ error, message }` envelope。Client-facing proxy 应把 workflows 5xx 当作 backend/platform failure，不应依赖 response body 中的 raw backend diagnostic message。 Workflow execution 使用两条 channel： diff --git a/docs/redis-key-layout.md b/docs/redis-key-layout.md index 7c4119a..1946f43 100644 --- a/docs/redis-key-layout.md +++ b/docs/redis-key-layout.md @@ -118,6 +118,15 @@ not base64. Typical fields include: } ``` +Control writes `__meta__` as a JSON object. Control-plane consumers parse required +bundle metadata through `control/lib.js::parseBundleMeta()`; malformed JSON, arrays, +and scalar values fail closed as `corrupt_meta`. Absence remains use-site-specific. +Paths that need metadata to compute a correct projection change, uniqueness proof, +lifecycle cleanup, workflow view, or environment budget fail closed while their +authoritative route or index still names the bundle. Deploy discovery/link preflight +does not classify absence as `corrupt_meta`; the watched commit remains authoritative +and rejects a missing pinned service target as `target_drift`. + Routes, crons, queue consumers, bindings, vars, exports, workflow definitions, and asset prefixes are version metadata. Rollback is a promote of an older immutable version. diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index 71e8921..36232bc 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -73,6 +73,8 @@ Pattern `slot` 是原始 wrangler pattern，例如 `/mcp` 或 `/mcp/*`；它也 } ``` +Control 把 `__meta__` 写为 JSON object。Control-plane consumer 通过 `control/lib.js::parseBundleMeta()` 解析必需的 bundle metadata；malformed JSON、array 和 scalar 值都会以 `corrupt_meta` fail closed。Bundle 缺失的语义仍由具体 use site 持有：当 projection 变更、唯一性证明、lifecycle cleanup、workflow view 或 environment budget 必须消费 metadata 才能产生正确结果时，只要权威 route 或 index 仍指向该 bundle，缺失就 fail closed。Deploy discovery/link preflight 不把缺失归类为 `corrupt_meta`；watched commit 仍是权威检查，并以 `target_drift` 拒绝缺失的 pinned service target。 + Routes、crons、queue consumers、bindings、vars、exports、workflow definitions 和 asset prefixes 都是 version metadata。Rollback 本质上是 promote 一个旧的 immutable version。 ## Feature Key Families diff --git a/docs/rust-sidecar-standards.md b/docs/rust-sidecar-standards.md index b657e8d..6860745 100644 --- a/docs/rust-sidecar-standards.md +++ b/docs/rust-sidecar-standards.md @@ -156,6 +156,10 @@ wrapper, and UTF-8-safe string truncation. It must not become a dumping ground f service behavior. Axum-facing helpers are feature-gated so non-HTTP users such as the D1/DO supervisor binaries do not pay for the HTTP stack. +The `test-support` feature exposes the single process-environment override helper used +by Rust service tests. It serializes overrides across modules in one test process and +restores all values during unwinding; production dependency builds leave it disabled. + Local explicit code is still preferable when sidecars have different behavior around: - service-specific shutdown/drain timing and shutdown log events diff --git a/docs/rust-sidecar-standards.zh.md b/docs/rust-sidecar-standards.zh.md index 762cc8b..811d459 100644 --- a/docs/rust-sidecar-standards.zh.md +++ b/docs/rust-sidecar-standards.zh.md @@ -88,6 +88,8 @@ Crate root 和目录级 `mod` glue 可以保留既有的显式本地 prelude， `wdl-rust-common`（`rust/common/`）是本批次唯一的 shared Rust helper crate。它只拥有必须跨 crate 保持一致的小 primitive，例如环境变量数字解析、日志等级解析、HTTP health probe、shutdown/in-flight tracking、通用 JSON log-line emission、wall-clock millisecond helper、短 non-cryptographic random hex suffix、稳定 non-cryptographic hash、queue Redis key builders、worker version / bundle-key parsing、Prometheus metric storage/formatting、Prometheus text response、结构化错误字段合并、internal-auth token/header matching、Redis command construction helper、中立的 Redis connection execution wrapper，以及 UTF-8 安全的字符串截断。它不能变成 service 行为的杂物箱。Axum-facing helper 必须 feature-gate，因此 D1/DO supervisor binaries 这类非 HTTP consumer 不需要付 HTTP stack 成本。 +`test-support` feature 暴露 Rust service tests 共用的唯一 process-environment override helper。它在同一 test process 的 module 之间串行化 override，并在 unwind 时恢复全部值；production dependency build 不启用该 feature。 + 当 sidecar/service 在这些方面行为不同，本地显式代码仍然更好： - service-specific shutdown/drain timing 和 shutdown log event diff --git a/docs/security.zh.md b/docs/security.zh.md index ef90532..21df4f9 100644 --- a/docs/security.zh.md +++ b/docs/security.zh.md @@ -94,7 +94,7 @@ Binding 是 tenant capability 的主要 surface： Terraform 在 ECS Fargate 上运行平台服务，包括执行 tenant 的 runtime task。Tenant-running task 的 cloud credential 暴露由 least-privilege task role、public-only workerd outbound binding 和 private mesh security group 约束。ECS Exec 只应在需要 platform operator access 的地方启用。 -Service Connect 和 security group 是 internal mesh 边界的一部分。共享 `WDL_INTERNAL_AUTH_TOKEN` 当前值必须在 runtime、d1-runtime、do-runtime、scheduler、workflows 和 redis-proxy sidecar 间保持一致；可选 previous 值只作为维护窗口轮换桥接被 receiver 接受。Caller 始终发送当前值，因此 token 轮换不是 rolling-safe，除非先暂停 traffic 或一起重启 private fleet。Scheduler 是 runtime internal dispatch 和 workflows tick 的 client；Workflows 会把 Durable Object alarm dispatch 到 do-runtime。Scheduler 和 workflows 都不是公开 service target。 +Service Connect 和 security group 是 internal mesh 边界的一部分。共享 `WDL_INTERNAL_AUTH_TOKEN` 当前值必须在 runtime、d1-runtime、do-runtime、scheduler、workflows 和 redis-proxy sidecar 间保持一致；可选的 `WDL_INTERNAL_AUTH_PREVIOUS_TOKEN` 只作为维护窗口轮换桥接被 receiver 接受。Caller 始终发送当前值，因此 token 轮换不是 rolling-safe，除非先暂停 traffic 或一起重启 private fleet。Scheduler 是 runtime internal dispatch 和 workflows tick 的 client；Workflows 会把 Durable Object alarm dispatch 到 do-runtime。Scheduler 和 workflows 都不是公开 service target。 ## Tenant-facing 合同 diff --git a/docs/source-map.md b/docs/source-map.md index f8b3c9b..6b6362a 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -36,8 +36,10 @@ are outside this map unless they own runtime or deployable service behavior. | `runtime/lib.js` | Pure runtime helpers such as bundle-to-worker-code, byte normalization, and dispatch body normalization. | | `control/index.js` | Thin HTTP dispatcher on system-runtime `:8082`; delegates to handlers after auth. | | `control/handlers/` | Endpoint handlers for deploy, promote, versions, workers, delete, secrets, hosts, reload, auth tokens, D1, R2, workflows, and log tail. | -| `control/shared.js` | Control singletons, auth wrapper, JSON/error helpers, Redis publish helpers, and shared lifecycle/delete helpers. Direct `state.*` access belongs here or in the dispatcher. | -| `control/lib.js` | Control route-to-action classifier, route utilities, delete-lock key helper, and referrer redaction. | +| `control/shared.js` | Control singletons, auth wrapper, Redis publish helpers, state-bound workflow transport wiring, and shared lifecycle/delete helpers. Direct `state.*` access belongs here or in the dispatcher. | +| `control/errors.js`, `control/json-body.js`, `control/optimistic.js` | Pure Control error-response and bounded JSON request-body contracts plus the strict `WatchError`/Redis-session adapter over the shared optimistic retry loop, re-exported by `control/shared.js`. | +| `control/workflows-client.js` | Internal Control-to-Workflows POST transport with explicit caller-owned timeout selection; callers own endpoint-specific response interpretation. | +| `control/lib.js` | Pure Control data-shaping: route-to-action classification, key helpers, canonical bundle `__meta__` parsing, and referrer redaction. | | `control/bundle.js` | Bundle/module normalization, compatibility metadata, vars, and emitted module manifest construction. | | `control/bindings.js` | Service/platform binding parsers, ACL evaluation, and linker helpers. | | `control/topology.js` | Route, pattern, cron, queue consumer, and workflow declaration parsing for deploy metadata. | @@ -56,23 +58,26 @@ are outside this map unless they own runtime or deployable service behavior. | Path | Responsibility | |---|---| | `shared/redis.js`, `shared/redis-*.js` | Public Redis import surface plus split RESP codec, per-call client, WATCH/MULTI session, and subscriber loop modules. Runtime hot paths prefer the Rust redis-proxy sidecar. | -| `shared/owner-lease.js`, `shared/owner-protocol.js`, `shared/owner-forwarder.js` | Shared owner lease parsing, generation counters, key derivation, fence matching, staged Redis owner writes, and owner-forwarding HTTP mechanics used by D1 and DO runtimes. | +| `shared/redis-lock.js` | Token-fenced Redis lock creation, acquire, renewal, and best-effort token-scoped release shared by Control and Auth. | +| `shared/owner-lease.js`, `shared/owner-protocol.js`, `shared/owner-forwarder.js` | Shared optimistic retry loop plus owner lease parsing, generation counters, key derivation, fence matching, staged Redis owner writes, and owner-forwarding HTTP mechanics used by Control and the D1/DO runtimes. | | `shared/auth-roles.js` | Role table, principal validation, reserved namespace policy, and auth action capabilities. | | `shared/auth-token.js` | Shared `x-admin-token` sanitizer used by control and auth. | | `shared/internal-auth.js` | Shared internal mesh auth header and token helpers used by JS callers and receivers. | | `shared/secret-envelope.js`, `shared/secret-keys.js` | Secret envelope encryption/decryption, canonical base64/JSON handling, AAD binding helpers, and secret Redis key construction. | +| `shared/base64.js` | Dependency-free byte/text base64 codec shared by workerd tiers, with a `Buffer` fast path under `nodejs_compat`. | | `shared/hex.js`, `shared/random-id.js`, `shared/errors.js` | Small dependency-free primitives for byte-to-hex rendering, random hex ids, and string-only error message extraction. | | `shared/observability.js` | Structured logger, metrics registry, request-id helpers, and log-level handling for JS tiers. | | `shared/respond.js` | Shared HTTP response, JSON error, Prometheus text, best-effort response body discard, and `x-request-id` echo helpers. | -| `shared/bounded-body.js` | Shared bounded request body byte/text readers; each tier maps limit errors to its own HTTP error contract. | -| `shared/ns-pattern.js` | Namespace, worker, binding, queue, KV id, module path, reserved object-key, and reserved namespace grammars. | -| `shared/version.js` | Worker version formatting and bundle key helpers. | +| `shared/bounded-body.js` | Shared bounded byte-stream and request-body readers; each tier maps limit errors to its own contract. | +| `shared/ns-pattern.js` | Platform-domain normalization plus namespace, worker, binding, queue, KV/D1/R2 id, module path, reserved object-key, and reserved namespace grammars. | +| `shared/version.js` | Worker version formatting plus worker and route-plane Redis key helpers. | | `shared/workerd-compat-flags.js` | Pinned mirror of upstream workerd experimental compatibility enable flags used to reject tenant metadata before cold-load. | | `shared/queue-keys.js` | JavaScript queue key helpers used by tests and cross-tier key-shape checks. | | `shared/route-projection.js` | Compact pattern-route projection encoding shared by control writers, delete checks, and gateway readers. | | `shared/d1-*.js`, `shared/sql-splitter.js` | D1 parameter, data-field, transport, timeout, query-wire, and SQL splitting utilities shared by runtime, d1-runtime, control, and tests. | | `shared/fnv1a32.js` | Shared JavaScript FNV-1a helpers for runtime-side shard and slot hashing. | | `shared/s3-query.js` | S3 query encoder used by the s3-cleanup system worker; runtime R2 keeps the same standalone helper in `runtime/r2-utils.js` because that file is injected as worker source. | +| `shared/s3-retry.js` | Bounded transient retry policy for idempotent S3 POST operations, shared by runtime R2 and the s3-cleanup worker. | | `shared/s3-xml.js` | Shared S3 XML parsing helpers used by control R2, runtime R2, and system cleanup paths. | | `shared/worker-id.js` | Shared `x-worker-id` formatting, parsing, and runtime-load identity grammar used by gateway, runtime, DO runtime, and tests. | | `shared/cron-time.js` | Control-side cron parsing and slot-alignment helpers; scheduler advancement uses Rust `croner`. | diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index 88eece2..ee5f220 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -33,8 +33,10 @@ | `runtime/lib.js` | 纯 runtime helpers，例如 bundle-to-worker-code、byte normalization 和 dispatch body normalization。 | | `control/index.js` | system-runtime `:8082` 上的薄 HTTP dispatcher；auth 后交给 handlers。 | | `control/handlers/` | deploy、promote、versions、workers、delete、secrets、hosts、reload、auth tokens、D1、R2、workflows 和 log tail endpoint handlers。 | -| `control/shared.js` | Control singletons、auth wrapper、JSON/error helpers、Redis publish helpers 和共享 lifecycle/delete helpers。Direct `state.*` access 只应在这里或 dispatcher。 | -| `control/lib.js` | Control route-to-action classifier、route utilities、delete-lock key helper 和 referrer redaction。 | +| `control/shared.js` | Control singletons、auth wrapper、Redis publish helpers、state-bound workflow transport wiring 和共享 lifecycle/delete helpers。Direct `state.*` access 只应在这里或 dispatcher。 | +| `control/errors.js`、`control/json-body.js`、`control/optimistic.js` | 由 `control/shared.js` re-export 的纯 Control error-response、bounded JSON request-body contract，以及 shared optimistic retry loop 上的 strict `WatchError`/Redis-session adapter。 | +| `control/workflows-client.js` | timeout 由 caller 显式选择的 Control-to-Workflows internal POST transport；endpoint-specific response interpretation 仍由 caller 持有。 | +| `control/lib.js` | 纯 Control data-shaping：route-to-action classification、key helpers、canonical bundle `__meta__` parsing 和 referrer redaction。 | | `control/bundle.js` | Bundle/module normalization、compatibility metadata、vars 和 emitted module manifest construction。 | | `control/bindings.js` | Service/platform binding parsers、ACL evaluation 和 linker helpers。 | | `control/topology.js` | Deploy metadata 中 routes、patterns、cron、queue consumer 和 workflow declaration parsing。 | @@ -53,23 +55,26 @@ | Path | 责任 | |---|---| | `shared/redis.js`、`shared/redis-*.js` | 公共 Redis import surface，以及拆分后的 RESP codec、per-call client、WATCH/MULTI session 和 subscriber loop modules。Runtime hot path 优先使用 Rust redis-proxy sidecar。 | -| `shared/owner-lease.js`、`shared/owner-protocol.js`、`shared/owner-forwarder.js` | D1 和 DO runtimes 共用的 owner lease parsing、generation counters、key derivation、fence matching、staged Redis owner writes 和 owner-forwarding HTTP mechanics。 | +| `shared/redis-lock.js` | Control 和 Auth 共用的 token-fenced Redis lock creation、acquire、renewal 和 best-effort token-scoped release。 | +| `shared/owner-lease.js`、`shared/owner-protocol.js`、`shared/owner-forwarder.js` | Control 与 D1/DO runtimes 共用的 optimistic retry loop、owner lease parsing、generation counters、key derivation、fence matching、staged Redis owner writes 和 owner-forwarding HTTP mechanics。 | | `shared/auth-roles.js` | Role table、principal validation、reserved namespace policy 和 auth action capabilities。 | | `shared/auth-token.js` | Control 和 auth 共用的 `x-admin-token` sanitizer。 | | `shared/internal-auth.js` | JS caller 和 receiver 共用的 internal mesh auth header / token helpers。 | | `shared/secret-envelope.js`、`shared/secret-keys.js` | Secret envelope encryption/decryption、canonical base64/JSON handling、AAD binding helpers 和 secret Redis key construction。 | +| `shared/base64.js` | Workerd tiers 共用的无依赖 byte/text base64 codec；在 `nodejs_compat` 下使用 `Buffer` fast path。 | | `shared/hex.js`、`shared/random-id.js`、`shared/errors.js` | byte-to-hex rendering、random hex ids 和 string-only error message extraction 的无依赖小 primitive。 | | `shared/observability.js` | JS tiers 的 structured logger、metrics registry、request-id helpers 和 log-level handling。 | | `shared/respond.js` | 共享 HTTP response、JSON error、Prometheus text、best-effort response body discard 和 `x-request-id` echo helpers。 | -| `shared/bounded-body.js` | 共享 bounded request body byte/text readers；各 tier 自己把 limit error 映射为对应 HTTP error contract。 | -| `shared/ns-pattern.js` | Namespace、worker、binding、queue、KV id、module path、reserved object-key 和 reserved namespace grammars。 | -| `shared/version.js` | Worker version formatting 和 bundle key helpers。 | +| `shared/bounded-body.js` | 共享 bounded byte-stream 和 request-body readers；各 tier 自己把 limit error 映射为对应 contract。 | +| `shared/ns-pattern.js` | Platform-domain normalization，以及 namespace、worker、binding、queue、KV/D1/R2 id、module path、reserved object-key 和 reserved namespace grammars。 | +| `shared/version.js` | Worker version formatting，以及 worker 和 route-plane Redis key helpers。 | | `shared/workerd-compat-flags.js` | 上游 workerd experimental compatibility enable flags 的 pinned mirror，用于在 cold-load 前拒绝 tenant metadata。 | | `shared/queue-keys.js` | JavaScript queue key helpers，供 tests 和 cross-tier key-shape checks 使用。 | | `shared/route-projection.js` | Control writer、delete check 和 gateway reader 共用的紧凑 pattern-route projection encoding。 | | `shared/d1-*.js`、`shared/sql-splitter.js` | Runtime、d1-runtime、control 和 tests 共用的 D1 parameter、data-field、transport、timeout、query-wire 和 SQL splitting utilities。 | | `shared/fnv1a32.js` | Runtime-side shard 和 slot hashing 共用的 JavaScript FNV-1a helpers。 | | `shared/s3-query.js` | s3-cleanup system worker 使用的 S3 query encoder；runtime R2 在 `runtime/r2-utils.js` 保留同一套 standalone helper，因为该文件会作为 worker source 注入。 | +| `shared/s3-retry.js` | runtime R2 与 s3-cleanup worker 共用的 idempotent S3 POST 有界瞬态重试策略。 | | `shared/s3-xml.js` | Control R2、runtime R2 和 system cleanup 路径共用的 S3 XML parsing helpers。 | | `shared/worker-id.js` | Gateway、runtime、DO runtime 和 tests 共用的 `x-worker-id` formatting、parsing 和 runtime-load identity grammar。 | | `shared/cron-time.js` | Control 侧 cron parsing 和 slot-alignment helpers；scheduler advancement 使用 Rust `croner`。 | diff --git a/docs/testing.md b/docs/testing.md index 503e24e..00f6a8f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -201,6 +201,9 @@ Prefer the narrow helper that matches the response or fixture source: static-analysis utilities (`source-scan.js`), and `mocks/` for mocked Cloudflare runtime surfaces. `load-shared-module.js` owns repository module source loading, data-URL construction, and import-specifier rewrite helpers; + load import-free contract owners such as `shared/version.js` and + `shared/ns-pattern.js` directly with `repositoryFileUrl(...)` instead of + reimplementing their grammar in a stub. use it instead of ad hoc `readFileSync(...).replace(...)` chains when a test rewrites repo module imports. Use `readRepositoryFile(...)` plus `applyModuleReplacements(...)` for local source rewrites, @@ -216,7 +219,9 @@ Prefer the narrow helper that matches the response or fixture source: `control/handlers/*` unit tests. Use it for state/env/log/metrics/backend injection instead of declaring another file-local `control-shared` data-URL stub when the handler under test follows the shared control entrypoint - shape. + shape. The harness stub imports pure response, bounded JSON body, coded abort, + and optimistic retry contracts from their production owners; keep only + state-bound wiring and test instrumentation local to the stub. `load-runtime-r2-binding.js` owns the `runtime/bindings/r2.js` host binding module graph, including SigV4Client/fetch recording hooks. Extend that loader for additional R2 host tests instead of copying the replacement graph back @@ -227,7 +232,10 @@ Prefer the narrow helper that matches the response or fixture source: `mocks/fake-redis.js` owns the reusable in-memory Redis session/multi subset for unit tests, including command tracing for reads, writes, batch helpers, and multi ops; extend it when multiple tests need the same Redis command - shape instead of adding another file-local mock. + shape instead of adding another file-local mock. Use + `sharedRedisStubUrl(...)` for data-URL `shared-redis` replacements so tests + share the fake `WatchError` constructor and production `decodeBulk` + semantics; append only graph-specific state or client exports. `request-body.js` owns typed JSON request-body parsing for mock backends; use it when tests capture `RequestInit.body` instead of open-coding `JSON.parse(...)`. `do-envelope.js` owns test decoding for Durable Object diff --git a/docs/testing.zh.md b/docs/testing.zh.md index 516782c..373d98c 100644 --- a/docs/testing.zh.md +++ b/docs/testing.zh.md @@ -164,10 +164,10 @@ npm install -g @wdl-dev/cli@1.4.0 - 被 JavaScript 和 Rust 两侧测试共同读取的 cross-language JSON fixture 放在 `tests/fixtures/`。JavaScript 测试用 `readRepositoryJson(...)`；Rust 测试用 `include_str!(...)` 读取同一个文件。这类 fixture 只 pin 测试合同和 drift guard，不新增 runtime shared owner。 - `tests/helpers/` 是 unit test helper 的归属： - 包含 `load-*.js` ESM data-URL loader 家族（每个被测 repo 模块一份 loader，例如 `load-auth-lib.js`、`load-control-lib.js`、`load-runtime-dispatch.js`）、共享 fixture（`runtime-dispatch-fixtures.js`、`control-shared-stub.js`）、静态分析工具（`source-scan.js`），以及 mock 化 Cloudflare runtime 表面的 `mocks/`。 - - `load-shared-module.js` 负责仓库模块源码读取、data-URL 构造和 import specifier rewrite helper；测试需要重写 repo module import 时，用它代替手写 `readFileSync(...).replace(...)` 链。局部 source rewrite 使用 `readRepositoryFile(...)` 加 `applyModuleReplacements(...)`；需要读取 repo module source 并一次性应用 replacements 时用 `readRepositoryModuleSource(...)`；可复用 repo module 用 `repositoryModuleDataUrl(...)`；import-map 形态的 stub 用 `importSpecifierReplacements(...)`。这是被测试锁住的约定：`tests/unit/test-helper-style-contracts.test.js` 会拒绝共享 loader helper 之外新增的 source-producer `.replace(...)` module rewrite 链，包括直接链、变量中转形态，以及对 `applyModuleReplacements(...)` 产物的二次 rewrite。 - - `control-handler-harness.js` 负责 `control/handlers/*` unit tests 共用的 `control-shared` harness；handler 遵循 shared control entrypoint 形态时，用它注入 state/env/log/metrics/backend service 和 Redis，不再新增 file-local `control-shared` data-URL stub。 + - `load-shared-module.js` 负责仓库模块源码读取、data-URL 构造和 import specifier rewrite helper；`shared/version.js`、`shared/ns-pattern.js` 这类无 import 的 contract owner 应通过 `repositoryFileUrl(...)` 直接加载，不在 stub 中重写其 grammar。测试需要重写 repo module import 时，用它代替手写 `readFileSync(...).replace(...)` 链。局部 source rewrite 使用 `readRepositoryFile(...)` 加 `applyModuleReplacements(...)`；需要读取 repo module source 并一次性应用 replacements 时用 `readRepositoryModuleSource(...)`；可复用 repo module 用 `repositoryModuleDataUrl(...)`；import-map 形态的 stub 用 `importSpecifierReplacements(...)`。这是被测试锁住的约定：`tests/unit/test-helper-style-contracts.test.js` 会拒绝共享 loader helper 之外新增的 source-producer `.replace(...)` module rewrite 链，包括直接链、变量中转形态，以及对 `applyModuleReplacements(...)` 产物的二次 rewrite。 + - `control-handler-harness.js` 负责 `control/handlers/*` unit tests 共用的 `control-shared` harness；handler 遵循 shared control entrypoint 形态时，用它注入 state/env/log/metrics/backend service 和 Redis，不再新增 file-local `control-shared` data-URL stub。Harness stub 从 production owner 导入纯 response、bounded JSON body、coded abort 和 optimistic retry contract；本地只保留 state-bound wiring 和 test instrumentation。 - `load-runtime-r2-binding.js` 负责 `runtime/bindings/r2.js` host binding module graph，包括 SigV4Client/fetch 记录钩子；新增 R2 host 测试时扩展这个 loader，不把 replacement graph 复制回测试文件。`load-auth-index.js` 负责 auth entrypoint mock state accessor；测试使用 `authMockState(...)`、`authLogs(...)` 和 `lastAuthLog(...)`，不直接触碰 `globalThis.__authMockState`。 - - `mocks/fake-redis.js` 负责 unit tests 可复用的 in-memory Redis session/multi 子集，并为读写、batch helper 和 multi ops 提供 command trace；多个测试需要同一 Redis command shape 时扩展它，不再新增 file-local mock。 + - `mocks/fake-redis.js` 负责 unit tests 可复用的 in-memory Redis session/multi 子集，并为读写、batch helper 和 multi ops 提供 command trace；多个测试需要同一 Redis command shape 时扩展它，不再新增 file-local mock。Data-URL `shared-redis` replacement 使用 `sharedRedisStubUrl(...)`，共享 fake `WatchError` constructor 和 production `decodeBulk` semantics；只追加 graph-specific state 或 client exports。 - `request-body.js` 负责 mock backend 捕获 `RequestInit.body` 时的 typed JSON request-body 解析；这类测试不要再直接手写 `JSON.parse(...)`。`do-envelope.js` 负责 Durable Object binary invoke envelope 的测试解码，不再新增 file-local `decodeDoEnvelope()`。style-contract 套件会拒绝新的直接 mock request-body `JSON.parse(...)` 和 file-local DO envelope decoder。 - `mock-global.js` 负责临时替换 global 和 global property；单个 lexical async scope 用 `withMockedGlobal(...)` / `withMockedProperty(...)`，只有文件需要 before/after hook 所有权时才用 install-style helper。`mock-fetch.js` 是这套 helper 的 typed fetch wrapper。style-contract 套件会扫描仓库作者维护的 `.js` 测试，并拒绝这些 helper 之外新增的直接非 `__` `globalThis. = ...` 赋值、直接 `console.log/warn/error/info = ...` 赋值，以及测试常用 built-in/global property hook（`process.stderr.*`、`AbortSignal.*`、`Object.*`、`Headers.prototype.*`、`Array.prototype.*`）的直接赋值。 - unit tests 不起 Docker，通过这些 loader 导入。 diff --git a/gateway/index.js b/gateway/index.js index c9c90a8..cc89f53 100644 --- a/gateway/index.js +++ b/gateway/index.js @@ -19,6 +19,7 @@ import { createHttpRequestScope, } from "shared-request-scope"; import { + deleteGatewayInternalHeaders, isWebSocketUpgrade, normalizeRequestHost, } from "gateway-lib"; @@ -37,6 +38,7 @@ import { runtimeForwardOutcome, } from "gateway-runtime"; import { formatWorkerId } from "shared-worker-id"; +import { platformDomainFromEnv } from "shared-ns-pattern"; // Re-exported so workerd's capnp `durableObjectNamespaces` entry resolves // the class name against this worker's exports. @@ -62,19 +64,6 @@ function notFoundResponse() { } const bindLogLevel = createLogLevelBinder(); -const INTERNAL_HEADER_PREFIX = "x-wdl-"; -const INTERNAL_FORWARD_HEADERS = [ - "x-worker-id", - "x-worker-prefix", -]; - -/** @param {Headers} headers */ -function stripClientInternalHeaders(headers) { - for (const header of INTERNAL_FORWARD_HEADERS) headers.delete(header); - for (const header of [...headers.keys()]) { - if (header.toLowerCase().startsWith(INTERNAL_HEADER_PREFIX)) headers.delete(header); - } -} export default { /** @@ -86,7 +75,6 @@ export default { bindLogLevel(env); const url = new URL(request.url); const redis = createGatewayRedis(env.REDIS_ADDR); - const platformDomain = normalizeRequestHost(env.PLATFORM_DOMAIN || "workers.local").toLowerCase(); /** @type {string | null} */ let namespace = null; /** @type {string | null} */ @@ -102,10 +90,11 @@ export default { extras: () => ({ namespace, worker, version }), }); - const subscriberStart = ensureGatewaySubscriber(env.REDIS_ADDR); - if (subscriberStart) ctx.waitUntil(subscriberStart); - try { + const platformDomain = platformDomainFromEnv(env); + const subscriberStart = ensureGatewaySubscriber(env.REDIS_ADDR); + if (subscriberStart) ctx.waitUntil(subscriberStart); + if (url.pathname === "/healthz" && request.method === "GET") { scope.setRoute("healthz"); return scope.respond(jsonResponse(200, { @@ -145,7 +134,7 @@ export default { url.pathname = dispatch.forwardPath; const forwardRequest = new Request(url.toString(), request); - stripClientInternalHeaders(forwardRequest.headers); + deleteGatewayInternalHeaders(forwardRequest.headers); // Loader branches carry worker identity + prefix; control is // infrastructure and has no worker id to inject. if (dispatch.bindingName !== "CONTROL") { diff --git a/gateway/lib.js b/gateway/lib.js index e083977..25aa0ec 100644 --- a/gateway/lib.js +++ b/gateway/lib.js @@ -8,6 +8,20 @@ * @typedef {{ slot: string, reason: string }} PatternError */ +const INTERNAL_HEADER_PREFIX = "x-wdl-"; +const INTERNAL_FORWARD_HEADERS = [ + "x-worker-id", + "x-worker-prefix", +]; + +/** @param {Headers} headers */ +export function deleteGatewayInternalHeaders(headers) { + for (const name of INTERNAL_FORWARD_HEADERS) headers.delete(name); + for (const name of [...headers.keys()]) { + if (name.toLowerCase().startsWith(INTERNAL_HEADER_PREFIX)) headers.delete(name); + } +} + /** @param {string} s */ export function escapeRegex(s) { return RegExp.escape(s); diff --git a/gateway/runtime.js b/gateway/runtime.js index fcb847c..1699fdc 100644 --- a/gateway/runtime.js +++ b/gateway/runtime.js @@ -11,7 +11,7 @@ import { recordRedisCommand, } from "shared-observability"; import { isValidRouteNs } from "shared-ns-pattern"; -import { patternsKey, routesKey } from "shared-version"; +import { DECLARED_HOSTS_KEY, NAMESPACES_KEY, patternsKey, routesKey } from "shared-version"; import { isCanonicalPatternHost, sortPatterns } from "gateway-lib"; /** @type {Set | null} */ @@ -28,7 +28,6 @@ let websocketProxyDetachedConnections = 0; let websocketProxyBufferedMessages = 0; const MAX_ROUTE_CACHE_ENTRIES = 10_000; const MAX_PATTERN_CACHE_ENTRIES = 10_000; -const DECLARED_HOSTS_KEY = "declared-hosts"; const utf8Decoder = new TextDecoder(); export const metrics = new MetricsRegistry(); @@ -56,7 +55,7 @@ export function createGatewayRedis(redisAddr) { /** @param {RedisClient} redis */ export async function ensureKnownNs(redis) { - if (knownNs === null) knownNs = new Set(await redis.sMembers("namespaces")); + if (knownNs === null) knownNs = new Set(await redis.sMembers(NAMESPACES_KEY)); return knownNs; } diff --git a/gateway/websocket.js b/gateway/websocket.js index ec9ec12..065042c 100644 --- a/gateway/websocket.js +++ b/gateway/websocket.js @@ -1,5 +1,6 @@ import { logStructured } from "shared-observability"; import { discardResponseBody } from "shared-respond"; +import { deleteGatewayInternalHeaders } from "gateway-lib"; /** * @typedef {{ @@ -41,11 +42,6 @@ function websocketClosedNormally(evt) { const RECONNECT_DELAYS_MS = [0, 100, 250, 500, 1000, 2000, 5000]; const MAX_BUFFERED_CLIENT_MESSAGES = 64; const MAX_BUFFERED_CLIENT_MESSAGES_CAP = 1024; -const INTERNAL_HEADER_PREFIX = "x-wdl-"; -const INTERNAL_FORWARD_HEADERS = [ - "x-worker-id", - "x-worker-prefix", -]; const proxyOptionsByEnv = new WeakMap(); /** @param {unknown} value */ @@ -58,10 +54,7 @@ function parseNonNegativeInteger(value) { /** @param {Headers} headers */ function publicWebSocketResponseHeaders(headers) { const out = new Headers(headers); - for (const name of INTERNAL_FORWARD_HEADERS) out.delete(name); - for (const name of [...out.keys()]) { - if (name.toLowerCase().startsWith(INTERNAL_HEADER_PREFIX)) out.delete(name); - } + deleteGatewayInternalHeaders(out); return out; } diff --git a/knip.jsonc b/knip.jsonc index 5b0f14d..3436be5 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -9,6 +9,10 @@ "d1-runtime/index.js", "do-runtime/index.js", "gateway/index.js", + // Injected into loaded workers and imported by the generated binding + // wrapper, so this is an external module entry rather than an ignored + // source file. Its static dependencies remain covered by Knip. + "runtime/d1-client.js", "runtime/do-owner-network.js", "runtime/index.js", "runtime/tail-worker.js", @@ -27,14 +31,6 @@ "**/*.mjs", "**/*.cjs" ], - // These sources are embedded as text by workerd capnp and injected into - // loaded workers, so no ordinary JS import points at the files. - "ignoreFiles": [ - "runtime/_wdl-d1-params.js", - "runtime/_wdl-d1-transport.js", - "runtime/_wdl-sql-splitter.js", - "runtime/d1-client.js" - ], "ignore": [ "shared/vendor/**", "examples/**", diff --git a/package-lock.json b/package-lock.json index fd5ab68..ce26d41 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,8 @@ "eslint": "^10.6.0", "globals": "^17.7.0", "knip": "^6.25.0", - "typescript": "^6.0.3" + "typescript": "^6.0.3", + "yaml": "^2.9.0" }, "engines": { "node": ">=24" diff --git a/package.json b/package.json index 4b1c8cf..09a6760 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "scripts": { "lint": "eslint .", - "lint:unused": "knip --include exports,types,files --no-config-hints", + "lint:unused": "knip --include exports,types,files --no-config-hints --treat-tag-hints-as-errors", "typecheck": "tsc --noEmit", "typecheck:strict": "tsc --project tsconfig.strict.json", "test:unit": "node --test --test-reporter=spec 'tests/unit/**/*.test.js'", @@ -42,7 +42,8 @@ "eslint": "^10.6.0", "globals": "^17.7.0", "knip": "^6.25.0", - "typescript": "^6.0.3" + "typescript": "^6.0.3", + "yaml": "^2.9.0" }, "dependencies": { "workerd": "1.20260701.1" diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index dd3b2a8..2623039 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -32,16 +32,19 @@ const DO_CONNECT_STRIP_HEADERS = [ ...DO_FETCH_STRIP_HEADERS, ]; const OWNER_ENDPOINT_UNAVAILABLE_STATUSES = new Set([502, 503, 504]); -const OWNER_RACE_RETRY_CODES = new Set([ +export const OWNER_RACE_RETRY_CODES = new Set([ "stale_owner_generation", "owner_claim_raced", + "owner_fence_missing", "owner_lease_expired", "owner_lease_too_short", ]); -const OWNER_HINT_STALE_CODES = new Set([ +export const OWNER_HINT_STALE_CODES = new Set([ "stale_owner_generation", "owner_claim_raced", + "owner_fence_missing", "owner_lease_expired", + "stale_owner_storage", "owner_lease_too_short", "owner_renew_raced", "owner_release_raced", @@ -67,6 +70,21 @@ const RPC_RESERVED_METHODS = new Set(["fetch", "alarm"]); * @typedef {{ ns: string, worker: string, version: string, doStorageId: string, className: string }} DoBindingProps * @typedef {{ ownerKey: string, taskId: string, endpoint: string, generation: number }} DoOwnerHint * @typedef {(url: string, init?: RequestInit) => Promise} DoFetch + * @typedef {{ + * get: (value: unknown) => unknown, + * set: (value: unknown, hint: unknown) => unknown, + * delete: (value: unknown) => unknown, + * }} DoOwnerHintCache + * @typedef {{ + * routerFetch: DoFetch, + * routerUrl: string, + * ownerFetch: DoFetch | typeof fetch | null | undefined, + * ownerPath: string, + * init: RequestInit, + * cache: DoOwnerHintCache, + * hintKey: string, + * replayOwnerUnavailable?: boolean, + * }} DoOwnerHintDispatchOptions */ /** @param {DoBindingProps} props @param {string} objectName */ @@ -361,6 +379,12 @@ export function isWebSocketUpgrade(request) { return (request.headers.get("Upgrade") || "").toLowerCase() === "websocket"; } +/** @param {Request} request */ +export function replayOwnerUnavailableForFetch(request) { + const method = request.method.toUpperCase(); + return method === "GET" || method === "HEAD"; +} + /** * @param {DoBindingProps} props * @param {string} objectName @@ -544,6 +568,55 @@ export async function dispatchDoRequestWithOwnerHint({ } } +/** + * @param {DoOwnerHintDispatchOptions} options + * @param {boolean} retryOwnerRace + */ +async function dispatchDoWithHintCache({ + routerFetch, + routerUrl, + ownerFetch, + ownerPath, + init, + cache, + hintKey, + replayOwnerUnavailable = false, +}, retryOwnerRace) { + let response = await dispatchDoRequestWithOwnerHint({ + routerFetch, + routerUrl, + ownerFetch, + ownerPath, + init, + cachedHint: /** @type {DoOwnerHint | null} */ (cache.get(hintKey)), + rememberHint: (hint) => cache.set(hintKey, hint), + clearHint: () => cache.delete(hintKey), + staleCachedResponse: staleDoOwnerHintResponse, + bypassOwnerHintResponse: retryOwnerRace + ? retryableOwnerRaceResponse + : async (_response) => false, + replayOwnerUnavailable, + }); + if (retryOwnerRace && await retryableOwnerRaceResponse(response)) { + cache.delete(hintKey); + response = await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); + } + return stripOwnerHintHeaders(response); +} + +/** @param {DoOwnerHintDispatchOptions} options */ +export async function dispatchDoInvokeWithHintCache(options) { + return await dispatchDoWithHintCache(options, true); +} + +/** @param {DoOwnerHintDispatchOptions} options */ +export async function dispatchDoConnectWithHintCache(options) { + return await dispatchDoWithHintCache({ + ...options, + replayOwnerUnavailable: false, + }, false); +} + /** @param {Response} response */ export function ownerEndpointUnavailableResponse(response) { return OWNER_ENDPOINT_UNAVAILABLE_STATUSES.has(response.status) && diff --git a/runtime/bindings/do.js b/runtime/bindings/do.js index ebc63ce..f82ef94 100644 --- a/runtime/bindings/do.js +++ b/runtime/bindings/do.js @@ -4,15 +4,13 @@ import { DO_INVOKE_URL, connectHeaders, doOwnerHintCacheKey, - dispatchDoRequestWithOwnerHint, + dispatchDoConnectWithHintCache, + dispatchDoInvokeWithHintCache, fetchInvokeInit, isWebSocketUpgrade, - retryableOwnerRaceResponse, + replayOwnerUnavailableForFetch, rpcInvokeInit, rpcResultFromResponse, - staleDoOwnerHintResponse, - stripOwnerHintHeaders, - withoutOwnerHintOptIn, } from "runtime-do-transport"; import { createOwnerHintCache } from "runtime-owner-hint-cache"; import { withInternalAuth } from "shared-internal-auth"; @@ -25,12 +23,6 @@ import { withInternalAuth } from "shared-internal-auth"; const ownerHintCache = createOwnerHintCache(); -/** @param {Request} request */ -function replayOwnerUnavailableForFetch(request) { - const method = request.method.toUpperCase(); - return method === "GET" || method === "HEAD"; -} - /** @param {DurableObjectNamespace} binding @returns {DoBinding} */ function doBinding(binding) { return /** @type {DoBinding} */ (/** @type {unknown} */ (binding)); @@ -59,24 +51,16 @@ async function dispatchInvokeWithOwnerHint(binding, init, objectName, { replayOw const backend = doBinding(binding).env.DO_BACKEND; const props = propsOf(binding); const hintKey = doOwnerHintCacheKey(props, objectName); - const response = await dispatchDoRequestWithOwnerHint({ + return await dispatchDoInvokeWithHintCache({ routerFetch: (url, requestInit) => backend.fetch(url, requestInit), routerUrl: DO_INVOKE_URL, ownerFetch: fetch, ownerPath: "/internal/do/invoke", init, - cachedHint: /** @type {import("runtime-do-transport").DoOwnerHint | null} */ (ownerHintCache.get(hintKey)), - rememberHint: (hint) => ownerHintCache.set(hintKey, hint), - clearHint: () => ownerHintCache.delete(hintKey), - staleCachedResponse: staleDoOwnerHintResponse, - bypassOwnerHintResponse: retryableOwnerRaceResponse, + cache: ownerHintCache, + hintKey, replayOwnerUnavailable, }); - if (await retryableOwnerRaceResponse(response)) { - ownerHintCache.delete(hintKey); - return stripOwnerHintHeaders(await backend.fetch(DO_INVOKE_URL, withoutOwnerHintOptIn(init))); - } - return stripOwnerHintHeaders(response); } export class DurableObjectNamespace extends WorkerEntrypoint { @@ -97,18 +81,15 @@ export class DurableObjectNamespace extends WorkerEntrypoint { headers: withInternalAuth(connectHeaders(props, objectName, request, requestId), doBinding(this).env), }; const hintKey = doOwnerHintCacheKey(props, objectName); - return stripOwnerHintHeaders(await dispatchDoRequestWithOwnerHint({ + return await dispatchDoConnectWithHintCache({ routerFetch: (url, requestInit) => doBinding(this).env.DO_BACKEND.fetch(url, requestInit), routerUrl: DO_CONNECT_URL, ownerFetch: fetch, ownerPath: "/internal/do/connect", init, - cachedHint: /** @type {import("runtime-do-transport").DoOwnerHint | null} */ (ownerHintCache.get(hintKey)), - rememberHint: (hint) => ownerHintCache.set(hintKey, hint), - clearHint: () => ownerHintCache.delete(hintKey), - staleCachedResponse: staleDoOwnerHintResponse, - replayOwnerUnavailable: false, - })); + cache: ownerHintCache, + hintKey, + }); } const init = await fetchInvokeInit(props, objectName, request, requestId); init.headers = withInternalAuth(init.headers, doBinding(this).env); diff --git a/runtime/bindings/kv.js b/runtime/bindings/kv.js index 0784e9a..f1bd6a5 100644 --- a/runtime/bindings/kv.js +++ b/runtime/bindings/kv.js @@ -14,8 +14,10 @@ // via JSRPC and execute through the Redis proxy sidecar. import { WorkerEntrypoint } from "cloudflare:workers"; -import { base64ToBytes, bytesToBase64, toBytes } from "runtime-lib"; +import { toBytes } from "runtime-lib"; import { recordBindingOperation } from "runtime-metrics"; +import { base64ToBytes, bytesToBase64 } from "shared-base64"; +import { readBoundedStreamBytes } from "shared-bounded-body"; import { discardResponseBody } from "shared-respond"; import { proxyEndpoint as buildProxyEndpoint, @@ -74,41 +76,6 @@ function stringifyKvMetadata(value) { return json; } -/** - * @param {ReadableStream} stream - * @returns {Promise} - */ -async function readStreamWithLimit(stream) { - const reader = stream.getReader(); - const chunks = []; - let total = 0; - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); - total += chunk.byteLength; - if (total > KV_VALUE_MAX_BYTES) { - try { - await reader.cancel(`KV put: value exceeds ${KV_VALUE_MAX_BYTES} byte limit`); - } catch {} - assertKvValueSize(total); - } - chunks.push(chunk); - } - } finally { - try { reader.releaseLock(); } catch {} - } - if (chunks.length === 1) return chunks[0]; - const out = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - out.set(chunk, offset); - offset += chunk.byteLength; - } - return out; -} - /** * @param {unknown} value * @returns {number} @@ -280,7 +247,11 @@ export class KV extends WorkerEntrypoint { return recordBindingOperation(serviceName(kv), "kv", "put", async () => { let bytes; if (value instanceof ReadableStream) { - bytes = await readStreamWithLimit(value); + bytes = await readBoundedStreamBytes( + value, + KV_VALUE_MAX_BYTES, + () => new TypeError(`KV put: value exceeds ${KV_VALUE_MAX_BYTES} byte limit`) + ); } else { bytes = toBytes(value); assertKvValueSize(bytes.byteLength); diff --git a/runtime/bindings/r2.js b/runtime/bindings/r2.js index 4265f72..5cd2ce6 100644 --- a/runtime/bindings/r2.js +++ b/runtime/bindings/r2.js @@ -3,6 +3,7 @@ import { SigV4Client } from "@wdl-dev/aws-sigv4"; import { recordBindingOperation } from "runtime-metrics"; import { serviceNameFromEnv } from "runtime-bindings-proxy"; import { discardResponseBody } from "shared-respond"; +import { S3_TRANSIENT_RETRIES, fetchRetryableS3Post } from "shared-s3-retry"; import { assertR2BufferSize, encodeS3KeyPath, @@ -27,9 +28,6 @@ import { parseListObjects, xmlEscape, xmlUnescape } from "runtime-bindings-r2-xm const DELETE_OBJECTS_BATCH_SIZE = 1000; const LIST_INCLUDE_HEAD_CONCURRENCY = 16; const S3_CLIENT_CACHE_MAX_ENTRIES = 128; -const S3_TRANSIENT_RETRIES = 10; -const S3_RETRY_BASE_MS = 50; -const S3_RETRY_MAX_MS = 5_000; const utf8Encoder = new TextEncoder(); const s3Cache = new Map(); const s3ByBucket = new WeakMap(); @@ -113,47 +111,6 @@ async function mapWithConcurrency(items, concurrency, fn) { return out; } -/** @param {number} ms */ -function delay(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** @param {number} attempt */ -function s3RetryDelayMs(attempt) { - return Math.min(S3_RETRY_BASE_MS * 2 ** attempt, S3_RETRY_MAX_MS); -} - -/** @param {Response} response */ -function isTransientS3Response(response) { - return response.status === 429 || response.status >= 500; -} - -/** - * DeleteObjects is a POST, but deleting the same key set is safe to retry. - * Keep this scoped to S3 POST calls with known idempotent semantics. - * @param {{ fetch(url: string, init?: RequestInit): Promise }} client - * @param {string} url - * @param {RequestInit} init - */ -async function fetchRetryableS3Post(client, url, init) { - for (let attempt = 0; attempt <= S3_TRANSIENT_RETRIES; attempt += 1) { - let response; - try { - response = await client.fetch(url, init); - } catch (err) { - if (attempt === S3_TRANSIENT_RETRIES) throw err; - await delay(Math.random() * s3RetryDelayMs(attempt)); - continue; - } - if (attempt === S3_TRANSIENT_RETRIES || !isTransientS3Response(response)) { - return response; - } - await discardResponseBody(response); - await delay(Math.random() * s3RetryDelayMs(attempt)); - } - throw new Error("unreachable S3 retry loop exit"); -} - /** @param {R2BucketBinding} bucket */ function serviceName(bucket) { return serviceNameFromEnv(bucket.env); diff --git a/runtime/bindings/r2/metadata.js b/runtime/bindings/r2/metadata.js index 0094b55..cdcb52f 100644 --- a/runtime/bindings/r2/metadata.js +++ b/runtime/bindings/r2/metadata.js @@ -1,4 +1,9 @@ -import { r2RangeAndSizeFromHeaders } from "runtime-r2-utils"; +import { + R2_HTTP_METADATA_FIELDS, + r2CacheExpiryFromHeaders, + r2RangeAndSizeFromHeaders, + setR2CacheExpiryHeader, +} from "runtime-r2-utils"; /** * @typedef {{ offset?: number, length?: number, suffix?: number, header?: string }} R2RangeOptions @@ -79,22 +84,12 @@ export function headersWithRequestId(requestMeta = {}) { function httpMetadataFromHeaders(headers) { /** @type {Record} */ const out = {}; - const fields = [ - ["content-type", "contentType"], - ["content-language", "contentLanguage"], - ["content-disposition", "contentDisposition"], - ["content-encoding", "contentEncoding"], - ["cache-control", "cacheControl"], - ]; - for (const [header, field] of fields) { + for (const [field, header] of R2_HTTP_METADATA_FIELDS) { const value = headers.get(header); if (value) out[field] = value; } - const expires = headers.get("expires"); - if (expires) { - const ms = new Date(expires).getTime(); - if (Number.isFinite(ms)) out.cacheExpiry = ms; - } + const cacheExpiry = r2CacheExpiryFromHeaders(headers); + if (cacheExpiry != null) out.cacheExpiry = cacheExpiry; return out; } @@ -171,19 +166,10 @@ export function metaFromPutResponse(userKey, headers, bodySize, options = {}) { * @param {R2Options} [httpMetadata] */ export function applyHttpMetadata(headers, httpMetadata = {}) { - const pairs = [ - ["contentType", "content-type"], - ["contentLanguage", "content-language"], - ["contentDisposition", "content-disposition"], - ["contentEncoding", "content-encoding"], - ["cacheControl", "cache-control"], - ]; - for (const [field, header] of pairs) { + for (const [field, header] of R2_HTTP_METADATA_FIELDS) { if (httpMetadata[field]) headers.set(header, String(httpMetadata[field])); } - if (httpMetadata.cacheExpiry != null) { - headers.set("expires", new Date(Number(httpMetadata.cacheExpiry)).toUTCString()); - } + setR2CacheExpiryHeader(headers, httpMetadata.cacheExpiry); } /** diff --git a/runtime/bindings/service.js b/runtime/bindings/service.js index c2c834b..06b7329 100644 --- a/runtime/bindings/service.js +++ b/runtime/bindings/service.js @@ -9,9 +9,8 @@ // entrypoint. Protected by service-binding RPC integration tests. import { WorkerEntrypoint } from "cloudflare:workers"; -import { createLoaderCallback } from "runtime-load"; -import { recordLoadedWorker } from "runtime-state"; -import { emitRuntimeTailEvent, fetchTailFields } from "runtime-tail-forwarder"; +import { getLoadedWorkerStub } from "runtime-load"; +import { fetchTailFields, startTailEnvelope } from "runtime-tail-forwarder"; import { INTERNAL_AUTH_HEADER } from "shared-internal-auth"; const SERVICE_BINDING_INTERNAL_HEADERS = [ @@ -72,49 +71,24 @@ export class ServiceBinding extends WorkerEntrypoint { requestId, }; const baseFields = fetchTailFields(forwarded); - const startedAt = Date.now(); - const startTailEvent = emitRuntimeTailEvent({ + const tail = startTailEnvelope({ env: self.env, ctx: self.ctx, identity, event: "worker_fetch", - phase: "start", fields: baseFields, }); try { const response = await targetStub(self, requestId) .getEntrypoint(targetEntrypoint ?? null, targetEntrypointOpts(self)) .fetch(forwarded); - emitRuntimeTailEvent({ - env: self.env, - ctx: self.ctx, - identity, - event: "worker_fetch", - phase: "finish", - after: startTailEvent, - fields: { - ...baseFields, - outcome: "ok", - status: response.status, - duration_ms: Date.now() - startedAt, - }, + tail.finish({ + outcome: "ok", + status: response.status, }); return response; } catch (err) { - emitRuntimeTailEvent({ - env: self.env, - ctx: self.ctx, - identity, - event: "worker_fetch", - phase: "finish", - after: startTailEvent, - fields: { - ...baseFields, - outcome: "error", - error: err instanceof Error ? err.message : undefined, - duration_ms: Date.now() - startedAt, - }, - }); + tail.finishError(err); throw err; } } @@ -151,7 +125,9 @@ function serviceBinding(binding) { function targetStub(self, requestId) { const { targetNs, targetWorker, targetVersion } = self.ctx.props; const targetId = targetIdOf(self.ctx.props); - const baseCallback = createLoaderCallback({ + // The pinned target version may not be active, so service-binding cold-loads + // record the isolate but deliberately leave sibling eviction disabled. + return getLoadedWorkerStub({ requestId, env: self.env, ctx: self.ctx, @@ -159,14 +135,7 @@ function targetStub(self, requestId) { worker: targetWorker, version: targetVersion, workerId: targetId, - }); - // Record but do not evict — the pinned version may not be the - // active one, so aborting siblings here would thrash gateway traffic. - return self.env.LOADER.get(targetId, async () => { - const code = await baseCallback(); - recordLoadedWorker(targetId); - return code; - }); + }).stub; } /** diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 4c54e77..c4795a8 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -91,11 +91,14 @@ const loaderWorker :Workerd.Worker = ( (name = "hex.js", esModule = embed "../shared/hex.js"), (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), + (name = "shared-base64", esModule = embed "../shared/base64.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), + (name = "shared-s3-retry", esModule = embed "../shared/s3-retry.js"), + (name = "respond.js", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-env", esModule = embed "../shared/env.js"), @@ -151,6 +154,11 @@ const controlWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "../control/index.js"), (name = "control-shared", esModule = embed "../control/shared.js"), + (name = "control-errors", esModule = embed "../control/errors.js"), + (name = "control-json-body", esModule = embed "../control/json-body.js"), + (name = "control-optimistic", esModule = embed "../control/optimistic.js"), + (name = "shared-owner-lease", esModule = embed "../shared/owner-lease.js"), + (name = "control-workflows-client", esModule = embed "../control/workflows-client.js"), (name = "control-lib", esModule = embed "../control/lib.js"), (name = "control-d1-model", esModule = embed "../control/d1-model.js"), (name = "control-d1-runtime-client", esModule = embed "../control/d1-runtime-client.js"), @@ -162,6 +170,7 @@ const controlWorker :Workerd.Worker = ( (name = "control-bindings", esModule = embed "../control/bindings.js"), (name = "control-topology", esModule = embed "../control/topology.js"), (name = "control-env-budget", esModule = embed "../control/env-budget.js"), + (name = "runtime-load-env-build", esModule = embed "load/env-build.js"), (name = "control-worker-code-budget", esModule = embed "../control/worker-code-budget.js"), (name = "do-runtime-load-code-budget", esModule = embed "../do-runtime/load-code-budget.js"), (name = "control-lifecycle-indexes", esModule = embed "../control/lifecycle-indexes.js"), @@ -195,15 +204,19 @@ const controlWorker :Workerd.Worker = ( (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-hex", esModule = embed "../shared/hex.js"), (name = "shared-random-id", esModule = embed "../shared/random-id.js"), + (name = "shared-redis-lock", esModule = embed "../shared/redis-lock.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "base64.js", esModule = embed "../shared/base64.js"), (name = "assets-token.js", esModule = embed "../shared/assets-token.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), + (name = "shared-s3-retry", esModule = embed "../shared/s3-retry.js"), + (name = "respond.js", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "ns-pattern.js", esModule = embed "../shared/ns-pattern.js"), (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-auth-token", esModule = embed "../shared/auth-token.js"), (name = "shared-auth-roles", esModule = embed "../shared/auth-roles.js"), @@ -300,8 +313,10 @@ const authWorker :Workerd.Worker = ( (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-auth-token", esModule = embed "../shared/auth-token.js"), (name = "shared-auth-roles", esModule = embed "../shared/auth-roles.js"), + (name = "shared-version", esModule = embed "../shared/version.js"), (name = "shared-hex", esModule = embed "../shared/hex.js"), (name = "shared-random-id", esModule = embed "../shared/random-id.js"), + (name = "shared-redis-lock", esModule = embed "../shared/redis-lock.js"), ], compatibilityDate = "2026-04-24", compatibilityFlags = ["nodejs_compat"], @@ -324,6 +339,7 @@ const tailWorker :Workerd.Worker = ( (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-bindings-proxy", esModule = embed "bindings/proxy.js"), + (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), ], diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index d83c9f4..9819b04 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -90,11 +90,14 @@ const loaderWorker :Workerd.Worker = ( (name = "hex.js", esModule = embed "../shared/hex.js"), (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), + (name = "shared-base64", esModule = embed "../shared/base64.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), + (name = "shared-s3-retry", esModule = embed "../shared/s3-retry.js"), + (name = "respond.js", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-env", esModule = embed "../shared/env.js"), @@ -153,6 +156,7 @@ const tailWorker :Workerd.Worker = ( (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-bindings-proxy", esModule = embed "bindings/proxy.js"), + (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), ], diff --git a/runtime/dispatch.js b/runtime/dispatch.js index a6313c0..97217b4 100644 --- a/runtime/dispatch.js +++ b/runtime/dispatch.js @@ -14,7 +14,11 @@ import { normalizeQueuedDispatchBody, normalizeScheduledDispatchBody, } from "runtime-lib"; -import { emitRuntimeTailEvent, fetchTailFields } from "runtime-tail-forwarder"; +import { + emitRuntimeTailEvent, + fetchTailFields, + startTailEnvelope, +} from "runtime-tail-forwarder"; import { _stringifyWorkflowBackendBodyForTest as stringifyWorkflowBackendBodyForTest, _stringifyWorkflowJsonForTest as stringifyWorkflowJsonForTest, @@ -135,35 +139,6 @@ function workflowTailFields(run) { }; } -/** @param {{ env: RuntimeEnv, ctx: RuntimeCtx, identity: RuntimeIdentity, event: string, fields: Record }} opts */ -function startTailEnvelope({ env, ctx, identity, event, fields }) { - const startedAt = Date.now(); - const startTailEvent = emitRuntimeTailEvent({ - env, ctx, identity, - event, - phase: "start", - fields, - }); - return { - /** @param {Record} extraFields */ - finish(extraFields) { - const durationMs = Date.now() - startedAt; - emitRuntimeTailEvent({ - env, ctx, identity, - event, - phase: "finish", - after: startTailEvent, - fields: { - ...fields, - ...extraFields, - duration_ms: durationMs, - }, - }); - return durationMs; - }, - }; -} - /** @param {WorkerDispatchArgs} args */ export async function handleFetchDispatch({ request, stub, scope, env, ctx, identity }) { const tail = startTailEnvelope({ @@ -180,10 +155,7 @@ export async function handleFetchDispatch({ request, stub, scope, env, ctx, iden return scope.respond(response); } catch (err) { scope.markError(err); - tail.finish({ - outcome: "error", - error: errorMessage(err), - }); + tail.finishError(err); return scope.respond(internalErrorResponse(502, "runtime_error", "Runtime error", scope.requestId)); } } diff --git a/runtime/dispatch/workflow-json.js b/runtime/dispatch/workflow-json.js index 4666959..03ea2e6 100644 --- a/runtime/dispatch/workflow-json.js +++ b/runtime/dispatch/workflow-json.js @@ -1,5 +1,6 @@ -const WORKFLOW_RESULT_BYTES_MAX = 1024 * 1024; -const WORKFLOW_BACKEND_REQUEST_BYTES_MAX = 2 * 1024 * 1024; +export const WORKFLOW_RESULT_BYTES_MAX = 1024 * 1024; +export const WORKFLOW_BACKEND_REQUEST_BYTES_MAX = 2 * 1024 * 1024; +export const WORKFLOW_PAYLOAD_TOO_LARGE_CODE = "workflow_payload_too_large"; const WORKFLOW_JSON_ENCODE_CHARS = 8192; const WORKFLOW_JSON_FLUSH_CHARS = 8192; const utf8Encoder = new TextEncoder(); @@ -20,7 +21,7 @@ export function workflowStepError(code, message) { */ function workflowPayloadTooLarge(kind, maxBytes = WORKFLOW_RESULT_BYTES_MAX) { return workflowStepError( - "workflow_payload_too_large", + WORKFLOW_PAYLOAD_TOO_LARGE_CODE, `Workflow ${kind} exceeds the ${maxBytes} byte limit` ); } @@ -255,7 +256,6 @@ export function stringifyWorkflowResult(value, kind) { return stringifyWorkflowJson(value, kind, WORKFLOW_RESULT_BYTES_MAX); } -/** @lintignore data-URL unit tests import this hook from a rewritten module. */ /** * @param {unknown} value * @param {number} [maxBytes] @@ -264,7 +264,6 @@ export function _stringifyWorkflowJsonForTest(value, maxBytes = WORKFLOW_RESULT_ return stringifyWorkflowJson(value, "test value", maxBytes); } -/** @lintignore data-URL unit tests import this hook from a rewritten module. */ /** @param {string} path @param {unknown} body */ export function _stringifyWorkflowBackendBodyForTest(path, body) { return workflowBackendBody(path, body); diff --git a/runtime/do-client.js b/runtime/do-client.js index 48e8451..e294ae6 100644 --- a/runtime/do-client.js +++ b/runtime/do-client.js @@ -3,27 +3,19 @@ import { DO_INVOKE_URL, connectHeaders, doOwnerHintCacheKey, - dispatchDoRequestWithOwnerHint, + dispatchDoConnectWithHintCache, + dispatchDoInvokeWithHintCache, fetchInvokeInit, isWebSocketUpgrade, - retryableOwnerRaceResponse, + replayOwnerUnavailableForFetch, rpcInvokeInit, rpcResultFromResponse, - staleDoOwnerHintResponse, - stripOwnerHintHeaders, - withoutOwnerHintOptIn, } from "./_wdl-do-transport.js"; import { createOwnerHintCache } from "./_wdl-owner-hint-cache.js"; import { requestIdFromOptions } from "./_wdl-request-id.js"; const ownerHintCache = createOwnerHintCache(); -/** @param {Request} request */ -function replayOwnerUnavailableForFetch(request) { - const method = request.method.toUpperCase(); - return method === "GET" || method === "HEAD"; -} - /** * @typedef {{ fetch(url: string, init?: RequestInit): Promise }} DoBackend * @typedef {{ @@ -201,28 +193,24 @@ export class DurableObjectNamespace { method: "GET", headers: connectHeaders(props, objectName, request, requestId), }; - const response = await this.#fetchWithOwnerHint( + return await this.#dispatchWithOwnerHint( + dispatchDoConnectWithHintCache, DO_CONNECT_URL, "/internal/do/connect", init, doOwnerHintCacheKey(props, objectName), false ); - return stripOwnerHintHeaders(response); } const init = await fetchInvokeInit(props, objectName, request, requestId); - const response = await this.#fetchWithOwnerHint( + return await this.#dispatchWithOwnerHint( + dispatchDoInvokeWithHintCache, DO_INVOKE_URL, "/internal/do/invoke", init, doOwnerHintCacheKey(props, objectName), replayOwnerUnavailableForFetch(request) ); - if (await retryableOwnerRaceResponse(response)) { - ownerHintCache.delete(doOwnerHintCacheKey(props, objectName)); - return stripOwnerHintHeaders(await backend.fetch(DO_INVOKE_URL, withoutOwnerHintOptIn(init))); - } - return stripOwnerHintHeaders(response); } /** @param {string} objectName @param {string} method @param {unknown[]} args @param {string | null} [requestId] */ @@ -236,32 +224,26 @@ export class DurableObjectNamespace { } const props = requireBindingProps(this.#props); const init = rpcInvokeInit(props, objectName, method, args, requestId); - const response = await this.#fetchWithOwnerHint( + const response = await this.#dispatchWithOwnerHint( + dispatchDoInvokeWithHintCache, DO_INVOKE_URL, "/internal/do/invoke", init, doOwnerHintCacheKey(props, objectName), false ); - let routed = response; - if (await retryableOwnerRaceResponse(response)) { - // Owner-hint following already happened above. This retry is only for - // generation races after reaching an owner, so it falls back through the - // router with hint opt-in disabled. - ownerHintCache.delete(doOwnerHintCacheKey(props, objectName)); - routed = await backend.fetch(DO_INVOKE_URL, withoutOwnerHintOptIn(init)); - } - return await rpcResultFromResponse(stripOwnerHintHeaders(routed)); + return await rpcResultFromResponse(response); } /** + * @param {typeof dispatchDoInvokeWithHintCache | typeof dispatchDoConnectWithHintCache} dispatch * @param {string} routerUrl * @param {string} ownerPath * @param {RequestInit} init * @param {string} hintKey * @param {boolean} replayOwnerUnavailable */ - async #fetchWithOwnerHint(routerUrl, ownerPath, init, hintKey, replayOwnerUnavailable) { + async #dispatchWithOwnerHint(dispatch, routerUrl, ownerPath, init, hintKey, replayOwnerUnavailable) { const backend = this.#backend; const ownerNetwork = this.#ownerNetwork; if (!backend || typeof backend.fetch !== "function") { @@ -273,18 +255,14 @@ export class DurableObjectNamespace { : null; /** @type {import("./_wdl-do-transport.js").DoFetch} */ const routerFetch = (url, requestInit) => backend.fetch(url, requestInit); - return await dispatchDoRequestWithOwnerHint({ + return await dispatch({ routerFetch, routerUrl, ownerFetch, ownerPath, init, - cachedHint: /** @type {import("./_wdl-do-transport.js").DoOwnerHint | null} */ ( - ownerHintCache.get(hintKey) - ), - rememberHint: (hint) => ownerHintCache.set(hintKey, hint), - clearHint: () => ownerHintCache.delete(hintKey), - staleCachedResponse: staleDoOwnerHintResponse, + cache: ownerHintCache, + hintKey, replayOwnerUnavailable, }); } diff --git a/runtime/index.js b/runtime/index.js index d5e79e2..20fc17c 100644 --- a/runtime/index.js +++ b/runtime/index.js @@ -11,11 +11,9 @@ import { parseDispatchWorkerId } from "shared-worker-id"; import { handleFetchDispatch, } from "runtime-dispatch"; -import { createLoaderCallback } from "runtime-load"; +import { getLoadedWorkerStub } from "runtime-load"; import { bindRuntime, - evictSiblings, - recordLoadedWorker, runtimeServiceAllowsNamespace, } from "runtime-state"; @@ -64,23 +62,13 @@ export default class Runtime extends WorkerEntrypoint { return scope.respond(jsonError(403, "runtime_pool_mismatch", "Worker namespace is not allowed in this runtime pool")); } - const baseCallback = createLoaderCallback({ + // Gateway-routed traffic is the authoritative "this version is active" + // signal. Service-binding cold-loads use the same helper without eviction. + const { stub } = getLoadedWorkerStub({ requestId: scope.requestId, env, ctx, ns: namespace, worker: workerName, version, workerId, metrics: runtime.metrics, log: runtime.log, - }); - // Factory fires only on cache miss. Gateway-routed traffic is the - // authoritative "this version is active" signal, so this is the - // one place we evict historical siblings; service-binding cold - // loads record but do not evict. - const stub = env.LOADER.get(workerId, async () => { - const code = await baseCallback(); - recordLoadedWorker(workerId); - ctx.waitUntil( - evictSiblings({ env, workerId, log: runtime.log }) - .catch(() => {}) - ); - return code; + evictOnLoad: true, }); const forwardRequest = new Request(request); diff --git a/runtime/internal.js b/runtime/internal.js index 6d90a57..51c0fe5 100644 --- a/runtime/internal.js +++ b/runtime/internal.js @@ -2,7 +2,7 @@ // runtime/index.js on :8081; scheduler/workflows call this worker on :8088. import { WorkerEntrypoint } from "cloudflare:workers"; -import { formatWorkerId, parseDispatchWorkerId } from "shared-worker-id"; +import { parseDispatchWorkerId } from "shared-worker-id"; import { internalErrorResponse, jsonError, @@ -21,11 +21,9 @@ import { readWorkflowNotifyDispatch, readWorkflowRunDispatch, } from "runtime-dispatch"; -import { createLoaderCallback } from "runtime-load"; +import { getLoadedWorkerStub } from "runtime-load"; import { bindRuntime, - evictSiblings, - recordLoadedWorker, runtimeServiceAllowsNamespace, } from "runtime-state"; @@ -51,24 +49,12 @@ export { InternalAuthBackend } from "runtime-bindings-internal-auth-backend"; * @param {{ env: RuntimeInternalEnv, ctx: { waitUntil(promise: Promise): void }, runtime: RuntimeBinding, scope: RuntimeScope, namespace: string, workerName: string, version: string, evictOnLoad?: boolean }} opts */ function loadStub({ env, ctx, runtime, scope, namespace, workerName, version, evictOnLoad = false }) { - const workerId = formatWorkerId({ namespace, worker: workerName, version }); - const baseCallback = createLoaderCallback({ + return getLoadedWorkerStub({ requestId: scope.requestId, env, ctx, - ns: namespace, worker: workerName, version, workerId, + ns: namespace, worker: workerName, version, metrics: runtime.metrics, log: runtime.log, + evictOnLoad, }); - const stub = env.LOADER.get(workerId, async () => { - const code = await baseCallback(); - recordLoadedWorker(workerId); - if (evictOnLoad) { - ctx.waitUntil( - evictSiblings({ env, workerId, log: runtime.log }) - .catch(() => {}) - ); - } - return code; - }); - return { workerId, stub }; } /** @param {{ serviceName: string }} runtime @param {string} namespace */ diff --git a/runtime/lib.js b/runtime/lib.js index 0d6b977..2e6f469 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -1,11 +1,13 @@ // Pure helpers for the runtime worker. +import { base64ToBytes, bytesToBase64 } from "shared-base64"; import { isWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; -const BASE64_CHUNK_SIZE = 0x8000; const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); +export { base64ToBytes, bytesToBase64 }; + // Coerce a KV.put value into Uint8Array. ReadableStream is handled by the // caller (await body); here we stick to sync inputs. /** @param {unknown} value */ @@ -19,23 +21,6 @@ export function toBytes(value) { throw new Error("KV put: value must be string | ArrayBuffer | typed array | ReadableStream"); } -/** @param {Uint8Array} bytes @returns {string} */ -export function bytesToBase64(bytes) { - let binary = ""; - for (let offset = 0; offset < bytes.length; offset += BASE64_CHUNK_SIZE) { - binary += String.fromCharCode(...bytes.subarray(offset, offset + BASE64_CHUNK_SIZE)); - } - return btoa(binary); -} - -/** @param {string} value @returns {Uint8Array} */ -export function base64ToBytes(value) { - const binary = atob(value); - const out = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i += 1) out[i] = binary.charCodeAt(i); - return out; -} - export const MAX_QUEUE_DELAY_SECONDS = 86_400; export const QUEUE_CONTENT_TYPES = Object.freeze({ JSON: "json", diff --git a/runtime/load.js b/runtime/load.js index d33574e..88904b0 100644 --- a/runtime/load.js +++ b/runtime/load.js @@ -5,7 +5,8 @@ import { bundleToWorkerCode } from "runtime-lib"; import { formatError } from "shared-observability"; import { withInternalAuth } from "shared-internal-auth"; import { discardResponseBody } from "shared-respond"; -import { parseRuntimeLoadWorkerId } from "shared-worker-id"; +import { formatWorkerId, parseRuntimeLoadWorkerId } from "shared-worker-id"; +import { evictSiblings, recordLoadedWorker } from "runtime-state"; import { analyzeRuntimeMeta, injectRuntimeModulesForHostBindings, @@ -348,3 +349,56 @@ export function createLoaderCallback({ requestId, env, ctx, ns, worker, version, return workerCode; }; } + +/** + * @template TStub + * @param {{ + * requestId: string | null, + * env: RuntimeLoaderEnv & { LOADER: { get(id: string, factory: () => Promise): TStub } }, + * ctx: { waitUntil(promise: Promise): void }, + * ns: string, + * worker: string, + * version: string, + * workerId?: string, + * metrics?: RuntimeLoaderMetrics | null, + * log?: ((level: string, event: string, fields?: Record) => void) | null, + * evictOnLoad?: boolean, + * }} options + * @returns {{ workerId: string, stub: TStub }} + */ +export function getLoadedWorkerStub({ + requestId, + env, + ctx, + ns, + worker, + version, + workerId = formatWorkerId({ namespace: ns, worker, version }), + metrics = null, + log = null, + evictOnLoad = false, +}) { + const baseCallback = createLoaderCallback({ + requestId, + env, + ctx, + ns, + worker, + version, + workerId, + metrics, + log, + }); + const stub = env.LOADER.get(workerId, async () => { + const code = await baseCallback(); + recordLoadedWorker(workerId); + if (evictOnLoad) { + ctx.waitUntil( + evictSiblings({ env, workerId, log }) + .catch(() => {}) + ); + } + return code; + }); + return { workerId, stub }; +} diff --git a/runtime/load/env-build.js b/runtime/load/env-build.js index 161aea9..40749a5 100644 --- a/runtime/load/env-build.js +++ b/runtime/load/env-build.js @@ -178,11 +178,16 @@ function materializeServiceBinding({ name, spec, ns, worker, nsSecrets, workerSe // Secrets-only (not vars): a credential moved from secrets → vars // stops reaching the target instead of silently continuing. // Missing keys are dropped; control warned at deploy. + const requiredCallerSecrets = Array.isArray(spec.requiredCallerSecrets) + ? spec.requiredCallerSecrets + : []; /** @type {Record | undefined} */ - let callerSecrets; - if (Array.isArray(spec.requiredCallerSecrets) && spec.requiredCallerSecrets.length) { - callerSecrets = {}; - for (const k of spec.requiredCallerSecrets) { + const callerSecrets = requiredCallerSecrets.length + // workerd's JSRPC props serializer rejects null-prototype objects. + ? {} + : undefined; + if (callerSecrets) { + for (const k of requiredCallerSecrets) { if (typeof k !== "string") continue; if (Object.hasOwn(workerSecrets, k)) { callerSecrets[k] = workerSecrets[k]; diff --git a/runtime/load/wrapper-generate.js b/runtime/load/wrapper-generate.js index 1235257..e18035b 100644 --- a/runtime/load/wrapper-generate.js +++ b/runtime/load/wrapper-generate.js @@ -1,3 +1,15 @@ +const WDL_ABORT_SHIM_SOURCE = `// Reserved name (WDL_RESERVED_ENTRYPOINT_RE) — control rejects user +// [[exports]] / [[services]] matches; implicit user exports collide +// silently via the export-* shadow rule. +export class __WdlAbort__ extends WorkerEntrypoint { + abort(reason) { + abortIsolate(reason ?? "wdl-evict"); + } +}`; + +const DEFAULT_EXPORT_SOURCE_SNIPPET = "const source = Function.prototype.toString.call(raw);"; +const DEFAULT_EXPORT_CLASS_TEST_SOURCE = "/^\\s*class\\b/.test(source)"; + /** @param {string} userMainSpecifier */ export function generateAbortShimWrapperModule(userMainSpecifier) { const userMain = JSON.stringify(`./${userMainSpecifier}`); @@ -6,14 +18,7 @@ import * as user from ${userMain}; import { WorkerEntrypoint, abortIsolate } from "cloudflare:workers"; export * from ${userMain}; -// Reserved name (WDL_RESERVED_ENTRYPOINT_RE) — control rejects user -// [[exports]] / [[services]] matches; implicit user exports collide -// silently via the export-* shadow rule. -export class __WdlAbort__ extends WorkerEntrypoint { - abort(reason) { - abortIsolate(reason ?? "wdl-evict"); - } -} +${WDL_ABORT_SHIM_SOURCE} export class __WdlWorkflowNotify__ extends WorkerEntrypoint { fetch() { @@ -25,8 +30,8 @@ const raw = user.default; let wrappedDefault = raw; if (typeof raw === "function") { - const source = Function.prototype.toString.call(raw); - if (!/^\\s*class\\b/.test(source)) { + ${DEFAULT_EXPORT_SOURCE_SNIPPET} + if (!${DEFAULT_EXPORT_CLASS_TEST_SOURCE}) { wrappedDefault = { fetch(request, env, ctx) { return raw.call(undefined, request, env, ctx); @@ -83,14 +88,7 @@ ${workflowImport} // declared entrypoints get facade-aware env even when star exports are present. ${starExport} -// Reserved name (WDL_RESERVED_ENTRYPOINT_RE) — control rejects user -// [[exports]] / [[services]] matches; implicit user exports collide -// silently via the export-* shadow rule. -export class __WdlAbort__ extends WorkerEntrypoint { - abort(reason) { - abortIsolate(reason ?? "wdl-evict"); - } -} +${WDL_ABORT_SHIM_SOURCE} export class __WdlWorkflowNotify__ extends WorkerEntrypoint { async fetch(request) { @@ -267,8 +265,8 @@ if (raw && typeof raw === "object") { wrapDefaultFunctionKey(key); } } else if (typeof raw === "function") { - const source = Function.prototype.toString.call(raw); - if (/^\\s*class\\b/.test(source)) { + ${DEFAULT_EXPORT_SOURCE_SNIPPET} + if (${DEFAULT_EXPORT_CLASS_TEST_SOURCE}) { wrappedDefault = class extends raw { constructor(ctx, env) { const requestContext = createRequestContext(); diff --git a/runtime/r2-client.js b/runtime/r2-client.js index 83e4e72..69dedb3 100644 --- a/runtime/r2-client.js +++ b/runtime/r2-client.js @@ -1,8 +1,11 @@ import { + R2_HTTP_METADATA_FIELDS, R2_OBJECT_MAX_BUFFER_BYTES, assertR2BufferSize, normalizeR2ListLimit, normalizeR2ObjectKey, + r2CacheExpiryFromHeaders, + setR2CacheExpiryHeader, } from "./_wdl-r2-utils.js"; import { requestIdFromOptions } from "./_wdl-request-id.js"; @@ -119,15 +122,13 @@ function cappedReadableStream(stream, operation) { /** @param {Headers} headers */ function headersToHttpMetadata(headers) { - const expires = headers.get("expires"); - return { - contentType: headers.get("content-type") || undefined, - contentLanguage: headers.get("content-language") || undefined, - contentDisposition: headers.get("content-disposition") || undefined, - contentEncoding: headers.get("content-encoding") || undefined, - cacheControl: headers.get("cache-control") || undefined, - cacheExpiry: expires ? new Date(expires).getTime() : undefined, - }; + /** @type {AnyRecord} */ + const out = {}; + for (const [field, header] of R2_HTTP_METADATA_FIELDS) { + out[field] = headers.get(header) || undefined; + } + out.cacheExpiry = r2CacheExpiryFromHeaders(headers); + return out; } /** @param {unknown} input */ @@ -140,14 +141,8 @@ function normalizeHttpMetadata(input) { /** @type {AnyRecord} */ const out = {}; const record = /** @type {AnyRecord} */ (input); - for (const key of [ - "contentType", - "contentLanguage", - "contentDisposition", - "contentEncoding", - "cacheControl", - ]) { - if (record[key] != null) out[key] = String(record[key]); + for (const [field] of R2_HTTP_METADATA_FIELDS) { + if (record[field] != null) out[field] = String(record[field]); } if (record.cacheExpiry != null) { const ms = record.cacheExpiry instanceof Date @@ -320,17 +315,11 @@ export class R2Object { /** @param {Headers} headers */ writeHttpMetadata(headers) { const h = this.httpMetadata || {}; - const contentType = stringOrUndefined(h.contentType); - const contentLanguage = stringOrUndefined(h.contentLanguage); - const contentDisposition = stringOrUndefined(h.contentDisposition); - const contentEncoding = stringOrUndefined(h.contentEncoding); - const cacheControl = stringOrUndefined(h.cacheControl); - if (contentType) headers.set("content-type", contentType); - if (contentLanguage) headers.set("content-language", contentLanguage); - if (contentDisposition) headers.set("content-disposition", contentDisposition); - if (contentEncoding) headers.set("content-encoding", contentEncoding); - if (cacheControl) headers.set("cache-control", cacheControl); - if (h.cacheExpiry) headers.set("expires", dateFromUnknown(h.cacheExpiry).toUTCString()); + for (const [field, header] of R2_HTTP_METADATA_FIELDS) { + const value = stringOrUndefined(h[field]); + if (value) headers.set(header, value); + } + setR2CacheExpiryHeader(headers, h.cacheExpiry); } } diff --git a/runtime/r2-utils.js b/runtime/r2-utils.js index d029ce4..6f14800 100644 --- a/runtime/r2-utils.js +++ b/runtime/r2-utils.js @@ -3,6 +3,28 @@ export const R2_LIST_LIMIT_MAX = 1000; // Keep in sync with shared/ns-pattern.js. This file is embedded into loaded // workers as _wdl-r2-utils.js, so it must stay standalone. export const R2_BUCKET_NAME_RE = /^[a-z0-9][a-z0-9-]{0,62}$/; +export const R2_HTTP_METADATA_FIELDS = Object.freeze([ + Object.freeze(["contentType", "content-type"]), + Object.freeze(["contentLanguage", "content-language"]), + Object.freeze(["contentDisposition", "content-disposition"]), + Object.freeze(["contentEncoding", "content-encoding"]), + Object.freeze(["cacheControl", "cache-control"]), +]); + +/** @param {Headers} headers */ +export function r2CacheExpiryFromHeaders(headers) { + const expires = headers.get("expires"); + if (!expires) return undefined; + const ms = new Date(expires).getTime(); + return Number.isFinite(ms) ? ms : undefined; +} + +/** @param {Headers} headers @param {unknown} value */ +export function setR2CacheExpiryHeader(headers, value) { + if (value == null) return; + const date = value instanceof Date ? value : new Date(Number(value)); + headers.set("expires", date.toUTCString()); +} /** * @param {unknown} bucketName diff --git a/runtime/runtime.js b/runtime/runtime.js index 74ae686..29147cf 100644 --- a/runtime/runtime.js +++ b/runtime/runtime.js @@ -1,7 +1,7 @@ // Runtime helpers for user-runtime and system-runtime. This module owns // service-name binding, logger setup, metrics access, request-scope creation, // and the per-process loaded-worker registry that drives historical-version -// eviction; runtime/index.js owns loader dispatch and worker event handling. +// eviction; runtime/load.js owns the shared workerLoader cold-load wrapper. import { createLogLevelBinder, diff --git a/runtime/tail-forwarder.js b/runtime/tail-forwarder.js index 00f7eaf..01f0fc7 100644 --- a/runtime/tail-forwarder.js +++ b/runtime/tail-forwarder.js @@ -6,6 +6,7 @@ import { optionalRedisProxyBaseUrl, proxyEndpoint, } from "runtime-bindings-proxy"; +import { errorMessage } from "shared-errors"; import { withInternalAuth } from "shared-internal-auth"; // Active-set cache. Keep the timing constants beside the implementation; @@ -298,3 +299,51 @@ export function emitRuntimeTailEvent({ env, ctx, identity, event, phase, fields ctx.waitUntil(task); return task; } + +/** + * @param {{ + * env: RuntimeTailEnv, + * ctx: { waitUntil?: (promise: Promise) => void }, + * identity: { namespace?: string, workerName?: string, workerId?: string, requestId?: string | null }, + * event: string, + * fields: Record, + * }} options + */ +export function startTailEnvelope({ env, ctx, identity, event, fields }) { + const startedAt = Date.now(); + const startTailEvent = emitRuntimeTailEvent({ + env, ctx, identity, + event, + phase: "start", + fields, + }); + + /** @param {Record} extraFields */ + function finish(extraFields) { + const durationMs = Date.now() - startedAt; + emitRuntimeTailEvent({ + env, ctx, identity, + event, + phase: "finish", + after: startTailEvent, + fields: { + ...fields, + ...extraFields, + duration_ms: durationMs, + }, + }); + return durationMs; + } + + return { + finish, + /** @param {unknown} err @param {Record} [extraFields] */ + finishError(err, extraFields = {}) { + return finish({ + ...extraFields, + outcome: "error", + error: errorMessage(err), + }); + }, + }; +} diff --git a/rust/common/Cargo.toml b/rust/common/Cargo.toml index 87a8aad..9c0ee7a 100644 --- a/rust/common/Cargo.toml +++ b/rust/common/Cargo.toml @@ -14,6 +14,7 @@ path = "src/lib.rs" [features] default = ["axum"] axum = ["dep:axum"] +test-support = [] [dependencies] axum = { workspace = true, features = ["http1"], optional = true } diff --git a/rust/common/src/internal_auth.rs b/rust/common/src/internal_auth.rs index c0f1407..6f690e5 100644 --- a/rust/common/src/internal_auth.rs +++ b/rust/common/src/internal_auth.rs @@ -1,12 +1,17 @@ use std::env; #[cfg(feature = "axum")] -use axum::http::HeaderMap; +use axum::{ + http::{HeaderMap, StatusCode, header::CONTENT_TYPE}, + response::{IntoResponse, Response}, +}; use subtle::ConstantTimeEq; pub const INTERNAL_AUTH_HEADER: &str = "x-wdl-internal-auth"; pub const INTERNAL_AUTH_ENV: &str = "WDL_INTERNAL_AUTH_TOKEN"; pub const INTERNAL_AUTH_PREVIOUS_ENV: &str = "WDL_INTERNAL_AUTH_PREVIOUS_TOKEN"; +pub const INTERNAL_AUTH_FAILURE_CODE: &str = "internal_auth_failed"; +pub const INTERNAL_AUTH_FAILURE_MESSAGE: &str = "Internal authentication failed"; #[derive(Clone, Debug, Eq, PartialEq)] pub struct InternalAuthTokens { @@ -81,10 +86,59 @@ pub fn internal_auth_headers_match(headers: &HeaderMap, expected: &InternalAuthT internal_auth_matches_any(internal_auth_header_value(headers), expected) } +#[cfg(feature = "axum")] +pub fn internal_auth_failure_response() -> Response { + let body = serde_json::json!({ + "error": INTERNAL_AUTH_FAILURE_CODE, + "message": INTERNAL_AUTH_FAILURE_MESSAGE, + }) + .to_string(); + ( + StatusCode::UNAUTHORIZED, + [(CONTENT_TYPE, "application/json")], + body, + ) + .into_response() +} + #[cfg(test)] mod tests { use super::*; + fn contract() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../tests/fixtures/internal-auth-contract.json" + )) + .expect("internal auth contract fixture") + } + + #[test] + fn internal_auth_literals_and_rotation_match_contract() { + let contract = contract(); + assert_eq!(contract["header"], INTERNAL_AUTH_HEADER); + assert_eq!(contract["currentEnv"], INTERNAL_AUTH_ENV); + assert_eq!(contract["previousEnv"], INTERNAL_AUTH_PREVIOUS_ENV); + assert_eq!(contract["failure"]["error"], INTERNAL_AUTH_FAILURE_CODE); + assert_eq!( + contract["failure"]["message"], + INTERNAL_AUTH_FAILURE_MESSAGE + ); + + for case in contract["rotationCases"] + .as_array() + .expect("rotationCases array") + { + let tokens = InternalAuthTokens { + current: case["current"].as_str().expect("current token").to_string(), + previous: case["previous"].as_str().map(str::to_string), + }; + assert_eq!( + internal_auth_matches_any(case["actual"].as_str(), &tokens), + case["accepted"].as_bool().expect("accepted boolean") + ); + } + } + #[test] fn internal_auth_matches_requires_exact_token() { assert!(internal_auth_matches(Some("secret"), "secret")); @@ -135,4 +189,24 @@ mod tests { assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, "".to_string()).is_err()); assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, "tokén".to_string()).is_err()); } + + #[cfg(feature = "axum")] + #[tokio::test] + async fn internal_auth_failure_response_matches_contract() { + let contract = contract(); + let response = internal_auth_failure_response(); + assert_eq!( + u64::from(response.status().as_u16()), + contract["failure"]["status"] + .as_u64() + .expect("failure status") + ); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("internal auth failure body"); + let body: serde_json::Value = + serde_json::from_slice(&body).expect("internal auth failure JSON"); + assert_eq!(body["error"], contract["failure"]["error"]); + assert_eq!(body["message"], contract["failure"]["message"]); + } } diff --git a/rust/common/src/lib.rs b/rust/common/src/lib.rs index bbc239b..cd2c5b0 100644 --- a/rust/common/src/lib.rs +++ b/rust/common/src/lib.rs @@ -5,8 +5,9 @@ //! health probes, shutdown/in-flight tracking, metric storage/formatting, FNV hashing, //! request-id sanitization, internal-auth constants/token matching, identity grammar, //! version and queue-key helpers, Redis connection and EVAL command helpers, time -//! helpers, structured error fields, and UTF-8-safe text helpers. It should not own -//! service protocols, Redis schemas, dispatch policy, or lifecycle behavior. +//! helpers, structured error fields, test-only process-environment overrides, and +//! UTF-8-safe text helpers. It should not own service protocols, Redis schemas, +//! dispatch policy, or lifecycle behavior. pub mod env; pub mod hash; @@ -21,12 +22,21 @@ pub mod redis_conn; pub mod redis_eval; pub mod request_id; pub mod shutdown; +#[cfg(any(test, feature = "test-support"))] +pub mod test_env; pub mod text; pub mod time; pub mod version; #[cfg(test)] pub(crate) mod test_fixtures { + pub(crate) fn observability_contract() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../tests/fixtures/observability-contract.json" + )) + .expect("observability contract fixture parses") + } + pub(crate) fn identity_cases(field: &str) -> Vec<(String, bool)> { let fixture: serde_json::Value = serde_json::from_str(include_str!( "../../../tests/fixtures/cross-language-identity.json" diff --git a/rust/common/src/log.rs b/rust/common/src/log.rs index 0adfec4..df484a2 100644 --- a/rust/common/src/log.rs +++ b/rust/common/src/log.rs @@ -2,6 +2,7 @@ use chrono::{SecondsFormat, Utc}; use serde_json::Value as JsonValue; #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +#[repr(u8)] pub enum LogLevel { Debug = 10, Info = 20, @@ -9,6 +10,24 @@ pub enum LogLevel { Error = 40, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LogStream { + Stdout, + Stderr, +} + +fn should_emit(level: LogLevel, min_level: LogLevel) -> bool { + level >= min_level +} + +fn stream_for_level(level: LogLevel) -> LogStream { + if level == LogLevel::Error { + LogStream::Stderr + } else { + LogStream::Stdout + } +} + impl LogLevel { pub fn parse(value: &str) -> Option { match value.to_ascii_lowercase().as_str() { @@ -45,14 +64,13 @@ pub fn emit_log_line( event: &str, fields: JsonValue, ) { - if level < min_level { + if !should_emit(level, min_level) { return; } let line = log_payload_line(service, level, event, fields); - if level >= LogLevel::Error { - eprintln!("{line}"); - } else { - println!("{line}"); + match stream_for_level(level) { + LogStream::Stdout => println!("{line}"), + LogStream::Stderr => eprintln!("{line}"), } } @@ -119,8 +137,16 @@ fn now_log_ts() -> String { #[cfg(test)] mod tests { use super::*; + use crate::test_fixtures::observability_contract; use serde_json::json; + fn timestamp_shape(value: &str) -> String { + value + .chars() + .map(|ch| if ch.is_ascii_digit() { '0' } else { ch }) + .collect() + } + #[test] fn level_parse_is_case_insensitive() { assert_eq!(LogLevel::parse("debug"), Some(LogLevel::Debug)); @@ -131,10 +157,50 @@ mod tests { } #[test] - fn level_ord_lets_us_gate() { - assert!(LogLevel::Debug < LogLevel::Info); - assert!(LogLevel::Info < LogLevel::Warn); - assert!(LogLevel::Warn < LogLevel::Error); + fn log_levels_gating_and_streams_match_cross_language_fixture() { + let contract = observability_contract(); + let levels = contract["logEnvelope"]["levels"] + .as_array() + .expect("logEnvelope.levels is an array"); + + for entry in levels { + let name = entry["name"].as_str().expect("log level name is a string"); + let priority = entry["priority"] + .as_u64() + .expect("log level priority is a number"); + let stream = entry["stream"] + .as_str() + .expect("log level stream is a string"); + let level = LogLevel::parse(name).expect("fixture log level is supported"); + let actual_stream = match stream_for_level(level) { + LogStream::Stdout => "stdout", + LogStream::Stderr => "stderr", + }; + assert_eq!(u64::from(level as u8), priority, "{name} priority"); + assert_eq!(actual_stream, stream, "{name} stream"); + } + + for min_entry in levels { + let min_name = min_entry["name"] + .as_str() + .expect("minimum log level name is a string"); + let min_priority = min_entry["priority"] + .as_u64() + .expect("minimum log level priority is a number"); + let min_level = LogLevel::parse(min_name).expect("minimum log level is supported"); + for entry in levels { + let name = entry["name"].as_str().expect("log level name is a string"); + let priority = entry["priority"] + .as_u64() + .expect("log level priority is a number"); + let level = LogLevel::parse(name).expect("fixture log level is supported"); + assert_eq!( + should_emit(level, min_level), + priority >= min_priority, + "{name} at minimum {min_name}" + ); + } + } } #[test] @@ -159,6 +225,10 @@ mod tests { #[test] fn log_payload_line_matches_js_envelope_order() { + let contract = observability_contract(); + let ordered_keys = contract["logEnvelope"]["orderedKeys"] + .as_array() + .expect("logEnvelope.orderedKeys is an array"); let line = log_payload_line( "scheduler", LogLevel::Info, @@ -166,25 +236,24 @@ mod tests { json!({ "port": 7070 }), ); - assert!( - line.starts_with(r#"{"ts":"#), - "Rust structured logs should match JS envelope order: {line}" - ); - assert!(line.contains(r#","service":"scheduler","level":"info","event":"started","#)); + let mut position = 0; + for (index, key) in ordered_keys.iter().enumerate() { + let key = key.as_str().expect("ordered log key is a string"); + let marker = format!("{}\"{key}\":", if index == 0 { "{" } else { "," }); + let offset = line[position..] + .find(&marker) + .unwrap_or_else(|| panic!("missing ordered marker {marker:?} in {line}")); + position += offset + marker.len(); + } } #[test] fn log_timestamp_matches_js_iso_millisecond_shape() { + let contract = observability_contract(); + let expected = contract["logEnvelope"]["timestampShape"] + .as_str() + .expect("logEnvelope.timestampShape is a string"); let ts = now_log_ts(); - assert_eq!(ts.len(), "2026-01-01T00:00:00.000Z".len()); - assert_eq!(&ts[4..5], "-"); - assert_eq!(&ts[7..8], "-"); - assert_eq!(&ts[10..11], "T"); - assert_eq!(&ts[13..14], ":"); - assert_eq!(&ts[16..17], ":"); - assert_eq!(&ts[19..20], "."); - assert_eq!(&ts[23..24], "Z"); - assert!(ts[20..23].chars().all(|ch| ch.is_ascii_digit())); - assert!(!ts.contains("+00:00")); + assert_eq!(timestamp_shape(&ts), expected); } } diff --git a/rust/common/src/metrics.rs b/rust/common/src/metrics.rs index b316acc..5986d36 100644 --- a/rust/common/src/metrics.rs +++ b/rust/common/src/metrics.rs @@ -348,16 +348,10 @@ fn summary_sample_cmp(left: &SummarySample, right: &SummarySample) -> std::cmp:: #[cfg(test)] mod tests { use super::*; + use crate::test_fixtures::observability_contract; static PANIC_HOOK_LOCK: Mutex<()> = Mutex::new(()); - fn observability_contract() -> serde_json::Value { - serde_json::from_str(include_str!( - "../../../tests/fixtures/observability-contract.json" - )) - .expect("observability contract fixture parses") - } - #[test] fn metrics_contract_matches_cross_language_fixture() { let contract = observability_contract(); diff --git a/rust/common/src/queue_keys.rs b/rust/common/src/queue_keys.rs index ac11571..27bff43 100644 --- a/rust/common/src/queue_keys.rs +++ b/rust/common/src/queue_keys.rs @@ -5,6 +5,9 @@ use crate::identity::is_valid_runtime_load_ns; pub const QUEUE_CONSUMER_INDEX_KEY: &str = "queue:index:consumers"; pub const QUEUE_STREAM_INDEX_KEY: &str = "queue:index:streams"; pub const QUEUE_DELAYED_INDEX_KEY: &str = "queue:index:delayed"; +pub const QUEUE_DELAYED_WAKE_STREAM: &str = "queue-delayed-wake"; +pub const QUEUE_DELAYED_WAKE_KEY_FIELD: &str = "delayed_key"; +pub const QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD: &str = "visible_at"; pub fn is_valid_queue_name(queue: &str) -> bool { let bytes = queue.as_bytes(); @@ -67,6 +70,25 @@ fn split_queue_key_rest(rest: &str) -> Option<(String, String)> { mod tests { use super::*; use crate::test_fixtures::identity_cases; + use serde_json::{Value as JsonValue, json}; + + fn assert_queue_key_parse_contract(kind: &str, parse: fn(&str) -> Option<(String, String)>) { + let fixture: JsonValue = + serde_json::from_str(include_str!("../../../tests/fixtures/queue-key-parse.json")) + .expect("queue key parse fixture parses"); + for entry in fixture[kind] + .as_array() + .expect("queue key parse fixture field is an array") + { + let key = entry["key"] + .as_str() + .expect("queue key parse fixture key is a string"); + let actual = parse(key) + .map(|(ns, queue)| json!({ "ns": ns, "queue": queue })) + .unwrap_or(JsonValue::Null); + assert_eq!(actual, entry["parsed"], "{kind}:{key:?}"); + } + } #[test] fn queue_name_grammar_matches_cross_language_fixture() { @@ -80,6 +102,9 @@ mod tests { assert_eq!(QUEUE_CONSUMER_INDEX_KEY, "queue:index:consumers"); assert_eq!(QUEUE_STREAM_INDEX_KEY, "queue:index:streams"); assert_eq!(QUEUE_DELAYED_INDEX_KEY, "queue:index:delayed"); + assert_eq!(QUEUE_DELAYED_WAKE_STREAM, "queue-delayed-wake"); + assert_eq!(QUEUE_DELAYED_WAKE_KEY_FIELD, "delayed_key"); + assert_eq!(QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD, "visible_at"); assert_eq!(queue_stream_key("demo", "jobs"), "queue:demo:jobs:s"); assert_eq!(queue_delayed_key("demo", "jobs"), "queue-delayed:demo:jobs"); assert_eq!(queue_dlq_key("demo", "jobs"), "queue:demo:jobs:dlq"); @@ -106,41 +131,9 @@ mod tests { } #[test] - fn parse_queue_keys() { - assert_eq!( - parse_stream_key("queue:demo:jobs:s"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_stream_key("queue:__platform__:jobs:s"), - Some(("__platform__".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_delayed_key("queue-delayed:demo:jobs"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_consumer_key("queue-consumer:demo:jobs"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_stream_key("queue:t:my-queue-1:s"), - Some(("t".to_string(), "my-queue-1".to_string())) - ); - assert_eq!(parse_stream_key("queue:demo:jobs"), None); - assert_eq!(parse_stream_key("queue-delayed:demo:jobs"), None); - assert_eq!(parse_stream_key("queue::jobs:s"), None); - assert_eq!(parse_stream_key("queue:admin:jobs:s"), None); - assert_eq!(parse_stream_key("queue:__community__:jobs:s"), None); - assert_eq!(parse_stream_key("queue:demo:Jobs:s"), None); - assert_eq!(parse_stream_key("queue:demo:jobs:extra:s"), None); - assert_eq!(parse_stream_key("notaqueue:demo:jobs:s"), None); - assert_eq!(parse_stream_key(""), None); - assert_eq!(parse_delayed_key("queue:demo:jobs:s"), None); - assert_eq!(parse_delayed_key("queue-delayed::jobs"), None); - assert_eq!(parse_delayed_key("queue-delayed:demo:"), None); - assert_eq!(parse_consumer_key("queue-consumer:demo"), None); - assert_eq!(parse_consumer_key("queue-consumer:"), None); - assert_eq!(parse_consumer_key("other:demo:jobs"), None); + fn queue_key_parsers_match_cross_language_fixture() { + assert_queue_key_parse_contract("stream", parse_stream_key); + assert_queue_key_parse_contract("delayed", parse_delayed_key); + assert_queue_key_parse_contract("consumer", parse_consumer_key); } } diff --git a/rust/common/src/request_id.rs b/rust/common/src/request_id.rs index a3ebeb9..b041fd1 100644 --- a/rust/common/src/request_id.rs +++ b/rust/common/src/request_id.rs @@ -1,10 +1,14 @@ pub fn sanitize_request_id(raw: &str) -> Option { - let first = raw.split(',').next()?.trim(); + let first = raw + .split(',') + .next()? + .trim_matches(|ch: char| ch.is_whitespace() || ch == '\u{feff}'); if first.is_empty() || first.len() > 128 { return None; } if first.chars().any(|ch| { - ch.is_ascii_whitespace() + ch.is_whitespace() + || ch == '\u{feff}' || ch == '"' || ch == '\\' || (ch as u32) < 0x20 diff --git a/rust/common/src/test_env.rs b/rust/common/src/test_env.rs new file mode 100644 index 0000000..994a335 --- /dev/null +++ b/rust/common/src/test_env.rs @@ -0,0 +1,73 @@ +//! Serialized process-environment overrides for Rust tests. + +use std::ffi::OsString; +use std::sync::Mutex; + +static ENV_LOCK: Mutex<()> = Mutex::new(()); + +struct EnvRestore(Vec<(String, Option)>); + +impl Drop for EnvRestore { + fn drop(&mut self) { + for (key, value) in self.0.iter().rev() { + // SAFETY: ENV_LOCK remains held until after this restore guard drops. + match value { + Some(value) => unsafe { std::env::set_var(key, value) }, + None => unsafe { std::env::remove_var(key) }, + } + } + } +} + +pub fn with_temp_env(key: &str, value: Option<&str>, f: impl FnOnce() -> R) -> R { + with_temp_envs(&[(key, value)], f) +} + +pub fn with_temp_envs(items: &[(&str, Option<&str>)], f: impl FnOnce() -> R) -> R { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _restore = EnvRestore( + items + .iter() + .map(|(key, _)| ((*key).to_string(), std::env::var_os(key))) + .collect(), + ); + for (key, value) in items { + // SAFETY: callers keep all environment reads inside the closure, and every + // WDL test environment override uses this crate-wide lock. + match value { + Some(value) => unsafe { std::env::set_var(key, value) }, + None => unsafe { std::env::remove_var(key) }, + } + } + f() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn restores_multiple_values_after_unwind_and_recovers_poisoned_lock() { + const FIRST: &str = "WDL_COMMON_TEST_ENV_FIRST"; + const SECOND: &str = "WDL_COMMON_TEST_ENV_SECOND"; + let before = [std::env::var_os(FIRST), std::env::var_os(SECOND)]; + + let result = std::panic::catch_unwind(|| { + with_temp_envs(&[(FIRST, Some("one")), (SECOND, None)], || { + assert_eq!(std::env::var(FIRST).as_deref(), Ok("one")); + assert_eq!(std::env::var_os(SECOND), None); + panic!("exercise panic-safe restore"); + }); + }); + assert!(result.is_err()); + assert_eq!(std::env::var_os(FIRST), before[0]); + assert_eq!(std::env::var_os(SECOND), before[1]); + + with_temp_env(FIRST, Some("two"), || { + assert_eq!(std::env::var(FIRST).as_deref(), Ok("two")); + }); + assert_eq!(std::env::var_os(FIRST), before[0]); + } +} diff --git a/rust/redis-proxy/src/kv.rs b/rust/redis-proxy/src/kv.rs index 072d866..052df02 100644 --- a/rust/redis-proxy/src/kv.rs +++ b/rust/redis-proxy/src/kv.rs @@ -12,6 +12,7 @@ use serde_json::{Value, json}; use std::collections::HashMap; use wdl_rust_common::hash::fnv1a32; use wdl_rust_common::identity::is_valid_runtime_load_ns; +use wdl_rust_common::redis_eval::eval_cmd; use crate::observability::Metrics; use crate::{AppError, AppResult, AppState, SERVICE, empty}; @@ -262,15 +263,11 @@ fn hmget_with_raw_byte_budget_command( remaining: usize, fields: &[String], ) -> redis::Cmd { - let mut cmd = redis::cmd("EVAL"); - cmd.arg(HMGET_WITH_RAW_BYTE_BUDGET_SCRIPT) - .arg(1) - .arg(redis_key) - .arg(remaining); - for field in fields { - cmd.arg(field); - } - cmd + let remaining_arg = remaining.to_string(); + let mut args = Vec::with_capacity(fields.len() + 1); + args.push(remaining_arg.as_str()); + args.extend(fields.iter().map(String::as_str)); + eval_cmd(HMGET_WITH_RAW_BYTE_BUDGET_SCRIPT, &[redis_key], &args) } async fn query_hmget_with_raw_byte_budget( @@ -884,6 +881,26 @@ mod tests { assert_eq!(err.code, "internal_error"); } + #[test] + fn kv_binding_id_grammar_matches_cross_language_fixture() { + let fixture: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/cross-language-identity.json" + )) + .expect("cross-language identity fixture must parse"); + let cases = fixture["kvIds"] + .as_array() + .expect("kvIds fixture field must be an array"); + for case in cases { + let value = case["value"] + .as_str() + .expect("kvIds fixture value must be a string"); + let valid = case["valid"] + .as_bool() + .expect("kvIds fixture valid flag must be a boolean"); + assert_eq!(is_valid_kv_binding_id(value), valid, "kvIds:{value:?}"); + } + } + #[test] fn hmget_budget_command_packs_script_key_budget_and_fields() { let fields = vec!["v:first".to_string(), "v:second".to_string()]; diff --git a/rust/redis-proxy/src/lib.rs b/rust/redis-proxy/src/lib.rs index 56b56fa..7cd4ac1 100644 --- a/rust/redis-proxy/src/lib.rs +++ b/rust/redis-proxy/src/lib.rs @@ -16,7 +16,8 @@ use serde_json::{Value, json}; use wdl_rust_common::env::env_u16; use wdl_rust_common::health::healthcheck_http_200; use wdl_rust_common::internal_auth::{ - InternalAuthTokens, internal_auth_headers_match, internal_auth_tokens_from_env, + INTERNAL_AUTH_FAILURE_CODE, INTERNAL_AUTH_FAILURE_MESSAGE, InternalAuthTokens, + internal_auth_failure_response, internal_auth_headers_match, internal_auth_tokens_from_env, }; use wdl_rust_common::metrics::prometheus_response; use wdl_rust_common::redis_conn::RedisConnection; @@ -311,14 +312,7 @@ async fn track_request( if !matches!(route, "healthz" | "metrics") && !internal_auth_headers_match(request.headers(), &state.internal_auth_tokens) { - let response = ( - StatusCode::UNAUTHORIZED, - Json(json!({ - "error": "internal_auth_failed", - "message": "Internal authentication failed", - })), - ) - .into_response(); + let response = internal_auth_failure_response(); record_request_complete( &state, method.as_str(), @@ -326,7 +320,7 @@ async fn track_request( response.status(), request_id.as_deref(), started_at, - Some(("internal_auth_failed", "Internal authentication failed")), + Some((INTERNAL_AUTH_FAILURE_CODE, INTERNAL_AUTH_FAILURE_MESSAGE)), ); return response; } diff --git a/rust/redis-proxy/src/queue.rs b/rust/redis-proxy/src/queue.rs index 7a68d58..b87448b 100644 --- a/rust/redis-proxy/src/queue.rs +++ b/rust/redis-proxy/src/queue.rs @@ -6,8 +6,9 @@ use redis::Pipeline; use serde::{Deserialize, Serialize}; use wdl_rust_common::identity::is_valid_runtime_load_ns; use wdl_rust_common::queue_keys::{ - QUEUE_DELAYED_INDEX_KEY, QUEUE_STREAM_INDEX_KEY, is_valid_queue_name, queue_delayed_key, - queue_stream_key, + QUEUE_DELAYED_INDEX_KEY, QUEUE_DELAYED_WAKE_KEY_FIELD, QUEUE_DELAYED_WAKE_STREAM, + QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD, QUEUE_STREAM_INDEX_KEY, is_valid_queue_name, + queue_delayed_key, queue_stream_key, }; use crate::{AppError, AppResult, AppState, empty}; @@ -15,7 +16,6 @@ use crate::{AppError, AppResult, AppState, empty}; pub(crate) const MAX_QUEUE_MESSAGE_BYTES: usize = 128_000; pub(crate) const MAX_QUEUE_BATCH_MESSAGES: usize = 100; pub(crate) const MAX_QUEUE_BATCH_BYTES: usize = 256_000; -pub(crate) const QUEUE_DELAYED_WAKE_STREAM: &str = "queue-delayed-wake"; const QUEUE_DELAYED_WAKE_MAX_LEN: usize = 1000; const MAX_SAFE_VISIBLE_AT: f64 = 9_007_199_254_740_991.0; @@ -63,9 +63,9 @@ pub(crate) fn pipe_delayed_wake(pipe: &mut Pipeline, delayed_key: &str, visible_ .arg("~") .arg(QUEUE_DELAYED_WAKE_MAX_LEN) .arg("*") - .arg("delayed_key") + .arg(QUEUE_DELAYED_WAKE_KEY_FIELD) .arg(delayed_key) - .arg("visible_at") + .arg(QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD) .arg(visible_at.to_string()); } @@ -399,9 +399,9 @@ mod tests { let packed = String::from_utf8(pipe.get_packed_pipeline()).unwrap(); assert!(packed.contains(QUEUE_DELAYED_WAKE_STREAM)); assert!(packed.contains("MAXLEN")); - assert!(packed.contains("delayed_key")); + assert!(packed.contains(QUEUE_DELAYED_WAKE_KEY_FIELD)); assert!(packed.contains("queue-delayed:demo:jobs")); - assert!(packed.contains("visible_at")); + assert!(packed.contains(QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD)); } #[test] diff --git a/rust/redis-proxy/src/secrets.rs b/rust/redis-proxy/src/secrets.rs index 3e1ce9e..d6b16fd 100644 --- a/rust/redis-proxy/src/secrets.rs +++ b/rust/redis-proxy/src/secrets.rs @@ -6,7 +6,7 @@ use aes_gcm::aead::AeadInOut; use aes_gcm::{Aes256Gcm, KeyInit, Nonce, Tag}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use wdl_rust_common::hash::fnv1a64; use zeroize::Zeroizing; @@ -28,7 +28,7 @@ const DEK_CACHE_SHARDS: usize = 16; const DEK_CACHE_SHARD_LIMIT: usize = DEK_CACHE_LIMIT / DEK_CACHE_SHARDS; const DEK_CACHE_TTL: Duration = Duration::from_secs(60); -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] struct SecretEnvelope { v: u8, @@ -123,6 +123,13 @@ impl SecretEnvelopeDecryptor { let json_bytes = &value[SECRET_ENVELOPE_PREFIX.len()..]; let envelope: SecretEnvelope = serde_json::from_slice(json_bytes) .map_err(|_| secret_decrypt_error("secret envelope JSON is invalid"))?; + let canonical_json = serde_json::to_vec(&envelope) + .map_err(|_| secret_decrypt_error("secret envelope JSON is invalid"))?; + if canonical_json != json_bytes { + return Err(secret_decrypt_error( + "secret envelope JSON is not canonical", + )); + } validate_envelope(&envelope)?; let provider = self.local.as_ref().ok_or_else(|| { secret_decrypt_error("secret envelope local provider is not configured") @@ -352,6 +359,59 @@ fn secret_config_error(message: impl Into) -> AppError { mod tests { use super::*; + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SecretEnvelopeParityFixture { + provider: SecretEnvelopeParityProvider, + vectors: Vec, + rejections: Vec, + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SecretEnvelopeParityProvider { + kid: String, + local_key_b64: String, + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SecretEnvelopeParityVector { + name: String, + plaintext: String, + hash_key: String, + field_name: String, + envelope: String, + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SecretEnvelopeParityRejection { + name: String, + envelope: String, + hash_key: String, + field_name: String, + configured_kid: String, + } + + fn parity_fixture() -> SecretEnvelopeParityFixture { + serde_json::from_str(include_str!( + "../../../tests/fixtures/secret-envelope-parity.json" + )) + .expect("secret envelope parity fixture must parse") + } + + fn parity_vector<'a>( + fixture: &'a SecretEnvelopeParityFixture, + name: &str, + ) -> &'a SecretEnvelopeParityVector { + fixture + .vectors + .iter() + .find(|vector| vector.name == name) + .expect("secret envelope parity vector must exist") + } + fn test_metrics() -> Arc { Arc::new(Metrics::default()) } @@ -364,11 +424,23 @@ mod tests { } } - fn decryptor_with_local(kid: &str) -> SecretEnvelopeDecryptor { + fn decryptor_with_provider( + provider: &SecretEnvelopeParityProvider, + kid: &str, + ) -> SecretEnvelopeDecryptor { + let decoded = decode_canonical_base64( + &provider.local_key_b64, + SECRET_ENVELOPE_LOCAL_KEY_ENV, + false, + ) + .expect("secret envelope parity provider key must be canonical base64"); + let key: [u8; AES_256_KEY_BYTES] = decoded + .try_into() + .expect("secret envelope parity provider key must be 32 bytes"); SecretEnvelopeDecryptor { local: Some(LocalProvider { kid: kid.to_string(), - key: Zeroizing::new(*b"0123456789abcdef0123456789abcdef"), + key: Zeroizing::new(key), }), dek_cache: Arc::new(dek_cache_shards()), metrics: test_metrics(), @@ -415,89 +487,62 @@ mod tests { } #[test] - fn decrypts_js_generated_envelope_vector() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); - let out = decryptor - .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], - ) - .unwrap(); - assert_eq!(out.get("TOKEN").unwrap(), "sensitive-value"); - } - - #[test] - fn decrypts_js_generated_empty_string_vector() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLSmHuPVe4uS7JepH053l0Yt","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"","tag":"TTVUTxM6DfSV+VZ/AODK1w=="}"# - ); - let out = decryptor - .decrypt_hash_entries( - "secrets:demo:api", - vec![("EMPTY".to_string(), envelope.as_bytes().to_vec())], - ) - .unwrap(); - assert_eq!(out.get("EMPTY").unwrap(), ""); - } - - #[test] - fn decrypt_rejects_storage_location_mismatch() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); - let err = decryptor - .decrypt_hash_entries( - "secrets:other:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], - ) - .unwrap_err(); - assert_eq!(err.code, "secret_decrypt_failed"); + fn decrypts_secret_envelope_parity_vectors() { + let fixture = parity_fixture(); + for vector in &fixture.vectors { + let decryptor = decryptor_with_provider(&fixture.provider, &fixture.provider.kid); + let out = decryptor + .decrypt_hash_entries( + &vector.hash_key, + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], + ) + .unwrap_or_else(|err| panic!("{}: {err:?}", vector.name)); + assert_eq!( + out.get(&vector.field_name), + Some(&vector.plaintext), + "{}", + vector.name + ); + } } #[test] - fn decrypt_rejects_unknown_kid() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v2"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); - let err = decryptor - .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], - ) - .unwrap_err(); - assert_eq!(err.code, "secret_decrypt_failed"); + fn rejects_secret_envelope_parity_vectors() { + let fixture = parity_fixture(); + for rejection in &fixture.rejections { + let decryptor = decryptor_with_provider(&fixture.provider, &rejection.configured_kid); + let err = decryptor + .decrypt_hash_entries( + &rejection.hash_key, + vec![( + rejection.field_name.clone(), + rejection.envelope.as_bytes().to_vec(), + )], + ) + .unwrap_err(); + assert_eq!(err.code, "secret_decrypt_failed", "{}", rejection.name); + } } #[test] fn dek_cache_is_scoped_to_storage_location() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); + let fixture = parity_fixture(); + let vector = parity_vector(&fixture, "value"); + let decryptor = decryptor_with_provider(&fixture.provider, &fixture.provider.kid); // First read at the matching location decrypts and caches the DEK. let out = decryptor .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], + &vector.hash_key, + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], ) .unwrap(); - assert_eq!(out.get("TOKEN").unwrap(), "sensitive-value"); + assert_eq!(out.get(&vector.field_name), Some(&vector.plaintext)); assert_eq!( decryptor .dek_cache @@ -510,18 +555,24 @@ mod tests { // Same envelope at the same location hits the cache. let again = decryptor .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], + &vector.hash_key, + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], ) .unwrap(); - assert_eq!(again.get("TOKEN").unwrap(), "sensitive-value"); + assert_eq!(again.get(&vector.field_name), Some(&vector.plaintext)); // Same edek at a different location is a cache miss (the key is scoped // to hash_key/field) and fails closed at DEK decryption via data_key_aad. let err = decryptor .decrypt_hash_entries( "secrets:other:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], ) .unwrap_err(); assert_eq!(err.code, "secret_decrypt_failed"); @@ -537,7 +588,9 @@ mod tests { #[test] fn decrypt_records_expired_dek_cache_evictions() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); + let fixture = parity_fixture(); + let vector = parity_vector(&fixture, "value"); + let decryptor = decryptor_with_provider(&fixture.provider, &fixture.provider.kid); { let mut cache = decryptor.dek_cache[0] .lock() @@ -552,18 +605,16 @@ mod tests { ); } } - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); let out = decryptor .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], + &vector.hash_key, + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], ) .unwrap(); - assert_eq!(out.get("TOKEN").unwrap(), "sensitive-value"); + assert_eq!(out.get(&vector.field_name), Some(&vector.plaintext)); let metrics = decryptor.metrics.render_prometheus(); assert!(metrics.contains(&format!( r#"wdl_secret_dek_cache_evictions_total{{reason="expired",service="redis-proxy"}} {DEK_CACHE_SHARD_LIMIT}"# diff --git a/rust/scheduler/src/cron/dispatch.rs b/rust/scheduler/src/cron/dispatch.rs index 10b1728..7fd6aee 100644 --- a/rust/scheduler/src/cron/dispatch.rs +++ b/rust/scheduler/src/cron/dispatch.rs @@ -7,8 +7,8 @@ use crate::{ redis_fields_with_error, scheduler_fields_with_error, }; -use super::reference::{CronEntry, RefVerdict, classify_ref, parse_ref}; -use super::slot::{lease_key, next_fire_ms, slot_key, slot_ms_for}; +use super::reference::{CronEntry, RefVerdict, classify_ref, cron_worker_key, parse_ref}; +use super::slot::{lease_key, next_fire_ms, slot_expire_at, slot_key, slot_ms_for}; const CRON_CLAIM_ADVANCE_SCRIPT: &str = r#" if redis.call('SET', KEYS[2], ARGV[1], 'NX', 'EX', ARGV[2]) then @@ -30,7 +30,7 @@ async fn claim_and_advance_ref( let from_key = slot_key(from_slot); let lease = lease_key(from_slot, &reference); let to_key = slot_key(to_slot); - let to_expire_at = to_slot / 1000 + 600; + let to_expire_at = slot_expire_at(to_slot); let instance_id = state.instance_id.clone(); let lease_ttl_s = state.config.lease_ttl_s; let lease_ttl_s_arg = lease_ttl_s.to_string(); @@ -107,7 +107,7 @@ async fn process_ref( remove_ref_from_slot(&state, slot_ms, &reference).await?; return Ok(()); }; - let cron_hash_key = format!("crons:{}:{}", parts.ns, parts.worker); + let cron_hash_key = cron_worker_key(&parts.ns, &parts.worker); let cron_id = parts.cron_id.clone(); let (meta_str, entry_str): (Option, Option) = state .redis diff --git a/rust/scheduler/src/cron/reference.rs b/rust/scheduler/src/cron/reference.rs index 4038f36..fb3e197 100644 --- a/rust/scheduler/src/cron/reference.rs +++ b/rust/scheduler/src/cron/reference.rs @@ -1,5 +1,8 @@ use serde::Deserialize; +const CRON_WORKER_KEY_PREFIX: &str = "crons:"; +pub(crate) const CRON_WORKER_KEY_SCAN_PATTERN: &str = "crons:*:*"; + #[derive(Clone, Debug)] pub(crate) struct RefParts { pub(crate) ns: String, @@ -29,6 +32,18 @@ pub(crate) enum RefVerdict { Corrupt, } +pub(crate) fn cron_worker_key(ns: &str, worker: &str) -> String { + format!("{CRON_WORKER_KEY_PREFIX}{ns}:{worker}") +} + +pub(crate) fn parse_cron_worker_key(key: &str) -> Option<(&str, &str)> { + let (ns, worker) = key.strip_prefix(CRON_WORKER_KEY_PREFIX)?.split_once(':')?; + if ns.is_empty() || worker.is_empty() || worker.contains(':') { + return None; + } + Some((ns, worker)) +} + pub(crate) fn ref_for(ns: &str, worker: &str, cron_id: &str, r#gen: i64) -> String { format!("{ns}:{worker}:{cron_id}:{gen}") } @@ -81,6 +96,36 @@ mod tests { use serde_json::json; use super::*; + use crate::test_fixtures::scheduler_projection_contract; + + #[test] + fn cron_projection_contract_matches_control_writer() { + let fixture = scheduler_projection_contract(); + let cron = fixture.cron; + + assert_eq!(cron_worker_key(&cron.ns, &cron.worker), cron.worker_key); + assert_eq!( + parse_cron_worker_key(&cron.worker_key), + Some((cron.ns.as_str(), cron.worker.as_str())) + ); + assert_eq!( + ref_for(&cron.ns, &cron.worker, &cron.cron_id, cron.r#gen), + cron.reference + ); + let parts = parse_ref(&cron.reference).expect("fixture cron reference must parse"); + match classify_ref(&parts, Some(cron.meta.json), Some(cron.entry.json)) { + RefVerdict::Fire { + entry, + active_version, + } => { + assert_eq!(active_version, cron.meta.version); + assert_eq!(entry.cron, cron.entry.cron); + assert_eq!(entry.timezone, cron.entry.timezone); + assert_eq!(entry.r#gen, cron.entry.r#gen); + } + _ => panic!("fixture cron projection must be fireable"), + } + } #[test] fn parse_ref_round_trip() { @@ -105,6 +150,15 @@ mod tests { assert!(parse_ref("demo:hello:abc123:").is_none()); } + #[test] + fn parse_cron_worker_key_rejects_non_worker_hash_keys() { + assert_eq!(parse_cron_worker_key("queue:demo:jobs"), None); + assert_eq!(parse_cron_worker_key("crons:demo"), None); + assert_eq!(parse_cron_worker_key("crons::worker"), None); + assert_eq!(parse_cron_worker_key("crons:demo:"), None); + assert_eq!(parse_cron_worker_key("crons:demo:worker:extra"), None); + } + #[test] fn classify_ref_fire_carries_entry_and_active_version() { let parts = RefParts { diff --git a/rust/scheduler/src/cron/slot.rs b/rust/scheduler/src/cron/slot.rs index d465964..cd91f6d 100644 --- a/rust/scheduler/src/cron/slot.rs +++ b/rust/scheduler/src/cron/slot.rs @@ -14,6 +14,10 @@ pub(crate) fn slot_key(slot_ms: i64) -> String { format!("cron-slot:{slot_ms}") } +pub(crate) fn slot_expire_at(slot_ms: i64) -> i64 { + slot_ms.div_euclid(1000) + 600 +} + pub(crate) fn wait_ms_until_next_slot(now_ms: i64) -> u64 { (slot_ms_for(now_ms) + 60_000 - now_ms).max(1) as u64 } @@ -45,6 +49,7 @@ mod tests { use super::*; use crate::cron::reference::ref_for; + use crate::test_fixtures::scheduler_projection_contract; #[derive(serde::Deserialize)] #[serde(rename_all = "camelCase")] @@ -60,8 +65,13 @@ mod tests { #[test] fn cron_key_builders_compose_scheduler_keys() { + let fixture = scheduler_projection_contract(); + assert_eq!(slot_key(fixture.cron.slot_ms), fixture.cron.slot_key); + assert_eq!( + slot_expire_at(fixture.cron.slot_ms), + fixture.cron.slot_expire_at + ); let reference = ref_for("demo", "hello", "abc123", 7); - assert_eq!(slot_key(1_710_000_000_000), "cron-slot:1710000000000"); assert_eq!( lease_key(1_710_000_000_000, &reference), "cron-lease:1710000000000:demo:hello:abc123:7" diff --git a/rust/scheduler/src/cron/sweep.rs b/rust/scheduler/src/cron/sweep.rs index 7acc5f6..63a5501 100644 --- a/rust/scheduler/src/cron/sweep.rs +++ b/rust/scheduler/src/cron/sweep.rs @@ -6,8 +6,8 @@ use crate::{ AppState, LogLevel, SERVICE, SchedulerResult, indexed_keys_after_backfill, log, now_ms, }; -use super::reference::{CronEntry, ref_for}; -use super::slot::{next_fire_ms, slot_key, slot_ms_for}; +use super::reference::{CRON_WORKER_KEY_SCAN_PATTERN, CronEntry, parse_cron_worker_key, ref_for}; +use super::slot::{next_fire_ms, slot_expire_at, slot_key, slot_ms_for}; const CRON_WORKER_INDEX_KEY: &str = "cron:index:workers"; const CRON_WORKER_INDEX_BACKFILLED_KEY: &str = "cron:index:workers:backfilled"; @@ -19,20 +19,11 @@ async fn cron_worker_keys(state: &AppState) -> Result, redis::RedisE state, CRON_WORKER_INDEX_KEY, CRON_WORKER_INDEX_BACKFILLED_KEY, - "crons:*:*", + CRON_WORKER_KEY_SCAN_PATTERN, ) .await } -fn parse_cron_worker_key(key: &str) -> Option<(&str, &str)> { - let parts = key.split(':').collect::>(); - if parts.len() == 3 && parts[0] == "crons" && !parts[1].is_empty() && !parts[2].is_empty() { - Some((parts[1], parts[2])) - } else { - None - } -} - async fn fetch_cron_hash_chunk( state: &AppState, keys: &[String], @@ -65,7 +56,7 @@ fn cron_slot_writes(slot_refs: BTreeMap>) -> Vec slot, key: slot_key(slot), refs, - expire_at: slot / 1000 + 600, + expire_at: slot_expire_at(slot), }) .collect() } @@ -194,22 +185,12 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { #[cfg(test)] mod tests { use super::*; - - #[test] - fn parse_cron_worker_key_accepts_only_scheduler_cron_hashes() { - assert_eq!( - parse_cron_worker_key("crons:demo:worker"), - Some(("demo", "worker")) - ); - assert_eq!(parse_cron_worker_key("queue:demo:jobs"), None); - assert_eq!(parse_cron_worker_key("crons:demo"), None); - assert_eq!(parse_cron_worker_key("crons::worker"), None); - assert_eq!(parse_cron_worker_key("crons:demo:"), None); - } + use crate::test_fixtures::scheduler_projection_contract; #[test] fn cron_worker_index_keys_are_stable() { - assert_eq!(CRON_WORKER_INDEX_KEY, "cron:index:workers"); + let fixture = scheduler_projection_contract(); + assert_eq!(CRON_WORKER_INDEX_KEY, fixture.cron.worker_index_key); assert_eq!( CRON_WORKER_INDEX_BACKFILLED_KEY, "cron:index:workers:backfilled" diff --git a/rust/scheduler/src/lib.rs b/rust/scheduler/src/lib.rs index 5de9227..cbc8e2e 100644 --- a/rust/scheduler/src/lib.rs +++ b/rust/scheduler/src/lib.rs @@ -13,6 +13,8 @@ mod runtime_client; mod server; mod state; mod tasks; +#[cfg(test)] +mod test_fixtures; mod time; mod workflows; diff --git a/rust/scheduler/src/queue/delayed.rs b/rust/scheduler/src/queue/delayed.rs index 678dc72..fd554ff 100644 --- a/rust/scheduler/src/queue/delayed.rs +++ b/rust/scheduler/src/queue/delayed.rs @@ -6,6 +6,7 @@ use redis::streams::StreamReadReply; use serde_json::json; use tokio::time::sleep; use wdl_rust_common::hash::fnv1a64; +use wdl_rust_common::queue_keys::{QUEUE_DELAYED_WAKE_KEY_FIELD, QUEUE_DELAYED_WAKE_STREAM}; use wdl_rust_common::redis_eval::append_eval_cmd; use crate::{ @@ -17,7 +18,6 @@ use super::{ parse_delayed_key, queue_orphaned_key, queue_stream_key, resolve_consumer, stream_id_to_entry, }; -pub(crate) const QUEUE_DELAYED_WAKE_STREAM: &str = "queue-delayed-wake"; const QUEUE_DELAYED_CLAIM_SAFETY_MS: u64 = 5_000; const QUEUE_DELAYED_NO_PROGRESS_BACKOFF_MS: u64 = 100; const QUEUE_DELAYED_WAKE_RETRY_BASE_MS: u64 = 1_000; @@ -500,7 +500,7 @@ pub(crate) async fn queue_delayed_wake_loop(state: AppState) -> SchedulerResult< for id in key.ids { last_id = id.id.clone(); let entry = stream_id_to_entry(id); - let Some(delayed_key) = entry.fields.get("delayed_key") else { + let Some(delayed_key) = entry.fields.get(QUEUE_DELAYED_WAKE_KEY_FIELD) else { continue; }; if parse_delayed_key(delayed_key).is_none() { diff --git a/rust/scheduler/src/queue/keys.rs b/rust/scheduler/src/queue/keys.rs index b4f1748..f976eae 100644 --- a/rust/scheduler/src/queue/keys.rs +++ b/rust/scheduler/src/queue/keys.rs @@ -3,58 +3,3 @@ pub(crate) use wdl_rust_common::queue_keys::{ parse_delayed_key, parse_stream_key, queue_consumer_key, queue_delayed_key, queue_dlq_key, queue_orphaned_key, queue_stream_key, }; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn queue_key_builders_compose_redis_key_shapes() { - assert_eq!(QUEUE_CONSUMER_INDEX_KEY, "queue:index:consumers"); - assert_eq!(QUEUE_STREAM_INDEX_KEY, "queue:index:streams"); - assert_eq!(QUEUE_DELAYED_INDEX_KEY, "queue:index:delayed"); - assert_eq!(queue_stream_key("demo", "jobs"), "queue:demo:jobs:s"); - assert_eq!(queue_delayed_key("demo", "jobs"), "queue-delayed:demo:jobs"); - assert_eq!(queue_dlq_key("demo", "jobs"), "queue:demo:jobs:dlq"); - assert_eq!( - queue_orphaned_key("demo", "jobs"), - "queue-orphaned:demo:jobs" - ); - assert_eq!( - queue_consumer_key("demo", "jobs"), - "queue-consumer:demo:jobs" - ); - assert_eq!(queue_stream_key("t", "my-queue-1"), "queue:t:my-queue-1:s"); - } - - #[test] - fn parse_queue_keys() { - assert_eq!( - parse_stream_key("queue:demo:jobs:s"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_delayed_key("queue-delayed:demo:jobs"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_consumer_key("queue-consumer:demo:jobs"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_stream_key("queue:t:my-queue-1:s"), - Some(("t".to_string(), "my-queue-1".to_string())) - ); - assert_eq!(parse_stream_key("queue:demo:jobs"), None); - assert_eq!(parse_stream_key("queue-delayed:demo:jobs"), None); - assert_eq!(parse_stream_key("queue::jobs:s"), None); - assert_eq!(parse_stream_key("notaqueue:demo:jobs:s"), None); - assert_eq!(parse_stream_key(""), None); - assert_eq!(parse_delayed_key("queue:demo:jobs:s"), None); - assert_eq!(parse_delayed_key("queue-delayed::jobs"), None); - assert_eq!(parse_delayed_key("queue-delayed:demo:"), None); - assert_eq!(parse_consumer_key("queue-consumer:demo"), None); - assert_eq!(parse_consumer_key("queue-consumer:"), None); - assert_eq!(parse_consumer_key("other:demo:jobs"), None); - } -} diff --git a/rust/scheduler/src/queue/registry.rs b/rust/scheduler/src/queue/registry.rs index a83cb89..c8a8651 100644 --- a/rust/scheduler/src/queue/registry.rs +++ b/rust/scheduler/src/queue/registry.rs @@ -247,6 +247,7 @@ async fn write_resolved_consumer( #[cfg(test)] mod tests { use super::*; + use crate::test_fixtures::scheduler_projection_contract; use std::sync::{Arc as StdArc, Mutex}; fn str_map(items: &[(&str, &str)]) -> HashMap { @@ -350,6 +351,24 @@ mod tests { assert_eq!(zeroes.retry_delay_secs, 0); } + #[test] + fn hydrate_consumer_matches_control_projection_fixture() { + let fixture = scheduler_projection_contract(); + let contract = fixture.queue_consumer; + let consumer = hydrate_consumer(&contract.ns, &contract.queue, &contract.fields) + .expect("fixture queue consumer projection must hydrate"); + + assert_eq!( + consumer.worker_id, + format!("{}:{}:{}", contract.ns, contract.worker, contract.version) + ); + assert_eq!(consumer.max_batch_size, 12); + assert_eq!(consumer.max_batch_timeout_ms, 250); + assert_eq!(consumer.max_retries, 4); + assert_eq!(consumer.retry_delay_secs, 17); + assert_eq!(consumer.dead_letter_queue.as_deref(), Some("jobs-dlq")); + } + #[test] fn hydrate_consumer_falls_back_for_invalid_or_incomplete_hashes() { let invalid = hydrate_consumer( diff --git a/rust/scheduler/src/test_fixtures.rs b/rust/scheduler/src/test_fixtures.rs new file mode 100644 index 0000000..47dfde6 --- /dev/null +++ b/rust/scheduler/src/test_fixtures.rs @@ -0,0 +1,57 @@ +use std::collections::HashMap; + +use serde::Deserialize; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SchedulerProjectionContract { + pub(crate) cron: CronProjectionContract, + pub(crate) queue_consumer: QueueConsumerProjectionContract, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CronProjectionContract { + pub(crate) worker_index_key: String, + pub(crate) ns: String, + pub(crate) worker: String, + pub(crate) worker_key: String, + pub(crate) slot_ms: i64, + pub(crate) slot_key: String, + pub(crate) slot_expire_at: i64, + pub(crate) cron_id: String, + pub(crate) r#gen: i64, + pub(crate) reference: String, + pub(crate) meta: CronMetaContract, + pub(crate) entry: CronEntryContract, +} + +#[derive(Deserialize)] +pub(crate) struct CronMetaContract { + pub(crate) version: String, + pub(crate) json: String, +} + +#[derive(Deserialize)] +pub(crate) struct CronEntryContract { + pub(crate) cron: String, + pub(crate) timezone: String, + pub(crate) r#gen: i64, + pub(crate) json: String, +} + +#[derive(Deserialize)] +pub(crate) struct QueueConsumerProjectionContract { + pub(crate) ns: String, + pub(crate) queue: String, + pub(crate) worker: String, + pub(crate) version: String, + pub(crate) fields: HashMap, +} + +pub(crate) fn scheduler_projection_contract() -> SchedulerProjectionContract { + serde_json::from_str(include_str!( + "../../../tests/fixtures/scheduler-projection-contract.json" + )) + .expect("scheduler projection contract fixture must parse") +} diff --git a/rust/supervisor/Cargo.toml b/rust/supervisor/Cargo.toml index 46c6b81..cae5d0c 100644 --- a/rust/supervisor/Cargo.toml +++ b/rust/supervisor/Cargo.toml @@ -31,3 +31,6 @@ serde_json.workspace = true # Supervisor bins use #[tokio::main(flavor = "current_thread")]. tokio = { workspace = true, features = ["macros", "process", "rt", "signal", "sync", "time"] } wdl-rust-common = { workspace = true, default-features = false } + +[dev-dependencies] +wdl-rust-common = { workspace = true, default-features = false, features = ["test-support"] } diff --git a/rust/supervisor/src/config.rs b/rust/supervisor/src/config.rs index a7e2b8e..35e93e8 100644 --- a/rust/supervisor/src/config.rs +++ b/rust/supervisor/src/config.rs @@ -258,71 +258,43 @@ pub(crate) fn pick_do_compiled_config() -> &'static str { #[cfg(test)] mod tests { - use std::sync::{Mutex, OnceLock}; - use super::*; - - static ENV_LOCK: OnceLock> = OnceLock::new(); - - fn temp_env(key: &str, value: Option<&str>, f: impl FnOnce() -> R) -> R { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env mutex poisoned"); - let prev = std::env::var(key).ok(); - // SAFETY: these tests serialize environment mutation with ENV_LOCK, so no - // concurrent test can read or write the same process environment while this - // temporary override is active. - match value { - Some(v) => unsafe { std::env::set_var(key, v) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - let result = f(); - // SAFETY: ENV_LOCK is still held, so restoring the process environment is - // serialized with all other test environment mutation in this module. - match prev { - Some(p) => unsafe { std::env::set_var(key, p) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - result - } + use wdl_rust_common::test_env::with_temp_env; #[test] fn positive_int_env_accepts_only_strict_positive_integer_strings() { - temp_env("WDLTEST_VAL", None, || { + with_temp_env("WDLTEST_VAL", None, || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some(""), || { + with_temp_env("WDLTEST_VAL", Some(""), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("0"), || { + with_temp_env("WDLTEST_VAL", Some("0"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("-3"), || { + with_temp_env("WDLTEST_VAL", Some("-3"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("nope"), || { + with_temp_env("WDLTEST_VAL", Some("nope"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("12.9"), || { + with_temp_env("WDLTEST_VAL", Some("12.9"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("1e3"), || { + with_temp_env("WDLTEST_VAL", Some("1e3"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("18446744073709551616"), || { + with_temp_env("WDLTEST_VAL", Some("18446744073709551616"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("42"), || { + with_temp_env("WDLTEST_VAL", Some("42"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 42); }); } #[test] fn owner_ttl_ms_saturates_extreme_env_values() { - temp_env( + with_temp_env( "WDLTEST_OWNER_TTL_SECONDS", Some("18446744073709552"), || { diff --git a/rust/supervisor/src/drain.rs b/rust/supervisor/src/drain.rs index 8c7535f..1d649d4 100644 --- a/rust/supervisor/src/drain.rs +++ b/rust/supervisor/src/drain.rs @@ -126,39 +126,12 @@ pub(crate) fn build_request_id(prefix: &str) -> String { #[cfg(test)] mod tests { - use std::sync::{Mutex, OnceLock}; - use super::*; - - static ENV_LOCK: OnceLock> = OnceLock::new(); - - fn temp_env(key: &str, value: Option<&str>, f: impl FnOnce() -> R) -> R { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env mutex poisoned"); - let prev = std::env::var(key).ok(); - // SAFETY: tests hold ENV_LOCK for the full override/restore window, so process - // environment mutation is serialized. - match value { - Some(v) => unsafe { std::env::set_var(key, v) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - let result = f(); - // SAFETY: ENV_LOCK is still held, so restoring the environment cannot race - // with another test in this module. - match prev { - Some(p) => unsafe { std::env::set_var(key, p) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - result - } + use wdl_rust_common::test_env::with_temp_env; #[test] fn build_request_id_starts_with_prefix() { - temp_env("HOSTNAME", Some("test-host"), || { + with_temp_env("HOSTNAME", Some("test-host"), || { let id = build_request_id("do-drain"); assert!( id.starts_with("do-drain-test-host-"), diff --git a/rust/supervisor/src/renew.rs b/rust/supervisor/src/renew.rs index 2554d70..652e5b0 100644 --- a/rust/supervisor/src/renew.rs +++ b/rust/supervisor/src/renew.rs @@ -156,51 +156,12 @@ fn log_renew_error( #[cfg(test)] mod tests { - use std::sync::{Mutex, OnceLock}; - use super::*; - - static ENV_LOCK: OnceLock> = OnceLock::new(); - - struct EnvRestore { - key: String, - prev: Option, - } - - impl Drop for EnvRestore { - fn drop(&mut self) { - // SAFETY: tests hold ENV_LOCK for the full override/restore window, so - // restoring the process environment is serialized. - match &self.prev { - Some(prev) => unsafe { std::env::set_var(&self.key, prev) }, - None => unsafe { std::env::remove_var(&self.key) }, - } - } - } - - fn temp_env(key: &str, value: Option<&str>, f: impl FnOnce() -> R) -> R { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env mutex poisoned"); - let prev = std::env::var(key).ok(); - let _restore = EnvRestore { - key: key.to_string(), - prev, - }; - // SAFETY: tests hold ENV_LOCK for the full override/restore window, so process - // environment mutation is serialized. - match value { - Some(v) => unsafe { std::env::set_var(key, v) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - f() - } + use wdl_rust_common::test_env::with_temp_env; #[test] fn start_fails_before_spawn_when_internal_auth_token_is_missing() { - temp_env("WDL_INTERNAL_AUTH_TOKEN", None, || { + with_temp_env("WDL_INTERNAL_AUTH_TOKEN", None, || { let client = Arc::new(reqwest::Client::new()); match start(&crate::D1_CONFIG, client) { Ok(_) => panic!("start must reject missing token"), diff --git a/rust/workflows/Cargo.toml b/rust/workflows/Cargo.toml index b4c70f0..1053537 100644 --- a/rust/workflows/Cargo.toml +++ b/rust/workflows/Cargo.toml @@ -25,3 +25,6 @@ serde_json.workspace = true sha2.workspace = true tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } wdl-rust-common = { workspace = true, features = ["axum"] } + +[dev-dependencies] +wdl-rust-common = { workspace = true, features = ["axum", "test-support"] } diff --git a/rust/workflows/src/api.rs b/rust/workflows/src/api.rs index ddb963d..5f154ae 100644 --- a/rust/workflows/src/api.rs +++ b/rust/workflows/src/api.rs @@ -55,7 +55,7 @@ use limits::{ MAX_WORKFLOW_EVENT_TYPE_BYTES, MAX_WORKFLOW_INSTANCE_PAYLOAD_BYTES, MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_PARAMS_BYTES, MAX_WORKFLOW_RESULT_BYTES, MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES, MAX_WORKFLOW_STEP_CONFIG_BYTES, - MAX_WORKFLOW_STEP_NAME_BYTES, READY_SHARDS, + MAX_WORKFLOW_STEP_NAME_BYTES, READY_SHARDS, WORKFLOW_PAYLOAD_TOO_LARGE_CODE, }; pub(crate) use model::{ CreateBatchResponse, EventRecord, InstanceIdentity, InstanceResponse, LifecycleBlocker, @@ -92,6 +92,18 @@ use status::{ }; pub(crate) use tick::tick_workflows; +fn runtime_endpoint(app: &AppState, ns: &str, path: &str) -> String { + let (host, port) = if ns == "__system__" { + ( + &app.config.system_runtime_host, + app.config.system_runtime_port, + ) + } else { + (&app.config.runtime_host, app.config.runtime_port) + }; + format!("http://{host}:{port}{path}") +} + #[cfg(test)] pub(crate) use execution::{ COMMIT_STEP_ERROR_SCRIPT, COMMIT_STEP_RECORD_SCRIPT, COMMIT_STEP_SUCCESS_SCRIPT, diff --git a/rust/workflows/src/api/do_alarms/model.rs b/rust/workflows/src/api/do_alarms/model.rs index 723aeb1..9acf357 100644 --- a/rust/workflows/src/api/do_alarms/model.rs +++ b/rust/workflows/src/api/do_alarms/model.rs @@ -1,6 +1,8 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; +use wdl_rust_common::identity::{is_valid_runtime_load_ns, is_valid_worker_name}; +use wdl_rust_common::version::parse_version_tag; use crate::{ DoAlarmJobKeys, WorkflowError, WorkflowResult, do_alarm_by_worker_key, do_alarm_job_id, @@ -176,39 +178,12 @@ fn is_class_name(value: &str) -> bool { && chars.all(|ch| ch == '_' || ch == '$' || ch.is_ascii_alphanumeric()) } -fn is_worker_name(value: &str) -> bool { - let bytes = value.as_bytes(); - matches!(bytes.first(), Some(first) if first.is_ascii_alphanumeric()) - && bytes.len() <= 255 - && bytes - .iter() - .all(|byte| byte.is_ascii_alphanumeric() || *byte == b'_' || *byte == b'-') -} - -fn is_ns_field(value: &str) -> bool { - let bytes = value.as_bytes(); - if value.starts_with("__") && value.ends_with("__") && bytes.len() > 4 { - return bytes[2..bytes.len() - 2].iter().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_' || *byte == b'-' - }); - } - bytes - .iter() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') -} - fn is_storage_id(value: &str) -> bool { value.bytes().all(|byte| { byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' || byte == b'-' }) } -fn is_version(value: &str) -> bool { - value.strip_prefix('v').is_some_and(|suffix| { - !suffix.is_empty() && suffix.bytes().all(|byte| byte.is_ascii_digit()) - }) -} - fn protocol_string_field<'a>( state: &'a HashMap, field: &'static str, @@ -260,9 +235,12 @@ pub(super) fn job_from_state( parse_positive_i64_field(&state, "dueAtMs")?; Ok(DoAlarmJob { job_id, - ns: protocol_string_field(&state, "ns", is_ns_field)?.to_string(), - worker: protocol_string_field(&state, "worker", is_worker_name)?.to_string(), - version: protocol_string_field(&state, "scheduledVersion", is_version)?.to_string(), + ns: protocol_string_field(&state, "ns", is_valid_runtime_load_ns)?.to_string(), + worker: protocol_string_field(&state, "worker", is_valid_worker_name)?.to_string(), + version: protocol_string_field(&state, "scheduledVersion", |value| { + parse_version_tag(value).is_ok() + })? + .to_string(), do_storage_id: protocol_string_field(&state, "doStorageId", is_storage_id)?.to_string(), class_name: protocol_string_field(&state, "className", is_class_name)?.to_string(), object_name: object_name_field(&state, "objectName")?.to_string(), @@ -316,10 +294,15 @@ mod tests { #[test] fn do_alarm_job_state_rejects_malformed_dispatch_identity() { for (field, value) in [ + ("ns", ""), ("ns", "Demo"), + ("ns", "admin"), + ("ns", "__community__"), ("ns", "____"), ("worker", "-bad"), ("scheduledVersion", "1"), + ("scheduledVersion", "v0"), + ("scheduledVersion", "v01"), ("doStorageId", "DO_ABC"), ("className", "bad-name"), ("objectName", "bad\nname"), diff --git a/rust/workflows/src/api/limits.rs b/rust/workflows/src/api/limits.rs index 5864c1e..83d3b90 100644 --- a/rust/workflows/src/api/limits.rs +++ b/rust/workflows/src/api/limits.rs @@ -1,6 +1,7 @@ pub(crate) const MAX_WORKFLOW_JSON_BODY_BYTES: usize = 2 * 1024 * 1024; pub(crate) const MAX_WORKFLOW_PARAMS_BYTES: usize = 1024 * 1024; pub(crate) const MAX_WORKFLOW_RESULT_BYTES: usize = 1024 * 1024; +pub(crate) const WORKFLOW_PAYLOAD_TOO_LARGE_CODE: &str = "workflow_payload_too_large"; pub(crate) const MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES: usize = MAX_WORKFLOW_RESULT_BYTES + 16 * 1024; pub(crate) const MAX_WORKFLOW_EVENT_BYTES: usize = 256 * 1024; pub(crate) const MAX_WORKFLOW_EVENT_TYPE_BYTES: usize = 512; @@ -10,3 +11,38 @@ pub(crate) const MAX_WORKFLOW_STEP_CONFIG_BYTES: usize = 64 * 1024; pub(crate) const MAX_CREATE_BATCH_SIZE: usize = 100; pub(crate) const READY_SHARDS: usize = crate::WORKFLOW_READY_SHARDS; pub(crate) const LIFECYCLE_BLOCKER_LIMIT: usize = 20; + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + use super::{ + MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_RESULT_BYTES, WORKFLOW_PAYLOAD_TOO_LARGE_CODE, + }; + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct WorkflowLimitsContract { + result_bytes_max: usize, + backend_request_bytes_max: usize, + payload_too_large_code: String, + } + + #[test] + fn workflow_limits_match_runtime_fixture() { + let contract: WorkflowLimitsContract = serde_json::from_str(include_str!( + "../../../../tests/fixtures/workflow-limits.json" + )) + .expect("workflow limits fixture"); + + assert_eq!(contract.result_bytes_max, MAX_WORKFLOW_RESULT_BYTES); + assert_eq!( + contract.backend_request_bytes_max, + MAX_WORKFLOW_JSON_BODY_BYTES + ); + assert_eq!( + contract.payload_too_large_code, + WORKFLOW_PAYLOAD_TOO_LARGE_CODE + ); + } +} diff --git a/rust/workflows/src/api/progress.rs b/rust/workflows/src/api/progress.rs index ed4f67e..ed6a958 100644 --- a/rust/workflows/src/api/progress.rs +++ b/rust/workflows/src/api/progress.rs @@ -1,4 +1,4 @@ -use super::{execution::WorkflowStepRequest, parse_positive_identity_i64}; +use super::{execution::WorkflowStepRequest, parse_positive_identity_i64, runtime_endpoint}; use serde_json::{Value as JsonValue, json}; use wdl_rust_common::internal_auth::INTERNAL_AUTH_HEADER; use wdl_rust_common::time::now_ms; @@ -9,18 +9,6 @@ use crate::{ const PROGRESS_CALLBACK_CACHE_LIMIT: usize = 4096; -fn runtime_endpoint(app: &AppState, ns: &str) -> String { - let (host, port) = if ns == "__system__" { - ( - &app.config.system_runtime_host, - app.config.system_runtime_port, - ) - } else { - (&app.config.runtime_host, app.config.runtime_port) - }; - format!("http://{host}:{port}/internal/workflows/notify") -} - fn progress_payload(event: &str, status: &str, step: Option) -> JsonValue { json!({ "event": event, @@ -83,7 +71,7 @@ async fn notify(app: &AppState, identity: InstanceIdentity, callback: String, pr ); return; }; - let url = runtime_endpoint(app, &identity.ns); + let url = runtime_endpoint(app, &identity.ns, "/internal/workflows/notify"); let response = app .http .post(url) diff --git a/rust/workflows/src/api/tick/dispatch.rs b/rust/workflows/src/api/tick/dispatch.rs index 2627b14..a0be147 100644 --- a/rust/workflows/src/api/tick/dispatch.rs +++ b/rust/workflows/src/api/tick/dispatch.rs @@ -10,10 +10,11 @@ use crate::{ }; use super::super::{ - InstanceRouteKeys, MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES, RunClaim, clear_suspended_run_claim, - eval_script, instance_payload_limit_arg, log_instance_event, observe_instance_duration, - parse_positive_identity_i64, read_state_by_id, result_json, spawn_progress_from_identity, - terminal_retention_ms, + InstanceRouteKeys, MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES, RunClaim, + WORKFLOW_PAYLOAD_TOO_LARGE_CODE, clear_suspended_run_claim, eval_script, + instance_payload_limit_arg, log_instance_event, observe_instance_duration, + parse_positive_identity_i64, read_state_by_id, result_json, runtime_endpoint, + spawn_progress_from_identity, terminal_retention_ms, }; pub(crate) const COMMIT_RUNTIME_TERMINAL_SCRIPT: &str = r#" @@ -226,17 +227,7 @@ pub(super) async fn dispatch_runtime( params: JsonValue, request_id: Option<&str>, ) -> WorkflowResult { - let host = if identity.ns == "__system__" { - &app.config.system_runtime_host - } else { - &app.config.runtime_host - }; - let port = if identity.ns == "__system__" { - app.config.system_runtime_port - } else { - app.config.runtime_port - }; - let url = format!("http://{host}:{port}/internal/workflows/run"); + let url = runtime_endpoint(app, &identity.ns, "/internal/workflows/run"); let generation = parse_positive_identity_i64(&identity.generation, "generation")?; let created_at_ms = parse_positive_identity_i64(&identity.created_at_ms, "createdAtMs")?; let mut request = app @@ -329,7 +320,7 @@ async fn commit_runtime_failed_payload( fn runtime_payload_too_large_error(err: &WorkflowError) -> JsonValue { json!({ "name": "WorkflowError", - "code": "workflow_payload_too_large", + "code": WORKFLOW_PAYLOAD_TOO_LARGE_CODE, "message": err.message, }) } diff --git a/rust/workflows/src/server.rs b/rust/workflows/src/server.rs index 9569566..aea9574 100644 --- a/rust/workflows/src/server.rs +++ b/rust/workflows/src/server.rs @@ -25,7 +25,10 @@ use serde::Serialize; use serde_json::{Value as JsonValue, json}; use tokio::sync::Semaphore; use wdl_rust_common::health::healthcheck_http_200; -use wdl_rust_common::internal_auth::internal_auth_headers_match; +use wdl_rust_common::internal_auth::{ + INTERNAL_AUTH_FAILURE_CODE, INTERNAL_AUTH_FAILURE_MESSAGE, internal_auth_failure_response, + internal_auth_headers_match, +}; use wdl_rust_common::metrics::prometheus_response; use wdl_rust_common::request_id::sanitize_request_id; use wdl_rust_common::shutdown::shutdown_signal; @@ -390,14 +393,7 @@ async fn track_request( if !matches!(route, "healthz" | "metrics") && !internal_auth_headers_match(request.headers(), &state.config.internal_auth_tokens) { - let response = ( - StatusCode::UNAUTHORIZED, - Json(json!({ - "error": "internal_auth_failed", - "message": "Internal authentication failed", - })), - ) - .into_response(); + let response = internal_auth_failure_response(); record_request_complete( &state, method.as_str(), @@ -405,7 +401,7 @@ async fn track_request( response.status(), request_id.as_deref(), started_at, - Some(("internal_auth_failed", "Internal authentication failed")), + Some((INTERNAL_AUTH_FAILURE_CODE, INTERNAL_AUTH_FAILURE_MESSAGE)), ); return response; } diff --git a/rust/workflows/src/tests.rs b/rust/workflows/src/tests.rs index 2cfe33c..f0881f7 100644 --- a/rust/workflows/src/tests.rs +++ b/rust/workflows/src/tests.rs @@ -7,22 +7,13 @@ use crate::{ InstanceKeys, config_from_env, due_key, ready_active_key, ready_key, schema_version_key, validate_instance_id_value, }; -use std::sync::{Mutex, OnceLock}; - -static ENV_LOCK: OnceLock> = OnceLock::new(); +use wdl_rust_common::test_env::with_temp_envs; // Source-contract tests below intentionally couple to these module paths. If // the files move, update the constants and the associated assertions together. const RUNTIME_DISPATCH_SOURCE: &str = include_str!("api/tick/dispatch.rs"); const EXPECTED_EVENT_INDEX_STALE_SCAN_LIMIT: usize = 256; fn temp_env(items: &[(&str, Option<&str>)], f: impl FnOnce() -> R) -> R { - // Environment variables are process-global. This helper only protects the - // workflows tests that opt into it; tests that read or mutate env - // directly must stay out of the same parallel window. - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env mutex poisoned"); let mut all_items = items.to_vec(); if !all_items .iter() @@ -30,31 +21,7 @@ fn temp_env(items: &[(&str, Option<&str>)], f: impl FnOnce() -> R) -> R { { all_items.push(("WDL_INTERNAL_AUTH_TOKEN", Some("test-internal-auth-token"))); } - let previous: Vec<(&str, Option)> = all_items - .iter() - .map(|(key, _)| (*key, std::env::var(key).ok())) - .collect(); - for (key, value) in &all_items { - match value { - // SAFETY: Rust marks environment mutation unsafe because other - // threads can read concurrently. ENV_LOCK serializes this test - // helper's callers, and the workflows env tests keep all - // config reads inside the locked closure. - Some(value) => unsafe { std::env::set_var(key, value) }, - // SAFETY: Same rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - } - let result = f(); - for (key, value) in previous { - match value { - // SAFETY: Same rationale as the initial set_var above. - Some(value) => unsafe { std::env::set_var(key, value) }, - // SAFETY: Same rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - } - result + with_temp_envs(&all_items, f) } #[test] diff --git a/scripts/_integration-pool.js b/scripts/_integration-pool.js index f0f659c..0ceb801 100644 --- a/scripts/_integration-pool.js +++ b/scripts/_integration-pool.js @@ -1,17 +1,21 @@ import { spawn } from "node:child_process"; -import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { renameSync, writeFileSync } from "node:fs"; import { COMPILE_WORKERD_LOCAL_ARGS, DOCKER_COMPOSE_BUILD_ARGS, + LOCAL_ADMIN_TOKEN, + LOCAL_CONNECT_HOST, ROOT, + localAssetsCdnBase, + localControlUrl, shouldPrepareIntegrationArtifacts, } from "./integration-environment.js"; +import { readIntegrationDurationReport } from "./integration-test-plan.js"; +import { pipeWithPrefix, writeWithPrefix } from "./_stream-prefix.js"; /** * @typedef {{ file: string, slot: number, durationMs: number, status: "passed" | "failed" }} FileDuration * @typedef {{ shardCount: number, runDurationMs: number, fileDurations: FileDuration[] }} IntegrationSummary - * @typedef {{ durationMs: number, status: string, updatedAt: string }} DurationRecord - * @typedef {{ updatedAt?: string, runDurationMs?: number, files?: Record }} DurationReport * @typedef {{ * files: string[], * shardCount: number, @@ -215,10 +219,10 @@ export function makeSlotEnv(idx, projectPrefix, basePort, s3PortBase, baseEnv = WDL_S3MOCK_HOST_PORT: String(s3mockPort), WDL_WORKERD_CONFIG_VARIANT: "local", WDL_INTEGRATION_NO_BUILD: "1", - ADMIN_TOKEN: "local-dev-token", - CONTROL_URL: `http://admin.test:${gatewayPort}`, - ASSETS_CDN_BASE: `http://localhost:${s3mockPort}/wdl-assets`, - CONTROL_CONNECT_HOST: "localhost", + ADMIN_TOKEN: LOCAL_ADMIN_TOKEN, + CONTROL_URL: localControlUrl(gatewayPort), + ASSETS_CDN_BASE: localAssetsCdnBase(s3mockPort), + CONTROL_CONNECT_HOST: LOCAL_CONNECT_HOST, }; return env; } @@ -251,8 +255,9 @@ function teardownSlot(env, prefix) { }); } -/** @param {DurationReport | null | undefined} existing @param {{ runDurationMs: number, fileDurations: FileDuration[] }} observed @param {Date} [now] */ +/** @param {ReturnType | undefined} existing @param {{ runDurationMs: number, fileDurations: FileDuration[] }} observed @param {Date} [now] */ export function buildDurationReport(existing, { runDurationMs, fileDurations }, now = new Date()) { + /** @type {Record} */ const files = { ...(existing?.files && typeof existing.files === "object" ? existing.files : {}) }; const updatedAt = now.toISOString(); for (const row of fileDurations) { @@ -315,44 +320,9 @@ export function persistDurationFile(file, observed) { /** @param {string} file @param {{ runDurationMs: number, fileDurations: FileDuration[] }} observed */ function updateDurationFile(file, observed) { - const existing = readDurationFile(file); + const existing = readIntegrationDurationReport(file); const next = buildDurationReport(existing, observed); const tmp = `${file}.tmp-${process.pid}`; writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`); renameSync(tmp, file); } - -/** @param {string} file @returns {DurationReport} */ -function readDurationFile(file) { - if (!existsSync(file)) return {}; - try { - return JSON.parse(readFileSync(file, "utf8")); - } catch (err) { - process.stderr.write( - `warning: ignoring unreadable integration duration file ${file}: ${errorMessage(err)}\n` - ); - return {}; - } -} - -/** @param {NodeJS.ReadableStream} src @param {NodeJS.WritableStream} dst @param {string} prefix */ -function pipeWithPrefix(src, dst, prefix) { - let buf = ""; - src.setEncoding("utf8"); - src.on("data", (chunk) => { - buf += String(chunk); - let nl; - while ((nl = buf.indexOf("\n")) >= 0) { - writeWithPrefix(dst, prefix, `${buf.slice(0, nl)}\n`); - buf = buf.slice(nl + 1); - } - }); - src.on("end", () => { - if (buf.length) writeWithPrefix(dst, prefix, `${buf}\n`); - }); -} - -/** @param {NodeJS.WritableStream} dst @param {string} prefix @param {string} text */ -function writeWithPrefix(dst, prefix, text) { - dst.write(`${prefix}${text}`); -} diff --git a/scripts/_stream-prefix.js b/scripts/_stream-prefix.js new file mode 100644 index 0000000..89d7669 --- /dev/null +++ b/scripts/_stream-prefix.js @@ -0,0 +1,21 @@ +/** @param {NodeJS.WritableStream} dst @param {string} prefix @param {string} text */ +export function writeWithPrefix(dst, prefix, text) { + dst.write(`${prefix}${text}`); +} + +/** @param {NodeJS.ReadableStream} src @param {NodeJS.WritableStream} dst @param {string} prefix */ +export function pipeWithPrefix(src, dst, prefix) { + let buffer = ""; + src.setEncoding("utf8"); + src.on("data", (chunk) => { + buffer += String(chunk); + let newline; + while ((newline = buffer.indexOf("\n")) >= 0) { + writeWithPrefix(dst, prefix, `${buffer.slice(0, newline)}\n`); + buffer = buffer.slice(newline + 1); + } + }); + src.on("end", () => { + if (buffer.length) writeWithPrefix(dst, prefix, `${buffer}\n`); + }); +} diff --git a/scripts/integration-environment.js b/scripts/integration-environment.js index ed377f3..5b96585 100644 --- a/scripts/integration-environment.js +++ b/scripts/integration-environment.js @@ -5,6 +5,19 @@ export const ROOT = path.resolve(import.meta.dirname, ".."); export const COMPILE_WORKERD_LOCAL_ARGS = ["scripts/compile-workerd-configs.js", "--local"]; export const DOCKER_COMPOSE_BUILD_ARGS = ["compose", "build", "gateway", "workflows"]; export const DEFAULT_WDL_CLI_COMMAND = "wdl"; +export const LOCAL_ADMIN_TOKEN = "local-dev-token"; +export const ADMIN_HOST_HEADER = "admin.test"; +export const LOCAL_CONNECT_HOST = "localhost"; + +/** @param {number} gatewayPort */ +export function localControlUrl(gatewayPort) { + return `http://${ADMIN_HOST_HEADER}:${gatewayPort}`; +} + +/** @param {number} s3mockPort */ +export function localAssetsCdnBase(s3mockPort) { + return `http://${LOCAL_CONNECT_HOST}:${s3mockPort}/wdl-assets`; +} /** @param {string} file */ function isExecutableFile(file) { diff --git a/scripts/integration-test-plan.js b/scripts/integration-test-plan.js index 9a2697a..fabd257 100644 --- a/scripts/integration-test-plan.js +++ b/scripts/integration-test-plan.js @@ -74,6 +74,7 @@ const CLI_INTEGRATION_MARKER_RE = new RegExp(`^//\\s*${RegExp.escape(CLI_INTEGRA /** * @typedef {{ durationMs?: unknown, status?: unknown, updatedAt?: unknown }} DurationRecord * @typedef {Record} DurationRecords + * @typedef {{ updatedAt?: unknown, runDurationMs?: unknown, files?: unknown }} DurationReport * @typedef {{ priority?: string[], durationRecords?: DurationRecords | null }} PrioritizeOptions */ @@ -82,21 +83,22 @@ function errorMessage(err) { return err instanceof Error ? err.message : String(err); } -/** @param {unknown} value @returns {DurationRecords | null} */ -function durationRecordsFromJson(value) { - if (!value || typeof value !== "object") return null; - const files = /** @type {{ files?: unknown }} */ (value).files; +/** @param {DurationReport | null} report @returns {DurationRecords | null} */ +function durationRecordsFromReport(report) { + const files = report?.files; return files && typeof files === "object" ? /** @type {DurationRecords} */ (files) : null; } -/** @param {string} [file] @returns {DurationRecords | null} */ -export function readIntegrationDurationRecords(file = DEFAULT_INTEGRATION_DURATIONS_FILE) { +/** @param {string} [file] @returns {DurationReport | null} */ +export function readIntegrationDurationReport(file = DEFAULT_INTEGRATION_DURATIONS_FILE) { if (!file || !existsSync(file)) return null; try { const parsed = JSON.parse(readFileSync(file, "utf8")); - return durationRecordsFromJson(parsed); + return parsed && typeof parsed === "object" + ? /** @type {DurationReport} */ (parsed) + : null; } catch (err) { process.stderr.write( `warning: ignoring unreadable integration duration file ${file}: ${errorMessage(err)}\n` @@ -105,6 +107,11 @@ export function readIntegrationDurationRecords(file = DEFAULT_INTEGRATION_DURATI } } +/** @param {string} [file] @returns {DurationRecords | null} */ +export function readIntegrationDurationRecords(file = DEFAULT_INTEGRATION_DURATIONS_FILE) { + return durationRecordsFromReport(readIntegrationDurationReport(file)); +} + /** @param {string[]} names @param {DurationRecords | null | undefined} durationRecords */ export function durationPriorityNames(names, durationRecords) { if (!durationRecords) return []; diff --git a/scripts/run-parallel.js b/scripts/run-parallel.js index 1192f97..2eeafe4 100644 --- a/scripts/run-parallel.js +++ b/scripts/run-parallel.js @@ -1,21 +1,5 @@ import { spawn } from "node:child_process"; - -/** @param {NodeJS.ReadableStream} src @param {NodeJS.WritableStream} dst @param {string} prefix */ -function pipeWithPrefix(src, dst, prefix) { - let buf = ""; - src.setEncoding("utf8"); - src.on("data", (chunk) => { - buf += String(chunk); - let nl; - while ((nl = buf.indexOf("\n")) >= 0) { - dst.write(`${prefix}${buf.slice(0, nl)}\n`); - buf = buf.slice(nl + 1); - } - }); - src.on("end", () => { - if (buf.length) dst.write(`${prefix}${buf}\n`); - }); -} +import { pipeWithPrefix } from "./_stream-prefix.js"; const tasks = process.argv.slice(2); if (tasks.length === 0) { diff --git a/shared/base64.js b/shared/base64.js index 9d49ff4..004b01b 100644 --- a/shared/base64.js +++ b/shared/base64.js @@ -3,8 +3,29 @@ // needed. const CHUNK_SIZE = 0x8000; +const BASE64_ALPHABET_RE = /^[A-Za-z0-9+/]*$/; +const BASE64_WHITESPACE_RE = /[\t\n\f\r ]/; +const BASE64_WHITESPACE_GLOBAL_RE = /[\t\n\f\r ]/g; const utf8Encoder = new TextEncoder(); +// Buffer.from(..., "base64") skips invalid bytes and accepts base64url. +// Validate with the same forgiving-base64 grammar as atob first so the +// nodejs_compat and web-platform branches fail closed identically. +/** @param {string} value */ +function validatedBufferBase64(value) { + let payload = BASE64_WHITESPACE_RE.test(value) + ? value.replace(BASE64_WHITESPACE_GLOBAL_RE, "") + : value; + if (payload.length % 4 === 0) { + if (payload.endsWith("==")) payload = payload.slice(0, -2); + else if (payload.endsWith("=")) payload = payload.slice(0, -1); + } + if (payload.length % 4 === 1 || !BASE64_ALPHABET_RE.test(payload)) { + throw new TypeError("Invalid base64 input"); + } + return payload; +} + /** @param {Uint8Array} bytes @returns {string} */ export function bytesToBase64(bytes) { if (typeof Buffer !== "undefined") return Buffer.from(bytes).toString("base64"); @@ -17,7 +38,9 @@ export function bytesToBase64(bytes) { /** @param {string} value @returns {Uint8Array} */ export function base64ToBytes(value) { - if (typeof Buffer !== "undefined") return new Uint8Array(Buffer.from(value, "base64")); + if (typeof Buffer !== "undefined") { + return new Uint8Array(Buffer.from(validatedBufferBase64(value), "base64")); + } const binary = atob(value); const out = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i += 1) out[i] = binary.charCodeAt(i); diff --git a/shared/bounded-body.js b/shared/bounded-body.js index 4116f1d..80d2218 100644 --- a/shared/bounded-body.js +++ b/shared/bounded-body.js @@ -10,21 +10,17 @@ export class BodyTooLargeError extends Error { } /** - * @param {Request} request + * @param {ReadableStream} stream * @param {number} maxBytes + * @param {() => Error} [overflowError] * @returns {Promise} */ -export async function readBoundedBytes(request, maxBytes) { - const contentLength = request.headers.get("content-length"); - if (contentLength != null && contentLength !== "") { - const declared = Number(contentLength); - if (Number.isFinite(declared) && declared > maxBytes) { - throw new BodyTooLargeError(maxBytes); - } - } - if (!request.body) return new Uint8Array(); - - const reader = request.body.getReader(); +export async function readBoundedStreamBytes( + stream, + maxBytes, + overflowError = () => new BodyTooLargeError(maxBytes) +) { + const reader = stream.getReader(); /** @type {Uint8Array[]} */ const chunks = []; let total = 0; @@ -32,17 +28,24 @@ export async function readBoundedBytes(request, maxBytes) { while (true) { const { done, value } = await reader.read(); if (done) break; - total += value.byteLength; + const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); + total += chunk.byteLength; if (total > maxBytes) { - await reader.cancel().catch(() => {}); - throw new BodyTooLargeError(maxBytes); + const error = overflowError(); + await reader.cancel(error).catch(() => {}); + throw error; } - chunks.push(value); + chunks.push(chunk); } } finally { - reader.releaseLock(); + try { reader.releaseLock(); } catch {} } + if (chunks.length === 1) { + const [chunk] = chunks; + if (chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength) return chunk; + return new Uint8Array(chunk); + } const bytes = new Uint8Array(total); let offset = 0; for (const chunk of chunks) { @@ -52,6 +55,23 @@ export async function readBoundedBytes(request, maxBytes) { return bytes; } +/** + * @param {Request} request + * @param {number} maxBytes + * @returns {Promise} + */ +export async function readBoundedBytes(request, maxBytes) { + const contentLength = request.headers.get("content-length"); + if (contentLength != null && contentLength !== "") { + const declared = Number(contentLength); + if (Number.isFinite(declared) && declared > maxBytes) { + throw new BodyTooLargeError(maxBytes); + } + } + if (!request.body) return new Uint8Array(); + return readBoundedStreamBytes(request.body, maxBytes); +} + /** * @param {Request} request * @param {number} maxBytes diff --git a/shared/d1-transport.js b/shared/d1-transport.js index c3f038e..66fe95b 100644 --- a/shared/d1-transport.js +++ b/shared/d1-transport.js @@ -110,7 +110,6 @@ function walkD1Transport(value, objectLeaf) { /** * @param {unknown} value * @returns {unknown} - * @lintignore runtime/_wdl-d1-transport.js re-exports this for loaded workers. */ export function decodeD1Transport(value) { return walkD1Transport(value, (item) => { diff --git a/shared/internal-auth.js b/shared/internal-auth.js index 692f4f7..b04e3a6 100644 --- a/shared/internal-auth.js +++ b/shared/internal-auth.js @@ -1,6 +1,8 @@ export const INTERNAL_AUTH_HEADER = "x-wdl-internal-auth"; export const INTERNAL_AUTH_ENV = "WDL_INTERNAL_AUTH_TOKEN"; export const INTERNAL_AUTH_PREVIOUS_ENV = "WDL_INTERNAL_AUTH_PREVIOUS_TOKEN"; +export const INTERNAL_AUTH_FAILURE_CODE = "internal_auth_failed"; +export const INTERNAL_AUTH_FAILURE_MESSAGE = "Internal authentication failed"; /** * @param {string} name @@ -117,7 +119,7 @@ export function stripInternalAuthHeader(headers) { export function internalAuthFailureResponse() { return Response.json({ - error: "internal_auth_failed", - message: "Internal authentication failed", + error: INTERNAL_AUTH_FAILURE_CODE, + message: INTERNAL_AUTH_FAILURE_MESSAGE, }, { status: 401 }); } diff --git a/shared/ns-pattern.js b/shared/ns-pattern.js index f47cb1e..c13bbba 100644 --- a/shared/ns-pattern.js +++ b/shared/ns-pattern.js @@ -5,6 +5,24 @@ // character set can never drift between the tier that accepts deploys and // the tier that routes traffic. export const NS_PATTERN = "[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?"; +export const DEFAULT_PLATFORM_DOMAIN = "workers.local"; + +/** @param {unknown} value */ +export function configuredHostname(value) { + if (typeof value !== "string" || !value.trim()) return null; + const host = value.trim().replace(/\.+$/, "").toLowerCase(); + if (!host || host.includes("/") || host.includes(":") || /\s/.test(host)) return null; + return host; +} + +/** @param {Record} env */ +export function platformDomainFromEnv(env) { + const raw = env.PLATFORM_DOMAIN; + if (raw == null || raw === "" || typeof raw !== "string") return DEFAULT_PLATFORM_DOMAIN; + const domain = configuredHostname(raw); + if (!domain) throw new TypeError("PLATFORM_DOMAIN must be a plain hostname"); + return domain; +} export const RESERVED_NS = new Set(["__system__", "__platform__", "__community__"]); @@ -193,6 +211,10 @@ export const WDL_RESERVED_ENTRYPOINT_RE = /^__Wdl[A-Za-z0-9_]*__$/; // key="bar") and (id="foo", key="v:bar"). export const KV_ID_RE = /^[a-z0-9][a-z0-9-]{0,62}$/; +// D1 API ids are stable control-plane references and may retain the +// mixed-case/underscore forms accepted by the existing D1 model. +export const D1_DATABASE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; + // Worker-scoped R2 virtual bucket name. Same Redis/S3-key hygiene class as // KV / Queue ids: lowercase, no slash, no colon, bounded length. export const R2_BUCKET_NAME_RE = /^[a-z0-9][a-z0-9-]{0,62}$/; diff --git a/shared/observability.js b/shared/observability.js index 4e58b6f..85e89b1 100644 --- a/shared/observability.js +++ b/shared/observability.js @@ -6,6 +6,7 @@ import { bytesToHex } from "./hex.js"; const CARDINALITY_WARN_LIMIT = 100; +const requestIdUtf8Encoder = new TextEncoder(); /** * @typedef {Record} Labels * @typedef {(level: string, event: string, fields?: Record) => void} Logger @@ -133,7 +134,7 @@ export function sanitizeRequestId(raw) { if (Array.isArray(raw)) raw = raw[0]; if (typeof raw !== "string") return null; const first = raw.split(",")[0].trim(); - if (!first || first.length > 128) return null; + if (!first || requestIdUtf8Encoder.encode(first).byteLength > 128) return null; if (/[\s"\\]/.test(first)) return null; for (let i = 0; i < first.length; i++) { const code = first.charCodeAt(i); diff --git a/shared/owner-lease.js b/shared/owner-lease.js index d2c576f..f7dd96e 100644 --- a/shared/owner-lease.js +++ b/shared/owner-lease.js @@ -144,6 +144,41 @@ export function boundedPositiveIntEnv(env, name, fallback, max) { return Math.max(1, Math.min(Math.trunc(raw), upper)); } +/** + * @template T + * @param {(attempt: number) => Promise} operation + * @param {{ + * attempts: number, + * isRetryableError?: (err: unknown) => boolean, + * onRetryableError?: (err: unknown, attempt: number) => void, + * shouldRetryResult?: (result: T, attempt: number) => boolean, + * onExhausted: () => T | Promise, + * }} options + * @returns {Promise} + */ +export async function withOptimisticRetries(operation, { + attempts, + isRetryableError = () => false, + onRetryableError, + shouldRetryResult, + onExhausted, +}) { + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + const result = await operation(attempt); + if (shouldRetryResult?.(result, attempt)) continue; + return result; + } catch (err) { + if (isRetryableError(err)) { + onRetryableError?.(err, attempt); + continue; + } + throw err; + } + } + return await onExhausted(); +} + /** * @template T * @param {() => Promise} operation @@ -164,16 +199,14 @@ export async function withOwnerWatchRetries(operation, { exhaustedMessage, }) { const maxAttempts = Number.isInteger(retries) && retries > 0 ? retries : 1; - for (let attempt = 0; attempt < maxAttempts; attempt += 1) { - try { - return await operation(); - } catch (err) { - if (typeof isWatchError === "function" && isWatchError(err)) { - if (attempt < maxAttempts - 1) continue; + return await withOptimisticRetries( + async () => await operation(), + { + attempts: maxAttempts, + isRetryableError: (err) => typeof isWatchError === "function" && isWatchError(err), + onExhausted: () => { throw createError(503, exhaustedCode, exhaustedMessage); - } - throw err; + }, } - } - throw createError(503, exhaustedCode, exhaustedMessage); + ); } diff --git a/shared/queue-keys.js b/shared/queue-keys.js index 46df402..ae9b395 100644 --- a/shared/queue-keys.js +++ b/shared/queue-keys.js @@ -2,6 +2,8 @@ // corrupt parseStreamKey's anchor, so inline construction is a drift // risk — every tier imports from here. +import { isValidRuntimeLoadNs, QUEUE_NAME_RE } from "./ns-pattern.js"; + /** * @typedef {{ ns: string, queue: string }} QueueKeyParts */ @@ -27,35 +29,41 @@ export const QUEUE_DELAYED_INDEX_KEY = "queue:index:delayed"; export function queueConsumerScanPrefix(ns) { return `queue-consumer:${ns}:`; } /** - * @lintignore kept as the JS mirror of Rust queue key parsers. + * @param {string} rest + * @returns {QueueKeyParts | null} + */ +function parseQueueKeyRest(rest) { + const separator = rest.indexOf(":"); + if (separator <= 0) return null; + const ns = rest.slice(0, separator); + const queue = rest.slice(separator + 1); + if (!isValidRuntimeLoadNs(ns) || !QUEUE_NAME_RE.test(queue)) return null; + return { ns, queue }; +} + +/** * @param {string} key * @returns {QueueKeyParts | null} */ export function parseStreamKey(key) { - const m = key.match(/^queue:([^:]+):(.+):s$/); - return m ? { ns: m[1], queue: m[2] } : null; + if (!key.startsWith("queue:") || !key.endsWith(":s")) return null; + return parseQueueKeyRest(key.slice("queue:".length, -":s".length)); } /** - * @lintignore kept as the JS mirror of Rust queue key parsers. * @param {string} key * @returns {QueueKeyParts | null} */ export function parseDelayedKey(key) { - const m = key.match(/^queue-delayed:([^:]+):(.+)$/); - return m ? { ns: m[1], queue: m[2] } : null; + if (!key.startsWith("queue-delayed:")) return null; + return parseQueueKeyRest(key.slice("queue-delayed:".length)); } /** - * @lintignore kept as the JS mirror of Rust queue key parsers. * @param {string} key * @returns {QueueKeyParts | null} */ export function parseConsumerKey(key) { - const parts = key.split(":"); - if (parts.length < 3 || parts[0] !== "queue-consumer") return null; - const ns = parts[1]; - const queue = parts.slice(2).join(":"); - if (!ns || !queue) return null; - return { ns, queue }; + if (!key.startsWith("queue-consumer:")) return null; + return parseQueueKeyRest(key.slice("queue-consumer:".length)); } diff --git a/shared/redis-lock.js b/shared/redis-lock.js new file mode 100644 index 0000000..358c25d --- /dev/null +++ b/shared/redis-lock.js @@ -0,0 +1,69 @@ +import { randomHex } from "shared-random-id"; + +/** + * @typedef {{ key: string, token: string }} TokenLock + * @typedef {{ set(key: string, value: string, options: { nx?: boolean, ttl?: number, ifeq?: string }): Promise }} TokenLockSetClient + * @typedef {{ multi(): { set(key: string, value: string, options: { nx: true, ttl: number }): { exec(): Promise } } }} TokenLockTransactionalClient + * @typedef {{ delIfEq(key: string, value: string): Promise }} TokenLockReleaseClient + */ + +/** @param {string} key @returns {TokenLock} */ +export function createTokenLock(key) { + return { key, token: randomHex(16) }; +} + +/** + * @param {TokenLockSetClient | TokenLockTransactionalClient} client + * @param {TokenLock} lock + * @param {{ ttlSeconds: number, transactional?: boolean }} options + */ +export async function acquireTokenLock(client, lock, { ttlSeconds, transactional = false }) { + if (transactional) { + if (!("multi" in client) || typeof client.multi !== "function") { + throw new TypeError("transactional token lock requires multi()"); + } + const reply = await client.multi() + .set(lock.key, lock.token, { nx: true, ttl: ttlSeconds }) + .exec(); + return Array.isArray(reply) && reply[0] === "OK"; + } + if (!("set" in client) || typeof client.set !== "function") { + throw new TypeError("token lock acquire requires set()"); + } + return await client.set(lock.key, lock.token, { + nx: true, + ttl: ttlSeconds, + }) === "OK"; +} + +/** + * @param {TokenLockSetClient} client + * @param {TokenLock} lock + * @param {number} ttlSeconds + */ +export async function renewTokenLock(client, lock, ttlSeconds) { + return await client.set(lock.key, lock.token, { + ttl: ttlSeconds, + ifeq: lock.token, + }) === "OK"; +} + +/** + * Release is best-effort by contract: expiry bounds a leaked lock, while a + * release exception must not replace the operation's real success or failure. + * @param {TokenLockReleaseClient} client + * @param {TokenLock | null | undefined} lock + * @param {{ onError?: (err: unknown) => void }} [options] + */ +export async function releaseTokenLock(client, lock, { onError } = {}) { + if (!lock) return; + try { + await client.delIfEq(lock.key, lock.token); + } catch (err) { + try { + onError?.(err); + } catch { + // Logging is also best-effort on the release path. + } + } +} diff --git a/shared/s3-retry.js b/shared/s3-retry.js new file mode 100644 index 0000000..cd49e7d --- /dev/null +++ b/shared/s3-retry.js @@ -0,0 +1,46 @@ +import { discardResponseBody } from "./respond.js"; + +export const S3_TRANSIENT_RETRIES = 10; +const S3_RETRY_BASE_MS = 50; +const S3_RETRY_MAX_MS = 5_000; + +/** @param {number} ms */ +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** @param {number} attempt */ +function s3RetryDelayMs(attempt) { + return Math.min(S3_RETRY_BASE_MS * 2 ** attempt, S3_RETRY_MAX_MS); +} + +/** @param {Response} response */ +export function isTransientS3Response(response) { + return response.status === 429 || response.status >= 500; +} + +/** + * DeleteObjects is a POST, but deleting the same key set is safe to retry. + * Keep this scoped to S3 POST calls with known idempotent semantics. + * @param {{ fetch(url: string, init?: RequestInit): Promise }} client + * @param {string} url + * @param {RequestInit} init + */ +export async function fetchRetryableS3Post(client, url, init) { + for (let attempt = 0; attempt <= S3_TRANSIENT_RETRIES; attempt += 1) { + let response; + try { + response = await client.fetch(url, init); + } catch (err) { + if (attempt === S3_TRANSIENT_RETRIES) throw err; + await delay(Math.random() * s3RetryDelayMs(attempt)); + continue; + } + if (attempt === S3_TRANSIENT_RETRIES || !isTransientS3Response(response)) { + return response; + } + await discardResponseBody(response); + await delay(Math.random() * s3RetryDelayMs(attempt)); + } + throw new Error("unreachable S3 retry loop exit"); +} diff --git a/shared/secret-envelope.js b/shared/secret-envelope.js index d6291a2..070bcf7 100644 --- a/shared/secret-envelope.js +++ b/shared/secret-envelope.js @@ -12,7 +12,8 @@ export const SECRET_ENVELOPE_KID_ENV = "SECRET_ENVELOPE_KID"; const AES_GCM_TAG_BYTES = 16; const AES_GCM_IV_BYTES = 12; const AES_256_KEY_BYTES = 32; -const ENVELOPE_FIELDS = ["alg", "ct", "edek", "iv", "kid", "tag", "v"]; +const ENVELOPE_CANONICAL_FIELDS = ["v", "alg", "kid", "edek", "iv", "ct", "tag"]; +const ENVELOPE_FIELDS = ENVELOPE_CANONICAL_FIELDS.toSorted(); const utf8Encoder = new TextEncoder(); const utf8FatalDecoder = new TextDecoder("utf-8", { fatal: true }); @@ -52,6 +53,14 @@ function bytesToText(bytes) { export { bytesToBase64 }; +/** @param {Record} envelope */ +function canonicalEnvelopeJson(envelope) { + /** @type {Record} */ + const ordered = {}; + for (const field of ENVELOPE_CANONICAL_FIELDS) ordered[field] = envelope[field]; + return JSON.stringify(ordered); +} + /** * @param {unknown} value * @param {string} [fieldName] @@ -218,7 +227,7 @@ export async function encryptSecretValue(plaintext, { env, hashKey, fieldName, r edekBytes.set(wrappedDek.tag, dekIv.length + wrappedDek.ct.length); const payload = await aesGcmEncrypt(dek, payloadIv, textBytes(plaintext), payloadAadBytes(hashKey, fieldName)); - return `${SECRET_ENVELOPE_PREFIX}${JSON.stringify({ + return `${SECRET_ENVELOPE_PREFIX}${canonicalEnvelopeJson({ v: SECRET_ENVELOPE_VERSION, alg: SECRET_ENVELOPE_ALG, kid, @@ -252,7 +261,7 @@ function parseEnvelope(value) { if (fields.length !== ENVELOPE_FIELDS.length || fields.some((field, idx) => field !== ENVELOPE_FIELDS[idx])) { throw new SecretEnvelopeError("invalid_envelope", "secret envelope has unknown or missing fields"); } - if (JSON.stringify(parsed) !== json) { + if (canonicalEnvelopeJson(envelope) !== json) { throw new SecretEnvelopeError("invalid_envelope", "secret envelope JSON is not canonical"); } if (envelope.v !== SECRET_ENVELOPE_VERSION || envelope.alg !== SECRET_ENVELOPE_ALG) { diff --git a/shared/version.js b/shared/version.js index 4f7e8ff..e7ed7c2 100644 --- a/shared/version.js +++ b/shared/version.js @@ -42,6 +42,12 @@ export function bundleKey(ns, name, version) { return `worker:${ns}:${name}:v:${n}`; } +// Monotonic immutable-version allocator for one logical worker. +/** @param {string} ns @param {string} name */ +export function nextVersionKey(ns, name) { + return `worker:${ns}:${name}:next_version`; +} + // Active-route hash for a namespace: field=workerName, value=`v`. Control // is the sole writer; centralized here so cross-tier readers cannot drift from // the key grammar (reader set in docs/redis-key-layout.md). @@ -57,6 +63,34 @@ export function patternsKey(host) { return `patterns:${host}`; } +export const NAMESPACES_KEY = "namespaces"; +export const DECLARED_HOSTS_KEY = "declared-hosts"; +const HOSTS_PREFIX = "hosts:"; +const NS_HOSTS_PREFIX = "ns-hosts:"; +const HOST_DECLARATIONS_PREFIX = "host-declarations:"; +export const HOSTS_SCAN_PATTERN = `${HOSTS_PREFIX}*`; +export const HOST_DECLARATIONS_SCAN_PATTERN = `${HOST_DECLARATIONS_PREFIX}*`; + +/** @param {string} ns */ +export function hostsKey(ns) { + return `${HOSTS_PREFIX}${ns}`; +} + +/** @param {string} key */ +export function namespaceFromHostsKey(key) { + return key.startsWith(HOSTS_PREFIX) ? key.slice(HOSTS_PREFIX.length) : ""; +} + +/** @param {string} ns */ +export function nsHostsKey(ns) { + return `${NS_HOSTS_PREFIX}${ns}`; +} + +/** @param {string} host */ +export function hostDeclarationsKey(host) { + return `${HOST_DECLARATIONS_PREFIX}${host}`; +} + // Retained-version ZSET for a worker: score=int version, member=`v`. /** @param {string} ns @param {string} worker @returns {string} */ export function workerVersionsKey(ns, worker) { diff --git a/system-workers/s3-cleanup/src/index.js b/system-workers/s3-cleanup/src/index.js index a75fd8e..2e3a765 100644 --- a/system-workers/s3-cleanup/src/index.js +++ b/system-workers/s3-cleanup/src/index.js @@ -11,7 +11,6 @@ import { createLogLevelBinder, logStructured as emitStructuredLog, } from "../../../shared/observability.js"; -import { discardResponseBody } from "../../../shared/respond.js"; import { errorMessage } from "../../../shared/errors.js"; import { bytesToBase64 } from "../../../shared/base64.js"; import { @@ -29,6 +28,10 @@ import { xmlUnescape, } from "../../../shared/s3-xml.js"; import { encodeS3Query } from "../../../shared/s3-query.js"; +import { + S3_TRANSIENT_RETRIES, + fetchRetryableS3Post, +} from "../../../shared/s3-retry.js"; const MAX_ATTEMPTS = 10; const BACKOFF_MAX_MS = 30 * 60_000; @@ -37,9 +40,6 @@ const CRON_BATCH = 100; const MAX_DELETE_PAGES_PER_RUN = 1; const MAX_LIST_PAGES = 1000; const PROCESSING_LEASE_MS = 30 * 60_000; -const S3_TRANSIENT_RETRIES = 10; -const S3_RETRY_BASE_MS = 50; -const S3_RETRY_MAX_MS = 5_000; const DB_BINDING = "S3_CLEANUP_DB"; const SERVICE = "s3-cleanup"; const utf8Encoder = new TextEncoder(); @@ -59,47 +59,6 @@ export function nextBackoffMs(attempts) { return Math.min(BACKOFF_BASE_MS * 2 ** (attempts - 1), BACKOFF_MAX_MS); } -/** @param {number} ms */ -function delay(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** @param {number} attempt */ -function s3RetryDelayMs(attempt) { - return Math.min(S3_RETRY_BASE_MS * 2 ** attempt, S3_RETRY_MAX_MS); -} - -/** @param {Response} response */ -function isTransientS3Response(response) { - return response.status === 429 || response.status >= 500; -} - -/** - * DeleteObjects is a POST, but deleting the same key set is safe to retry. - * Keep this scoped to S3 POST calls with known idempotent semantics. - * @param {SigV4Client} client - * @param {string} url - * @param {RequestInit} init - */ -async function fetchRetryableS3Post(client, url, init) { - for (let attempt = 0; attempt <= S3_TRANSIENT_RETRIES; attempt += 1) { - let response; - try { - response = await client.fetch(url, init); - } catch (err) { - if (attempt === S3_TRANSIENT_RETRIES) throw err; - await delay(Math.random() * s3RetryDelayMs(attempt)); - continue; - } - if (attempt === S3_TRANSIENT_RETRIES || !isTransientS3Response(response)) { - return response; - } - await discardResponseBody(response); - await delay(Math.random() * s3RetryDelayMs(attempt)); - } - throw new Error("unreachable S3 retry loop exit"); -} - /** * @param {"debug" | "info" | "warn" | "error"} level * @param {string} event diff --git a/terraform/modules/compute/security_groups.tf b/terraform/modules/compute/security_groups.tf index 5ccef91..f901ecb 100644 --- a/terraform/modules/compute/security_groups.tf +++ b/terraform/modules/compute/security_groups.tf @@ -36,6 +36,10 @@ resource "aws_security_group" "runtime" { } } +# ECS intentionally shares this SG across user, system, D1, and DO tasks. Rules +# sourced from or targeting it are therefore coarser than Kubernetes per-component +# NetworkPolicies; splitting those caller sets requires splitting this SG first. + # Range covers :8081 (loader sockets) + :8082 (system-runtime control). resource "aws_security_group_rule" "runtime_from_gateway" { type = "ingress" diff --git a/tests/fixtures/cross-language-identity.json b/tests/fixtures/cross-language-identity.json index 4c6d227..248df6f 100644 --- a/tests/fixtures/cross-language-identity.json +++ b/tests/fixtures/cross-language-identity.json @@ -50,6 +50,20 @@ { "value": "jobs:tail", "valid": false }, { "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "valid": false } ], + "kvIds": [ + { "value": "cache", "valid": true }, + { "value": "9cache", "valid": true }, + { "value": "session-store", "valid": true }, + { "value": "a-", "valid": true }, + { "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "valid": true }, + { "value": "", "valid": false }, + { "value": "-cache", "valid": false }, + { "value": "Cache", "valid": false }, + { "value": "cache_id", "valid": false }, + { "value": "cache:id", "valid": false }, + { "value": "cache/id", "valid": false }, + { "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "valid": false } + ], "workflowInstanceIds": [ { "value": "inst-1", "valid": true }, { "value": "a_1", "valid": true }, diff --git a/tests/fixtures/internal-auth-contract.json b/tests/fixtures/internal-auth-contract.json new file mode 100644 index 0000000..72724ab --- /dev/null +++ b/tests/fixtures/internal-auth-contract.json @@ -0,0 +1,18 @@ +{ + "header": "x-wdl-internal-auth", + "currentEnv": "WDL_INTERNAL_AUTH_TOKEN", + "previousEnv": "WDL_INTERNAL_AUTH_PREVIOUS_TOKEN", + "failure": { + "status": 401, + "error": "internal_auth_failed", + "message": "Internal authentication failed" + }, + "rotationCases": [ + { "actual": "current", "current": "current", "previous": null, "accepted": true }, + { "actual": "current", "current": "current", "previous": "previous", "accepted": true }, + { "actual": "previous", "current": "current", "previous": "previous", "accepted": true }, + { "actual": "wrong", "current": "current", "previous": "previous", "accepted": false }, + { "actual": null, "current": "current", "previous": "previous", "accepted": false }, + { "actual": "previous", "current": "current", "previous": null, "accepted": false } + ] +} diff --git a/tests/fixtures/observability-contract.json b/tests/fixtures/observability-contract.json index b3698ad..fef5f54 100644 --- a/tests/fixtures/observability-contract.json +++ b/tests/fixtures/observability-contract.json @@ -18,5 +18,15 @@ { "raw": "a\rb", "escaped": "a\\rb" }, { "raw": "a\"b", "escaped": "a\\\"b" }, { "raw": "\r\n\"\\", "escaped": "\\r\\n\\\"\\\\" } - ] + ], + "logEnvelope": { + "orderedKeys": ["ts", "service", "level", "event"], + "timestampShape": "0000-00-00T00:00:00.000Z", + "levels": [ + { "name": "debug", "priority": 10, "stream": "stdout" }, + { "name": "info", "priority": 20, "stream": "stdout" }, + { "name": "warn", "priority": 30, "stream": "stdout" }, + { "name": "error", "priority": 40, "stream": "stderr" } + ] + } } diff --git a/tests/fixtures/queue-key-parse.json b/tests/fixtures/queue-key-parse.json new file mode 100644 index 0000000..5a85d1b --- /dev/null +++ b/tests/fixtures/queue-key-parse.json @@ -0,0 +1,34 @@ +{ + "stream": [ + { "key": "queue:demo:jobs:s", "parsed": { "ns": "demo", "queue": "jobs" } }, + { "key": "queue:__platform__:jobs:s", "parsed": { "ns": "__platform__", "queue": "jobs" } }, + { "key": "queue:t:my-queue-1:s", "parsed": { "ns": "t", "queue": "my-queue-1" } }, + { "key": "queue:demo:jobs", "parsed": null }, + { "key": "queue-delayed:demo:jobs", "parsed": null }, + { "key": "queue::jobs:s", "parsed": null }, + { "key": "queue:admin:jobs:s", "parsed": null }, + { "key": "queue:__community__:jobs:s", "parsed": null }, + { "key": "queue:demo:Jobs:s", "parsed": null }, + { "key": "queue:demo:jobs:extra:s", "parsed": null }, + { "key": "notaqueue:demo:jobs:s", "parsed": null }, + { "key": "", "parsed": null } + ], + "delayed": [ + { "key": "queue-delayed:demo:jobs", "parsed": { "ns": "demo", "queue": "jobs" } }, + { "key": "queue-delayed:__system__:jobs-1", "parsed": { "ns": "__system__", "queue": "jobs-1" } }, + { "key": "queue:demo:jobs:s", "parsed": null }, + { "key": "queue-delayed::jobs", "parsed": null }, + { "key": "queue-delayed:demo:", "parsed": null }, + { "key": "queue-delayed:demo:Jobs", "parsed": null }, + { "key": "queue-delayed:demo:jobs:extra", "parsed": null } + ], + "consumer": [ + { "key": "queue-consumer:demo:jobs", "parsed": { "ns": "demo", "queue": "jobs" } }, + { "key": "queue-consumer:__platform__:jobs-1", "parsed": { "ns": "__platform__", "queue": "jobs-1" } }, + { "key": "queue-consumer:demo", "parsed": null }, + { "key": "queue-consumer:", "parsed": null }, + { "key": "queue-consumer:admin:jobs", "parsed": null }, + { "key": "queue-consumer:demo:jobs:extra", "parsed": null }, + { "key": "other:demo:jobs", "parsed": null } + ] +} diff --git a/tests/fixtures/request-id-sanitizer.json b/tests/fixtures/request-id-sanitizer.json index 3164db7..4fbecda 100644 --- a/tests/fixtures/request-id-sanitizer.json +++ b/tests/fixtures/request-id-sanitizer.json @@ -8,6 +8,12 @@ { "raw": "a\\b", "sanitized": null }, { "raw": "ok\nbad", "sanitized": null }, { "raw": "ok\u0085bad", "sanitized": null }, + { "raw": "ok\u00a0bad", "sanitized": null }, + { "raw": "ok\u2028bad", "sanitized": null }, + { "raw": "ok\u3000bad", "sanitized": null }, + { "raw": "ok\ufeffbad", "sanitized": null }, + { "raw": "\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9", "sanitized": "\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9" }, + { "raw": "\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9", "sanitized": null }, { "raw": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "sanitized": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, { "raw": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "sanitized": null } ] diff --git a/tests/fixtures/scheduler-projection-contract.json b/tests/fixtures/scheduler-projection-contract.json new file mode 100644 index 0000000..ff84136 --- /dev/null +++ b/tests/fixtures/scheduler-projection-contract.json @@ -0,0 +1,48 @@ +{ + "cron": { + "workerIndexKey": "cron:index:workers", + "ns": "tenant", + "worker": "worker", + "workerKey": "crons:tenant:worker", + "slotMs": 1778856600000, + "slotKey": "cron-slot:1778856600000", + "slotExpireAt": 1778857200, + "cronId": "cron-1", + "gen": 7, + "reference": "tenant:worker:cron-1:7", + "meta": { + "version": "v3", + "seq": 7, + "json": "{\"version\":\"v3\",\"seq\":7}" + }, + "entry": { + "cron": "*/5 * * * *", + "timezone": "UTC", + "gen": 7, + "json": "{\"cron\":\"*/5 * * * *\",\"timezone\":\"UTC\",\"gen\":7}" + } + }, + "queueConsumer": { + "ns": "tenant", + "queue": "jobs", + "worker": "worker", + "version": "v3", + "input": { + "queue": "jobs", + "maxBatchSize": 12, + "maxBatchTimeoutMs": 250, + "maxRetries": 4, + "deadLetterQueue": "jobs-dlq", + "retryDelaySeconds": 17 + }, + "fields": { + "worker": "worker", + "version": "v3", + "max_batch_size": "12", + "max_batch_timeout_ms": "250", + "max_retries": "4", + "dead_letter_queue": "jobs-dlq", + "retry_delay_secs": "17" + } + } +} diff --git a/tests/fixtures/secret-envelope-parity.json b/tests/fixtures/secret-envelope-parity.json new file mode 100644 index 0000000..be1a176 --- /dev/null +++ b/tests/fixtures/secret-envelope-parity.json @@ -0,0 +1,74 @@ +{ + "provider": { + "kid": "local:test:secret-envelope:v1", + "localKeyB64": "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" + }, + "vectors": [ + { + "name": "value", + "plaintext": "sensitive-value", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "randomStart": 0, + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}" + }, + { + "name": "empty", + "plaintext": "", + "hashKey": "secrets:demo:api", + "fieldName": "EMPTY", + "randomStart": 0, + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLSmHuPVe4uS7JepH053l0Yt\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"\",\"tag\":\"TTVUTxM6DfSV+VZ/AODK1w==\"}" + } + ], + "rejections": [ + { + "name": "wrong-hash", + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:other:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "secret_decrypt_failed" + }, + { + "name": "wrong-field", + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:demo:api", + "fieldName": "OTHER", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "secret_decrypt_failed" + }, + { + "name": "wrong-kid", + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v2", + "jsErrorCode": "unknown_kid" + }, + { + "name": "non-canonical-json", + "envelope": "WDL-ENC:{ \"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "invalid_envelope" + }, + { + "name": "non-canonical-field-order", + "envelope": "WDL-ENC:{\"alg\":\"AES-256-GCM\",\"v\":1,\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "invalid_envelope" + }, + { + "name": "non-canonical-base64", + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg\"}", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "invalid_envelope" + } + ] +} diff --git a/tests/fixtures/workflow-limits.json b/tests/fixtures/workflow-limits.json new file mode 100644 index 0000000..29e1509 --- /dev/null +++ b/tests/fixtures/workflow-limits.json @@ -0,0 +1,5 @@ +{ + "resultBytesMax": 1048576, + "backendRequestBytesMax": 2097152, + "payloadTooLargeCode": "workflow_payload_too_large" +} diff --git a/tests/helpers/control-handler-harness.js b/tests/helpers/control-handler-harness.js index 8145e92..b290f02 100644 --- a/tests/helpers/control-handler-harness.js +++ b/tests/helpers/control-handler-harness.js @@ -4,7 +4,6 @@ import { importSpecifierReplacements, } from "./load-shared-module.js"; import { createFakeRedis } from "./mocks/fake-redis.js"; -import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; const DEFAULT_GLOBAL_NAME = "__controlHandlerHarnessState"; @@ -79,7 +78,6 @@ export const metrics = { increment(...args) { harnessState().metrics.increment?.(...args); }, observe(...args) { harnessState().metrics.observe?.(...args); }, }; -export { formatError } from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}; ${extraSource} `); } diff --git a/tests/helpers/control-shared-stub.js b/tests/helpers/control-shared-stub.js index dccc3f2..d310fec 100644 --- a/tests/helpers/control-shared-stub.js +++ b/tests/helpers/control-shared-stub.js @@ -2,100 +2,62 @@ // tail so each test wires its own redis fakes without re-declaring the base // helpers. -import { moduleDataUrl } from "./load-shared-module.js"; +import { + freshRepositoryModuleDataUrl, + moduleDataUrl, + repositoryFileUrl, +} from "./load-shared-module.js"; +import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; +import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; + +const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); +const SHARED_ENV_URL = repositoryFileUrl("shared/env.js"); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); +const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const CONTROL_ERRORS_URL = freshRepositoryModuleDataUrl("control/errors.js", [ + [/from "shared-respond"/g, `from ${JSON.stringify(SHARED_RESPOND_URL)}`], +]); +const CONTROL_WORKFLOWS_CLIENT_URL = freshRepositoryModuleDataUrl("control/workflows-client.js", [ + [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], + [/from "control-errors"/g, `from ${JSON.stringify(CONTROL_ERRORS_URL)}`], +]); +const SHARED_OWNER_LEASE_URL = freshRepositoryModuleDataUrl("shared/owner-lease.js", [ + [/from "shared-env"/g, `from ${JSON.stringify(SHARED_ENV_URL)}`], +]); +const CONTROL_OPTIMISTIC_REDIS_URL = sharedRedisStubUrl(); +const CONTROL_OPTIMISTIC_URL = freshRepositoryModuleDataUrl("control/optimistic.js", [ + [/from "shared-redis"/g, `from ${JSON.stringify(CONTROL_OPTIMISTIC_REDIS_URL)}`], + [/from "shared-owner-lease"/g, `from ${JSON.stringify(SHARED_OWNER_LEASE_URL)}`], +]); +const CONTROL_JSON_BODY_URL = freshRepositoryModuleDataUrl("control/json-body.js", [ + [/from "shared-bounded-body"/g, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)}`], + [/from "shared-respond"/g, `from ${JSON.stringify(SHARED_RESPOND_URL)}`], +]); const CONTROL_SHARED_BASE = ` -export function jsonResponse(status, data, extraHeaders = {}) { - return new Response(JSON.stringify(data), { status, headers: { "content-type": "application/json", ...extraHeaders } }); -} -function sanitizeJsonErrorDetails(value) { - if (Array.isArray(value)) return value; - if (!value || typeof value !== "object") return value; - const out = {}; - for (const [key, entry] of Object.entries(value)) { - if (key === "error" || key === "message" || key === "reason") continue; - if (entry !== undefined) { - Object.defineProperty(out, key, { - value: entry, - enumerable: true, - configurable: true, - writable: true, - }); - } - } - return Object.keys(out).length === 0 ? undefined : out; -} -export function jsonError(status, error, message, details = {}, extraHeaders = {}) { - const body = { error }; - if (message) body.message = message; - const sanitized = sanitizeJsonErrorDetails(details); - const safeDetails = sanitized && typeof sanitized === "object" && !Array.isArray(sanitized) ? sanitized : {}; - return jsonResponse(status, { ...safeDetails, ...body }, extraHeaders); -} -export async function readJsonBody(request, { requireObject = false, allowEmpty = false, maxBytes = 1024 * 1024 } = {}) { - let body; - try { - const declared = Number(request.headers.get("content-length") || 0); - if (Number.isFinite(declared) && declared > maxBytes) { - return { response: jsonError(413, "request_body_too_large", "Body must be at most " + maxBytes + " bytes") }; - } - const text = await request.text(); - if (Buffer.byteLength(text, "utf8") > maxBytes) { - return { response: jsonError(413, "request_body_too_large", "Body must be at most " + maxBytes + " bytes") }; - } - if (text === "") { - if (!allowEmpty) { - return { response: jsonError(400, "invalid_json", "Body must be valid JSON") }; - } - body = {}; - } else { - body = JSON.parse(text); - } - } catch { - return { response: jsonError(400, "invalid_json", "Body must be valid JSON") }; - } - if (requireObject && (!body || typeof body !== "object" || Array.isArray(body))) { - return { response: jsonError(400, "invalid_json_object", "Body must be a JSON object") }; - } - return { body }; -} -export class ControlAbort extends Error { - constructor(status, code, details = {}) { - super(details?.message || code); - this.status = status; - this.code = code; - this.details = details; - } -} -export function codedErrorResponse(err, fallbackCode = "internal_error", extraDetails = {}) { - const record = err && typeof err === "object" && !Array.isArray(err) ? err : {}; - const details = record.details && typeof record.details === "object" && !Array.isArray(record.details) - ? record.details - : {}; - const status = typeof record.status === "number" ? record.status : 500; - const code = typeof record.code === "string" && record.code ? record.code : fallbackCode; - const recordMessage = typeof record.message === "string" && record.message ? record.message : undefined; - const message = err instanceof ControlAbort - ? (typeof err.details.message === "string" && err.details.message) || err.message || code - : recordMessage || (typeof details.message === "string" && details.message) || code; - return jsonError( - status, - code, - message, - { ...details, ...extraDetails }, - ); -} -export function controlAbortResponse(err, extraDetails = {}) { - return codedErrorResponse(err, err.code, extraDetails); -} -export function errMessage(err) { - return err instanceof Error ? err.message : String(err); -} -export function randomHex(bytes = 16) { - const data = new Uint8Array(bytes); - crypto.getRandomValues(data); - return Array.from(data, (b) => b.toString(16).padStart(2, "0")).join(""); -} +import { jsonError, jsonResponse, sanitizeJsonErrorDetails } from ${JSON.stringify(SHARED_RESPOND_URL)}; +import { createPostWorkflowsInternal } from ${JSON.stringify(CONTROL_WORKFLOWS_CLIENT_URL)}; +import { ControlAbort, codedErrorResponse, controlAbortResponse } from ${JSON.stringify(CONTROL_ERRORS_URL)}; +import { runOptimistic, withOptimisticRetries } from ${JSON.stringify(CONTROL_OPTIMISTIC_URL)}; +import { readJsonBody } from ${JSON.stringify(CONTROL_JSON_BODY_URL)}; +import { errorMessage as errMessage } from ${JSON.stringify(SHARED_ERRORS_URL)}; +import { randomHex } from ${JSON.stringify(SHARED_RANDOM_ID_URL)}; +import { formatError } from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}; +export { + ControlAbort, + codedErrorResponse, + controlAbortResponse, + errMessage, + formatError, + jsonError, + jsonResponse, + randomHex, + readJsonBody, + runOptimistic, + sanitizeJsonErrorDetails, + withOptimisticRetries, +}; export function prefixedId(prefix, bytes = 16) { return prefix + randomHex(bytes); } @@ -125,37 +87,17 @@ export function getControlS3() { export function getControlR2() { return state.r2; } -export function getControlWorkflows() { - return state.workflows; -} export function controlInternalJsonHeaders() { const token = state.env?.WDL_INTERNAL_AUTH_TOKEN; return token ? { "content-type": "application/json", "x-wdl-internal-auth": token } : { "content-type": "application/json" }; } -function isWatchError(err) { - return err instanceof Error && ( - err.name === "WatchError" - || err.constructor?.name === "WatchError" - ); -} -export async function runOptimistic(redis, { attempts = 5, onExhausted, onWatchError, shouldRetryResult }, fn) { - for (let attempt = 0; attempt < attempts; attempt += 1) { - try { - const result = await redis.session((session) => fn(session, attempt)); - if (shouldRetryResult?.(result, attempt)) continue; - return result; - } catch (err) { - if (isWatchError(err)) { - onWatchError?.(err, attempt); - continue; - } - throw err; - } - } - return await onExhausted(); -} +export const postWorkflowsInternal = createPostWorkflowsInternal({ + getWorkflows: () => state.workflows, + headers: controlInternalJsonHeaders, + getLog: () => state.log, +}); export async function recordCleanupIntentOrWarn({ cleanupIntent, cleanupTaskId, diff --git a/tests/helpers/load-auth-index.js b/tests/helpers/load-auth-index.js index 1f6b99d..afb3eeb 100644 --- a/tests/helpers/load-auth-index.js +++ b/tests/helpers/load-auth-index.js @@ -12,16 +12,19 @@ import { } from "./load-shared-module.js"; import { compileSharedAuthRoles } from "./load-auth-roles.js"; import { CLOUDFLARE_WORKERS_URL } from "./mocks/cloudflare-workers.js"; +import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; const SHARED_NS_URL = repositoryFileUrl("shared/ns-pattern.js"); const SHARED_AUTH_TOKEN_URL = repositoryFileUrl("shared/auth-token.js"); const SHARED_HEX_URL = repositoryFileUrl("shared/hex.js"); const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); +const SHARED_REDIS_LOCK_URL = freshRepositoryModuleDataUrl("shared/redis-lock.js", [ + [/from "shared-random-id"/g, `from ${JSON.stringify(SHARED_RANDOM_ID_URL)}`], +]); const SHARED_OBSERVABILITY_URL = repositoryFileUrl("shared/observability.js"); +const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const SHARED_REDIS_MOCK = ` -export class WatchError extends Error {} - function ensureState() { if (!globalThis.__authMockState) { throw new Error("auth-index harness: globalThis.__authMockState not initialized"); @@ -301,7 +304,7 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel const authLib = await import(authLibUrl); // Mocks have no module-level state; cache them across loads. - const sharedRedisUrl = moduleDataUrl(SHARED_REDIS_MOCK); + const sharedRedisUrl = sharedRedisStubUrl(SHARED_REDIS_MOCK); const sharedObservabilityUrl = moduleDataUrl(SHARED_OBSERVABILITY_MOCK); const authRuntimeUrl = freshRepositoryModuleDataUrl("auth/runtime.js", [ @@ -316,6 +319,8 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel [/from "auth-lib"/g, `from ${JSON.stringify(authLibUrl)}`], [/from "auth-runtime"/g, `from ${JSON.stringify(authRuntimeUrl)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], + [/from "shared-redis-lock"/g, `from ${JSON.stringify(SHARED_REDIS_LOCK_URL)}`], + [/from "shared-version"/g, `from ${JSON.stringify(SHARED_VERSION_URL)}`], ]); const indexMod = await import(indexUrl); diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index d66bc81..5e937b2 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -41,6 +41,7 @@ export async function compileControlGraph(opts = {}) { const libUrl = freshRepositoryModuleDataUrl("control/lib.js", [ [/from "shared-ns-pattern"/g, `from ${JSON.stringify(SHARED_NS_URL)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], + [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], [/from "shared-version"/g, `from ${JSON.stringify(SHARED_VERSION_URL)}`], ]); @@ -78,6 +79,7 @@ export async function compileControlGraph(opts = {}) { ...importSpecifierReplacements({ "shared-ns-pattern": SHARED_NS_URL, "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, + "control-lib": libUrl, }), [/from "control-bindings"/g, `from ${JSON.stringify(bindingsUrl)}`], [/from "wdl-package-json-source"/g, `from ${JSON.stringify(packageJsonSourceUrl)}`], diff --git a/tests/helpers/load-control-routing.js b/tests/helpers/load-control-routing.js index 2ad9100..66b5f3d 100644 --- a/tests/helpers/load-control-routing.js +++ b/tests/helpers/load-control-routing.js @@ -21,10 +21,7 @@ const { routePlanUrl, } = await compileControlGraph(); -const controlSharedUrl = controlSharedStubUrl(` -export const DECLARED_HOSTS_KEY = "declared-hosts"; -export const HOST_DECLARATIONS_PREFIX = "host-declarations:"; -`); +const controlSharedUrl = controlSharedStubUrl(); export const CONTROL_ROUTING_TEST_URL = moduleDataUrl(applyModuleReplacements(readRepositoryFile("control/routing.js"), [ ...importSpecifierReplacements({ diff --git a/tests/helpers/load-d1-owner-registry.js b/tests/helpers/load-d1-owner-registry.js index 63b2a80..610de4c 100644 --- a/tests/helpers/load-d1-owner-registry.js +++ b/tests/helpers/load-d1-owner-registry.js @@ -7,7 +7,11 @@ import { repositoryModuleDataUrl, sharedModuleDataUrl, } from "./load-shared-module.js"; -import { createFakeRedisState, resetFakeRedisState } from "./mocks/fake-redis.js"; +import { + createFakeRedisState, + resetFakeRedisState, + sharedRedisStubUrl, +} from "./mocks/fake-redis.js"; import { delay } from "./timing.js"; const FAKE_REDIS_URL = repositoryFileUrl("tests/helpers/mocks/fake-redis.js"); @@ -87,16 +91,7 @@ export async function resolveTaskIdentity() { } `); -const redisUrl = moduleDataUrl(` -const D1_TEST_STATE = /** @type {any} */ (globalThis).__d1OwnerRegistryTestState; -export { FakeRedisWatchError as WatchError } from ${JSON.stringify(FAKE_REDIS_URL)}; -export function decodeBulk(value) { - if (value == null) return value; - if (typeof value === "string") return value; - if (value instanceof Uint8Array) return new TextDecoder().decode(value); - return String(value); -} -`); +const redisUrl = sharedRedisStubUrl(); const redisClientUrl = moduleDataUrl(` import { createFakeRedisClient, createFakeRedisSession } from ${JSON.stringify(FAKE_REDIS_URL)}; diff --git a/tests/helpers/load-do-host-actor.js b/tests/helpers/load-do-host-actor.js index 65ef04b..03d59c4 100644 --- a/tests/helpers/load-do-host-actor.js +++ b/tests/helpers/load-do-host-actor.js @@ -4,6 +4,7 @@ import { moduleDataUrl, repositoryFileUrl, } from "./load-shared-module.js"; +import { doProtocolDataUrl } from "./load-do-protocol.js"; import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; /** @@ -38,6 +39,7 @@ const DO_ACTOR_TEST_STATE = { doActorGlobal.__doActorTestState = DO_ACTOR_TEST_STATE; const durableObjectStub = "class DurableObject { constructor(ctx, env) { this.ctx = ctx; this.env = env; } }"; +const productionProtocolUrl = doProtocolDataUrl(); const loadUrl = moduleDataUrl(` export function loadDoWorkerCode() { @@ -56,6 +58,7 @@ export async function rememberDoObject(env, invoke) { `); const protocolUrl = moduleDataUrl(` +export { DO_OWNERSHIP_CODE } from ${JSON.stringify(productionProtocolUrl)}; export class DoRuntimeError extends Error { constructor(status, code, message) { super(message); diff --git a/tests/helpers/load-do-owner-registry.js b/tests/helpers/load-do-owner-registry.js index 1bc85d2..1683246 100644 --- a/tests/helpers/load-do-owner-registry.js +++ b/tests/helpers/load-do-owner-registry.js @@ -7,7 +7,11 @@ import { repositoryModuleDataUrl, sharedModuleDataUrl, } from "./load-shared-module.js"; -import { createFakeRedisState, resetFakeRedisState } from "./mocks/fake-redis.js"; +import { + createFakeRedisState, + resetFakeRedisState, + sharedRedisStubUrl, +} from "./mocks/fake-redis.js"; const PROTOCOL_URL = doProtocolDataUrl(); const FAKE_REDIS_URL = repositoryFileUrl("tests/helpers/mocks/fake-redis.js"); @@ -97,14 +101,7 @@ export async function resolveTaskIdentity() { } `); -const sharedRedisUrl = moduleDataUrl(` -export { FakeRedisWatchError as WatchError } from ${JSON.stringify(FAKE_REDIS_URL)}; -export function decodeBulk(value) { - if (value == null || typeof value === "string") return value; - if (value instanceof Uint8Array) return new TextDecoder().decode(value); - return String(value); -} -`); +const sharedRedisUrl = sharedRedisStubUrl(); const redisUrl = moduleDataUrl(` import { createFakeRedisClient } from ${JSON.stringify(FAKE_REDIS_URL)}; diff --git a/tests/helpers/load-runtime-dispatch.js b/tests/helpers/load-runtime-dispatch.js index adf5154..2e5d97b 100644 --- a/tests/helpers/load-runtime-dispatch.js +++ b/tests/helpers/load-runtime-dispatch.js @@ -41,6 +41,7 @@ export async function loadRuntimeDispatch() { ]); const tailForwarderUrl = repositoryModuleDataUrl("runtime/tail-forwarder.js", [ [/from "runtime-bindings-proxy";/, `from ${JSON.stringify(PROXY_BINDING_URL)};`], + [/from "shared-errors";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "shared-internal-auth";/, `from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)};`], ]); diff --git a/tests/helpers/load-runtime-r2-binding.js b/tests/helpers/load-runtime-r2-binding.js index 31ff661..face7a1 100644 --- a/tests/helpers/load-runtime-r2-binding.js +++ b/tests/helpers/load-runtime-r2-binding.js @@ -13,6 +13,7 @@ const R2_UTILS_URL = repositoryFileUrl("runtime/r2-utils.js"); const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); const SHARED_S3_XML_URL = repositoryFileUrl("shared/s3-xml.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_S3_RETRY_URL = repositoryFileUrl("shared/s3-retry.js"); const R2_METADATA_URL = repositoryModuleDataUrl("runtime/bindings/r2/metadata.js", [ [/from "runtime-r2-utils";/, `from ${JSON.stringify(R2_UTILS_URL)};`], ]); @@ -51,6 +52,7 @@ const mod = await importRepositoryModule("runtime/bindings/r2.js", [ [/from "runtime-bindings-r2-metadata";/, `from ${JSON.stringify(R2_METADATA_URL)};`], [/from "runtime-bindings-r2-xml";/, `from ${JSON.stringify(R2_XML_URL)};`], [/from "shared-respond";/, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], + [/from "shared-s3-retry";/, `from ${JSON.stringify(SHARED_S3_RETRY_URL)};`], ]); export const { R2Bucket } = mod; diff --git a/tests/helpers/load-shared-module.js b/tests/helpers/load-shared-module.js index 949dc48..f72082f 100644 --- a/tests/helpers/load-shared-module.js +++ b/tests/helpers/load-shared-module.js @@ -10,6 +10,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(__dirname, "../.."); const SHARED_ENV_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/env.js")).href; const SHARED_ERRORS_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/errors.js")).href; +const SHARED_BASE64_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/base64.js")).href; const SHARED_HEX_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/hex.js")).href; const SHARED_WORKERD_COMPAT_FLAGS_URL = pathToFileURL( path.resolve(REPO_ROOT, "shared/workerd-compat-flags.js") @@ -87,6 +88,7 @@ export function repositoryModuleDataUrl(relativePath, replacements = []) { export function runtimeLibModuleDataUrl() { return repositoryModuleDataUrl("runtime/lib.js", importSpecifierReplacements({ + "shared-base64": SHARED_BASE64_URL, "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, })); } diff --git a/tests/helpers/mocks/fake-redis.js b/tests/helpers/mocks/fake-redis.js index 7ecab29..f7a7849 100644 --- a/tests/helpers/mocks/fake-redis.js +++ b/tests/helpers/mocks/fake-redis.js @@ -1,3 +1,16 @@ +import { + moduleDataUrl, + repositoryFileUrl, + repositoryModuleDataUrl, +} from "../load-shared-module.js"; + +const SHARED_OBSERVABILITY_URL = repositoryFileUrl("shared/observability.js"); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_REDIS_RESP_URL = repositoryModuleDataUrl("shared/redis-resp.js", [ + [/from "shared-observability";/, `from ${JSON.stringify(SHARED_OBSERVABILITY_URL)};`], + [/from "\.\/errors\.js";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], +]); + /** @typedef {Map} FakeRedisStrings */ /** @typedef {Map>} FakeRedisHashes */ /** @typedef {Map>} FakeRedisSets */ @@ -26,6 +39,20 @@ export class FakeRedisWatchError extends Error { } } +/** + * Canonical `shared-redis` test surface. Callers may append only the + * state-bound exports their module graph needs. + * @param {string} [extraSource] + */ +export function sharedRedisStubUrl(extraSource = "") { + return moduleDataUrl(` +import { FakeRedisWatchError as WatchError } from ${JSON.stringify(import.meta.url)}; +export { WatchError }; +export { decodeBulk } from ${JSON.stringify(SHARED_REDIS_RESP_URL)}; +${extraSource} +`); +} + /** @returns {FakeRedisState} */ export function createFakeRedisState() { return { diff --git a/tests/helpers/mocks/observability.js b/tests/helpers/mocks/observability.js index 2dcf611..8e5406b 100644 --- a/tests/helpers/mocks/observability.js +++ b/tests/helpers/mocks/observability.js @@ -1,9 +1,15 @@ // Tests that drive logger or metrics spies inline their own variant — the // spy shape is per-test. -import { moduleDataUrl } from "../load-shared-module.js"; +import { moduleDataUrl, repositoryFileUrl } from "../load-shared-module.js"; + +const SHARED_OBSERVABILITY_URL = repositoryFileUrl("shared/observability.js"); const OBSERVABILITY_NOOP_SOURCE = String.raw` +import { formatError, sanitizeRequestId } from ${JSON.stringify(SHARED_OBSERVABILITY_URL)}; + +export { formatError, sanitizeRequestId }; + export class MetricsRegistry { increment() {} observe() {} @@ -15,19 +21,6 @@ export function createLogger() { export function createLogLevelBinder() { return function bindLogLevel() {}; } -export function formatError(err) { - if (!err) return { error_message: "Unknown error" }; - if (err instanceof Error) { - const out = { error_name: err.name, error_message: err.message }; - if (typeof /** @type {any} */ (err).code === "string") { - out.error_code = /** @type {any} */ (err).code; - } else if (typeof /** @type {any} */ (err).reason === "string") { - out.error_code = /** @type {any} */ (err).reason; - } - return out; - } - return { error_message: String(err) }; -} export function recordRedisCommand() {} export function recordRequestComplete() {} export function ensureRequestId(headersLike) { @@ -37,18 +30,6 @@ export function ensureRequestId(headersLike) { : headersLike["x-request-id"]; return sanitizeRequestId(raw) || generateRequestId(); } -export function sanitizeRequestId(raw) { - if (Array.isArray(raw)) raw = raw[0]; - if (typeof raw !== "string") return null; - const first = raw.split(",")[0].trim(); - if (!first || first.length > 128) return null; - if ([...first].some((ch) => ch.trim() === "" || ch === '"' || ch === "\\")) return null; - for (let i = 0; i < first.length; i++) { - const code = first.charCodeAt(i); - if (code < 0x20 || code === 0x7f || (code >= 0x80 && code <= 0x9f)) return null; - } - return first; -} export function generateRequestId() { return "rid"; } diff --git a/tests/helpers/style-contract-scanner.js b/tests/helpers/style-contract-scanner.js index 950dfc5..e2c7400 100644 --- a/tests/helpers/style-contract-scanner.js +++ b/tests/helpers/style-contract-scanner.js @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { readdirSync } from "node:fs"; import path from "node:path"; +import { isAlias, isSeq, parseAllDocuments, parseDocument, visit } from "yaml"; import { readRepoFile, repoPath } from "./source-scan.js"; @@ -52,6 +53,49 @@ export function testSourceFiles(dir) { return out; } +/** + * @param {string} file + * @param {{ merge?: boolean }} [options] + */ +export function yamlDocument(file, options = {}) { + const document = parseDocument(readRepoFile(file), { merge: options.merge === true }); + assert.equal(document.errors.length, 0, `${file} must parse as YAML`); + return document; +} + +/** @param {string} file */ +export function yamlDocuments(file) { + const documents = parseAllDocuments(readRepoFile(file)); + for (const document of documents) { + assert.equal(document.errors.length, 0, `${file} must parse as YAML`); + } + return documents; +} + +/** + * @param {import("yaml").Document} document + * @param {(string | number)[]} pathParts + * @returns {string[]} + */ +export function yamlMergeAliases(document, pathParts) { + const merge = document.getIn([...pathParts, "<<"], true); + if (isAlias(merge)) return [merge.source]; + assert.ok(isSeq(merge), `YAML path ${pathParts.join(".")} must contain a merge alias`); + assert.ok(merge.items.every(isAlias), `YAML path ${pathParts.join(".")} merge entries must be aliases`); + return merge.items.map((item) => /** @type {import("yaml").Alias} */ (item).source); +} + +/** @param {import("yaml").Document} document @param {string} source */ +export function yamlAliasCount(document, source) { + let count = 0; + visit(document, { + Alias(_key, alias) { + if (alias.source === source) count += 1; + }, + }); + return count; +} + /** @param {Set} [exempt] */ export function scannedTestFiles(exempt = new Set()) { const scanned = [ @@ -153,16 +197,72 @@ export function extractRegex(file, name) { } /** - * @param {string} service - * @param {string} anchor + * @param {string} source + * @param {number} openIndex + * @param {string} open + * @param {string} close + */ +function findClosingDelimiter(source, openIndex, open, close) { + let depth = 0; + let quote = ""; + for (let index = openIndex; index < source.length; index += 1) { + const char = source[index]; + if (quote) { + if (char === "\\") index += 1; + else if (char === quote) quote = ""; + continue; + } + if (char === '"') { + quote = char; + } else if (char === open) { + depth += 1; + } else if (char === close) { + depth -= 1; + if (depth === 0) return index; + } + } + return -1; +} + +/** + * Extracts a Cap'n Proto const struct while ignoring comments and formatting. + * @param {string} source + * @param {string} name */ -export function serviceAnchorRegex(service, anchor) { - return new RegExp(`\\n ${RegExp.escape(service)}:\\n <<: \\*${RegExp.escape(anchor)}\\b`); +export function extractCapnpConstBlock(source, name) { + const clean = source.replace(/#.*$/gm, ""); + const declaration = new RegExp(`const\\s+${RegExp.escape(name)}\\s+:[^=]+=`).exec(clean); + assert.ok(declaration, `Cap'n Proto const ${name} must be present`); + const start = clean.indexOf("(", declaration.index + declaration[0].length); + assert.notEqual(start, -1, `Cap'n Proto const ${name} must contain a struct value`); + const end = findClosingDelimiter(clean, start, "(", ")"); + assert.notEqual(end, -1, `Cap'n Proto const ${name} must have balanced parentheses`); + return clean.slice(start, end + 1); } -/** @param {string} service */ -export function d1RuntimeServiceRegex(service) { - return new RegExp(`\\n ${RegExp.escape(service)}:\\n(?: profiles: \\["d1-multi"\\]\\n)? <<: \\*d1-runtime-service\\b`); +/** + * Returns normalized top-level tuple entries from a list field in a Cap'n Proto struct. + * @param {string} block + * @param {string} field + */ +export function extractCapnpListEntries(block, field) { + const assignment = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*\\[`).exec(block); + assert.ok(assignment, `Cap'n Proto field ${field} must be present`); + const listStart = block.indexOf("[", assignment.index); + const listEnd = findClosingDelimiter(block, listStart, "[", "]"); + assert.notEqual(listEnd, -1, `Cap'n Proto field ${field} must have balanced brackets`); + + const entries = []; + let index = listStart + 1; + while (index < listEnd) { + const start = block.indexOf("(", index); + if (start === -1 || start >= listEnd) break; + const end = findClosingDelimiter(block, start, "(", ")"); + assert.notEqual(end, -1, `Cap'n Proto field ${field} entry must have balanced parentheses`); + entries.push(block.slice(start, end + 1).replace(/\s+/g, " ").trim()); + index = end + 1; + } + return entries; } /** diff --git a/tests/integration/helpers/env.js b/tests/integration/helpers/env.js index d982bc2..70cd744 100644 --- a/tests/integration/helpers/env.js +++ b/tests/integration/helpers/env.js @@ -2,7 +2,12 @@ // tests. Imported by nearly every other helper; never imports sibling helpers. import { + ADMIN_HOST_HEADER, + LOCAL_ADMIN_TOKEN, + LOCAL_CONNECT_HOST, ROOT, + localAssetsCdnBase, + localControlUrl, resolveWdlCliBin, } from "../../../scripts/integration-environment.js"; @@ -10,22 +15,22 @@ export { ROOT }; // Control reaches through gateway with Host: admin.test — // gateway's admin-host short-circuit picks it up. Matches docker-compose.yml. -export const GATEWAY_HOST = "localhost"; +export const GATEWAY_HOST = LOCAL_CONNECT_HOST; export const GATEWAY_PORT = Number(process.env.WDL_GATEWAY_HOST_PORT || 8080); -export const S3MOCK_HOST = "localhost"; +export const S3MOCK_HOST = LOCAL_CONNECT_HOST; export const S3MOCK_PORT = Number(process.env.WDL_S3MOCK_HOST_PORT || 19500); -export const ASSETS_CDN_BASE = `http://${S3MOCK_HOST}:${S3MOCK_PORT}/wdl-assets`; -export const ADMIN_HOST_HEADER = "admin.test"; -export const ADMIN_TOKEN = "local-dev-token"; +export const ASSETS_CDN_BASE = localAssetsCdnBase(S3MOCK_PORT); +export { ADMIN_HOST_HEADER }; +export const ADMIN_TOKEN = LOCAL_ADMIN_TOKEN; export const WDL_CLI_BIN = resolveWdlCliBin(); // Integration tests are local-compose only. Force the control env so host // shell credentials (staging/production ADMIN_TOKEN, CONTROL_URL, WDL_NS, etc.) cannot // leak into adminFetch() or CLI child processes and produce misleading 401s. process.env.ADMIN_TOKEN = ADMIN_TOKEN; -export const CONTROL_URL = `http://${ADMIN_HOST_HEADER}:${GATEWAY_PORT}`; +export const CONTROL_URL = localControlUrl(GATEWAY_PORT); process.env.CONTROL_URL = CONTROL_URL; process.env.ASSETS_CDN_BASE = ASSETS_CDN_BASE; // admin.test has no DNS on most machines — aim the socket at localhost. -process.env.CONTROL_CONNECT_HOST = "localhost"; +process.env.CONTROL_CONNECT_HOST = LOCAL_CONNECT_HOST; delete process.env.WDL_NS; delete process.env.CLOUDFLARE_ENV; diff --git a/tests/integration/queue-native-dispatch.test.js b/tests/integration/queue-native-dispatch.test.js index 2742001..b8cbca0 100644 --- a/tests/integration/queue-native-dispatch.test.js +++ b/tests/integration/queue-native-dispatch.test.js @@ -145,4 +145,21 @@ test("/_queued: rejects malformed requests with 400", async () => { { queue: "x", messages: [{ id: "m", first_seen_ms: 0, attempts: 0, body_b64: "", content_type: "v8" }] } ); assertStatus(badContentType, 400, "bad content type request"); + + const badBase64 = runtimeDispatchPost( + "/_queued", + { "x-worker-id": workerId }, + { + queue: "x", + messages: [{ + id: "m", + first_seen_ms: 0, + attempts: 0, + body_b64: "%%%", + content_type: "bytes", + }], + } + ); + assertStatus(badBase64, 400, "invalid base64 request"); + assert.equal(responseJson(badBase64).error, "queue_message_decode_failed"); }); diff --git a/tests/integration/service-bindings-rpc.test.js b/tests/integration/service-bindings-rpc.test.js index f5da0fe..9000f17 100644 --- a/tests/integration/service-bindings-rpc.test.js +++ b/tests/integration/service-bindings-rpc.test.js @@ -511,11 +511,9 @@ const CHAIN_INNER_SRC = ` throw err; } async boomPlainNoMessage() { - // eslint-disable-next-line no-throw-literal throw { code: "E_PLAIN", detail: "plain-obj" }; } async boomPlainWithMessage() { - // eslint-disable-next-line no-throw-literal throw { message: "from-inner-plain", code: "E_PLAIN_MSG", detail: "plain-msg" }; } } diff --git a/tests/unit/bounded-body.test.js b/tests/unit/bounded-body.test.js index 5e878b3..e51d6b6 100644 --- a/tests/unit/bounded-body.test.js +++ b/tests/unit/bounded-body.test.js @@ -1,6 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { BodyTooLargeError, readBoundedBytes, readBoundedText } from "../../shared/bounded-body.js"; +import { + BodyTooLargeError, + readBoundedBytes, + readBoundedStreamBytes, + readBoundedText, +} from "../../shared/bounded-body.js"; test("readBoundedText rejects oversized declared content length", async () => { const request = new Request("https://demo.workers.example", { @@ -39,3 +44,36 @@ test("readBoundedBytes returns an empty body when no request body exists", async const bytes = await readBoundedBytes(request, 1); assert.equal(bytes.byteLength, 0); }); + +test("readBoundedStreamBytes copies a view backed by a larger buffer", async () => { + const backing = new Uint8Array([0, 1, 2, 3]); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(backing.subarray(1, 3)); + controller.close(); + }, + }); + + const bytes = await readBoundedStreamBytes(stream, 2); + + assert.deepEqual([...bytes], [1, 2]); + assert.equal(bytes.buffer.byteLength, 2); +}); + +test("readBoundedStreamBytes supports caller-owned overflow errors", async () => { + let cancelled = false; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2, 3])); + }, + cancel() { + cancelled = true; + }, + }); + + await assert.rejects( + () => readBoundedStreamBytes(stream, 2, () => new TypeError("custom limit")), + /custom limit/ + ); + assert.equal(cancelled, true); +}); diff --git a/tests/unit/control-d1-migrations.test.js b/tests/unit/control-d1-migrations.test.js index 36a691b..c112382 100644 --- a/tests/unit/control-d1-migrations.test.js +++ b/tests/unit/control-d1-migrations.test.js @@ -5,6 +5,7 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { readJsonResponse } from "../helpers/response-json.js"; @@ -15,12 +16,19 @@ const D1_MIGRATIONS_TEST_STATE = { runtimeCalls: null, runtimeResult: null, lockToken: null, + logs: [], splitStatements: null, }; /** @type {typeof globalThis & { __d1MigrationTestState?: typeof D1_MIGRATIONS_TEST_STATE }} */ const d1MigrationGlobal = globalThis; d1MigrationGlobal.__d1MigrationTestState = D1_MIGRATIONS_TEST_STATE; +const sharedRandomIdUrl = repositoryFileUrl("shared/random-id.js"); +const sharedRedisLockUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("shared/redis-lock.js"), + [[/from "shared-random-id"/g, `from ${JSON.stringify(sharedRandomIdUrl)}`]] +)); + const controlSharedUrl = controlSharedStubUrl(` export const state = { redis: { @@ -41,7 +49,9 @@ export const state = { return await fn(globalThis.__d1MigrationTestState.session); }, }, - log() {}, + log(level, event, fields) { + /** @type {any} */ (globalThis).__d1MigrationTestState.logs.push({ level, event, fields }); + }, }; `); @@ -108,6 +118,7 @@ const src = applyModuleReplacements(readRepositoryFile("control/d1-migrations.js [/from "control-d1-model";/, `from ${JSON.stringify(modelUrl)};`], [/from "control-d1-runtime-client";/, `from ${JSON.stringify(backendUrl)};`], [/from "control-d1-store";/, `from ${JSON.stringify(storeUrl)};`], + [/from "shared-redis-lock";/, `from ${JSON.stringify(sharedRedisLockUrl)};`], ]); const { @@ -123,6 +134,7 @@ afterEach(() => { D1_MIGRATIONS_TEST_STATE.runtimeCalls = null; D1_MIGRATIONS_TEST_STATE.runtimeResult = null; D1_MIGRATIONS_TEST_STATE.lockToken = null; + D1_MIGRATIONS_TEST_STATE.logs = []; D1_MIGRATIONS_TEST_STATE.splitStatements = null; }); @@ -265,6 +277,7 @@ test("applyMigrations maps lock renewal loss to 409", async () => { test("applyMigrations includes empty progress when reading existing migrations fails", async () => { /** @type {any} */ (globalThis).__d1MigrationTestState.redis = { async set() { return "OK"; }, + async delIfEq() { throw new Error("migration lock release unavailable"); }, }; /** @type {any} */ (globalThis).__d1MigrationTestState.runtimeResult = { ok: false, @@ -290,6 +303,17 @@ test("applyMigrations includes empty progress when reading existing migrations f assert.equal(body.error, "d1_migrations_apply_failed"); assert.deepEqual(body.applied, []); assert.deepEqual(body.skipped, []); + assert.deepEqual(/** @type {any} */ (globalThis).__d1MigrationTestState.logs, [{ + level: "warn", + event: "d1_migration_lock_release_failed", + fields: { + request_id: "rid-migration-read-failure", + namespace: "demo", + database_id: "d1_main", + error_name: "Error", + error_message: "migration lock release unavailable", + }, + }]); } finally { /** @type {any} */ (globalThis).__d1MigrationTestState.redis = null; /** @type {any} */ (globalThis).__d1MigrationTestState.runtimeResult = null; diff --git a/tests/unit/control-d1-model.test.js b/tests/unit/control-d1-model.test.js index d7bba76..3a5ace7 100644 --- a/tests/unit/control-d1-model.test.js +++ b/tests/unit/control-d1-model.test.js @@ -4,6 +4,7 @@ import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-share const SQL_SPLITTER_URL = repositoryFileUrl("shared/sql-splitter.js"); const SHARED_HEX_URL = repositoryFileUrl("shared/hex.js"); +const SHARED_NS_URL = repositoryFileUrl("shared/ns-pattern.js"); const { decodeDatabaseHash, isReadyDatabase, @@ -18,6 +19,7 @@ const { [/export \{ splitSqlStatements \} from "shared-sql-splitter";/, `export { splitSqlStatements } from ${JSON.stringify(SQL_SPLITTER_URL)};`], [/from "shared-hex";/, `from ${JSON.stringify(SHARED_HEX_URL)};`], + [/from "shared-ns-pattern";/, `from ${JSON.stringify(SHARED_NS_URL)};`], ]); test("control D1: database metadata state is explicit", () => { diff --git a/tests/unit/control-d1-store.test.js b/tests/unit/control-d1-store.test.js index a9dc1fe..dbf0c1d 100644 --- a/tests/unit/control-d1-store.test.js +++ b/tests/unit/control-d1-store.test.js @@ -1,7 +1,7 @@ import { afterEach, test } from "node:test"; import assert from "node:assert/strict"; import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; -import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; +import { createFakeRedis, sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { applyModuleReplacements, moduleDataUrl, @@ -41,10 +41,7 @@ export function d1DatabaseTombstonesKey(ns) { return "d1:database-tombstones:" + export function formatD1ReferrerBlockers() { return { blockers: [], malformedReferrerCount: 0 }; } `); -const sharedRedisUrl = moduleDataUrl(` -export function decodeBulk(value) { return value; } -export class WatchError extends Error {} -`); +const sharedRedisUrl = sharedRedisStubUrl(); const modelUrl = repositoryModuleDataUrl("control/d1-model.js", [ [ @@ -52,6 +49,7 @@ const modelUrl = repositoryModuleDataUrl("control/d1-model.js", [ `export { splitSqlStatements } from ${JSON.stringify(repositoryFileUrl("shared/sql-splitter.js"))};` ], [/from "shared-hex";/, `from ${JSON.stringify(repositoryFileUrl("shared/hex.js"))};`], + [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], ]); const src = applyModuleReplacements(readRepositoryFile("control/d1-store.js"), [ [/from "control-shared";/, `from ${JSON.stringify(controlSharedUrl)};`], diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index 92dfd1b..339ed08 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -11,9 +11,10 @@ import { readRepositoryFile, repositoryFileUrl, } from "../helpers/load-shared-module.js"; +import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; -const { lifecycleIndexesUrl } = await compileControlGraph(); +const { libUrl: productionControlLibUrl, lifecycleIndexesUrl } = await compileControlGraph(); const controlSharedExtraSource = ` export const ROUTES_CHANNEL = "routes:invalidate"; @@ -62,50 +63,30 @@ export async function recordS3CleanupIntent(intent) { `; const controlLibUrl = moduleDataUrl(` -export function workersIndexKey(ns) { return "workers:" + ns; } -export function workerVersionsKey(ns, name) { return "worker-versions:" + ns + ":" + name; } -export function referrersKey(ns, name, version) { - return "worker-version-referrers:" + ns + ":" + name + ":" + version; -} -export function d1DatabaseReferrersKey(ns, databaseId) { - return "d1:database-referrers:" + ns + ":" + databaseId; -} -export function deleteLockKey(ns, worker) { return "worker-delete-lock:" + ns + ":" + worker; } -export function encodeReferrerMember(ref) { return JSON.stringify(ref); } -export function doObjectRegistryKey(id) { return "do:objects:" + id; } -export function doOwnerScopeScanPatternForStorage(id) { return "do:owner:" + id + ":*"; } -export function doStorageIdKey(ns, worker) { return "worker:do-storage:" + ns + ":" + worker; } -export function workflowDefsKey(ns, worker) { return "wf:defs:" + ns + ":" + worker; } +export { + bundleAssetPrefix, + d1DatabaseReferrersKey, + deleteLockKey, + doObjectRegistryKey, + doOwnerScopeScanPatternForStorage, + doStorageIdKey, + encodeReferrerMember, + parseBundleMeta, + referrersKey, + workerVersionsKey, + workersIndexKey, + workflowDefsKey, +} from ${JSON.stringify(productionControlLibUrl)}; export function extractD1Refs() { return []; } export function extractOutgoingRefs() { return []; } export function formatReferrerBlocker(members) { return { referrers: members }; } `); -const sharedVersionUrl = moduleDataUrl(` -export function routesKey(ns) { return "routes:" + ns; } -export function patternsKey(host) { return "patterns:" + host; } -export function formatVersion(n) { return "v" + n; } -export function parseVersion(tag) { - const match = /^v([1-9][0-9]*)$/.exec(tag); - return match ? Number(match[1]) : null; -} -export function bundleKey(ns, worker, version) { - const n = parseVersion(version); - if (n == null) throw new Error("invalid version tag " + JSON.stringify(version)); - return "worker:" + ns + ":" + worker + ":v:" + n; -} -`); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const sharedSecretKeysUrl = repositoryFileUrl("shared/secret-keys.js"); -const sharedRedisUrl = moduleDataUrl(` -const decoder = new TextDecoder(); -export function decodeBulk(value) { - if (value == null) return null; - return value instanceof Uint8Array ? decoder.decode(value) : String(value); -} -export class WatchError extends Error {} -`); +const sharedRedisUrl = sharedRedisStubUrl(); const sharedQueueKeysUrl = moduleDataUrl(` export const QUEUE_CONSUMER_INDEX_KEY = "queue:index:consumers"; @@ -170,6 +151,7 @@ const { handle: handleVersions } = await importControlHandler("control/handlers/ * queueConsumerWorkerDuringExec?: string | null, * referrersByVersion?: Record, * retainedVersions?: string[], + * bundleMetaRaw?: string | null, * }} [opts] */ function resetDeleteHandlerState({ @@ -184,13 +166,16 @@ function resetDeleteHandlerState({ queueConsumerWorkerDuringExec = queueConsumerWorker, referrersByVersion = {}, retainedVersions = ["v1"], + bundleMetaRaw, } = {}) { const referrers = /** @type {Record} */ (referrersByVersion); - const meta = JSON.stringify({ - ...(assetPrefix ? { assets: { prefix: assetPrefix } } : {}), - bindings: {}, - routes: [], - }); + const meta = bundleMetaRaw === undefined + ? JSON.stringify({ + ...(assetPrefix ? { assets: { prefix: assetPrefix } } : {}), + bindings: {}, + routes: [], + }) + : bundleMetaRaw; /** @type {unknown[][]} */ const multiCalls = []; /** @type {unknown[][]} */ @@ -237,7 +222,7 @@ function resetDeleteHandlerState({ }, /** @param {string} _cursor @param {string} pattern */ async scan(_cursor, pattern) { - if (pattern === "do:owner:do_old:*") return ["0", doOwnerKeysDuringExec]; + if (pattern === "do:owner:scope:do_old%3A*") return ["0", doOwnerKeysDuringExec]; if (pattern === "queue-consumer:demo:*" && queueConsumerWorkerDuringExec) { return ["0", queueConsumerKeys]; } @@ -270,7 +255,7 @@ function resetDeleteHandlerState({ }, /** @param {string} _cursor @param {string} pattern */ async scan(_cursor, pattern) { - if (pattern === "do:owner:do_old:*") return ["0", doOwnerKeys]; + if (pattern === "do:owner:scope:do_old%3A*") return ["0", doOwnerKeys]; if (pattern === "queue-consumer:demo:*" && queueConsumerWorker) { return ["0", queueConsumerKeys]; } @@ -283,6 +268,7 @@ function resetDeleteHandlerState({ }, /** @param {string} key @param {string} field */ async hGet(key, field) { + if (key === "routes:demo" && field === "api") return activeVersion; if (/^worker:demo:api:v:\d+$/.test(key) && field === "__meta__") return meta; if (queueConsumerKeys.includes(key) && field === "worker") return queueConsumerWorker; return null; @@ -332,12 +318,19 @@ function resetDeleteHandlerState({ }); } -/** @param {{ assetPrefix?: string | null }} [opts] */ -function resetVersionDeleteHandlerState({ assetPrefix = "assets/demo/api/v1/" } = {}) { - const meta = JSON.stringify({ - ...(assetPrefix ? { assets: { prefix: assetPrefix } } : {}), - bindings: {}, - }); +/** @param {{ assetPrefix?: string | null, bundleMetaRaw?: string | null, retainedVersions?: string[], siblingMetaRaw?: string | null }} [opts] */ +function resetVersionDeleteHandlerState({ + assetPrefix = "assets/demo/api/v1/", + bundleMetaRaw, + retainedVersions = ["v1"], + siblingMetaRaw = null, +} = {}) { + const meta = bundleMetaRaw === undefined + ? JSON.stringify({ + ...(assetPrefix ? { assets: { prefix: assetPrefix } } : {}), + bindings: {}, + }) + : bundleMetaRaw; /** @type {unknown[][]} */ const multiCalls = []; const session = { @@ -347,11 +340,12 @@ function resetVersionDeleteHandlerState({ assetPrefix = "assets/demo/api/v1/" } async hGet(key, field) { if (key === "routes:demo" && field === "api") return null; if (key === "worker:demo:api:v:1" && field === "__meta__") return meta; + if (key === "worker:demo:api:v:2" && field === "__meta__") return siblingMetaRaw; return null; }, /** @param {string} key */ async zRange(key) { - if (key === "worker-versions:demo:api") return ["v1"]; + if (key === "worker-versions:demo:api") return retainedVersions; return []; }, async sMembers() { return []; }, @@ -507,6 +501,75 @@ test("worker delete reports cleanup_queue_failed when data-plane cleanup enqueue )); }); +test("worker delete classifies non-object retained bundle metadata as corrupt", async () => { + const testState = resetDeleteHandlerState({ bundleMetaRaw: "[]" }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, "retained_meta_parse"); + assert.equal(body.detail, "__meta__ must be a JSON object"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.equal(testState.releaseCalls, 1); +}); + +test("worker delete fails closed when indexed active bundle metadata is missing", async () => { + const testState = resetDeleteHandlerState({ + retainedVersions: [], + bundleMetaRaw: null, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-missing-active-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, "active_meta_parse"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.equal(testState.releaseCalls, 1); +}); + +for (const { label, bundleMetaRaw } of [ + { label: "missing", bundleMetaRaw: null }, + { label: "empty", bundleMetaRaw: "" }, +]) { + test(`worker delete fails closed when retained bundle metadata is ${label}`, async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + bundleMetaRaw, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: `rid-delete-${label}-retained-meta`, + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, "retained_meta_parse"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.equal(testState.releaseCalls, 1); + }); +} + test("worker delete collects retained version metadata without serial round trips", async () => { const testState = resetDeleteHandlerState({ activeVersion: null, @@ -550,7 +613,7 @@ test("worker delete retries instead of committing when DO owner keys drift after const testState = resetDeleteHandlerState({ assetPrefix: null, doOwnerKeys: [], - doOwnerKeysDuringExec: ["do:owner:do_old:Room:shard0"], + doOwnerKeysDuringExec: ["do:owner:scope:do_old%3ARoom%3Ashard0"], }); const response = await handle({ @@ -632,6 +695,37 @@ test("worker delete dry-run includes workflow lifecycle blockers", async () => { assert.equal(testState.releaseCalls, 0); }); +test("worker delete dry-run retries when a retained version is deleted during collection", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + bundleMetaRaw: null, + doStorageId: null, + queueConsumerWorker: null, + }); + let versionReads = 0; + testState.redis.zRange = async (/** @type {string} */ key) => { + assert.equal(key, "worker-versions:demo:api"); + versionReads += 1; + return versionReads === 1 ? ["v1"] : []; + }; + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-dry-run-version-delete-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, false); + assert.equal(body.noop, true); + assert.deepEqual(body.versionsDeleted, []); + assert.equal(versionReads, 3); + assert.equal(testState.releaseCalls, 0); +}); + test("worker delete reports workflow and version referrer blockers together", async () => { const referrer = JSON.stringify({ callerNs: "demo", @@ -908,6 +1002,75 @@ test("version delete reports cleanup_queue_failed when data-plane cleanup enqueu )); }); +test("version delete classifies non-object bundle metadata as corrupt", async () => { + const testState = resetVersionDeleteHandlerState({ bundleMetaRaw: "null" }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: "rid-version-delete-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.namespace, "demo"); + assert.equal(body.name, "api"); + assert.equal(body.version, "v1"); + assert.deepEqual(testState.cleanupIntents, []); + assert.equal(testState.releaseCalls, 1); +}); + +test("version delete classifies indexed target metadata absence as corrupt", async () => { + const testState = resetVersionDeleteHandlerState({ bundleMetaRaw: null }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: "rid-version-delete-missing-target-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.version, "v1"); + assert.deepEqual(testState.cleanupIntents, []); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.equal(testState.releaseCalls, 1); +}); + +for (const [label, siblingMetaRaw] of [ + ["missing", null], + ["empty", ""], +]) { + test(`version delete fails closed when indexed sibling metadata is ${label}`, async () => { + const testState = resetVersionDeleteHandlerState({ + retainedVersions: ["v1", "v2"], + siblingMetaRaw, + }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: `rid-version-delete-${label}-sibling-meta`, + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.version, "v2"); + assert.equal(body.stage, "sibling_meta_parse"); + assert.deepEqual(testState.cleanupIntents, []); + assert.equal(testState.releaseCalls, 1); + }); +} + test("version delete reports queueHint none when no content cleanup is needed", async () => { const testState = resetVersionDeleteHandlerState({ assetPrefix: null }); diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 8add919..5d1046e 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -11,11 +11,15 @@ import { readRepositoryModuleSource, repositoryFileUrl, } from "../helpers/load-shared-module.js"; -import { createFakeRedisSession } from "../helpers/mocks/fake-redis.js"; +import { createFakeRedisSession, sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { readJsonResponse } from "../helpers/response-json.js"; import { realRuntimeInjectionSourcesUrl } from "../helpers/runtime-injection-sources.js"; -const { libUrl: controlLibUrl, lifecycleIndexesUrl } = await compileControlGraph(); +const { + libUrl: controlLibUrl, + lifecycleIndexesUrl, + sharedAuthRolesUrl, +} = await compileControlGraph(); /** @type {any} */ const CONTROL_DEPLOY_TEST_STATE = { @@ -193,36 +197,11 @@ export function parseQueueConsumers() { } `); -const sharedVersionUrl = moduleDataUrl(` -export function routesKey(ns) { return "routes:" + ns; } -export function formatVersion(n) { return "v" + n; } -export function parseVersion(tag) { - const match = /^v([1-9][0-9]*)$/.exec(tag); - return match ? Number(match[1]) : null; -} -export function bundleKey(ns, worker, version) { - const n = parseVersion(version); - if (n == null) throw new Error("invalid version tag " + JSON.stringify(version)); - return "worker:" + ns + ":" + worker + ":v:" + n; -} -`); - -const sharedRedisUrl = moduleDataUrl(` -export class WatchError extends Error {} -`); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); -const sharedNsUrl = moduleDataUrl(` -export function isReservedNs(ns) { return typeof ns === "string" && ns.startsWith("__"); } -export function isValidRouteNs(ns) { - return typeof ns === "string" && - (/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(ns) || ns === "__system__"); -} -export const ROUTES_ALLOWED_RESERVED_NS = new Set(["__system__"]); -`); +const sharedRedisUrl = sharedRedisStubUrl(); -const sharedAuthRolesUrl = moduleDataUrl(` -export const PLATFORM_TIER_RESERVED_NS = new Set(["__platform__"]); -`); +const sharedNsUrl = repositoryFileUrl("shared/ns-pattern.js"); const controlS3Url = moduleDataUrl(` export async function putAsset() { @@ -691,6 +670,69 @@ test("deploy handler resolves cross-namespace service-binding meta from the targ } }); +test("deploy handler classifies empty service target metadata before commit", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:other" && field === "api") return "v1"; + if (key === "worker:other:api:v:1" && field === "__meta__") return ""; + return null; + }, + async incr() { + throw new Error("corrupt target metadata must fail before version allocation"); + }, + }; + + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/caller/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + bindings: { + API: { type: "service", ns: "other", service: "api" }, + }, + }), + }), + env: {}, + ns: "tenant-a", + name: "caller", + requestId: "rid-corrupt-service-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.message, "Corrupt __meta__ for other/api/v1"); +}); + +test("deploy handler fails closed instead of hiding empty platform export metadata", async () => { + installPlatformAuthWarningFixture(); + /** @param {string} key @param {string} field */ + const corruptPlatformMetaHGet = async (key, field) => { + if (key === "worker:__platform__:auth:v:1" && field === "__meta__") return ""; + return null; + }; + /** @type {any} */ (globalThis).__controlDeployTestState.redis.hGet = corruptPlatformMetaHGet; + + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/caller/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "caller", + requestId: "rid-corrupt-platform-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.message, "Corrupt __meta__ for __platform__/auth/v1"); +}); + test("deploy handler schedules cleanup when the first asset upload fails", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index 20dd10c..020fe6e 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -3,26 +3,28 @@ import assert from "node:assert/strict"; import { importRepositoryModule, importSpecifierReplacements, - moduleDataUrl, repositoryFileUrl, + repositoryModuleDataUrl, } from "../helpers/load-shared-module.js"; +import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { encryptSecretValue } from "../../shared/secret-envelope.js"; +const { libUrl: controlLibUrl } = await compileControlGraph(); const secretEnvelopeUrl = repositoryFileUrl("shared/secret-envelope.js"); -const sharedErrorsUrl = repositoryFileUrl("shared/errors.js"); const sharedVersionUrl = repositoryFileUrl("shared/version.js"); -const sharedRedisUrl = moduleDataUrl(` -export class WatchError extends Error { - constructor(message = "watched key changed") { - super(message); - this.name = "WatchError"; - } -} -`); +const sharedRedisUrl = sharedRedisStubUrl(); +const runtimeEnvBuildUrl = repositoryModuleDataUrl( + "runtime/load/env-build.js", + importSpecifierReplacements({ + "shared-ns-pattern": repositoryFileUrl("shared/ns-pattern.js"), + }) +); const { UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, WORKER_LOADER_ENV_HEADROOM_BYTES, WORKER_LOADER_ENV_MAX_BYTES, + BundleMetaError, WorkerEnvBudgetError, assertWorkerLoaderUserEnvBudget, assertWorkerVersionsUserEnvBudget, @@ -31,8 +33,9 @@ const { estimatedWorkerLoaderEnvBytes, WORKER_LOADER_ENV_VERSION_PLACEHOLDER, } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ + "control-lib": controlLibUrl, + "runtime-load-env-build": runtimeEnvBuildUrl, "shared-secret-envelope": secretEnvelopeUrl, - "shared-errors": sharedErrorsUrl, "shared-version": sharedVersionUrl, "shared-redis": sharedRedisUrl, })); @@ -42,6 +45,19 @@ const envelopeEnv = { SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", }; +/** @param {unknown} err @param {string} reason */ +function assertBundleMetaError(err, reason) { + assert.ok(err instanceof BundleMetaError); + const bundleError = /** @type {Error & { status: number, code: string, details: Record, cause: unknown }} */ (err); + assert.equal(bundleError.status, 500); + assert.equal(bundleError.code, "corrupt_meta"); + assert.equal(bundleError.message, "Corrupt __meta__ for demo/api/v1"); + assert.deepEqual(bundleError.details, { namespace: "demo", worker: "api", version: "v1" }); + assert.ok(bundleError.cause instanceof Error); + assert.equal(bundleError.cause.message, reason); + return true; +} + test("worker env budget counts merged vars and secrets with worker-secret precedence", () => { const estimated = estimatedWorkerLoaderEnv({ ns: "demo", @@ -166,11 +182,11 @@ test("worker env budget accounts for V8 two-byte strings on mixed non-Latin-1 en ); }); -test("worker env budget stores required caller secrets in a null-prototype map", () => { +test("worker env budget stores required caller secrets in a JSRPC-serializable object", () => { const estimated = estimatedWorkerLoaderEnv({ ns: "demo", worker: "caller", - nsSecrets: JSON.parse('{"__proto__":"secret"}'), + nsSecrets: { API_TOKEN: "secret" }, meta: { bindings: { PLATFORM: { @@ -178,16 +194,15 @@ test("worker env budget stores required caller secrets in a null-prototype map", ns: "__platform__", service: "platformApi", version: "v1", - requiredCallerSecrets: ["__proto__"], + requiredCallerSecrets: ["API_TOKEN"], }, }, }, }); const callerSecrets = /** @type {any} */ (estimated.PLATFORM).props.callerSecrets; - assert.equal(Object.getPrototypeOf(callerSecrets), null); - assert.equal(Object.hasOwn(callerSecrets, "__proto__"), true); - assert.equal(callerSecrets.__proto__, "secret"); + assert.equal(Object.getPrototypeOf(callerSecrets), Object.prototype); + assert.deepEqual(callerSecrets, { API_TOKEN: "secret" }); }); test("worker env budget counts configured assets CDN base", () => { @@ -471,7 +486,35 @@ test("worker env budget reports bundle metadata parse context", async () => { worker: "api", versions: ["v1"], }), - /invalid bundle metadata for demo\/api@v1/ + (err) => { + assert.ok(err instanceof BundleMetaError); + const bundleError = /** @type {Error & { cause: unknown }} */ (err); + assert.ok(bundleError.cause instanceof SyntaxError); + return assertBundleMetaError(err, bundleError.cause.message); + } + ); +}); + +test("worker env budget maps strict retained metadata failures to corrupt_meta", async () => { + const redis = { + async hGet() { + return JSON.stringify({ + workflows: [{ binding: "FLOW", name: "flow", className: "Flow" }], + }); + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + }), + (err) => assertBundleMetaError( + err, + 'Workflow binding "FLOW" is missing workflow metadata (redeploy demo/api)' + ) ); }); @@ -492,7 +535,7 @@ test("worker env budget fails closed when retained bundle metadata is missing", worker: "api", versions: ["v1"], }), - /bundle metadata missing for demo\/api@v1/ + (err) => assertBundleMetaError(err, "__meta__ must be a JSON string") ); }); @@ -535,6 +578,6 @@ test("worker env budget fails closed when retained bundle metadata is not an obj worker: "api", versions: ["v1"], }), - /__meta__ must be a JSON object/ + (err) => assertBundleMetaError(err, "__meta__ must be a JSON object") ); }); diff --git a/tests/unit/control-handlers-workers.test.js b/tests/unit/control-handlers-workers.test.js index 8da5ebd..66af90b 100644 --- a/tests/unit/control-handlers-workers.test.js +++ b/tests/unit/control-handlers-workers.test.js @@ -11,6 +11,7 @@ import { readRepositoryFile, repositoryFileUrl, } from "../helpers/load-shared-module.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; import { assertJsonResponse } from "../helpers/response-json.js"; @@ -23,12 +24,7 @@ const workersHandlerState = installControlHandlerState( }) ); const controlSharedUrl = controlSharedHarnessUrl(WORKERS_HANDLER_STATE_GLOBAL); - -const controlLibUrl = moduleDataUrl(` -export function routesKey(ns) { return "routes:" + ns; } -export function workersIndexKey(ns) { return "workers:" + ns; } -export function workerVersionsKey(ns, name) { return "worker-versions:" + ns + ":" + name; } -`); +const { libUrl: controlLibUrl } = await compileControlGraph(); const sharedSecretKeysUrl = repositoryFileUrl("shared/secret-keys.js"); diff --git a/tests/unit/control-handlers-workflows.test.js b/tests/unit/control-handlers-workflows.test.js index ca23002..8221b81 100644 --- a/tests/unit/control-handlers-workflows.test.js +++ b/tests/unit/control-handlers-workflows.test.js @@ -6,40 +6,22 @@ import { installControlHandlerState, } from "../helpers/control-handler-harness.js"; import { - moduleDataUrl, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; import { assertJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; const WORKFLOWS_HANDLER_GLOBAL = "__workflowsHandlerState"; - -const controlLibUrl = moduleDataUrl(` -export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; -export const WORKFLOW_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; -export function isValidWorkerName(name) { return typeof name === "string" && WORKER_NAME_RE.test(name); } -export function isValidWorkflowName(name) { return typeof name === "string" && WORKFLOW_NAME_RE.test(name); } -export function workflowDefsKey(ns, worker) { return "wf:defs:" + ns + ":" + worker; } -`); - -const sharedVersionUrl = moduleDataUrl(` -export function parseVersion(tag) { - const match = /^v([1-9][0-9]*)$/.exec(tag); - return match ? Number(match[1]) : null; -} -export function bundleKey(ns, worker, version) { - const n = parseVersion(version); - if (n == null) throw new Error("invalid version tag " + JSON.stringify(version)); - return "worker:" + ns + ":" + worker + ":v:" + n; -} -export function routesKey(ns) { return "routes:" + ns; } -`); +const { libUrl: productionControlLibUrl } = await compileControlGraph(); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const { handle } = await importControlHandler("control/handlers/workflows.js", { globalName: WORKFLOWS_HANDLER_GLOBAL, replacements: { - "control-lib": controlLibUrl, + "control-lib": productionControlLibUrl, "shared-internal-auth": sharedInternalAuthUrl(), "shared-version": sharedVersionUrl, }, @@ -61,8 +43,9 @@ function resetWorkflowsHandlerState() { redis.hashes.set("routes:demo", { api: "v2" }); redis.hashes.set("worker:demo:api:v:2", { "__meta__": meta }); state.workflows = { - /** @param {string} url @param {{ body: string, headers?: HeadersInit }} init */ + /** @param {string} url @param {{ body: string, headers?: HeadersInit, signal?: AbortSignal | null }} init */ async fetch(url, init) { + assert.equal(init.signal, undefined); assert.equal(new Headers(init.headers).get("x-wdl-internal-auth"), TEST_INTERNAL_AUTH_TOKEN); redis.commands.push(["fetch", url, parseJsonObjectRequestBody(init, "workflows backend request body")]); return new Response(JSON.stringify({ id: "order-1", status: "paused" }), { @@ -110,6 +93,34 @@ test("workflows handler lists active workflow definitions from bundle metadata", }]); }); +for (const [label, rawMeta] of [ + ["missing", null], + ["empty", ""], + ["malformed", "{bad"], + ["non-object", "[]"], +]) { + test(`workflows handler fails closed on ${label} active bundle metadata`, async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set( + "worker:demo:api:v:2", + rawMeta === null ? {} : { "__meta__": rawMeta } + ); + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: `rid-${label}-meta`, + }); + + await assertJsonResponse(response, 500, { + error: "corrupt_meta", + message: "Corrupt __meta__ for demo/api/v2", + }); + }); +} + test("workflows handler lists empty namespaces without batch reads", async () => { const state = resetWorkflowsHandlerState(); state.redis.hashes.set("routes:demo", {}); diff --git a/tests/unit/control-index.test.js b/tests/unit/control-index.test.js index fdbb51c..457e405 100644 --- a/tests/unit/control-index.test.js +++ b/tests/unit/control-index.test.js @@ -4,15 +4,17 @@ import { importRepositoryModule, importSpecifierReplacements, moduleDataUrl, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; /** @type {any} */ (globalThis).__controlIndexState = null; +const sharedNsPatternUrl = repositoryFileUrl("shared/ns-pattern.js"); const controlLibUrl = moduleDataUrl(` export const NS_RE = /^[a-z0-9-]+$/; export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; -export function configuredHostname(value) { return typeof value === "string" && value ? value : null; } +export { configuredHostname } from ${JSON.stringify(sharedNsPatternUrl)}; export function configuredPublicUrl() { return null; } export function platformVersionFromPackageJson() { return "wdl.test"; } export function projectAccessPrincipal(principal) { return principal || null; } @@ -172,6 +174,20 @@ test("control whoami uses sanitized forwarded proto for public URL hints", async }); }); +test("control whoami omits namespace URL when no public platform domain is configured", async () => { + const state = resetControlIndexState(); + state.authResult.principal = { kind: "ns", ns: "tenant-a" }; + + const response = await controlIndex.fetch( + new Request("http://control.example/whoami"), + {}, + /** @type {ExecutionContext} */ ({}) + ); + + const body = await readJsonResponse(response, 200); + assert.deepEqual(body.urls, { control: "http://control.example" }); +}); + test("control whoami ignores malformed forwarded proto values", async () => { resetControlIndexState(); diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 945ec9d..7a63c79 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -27,8 +27,13 @@ const { NS_RE, parseControlRoute, configuredHostname, + platformDomainFromEnv, configuredPublicUrl, + parseWorkerdDependencyVersion, platformVersionFromPackageJson, + BundleMetaError, + parseBundleMeta, + bundleAssetPrefix, projectAccessPrincipal, parseR2DispatchPath, isAdminAcceptableNs, @@ -97,6 +102,82 @@ function deletePlatformTierFixture(ns) { RESERVED_NS.delete(ns); } +test("parseBundleMeta returns JSON objects", () => { + assert.deepEqual( + parseBundleMeta('{"routes":[],"vars":{"MODE":"test"}}', { + ns: "demo", + worker: "api", + version: "v3", + }), + { routes: [], vars: { MODE: "test" } } + ); +}); + +test("parseBundleMeta rejects missing, malformed, and non-object metadata with context", () => { + for (const raw of [undefined, null, "", "{bad", "null", "[]", "1", '"text"']) { + assert.throws( + () => parseBundleMeta(raw, { ns: "demo", worker: "api", version: "v3" }), + (err) => { + assert.ok(err instanceof BundleMetaError, `expected BundleMetaError for ${String(raw)}`); + const bundleError = /** @type {Error & { status: number, code: string, details: Record, cause: unknown }} */ (err); + assert.equal(bundleError.status, 500); + assert.equal(bundleError.code, "corrupt_meta"); + assert.equal(bundleError.message, "Corrupt __meta__ for demo/api/v3"); + assert.deepEqual(bundleError.details, { namespace: "demo", worker: "api", version: "v3" }); + assert.ok(bundleError.cause instanceof Error); + return true; + } + ); + } +}); + +test("bundleAssetPrefix reads only string prefixes from object-shaped assets", () => { + assert.equal(bundleAssetPrefix({ assets: { prefix: "assets/demo/" } }), "assets/demo/"); + assert.equal(bundleAssetPrefix({ assets: { prefix: "" } }), ""); + for (const assets of [null, [], "assets/demo/", 1, { prefix: 1 }]) { + assert.equal(bundleAssetPrefix({ assets }), null); + } +}); + +test("parseBundleMeta delegates error construction without changing parse context", () => { + const expected = new Error("routing-owned error"); + /** @type {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown } | null} */ + let failure = null; + assert.throws( + () => parseBundleMeta("[]", { + ns: "demo", + worker: "api", + version: "v3", + makeError(/** @type {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown }} */ value) { + failure = value; + return expected; + }, + }), + (err) => err === expected + ); + const captured = /** @type {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown } | null} */ ( + /** @type {unknown} */ (failure) + ); + assert.ok(captured); + assert.deepEqual( + { + namespace: captured.namespace, + worker: captured.worker, + version: captured.version, + message: captured.message, + reason: captured.reason, + }, + { + namespace: "demo", + worker: "api", + version: "v3", + message: "Corrupt __meta__ for demo/api/v3", + reason: "__meta__ must be a JSON object", + } + ); + assert.ok(captured.cause instanceof TypeError); +}); + test("encodeReferrerMember: key order is canonical (alphabetical) regardless of caller shape", () => { const a = encodeReferrerMember({ binding: "AUTH", callerNs: "demo", callerWorker: "api", callerVersion: "v3", @@ -768,15 +849,12 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( }); test("MAX_WORKER_COMPATIBILITY_DATE matches pinned workerd release plus seven days", () => { - assert.match( - PACKAGE_WORKERD_DEP, - /^[~^]?1\.\d{8}\.\d+$/, - `unexpected workerd dependency format ${PACKAGE_WORKERD_DEP}` - ); - assert.equal(PACKAGE_WORKERD_DEP.replace(/^[~^]/, ""), WORKERD_VERSION); - const match = /^1\.(\d{4})(\d{2})(\d{2})\.\d+$/.exec(WORKERD_VERSION); - assert.ok(match, `unexpected workerd version ${WORKERD_VERSION}`); - const max = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]) + 7)); + const parsed = parseWorkerdDependencyVersion(JSON.stringify({ + dependencies: { workerd: PACKAGE_WORKERD_DEP }, + })); + assert.ok(parsed, `unexpected workerd dependency format ${PACKAGE_WORKERD_DEP}`); + assert.equal(parsed.version, WORKERD_VERSION); + const max = new Date(Date.UTC(parsed.year, parsed.month - 1, parsed.day + 7)); const expected = [ String(max.getUTCFullYear()).padStart(4, "0"), String(max.getUTCMonth() + 1).padStart(2, "0"), @@ -1975,6 +2053,15 @@ test("configuredHostname accepts plain hosts and rejects URL injection shapes", } }); +test("platformDomainFromEnv normalizes configured values and owns the default", () => { + assert.equal(platformDomainFromEnv({}), "workers.local"); + assert.equal(platformDomainFromEnv({ PLATFORM_DOMAIN: " Workers.Example. " }), "workers.example"); + assert.throws( + () => platformDomainFromEnv({ PLATFORM_DOMAIN: "workers.example:8443" }), + /plain hostname/ + ); +}); + test("configuredPublicUrl returns a safe absolute base URL hint", () => { assert.equal(configuredPublicUrl(" https://assets.example/base/?token=secret#frag "), "https://assets.example/base"); assert.equal(configuredPublicUrl("http://assets.example/"), "http://assets.example"); @@ -2011,6 +2098,21 @@ test("platformVersionFromPackageJson derives WDL version from bundled workerd de ); }); +test("parseWorkerdDependencyVersion owns the bundled dependency grammar", () => { + assert.deepEqual( + parseWorkerdDependencyVersion(JSON.stringify({ dependencies: { workerd: "^1.20260531.12" } })), + { version: "1.20260531.12", year: 2026, month: 5, day: 31, patch: 12 } + ); + for (const source of [ + "{", + JSON.stringify({ dependencies: {} }), + JSON.stringify({ dependencies: { workerd: "2.20260531.1" } }), + JSON.stringify({ dependencies: { workerd: "1.20260531" } }), + ]) { + assert.equal(parseWorkerdDependencyVersion(source), null); + } +}); + test("normalizeWorkflows: validates deploy-wire workflow declarations", () => { assert.deepEqual(normalizeWorkflows(undefined), []); assert.deepEqual(normalizeWorkflows([ @@ -2214,6 +2316,20 @@ test("linkServiceBinding: target meta lookup throws → 502", async () => { 502, "failed to read target meta: redis blew up", "service_binding_target_meta_unavailable"); }); +test("linkServiceBinding: preserves target metadata domain errors", async () => { + const corruptMeta = new LinkError(500, "corrupt_meta", "Corrupt __meta__ for other/t/v1"); + await expectLinkError( + () => linkServiceBinding({ + callerNs: "caller", callerName: "c", bindingName: "T", + spec: { type: "service", service: "t", ns: "other" }, + ...makeLookups({ + versions: { "other/t": "v1" }, + metaThrows: corruptMeta, + }), + }), + 500, "Corrupt __meta__ for other/t/v1", "corrupt_meta"); +}); + test("linkServiceBinding: rejects runtime-reserved entrypoint at link time (defense in depth)", async () => { // Even if a manually-crafted binding makes it past validateBindings // (e.g. direct Redis writes during ops recovery), the linker is the diff --git a/tests/unit/control-lifecycle-indexes.test.js b/tests/unit/control-lifecycle-indexes.test.js index 2282322..0a57c4b 100644 --- a/tests/unit/control-lifecycle-indexes.test.js +++ b/tests/unit/control-lifecycle-indexes.test.js @@ -1,12 +1,21 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { importRepositoryModule } from "../helpers/load-shared-module.js"; +import { importRepositoryModule, readRepositoryJson } from "../helpers/load-shared-module.js"; + +const SCHEDULER_PROJECTION_CONTRACT = readRepositoryJson( + "tests/fixtures/scheduler-projection-contract.json" +); const { + cronEntryJson, + cronMetaJson, cronRefMember, + cronSlotExpireAt, cronSlotKey, cronWorkerKey, + queueConsumerFields, stageCronSlotRef, + stageCronProjection, stageCronWorkerIndexed, stageCronWorkerRemoved, stageD1ReferrerAdds, @@ -41,6 +50,11 @@ function recordMulti() { return this; }, /** @param {unknown[]} args */ + hDel(...args) { + calls.push(["hDel", ...args]); + return this; + }, + /** @param {unknown[]} args */ expireAt(...args) { calls.push(["expireAt", ...args]); return this; @@ -71,52 +85,79 @@ test("cron worker index helpers maintain non-authoritative discovery members", ( }); test("cron key helpers compose worker, slot, and ref members", () => { - assert.equal(cronWorkerKey("tenant", "worker"), "crons:tenant:worker"); - assert.equal(cronSlotKey(1_778_856_600_000), "cron-slot:1778856600000"); - assert.equal(cronRefMember("tenant", "worker", "cron-1", 7), "tenant:worker:cron-1:7"); + const cron = SCHEDULER_PROJECTION_CONTRACT.cron; + assert.equal(cronWorkerKey(cron.ns, cron.worker), cron.workerKey); + assert.equal(cronSlotKey(cron.slotMs), cron.slotKey); + assert.equal(cronSlotExpireAt(cron.slotMs), cron.slotExpireAt); + assert.equal(cronRefMember(cron.ns, cron.worker, cron.cronId, cron.gen), cron.reference); }); test("stageCronSlotRef writes ref and expiry together", () => { + const cron = SCHEDULER_PROJECTION_CONTRACT.cron; const multi = recordMulti(); - stageCronSlotRef(multi, "tenant", "worker", { - id: "cron-1", - gen: 7, - slot: 1_778_856_600_000, + stageCronSlotRef(multi, cron.ns, cron.worker, { + id: cron.cronId, + gen: cron.gen, + slot: cron.slotMs, }); assert.deepEqual(multi.calls, [ - ["sAdd", "cron-slot:1778856600000", "tenant:worker:cron-1:7"], - ["expireAt", "cron-slot:1778856600000", 1_778_857_200], + ["sAdd", cron.slotKey, cron.reference], + ["expireAt", cron.slotKey, cron.slotExpireAt], ]); }); -test("stageQueueConsumerProjection writes full optional queue consumer fields", () => { +test("stageCronProjection writes the shared scheduler projection contract", () => { + const cron = SCHEDULER_PROJECTION_CONTRACT.cron; const multi = recordMulti(); - stageQueueConsumerProjection(multi, "tenant", "worker", "v3", { - queue: "jobs", - maxBatchSize: 10, - maxBatchTimeoutMs: 250, - maxRetries: 4, - deadLetterQueue: "jobs-dlq", - retryDelaySeconds: 0, + const entry = { + id: cron.cronId, + cron: cron.entry.cron, + timezone: cron.entry.timezone, + gen: cron.entry.gen, + slot: cron.slotMs, + }; + stageCronProjection(multi, { + ns: cron.ns, + worker: cron.worker, + version: cron.meta.version, + cronKey: cron.workerKey, + existingHash: {}, + crons: [{ cron: entry.cron, timezone: entry.timezone }], + plan: { cronSeq: cron.meta.seq, addedWithPlacement: [entry], removed: [] }, }); + assert.equal(cronMetaJson(cron.meta.version, cron.meta.seq), cron.meta.json); + assert.equal(cronEntryJson(entry), cron.entry.json); + assert.deepEqual(multi.calls, [ + ["sAdd", cron.workerIndexKey, cron.workerKey], + ["hSet", cron.workerKey, "__meta__", cron.meta.json], + ["hSet", cron.workerKey, cron.cronId, cron.entry.json], + ["sAdd", cron.slotKey, cron.reference], + ["expireAt", cron.slotKey, cron.slotExpireAt], + ]); +}); + +test("stageQueueConsumerProjection writes full optional queue consumer fields", () => { + const contract = SCHEDULER_PROJECTION_CONTRACT.queueConsumer; + const multi = recordMulti(); + assert.equal(contract.input.queue, contract.queue); + assert.deepEqual( + queueConsumerFields(contract.worker, contract.version, contract.input), + contract.fields + ); + stageQueueConsumerProjection( + multi, + contract.ns, + contract.worker, + contract.version, + contract.input + ); + assert.deepEqual(multi.calls, [ - ["del", "queue-consumer:tenant:jobs"], - [ - "hSet", - "queue-consumer:tenant:jobs", - { - worker: "worker", - version: "v3", - max_batch_size: "10", - max_batch_timeout_ms: "250", - max_retries: "4", - dead_letter_queue: "jobs-dlq", - retry_delay_secs: "0", - }, - ], - ["sAdd", "queue-consumer-index", "queue-consumer:tenant:jobs"], + ["del", `queue-consumer:${contract.ns}:${contract.input.queue}`], + ["hSet", `queue-consumer:${contract.ns}:${contract.input.queue}`, contract.fields], + ["sAdd", "queue-consumer-index", `queue-consumer:${contract.ns}:${contract.input.queue}`], ]); }); diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index 2c1818c..b110279 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -1,8 +1,15 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModuleFresh } from "../helpers/load-shared-module.js"; +import { + importRepositoryModuleFresh, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; +import { compileSharedAuthRoles } from "../helpers/load-auth-roles.js"; import { delay, waitUntil } from "../helpers/timing.js"; +const { sharedAuthRolesUrl } = await compileSharedAuthRoles(); +const SHARED_NS_PATTERN_URL = repositoryFileUrl("shared/ns-pattern.js"); + /** @param {{ keepaliveMs?: number }} [options] */ function loadLogsTailHandler(options = {}) { /** @type {Array<[RegExp | string, string]>} */ @@ -44,12 +51,12 @@ function loadLogsTailHandler(options = {}) { const redisDbFromEnv = (env, name) => Number(env?.[name] || 0);`, ], [ - /import \{ PLATFORM_TIER_RESERVED_NS \} from "shared-auth-roles";/, - "const PLATFORM_TIER_RESERVED_NS = new Set(['__platform__']);", + /from "shared-auth-roles";/, + `from ${JSON.stringify(sharedAuthRolesUrl)};`, ], [ - /import \{ isReservedNs \} from "shared-ns-pattern";/, - "const isReservedNs = (ns) => ns === '__system__' || ns === '__platform__';", + /from "shared-ns-pattern";/, + `from ${JSON.stringify(SHARED_NS_PATTERN_URL)};`, ], [ /import \{\n {2}WORKER_NAME_RE,\n {2}compareStreamIds,\n {2}isValidResumeId,\n {2}isValidWorkerName,\n\} from "control-lib";/, diff --git a/tests/unit/control-r2.test.js b/tests/unit/control-r2.test.js index c5bcf3f..96994d0 100644 --- a/tests/unit/control-r2.test.js +++ b/tests/unit/control-r2.test.js @@ -15,6 +15,9 @@ export class SigV4Client { `); const { + deleteR2Object, + getR2Object, + headR2Object, listR2Buckets, listR2Objects, makeR2AdminClient, @@ -22,6 +25,7 @@ const { "@wdl-dev/aws-sigv4": AWS_SIGV4_STUB_URL, "runtime-r2-utils": repositoryFileUrl("runtime/r2-utils.js"), "shared-s3-xml": repositoryFileUrl("shared/s3-xml.js"), + "shared-s3-retry": repositoryFileUrl("shared/s3-retry.js"), "shared-respond": repositoryFileUrl("shared/respond.js"), })); @@ -114,3 +118,41 @@ test("control R2 admin client restores S3 transient retry budget", () => { assert.ok(r2); assert.equal(r2.client.config.retries, 10); }); + +test("control R2 object methods share request, error, and not-found handling", async () => { + const getMock = r2AdminMock(new Response("missing", { status: 404 })); + assert.equal(await getR2Object({ + r2: getMock.r2, + ns: "demo", + bucketName: "uploads", + key: "a.txt", + requestId: "rid-get", + }), null); + assert.equal(getMock.calls[0].init?.method, "GET"); + assert.equal(new Headers(getMock.calls[0].init?.headers).get("x-request-id"), "rid-get"); + + const headMock = r2AdminMock(new Response("backend detail", { status: 503 })); + await assert.rejects( + () => headR2Object({ + r2: headMock.r2, + ns: "demo", + bucketName: "uploads", + key: "a.txt", + }), + /R2 admin HEAD failed with 503: backend detail/ + ); + + const deleteMock = r2AdminMock(new Response("missing", { status: 404 })); + assert.deepEqual(await deleteR2Object({ + r2: deleteMock.r2, + ns: "demo", + bucketName: "uploads", + key: "a.txt", + }), { + namespace: "demo", + bucket: "uploads", + key: "a.txt", + status: "ok", + }); + assert.equal(deleteMock.calls[0].init?.method, "DELETE"); +}); diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index bc2a7a3..9200c67 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -116,6 +116,42 @@ test("promoteWithRoutes removes queue consumer discovery index entries for remov )); }); +for (const [label, rawMeta] of [ + ["missing", null], + ["empty", ""], +]) { + test(`promoteWithRoutes rejects ${label} active bundle metadata before changing projections`, async () => { + const redis = makeRedis(); + seedBundle(redis, "v2", { routes: [], queueConsumers: [] }); + if (rawMeta !== null) { + redis.state.hashes.set(productionBundleKey("demo", "worker", "v1"), { + __meta__: rawMeta, + }); + } + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.hashes.set("patterns:api.example", { + "/": patternProjection("demo", "worker", "v1", "exact", "/"), + }); + redis.state.hashes.set("queue-consumer:demo:jobs", { + worker: "worker", + version: "v1", + }); + + await assert.rejects( + promoteWithRoutes(redis, "demo", "worker", "v2"), + (err) => { + assertRoutingErrorShape(err, 500, "corrupt_meta"); + return true; + } + ); + + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v1"); + assert.equal(redis.state.hashes.get("queue-consumer:demo:jobs")?.version, "v1"); + assert.equal(redis.state.hashes.get("patterns:api.example")?.["/"], + patternProjection("demo", "worker", "v1", "exact", "/")); + }); +} + test("promoteWithRoutes skips empty platform route versions while checking exported as names", async () => { const redis = makeRedis(); redis.state.hashes.set(productionBundleKey("__platform__", "api", "v1"), { @@ -135,6 +171,39 @@ test("promoteWithRoutes skips empty platform route versions while checking expor ); }); +test("promoteWithRoutes rejects a platform route whose active metadata is missing", async () => { + const redis = makeRedis(); + redis.state.hashes.set(productionBundleKey("__platform__", "api", "v1"), { + __meta__: JSON.stringify({ exports: [{ entrypoint: "default", as: "demo" }] }), + }); + redis.state.hashes.set("routes:__platform__", { other: "v2" }); + + await assert.rejects( + promoteWithRoutes(redis, "__platform__", "api", "v1"), + (err) => { + assertRoutingErrorShape(err, 500, "corrupt_meta"); + return true; + } + ); + + assert.equal(redis.state.hashes.get("routes:__platform__")?.api, undefined); +}); + +test("promoteWithRoutes rejects non-object candidate bundle metadata", async () => { + const redis = makeRedis(); + redis.state.hashes.set(productionBundleKey("demo", "worker", "v1"), { + __meta__: "[]", + }); + + await assert.rejects( + promoteWithRoutes(redis, "demo", "worker", "v1"), + (err) => { + assertRoutingErrorShape(err, 500, "corrupt_meta"); + return true; + } + ); +}); + test("promoteWithRoutes rejects malformed cron metadata", async () => { const redis = makeRedis(); /** @type {Array<{ level: string, event: string, fields: any }>} */ @@ -381,6 +450,21 @@ test("bumpActiveAndPromote aborts copy and route flip when staged writes fail", assert.ok(redis.state.watched.includes("secrets:demo:worker")); }); +test("bumpActiveAndPromote preserves bundle_copy_failed for a missing source bundle", async () => { + const redis = makeRedis(); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.strings.set("worker:demo:worker:next_version", "1"); + + await assert.rejects( + bumpActiveAndPromote(redis, "demo", "worker"), + (err) => { + assertRoutingErrorShape(err, 500, "bundle_copy_failed"); + return true; + } + ); + assert.equal(redis.state.hashes.has(productionBundleKey("demo", "worker", "v2")), false); +}); + test("bumpActiveAndPromote rejects active routes that no longer declare their hosts", async () => { const redis = makeRedis(); seedBundle(redis, "v1", { diff --git a/tests/unit/control-s3.test.js b/tests/unit/control-s3.test.js index db7caec..bd79be6 100644 --- a/tests/unit/control-s3.test.js +++ b/tests/unit/control-s3.test.js @@ -9,6 +9,7 @@ import { const { makeS3Client } = await importRepositoryModule("control/s3.js", [ [/import \{ SigV4Client \} from "@wdl-dev\/aws-sigv4";/, "class SigV4Client { constructor(options) { this.options = options; } }"], [/from "runtime-r2-utils";/g, `from ${JSON.stringify(repositoryFileUrl("runtime/r2-utils.js"))};`], + [/from "shared-s3-retry";/g, `from ${JSON.stringify(repositoryFileUrl("shared/s3-retry.js"))};`], ]); test("makeS3Client requires credentials outside explicit local/mock endpoints", () => { diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 166710d..3af2492 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -11,15 +11,20 @@ import { moduleDataUrl, readRepositoryFile, repositoryFileUrl, + repositoryModuleDataUrl, } from "../helpers/load-shared-module.js"; -import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; +import { createFakeRedis, sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { CONTROL_ROUTING_TEST_URL } from "../helpers/load-control-routing.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { readJsonResponse } from "../helpers/response-json.js"; const SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); -const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const SHARED_SECRET_KEYS_URL = repositoryFileUrl("shared/secret-keys.js"); +const RUNTIME_ENV_BUILD_URL = repositoryModuleDataUrl("runtime/load/env-build.js", [ + [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], +]); +const { libUrl: PRODUCTION_CONTROL_LIB_URL } = await compileControlGraph(); const env = { SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", @@ -137,17 +142,11 @@ function secretPutUrl(controlSharedUrl, controlLibUrl) { } function envBudgetUrl() { - const sharedRedisUrl = moduleDataUrl(` -export class WatchError extends Error { - constructor(message = "watched key changed") { - super(message); - this.name = "WatchError"; - } -} -`); + const sharedRedisUrl = sharedRedisStubUrl(); const source = applyModuleReplacements(readRepositoryFile("control/env-budget.js"), [ + [/from "control-lib";/, `from ${JSON.stringify(PRODUCTION_CONTROL_LIB_URL)};`], + [/from "runtime-load-env-build";/, `from ${JSON.stringify(RUNTIME_ENV_BUILD_URL)};`], [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], - [/from "shared-errors";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], [/from "shared-redis";/, `from ${JSON.stringify(sharedRedisUrl)};`], ]); @@ -367,6 +366,50 @@ test("namespace secret PUT checks retained worker versions before storing", asyn }); }); +test("namespace secret PUT returns corrupt_meta for invalid retained bundle metadata", async () => { + const logStart = namespaceSecretState.logs.length; + await withNamespaceSecretRedis(namespaceSecretState, (redis) => { + redis.sets.set("workers:demo", new Set(["api"])); + seedWorkerSecretVersions(redis, ["v1"]); + redis.hashes.set("worker:demo:api:v:1", { __meta__: "[]" }); + }, async (redis) => { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-ns-secret-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.namespace, "demo"); + assert.equal(body.worker, "api"); + assert.equal(body.version, "v1"); + assert.equal(body.detail, undefined); + assert.deepEqual(namespaceSecretState.logs.slice(logStart).find((entry) => + entry.event === "ns_secret_mutation_rejected" + ), { + level: "error", + event: "ns_secret_mutation_rejected", + fields: { + request_id: "rid-ns-secret-corrupt-meta", + namespace: "demo", + key: "TOKEN", + method: "PUT", + status: 500, + reason: "corrupt_meta", + error_detail: "__meta__ must be a JSON object", + }, + }); + assert.deepEqual(redisHSetAttempts(redis, "secrets:demo"), []); + }); +}); + test("namespace secret PUT budgets active worker versions with their real active version", async () => { const nsSecrets = { TOKEN: "plain-secret" }; const padLength = WORKER_LOADER_ENV_MAX_BYTES - @@ -605,6 +648,12 @@ const workerSecretState = /** @type {ReturnType entry.event === "secret_mutation_rejected"); +} + const workerControlSharedUrl = controlSharedHarnessUrl(WORKER_SECRET_STATE_GLOBAL); const workerLibStubUrl = moduleDataUrl(` ${validateSecretKeyStubSource} @@ -681,6 +730,76 @@ test("worker secret PUT encrypts before WATCH retries and reuses the envelope", }); }); +test("worker secret PUT returns corrupt_meta for invalid active bundle metadata", async () => { + const state = workerSecretState; + await withWorkerSecretRedis(state, (redis) => { + redis.hashes.set("worker:demo:api:v:1", { __meta__: "[]" }); + }, async (redis) => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.message, "Corrupt __meta__ for demo/api/v1"); + assert.equal(body.detail, undefined); + assert.deepEqual(redisHSetAttempts(redis, "secrets:demo:api"), []); + }); +}); + +test("worker secret PUT logs retained metadata diagnostics without exposing them", async () => { + const state = workerSecretState; + const logStart = state.logs.length; + await withWorkerSecretRedis(state, (redis) => { + redis.hashes.delete("routes:demo"); + seedWorkerSecretVersions(redis, ["v1"]); + redis.hashes.set("worker:demo:api:v:1", { __meta__: "[]" }); + }, async (redis) => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-retained-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.detail, undefined); + assert.deepEqual(state.logs.slice(logStart).find((entry) => + entry.event === "secret_mutation_rejected" + ), { + level: "error", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-retained-corrupt-meta", + namespace: "demo", + worker: "api", + key: "TOKEN", + method: "PUT", + status: 500, + reason: "corrupt_meta", + error_detail: "__meta__ must be a JSON object", + }, + }); + assert.deepEqual(redisHSetAttempts(redis, "secrets:demo:api"), []); + }); +}); + test("worker secret mutation rejects invalid keys through shared validator", async () => { const state = workerSecretState; const opsBefore = state.redis.ops.length; @@ -737,6 +856,7 @@ test("worker secret PUT active precheck reads the shared fake Redis state", asyn test("worker secret PUT maps active bump delete-lock errors to deleting", async () => { const state = workerSecretState; + const logStart = state.logs.length; await withWorkerSecretRedis(state, (redis) => { redis.strings.set("worker-delete-lock:demo:api", "holder-token"); }, async (redis) => { @@ -756,11 +876,25 @@ test("worker secret PUT maps active bump delete-lock errors to deleting", async const body = await readJsonResponse(response, 409); assert.equal(body.error, "deleting"); assert.equal(redis.ops.length, 0); + assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ + level: "warn", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-delete-lock", + namespace: "demo", + worker: "api", + key: "KEY", + method: "PUT", + status: 409, + reason: "deleting", + }, + }]); }); }); test("worker secret PUT maps active bump contention to secret mutation contention", async () => { const state = workerSecretState; + const logStart = state.logs.length; await withWorkerSecretRedis(state, (redis) => { redis.execFailures = 5; }, async (redis) => { @@ -780,6 +914,55 @@ test("worker secret PUT maps active bump contention to secret mutation contentio const body = await readJsonResponse(response, 503); assert.equal(body.error, "secret_mutation_contention"); assert.equal(redis.ops.some((op) => op[0] === "hSet" && op[1] === "secrets:demo:api"), false); + assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ + level: "warn", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-bump-contention", + namespace: "demo", + worker: "api", + key: "KEY", + method: "PUT", + status: 503, + reason: "secret_mutation_contention", + }, + }]); + }); +}); + +test("worker secret DELETE precheck logs direct mutation aborts through the shared rejection shape", async () => { + const state = workerSecretState; + const logStart = state.logs.length; + await withWorkerSecretRedis(state, (redis) => { + redis.strings.set("worker-delete-lock:demo:api", "holder-token"); + }, async () => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/KEY", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["KEY"], + requestId: "rid-worker-secret-delete-precheck", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ + level: "warn", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-delete-precheck", + namespace: "demo", + worker: "api", + key: "KEY", + method: "DELETE", + status: 409, + reason: "deleting", + }, + }]); }); }); diff --git a/tests/unit/control-shared-stub.test.js b/tests/unit/control-shared-stub.test.js new file mode 100644 index 0000000..3255b12 --- /dev/null +++ b/tests/unit/control-shared-stub.test.js @@ -0,0 +1,40 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + jsonError as productionJsonError, + jsonResponse as productionJsonResponse, + sanitizeJsonErrorDetails as productionSanitizeJsonErrorDetails, +} from "../../shared/respond.js"; +import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; +import { readJsonResponse } from "../helpers/response-json.js"; + +const stub = await import(controlSharedStubUrl()); + +test("control shared stub re-exports the production JSON response contract", () => { + assert.equal(stub.jsonError, productionJsonError); + assert.equal(stub.jsonResponse, productionJsonResponse); + assert.equal(stub.sanitizeJsonErrorDetails, productionSanitizeJsonErrorDetails); +}); + +test("control shared stub enforces streamed request limits through shared bounded-body", async () => { + let canceled = false; + const requestInit = /** @type {RequestInit} */ (/** @type {unknown} */ ({ + method: "POST", + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{"a":')); + controller.enqueue(new TextEncoder().encode('"too large"}')); + }, + cancel() { + canceled = true; + }, + }), + duplex: "half", + })); + const request = new Request("http://control.test/", requestInit); + + const result = await stub.readJsonBody(request, { maxBytes: 8 }); + assert.ok("response" in result); + await readJsonResponse(result.response, 413); + assert.equal(canceled, true); +}); diff --git a/tests/unit/control-shared.test.js b/tests/unit/control-shared.test.js index 6f0fb43..7afdcdc 100644 --- a/tests/unit/control-shared.test.js +++ b/tests/unit/control-shared.test.js @@ -12,6 +12,7 @@ import { readRepositoryFile, repositoryFileUrl, } from "../helpers/load-shared-module.js"; +import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { installMockProperty } from "../helpers/mock-global.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; import { assertJsonResponse } from "../helpers/response-json.js"; @@ -19,9 +20,8 @@ import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; -const sharedRedisUrl = moduleDataUrl(` +const sharedRedisUrl = sharedRedisStubUrl(` export class RedisClient {} - export class WatchError extends Error {} export function redisDbFromEnv() { return 0; } `); const controlS3Url = moduleDataUrl(`export function makeS3Client() { return null; }`); @@ -29,12 +29,48 @@ const controlR2Url = moduleDataUrl(`export function makeR2AdminClient() { return const sharedAuthTokenUrl = moduleDataUrl(`export function extractToken() { return null; }`); const sharedAuthRolesUrl = moduleDataUrl(`export function validatePrincipalShape() { return false; }`); const sharedBoundedBodyUrl = repositoryFileUrl("shared/bounded-body.js"); +const sharedEnvUrl = repositoryFileUrl("shared/env.js"); const sharedErrorsUrl = repositoryFileUrl("shared/errors.js"); const sharedRandomIdUrl = repositoryFileUrl("shared/random-id.js"); +const sharedRedisLockUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("shared/redis-lock.js"), + [[/from "shared-random-id"/g, `from ${JSON.stringify(sharedRandomIdUrl)}`]] +)); const sharedQueueKeysUrl = moduleDataUrl(`export function queueStreamKey() { return ""; }`); const sharedRespondUrl = moduleDataUrl(readRepositoryFile("shared/respond.js")); const controlLibUrl = moduleDataUrl(`export function deleteLockKey(ns, worker) { return \`worker-delete-lock:\${ns}:\${worker}\`; }`); const sharedS3CleanupLifecycleUrl = repositoryFileUrl("shared/s3-cleanup-lifecycle.js"); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const controlErrorsUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("control/errors.js"), + [[/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`]] +)); +const controlWorkflowsClientUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("control/workflows-client.js"), + [ + [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], + [/from "control-errors"/g, `from ${JSON.stringify(controlErrorsUrl)}`], + ] +)); +const { postWorkflowsInternalRequest } = await import(controlWorkflowsClientUrl); +const sharedOwnerLeaseUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("shared/owner-lease.js"), + [[/from "shared-env"/g, `from ${JSON.stringify(sharedEnvUrl)}`]] +)); +const controlOptimisticUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("control/optimistic.js"), + [ + [/from "shared-redis"/g, `from ${JSON.stringify(sharedRedisUrl)}`], + [/from "shared-owner-lease"/g, `from ${JSON.stringify(sharedOwnerLeaseUrl)}`], + ] +)); +const controlJsonBodyUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("control/json-body.js"), + [ + [/from "shared-bounded-body"/g, `from ${JSON.stringify(sharedBoundedBodyUrl)}`], + [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], + ] +)); // Use the same stub constructor imported by control/shared.js so // runOptimistic's `instanceof WatchError` check observes the test error. const { WatchError: ControlSharedWatchError } = await import(sharedRedisUrl); @@ -48,12 +84,18 @@ const rewritten = applyModuleReplacements(readRepositoryFile("control/shared.js" [/from "shared-bounded-body"/g, `from ${JSON.stringify(sharedBoundedBodyUrl)}`], [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], [/from "shared-random-id"/g, `from ${JSON.stringify(sharedRandomIdUrl)}`], + [/from "shared-redis-lock"/g, `from ${JSON.stringify(sharedRedisLockUrl)}`], [/from "shared-queue-keys"/g, `from ${JSON.stringify(sharedQueueKeysUrl)}`], [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], [/from "control-lib"/g, `from ${JSON.stringify(controlLibUrl)}`], [/from "shared-s3-cleanup-lifecycle"/g, `from ${JSON.stringify(sharedS3CleanupLifecycleUrl)}`], + [/from "shared-version"/g, `from ${JSON.stringify(sharedVersionUrl)}`], [/from "shared-internal-auth"/g, `from ${JSON.stringify(sharedInternalAuthUrl())}`], [/from "shared-observability"/g, `from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}`], + [/from "control-workflows-client"/g, `from ${JSON.stringify(controlWorkflowsClientUrl)}`], + [/from "control-errors"/g, `from ${JSON.stringify(controlErrorsUrl)}`], + [/from "control-optimistic"/g, `from ${JSON.stringify(controlOptimisticUrl)}`], + [/from "control-json-body"/g, `from ${JSON.stringify(controlJsonBodyUrl)}`], ]); const { @@ -612,21 +654,21 @@ test("assertWorkflowDeleteAllowed preserves active workflow blockers", async (t) ); }); -test("cleanupDoAlarmsForWorker uses a bounded workflows fetch signal", async (t) => { +test("shared workflows calls preserve endpoint-specific timeout behavior", async (t) => { restoreControlSharedStateAfter(t); state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; - /** @type {number | null} */ - let timeoutMs = null; + /** @type {number[]} */ + const timeoutMs = []; /** @type {AbortSignal[]} */ const timeoutSignals = []; /** @type {ReturnType[]} */ const timeoutHandles = []; - /** @type {AbortSignal | undefined} */ - let fetchSignal; - /** @type {unknown} */ - let cleanupBody = null; + /** @type {AbortSignal[]} */ + const fetchSignals = []; + /** @type {Record} */ + const requestBodies = {}; const restoreTimeout = installMockProperty(AbortSignal, "timeout", (ms) => { - timeoutMs = ms; + timeoutMs.push(ms); const controller = new AbortController(); const handle = setTimeout(() => { controller.abort(new DOMException("The operation timed out.", "TimeoutError")); @@ -637,28 +679,61 @@ test("cleanupDoAlarmsForWorker uses a bounded workflows fetch signal", async (t) }); state.workflows = { /** - * @param {RequestInfo | URL} _url + * @param {RequestInfo | URL} url * @param {RequestInit | undefined} init */ - async fetch(_url, init) { - fetchSignal = init?.signal ?? undefined; + async fetch(url, init) { assert.equal(new Headers(init?.headers).get("x-wdl-internal-auth"), TEST_INTERNAL_AUTH_TOKEN); - cleanupBody = parseJsonObjectRequestBody(init, "DO alarm cleanup request body"); + const endpoint = String(url); + if (endpoint.endsWith("/lifecycle/check-delete")) { + assert.equal(init?.signal, undefined); + requestBodies.lifecycle = parseJsonObjectRequestBody(init, "workflow lifecycle request body"); + return Response.json({ allowed: true }); + } + assert.ok(init?.signal instanceof AbortSignal); + fetchSignals.push(init.signal); + requestBodies.cleanup = parseJsonObjectRequestBody(init, "DO alarm cleanup request body"); return Response.json({ ok: true }); }, }; try { + await assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", version: "v2" }); await cleanupDoAlarmsForWorker({ ns: "demo", worker: "api", doStorageId: "do_old" }); } finally { for (const handle of timeoutHandles) clearTimeout(handle); restoreTimeout(); } - assert.equal(timeoutMs, 5_000); + assert.deepEqual(timeoutMs, [5_000]); assert.equal(timeoutSignals.length, 1); - assert.equal(timeoutSignals[0], fetchSignal); - assert.deepEqual(cleanupBody, { ns: "demo", worker: "api", doStorageId: "do_old" }); + assert.deepEqual(timeoutSignals, fetchSignals); + assert.deepEqual(requestBodies, { + lifecycle: { ns: "demo", worker: "api", version: "v2" }, + cleanup: { ns: "demo", worker: "api", doStorageId: "do_old" }, + }); +}); + +test("workflows transport requires an explicit timeout selection at runtime", async () => { + let fetchCalls = 0; + await assert.rejects( + postWorkflowsInternalRequest({ + workflows: { + async fetch() { + fetchCalls += 1; + return Response.json({ ok: true }); + }, + }, + headers: () => ({ "content-type": "application/json" }), + endpoint: "workflows/test", + body: {}, + logEvent: "workflow_test_failed", + timeoutMs: /** @type {any} */ (undefined), + makeError: (/** @type {"unavailable" | "request_failed"} */ failure) => new Error(failure), + }), + /request_failed/ + ); + assert.equal(fetchCalls, 0); }); test("codedErrorResponse preserves semantic status/code with a fallback code", async () => { diff --git a/tests/unit/cross-language-identity.test.js b/tests/unit/cross-language-identity.test.js index 4e0a99a..ed5e402 100644 --- a/tests/unit/cross-language-identity.test.js +++ b/tests/unit/cross-language-identity.test.js @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + KV_ID_RE, QUEUE_NAME_RE, WORKER_NAME_RE, WORKFLOW_INSTANCE_ID_RE, @@ -28,5 +29,6 @@ test("cross-language identity grammar fixture matches JS owners", () => { assertIdentityCases("runtimeLoadNs", isValidRuntimeLoadNs); assertIdentityCases("workerNames", (value) => WORKER_NAME_RE.test(value)); assertIdentityCases("queueNames", (value) => QUEUE_NAME_RE.test(value)); + assertIdentityCases("kvIds", (value) => KV_ID_RE.test(value)); assertIdentityCases("workflowInstanceIds", (value) => WORKFLOW_INSTANCE_ID_RE.test(value)); }); diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 60790cf..20038f8 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -4,10 +4,19 @@ import { test } from "node:test"; import { decodeDoEnvelope } from "../helpers/do-envelope.js"; import { loadDoProtocol } from "../helpers/load-do-protocol.js"; import { assertJsonResponse } from "../helpers/response-json.js"; +import { + OWNER_HINT_STALE_CODES, + OWNER_RACE_RETRY_CODES, + dispatchDoInvokeWithHintCache, + retryableOwnerRaceResponse, + staleDoOwnerHintResponse, +} from "../../runtime/_wdl-do-transport.js"; +import { doOwnerHintHeaders } from "../helpers/do-owner-hint.js"; const { DoRuntimeError, DO_INVOKE_CONTENT_TYPE, + DO_OWNERSHIP_CODE, buildFacetName, buildAlarmRequest, buildForwardRequest, @@ -23,6 +32,80 @@ const { readJsonBody, } = await loadDoProtocol(); +test("DO ownership errors stay aligned with runtime hint and retry handling", async () => { + const ownershipCodes = new Set(Object.values(DO_OWNERSHIP_CODE)); + /** @type {Set} */ + const ownerRaceRetryCodes = new Set([ + DO_OWNERSHIP_CODE.STALE_OWNER_GENERATION, + DO_OWNERSHIP_CODE.OWNER_CLAIM_RACED, + DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, + DO_OWNERSHIP_CODE.OWNER_LEASE_EXPIRED, + DO_OWNERSHIP_CODE.OWNER_LEASE_TOO_SHORT, + ]); + assert.deepEqual( + [...OWNER_HINT_STALE_CODES].toSorted(), + [...ownershipCodes].toSorted() + ); + assert.deepEqual( + [...OWNER_RACE_RETRY_CODES].toSorted(), + [...ownerRaceRetryCodes].toSorted() + ); + + for (const code of ownershipCodes) { + const response = Response.json({ error: code }, { status: 503 }); + assert.equal(await staleDoOwnerHintResponse(response), true, code); + assert.equal( + await retryableOwnerRaceResponse(response), + ownerRaceRetryCodes.has(code), + code + ); + } +}); + +test("DO invoke dispatch does not cache hints carried by owner-race responses", async () => { + /** @type {Array<{ url: string, init: RequestInit | undefined }>} */ + const routerCalls = []; + /** @type {unknown[]} */ + const cachedHints = []; + const init = { + method: "POST", + headers: { "x-wdl-do-accept-owner-hint": "1" }, + }; + + const response = await dispatchDoInvokeWithHintCache({ + async routerFetch(url, requestInit) { + routerCalls.push({ url, init: requestInit }); + return routerCalls.length === 1 + ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnerHintHeaders(), + }) + : new Response("retried"); + }, + routerUrl: "http://do-runtime/internal/do/invoke", + async ownerFetch() { + assert.fail("owner endpoint must not receive an owner-race response hint"); + }, + ownerPath: "/internal/do/invoke", + init, + cache: { + get() { + return null; + }, + set(_key, hint) { + cachedHints.push(hint); + }, + delete() {}, + }, + hintKey: "room-a", + }); + + assert.equal(await response.text(), "retried"); + assert.equal(routerCalls.length, 2); + assert.deepEqual(cachedHints, []); + assert.equal(new Headers(routerCalls[1].init?.headers).get("x-wdl-do-accept-owner-hint"), null); +}); + const DO_STORAGE_ID = "do_0123456789abcdef0123456789abcdef"; const CHAT_ROOM_HOST_ID = `${DO_STORAGE_ID}:ChatRoom:shard12`; diff --git a/tests/unit/fake-redis.test.js b/tests/unit/fake-redis.test.js index ecdb171..4399466 100644 --- a/tests/unit/fake-redis.test.js +++ b/tests/unit/fake-redis.test.js @@ -1,7 +1,21 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { redisConformanceCases } from "../helpers/redis-conformance-cases.js"; -import { createFakeRedis, FakeRedisWatchError } from "../helpers/mocks/fake-redis.js"; +import { + createFakeRedis, + FakeRedisWatchError, + sharedRedisStubUrl, +} from "../helpers/mocks/fake-redis.js"; + +test("shared Redis test modules use the fake WatchError and production bulk decoder", async () => { + const { WatchError, decodeBulk } = await import(sharedRedisStubUrl()); + assert.equal(WatchError, FakeRedisWatchError); + assert.equal(decodeBulk(undefined), undefined); + assert.equal(decodeBulk(null), null); + assert.equal(decodeBulk("value"), "value"); + assert.equal(decodeBulk(new TextEncoder().encode("bytes")), "bytes"); + assert.equal(decodeBulk(42), "42"); +}); /** * @param {ReturnType} redis diff --git a/tests/unit/gateway-lib.test.js b/tests/unit/gateway-lib.test.js index 7a588e2..e2e1f95 100644 --- a/tests/unit/gateway-lib.test.js +++ b/tests/unit/gateway-lib.test.js @@ -1,6 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { + deleteGatewayInternalHeaders, escapeRegex, classifyHost, isCanonicalPatternHost, @@ -11,6 +12,24 @@ import { } from "../../gateway/lib.js"; import { NS_PATTERN, isValidRouteNs } from "../../shared/ns-pattern.js"; +test("deleteGatewayInternalHeaders removes fixed and prefixed private headers", () => { + const headers = new Headers({ + "x-worker-id": "tenant:worker:v1", + "x-worker-prefix": "/worker", + "x-wdl-upstream-binding": "RUNTIME_USER", + "x-wdl-future-private": "private", + "x-request-id": "keep-request-id", + "sec-websocket-protocol": "keep-protocol", + }); + + deleteGatewayInternalHeaders(headers); + + assert.deepEqual(Object.fromEntries(headers), { + "sec-websocket-protocol": "keep-protocol", + "x-request-id": "keep-request-id", + }); +}); + /** * @param {any} sorted * @param {string} pathname diff --git a/tests/unit/gateway-websocket.test.js b/tests/unit/gateway-websocket.test.js index 0de506a..b2d55e9 100644 --- a/tests/unit/gateway-websocket.test.js +++ b/tests/unit/gateway-websocket.test.js @@ -77,6 +77,7 @@ const { proxyGatewayWebSocket, webSocketProxyOptionsFromEnv } = await importRepo [ [/from "shared-observability";/, `from ${JSON.stringify(repositoryFileUrl("shared/observability.js"))};`], [/from "shared-respond";/, `from ${JSON.stringify(repositoryFileUrl("shared/respond.js"))};`], + [/from "gateway-lib";/, `from ${JSON.stringify(repositoryFileUrl("gateway/lib.js"))};`], ] ); diff --git a/tests/unit/integration-pool.test.js b/tests/unit/integration-pool.test.js index 9540aa1..cd26a5c 100644 --- a/tests/unit/integration-pool.test.js +++ b/tests/unit/integration-pool.test.js @@ -11,6 +11,12 @@ import { makeSlotEnv, persistDurationFile, } from "../../scripts/_integration-pool.js"; +import { + LOCAL_ADMIN_TOKEN, + LOCAL_CONNECT_HOST, + localAssetsCdnBase, + localControlUrl, +} from "../../scripts/integration-environment.js"; import { installStreamWriteCapture } from "../helpers/output-capture.js"; test("integration shard env ignores host control credentials", () => { @@ -24,10 +30,10 @@ test("integration shard env ignores host control credentials", () => { assert.equal(env.COMPOSE_PROJECT_NAME, "wdl-it-2"); assert.equal(env.WDL_GATEWAY_HOST_PORT, "18082"); - assert.equal(env.ADMIN_TOKEN, "local-dev-token"); - assert.equal(env.CONTROL_URL, "http://admin.test:18082"); - assert.equal(env.CONTROL_CONNECT_HOST, "localhost"); - assert.equal(env.ASSETS_CDN_BASE, "http://localhost:29502/wdl-assets"); + assert.equal(env.ADMIN_TOKEN, LOCAL_ADMIN_TOKEN); + assert.equal(env.CONTROL_URL, localControlUrl(18082)); + assert.equal(env.CONTROL_CONNECT_HOST, LOCAL_CONNECT_HOST); + assert.equal(env.ASSETS_CDN_BASE, localAssetsCdnBase(29502)); assert.equal(env.WDL_WORKERD_CONFIG_VARIANT, "local"); assert.equal("WDL_NS" in env, false); assert.equal("CLOUDFLARE_ENV" in env, false); diff --git a/tests/unit/integration-test-plan.test.js b/tests/unit/integration-test-plan.test.js index acb6118..c33b8ef 100644 --- a/tests/unit/integration-test-plan.test.js +++ b/tests/unit/integration-test-plan.test.js @@ -1,12 +1,17 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; import { CLI_INTEGRATION_MARKER, durationPriorityNames, hasCliIntegrationMarker, prioritizeDefaultFiles, + readIntegrationDurationRecords, + readIntegrationDurationReport, } from "../../scripts/integration-test-plan.js"; +import { installStreamWriteCapture } from "../helpers/output-capture.js"; +import { withTempDir } from "../helpers/temp-dir.js"; test("prioritizeDefaultFiles runs configured slow files first and preserves the rest", () => { assert.deepEqual( @@ -94,3 +99,40 @@ test("hasCliIntegrationMarker only accepts line-start marker comments", () => { assert.equal(hasCliIntegrationMarker(`const marker = "// ${CLI_INTEGRATION_MARKER}";\n`), false); assert.equal(hasCliIntegrationMarker(`/* ${CLI_INTEGRATION_MARKER} */\n`), false); }); + +test("integration duration reader owns the full tolerant report schema", async () => { + await withTempDir("wdl-duration-reader-", async (dir) => { + const file = `${dir}/durations.json`; + const report = { + updatedAt: "2026-07-10T00:00:00.000Z", + runDurationMs: 123, + files: { + "tests/integration/a.test.js": { + durationMs: 42, + status: "passed", + updatedAt: "2026-07-10T00:00:00.000Z", + }, + }, + }; + writeFileSync(file, JSON.stringify(report)); + + assert.deepEqual(readIntegrationDurationReport(file), report); + assert.deepEqual(readIntegrationDurationRecords(file), report.files); + }); +}); + +test("integration duration reader warns once and returns null for unreadable JSON", async () => { + await withTempDir("wdl-duration-reader-", async (dir) => { + const file = `${dir}/durations.json`; + writeFileSync(file, "not-json"); + /** @type {string[]} */ + const writes = []; + const restoreWrite = installStreamWriteCapture(process.stderr, writes); + try { + assert.equal(readIntegrationDurationReport(file), null); + } finally { + restoreWrite(); + } + assert.match(writes.join(""), /warning: ignoring unreadable integration duration file/); + }); +}); diff --git a/tests/unit/internal-auth.test.js b/tests/unit/internal-auth.test.js index 29b9712..003c87c 100644 --- a/tests/unit/internal-auth.test.js +++ b/tests/unit/internal-auth.test.js @@ -4,6 +4,8 @@ import { test } from "node:test"; import { INTERNAL_AUTH_HEADER, INTERNAL_AUTH_ENV, + INTERNAL_AUTH_FAILURE_CODE, + INTERNAL_AUTH_FAILURE_MESSAGE, INTERNAL_AUTH_PREVIOUS_ENV, internalAuthFailureResponse, internalAuthPreviousToken, @@ -14,10 +16,32 @@ import { withInternalAuthEntries, } from "../../shared/internal-auth.js"; import { withMockedProperty } from "../helpers/mock-global.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { assertJsonResponse } from "../helpers/response-json.js"; const TOKEN = "test-internal-auth-token"; const ENV = { [INTERNAL_AUTH_ENV]: TOKEN }; +const contract = /** @type {{ header: string, currentEnv: string, previousEnv: string, failure: { status: number, error: string, message: string }, rotationCases: Array<{ actual: string | null, current: string, previous: string | null, accepted: boolean }> }} */ ( + readRepositoryJson("tests/fixtures/internal-auth-contract.json") +); + +test("internal auth literals and rotation match the shared Rust/JS contract", () => { + assert.equal(INTERNAL_AUTH_HEADER, contract.header); + assert.equal(INTERNAL_AUTH_ENV, contract.currentEnv); + assert.equal(INTERNAL_AUTH_PREVIOUS_ENV, contract.previousEnv); + assert.equal(INTERNAL_AUTH_FAILURE_CODE, contract.failure.error); + assert.equal(INTERNAL_AUTH_FAILURE_MESSAGE, contract.failure.message); + + for (const entry of contract.rotationCases) { + const headers = new Headers(); + if (entry.actual !== null) headers.set(INTERNAL_AUTH_HEADER, entry.actual); + const env = { + [INTERNAL_AUTH_ENV]: entry.current, + ...(entry.previous === null ? {} : { [INTERNAL_AUTH_PREVIOUS_ENV]: entry.previous }), + }; + assert.equal(verifyInternalAuthHeaders(headers, env), entry.accepted); + } +}); test("internal auth token requires a non-empty configured string", () => { assert.equal(internalAuthToken(ENV), TOKEN); @@ -110,8 +134,8 @@ test("internal auth strip removes only the internal header", () => { test("internal auth failure response has stable public shape", async () => { const response = internalAuthFailureResponse(); - await assertJsonResponse(response, 401, { - error: "internal_auth_failed", - message: "Internal authentication failed", + await assertJsonResponse(response, contract.failure.status, { + error: contract.failure.error, + message: contract.failure.message, }); }); diff --git a/tests/unit/observability.test.js b/tests/unit/observability.test.js index f4d9b0d..d7500d8 100644 --- a/tests/unit/observability.test.js +++ b/tests/unit/observability.test.js @@ -15,10 +15,12 @@ import { } from "../../shared/observability.js"; import { parseStdoutJson } from "../helpers/json-payload.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; +import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; import { withCapturedConsole } from "../helpers/output-capture.js"; const REQUEST_ID_FIXTURES = readRepositoryJson("tests/fixtures/request-id-sanitizer.json"); const OBSERVABILITY_CONTRACT = readRepositoryJson("tests/fixtures/observability-contract.json"); +const OBSERVABILITY_NOOP = await import(OBSERVABILITY_NOOP_URL); test("generateRequestId produces 16 lowercase hex chars", () => { for (let i = 0; i < 20; i++) { @@ -82,6 +84,12 @@ test("sanitizeRequestId follows the cross-language fixture contract", () => { } }); +test("observability noop uses production request-id and error normalization", () => { + assert.equal(OBSERVABILITY_NOOP.sanitizeRequestId, sanitizeRequestId); + assert.equal(OBSERVABILITY_NOOP.formatError, formatError); + assert.equal(OBSERVABILITY_NOOP.ensureRequestId({ "x-request-id": "bad id" }), "rid"); +}); + test("formatError normalizes null, Error objects, and primitive throwables", () => { assert.deepEqual(formatError(null), { error_message: "Unknown error" }); assert.deepEqual(formatError(new TypeError("boom")), { @@ -245,6 +253,53 @@ test("MetricsRegistry: cardinality warning fires per-metric-name as structured J }); }); +test("structured log envelope follows the cross-language contract", () => { + const { orderedKeys, timestampShape } = OBSERVABILITY_CONTRACT.logEnvelope; + /** @type {Array<{ name: string, priority: number, stream: string }>} */ + const levels = OBSERVABILITY_CONTRACT.logEnvelope.levels; + setLogLevel("debug"); + try { + const log = createLogger("test-svc"); + withCapturedConsole(({ stdout, stderr }) => { + for (const { name } of levels) log(name, `event_${name}`, { probe: true }); + + const emitted = [ + ...stdout.map((line) => ({ line, stream: "stdout" })), + ...stderr.map((line) => ({ line, stream: "stderr" })), + ].map((record) => ({ + ...record, + payload: parseStdoutJson(record.line, `${record.stream} log payload`), + })); + assert.equal(emitted.length, levels.length); + for (const expected of levels) { + const record = emitted.find(({ payload }) => payload.level === expected.name); + assert.ok(record, `missing ${expected.name} log line`); + assert.equal(record.stream, expected.stream, `${expected.name} stream`); + const { payload } = record; + assert.deepEqual(Object.keys(payload).slice(0, orderedKeys.length), orderedKeys); + assert.equal(payload.ts.replace(/\d/g, "0"), timestampShape); + } + }); + + for (const minimum of levels) { + setLogLevel(minimum.name); + withCapturedConsole(({ stdout, stderr }) => { + for (const { name } of levels) log(name, `gated_${name}`); + const actual = [...stdout, ...stderr] + .map((line) => parseStdoutJson(line, `minimum=${minimum.name} log payload`).level) + .toSorted(); + const expected = levels + .filter(({ priority }) => priority >= minimum.priority) + .map(({ name }) => name) + .toSorted(); + assert.deepEqual(actual, expected, `minimum=${minimum.name}`); + }); + } + } finally { + setLogLevel("info"); + } +}); + test("setLogLevel=warn suppresses info but passes error through", () => { setLogLevel("warn"); try { diff --git a/tests/unit/queue-keys.test.js b/tests/unit/queue-keys.test.js new file mode 100644 index 0000000..23ed34a --- /dev/null +++ b/tests/unit/queue-keys.test.js @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + parseConsumerKey, + parseDelayedKey, + parseStreamKey, + queueConsumerKey, + queueDelayedKey, + queueStreamKey, +} from "../../shared/queue-keys.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; + +const QUEUE_KEY_PARSE_CASES = readRepositoryJson("tests/fixtures/queue-key-parse.json"); + +test("queue key builders compose canonical Redis shapes", () => { + assert.equal(queueStreamKey("demo", "jobs"), "queue:demo:jobs:s"); + assert.equal(queueDelayedKey("demo", "jobs"), "queue-delayed:demo:jobs"); + assert.equal(queueConsumerKey("demo", "jobs"), "queue-consumer:demo:jobs"); +}); + +test("queue key parsers match the cross-language fixture", () => { + const parsers = { + stream: parseStreamKey, + delayed: parseDelayedKey, + consumer: parseConsumerKey, + }; + for (const [kind, parser] of Object.entries(parsers)) { + for (const { key, parsed } of QUEUE_KEY_PARSE_CASES[kind]) { + assert.deepEqual(parser(key), parsed, `${kind}:${JSON.stringify(key)}`); + } + } +}); diff --git a/tests/unit/r2-utils.test.js b/tests/unit/r2-utils.test.js index ca4e553..c55f11a 100644 --- a/tests/unit/r2-utils.test.js +++ b/tests/unit/r2-utils.test.js @@ -1,6 +1,8 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { + R2_BUCKET_NAME_RE as RUNTIME_R2_BUCKET_NAME_RE, + R2_HTTP_METADATA_FIELDS, R2_OBJECT_MAX_BUFFER_BYTES, assertR2BufferSize, encodeS3KeyPath, @@ -8,9 +10,12 @@ import { r2PhysicalKey, r2PhysicalPrefix, r2RangeAndSizeFromHeaders, + r2CacheExpiryFromHeaders, + setR2CacheExpiryHeader, stripR2PhysicalPrefix, validateR2BucketName, } from "../../runtime/r2-utils.js"; +import { R2_BUCKET_NAME_RE as CONTROL_R2_BUCKET_NAME_RE } from "../../shared/ns-pattern.js"; test("r2PhysicalPrefix scopes virtual buckets under namespace", () => { assert.equal( @@ -49,6 +54,27 @@ test("validateR2BucketName enforces prefix-safe virtual bucket names", () => { assert.throws(() => validateR2BucketName("bad/name"), /bucket_name must match/); }); +test("injected and control R2 bucket grammars stay identical", () => { + assert.equal(RUNTIME_R2_BUCKET_NAME_RE.source, CONTROL_R2_BUCKET_NAME_RE.source); + assert.equal(RUNTIME_R2_BUCKET_NAME_RE.flags, CONTROL_R2_BUCKET_NAME_RE.flags); +}); + +test("R2 HTTP metadata fields and cache expiry use one mapping", () => { + assert.deepEqual(R2_HTTP_METADATA_FIELDS, [ + ["contentType", "content-type"], + ["contentLanguage", "content-language"], + ["contentDisposition", "content-disposition"], + ["contentEncoding", "content-encoding"], + ["cacheControl", "cache-control"], + ]); + const headers = new Headers(); + setR2CacheExpiryHeader(headers, 0); + assert.equal(headers.get("expires"), "Thu, 01 Jan 1970 00:00:00 GMT"); + assert.equal(r2CacheExpiryFromHeaders(headers), 0); + headers.set("expires", "not-a-date"); + assert.equal(r2CacheExpiryFromHeaders(headers), undefined); +}); + test("encodeS3KeyPath percent-encodes key segments while preserving slashes", () => { assert.equal( encodeS3KeyPath("r2/demo/uploads/a b/?.txt"), diff --git a/tests/unit/redis-lock.test.js b/tests/unit/redis-lock.test.js new file mode 100644 index 0000000..185c826 --- /dev/null +++ b/tests/unit/redis-lock.test.js @@ -0,0 +1,95 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + importRepositoryModule, + importSpecifierReplacements, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; + +const { + acquireTokenLock, + createTokenLock, + releaseTokenLock, + renewTokenLock, +} = await importRepositoryModule("shared/redis-lock.js", importSpecifierReplacements({ + "shared-random-id": repositoryFileUrl("shared/random-id.js"), +})); + +test("token lock owner acquires and renews with the canonical Redis options", async () => { + /** @type {unknown[][]} */ + const calls = []; + const client = { + /** @param {unknown[]} args */ + async set(...args) { + calls.push(["set", ...args]); + return "OK"; + }, + async delIfEq() { return 1; }, + }; + const lock = createTokenLock("lock:demo"); + + assert.match(lock.token, /^[0-9a-f]{32}$/); + assert.equal(await acquireTokenLock(client, lock, { ttlSeconds: 30 }), true); + assert.equal(await renewTokenLock(client, lock, 60), true); + assert.deepEqual(calls, [ + ["set", "lock:demo", lock.token, { nx: true, ttl: 30 }], + ["set", "lock:demo", lock.token, { ttl: 60, ifeq: lock.token }], + ]); +}); + +test("token lock owner supports session-local transactional acquisition", async () => { + /** @type {unknown[][]} */ + const calls = []; + const multi = { + /** @param {string} key @param {string} token @param {Record} options */ + set(key, token, options) { + calls.push(["set", key, token, options]); + return this; + }, + async exec() { + calls.push(["exec"]); + return ["OK"]; + }, + }; + const client = { + async set() { throw new Error("direct SET must not run"); }, + async delIfEq() { return 1; }, + multi() { return multi; }, + }; + const lock = { key: "lock:auth", token: "token" }; + + assert.equal(await acquireTokenLock(client, lock, { + ttlSeconds: 30, + transactional: true, + }), true); + assert.deepEqual(calls, [ + ["set", "lock:auth", "token", { nx: true, ttl: 30 }], + ["exec"], + ]); +}); + +test("token lock release is token-scoped and never masks the primary outcome", async () => { + /** @type {unknown[][]} */ + const seen = []; + const releaseError = new Error("redis unavailable"); + const client = { + async set() { return "OK"; }, + /** @param {string} key @param {string} token */ + async delIfEq(key, token) { + seen.push([key, token]); + throw releaseError; + }, + }; + + await assert.doesNotReject(() => releaseTokenLock( + client, + { key: "lock:demo", token: "token" }, + { + onError(/** @type {unknown} */ err) { + assert.equal(err, releaseError); + throw new Error("logger unavailable"); + }, + } + )); + assert.deepEqual(seen, [["lock:demo", "token"]]); +}); diff --git a/tests/unit/runtime-binding-surface.test.js b/tests/unit/runtime-binding-surface.test.js index 2dee555..1f79981 100644 --- a/tests/unit/runtime-binding-surface.test.js +++ b/tests/unit/runtime-binding-surface.test.js @@ -6,6 +6,8 @@ import { RUNTIME_METRICS_NOOP_URL } from "../helpers/mocks/runtime-metrics.js"; import { runtimeProxyBindingStubUrl } from "../helpers/runtime-proxy-stub.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); +const SHARED_BASE64_URL = repositoryFileUrl("shared/base64.js"); +const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); const toBytesStub = `const toBytes = (value) => { @@ -16,13 +18,7 @@ const toBytesStub = `const toBytes = (value) => { return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); } throw new Error("KV put: value must be string | ArrayBuffer | typed array | ReadableStream"); -}; -const bytesToBase64 = (bytes) => { - let binary = ""; - for (const byte of bytes) binary += String.fromCharCode(byte); - return btoa(binary); -}; -const base64ToBytes = (value) => Uint8Array.from(atob(value), (ch) => ch.charCodeAt(0));`; +};`; const buildAssetUrlStub = `const buildAssetUrl = (cdnBase, prefix, path) => { if (!cdnBase) throw new Error("ASSETS.url: cdnBase is not configured"); @@ -45,8 +41,10 @@ const buildAssetUrlStub = `const buildAssetUrl = (cdnBase, prefix, path) => { test("KV host RPC surface exposes only public namespace methods", async () => { const { KV } = await importRepositoryModule("runtime/bindings/kv.js", [ [/from "cloudflare:workers";/, `from ${JSON.stringify(CLOUDFLARE_WORKERS_URL)};`], - [/import \{ base64ToBytes, bytesToBase64, toBytes \} from "runtime-lib";/, toBytesStub], + [/import \{ toBytes \} from "runtime-lib";/, toBytesStub], [/from "runtime-metrics";/, `from ${JSON.stringify(RUNTIME_METRICS_NOOP_URL)};`], + [/from "shared-base64";/, `from ${JSON.stringify(SHARED_BASE64_URL)};`], + [/from "shared-bounded-body";/, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)};`], [/from "runtime-bindings-proxy";/, `from ${JSON.stringify(PROXY_BINDING_URL)};`], [/from "shared-respond";/, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], ]); diff --git a/tests/unit/runtime-dispatch-handlers.test.js b/tests/unit/runtime-dispatch-handlers.test.js index 6377ed7..59a4c77 100644 --- a/tests/unit/runtime-dispatch-handlers.test.js +++ b/tests/unit/runtime-dispatch-handlers.test.js @@ -258,11 +258,11 @@ test("handleFetchDispatch truncates tail path fields before append", async () => } }); -test("handleFetchDispatch emits error finish tail event while returning runtime_error", async () => { +test("handleFetchDispatch preserves falsey thrown values in the error finish tail event", async () => { const fetchSpy = installTailFetchSpy(["demo:fetch-error-worker"]); const ctx = makeCtx(); const scope = makeScope(); - const err = new Error("fetch failed"); + const err = 0; try { const res = await handleFetchDispatch({ @@ -292,7 +292,7 @@ test("handleFetchDispatch emits error finish tail event while returning runtime_ assert.equal(events[1].phase, "finish"); assert.equal(events[1].path, "/app"); assert.equal(events[1].outcome, "error"); - assert.equal(events[1].error, "fetch failed"); + assert.equal(events[1].error, "0"); } finally { fetchSpy.restore(); } @@ -379,6 +379,35 @@ test("handleQueuedDispatch decodes queue messages before JSRPC dispatch", async assert.equal(calls[0].messages[0].attempts, 3); }); +test("handleQueuedDispatch maps invalid base64 to a permanent decode failure", async () => { + let dispatchCalls = 0; + const scope = makeScope(); + const res = await handleQueuedDispatch({ + request: jsonRequest({ + queue: "jobs", + messages: [{ + id: "m1", + first_seen_ms: "123", + attempts: "0", + body_b64: "%%%", + content_type: "bytes", + }], + }), + scope, + stub: makeStub({ + async queue() { + dispatchCalls += 1; + return { ackAll: true }; + }, + }), + }); + + const body = await readJsonResponse(res, 400); + assert.equal(body.error, "queue_message_decode_failed"); + assert.match(body.message, /Invalid base64 input/); + assert.equal(dispatchCalls, 0); +}); + test("handleQueuedDispatch accepts legal queue batches above the small dispatch cap", async () => { /** @type {any[]} */ const calls = []; diff --git a/tests/unit/runtime-dispatch-workflows.test.js b/tests/unit/runtime-dispatch-workflows.test.js index 9038fbb..fe2d9f5 100644 --- a/tests/unit/runtime-dispatch-workflows.test.js +++ b/tests/unit/runtime-dispatch-workflows.test.js @@ -2,6 +2,10 @@ import { beforeEach, test } from "node:test"; import assert from "node:assert/strict"; import { parseJsonText } from "../helpers/json-payload.js"; import { loadRuntimeDispatch } from "../helpers/load-runtime-dispatch.js"; +import { + importRepositoryModule, + readRepositoryJson, +} from "../helpers/load-shared-module.js"; import { jsonRequest, makeScope, @@ -12,6 +16,10 @@ import { readJsonResponse } from "../helpers/response-json.js"; import { delay } from "../helpers/timing.js"; const { runtimeDispatch, runtimeDispatchWorkflowStep } = await loadRuntimeDispatch(); +const workflowJson = await importRepositoryModule("runtime/dispatch/workflow-json.js"); +const workflowLimits = /** @type {{ resultBytesMax: number, backendRequestBytesMax: number, payloadTooLargeCode: string }} */ ( + readRepositoryJson("tests/fixtures/workflow-limits.json") +); const { _resetWorkflowReplayCacheForTest, _stringifyWorkflowBackendBodyForTest, @@ -38,6 +46,22 @@ beforeEach(() => { _resetWorkflowReplayCacheForTest(); }); +test("workflow payload limits match the shared Rust/JS contract", () => { + assert.equal(workflowJson.WORKFLOW_RESULT_BYTES_MAX, workflowLimits.resultBytesMax); + assert.equal( + workflowJson.WORKFLOW_BACKEND_REQUEST_BYTES_MAX, + workflowLimits.backendRequestBytesMax + ); + assert.equal( + workflowJson.WORKFLOW_PAYLOAD_TOO_LARGE_CODE, + workflowLimits.payloadTooLargeCode + ); + assert.throws( + () => workflowJson._stringifyWorkflowJsonForTest("x", 0), + (err) => err instanceof Error && err.name === workflowLimits.payloadTooLargeCode + ); +}); + test("workflow bounded JSON serializer matches JSON.stringify for supported values", () => { const inherited = Object.create({ hidden: true }); inherited.visible = "yes"; diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index b6cbd60..733ae00 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -570,6 +570,42 @@ test("DurableObjectNamespace direct backend retries owner claim races once", asy assert.equal(calls.length, 2); }); +test("DurableObjectNamespace direct backend ignores hints attached to owner-race responses", async () => { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + const backend = { + fetch: makeRecordingFetch(routerCalls, { + response: () => routerCalls.length === 1 + ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnerHintResponse().headers, + }) + : new Response("ok"), + }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { response: new Response("owner-should-not-be-called") }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend, ownerNetwork }); + + const response = await ns.get(ns.idFromName("room-race-hint")).fetch("https://demo.workers.example/send"); + + assert.equal(await response.text(), "ok"); + assert.equal(routerCalls.length, 2); + assert.equal(ownerCalls.length, 0); + assert.equal(headerValue(routerCalls[0].init.headers, "x-wdl-do-accept-owner-hint"), "1"); + assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); +}); + test("DurableObjectNamespace direct backend retries owner lease budget errors once", async () => { for (const error of ["owner_lease_expired", "owner_lease_too_short"]) { /** @type {any[]} */ diff --git a/tests/unit/runtime-internal-auth.test.js b/tests/unit/runtime-internal-auth.test.js index c6de062..9653402 100644 --- a/tests/unit/runtime-internal-auth.test.js +++ b/tests/unit/runtime-internal-auth.test.js @@ -50,8 +50,8 @@ export async function readWorkflowNotifyDispatch() { throw new Error("unexpected export async function readWorkflowRunDispatch() { throw new Error("unexpected workflow run body read"); } `); const runtimeLoadUrl = moduleDataUrl(` -export function createLoaderCallback() { - return async () => { throw new Error("unexpected worker load"); }; +export function getLoadedWorkerStub() { + throw new Error("unexpected worker load"); } `); const emptyBindingUrl = moduleDataUrl(` diff --git a/tests/unit/runtime-kv-binding.test.js b/tests/unit/runtime-kv-binding.test.js index 3d1bc68..4d8b3f8 100644 --- a/tests/unit/runtime-kv-binding.test.js +++ b/tests/unit/runtime-kv-binding.test.js @@ -13,7 +13,9 @@ import { runtimeProxyBindingStubUrl } from "../helpers/runtime-proxy-stub.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); const RUNTIME_LIB_URL = runtimeLibModuleDataUrl(); +const SHARED_BASE64_URL = repositoryFileUrl("shared/base64.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); /** @param {Array<[RegExp | string, string]>} [replacements] */ async function loadKvBinding(replacements = []) { @@ -21,10 +23,12 @@ async function loadKvBinding(replacements = []) { [/from "cloudflare:workers";/, `from ${JSON.stringify(CLOUDFLARE_WORKERS_URL)};`], [/from "runtime-lib";/, `from ${JSON.stringify(RUNTIME_LIB_URL)};`], [/from "runtime-metrics";/, `from ${JSON.stringify(RUNTIME_METRICS_NOOP_URL)};`], + [/from "shared-base64";/, `from ${JSON.stringify(SHARED_BASE64_URL)};`], [ /from "runtime-bindings-proxy";/, `from ${JSON.stringify(PROXY_BINDING_URL)};`, ], + [/from "shared-bounded-body";/, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)};`], [/from "shared-respond";/, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], ]; return importRepositoryModule("runtime/bindings/kv.js", /** @type {Array<[RegExp | string, string]>} */ ([...baseReplacements, ...replacements])); diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index 76b96bb..61f7bd9 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -4,6 +4,8 @@ import { parseBase64Json } from "../helpers/json-payload.js"; import { runtimeLibModuleDataUrl } from "../helpers/load-shared-module.js"; const { + base64ToBytes, + bytesToBase64, toBytes, bundleToWorkerCode, buildAssetUrl, @@ -16,9 +18,25 @@ const { normalizeQueuedDispatchBody, normalizeScheduledDispatchBody, } = await import(runtimeLibModuleDataUrl()); +const sharedBase64 = await import("../../shared/base64.js"); const enc = new TextEncoder(); +test("runtime base64 exports use the shared codec owner", () => { + assert.equal(bytesToBase64, sharedBase64.bytesToBase64); + assert.equal(base64ToBytes, sharedBase64.base64ToBytes); + const bytes = Uint8Array.from({ length: 70_000 }, (_, index) => index % 256); + assert.deepEqual(base64ToBytes(bytesToBase64(bytes)), bytes); +}); + +test("shared base64 decoder matches atob rejection semantics with Buffer present", () => { + assert.deepEqual(base64ToBytes("Z g =="), new Uint8Array([0x66])); + assert.deepEqual(base64ToBytes("Zg"), new Uint8Array([0x66])); + for (const invalid of ["%%%", "Zg=", "Zm9v=", "-_8=", "YWJjZA==x"]) { + assert.throws(() => base64ToBytes(invalid), /Invalid base64 input/); + } +}); + test("toBytes: string → Uint8Array utf8", () => { const r = toBytes("hi"); assert.ok(r instanceof Uint8Array); @@ -610,6 +628,10 @@ test("decodeQueueBody: bytes contentType preserves binary payload", () => { assert.deepEqual(Array.from(decoded), Array.from(payload)); }); +test("decodeQueueBody: invalid base64 fails closed before handler dispatch", () => { + assert.throws(() => decodeQueueBody("%%%", "bytes"), /Invalid base64 input/); +}); + test("decodeQueueBody: unknown contentType throws (v8 path must not silently pass)", () => { assert.throws( () => decodeQueueBody(b64FromUtf8("x"), "v8"), diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index fff1ccf..8583c16 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -6,13 +6,15 @@ import { pathToFileURL } from "node:url"; import { importRepositoryModule, importSpecifierReplacements, - moduleDataUrl, readRepositoryModuleSource, repositoryFileUrl, + repositoryModuleDataUrl, } from "../helpers/load-shared-module.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { makeRecordingFetch, withMockedFetch } from "../helpers/mock-fetch.js"; import { withMockedProperty } from "../helpers/mock-global.js"; import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; +import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { assertJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; import { @@ -26,20 +28,18 @@ const SHARED_WORKER_ID_URL = repositoryFileUrl("shared/worker-id.js"); const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); const SHARED_SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); -const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); -const SHARED_REDIS_URL = moduleDataUrl(` -export class WatchError extends Error { - constructor(message = "watched key changed") { - super(message); - this.name = "WatchError"; - } -} -`); +const SHARED_REDIS_URL = sharedRedisStubUrl(); +const { libUrl: CONTROL_LIB_URL } = await compileControlGraph(); +const RUNTIME_ENV_BUILD_URL = repositoryModuleDataUrl( + "runtime/load/env-build.js", + importSpecifierReplacements({ "shared-ns-pattern": SHARED_NS_PATTERN_URL }) +); const FETCH_STUB = { fetch() {} }; const { estimatedWorkerLoaderEnv } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ + "control-lib": CONTROL_LIB_URL, + "runtime-load-env-build": RUNTIME_ENV_BUILD_URL, "shared-secret-envelope": SHARED_SECRET_ENVELOPE_URL, - "shared-errors": SHARED_ERRORS_URL, "shared-version": SHARED_VERSION_URL, "shared-redis": SHARED_REDIS_URL, })); @@ -68,6 +68,15 @@ const src = readRepositoryModuleSource("runtime/load.js", [ "shared-internal-auth": SHARED_INTERNAL_AUTH_URL, "shared-respond": SHARED_RESPOND_URL, }), + [ + /import \{ evictSiblings, recordLoadedWorker \} from "runtime-state";/, + `const recordLoadedWorker = (workerId) => { + /** @type {any[]} */ (globalThis).__runtimeLoadRecordedWorkers.push(workerId); + }; + const evictSiblings = async (options) => { + /** @type {any[]} */ (globalThis).__runtimeLoadEvictions.push(options); + };` + ], [ /from "runtime-load-code-budget";/, 'from "./load/code-budget.js";' @@ -113,12 +122,16 @@ for (const name of ["env-build.js", "code-budget.js", "module-rewrite.js", "wrap } after(() => removeTempDir(LOAD_TEST_DIR)); +/** @type {any} */ (globalThis).__runtimeLoadRecordedWorkers = []; +/** @type {any} */ (globalThis).__runtimeLoadEvictions = []; + const mod = await import(pathToFileURL(path.join(LOAD_TEST_RUNTIME_DIR, "load.js")).href); const codeBudgetMod = await import(pathToFileURL(path.join(LOAD_TEST_SUBMODULE_DIR, "code-budget.js")).href); const { buildWorkerEnv, createLoaderCallback, decodeRuntimeLoadPayload, + getLoadedWorkerStub, runtimeLoadContentTypeMatches, wrapWorkerCodeForHostBindings, } = mod; @@ -625,6 +638,7 @@ test("buildWorkerEnv: service binding with requiredCallerSecrets filters from ns makeCtx() ); // Listed keys only, worker wins over ns, missing key silently absent. + assert.equal(Object.getPrototypeOf(env.JSJ.props.callerSecrets), Object.prototype); assert.deepEqual(env.JSJ.props.callerSecrets, { JSJ_API_KEY: "worker-level-key" }); assert.equal(env.JSJ.props.callerNs, "demo"); assert.equal(env.JSJ.props.targetNs, "__platform__"); @@ -653,6 +667,7 @@ test("buildWorkerEnv: requiredCallerSecrets does NOT fall back to vars", () => { "https://assets.example", makeCtx() ); + assert.equal(Object.getPrototypeOf(env.JSJ.props.callerSecrets), Object.prototype); assert.deepEqual(env.JSJ.props.callerSecrets, {}); }); @@ -1161,6 +1176,77 @@ test("createLoaderCallback: attaches configured tail worker and always wraps mai ); }); +test("getLoadedWorkerStub records cache misses before scheduling active-version eviction", async () => { + /** @type {Array<{ id: string, factory: () => Promise }>} */ + const loaderCalls = []; + /** @type {Promise[]} */ + const backgroundTasks = []; + /** @type {any} */ (globalThis).__runtimeLoadRecordedWorkers.length = 0; + /** @type {any} */ (globalThis).__runtimeLoadEvictions.length = 0; + const stub = { getEntrypoint() {} }; + const env = { + REDIS_PROXY_URL: "http://redis-proxy.local", + WDL_INTERNAL_AUTH_TOKEN: "test-internal-auth-token", + ASSETS_CDN_BASE: "https://assets.example", + PUBLIC_NETWORK: { kind: "public-network" }, + LOADER: { + get(/** @type {string} */ id, /** @type {() => Promise} */ factory) { + loaderCalls.push({ id, factory }); + return stub; + }, + }, + }; + const ctx = { + ...makeCtx(), + waitUntil(/** @type {Promise} */ promise) { + backgroundTasks.push(promise); + }, + }; + + await withMockedFetch( + async () => new Response(encodeRuntimeLoadPayload({ + bundle: { + "__meta__": JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": { type: "module" } }, + }), + "worker.js": "export default {};", + }, + }), { + status: 200, + headers: { "content-type": RUNTIME_LOAD_CONTENT_TYPE }, + }), + async () => { + const loaded = getLoadedWorkerStub({ + requestId: "rid-loader-wrapper", + env, + ctx, + ns: "demo", + worker: "app", + version: "v2", + evictOnLoad: true, + }); + + assert.equal(loaded.workerId, "demo:app:v2"); + assert.equal(loaded.stub, stub); + assert.equal(loaderCalls.length, 1); + assert.equal(loaderCalls[0].id, "demo:app:v2"); + assert.deepEqual(/** @type {any} */ (globalThis).__runtimeLoadRecordedWorkers, []); + + await loaderCalls[0].factory(); + await Promise.all(backgroundTasks); + + assert.deepEqual(/** @type {any} */ (globalThis).__runtimeLoadRecordedWorkers, ["demo:app:v2"]); + assert.deepEqual( + /** @type {any} */ (globalThis).__runtimeLoadEvictions.map( + (/** @type {{ workerId: string }} */ entry) => entry.workerId + ), + ["demo:app:v2"] + ); + } + ); +}); + test("createLoaderCallback: DO bindings are host proxies and workflow auth stays out of generated facade code", async () => { /** @type {Array<{ service: string, headers: HeadersInit | undefined }>} */ const privateCalls = []; diff --git a/tests/unit/runtime-loaded-facade-surface.test.js b/tests/unit/runtime-loaded-facade-surface.test.js index a7f2509..2531c00 100644 --- a/tests/unit/runtime-loaded-facade-surface.test.js +++ b/tests/unit/runtime-loaded-facade-surface.test.js @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModule } from "../helpers/load-shared-module.js"; +import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; const requestIdFromOptionsStub = `const requestIdFromOptions = (options) => { if (!options || typeof options !== "object") return null; @@ -20,13 +20,8 @@ const d1Facade = await importRepositoryModule("runtime/d1-client.js", [ const r2Facade = await importRepositoryModule("runtime/r2-client.js", [ [ - /import \{\s*R2_OBJECT_MAX_BUFFER_BYTES,\s*assertR2BufferSize,\s*normalizeR2ListLimit,\s*normalizeR2ObjectKey,\s*\} from "\.\/_wdl-r2-utils\.js";/, - `const R2_OBJECT_MAX_BUFFER_BYTES = 26214400; - const assertR2BufferSize = (size, operation) => { - if (size > R2_OBJECT_MAX_BUFFER_BYTES) throw new TypeError("R2 " + operation + ": object too large"); - }; - const normalizeR2ListLimit = (limit) => limit == null ? undefined : Number(limit); - const normalizeR2ObjectKey = (key) => String(key);`, + /from "\.\/_wdl-r2-utils\.js";/, + `from ${JSON.stringify(repositoryFileUrl("runtime/r2-utils.js"))};`, ], [/import \{ requestIdFromOptions \} from "\.\/_wdl-request-id\.js";/, requestIdFromOptionsStub], ]); diff --git a/tests/unit/runtime-pool-boundary.test.js b/tests/unit/runtime-pool-boundary.test.js index 0075a4f..04ab3f9 100644 --- a/tests/unit/runtime-pool-boundary.test.js +++ b/tests/unit/runtime-pool-boundary.test.js @@ -54,8 +54,9 @@ export async function readWorkflowNotifyDispatch(request) { return { body: await export async function readWorkflowRunDispatch(request) { return { body: await request.json() }; } `); const runtimeLoadUrl = moduleDataUrl(` -export function createLoaderCallback() { - return async () => ({}); +export function getLoadedWorkerStub({ env, ns, worker, version }) { + const workerId = \`\${ns}:\${worker}:\${version}\`; + return { workerId, stub: env.LOADER.get(workerId, async () => ({})) }; } `); const emptyBindingUrl = moduleDataUrl(` diff --git a/tests/unit/runtime-r2-client.test.js b/tests/unit/runtime-r2-client.test.js index 0776247..e202b46 100644 --- a/tests/unit/runtime-r2-client.test.js +++ b/tests/unit/runtime-r2-client.test.js @@ -133,6 +133,61 @@ test("R2Bucket.put preserves onlyIf etag arrays for host binding", async () => { assert.deepEqual(calls[0].requestMeta, { requestId: "rid-put" }); }); +test("R2Bucket.put normalizes every Headers httpMetadata field", async () => { + /** @type {any[]} */ + const calls = []; + const bucket = new R2Bucket({ + async put(/** @type {string} */ _key, /** @type {any} */ _value, /** @type {any} */ options) { + calls.push(options); + return null; + }, + }); + const headers = new Headers({ + "content-type": "text/plain", + "content-language": "en", + "content-disposition": "inline", + "content-encoding": "gzip", + "cache-control": "max-age=60", + expires: "Thu, 01 Jan 1970 00:00:00 GMT", + }); + + await bucket.put("a.txt", "hello", { httpMetadata: headers }); + + assert.deepEqual(calls[0].httpMetadata, { + contentType: "text/plain", + contentLanguage: "en", + contentDisposition: "inline", + contentEncoding: "gzip", + cacheControl: "max-age=60", + cacheExpiry: 0, + }); +}); + +test("R2Object.writeHttpMetadata writes every mapped field including epoch expiry", () => { + const object = new R2Object({ + httpMetadata: { + contentType: "text/plain", + contentLanguage: "en", + contentDisposition: "inline", + contentEncoding: "gzip", + cacheControl: "max-age=60", + cacheExpiry: 0, + }, + }); + const headers = new Headers(); + + object.writeHttpMetadata(headers); + + assert.deepEqual(Object.fromEntries(headers), { + "cache-control": "max-age=60", + "content-disposition": "inline", + "content-encoding": "gzip", + "content-language": "en", + "content-type": "text/plain", + expires: "Thu, 01 Jan 1970 00:00:00 GMT", + }); +}); + test("R2Bucket.put rejects timestamp onlyIf conditions", async () => { const bucket = new R2Bucket({ async put() { diff --git a/tests/unit/runtime-r2-host.test.js b/tests/unit/runtime-r2-host.test.js index 0fec3ec..b99c1b8 100644 --- a/tests/unit/runtime-r2-host.test.js +++ b/tests/unit/runtime-r2-host.test.js @@ -121,6 +121,33 @@ test("R2 host put returns metadata from PUT response without a follow-up HEAD", } }); +test("R2 host reads every mapped HTTP metadata response header", async () => { + const restore = installR2FetchMock(async () => new Response(null, { + status: 200, + headers: { + "content-type": "text/plain", + "content-language": "en", + "content-disposition": "inline", + "content-encoding": "gzip", + "cache-control": "max-age=60", + expires: "Thu, 01 Jan 1970 00:00:00 GMT", + }, + })); + try { + const meta = await makeR2Bucket().head("metadata.txt"); + assert.deepEqual(meta.httpMetadata, { + contentType: "text/plain", + contentLanguage: "en", + contentDisposition: "inline", + contentEncoding: "gzip", + cacheControl: "max-age=60", + cacheExpiry: 0, + }); + } finally { + restore(); + } +}); + test("R2 host metadata preserves magic custom metadata keys as data fields", async () => { const restore = installR2FetchMock(async () => new Response(null, { status: 200, diff --git a/tests/unit/runtime-service-binding.test.js b/tests/unit/runtime-service-binding.test.js index 9ed634a..0fe4fd6 100644 --- a/tests/unit/runtime-service-binding.test.js +++ b/tests/unit/runtime-service-binding.test.js @@ -14,20 +14,19 @@ const { ServiceBinding } = await importRepositoryModule("runtime/bindings/servic `const INTERNAL_AUTH_HEADER = ${JSON.stringify(TEST_INTERNAL_AUTH_HEADER)};` ], [ - /import \{ createLoaderCallback \} from "runtime-load";/, - `const createLoaderCallback = (options) => { + /import \{ getLoadedWorkerStub \} from "runtime-load";/, + `const getLoadedWorkerStub = (options) => { /** @type {any} */ (globalThis).__serviceBindingLoaderCallbackOptions.push(options); - return async () => ({}); + return { + workerId: options.workerId, + stub: options.env.LOADER.get(options.workerId, async () => ({})), + }; };` ], [ - /import \{ recordLoadedWorker \} from "runtime-state";/, - `const recordLoadedWorker = () => {};` - ], - [ - /import \{ emitRuntimeTailEvent, fetchTailFields \} from "runtime-tail-forwarder";/, - `const emitRuntimeTailEvent = () => null; - const fetchTailFields = () => ({});` + /import \{ fetchTailFields, startTailEnvelope \} from "runtime-tail-forwarder";/, + `const fetchTailFields = () => ({}); + const startTailEnvelope = () => ({ finish() {}, finishError() {} });` ], ]); @@ -102,6 +101,7 @@ test("ServiceBinding RPC cold-load does not mint a service-binding request id", const options = loaderCallbackOptions(); assert.equal(options.length, 1); assert.equal(options[0].requestId, null); + assert.equal(options[0].evictOnLoad, undefined); }); test("ServiceBinding fetch strips caller-supplied platform forwarding headers", async () => { diff --git a/tests/unit/runtime-tail-worker.test.js b/tests/unit/runtime-tail-worker.test.js index f53f041..6a13cf5 100644 --- a/tests/unit/runtime-tail-worker.test.js +++ b/tests/unit/runtime-tail-worker.test.js @@ -4,6 +4,7 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { installMockFetch, makeRecordingFetch } from "../helpers/mock-fetch.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; @@ -12,6 +13,7 @@ import { delay } from "../helpers/timing.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const OBSERVABILITY_STUB_SRC = `const createLogger = (service) => (level, event, fields) => /** @type {any} */ (globalThis).__runtimeTailLogs.push({ service, level, event, fields }); const createLogLevelBinder = () => { let logLevelSet = false; @@ -23,6 +25,7 @@ const createLogLevelBinder = () => { };`; const forwarderSrc = applyModuleReplacements(readRepositoryFile("runtime/tail-forwarder.js"), [ [/from "runtime-bindings-proxy";/, `from ${JSON.stringify(PROXY_BINDING_URL)};`], + [/from "shared-errors";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "shared-internal-auth";/, `from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)};`], ]); const tailWorkerRawSrc = applyModuleReplacements(readRepositoryFile("runtime/tail-worker.js"), [ diff --git a/tests/unit/runtime-wrapper-generate.test.js b/tests/unit/runtime-wrapper-generate.test.js new file mode 100644 index 0000000..8e50b43 --- /dev/null +++ b/tests/unit/runtime-wrapper-generate.test.js @@ -0,0 +1,48 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + generateAbortShimWrapperModule, + generateHostBindingWrapperModule, +} from "../../runtime/load/wrapper-generate.js"; + +function generatedWrappers() { + return { + abortOnly: generateAbortShimWrapperModule("worker.js"), + hostBindings: generateHostBindingWrapperModule("worker.js", [], [], [], {}, []), + }; +} + +/** + * @param {string} source + * @param {string} startMarker + * @param {string} endMarker + */ +function sourceFragment(source, startMarker, endMarker) { + const start = source.indexOf(startMarker); + assert.notEqual(start, -1, `missing generated source marker: ${startMarker}`); + const end = source.indexOf(endMarker, start); + assert.notEqual(end, -1, `missing generated source marker: ${endMarker}`); + return source.slice(start, end); +} + +test("generated wrapper flavors share the exact abort shim", () => { + const { abortOnly, hostBindings } = generatedWrappers(); + const start = "// Reserved name (WDL_RESERVED_ENTRYPOINT_RE)"; + const end = "\n\nexport class __WdlWorkflowNotify__"; + + assert.equal( + sourceFragment(abortOnly, start, end), + sourceFragment(hostBindings, start, end) + ); +}); + +test("generated wrapper flavors share default-export class detection", () => { + const { abortOnly, hostBindings } = generatedWrappers(); + const sourceLine = "const source = Function.prototype.toString.call(raw);"; + const classTest = "/^\\s*class\\b/.test(source)"; + + assert.equal(abortOnly.split(sourceLine).length - 1, 1); + assert.equal(hostBindings.split(sourceLine).length - 1, 1); + assert.match(abortOnly, new RegExp(`if \\(!${RegExp.escape(classTest)}\\)`)); + assert.match(hostBindings, new RegExp(`if \\(${RegExp.escape(classTest)}\\)`)); +}); diff --git a/tests/unit/s3-retry.test.js b/tests/unit/s3-retry.test.js new file mode 100644 index 0000000..91843a4 --- /dev/null +++ b/tests/unit/s3-retry.test.js @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + S3_TRANSIENT_RETRIES, + fetchRetryableS3Post, + isTransientS3Response, +} from "../../shared/s3-retry.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; + +test("S3 transient retry policy classifies throttling and server errors", () => { + assert.equal(S3_TRANSIENT_RETRIES, 10); + assert.equal(isTransientS3Response(new Response(null, { status: 429 })), true); + assert.equal(isTransientS3Response(new Response(null, { status: 503 })), true); + assert.equal(isTransientS3Response(new Response(null, { status: 408 })), false); + assert.equal(isTransientS3Response(new Response(null, { status: 400 })), false); +}); + +test("fetchRetryableS3Post retries transport and transient response failures", async () => { + let calls = 0; + const client = { + async fetch() { + calls += 1; + if (calls === 1) throw new Error("transport down"); + if (calls === 2) return new Response("slow down", { status: 500 }); + return new Response("ok"); + }, + }; + + await withMockedProperty(Math, "random", () => 0, async () => { + const response = await fetchRetryableS3Post(client, "https://s3.test/bucket?delete", { + method: "POST", + }); + assert.equal(await response.text(), "ok"); + }); + assert.equal(calls, 3); +}); diff --git a/tests/unit/secret-envelope.test.js b/tests/unit/secret-envelope.test.js index 954f47f..95c3f5c 100644 --- a/tests/unit/secret-envelope.test.js +++ b/tests/unit/secret-envelope.test.js @@ -7,14 +7,16 @@ import { encryptSecretValue, isSecretEnvelope, } from "../../shared/secret-envelope.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; +const SECRET_ENVELOPE_PARITY = readRepositoryJson("tests/fixtures/secret-envelope-parity.json"); const env = { - SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", - SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", + SECRET_ENVELOPE_LOCAL_KEY_B64: SECRET_ENVELOPE_PARITY.provider.localKeyB64, + SECRET_ENVELOPE_KID: SECRET_ENVELOPE_PARITY.provider.kid, }; -function deterministicRandomFactory() { - let next = 0; +function deterministicRandomFactory(start = 0) { + let next = start; return (/** @type {number} */ length) => { const out = new Uint8Array(length); for (let i = 0; i < length; i++) out[i] = next++ & 0xff; @@ -22,70 +24,42 @@ function deterministicRandomFactory() { }; } -test("secret envelope encrypts and decrypts with storage-location AAD", async () => { - const envelope = await encryptSecretValue("sensitive-value", { - env, - hashKey: "secrets:demo:api", - fieldName: "TOKEN", - random: deterministicRandomFactory(), - }); - - assert.equal(isSecretEnvelope(envelope), true); - assert.ok(envelope.startsWith(SECRET_ENVELOPE_PREFIX)); - assert.equal(envelope.includes("sensitive-value"), false); - assert.equal( - await decryptSecretValue(envelope, { - env, - hashKey: "secrets:demo:api", - fieldName: "TOKEN", - }), - "sensitive-value" - ); -}); - -test("secret envelope preserves empty string secrets", async () => { - const envelope = await encryptSecretValue("", { - env, - hashKey: "secrets:demo:api", - fieldName: "EMPTY", - random: deterministicRandomFactory(), - }); - - assert.equal(isSecretEnvelope(envelope), true); - assert.equal( - await decryptSecretValue(envelope, { +test("secret envelope generation and decrypt match the shared parity vectors", async () => { + for (const vector of SECRET_ENVELOPE_PARITY.vectors) { + const envelope = await encryptSecretValue(vector.plaintext, { env, - hashKey: "secrets:demo:api", - fieldName: "EMPTY", - }), - "" - ); + hashKey: vector.hashKey, + fieldName: vector.fieldName, + random: deterministicRandomFactory(vector.randomStart), + }); + + assert.equal(envelope, vector.envelope, vector.name); + assert.equal(isSecretEnvelope(envelope), true, vector.name); + if (vector.plaintext !== "") assert.equal(envelope.includes(vector.plaintext), false, vector.name); + assert.equal( + await decryptSecretValue(envelope, { + env, + hashKey: vector.hashKey, + fieldName: vector.fieldName, + }), + vector.plaintext, + vector.name + ); + } }); -test("secret envelope rejects copy to another hash or field", async () => { - const envelope = await encryptSecretValue("value", { - env, - hashKey: "secrets:demo", - fieldName: "TOKEN", - random: deterministicRandomFactory(), - }); - - await assert.rejects( - decryptSecretValue(envelope, { - env, - hashKey: "secrets:other", - fieldName: "TOKEN", - }), - { code: "secret_decrypt_failed" } - ); - await assert.rejects( - decryptSecretValue(envelope, { - env, - hashKey: "secrets:demo", - fieldName: "OTHER", - }), - { code: "secret_decrypt_failed" } - ); +test("secret envelope rejection behavior matches the shared parity vectors", async () => { + for (const rejection of SECRET_ENVELOPE_PARITY.rejections) { + await assert.rejects( + decryptSecretValue(rejection.envelope, { + env: { ...env, SECRET_ENVELOPE_KID: rejection.configuredKid }, + hashKey: rejection.hashKey, + fieldName: rejection.fieldName, + }), + { code: rejection.jsErrorCode }, + rejection.name + ); + } }); test("secret envelope requires explicit local provider configuration", async () => { @@ -110,24 +84,6 @@ test("secret envelope rejects unprefixed values", async () => { ); }); -test("secret envelope rejects unknown kid", async () => { - const envelope = await encryptSecretValue("value", { - env, - hashKey: "secrets:demo", - fieldName: "TOKEN", - random: deterministicRandomFactory(), - }); - - await assert.rejects( - decryptSecretValue(envelope, { - env: { ...env, SECRET_ENVELOPE_KID: "local:test:secret-envelope:v2" }, - hashKey: "secrets:demo", - fieldName: "TOKEN", - }), - { code: "unknown_kid" } - ); -}); - test("secret envelope reports invalid local provider key as configuration error", async () => { await assert.rejects( encryptSecretValue("value", { diff --git a/tests/unit/stream-prefix.test.js b/tests/unit/stream-prefix.test.js new file mode 100644 index 0000000..b2fd8d1 --- /dev/null +++ b/tests/unit/stream-prefix.test.js @@ -0,0 +1,25 @@ +import { once } from "node:events"; +import { PassThrough, Writable } from "node:stream"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { pipeWithPrefix } from "../../scripts/_stream-prefix.js"; + +test("pipeWithPrefix prefixes complete lines and flushes a partial tail", async () => { + const source = new PassThrough(); + /** @type {string[]} */ + const chunks = []; + const destination = new Writable({ + write(chunk, _encoding, callback) { + chunks.push(String(chunk)); + callback(); + }, + }); + pipeWithPrefix(source, destination, "[task] "); + + source.write("first"); + source.write(" line\nsecond\n"); + source.end("tail"); + await once(source, "end"); + + assert.equal(chunks.join(""), "[task] first line\n[task] second\n[task] tail\n"); +}); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 1b1bc65..209a419 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -2,9 +2,10 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import path from "node:path"; import { - d1RuntimeServiceRegex, extractAssignedConstant, extractBraceBlock, + extractCapnpConstBlock, + extractCapnpListEntries, extractExportedStringConst, extractRegex, extractStringConst, @@ -13,8 +14,11 @@ import { markdownFiles, objectJsonPayloads, scannedTestFiles, - serviceAnchorRegex, withoutStringAndTemplateLiterals, + yamlAliasCount, + yamlDocument, + yamlDocuments, + yamlMergeAliases, } from "../helpers/style-contract-scanner.js"; import { jsFiles, readRepoFile, rustFiles, sourceFiles, withoutLineComments } from "../helpers/source-scan.js"; @@ -38,6 +42,7 @@ const PRODUCTION_JS_FILES = [ const PLATFORM_HTTP_ERROR_FILES = [ // auth/ is a socket-less JSRPC worker; it returns method payloads instead of // platform HTTP responses. + ...CONTROL_FILES, ...GATEWAY_FILES, ...RUNTIME_FILES, ...D1_RUNTIME_FILES, @@ -52,6 +57,17 @@ const OFFICIAL_DOC_FILES = [ // one-language exception is approved. const INTENTIONAL_ONE_LANGUAGE_ACTIVE_DOCS = new Set(); +function activeBilingualDocFiles() { + const notesDirName = ["arch", "ive"].join(""); + return [ + "README.md", + "README.zh.md", + ...markdownFiles("docs", { ignoreDirs: new Set([notesDirName]) }), + "deploy/kubernetes/README.md", + "deploy/kubernetes/README.zh.md", + ]; +} + // Heuristic multiline scanners for object-literal JSON payload helpers. // They match a first object argument and an optional second object argument // (init/options) without attempting full nested-brace parsing. @@ -312,6 +328,7 @@ test("route/version registry keys go through shared/version.js helpers", () => { // and comments are skipped. const files = [ ...CONTROL_FILES, + ...AUTH_FILES, ...GATEWAY_FILES, ...DO_RUNTIME_FILES, ...RUNTIME_FILES, @@ -327,6 +344,18 @@ test("route/version registry keys go through shared/version.js helpers", () => { if (/`worker:do-storage:\$\{/.test(source)) { offenders.push(`${file}: inline worker:do-storage: — use doStorageIdKey(ns, worker)`); } + if (/`hosts:\$\{/.test(source)) offenders.push(`${file}: inline hosts: — use hostsKey(ns)`); + if (/`ns-hosts:\$\{/.test(source)) offenders.push(`${file}: inline ns-hosts: — use nsHostsKey(ns)`); + if (/`worker:\$\{[^`]*:next_version`/.test(source)) { + offenders.push(`${file}: inline next_version counter — use nextVersionKey(ns, worker)`); + } + if (/"namespaces"/.test(source)) offenders.push(`${file}: inline namespaces set — use NAMESPACES_KEY`); + if (/"declared-hosts"/.test(source)) { + offenders.push(`${file}: inline declared-hosts set — use DECLARED_HOSTS_KEY`); + } + if (/"host-declarations:"/.test(source)) { + offenders.push(`${file}: inline host-declarations prefix — use hostDeclarationsKey(host)`); + } } // Rust services are production readers too; rust/common/version.rs holds the // only literals (mirror of shared/version.js), every other crate must call @@ -347,6 +376,21 @@ test("route/version registry keys go through shared/version.js helpers", () => { assert.deepEqual(offenders, [], `route/version key literals must use shared helpers:\n${offenders.join("\n")}`); }); +test("platform domain configuration uses the shared normalized owner", () => { + const owner = withoutLineComments(readRepoFile("shared/ns-pattern.js")); + assert.match(owner, /DEFAULT_PLATFORM_DOMAIN = "workers\.local"/); + for (const file of [ + "control/handlers/deploy.js", + "control/handlers/promote.js", + "control/handlers/hosts.js", + "gateway/index.js", + ]) { + const source = withoutLineComments(readRepoFile(file)); + assert.match(source, /\bplatformDomainFromEnv\b/, file); + assert.doesNotMatch(source, /"workers\.local"/, file); + } +}); + test("secret Redis key literals stay aligned across JS and redis-proxy", () => { const js = readRepoFile("shared/secret-keys.js"); const rust = readRepoFile("rust/redis-proxy/src/runtime.rs"); @@ -357,34 +401,6 @@ test("secret Redis key literals stay aligned across JS and redis-proxy", () => { assert.match(rust, /format!\("secrets:\{\}:\{\}", q\.ns, q\.worker\)/); }); -test("test bundleKey stubs stay production-faithful", () => { - const offenders = []; - const bundleKeyStubBodies = (/** @type {string} */ source) => { - const bodies = []; - for (const match of source.matchAll(/export function bundleKey\(|const bundleKey\s*=/g)) { - const start = match.index ?? 0; - const nextExport = source.indexOf("\nexport function ", start + 1); - const nextConst = source.indexOf("\nconst ", start + 1); - const nextBoundary = [nextExport, nextConst] - .filter((idx) => idx !== -1) - .sort((a, b) => a - b)[0] ?? source.length; - bodies.push(source.slice(start, nextBoundary)); - } - return bodies; - }; - for (const file of jsFiles("tests/unit")) { - const source = withoutLineComments(readRepoFile(file)); - for (const body of bundleKeyStubBodies(source)) { - const hasVersionSegmentComposition = - /":v:"\s*\+\s*n/.test(body) || /:v:\$\{?\s*n\s*\}?/.test(body); - if (!/\bparseVersion\(version\)/.test(body) || !hasVersionSegmentComposition) { - offenders.push(`${file}: bundleKey stub must parse vN and compose worker:::v:`); - } - } - } - assert.deepEqual(offenders, []); -}); - test("worker delete lock key stays aligned across control and workflows", () => { const controlLib = withoutLineComments(readRepoFile("control/lib.js")); const controlShared = withoutLineComments(readRepoFile("control/shared.js")); @@ -914,17 +930,19 @@ test("route invalidation channel literals stay aligned across control and gatewa }); test("declared-host Redis key literals stay aligned across control, gateway, and docs", () => { + const version = withoutLineComments(readRepoFile("shared/version.js")); const shared = withoutLineComments(readRepoFile("control/shared.js")); const routing = withoutLineComments(readRepoFile("control/routing.js")); const gateway = withoutLineComments(readRepoFile("gateway/runtime.js")); const layoutEn = readRepoFile("docs/redis-key-layout.md"); const layoutZh = readRepoFile("docs/redis-key-layout.zh.md"); - const declaredHostsKey = extractStringConst(shared, "DECLARED_HOSTS_KEY"); - const hostDeclarationsPrefix = extractStringConst(shared, "HOST_DECLARATIONS_PREFIX"); + const declaredHostsKey = extractStringConst(version, "DECLARED_HOSTS_KEY"); + const hostDeclarationsPrefix = extractStringConst(version, "HOST_DECLARATIONS_PREFIX"); assert.match(routing, new RegExp(`\\bDECLARED_HOSTS_KEY\\b`)); - assert.match(routing, new RegExp(`\\bHOST_DECLARATIONS_PREFIX\\b`)); - assert.equal(extractStringConst(gateway, "DECLARED_HOSTS_KEY"), declaredHostsKey); + assert.match(routing, /\bhostDeclarationsKey\b/); + assert.match(shared, /\bhostDeclarationsKey\b/); + assert.match(gateway, /\bDECLARED_HOSTS_KEY\b/); for (const source of [layoutEn, layoutZh]) { assert.match(source, new RegExp(`\\b${RegExp.escape(declaredHostsKey)}\\b`)); assert.match(source, new RegExp(`\\b${RegExp.escape(hostDeclarationsPrefix)}`)); @@ -932,12 +950,13 @@ test("declared-host Redis key literals stay aligned across control, gateway, and }); test("gateway strips every client-supplied platform header it injects", () => { - const source = withoutLineComments(readRepoFile("gateway/index.js")); - const headerList = /const INTERNAL_FORWARD_HEADERS = \[([\s\S]*?)\];/.exec(source); + const gateway = withoutLineComments(readRepoFile("gateway/index.js")); + const owner = withoutLineComments(readRepoFile("gateway/lib.js")); + const headerList = /const INTERNAL_FORWARD_HEADERS = \[([\s\S]*?)\];/.exec(owner); assert.ok(headerList, "gateway INTERNAL_FORWARD_HEADERS list must be present"); const stripped = new Set([...headerList[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])); const injectedNonPrefixedInternal = new Set( - [...source.matchAll(/forwardRequest\.headers\.set\("(x-worker[^"]+)"/g)] + [...gateway.matchAll(/forwardRequest\.headers\.set\("(x-worker[^"]+)"/g)] .map((match) => match[1]) ); const missing = [...injectedNonPrefixedInternal].filter((header) => !stripped.has(header)).toSorted(); @@ -946,22 +965,13 @@ test("gateway strips every client-supplied platform header it injects", () => { [], `gateway INTERNAL_FORWARD_HEADERS must include injected non-prefixed internal headers: ${missing.join(", ")}` ); - assert.match(source, /const INTERNAL_HEADER_PREFIX = "x-wdl-"/); - assert.match(source, /header\.toLowerCase\(\)\.startsWith\(INTERNAL_HEADER_PREFIX\)/); - assert.match(source, /headers\.delete\(header\)/); + assert.match(owner, /const INTERNAL_HEADER_PREFIX = "x-wdl-"/); + assert.match(owner, /name\.toLowerCase\(\)\.startsWith\(INTERNAL_HEADER_PREFIX\)/); + assert.match(owner, /headers\.delete\(name\)/); + assert.match(gateway, /deleteGatewayInternalHeaders\(forwardRequest\.headers\)/); const websocket = withoutLineComments(readRepoFile("gateway/websocket.js")); - const websocketHeaderList = /const INTERNAL_FORWARD_HEADERS = \[([\s\S]*?)\];/.exec(websocket); - assert.ok(websocketHeaderList, "gateway websocket INTERNAL_FORWARD_HEADERS list must be present"); - const websocketStripped = new Set([...websocketHeaderList[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])); - assert.deepEqual( - ["x-worker-id", "x-worker-prefix"].filter((header) => !websocketStripped.has(header)), - [], - "gateway websocket must strip non-prefixed internal headers from upgrade responses" - ); - assert.match(websocket, /const INTERNAL_HEADER_PREFIX = "x-wdl-"/); - assert.match(websocket, /name\.toLowerCase\(\)\.startsWith\(INTERNAL_HEADER_PREFIX\)/); - assert.match(websocket, /out\.delete\(name\)/); + assert.match(websocket, /deleteGatewayInternalHeaders\(out\)/); }); test("D1 owner protocol errors stay aligned with runtime stale-hint handling", () => { @@ -1181,12 +1191,7 @@ test("active docs and sources do not point at note paths", () => { }); test("active bilingual docs stay paired", () => { - const notesDirName = ["arch", "ive"].join(""); - const files = [ - "README.md", - "README.zh.md", - ...markdownFiles("docs", { ignoreDirs: new Set([notesDirName]) }), - ]; + const files = activeBilingualDocFiles(); const fileSet = new Set(files); const offenders = []; for (const file of files) { @@ -1202,6 +1207,44 @@ test("active bilingual docs stay paired", () => { assert.deepEqual(offenders.toSorted(), []); }); +test("active bilingual docs keep normative protocol tokens aligned", () => { + const files = activeBilingualDocFiles(); + const fileSet = new Set(files); + const tokenPatterns = [ + /\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b/g, + /\bx-[a-z0-9]+(?:-[a-z0-9]+)+\b/g, + /:\d{2,5}\b/g, + /\bDB\s*[0-9]+\b/gi, + ]; + const tokens = (/** @type {string} */ file) => { + const found = new Set(); + const source = readRepoFile(file); + for (const pattern of tokenPatterns) { + for (const match of source.matchAll(pattern)) { + found.add(match[0].toUpperCase().replace(/\s+/g, "")); + } + } + return found; + }; + const offenders = []; + + for (const english of files.filter((file) => file.endsWith(".md") && !file.endsWith(".zh.md"))) { + if (INTENTIONAL_ONE_LANGUAGE_ACTIVE_DOCS.has(english)) continue; + const chinese = english.replace(/\.md$/, ".zh.md"); + if (!fileSet.has(chinese)) continue; + const englishTokens = tokens(english); + const chineseTokens = tokens(chinese); + for (const token of englishTokens) { + if (!chineseTokens.has(token)) offenders.push(`${chinese}: missing ${token}`); + } + for (const token of chineseTokens) { + if (!englishTokens.has(token)) offenders.push(`${english}: missing ${token}`); + } + } + + assert.deepEqual(offenders.toSorted(), []); +}); + test("protocol and contributor contracts stay discoverable from active docs", () => { const links = [ ["CLAUDE.md", "docs/protocol-contracts.md"], @@ -1438,26 +1481,157 @@ test("owner forward metrics classify non-error HTTP statuses consistently", () = }); test("local compose services inherit shared image anchors", () => { - const source = readRepoFile("docker-compose.yml"); - const offenders = []; - for (const service of ["redis-proxy-user", "redis-proxy-system", "scheduler"]) { - if (!serviceAnchorRegex(service, "rust-sidecar-image").test(source)) { - offenders.push(service); - } + const document = yamlDocument("docker-compose.yml"); + const merged = /** @type {{ services: Record }} */ ( + yamlDocument("docker-compose.yml", { merge: true }).toJS() + ); + assert.deepEqual(yamlMergeAliases(document, ["x-redis-proxy-service"]), ["rust-sidecar-image"]); + assert.deepEqual(yamlMergeAliases(document, ["x-d1-runtime-service"]), ["workerd-image"]); + assert.deepEqual(yamlMergeAliases(document, ["x-do-runtime-service"]), ["workerd-image"]); + assert.deepEqual( + yamlMergeAliases(document, ["x-redis-proxy-service", "environment"]), + ["internal-auth-env", "secret-envelope-env"] + ); + assert.deepEqual( + yamlMergeAliases(document, ["services", "system-runtime", "environment"]), + ["internal-auth-env", "secret-envelope-env"] + ); + assert.equal(yamlAliasCount(document, "secret-envelope-env"), 2); + + /** @type {Map} */ + const serviceAnchors = new Map(); + for (const service of ["redis-proxy-user", "redis-proxy-system", "redis-proxy-do"]) { + serviceAnchors.set(service, "redis-proxy-service"); } + for (const service of ["scheduler", "workflows"]) serviceAnchors.set(service, "rust-sidecar-image"); for (const service of ["user-runtime", "system-runtime", "gateway"]) { - if (!serviceAnchorRegex(service, "workerd-image").test(source)) { - offenders.push(service); - } + serviceAnchors.set(service, "workerd-image"); } - // Strict by design: d1-multi profile filters stay before anchor inheritance - // so profile-only variants remain visually distinct from the default task. for (const service of ["d1-runtime", "d1-runtime-a", "d1-runtime-b", "d1-runtime-c"]) { - if (!d1RuntimeServiceRegex(service).test(source)) { - offenders.push(service); + serviceAnchors.set(service, "d1-runtime-service"); + } + for (const service of ["do-runtime", "do-runtime-a", "do-runtime-b", "do-runtime-c"]) { + serviceAnchors.set(service, "do-runtime-service"); + } + for (const [service, anchor] of serviceAnchors) { + assert.deepEqual(yamlMergeAliases(document, ["services", service]), [anchor], service); + } + for (const family of ["d1", "do"]) { + assert.equal(merged.services[`${family}-runtime`].profiles, undefined); + for (const suffix of ["a", "b", "c"]) { + assert.deepEqual(merged.services[`${family}-runtime-${suffix}`].profiles, [`${family}-multi`]); } } - assert.deepEqual(offenders, []); +}); + +test("published Compose overrides cover every locally built image service", () => { + const local = /** @type {{ + * services: Record, + * }} */ (yamlDocument("docker-compose.yml", { merge: true }).toJS()); + const publishedDocument = yamlDocument("docker-compose.images.yml"); + const published = /** @type {{ + * services: Record, + * }} */ (yamlDocument("docker-compose.images.yml", { merge: true }).toJS()); + const familyByDockerfile = new Map([ + ["Dockerfile.rust", { alias: "rust-published", image: "docker.io/getwdl/wdl-rust:latest" }], + ["Dockerfile.workerd", { alias: "workerd-published", image: "docker.io/getwdl/wdl-workerd:latest" }], + ]); + const expected = new Map(); + for (const [service, config] of Object.entries(local.services)) { + if (!config.build) continue; + assert.equal(config.build.context, ".", `${service} build context`); + const family = familyByDockerfile.get(config.build.dockerfile || ""); + assert.ok(family, `${service} must use a recognized shared Dockerfile`); + expected.set(service, family); + } + + assert.deepEqual(Object.keys(published.services).toSorted(), [...expected.keys()].toSorted()); + for (const [service, family] of expected) { + assert.deepEqual(yamlMergeAliases(publishedDocument, ["services", service]), [family.alias], service); + assert.deepEqual(published.services[service], { + image: family.image, + pull_policy: "always", + }); + } +}); + +test("Kubernetes runtime families share the exact redis-proxy sidecar spec", () => { + const files = [ + "deploy/kubernetes/base/user-runtime.yaml", + "deploy/kubernetes/base/system-runtime.yaml", + "deploy/kubernetes/base/do-runtime.yaml", + ]; + const sidecars = files.map((file) => { + const resources = /** @type {Array<{ + * kind?: string, + * spec?: { template?: { spec?: { containers?: Array> } } }, + * }>} */ (yamlDocuments(file).map((document) => document.toJS())); + const workload = resources.find((resource) => + resource.kind === "Deployment" || resource.kind === "StatefulSet" + ); + assert.ok(workload, `${file} must contain a Deployment or StatefulSet`); + const sidecar = workload.spec?.template?.spec?.containers + ?.find((container) => container.name === "redis-proxy"); + assert.ok(sidecar, `${file} must contain the redis-proxy sidecar`); + return sidecar; + }); + + assert.deepEqual(sidecars[1], sidecars[0], `${files[1]} redis-proxy sidecar drifted`); + assert.deepEqual(sidecars[2], sidecars[0], `${files[2]} redis-proxy sidecar drifted`); +}); + +test("Kubernetes NetworkPolicies pin the per-component ingress matrix", () => { + const documents = yamlDocuments("deploy/kubernetes/base/network-policy.yaml"); + /** @type {Record} */ + const actual = {}; + for (const document of documents) { + const policy = /** @type {{ + * spec?: { + * podSelector?: { matchLabels?: Record }, + * ingress?: Array<{ + * from?: Array<{ podSelector?: { matchLabels?: Record } }>, + * ports?: Array<{ port?: unknown }>, + * }>, + * }, + * }} */ (document.toJS()); + const receiver = policy?.spec?.podSelector?.matchLabels?.["app.kubernetes.io/component"]; + if (typeof receiver !== "string") continue; + assert.equal(actual[receiver], undefined, `duplicate NetworkPolicy receiver ${receiver}`); + + const edges = []; + for (const ingress of policy.spec?.ingress || []) { + const rawCallers = ingress.from === undefined + ? ["*"] + : ingress.from.map((peer) => peer?.podSelector?.matchLabels?.["app.kubernetes.io/component"]); + const ports = (ingress.ports || []).map((entry) => Number(entry.port)); + assert.ok(rawCallers.every((caller) => typeof caller === "string"), `${receiver} callers must use component selectors`); + assert.ok(ports.every(Number.isInteger), `${receiver} ingress ports must be integers`); + const callers = /** @type {string[]} */ (rawCallers); + for (const caller of callers) { + for (const port of ports) edges.push(`${caller}:${port}`); + } + } + actual[receiver] = edges.toSorted(); + } + + assert.deepEqual(actual, { + gateway: ["*:8080"], + "user-runtime": ["gateway:8081", "scheduler:8088", "workflows:8088"], + "system-runtime": ["gateway:8081", "gateway:8082", "scheduler:8088", "workflows:8088"], + "d1-runtime": ["d1-runtime:8787", "do-runtime:8787", "system-runtime:8787", "user-runtime:8787"], + "do-runtime": ["do-runtime:8788", "system-runtime:8788", "user-runtime:8788", "workflows:8788"], + workflows: ["do-runtime:9120", "scheduler:9120", "system-runtime:9120", "user-runtime:9120"], + valkey: [ + "d1-runtime:6379", + "do-runtime:6379", + "gateway:6379", + "scheduler:6379", + "system-runtime:6379", + "user-runtime:6379", + "workflows:6379", + ], + s3mock: ["do-runtime:9090", "system-runtime:9090", "user-runtime:9090"], + }); }); test("local compose routes private HTTP hops through Envoy only", () => { @@ -1626,6 +1800,126 @@ test("local compose routes private HTTP hops through Envoy only", () => { assert.ok((envoy.match(/preserve_external_request_id: true/g) || []).length >= 5); }); +test("user and system runtime worker contracts differ only at privilege fields", () => { + const user = readRepoFile("runtime/config-user.capnp"); + const system = readRepoFile("runtime/config-system.capnp"); + + const entryMap = (/** @type {string[]} */ entries) => Object.fromEntries( + entries.map((entry) => { + const name = /\(name\s*=\s*"([^"]+)"/.exec(entry); + assert.ok(name, `Cap'n Proto tuple must have a name: ${entry}`); + return [name[1], entry]; + }).toSorted(([left], [right]) => left.localeCompare(right)) + ); + const stringField = (/** @type {string} */ block, /** @type {string} */ field) => { + const match = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*"([^"]+)"`).exec(block); + assert.ok(match, `Cap'n Proto field ${field} must be present`); + return match[1]; + }; + const listField = (/** @type {string} */ block, /** @type {string} */ field) => { + const match = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*(\\[[^\\]]*\\])`).exec(block); + assert.ok(match, `Cap'n Proto field ${field} must be present`); + return match[1].replace(/\s+/g, ""); + }; + const withoutEntries = (/** @type {Record} */ entries, /** @type {string[]} */ names) => Object.fromEntries( + Object.entries(entries).filter(([name]) => !names.includes(name)) + ); + + const userLoader = extractCapnpConstBlock(user, "loaderWorker"); + const systemLoader = extractCapnpConstBlock(system, "loaderWorker"); + assert.deepEqual( + entryMap(extractCapnpListEntries(userLoader, "modules")), + entryMap(extractCapnpListEntries(systemLoader, "modules")) + ); + assert.equal(stringField(userLoader, "compatibilityDate"), stringField(systemLoader, "compatibilityDate")); + assert.equal(listField(userLoader, "compatibilityFlags"), listField(systemLoader, "compatibilityFlags")); + + const userBindings = entryMap(extractCapnpListEntries(userLoader, "bindings")); + const systemBindings = entryMap(extractCapnpListEntries(systemLoader, "bindings")); + assert.deepEqual( + withoutEntries(userBindings, ["SERVICE_NAME", "PUBLIC_NETWORK"]), + withoutEntries(systemBindings, ["SERVICE_NAME", "PUBLIC_NETWORK"]) + ); + assert.match(userBindings.SERVICE_NAME, /text = "user-runtime"/); + assert.match(systemBindings.SERVICE_NAME, /text = "system-runtime"/); + assert.match(userBindings.PUBLIC_NETWORK, /service = "public-network"/); + assert.match(systemBindings.PUBLIC_NETWORK, /service = "network"/); + assert.equal(stringField(userLoader, "globalOutbound"), "internal-network"); + assert.equal(stringField(systemLoader, "globalOutbound"), "network"); + + const userOwner = extractCapnpConstBlock(user, "doOwnerNetworkWorker"); + const systemOwner = extractCapnpConstBlock(system, "doOwnerNetworkWorker"); + assert.deepEqual( + entryMap(extractCapnpListEntries(userOwner, "modules")), + entryMap(extractCapnpListEntries(systemOwner, "modules")) + ); + assert.deepEqual( + entryMap(extractCapnpListEntries(userOwner, "bindings")), + entryMap(extractCapnpListEntries(systemOwner, "bindings")) + ); + assert.equal(stringField(userOwner, "compatibilityDate"), stringField(systemOwner, "compatibilityDate")); + assert.equal(stringField(userOwner, "globalOutbound"), "internal-network"); + assert.equal(stringField(systemOwner, "globalOutbound"), "network"); + + const userTail = extractCapnpConstBlock(user, "tailWorker"); + const systemTail = extractCapnpConstBlock(system, "tailWorker"); + assert.deepEqual( + entryMap(extractCapnpListEntries(userTail, "modules")), + entryMap(extractCapnpListEntries(systemTail, "modules")) + ); + const userTailBindings = entryMap(extractCapnpListEntries(userTail, "bindings")); + const systemTailBindings = entryMap(extractCapnpListEntries(systemTail, "bindings")); + assert.deepEqual( + withoutEntries(userTailBindings, ["SERVICE_NAME"]), + withoutEntries(systemTailBindings, ["SERVICE_NAME"]) + ); + assert.match(userTailBindings.SERVICE_NAME, /text = "user-runtime-tail"/); + assert.match(systemTailBindings.SERVICE_NAME, /text = "system-runtime-tail"/); + assert.equal(stringField(userTail, "compatibilityDate"), stringField(systemTail, "compatibilityDate")); + assert.equal(stringField(userTail, "globalOutbound"), "internal-network"); + assert.equal(stringField(systemTail, "globalOutbound"), "network"); +}); + +test("local workerd configs preserve base service and socket topology", () => { + const pairs = [ + ["gateway/config.capnp", "gateway/config-local.capnp"], + ["runtime/config-user.capnp", "runtime/config-user-local.capnp"], + ["runtime/config-system.capnp", "runtime/config-system-local.capnp"], + ["do-runtime/config.capnp", "do-runtime/config-local.capnp"], + ]; + const entryMap = (/** @type {string[]} */ entries) => Object.fromEntries( + entries.map((entry) => { + const name = /\(name\s*=\s*"([^"]+)"/.exec(entry); + assert.ok(name, `Cap'n Proto tuple must have a name: ${entry}`); + return [name[1], entry]; + }).toSorted(([left], [right]) => left.localeCompare(right)) + ); + const normalizeBaseReference = (/** @type {string} */ entry) => entry.replace(/\.Base\./g, "."); + + for (const [baseFile, localFile] of pairs) { + const base = extractCapnpConstBlock(readRepoFile(baseFile), "config"); + const local = extractCapnpConstBlock(readRepoFile(localFile), "config"); + const baseServices = entryMap(extractCapnpListEntries(base, "services")); + const localServices = entryMap(extractCapnpListEntries(local, "services")); + assert.deepEqual(Object.keys(localServices), Object.keys(baseServices), `${localFile} service names`); + + for (const [name, baseService] of Object.entries(baseServices)) { + const localService = localServices[name]; + if (baseService.includes("external =")) { + assert.match(localService, /external = \(address = "[^"]+", http = \(\)\)/, `${localFile}:${name}`); + } else { + assert.equal(normalizeBaseReference(localService), baseService, `${localFile}:${name}`); + } + } + + assert.deepEqual( + entryMap(extractCapnpListEntries(local, "sockets")), + entryMap(extractCapnpListEntries(base, "sockets")), + `${localFile} sockets` + ); + } +}); + test("S3 query encoding stays aligned between shared and injected runtime helpers", () => { const extract = (/** @type {string} */ file) => { const source = readRepoFile(file); @@ -1733,9 +2027,9 @@ test("workflow instance state is owned by workflows DB2", () => { "tests/integration/workflows-runtime-core.test.js", "tests/integration/workflows-runtime-scheduler.test.js", "tests/integration/workflows-runtime-pausing.test.js", - "tests/integration/workflows-runtime-retention.test.js", "tests/unit/style-contracts.test.js", "rust/workflows/src/keys.rs", + "rust/workflows/src/schema.rs", "rust/workflows/src/tests.rs", ]); const allowedFilePrefixes = new Set([ @@ -1755,6 +2049,7 @@ test("workflow instance state is owned by workflows DB2", () => { ...jsFiles("do-runtime"), ...jsFiles("tests/unit"), ...jsFiles("tests/integration"), + ...rustFiles("rust"), ]; const offenders = []; for (const file of files) { @@ -1820,7 +2115,7 @@ test("cron discovery index literals stay aligned across scheduler and control", assert.ok(control.includes(workerIndex), `control cron indexes must contain ${workerIndex}`); assert.ok(scheduler.includes(workerIndex), `scheduler cron indexes must contain ${workerIndex}`); assert.ok(integration.includes(workerIndex), `cron integration tests must contain ${workerIndex}`); - assert.ok(routing.includes("stageCronSlotRef"), "control routing should stage cron refs through the shared helper"); + assert.ok(routing.includes("stageCronProjection"), "control routing should stage the scheduler projection through its owner"); assert.equal(/`crons:\$\{/.test(routing), false, "control routing should not hand-roll cron hash keys"); assert.equal(/`cron-slot:\$\{/.test(routing), false, "control routing should not hand-roll cron slot keys"); @@ -2171,12 +2466,10 @@ test("DO invoke request-size caps stay aligned across client and server", () => test("internal binary protocol content-types stay centralized in transport constants", () => { const files = [ - ...jsFiles("shared"), - ...jsFiles("runtime"), - ...jsFiles("d1-runtime"), - ...jsFiles("do-runtime"), + ...PRODUCTION_JS_FILES, ...jsFiles("tests/unit"), ...jsFiles("tests/integration"), + ...rustFiles("rust"), ]; const literals = [ "application/vnd.wdl.d1-query", @@ -2191,6 +2484,7 @@ test("internal binary protocol content-types stay centralized in transport const "d1-runtime/protocol.js", "runtime/_wdl-do-transport.js", "do-runtime/protocol.js", + "rust/redis-proxy/src/runtime.rs", "tests/unit/runtime-load.test.js", "tests/unit/style-contracts.test.js", ]); @@ -2301,6 +2595,7 @@ test("queue delay cap literals stay aligned across standalone tiers", () => { test("Docker images build from pinned public base images", () => { const workerdDockerfile = withoutLineComments(readRepoFile("Dockerfile.workerd")); const rustDockerfile = withoutLineComments(readRepoFile("Dockerfile.rust")); + const cargoWorkspace = withoutLineComments(readRepoFile("rust/Cargo.toml")); const integrationEnvironment = withoutLineComments(readRepoFile("scripts/integration-environment.js")); assert.match(workerdDockerfile, /FROM rust:1-alpine AS supervisor-build/); @@ -2317,6 +2612,54 @@ test("Docker images build from pinned public base images", () => { assert.doesNotMatch(workerdDockerfile, /\bwget\b/); assert.doesNotMatch(workerdDockerfile, /^FROM\s+\S+:latest\b/m); assert.match(integrationEnvironment, /DOCKER_COMPOSE_BUILD_ARGS = \["compose", "build", "gateway", "workflows"\]/); + + const workspaceMembersMatch = /\bmembers\s*=\s*\[([\s\S]*?)\]/.exec(cargoWorkspace); + assert.ok(workspaceMembersMatch, "rust/Cargo.toml must declare workspace members"); + const workspaceMembers = [...workspaceMembersMatch[1].matchAll(/"([^"]+)"/g)] + .map((match) => match[1]) + .toSorted(); + const copiedManifests = (/** @type {string} */ source) => { + const copies = [...source.matchAll(/^COPY rust\/([^/\s]+)\/Cargo\.toml \.\/([^/\s]+)\/$/gm)]; + for (const copy of copies) { + assert.equal(copy[2], copy[1], `Cargo manifest destination must preserve ${copy[1]}`); + } + return copies.map((copy) => copy[1]).toSorted(); + }; + assert.deepEqual(copiedManifests(rustDockerfile), workspaceMembers); + assert.deepEqual(copiedManifests(workerdDockerfile), workspaceMembers); + + const imageLabels = (/** @type {string} */ source) => Object.fromEntries( + [...source.matchAll(/org\.opencontainers\.image\.([a-z]+)="([^"]*)"/g)] + .map((match) => [match[1], match[2]]) + ); + const rustLabels = imageLabels(rustDockerfile); + const workerdLabels = imageLabels(workerdDockerfile); + const sharedLabelKeys = [ + "source", + "documentation", + "vendor", + "authors", + "licenses", + "version", + "revision", + "created", + ]; + assert.deepEqual( + Object.fromEntries(sharedLabelKeys.map((key) => [key, rustLabels[key]])), + Object.fromEntries(sharedLabelKeys.map((key) => [key, workerdLabels[key]])) + ); + assert.notEqual(rustLabels.title, workerdLabels.title); + assert.notEqual(rustLabels.description, workerdLabels.description); + + for (const source of [rustDockerfile, workerdDockerfile]) { + for (const arg of ["WDL_VERSION", "VCS_REF", "BUILD_DATE"]) { + assert.equal( + [...source.matchAll(new RegExp(`^ARG ${arg}(?:=[^\\n]+)?$`, "gm"))].length, + 2, + `${arg} must be declared globally and in the final image stage` + ); + } + } }); test("workerd deploy configs use the supervisor-owned health check binary", () => { diff --git a/tests/unit/test-helper-style-contracts.test.js b/tests/unit/test-helper-style-contracts.test.js index 68ed4cf..458244a 100644 --- a/tests/unit/test-helper-style-contracts.test.js +++ b/tests/unit/test-helper-style-contracts.test.js @@ -34,11 +34,7 @@ test("test files use moduleDataUrl/repositoryFileUrl instead of raw fs+URL boile }); test("test module source rewrites go through load-shared-module helpers", () => { - const EXEMPT = new Set([ - "tests/helpers/load-shared-module.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT); + const testFiles = scannedTestFiles(); /** @type {string[]} */ const offenders = []; const sourceReader = String.raw`(?:readFileSync|readRepositoryFile|readRepositoryModuleSource)`; @@ -87,8 +83,7 @@ test("test module source rewrites go through load-shared-module helpers", () => }); test("test global mocks go through mock-global helpers", () => { - const EXEMPT = new Set(["tests/helpers/mock-global.js"]); - const testFiles = scannedTestFiles(EXEMPT); + const testFiles = scannedTestFiles(); /** @type {string[]} */ const offenders = []; for (const file of testFiles) { @@ -115,7 +110,6 @@ test("test global mocks go through mock-global helpers", () => { test("auth index tests access mock state through the harness", () => { const EXEMPT = new Set([ "tests/helpers/load-auth-index.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -132,8 +126,6 @@ test("auth index tests access mock state through the harness", () => { test("test output capture goes through output-capture helpers", () => { const EXEMPT = new Set([ "tests/helpers/output-capture.js", - "tests/unit/output-capture.test.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -153,9 +145,7 @@ test("test output capture goes through output-capture helpers", () => { test("test request-body and DO envelope decoders use shared helpers", () => { const EXEMPT = new Set([ - "tests/helpers/request-body.js", "tests/helpers/do-envelope.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -181,11 +171,7 @@ test("test request-body and DO envelope decoders use shared helpers", () => { }); test("integration response JSON parsing uses response accessors", () => { - const EXEMPT = new Set([ - "tests/integration/helpers/http-response.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT) + const testFiles = scannedTestFiles() .filter((file) => file.startsWith("tests/integration/")); /** @type {string[]} */ const offenders = []; @@ -284,11 +270,7 @@ test("integration Redis commands use typed Redis helpers", () => { }); test("integration status assertions use status helpers for structured diagnostics", () => { - const EXEMPT = new Set([ - "tests/integration/helpers/assertions.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT) + const testFiles = scannedTestFiles() .filter((file) => file.startsWith("tests/integration/")); /** @type {string[]} */ const offenders = []; @@ -304,13 +286,7 @@ test("integration status assertions use status helpers for structured diagnostic }); test("unit JSON response assertions use response-json helper", () => { - const EXEMPT = new Set([ - "tests/helpers/response-json.js", - "tests/unit/mock-fetch.test.js", - "tests/unit/response-json.test.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT) + const testFiles = scannedTestFiles() .filter((file) => file.startsWith("tests/unit/")); /** @type {string[]} */ const offenders = []; @@ -343,12 +319,7 @@ test("unit JSON response assertions use response-json helper", () => { }); test("tests use delay helper for simple sleep promises", () => { - const EXEMPT = new Set([ - "tests/helpers/timing.js", - "tests/integration/helpers/stack.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT) + const testFiles = scannedTestFiles() .filter((file) => !file.includes("/manual/")); /** @type {string[]} */ const offenders = []; @@ -366,7 +337,6 @@ test("tests use delay helper for simple sleep promises", () => { test("test temporary directories use temp-dir helper", () => { const EXEMPT = new Set([ "tests/helpers/temp-dir.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -386,9 +356,7 @@ test("test temporary directories use temp-dir helper", () => { test("DO owner hint test fixtures use shared helper", () => { const EXEMPT = new Set([ - "tests/helpers/do-owner-hint.js", "tests/unit/style-contracts.test.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -404,11 +372,7 @@ test("DO owner hint test fixtures use shared helper", () => { }); test("repository JSON fixtures use readRepositoryJson", () => { - const EXEMPT = new Set([ - "tests/helpers/load-shared-module.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT); + const testFiles = scannedTestFiles(); /** @type {string[]} */ const offenders = []; const fixtureJsonParse = /JSON\.parse\(\s*readFileSync\(\s*new URL\([^)]*fixtures\/[^)]*\.json/; @@ -422,15 +386,7 @@ test("repository JSON fixtures use readRepositoryJson", () => { }); test("structured JSON payloads use protocol-specific helpers", () => { - const EXEMPT = new Set([ - "tests/integration/helpers/http-response.js", - "tests/integration/helpers/internal-http.js", - "tests/integration/helpers/json-payload.js", - "tests/helpers/json-payload.js", - "tests/integration/helpers/ws-roundtrip-runner.cjs", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT); + const testFiles = scannedTestFiles(); /** @type {string[]} */ const offenders = []; const structuredJsonParsers = [ diff --git a/tests/unit/version.test.js b/tests/unit/version.test.js index c263668..ac481dd 100644 --- a/tests/unit/version.test.js +++ b/tests/unit/version.test.js @@ -1,6 +1,19 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { formatVersion, parseVersion, bundleKey } from "../../shared/version.js"; +import { + DECLARED_HOSTS_KEY, + HOST_DECLARATIONS_SCAN_PATTERN, + HOSTS_SCAN_PATTERN, + NAMESPACES_KEY, + bundleKey, + formatVersion, + hostDeclarationsKey, + hostsKey, + namespaceFromHostsKey, + nextVersionKey, + nsHostsKey, + parseVersion, +} from "../../shared/version.js"; test("formatVersion: integer → v", () => { assert.equal(formatVersion(1), "v1"); @@ -43,3 +56,19 @@ test("bundleKey: rejects malformed version tags", () => { assert.throws(() => bundleKey("demo", "hello", ""), /invalid version/); assert.throws(() => bundleKey("demo", "hello", null), /invalid version/); }); + +test("route-plane registry key helpers compose and parse canonical keys", () => { + assert.equal(NAMESPACES_KEY, "namespaces"); + assert.equal(DECLARED_HOSTS_KEY, "declared-hosts"); + assert.equal(HOSTS_SCAN_PATTERN, "hosts:*"); + assert.equal(HOST_DECLARATIONS_SCAN_PATTERN, "host-declarations:*"); + assert.equal(hostsKey("demo"), "hosts:demo"); + assert.equal(namespaceFromHostsKey("hosts:demo"), "demo"); + assert.equal(namespaceFromHostsKey("routes:demo"), ""); + assert.equal(nsHostsKey("demo"), "ns-hosts:demo"); + assert.equal(hostDeclarationsKey("app.example"), "host-declarations:app.example"); +}); + +test("nextVersionKey composes the worker version counter key", () => { + assert.equal(nextVersionKey("demo", "hello"), "worker:demo:hello:next_version"); +}); From 09a38bff6c0e1e32109436d17828f70c2f6efec3 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Sun, 12 Jul 2026 16:53:55 +0800 Subject: [PATCH 03/23] Validate R2 cache expiry metadata Signed-off-by: Lu Zhang --- runtime/r2-utils.js | 13 ++++++++++++- tests/unit/r2-utils.test.js | 13 +++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/runtime/r2-utils.js b/runtime/r2-utils.js index 6f14800..91db592 100644 --- a/runtime/r2-utils.js +++ b/runtime/r2-utils.js @@ -22,7 +22,18 @@ export function r2CacheExpiryFromHeaders(headers) { /** @param {Headers} headers @param {unknown} value */ export function setR2CacheExpiryHeader(headers, value) { if (value == null) return; - const date = value instanceof Date ? value : new Date(Number(value)); + let date; + if (value instanceof Date || typeof value === "number") { + date = new Date(value); + } else if (typeof value === "string" && value.trim() !== "") { + const timestamp = Number(value); + date = new Date(Number.isFinite(timestamp) ? timestamp : value); + } else { + throw new TypeError("R2 httpMetadata.cacheExpiry must be a valid Date or timestamp"); + } + if (!Number.isFinite(date.getTime())) { + throw new TypeError("R2 httpMetadata.cacheExpiry must be a valid Date or timestamp"); + } headers.set("expires", date.toUTCString()); } diff --git a/tests/unit/r2-utils.test.js b/tests/unit/r2-utils.test.js index c55f11a..a697c05 100644 --- a/tests/unit/r2-utils.test.js +++ b/tests/unit/r2-utils.test.js @@ -71,10 +71,23 @@ test("R2 HTTP metadata fields and cache expiry use one mapping", () => { setR2CacheExpiryHeader(headers, 0); assert.equal(headers.get("expires"), "Thu, 01 Jan 1970 00:00:00 GMT"); assert.equal(r2CacheExpiryFromHeaders(headers), 0); + setR2CacheExpiryHeader(headers, "2026-07-12T00:00:00.000Z"); + assert.equal(headers.get("expires"), "Sun, 12 Jul 2026 00:00:00 GMT"); headers.set("expires", "not-a-date"); assert.equal(r2CacheExpiryFromHeaders(headers), undefined); }); +test("setR2CacheExpiryHeader rejects invalid expiry values", () => { + for (const value of ["", "not-a-date", new Date(NaN), true]) { + const headers = new Headers(); + assert.throws( + () => setR2CacheExpiryHeader(headers, value), + /cacheExpiry must be a valid Date or timestamp/ + ); + assert.equal(headers.has("expires"), false); + } +}); + test("encodeS3KeyPath percent-encodes key segments while preserving slashes", () => { assert.equal( encodeS3KeyPath("r2/demo/uploads/a b/?.txt"), From 79b605fb164e93be2b761666c8e036752927e88b Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Mon, 13 Jul 2026 10:46:34 +0800 Subject: [PATCH 04/23] Harden cross-tier runtime contracts Pin internal auth and request-id contracts, protect DO transport and wrapper intrinsics from tenant mutation, tighten secret and error handling, and align tests and module documentation. Signed-off-by: Lu Zhang --- control/lib.js | 4 + do-runtime/actor.js | 22 +- do-runtime/index.js | 15 +- do-runtime/protocol.js | 69 +- docs/modules/control-auth.md | 4 + docs/modules/control-auth.zh.md | 2 + docs/modules/durable-objects.md | 22 +- docs/modules/durable-objects.zh.md | 6 +- docs/modules/infra.md | 15 +- docs/modules/infra.zh.md | 2 +- docs/modules/log-tail-observability.md | 11 +- docs/modules/log-tail-observability.zh.md | 4 +- docs/modules/runtime.md | 12 +- docs/modules/runtime.zh.md | 6 +- runtime/_wdl-do-transport.js | 521 ++++++++++--- runtime/config-system.capnp | 1 + runtime/config-user.capnp | 1 + runtime/load/code-budget.js | 5 + runtime/load/module-rewrite.js | 1 + runtime/load/wrapper-generate.js | 140 +++- runtime/r2-client.js | 15 +- runtime/r2-utils.js | 10 +- rust/common/src/internal_auth.rs | 57 +- rust/common/src/request_id.rs | 44 +- rust/redis-proxy/src/lib.rs | 12 +- rust/workflows/src/server.rs | 9 +- shared/errors.js | 6 +- shared/internal-auth.js | 13 +- shared/observability.js | 58 +- shared/respond.js | 39 +- .../do-intrinsic-guard-worker.mjs.txt | 70 ++ tests/fixtures/internal-auth-contract.json | 21 + tests/fixtures/request-id-sanitizer.json | 6 +- tests/helpers/do-owner-hint.js | 7 + tests/helpers/load-do-host-actor.js | 18 +- tests/helpers/mock-global.js | 23 + .../integration/durable-objects-core.test.js | 30 + .../durable-objects-ownership.test.js | 5 +- tests/unit/control-lib.test.js | 7 + .../control-secret-envelope-handlers.test.js | 22 + tests/unit/do-runtime-actor.test.js | 20 + tests/unit/do-runtime-protocol.test.js | 85 +- tests/unit/internal-auth.test.js | 24 +- tests/unit/observability.test.js | 28 + tests/unit/r2-utils.test.js | 4 + tests/unit/respond.test.js | 16 + tests/unit/runtime-bindings-do.test.js | 54 +- tests/unit/runtime-do-client.test.js | 732 +++++++++++++++++- tests/unit/runtime-load.test.js | 34 +- tests/unit/runtime-r2-client.test.js | 24 + tests/unit/runtime-wrapper-generate.test.js | 73 +- tests/unit/shared-primitives.test.js | 11 + tests/unit/style-contracts.test.js | 8 +- 53 files changed, 2099 insertions(+), 349 deletions(-) create mode 100644 tests/fixtures/do-intrinsic-guard-worker.mjs.txt diff --git a/control/lib.js b/control/lib.js index e5ef8c2..3c2be48 100644 --- a/control/lib.js +++ b/control/lib.js @@ -4,6 +4,7 @@ import { NS_PATTERN, + RESERVED_OBJECT_KEYS, isReservedNs, RESERVED_TENANT_NS, WORKER_NAME_RE, @@ -695,5 +696,8 @@ export function validateSecretKey(key) { if (WDL_RESERVED_BINDING_RE.test(key)) { throw new Error("secret key is reserved for runtime-internal bindings"); } + if (RESERVED_OBJECT_KEYS.has(key)) { + throw new Error("secret key is a reserved Object.prototype key"); + } if (key.length > 128) throw new Error("secret key too long (max 128)"); } diff --git a/do-runtime/actor.js b/do-runtime/actor.js index f00762e..54e9d46 100644 --- a/do-runtime/actor.js +++ b/do-runtime/actor.js @@ -5,8 +5,9 @@ import { buildAlarmRequest, buildFacetName, buildForwardRequest, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, DO_OWNERSHIP_CODE, - doErrorResponse, + doPlatformErrorResponse, DoRuntimeError, normalizeDoConnectRequest, readLocalActorInvokeRequest, @@ -26,6 +27,7 @@ import { } from "do-runtime-state"; import { errorMessage } from "shared-errors"; import { formatError } from "shared-observability"; +import { rebuildResponseWithHeaders } from "shared-respond"; /** * @typedef {{ LOADER: { get(key: string, loader: () => Promise): DoWorkerStub }, DO_TEST_HOOKS?: unknown }} DoEnv @@ -167,7 +169,7 @@ export class WdlDoHostActor extends DurableObject { return await facet.fetch(buildForwardRequest(invoke.request)); }); } catch (err) { - return doErrorResponse(err); + return doPlatformErrorResponse(err); } } @@ -182,7 +184,9 @@ export class WdlDoHostActor extends DurableObject { try { const { owner, leaseRemainingMs } = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); await this.rememberObject(invoke); - return await this.dispatchWithLeaseBudget(invoke, owner, leaseRemainingMs, run); + return withoutOwnershipErrorControlHeader( + await this.dispatchWithLeaseBudget(invoke, owner, leaseRemainingMs, run) + ); } finally { endInFlightDispatch(); } @@ -257,6 +261,14 @@ export class WdlDoHostActor extends DurableObject { } } +/** @param {Response} response */ +function withoutOwnershipErrorControlHeader(response) { + if (!response.headers.has(DO_OWNERSHIP_ERROR_CONTROL_HEADER)) return response; + const headers = new Headers(response.headers); + headers.delete(DO_OWNERSHIP_ERROR_CONTROL_HEADER); + return rebuildResponseWithHeaders(response, headers); +} + /** * @param {DoFacet} facet * @param {{ method: string, args: unknown[] }} rpc @@ -278,8 +290,8 @@ async function dispatchRpc(facet, rpc) { const errorObject = err && typeof err === "object" ? err : {}; // The stack captured from fn(...) is tenant method execution, not // do-runtime framework internals, so it belongs on the tenant RPC boundary. - // Runtime failures still go through doErrorResponse(), which intentionally - // hides internal stack traces and infrastructure details. + // Runtime failures still go through the platform error mapper, which + // intentionally hides internal stack traces and infrastructure details. return Response.json({ error: "do_rpc_error", name: "name" in errorObject && typeof errorObject.name === "string" ? errorObject.name : "Error", diff --git a/do-runtime/index.js b/do-runtime/index.js index b5be45a..a50ae22 100644 --- a/do-runtime/index.js +++ b/do-runtime/index.js @@ -4,14 +4,14 @@ import { verifyInternalAuthHeaders, } from "shared-internal-auth"; import { createHttpRequestScope } from "shared-request-scope"; -import { discardResponseBody, prometheusResponse } from "shared-respond"; +import { discardResponseBody, prometheusResponse, rebuildResponseWithHeaders } from "shared-respond"; import { boundedPositiveIntEnv } from "shared-owner-lease"; import { formatWorkerId } from "shared-worker-id"; import { WdlDoHostActor } from "do-runtime-actor"; import { handleAlarmDispatch } from "do-runtime-alarm-dispatch"; import { buildLocalActorRequest, - doErrorResponse, + doPlatformErrorResponse, DoRuntimeError, hostIdForObject, hostIdForShard, @@ -153,14 +153,7 @@ function withOwnerHintHeaders(response, owner) { for (const [name, value] of Object.entries(ownerHintHeaders(owner))) { headers.set(name, value); } - const init = /** @type {ResponseInit & { webSocket?: WebSocket }} */ ({ - status: response.status, - statusText: response.statusText, - headers, - }); - const webSocket = /** @type {{ webSocket?: WebSocket }} */ (response).webSocket; - if (webSocket) init.webSocket = webSocket; - return new Response(response.status === 101 ? null : response.body, init); + return rebuildResponseWithHeaders(response, headers); } /** @param {Request} request */ @@ -543,7 +536,7 @@ export default { return scope.respond(jsonError(404, "not_found", "Not found")); } catch (err) { scope.markError(err); - return scope.respond(doErrorResponse(err)); + return scope.respond(doPlatformErrorResponse(err)); } finally { scope.complete(); } diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index 2771d70..cced13b 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -10,7 +10,7 @@ import { DO_HOST_SHARD_COUNT, MAX_ID_BYTES, } from "do-runtime-protocol-wire-grammar"; -import { DoRuntimeError } from "do-runtime-protocol-errors"; +import { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; import { hostIdForObject, shardForObjectName } from "do-runtime-protocol-identity"; import { formatWorkerId } from "shared-worker-id"; import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; @@ -39,6 +39,18 @@ export const DO_OWNERSHIP_CODE = Object.freeze({ FORWARD_HOP_EXHAUSTED: "forward_hop_exhausted", TASK_DRAINING: "task_draining", }); +export const DO_OWNERSHIP_ERROR_CONTROL_HEADER = "x-wdl-do-ownership-error"; +/** @type {Set} */ +const DO_OWNERSHIP_CODES = new Set(Object.values(DO_OWNERSHIP_CODE)); + +/** @param {unknown} err */ +export function doPlatformErrorResponse(err) { + const response = doErrorResponse(err); + if (err instanceof DoRuntimeError && DO_OWNERSHIP_CODES.has(err.code)) { + response.headers.set(DO_OWNERSHIP_ERROR_CONTROL_HEADER, err.code); + } + return response; +} const MAX_MODULE_COUNT = 128; const MAX_MODULE_SOURCE_BYTES = 1024 * 1024; @@ -47,6 +59,19 @@ const MAX_INVOKE_ENVELOPE_BYTES = 2 * 1024 * 1024; const MAX_REQUEST_HEADER_COUNT = 128; const MAX_REQUEST_HEADER_BYTES = 64 * 1024; export const DO_INVOKE_CONTENT_TYPE = "application/vnd.wdl.do-invoke"; +const IntrinsicArray = Array; +const IntrinsicNumber = Number; +const IntrinsicObject = Object; +const IntrinsicWeakSet = WeakSet; +const intrinsicReflectApply = Reflect.apply; +const intrinsicArrayIsArray = Array.isArray; +const intrinsicNumberIsFinite = Number.isFinite; +const intrinsicObjectEntries = Object.entries; +const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; +const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; +const intrinsicWeakSetAdd = WeakSet.prototype.add; +const intrinsicWeakSetDelete = WeakSet.prototype.delete; +const intrinsicWeakSetHas = WeakSet.prototype.has; const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); const ALARM_INTERNAL_URL = "https://do.internal/__wdl_alarm"; @@ -211,35 +236,47 @@ function requireJsonSerializedSize(value, field, maxBytes) { * @param {WeakSet} [seen] * @returns {string | null} */ -function jsonDataError(value, field, seen = new WeakSet()) { +function jsonDataError(value, field, seen = new IntrinsicWeakSet()) { if (value === null) return null; const type = typeof value; if (type === "string" || type === "boolean") return null; - if (type === "number") return Number.isFinite(value) ? null : `${field} must be a finite number`; + if (type === "number") { + return intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [value]) + ? null + : `${field} must be a finite number`; + } if (type === "bigint" || type === "function" || type === "symbol" || type === "undefined") { return `${field} must be JSON data`; } if (type !== "object") return `${field} must be JSON data`; const objectValue = /** @type {Record | unknown[]} */ (value); - if (seen.has(objectValue)) return `${field} must not be circular`; - seen.add(objectValue); - if (Array.isArray(objectValue)) { - for (let i = 0; i < objectValue.length; i++) { - if (!(i in objectValue)) return `${field} must not be sparse`; - const error = jsonDataError(objectValue[i], `${field}[${i}]`, seen); + if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { + return `${field} must not be circular`; + } + intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); + if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { + const arrayValue = /** @type {unknown[]} */ (objectValue); + for (let i = 0; i < arrayValue.length; i++) { + if (!(i in arrayValue)) return `${field} must not be sparse`; + const error = jsonDataError(arrayValue[i], `${field}[${i}]`, seen); if (error) return error; } - seen.delete(objectValue); + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); return null; } - const proto = Object.getPrototypeOf(objectValue); - if (proto !== Object.prototype && proto !== null) return `${field} must be a plain JSON object`; - if (Object.getOwnPropertySymbols(objectValue).length > 0) return `${field} must not contain symbol keys`; - for (const [key, entry] of Object.entries(objectValue)) { - const error = jsonDataError(entry, `${field}.${key}`, seen); + const proto = intrinsicReflectApply(intrinsicObjectGetPrototypeOf, IntrinsicObject, [objectValue]); + if (proto !== IntrinsicObject.prototype && proto !== null) return `${field} must be a plain JSON object`; + const symbols = intrinsicReflectApply(intrinsicObjectGetOwnPropertySymbols, IntrinsicObject, [objectValue]); + if (symbols.length > 0) return `${field} must not contain symbol keys`; + const entries = /** @type {[string, unknown][]} */ ( + intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) + ); + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + const error = jsonDataError(entry[1], `${field}.${entry[0]}`, seen); if (error) return error; } - seen.delete(objectValue); + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); return null; } diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index 299033a..2a43b60 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -172,6 +172,10 @@ Control uses WATCH/MULTI/EXEC for active-version flips, routing changes, delete and lifecycle index updates. Worker lifecycle indexes are authoritative; handlers should not add fallback scans of bundle state. +Secret mutation validates env-var grammar, runtime-reserved names, and reserved +`Object.prototype` keys before persistence so every accepted key can be materialized by +the runtime env builder. + Auth stores token records in Redis and evaluates against `shared/auth-roles.js`. Key families: diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index a0853d2..e105529 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -90,6 +90,8 @@ Control 拥有 DB 0 metadata： Control 对 active-version flip、routing change、delete lock 和 lifecycle index update 使用 WATCH/MULTI/EXEC。Worker lifecycle index 是权威状态；handler 不应增加扫描 bundle state 的 fallback。 +Secret mutation 在写入前校验 env-var grammar、runtime-reserved name 和保留的 `Object.prototype` key，确保每个被接受的 key 都能由 runtime env builder materialize。 + Auth 在 Redis 中存 token record，并按 `shared/auth-roles.js` 评估权限。 Key families： diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index b20e689..ebb5d98 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -110,9 +110,9 @@ when the logical worker is gone or now points at a different `doStorageId`. - One task owns a class shard at a time. - Generation fencing prevents stale owners from committing after ownership moves. -- `do-runtime/protocol.js` owns the DO ownership error vocabulary and the subset that - proves a request was fence-rejected and is safe to retry through the router. The - injected runtime transport mirrors both sets under a direct parity test. +- `do-runtime/protocol.js` owns the DO ownership error vocabulary. The injected runtime + transport keeps its retry and stale-hint subsets private and pins them through + response-classification tests. - Facet identity is `className:objectName` inside stable `doStorageId`, so worker promotion preserves object state. - Existing native facets keep the constructed class version until host actor restart or @@ -134,11 +134,12 @@ when the logical worker is gone or now points at a different `doStorageId`. Client messages queued under an older backend reconnect epoch may be discarded without per-frame ack/nack when the gateway resets that epoch. - Ordinary fetch/RPC can fall back through the router after explicit stale-owner or - owner-race responses. A direct owner transport failure, or a 502/503/504 without a - fresh owner-hint header, evicts the cached hint. Safe `GET`/`HEAD` requests may replay - through the router to rediscover the owner; non-idempotent methods and RPC return - `owner_unavailable` without replay because the owner may already have applied the - request. + owner-race responses carrying do-runtime's private ownership-error control header. + Tenant response bodies cannot opt into replay. A direct owner transport failure, or a + 502/503/504 without a fresh owner-hint header, evicts the cached hint. Safe `GET`/`HEAD` + requests may replay through the router to rediscover the owner; non-idempotent methods + and RPC return `owner_unavailable` without replay because the owner may already have + applied the request. - The shared runtime transport owns owner-hint cache wiring, invoke race retry, and response-header stripping for both the host binding and injected facade. Its connect wrapper deliberately omits the invoke-only router fallback required to preserve @@ -218,8 +219,9 @@ interrupt. endpoints, or raw transport error text. - Owner hints are trusted only when returned by do-runtime headers and validated against endpoint grammar. -- Owner-hint defense is layered: tenant response bodies are ignored, only do-runtime - control headers are trusted, and endpoint grammar/acceptable-address checks must pass. +- Owner-hint and ownership-error defense is layered: tenant response bodies and + tenant-supplied control headers are ignored, only do-runtime control headers are + trusted, and endpoint grammar/acceptable-address checks must pass for hints. - do-runtime supervisor must call local `127.0.0.1:8788` drain/renew endpoints; Service Connect aliases may hit a different task. diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 57601aa..6286eae 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -67,13 +67,13 @@ workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespa - 同一时间只有一个 task 拥有一个 class shard。 - Generation fence 防止 stale owner 在 ownership 移动后继续 commit。 -- `do-runtime/protocol.js` 持有 DO ownership error vocabulary，以及能够证明 request 已被 fence 拒绝、可安全通过 router 重试的子集。Injected runtime transport 在直接 parity test 下 mirror 这两个 Set。 +- `do-runtime/protocol.js` 持有 DO ownership error vocabulary。Injected runtime transport 将 retry 和 stale-hint 子集保持为私有策略，并通过 response-classification test pin 住。 - Facet identity 是 stable `doStorageId` 内的 `className:objectName`，因此 worker promotion 保留 object state。 - 已构造的 native facet 会保留构造时的 class version，直到 host actor restart 或 facet deletion。Promotion 改变未来 load 和 routing metadata，不会替换当前 host actor 中已经构造的 facet。 - Whole-worker delete 后 redeploy 会分配新的 `doStorageId`；旧 native storage tombstone 给后续 cleanup，而不是立即物理删除。 - WebSocket upgrade 必须在 owner endpoint 上完成。Owner-hinted WebSocket direct retry 不能 fall back 到 router-established 101。 - WDL 会尽量让 client-facing WebSocket 由 gateway 持有并保持连接，包括 user-runtime 或 do-runtime restart 后的 backend reconnect。这个连接连续性目标强于 Cloudflare shutdown 行为；Cloudflare 可以终止 WebSocket 让新 Durable Object instance 接管。当前 backend facet 仍属于 owner scope：初始 `101` 之后，WebSocket message / close event 不会在每一帧重新校验 Redis owner generation。未来增强应在尽量保持 client 连接的同时 rebinding 或拒绝 stale backend owner facet，而不是把 client disconnect 作为主要安全机制。Gateway 重置 backend reconnect epoch 时，旧 epoch 下排队的 client message 可能被丢弃，且没有逐帧 ack/nack。 -- Ordinary fetch/RPC 只有在明确的 stale-owner 或 owner-race response 后才会回退到 router。Direct owner transport failure，或 direct owner hop 返回 502/503/504 且没有 fresh owner-hint header 时，会清除 cached hint。安全的 `GET`/`HEAD` request 可以通过 router 重放以重新发现 owner；非幂等 method 和 RPC 会返回 `owner_unavailable`，不会重放，因为 owner 可能已经应用了该请求。 +- Ordinary fetch/RPC 只有在明确的 stale-owner 或 owner-race response 同时携带 do-runtime 私有 ownership-error control header 时才会回退到 router；tenant response body 不能触发重放。Direct owner transport failure，或 direct owner hop 返回 502/503/504 且没有 fresh owner-hint header 时，会清除 cached hint。安全的 `GET`/`HEAD` request 可以通过 router 重放以重新发现 owner；非幂等 method 和 RPC 会返回 `owner_unavailable`，不会重放，因为 owner 可能已经应用了该请求。 - Shared runtime transport 统一持有 host binding 与 injected facade 的 owner-hint cache wiring、invoke race retry 和 response-header stripping。Connect wrapper 刻意不包含 invoke-only router fallback，以保留 owner-established WebSocket upgrade 语义。 - `WEBSOCKET_RECONNECT_DELAYS_MS` 和 `WEBSOCKET_MAX_BUFFERED_MESSAGES` 可以在不改代码的情况下调整 gateway backend reconnect budget 和 client-message buffer cap。 - Alarm delivery 是 at-least-once。Scheduler 唤醒 Workflows；Workflows 把到期 internal alarm job promote 到 ready，在 DB 2 run token 下 claim，然后调用 do-runtime `/internal/do/alarms/dispatch`。do-runtime 仍然构造原来的 `DoInvoke{kind:"alarm"}` 请求，并走正常 owner router/fence 路径。 @@ -100,7 +100,7 @@ Terraform 除了 Fargate task memory limit，还会给 do-runtime workerd contai - Tenant code 只能通过 runtime 生成的 facade 和 frozen metadata 访问 DO。 - Tenant-visible DO metadata 和 error 不得包含 owner task id、backend endpoint 或原始 transport error 文本。 - Owner hint 只信任 do-runtime header，并且要通过 endpoint grammar validation。 -- Owner-hint 防御是分层的：忽略 tenant response body，只信任 do-runtime control header，并且必须通过 endpoint grammar / acceptable-address 检查。 +- Owner-hint 与 ownership-error 防御是分层的：忽略 tenant response body 和 tenant-supplied control header，只信任 do-runtime control header；hint 还必须通过 endpoint grammar / acceptable-address 检查。 - do-runtime supervisor 必须调用本地 `127.0.0.1:8788` drain/renew endpoint；Service Connect alias 可能打到其他 task。 ## 可观测性 diff --git a/docs/modules/infra.md b/docs/modules/infra.md index 60280e0..32f50ea 100644 --- a/docs/modules/infra.md +++ b/docs/modules/infra.md @@ -99,13 +99,14 @@ Private mesh callers and receivers share `WDL_INTERNAL_AUTH_TOKEN` as the curren internal token. Calls between runtime, d1-runtime, do-runtime, scheduler, workflows, and redis-proxy sidecars carry it as `x-wdl-internal-auth`. Receivers also accept optional `WDL_INTERNAL_AUTH_PREVIOUS_TOKEN` during rotation; callers -always send only the current token. Both token values must be non-empty ASCII -strings so JS and Rust compare the same header representation. Health and -metrics endpoints are the only unauthenticated service endpoints. The token is -platform plumbing, not a tenant binding: runtime wrapper code strips it from -tenant-visible `env`, host-owned DO proxies and host-side backend capabilities -add it for DO forwarding, and spoofed tenant headers are removed before -forwarding. +always send only the current token. Receivers require exactly one auth header. +Both token values must contain only visible ASCII bytes, with no whitespace or commas, +so Fetch header normalization preserves them exactly and header joining cannot make +repeated headers look like one configured token. Health and metrics endpoints are the +only unauthenticated service endpoints. The token is platform plumbing, +not a tenant binding: runtime wrapper code strips it from tenant-visible `env`, host-owned +DO proxies and host-side backend capabilities add it for DO forwarding, and spoofed tenant +headers are removed before forwarding. ## Redis / Storage Contracts diff --git a/docs/modules/infra.zh.md b/docs/modules/infra.zh.md index 144f222..6515821 100644 --- a/docs/modules/infra.zh.md +++ b/docs/modules/infra.zh.md @@ -59,7 +59,7 @@ Gateway 和 runtime 在不同环境中使用稳定的内部端口合同： 不要把 private mesh endpoint 暴露到 public ingress。K8s 和 ECS 交付都必须通过 Service Connect、ClusterIP Services、NetworkPolicy 或等价控制保持这个假设成立。 -Private mesh caller 和 receiver 共享 `WDL_INTERNAL_AUTH_TOKEN` 作为当前 internal token。runtime、d1-runtime、do-runtime、scheduler、workflows 和 redis-proxy sidecar 之间的调用会携带 `x-wdl-internal-auth`。轮换期间 receiver 还会接受可选的 `WDL_INTERNAL_AUTH_PREVIOUS_TOKEN`；caller 始终只发送当前 token。两个 token 值都必须是非空 ASCII string，确保 JS 和 Rust 比较同一种 header 表示。Health 和 metrics endpoint 是唯一不要求该 token 的 service endpoint。这个 token 是平台 plumbing，不是 tenant binding：runtime wrapper code 会从 tenant-visible `env` 中剥离它，host-owned DO proxy 和 host-side backend capability 会在 DO forwarding 时添加它，并在 forwarding 前删除租户伪造的同名 header。 +Private mesh caller 和 receiver 共享 `WDL_INTERNAL_AUTH_TOKEN` 作为当前 internal token。runtime、d1-runtime、do-runtime、scheduler、workflows 和 redis-proxy sidecar 之间的调用会携带 `x-wdl-internal-auth`。轮换期间 receiver 还会接受可选的 `WDL_INTERNAL_AUTH_PREVIOUS_TOKEN`；caller 始终只发送当前 token。Receiver 要求 auth header 恰好出现一次；两个 token 值都只能包含 visible ASCII byte，不能包含空白或逗号，确保 Fetch header normalization 不改变 token，并避免 header joining 把重复 header 伪装成单个已配置 token。Health 和 metrics endpoint 是唯一不要求该 token 的 service endpoint。这个 token 是平台 plumbing，不是 tenant binding：runtime wrapper code 会从 tenant-visible `env` 中剥离它，host-owned DO proxy 和 host-side backend capability 会在 DO forwarding 时添加它，并在 forwarding 前删除租户伪造的同名 header。 ## Redis / Storage 合同 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index 81020f1..6850cb6 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -138,9 +138,9 @@ Central owners: Common rules: - `x-request-id` propagates across gateway, control/runtime, loaded workers, and D1 - where possible. Missing inbound ids are minted at ingress; dirty ids such as - multi-valued ids, ids containing Unicode whitespace or control characters, or ids - over 128 UTF-8 bytes are treated as absent. JS entrypoints using + where possible. Missing inbound ids are minted at ingress. Header adapters consider + only the first comma-delimited token; ids containing bytes outside visible ASCII, + quotes, backslashes, or more than 128 bytes are treated as absent. JS entrypoints using `shared/request-scope.js` echo the sanitized id on responses and log it as `request_id` on `request_complete`; Rust sidecars sanitize inbound ids and log them where request middleware owns completion. Control's Redis `PUBLISH` path logs the id @@ -153,8 +153,9 @@ Common rules: `ts`, `service`, `level`, `event`, plus snake_case fields. JS services use `shared/observability.js`; Rust services use `wdl-rust-common::log::emit_log_line`. Error text is emitted as `error_message`; - secret values, raw credentials, token material, raw Redis keys, and unbounded payloads - are not emitted. + throwables that cannot be converted to text degrade to `Unknown error` instead of + replacing the original failure. Secret values, raw credentials, token material, raw + Redis keys, and unbounded payloads are not emitted. - Product API response bodies should use camelCase unless an endpoint explicitly documents a different wire contract. - Metrics should use bounded enumerated labels only. diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index 309da3a..62ae958 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -91,9 +91,9 @@ WDL 在 JS workerd tier 和 Rust 服务中使用同一套可观测性策略： 通用规则： -- `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker 和 D1 传播。缺失的 inbound id 会在入口生成；multi-valued、包含 Unicode whitespace/control char 或超过 128 UTF-8 bytes 的脏 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 +- `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker 和 D1 传播。缺失的 inbound id 会在入口生成；header adapter 只考虑第一个逗号分隔 token，包含 visible ASCII 以外字节、引号、反斜杠或超过 128 字节的 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 - 日志字段使用 snake_case。只有 `level=error` 写入 stderr；debug/info/warn JSON log line 都写入 stdout，这样 JS、Rust 和 embedded workerd shim 的日志路由保持一致。 -- 内部运维日志，包括 system-worker cleanup 日志和防御性的 Redis callback warning，也使用同一套单行 JSON envelope：`ts`、`service`、`level`、`event`，再加 snake_case 字段。JS 服务使用 `shared/observability.js`；Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`；不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 +- 内部运维日志，包括 system-worker cleanup 日志和防御性的 Redis callback warning，也使用同一套单行 JSON envelope：`ts`、`service`、`level`、`event`，再加 snake_case 字段。JS 服务使用 `shared/observability.js`；Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`；无法转成文本的 throwable 降级为 `Unknown error`，不能反向覆盖原始失败。不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 - 产品 API response body 默认使用 camelCase，除非 endpoint 明确记录不同 wire contract。 - Metrics 只使用有界枚举 label。 - 暴露 request metrics 的 Rust HTTP sidecar 使用同一组 `requests`、`request_duration_ms` 和 `request_errors` metric family，以及有界的 `service`/`route`/`status` labels；per-route error context 只进入 `request_complete` 日志中的 `error_code` / `error_message`。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index c8b4830..0aefed6 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -132,7 +132,9 @@ are isolated by prefix. Runtime supports the common `head`, `get`, `put`, `delet 25 MiB cap. `put(stream, ...)` currently buffers and sends one S3 PUT with the same cap; multipart upload, SSE-C, and checksum selection are not supported. Conditional requests and range GETs implement the common R2 behavior. `list({ include: [...] })` performs -extra HEAD requests for metadata fields and applies a concurrency cap. +extra HEAD requests for metadata fields and applies a concurrency cap. Tenant-supplied +`Headers` metadata must carry a canonical IMF-fixdate `Expires` value when that header +is present; malformed write metadata is rejected before the host binding call. Tenant-facing R2 errors expose operation/status plus virtual object keys where useful, but not raw S3 response bodies or physical `r2///...` keys. Control-plane R2 admin errors may retain backend detail for operators. @@ -200,12 +202,16 @@ opaque cold-load failures under the current stock binary. - Internal active-version scheduled/queue dispatches opt into sibling eviction; frozen workflow dispatches do not. - Wrapper generation hides raw env from unwrapped entrypoints whenever privileged - internal Fetchers are injected. + internal Fetchers are injected. Its host-wrapper runtime evaluates before tenant + modules and captures every intrinsic used to decide handler or env wrapping and to + settle request-context cleanup, so tenant top-level prototype mutation cannot bypass + that boundary or retain stale request context. - Request context wrappers swap facade objects into env and propagate request id where that event class can carry it. - An uncaught tenant `fetch()` exception maps to a platform `502 runtime_error` response with the request id. Exception details are emitted to structured logs/live - tail and are not copied into the client body. + tail and are not copied into the client body. Tail formatting cannot replace the + original throwable when its string conversion fails. - Internal scheduled, queue, and workflow dispatch routes use result envelopes for handler outcomes. A tenant handler error is represented as outcome state for the scheduler/workflow protocol, not as a generic platform transport error. diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index c1963f7..06039bd 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -53,7 +53,7 @@ KV 是最直接的例子。Runtime 把 `KV` 导出成 named entrypoint，并用 KV 支持常用 `KVNamespace` 调用：`get`、batch `get`、`getWithMetadata`、batch `getWithMetadata`、`put`、`delete` 和 `list`。`get` 支持 text、JSON、arrayBuffer 和 stream 形状；batch read 支持 text 和 JSON。Runtime shim 在 proxy 前把 value 限制到 25 MiB，stream value 也用同一个 cap 读取。所有 KV 操作的 key 都在 redis-proxy 边界限制为 512 个 UTF-8 字节，包括 list prefix 和 batch read。`list()` 基于 Redis `HSCAN`，不是 Cloudflare 有序 B-tree：key 不排序，cursor 是 opaque WDL cursor，并发写可能乱序出现或被再次看到。`limit` 上限是 1000。`cacheTtl` 只是 API shape；没有 Cloudflare edge read cache 或 global eventual-consistency window。 -R2 binding 把 `bucket_name` 映射到平台 S3-compatible bucket 下的 namespace-scoped virtual bucket：`r2///`。同一个 namespace 中使用同一 `bucket_name` 的 worker 会有意共享数据；不同 namespace 通过前缀隔离。Runtime 支持常用 `head`、`get`、`put`、`delete` 和 `list` 路径。`get()` 返回 streaming body，便捷 reader 执行 25 MiB cap。`put(stream, ...)` 目前会先 buffer，再发单个 S3 PUT，并使用同一个 cap；不支持 multipart upload、SSE-C 和 checksum selection。Conditional requests 和 range GET 实现常用 R2 行为。`list({ include: [...] })` 为 metadata fields 额外执行 HEAD，并使用并发 cap。Tenant-facing R2 error 只暴露 operation/status，以及有帮助的 virtual object key；不会暴露原始 S3 response body 或 physical `r2///...` key。Control-plane R2 admin error 可以为 operator 保留 backend detail。 +R2 binding 把 `bucket_name` 映射到平台 S3-compatible bucket 下的 namespace-scoped virtual bucket：`r2///`。同一个 namespace 中使用同一 `bucket_name` 的 worker 会有意共享数据；不同 namespace 通过前缀隔离。Runtime 支持常用 `head`、`get`、`put`、`delete` 和 `list` 路径。`get()` 返回 streaming body，便捷 reader 执行 25 MiB cap。`put(stream, ...)` 目前会先 buffer，再发单个 S3 PUT，并使用同一个 cap；不支持 multipart upload、SSE-C 和 checksum selection。Conditional requests 和 range GET 实现常用 R2 行为。`list({ include: [...] })` 为 metadata fields 额外执行 HEAD，并使用并发 cap。Tenant 提供的 `Headers` metadata 如果包含 `Expires`，其值必须是 canonical IMF-fixdate；malformed write metadata 会在 host binding call 前被拒绝。Tenant-facing R2 error 只暴露 operation/status，以及有帮助的 virtual object key；不会暴露原始 S3 response body 或 physical `r2///...` key。Control-plane R2 admin error 可以为 operator 保留 backend detail。 ASSETS 是 deploy-artifact helper，不是完整 Cloudflare Pages asset pipeline。Control 把文件上传到 `assets////`，注入 `ASSETS` binding，runtime 暴露同步的 `env.ASSETS.url(path)`。该方法在 runtime 中不做 IO，并用 `ASSETS_CDN_BASE` 返回浏览器可访问的 CDN URL。Path 按 `/` 切段，空段、`.` 和 `..` 被拒绝，每段会 percent-encode。Version 在 load 时绑定，因此 rollback 会切换 asset URL。需要对静态文件做 auth 或 rewrite 的 worker 应把文件留在 bundle 里，而不是使用 declared `assets`。 @@ -82,9 +82,9 @@ Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materializ - workerLoader cache 没有 LRU。Runtime 给每个 loaded worker 注入 `__WdlAbort__`，并在 active-version cold-load 时 evict sibling historical versions。 - Service-binding cold load 会记录 loaded version，但不 evict sibling，因为 service binding 可能有意指向 frozen historical version。 - Internal active-version scheduled/queue dispatch 会 opt into sibling eviction；frozen workflow dispatch 不会。 -- 只要注入 privileged internal Fetcher，wrapper generation 就会避免 raw env 暴露给未包装 entrypoint。 +- 只要注入 privileged internal Fetcher，wrapper generation 就会避免 raw env 暴露给未包装 entrypoint。Host-wrapper runtime 会在 tenant module 前执行，并捕获所有参与 handler/env wrapping 决策和 request-context cleanup 的 intrinsic，因此 tenant 顶层 prototype mutation 不能绕过这条边界或遗留 stale request context。 - Request context wrapper 会把 facade object 换进 env，并在事件类型允许时传播 request id。 -- Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error` response，并带 request id。异常细节输出到结构化日志/live tail，不复制进客户端 body。 +- Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error` response，并带 request id。异常细节输出到结构化日志/live tail，不复制进客户端 body；throwable 自身无法转成字符串时，tail formatting 也不能反向覆盖原始异常。 - Internal scheduled、queue 和 workflow dispatch route 使用 result envelope 表达 handler outcome。Tenant handler error 是 scheduler/workflow 协议中的 outcome state，不是 generic platform transport error。 - Runtime 没有 route-cache invalidation protocol。`workerLoader` cache key 是不可变 worker id，因此 promote 后的新 version 是新 key，会自然 cold-load。 - `runtime/tail-worker.js` 通过 `workerCode.tails` 附加到每次 dynamic load。它总是输出结构化 stdout；只有 shared tail forwarder 看到 active subscription 后，才转发到 `wdl tail`。 diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index 2623039..e979b9e 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -9,6 +9,7 @@ export const MAX_DO_REQUEST_HEADER_COUNT = 128; export const MAX_DO_REQUEST_HEADER_BYTES = 64 * 1024; export const DO_ACCEPT_OWNER_HINT_HEADER = "x-wdl-do-accept-owner-hint"; export const DO_OWNER_HINT_CONTROL_HEADER = "x-wdl-do-owner-hint"; +export const DO_OWNERSHIP_ERROR_CONTROL_HEADER = "x-wdl-do-ownership-error"; export const DO_OWNER_HINT_CODE = "do_owner_hint"; export const INTERNAL_AUTH_HEADER = "x-wdl-internal-auth"; @@ -21,6 +22,7 @@ const DO_OWNER_HINT_HEADERS = { const DO_OWNER_HINT_STRIP_HEADERS = [ ...Object.values(DO_OWNER_HINT_HEADERS), DO_OWNER_HINT_CONTROL_HEADER, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, ]; const DO_FETCH_STRIP_HEADERS = [ ...DO_OWNER_HINT_STRIP_HEADERS, @@ -32,14 +34,14 @@ const DO_CONNECT_STRIP_HEADERS = [ ...DO_FETCH_STRIP_HEADERS, ]; const OWNER_ENDPOINT_UNAVAILABLE_STATUSES = new Set([502, 503, 504]); -export const OWNER_RACE_RETRY_CODES = new Set([ +const OWNER_RACE_RETRY_CODES = new Set([ "stale_owner_generation", "owner_claim_raced", "owner_fence_missing", "owner_lease_expired", "owner_lease_too_short", ]); -export const OWNER_HINT_STALE_CODES = new Set([ +const OWNER_HINT_STALE_CODES = new Set([ "stale_owner_generation", "owner_claim_raced", "owner_fence_missing", @@ -53,6 +55,81 @@ export const OWNER_HINT_STALE_CODES = new Set([ "forward_hop_exhausted", "task_draining", ]); +const IntrinsicArray = Array; +const IntrinsicDataView = DataView; +const IntrinsicHeaders = Headers; +const IntrinsicJSON = JSON; +const IntrinsicNumber = Number; +const IntrinsicObject = Object; +const IntrinsicRequest = Request; +const IntrinsicResponse = Response; +const IntrinsicUint8Array = Uint8Array; +const IntrinsicWeakSet = WeakSet; +const intrinsicReflectApply = Reflect.apply; +const intrinsicArrayForEach = Array.prototype.forEach; +const intrinsicArrayIsArray = Array.isArray; +const intrinsicArrayPush = Array.prototype.push; +const intrinsicDataViewSetUint32 = DataView.prototype.setUint32; +const intrinsicHeadersAppend = Headers.prototype.append; +const intrinsicHeadersDelete = Headers.prototype.delete; +const intrinsicHeadersForEach = Headers.prototype.forEach; +const intrinsicHeadersGet = Headers.prototype.get; +const intrinsicHeadersHas = Headers.prototype.has; +const intrinsicHeadersSet = Headers.prototype.set; +const intrinsicJsonStringify = JSON.stringify; +const intrinsicNumberIsFinite = Number.isFinite; +const intrinsicObjectCreate = Object.create; +const intrinsicObjectEntries = Object.entries; +const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; +const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; +const intrinsicObjectSetPrototypeOf = Object.setPrototypeOf; +const intrinsicPromiseThen = Promise.prototype.then; +const intrinsicReadableStreamGetReader = ReadableStream.prototype.getReader; +const intrinsicReadableStreamReaderCancel = ReadableStreamDefaultReader.prototype.cancel; +const intrinsicReadableStreamReaderRead = ReadableStreamDefaultReader.prototype.read; +const intrinsicReadableStreamReaderReleaseLock = ReadableStreamDefaultReader.prototype.releaseLock; +const intrinsicRegExpTest = RegExp.prototype.test; +const intrinsicSetHas = Set.prototype.has; +const intrinsicStringToLowerCase = String.prototype.toLowerCase; +const intrinsicStringToUpperCase = String.prototype.toUpperCase; +const intrinsicTextEncoderEncode = TextEncoder.prototype.encode; +const intrinsicUint8ArraySet = Uint8Array.prototype.set; +const intrinsicUint8ArrayBufferGet = /** @type {(this: Uint8Array) => ArrayBufferLike} */ ( + prototypeGetter(Uint8Array.prototype, "buffer") +); +const intrinsicUint8ArrayByteLengthGet = /** @type {(this: Uint8Array) => number} */ ( + prototypeGetter(Uint8Array.prototype, "byteLength") +); +const intrinsicWeakSetAdd = WeakSet.prototype.add; +const intrinsicWeakSetDelete = WeakSet.prototype.delete; +const intrinsicWeakSetHas = WeakSet.prototype.has; +const intrinsicRequestHeadersGet = /** @type {(this: Request) => Headers} */ ( + prototypeGetter(Request.prototype, "headers") +); +const intrinsicRequestBodyGet = /** @type {(this: Request) => ReadableStream | null} */ ( + prototypeGetter(Request.prototype, "body") +); +const intrinsicRequestMethodGet = /** @type {(this: Request) => string} */ ( + prototypeGetter(Request.prototype, "method") +); +const intrinsicRequestUrlGet = /** @type {(this: Request) => string} */ ( + prototypeGetter(Request.prototype, "url") +); +const intrinsicResponseBodyGet = /** @type {(this: Response) => ReadableStream | null} */ ( + prototypeGetter(Response.prototype, "body") +); +const intrinsicResponseHeadersGet = /** @type {(this: Response) => Headers} */ ( + prototypeGetter(Response.prototype, "headers") +); +const intrinsicResponseStatusGet = /** @type {(this: Response) => number} */ ( + prototypeGetter(Response.prototype, "status") +); +const intrinsicResponseStatusTextGet = /** @type {(this: Response) => string} */ ( + prototypeGetter(Response.prototype, "statusText") +); +const intrinsicResponseWebSocketGet = /** @type {((this: Response) => WebSocket | null) | undefined} */ ( + prototypeGetter(Response.prototype, "webSocket") +); const utf8Encoder = new TextEncoder(); const DO_CONNECT_HEADERS = { @@ -87,6 +164,19 @@ const RPC_RESERVED_METHODS = new Set(["fetch", "alarm"]); * }} DoOwnerHintDispatchOptions */ +// workerd places some WebIDL mixin getters on a parent prototype. Resolve +// them once before tenant module evaluation rather than reading patched getters. +/** @param {object | null} prototype @param {string} name */ +function prototypeGetter(prototype, name) { + let current = prototype; + while (current) { + const getter = Object.getOwnPropertyDescriptor(current, name)?.get; + if (getter) return getter; + current = Object.getPrototypeOf(current); + } + return undefined; +} + /** @param {DoBindingProps} props @param {string} objectName */ export function doOwnerHintCacheKey(props, objectName) { return `${props.doStorageId}:${props.className}:${objectName}`; @@ -94,18 +184,219 @@ export function doOwnerHintCacheKey(props, objectName) { /** @param {Response} response */ export async function staleDoOwnerHintResponse(response) { - if (response.status < 400) return false; + if (responseStatus(response) < 400) return false; + const code = responseHeader(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER); + return code !== null && setHas(OWNER_HINT_STALE_CODES, code); +} + +/** @template T @param {T[]} values @param {(value: T) => void} callback */ +function forEachArray(values, callback) { + intrinsicReflectApply(intrinsicArrayForEach, values, [callback]); +} + +/** @template T @param {T[]} values @param {T} value */ +function pushArray(values, value) { + intrinsicReflectApply(intrinsicArrayPush, values, [value]); +} + +/** @param {Uint8Array} target @param {Uint8Array} source @param {number} offset */ +function setBytes(target, source, offset) { + intrinsicReflectApply(intrinsicUint8ArraySet, target, [source, offset]); +} + +/** @param {Uint8Array} value */ +function byteArrayBuffer(value) { + return intrinsicReflectApply(intrinsicUint8ArrayBufferGet, value, []); +} + +/** @param {Uint8Array} value */ +function byteArrayLength(value) { + return intrinsicReflectApply(intrinsicUint8ArrayByteLengthGet, value, []); +} + +/** @param {Headers} headers @param {string} name @param {string} value */ +function headerAppend(headers, name, value) { + intrinsicReflectApply(intrinsicHeadersAppend, headers, [name, value]); +} + +/** @param {Headers} headers @param {string} name */ +function headerDelete(headers, name) { + intrinsicReflectApply(intrinsicHeadersDelete, headers, [name]); +} + +/** @param {Headers} headers @param {string} name */ +function headerValue(headers, name) { + return intrinsicReflectApply(intrinsicHeadersGet, headers, [name]); +} + +/** @param {Headers} headers @param {string} name */ +function headerHas(headers, name) { + return intrinsicReflectApply(intrinsicHeadersHas, headers, [name]); +} + +/** @param {Headers} headers @param {string} name @param {string} value */ +function headerSet(headers, name, value) { + intrinsicReflectApply(intrinsicHeadersSet, headers, [name, value]); +} + +/** @param {Headers} source */ +function copyHeaders(source) { + const out = new IntrinsicHeaders(); + intrinsicReflectApply(intrinsicHeadersForEach, source, [ + /** @param {string} value @param {string} name */ + (value, name) => headerAppend(out, name, value), + ]); + return out; +} + +/** @param {Headers} headers */ +function headerEntries(headers) { + /** @type {[string, string][]} */ + const out = []; + intrinsicReflectApply(intrinsicHeadersForEach, headers, [ + /** @param {string} value @param {string} name */ + (value, name) => pushArray(out, [name, value]), + ]); + return out; +} + +/** @template T @param {Set} set @param {T} value */ +function setHas(set, value) { + return intrinsicReflectApply(intrinsicSetHas, set, [value]); +} + +/** @param {string} value */ +function stringToLowerCase(value) { + return intrinsicReflectApply(intrinsicStringToLowerCase, value, []); +} + +/** @param {string} value */ +function stringToUpperCase(value) { + return intrinsicReflectApply(intrinsicStringToUpperCase, value, []); +} + +/** @param {RegExp} regexp @param {string} value */ +function regexpTest(regexp, value) { + return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); +} + +/** @param {Request} request */ +function requestHeaders(request) { + return intrinsicReflectApply(intrinsicRequestHeadersGet, request, []); +} + +/** @param {Request} request */ +function requestBody(request) { + return intrinsicReflectApply(intrinsicRequestBodyGet, request, []); +} + +/** @param {Request} request */ +function requestMethod(request) { + return intrinsicReflectApply(intrinsicRequestMethodGet, request, []); +} + +/** @param {Request} request */ +function requestUrl(request) { + return intrinsicReflectApply(intrinsicRequestUrlGet, request, []); +} + +/** @param {ReadableStream} stream */ +function streamReader(stream) { + return /** @type {ReadableStreamDefaultReader} */ ( + intrinsicReflectApply(intrinsicReadableStreamGetReader, stream, []) + ); +} + +/** @param {ReadableStreamDefaultReader} reader */ +function readStreamChunk(reader) { + return intrinsicReflectApply(intrinsicReadableStreamReaderRead, reader, []); +} + +/** @param {ReadableStreamDefaultReader} reader */ +function releaseStreamReader(reader) { + intrinsicReflectApply(intrinsicReadableStreamReaderReleaseLock, reader, []); +} + +/** @param {ReadableStreamDefaultReader} reader */ +function cancelStreamReader(reader) { try { - const body = await response.clone().json(); - return OWNER_HINT_STALE_CODES.has(body?.error); + const promise = intrinsicReflectApply(intrinsicReadableStreamReaderCancel, reader, []); + intrinsicReflectApply(intrinsicPromiseThen, promise, [undefined, () => {}]); } catch { - return false; + // Cancellation is best-effort after the bounded reader has already rejected. } } +/** @param {Response} response */ +function responseBody(response) { + return intrinsicReflectApply(intrinsicResponseBodyGet, response, []); +} + +/** @param {Response} response */ +function responseHeaders(response) { + return intrinsicReflectApply(intrinsicResponseHeadersGet, response, []); +} + +/** @param {Response} response */ +function responseStatus(response) { + return intrinsicReflectApply(intrinsicResponseStatusGet, response, []); +} + +/** @param {Response} response */ +function responseStatusText(response) { + return intrinsicReflectApply(intrinsicResponseStatusTextGet, response, []); +} + +/** @param {Response} response */ +function responseWebSocket(response) { + if (!intrinsicResponseWebSocketGet) return null; + return intrinsicReflectApply(intrinsicResponseWebSocketGet, response, []); +} + +/** @param {Response} response @param {string} name */ +function responseHeader(response, name) { + return headerValue(responseHeaders(response), name); +} + /** @param {string} value */ function encodeUtf8(value) { - return utf8Encoder.encode(value); + return intrinsicReflectApply(intrinsicTextEncoderEncode, utf8Encoder, [value]); +} + +/** @param {unknown} value */ +function stringifyJson(value) { + return intrinsicReflectApply(intrinsicJsonStringify, IntrinsicJSON, [cloneJsonData(value)]); +} + +/** @param {unknown} value @param {WeakSet} [seen] */ +function cloneJsonData(value, seen = new IntrinsicWeakSet()) { + if (value === null || typeof value !== "object") return value; + const objectValue = /** @type {object} */ (value); + if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { + throw new TypeError("Cannot serialize circular JSON data"); + } + intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); + try { + if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { + const source = /** @type {unknown[]} */ (objectValue); + const out = new IntrinsicArray(source.length); + intrinsicReflectApply(intrinsicObjectSetPrototypeOf, IntrinsicObject, [out, null]); + for (let i = 0; i < source.length; i++) out[i] = cloneJsonData(source[i], seen); + return out; + } + const out = /** @type {Record} */ ( + intrinsicReflectApply(intrinsicObjectCreate, IntrinsicObject, [null]) + ); + const entries = /** @type {[string, unknown][]} */ ( + intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) + ); + forEachArray(entries, (entry) => { + out[entry[0]] = cloneJsonData(entry[1], seen); + }); + return out; + } finally { + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); + } } /** @@ -114,21 +405,25 @@ function encodeUtf8(value) { * @returns {Uint8Array} */ function encodeDoInvokeEnvelope(metadata, bodyBytes = null) { - const metadataBytes = encodeUtf8(JSON.stringify(metadata)); - const body = bodyBytes == null ? new Uint8Array() : bodyBytes; - if (metadataBytes.length + body.length + 4 > MAX_DO_INVOKE_ENVELOPE_BYTES) { + const metadataBytes = encodeUtf8(stringifyJson(metadata)); + const body = bodyBytes == null ? new IntrinsicUint8Array() : bodyBytes; + const metadataLength = byteArrayLength(metadataBytes); + const bodyLength = byteArrayLength(body); + const envelopeLength = 4 + metadataLength + bodyLength; + if (envelopeLength > MAX_DO_INVOKE_ENVELOPE_BYTES) { throw new TypeError(`Durable Object invoke envelope exceeds ${MAX_DO_INVOKE_ENVELOPE_BYTES} bytes`); } - const envelope = new Uint8Array(4 + metadataBytes.length + body.length); - new DataView(envelope.buffer, envelope.byteOffset, envelope.byteLength).setUint32(0, metadataBytes.length, false); - envelope.set(metadataBytes, 4); - envelope.set(body, 4 + metadataBytes.length); + const envelope = new IntrinsicUint8Array(envelopeLength); + const view = new IntrinsicDataView(byteArrayBuffer(envelope), 0, envelopeLength); + intrinsicReflectApply(intrinsicDataViewSetUint32, view, [0, metadataLength, false]); + setBytes(envelope, metadataBytes, 4); + setBytes(envelope, body, 4 + metadataLength); return envelope; } /** @param {string} value */ function byteLength(value) { - return utf8Encoder.encode(value).byteLength; + return byteArrayLength(encodeUtf8(value)); } /** @@ -138,40 +433,41 @@ function byteLength(value) { async function readRequestBodyBytes(request) { // This source is injected into loaded workers, so keep the bounded reader // local instead of importing a helper that would add another facade module. - const contentLength = request.headers.get("content-length"); + const contentLength = headerValue(requestHeaders(request), "content-length"); if (contentLength != null && contentLength !== "") { const declared = Number(contentLength); if (Number.isFinite(declared) && declared > MAX_DO_REQUEST_BODY_BYTES) { throw new TypeError(`Durable Object fetch body exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } } - if (!request.body) return new Uint8Array(); + const requestStream = requestBody(request); + if (!requestStream) return new IntrinsicUint8Array(); - const reader = request.body.getReader(); + const reader = streamReader(requestStream); /** @type {Uint8Array[]} */ const chunks = []; let total = 0; try { while (true) { - const { done, value } = await reader.read(); + const { done, value } = await readStreamChunk(reader); if (done) break; - total += value.byteLength; + total += byteArrayLength(value); if (total > MAX_DO_REQUEST_BODY_BYTES) { - reader.cancel().catch(() => {}); + cancelStreamReader(reader); throw new TypeError(`Durable Object fetch body exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } - chunks.push(value); + pushArray(chunks, value); } } finally { - reader.releaseLock(); + releaseStreamReader(reader); } - const body = new Uint8Array(total); + const body = new IntrinsicUint8Array(total); let offset = 0; - for (const chunk of chunks) { - body.set(chunk, offset); - offset += chunk.byteLength; - } + forEachArray(chunks, (chunk) => { + setBytes(body, chunk, offset); + offset += byteArrayLength(chunk); + }); return body; } @@ -181,51 +477,65 @@ async function readRequestBodyBytes(request) { * @param {WeakSet} [seen] * @returns {string | null} */ -function jsonDataError(value, field, seen = new WeakSet()) { +function jsonDataError(value, field, seen = new IntrinsicWeakSet()) { if (value === null) return null; const type = typeof value; if (type === "string" || type === "boolean") return null; - if (type === "number") return Number.isFinite(value) ? null : `${field} must be a finite number`; + if (type === "number") { + return intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [value]) + ? null + : `${field} must be a finite number`; + } if (type === "bigint" || type === "function" || type === "symbol" || type === "undefined") { return `${field} must be JSON data`; } if (type !== "object") return `${field} must be JSON data`; const objectValue = /** @type {Record | unknown[]} */ (value); - if (seen.has(objectValue)) return `${field} must not be circular`; - seen.add(objectValue); - if (Array.isArray(objectValue)) { - for (let i = 0; i < objectValue.length; i++) { - if (!(i in objectValue)) return `${field} must not be sparse`; - const error = jsonDataError(objectValue[i], `${field}[${i}]`, seen); + if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { + return `${field} must not be circular`; + } + intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); + if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { + const arrayValue = /** @type {unknown[]} */ (objectValue); + for (let i = 0; i < arrayValue.length; i++) { + if (!(i in arrayValue)) return `${field} must not be sparse`; + const error = jsonDataError(arrayValue[i], `${field}[${i}]`, seen); if (error) return error; } - seen.delete(objectValue); + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); return null; } - const proto = Object.getPrototypeOf(objectValue); - if (proto !== Object.prototype && proto !== null) return `${field} must be a plain JSON object`; - if (Object.getOwnPropertySymbols(objectValue).length > 0) return `${field} must not contain symbol keys`; - for (const [key, entry] of Object.entries(objectValue)) { - const error = jsonDataError(entry, `${field}.${key}`, seen); + const proto = intrinsicReflectApply(intrinsicObjectGetPrototypeOf, IntrinsicObject, [objectValue]); + if (proto !== IntrinsicObject.prototype && proto !== null) return `${field} must be a plain JSON object`; + const symbols = intrinsicReflectApply(intrinsicObjectGetOwnPropertySymbols, IntrinsicObject, [objectValue]); + if (symbols.length > 0) return `${field} must not contain symbol keys`; + const entries = /** @type {[string, unknown][]} */ ( + intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) + ); + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + const error = jsonDataError(entry[1], `${field}.${entry[0]}`, seen); if (error) return error; } - seen.delete(objectValue); + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); return null; } /** @param {unknown} args */ export function assertJsonRpcArgs(args) { - if (!Array.isArray(args)) throw new TypeError("rpc.args must be an array"); + if (!intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [args])) { + throw new TypeError("rpc.args must be an array"); + } const error = jsonDataError(args, "rpc.args"); if (error) throw new TypeError(error); } /** @param {unknown} method */ export function assertRpcMethod(method) { - if (typeof method !== "string" || !/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(method)) { + if (typeof method !== "string" || !regexpTest(/^[A-Za-z_$][A-Za-z0-9_$]*$/, method)) { throw new TypeError("rpc.method is not valid"); } - if (RPC_RESERVED_METHODS.has(method)) { + if (setHas(RPC_RESERVED_METHODS, method)) { throw new TypeError("rpc.method is reserved"); } } @@ -240,7 +550,7 @@ export function assertRpcMethod(method) { export function rpcInvokeBody(props, objectName, method, args) { assertRpcMethod(method); assertJsonRpcArgs(args); - if (byteLength(JSON.stringify(args)) > MAX_DO_REQUEST_BODY_BYTES) { + if (byteLength(stringifyJson(args)) > MAX_DO_REQUEST_BODY_BYTES) { throw new TypeError(`rpc.args exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } return encodeDoInvokeEnvelope({ @@ -311,7 +621,7 @@ export function rpcInvokeInit(props, objectName, method, args, requestId) { export async function rpcResultFromResponse(response) { const body = await response.json().catch(() => null); if (!response.ok) { - const err = new Error(body?.message || `Durable Object RPC failed with status ${response.status}`); + const err = new Error(body?.message || `Durable Object RPC failed with status ${responseStatus(response)}`); if (body?.name) err.name = body.name; if (body?.error) Object.defineProperty(err, "code", { value: body.error }); if (typeof body?.stack === "string" && body.stack) err.stack = body.stack; @@ -322,13 +632,9 @@ export async function rpcResultFromResponse(response) { /** @param {Response} response */ export async function retryableOwnerRaceResponse(response) { - if (response.status !== 503) return false; - try { - const body = await response.clone().json(); - return OWNER_RACE_RETRY_CODES.has(body?.error); - } catch { - return false; - } + if (responseStatus(response) !== 503) return false; + const code = responseHeader(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER); + return code !== null && setHas(OWNER_RACE_RETRY_CODES, code); } /** @@ -337,25 +643,26 @@ export async function retryableOwnerRaceResponse(response) { * @returns {Promise<{ spec: { method: string, url: string, headers: [string, string][] }, bodyBytes: Uint8Array | null }>} */ export async function requestSpec(request, requestId) { - const forwarded = new Request(request); - for (const name of DO_FETCH_STRIP_HEADERS) { - forwarded.headers.delete(name); - } - if (requestId && !forwarded.headers.has("x-request-id")) { - forwarded.headers.set("x-request-id", requestId); + const forwarded = new IntrinsicRequest(request); + const forwardedHeaders = requestHeaders(forwarded); + forEachArray(DO_FETCH_STRIP_HEADERS, (name) => { + headerDelete(forwardedHeaders, name); + }); + if (requestId && !headerHas(forwardedHeaders, "x-request-id")) { + headerSet(forwardedHeaders, "x-request-id", requestId); } - const method = forwarded.method.toUpperCase(); - const headers = [...forwarded.headers.entries()]; + const method = stringToUpperCase(requestMethod(forwarded)); + const headers = headerEntries(forwardedHeaders); enforceRequestHeadersBudget(headers); const spec = { method, - url: forwarded.url, + url: requestUrl(forwarded), headers, }; let bodyBytes = null; if (method !== "GET" && method !== "HEAD") { const body = await readRequestBodyBytes(forwarded); - if (body.byteLength > 0) bodyBytes = body; + if (byteArrayLength(body) > 0) bodyBytes = body; } return { spec, bodyBytes }; } @@ -366,22 +673,24 @@ function enforceRequestHeadersBudget(headers) { throw new TypeError(`Durable Object fetch headers exceed ${MAX_DO_REQUEST_HEADER_COUNT} entries`); } let total = 0; - for (const [name, value] of headers) { + forEachArray(headers, (entry) => { + const name = entry[0]; + const value = entry[1]; total += byteLength(name) + byteLength(value); if (total > MAX_DO_REQUEST_HEADER_BYTES) { throw new TypeError(`Durable Object fetch headers exceed ${MAX_DO_REQUEST_HEADER_BYTES} bytes`); } - } + }); } /** @param {Request} request */ export function isWebSocketUpgrade(request) { - return (request.headers.get("Upgrade") || "").toLowerCase() === "websocket"; + return stringToLowerCase(headerValue(requestHeaders(request), "Upgrade") || "") === "websocket"; } /** @param {Request} request */ export function replayOwnerUnavailableForFetch(request) { - const method = request.method.toUpperCase(); + const method = stringToUpperCase(requestMethod(request)); return method === "GET" || method === "HEAD"; } @@ -392,17 +701,17 @@ export function replayOwnerUnavailableForFetch(request) { * @param {string | null | undefined} requestId */ export function connectHeaders(props, objectName, request, requestId) { - const headers = new Headers(request.headers); - for (const name of DO_CONNECT_STRIP_HEADERS) headers.delete(name); - headers.set(DO_CONNECT_HEADERS.ns, props.ns); - headers.set(DO_CONNECT_HEADERS.worker, props.worker); - headers.set(DO_CONNECT_HEADERS.version, props.version); - headers.set(DO_CONNECT_HEADERS.doStorageId, props.doStorageId); - headers.set(DO_CONNECT_HEADERS.className, props.className); - headers.set(DO_CONNECT_HEADERS.objectName, objectName); - headers.set(DO_CONNECT_HEADERS.requestUrl, request.url); - if (requestId && !headers.has("x-request-id")) headers.set("x-request-id", requestId); - headers.set(DO_ACCEPT_OWNER_HINT_HEADER, "1"); + const headers = copyHeaders(requestHeaders(request)); + forEachArray(DO_CONNECT_STRIP_HEADERS, (name) => headerDelete(headers, name)); + headerSet(headers, DO_CONNECT_HEADERS.ns, props.ns); + headerSet(headers, DO_CONNECT_HEADERS.worker, props.worker); + headerSet(headers, DO_CONNECT_HEADERS.version, props.version); + headerSet(headers, DO_CONNECT_HEADERS.doStorageId, props.doStorageId); + headerSet(headers, DO_CONNECT_HEADERS.className, props.className); + headerSet(headers, DO_CONNECT_HEADERS.objectName, objectName); + headerSet(headers, DO_CONNECT_HEADERS.requestUrl, requestUrl(request)); + if (requestId && !headerHas(headers, "x-request-id")) headerSet(headers, "x-request-id", requestId); + headerSet(headers, DO_ACCEPT_OWNER_HINT_HEADER, "1"); return headers; } @@ -438,25 +747,26 @@ export function ownerHintHeaders(owner, { control = false } = {}) { * @returns {DoOwnerHint | null} */ export function ownerHintFromHeaders(headers) { - const ownerKey = headers.get(DO_OWNER_HINT_HEADERS.ownerKey); - const taskId = headers.get(DO_OWNER_HINT_HEADERS.taskId); - const endpoint = headers.get(DO_OWNER_HINT_HEADERS.endpoint); - const rawGeneration = headers.get(DO_OWNER_HINT_HEADERS.generation); + const ownerKey = headerValue(headers, DO_OWNER_HINT_HEADERS.ownerKey); + const taskId = headerValue(headers, DO_OWNER_HINT_HEADERS.taskId); + const endpoint = headerValue(headers, DO_OWNER_HINT_HEADERS.endpoint); + const rawGeneration = headerValue(headers, DO_OWNER_HINT_HEADERS.generation); if (rawGeneration == null || rawGeneration === "") return null; const generation = Number(rawGeneration); if (!ownerKey || !taskId || !validOwnerEndpoint(endpoint) || !Number.isInteger(generation) || generation < 0) { return null; } - return { ownerKey, taskId, endpoint: String(endpoint), generation }; + return { ownerKey, taskId, endpoint: /** @type {string} */ (endpoint), generation }; } /** @param {Response} response */ export async function ownerHintFromResponse(response) { - if (response.status !== 409) return null; + if (responseStatus(response) !== 409) return null; // Only do-runtime-authored headers are trusted. Tenant DO code controls the // response body and may intentionally return a do_owner_hint-shaped 409. - if (response.headers.get(DO_OWNER_HINT_CONTROL_HEADER) !== "1") return null; - return ownerHintFromHeaders(response.headers); + const headers = responseHeaders(response); + if (headerValue(headers, DO_OWNER_HINT_CONTROL_HEADER) !== "1") return null; + return ownerHintFromHeaders(headers); } /** @param {unknown} endpoint */ @@ -475,7 +785,7 @@ export async function followOwnerHint(response, ownerFetch, pathname, init) { if (!hint || typeof ownerFetch !== "function") return response; const direct = await ownerFetch(ownerRequestUrl(hint, pathname), init); if (ownerEndpointUnavailableResponse(direct)) { - throw new Error(`DO owner endpoint returned ${direct.status}`); + throw new Error(`DO owner endpoint returned ${responseStatus(direct)}`); } return direct; } @@ -529,7 +839,7 @@ export async function dispatchDoRequestWithOwnerHint({ } return ownerUnavailableResponse(); } else { - const learned = ownerHintFromHeaders(direct.headers); + const learned = ownerHintFromHeaders(responseHeaders(direct)); if (learned) rememberHint(learned); return direct; } @@ -548,7 +858,7 @@ export async function dispatchDoRequestWithOwnerHint({ if (!hinted || typeof ownerFetch !== "function") { if (hinted) rememberHint(hinted); else { - const learned = ownerHintFromHeaders(routed.headers); + const learned = ownerHintFromHeaders(responseHeaders(routed)); if (learned) rememberHint(learned); } return routed; @@ -556,7 +866,7 @@ export async function dispatchDoRequestWithOwnerHint({ rememberHint(hinted); try { const direct = await followOwnerHint(routed, ownerFetch, ownerPath, init); - const learned = ownerHintFromHeaders(direct.headers); + const learned = ownerHintFromHeaders(responseHeaders(direct)); if (learned) rememberHint(learned); return direct; } catch { @@ -619,29 +929,32 @@ export async function dispatchDoConnectWithHintCache(options) { /** @param {Response} response */ export function ownerEndpointUnavailableResponse(response) { - return OWNER_ENDPOINT_UNAVAILABLE_STATUSES.has(response.status) && - !ownerHintFromHeaders(response.headers); + return setHas(OWNER_ENDPOINT_UNAVAILABLE_STATUSES, responseStatus(response)) && + !ownerHintFromHeaders(responseHeaders(response)); } /** @param {RequestInit} init */ export function withoutOwnerHintOptIn(init) { - const headers = new Headers(init.headers || {}); - headers.delete(DO_ACCEPT_OWNER_HINT_HEADER); + const headers = init.headers instanceof IntrinsicHeaders + ? copyHeaders(init.headers) + : new IntrinsicHeaders(init.headers || {}); + headerDelete(headers, DO_ACCEPT_OWNER_HINT_HEADER); return { ...init, headers }; } /** @param {Response} response */ export function stripOwnerHintHeaders(response) { - const headers = new Headers(response.headers); - for (const name of DO_OWNER_HINT_STRIP_HEADERS) { - headers.delete(name); - } + const headers = copyHeaders(responseHeaders(response)); + forEachArray(DO_OWNER_HINT_STRIP_HEADERS, (name) => { + headerDelete(headers, name); + }); + const status = responseStatus(response); const init = /** @type {ResponseInit & { webSocket?: WebSocket }} */ ({ - status: response.status, - statusText: response.statusText, + status, + statusText: responseStatusText(response), headers, }); - const webSocket = /** @type {{ webSocket?: WebSocket }} */ (response).webSocket; + const webSocket = responseWebSocket(response); if (webSocket) init.webSocket = webSocket; - return new Response(response.status === 101 ? null : response.body, init); + return new IntrinsicResponse(status === 101 ? null : responseBody(response), init); } diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index c4795a8..145a414 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -336,6 +336,7 @@ const tailWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "tail-worker.js"), (name = "hex.js", esModule = embed "../shared/hex.js"), + (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-bindings-proxy", esModule = embed "bindings/proxy.js"), diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index 9819b04..110ea95 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -153,6 +153,7 @@ const tailWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "tail-worker.js"), (name = "hex.js", esModule = embed "../shared/hex.js"), + (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-bindings-proxy", esModule = embed "bindings/proxy.js"), diff --git a/runtime/load/code-budget.js b/runtime/load/code-budget.js index 120714a..39a9c84 100644 --- a/runtime/load/code-budget.js +++ b/runtime/load/code-budget.js @@ -10,6 +10,8 @@ import { rewriteCloudflareWorkflowsImports, } from "runtime-load-module-rewrite"; import { + HOST_BINDING_RUNTIME_MODULE_NAME, + HOST_BINDING_RUNTIME_SOURCE, generateAbortShimWrapperModule, generateHostBindingWrapperModule, } from "runtime-load-wrapper-generate"; @@ -281,6 +283,9 @@ function runtimeInjectedModuleSources(mainModule, meta, runtimeSources, plan = a addModules(injections.workflowsModuleInjections); } out.set(WORKFLOWS_MODULE_NAME, WORKFLOWS_MODULE_SOURCE); + if (plan.needsHostBindingWrapper) { + out.set(HOST_BINDING_RUNTIME_MODULE_NAME, HOST_BINDING_RUNTIME_SOURCE); + } // `_wdl-wrapper.js` is always injected: host bindings use the larger wrapper, // and otherwise the abort shim still rewrites the user main module. out.set( diff --git a/runtime/load/module-rewrite.js b/runtime/load/module-rewrite.js index ca85636..7097513 100644 --- a/runtime/load/module-rewrite.js +++ b/runtime/load/module-rewrite.js @@ -26,6 +26,7 @@ export const HOST_BINDING_RESERVED_MODULE_NAMES = Object.freeze([ "_wdl-owner-hint-cache.js", "_wdl-request-id.js", "_wdl-workflows-client.js", + "_wdl-host-wrapper-runtime.js", "_wdl-wrapper.js", ]); export const HOST_BINDING_RESERVED_MODULES = new Set(HOST_BINDING_RESERVED_MODULE_NAMES); diff --git a/runtime/load/wrapper-generate.js b/runtime/load/wrapper-generate.js index e18035b..611d44c 100644 --- a/runtime/load/wrapper-generate.js +++ b/runtime/load/wrapper-generate.js @@ -10,6 +10,80 @@ export class __WdlAbort__ extends WorkerEntrypoint { const DEFAULT_EXPORT_SOURCE_SNIPPET = "const source = Function.prototype.toString.call(raw);"; const DEFAULT_EXPORT_CLASS_TEST_SOURCE = "/^\\s*class\\b/.test(source)"; +export const HOST_BINDING_RUNTIME_MODULE_NAME = "_wdl-host-wrapper-runtime.js"; +export const HOST_BINDING_RUNTIME_SOURCE = ` +const IntrinsicObject = Object; +const IntrinsicPromise = Promise; +const IntrinsicProxy = Proxy; +const IntrinsicReflect = Reflect; +const IntrinsicSymbol = Symbol; +const intrinsicArrayForEach = Array.prototype.forEach; +const intrinsicFunctionToString = Function.prototype.toString; +const intrinsicObjectDefineProperty = Object.defineProperty; +const intrinsicObjectEntries = Object.entries; +const intrinsicObjectKeys = Object.keys; +const intrinsicPromiseResolve = Promise.resolve; +const intrinsicPromiseThen = Promise.prototype.then; +const intrinsicReflectApply = Reflect.apply; +const intrinsicReflectGet = Reflect.get; +const intrinsicRegExpTest = RegExp.prototype.test; + +export function applyFunction(fn, receiver, args) { + return intrinsicReflectApply(fn, receiver, args); +} + +export function createPrivateSymbol(description) { + return IntrinsicSymbol(description); +} + +export function createProxy(target, handler) { + return new IntrinsicProxy(target, handler); +} + +export function defineProperty(target, property, descriptor) { + return intrinsicReflectApply(intrinsicObjectDefineProperty, IntrinsicObject, [target, property, descriptor]); +} + +export function forEachArray(values, callback) { + intrinsicReflectApply(intrinsicArrayForEach, values, [callback]); +} + +export function forEachObjectEntry(record, callback) { + const entries = intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [record]); + forEachArray(entries, (entry) => callback(entry[0], entry[1])); +} + +export function functionSource(fn) { + return intrinsicReflectApply(intrinsicFunctionToString, fn, []); +} + +export function objectKeys(record) { + return intrinsicReflectApply(intrinsicObjectKeys, IntrinsicObject, [record]); +} + +export function reflectGet(target, property, receiver) { + return intrinsicReflectApply(intrinsicReflectGet, IntrinsicReflect, [target, property, receiver]); +} + +export function regexpTest(regexp, value) { + return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); +} + +export function settleWithFinally(value, callback) { + const promise = intrinsicReflectApply(intrinsicPromiseResolve, IntrinsicPromise, [value]); + return intrinsicReflectApply(intrinsicPromiseThen, promise, [ + (result) => { + callback(); + return result; + }, + (error) => { + callback(); + throw error; + }, + ]); +} +`; + /** @param {string} userMainSpecifier */ export function generateAbortShimWrapperModule(userMainSpecifier) { const userMain = JSON.stringify(`./${userMainSpecifier}`); @@ -78,12 +152,25 @@ export class ${name} extends user.${name} { } `).join(""); return ` -import * as user from ${userMain}; import { WorkerEntrypoint, abortIsolate } from "cloudflare:workers"; +import { + applyFunction, + createPrivateSymbol, + createProxy, + defineProperty, + forEachArray, + forEachObjectEntry, + functionSource, + objectKeys, + reflectGet, + regexpTest, + settleWithFinally, +} from "./${HOST_BINDING_RUNTIME_MODULE_NAME}"; ${d1Import} ${r2Import} ${doImport} ${workflowImport} +import * as user from ${userMain}; // Local wrapper subclasses intentionally shadow same-name user exports, so // declared entrypoints get facade-aware env even when star exports are present. ${starExport} @@ -103,7 +190,7 @@ const WORKFLOW_BINDINGS = ${workflowBindingJson}; const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; -const HOST_BINDINGS_WRAPPED = Symbol("wdl.host-bindings-wrapped"); +const HOST_BINDINGS_WRAPPED = createPrivateSymbol("wdl.host-bindings-wrapped"); const INTERNAL_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; function requestIdFromEventArg(arg) { @@ -139,7 +226,7 @@ function withRequestContext(context, arg, fn) { try { const result = fn(); if (result && typeof result.then === "function") { - return Promise.resolve(result).finally(() => { + return settleWithFinally(result, () => { context.requestId = previous; }); } @@ -152,12 +239,12 @@ function withRequestContext(context, arg, fn) { } function wrapClassInstance(instance, requestContext) { - return new Proxy(instance, { + return createProxy(instance, { get(target, prop, receiver) { - const value = Reflect.get(target, prop, receiver); + const value = reflectGet(target, prop, receiver); if (typeof value !== "function") return value; return function(...args) { - return withRequestContext(requestContext, args[0], () => value.apply(target, args)); + return withRequestContext(requestContext, args[0], () => applyFunction(value, target, args)); }; }, }); @@ -175,26 +262,26 @@ function wrapEnv(env, requestIdOrContext = null) { delete out[DO_BACKEND_BINDING]; delete out[DO_OWNER_NETWORK_BINDING]; delete out[WORKFLOWS_BACKEND_BINDING]; - for (const name of Object.keys(out)) { - if (INTERNAL_BINDING_RE.test(name)) delete out[name]; - } - for (const name of D1_BINDINGS) { + forEachArray(objectKeys(out), (name) => { + if (regexpTest(INTERNAL_BINDING_RE, name)) delete out[name]; + }); + forEachArray(D1_BINDINGS, (name) => { if (out[name] !== undefined) out[name] = new D1Database(out[name], requestIdOptions(requestIdOrContext)); - } - for (const name of R2_BINDINGS) { + }); + forEachArray(R2_BINDINGS, (name) => { if (out[name] !== undefined) out[name] = new R2Bucket(out[name], requestIdOptions(requestIdOrContext)); - } - for (const name of DO_BINDINGS) { + }); + forEachArray(DO_BINDINGS, (name) => { if (out[name] !== undefined) { out[name] = new DurableObjectNamespace(out[name], doOptions(requestIdOrContext, doBackend, doOwnerNetwork)); } - } - for (const [name, metadata] of Object.entries(WORKFLOW_BINDINGS)) { + }); + forEachObjectEntry(WORKFLOW_BINDINGS, (name, metadata) => { if (out[name] !== undefined) { out[name] = new Workflow(out[name] || metadata, workflowOptions(requestIdOrContext, workflowsBackend)); } - } - Object.defineProperty(out, HOST_BINDINGS_WRAPPED, { value: true }); + }); + defineProperty(out, HOST_BINDINGS_WRAPPED, { value: true }); return out; } @@ -244,7 +331,7 @@ async function notifyWorkflowCallback(request, env) { function wrapHandler(owner, fn) { return function(arg1, env, ctx) { - return fn.call(owner, arg1, wrapEnv(env, requestIdFromEventArg(arg1)), ctx); + return applyFunction(fn, owner, [arg1, wrapEnv(env, requestIdFromEventArg(arg1)), ctx]); }; } @@ -258,15 +345,20 @@ if (raw && typeof raw === "object") { const wrapDefaultFunctionKey = (key) => { const fn = raw[key]; if (typeof fn === "function") { - wrappedDefault[key] = wrapHandler(raw, fn); + defineProperty(wrappedDefault, key, { + value: wrapHandler(raw, fn), + writable: true, + enumerable: true, + configurable: true, + }); } }; - for (const key of HOST_WRAPPED_HANDLER_KEYS) { + forEachArray(HOST_WRAPPED_HANDLER_KEYS, (key) => { wrapDefaultFunctionKey(key); - } + }); } else if (typeof raw === "function") { - ${DEFAULT_EXPORT_SOURCE_SNIPPET} - if (${DEFAULT_EXPORT_CLASS_TEST_SOURCE}) { + const source = functionSource(raw); + if (regexpTest(/^\\s*class\\b/, source)) { wrappedDefault = class extends raw { constructor(ctx, env) { const requestContext = createRequestContext(); diff --git a/runtime/r2-client.js b/runtime/r2-client.js index 69dedb3..67d849f 100644 --- a/runtime/r2-client.js +++ b/runtime/r2-client.js @@ -120,21 +120,25 @@ function cappedReadableStream(stream, operation) { }); } -/** @param {Headers} headers */ -function headersToHttpMetadata(headers) { +/** @param {Headers} headers @param {{ strictExpiry?: boolean }} [options] */ +function headersToHttpMetadata(headers, { strictExpiry = false } = {}) { /** @type {AnyRecord} */ const out = {}; for (const [field, header] of R2_HTTP_METADATA_FIELDS) { out[field] = headers.get(header) || undefined; } - out.cacheExpiry = r2CacheExpiryFromHeaders(headers); + const cacheExpiry = r2CacheExpiryFromHeaders(headers, { canonical: strictExpiry }); + if (strictExpiry && headers.has("expires") && cacheExpiry === undefined) { + throw new TypeError("R2 httpMetadata Expires header must be canonical IMF-fixdate"); + } + out.cacheExpiry = cacheExpiry; return out; } /** @param {unknown} input */ function normalizeHttpMetadata(input) { if (input == null) return undefined; - if (input instanceof Headers) return headersToHttpMetadata(input); + if (input instanceof Headers) return headersToHttpMetadata(input, { strictExpiry: true }); if (typeof input !== "object" || Array.isArray(input)) { throw new TypeError("R2 httpMetadata must be an object or Headers"); } @@ -424,6 +428,7 @@ export class R2Bucket { /** @param {string} key @param {unknown} value @param {AnyRecord} [options] */ async put(key, value, options) { + const normalizedOptions = normalizePutOptions(options); const bytes = await valueToBytes(value); assertR2BufferSize(bytes.byteLength, "put"); const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); @@ -431,7 +436,7 @@ export class R2Bucket { const meta = await stub.put( normalizeR2ObjectKey(key), bytes, - normalizePutOptions(options), + normalizedOptions, bucketRequestMeta(this) ); return meta ? new R2Object(meta) : null; diff --git a/runtime/r2-utils.js b/runtime/r2-utils.js index 91db592..d25aeca 100644 --- a/runtime/r2-utils.js +++ b/runtime/r2-utils.js @@ -10,12 +10,18 @@ export const R2_HTTP_METADATA_FIELDS = Object.freeze([ Object.freeze(["contentEncoding", "content-encoding"]), Object.freeze(["cacheControl", "cache-control"]), ]); +const R2_IMF_FIXDATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/; -/** @param {Headers} headers */ -export function r2CacheExpiryFromHeaders(headers) { +/** @param {Headers} headers @param {{ canonical?: boolean }} [options] */ +export function r2CacheExpiryFromHeaders(headers, { canonical = false } = {}) { const expires = headers.get("expires"); if (!expires) return undefined; const ms = new Date(expires).getTime(); + if (canonical && ( + !R2_IMF_FIXDATE_RE.test(expires) || + !Number.isFinite(ms) || + new Date(ms).toUTCString() !== expires + )) return undefined; return Number.isFinite(ms) ? ms : undefined; } diff --git a/rust/common/src/internal_auth.rs b/rust/common/src/internal_auth.rs index 6f690e5..d7e29ad 100644 --- a/rust/common/src/internal_auth.rs +++ b/rust/common/src/internal_auth.rs @@ -20,9 +20,13 @@ pub struct InternalAuthTokens { } fn validate_internal_auth_token(name: &str, token: String) -> Result { - if token.is_empty() || !token.is_ascii() { + if token.is_empty() + || token + .bytes() + .any(|byte| !(0x21..=0x7e).contains(&byte) || byte == b',') + { return Err(format!( - "{name} must be configured as a non-empty ASCII string" + "{name} must be configured as visible ASCII without whitespace or commas" )); } Ok(token) @@ -76,9 +80,12 @@ pub fn internal_auth_matches_any(actual: Option<&str>, expected: &InternalAuthTo #[cfg(feature = "axum")] pub fn internal_auth_header_value(headers: &HeaderMap) -> Option<&str> { - headers - .get(INTERNAL_AUTH_HEADER) - .and_then(|value| value.to_str().ok()) + let mut values = headers.get_all(INTERNAL_AUTH_HEADER).iter(); + let value = values.next()?.to_str().ok()?; + if values.next().is_some() { + return None; + } + Some(value) } #[cfg(feature = "axum")] @@ -124,6 +131,14 @@ mod tests { INTERNAL_AUTH_FAILURE_MESSAGE ); + for case in contract["tokenCases"].as_array().expect("tokenCases array") { + let value = case["value"].as_str().expect("token value").to_string(); + assert_eq!( + validate_internal_auth_token(INTERNAL_AUTH_ENV, value).is_ok(), + case["accepted"].as_bool().expect("accepted boolean") + ); + } + for case in contract["rotationCases"] .as_array() .expect("rotationCases array") @@ -137,6 +152,28 @@ mod tests { case["accepted"].as_bool().expect("accepted boolean") ); } + + #[cfg(feature = "axum")] + for case in contract["headerCases"] + .as_array() + .expect("headerCases array") + { + let tokens = InternalAuthTokens { + current: case["current"].as_str().expect("current token").to_string(), + previous: case["previous"].as_str().map(str::to_string), + }; + let mut headers = HeaderMap::new(); + for value in case["values"].as_array().expect("header values") { + headers.append( + INTERNAL_AUTH_HEADER, + value.as_str().expect("header value").parse().unwrap(), + ); + } + assert_eq!( + internal_auth_headers_match(&headers, &tokens), + case["accepted"].as_bool().expect("accepted boolean") + ); + } } #[test] @@ -181,13 +218,21 @@ mod tests { } #[test] - fn internal_auth_tokens_must_be_ascii() { + fn internal_auth_tokens_must_be_visible_ascii_without_commas() { assert_eq!( validate_internal_auth_token(INTERNAL_AUTH_ENV, "ascii-token".to_string()).unwrap(), "ascii-token" ); assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, "".to_string()).is_err()); assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, "tokén".to_string()).is_err()); + assert!( + validate_internal_auth_token(INTERNAL_AUTH_ENV, "token,other".to_string()).is_err() + ); + assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, " token".to_string()).is_err()); + assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, "token ".to_string()).is_err()); + assert!( + validate_internal_auth_token(INTERNAL_AUTH_ENV, "token\0value".to_string()).is_err() + ); } #[cfg(feature = "axum")] diff --git a/rust/common/src/request_id.rs b/rust/common/src/request_id.rs index b041fd1..7de1db5 100644 --- a/rust/common/src/request_id.rs +++ b/rust/common/src/request_id.rs @@ -1,25 +1,34 @@ +#[cfg(feature = "axum")] +use axum::http::HeaderMap; + +#[cfg(feature = "axum")] +const REQUEST_ID_HEADER: &str = "x-request-id"; + pub fn sanitize_request_id(raw: &str) -> Option { let first = raw .split(',') .next()? - .trim_matches(|ch: char| ch.is_whitespace() || ch == '\u{feff}'); + .trim_matches(|ch: char| ch.is_ascii_whitespace()); if first.is_empty() || first.len() > 128 { return None; } - if first.chars().any(|ch| { - ch.is_whitespace() - || ch == '\u{feff}' - || ch == '"' - || ch == '\\' - || (ch as u32) < 0x20 - || ch == '\u{7f}' - || ('\u{80}'..='\u{9f}').contains(&ch) - }) { + if first + .bytes() + .any(|byte| !(0x21..=0x7e).contains(&byte) || byte == b'"' || byte == b'\\') + { return None; } Some(first.to_string()) } +#[cfg(feature = "axum")] +pub fn request_id_from_headers(headers: &HeaderMap) -> Option { + headers + .get(REQUEST_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .and_then(sanitize_request_id) +} + #[cfg(test)] mod tests { use super::*; @@ -48,6 +57,21 @@ mod tests { assert_eq!(sanitize_request_id("bad\"id"), None); assert_eq!(sanitize_request_id("bad\\id"), None); assert_eq!(sanitize_request_id("bad\nid"), None); + assert_eq!(sanitize_request_id("café"), None); + assert_eq!(sanitize_request_id("\u{85}rid"), None); + } + + #[cfg(feature = "axum")] + #[test] + fn request_id_header_adapter_rejects_non_ascii_wire_values() { + use axum::http::HeaderValue; + + let mut headers = HeaderMap::new(); + headers.insert( + REQUEST_ID_HEADER, + HeaderValue::from_bytes("é".as_bytes()).expect("obs-text header value"), + ); + assert_eq!(request_id_from_headers(&headers), None); } #[test] diff --git a/rust/redis-proxy/src/lib.rs b/rust/redis-proxy/src/lib.rs index 7cd4ac1..b66e38e 100644 --- a/rust/redis-proxy/src/lib.rs +++ b/rust/redis-proxy/src/lib.rs @@ -7,7 +7,7 @@ use axum::body::Body; use axum::extract::DefaultBodyLimit; use axum::extract::State; use axum::http::header::CONTENT_LENGTH; -use axum::http::{HeaderMap, HeaderValue, Request, StatusCode}; +use axum::http::{HeaderValue, Request, StatusCode}; use axum::middleware::{self, Next}; use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post, put}; @@ -21,7 +21,7 @@ use wdl_rust_common::internal_auth::{ }; use wdl_rust_common::metrics::prometheus_response; use wdl_rust_common::redis_conn::RedisConnection; -use wdl_rust_common::request_id::sanitize_request_id; +use wdl_rust_common::request_id::request_id_from_headers; use wdl_rust_common::time::duration_ms_for_log; // CF KV allows 25 MiB values; set Axum's body cap above that instead of @@ -293,13 +293,6 @@ fn response_error(response: &Response) -> Option<(&'static str, &str)> { .map(|err| (err.code, err.message.as_str())) } -fn request_id_from_headers(headers: &HeaderMap) -> Option { - headers - .get("x-request-id") - .and_then(|value| value.to_str().ok()) - .and_then(sanitize_request_id) -} - async fn track_request( State(state): State, request: Request, @@ -411,6 +404,7 @@ pub async fn run() -> Result<(), Box> { #[cfg(test)] mod tests { use axum::body::to_bytes; + use axum::http::HeaderMap; use axum::response::IntoResponse; use serde_json::json; diff --git a/rust/workflows/src/server.rs b/rust/workflows/src/server.rs index aea9574..1f64e18 100644 --- a/rust/workflows/src/server.rs +++ b/rust/workflows/src/server.rs @@ -30,7 +30,7 @@ use wdl_rust_common::internal_auth::{ internal_auth_headers_match, }; use wdl_rust_common::metrics::prometheus_response; -use wdl_rust_common::request_id::sanitize_request_id; +use wdl_rust_common::request_id::request_id_from_headers; use wdl_rust_common::shutdown::shutdown_signal; use wdl_rust_common::time::duration_ms_for_log; @@ -374,13 +374,6 @@ fn response_error(response: &Response) -> Option<(&'static str, &str)> { .map(|err| (err.code, err.message.as_str())) } -fn request_id_from_headers(headers: &HeaderMap) -> Option { - headers - .get("x-request-id") - .and_then(|value| value.to_str().ok()) - .and_then(sanitize_request_id) -} - async fn track_request( State(state): State, request: Request, diff --git a/shared/errors.js b/shared/errors.js index 783c9b9..015e6a1 100644 --- a/shared/errors.js +++ b/shared/errors.js @@ -1,4 +1,8 @@ /** @param {unknown} err */ export function errorMessage(err) { - return err instanceof Error ? err.message : String(err); + try { + return String(err instanceof Error ? err.message : err); + } catch { + return "Unknown error"; + } } diff --git a/shared/internal-auth.js b/shared/internal-auth.js index b04e3a6..defc864 100644 --- a/shared/internal-auth.js +++ b/shared/internal-auth.js @@ -8,13 +8,14 @@ export const INTERNAL_AUTH_FAILURE_MESSAGE = "Internal authentication failed"; * @param {string} name * @param {string} token */ -function assertAsciiToken(name, token) { +function assertHeaderToken(name, token) { if (token === "") { - throw new Error(`${name} must be configured as a non-empty ASCII string`); + throw new Error(`${name} must be configured as visible ASCII without whitespace or commas`); } for (let i = 0; i < token.length; i++) { - if (token.charCodeAt(i) > 127) { - throw new Error(`${name} must be configured as a non-empty ASCII string`); + const code = token.charCodeAt(i); + if (code < 0x21 || code > 0x7e || code === 0x2c) { + throw new Error(`${name} must be configured as visible ASCII without whitespace or commas`); } } return token; @@ -28,7 +29,7 @@ export function internalAuthToken(env) { if (typeof token !== "string" || token === "") { throw new Error(`${INTERNAL_AUTH_ENV} must be configured`); } - return assertAsciiToken(INTERNAL_AUTH_ENV, token); + return assertHeaderToken(INTERNAL_AUTH_ENV, token); } /** @@ -36,7 +37,7 @@ export function internalAuthToken(env) { */ export function internalAuthPreviousToken(env) { const token = env?.[INTERNAL_AUTH_PREVIOUS_ENV]; - return typeof token === "string" && token !== "" ? assertAsciiToken(INTERNAL_AUTH_PREVIOUS_ENV, token) : null; + return typeof token === "string" && token !== "" ? assertHeaderToken(INTERNAL_AUTH_PREVIOUS_ENV, token) : null; } /** diff --git a/shared/observability.js b/shared/observability.js index 85e89b1..1993d31 100644 --- a/shared/observability.js +++ b/shared/observability.js @@ -4,9 +4,9 @@ // Label discipline + cardinality rules: see CLAUDE.md. import { bytesToHex } from "./hex.js"; +import { errorMessage } from "./errors.js"; const CARDINALITY_WARN_LIMIT = 100; -const requestIdUtf8Encoder = new TextEncoder(); /** * @typedef {Record} Labels * @typedef {(level: string, event: string, fields?: Record) => void} Logger @@ -120,10 +120,21 @@ export function generateRequestId() { return bytesToHex(bytes); } +/** @param {string} value */ +function trimAsciiWhitespace(value) { + let start = 0; + let end = value.length; + /** @param {number} code */ + const isWhitespace = (code) => code === 0x20 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d; + while (start < end && isWhitespace(value.charCodeAt(start))) start += 1; + while (end > start && isWhitespace(value.charCodeAt(end - 1))) end -= 1; + return value.slice(start, end); +} + // An inbound id flows into log fields AND response headers AND downstream -// subrequests, so anything multi-valued (Node joins dup headers as "v1, v2" -// or sometimes string[]) or containing CRLF/quotes/control chars must be -// dropped — pass-through would corrupt header framing or JSON escaping. +// subrequests. Header adapters may join repeated values with commas, so only +// the first token is considered; non-visible-ASCII and quote/backslash bytes +// are rejected before pass-through. // Dirty → mint fresh; preserving a maybe-poisoned upstream id defeats the // "single correlation token" goal. /** @@ -133,12 +144,11 @@ export function generateRequestId() { export function sanitizeRequestId(raw) { if (Array.isArray(raw)) raw = raw[0]; if (typeof raw !== "string") return null; - const first = raw.split(",")[0].trim(); - if (!first || requestIdUtf8Encoder.encode(first).byteLength > 128) return null; - if (/[\s"\\]/.test(first)) return null; + const first = trimAsciiWhitespace(raw.split(",")[0]); + if (!first || first.length > 128) return null; for (let i = 0; i < first.length; i++) { const code = first.charCodeAt(i); - if (code < 0x20 || code === 0x7f || (code >= 0x80 && code <= 0x9f)) return null; + if (code < 0x21 || code > 0x7e || code === 0x22 || code === 0x5c) return null; } return first; } @@ -221,18 +231,36 @@ export function recordRequestComplete({ */ export function formatError(err) { if (!err) return { error_message: "Unknown error" }; - if (err instanceof Error) { + if (isErrorObject(err)) { /** @type {Record} */ const out = { - error_name: err.name, - error_message: err.message, + error_name: errorStringField(err, "name") ?? "Error", + error_message: errorMessage(err), }; - const coded = /** @type {{ code?: unknown, reason?: unknown }} */ (err); - if (typeof coded.code === "string") out.error_code = coded.code; - else if (typeof coded.reason === "string") out.error_code = coded.reason; + const code = errorStringField(err, "code") ?? errorStringField(err, "reason"); + if (code !== null) out.error_code = code; return out; } - return { error_message: String(err) }; + return { error_message: errorMessage(err) }; +} + +/** @param {unknown} value @returns {value is Error} */ +function isErrorObject(value) { + try { + return value instanceof Error; + } catch { + return false; + } +} + +/** @param {Error} err @param {string} field */ +function errorStringField(err, field) { + try { + const value = /** @type {Record} */ (/** @type {unknown} */ (err))[field]; + return typeof value === "string" ? value : null; + } catch { + return null; + } } // Metrics bypass this gate entirely (in-memory registry, separate scrape diff --git a/shared/respond.js b/shared/respond.js index 8ae066f..9a06428 100644 --- a/shared/respond.js +++ b/shared/respond.js @@ -1,26 +1,31 @@ -// 101 upgrades can't be re-wrapped with the original body: workerd -// rejects a body on 101 (http.c++:1136-1149) and Response.clone() rejects -// WS handshakes (http.c++:1269). Rebuild with a null body + the hijacked -// WebSocket so the upgrade flows through and x-request-id still lands on -// the 101 headers. +// 101 upgrades can't be re-wrapped with the original body: workerd rejects a +// body on 101 and Response.clone() rejects WS handshakes. Rebuild with a null +// body plus the hijacked WebSocket so callers can safely replace headers. +/** + * @param {Response & { webSocket?: WebSocket | null }} response + * @param {HeadersInit} headers + * @returns {Response} + */ +export function rebuildResponseWithHeaders(response, headers) { + const init = /** @type {ResponseInit & { webSocket?: WebSocket }} */ ({ + status: response.status, + statusText: response.statusText, + headers, + }); + const webSocket = response.webSocket; + if (webSocket) init.webSocket = webSocket; + return new Response(response.status === 101 ? null : response.body, init); +} + /** * @param {Response & { webSocket?: WebSocket | null }} response * @param {string} requestId * @returns {Response} */ export function echoResponseWithRequestId(response, requestId) { - if (response.status === 101) { - const rebuilt = new Response(null, { - status: 101, - headers: response.headers, - webSocket: response.webSocket, - }); - rebuilt.headers.set("x-request-id", requestId); - return rebuilt; - } - const echoed = new Response(response.body, response); - echoed.headers.set("x-request-id", requestId); - return echoed; + const headers = new Headers(response.headers); + headers.set("x-request-id", requestId); + return rebuildResponseWithHeaders(response, headers); } /** diff --git a/tests/fixtures/do-intrinsic-guard-worker.mjs.txt b/tests/fixtures/do-intrinsic-guard-worker.mjs.txt new file mode 100644 index 0000000..a981a52 --- /dev/null +++ b/tests/fixtures/do-intrinsic-guard-worker.mjs.txt @@ -0,0 +1,70 @@ +const iteratorSymbol = Symbol.iterator; +const originalArrayIterator = Array.prototype[iteratorSymbol]; +const originalFunctionToString = Function.prototype.toString; +const originalObjectDefineProperty = Object.defineProperty; +const originalObjectEntries = Object.entries; +const originalObjectKeys = Object.keys; +const originalPromiseThen = Promise.prototype.then; +const originalProxy = Proxy; +const originalReflectGet = Reflect.get; +const originalSymbol = Symbol; + +Array.prototype[iteratorSymbol] = function* () {}; +Function.prototype.toString = () => "function fetch() {}"; +Object.defineProperty = () => {}; +Object.entries = () => []; +Object.keys = () => []; +Promise.prototype.then = () => { throw new Error("tenant Promise.then must not run"); }; +globalThis.Proxy = class BrokenProxy { constructor(target) { return target; } }; +Reflect.get = () => undefined; +globalThis.Symbol = () => "BYPASS"; + +function restoreIntrinsics() { + globalThis.Symbol = originalSymbol; + Reflect.get = originalReflectGet; + globalThis.Proxy = originalProxy; + Object.keys = originalObjectKeys; + Object.entries = originalObjectEntries; + Object.defineProperty = originalObjectDefineProperty; + Promise.prototype.then = originalPromiseThen; + Function.prototype.toString = originalFunctionToString; +} + +function restorePromiseThen() { + Promise.prototype.then = originalPromiseThen; +} + +function restoreArrayIterator() { + Array.prototype[iteratorSymbol] = originalArrayIterator; +} + +export class Room { + async fetch(request) { + queueMicrotask(restorePromiseThen); + return new Response(await request.text()); + } +} + +export default { + async fetch(_request, env) { + restoreIntrinsics(); + const room = env.ROOM.get(env.ROOM.idFromName("guard")); + const roomResponse = await room.fetch("https://room.local/", { + method: "POST", + body: "room-ok", + }); + const ownerMetadataVisible = + roomResponse.headers.has("x-wdl-do-owner-task-id") || + roomResponse.headers.has("x-wdl-do-owner-endpoint") || + roomResponse.headers.has("x-wdl-do-ownership-error"); + restoreArrayIterator(); + return Response.json({ + room: await roomResponse.text(), + roomFacade: env.ROOM.constructor.name, + ownerMetadataVisible, + doBackendVisible: "__WDL_DO_BACKEND__" in env, + ownerNetworkVisible: "__WDL_DO_OWNER_NETWORK__" in env, + workflowsBackendVisible: "__WDL_WORKFLOWS_BACKEND__" in env, + }); + }, +}; diff --git a/tests/fixtures/internal-auth-contract.json b/tests/fixtures/internal-auth-contract.json index 72724ab..3f4a065 100644 --- a/tests/fixtures/internal-auth-contract.json +++ b/tests/fixtures/internal-auth-contract.json @@ -7,6 +7,20 @@ "error": "internal_auth_failed", "message": "Internal authentication failed" }, + "tokenCases": [ + { "value": "current", "accepted": true }, + { "value": "token-._~+/=", "accepted": true }, + { "value": "", "accepted": false }, + { "value": " leading", "accepted": false }, + { "value": "trailing ", "accepted": false }, + { "value": "inner space", "accepted": false }, + { "value": "token\tvalue", "accepted": false }, + { "value": "token\nvalue", "accepted": false }, + { "value": "token\u0000value", "accepted": false }, + { "value": "token\u007fvalue", "accepted": false }, + { "value": "token,other", "accepted": false }, + { "value": "tokén", "accepted": false } + ], "rotationCases": [ { "actual": "current", "current": "current", "previous": null, "accepted": true }, { "actual": "current", "current": "current", "previous": "previous", "accepted": true }, @@ -14,5 +28,12 @@ { "actual": "wrong", "current": "current", "previous": "previous", "accepted": false }, { "actual": null, "current": "current", "previous": "previous", "accepted": false }, { "actual": "previous", "current": "current", "previous": null, "accepted": false } + ], + "headerCases": [ + { "values": ["current"], "current": "current", "previous": "previous", "accepted": true }, + { "values": ["previous"], "current": "current", "previous": "previous", "accepted": true }, + { "values": ["current", "wrong"], "current": "current", "previous": "previous", "accepted": false }, + { "values": ["wrong", "current"], "current": "current", "previous": "previous", "accepted": false }, + { "values": ["current", "current"], "current": "current", "previous": null, "accepted": false } ] } diff --git a/tests/fixtures/request-id-sanitizer.json b/tests/fixtures/request-id-sanitizer.json index 4fbecda..1d4e77a 100644 --- a/tests/fixtures/request-id-sanitizer.json +++ b/tests/fixtures/request-id-sanitizer.json @@ -7,13 +7,15 @@ { "raw": "a\"b", "sanitized": null }, { "raw": "a\\b", "sanitized": null }, { "raw": "ok\nbad", "sanitized": null }, + { "raw": "\u000bxyz", "sanitized": null }, { "raw": "ok\u0085bad", "sanitized": null }, + { "raw": "\u0085rid", "sanitized": null }, + { "raw": "rid\u0085", "sanitized": null }, { "raw": "ok\u00a0bad", "sanitized": null }, { "raw": "ok\u2028bad", "sanitized": null }, { "raw": "ok\u3000bad", "sanitized": null }, { "raw": "ok\ufeffbad", "sanitized": null }, - { "raw": "\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9", "sanitized": "\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9" }, - { "raw": "\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9", "sanitized": null }, + { "raw": "caf\u00e9", "sanitized": null }, { "raw": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "sanitized": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, { "raw": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "sanitized": null } ] diff --git a/tests/helpers/do-owner-hint.js b/tests/helpers/do-owner-hint.js index cb66546..368daf2 100644 --- a/tests/helpers/do-owner-hint.js +++ b/tests/helpers/do-owner-hint.js @@ -31,6 +31,13 @@ export function doOwnerHintResponse(options = {}) { }); } +/** @param {string} code @param {HeadersInit | undefined} [headers] */ +export function doOwnershipErrorHeaders(code, headers = undefined) { + const out = new Headers(headers); + out.set("x-wdl-do-ownership-error", code); + return out; +} + export function tenantBodyDoOwnerHintResponse() { return Response.json({ error: "do_owner_hint", diff --git a/tests/helpers/load-do-host-actor.js b/tests/helpers/load-do-host-actor.js index 03d59c4..105990f 100644 --- a/tests/helpers/load-do-host-actor.js +++ b/tests/helpers/load-do-host-actor.js @@ -58,22 +58,17 @@ export async function rememberDoObject(env, invoke) { `); const protocolUrl = moduleDataUrl(` -export { DO_OWNERSHIP_CODE } from ${JSON.stringify(productionProtocolUrl)}; -export class DoRuntimeError extends Error { - constructor(status, code, message) { - super(message); - this.status = status; - this.code = code; - } -} +export { + DO_OWNERSHIP_CODE, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, + DoRuntimeError, + doPlatformErrorResponse, +} from ${JSON.stringify(productionProtocolUrl)}; export function buildFacetName(invoke) { return [invoke.className, invoke.objectName].join(":"); } export function buildAlarmRequest() { throw new Error("unexpected alarm request"); } export function buildForwardRequest() { throw new Error("unexpected forward request"); } -export function doErrorResponse(err) { - return Response.json({ error: err?.code || "internal_error", message: err?.message || String(err) }, { status: err?.status || 500 }); -} export function normalizeDoConnectRequest() { throw new Error("unexpected connect normalize"); } export async function readLocalActorInvokeRequest() { throw new Error("unexpected actor request read"); } `); @@ -134,6 +129,7 @@ const source = applyModuleReplacements(readRepositoryFile("do-runtime/actor.js") [/from "do-runtime-state";/g, `from ${JSON.stringify(stateUrl)};`], [/from "shared-errors";/g, `from ${JSON.stringify(repositoryFileUrl("shared/errors.js"))};`], [/from "shared-observability";/g, `from ${JSON.stringify(OBSERVABILITY_NOOP_URL)};`], + [/from "shared-respond";/g, `from ${JSON.stringify(repositoryFileUrl("shared/respond.js"))};`], ]); /** @returns {DoHostActorHarnessState} */ diff --git a/tests/helpers/mock-global.js b/tests/helpers/mock-global.js index a324449..162ae3e 100644 --- a/tests/helpers/mock-global.js +++ b/tests/helpers/mock-global.js @@ -81,3 +81,26 @@ export async function withMockedProperty(target, name, mockImpl, callback) { restore(); } } + +/** + * Temporarily replaces an accessor or other descriptor for one async scope. + * + * @template {object} T + * @template {keyof T} K + * @template {() => unknown | Promise} TCallback + * @param {T} target + * @param {K} name + * @param {PropertyDescriptor} descriptor + * @param {TCallback} callback + * @returns {Promise>>} + */ +export async function withMockedPropertyDescriptor(target, name, descriptor, callback) { + const originalDescriptor = Object.getOwnPropertyDescriptor(target, name); + Object.defineProperty(target, name, descriptor); + try { + return /** @type {Awaited>} */ (await callback()); + } finally { + if (originalDescriptor) Object.defineProperty(target, name, originalDescriptor); + else delete target[name]; + } +} diff --git a/tests/integration/durable-objects-core.test.js b/tests/integration/durable-objects-core.test.js index b25cfe1..c7c0ee1 100644 --- a/tests/integration/durable-objects-core.test.js +++ b/tests/integration/durable-objects-core.test.js @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { test } from "node:test"; import assert from "node:assert/strict"; import { @@ -16,6 +17,11 @@ import { setupIntegrationSuite(); +const DO_INTRINSIC_GUARD_WORKER = readFileSync( + new URL("../fixtures/do-intrinsic-guard-worker.mjs.txt", import.meta.url), + "utf8" +); + test("worker Durable Object binding routes through do-runtime and preserves object state", async () => { const ns = uniqueNs("do"); await deployAndPromote(ns, "counter", { @@ -57,6 +63,30 @@ test("worker Durable Object binding routes through do-runtime and preserves obje }); }); +test("tenant top-level intrinsic patches cannot bypass host env wrapping", async () => { + const ns = uniqueNs("do-intrinsic-guard"); + await deployAndPromote(ns, "guard", { + mainModule: "worker.js", + modules: { "worker.js": DO_INTRINSIC_GUARD_WORKER }, + vars: { BYPASS: true }, + bindings: { + ROOM: { type: "do", className: "Room" }, + }, + }); + + const response = await gatewayFetch(ns, "/guard"); + const text = await response.text(); + assert.equal(response.status, 200, text); + assert.deepEqual(responseJson({ body: text }), { + room: "room-ok", + roomFacade: "DurableObjectNamespace", + ownerMetadataVisible: false, + doBackendVisible: false, + ownerNetworkVisible: false, + workflowsBackendVisible: false, + }); +}); + test("Durable Object classes receive ordinary Worker bindings", async () => { const ns = uniqueNs("do-bindings"); await deployAndPromote(ns, "rooms", { diff --git a/tests/integration/durable-objects-ownership.test.js b/tests/integration/durable-objects-ownership.test.js index c3d2286..88f1bfd 100644 --- a/tests/integration/durable-objects-ownership.test.js +++ b/tests/integration/durable-objects-ownership.test.js @@ -72,7 +72,7 @@ test("Durable Object takeover preserves committed SQLite state after owner loss" assert.equal(owner.taskId, "do-runtime-a"); const shortLeaseOwner = { ...owner, - leaseExpiresAt: Date.now() + 1000, + leaseExpiresAt: Date.now() + 30_000, }; redisSetDoOwner(ownerKey, shortLeaseOwner); const renew = serviceInternalPost("do-runtime-a", 8788, "/internal/do/renew", {}); @@ -122,6 +122,9 @@ test("Durable Object takeover preserves committed SQLite state after owner loss" storage: 3, body: "from-worker", }); + }, { + // Keep heartbeat renewal outside this explicit-renew assertion window. + renewStartDelayMs: 600_000, }); }); diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 7a63c79..81beece 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -932,6 +932,13 @@ test("validateSecretKey: env-var grammar only", () => { assert.throws(() => validateSecretKey(bad), `"${bad}" should be rejected`); } assert.throws(() => validateSecretKey("__WDL_DO_BACKEND__"), /runtime-internal bindings/); + for (const reserved of ["__proto__", "constructor", "prototype", "toString", "valueOf"]) { + assert.throws( + () => validateSecretKey(reserved), + /reserved Object\.prototype key/, + `"${reserved}" should be rejected before persistence` + ); + } assert.throws(() => validateSecretKey("A".repeat(129)), /too long/); }); diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 3af2492..4f17b2d 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -119,11 +119,13 @@ async function withNamespaceSecretRedis(state, setup, callback) { } const validateSecretKeyStubSource = ` +import { RESERVED_OBJECT_KEYS } from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))}; const SECRET_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; const WDL_RESERVED_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; export function validateSecretKey(key) { if (typeof key !== "string" || !SECRET_KEY_RE.test(key)) throw new Error("bad key"); if (WDL_RESERVED_BINDING_RE.test(key)) throw new Error("reserved key"); + if (RESERVED_OBJECT_KEYS.has(key)) throw new Error("reserved object key"); if (key.length > 128) throw new Error("key too long"); } `; @@ -290,6 +292,26 @@ test("namespace secret mutation rejects invalid keys through shared validator", }); }); +test("namespace secret mutation rejects Object.prototype keys before persistence", async () => { + await withNamespaceSecretRedis(namespaceSecretState, () => {}, async (redis) => { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/constructor", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "constructor", + requestId: "rid-secret", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "invalid_request"); + assert.equal(redis.ops.length, 0); + }); +}); + test("namespace secret PUT accepts lowercase secret keys like production", async () => { await withNamespaceSecretRedis(namespaceSecretState, () => {}, async (redis) => { const response = await handle({ diff --git a/tests/unit/do-runtime-actor.test.js b/tests/unit/do-runtime-actor.test.js index ca1fee4..241f2d1 100644 --- a/tests/unit/do-runtime-actor.test.js +++ b/tests/unit/do-runtime-actor.test.js @@ -231,3 +231,23 @@ test("DO host actor: registry remember failure is best-effort and does not fail assert.equal(harness.logs.at(-1).fields.member, "Room:alice"); assert.equal(harness.logs.at(-1).fields.worker_id, "demo:room:v1"); }); + +test("DO host actor strips tenant-supplied ownership error control markers", async () => { + const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); + harness.assertResponses = [{ + ownerKey: "do_0123456789abcdef0123456789abcdef:Room:shard0", + taskId: "task-a", + generation: 7, + leaseExpiresAt: Date.now() + 60_000, + }]; + + const response = await host.dispatchWithFence(invoke(), () => Promise.resolve( + new Response("tenant response", { + status: 503, + headers: { "x-wdl-do-ownership-error": "owner_fence_missing" }, + }) + )); + + assert.equal(await response.text(), "tenant response"); + assert.equal(response.headers.get("x-wdl-do-ownership-error"), null); +}); diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 20038f8..35da117 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -5,17 +5,20 @@ import { decodeDoEnvelope } from "../helpers/do-envelope.js"; import { loadDoProtocol } from "../helpers/load-do-protocol.js"; import { assertJsonResponse } from "../helpers/response-json.js"; import { - OWNER_HINT_STALE_CODES, - OWNER_RACE_RETRY_CODES, dispatchDoInvokeWithHintCache, retryableOwnerRaceResponse, staleDoOwnerHintResponse, } from "../../runtime/_wdl-do-transport.js"; -import { doOwnerHintHeaders } from "../helpers/do-owner-hint.js"; +import { + doOwnerHintHeaders, + doOwnershipErrorHeaders, +} from "../helpers/do-owner-hint.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; const { DoRuntimeError, DO_INVOKE_CONTENT_TYPE, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, DO_OWNERSHIP_CODE, buildFacetName, buildAlarmRequest, @@ -23,6 +26,7 @@ const { encodeDoInvokeRequest, buildLocalActorRequest, doErrorResponse, + doPlatformErrorResponse, hostIdForObject, hostIdForShard, normalizeDoConnectRequest, @@ -42,17 +46,15 @@ test("DO ownership errors stay aligned with runtime hint and retry handling", as DO_OWNERSHIP_CODE.OWNER_LEASE_EXPIRED, DO_OWNERSHIP_CODE.OWNER_LEASE_TOO_SHORT, ]); - assert.deepEqual( - [...OWNER_HINT_STALE_CODES].toSorted(), - [...ownershipCodes].toSorted() - ); - assert.deepEqual( - [...OWNER_RACE_RETRY_CODES].toSorted(), - [...ownerRaceRetryCodes].toSorted() - ); - for (const code of ownershipCodes) { - const response = Response.json({ error: code }, { status: 503 }); + const untrusted = Response.json({ error: code }, { status: 503 }); + assert.equal(await staleDoOwnerHintResponse(untrusted), false, code); + assert.equal(await retryableOwnerRaceResponse(untrusted), false, code); + + const response = Response.json({ error: code }, { + status: 503, + headers: doOwnershipErrorHeaders(code), + }); assert.equal(await staleDoOwnerHintResponse(response), true, code); assert.equal( await retryableOwnerRaceResponse(response), @@ -78,7 +80,7 @@ test("DO invoke dispatch does not cache hints carried by owner-race responses", return routerCalls.length === 1 ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503, - headers: doOwnerHintHeaders(), + headers: doOwnershipErrorHeaders("stale_owner_generation", doOwnerHintHeaders()), }) : new Response("retried"); }, @@ -312,6 +314,29 @@ test("rejects invalid do rpc shapes", () => { ); }); +test("do-runtime RPC JSON validation uses captured intrinsics", async () => { + await withMockedProperty(Object, "entries", () => [], () => { + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + kind: "rpc", + rpc: { method: "addMessage", args: [{ fn() {} }] }, + }), + (error) => error instanceof Error && error.message === "rpc.args[0].fn must be JSON data" + ); + }); + await withMockedProperty(Number, "isFinite", () => true, () => { + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + kind: "rpc", + rpc: { method: "addMessage", args: [Number.NaN] }, + }), + (error) => error instanceof Error && error.message === "rpc.args[0] must be a finite number" + ); + }); +}); + test("normalizes inline workerCode only for test hooks", () => { assert.throws( () => normalizeDoInvokeRequest(INLINE_BODY), @@ -758,6 +783,38 @@ test("do runtime error responses keep internal exception messages out of the wir }); }); +test("do runtime ownership errors carry a private control marker", async () => { + const ownershipResponse = doPlatformErrorResponse(new DoRuntimeError( + 503, + DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, + "DO owner fence is missing" + )); + assert.equal( + ownershipResponse.headers.get(DO_OWNERSHIP_ERROR_CONTROL_HEADER), + DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING + ); + + const tenantResponse = doPlatformErrorResponse(new DoRuntimeError( + 503, + "tenant_failure", + "Tenant failure" + )); + assert.equal( + tenantResponse.headers.get(DO_OWNERSHIP_ERROR_CONTROL_HEADER), + null + ); + + const forgedResponse = doPlatformErrorResponse({ + status: 503, + code: DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, + message: "forged ownership error", + }); + assert.equal( + forgedResponse.headers.get(DO_OWNERSHIP_ERROR_CONTROL_HEADER), + null + ); +}); + test("do runtime error responses keep top-level fields stable while preserving nested details", async () => { const response = doErrorResponse(new DoRuntimeError(503, "owner_unavailable", "DO owner is unavailable", { error: "socket leaked", diff --git a/tests/unit/internal-auth.test.js b/tests/unit/internal-auth.test.js index 003c87c..2aad6fa 100644 --- a/tests/unit/internal-auth.test.js +++ b/tests/unit/internal-auth.test.js @@ -21,7 +21,7 @@ import { assertJsonResponse } from "../helpers/response-json.js"; const TOKEN = "test-internal-auth-token"; const ENV = { [INTERNAL_AUTH_ENV]: TOKEN }; -const contract = /** @type {{ header: string, currentEnv: string, previousEnv: string, failure: { status: number, error: string, message: string }, rotationCases: Array<{ actual: string | null, current: string, previous: string | null, accepted: boolean }> }} */ ( +const contract = /** @type {{ header: string, currentEnv: string, previousEnv: string, failure: { status: number, error: string, message: string }, tokenCases: Array<{ value: string, accepted: boolean }>, rotationCases: Array<{ actual: string | null, current: string, previous: string | null, accepted: boolean }>, headerCases: Array<{ values: string[], current: string, previous: string | null, accepted: boolean }> }} */ ( readRepositoryJson("tests/fixtures/internal-auth-contract.json") ); @@ -32,6 +32,11 @@ test("internal auth literals and rotation match the shared Rust/JS contract", () assert.equal(INTERNAL_AUTH_FAILURE_CODE, contract.failure.error); assert.equal(INTERNAL_AUTH_FAILURE_MESSAGE, contract.failure.message); + for (const entry of contract.tokenCases) { + const read = () => internalAuthToken({ [INTERNAL_AUTH_ENV]: entry.value }); + if (entry.accepted) assert.equal(read(), entry.value); + else assert.throws(read); + } for (const entry of contract.rotationCases) { const headers = new Headers(); if (entry.actual !== null) headers.set(INTERNAL_AUTH_HEADER, entry.actual); @@ -41,6 +46,15 @@ test("internal auth literals and rotation match the shared Rust/JS contract", () }; assert.equal(verifyInternalAuthHeaders(headers, env), entry.accepted); } + for (const entry of contract.headerCases) { + const headers = new Headers(); + for (const value of entry.values) headers.append(INTERNAL_AUTH_HEADER, value); + const env = { + [INTERNAL_AUTH_ENV]: entry.current, + ...(entry.previous === null ? {} : { [INTERNAL_AUTH_PREVIOUS_ENV]: entry.previous }), + }; + assert.equal(verifyInternalAuthHeaders(headers, env), entry.accepted); + } }); test("internal auth token requires a non-empty configured string", () => { @@ -52,11 +66,15 @@ test("internal auth token requires a non-empty configured string", () => { assert.throws(() => internalAuthToken({ [INTERNAL_AUTH_ENV]: "" }), /WDL_INTERNAL_AUTH_TOKEN must be configured/); assert.throws( () => internalAuthToken({ [INTERNAL_AUTH_ENV]: "tokén" }), - /WDL_INTERNAL_AUTH_TOKEN must be configured as a non-empty ASCII string/ + /WDL_INTERNAL_AUTH_TOKEN must be configured as visible ASCII/ ); assert.throws( () => internalAuthPreviousToken({ [INTERNAL_AUTH_PREVIOUS_ENV]: "prévious" }), - /WDL_INTERNAL_AUTH_PREVIOUS_TOKEN must be configured as a non-empty ASCII string/ + /WDL_INTERNAL_AUTH_PREVIOUS_TOKEN must be configured as visible ASCII/ + ); + assert.throws( + () => internalAuthToken({ [INTERNAL_AUTH_ENV]: "token,other" }), + /without whitespace or commas/ ); }); diff --git a/tests/unit/observability.test.js b/tests/unit/observability.test.js index d7500d8..19e73b2 100644 --- a/tests/unit/observability.test.js +++ b/tests/unit/observability.test.js @@ -97,6 +97,7 @@ test("formatError normalizes null, Error objects, and primitive throwables", () error_message: "boom", }); assert.deepEqual(formatError(42), { error_message: "42" }); + assert.deepEqual(formatError(Object.create(null)), { error_message: "Unknown error" }); }); test("formatError promotes stable code/reason fields to error_code", () => { @@ -113,6 +114,33 @@ test("formatError promotes stable code/reason fields to error_code", () => { assert.equal(formatError(reasoned).error_code, "auth_unavailable"); }); +test("formatError contains throwing Error field getters", () => { + const broken = new Error("original"); + for (const field of ["name", "message", "code"]) { + Object.defineProperty(broken, field, { + configurable: true, + get() { throw new Error(`formatter read ${field}`); }, + }); + } + Object.defineProperty(broken, "reason", { value: "tenant_failure" }); + + assert.deepEqual(formatError(broken), { + error_name: "Error", + error_message: "Unknown error", + error_code: "tenant_failure", + }); +}); + +test("formatError contains a throwing Error classification trap", () => { + const throwable = new Proxy(Object.create(null), { + getPrototypeOf() { throw new Error("prototype trap"); }, + }); + + assert.deepEqual(formatError(throwable), { + error_message: "Unknown error", + }); +}); + test("MetricsRegistry: renderPrometheus emits counter TYPE + totals", () => { const reg = new MetricsRegistry(); reg.increment("requests", { service: "test", status: "200" }); diff --git a/tests/unit/r2-utils.test.js b/tests/unit/r2-utils.test.js index a697c05..076ecc7 100644 --- a/tests/unit/r2-utils.test.js +++ b/tests/unit/r2-utils.test.js @@ -71,8 +71,12 @@ test("R2 HTTP metadata fields and cache expiry use one mapping", () => { setR2CacheExpiryHeader(headers, 0); assert.equal(headers.get("expires"), "Thu, 01 Jan 1970 00:00:00 GMT"); assert.equal(r2CacheExpiryFromHeaders(headers), 0); + assert.equal(r2CacheExpiryFromHeaders(headers, { canonical: true }), 0); setR2CacheExpiryHeader(headers, "2026-07-12T00:00:00.000Z"); assert.equal(headers.get("expires"), "Sun, 12 Jul 2026 00:00:00 GMT"); + headers.set("expires", "0"); + assert.equal(Number.isFinite(r2CacheExpiryFromHeaders(headers)), true); + assert.equal(r2CacheExpiryFromHeaders(headers, { canonical: true }), undefined); headers.set("expires", "not-a-date"); assert.equal(r2CacheExpiryFromHeaders(headers), undefined); }); diff --git a/tests/unit/respond.test.js b/tests/unit/respond.test.js index 17d99c3..384f4b2 100644 --- a/tests/unit/respond.test.js +++ b/tests/unit/respond.test.js @@ -9,6 +9,7 @@ import { jsonInitResponse, jsonResponse, prometheusResponse, + rebuildResponseWithHeaders, sanitizeJsonErrorDetails, } from "../../shared/respond.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; @@ -63,6 +64,21 @@ test("echoResponseWithRequestId overwrites a pre-existing x-request-id", () => { assert.equal(out.headers.get("x-request-id"), "ours-id"); }); +test("rebuildResponseWithHeaders preserves response fields while replacing headers", async () => { + const src = new Response("hello", { + status: 202, + statusText: "Accepted", + headers: { "x-old": "hidden" }, + }); + const out = rebuildResponseWithHeaders(src, { "x-new": "visible" }); + + assert.equal(out.status, 202); + assert.equal(out.statusText, "Accepted"); + assert.equal(out.headers.get("x-old"), null); + assert.equal(out.headers.get("x-new"), "visible"); + assert.equal(await out.text(), "hello"); +}); + test("jsonResponse returns JSON with the canonical content-type", async () => { const out = jsonResponse(201, { ok: true }); assert.equal(out.headers.get("content-type"), "application/json"); diff --git a/tests/unit/runtime-bindings-do.test.js b/tests/unit/runtime-bindings-do.test.js index 0524403..14712a0 100644 --- a/tests/unit/runtime-bindings-do.test.js +++ b/tests/unit/runtime-bindings-do.test.js @@ -5,10 +5,12 @@ import { decodeDoEnvelopeMetadata as decodeDoEnvelope } from "../helpers/do-enve import { doOwnerHintHeaders, doOwnerHintResponse, + doOwnershipErrorHeaders, tenantBodyDoOwnerHintResponse, } from "../helpers/do-owner-hint.js"; import { CLOUDFLARE_WORKERS_URL } from "../helpers/mocks/cloudflare-workers.js"; import { makeRecordingFetch, withRecordingFetch } from "../helpers/mock-fetch.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; @@ -72,6 +74,7 @@ test("DO fetch requestSpec strips tenant internal owner routing headers", async "x-wdl-do-hop-count": "99", "x-wdl-do-accept-owner-hint": "1", "x-wdl-do-owner-hint": "1", + "x-wdl-do-ownership-error": "owner_fence_missing", "x-wdl-do-owner-key": "tenant", "x-wdl-do-owner-generation": "123", "x-request-id": "tenant-rid", @@ -82,11 +85,45 @@ test("DO fetch requestSpec strips tenant internal owner routing headers", async assert.equal(headers.get("x-wdl-do-hop-count"), null); assert.equal(headers.get("x-wdl-do-accept-owner-hint"), null); assert.equal(headers.get("x-wdl-do-owner-hint"), null); + assert.equal(headers.get("x-wdl-do-ownership-error"), null); assert.equal(headers.get("x-wdl-do-owner-key"), null); assert.equal(headers.get("x-wdl-do-owner-generation"), null); assert.equal(headers.get("x-request-id"), "tenant-rid"); }); +test("DO fetch requestSpec uses captured header mutation intrinsics", async () => { + const request = new Request("https://tenant.workers.example/send", { + method: "POST", + headers: { + "x-wdl-do-owner-key": "tenant", + "x-wdl-do-ownership-error": "owner_fence_missing", + }, + body: "hello", + }); + + const { spec } = await withMockedProperty( + Headers.prototype, + "delete", + function mockedDelete() {}, + () => withMockedProperty( + Headers.prototype, + "has", + function mockedHas() { return true; }, + () => withMockedProperty( + Headers.prototype, + "set", + function mockedSet() {}, + () => requestSpec(request, "scope-rid") + ) + ) + ); + + const headers = new Headers(spec.headers); + assert.equal(headers.get("x-wdl-do-owner-key"), null); + assert.equal(headers.get("x-wdl-do-ownership-error"), null); + assert.equal(headers.get("x-request-id"), "scope-rid"); +}); + test("DO owner hint parser requires integer owner generation", () => { assert.deepEqual(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 0 }))), { ownerKey: "do_0123456789abcdef0123456789abcdef:Room:shard0", @@ -232,7 +269,10 @@ test("DO-to-DO fetch retries owner generation races without hint opt-in", async const binding = bindingWithBackend({ fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503 }) + ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnershipErrorHeaders("stale_owner_generation"), + }) : new Response("retried"), }), }); @@ -262,7 +302,10 @@ test("DO-to-DO fetch ignores owner hints attached to race responses", async () = response: () => calls.length === 1 ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503, - headers: doOwnerHintResponse().headers, + headers: doOwnershipErrorHeaders( + "stale_owner_generation", + doOwnerHintResponse().headers + ), }) : new Response("retried"), }), @@ -286,7 +329,10 @@ test("DO-to-DO RPC retries owner claim races without hint opt-in", async () => { const binding = bindingWithBackend({ fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error: "owner_claim_raced", message: "retry" }, { status: 503 }) + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) : Response.json({ ok: true, result: "retried-rpc" }), }), }); @@ -384,6 +430,7 @@ test("DO-to-DO websocket strips owner hint headers from successful upgrades", as "x-wdl-do-owner-endpoint": "do-runtime-a:8788", "x-wdl-do-owner-generation": "3", "x-wdl-do-owner-hint": "1", + "x-wdl-do-ownership-error": "owner_fence_missing", }, }); }, @@ -400,4 +447,5 @@ test("DO-to-DO websocket strips owner hint headers from successful upgrades", as assert.equal(await response.text(), "upgrade"); assert.equal(response.headers.get("x-wdl-do-owner-key"), null); assert.equal(response.headers.get("x-wdl-do-owner-hint"), null); + assert.equal(response.headers.get("x-wdl-do-ownership-error"), null); }); diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index 733ae00..afa408d 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -6,11 +6,24 @@ import { clearDoOwnerHintsForTest, setDoOwnerHintMaxEntriesForTest, } from "../../runtime/do-client.js"; -import { requestSpec } from "../../runtime/_wdl-do-transport.js"; +import { + MAX_DO_REQUEST_HEADER_BYTES, + fetchInvokeInit, + ownerHintFromHeaders, + requestSpec, + rpcInvokeBody, +} from "../../runtime/_wdl-do-transport.js"; import { decodeDoEnvelope } from "../helpers/do-envelope.js"; -import { doOwnerHintHeaders, doOwnerHintResponse } from "../helpers/do-owner-hint.js"; +import { + doOwnerHintHeaders, + doOwnerHintResponse, + doOwnershipErrorHeaders, +} from "../helpers/do-owner-hint.js"; import { makeRecordingFetch } from "../helpers/mock-fetch.js"; -import { withMockedProperty } from "../helpers/mock-global.js"; +import { + withMockedProperty, + withMockedPropertyDescriptor, +} from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; test.afterEach(() => { @@ -353,6 +366,35 @@ test("DurableObjectNamespace RPC rejects invalid and reserved methods before tra await assert.rejects(Reflect.get(stub, "alarm")(), /rpc\.method is reserved/); }); +test("DO RPC validation uses captured JSON and method intrinsics", async () => { + const props = { + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }; + + await withMockedProperty(Object, "entries", () => [], () => { + assert.throws( + () => rpcInvokeBody(props, "room-a", "save", [{ fn() {} }]), + (error) => error instanceof TypeError && error.message === "rpc.args[0].fn must be JSON data" + ); + }); + await withMockedProperty(Number, "isFinite", () => true, () => { + assert.throws( + () => rpcInvokeBody(props, "room-a", "save", [Number.NaN]), + (error) => error instanceof TypeError && error.message === "rpc.args[0] must be a finite number" + ); + }); + await withMockedProperty(RegExp.prototype, "test", () => true, () => { + assert.throws( + () => rpcInvokeBody(props, "room-a", "not-valid-method", []), + (error) => error instanceof TypeError && error.message === "rpc.method is not valid" + ); + }); +}); + test("DurableObjectNamespace RPC rejects oversized args before transport", async () => { const backend = { async fetch() { @@ -411,7 +453,10 @@ test("DurableObjectNamespace RPC retries stale owner generation once", async () fetch: makeRecordingFetch(calls, { response: () => { if (calls.length === 1) { - return Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503 }); + return Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnershipErrorHeaders("stale_owner_generation"), + }); } return Response.json({ ok: true, result: "rpc-ok" }); }, @@ -441,7 +486,10 @@ test("DurableObjectNamespace RPC retries owner claim races once", async () => { const backend = { fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error: "owner_claim_raced", message: "retry" }, { status: 503 }) + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) : Response.json({ ok: true, result: "claimed" }), }), }; @@ -492,7 +540,10 @@ test("DurableObjectNamespace direct backend retries stale owner generation once" fetch: makeRecordingFetch(calls, { response: () => { if (calls.length === 1) { - return Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503 }); + return Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnershipErrorHeaders("stale_owner_generation"), + }); } return new Response("ok"); }, @@ -519,12 +570,15 @@ test("DurableObjectNamespace direct backend retries stale owner generation once" assert.equal(calls[1].init.headers.get("x-wdl-do-accept-owner-hint"), null); }); -test("DurableObjectNamespace direct backend does not retry tenant 503 responses", async () => { +test("DurableObjectNamespace does not replay tenant responses that mimic ownership errors", async () => { /** @type {any[]} */ const calls = []; const backend = { fetch: makeRecordingFetch(calls, { - response: Response.json({ error: "tenant_failure", message: "no retry" }, { status: 503 }), + response: Response.json({ + error: "owner_fence_missing", + message: "tenant response after side effect", + }, { status: 503 }), }), }; const ns = new DurableObjectNamespace({ @@ -536,22 +590,613 @@ test("DurableObjectNamespace direct backend does not retry tenant 503 responses" className: "Room", }, { backend }); - const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { - method: "POST", - body: "hello", + const originalGet = Headers.prototype.get; + await withMockedProperty(Headers.prototype, "get", /** @this {Headers} */ function get(name) { + if (String(name).toLowerCase() === "x-wdl-do-ownership-error") { + return "owner_fence_missing"; + } + return originalGet.call(this, name); + }, async () => { + const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "hello", + }); + + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { + error: "owner_fence_missing", + message: "tenant response after side effect", + }); + assert.equal(calls.length, 1); }); +}); + +test("DurableObjectNamespace owner-race classification uses the captured status getter", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: () => calls.length === 1 + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) + : new Response("retried"), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + + const response = await withMockedPropertyDescriptor( + Response.prototype, + "status", + { configurable: true, get: () => 200 }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send") + ); + + assert.equal(await response.text(), "retried"); + assert.equal(calls.length, 2); +}); + +test("DurableObjectNamespace retry policy uses the captured Set membership check", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: () => calls.length === 1 + ? Response.json({ error: "owner_unavailable", message: "outcome unknown" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_unavailable"), + }) + : new Response("replayed"), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + + const response = await withMockedProperty( + Set.prototype, + "has", + function mockedHas() { return true; }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "side effect", + }) + ); assert.equal(response.status, 503); assert.equal(calls.length, 1); }); +test("DurableObjectNamespace retry policy uses the captured Request method getter", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: Response.json({ error: "owner_unavailable", message: "outcome unknown" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_unavailable"), + }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + let reads = 0; + + const response = await withMockedPropertyDescriptor( + Request.prototype, + "method", + { + configurable: true, + get() { + reads += 1; + return reads === 1 ? "POST" : "GET"; + }, + }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "side effect", + }) + ); + + assert.equal(response.status, 503); + assert.equal(calls.length, 1); + assert.equal(reads, 0); + const { metadata } = decodeDoEnvelope(calls[0].init.body); + assert.equal(/** @type {any} */ (metadata.request).method, "POST"); +}); + +test("DurableObjectNamespace retry policy uses captured string case intrinsics", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: Response.json({ error: "owner_unavailable", message: "outcome unknown" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_unavailable"), + }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + let upperReads = 0; + + const response = await withMockedProperty( + String.prototype, + "toUpperCase", + function mockedToUpperCase() { + upperReads += 1; + return upperReads === 1 ? "POST" : "GET"; + }, + () => withMockedProperty( + String.prototype, + "toLowerCase", + function mockedToLowerCase() { + return "websocket"; + }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "side effect", + }) + ) + ); + + assert.equal(response.status, 503); + assert.equal(calls.length, 1); + assert.equal(upperReads, 0); + const { metadata } = decodeDoEnvelope(calls[0].init.body); + assert.equal(/** @type {any} */ (metadata.request).method, "POST"); +}); + +test("DurableObjectNamespace strips owner metadata with patched Headers methods", async () => { + const backend = { + fetch: makeRecordingFetch([], { + response: new Response("ok", { + headers: doOwnershipErrorHeaders("owner_fence_missing", doOwnerHintHeaders()), + }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + + const iteratorSymbol = Symbol.iterator; + const arrayIterator = Array.prototype[Symbol.iterator]; + const arrayIncludes = Array.prototype.includes; + const response = await withMockedProperty( + Headers.prototype, + "delete", + function mockedDelete() {}, + () => withMockedProperty( + /** @type {any} */ (Array.prototype), + iteratorSymbol, + /** @this {unknown[]} */ + function targetedIterator() { + if (Reflect.apply(arrayIncludes, this, ["x-wdl-do-owner-task-id"])) { + return { + next: () => ({ done: true, value: undefined }), + [iteratorSymbol]() { return this; }, + }; + } + return Reflect.apply(arrayIterator, this, []); + }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send") + ) + ); + + assert.equal(await response.text(), "ok"); + assert.equal(response.headers.get("x-wdl-do-owner-task-id"), null); + assert.equal(response.headers.get("x-wdl-do-owner-endpoint"), null); + assert.equal(response.headers.get("x-wdl-do-ownership-error"), null); +}); + +test("DO requestSpec header budget uses captured array iteration", async () => { + const iteratorSymbol = Symbol.iterator; + const arrayIterator = Array.prototype[Symbol.iterator]; + const request = new Request("https://demo.workers.example/send", { + headers: { "x-oversized": "a".repeat(MAX_DO_REQUEST_HEADER_BYTES + 1) }, + }); + + await withMockedProperty( + /** @type {any} */ (Array.prototype), + iteratorSymbol, + /** @this {unknown[]} */ + function targetedIterator() { + const first = this[0]; + if ( + (Array.isArray(first) && first[0] === "x-oversized") || + first === "x-oversized" + ) { + return { + next: () => ({ done: true, value: undefined }), + [iteratorSymbol]() { return this; }, + }; + } + return Reflect.apply(arrayIterator, this, []); + }, + async () => { + await assert.rejects( + () => requestSpec(request, null), + /fetch headers exceed 65536 bytes/ + ); + } + ); +}); + +test("DO requestSpec header budget uses captured TextEncoder.encode", async () => { + const textEncode = TextEncoder.prototype.encode; + let hostileEncodeCalls = 0; + const request = new Request("https://demo.workers.example/send", { + headers: { "x-oversized": "a".repeat(MAX_DO_REQUEST_HEADER_BYTES + 1) }, + }); + + await withMockedProperty( + TextEncoder.prototype, + "encode", + /** @this {TextEncoder} */ + function targetedEncode(value = "") { + if (value.length > MAX_DO_REQUEST_HEADER_BYTES) { + hostileEncodeCalls += 1; + return new Uint8Array(); + } + return Reflect.apply(textEncode, this, [value]); + }, + async () => { + await assert.rejects( + () => requestSpec(request, null), + /fetch headers exceed 65536 bytes/ + ); + } + ); + assert.equal(hostileEncodeCalls, 0); +}); + +function chunkedBodyRequest() { + return new Request("https://demo.workers.example/send", /** @type {RequestInit} */ (/** @type {unknown} */ ({ + method: "POST", + body: new ReadableStream({ + start(controller) { + controller.enqueue(Uint8Array.of(1, 2)); + controller.enqueue(Uint8Array.of(3, 4)); + controller.close(); + }, + }), + duplex: "half", + }))); +} + +test("DO requestSpec body collection uses captured Array.push", async () => { + const arrayPush = Array.prototype.push; + let hostilePushCalls = 0; + + const { bodyBytes } = await withMockedProperty( + Array.prototype, + "push", + /** @this {unknown[]} */ + function targetedPush(value) { + if (value instanceof Uint8Array) { + hostilePushCalls += 1; + return this.length; + } + return Reflect.apply(arrayPush, this, [value]); + }, + () => requestSpec(chunkedBodyRequest(), null) + ); + + assert.equal(hostilePushCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO requestSpec body copying uses captured array iteration", async () => { + const iteratorSymbol = Symbol.iterator; + const arrayIterator = Array.prototype[Symbol.iterator]; + let hostileIteratorCalls = 0; + + const { bodyBytes } = await withMockedProperty( + /** @type {any} */ (Array.prototype), + iteratorSymbol, + /** @this {unknown[]} */ + function targetedIterator() { + if (this[0] instanceof Uint8Array) { + hostileIteratorCalls += 1; + return { + next: () => ({ done: true, value: undefined }), + [iteratorSymbol]() { return this; }, + }; + } + return Reflect.apply(arrayIterator, this, []); + }, + () => requestSpec(chunkedBodyRequest(), null) + ); + + assert.equal(hostileIteratorCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO requestSpec body copying uses captured Uint8Array.set", async () => { + const uint8ArraySet = Uint8Array.prototype.set; + let hostileSetCalls = 0; + + const { bodyBytes } = await withMockedProperty( + Uint8Array.prototype, + "set", + /** @this {Uint8Array} */ + function targetedSet(source, offset) { + if (this.length === 4 && source instanceof Uint8Array) { + hostileSetCalls += 1; + return; + } + return Reflect.apply(uint8ArraySet, this, [source, offset]); + }, + () => requestSpec(chunkedBodyRequest(), null) + ); + + assert.equal(hostileSetCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO requestSpec body reading uses the captured Request body getter", async () => { + let hostileBodyGetterCalls = 0; + const request = chunkedBodyRequest(); + + const { bodyBytes } = await withMockedPropertyDescriptor( + Request.prototype, + "body", + { + configurable: true, + enumerable: true, + get() { + hostileBodyGetterCalls += 1; + return null; + }, + }, + () => requestSpec(request, null) + ); + + assert.equal(hostileBodyGetterCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO requestSpec body reading uses captured stream reader methods", async () => { + const getReader = ReadableStream.prototype.getReader; + const read = ReadableStreamDefaultReader.prototype.read; + const releaseLock = ReadableStreamDefaultReader.prototype.releaseLock; + let hostileMethodCalls = 0; + + const { bodyBytes } = await withMockedProperty( + ReadableStream.prototype, + "getReader", + /** @type {any} */ (/** @this {ReadableStream} @param {ReadableStreamGetReaderOptions | undefined} options */ + function targetedGetReader(options) { + hostileMethodCalls += 1; + return Reflect.apply(getReader, this, [options]); + }), + () => withMockedProperty( + ReadableStreamDefaultReader.prototype, + "read", + /** @this {ReadableStreamDefaultReader} */ + function targetedRead() { + hostileMethodCalls += 1; + return Reflect.apply(read, this, []); + }, + () => withMockedProperty( + ReadableStreamDefaultReader.prototype, + "releaseLock", + /** @this {ReadableStreamDefaultReader} */ + function targetedReleaseLock() { + hostileMethodCalls += 1; + return Reflect.apply(releaseLock, this, []); + }, + () => requestSpec(chunkedBodyRequest(), null) + ) + ) + ); + + assert.equal(hostileMethodCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO invoke envelope uses captured serialization intrinsics", async () => { + const jsonStringify = JSON.stringify; + const dataViewSetUint32 = DataView.prototype.setUint32; + const uint8ArraySet = Uint8Array.prototype.set; + let hostileJsonCalls = 0; + let hostileLengthCalls = 0; + let hostileSetCalls = 0; + + const init = await withMockedProperty( + JSON, + "stringify", + /** @this {typeof JSON} */ + function targetedStringify(value) { + if (value && typeof value === "object" && "doStorageId" in value) { + hostileJsonCalls += 1; + return "{}"; + } + return Reflect.apply(jsonStringify, this, [value]); + }, + () => withMockedProperty( + DataView.prototype, + "setUint32", + /** @this {DataView} */ + function targetedSetUint32(offset, value, littleEndian) { + if (offset === 0) { + hostileLengthCalls += 1; + return; + } + return Reflect.apply(dataViewSetUint32, this, [offset, value, littleEndian]); + }, + () => withMockedProperty( + Uint8Array.prototype, + "set", + /** @this {Uint8Array} */ + function targetedSet(source, offset) { + if (offset === 4) { + hostileSetCalls += 1; + return; + } + return Reflect.apply(uint8ArraySet, this, [source, offset]); + }, + () => fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", new Request("https://demo.workers.example/send"), null) + ) + ) + ); + + assert.equal(hostileJsonCalls, 0); + assert.equal(hostileLengthCalls, 0); + assert.equal(hostileSetCalls, 0); + const { metadata } = decodeDoEnvelope(/** @type {Uint8Array} */ (init.body)); + assert.equal(/** @type {any} */ (metadata).ns, "tenant"); + assert.equal(/** @type {any} */ (metadata).objectName, "room-a"); +}); + +test("DO invoke envelope ignores inherited object toJSON hooks", async () => { + const init = await withMockedProperty( + /** @type {any} */ (Object.prototype), + "toJSON", + function hostileToJSON() { + return { + ns: "attacker", + worker: "attacker", + version: "v9", + doStorageId: "do_ffffffffffffffffffffffffffffffff", + className: "Room", + objectName: "other", + request: { method: "GET", url: "https://evil.example/", headers: [] }, + }; + }, + () => fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", new Request("https://demo.workers.example/send"), null) + ); + + const { metadata } = decodeDoEnvelope(/** @type {Uint8Array} */ (init.body)); + assert.equal(/** @type {any} */ (metadata).ns, "tenant"); + assert.equal(/** @type {any} */ (metadata).objectName, "room-a"); +}); + +test("DO invoke envelope ignores inherited array toJSON hooks", async () => { + const init = await withMockedProperty( + /** @type {any} */ (Array.prototype), + "toJSON", + function hostileToJSON() { + return []; + }, + () => fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", new Request("https://demo.workers.example/send", { + headers: { "x-proof": "preserved" }, + }), null) + ); + + const { metadata } = decodeDoEnvelope(/** @type {Uint8Array} */ (init.body)); + assert.deepEqual(/** @type {any} */ (metadata).request.headers, [["x-proof", "preserved"]]); +}); + +test("DO invoke envelope uses captured typed-array getters", async () => { + const typedArrayPrototype = /** @type {any} */ (Object.getPrototypeOf(Uint8Array.prototype)); + let hostileGetterCalls = 0; + const request = new Request("https://demo.workers.example/send"); + + const init = await withMockedPropertyDescriptor( + typedArrayPrototype, + "length", + { + configurable: true, + get() { + hostileGetterCalls += 1; + return 0; + }, + }, + () => withMockedPropertyDescriptor( + typedArrayPrototype, + "byteLength", + { + configurable: true, + get() { + hostileGetterCalls += 1; + return 0; + }, + }, + () => withMockedPropertyDescriptor( + typedArrayPrototype, + "buffer", + { + configurable: true, + get() { + hostileGetterCalls += 1; + return new ArrayBuffer(0); + }, + }, + () => fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", request, null) + ) + ) + ); + + assert.ok(hostileGetterCalls > 0); + const { metadata } = decodeDoEnvelope(/** @type {Uint8Array} */ (init.body)); + assert.equal(/** @type {any} */ (metadata).ns, "tenant"); +}); + test("DurableObjectNamespace direct backend retries owner claim races once", async () => { /** @type {any[]} */ const calls = []; const backend = { fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error: "owner_claim_raced", message: "retry" }, { status: 503 }) + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) : new Response("ok"), }), }; @@ -580,7 +1225,10 @@ test("DurableObjectNamespace direct backend ignores hints attached to owner-race response: () => routerCalls.length === 1 ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503, - headers: doOwnerHintResponse().headers, + headers: doOwnershipErrorHeaders( + "stale_owner_generation", + doOwnerHintResponse().headers + ), }) : new Response("ok"), }), @@ -613,7 +1261,10 @@ test("DurableObjectNamespace direct backend retries owner lease budget errors on const backend = { fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error, message: "owner lease unavailable" }, { status: 503 }) + ? Response.json({ error, message: "owner lease unavailable" }, { + status: 503, + headers: doOwnershipErrorHeaders(error), + }) : new Response("ok"), }), }; @@ -859,6 +1510,28 @@ test("DurableObjectNamespace direct backend accepts VPC-local IPv4 owner hint en assert.equal(ownerCalls[1].url, "http://100.64.30.52:8788/internal/do/invoke"); }); +test("DO owner hint parsing keeps the validated endpoint with patched String", async () => { + const headers = new Headers(doOwnerHintHeaders({ endpoint: "do-runtime-a:8788" })); + const nativeString = String; + let hostileStringCalls = 0; + + const hint = await withMockedProperty( + globalThis, + "String", + /** @type {StringConstructor} */ (function hostileString(value) { + if (value === "do-runtime-a:8788") { + hostileStringCalls += 1; + return "10.0.0.5:8788"; + } + return nativeString(value); + }), + () => ownerHintFromHeaders(headers) + ); + + assert.equal(hostileStringCalls, 0); + assert.equal(hint?.endpoint, "do-runtime-a:8788"); +}); + test("DurableObjectNamespace direct backend rejects unsafe IPv4 owner hint endpoints", async () => { for (const endpoint of ["0.0.0.0:8788", "127.0.0.1:8788", "169.254.169.254:8788", "224.0.0.1:8788"]) { /** @type {any[]} */ @@ -1122,20 +1795,30 @@ test("DO requestSpec rejects oversized streams without waiting for cancel", asyn })); /** @type {ReturnType | undefined} */ let timeout; + let hostileCatchCalls = 0; try { - await assert.rejects( - Promise.race([ - requestSpec(request, "rid-cancel"), - new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error("requestSpec waited for stream cancel")), 1000); - }), - ]), - /Durable Object fetch body exceeds/ + await withMockedProperty( + Promise.prototype, + "catch", + function hostileCatch() { + hostileCatchCalls += 1; + throw new Error("tenant Promise.catch must not run"); + }, + () => assert.rejects( + Promise.race([ + requestSpec(request, "rid-cancel"), + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error("requestSpec waited for stream cancel")), 1000); + }), + ]), + /Durable Object fetch body exceeds/ + ) ); } finally { if (timeout) clearTimeout(timeout); } + assert.equal(hostileCatchCalls, 0); }); test("DurableObjectNamespace facade uses direct upgrade path for websockets", async () => { @@ -1451,7 +2134,10 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu fetch: makeRecordingFetch(ownerCalls, { response: () => ownerCalls.length === 1 ? new Response("owner-ok", { headers: doOwnerHintHeaders() }) - : Response.json({ error, message: "owner lease unavailable" }, { status: 503 }), + : Response.json({ error, message: "owner lease unavailable" }, { + status: 503, + headers: doOwnershipErrorHeaders(error), + }), }), }; const ns = new DurableObjectNamespace({ diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index 8583c16..d056b2a 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -1632,21 +1632,23 @@ test("workerLoader code estimator does not rewrite CommonJS workflow strings", ( }); test("wrapWorkerCodeForHostBindings rejects empty reserved module collisions", () => { - const workerCode = { - mainModule: "worker.js", - modules: { - "worker.js": "export default {};", - "_wdl-wrapper.js": "", - }, - }; - - assert.throws( - () => injectRuntimeModulesForHostBindings(workerCode, { + for (const reserved of ["_wdl-wrapper.js", "_wdl-host-wrapper-runtime.js"]) { + const workerCode = { mainModule: "worker.js", - modules: { "worker.js": { type: "module" } }, - }, STUB_RUNTIME_INJECTION_SOURCES), - /reserved module names/ - ); + modules: { + "worker.js": "export default {};", + [reserved]: "", + }, + }; + + assert.throws( + () => injectRuntimeModulesForHostBindings(workerCode, { + mainModule: "worker.js", + modules: { "worker.js": { type: "module" } }, + }, STUB_RUNTIME_INJECTION_SOURCES), + /reserved module names/ + ); + } }); test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserves original main module", () => { @@ -1680,6 +1682,7 @@ test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserv assert.match(/** @type {any} */ (workerCode.modules)["_wdl-d1-transport.js"], /from "\.\/_wdl-d1-data-field\.js";/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-d1-client.js"], /class D1Database/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], /intrinsicArrayForEach/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as user from "\.\/src\/worker\.js";/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /Local wrapper subclasses intentionally shadow/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const D1_BINDINGS = \["DB"\];/); @@ -1688,7 +1691,7 @@ test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserv assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new D1Database\(out\[name\], requestIdOptions\(requestIdOrContext\)\)/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /requestIdFromEventArg/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /wrapClassInstance\(this, requestContext\)/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const HOST_WRAPPED_HANDLER_KEYS = \["fetch", "scheduled", "queue", "tail"\]/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /forEachArray\(HOST_WRAPPED_HANDLER_KEYS/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class Api extends user\.Api/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class Admin extends user\.Admin/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class default/); @@ -1823,6 +1826,7 @@ test("wrapWorkerCodeForHostBindings: workers without host facades get only the a assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-d1-client.js"], undefined); assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-r2-client.js"], undefined); assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-do-client.js"], undefined); + assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], undefined); assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-do-transport.js"], undefined); assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], undefined); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-cloudflare-workflows.js"], /class NonRetryableError extends Error/); diff --git a/tests/unit/runtime-r2-client.test.js b/tests/unit/runtime-r2-client.test.js index e202b46..6199328 100644 --- a/tests/unit/runtime-r2-client.test.js +++ b/tests/unit/runtime-r2-client.test.js @@ -163,6 +163,30 @@ test("R2Bucket.put normalizes every Headers httpMetadata field", async () => { }); }); +test("R2Bucket.put rejects non-canonical Headers expiry before the host call", async () => { + let calls = 0; + const bucket = new R2Bucket({ + async put() { + calls += 1; + return null; + }, + }); + for (const expires of [ + "not-a-date", + "0", + "2026-07-13T00:00:00.000Z", + "July 13, 2026", + "Tue, 13 Jul 2026 00:00:00 GMT", + ]) { + const headers = new Headers({ expires }); + await assert.rejects( + () => bucket.put("a.txt", "hello", { httpMetadata: headers }), + /Expires header must be canonical IMF-fixdate/ + ); + } + assert.equal(calls, 0); +}); + test("R2Object.writeHttpMetadata writes every mapped field including epoch expiry", () => { const object = new R2Object({ httpMetadata: { diff --git a/tests/unit/runtime-wrapper-generate.test.js b/tests/unit/runtime-wrapper-generate.test.js index 8e50b43..e51e785 100644 --- a/tests/unit/runtime-wrapper-generate.test.js +++ b/tests/unit/runtime-wrapper-generate.test.js @@ -1,9 +1,15 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { + HOST_BINDING_RUNTIME_MODULE_NAME, + HOST_BINDING_RUNTIME_SOURCE, generateAbortShimWrapperModule, generateHostBindingWrapperModule, } from "../../runtime/load/wrapper-generate.js"; +import { moduleDataUrl } from "../helpers/load-shared-module.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; + +const hostBindingRuntime = await import(moduleDataUrl(HOST_BINDING_RUNTIME_SOURCE)); function generatedWrappers() { return { @@ -36,13 +42,74 @@ test("generated wrapper flavors share the exact abort shim", () => { ); }); -test("generated wrapper flavors share default-export class detection", () => { +test("generated wrapper flavors preserve default-export class detection", () => { const { abortOnly, hostBindings } = generatedWrappers(); const sourceLine = "const source = Function.prototype.toString.call(raw);"; const classTest = "/^\\s*class\\b/.test(source)"; assert.equal(abortOnly.split(sourceLine).length - 1, 1); - assert.equal(hostBindings.split(sourceLine).length - 1, 1); assert.match(abortOnly, new RegExp(`if \\(!${RegExp.escape(classTest)}\\)`)); - assert.match(hostBindings, new RegExp(`if \\(${RegExp.escape(classTest)}\\)`)); + assert.match(hostBindings, /const source = functionSource\(raw\);/); + assert.match(hostBindings, /if \(regexpTest\(\/\^\\s\*class\\b\/, source\)\)/); +}); + +test("host wrapper runtime captures platform intrinsics before user module evaluation", () => { + const source = generateHostBindingWrapperModule("worker.js", [], [], ["ROOM"], {}, []); + assert.ok( + source.indexOf(`from "./${HOST_BINDING_RUNTIME_MODULE_NAME}";`) < + source.indexOf('import * as user from "./worker.js";') + ); + for (const intrinsic of [ + "Array.prototype.forEach", + "Function.prototype.toString", + "Object.defineProperty", + "Object.entries", + "Object.keys", + "Promise.prototype.then", + "Reflect.apply", + "Reflect.get", + "RegExp.prototype.test", + ]) { + assert.match(HOST_BINDING_RUNTIME_SOURCE, new RegExp(RegExp.escape(intrinsic))); + } + assert.doesNotMatch(source, /Object\.(?:defineProperty|entries|keys)\(/); + assert.doesNotMatch(source, /for \(const .* of /); + assert.doesNotMatch(source, /Function\.prototype\.toString\.call/); +}); + +test("host wrapper cleanup ignores tenant-patched Promise.then", async () => { + let cleaned = false; + await withMockedProperty( + Promise.prototype, + "then", + () => { + throw new Error("tenant Promise.then must not run"); + }, + async () => { + const result = await hostBindingRuntime.settleWithFinally( + Promise.resolve("settled"), + () => { + cleaned = true; + } + ); + assert.equal(result, "settled"); + assert.equal(cleaned, true); + + cleaned = false; + const failure = new Error("rejected"); + let rejection; + try { + await hostBindingRuntime.settleWithFinally( + Promise.reject(failure), + () => { + cleaned = true; + } + ); + } catch (error) { + rejection = error; + } + assert.equal(rejection, failure); + assert.equal(cleaned, true); + } + ); }); diff --git a/tests/unit/shared-primitives.test.js b/tests/unit/shared-primitives.test.js index f21ef84..b5bcfef 100644 --- a/tests/unit/shared-primitives.test.js +++ b/tests/unit/shared-primitives.test.js @@ -19,3 +19,14 @@ test("errorMessage extracts string messages without structured log shape", () => assert.equal(errorMessage(42), "42"); assert.equal(errorMessage(null), "null"); }); + +test("errorMessage never replaces a pathological thrown value", () => { + const throwable = Object.create(null); + assert.equal(errorMessage(throwable), "Unknown error"); + + const brokenError = new Error("original"); + Object.defineProperty(brokenError, "message", { + get() { throw new Error("message getter failed"); }, + }); + assert.equal(errorMessage(brokenError), "Unknown error"); +}); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 209a419..81bfc49 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -2524,7 +2524,8 @@ test("DO owner hints are trusted only from do-runtime headers", () => { assert.ok(match, "runtime DO transport must keep ownerHintFromResponse next to followOwnerHint"); const ownerHintFromResponse = match[0]; - assert.match(ownerHintFromResponse, /ownerHintFromHeaders\(response\.headers\)/); + assert.match(ownerHintFromResponse, /const headers = responseHeaders\(response\)/); + assert.match(ownerHintFromResponse, /ownerHintFromHeaders\(headers\)/); assert.doesNotMatch(ownerHintFromResponse, /response\.clone\(\)\.json\(\)/); assert.doesNotMatch(ownerHintFromResponse, /\.json\(\)/); @@ -2539,6 +2540,11 @@ test("DO owner hints are trusted only from do-runtime headers", () => { assert.match(ownerHintHeadersFn, /\[DO_OWNER_HINT_HEADERS\.endpoint\]/); assert.match(ownerHintHeadersFn, /\[DO_OWNER_HINT_HEADERS\.generation\]/); assert.match(ownerHintHeadersFn, /if \(control\) headers\[DO_OWNER_HINT_CONTROL_HEADER\] = "1";/); + + assert.doesNotMatch(runtime, /export const OWNER_(?:HINT_STALE|RACE_RETRY)_CODES/); + assert.match(runtime, /const code = responseHeader\(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER\)/); + assert.match(runtime, /setHas\(OWNER_RACE_RETRY_CODES, code\)/); + assert.match(doRuntime, /doPlatformErrorResponse\(err\)/); }); test("owner endpoint validation lives in neutral runtime helper", () => { From 3ccd385aba7273ce76f782bdc2f3d0c780509a05 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Mon, 13 Jul 2026 11:29:22 +0800 Subject: [PATCH 05/23] Harden owner endpoint validation Capture owner-endpoint parsing intrinsics before tenant code runs, add adversarial regression coverage, and align compatibility, module, and changelog documentation with the final branch behavior. Signed-off-by: Lu Zhang --- CHANGELOG.md | 5 ++ docs/compatibility.md | 2 +- docs/compatibility.zh.md | 2 +- docs/modules/durable-objects.md | 4 + docs/modules/durable-objects.zh.md | 1 + runtime/_wdl-owner-endpoint.js | 91 +++++++++++++++++---- tests/unit/runtime-do-owner-network.test.js | 76 +++++++++++++++++ 7 files changed, 165 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c3661..c36e666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## Unreleased - Updated maintenance dependencies and image baselines: workerd runtime images now use distroless `base-debian13`, local and Kubernetes Valkey use `valkey/valkey:9.1-alpine`, and root development tooling moved to current patch/minor releases. +- Consolidated duplicated Control request/error/retry paths, Redis locks and key grammars, queue and cron projections, S3 retry policy, observability helpers, and Rust sidecar contracts into canonical owners with shared behavioral and cross-language fixture coverage. +- Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and Durable Object transports resist tenant prototype mutation, and DO ownership retries now rely on private do-runtime markers plus captured endpoint validation without replaying unknown-outcome non-idempotent requests. +- Tightened forward-only input contracts: internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. +- Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, scheduler-projection, and workflow-limit fixtures. +- Normalized `PLATFORM_DOMAIN` across routing tiers, stopped `/whoami` from advertising the internal `workers.local` fallback as a public namespace URL, and aligned Compose/Kubernetes/test wiring with the consolidated owners. ## wdl.20260701.1 - 2026-07-08 diff --git a/docs/compatibility.md b/docs/compatibility.md index e86b71c..6693d11 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -48,7 +48,7 @@ bundled workerd behavior for all dates. | Surface | Status | What workerd provides | Stronger / added in WDL | Different from Cloudflare | Not implemented / gaps | |---|---|---|---|---|---| | KV namespace | Supported | A binding object can be exposed to user code through workerd entrypoint/JSRPC mechanics. | Runtime `KV` facade calls redis-proxy; redis-proxy stores values and metadata in DB 1 hash buckets `kvh:::b:`, with `v:` and `m:` fields, 512-byte key/list-prefix cap, 25 MiB value cap, batch raw-byte budget, TTL/EXAT, and prefix list cursors. | KV storage is Redis-backed and namespace-scoped in a WDL deployment. `cacheTtl` is not a storage freshness contract. | No Cloudflare global edge replication or eventual consistency model. | -| R2 bucket | Supported | Fetch/stream primitives in workerd. | Runtime R2 facade signs S3-compatible requests with platform credentials; CLI parses `[[r2_buckets]]`. | Bucket lifecycle and placement are the S3-compatible backend's responsibility. | `preview_bucket_name` and `jurisdiction` are not supported. | +| R2 bucket | Supported | Fetch/stream primitives in workerd. | Runtime R2 facade signs S3-compatible requests with platform credentials; CLI parses `[[r2_buckets]]`. | Bucket lifecycle and placement are the S3-compatible backend's responsibility. `Headers`-form `httpMetadata` accepts `Expires` only as a canonical IMF-fixdate and rejects malformed values before the host call. | `preview_bucket_name` and `jurisdiction` are not supported. | | ASSETS | Partial | Worker can receive a platform-provided binding object. | CLI uploads assets to S3-compatible storage and runtime exposes the WDL `env.ASSETS.url(path)` helper for tokenized CDN URLs. | WDL assets are an S3/CDN helper model, not Cloudflare Pages asset hosting. | WDL does not provide a full Cloudflare Pages asset pipeline or a fetch-style assets binding contract. | | D1 database | Partial | workerd can host a D1-like binding facade and localDisk-backed SQLite actor code. | WDL implements control-plane metadata, d1-runtime, owner lease/generation fencing, migrations, SQL execution, drain/renew, deploy-time alias freezing, and caps on query bodies, decoded statement payloads, rows, and result bytes before SQLite work or response emission. | D1 storage is WDL-owned SQLite with owner routing, not Cloudflare global D1. Physical SQLite files are not deleted on metadata delete. | No Cloudflare global replication/bookmark semantics. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | | Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql`, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-held public WebSocket forwarding, storage ids, and cleanup tombstones. | WDL DO identity, owner routing, and cleanup are WDL-managed rather than Cloudflare migration-compatible. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index bcd10f8..dc7f5de 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -35,7 +35,7 @@ Node.js TLS 行为跟随 bundled workerd binary。在 2026-07-01 workerd pin 下 | Surface | 状态 | workerd 提供什么 | WDL 增强 / 新增什么 | 与 Cloudflare 的模型差异 | 未实现 / 缺口 | |---|---|---|---|---|---| | KV namespace | Supported | 通过 workerd entrypoint/JSRPC 机制把 binding object 暴露给用户代码。 | Runtime `KV` facade 调 redis-proxy；redis-proxy 在 DB 1 hash bucket `kvh:::b:` 存 value/metadata，字段为 `v:` 和 `m:`，支持 512-byte key/list-prefix cap、25 MiB value cap、batch raw-byte budget、TTL/EXAT、prefix list cursor。 | KV storage 是 WDL deployment 内的 Redis-backed、namespace-scoped 存储；`cacheTtl` 不是存储新鲜度合同。 | 没有 Cloudflare 全局边缘复制 / eventual consistency 模型。 | -| R2 bucket | Supported | workerd 的 fetch/stream primitives。 | Runtime R2 facade 用平台 credentials 签 S3-compatible request；CLI 解析 `[[r2_buckets]]`。 | Bucket lifecycle/placement 由 S3-compatible backend 负责。 | 不支持 `preview_bucket_name` 和 `jurisdiction`。 | +| R2 bucket | Supported | workerd 的 fetch/stream primitives。 | Runtime R2 facade 用平台 credentials 签 S3-compatible request；CLI 解析 `[[r2_buckets]]`。 | Bucket lifecycle/placement 由 S3-compatible backend 负责。`Headers` 形态的 `httpMetadata` 只接受 canonical IMF-fixdate `Expires`，malformed value 会在 host call 前被拒绝。 | 不支持 `preview_bucket_name` 和 `jurisdiction`。 | | ASSETS | Partial | Worker 可接收平台提供的 binding object。 | CLI 把 assets 上传到 S3-compatible storage；runtime 暴露 WDL 的 `env.ASSETS.url(path)` helper，用于生成 tokenized CDN URL。 | WDL assets 是 S3/CDN helper 模型，不是 Cloudflare Pages asset hosting。 | 不是完整 Cloudflare Pages assets pipeline，也不提供 fetch-style assets binding 合同。 | | D1 database | Partial | workerd 可承载 D1-like facade 和 localDisk-backed SQLite actor 代码。 | WDL 实现 control-plane metadata、d1-runtime、owner lease/generation fencing、migrations、SQL execution、drain/renew、deploy-time alias freezing，并在 SQLite work 或 response emission 前限制 query body、decoded statement payload、row 和 result bytes。 | D1 storage 是 WDL-owned SQLite + owner routing，不是 Cloudflare global D1；metadata delete 不删除物理 SQLite 文件。 | 没有 Cloudflare global replication/bookmark 语义。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | | Durable Objects | Partial | Native Durable Object class execution、facet identity、SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-held public WebSocket forwarding、storage id、cleanup tombstone。 | WDL DO identity、owner routing 和 cleanup 由 WDL 管理，不兼容 Cloudflare migration 模型。 | 只支持同 worker class；不支持 `script_name`、rename/delete migrations、平台级 WebSocket session recovery。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index ebb5d98..345d217 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -222,6 +222,10 @@ interrupt. - Owner-hint and ownership-error defense is layered: tenant response bodies and tenant-supplied control headers are ignored, only do-runtime control headers are trusted, and endpoint grammar/acceptable-address checks must pass for hints. +- The injected DO transport and shared D1/DO endpoint validator evaluate before tenant + modules and capture the intrinsics used for private-header stripping, request bounds, + invoke serialization, replay classification, and endpoint validation. Tenant prototype + mutation cannot rewrite a trusted target or replay policy after those checks. - do-runtime supervisor must call local `127.0.0.1:8788` drain/renew endpoints; Service Connect aliases may hit a different task. diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 6286eae..509198b 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -101,6 +101,7 @@ Terraform 除了 Fargate task memory limit，还会给 do-runtime workerd contai - Tenant-visible DO metadata 和 error 不得包含 owner task id、backend endpoint 或原始 transport error 文本。 - Owner hint 只信任 do-runtime header，并且要通过 endpoint grammar validation。 - Owner-hint 与 ownership-error 防御是分层的：忽略 tenant response body 和 tenant-supplied control header，只信任 do-runtime control header；hint 还必须通过 endpoint grammar / acceptable-address 检查。 +- 注入的 DO transport 与共享 D1/DO endpoint validator 会在 tenant module 前执行，并捕获 private-header stripping、request bound、invoke serialization、replay classification 和 endpoint validation 使用的 intrinsic。Tenant prototype mutation 不能在校验后改写受信 target 或 replay policy。 - do-runtime supervisor 必须调用本地 `127.0.0.1:8788` drain/renew endpoint；Service Connect alias 可能打到其他 task。 ## 可观测性 diff --git a/runtime/_wdl-owner-endpoint.js b/runtime/_wdl-owner-endpoint.js index 4cfdb54..cc3ce51 100644 --- a/runtime/_wdl-owner-endpoint.js +++ b/runtime/_wdl-owner-endpoint.js @@ -7,6 +7,58 @@ const OWNER_ENDPOINT_HEADLESS_RE = { "d1-runtime": /^d1-runtime-[0-9]+\.d1-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, "do-runtime": /^do-runtime-[0-9]+\.do-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, }; +const INVALID_OWNER_ENDPOINT_RE = /[/?#@[\]\\]/; +const IPV4_OCTET_RE = /^(?:0|[1-9]\d{0,2})$/; +const IntrinsicNumber = Number; +const IntrinsicString = String; +const IntrinsicURL = URL; +const intrinsicReflectApply = Reflect.apply; +const intrinsicRegExpTest = RegExp.prototype.test; +const intrinsicStringSplit = String.prototype.split; +const intrinsicUrlHashGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "hash")); +const intrinsicUrlHostGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "host")); +const intrinsicUrlHostnameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "hostname")); +const intrinsicUrlPasswordGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "password")); +const intrinsicUrlPathnameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "pathname")); +const intrinsicUrlPortGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "port")); +const intrinsicUrlSearchGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "search")); +const intrinsicUrlUsernameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "username")); + +/** @param {object | null} prototype @param {string} name */ +function prototypeGetter(prototype, name) { + let current = prototype; + while (current) { + const getter = Object.getOwnPropertyDescriptor(current, name)?.get; + if (getter) return getter; + current = Object.getPrototypeOf(current); + } + return undefined; +} + +/** @param {RegExp} regexp @param {string} value */ +function regexpTest(regexp, value) { + return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); +} + +/** @param {string} value */ +function splitDots(value) { + return intrinsicReflectApply(intrinsicStringSplit, value, ["."]); +} + +/** @param {unknown} value */ +function numberValue(value) { + return intrinsicReflectApply(IntrinsicNumber, undefined, [value]); +} + +/** @param {unknown} value */ +function stringValue(value) { + return intrinsicReflectApply(IntrinsicString, undefined, [value]); +} + +/** @param {(this: URL) => string} getter @param {URL} url */ +function urlValue(getter, url) { + return intrinsicReflectApply(getter, url, []); +} /** * @param {unknown} endpoint @@ -19,22 +71,30 @@ export function validOwnerEndpointForService(endpoint, port, serviceName) { const serviceRe = OWNER_ENDPOINT_SERVICE_RE[serviceName]; const headlessRe = OWNER_ENDPOINT_HEADLESS_RE[serviceName]; if (!serviceRe) throw new Error(`Unknown owner endpoint service ${serviceName}`); - if (/[/?#@[\]\\]/.test(endpoint)) return false; + if (regexpTest(INVALID_OWNER_ENDPOINT_RE, endpoint)) return false; let url; try { - url = new URL(`http://${endpoint}`); + url = new IntrinsicURL(`http://${endpoint}`); } catch { return false; } - if (url.host !== endpoint || url.username || url.password || url.pathname !== "/" || url.search || url.hash) { + if ( + urlValue(intrinsicUrlHostGet, url) !== endpoint || + urlValue(intrinsicUrlUsernameGet, url) || + urlValue(intrinsicUrlPasswordGet, url) || + urlValue(intrinsicUrlPathnameGet, url) !== "/" || + urlValue(intrinsicUrlSearchGet, url) || + urlValue(intrinsicUrlHashGet, url) + ) { return false; } - if (url.port !== String(port)) return false; + if (urlValue(intrinsicUrlPortGet, url) !== stringValue(port)) return false; + const hostname = urlValue(intrinsicUrlHostnameGet, url); // ECS task ENI addresses are VPC-local but not necessarily RFC1918; some // deployments may use non-RFC1918 VPC-local ranges. Trust comes from // runtime-authored owner-hint headers, while this check rejects URL injection // and obviously unsafe endpoint classes. - return serviceRe.test(url.hostname) || headlessRe.test(url.hostname) || acceptableIpv4(url.hostname); + return regexpTest(serviceRe, hostname) || regexpTest(headlessRe, hostname) || acceptableIpv4(hostname); } /** @@ -42,16 +102,19 @@ export function validOwnerEndpointForService(endpoint, port, serviceName) { * @returns {boolean} */ function acceptableIpv4(hostname) { - const parts = hostname.split("."); + const parts = splitDots(hostname); if (parts.length !== 4) return false; - const octets = parts.map(/** @param {string} part */ (part) => { - if (!/^(?:0|[1-9]\d{0,2})$/.test(part)) return null; - const value = Number(part); - return value >= 0 && value <= 255 ? value : null; - }); - if (octets.some(/** @param {number | null} octet */ (octet) => octet === null)) return false; - const [a, b] = octets; - if (a == null || b == null) return false; + /** @type {number[]} */ + const octets = []; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (!regexpTest(IPV4_OCTET_RE, part)) return false; + const value = numberValue(part); + if (value < 0 || value > 255) return false; + octets[i] = value; + } + const a = octets[0]; + const b = octets[1]; if (a === 0 || a === 127) return false; if (a === 169 && b === 254) return false; if (a >= 224) return false; diff --git a/tests/unit/runtime-do-owner-network.test.js b/tests/unit/runtime-do-owner-network.test.js index 447f7f7..65b7b4a 100644 --- a/tests/unit/runtime-do-owner-network.test.js +++ b/tests/unit/runtime-do-owner-network.test.js @@ -7,8 +7,13 @@ import { repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { withMockedFetch } from "../helpers/mock-fetch.js"; +import { + withMockedProperty, + withMockedPropertyDescriptor, +} from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; +import { validOwnerEndpointForService } from "../../runtime/_wdl-owner-endpoint.js"; const OWNER_ENDPOINT_URL = repositoryFileUrl("runtime/_wdl-owner-endpoint.js"); const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; @@ -69,6 +74,77 @@ test("do owner network rejects invalid owner endpoints before forwarding", async }); }); +test("owner endpoint validation uses captured tenant-realm intrinsics", async () => { + const invalidEndpoint = "redis-proxy-user:7070"; + + let hostileUrlCalls = 0; + await withMockedProperty( + globalThis, + "URL", + /** @type {typeof URL} */ (/** @type {unknown} */ (class HostileURL { + constructor() { + hostileUrlCalls += 1; + this.host = invalidEndpoint; + this.hostname = "do-runtime-a"; + this.port = "8788"; + this.username = ""; + this.password = ""; + this.pathname = "/"; + this.search = ""; + this.hash = ""; + } + })), + () => { + assert.equal(validOwnerEndpointForService(invalidEndpoint, 8788, "do-runtime"), false); + } + ); + assert.equal(hostileUrlCalls, 0); + + let hostileGetterCalls = 0; + await withMockedPropertyDescriptor(URL.prototype, "port", { + configurable: true, + get() { + hostileGetterCalls += 1; + return "8788"; + }, + }, () => withMockedPropertyDescriptor(URL.prototype, "hostname", { + configurable: true, + get() { + hostileGetterCalls += 1; + return "do-runtime-a"; + }, + }, () => { + assert.equal(validOwnerEndpointForService(invalidEndpoint, 8788, "do-runtime"), false); + })); + assert.equal(hostileGetterCalls, 0); + + const nativeRegExpTest = RegExp.prototype.test; + let hostileRegExpCalls = 0; + await withMockedProperty(RegExp.prototype, "test", /** @this {RegExp} */ function hostileTest(value) { + if (value === "redis-proxy-user") { + hostileRegExpCalls += 1; + return true; + } + return Reflect.apply(nativeRegExpTest, this, [value]); + }, () => { + assert.equal(validOwnerEndpointForService("redis-proxy-user:8788", 8788, "do-runtime"), false); + }); + assert.equal(hostileRegExpCalls, 0); + + const nativeString = String; + let hostileStringCalls = 0; + await withMockedProperty(globalThis, "String", /** @type {StringConstructor} */ (function hostileString(value) { + if (value === 8788) { + hostileStringCalls += 1; + return "7070"; + } + return nativeString(value); + }), () => { + assert.equal(validOwnerEndpointForService("do-runtime-a:7070", 8788, "do-runtime"), false); + }); + assert.equal(hostileStringCalls, 0); +}); + test("do owner network rejects non-owner-dispatch paths before forwarding", async () => { const { default: worker } = await loadWorker(); let called = false; From 43bcda1fb7e7c1438247c219b28831512ca00c98 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Mon, 13 Jul 2026 14:35:10 +0800 Subject: [PATCH 06/23] Make published Compose overrides pull-only Reset inherited build definitions in published-image overrides and pin the Compose configuration contract. Signed-off-by: Lu Zhang --- docker-compose.images.yml | 2 ++ tests/unit/style-contracts.test.js | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-compose.images.yml b/docker-compose.images.yml index 0bda468..8b296a5 100644 --- a/docker-compose.images.yml +++ b/docker-compose.images.yml @@ -1,10 +1,12 @@ x-rust-published: &rust-published image: docker.io/getwdl/wdl-rust:latest pull_policy: always + build: !reset null x-workerd-published: &workerd-published image: docker.io/getwdl/wdl-workerd:latest pull_policy: always + build: !reset null services: redis-proxy-user: diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 81bfc49..cb4022b 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1530,12 +1530,19 @@ test("published Compose overrides cover every locally built image service", () = * }} */ (yamlDocument("docker-compose.yml", { merge: true }).toJS()); const publishedDocument = yamlDocument("docker-compose.images.yml"); const published = /** @type {{ - * services: Record, + * services: Record, * }} */ (yamlDocument("docker-compose.images.yml", { merge: true }).toJS()); const familyByDockerfile = new Map([ ["Dockerfile.rust", { alias: "rust-published", image: "docker.io/getwdl/wdl-rust:latest" }], ["Dockerfile.workerd", { alias: "workerd-published", image: "docker.io/getwdl/wdl-workerd:latest" }], ]); + for (const family of familyByDockerfile.values()) { + const build = /** @type {{ tag?: string, value?: unknown } | undefined} */ ( + publishedDocument.getIn([`x-${family.alias}`, "build"], true) + ); + assert.equal(build?.tag, "!reset", `${family.alias} must reset inherited build config`); + assert.equal(build?.value, "null", `${family.alias} build reset must use null`); + } const expected = new Map(); for (const [service, config] of Object.entries(local.services)) { if (!config.build) continue; @@ -1551,6 +1558,7 @@ test("published Compose overrides cover every locally built image service", () = assert.deepEqual(published.services[service], { image: family.image, pull_policy: "always", + build: "null", }); } }); From e6693361e0e3d39c92eaf891d81b1c0e266588e2 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Tue, 14 Jul 2026 18:06:04 +0800 Subject: [PATCH 07/23] Harden owner records and runtime boundaries - Validate owner endpoints, scoped records, generations, and canonical identities. - Harden D1, R2, DO alarm, request-id, and RPC trust boundaries. - Align JS/Rust scheduler contracts, Terraform network checks, tests, and docs. Signed-off-by: Lu Zhang --- CHANGELOG.md | 5 +- control/d1-runtime-client.js | 4 +- control/optimistic.js | 2 +- d1-runtime/config.capnp | 4 +- d1-runtime/owner-client.js | 9 + d1-runtime/owner-registry.js | 55 +++- d1-runtime/protocol.js | 11 +- d1-runtime/task-identity.js | 2 + do-runtime/actor.js | 15 +- do-runtime/alarm-dispatch.js | 12 +- do-runtime/alarm-shim-source.js | 246 +++++++++----- do-runtime/config.capnp | 13 +- do-runtime/index.js | 14 +- do-runtime/load-code-budget.js | 4 +- do-runtime/owner-client.js | 10 + do-runtime/owner-registry.js | 96 +++++- do-runtime/protocol.js | 150 ++------- do-runtime/protocol/identity.js | 5 +- do-runtime/protocol/wire-grammar.js | 8 +- do-runtime/task-identity.js | 2 + docs/modules/d1.md | 10 +- docs/modules/d1.zh.md | 4 +- docs/modules/durable-objects.md | 24 +- docs/modules/durable-objects.zh.md | 8 +- docs/modules/infra.md | 6 + docs/modules/infra.zh.md | 1 + docs/modules/queues-cron.md | 14 +- docs/modules/queues-cron.zh.md | 7 +- docs/modules/runtime.md | 4 +- docs/modules/runtime.zh.md | 2 +- docs/modules/workflows.md | 12 +- docs/modules/workflows.zh.md | 4 +- docs/redis-key-layout.md | 16 +- docs/redis-key-layout.zh.md | 9 +- docs/source-map.md | 3 +- docs/source-map.zh.md | 3 +- docs/testing.md | 2 +- runtime/_wdl-do-transport.js | 111 ++++--- runtime/_wdl-owner-endpoint.js | 124 +------ runtime/bindings/d1.js | 8 +- runtime/bindings/service.js | 5 +- runtime/config-system.capnp | 18 +- runtime/config-user.capnp | 13 +- runtime/d1-client.js | 49 ++- runtime/do-client.js | 19 +- runtime/do-owner-network.js | 2 +- runtime/lib.js | 8 +- runtime/load/env-build.js | 13 +- runtime/load/wrapper-generate.js | 4 +- runtime/r2-client.js | 27 +- rust/common/src/queue_keys.rs | 6 + rust/common/src/version.rs | 25 +- rust/scheduler/src/cron/reference.rs | 54 +++- rust/scheduler/src/cron/sweep.rs | 83 ++++- rust/scheduler/src/queue/keys.rs | 7 +- rust/scheduler/src/queue/registry.rs | 17 +- rust/scheduler/src/test_fixtures.rs | 20 ++ rust/workflows/src/api/do_alarms/dispatch.rs | 6 +- rust/workflows/src/api/do_alarms/model.rs | 38 +++ shared/optimistic-retry.js | 34 ++ shared/owner-endpoint.js | 120 +++++++ shared/owner-forwarder.js | 10 + shared/owner-lease.js | 44 +-- shared/task-identity.js | 23 +- shared/version.js | 6 +- terraform/README.md | 15 +- terraform/foundation/main.tf | 37 ++- terraform/main.tf | 49 ++- .../modules/compute/do_runtime_service.tf | 8 +- terraform/modules/compute/variables.tf | 8 +- terraform/terraform.tfvars.example | 1 - terraform/variables.tf | 15 +- tests/fixtures/version-tags.json | 16 + tests/helpers/control-shared-stub.js | 7 +- tests/helpers/load-d1-owner-client.js | 2 + tests/helpers/load-d1-owner-registry.js | 3 +- tests/helpers/load-d1-protocol.js | 2 + tests/helpers/load-do-owner-client.js | 1 + tests/helpers/load-do-owner-registry.js | 7 +- tests/helpers/load-do-protocol.js | 6 +- tests/helpers/load-owner-client-harness.js | 6 + tests/helpers/load-shared-module.js | 6 + tests/helpers/runtime-injection-sources.js | 2 +- tests/helpers/style-contract-scanner.js | 36 ++- .../durable-objects-alarms.test.js | 76 +++++ .../durable-objects-ownership.test.js | 4 +- tests/integration/helpers/durable-objects.js | 3 +- tests/integration/helpers/misc.js | 3 +- tests/unit/control-d1-runtime-client.test.js | 44 ++- tests/unit/control-env-budget.test.js | 1 + .../control-secret-envelope-handlers.test.js | 1 + tests/unit/control-shared.test.js | 8 +- tests/unit/d1-owner-client.test.js | 12 + .../d1-owner-registry-regressions.test.js | 83 +++-- tests/unit/d1-owner-registry.test.js | 32 +- tests/unit/d1-protocol.test.js | 13 +- tests/unit/d1-router.test.js | 10 +- tests/unit/d1-task-identity.test.js | 17 +- tests/unit/do-alarm-shim.test.js | 302 +++++++++++++++++- tests/unit/do-owner-client.test.js | 8 + tests/unit/do-owner-registry.test.js | 78 ++++- tests/unit/do-runtime-index.test.js | 50 +++ tests/unit/do-runtime-load.test.js | 6 + tests/unit/do-runtime-protocol.test.js | 175 +++++----- tests/unit/do-task-identity.test.js | 17 +- tests/unit/owner-lease.test.js | 14 + tests/unit/runtime-bindings-do.test.js | 11 +- tests/unit/runtime-d1-stub.test.js | 103 ++---- tests/unit/runtime-do-client.test.js | 110 ++++++- tests/unit/runtime-do-owner-network.test.js | 7 +- tests/unit/runtime-lib.test.js | 16 +- tests/unit/runtime-load.test.js | 59 +++- .../runtime-loaded-facade-surface.test.js | 18 ++ tests/unit/runtime-service-binding.test.js | 22 +- tests/unit/style-contracts.test.js | 178 +++++------ .../unit/test-helper-style-contracts.test.js | 16 + tests/unit/version.test.js | 9 + tsconfig.json | 1 - 118 files changed, 2421 insertions(+), 1003 deletions(-) create mode 100644 shared/optimistic-retry.js create mode 100644 shared/owner-endpoint.js create mode 100644 tests/fixtures/version-tags.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c36e666..057f695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,11 @@ - Updated maintenance dependencies and image baselines: workerd runtime images now use distroless `base-debian13`, local and Kubernetes Valkey use `valkey/valkey:9.1-alpine`, and root development tooling moved to current patch/minor releases. - Consolidated duplicated Control request/error/retry paths, Redis locks and key grammars, queue and cron projections, S3 retry policy, observability helpers, and Rust sidecar contracts into canonical owners with shared behavioral and cross-language fixture coverage. -- Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and Durable Object transports resist tenant prototype mutation, and DO ownership retries now rely on private do-runtime markers plus captured endpoint validation without replaying unknown-outcome non-idempotent requests. +- Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and D1/R2/DO facades resist tenant prototype mutation, owner records and forwarding targets require service-specific private endpoints, and DO ownership retries rely on private do-runtime markers without replaying unknown-outcome non-idempotent requests. - Tightened forward-only input contracts: internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. -- Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, scheduler-projection, and workflow-limit fixtures. +- Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, worker-version, scheduler-projection, and workflow-limit fixtures. - Normalized `PLATFORM_DOMAIN` across routing tiers, stopped `/whoami` from advertising the internal `workers.local` fallback as a public namespace URL, and aligned Compose/Kubernetes/test wiring with the consolidated owners. +- Removed the unreachable DO inline worker-code test hook and its Terraform switch; do-runtime invoke paths now accept only canonical persisted bundle identities and reject non-canonical or oversized host ids. ## wdl.20260701.1 - 2026-07-08 diff --git a/control/d1-runtime-client.js b/control/d1-runtime-client.js index c931539..0ff5d88 100644 --- a/control/d1-runtime-client.js +++ b/control/d1-runtime-client.js @@ -13,7 +13,7 @@ import { } from "shared-d1-query-wire"; import { sanitizeJsonErrorDetails } from "shared-respond"; import { withInternalAuth } from "shared-internal-auth"; -import { validOwnerEndpointForService } from "runtime-owner-endpoint"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; /** * @typedef {{ fetch(input: RequestInfo | URL, init?: RequestInit): Promise }} Fetcher @@ -33,7 +33,7 @@ function d1OwnerGenerationFromHeaders(headers) { const raw = headers.get("x-wdl-d1-owner-generation"); if (raw == null || raw === "") return null; const generation = Number(raw); - return Number.isInteger(generation) && generation >= 0 ? generation : null; + return Number.isSafeInteger(generation) && generation > 0 ? generation : null; } /** diff --git a/control/optimistic.js b/control/optimistic.js index 14d21b5..5573119 100644 --- a/control/optimistic.js +++ b/control/optimistic.js @@ -1,5 +1,5 @@ import { WatchError } from "shared-redis"; -import { withOptimisticRetries } from "shared-owner-lease"; +import { withOptimisticRetries } from "shared-optimistic-retry"; export { withOptimisticRetries }; diff --git a/d1-runtime/config.capnp b/d1-runtime/config.capnp index 420ab67..820714b 100644 --- a/d1-runtime/config.capnp +++ b/d1-runtime/config.capnp @@ -44,8 +44,11 @@ const d1RuntimeWorker :Workerd.Worker = ( (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-env", esModule = embed "../shared/env.js"), (name = "shared-fnv1a32", esModule = embed "../shared/fnv1a32.js"), + (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-task-identity", esModule = embed "../shared/task-identity.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), (name = "shared-owner-lease", esModule = embed "../shared/owner-lease.js"), + (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), (name = "shared-owner-protocol", esModule = embed "../shared/owner-protocol.js"), (name = "shared-owner-forwarder", esModule = embed "../shared/owner-forwarder.js"), (name = "shared-redis-client", esModule = embed "../shared/redis-client.js"), @@ -71,7 +74,6 @@ const d1RuntimeWorker :Workerd.Worker = ( (name = "D1_TASK_ID", fromEnvironment = "D1_TASK_ID"), (name = "D1_TASK_ENDPOINT", fromEnvironment = "D1_TASK_ENDPOINT"), (name = "D1_TASK_CONTAINER_NAME", fromEnvironment = "D1_TASK_CONTAINER_NAME"), - (name = "D1_TASK_PORT", fromEnvironment = "D1_TASK_PORT"), (name = "D1_TASK_IDENTITY_TIMEOUT_MS", fromEnvironment = "D1_TASK_IDENTITY_TIMEOUT_MS"), (name = "ECS_CONTAINER_METADATA_URI_V4", fromEnvironment = "ECS_CONTAINER_METADATA_URI_V4"), (name = "D1_OWNER_TTL_SECONDS", fromEnvironment = "D1_OWNER_TTL_SECONDS"), diff --git a/d1-runtime/owner-client.js b/d1-runtime/owner-client.js index e0bba63..b74a21a 100644 --- a/d1-runtime/owner-client.js +++ b/d1-runtime/owner-client.js @@ -21,6 +21,7 @@ import { } from "d1-runtime-state"; import { withInternalAuth } from "shared-internal-auth"; import { forwardOwnerRequest } from "shared-owner-forwarder"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; /** * @typedef {Record & { D1_QUERY_TIMEOUT_MS?: unknown }} D1Env @@ -30,6 +31,7 @@ import { forwardOwnerRequest } from "shared-owner-forwarder"; */ const OWNER_UNAVAILABLE_STATUSES = new Set([502, 503, 504]); +const D1_OWNER_PORT = 8787; /** @param {Headers} headers */ function isD1QueryResponse(headers) { @@ -53,6 +55,9 @@ function resultUnknownError(query, detail) { /** @param {D1Env} env @param {D1Query} query @param {D1Owner} owner */ export async function probeOwner(env, query, owner) { if (!owner.endpoint) return { outcome: "owner-endpoint-missing" }; + if (!validOwnerEndpointForService(owner.endpoint, D1_OWNER_PORT, "d1-runtime")) { + return { outcome: "owner-endpoint-invalid" }; + } try { const res = await fetch( `http://${owner.endpoint}/internal/d1/probe?dbKey=${encodeURIComponent(query.dbKey)}&generation=${owner.generation}`, @@ -83,6 +88,8 @@ export async function forwardToOwner(query, env, owner, requestId = null, hopCou const response = await forwardOwnerRequest({ env, endpoint: owner.endpoint, + endpointPort: D1_OWNER_PORT, + endpointService: "d1-runtime", pathname: "/internal/d1/query", requestId, hopCount, @@ -113,6 +120,8 @@ export async function forwardToOwner(query, env, owner, requestId = null, hopCou }), missingEndpointError: () => new D1ProtocolError(503, "owner-endpoint-missing", `D1 database ${query.dbKey} owner has no endpoint`), + invalidEndpointError: () => + new D1ProtocolError(503, "owner-endpoint-invalid", `D1 database ${query.dbKey} owner endpoint is invalid`), hopExhaustedError: () => new D1ProtocolError( 503, diff --git a/d1-runtime/owner-registry.js b/d1-runtime/owner-registry.js index 795d527..195d2f1 100644 --- a/d1-runtime/owner-registry.js +++ b/d1-runtime/owner-registry.js @@ -10,6 +10,7 @@ import { decodeBulk, WatchError } from "shared-redis"; import { envValueOr } from "shared-env"; import { errorMessage } from "shared-errors"; import { createRequiredRedisClient } from "shared-redis-client"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { boundedPositiveIntEnv, currentOwnerGenerationCounter, @@ -55,6 +56,7 @@ const DRAIN_WAIT_POLL_MS = 25; const OWNER_PREFIX = "d1:owner:db:"; const OWNER_CLAIM_RETRIES = 3; const OWNER_CLAIM_WINNER_READ_DELAYS_MS = [0, 10, 25, 50, 100]; +const D1_OWNER_PORT = 8787; /** * @typedef {{ idFromName(name: string): unknown, get(id: unknown): { fetch(url: string, init?: RequestInit): Promise } }} D1Namespace @@ -127,9 +129,39 @@ export function ownerGenerationKeyOf(dbKey) { return ownerProtocolKeys(OWNER_PREFIX, dbKey).generationKey; } -/** @param {string | null | undefined} raw @returns {D1Owner | null} */ -export function parseOwner(raw) { - return /** @type {D1Owner | null} */ (parseOwnerRecord(raw)); +/** + * @param {D1Owner | null} owner + * @param {string} expectedDbKey + */ +function ownerMatchesScope(owner, expectedDbKey) { + if ( + !owner || + typeof owner.namespace !== "string" || + typeof owner.databaseId !== "string" || + typeof owner.dbKey !== "string" || + typeof owner.slot !== "number" || + !Number.isSafeInteger(owner.slot) || owner.slot < 0 + ) return false; + try { + return owner.dbKey === expectedDbKey && dbKeyOf(owner.namespace, owner.databaseId) === expectedDbKey; + } catch { + return false; + } +} + +/** @param {string | null | undefined} raw @param {string} expectedDbKey @returns {D1Owner | null} */ +export function parseOwner(raw, expectedDbKey) { + if (raw == null) return null; + const owner = /** @type {D1Owner | null} */ (parseOwnerRecord(raw)); + if ( + !owner || + !ownerMatchesScope(owner, expectedDbKey) || + typeof owner.taskId !== "string" || !owner.taskId || + !validOwnerEndpointForService(owner.endpoint, D1_OWNER_PORT, "d1-runtime") + ) { + throw new D1ProtocolError(503, "owner-record-invalid", "D1 owner record is invalid"); + } + return owner; } /** @param {string} dbKey @returns {never} */ @@ -148,12 +180,12 @@ export async function readOwner(env, dbKey) { /** @param {RedisClient} client @param {string} dbKey */ async function readOwnerFromClient(client, dbKey) { - return await readOwnerRecord(client, ownerKeyOf(dbKey), (raw) => parseOwner(decodeBulk(raw))); + return await readOwnerRecord(client, ownerKeyOf(dbKey), (raw) => parseOwner(decodeBulk(raw), dbKey)); } /** @param {{ getWithTime(key: string): Promise<{ value: string | Uint8Array | null | undefined, nowMs: number }> }} client @param {string} dbKey */ async function readOwnerWithTimeFromClient(client, dbKey) { - return await readOwnerRecordWithRedisTime(client, ownerKeyOf(dbKey), (raw) => parseOwner(decodeBulk(raw))); + return await readOwnerRecordWithRedisTime(client, ownerKeyOf(dbKey), (raw) => parseOwner(decodeBulk(raw), dbKey)); } /** @param {RedisClient} client @param {string} dbKey */ @@ -168,6 +200,9 @@ async function readOwnerWithTimeAfterClaimRace(client, dbKey) { /** @param {D1Env} env @param {D1Identity} identity @param {number} generation @param {D1Target} target @param {number} nowMs */ function ownerRecordFor(env, identity, generation, target, nowMs) { + if (!validOwnerEndpointForService(target.endpoint, D1_OWNER_PORT, "d1-runtime")) { + throw new D1ProtocolError(503, "owner-record-invalid", "D1 owner target endpoint is invalid"); + } const ttl = ownerTtlSeconds(env); return { namespace: identity.namespace, @@ -465,7 +500,7 @@ export async function releaseOwner(env, owner) { const key = ownerKeyOf(owner.dbKey); return await withWatchRetries(async () => client.session(async (session) => { await session.watch(key); - const current = parseOwner(await session.get(key)); + const current = parseOwner(await session.get(key), owner.dbKey); if (!ownerFenceMatches(current, owner)) { await session.unwatch(); forgetOwnedDb(owner.dbKey); @@ -645,9 +680,6 @@ export function normalizeDatabases(databases) { const record = /** @type {Record} */ (database); const namespace = record.namespace; const databaseId = record.databaseId; - if (typeof databaseId === "string" && databaseId.includes(":")) { - throw new D1ProtocolError(400, "invalid-database-id", "databaseId must not contain ':'"); - } const dbKey = dbKeyOf(namespace, databaseId); const normalizedNamespace = /** @type {string} */ (namespace); const normalizedDatabaseId = /** @type {string} */ (databaseId); @@ -673,6 +705,9 @@ export function normalizeTarget(target) { if (typeof record.endpoint !== "string" || !record.endpoint) { throw new D1ProtocolError(400, "invalid-target", "target.endpoint is required"); } + if (!validOwnerEndpointForService(record.endpoint, D1_OWNER_PORT, "d1-runtime")) { + throw new D1ProtocolError(400, "invalid-target", "target.endpoint is invalid"); + } return { taskId: record.taskId, endpoint: record.endpoint }; } @@ -684,7 +719,7 @@ export async function rebalanceDatabase(env, database, target) { const localTask = await resolveTaskIdentity(env); return await withWatchRetries(async () => client.session(async (session) => { await session.watch(key, generationKey); - const current = parseOwner(await session.get(key)); + const current = parseOwner(await session.get(key), database.dbKey); if (!current || current.taskId !== localTask.taskId) { await session.unwatch(); forgetOwnedDb(database.dbKey); diff --git a/d1-runtime/protocol.js b/d1-runtime/protocol.js index cb1e059..4dbb5d2 100644 --- a/d1-runtime/protocol.js +++ b/d1-runtime/protocol.js @@ -2,6 +2,7 @@ import { normalizeD1Param } from "shared-d1-params"; import { fnv1a32CodeUnits } from "shared-fnv1a32"; import { BodyTooLargeError, readBoundedBytes as readRequestBoundedBytes } from "shared-bounded-body"; import { errorMessage as sharedErrorMessage } from "shared-errors"; +import { D1_DATABASE_ID_RE, isValidRuntimeLoadNs } from "shared-ns-pattern"; import { D1_QUERY_CONTENT_TYPE, D1_QUERY_RESPONSE_CONTENT_TYPE, @@ -96,11 +97,11 @@ export class D1ProtocolError extends Error { * @param {unknown} databaseId */ export function dbKeyOf(namespace, databaseId) { - if (typeof namespace !== "string" || !namespace) { - throw new D1ProtocolError(400, "invalid-namespace", "namespace is required"); + if (!isValidRuntimeLoadNs(namespace)) { + throw new D1ProtocolError(400, "invalid-namespace", "namespace is invalid"); } - if (typeof databaseId !== "string" || !databaseId) { - throw new D1ProtocolError(400, "invalid-database-id", "databaseId is required"); + if (typeof databaseId !== "string" || !D1_DATABASE_ID_RE.test(databaseId)) { + throw new D1ProtocolError(400, "invalid-database-id", "databaseId is invalid"); } return `${namespace}:${databaseId}`; } @@ -365,7 +366,9 @@ const OWNERSHIP_CODES = new Set([ "not-owner", "owner-not-ready", "owner-unavailable", + "owner-record-invalid", "owner-endpoint-missing", + "owner-endpoint-invalid", "forward-hop-exhausted", "owner-claim-raced", "owner-takeover-raced", diff --git a/d1-runtime/task-identity.js b/d1-runtime/task-identity.js index b7590db..af2b554 100644 --- a/d1-runtime/task-identity.js +++ b/d1-runtime/task-identity.js @@ -1,4 +1,5 @@ import { createTaskIdentityResolver } from "shared-task-identity"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { D1ProtocolError } from "d1-runtime-protocol"; const resolver = createTaskIdentityResolver({ @@ -8,6 +9,7 @@ const resolver = createTaskIdentityResolver({ serviceLabel: "D1", unavailableCode: "task-identity-unavailable", createError: (status, code, message) => new D1ProtocolError(status, code, message), + validateEndpoint: (endpoint, port) => validOwnerEndpointForService(endpoint, port, "d1-runtime"), }); export const { diff --git a/do-runtime/actor.js b/do-runtime/actor.js index 54e9d46..ffb64b7 100644 --- a/do-runtime/actor.js +++ b/do-runtime/actor.js @@ -30,7 +30,7 @@ import { formatError } from "shared-observability"; import { rebuildResponseWithHeaders } from "shared-respond"; /** - * @typedef {{ LOADER: { get(key: string, loader: () => Promise): DoWorkerStub }, DO_TEST_HOOKS?: unknown }} DoEnv + * @typedef {{ LOADER: { get(key: string, loader: () => Promise): DoWorkerStub } }} DoEnv * @typedef {{ getDurableObjectClass(className: string, options: { props: Record }): DurableObjectClass }} DoWorkerStub * @typedef {{ fetch(request: Request): Promise }} DoFacet * @typedef {import("do-runtime-protocol").DoInvoke} DoInvoke @@ -66,12 +66,11 @@ export class WdlDoHostActor extends DurableObject { tenantWorker(invoke, requestId) { const existing = this.workers.get(invoke.workerId); if (existing) return existing; - const workerCode = "workerCode" in invoke ? invoke.workerCode : undefined; const worker = this.env.LOADER.get(invoke.workerId, () => ( - workerCode || loadDoWorkerCode( + loadDoWorkerCode( this.env, this.ctx, - /** @type {DoInvoke & { ns: string, worker: string, version: string, doStorageId: string }} */ (invoke), + invoke, requestId ) )); @@ -136,9 +135,7 @@ export class WdlDoHostActor extends DurableObject { }); } if (url.pathname === "/delete-storage") { - const invoke = /** @type {DoInvoke} */ (await readLocalActorInvokeRequest(request, { - allowInlineWorkerCode: false, - })); + const invoke = /** @type {DoInvoke} */ (await readLocalActorInvokeRequest(request)); await assertCurrentOwner(this.env, invoke.owner); const facetName = buildFacetName(invoke); this.ctx.facets.delete(facetName); @@ -148,9 +145,7 @@ export class WdlDoHostActor extends DurableObject { metrics.setGauge("do_host_actor_object_registry_size", { service: SERVICE }, this.registeredObjectMembers.size); return Response.json({ ok: true }); } - const invoke = /** @type {DoInvoke} */ (await readLocalActorInvokeRequest(request, { - allowInlineWorkerCode: this.env.DO_TEST_HOOKS === "1", - })); + const invoke = /** @type {DoInvoke} */ (await readLocalActorInvokeRequest(request)); return await this.dispatchWithFence(invoke, async () => { const requestId = request.headers.get("x-request-id") || null; const cls = this.tenantWorker(invoke, requestId).getDurableObjectClass(invoke.className, { diff --git a/do-runtime/alarm-dispatch.js b/do-runtime/alarm-dispatch.js index 28809e1..355e4e2 100644 --- a/do-runtime/alarm-dispatch.js +++ b/do-runtime/alarm-dispatch.js @@ -1,4 +1,5 @@ import { formatWorkerId } from "shared-worker-id"; +import { parseVersion } from "shared-version"; import { DoRuntimeError, hostIdForObject, nonEmptyAlarmString, readJsonBody } from "do-runtime-protocol"; import { json } from "do-runtime-http"; @@ -20,6 +21,15 @@ function requiredAlarmString(value, field) { return text; } +/** @param {unknown} value */ +function requiredAlarmVersion(value) { + const version = requiredAlarmString(value, "version"); + if (parseVersion(version) == null) { + throw new DoRuntimeError(400, "invalid_do_alarm_dispatch", "DO alarm version is invalid"); + } + return version; +} + /** @param {unknown} value */ function alarmRetryCount(value) { if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { @@ -41,7 +51,7 @@ export async function handleAlarmDispatch(request, env, dispatchInvoke, requestI : {}; const ns = requiredAlarmString(record.ns, "ns"); const worker = requiredAlarmString(record.worker, "worker"); - const version = requiredAlarmString(record.version, "version"); + const version = requiredAlarmVersion(record.version); const doStorageId = requiredAlarmString(record.doStorageId, "doStorageId"); const className = requiredAlarmString(record.className, "className"); const objectName = requiredAlarmString(record.objectName, "objectName"); diff --git a/do-runtime/alarm-shim-source.js b/do-runtime/alarm-shim-source.js index 15b93b1..30d6c47 100644 --- a/do-runtime/alarm-shim-source.js +++ b/do-runtime/alarm-shim-source.js @@ -3,6 +3,36 @@ const ALARM_HEADER = "x-wdl-do-internal-alarm"; const ALARMS_BINDING = "__WDL_DO_ALARMS__"; const ALARM_TABLE = "_wdl_do_alarms"; +// This module is evaluated before tenant code. Keep the small set of intrinsics +// that controls alarm classification, state transitions, and facade installation +// stable after tenant top-level evaluation mutates the shared isolate realm. +const NativeDate = Date; +const NativeNumber = Number; +const NativePromise = Promise; +const NativeProxy = Proxy; +const NativeResponse = Response; +const NativeString = String; +const nativeCrypto = crypto; +const arrayAt = Array.prototype.at; +const arrayPush = Array.prototype.push; +const cryptoRandomUUID = crypto.randomUUID; +const dateGetTime = Date.prototype.getTime; +const dateNow = Date.now; +const headersGet = Headers.prototype.get; +const mathTrunc = Math.trunc; +const numberIsFinite = Number.isFinite; +const numberIsInteger = Number.isInteger; +const objectDefineProperty = Object.defineProperty; +const promiseResolve = Promise.resolve; +const reflectApply = Reflect.apply; +const reflectGet = Reflect.get; +const requestHeadersGetter = Object.getOwnPropertyDescriptor(Request.prototype, "headers").get; +const requestJson = Request.prototype.json; +const responseJson = Response.json; +const stringReplaceAll = String.prototype.replaceAll; +const stringStartsWith = String.prototype.startsWith; +const stringToLowerCase = String.prototype.toLowerCase; + function withoutInternalEnv(env) { if (!env || typeof env !== "object" || !(ALARMS_BINDING in env)) return env; const out = { ...env }; @@ -11,57 +41,87 @@ function withoutInternalEnv(env) { } function objectNameFromCtx(ctx) { - return String(ctx.id); + return NativeString(ctx.id); } function scheduledTimeFromInput(value) { - const scheduledTime = value instanceof Date ? value.getTime() : Number(value); - if (!Number.isFinite(scheduledTime) || scheduledTime <= 0) { + let scheduledTime; + try { + scheduledTime = reflectApply(dateGetTime, value, []); + } catch { + scheduledTime = NativeNumber(value); + } + if (!numberIsFinite(scheduledTime) || scheduledTime <= 0) { throw new TypeError("setAlarm() cannot be called with an alarm time <= 0"); } return scheduledTime; } function retryCountFromInput(value) { - const retryCount = Number(value ?? 0); - if (!Number.isInteger(retryCount) || retryCount < 0) { + const retryCount = NativeNumber(value ?? 0); + if (!numberIsInteger(retryCount) || retryCount < 0) { throw new TypeError("DO alarm retryCount must be a non-negative integer"); } return retryCount; } function alarmFieldsFromRow(row) { - const scheduledTime = Number(row.scheduled_time); - const retryCount = Number(row.retry_count); - if (!Number.isFinite(scheduledTime) || scheduledTime <= 0) return null; - if (!Number.isInteger(retryCount) || retryCount < 0) return null; + const scheduledTime = NativeNumber(row.scheduled_time); + const retryCount = NativeNumber(row.retry_count); + if (!numberIsFinite(scheduledTime) || scheduledTime <= 0) return null; + if (!numberIsInteger(retryCount) || retryCount < 0) return null; return { scheduledTime, retryCount }; } function alarmToken() { - return crypto.randomUUID(); + return reflectApply(cryptoRandomUUID, nativeCrypto, []); +} + +function safeErrorField(err, field) { + try { + return err == null ? undefined : err[field]; + } catch { + return undefined; + } +} + +function safeErrorString(value) { + try { + return value == null ? null : NativeString(value); + } catch { + return null; + } } function formatWrappedError(err) { - const out = { - error_name: err?.name || "Error", - error_message: err instanceof Error ? err.message : String(err), - }; - if (err?.code != null) out.error_code = String(err.code); - return out; + try { + const out = { + error_name: safeErrorString(safeErrorField(err, "name")) || "Error", + error_message: safeErrorString(safeErrorField(err, "message")) || safeErrorString(err) || "Unknown error", + }; + const code = safeErrorString(safeErrorField(err, "code")); + if (code != null) out.error_code = code; + return out; + } catch { + return { error_name: "Error", error_message: "Unknown error" }; + } } function logStructured(level, event, fields = {}) { - const payload = { - ts: new Date().toISOString(), - service: "do-runtime", - level, - event, - ...fields, - }; - const line = JSON.stringify(payload); - if (level === "error") console.error(line); - else console.log(line); + try { + const payload = { + ts: new Date().toISOString(), + service: "do-runtime", + level, + event, + ...fields, + }; + const line = JSON.stringify(payload); + if (level === "error") console.error(line); + else console.log(line); + } catch { + // Tenant mutations must not turn best-effort logging into storage behavior. + } } function ensureAlarmTable(storage) { @@ -95,11 +155,11 @@ function writeAlarmRow(storage, row) { "in_flight = excluded.in_flight, " + "token = excluded.token, " + "last_error = excluded.last_error", - Math.trunc(scheduledTimeFromInput(row.scheduledTime)), + mathTrunc(scheduledTimeFromInput(row.scheduledTime)), retryCountFromInput(row.retryCount), row.inFlight ? 1 : 0, - String(row.token), - row.lastError == null ? null : String(row.lastError) + NativeString(row.token), + row.lastError == null ? null : NativeString(row.lastError) ); } @@ -109,13 +169,13 @@ function deleteAlarmRow(storage, token = null) { storage.sql.exec("DELETE FROM " + ALARM_TABLE + " WHERE id = 1"); return; } - storage.sql.exec("DELETE FROM " + ALARM_TABLE + " WHERE id = 1 AND token = ?", String(token)); + storage.sql.exec("DELETE FROM " + ALARM_TABLE + " WHERE id = 1 AND token = ?", NativeString(token)); } async function flushAlarmSideEffects(sideEffects) { // One object has one alarm row. Transactional alarm updates coalesce to the // final SQLite row, so only the final backend index side effect should run. - const finalEffect = sideEffects.at(-1); + const finalEffect = reflectApply(arrayAt, sideEffects, [-1]); if (finalEffect) await finalEffect(); } @@ -136,7 +196,7 @@ async function setStorageAlarm(storage, alarmBinding, className, objectName, sch token, }); if (sideEffects) { - sideEffects.push(async () => { + reflectApply(arrayPush, sideEffects, [async () => { try { await effect(); } catch (err) { @@ -145,7 +205,7 @@ async function setStorageAlarm(storage, alarmBinding, className, objectName, sch deleteAlarmRow(storage, token); throw err; } - }); + }]); } else { try { await effect(); @@ -163,12 +223,12 @@ async function deleteStorageAlarm(storage, alarmBinding, className, objectName, deleteAlarmRow(storage); const token = sideEffects?.baselineAlarmToken != null ? sideEffects.baselineAlarmToken - : row?.token == null ? null : String(row.token); + : row?.token == null ? null : NativeString(row.token); const effect = () => token ? alarmBinding.deleteAlarmIndex({ className, objectName, token }) - : Promise.resolve("skipped"); + : reflectApply(promiseResolve, NativePromise, ["skipped"]); if (sideEffects) { - sideEffects.push(async () => { + reflectApply(arrayPush, sideEffects, [async () => { try { await effect(); } catch (err) { @@ -176,13 +236,13 @@ async function deleteStorageAlarm(storage, alarmBinding, className, objectName, writeAlarmRow(storage, { scheduledTime: row.scheduled_time, retryCount: row.retry_count, - inFlight: Number(row.in_flight) === 1, + inFlight: NativeNumber(row.in_flight) === 1, token: row.token, }); } throw err; } - }); + }]); } else { try { await effect(); @@ -191,7 +251,7 @@ async function deleteStorageAlarm(storage, alarmBinding, className, objectName, writeAlarmRow(storage, { scheduledTime: row.scheduled_time, retryCount: row.retry_count, - inFlight: Number(row.in_flight) === 1, + inFlight: NativeNumber(row.in_flight) === 1, token: row.token, }); } @@ -202,7 +262,7 @@ async function deleteStorageAlarm(storage, alarmBinding, className, objectName, async function getStorageAlarm(storage, alarmBinding, className, objectName) { const row = readAlarmRow(storage); - if (!row || Number(row.in_flight) === 1) return null; + if (!row || NativeNumber(row.in_flight) === 1) return null; const fields = alarmFieldsFromRow(row); if (!fields) { deleteAlarmRow(storage); @@ -214,7 +274,7 @@ async function getStorageAlarm(storage, alarmBinding, className, objectName) { objectName, scheduledTime: fields.scheduledTime, retryCount: fields.retryCount, - token: String(row.token), + token: NativeString(row.token), }); } catch (err) { logStructured("warn", "do_alarm_index_repair_failed", { @@ -234,11 +294,11 @@ function claimStorageAlarm(storage, alarm) { deleteAlarmRow(storage); return null; } - const rowToken = String(row.token); - const alarmTokenValue = alarm?.token == null ? null : String(alarm.token); + const rowToken = NativeString(row.token); + const alarmTokenValue = alarm?.token == null ? null : NativeString(alarm.token); if (alarmTokenValue && alarmTokenValue !== rowToken) return null; const retryCount = retryCountFromInput(alarm?.retryCount ?? fields.retryCount); - if (Number(row.in_flight) !== 1 && fields.scheduledTime > Date.now()) return null; + if (NativeNumber(row.in_flight) !== 1 && fields.scheduledTime > reflectApply(dateNow, NativeDate, [])) return null; writeAlarmRow(storage, { scheduledTime: fields.scheduledTime, retryCount, @@ -253,14 +313,17 @@ function completeStorageAlarm(storage, token) { } function quoteSqlIdentifier(name) { - return '"' + String(name).replaceAll('"', '""') + '"'; + return '"' + reflectApply(stringReplaceAll, NativeString(name), ['"', '""']) + '"'; } function sqlObjectDropStatement(row) { - const type = String(row.type); - const name = String(row.name); - const lowerName = name.toLowerCase(); - if (lowerName.startsWith("sqlite_") || lowerName.startsWith("_cf_")) return null; + const type = NativeString(row.type); + const name = NativeString(row.name); + const lowerName = reflectApply(stringToLowerCase, name, []); + if ( + reflectApply(stringStartsWith, lowerName, ["sqlite_"]) || + reflectApply(stringStartsWith, lowerName, ["_cf_"]) + ) return null; if (type === "table") return "DROP TABLE IF EXISTS " + quoteSqlIdentifier(name); if (type === "view") return "DROP VIEW IF EXISTS " + quoteSqlIdentifier(name); if (type === "trigger") return "DROP TRIGGER IF EXISTS " + quoteSqlIdentifier(name); @@ -285,7 +348,7 @@ function deleteAllSqlStorage(storage, deleteAlarm) { storage.sql.exec("PRAGMA foreign_keys = OFF"); try { for (const row of rows) { - if (String(row.name) === ALARM_TABLE && !deleteAlarm) continue; + if (NativeString(row.name) === ALARM_TABLE && !deleteAlarm) continue; const statement = sqlObjectDropStatement(row); if (statement) storage.sql.exec(statement); } @@ -298,7 +361,7 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = if (!storage || !alarmBinding) return storage; let syncTransactionSideEffects = null; const activeSideEffects = () => syncTransactionSideEffects || sideEffects; - return new Proxy(storage, { + return new NativeProxy(storage, { get(target, prop, receiver) { if (prop === "setAlarm") { return (scheduledTime, _options = undefined) => { @@ -325,11 +388,11 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = return async (callback, ...rest) => { const txSideEffects = []; const baselineAlarm = readAlarmRow(alarmStorage); - txSideEffects.baselineAlarmToken = baselineAlarm?.token == null ? null : String(baselineAlarm.token); + txSideEffects.baselineAlarmToken = baselineAlarm?.token == null ? null : NativeString(baselineAlarm.token); const wrapped = typeof callback === "function" ? (txn) => callback(wrapStorage(txn, alarmBinding, className, objectName, txSideEffects, target)) : callback; - const result = await Reflect.apply(Reflect.get(target, prop, receiver), target, [wrapped, ...rest]); + const result = await reflectApply(reflectGet(target, prop, receiver), target, [wrapped, ...rest]); await flushAlarmSideEffects(txSideEffects); return result; }; @@ -347,7 +410,7 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = syncTransactionSideEffects = previousSideEffects; } } : callback; - const result = Reflect.apply(Reflect.get(target, prop, receiver), target, [wrapped, ...rest]); + const result = reflectApply(reflectGet(target, prop, receiver), target, [wrapped, ...rest]); return result; }; } @@ -370,25 +433,25 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = const sideEffects = activeSideEffects(); const token = sideEffects?.baselineAlarmToken != null ? sideEffects.baselineAlarmToken - : alarmRow?.token == null ? null : String(alarmRow.token); + : alarmRow?.token == null ? null : NativeString(alarmRow.token); const effect = () => token ? alarmBinding.deleteAlarmIndex({ className, objectName, token }) - : Promise.resolve("skipped"); - if (sideEffects) sideEffects.push(effect); + : reflectApply(promiseResolve, NativePromise, ["skipped"]); + if (sideEffects) reflectApply(arrayPush, sideEffects, [effect]); else await effect(); } else if (preservedAlarm) { writeAlarmRow(alarmStorage, { scheduledTime: preservedAlarm.scheduled_time, retryCount: preservedAlarm.retry_count, - inFlight: Number(preservedAlarm.in_flight) === 1, + inFlight: NativeNumber(preservedAlarm.in_flight) === 1, token: preservedAlarm.token, }); } })(); }; } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; + const value = reflectGet(target, prop, receiver); + return typeof value === "function" ? (...args) => reflectApply(value, target, args) : value; }, }); } @@ -397,15 +460,15 @@ function wrapCtx(ctx, alarmBinding, className) { if (!ctx || !alarmBinding) return ctx; const objectName = objectNameFromCtx(ctx); let storageProxy = null; - return new Proxy(ctx, { + return new NativeProxy(ctx, { get(target, prop, receiver) { if (prop === "storage") { - const storage = Reflect.get(target, prop, receiver); + const storage = reflectGet(target, prop, receiver); if (!storageProxy) storageProxy = wrapStorage(storage, alarmBinding, className, objectName); return storageProxy; } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; + const value = reflectGet(target, prop, receiver); + return typeof value === "function" ? (...args) => reflectApply(value, target, args) : value; }, }); } @@ -415,7 +478,7 @@ function installStorageProxy(ctx, alarmBinding, className) { const objectName = objectNameFromCtx(ctx); const storageProxy = wrapStorage(ctx.storage, alarmBinding, className, objectName); try { - Object.defineProperty(ctx, "storage", { + objectDefineProperty(ctx, "storage", { value: storageProxy, configurable: true, }); @@ -440,9 +503,12 @@ export function wrapDurableObjectClass(Base, className) { // only the alarm binding here so __WDL_HOST_BINDINGS_WRAPPED can survive // through the two-layer wrapper contract. super(constructorCtx, withoutInternalEnv(env)); + // Resolve through the host wrapper so prototype methods, class fields, + // and accessors retain the real instance as their receiver. + const tenantFetch = reflectGet(this, "fetch", this); const wrappedCtx = wrapCtx(constructorCtx, alarmBinding, className); try { - Object.defineProperty(this, "ctx", { + objectDefineProperty(this, "ctx", { value: wrappedCtx, configurable: true, writable: true, @@ -450,26 +516,34 @@ export function wrapDurableObjectClass(Base, className) { } catch { this.ctx = wrappedCtx; } - } - - async fetch(request) { - if (request.headers.get(ALARM_HEADER) === "1") { - const alarm = await request.json(); - const claim = claimStorageAlarm(this.ctx.storage, alarm); - if (!claim) return Response.json({ ok: true, ignored: true }); - if (typeof super.alarm === "function") { - await super.alarm({ - retryCount: claim.retryCount, - isRetry: claim.retryCount > 0, - }); - } - completeStorageAlarm(this.ctx.storage, claim.token); - return Response.json({ ok: true }); - } - if (typeof super.fetch !== "function") { - return new Response("Durable Object class has no fetch handler", { status: 500 }); - } - return await super.fetch(request); + // workerd wraps instance handlers after construction, so this platform + // dispatch must remain writable and configurable after replacing class fields. + objectDefineProperty(this, "fetch", { + value: async function(request) { + const headers = reflectApply(requestHeadersGetter, request, []); + if (reflectApply(headersGet, headers, [ALARM_HEADER]) === "1") { + const alarm = await reflectApply(requestJson, request, []); + const claim = claimStorageAlarm(this.ctx.storage, alarm); + if (!claim) return reflectApply(responseJson, NativeResponse, [{ ok: true, ignored: true }]); + // Alarm accessors may depend on initialized instance state. + const tenantAlarm = reflectGet(this, "alarm", this); + if (typeof tenantAlarm === "function") { + await reflectApply(tenantAlarm, this, [{ + retryCount: claim.retryCount, + isRetry: claim.retryCount > 0, + }]); + } + completeStorageAlarm(this.ctx.storage, claim.token); + return reflectApply(responseJson, NativeResponse, [{ ok: true }]); + } + if (typeof tenantFetch !== "function") { + return new NativeResponse("Durable Object class has no fetch handler", { status: 500 }); + } + return await reflectApply(tenantFetch, this, [request]); + }, + configurable: true, + writable: true, + }); } }; } diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index d8e749c..0006c91 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -63,10 +63,10 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "../runtime/bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "../runtime/bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "../runtime/_wdl-do-transport.js"), - (name = "runtime-owner-endpoint", esModule = embed "../runtime/_wdl-owner-endpoint.js"), - # _wdl-do-transport.js imports the same helper by relative file path, while - # host bindings import it by bare module name. Workerd needs both entries. - (name = "_wdl-owner-endpoint.js", esModule = embed "../runtime/_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), + # Injected DO transport uses a relative module name; host modules use the + # shared bare name. Both resolve to the same shared contract owner. + (name = "_wdl-owner-endpoint.js", esModule = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache", esModule = embed "../runtime/_wdl-owner-hint-cache.js"), (name = "runtime-metrics", esModule = embed "../runtime/metrics.js"), (name = "runtime-state", esModule = embed "../runtime/runtime.js"), @@ -80,7 +80,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "runtime-r2-utils-source", text = embed "../runtime/r2-utils.js"), (name = "runtime-do-client-source", text = embed "../runtime/do-client.js"), (name = "runtime-do-transport-source", text = embed "../runtime/_wdl-do-transport.js"), - (name = "runtime-owner-endpoint-source", text = embed "../runtime/_wdl-owner-endpoint.js"), + (name = "runtime-owner-endpoint-source", text = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache-source", text = embed "../runtime/_wdl-owner-hint-cache.js"), (name = "runtime-request-id-source", text = embed "../runtime/_wdl-request-id.js"), (name = "runtime-workflows-client-source", text = embed "../runtime/workflows-client.js"), @@ -98,6 +98,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-owner-forwarder", esModule = embed "../shared/owner-forwarder.js"), (name = "shared-owner-lease", esModule = embed "../shared/owner-lease.js"), + (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), (name = "shared-owner-protocol", esModule = embed "../shared/owner-protocol.js"), (name = "shared-redis", esModule = embed "../shared/redis.js"), (name = "shared-redis-command-client", esModule = embed "../shared/redis-command-client.js"), @@ -152,7 +153,6 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "R2_S3_REGION", fromEnvironment = "R2_S3_REGION"), (name = "DO_TASK_ID", fromEnvironment = "DO_TASK_ID"), (name = "DO_TASK_ENDPOINT", fromEnvironment = "DO_TASK_ENDPOINT"), - (name = "DO_TASK_PORT", fromEnvironment = "DO_TASK_PORT"), (name = "DO_TASK_IDENTITY_TIMEOUT_MS", fromEnvironment = "DO_TASK_IDENTITY_TIMEOUT_MS"), (name = "DO_TASK_CONTAINER_NAME", fromEnvironment = "DO_TASK_CONTAINER_NAME"), (name = "ECS_CONTAINER_METADATA_URI_V4", fromEnvironment = "ECS_CONTAINER_METADATA_URI_V4"), @@ -160,7 +160,6 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "DO_OWNER_LEASE_GUARD_MS", fromEnvironment = "DO_OWNER_LEASE_GUARD_MS"), (name = "DO_RENEW_CONCURRENCY", fromEnvironment = "DO_RENEW_CONCURRENCY"), (name = "DO_DRAIN_IN_FLIGHT_TIMEOUT_MS", fromEnvironment = "DO_DRAIN_IN_FLIGHT_TIMEOUT_MS"), - (name = "DO_TEST_HOOKS", fromEnvironment = "DO_TEST_HOOKS"), (name = "LOG_LEVEL", fromEnvironment = "LOG_LEVEL"), ], ); diff --git a/do-runtime/index.js b/do-runtime/index.js index a50ae22..4828836 100644 --- a/do-runtime/index.js +++ b/do-runtime/index.js @@ -6,6 +6,7 @@ import { import { createHttpRequestScope } from "shared-request-scope"; import { discardResponseBody, prometheusResponse, rebuildResponseWithHeaders } from "shared-respond"; import { boundedPositiveIntEnv } from "shared-owner-lease"; +import { parseVersion } from "shared-version"; import { formatWorkerId } from "shared-worker-id"; import { WdlDoHostActor } from "do-runtime-actor"; import { handleAlarmDispatch } from "do-runtime-alarm-dispatch"; @@ -72,7 +73,7 @@ const STORAGE_DELETE_REQUEST = { }; /** - * @typedef {Record & { LOG_LEVEL?: unknown, REDIS_ADDR?: string, DO_HOSTS: DurableObjectNamespace, DO_TEST_HOOKS?: unknown, DO_DRAIN_IN_FLIGHT_TIMEOUT_MS?: unknown, DO_RENEW_INTERVAL_MS?: unknown }} DoEnv + * @typedef {Record & { LOG_LEVEL?: unknown, REDIS_ADDR?: string, DO_HOSTS: DurableObjectNamespace, DO_DRAIN_IN_FLIGHT_TIMEOUT_MS?: unknown, DO_RENEW_INTERVAL_MS?: unknown }} DoEnv * @typedef {import("do-runtime-protocol").DoInvoke} DoInvoke * @typedef {{ ownerKey: string, hostId?: string, className?: string, ns: string, worker: string, doStorageId: string, taskId: string, endpoint: string, generation: number, leaseExpiresAt?: number }} DoOwner * @typedef {{ requestId?: string | null, hopCount?: number, forwardPath?: string, localUrl?: string, request?: { method: string, url: string, headers: Array<[string, string]> } | null, metricKind?: string | null, acceptOwnerHint?: boolean }} DispatchOptions @@ -238,9 +239,7 @@ async function dispatchStorageDelete(env, invoke, requestId = null, hopCount = 0 * @param {string} requestId */ async function handleInvoke(request, env, requestId) { - const invoke = await readDoInvokeRequest(request, { - allowInlineWorkerCode: env.DO_TEST_HOOKS === "1", - }); + const invoke = await readDoInvokeRequest(request); return await dispatchInvoke( env, invoke, @@ -357,9 +356,7 @@ async function handleDrain(env) { * @param {string} requestId */ async function handleStorageDelete(request, env, requestId) { - const invoke = await readDoInvokeRequest(request, { - allowInlineWorkerCode: false, - }); + const invoke = await readDoInvokeRequest(request); return await dispatchStorageDelete( env, invoke, @@ -423,6 +420,9 @@ async function handleStorageDeleteWorker(env, request, requestId = null) { if (!ns || !worker || !version || !doStorageId) { return jsonError(400, "invalid_request", "ns, worker, version, and doStorageId are required"); } + if (parseVersion(version) == null) { + return jsonError(400, "invalid_request", "version is invalid"); + } let deleted = 0; /** @type {Array>} */ diff --git a/do-runtime/load-code-budget.js b/do-runtime/load-code-budget.js index 7e989c0..be0f421 100644 --- a/do-runtime/load-code-budget.js +++ b/do-runtime/load-code-budget.js @@ -35,11 +35,11 @@ function generateDoRuntimeWrapperModule(userMainSpecifier, classNames) { export class ${name} extends wrapDurableObjectClass(user.${name}, ${JSON.stringify(name)}) {} `).join(""); return ` +import { wrapDurableObjectClass } from ${alarmShim}; + import * as user from ${userMain}; export * from ${userMain}; -import { wrapDurableObjectClass } from ${alarmShim}; - ${wrappedClasses} `; } diff --git a/do-runtime/owner-client.js b/do-runtime/owner-client.js index b000b28..70ea4e1 100644 --- a/do-runtime/owner-client.js +++ b/do-runtime/owner-client.js @@ -24,6 +24,8 @@ export function parseHopCount(value) { return parseForwardHopCount(value); } +const DO_OWNER_PORT = 8788; + /** @param {DoOwner} owner */ function ownerFence(owner) { return { @@ -45,6 +47,8 @@ export async function forwardToOwner(invoke, env, owner, requestId = null, hopCo return await forwardOwnerRequest({ env, endpoint: owner.endpoint, + endpointPort: DO_OWNER_PORT, + endpointService: "do-runtime", pathname, requestId, hopCount, @@ -70,6 +74,8 @@ export async function forwardToOwner(invoke, env, owner, requestId = null, hopCo }), missingEndpointError: () => new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_ENDPOINT_MISSING, `DO scope ${owner.ownerKey} owner has no endpoint`), + invalidEndpointError: () => + new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, `DO scope ${owner.ownerKey} owner endpoint is invalid`), hopExhaustedError: () => new DoRuntimeError( 503, @@ -92,6 +98,8 @@ export async function forwardConnectToOwner(request, invoke, env, owner, request return await forwardOwnerRequest({ env, endpoint: owner.endpoint, + endpointPort: DO_OWNER_PORT, + endpointService: "do-runtime", pathname: "/internal/do/connect", method: request.method, requestId, @@ -120,6 +128,8 @@ export async function forwardConnectToOwner(request, invoke, env, owner, request }), missingEndpointError: () => new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_ENDPOINT_MISSING, `DO scope ${owner.ownerKey} owner has no endpoint`), + invalidEndpointError: () => + new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, `DO scope ${owner.ownerKey} owner endpoint is invalid`), hopExhaustedError: () => new DoRuntimeError( 503, diff --git a/do-runtime/owner-registry.js b/do-runtime/owner-registry.js index bab7aee..b7be7c0 100644 --- a/do-runtime/owner-registry.js +++ b/do-runtime/owner-registry.js @@ -6,6 +6,7 @@ import { buildOwnerKey, DO_OWNERSHIP_CODE, DoRuntimeError, + hostIdForShard, } from "do-runtime-protocol"; import { resolveTaskIdentity, @@ -21,6 +22,8 @@ import { import { createRedisClient } from "do-runtime-redis"; import { envValueOr } from "shared-env"; import { errorMessage } from "shared-errors"; +import { isValidRuntimeLoadNs, WORKER_NAME_RE } from "shared-ns-pattern"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { boundedPositiveIntEnv, currentOwnerGenerationCounter, @@ -48,12 +51,14 @@ const DEFAULT_OWNER_LEASE_GUARD_MS = 1_000; const OWNER_PREFIX = "do:owner:scope:"; const OWNER_CLAIM_RETRIES = 3; const OWNER_RENEW_FRACTION = 0.5; +const DO_OWNER_PORT = 8788; /** * @typedef {Record & { REDIS_ADDR?: unknown, REDIS_DB?: unknown, DO_OWNER_TTL_SECONDS?: unknown, DO_OWNER_LEASE_GUARD_MS?: unknown, DO_RENEW_CONCURRENCY?: unknown }} DoEnv * @typedef {{ taskId: string, endpoint: string }} LocalTask * @typedef {{ ownerKey: string, hostId?: string, className?: string, ns: string, worker: string, doStorageId: string, taskId: string, endpoint: string, generation: number, leaseExpiresAt?: number }} DoOwner * @typedef {{ ownerKey: string, taskId: string, generation: number }} OwnerFence + * @typedef {{ ns: string, worker: string }} BundleScope * @typedef {import("do-runtime-protocol").DoInvoke} DoInvoke * @typedef {{ hostId: string, className?: string, ns: string, worker: string, doStorageId: string }} InvokeScope * @typedef {{ get(key: string): Promise, getWithTime(key: string): Promise<{ value: string | Uint8Array | null | undefined, nowMs: number }>, time(): Promise, watch?(...keys: string[]): Promise, unwatch?(): Promise, multi?(): import("shared-redis").RedisMulti }} RedisLike @@ -91,16 +96,23 @@ export function ownerGenerationKeyOf(ownerKey) { return ownerProtocolKeys(OWNER_PREFIX, ownerKey).generationKey; } -/** @param {DoInvoke} invoke */ +/** @param {unknown} value @returns {value is BundleScope & Record} */ +function isValidBundleScope(value) { + const scope = /** @type {Record} */ (value); + return isValidRuntimeLoadNs(scope?.ns) && + typeof scope?.worker === "string" && + WORKER_NAME_RE.test(scope.worker); +} + +/** @param {DoInvoke} invoke @returns {InvokeScope} */ function requireInvokeScope(invoke) { const record = /** @type {Record} */ (invoke); if ( typeof record.hostId !== "string" || !record.hostId || - typeof record.ns !== "string" || !record.ns || - typeof record.worker !== "string" || !record.worker || + !isValidBundleScope(record) || typeof record.doStorageId !== "string" || !record.doStorageId ) { - throw new DoRuntimeError(400, "invalid_request", "DO owner request is missing bundle scope"); + throw new DoRuntimeError(400, "invalid_request", "DO owner request has missing or invalid bundle scope"); } return { hostId: record.hostId, @@ -111,11 +123,57 @@ function requireInvokeScope(invoke) { }; } -/** @param {unknown} raw @returns {DoOwner | null} */ -export function parseOwner(raw) { - return /** @type {DoOwner | null} */ ( +/** + * @param {DoOwner | null} owner + * @param {string} expectedOwnerKey + * @param {BundleScope | null | undefined} expectedBundleScope + */ +function ownerMatchesScope(owner, expectedOwnerKey, expectedBundleScope) { + if ( + !owner || + typeof owner.ownerKey !== "string" || owner.ownerKey !== expectedOwnerKey || + typeof owner.hostId !== "string" || owner.hostId !== expectedOwnerKey || + typeof owner.className !== "string" || !owner.className || + !isValidBundleScope(owner) || + typeof owner.doStorageId !== "string" || !owner.doStorageId + ) return false; + if ( + expectedBundleScope && + (!isValidBundleScope(expectedBundleScope) || + owner.ns !== expectedBundleScope.ns || + owner.worker !== expectedBundleScope.worker) + ) return false; + const shardMarker = ":shard"; + const markerIndex = expectedOwnerKey.lastIndexOf(shardMarker); + if (markerIndex < 0) return false; + const shard = Number(expectedOwnerKey.slice(markerIndex + shardMarker.length)); + try { + return hostIdForShard(owner.doStorageId, owner.className, shard) === expectedOwnerKey; + } catch { + return false; + } +} + +/** + * @param {unknown} raw + * @param {string} expectedOwnerKey + * @param {BundleScope | null} [expectedBundleScope] + * @returns {DoOwner | null} + */ +export function parseOwner(raw, expectedOwnerKey, expectedBundleScope = null) { + if (raw == null) return null; + const owner = /** @type {DoOwner | null} */ ( parseOwnerRecord(/** @type {string | BufferSource | null | undefined} */ (raw)) ); + if ( + !owner || + !ownerMatchesScope(owner, expectedOwnerKey, expectedBundleScope) || + typeof owner.taskId !== "string" || !owner.taskId || + !validOwnerEndpointForService(owner.endpoint, DO_OWNER_PORT, "do-runtime") + ) { + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner record is invalid"); + } + return owner; } /** @@ -127,6 +185,9 @@ export function parseOwner(raw) { * @returns {DoOwner} */ function ownerRecordFor(env, invoke, localTask, generation, nowMs) { + if (!validOwnerEndpointForService(localTask.endpoint, DO_OWNER_PORT, "do-runtime")) { + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner endpoint is invalid"); + } const ttl = ownerTtlSeconds(env); const scope = requireInvokeScope(invoke); return { @@ -151,6 +212,9 @@ function ownerRecordFor(env, invoke, localTask, generation, nowMs) { * @returns {DoOwner} */ function renewedOwnerRecordFor(env, current, localTask, nowMs) { + if (!validOwnerEndpointForService(localTask.endpoint, DO_OWNER_PORT, "do-runtime")) { + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner endpoint is invalid"); + } const ttl = ownerTtlSeconds(env); return { ...current, @@ -166,7 +230,7 @@ function renewedOwnerRecordFor(env, current, localTask, nowMs) { * @returns {Promise} */ async function readOwnerFromClient(client, ownerKey) { - return await readOwnerRecord(client, ownerKeyOf(ownerKey), parseOwner); + return await readOwnerRecord(client, ownerKeyOf(ownerKey), (raw) => parseOwner(raw, ownerKey)); } /** @@ -175,7 +239,11 @@ async function readOwnerFromClient(client, ownerKey) { * @returns {Promise<{ owner: DoOwner | null, nowMs: number }>} */ async function readOwnerWithTimeFromClient(client, ownerKey) { - return await readOwnerRecordWithRedisTime(client, ownerKeyOf(ownerKey), parseOwner); + return await readOwnerRecordWithRedisTime( + client, + ownerKeyOf(ownerKey), + (raw) => parseOwner(raw, ownerKey) + ); } /** @@ -183,7 +251,7 @@ async function readOwnerWithTimeFromClient(client, ownerKey) { * @param {DoOwner | null | undefined} owner */ async function ownerStoragePointerCurrent(session, owner) { - if (!owner?.ns || !owner.worker || !owner.doStorageId) return false; + if (!owner?.doStorageId || !isValidBundleScope(owner)) return false; const current = decodeBulk(await session.get(doStorageIdKey(owner.ns, owner.worker))); return current === owner.doStorageId; } @@ -278,7 +346,11 @@ export async function resolveDoOwner(env, invoke) { return await withWatchRetries(async () => client.session(async (session) => { await session.watch(key, generationKey); - const { owner: current, nowMs } = await readOwnerWithTimeFromClient(session, ownerKey); + const { owner: current, nowMs } = await readOwnerRecordWithRedisTime( + session, + key, + (raw) => parseOwner(raw, ownerKey, scope) + ); if (current && !ownerLeaseExpired(current, nowMs)) { await session.watch(doStorageIdKey(scope.ns, scope.worker)); if (!await ownerStoragePointerCurrent(session, current)) { @@ -502,7 +574,7 @@ export async function releaseOwner(env, owner) { const key = ownerKeyOf(owner.ownerKey); return await withWatchRetries(async () => client.session(async (session) => { await session.watch(key); - const current = parseOwner(await session.get(key)); + const current = parseOwner(await session.get(key), owner.ownerKey); if (!ownerFenceMatches(current, owner)) { await session.unwatch(); forgetOwnedScope(owner.ownerKey); diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index cced13b..aba3603 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -2,10 +2,7 @@ import { CLASS_NAME_RE, METHOD_NAME_RE, HEADER_NAME_RE, - WORKER_NAME_RE, - NS_FIELD_RE, STORAGE_ID_RE, - VERSION_RE, HOST_ID_RE, DO_HOST_SHARD_COUNT, MAX_ID_BYTES, @@ -13,13 +10,14 @@ import { import { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; import { hostIdForObject, shardForObjectName } from "do-runtime-protocol-identity"; import { formatWorkerId } from "shared-worker-id"; -import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; import { BodyTooLargeError, readBoundedBytes as readRequestBoundedBytes, readBoundedText as readRequestBoundedText, } from "shared-bounded-body"; import { INTERNAL_AUTH_HEADER } from "shared-internal-auth"; +import { isValidRuntimeLoadNs, WORKER_NAME_RE } from "shared-ns-pattern"; +import { parseVersion } from "shared-version"; export { DO_HOST_SHARD_COUNT } from "do-runtime-protocol-wire-grammar"; export { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; @@ -52,8 +50,6 @@ export function doPlatformErrorResponse(err) { return response; } -const MAX_MODULE_COUNT = 128; -const MAX_MODULE_SOURCE_BYTES = 1024 * 1024; const MAX_REQUEST_BODY_BYTES = 1024 * 1024; const MAX_INVOKE_ENVELOPE_BYTES = 2 * 1024 * 1024; const MAX_REQUEST_HEADER_COUNT = 128; @@ -110,18 +106,15 @@ const LOCAL_ACTOR_ENVELOPE_MARKER = "binary"; * @typedef {Record} JsonRecord * @typedef {{ ownerKey: string, taskId: string, generation: number }} OwnerFence * @typedef {{ ns: string, worker: string, version: string, doStorageId: string, workerId: string }} BundleSource - * @typedef {{ workerId: string, workerCode: JsonRecord }} InlineWorkerSource - * @typedef {BundleSource | InlineWorkerSource} InvokeSource * @typedef {{ className: string, objectName: string }} ObjectTarget * @typedef {{ method: string, url: string, headers: Array<[string, string]>, bodyBytes?: Uint8Array, bodyBase64?: undefined, bodyText?: undefined }} RequestSpec * @typedef {{ retryCount: number, isRetry: boolean, token?: string }} AlarmInfo * @typedef {{ method: string, args: unknown[] }} RpcInfo - * @typedef {InvokeSource & { kind: "fetch", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, request: RequestSpec }} FetchInvoke - * @typedef {InvokeSource & { kind: "alarm", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, alarm: AlarmInfo }} AlarmInvoke - * @typedef {InvokeSource & { kind: "rpc", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, rpc: RpcInfo }} RpcInvoke + * @typedef {BundleSource & { kind: "fetch", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, request: RequestSpec }} FetchInvoke + * @typedef {BundleSource & { kind: "alarm", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, alarm: AlarmInfo }} AlarmInvoke + * @typedef {BundleSource & { kind: "rpc", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, rpc: RpcInfo }} RpcInvoke * @typedef {FetchInvoke | AlarmInvoke | RpcInvoke} DoInvoke * @typedef {Record & { request?: Record & { bodyBytes?: Uint8Array } }} EnvelopeInvoke - * @typedef {{ allowInlineWorkerCode?: boolean }} NormalizeOptions */ /** @@ -442,8 +435,8 @@ function normalizeOwnerFence(value) { if (value == null) return null; const input = requireRecord(value, "owner"); const generation = Number(input.generation); - if (!Number.isInteger(generation) || generation < 0) { - throw new DoRuntimeError(400, "invalid_request", "owner.generation must be a non-negative integer"); + if (!Number.isSafeInteger(generation) || generation <= 0) { + throw new DoRuntimeError(400, "invalid_request", "owner.generation must be a positive safe integer"); } return { ownerKey: requireString(input.ownerKey, "owner.ownerKey"), @@ -473,89 +466,35 @@ function parseHostId(value) { /** * @param {unknown} value - * @param {InvokeSource} source + * @param {BundleSource} source * @param {ObjectTarget} input */ function normalizeHostId(value, source, input) { const parsed = parseHostId(value); if ( - "ns" in source && - ( - parsed.doStorageId !== source.doStorageId || - parsed.className !== input.className || - parsed.shard !== shardForObjectName(input.objectName) - ) + parsed.doStorageId !== source.doStorageId || + parsed.className !== input.className || + parsed.shard !== shardForObjectName(input.objectName) ) { throw new DoRuntimeError(400, "invalid_request", "hostId does not match object shard"); } return parsed.hostId; } -/** - * @param {unknown} value - * @returns {JsonRecord} - */ -function normalizeWorkerCode(value) { - const input = requireRecord(value, "workerCode"); - const modules = requireRecord(input.modules, "workerCode.modules"); - const entries = Object.entries(modules); - if (entries.length === 0) { - throw new DoRuntimeError(400, "invalid_request", "workerCode.modules must not be empty"); - } - if (entries.length > MAX_MODULE_COUNT) { - throw new DoRuntimeError(400, "invalid_request", "workerCode.modules has too many modules"); - } - /** @type {Record} */ - const normalizedModules = {}; - for (const [name, source] of entries) { - const moduleName = requireString(name, "workerCode module name", { maxBytes: 512 }); - if (moduleName.includes("..")) { - throw new DoRuntimeError(400, "invalid_request", "workerCode module names must not contain .."); - } - if (typeof source !== "string") { - throw new DoRuntimeError(400, "invalid_request", `workerCode.modules.${moduleName} must be a string`); - } - if (byteLength(source) > MAX_MODULE_SOURCE_BYTES) { - throw new DoRuntimeError(400, "invalid_request", `workerCode.modules.${moduleName} is too large`); - } - normalizedModules[moduleName] = source; - } - const mainModule = requireString(input.mainModule, "workerCode.mainModule", { maxBytes: 512 }); - if (!Object.hasOwn(normalizedModules, mainModule)) { - throw new DoRuntimeError(400, "invalid_request", "workerCode.mainModule must reference a module"); - } - const compatibilityDate = input.compatibilityDate == null - ? "2026-04-24" - : requireString(input.compatibilityDate, "workerCode.compatibilityDate", { maxBytes: 64 }); - const compatibilityFlags = Array.isArray(input.compatibilityFlags) - ? input.compatibilityFlags.map((flag, index) => requireString(flag, `workerCode.compatibilityFlags[${index}]`, { maxBytes: 128 })) - : ["nodejs_compat"]; - const experimentalFlag = firstWorkerdExperimentalCompatFlag(compatibilityFlags); - if (experimentalFlag) { - throw new DoRuntimeError( - 400, - "experimental_compat_flag_unsupported", - `workerCode.compatibilityFlags contains experimental workerd flag ${JSON.stringify(experimentalFlag)}, which WDL does not support for tenant workers` - ); - } - const env = input.env == null ? {} : requireRecord(input.env, "workerCode.env"); - return { - compatibilityDate, - compatibilityFlags, - mainModule, - modules: normalizedModules, - env, - }; -} - /** * @param {JsonRecord} input * @returns {BundleSource} */ function normalizeBundleSource(input) { - const ns = requireString(input.ns, "ns", { pattern: NS_FIELD_RE }); + const ns = requireString(input.ns, "ns"); + if (!isValidRuntimeLoadNs(ns)) { + throw new DoRuntimeError(400, "invalid_request", "ns is not valid"); + } const worker = requireString(input.worker, "worker", { pattern: WORKER_NAME_RE }); - const version = requireString(String(input.version ?? ""), "version", { pattern: VERSION_RE }); + const version = requireString(input.version, "version"); + if (parseVersion(version) == null) { + throw new DoRuntimeError(400, "invalid_request", "version is not valid"); + } const doStorageId = requireString(input.doStorageId, "doStorageId", { pattern: STORAGE_ID_RE }); return { ns, @@ -568,40 +507,31 @@ function normalizeBundleSource(input) { /** * @param {unknown} value - * @param {NormalizeOptions} [options] * @returns {DoInvoke} */ -export function normalizeDoInvokeRequest(value, options = {}) { +export function normalizeDoInvokeRequest(value) { const input = requireRecord(value, "body"); - const allowInlineWorkerCode = options.allowInlineWorkerCode === true; - const hasInlineWorkerCode = input.workerCode != null; - if (hasInlineWorkerCode && !allowInlineWorkerCode) { - throw new DoRuntimeError(400, "invalid_request", "workerCode is only accepted when DO_TEST_HOOKS=1"); - } - /** @type {InvokeSource} */ - const source = hasInlineWorkerCode - ? { - workerId: requireString(input.workerId, "workerId"), - workerCode: normalizeWorkerCode(input.workerCode), - } - : normalizeBundleSource(input); + if (input.workerCode != null) { + throw new DoRuntimeError(400, "invalid_request", "workerCode is not accepted by the DO invoke protocol"); + } + const source = normalizeBundleSource(input); const kind = normalizeInvokeKind(input.kind); const className = requireString(input.className, "className", { pattern: CLASS_NAME_RE }); const objectName = requireString(input.objectName, "objectName"); const base = { kind, hostId: input.hostId == null - ? defaultHostId(source, { className, objectName }) + ? hostIdForObject(source.doStorageId, className, objectName) : normalizeHostId(input.hostId, source, { className, objectName }), className, objectName, - props: "ns" in source ? { + props: { ns: source.ns, worker: source.worker, version: source.version, doStorageId: source.doStorageId, className, - } : {}, + }, ...(input.owner == null ? {} : { owner: /** @type {OwnerFence} */ (normalizeOwnerFence(input.owner)) }), ...source, }; @@ -643,17 +573,6 @@ export function normalizeDoConnectRequest(request) { return normalizeDoInvokeRequest(body); } -/** - * @param {InvokeSource} source - * @param {ObjectTarget} input - */ -function defaultHostId(source, input) { - if (!("ns" in source) || !source.ns || !source.worker) { - throw new DoRuntimeError(400, "invalid_request", "hostId is required for inline workerCode test hooks"); - } - return hostIdForObject(source.doStorageId, input.className, input.objectName); -} - /** @param {DoInvoke} invoke */ export function buildFacetName(invoke) { return `${invoke.className}:${invoke.objectName}`; @@ -741,9 +660,8 @@ export function buildLocalActorRequest(url, invoke, requestId = null) { /** * @param {Request} request - * @param {NormalizeOptions} [options] */ -export async function readLocalActorInvokeRequest(request, options = {}) { +export async function readLocalActorInvokeRequest(request) { if (request.headers.get(LOCAL_ACTOR_ENVELOPE_HEADER) !== LOCAL_ACTOR_ENVELOPE_MARKER) { throw new DoRuntimeError(415, "unsupported_media_type", "DO host actor requests require the local envelope"); } @@ -761,7 +679,7 @@ export async function readLocalActorInvokeRequest(request, options = {}) { } catch { throw new DoRuntimeError(400, "invalid_json", "Request body must be valid JSON"); } - const invoke = normalizeDoInvokeRequest(metadata, options); + const invoke = normalizeDoInvokeRequest(metadata); const bodyBytes = bytes.subarray(4 + metadataLength); if (bodyBytes.length === 0) return invoke; if (!("request" in invoke)) { @@ -797,10 +715,9 @@ function decodeInvokeEnvelope(bytes) { /** * @param {unknown} metadata * @param {Uint8Array} bodyBytes - * @param {NormalizeOptions} options */ -function invokeWithEnvelopeBody(metadata, bodyBytes, options) { - const invoke = normalizeDoInvokeRequest(metadata, options); +function invokeWithEnvelopeBody(metadata, bodyBytes) { + const invoke = normalizeDoInvokeRequest(metadata); if (bodyBytes.length === 0) return invoke; if (!("request" in invoke)) { throw new DoRuntimeError(400, "invalid_request", "DO invoke envelope body is only valid for fetch requests"); @@ -816,9 +733,8 @@ function invokeWithEnvelopeBody(metadata, bodyBytes, options) { /** * @param {Request} request - * @param {NormalizeOptions} [options] */ -export async function readDoInvokeRequest(request, options = {}) { +export async function readDoInvokeRequest(request) { const contentType = request.headers.get("content-type") || ""; if (contentType.split(";", 1)[0].trim().toLowerCase() !== DO_INVOKE_CONTENT_TYPE) { throw new DoRuntimeError( @@ -828,7 +744,7 @@ export async function readDoInvokeRequest(request, options = {}) { ); } const { metadata, bodyBytes } = decodeInvokeEnvelope(await readBoundedBytes(request, MAX_INVOKE_ENVELOPE_BYTES)); - return invokeWithEnvelopeBody(metadata, bodyBytes, options); + return invokeWithEnvelopeBody(metadata, bodyBytes); } /** @param {EnvelopeInvoke} invoke */ diff --git a/do-runtime/protocol/identity.js b/do-runtime/protocol/identity.js index 6bcf36f..722e883 100644 --- a/do-runtime/protocol/identity.js +++ b/do-runtime/protocol/identity.js @@ -1,6 +1,7 @@ import { CLASS_NAME_RE, DO_HOST_SHARD_COUNT, + HOST_ID_RE, MAX_ID_BYTES, STORAGE_ID_RE, } from "do-runtime-protocol-wire-grammar"; @@ -61,7 +62,9 @@ export function hostIdForShard(doStorageId, className, shard) { } requireIdentityString(doStorageId, "doStorageId", STORAGE_ID_RE); requireIdentityString(className, "className", CLASS_NAME_RE); - return `${doStorageId}:${className}:shard${parsed}`; + const hostId = `${doStorageId}:${className}:shard${parsed}`; + requireIdentityString(hostId, "hostId", HOST_ID_RE); + return hostId; } /** diff --git a/do-runtime/protocol/wire-grammar.js b/do-runtime/protocol/wire-grammar.js index 8d7778b..f36be22 100644 --- a/do-runtime/protocol/wire-grammar.js +++ b/do-runtime/protocol/wire-grammar.js @@ -2,12 +2,6 @@ export const MAX_ID_BYTES = 512; export const CLASS_NAME_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/; export const METHOD_NAME_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/; export const HEADER_NAME_RE = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; -export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,254}$/; -// ns field accepts the same shape gateway does (tenant grammar OR reserved -// ____ form). Without this, DO bindings deployed to reserved ns like -// __system__ fail at every host_invoke / actor_create. -export const NS_FIELD_RE = /^(?:[a-z0-9-]+|__[a-z0-9_-]+__)$/; export const STORAGE_ID_RE = /^[a-z0-9_-]+$/; -export const VERSION_RE = /^v[0-9]+$/; -export const HOST_ID_RE = /^[a-z0-9_-]+:[A-Za-z_$][A-Za-z0-9_$]*:shard[0-9]+$/; +export const HOST_ID_RE = /^[a-z0-9_-]+:[A-Za-z_$][A-Za-z0-9_$]*:shard(?:0|[1-9][0-9]*)$/; export const DO_HOST_SHARD_COUNT = 16; diff --git a/do-runtime/task-identity.js b/do-runtime/task-identity.js index c4a8748..7f393f8 100644 --- a/do-runtime/task-identity.js +++ b/do-runtime/task-identity.js @@ -1,4 +1,5 @@ import { createTaskIdentityResolver } from "shared-task-identity"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { DoRuntimeError } from "do-runtime-protocol"; const resolver = createTaskIdentityResolver({ @@ -8,6 +9,7 @@ const resolver = createTaskIdentityResolver({ serviceLabel: "DO", unavailableCode: "task_identity_unavailable", createError: (status, code, message) => new DoRuntimeError(status, code, message), + validateEndpoint: (endpoint, port) => validOwnerEndpointForService(endpoint, port, "do-runtime"), }); export const { diff --git a/docs/modules/d1.md b/docs/modules/d1.md index b616898..7a441c0 100644 --- a/docs/modules/d1.md +++ b/docs/modules/d1.md @@ -66,6 +66,8 @@ Control metadata is explicit: - Delete writes tombstones and does not physically remove SQLite files. - Deploy input may use an alias or physical id; control freezes physical `databaseId` and display `databaseName` into bundle metadata. +- Runtime materialization, query decoding, and rebalance revalidate namespace and + physical `databaseId` against the shared canonical grammar. Runtime ownership: @@ -162,7 +164,13 @@ failover. - Tenant code does not receive backend endpoints or owner credentials. - Tenant-visible D1 metadata and errors must not include owner task ids, backend endpoints, or raw transport error text. -- D1 owner hints must pass service-specific endpoint grammar and generation/task checks. +- D1 owner hints must pass service-specific endpoint grammar and carry a task id plus a + positive JavaScript-safe-integer generation. +- Task identities, rebalance targets, and persisted owner records are validated on write + and read. A persisted record must reconstruct the physical database key under which it + was read; records stored under a different database scope fail closed. Owner forwarding + accepts only the D1 service/headless DNS forms or private RFC1918/100.64 IPv4 addresses + on port 8787; invalid records fail closed before internal auth is attached. - Test hooks are opt-in and must not be enabled in live-ready services. ## Observability diff --git a/docs/modules/d1.zh.md b/docs/modules/d1.zh.md index 2f4623f..f46fc04 100644 --- a/docs/modules/d1.zh.md +++ b/docs/modules/d1.zh.md @@ -39,6 +39,7 @@ Control metadata 是显式 lifecycle： - 用户可见路径只接受 ready database。 - Delete 写 tombstone，不物理删除 SQLite 文件。 - Deploy input 可使用 alias 或 physical id；control 将 physical `databaseId` 和显示用 `databaseName` 冻结进 bundle metadata。 +- Runtime materialization、query decoding 和 rebalance 使用共享 canonical grammar 重新校验 namespace 与 physical `databaseId`。 Runtime ownership： @@ -88,7 +89,8 @@ Key families： - D1 binding metadata 由 control 在 deploy time 冻结。 - Tenant code 不获得 backend endpoint 或 owner credential。 - Tenant-visible D1 metadata 和 error 不得包含 owner task id、backend endpoint 或原始 transport error 文本。 -- D1 owner hint 必须通过 service-specific endpoint grammar 和 generation/task 检查。 +- D1 owner hint 必须通过 service-specific endpoint grammar，并携带 task id 和正的 JavaScript-safe-integer generation。 +- Task identity、rebalance target 和 persisted owner record 在写入和读取时都会校验。Persisted record 必须能重建出读取它的 physical database key；错放在其他 database scope 下的记录会 fail closed。Owner forwarding 只接受 8787 端口上的 D1 service/headless DNS，或 RFC1918/100.64 私网 IPv4；非法记录在附加 internal auth 前 fail closed。 - Test hooks 是 opt-in，不应在 live-ready service 中开启。 ## 可观测性 diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 345d217..7fd9626 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -64,6 +64,12 @@ partial batch result; that is a result envelope, not a generic JSON error envelo Tenant-originated DO fetch bodies are capped at 1 MiB in the runtime facade. The facade rejects an oversized `Content-Length` before reading, and streamed bodies are read incrementally so the cap is enforced before buffering the full body. +DO RPC method names use the JavaScript identifier grammar and are capped at 256 ASCII +bytes by both the runtime client and do-runtime protocol reader. +DO invoke envelopes identify persisted bundles by canonical namespace, worker, version, +and storage id. They do not accept inline worker source. +DO host ids are capped at 512 UTF-8 bytes and use canonical `shardN` suffixes without +leading zeroes. ## Redis / Storage Contracts @@ -150,6 +156,9 @@ when the logical worker is gone or now points at a different `doStorageId`. internal alarm jobs to ready, claims one job under a DB 2 run token, and calls do-runtime `/internal/do/alarms/dispatch`. do-runtime still constructs the native `DoInvoke{kind:"alarm"}` request and uses the normal owner router/fence path. +- Alarm mutation, retarget, dispatch, and whole-worker storage cleanup accept only the + canonical positive JavaScript-safe-integer worker version grammar. Invalid internal or + persisted versions fail before a job is stored or a worker invoke is attempted. - Alarm due times are Unix millisecond timestamps supplied to `setAlarm()`. Workflows and do-runtime both evaluate those timestamps with their local wall clocks; if a backend ready hint reaches do-runtime before the SQLite alarm row is locally due, @@ -218,7 +227,15 @@ interrupt. - Tenant-visible DO metadata and errors must not include owner task ids, backend endpoints, or raw transport error text. - Owner hints are trusted only when returned by do-runtime headers and validated against - endpoint grammar. + endpoint grammar. Owner hints and invoke fences must carry a positive + JavaScript-safe-integer generation. +- Task identities and persisted owner records are validated on write and read. A + persisted record's `ownerKey`, `hostId`, storage id, class, and shard must reconstruct + the Redis scope under which it was read. During owner resolution, its canonical + namespace and worker must also match the invoking bundle before do-runtime reads that + bundle's active storage pointer. Owner forwarding accepts only the DO service/headless + DNS forms or private RFC1918/100.64 IPv4 addresses on port 8788; invalid records fail + closed before internal auth is attached. - Owner-hint and ownership-error defense is layered: tenant response bodies and tenant-supplied control headers are ignored, only do-runtime control headers are trusted, and endpoint grammar/acceptable-address checks must pass for hints. @@ -226,6 +243,11 @@ interrupt. modules and capture the intrinsics used for private-header stripping, request bounds, invoke serialization, replay classification, and endpoint validation. Tenant prototype mutation cannot rewrite a trusted target or replay policy after those checks. +- The injected alarm shim also evaluates before the tenant module and captures the + request, response, numeric, proxy, and reflection operations that classify internal + alarms, update their SQLite state, and install the storage facade. Tenant top-level + mutation of those intrinsics cannot redirect an internal alarm to the tenant fetch + handler or prevent that facade installation. - do-runtime supervisor must call local `127.0.0.1:8788` drain/renew endpoints; Service Connect aliases may hit a different task. diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 509198b..474709e 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -35,6 +35,9 @@ Storage cleanup endpoint 是 native facet storage cleanup 和 worker storage cle DO protocol error 使用 `{ error, message, details? }`。不同于 admin HTTP 的 flat additive error shape，DO protocol detail 会嵌套在 `details` 下，因为消费者是 runtime/DO client protocol，不是通用 admin JSON parser。未知 internal exception 仍会降级成安全的 `internal_error` / `Internal error` message。Storage delete-worker 在 partial batch result 时可能返回 HTTP 207 和 `{ ok:false, deleted, errors }`；这是 result envelope，不是 generic JSON error envelope。 Tenant-originated DO fetch body 在 runtime facade 中限制为 1 MiB。Facade 会在读取前拒绝超限的 `Content-Length`，streamed body 会增量读取，因此 limit 会在完整 buffering 前生效。 +DO RPC method name 使用 JavaScript identifier grammar，并由 runtime client 和 do-runtime protocol reader 同时限制为最多 256 ASCII bytes。 +DO invoke envelope 通过 canonical namespace、worker、version 和 storage id 标识 persisted bundle，不接受 inline worker source。 +DO host id 最多 512 UTF-8 bytes，并使用不带前导零的 canonical `shardN` suffix。 ## Redis / Storage 合同 @@ -77,6 +80,7 @@ workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespa - Shared runtime transport 统一持有 host binding 与 injected facade 的 owner-hint cache wiring、invoke race retry 和 response-header stripping。Connect wrapper 刻意不包含 invoke-only router fallback，以保留 owner-established WebSocket upgrade 语义。 - `WEBSOCKET_RECONNECT_DELAYS_MS` 和 `WEBSOCKET_MAX_BUFFERED_MESSAGES` 可以在不改代码的情况下调整 gateway backend reconnect budget 和 client-message buffer cap。 - Alarm delivery 是 at-least-once。Scheduler 唤醒 Workflows；Workflows 把到期 internal alarm job promote 到 ready，在 DB 2 run token 下 claim，然后调用 do-runtime `/internal/do/alarms/dispatch`。do-runtime 仍然构造原来的 `DoInvoke{kind:"alarm"}` 请求，并走正常 owner router/fence 路径。 +- Alarm mutation、retarget、dispatch 和 whole-worker storage cleanup 只接受 canonical positive JavaScript-safe-integer worker version grammar。非法 internal 或 persisted version 会在写入 job 或尝试 worker invoke 前失败。 - Alarm due time 是传给 `setAlarm()` 的 Unix millisecond timestamp。Workflows 和 do-runtime 都用各自本地 wall clock 判断这些 timestamp；如果 backend ready hint 在 SQLite alarm row 对 do-runtime 来说尚未到期时抵达，do-runtime 会 ignore 这次 dispatch，但不清 row，让 backend due-index repair 路径之后继续投递。这是 alarm compatibility 边界，不属于 Redis-time owner lease fence。 - Failed alarm 使用 `WORKFLOWS_DO_ALARM_RETRY_DELAY_MS`、`WORKFLOWS_DO_ALARM_RETRY_MAX_DELAY_MS` 和 `WORKFLOWS_DO_ALARM_RETRY_JITTER` 的 exponential backoff 和 jitter 重试，最多到 `WORKFLOWS_DO_ALARM_RETRY_MAX_TRIES`（默认 `6`），之后 discard 并增加 `do_alarm_dispatches{outcome="discarded"}`。 - 如果 Workflows client 调用 do-runtime 后 timeout，backend 会保留 running claim 到 `WORKFLOWS_DO_ALARM_CLAIM_LEASE_MS` 过期，而不是立即调度 retry。默认值是五分钟，且配置值会被 clamp 到高于 `WORKFLOWS_DISPATCH_TIMEOUT_MS`，这样正常 timeout 处理可以避免 do-runtime 仍在执行原 dispatch 时并发执行重叠 alarm body。Operator 应按最长预期 alarm handler body 配置 claim lease，而不只是按 HTTP dispatch timeout；alarm body 仍是 at-least-once，claim lease 过期后可能重叠执行。 @@ -99,9 +103,11 @@ Terraform 除了 Fargate task memory limit，还会给 do-runtime workerd contai - do-runtime internal endpoints 只在 private mesh 内可达，并要求共享的 `WDL_INTERNAL_AUTH_TOKEN` / `x-wdl-internal-auth` 内部认证 header。Health 和 metrics endpoint 例外。 - Tenant code 只能通过 runtime 生成的 facade 和 frozen metadata 访问 DO。 - Tenant-visible DO metadata 和 error 不得包含 owner task id、backend endpoint 或原始 transport error 文本。 -- Owner hint 只信任 do-runtime header，并且要通过 endpoint grammar validation。 +- Owner hint 只信任 do-runtime header，并且要通过 endpoint grammar validation。Owner hint 和 invoke fence 必须携带正的 JavaScript-safe-integer generation。 +- Task identity 和 persisted owner record 在写入和读取时都会校验。Persisted record 的 `ownerKey`、`hostId`、storage id、class 和 shard 必须能重建出读取它的 Redis scope；owner resolution 还必须在 do-runtime 读取 invoking bundle 的 active storage pointer 前，确认 record 的 canonical namespace 和 worker 与该 bundle 一致。Owner forwarding 只接受 8788 端口上的 DO service/headless DNS，或 RFC1918/100.64 私网 IPv4；非法记录在附加 internal auth 前 fail closed。 - Owner-hint 与 ownership-error 防御是分层的：忽略 tenant response body 和 tenant-supplied control header，只信任 do-runtime control header；hint 还必须通过 endpoint grammar / acceptable-address 检查。 - 注入的 DO transport 与共享 D1/DO endpoint validator 会在 tenant module 前执行，并捕获 private-header stripping、request bound、invoke serialization、replay classification 和 endpoint validation 使用的 intrinsic。Tenant prototype mutation 不能在校验后改写受信 target 或 replay policy。 +- 注入的 alarm shim 也会在 tenant module 前执行，并捕获 internal alarm 分类、SQLite 状态更新和 storage facade 安装依赖的 request、response、number、proxy 与 reflection 操作。Tenant 顶层对这些 intrinsic 的修改不能把 internal alarm 重定向到 tenant fetch handler，也不能阻止 facade 安装。 - do-runtime supervisor 必须调用本地 `127.0.0.1:8788` drain/renew endpoint；Service Connect alias 可能打到其他 task。 ## 可观测性 diff --git a/docs/modules/infra.md b/docs/modules/infra.md index 32f50ea..30f970b 100644 --- a/docs/modules/infra.md +++ b/docs/modules/infra.md @@ -143,6 +143,12 @@ Stateful storage: changes should document which services can use `FARGATE_SPOT`; stateful runtimes and singleton control loops should stay on on-demand Fargate unless their interruption semantics are re-reviewed. +- The application Terraform root accepts at least two existing subnets. Every subnet + must belong to the configured VPC, use an RFC1918 or `100.64.0.0/10` IPv4 CIDR, and + occupy a distinct Availability Zone because each D1/DO EFS file system creates one + mount target per supplied subnet. Foundation keeps a three-AZ default. The subnet + `map_public_ip_on_launch` flag is not used as a routing classifier; ECS services + explicitly disable public task IP assignment. - In addition to the Fargate task memory limit, D1 and DO workerd containers set explicit container memory hard limits. DO also reserves memory for its local redis-proxy sidecar. diff --git a/docs/modules/infra.zh.md b/docs/modules/infra.zh.md index 6515821..4cd55e0 100644 --- a/docs/modules/infra.zh.md +++ b/docs/modules/infra.zh.md @@ -84,6 +84,7 @@ Stateful storage： - D1/DO 使用 owner lease、monotonic generation fence 和本地 drain/renew。超过 1 个 task 时需要稳定的 per-replica storage identity，并确保 supervisor drain/renew 只走私有本地入口，不能通过 service alias 打到其他 replica。 - 普通 D1/DO task 丢失会退回到 lease expiry，再由其他 replica takeover；graceful rollout 应优先走 supervisor drain，在 child workerd process 退出前释放 ownership。 - Terraform 在 ECS Fargate 上运行这套应用 stack 的服务。修改 capacity policy 时应记录哪些服务可以使用 `FARGATE_SPOT`；stateful runtime 和 singleton control loop 除非重新评估 interruption 语义，否则应保持 on-demand Fargate。 +- Application Terraform root 接受至少两个既有 subnet。每个 subnet 都必须属于指定 VPC、使用 RFC1918 或 `100.64.0.0/10` IPv4 CIDR，并且位于不同 Availability Zone，因为 D1/DO 的每个 EFS file system 都会为每个传入 subnet 创建一个 mount target。Foundation 仍默认创建三个 AZ。Subnet 的 `map_public_ip_on_launch` flag 不用于判断 routing 类型；ECS service 会显式关闭 task public IP assignment。 - 除了 Fargate task memory limit，D1 和 DO 的 workerd container 还会设置显式 container memory hard limit。DO 还会给本地 redis-proxy sidecar 保留内存。 - Terraform Fargate service 在服务能容忍 overlapping capacity 时应使用 rolling replacement。D1/DO 使用 sequential replacement；scheduler 作为 singleton control loop 保持 stop-before-start。D1/DO 和 scheduler 都关闭 Availability Zone rebalancing，让 replacement 遵循各自显式 deployment strategy。 diff --git a/docs/modules/queues-cron.md b/docs/modules/queues-cron.md index bdac1a6..e32d6ef 100644 --- a/docs/modules/queues-cron.md +++ b/docs/modules/queues-cron.md @@ -36,6 +36,12 @@ Scheduler owns runtime delivery: - Cron code lives under `rust/scheduler/src/cron/`. - Queue registry, consume, delayed delivery, DLQ, and orphan handling live under `rust/scheduler/src/queue/`. +- Before constructing a cron runtime worker id, scheduler revalidates the projection's + route namespace, worker name, and version with the canonical `wdl-rust-common` + grammar. Invalid cron refs are removed before slot advance or runtime dispatch, and + cron sweep does not recreate refs from malformed worker metadata. Queue consumer + projections remain trusted Control-owned internal state; scheduler does not add a + second worker identity/version validator for unsupported direct Redis writes. - Scheduler defaults to one replica in deployment, and current dispatch paths are multi-replica safe. Extra replicas improve runtime concurrency but do not imply zero-gap deployment: production rollout may still use stop-before-start semantics and @@ -91,7 +97,9 @@ Cron uses wall-clock minute slots: computes each entry's next fire time with croner and the configured timezone, rounds it to the minute slot, and writes refs into slot buckets. This is repair logic, not the authority. Control uses JavaScript `croner` only for the initial promote-time - slot placement; scheduler uses Rust `croner` for repair and advancement. + slot placement; scheduler uses Rust `croner` for repair and advancement. Worker + hashes without canonical identity or metadata are skipped instead of reseeding + invalid refs, and emit a bounded `invalid_identity` or `invalid_meta` reason. 3. Cron refs carry the entry generation. At fire time scheduler re-reads `crons::` and compares `gen`; missing metadata, corrupt JSON, or generation mismatch makes the ref stale and removes it from the slot. @@ -260,8 +268,10 @@ Important cron signals: - `cron_queue_lag_ms{outcome=...}` - `cron_bucket_size` - `cron_stale_refs_cleaned` +- `cron_sweep_entries_skipped` +- `cron_sweep_workers_skipped` - Logs: `cron_fired`, `cron_lease_lost`, `cron_ref_stale`, `cron_ref_stale_advanced`, - `cron_reconcile` + `cron_sweep_entry_skipped`, `cron_sweep_worker_skipped`, `cron_reconcile` Important queue signals: diff --git a/docs/modules/queues-cron.zh.md b/docs/modules/queues-cron.zh.md index 20058a4..71df338 100644 --- a/docs/modules/queues-cron.zh.md +++ b/docs/modules/queues-cron.zh.md @@ -23,6 +23,7 @@ Scheduler 负责实际投递： - Cron 代码在 `rust/scheduler/src/cron/`。 - Queue registry、consume、delayed delivery、DLQ 和 orphan 处理在 `rust/scheduler/src/queue/`。 +- Scheduler 在构造 cron runtime worker id 前会用 canonical `wdl-rust-common` grammar 重新校验 projection 的 route namespace、worker name 和 version。非法 cron ref 会在 slot advance 或 runtime dispatch 前移除，cron sweep 也不会从 malformed worker metadata 重新播种 ref。Queue consumer projection 仍是受信任的 Control-owned internal state；scheduler 不为不受支持的 direct Redis write 增加第二套 worker identity/version validator。 - 部署上 scheduler 默认 1 个副本，当前 dispatch 路径具备多副本安全性。增加副本可以提高运行时并发，但不等于部署零中断：生产 rollout 仍可采用 stop-before-start 语义，并短暂暂停调度。 ## 接口 @@ -60,7 +61,7 @@ Scheduler 不是通用 job runner。它把 Redis projection 转成 runtime call Cron 使用 wall-clock 分钟 slot： 1. `wait_ms_until_next_slot()` 睡到下一个 UTC 分钟边界。tick loop 随后扫描当前 `cron-slot:` bucket 和前一个 bucket。扫描前一个 bucket 是为了覆盖分钟 rollover 附近刚写入的 ref。 -2. 另一条 sweep/reconcile 路径从 `cron:index:workers` 读取 active cron hash，用 croner 和配置的 timezone 计算每个 entry 的下一次触发时间，再 round 到分钟 slot，并把 ref 写入 slot bucket。这个流程是 repair 逻辑，不是权威状态。Control 只在 promote-time 初始 slot placement 使用 JavaScript `croner`；scheduler repair 和 advance 使用 Rust `croner`。 +2. 另一条 sweep/reconcile 路径从 `cron:index:workers` 读取 active cron hash，用 croner 和配置的 timezone 计算每个 entry 的下一次触发时间，再 round 到分钟 slot，并把 ref 写入 slot bucket。这个流程是 repair 逻辑，不是权威状态。Control 只在 promote-time 初始 slot placement 使用 JavaScript `croner`；scheduler repair 和 advance 使用 Rust `croner`。缺少 canonical identity 或 metadata 的 worker hash 会被跳过，不会重新播种非法 ref，并输出 bounded `invalid_identity` 或 `invalid_meta` reason。 3. Cron ref 带 entry generation。真正 fire 前，scheduler 会重新读取 `crons::` 并比较 `gen`；metadata 缺失、JSON 损坏或 generation 不匹配都会让 ref 变成 stale，并从 slot 中移除。 4. Scheduler 会先原子地 lease ref、从当前 slot 移除、加入下一个 slot，然后才调用 runtime。这个顺序保证每个 slot 只 fire 一次，并且 runtime/network 失败不会变成自动 cron retry。 5. 如果某个 ref 滞留在早于当前 wall-clock slot 的旧 slot 中，scheduler 只把它 advance 到下一个 future slot，不会 fire。也就是说 outage 或 scheduler 长时间停顿期间错过的 cron event 会被跳过，而不是补发。 @@ -160,7 +161,9 @@ Scheduler 为 cron 和 queue outcome 输出结构化日志和 Prometheus metrics - `cron_queue_lag_ms{outcome=...}` - `cron_bucket_size` - `cron_stale_refs_cleaned` -- Logs：`cron_fired`、`cron_lease_lost`、`cron_ref_stale`、`cron_ref_stale_advanced`、`cron_reconcile` +- `cron_sweep_entries_skipped` +- `cron_sweep_workers_skipped` +- Logs：`cron_fired`、`cron_lease_lost`、`cron_ref_stale`、`cron_ref_stale_advanced`、`cron_sweep_entry_skipped`、`cron_sweep_worker_skipped`、`cron_reconcile` 重要 queue 信号： diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 0aefed6..8ad7263 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -156,7 +156,9 @@ Service bindings are frozen at caller deploy time. Control resolves target names worker, version, and entrypoint, stores them in caller metadata, and runtime loads that exact target version on first use. Promoting the target later does not move existing callers; caller redeploy is the refresh boundary. This version pinning makes rollback -and version delete referrer checks deterministic. +and version delete referrer checks deterministic. Runtime revalidates the persisted +pinned version with the canonical positive JavaScript-safe-integer grammar before +materializing the binding. Cross-namespace service bindings require a target `[[exports]]` entry for the bound entrypoint, including `entrypoint = "default"` for the default export. The entry's diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 06039bd..aeb04f7 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -59,7 +59,7 @@ ASSETS 是 deploy-artifact helper，不是完整 Cloudflare Pages asset pipeline R2 和 ASSETS 生命周期语义故意不同。ASSETS 是 deploy artifact，version/worker delete 会 stage `worker-delete-s3-cleanup` work。R2 是 tenant runtime data，worker delete 永远不删除 R2 object。 -Service binding 在 caller deploy 时冻结。Control 解析 target namespace、worker、version 和 entrypoint，存入 caller metadata，runtime 第一次使用时加载该精确 target version。之后 target promote 不会移动已有 caller；caller redeploy 才是 refresh 边界。这种 version pinning 让 rollback 和 version delete referrer check 都是确定的。 +Service binding 在 caller deploy 时冻结。Control 解析 target namespace、worker、version 和 entrypoint，存入 caller metadata，runtime 第一次使用时加载该精确 target version。之后 target promote 不会移动已有 caller；caller redeploy 才是 refresh 边界。这种 version pinning 让 rollback 和 version delete referrer check 都是确定的。Runtime 会在 materialize binding 前用 canonical positive JavaScript-safe-integer grammar 重新校验 persisted pinned version。 Cross-namespace service binding 要求 target 为被绑定的 entrypoint 声明 `[[exports]]` entry；default export 也要声明 `entrypoint = "default"`。该 entry 的 `allowed_callers` 控制 cross-namespace 访问；`["*"]` 对所有 namespace 开放，空数组则关闭 cross-namespace caller。没有 `[[exports]]` 的 target 只向 same-namespace caller 暴露 default entrypoint。Same-namespace caller 绕过 ACL，但 target 一旦声明 `[[exports]]`，仍受 strict entrypoint visibility 约束。ACL 变化是 deploy-time，不是 call-time；既有 caller 会保持 pin，直到 redeploy。 diff --git a/docs/modules/workflows.md b/docs/modules/workflows.md index 7e0a85f..7dcc4e9 100644 --- a/docs/modules/workflows.md +++ b/docs/modules/workflows.md @@ -133,10 +133,11 @@ Key families: - Scheduler also wakes Workflows-owned internal DO alarm jobs through the same `/internal/workflows/tick` endpoint; scheduler never reads or writes DO alarm state directly. -- Workflows revalidates persisted DO alarm namespace, worker, and version identity with - the `wdl-rust-common` owners before dispatch. Runtime run dispatch and progress - callbacks share one system-vs-user runtime endpoint selector inside the workflows - crate. +- Workflows rejects non-canonical worker versions before persisting DO alarm jobs, + revalidates persisted alarm identity before dispatch, and validates an active route + version before using it as a retarget. These checks use the `wdl-rust-common` owners. + Runtime run dispatch and progress callbacks share one system-vs-user runtime endpoint + selector inside the workflows crate. - 32 scheduling shards partition ready/due work. - Ready tokens are deduplicated hints; instance hash state is authority. - Execution commits are fenced by `generation`, `runToken`, active instance status, and @@ -184,6 +185,9 @@ canonicalizes against the current active route before writing DB 2, so new durab business processes start on the active version. Existing instances replay against their stored `frozenVersion`; promotion does not change their code. Worker-version delete is blocked by `wf:by-version` while non-expired instances still reference the version. +Runtime validates every dispatched `frozenVersion` with the same positive +JavaScript-safe-integer version parser used by bundle keys; malformed persisted tags +fail before worker loading. Scheduling is hint-based but state-authoritative: diff --git a/docs/modules/workflows.zh.md b/docs/modules/workflows.zh.md index f7c5887..f5403ba 100644 --- a/docs/modules/workflows.zh.md +++ b/docs/modules/workflows.zh.md @@ -83,7 +83,7 @@ Key families： - Instance 冻结创建时的 worker version/class identity。 - Scheduler 只负责唤醒 workflows；admission、fairness、shard tick、ready/due movement 和 runtime dispatch 都由 workflows 负责。 - Scheduler 也通过同一个 `/internal/workflows/tick` endpoint 唤醒 Workflows-owned internal DO alarm jobs；scheduler 不直接读写 DO alarm state。 -- Workflows 在 dispatch 前用 `wdl-rust-common` owner 重新校验持久化 DO alarm 的 namespace、worker 和 version identity。Runtime run dispatch 与 progress callback 在 workflows crate 内共用同一个 system-vs-user runtime endpoint selector。 +- Workflows 在持久化 DO alarm job 前拒绝 non-canonical worker version，在 dispatch 前重新校验持久化 alarm identity，并在把 active route 用作 retarget 前校验其 version。这些检查共用 `wdl-rust-common` owner。Runtime run dispatch 与 progress callback 在 workflows crate 内共用同一个 system-vs-user runtime endpoint selector。 - 32 个 scheduling shards 划分 ready/due work。 - Ready token 是去重 hint；instance hash state 是权威状态。 - Execution commit 同时用 `generation`、`runToken`、active instance status 和未过期 run lease fence。Step commit/register 接受同一 run 的 `running` 或 `waiting` 状态，因此一个并行 sibling 进入 retry/wait 后，另一个 sibling 仍可完成；completed runtime terminal 要求 `running`，failed runtime terminal 在 run lease 仍有效时也可以关闭由非法未 await suspending step 造成的同一 run `waiting` 状态。如果 lease 已过期，workflows 只恢复 ready hint，让下一次 claim 在新 lease 下 replay。Lifecycle commit 只用 generation fence，并在同一个 Lua commit 内 rotate `generation`。 @@ -98,7 +98,7 @@ Workflow execution 使用两条 channel： 1. Loaded worker 通过 reserved `__WDL_WORKFLOWS_BACKEND__` Fetcher binding 调 workflows。Runtime 从 bundle metadata 附加 identity；workflows 不信任 tenant body 中的 namespace、worker、version、workflow key、class 或 instance identity。 2. workflows 把已 claim 的 run dispatch 回 runtime `:8088` 上的 `/internal/workflows/run`。Runtime 加载 frozen worker version 并调用 `className.run(event, stepFacade)`。 -Create/restart 与 replay 的 version pinning 不同。新的 `create()` 或 `restart()` 写 DB 2 前会按当前 active route canonicalize，因此新的 durable business process 从 active version 开始。已有 instance 使用自己存的 `frozenVersion` replay；promotion 不会改变它的代码。只要 non-expired instance 仍引用某个 version，`wf:by-version` 就会阻止 worker-version delete。 +Create/restart 与 replay 的 version pinning 不同。新的 `create()` 或 `restart()` 写 DB 2 前会按当前 active route canonicalize，因此新的 durable business process 从 active version 开始。已有 instance 使用自己存的 `frozenVersion` replay；promotion 不会改变它的代码。只要 non-expired instance 仍引用某个 version，`wf:by-version` 就会阻止 worker-version delete。Runtime 会用 bundle key 共用的正 JavaScript-safe-integer version parser 校验每个 dispatch 的 `frozenVersion`；malformed persisted tag 会在加载 worker 前失败。 Scheduling 是 hint-based，但状态权威在 instance hash： diff --git a/docs/redis-key-layout.md b/docs/redis-key-layout.md index 1946f43..d86c0b8 100644 --- a/docs/redis-key-layout.md +++ b/docs/redis-key-layout.md @@ -8,12 +8,12 @@ families, and ownership rules that span modules. WDL uses a deliberate logical split: -- **DB 0, control plane:** bundles, routes/patterns, auth, D1/DO owner state, cron +- **`DB 0`, control plane:** bundles, routes/patterns, auth, D1/DO owner state, cron config, queue-consumer config, lifecycle metadata, and workflow definitions (`wf:defs:*`). -- **DB 1, data plane:** KV hash buckets, queue streams, delayed queues, orphan streams, +- **`DB 1`, data plane:** KV hash buckets, queue streams, delayed queues, orphan streams, and live log-tail streams. -- **DB 2, workflows:** `wf:schema_version`, instance state, step records/summaries, +- **`DB 2`, workflows:** `wf:schema_version`, instance state, step records/summaries, ready/due shards, events and event-type indexes, payload refs, retention indexes, and run leases. @@ -52,8 +52,9 @@ secrets: Hash, namespace-level WDL-ENC envelopes secrets:: Hash, worker-level WDL-ENC envelopes ``` -`worker:::v:` uses the integer version in the key, not the `"v"` -tag. Test fixtures that seed Redis directly must use `shared/version.js#bundleKey`. +`worker:::v:` uses a positive JavaScript-safe integer version in the +key, not the `"v"` tag. Test fixtures that seed Redis directly must use +`shared/version.js#bundleKey`. `namespaces` is an active worker gate. It is populated when a namespace has an active worker route and may be removed when the last active worker is deleted. @@ -144,6 +145,11 @@ Feature modules own the detailed contracts: Cross-cutting constraints: +- Persisted D1/DO owner records must reconstruct the encoded scope of the Redis key + under which they are read. A syntactically valid record stored under another scope + fails closed before forwarding, takeover, renewal, or release. DO owner resolution + also binds the record's canonical namespace and worker to the invoking bundle before + reading that bundle's active storage pointer. - Indexes are usually repairable projections, not authority. The module doc must state which key is authoritative before adding a writer. - Lifecycle and delete blocker indexes are authoritative where the module says they are; diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index 36232bc..51350c3 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -6,9 +6,9 @@ WDL 使用明确的逻辑切分： -- **DB 0，控制面：**bundle、routes/patterns、auth、D1/DO owner state、cron config、queue-consumer config、lifecycle metadata，以及 workflow definition（`wf:defs:*`）。 -- **DB 1，数据面：**KV hash bucket、queue stream、delayed queue、orphan stream 和 live log-tail stream。 -- **DB 2，workflows：**`wf:schema_version`、instance state、step record/summary、ready/due shard、event 和 event-type index、payload ref、retention index、run lease。 +- **`DB 0`，控制面：**bundle、routes/patterns、auth、D1/DO owner state、cron config、queue-consumer config、lifecycle metadata，以及 workflow definition（`wf:defs:*`）。 +- **`DB 1`，数据面：**KV hash bucket、queue stream、delayed queue、orphan stream 和 live log-tail stream。 +- **`DB 2`，workflows：**`wf:schema_version`、instance state、step record/summary、ready/due shard、event 和 event-type index、payload ref、retention index、run lease。 Local compose、Kubernetes 和 Terraform 都启用这个切分。Rust service 和 Rust `redis-proxy` 使用 `DATA_REDIS_URL` / `DATA_REDIS_DB` 选择 data-plane Redis connection/database；嵌入的 JS control/log-tail 路径使用 `DATA_REDIS_ADDR` 加 `DATA_REDIS_DB`，因为它们的 RESP client 接收 host:port address。未设置这些 data-plane 变量的部署会把数据面 key 留在 control Redis connection/database，直到显式 opt in。Workflows 不同：未设置 `WORKFLOWS_REDIS_URL` 时 workflows service 仍默认使用 DB 2；只有显式设置 `WORKFLOWS_REDIS_DB=0` 时才使用 DB 0。 @@ -37,7 +37,7 @@ secrets: Hash, namespace-level WDL-ENC envelope secrets:: Hash, worker-level WDL-ENC envelope ``` -`worker:::v:` 的 key 使用整数 version，而不是 `"v"` tag。直接 seed Redis 的测试 fixture 必须使用 `shared/version.js#bundleKey`。 +`worker:::v:` 的 key 使用 JavaScript safe-integer 范围内的正整数 version，而不是 `"v"` tag。直接 seed Redis 的测试 fixture 必须使用 `shared/version.js#bundleKey`。 `namespaces` 是 active worker gate。有 active worker route 时会加入，最后一个 active worker 删除时可能移除。Namespace-level secrets 和 data-plane state 等资源可以比这个 set membership 活得更久。Auth 在 delegated token issue 时只把它作为 generated-namespace collision 的 best-effort 信号读取，而不是永久 namespace registry。 @@ -91,6 +91,7 @@ Routes、crons、queue consumers、bindings、vars、exports、workflow definiti 跨模块约束： +- Persisted D1/DO owner record 必须能重建出读取它的 Redis key 所编码的 scope。语法合法但错放在其他 scope 下的记录会在 forwarding、takeover、renew 或 release 前 fail closed；DO owner resolution 还必须在读取 invoking bundle 的 active storage pointer 前，把 record 的 canonical namespace 和 worker 绑定到该 bundle。 - Index 通常是可修复 projection，不是 authority。新增 writer 前，模块文档必须说明哪个 key 是权威状态。 - Lifecycle 和 delete blocker index 在模块文档声明为权威时就是权威；不要增加绕过这些 index 的 request-path fallback scan。 - Queue main stream 不做 trim，因为 at-least-once delivery 是合同。DLQ、orphan、log-tail 这类诊断 stream 可以使用有界 approximate trim。 diff --git a/docs/source-map.md b/docs/source-map.md index 6b6362a..720e073 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -59,7 +59,8 @@ are outside this map unless they own runtime or deployable service behavior. |---|---| | `shared/redis.js`, `shared/redis-*.js` | Public Redis import surface plus split RESP codec, per-call client, WATCH/MULTI session, and subscriber loop modules. Runtime hot paths prefer the Rust redis-proxy sidecar. | | `shared/redis-lock.js` | Token-fenced Redis lock creation, acquire, renewal, and best-effort token-scoped release shared by Control and Auth. | -| `shared/owner-lease.js`, `shared/owner-protocol.js`, `shared/owner-forwarder.js` | Shared optimistic retry loop plus owner lease parsing, generation counters, key derivation, fence matching, staged Redis owner writes, and owner-forwarding HTTP mechanics used by Control and the D1/DO runtimes. | +| `shared/optimistic-retry.js` | Generic bounded optimistic retry loop used by Control and the D1/DO owner-lease adapters. | +| `shared/owner-endpoint.js`, `shared/owner-lease.js`, `shared/owner-protocol.js`, `shared/owner-forwarder.js` | Shared owner endpoint grammar, lease parsing, generation counters, key derivation, fence matching, staged Redis owner writes, and authenticated forwarding mechanics used by Control and the D1/DO runtimes. | | `shared/auth-roles.js` | Role table, principal validation, reserved namespace policy, and auth action capabilities. | | `shared/auth-token.js` | Shared `x-admin-token` sanitizer used by control and auth. | | `shared/internal-auth.js` | Shared internal mesh auth header and token helpers used by JS callers and receivers. | diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index ee5f220..7b9208b 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -56,7 +56,8 @@ |---|---| | `shared/redis.js`、`shared/redis-*.js` | 公共 Redis import surface，以及拆分后的 RESP codec、per-call client、WATCH/MULTI session 和 subscriber loop modules。Runtime hot path 优先使用 Rust redis-proxy sidecar。 | | `shared/redis-lock.js` | Control 和 Auth 共用的 token-fenced Redis lock creation、acquire、renewal 和 best-effort token-scoped release。 | -| `shared/owner-lease.js`、`shared/owner-protocol.js`、`shared/owner-forwarder.js` | Control 与 D1/DO runtimes 共用的 optimistic retry loop、owner lease parsing、generation counters、key derivation、fence matching、staged Redis owner writes 和 owner-forwarding HTTP mechanics。 | +| `shared/optimistic-retry.js` | Control 和 D1/DO owner-lease adapter 共用的通用有界 optimistic retry loop。 | +| `shared/owner-endpoint.js`、`shared/owner-lease.js`、`shared/owner-protocol.js`、`shared/owner-forwarder.js` | Control 与 D1/DO runtimes 共用的 owner endpoint grammar、owner lease parsing、generation counters、key derivation、fence matching、staged Redis owner writes 和 authenticated forwarding mechanics。 | | `shared/auth-roles.js` | Role table、principal validation、reserved namespace policy 和 auth action capabilities。 | | `shared/auth-token.js` | Control 和 auth 共用的 `x-admin-token` sanitizer。 | | `shared/internal-auth.js` | JS caller 和 receiver 共用的 internal mesh auth header / token helpers。 | diff --git a/docs/testing.md b/docs/testing.md index 00f6a8f..1e0d519 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -204,7 +204,7 @@ Prefer the narrow helper that matches the response or fixture source: load import-free contract owners such as `shared/version.js` and `shared/ns-pattern.js` directly with `repositoryFileUrl(...)` instead of reimplementing their grammar in a stub. - use it instead of ad hoc `readFileSync(...).replace(...)` chains when a test + Use it instead of ad hoc `readFileSync(...).replace(...)` chains when a test rewrites repo module imports. Use `readRepositoryFile(...)` plus `applyModuleReplacements(...)` for local source rewrites, `readRepositoryModuleSource(...)` when reading repository module source and diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index e979b9e..364005c 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -7,6 +7,7 @@ export const MAX_DO_REQUEST_BODY_BYTES = 1024 * 1024; export const MAX_DO_INVOKE_ENVELOPE_BYTES = 2 * 1024 * 1024; export const MAX_DO_REQUEST_HEADER_COUNT = 128; export const MAX_DO_REQUEST_HEADER_BYTES = 64 * 1024; +const MAX_DO_RPC_METHOD_BYTES = 256; export const DO_ACCEPT_OWNER_HINT_HEADER = "x-wdl-do-accept-owner-hint"; export const DO_OWNER_HINT_CONTROL_HEADER = "x-wdl-do-owner-hint"; export const DO_OWNERSHIP_ERROR_CONTROL_HEADER = "x-wdl-do-ownership-error"; @@ -78,6 +79,7 @@ const intrinsicHeadersHas = Headers.prototype.has; const intrinsicHeadersSet = Headers.prototype.set; const intrinsicJsonStringify = JSON.stringify; const intrinsicNumberIsFinite = Number.isFinite; +const intrinsicNumberIsSafeInteger = Number.isSafeInteger; const intrinsicObjectCreate = Object.create; const intrinsicObjectEntries = Object.entries; const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; @@ -363,6 +365,11 @@ function encodeUtf8(value) { return intrinsicReflectApply(intrinsicTextEncoderEncode, utf8Encoder, [value]); } +/** @param {unknown} value */ +function numberValue(value) { + return intrinsicReflectApply(IntrinsicNumber, undefined, [value]); +} + /** @param {unknown} value */ function stringifyJson(value) { return intrinsicReflectApply(intrinsicJsonStringify, IntrinsicJSON, [cloneJsonData(value)]); @@ -435,8 +442,8 @@ async function readRequestBodyBytes(request) { // local instead of importing a helper that would add another facade module. const contentLength = headerValue(requestHeaders(request), "content-length"); if (contentLength != null && contentLength !== "") { - const declared = Number(contentLength); - if (Number.isFinite(declared) && declared > MAX_DO_REQUEST_BODY_BYTES) { + const declared = numberValue(contentLength); + if (intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [declared]) && declared > MAX_DO_REQUEST_BODY_BYTES) { throw new TypeError(`Durable Object fetch body exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } } @@ -475,64 +482,68 @@ async function readRequestBodyBytes(request) { * @param {unknown} value * @param {string} field * @param {WeakSet} [seen] - * @returns {string | null} + * @returns {unknown} */ -function jsonDataError(value, field, seen = new IntrinsicWeakSet()) { +function cloneJsonRpcData(value, field, seen = new IntrinsicWeakSet()) { if (value === null) return null; const type = typeof value; - if (type === "string" || type === "boolean") return null; + if (type === "string" || type === "boolean") return value; if (type === "number") { - return intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [value]) - ? null - : `${field} must be a finite number`; + if (!intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [value])) { + throw new TypeError(`${field} must be a finite number`); + } + return value; } if (type === "bigint" || type === "function" || type === "symbol" || type === "undefined") { - return `${field} must be JSON data`; + throw new TypeError(`${field} must be JSON data`); } - if (type !== "object") return `${field} must be JSON data`; + if (type !== "object") throw new TypeError(`${field} must be JSON data`); const objectValue = /** @type {Record | unknown[]} */ (value); if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { - return `${field} must not be circular`; + throw new TypeError(`${field} must not be circular`); } intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); - if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { - const arrayValue = /** @type {unknown[]} */ (objectValue); - for (let i = 0; i < arrayValue.length; i++) { - if (!(i in arrayValue)) return `${field} must not be sparse`; - const error = jsonDataError(arrayValue[i], `${field}[${i}]`, seen); - if (error) return error; + try { + if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { + const arrayValue = /** @type {unknown[]} */ (objectValue); + const length = arrayValue.length; + const out = new IntrinsicArray(length); + intrinsicReflectApply(intrinsicObjectSetPrototypeOf, IntrinsicObject, [out, null]); + for (let i = 0; i < length; i++) { + if (!(i in arrayValue)) throw new TypeError(`${field} must not be sparse`); + out[i] = cloneJsonRpcData(arrayValue[i], `${field}[${i}]`, seen); + } + return out; + } + const proto = intrinsicReflectApply(intrinsicObjectGetPrototypeOf, IntrinsicObject, [objectValue]); + if (proto !== IntrinsicObject.prototype && proto !== null) { + throw new TypeError(`${field} must be a plain JSON object`); + } + const symbols = intrinsicReflectApply(intrinsicObjectGetOwnPropertySymbols, IntrinsicObject, [objectValue]); + if (symbols.length > 0) throw new TypeError(`${field} must not contain symbol keys`); + const out = /** @type {Record} */ ( + intrinsicReflectApply(intrinsicObjectCreate, IntrinsicObject, [null]) + ); + const entries = /** @type {[string, unknown][]} */ ( + intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) + ); + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + out[entry[0]] = cloneJsonRpcData(entry[1], `${field}.${entry[0]}`, seen); } + return out; + } finally { intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); - return null; - } - const proto = intrinsicReflectApply(intrinsicObjectGetPrototypeOf, IntrinsicObject, [objectValue]); - if (proto !== IntrinsicObject.prototype && proto !== null) return `${field} must be a plain JSON object`; - const symbols = intrinsicReflectApply(intrinsicObjectGetOwnPropertySymbols, IntrinsicObject, [objectValue]); - if (symbols.length > 0) return `${field} must not contain symbol keys`; - const entries = /** @type {[string, unknown][]} */ ( - intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) - ); - for (let i = 0; i < entries.length; i++) { - const entry = entries[i]; - const error = jsonDataError(entry[1], `${field}.${entry[0]}`, seen); - if (error) return error; } - intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); - return null; -} - -/** @param {unknown} args */ -export function assertJsonRpcArgs(args) { - if (!intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [args])) { - throw new TypeError("rpc.args must be an array"); - } - const error = jsonDataError(args, "rpc.args"); - if (error) throw new TypeError(error); } /** @param {unknown} method */ export function assertRpcMethod(method) { - if (typeof method !== "string" || !regexpTest(/^[A-Za-z_$][A-Za-z0-9_$]*$/, method)) { + if ( + typeof method !== "string" || + byteLength(method) > MAX_DO_RPC_METHOD_BYTES || + !regexpTest(/^[A-Za-z_$][A-Za-z0-9_$]*$/, method) + ) { throw new TypeError("rpc.method is not valid"); } if (setHas(RPC_RESERVED_METHODS, method)) { @@ -549,8 +560,11 @@ export function assertRpcMethod(method) { */ export function rpcInvokeBody(props, objectName, method, args) { assertRpcMethod(method); - assertJsonRpcArgs(args); - if (byteLength(stringifyJson(args)) > MAX_DO_REQUEST_BODY_BYTES) { + if (!intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [args])) { + throw new TypeError("rpc.args must be an array"); + } + const stableArgs = /** @type {unknown[]} */ (cloneJsonRpcData(args, "rpc.args")); + if (byteLength(stringifyJson(stableArgs)) > MAX_DO_REQUEST_BODY_BYTES) { throw new TypeError(`rpc.args exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } return encodeDoInvokeEnvelope({ @@ -561,7 +575,7 @@ export function rpcInvokeBody(props, objectName, method, args) { className: props.className, objectName, kind: "rpc", - rpc: { method, args }, + rpc: { method, args: stableArgs }, }); } @@ -752,8 +766,11 @@ export function ownerHintFromHeaders(headers) { const endpoint = headerValue(headers, DO_OWNER_HINT_HEADERS.endpoint); const rawGeneration = headerValue(headers, DO_OWNER_HINT_HEADERS.generation); if (rawGeneration == null || rawGeneration === "") return null; - const generation = Number(rawGeneration); - if (!ownerKey || !taskId || !validOwnerEndpoint(endpoint) || !Number.isInteger(generation) || generation < 0) { + const generation = numberValue(rawGeneration); + if ( + !ownerKey || !taskId || !validOwnerEndpoint(endpoint) || + !intrinsicReflectApply(intrinsicNumberIsSafeInteger, IntrinsicNumber, [generation]) || generation <= 0 + ) { return null; } return { ownerKey, taskId, endpoint: /** @type {string} */ (endpoint), generation }; diff --git a/runtime/_wdl-owner-endpoint.js b/runtime/_wdl-owner-endpoint.js index cc3ce51..94625ed 100644 --- a/runtime/_wdl-owner-endpoint.js +++ b/runtime/_wdl-owner-endpoint.js @@ -1,122 +1,2 @@ -const OWNER_ENDPOINT_SERVICE_RE = { - "d1-runtime": /^d1-runtime(?:-[a-z0-9-]+)?$/, - "do-runtime": /^do-runtime(?:-[a-z0-9-]+)?$/, -}; - -const OWNER_ENDPOINT_HEADLESS_RE = { - "d1-runtime": /^d1-runtime-[0-9]+\.d1-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, - "do-runtime": /^do-runtime-[0-9]+\.do-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, -}; -const INVALID_OWNER_ENDPOINT_RE = /[/?#@[\]\\]/; -const IPV4_OCTET_RE = /^(?:0|[1-9]\d{0,2})$/; -const IntrinsicNumber = Number; -const IntrinsicString = String; -const IntrinsicURL = URL; -const intrinsicReflectApply = Reflect.apply; -const intrinsicRegExpTest = RegExp.prototype.test; -const intrinsicStringSplit = String.prototype.split; -const intrinsicUrlHashGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "hash")); -const intrinsicUrlHostGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "host")); -const intrinsicUrlHostnameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "hostname")); -const intrinsicUrlPasswordGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "password")); -const intrinsicUrlPathnameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "pathname")); -const intrinsicUrlPortGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "port")); -const intrinsicUrlSearchGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "search")); -const intrinsicUrlUsernameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "username")); - -/** @param {object | null} prototype @param {string} name */ -function prototypeGetter(prototype, name) { - let current = prototype; - while (current) { - const getter = Object.getOwnPropertyDescriptor(current, name)?.get; - if (getter) return getter; - current = Object.getPrototypeOf(current); - } - return undefined; -} - -/** @param {RegExp} regexp @param {string} value */ -function regexpTest(regexp, value) { - return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); -} - -/** @param {string} value */ -function splitDots(value) { - return intrinsicReflectApply(intrinsicStringSplit, value, ["."]); -} - -/** @param {unknown} value */ -function numberValue(value) { - return intrinsicReflectApply(IntrinsicNumber, undefined, [value]); -} - -/** @param {unknown} value */ -function stringValue(value) { - return intrinsicReflectApply(IntrinsicString, undefined, [value]); -} - -/** @param {(this: URL) => string} getter @param {URL} url */ -function urlValue(getter, url) { - return intrinsicReflectApply(getter, url, []); -} - -/** - * @param {unknown} endpoint - * @param {number} port - * @param {"d1-runtime" | "do-runtime"} serviceName - * @returns {boolean} - */ -export function validOwnerEndpointForService(endpoint, port, serviceName) { - if (typeof endpoint !== "string" || !endpoint) return false; - const serviceRe = OWNER_ENDPOINT_SERVICE_RE[serviceName]; - const headlessRe = OWNER_ENDPOINT_HEADLESS_RE[serviceName]; - if (!serviceRe) throw new Error(`Unknown owner endpoint service ${serviceName}`); - if (regexpTest(INVALID_OWNER_ENDPOINT_RE, endpoint)) return false; - let url; - try { - url = new IntrinsicURL(`http://${endpoint}`); - } catch { - return false; - } - if ( - urlValue(intrinsicUrlHostGet, url) !== endpoint || - urlValue(intrinsicUrlUsernameGet, url) || - urlValue(intrinsicUrlPasswordGet, url) || - urlValue(intrinsicUrlPathnameGet, url) !== "/" || - urlValue(intrinsicUrlSearchGet, url) || - urlValue(intrinsicUrlHashGet, url) - ) { - return false; - } - if (urlValue(intrinsicUrlPortGet, url) !== stringValue(port)) return false; - const hostname = urlValue(intrinsicUrlHostnameGet, url); - // ECS task ENI addresses are VPC-local but not necessarily RFC1918; some - // deployments may use non-RFC1918 VPC-local ranges. Trust comes from - // runtime-authored owner-hint headers, while this check rejects URL injection - // and obviously unsafe endpoint classes. - return regexpTest(serviceRe, hostname) || regexpTest(headlessRe, hostname) || acceptableIpv4(hostname); -} - -/** - * @param {string} hostname - * @returns {boolean} - */ -function acceptableIpv4(hostname) { - const parts = splitDots(hostname); - if (parts.length !== 4) return false; - /** @type {number[]} */ - const octets = []; - for (let i = 0; i < parts.length; i++) { - const part = parts[i]; - if (!regexpTest(IPV4_OCTET_RE, part)) return false; - const value = numberValue(part); - if (value < 0 || value > 255) return false; - octets[i] = value; - } - const a = octets[0]; - const b = octets[1]; - if (a === 0 || a === 127) return false; - if (a === 169 && b === 254) return false; - if (a >= 224) return false; - return true; -} +// Local adapter for the injected DO transport's relative module name. +export { validOwnerEndpointForService } from "../shared/owner-endpoint.js"; diff --git a/runtime/bindings/d1.js b/runtime/bindings/d1.js index c921b11..1b99e2e 100644 --- a/runtime/bindings/d1.js +++ b/runtime/bindings/d1.js @@ -19,7 +19,7 @@ import { } from "shared-d1-query-wire"; import { metrics } from "runtime-metrics"; import { createOwnerHintCache } from "runtime-owner-hint-cache"; -import { validOwnerEndpointForService } from "runtime-owner-endpoint"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { serviceNameFromEnv } from "runtime-bindings-proxy"; import { withInternalAuth } from "shared-internal-auth"; import { errorMessage } from "shared-errors"; @@ -33,7 +33,9 @@ const OWNER_HINT_STALE_CODES = new Set([ "not-owner", "owner-not-ready", "owner-unavailable", + "owner-record-invalid", "owner-endpoint-missing", + "owner-endpoint-invalid", "forward-hop-exhausted", "owner-claim-raced", "owner-takeover-raced", @@ -147,8 +149,8 @@ function ownerHintFromHeaders(headers) { if ( !taskId || !validOwnerEndpointForService(endpoint, 8787, "d1-runtime") || - !Number.isInteger(generation) || - generation < 0 + !Number.isSafeInteger(generation) || + generation <= 0 ) { return null; } diff --git a/runtime/bindings/service.js b/runtime/bindings/service.js index 06b7329..f7b2c5c 100644 --- a/runtime/bindings/service.js +++ b/runtime/bindings/service.js @@ -12,6 +12,7 @@ import { WorkerEntrypoint } from "cloudflare:workers"; import { getLoadedWorkerStub } from "runtime-load"; import { fetchTailFields, startTailEnvelope } from "runtime-tail-forwarder"; import { INTERNAL_AUTH_HEADER } from "shared-internal-auth"; +import { sanitizeRequestId } from "shared-observability"; const SERVICE_BINDING_INTERNAL_HEADERS = [ "x-worker-id", @@ -63,7 +64,9 @@ export class ServiceBinding extends WorkerEntrypoint { forwarded.headers.set("x-worker-id", targetId); // ALS doesn't cross JSRPC in workerd — the outer rid isn't visible here, // so we can only preserve what the caller forwarded, never mint one. - const requestId = forwarded.headers.get("x-request-id") || null; + const requestId = sanitizeRequestId(forwarded.headers.get("x-request-id")); + if (requestId) forwarded.headers.set("x-request-id", requestId); + else forwarded.headers.delete("x-request-id"); const identity = { namespace: self.ctx.props.targetNs, workerName: self.ctx.props.targetWorker, diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 145a414..48c6006 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -66,10 +66,10 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), - # _wdl-do-transport.js imports the same helper by relative file path, while - # host bindings import it by bare module name. Workerd needs both entries. - (name = "_wdl-owner-endpoint.js", esModule = embed "_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), + # Injected DO transport uses a relative module name; host modules use the + # shared bare name. Both resolve to the same shared contract owner. + (name = "_wdl-owner-endpoint.js", esModule = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache", esModule = embed "_wdl-owner-hint-cache.js"), (name = "shared-d1-data-field", esModule = embed "../shared/d1-data-field.js"), (name = "shared-d1-params", esModule = embed "../shared/d1-params.js"), @@ -83,7 +83,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-r2-utils-source", text = embed "r2-utils.js"), (name = "runtime-do-client-source", text = embed "do-client.js"), (name = "runtime-do-transport-source", text = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint-source", text = embed "_wdl-owner-endpoint.js"), + (name = "runtime-owner-endpoint-source", text = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache-source", text = embed "_wdl-owner-hint-cache.js"), (name = "runtime-request-id-source", text = embed "_wdl-request-id.js"), (name = "runtime-workflows-client-source", text = embed "workflows-client.js"), @@ -92,6 +92,7 @@ const loaderWorker :Workerd.Worker = ( (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "shared-base64", esModule = embed "../shared/base64.js"), + (name = "shared-version", esModule = embed "../shared/version.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), @@ -140,7 +141,7 @@ const loaderWorker :Workerd.Worker = ( const doOwnerNetworkWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "do-owner-network.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), ], compatibilityDate = "2026-04-24", @@ -158,11 +159,12 @@ const controlWorker :Workerd.Worker = ( (name = "control-json-body", esModule = embed "../control/json-body.js"), (name = "control-optimistic", esModule = embed "../control/optimistic.js"), (name = "shared-owner-lease", esModule = embed "../shared/owner-lease.js"), + (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), (name = "control-workflows-client", esModule = embed "../control/workflows-client.js"), (name = "control-lib", esModule = embed "../control/lib.js"), (name = "control-d1-model", esModule = embed "../control/d1-model.js"), (name = "control-d1-runtime-client", esModule = embed "../control/d1-runtime-client.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), (name = "control-d1-lifecycle", esModule = embed "../control/d1-lifecycle.js"), (name = "control-d1-migrations", esModule = embed "../control/d1-migrations.js"), (name = "control-d1-store", esModule = embed "../control/d1-store.js"), @@ -249,7 +251,7 @@ const controlWorker :Workerd.Worker = ( (name = "runtime-r2-utils-source", text = embed "r2-utils.js"), (name = "runtime-do-client-source", text = embed "do-client.js"), (name = "runtime-do-transport-source", text = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint-source", text = embed "_wdl-owner-endpoint.js"), + (name = "runtime-owner-endpoint-source", text = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache-source", text = embed "_wdl-owner-hint-cache.js"), (name = "runtime-request-id-source", text = embed "_wdl-request-id.js"), (name = "runtime-workflows-client-source", text = embed "workflows-client.js"), diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index 110ea95..ea34afc 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -65,10 +65,10 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), - # _wdl-do-transport.js imports the same helper by relative file path, while - # host bindings import it by bare module name. Workerd needs both entries. - (name = "_wdl-owner-endpoint.js", esModule = embed "_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), + # Injected DO transport uses a relative module name; host modules use the + # shared bare name. Both resolve to the same shared contract owner. + (name = "_wdl-owner-endpoint.js", esModule = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache", esModule = embed "_wdl-owner-hint-cache.js"), (name = "shared-d1-data-field", esModule = embed "../shared/d1-data-field.js"), (name = "shared-d1-params", esModule = embed "../shared/d1-params.js"), @@ -82,7 +82,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-r2-utils-source", text = embed "r2-utils.js"), (name = "runtime-do-client-source", text = embed "do-client.js"), (name = "runtime-do-transport-source", text = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint-source", text = embed "_wdl-owner-endpoint.js"), + (name = "runtime-owner-endpoint-source", text = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache-source", text = embed "_wdl-owner-hint-cache.js"), (name = "runtime-request-id-source", text = embed "_wdl-request-id.js"), (name = "runtime-workflows-client-source", text = embed "workflows-client.js"), @@ -91,6 +91,7 @@ const loaderWorker :Workerd.Worker = ( (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "shared-base64", esModule = embed "../shared/base64.js"), + (name = "shared-version", esModule = embed "../shared/version.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), @@ -139,7 +140,7 @@ const loaderWorker :Workerd.Worker = ( const doOwnerNetworkWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "do-owner-network.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), ], compatibilityDate = "2026-04-24", diff --git a/runtime/d1-client.js b/runtime/d1-client.js index 4081910..c907376 100644 --- a/runtime/d1-client.js +++ b/runtime/d1-client.js @@ -9,6 +9,10 @@ import { requestIdFromOptions } from "./_wdl-request-id.js"; const D1_SESSION_CONSTRAINT_FIRST_PRIMARY = "first-primary"; const D1_SESSION_CONSTRAINT_FIRST_UNCONSTRAINED = "first-unconstrained"; +const intrinsicReflectApply = Reflect.apply; +const intrinsicWeakMapGet = WeakMap.prototype.get; +const intrinsicWeakMapHas = WeakMap.prototype.has; +const intrinsicWeakMapSet = WeakMap.prototype.set; /** * @typedef {string | number | null | undefined | number[]} D1Param @@ -92,10 +96,25 @@ const databaseState = /** @type {WeakMap} */ (new WeakM const sessionState = /** @type {WeakMap} */ (new WeakMap()); const statementState = /** @type {WeakMap} */ (new WeakMap()); +/** @param {WeakMap} map @param {object} key */ +function weakMapGet(map, key) { + return intrinsicReflectApply(intrinsicWeakMapGet, map, [key]); +} + +/** @param {WeakMap} map @param {object} key */ +function weakMapHas(map, key) { + return intrinsicReflectApply(intrinsicWeakMapHas, map, [key]); +} + +/** @param {WeakMap} map @param {object} key @param {unknown} value */ +function weakMapSet(map, key, value) { + intrinsicReflectApply(intrinsicWeakMapSet, map, [key, value]); +} + /** @param {object} dbOrSession */ function rootDatabase(dbOrSession) { - if (databaseState.has(dbOrSession)) return dbOrSession; - const session = sessionState.get(dbOrSession); + if (weakMapHas(databaseState, dbOrSession)) return dbOrSession; + const session = /** @type {D1SessionState | undefined} */ (weakMapGet(sessionState, dbOrSession)); if (session) return session.db; throw new D1Error("D1_ERROR: invalid D1 database/session object", { code: "invalid-database", @@ -105,14 +124,14 @@ function rootDatabase(dbOrSession) { /** @param {object} db */ function requestIdFor(db) { - const state = databaseState.get(db); + const state = /** @type {D1DatabaseState | undefined} */ (weakMapGet(databaseState, db)); if (!state) return null; return requestIdFromOptions(state.requestIdOptions); } /** @param {object} statement */ function serializeStatement(statement) { - const state = statementState.get(statement); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, statement)); if (!state) { throw new D1Error("D1_ERROR: batch() expects D1PreparedStatement values", { code: "invalid-batch", @@ -143,7 +162,7 @@ function prepareFor(dbOrSession, statement) { */ async function sendFor(dbOrSession, mode, statements) { const db = rootDatabase(dbOrSession); - const state = databaseState.get(db); + const state = /** @type {D1DatabaseState | undefined} */ (weakMapGet(databaseState, db)); if (!state) throw new D1Error("D1_ERROR: invalid D1 database object", { code: "invalid-database", category: "invalid-request", @@ -181,7 +200,7 @@ async function batchFor(dbOrSession, statements) { /** @type {SerializedStatement[]} */ const serialized = []; for (const statement of statements) { - const state = statementState.get(statement); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, statement)); if (!state) { throw new D1Error("D1_ERROR: batch() expects D1PreparedStatement values", { code: "invalid-batch", @@ -208,13 +227,13 @@ export class D1PreparedStatement { * @param {object} [owner] */ constructor(dbOrSession, statement, params = [], owner = dbOrSession) { - statementState.set(this, { dbOrSession, statement, params, owner }); + weakMapSet(statementState, this, { dbOrSession, statement, params, owner }); } /** @param {...unknown} values */ bind(...values) { try { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new Error("invalid prepared statement"); return new D1PreparedStatement( state.dbOrSession, @@ -232,7 +251,7 @@ export class D1PreparedStatement { /** @param {string} [columnName] */ async first(columnName) { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new D1Error("D1_ERROR: invalid prepared statement", { code: "invalid-statement", category: "invalid-request", @@ -255,7 +274,7 @@ export class D1PreparedStatement { } async run() { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new D1Error("D1_ERROR: invalid prepared statement", { code: "invalid-statement", category: "invalid-request", @@ -264,7 +283,7 @@ export class D1PreparedStatement { } async all() { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new D1Error("D1_ERROR: invalid prepared statement", { code: "invalid-statement", category: "invalid-request", @@ -274,7 +293,7 @@ export class D1PreparedStatement { /** @param {{ columnNames?: boolean }} [options] */ async raw(options = {}) { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new D1Error("D1_ERROR: invalid prepared statement", { code: "invalid-statement", category: "invalid-request", @@ -297,7 +316,7 @@ export class D1DatabaseSession { */ constructor(db, constraintOrBookmark = D1_SESSION_CONSTRAINT_FIRST_UNCONSTRAINED) { const normalized = String(constraintOrBookmark ?? "").trim() || D1_SESSION_CONSTRAINT_FIRST_UNCONSTRAINED; - sessionState.set(this, { db, bookmarkOrConstraint: normalized }); + weakMapSet(sessionState, this, { db, bookmarkOrConstraint: normalized }); } /** @param {unknown} statement */ @@ -311,7 +330,7 @@ export class D1DatabaseSession { } getBookmark() { - const state = sessionState.get(this); + const state = /** @type {D1SessionState | undefined} */ (weakMapGet(sessionState, this)); if (!state) return null; const { bookmarkOrConstraint } = state; if ( @@ -330,7 +349,7 @@ export class D1Database { * @param {{ requestId?: unknown, requestIdProvider?: unknown }} [options] */ constructor(stub, options = {}) { - databaseState.set(this, { + weakMapSet(databaseState, this, { stub, requestIdOptions: options, }); diff --git a/runtime/do-client.js b/runtime/do-client.js index e294ae6..2d5e6cb 100644 --- a/runtime/do-client.js +++ b/runtime/do-client.js @@ -15,6 +15,13 @@ import { createOwnerHintCache } from "./_wdl-owner-hint-cache.js"; import { requestIdFromOptions } from "./_wdl-request-id.js"; const ownerHintCache = createOwnerHintCache(); +const intrinsicObjectHasOwn = Object.hasOwn; +const intrinsicReflectApply = Reflect.apply; + +/** @param {object} object @param {PropertyKey} key */ +function objectHasOwn(object, key) { + return intrinsicReflectApply(intrinsicObjectHasOwn, undefined, [object, key]); +} /** * @typedef {{ fetch(url: string, init?: RequestInit): Promise }} DoBackend @@ -139,12 +146,12 @@ export class DurableObjectNamespace { */ constructor(binding, options = {}) { const isMetadataBinding = binding && typeof binding === "object" && ( - Object.hasOwn(binding, "ns") || - Object.hasOwn(binding, "worker") || - Object.hasOwn(binding, "version") || - Object.hasOwn(binding, "doStorageId") || - Object.hasOwn(binding, "binding") || - Object.hasOwn(binding, "className") + objectHasOwn(binding, "ns") || + objectHasOwn(binding, "worker") || + objectHasOwn(binding, "version") || + objectHasOwn(binding, "doStorageId") || + objectHasOwn(binding, "binding") || + objectHasOwn(binding, "className") ); if (binding && !isMetadataBinding) { this.#setBindingProxy(/** @type {DurableObjectBindingProxy} */ (binding)); diff --git a/runtime/do-owner-network.js b/runtime/do-owner-network.js index 3eb9b14..4151274 100644 --- a/runtime/do-owner-network.js +++ b/runtime/do-owner-network.js @@ -1,4 +1,4 @@ -import { validOwnerEndpointForService } from "runtime-owner-endpoint"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { withInternalAuth } from "shared-internal-auth"; /** diff --git a/runtime/lib.js b/runtime/lib.js index 2e6f469..28a6a95 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -1,6 +1,7 @@ // Pure helpers for the runtime worker. import { base64ToBytes, bytesToBase64 } from "shared-base64"; +import { parseVersion } from "shared-version"; import { isWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; const utf8Encoder = new TextEncoder(); @@ -111,7 +112,6 @@ const ENHANCED_ERROR_SERIALIZATION_FLAG = "enhanced_error_serialization"; const ROUTE_NS_RE = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?|__system__)$/; const RESERVED_TENANT_NS = new Set(["admin"]); const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,254}$/; -const VERSION_RE = /^v[1-9][0-9]*$/; // Loaded workers may declare an older compatibilityDate than the platform // workers. Keep enhanced error serialization as a floor only before the date @@ -312,7 +312,11 @@ function requiredWorkerName(body) { /** @param {Record | null} body */ function requiredVersion(body) { - return requiredPatternString(body, "frozenVersion", VERSION_RE, "an immutable worker version"); + const version = requiredString(body, "frozenVersion"); + if (parseVersion(version) == null) { + throw new Error("frozenVersion must be an immutable worker version"); + } + return version; } /** @param {Record | null} body @param {string} field */ diff --git a/runtime/load/env-build.js b/runtime/load/env-build.js index 40749a5..c478f08 100644 --- a/runtime/load/env-build.js +++ b/runtime/load/env-build.js @@ -1,11 +1,14 @@ import { BINDING_NAME_RE, + D1_DATABASE_ID_RE, RESERVED_OBJECT_KEYS, WDL_RESERVED_BINDING_RE, WDL_RESERVED_ENTRYPOINT_RE, isValidJsIdentifier, isValidJsClassDeclarationName, + isValidRuntimeLoadNs, } from "shared-ns-pattern"; +import { parseVersion } from "shared-version"; const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; @@ -91,8 +94,11 @@ function materializeQueueBinding({ name, spec, ns, ctx }) { /** @param {RuntimeBindingMaterializerArgs} args */ function materializeD1Binding({ name, spec, ns, ctx }) { const databaseId = spec.databaseId; - if (typeof databaseId !== "string" || !databaseId) { - throw new Error(`Binding "${name}" is a D1 binding but missing databaseId`); + if (!isValidRuntimeLoadNs(ns)) { + throw new Error(`Binding "${name}" is a D1 binding but has invalid namespace`); + } + if (typeof databaseId !== "string" || !D1_DATABASE_ID_RE.test(databaseId)) { + throw new Error(`Binding "${name}" is a D1 binding but has invalid databaseId`); } return { value: ctx.exports.D1Database({ @@ -164,6 +170,9 @@ function materializeServiceBinding({ name, spec, ns, worker, nsSecrets, workerSe `Binding "${name}" is a service binding but missing service/version (control should have pinned these at deploy time)` ); } + if (parseVersion(spec.version) == null) { + throw new Error(`Binding "${name}" is a service binding but has invalid version (redeploy ${ns}/${worker})`); + } if (spec.ns != null && (typeof spec.ns !== "string" || !spec.ns)) { throw new Error(`Binding "${name}" is a service binding but has invalid ns (redeploy ${ns}/${worker})`); } diff --git a/runtime/load/wrapper-generate.js b/runtime/load/wrapper-generate.js index 611d44c..8b79fcf 100644 --- a/runtime/load/wrapper-generate.js +++ b/runtime/load/wrapper-generate.js @@ -240,8 +240,8 @@ function withRequestContext(context, arg, fn) { function wrapClassInstance(instance, requestContext) { return createProxy(instance, { - get(target, prop, receiver) { - const value = reflectGet(target, prop, receiver); + get(target, prop) { + const value = reflectGet(target, prop, target); if (typeof value !== "function") return value; return function(...args) { return withRequestContext(requestContext, args[0], () => applyFunction(value, target, args)); diff --git a/runtime/r2-client.js b/runtime/r2-client.js index 67d849f..d3b6a64 100644 --- a/runtime/r2-client.js +++ b/runtime/r2-client.js @@ -22,6 +22,9 @@ import { requestIdFromOptions } from "./_wdl-request-id.js"; */ const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); +const intrinsicReflectApply = Reflect.apply; +const intrinsicWeakMapGet = WeakMap.prototype.get; +const intrinsicWeakMapSet = WeakMap.prototype.set; /** @param {unknown} value */ function dateFromUnknown(value) { @@ -279,9 +282,19 @@ function metaWithDates(meta) { /** @type {WeakMap} */ const bucketState = new WeakMap(); +/** @param {WeakMap} map @param {object} key */ +function weakMapGet(map, key) { + return intrinsicReflectApply(intrinsicWeakMapGet, map, [key]); +} + +/** @param {WeakMap} map @param {object} key @param {unknown} value */ +function weakMapSet(map, key, value) { + intrinsicReflectApply(intrinsicWeakMapSet, map, [key, value]); +} + /** @param {R2Bucket} bucket */ function bucketRequestMeta(bucket) { - const state = bucketState.get(bucket); + const state = /** @type {R2BucketState | undefined} */ (weakMapGet(bucketState, bucket)); if (!state) return {}; const requestId = requestIdFromOptions(state.requestIdOptions); return requestId ? { requestId } : {}; @@ -398,7 +411,7 @@ export class R2Bucket { constructor(stub, options = {}) { // Provider is used by class-style entrypoints where the same env wrapper // can serve different fetch/queue/scheduled invocations with different ids. - bucketState.set(this, { + weakMapSet(bucketState, this, { stub, requestIdOptions: options, }); @@ -406,7 +419,7 @@ export class R2Bucket { /** @param {string} key */ async head(key) { - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.head !== "function") throw new TypeError("R2 stub head is not configured"); const meta = await stub.head(normalizeR2ObjectKey(key), bucketRequestMeta(this)); return meta ? new R2Object(meta) : null; @@ -414,7 +427,7 @@ export class R2Bucket { /** @param {string} key @param {AnyRecord} [options] */ async get(key, options) { - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.get !== "function") throw new TypeError("R2 stub get is not configured"); const result = await stub.get( normalizeR2ObjectKey(key), @@ -431,7 +444,7 @@ export class R2Bucket { const normalizedOptions = normalizePutOptions(options); const bytes = await valueToBytes(value); assertR2BufferSize(bytes.byteLength, "put"); - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.put !== "function") throw new TypeError("R2 stub put is not configured"); const meta = await stub.put( normalizeR2ObjectKey(key), @@ -454,7 +467,7 @@ export class R2Bucket { /** @param {string | string[]} keys */ async delete(keys) { - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.delete !== "function") throw new TypeError("R2 stub delete is not configured"); if (Array.isArray(keys)) { await stub.delete( @@ -468,7 +481,7 @@ export class R2Bucket { /** @param {AnyRecord} [options] */ async list(options = {}) { - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.list !== "function") throw new TypeError("R2 stub list is not configured"); const out = await stub.list({ prefix: options.prefix == null ? undefined : String(options.prefix), diff --git a/rust/common/src/queue_keys.rs b/rust/common/src/queue_keys.rs index 27bff43..d9d5b30 100644 --- a/rust/common/src/queue_keys.rs +++ b/rust/common/src/queue_keys.rs @@ -5,6 +5,9 @@ use crate::identity::is_valid_runtime_load_ns; pub const QUEUE_CONSUMER_INDEX_KEY: &str = "queue:index:consumers"; pub const QUEUE_STREAM_INDEX_KEY: &str = "queue:index:streams"; pub const QUEUE_DELAYED_INDEX_KEY: &str = "queue:index:delayed"; +pub const QUEUE_CONSUMER_SCAN_PATTERN: &str = "queue-consumer:*:*"; +pub const QUEUE_STREAM_SCAN_PATTERN: &str = "queue:*:*:s"; +pub const QUEUE_DELAYED_SCAN_PATTERN: &str = "queue-delayed:*:*"; pub const QUEUE_DELAYED_WAKE_STREAM: &str = "queue-delayed-wake"; pub const QUEUE_DELAYED_WAKE_KEY_FIELD: &str = "delayed_key"; pub const QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD: &str = "visible_at"; @@ -102,6 +105,9 @@ mod tests { assert_eq!(QUEUE_CONSUMER_INDEX_KEY, "queue:index:consumers"); assert_eq!(QUEUE_STREAM_INDEX_KEY, "queue:index:streams"); assert_eq!(QUEUE_DELAYED_INDEX_KEY, "queue:index:delayed"); + assert_eq!(QUEUE_CONSUMER_SCAN_PATTERN, "queue-consumer:*:*"); + assert_eq!(QUEUE_STREAM_SCAN_PATTERN, "queue:*:*:s"); + assert_eq!(QUEUE_DELAYED_SCAN_PATTERN, "queue-delayed:*:*"); assert_eq!(QUEUE_DELAYED_WAKE_STREAM, "queue-delayed-wake"); assert_eq!(QUEUE_DELAYED_WAKE_KEY_FIELD, "delayed_key"); assert_eq!(QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD, "visible_at"); diff --git a/rust/common/src/version.rs b/rust/common/src/version.rs index 2780690..72b6a9b 100644 --- a/rust/common/src/version.rs +++ b/rust/common/src/version.rs @@ -1,7 +1,8 @@ //! Worker version and bundle-key helpers shared by Rust services. //! //! JavaScript control code owns the canonical version tag grammar in -//! `shared/version.js`: `v[1-9][0-9]*`. Rust services that read bundle hashes +//! `shared/version.js`: `v[1-9][0-9]*`, bounded to JavaScript's safe-integer +//! range. Rust services that read bundle hashes //! must use the same grammar so malformed Redis state fails closed instead of //! silently normalizing to another worker version. //! @@ -12,6 +13,8 @@ use std::fmt; +const MAX_SAFE_VERSION: u64 = 9_007_199_254_740_991; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct InvalidVersionTag; @@ -28,7 +31,10 @@ pub fn parse_version_tag(version: &str) -> Result { if raw.is_empty() || raw.starts_with('0') || !raw.bytes().all(|byte| byte.is_ascii_digit()) { return Err(InvalidVersionTag); } - raw.parse::().map_err(|_| InvalidVersionTag) + let parsed = raw.parse::().map_err(|_| InvalidVersionTag)?; + (parsed <= MAX_SAFE_VERSION) + .then_some(parsed) + .ok_or(InvalidVersionTag) } pub fn worker_bundle_key( @@ -66,6 +72,21 @@ mod tests { assert_eq!(parse_version_tag("v42").unwrap(), 42); } + #[test] + fn matches_cross_language_version_fixture() { + let fixture: serde_json::Value = + serde_json::from_str(include_str!("../../../tests/fixtures/version-tags.json")) + .expect("version tag fixture parses"); + for case in fixture["cases"] + .as_array() + .expect("version tag fixture cases is an array") + { + let tag = case["tag"].as_str().expect("version tag is a string"); + let expected = case["parsed"].as_u64(); + assert_eq!(parse_version_tag(tag).ok(), expected, "{tag:?}"); + } + } + #[test] fn rejects_malformed_version_tags() { for version in ["", "v", "v0", "v01", "1", "V1", "v1a"] { diff --git a/rust/scheduler/src/cron/reference.rs b/rust/scheduler/src/cron/reference.rs index fb3e197..3c2ad57 100644 --- a/rust/scheduler/src/cron/reference.rs +++ b/rust/scheduler/src/cron/reference.rs @@ -1,4 +1,8 @@ use serde::Deserialize; +use wdl_rust_common::{ + identity::{is_valid_route_ns, is_valid_worker_name}, + version::parse_version_tag, +}; const CRON_WORKER_KEY_PREFIX: &str = "crons:"; pub(crate) const CRON_WORKER_KEY_SCAN_PATTERN: &str = "crons:*:*"; @@ -38,7 +42,7 @@ pub(crate) fn cron_worker_key(ns: &str, worker: &str) -> String { pub(crate) fn parse_cron_worker_key(key: &str) -> Option<(&str, &str)> { let (ns, worker) = key.strip_prefix(CRON_WORKER_KEY_PREFIX)?.split_once(':')?; - if ns.is_empty() || worker.is_empty() || worker.contains(':') { + if !is_valid_route_ns(ns) || !is_valid_worker_name(worker) { return None; } Some((ns, worker)) @@ -56,6 +60,9 @@ pub(crate) fn parse_ref(reference: &str) -> Option { if parts.iter().any(|part| part.is_empty()) { return None; } + if !is_valid_route_ns(parts[0]) || !is_valid_worker_name(parts[1]) { + return None; + } let r#gen = parts[3].parse::().ok()?; Some(RefParts { ns: parts[0].to_string(), @@ -65,6 +72,15 @@ pub(crate) fn parse_ref(reference: &str) -> Option { }) } +fn validated_cron_meta_version(meta: CronMeta) -> Option { + parse_version_tag(&meta.version).ok()?; + Some(meta.version) +} + +pub(crate) fn cron_meta_version(raw: &str) -> Option { + validated_cron_meta_version(serde_json::from_str::(raw).ok()?) +} + pub(crate) fn classify_ref( parts: &RefParts, meta_str: Option, @@ -82,12 +98,15 @@ pub(crate) fn classify_ref( let Ok(entry) = serde_json::from_str::(&entry_str) else { return RefVerdict::Corrupt; }; + let Some(active_version) = validated_cron_meta_version(meta) else { + return RefVerdict::Corrupt; + }; if entry.r#gen != parts.r#gen { return RefVerdict::Stale("gen_mismatch"); } RefVerdict::Fire { entry, - active_version: meta.version, + active_version, } } @@ -96,7 +115,7 @@ mod tests { use serde_json::json; use super::*; - use crate::test_fixtures::scheduler_projection_contract; + use crate::test_fixtures::{scheduler_projection_contract, version_tag_cases}; #[test] fn cron_projection_contract_matches_control_writer() { @@ -148,6 +167,9 @@ mod tests { assert!(parse_ref("demo:hello::7").is_none()); assert!(parse_ref("demo:hello:abc123:x").is_none()); assert!(parse_ref("demo:hello:abc123:").is_none()); + assert!(parse_ref("__platform__:hello:abc123:7").is_none()); + assert!(parse_ref("Demo:hello:abc123:7").is_none()); + assert!(parse_ref("demo:/bad:abc123:7").is_none()); } #[test] @@ -157,6 +179,9 @@ mod tests { assert_eq!(parse_cron_worker_key("crons::worker"), None); assert_eq!(parse_cron_worker_key("crons:demo:"), None); assert_eq!(parse_cron_worker_key("crons:demo:worker:extra"), None); + assert_eq!(parse_cron_worker_key("crons:__platform__:worker"), None); + assert_eq!(parse_cron_worker_key("crons:Demo:worker"), None); + assert_eq!(parse_cron_worker_key("crons:demo:/bad"), None); } #[test] @@ -184,6 +209,29 @@ mod tests { } } + #[test] + fn classify_ref_versions_match_cross_language_fixture() { + let parts = RefParts { + ns: "demo".to_string(), + worker: "hello".to_string(), + cron_id: "abc".to_string(), + r#gen: 3, + }; + let entry = Some(json!({ "cron": "*/5 * * * *", "timezone": "UTC", "gen": 3 }).to_string()); + for (version, valid) in version_tag_cases() { + let verdict = classify_ref( + &parts, + Some(json!({ "version": version }).to_string()), + entry.clone(), + ); + assert_eq!( + matches!(verdict, RefVerdict::Fire { .. }), + valid, + "{version:?}" + ); + } + } + #[test] fn classify_ref_marks_missing_or_mismatched_state_stale() { let parts = RefParts { diff --git a/rust/scheduler/src/cron/sweep.rs b/rust/scheduler/src/cron/sweep.rs index 63a5501..84263ac 100644 --- a/rust/scheduler/src/cron/sweep.rs +++ b/rust/scheduler/src/cron/sweep.rs @@ -6,7 +6,9 @@ use crate::{ AppState, LogLevel, SERVICE, SchedulerResult, indexed_keys_after_backfill, log, now_ms, }; -use super::reference::{CRON_WORKER_KEY_SCAN_PATTERN, CronEntry, parse_cron_worker_key, ref_for}; +use super::reference::{ + CRON_WORKER_KEY_SCAN_PATTERN, CronEntry, cron_meta_version, parse_cron_worker_key, ref_for, +}; use super::slot::{next_fire_ms, slot_expire_at, slot_key, slot_ms_for}; const CRON_WORKER_INDEX_KEY: &str = "cron:index:workers"; @@ -69,6 +71,23 @@ fn count_sadd_replies(replies: &[i64]) -> u64 { .sum() } +fn has_dispatchable_cron_meta(hash: &HashMap) -> bool { + hash.get("__meta__") + .and_then(|raw| cron_meta_version(raw)) + .is_some() +} + +fn dispatchable_cron_worker<'a>( + key: &'a str, + hash: &HashMap, +) -> Result<(&'a str, &'a str), &'static str> { + let (ns, worker) = parse_cron_worker_key(key).ok_or("invalid_identity")?; + if !has_dispatchable_cron_meta(hash) { + return Err("invalid_meta"); + } + Ok((ns, worker)) +} + async fn write_cron_slot_refs( state: &AppState, slot_refs: BTreeMap>, @@ -98,17 +117,25 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { let started = now_ms(); let mut workers = 0_u64; let mut skipped = 0_u64; + let mut skipped_workers = 0_u64; let keys = cron_worker_keys(&state).await?; let mut slot_refs: BTreeMap> = BTreeMap::new(); for chunk in keys.chunks(CRON_SWEEP_READ_CHUNK_SIZE) { let hashes = fetch_cron_hash_chunk(&state, chunk).await?; for (key, hash) in chunk.iter().zip(hashes) { workers += 1; - if !hash.contains_key("__meta__") { - continue; - } - let Some((ns, worker)) = parse_cron_worker_key(key) else { - continue; + let (ns, worker) = match dispatchable_cron_worker(key, &hash) { + Ok(identity) => identity, + Err(reason) => { + skipped_workers += 1; + log( + &state, + LogLevel::Warn, + "cron_sweep_worker_skipped", + json!({ "worker_key": key, "reason": reason }), + ); + continue; + } }; let now = now_ms(); for (id, raw) in hash { @@ -168,6 +195,11 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { &[("service", SERVICE)], skipped as f64, ); + state.metrics.increment( + "cron_sweep_workers_skipped", + &[("service", SERVICE)], + skipped_workers as f64, + ); log( &state, LogLevel::Info, @@ -176,6 +208,8 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { "workers": workers, "re_added": re_added, "skipped": skipped, + "entries_skipped": skipped, + "workers_skipped": skipped_workers, "duration_ms": now_ms() - started, }), ); @@ -229,4 +263,41 @@ mod tests { fn count_sadd_replies_ignores_expireat_replies() { assert_eq!(count_sadd_replies(&[2, 1, 0, 1, 4, 1]), 6); } + + #[test] + fn cron_sweep_requires_dispatchable_worker_metadata() { + let valid = HashMap::from([( + "__meta__".to_string(), + json!({ "version": "v1" }).to_string(), + )]); + assert_eq!( + dispatchable_cron_worker("crons:demo:worker", &valid), + Ok(("demo", "worker")) + ); + assert_eq!( + dispatchable_cron_worker("crons:__platform__:worker", &valid), + Err("invalid_identity") + ); + assert_eq!( + dispatchable_cron_worker("crons:demo:worker", &HashMap::new()), + Err("invalid_meta") + ); + assert_eq!( + dispatchable_cron_worker( + "crons:demo:worker", + &HashMap::from([("__meta__".to_string(), "{broken".to_string(),)]) + ), + Err("invalid_meta") + ); + assert_eq!( + dispatchable_cron_worker( + "crons:demo:worker", + &HashMap::from([( + "__meta__".to_string(), + json!({ "version": "v9007199254740992" }).to_string(), + )]) + ), + Err("invalid_meta") + ); + } } diff --git a/rust/scheduler/src/queue/keys.rs b/rust/scheduler/src/queue/keys.rs index f976eae..001de6f 100644 --- a/rust/scheduler/src/queue/keys.rs +++ b/rust/scheduler/src/queue/keys.rs @@ -1,5 +1,6 @@ pub(crate) use wdl_rust_common::queue_keys::{ - QUEUE_CONSUMER_INDEX_KEY, QUEUE_DELAYED_INDEX_KEY, QUEUE_STREAM_INDEX_KEY, parse_consumer_key, - parse_delayed_key, parse_stream_key, queue_consumer_key, queue_delayed_key, queue_dlq_key, - queue_orphaned_key, queue_stream_key, + QUEUE_CONSUMER_INDEX_KEY, QUEUE_CONSUMER_SCAN_PATTERN, QUEUE_DELAYED_INDEX_KEY, + QUEUE_DELAYED_SCAN_PATTERN, QUEUE_STREAM_INDEX_KEY, QUEUE_STREAM_SCAN_PATTERN, + parse_consumer_key, parse_delayed_key, parse_stream_key, queue_consumer_key, queue_delayed_key, + queue_dlq_key, queue_orphaned_key, queue_stream_key, }; diff --git a/rust/scheduler/src/queue/registry.rs b/rust/scheduler/src/queue/registry.rs index c8a8651..b19a870 100644 --- a/rust/scheduler/src/queue/registry.rs +++ b/rust/scheduler/src/queue/registry.rs @@ -9,7 +9,8 @@ use crate::{ }; use super::{ - Consumer, QUEUE_CONSUMER_INDEX_KEY, QUEUE_DELAYED_INDEX_KEY, QUEUE_STREAM_INDEX_KEY, + Consumer, QUEUE_CONSUMER_INDEX_KEY, QUEUE_CONSUMER_SCAN_PATTERN, QUEUE_DELAYED_INDEX_KEY, + QUEUE_DELAYED_SCAN_PATTERN, QUEUE_STREAM_INDEX_KEY, QUEUE_STREAM_SCAN_PATTERN, parse_consumer_key, queue_consumer_key, queue_stream_key, redis_error_is_busygroup, }; @@ -72,8 +73,12 @@ async fn flush_indexed_streams( pub(crate) async fn queue_reconcile(state: AppState) -> SchedulerResult<()> { let mut seen = HashSet::new(); let mut registry_changed = false; - let consumer_keys = - indexed_keys(&state, QUEUE_CONSUMER_INDEX_KEY, "queue-consumer:*:*").await?; + let consumer_keys = indexed_keys( + &state, + QUEUE_CONSUMER_INDEX_KEY, + QUEUE_CONSUMER_SCAN_PATTERN, + ) + .await?; let mut indexed_streams = Vec::new(); for consumer_key_chunk in consumer_keys.chunks(QUEUE_RECONCILE_CONSUMER_HASH_BATCH_SIZE) { let consumer_hashes: Vec> = state @@ -148,13 +153,15 @@ pub(crate) async fn queue_reconcile(state: AppState) -> SchedulerResult<()> { // strand a stream outside the consumer loop. refresh_consumer_streams(&state).await; - let streams = indexed_data_keys(&state, QUEUE_STREAM_INDEX_KEY, "queue:*:*:s").await?; + let streams = + indexed_data_keys(&state, QUEUE_STREAM_INDEX_KEY, QUEUE_STREAM_SCAN_PATTERN).await?; { let mut known = state.queues.known_streams.write().await; known.clear(); known.extend(streams); } - let delayed = indexed_data_keys(&state, QUEUE_DELAYED_INDEX_KEY, "queue-delayed:*:*").await?; + let delayed = + indexed_data_keys(&state, QUEUE_DELAYED_INDEX_KEY, QUEUE_DELAYED_SCAN_PATTERN).await?; let delayed_changed = { let delayed = delayed.into_iter().collect::>(); let mut known = state.queues.known_delayed.write().await; diff --git a/rust/scheduler/src/test_fixtures.rs b/rust/scheduler/src/test_fixtures.rs index 47dfde6..37fb0e9 100644 --- a/rust/scheduler/src/test_fixtures.rs +++ b/rust/scheduler/src/test_fixtures.rs @@ -55,3 +55,23 @@ pub(crate) fn scheduler_projection_contract() -> SchedulerProjectionContract { )) .expect("scheduler projection contract fixture must parse") } + +pub(crate) fn version_tag_cases() -> Vec<(String, bool)> { + let fixture: serde_json::Value = + serde_json::from_str(include_str!("../../../tests/fixtures/version-tags.json")) + .expect("version tag fixture must parse"); + fixture["cases"] + .as_array() + .expect("version tag fixture cases must be an array") + .iter() + .map(|entry| { + ( + entry["tag"] + .as_str() + .expect("version tag must be a string") + .to_string(), + entry["parsed"].as_u64().is_some(), + ) + }) + .collect() +} diff --git a/rust/workflows/src/api/do_alarms/dispatch.rs b/rust/workflows/src/api/do_alarms/dispatch.rs index 887defe..fd4a908 100644 --- a/rust/workflows/src/api/do_alarms/dispatch.rs +++ b/rust/workflows/src/api/do_alarms/dispatch.rs @@ -4,7 +4,9 @@ use serde::Serialize; use serde_json::{Value as JsonValue, json}; use wdl_rust_common::internal_auth::INTERNAL_AUTH_HEADER; use wdl_rust_common::time::now_ms; -use wdl_rust_common::version::{do_storage_id_key, routes_key, worker_versions_key}; +use wdl_rust_common::version::{ + do_storage_id_key, parse_version_tag, routes_key, worker_versions_key, +}; use crate::{ AppState, DoAlarmJobKeys, LogLevel, WorkflowError, WorkflowResult, do_alarm_shard_queue_keys, @@ -241,6 +243,8 @@ async fn resolve_alarm_dispatch_version( } if let Some(active_version) = active_version { + parse_version_tag(&active_version) + .map_err(|_| WorkflowError::invalid_state("Active worker version is invalid"))?; log( app, LogLevel::Info, diff --git a/rust/workflows/src/api/do_alarms/model.rs b/rust/workflows/src/api/do_alarms/model.rs index 9acf357..1b611f0 100644 --- a/rust/workflows/src/api/do_alarms/model.rs +++ b/rust/workflows/src/api/do_alarms/model.rs @@ -102,6 +102,9 @@ pub(super) fn validate_set_request(req: &DoAlarmSetRequest) -> WorkflowResult<() validate_non_empty(&req.ns, "ns")?; validate_non_empty(&req.worker, "worker")?; validate_non_empty(&req.version, "version")?; + if parse_version_tag(&req.version).is_err() { + return Err(WorkflowError::invalid_request("version is invalid")); + } validate_non_empty(&req.do_storage_id, "doStorageId")?; validate_non_empty(&req.class_name, "className")?; validate_non_empty(&req.object_name, "objectName")?; @@ -254,6 +257,20 @@ pub(super) fn job_from_state( mod tests { use super::*; + fn valid_set_request() -> DoAlarmSetRequest { + DoAlarmSetRequest { + ns: "demo".to_string(), + worker: "alarms".to_string(), + version: "v1".to_string(), + do_storage_id: "do_abc".to_string(), + class_name: "AlarmCounter".to_string(), + object_name: "alpha".to_string(), + scheduled_time: 123_456_789, + retry_count: 0, + token: "row-token".to_string(), + } + } + fn valid_job_state() -> HashMap { HashMap::from([ ("ns".to_string(), "demo".to_string()), @@ -321,4 +338,25 @@ mod tests { ); } } + + #[test] + fn do_alarm_set_version_matches_cross_language_fixture() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../../tests/fixtures/version-tags.json" + )) + .expect("version tag fixture parses"); + for case in fixture["cases"] + .as_array() + .expect("version tag fixture cases is an array") + { + let tag = case["tag"].as_str().expect("version tag is a string"); + let mut req = valid_set_request(); + req.version = tag.to_string(); + assert_eq!( + validate_set_request(&req).is_ok(), + case["parsed"].as_u64().is_some(), + "{tag:?}" + ); + } + } } diff --git a/shared/optimistic-retry.js b/shared/optimistic-retry.js new file mode 100644 index 0000000..bae79d5 --- /dev/null +++ b/shared/optimistic-retry.js @@ -0,0 +1,34 @@ +/** + * @template T + * @param {(attempt: number) => Promise} operation + * @param {{ + * attempts: number, + * isRetryableError?: (err: unknown) => boolean, + * onRetryableError?: (err: unknown, attempt: number) => void, + * shouldRetryResult?: (result: T, attempt: number) => boolean, + * onExhausted: () => T | Promise, + * }} options + * @returns {Promise} + */ +export async function withOptimisticRetries(operation, { + attempts, + isRetryableError = () => false, + onRetryableError, + shouldRetryResult, + onExhausted, +}) { + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + const result = await operation(attempt); + if (shouldRetryResult?.(result, attempt)) continue; + return result; + } catch (err) { + if (isRetryableError(err)) { + onRetryableError?.(err, attempt); + continue; + } + throw err; + } + } + return await onExhausted(); +} diff --git a/shared/owner-endpoint.js b/shared/owner-endpoint.js new file mode 100644 index 0000000..97071b6 --- /dev/null +++ b/shared/owner-endpoint.js @@ -0,0 +1,120 @@ +const OWNER_ENDPOINT_SERVICE_RE = { + "d1-runtime": /^d1-runtime(?:-[a-z0-9-]+)?$/, + "do-runtime": /^do-runtime(?:-[a-z0-9-]+)?$/, +}; + +const OWNER_ENDPOINT_HEADLESS_RE = { + "d1-runtime": /^d1-runtime-[0-9]+\.d1-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, + "do-runtime": /^do-runtime-[0-9]+\.do-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, +}; +const INVALID_OWNER_ENDPOINT_RE = /[/?#@[\]\\]/; +const IPV4_OCTET_RE = /^(?:0|[1-9]\d{0,2})$/; +const IntrinsicNumber = Number; +const IntrinsicString = String; +const IntrinsicURL = URL; +const intrinsicReflectApply = Reflect.apply; +const intrinsicRegExpTest = RegExp.prototype.test; +const intrinsicStringSplit = String.prototype.split; +const intrinsicUrlHashGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "hash")); +const intrinsicUrlHostGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "host")); +const intrinsicUrlHostnameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "hostname")); +const intrinsicUrlPasswordGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "password")); +const intrinsicUrlPathnameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "pathname")); +const intrinsicUrlPortGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "port")); +const intrinsicUrlSearchGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "search")); +const intrinsicUrlUsernameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "username")); + +/** @param {object | null} prototype @param {string} name */ +function prototypeGetter(prototype, name) { + let current = prototype; + while (current) { + const getter = Object.getOwnPropertyDescriptor(current, name)?.get; + if (getter) return getter; + current = Object.getPrototypeOf(current); + } + return undefined; +} + +/** @param {RegExp} regexp @param {string} value */ +function regexpTest(regexp, value) { + return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); +} + +/** @param {string} value */ +function splitDots(value) { + return intrinsicReflectApply(intrinsicStringSplit, value, ["."]); +} + +/** @param {unknown} value */ +function numberValue(value) { + return intrinsicReflectApply(IntrinsicNumber, undefined, [value]); +} + +/** @param {unknown} value */ +function stringValue(value) { + return intrinsicReflectApply(IntrinsicString, undefined, [value]); +} + +/** @param {(this: URL) => string} getter @param {URL} url */ +function urlValue(getter, url) { + return intrinsicReflectApply(getter, url, []); +} + +/** + * @param {unknown} endpoint + * @param {number} port + * @param {"d1-runtime" | "do-runtime"} serviceName + * @returns {boolean} + */ +export function validOwnerEndpointForService(endpoint, port, serviceName) { + if (typeof endpoint !== "string" || !endpoint) return false; + const serviceRe = OWNER_ENDPOINT_SERVICE_RE[serviceName]; + const headlessRe = OWNER_ENDPOINT_HEADLESS_RE[serviceName]; + if (!serviceRe) throw new Error(`Unknown owner endpoint service ${serviceName}`); + if (regexpTest(INVALID_OWNER_ENDPOINT_RE, endpoint)) return false; + let url; + try { + url = new IntrinsicURL(`http://${endpoint}`); + } catch { + return false; + } + if ( + urlValue(intrinsicUrlHostGet, url) !== endpoint || + urlValue(intrinsicUrlUsernameGet, url) || + urlValue(intrinsicUrlPasswordGet, url) || + urlValue(intrinsicUrlPathnameGet, url) !== "/" || + urlValue(intrinsicUrlSearchGet, url) || + urlValue(intrinsicUrlHashGet, url) + ) { + return false; + } + if (urlValue(intrinsicUrlPortGet, url) !== stringValue(port)) return false; + const hostname = urlValue(intrinsicUrlHostnameGet, url); + return regexpTest(serviceRe, hostname) || + regexpTest(headlessRe, hostname) || + acceptablePrivateIpv4(hostname); +} + +/** + * @param {string} hostname + * @returns {boolean} + */ +function acceptablePrivateIpv4(hostname) { + const parts = splitDots(hostname); + if (parts.length !== 4) return false; + /** @type {number[]} */ + const octets = []; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (!regexpTest(IPV4_OCTET_RE, part)) return false; + const value = numberValue(part); + if (value < 0 || value > 255) return false; + octets[i] = value; + } + const a = octets[0]; + const b = octets[1]; + return a === 10 || + (a === 100 && b >= 64 && b <= 127) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168); +} diff --git a/shared/owner-forwarder.js b/shared/owner-forwarder.js index 60a90c1..63d6aa6 100644 --- a/shared/owner-forwarder.js +++ b/shared/owner-forwarder.js @@ -1,5 +1,6 @@ import { withInternalAuth } from "shared-internal-auth"; import { errorMessage } from "shared-errors"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; export const MAX_OWNER_FORWARD_HOPS = 2; @@ -35,6 +36,8 @@ function withRequestId(headers, requestId) { * @param {{ * env: Record, * endpoint?: string | null, + * endpointPort: number, + * endpointService: "d1-runtime" | "do-runtime", * pathname: string, * method?: string, * requestId?: string | null, @@ -49,6 +52,7 @@ function withRequestId(headers, requestId) { * buildHeaders(nextHopCount: number): HeadersInit, * logFields(): Record, * missingEndpointError(): Error, + * invalidEndpointError(): Error, * hopExhaustedError(): Error, * unavailableError(err: unknown): Error, * isTimeoutError?: (err: unknown) => boolean, @@ -58,6 +62,8 @@ function withRequestId(headers, requestId) { export async function forwardOwnerRequest({ env, endpoint, + endpointPort, + endpointService, pathname, method = "POST", requestId = null, @@ -72,11 +78,15 @@ export async function forwardOwnerRequest({ buildHeaders, logFields, missingEndpointError, + invalidEndpointError, hopExhaustedError, unavailableError, isTimeoutError = undefined, }) { if (!endpoint) throw missingEndpointError(); + if (!validOwnerEndpointForService(endpoint, endpointPort, endpointService)) { + throw invalidEndpointError(); + } const nextHopCount = hopCount + 1; if (nextHopCount > MAX_OWNER_FORWARD_HOPS) throw hopExhaustedError(); try { diff --git a/shared/owner-lease.js b/shared/owner-lease.js index f7dd96e..7407011 100644 --- a/shared/owner-lease.js +++ b/shared/owner-lease.js @@ -1,4 +1,5 @@ import { envValueOr } from "shared-env"; +import { withOptimisticRetries } from "shared-optimistic-retry"; const utf8Decoder = new TextDecoder(); const OWNER_GENERATION_COUNTER = /^(?:0|[1-9]\d*)$/; @@ -44,7 +45,7 @@ export function parseOwnerRecord(raw) { } if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; const record = /** @type {Record} */ (parsed); - if (!Number.isInteger(record.generation) || /** @type {number} */ (record.generation) < 0) { + if (!Number.isSafeInteger(record.generation) || /** @type {number} */ (record.generation) <= 0) { return null; } if ( @@ -127,7 +128,11 @@ export async function currentOwnerGenerationCounter(session, generationKey) { */ export async function nextOwnerGeneration(session, generationKey, currentGeneration = 0) { const currentCounter = await currentOwnerGenerationCounter(session, generationKey); - return Math.max(currentCounter + 1, currentGeneration + 1); + const previousGeneration = Math.max(currentCounter, currentGeneration); + if (!Number.isSafeInteger(previousGeneration) || previousGeneration >= Number.MAX_SAFE_INTEGER) { + throw new Error(`Owner generation counter is exhausted: ${generationKey}`); + } + return previousGeneration + 1; } /** @@ -144,41 +149,6 @@ export function boundedPositiveIntEnv(env, name, fallback, max) { return Math.max(1, Math.min(Math.trunc(raw), upper)); } -/** - * @template T - * @param {(attempt: number) => Promise} operation - * @param {{ - * attempts: number, - * isRetryableError?: (err: unknown) => boolean, - * onRetryableError?: (err: unknown, attempt: number) => void, - * shouldRetryResult?: (result: T, attempt: number) => boolean, - * onExhausted: () => T | Promise, - * }} options - * @returns {Promise} - */ -export async function withOptimisticRetries(operation, { - attempts, - isRetryableError = () => false, - onRetryableError, - shouldRetryResult, - onExhausted, -}) { - for (let attempt = 0; attempt < attempts; attempt += 1) { - try { - const result = await operation(attempt); - if (shouldRetryResult?.(result, attempt)) continue; - return result; - } catch (err) { - if (isRetryableError(err)) { - onRetryableError?.(err, attempt); - continue; - } - throw err; - } - } - return await onExhausted(); -} - /** * @template T * @param {() => Promise} operation diff --git a/shared/task-identity.js b/shared/task-identity.js index 3afce15..272960a 100644 --- a/shared/task-identity.js +++ b/shared/task-identity.js @@ -15,6 +15,7 @@ const DEFAULT_IDENTITY_TIMEOUT_MS = 1000; * serviceLabel: string, * unavailableCode: string, * createError: ErrorFactory, + * validateEndpoint?: (endpoint: string, port: number) => boolean, * }} TaskIdentityResolverOptions * @typedef {Record} EnvLike */ @@ -55,6 +56,7 @@ export function createTaskIdentityResolver({ serviceLabel, unavailableCode, createError, + validateEndpoint = undefined, }) { /** @type {TaskIdentity | null} */ let cachedIdentity = null; @@ -69,13 +71,12 @@ export function createTaskIdentityResolver({ return createError(503, unavailableCode, message); } - /** - * @param {EnvLike} env - * @returns {number} - */ - function taskPort(env) { - const raw = Number(envValueOr(env[`${envPrefix}_TASK_PORT`], defaultPort)); - return Number.isFinite(raw) && raw > 0 ? raw : defaultPort; + /** @param {TaskIdentity} identity */ + function requireValidEndpoint(identity) { + if (validateEndpoint?.(identity.endpoint, defaultPort) === false) { + throw error(`${serviceLabel} task endpoint is invalid`); + } + return identity; } /** @@ -140,7 +141,7 @@ export function createTaskIdentityResolver({ if (!taskId || !endpoint) { throw error(`${envPrefix}_TASK_ID and ${envPrefix}_TASK_ENDPOINT must be configured together`); } - return { taskId, endpoint, source: "env" }; + return requireValidEndpoint({ taskId, endpoint, source: "env" }); } /** @@ -154,11 +155,11 @@ export function createTaskIdentityResolver({ if (!taskId) throw error("ECS task metadata did not include TaskARN"); const privateIpv4 = firstPrivateIpv4(metadata, env); if (!privateIpv4) throw error("ECS task metadata did not include a private IPv4 address"); - return { + return requireValidEndpoint({ taskId, - endpoint: `${privateIpv4}:${taskPort(env)}`, + endpoint: `${privateIpv4}:${defaultPort}`, source: "ecs-metadata", - }; + }); } /** diff --git a/shared/version.js b/shared/version.js index e7ed7c2..725d536 100644 --- a/shared/version.js +++ b/shared/version.js @@ -10,7 +10,7 @@ const VERSION_RE = /^v([1-9][0-9]*)$/; * @returns {string} */ export function formatVersion(n) { - if (typeof n !== "number" || !Number.isInteger(n) || n < 1) { + if (typeof n !== "number" || !Number.isSafeInteger(n) || n < 1) { throw new Error(`invalid version number ${n}`); } return `v${n}`; @@ -24,7 +24,9 @@ export function formatVersion(n) { export function parseVersion(tag) { if (typeof tag !== "string") return null; const m = tag.match(VERSION_RE); - return m ? Number.parseInt(m[1], 10) : null; + if (!m) return null; + const parsed = Number.parseInt(m[1], 10); + return Number.isSafeInteger(parsed) ? parsed : null; } // `v:` infix separates the integer-indexed bundle namespace from sibling diff --git a/terraform/README.md b/terraform/README.md index 9447031..477ee6d 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -147,6 +147,16 @@ assets_cdn_domain = "" assets_cdn_acm_certificate_arn = "" ``` +The application stack accepts at least two pre-existing subnet ids. It validates +that each subnet belongs to `vpc_id`, uses an RFC1918 or `100.64.0.0/10` IPv4 CIDR, +and occupies a different Availability Zone so each D1/DO EFS file system creates +at most one mount target per AZ. The subnet `map_public_ip_on_launch` flag is not a +private-routing proof; ECS services explicitly disable public task IP assignment. +Foundation keeps its three-AZ default and applies the same address contract to the +VPC and every subnet before provisioning. Unsupported ranges or duplicate-AZ +subnets fail during planning rather than after tasks start claiming owner records +or creating EFS mount targets. + Initialize the application root with its own `terraform/backend.hcl` and apply: ```sh @@ -411,9 +421,8 @@ Manager secrets. ### Test Hooks -`d1_test_hooks_enabled` and `do_test_hooks_enabled` default to `false` and are -guarded to test-named compute stacks. Keep both disabled for normal live-ready -service state. +`d1_test_hooks_enabled` defaults to `false` and is guarded to test-named compute +stacks. Keep it disabled for normal live-ready service state. ## Validation diff --git a/terraform/foundation/main.tf b/terraform/foundation/main.tf index 359ed22..ab3e5c8 100644 --- a/terraform/foundation/main.tf +++ b/terraform/foundation/main.tf @@ -3,9 +3,33 @@ data "aws_availability_zones" "available" { } locals { - azs = slice(data.aws_availability_zones.available.names, 0, 3) - public_subnets = { for idx, cidr in var.public_subnet_cidrs : idx => cidr } - private_subnets = { for idx, cidr in var.private_subnet_cidrs : idx => cidr } + azs = slice(data.aws_availability_zones.available.names, 0, 3) + public_subnets = { for idx, cidr in var.public_subnet_cidrs : idx => cidr } + private_subnets = { for idx, cidr in var.private_subnet_cidrs : idx => cidr } + network_cidrs = concat([var.vpc_cidr], var.public_subnet_cidrs, var.private_subnet_cidrs) + private_cidr_contract = { + for cidr in local.network_cidrs : cidr => try( + ( + tonumber(split(".", cidrhost(cidr, 0))[0]) == 10 && + tonumber(split("/", cidr)[1]) >= 8 + ) || ( + tonumber(split(".", cidrhost(cidr, 0))[0]) == 100 && + tonumber(split(".", cidrhost(cidr, 0))[1]) >= 64 && + tonumber(split(".", cidrhost(cidr, 0))[1]) <= 127 && + tonumber(split("/", cidr)[1]) >= 10 + ) || ( + tonumber(split(".", cidrhost(cidr, 0))[0]) == 172 && + tonumber(split(".", cidrhost(cidr, 0))[1]) >= 16 && + tonumber(split(".", cidrhost(cidr, 0))[1]) <= 31 && + tonumber(split("/", cidr)[1]) >= 12 + ) || ( + tonumber(split(".", cidrhost(cidr, 0))[0]) == 192 && + tonumber(split(".", cidrhost(cidr, 0))[1]) == 168 && + tonumber(split("/", cidr)[1]) >= 16 + ), + false, + ) + } assets_cdn_enabled = var.assets_cdn_domain != "" site_host_enabled = var.site_host != "" site_www_host = local.site_host_enabled ? "www.${var.site_host}" : "" @@ -26,6 +50,13 @@ resource "aws_vpc" "this" { enable_dns_hostnames = true enable_dns_support = true + lifecycle { + precondition { + condition = alltrue(values(local.private_cidr_contract)) + error_message = "vpc_cidr and subnet CIDRs must be IPv4 ranges within RFC1918 or 100.64.0.0/10." + } + } + tags = { Name = "${var.name_prefix}-vpc" } diff --git a/terraform/main.tf b/terraform/main.tf index 02bee91..f1e4f26 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -2,6 +2,43 @@ locals { name = var.name_prefix } +data "aws_subnet" "private_contract" { + for_each = toset(var.private_subnet_ids) + + id = each.value + + lifecycle { + postcondition { + condition = self.vpc_id == var.vpc_id + error_message = "Private subnet ${self.id} must belong to vpc_id ${var.vpc_id}." + } + postcondition { + condition = try( + ( + tonumber(split(".", cidrhost(self.cidr_block, 0))[0]) == 10 && + tonumber(split("/", self.cidr_block)[1]) >= 8 + ) || ( + tonumber(split(".", cidrhost(self.cidr_block, 0))[0]) == 100 && + tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) >= 64 && + tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) <= 127 && + tonumber(split("/", self.cidr_block)[1]) >= 10 + ) || ( + tonumber(split(".", cidrhost(self.cidr_block, 0))[0]) == 172 && + tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) >= 16 && + tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) <= 31 && + tonumber(split("/", self.cidr_block)[1]) >= 12 + ) || ( + tonumber(split(".", cidrhost(self.cidr_block, 0))[0]) == 192 && + tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) == 168 && + tonumber(split("/", self.cidr_block)[1]) >= 16 + ), + false, + ) + error_message = "Private subnet ${self.id} must use an IPv4 CIDR within RFC1918 or 100.64.0.0/10." + } + } +} + module "data" { source = "./modules/data" @@ -15,10 +52,13 @@ module "data" { module "compute" { source = "./modules/compute" - name = local.name - region = var.region - vpc_id = var.vpc_id - private_subnet_ids = var.private_subnet_ids + name = local.name + region = var.region + vpc_id = var.vpc_id + private_subnet_ids = var.private_subnet_ids + private_subnet_availability_zone_ids = toset([ + for subnet in data.aws_subnet.private_contract : subnet.availability_zone_id + ]) alb_https_listener_arn = var.alb_https_listener_arn alb_security_group_id = var.alb_security_group_id platform_domain = var.platform_domain @@ -45,7 +85,6 @@ module "compute" { d1_runtime_desired_count = var.d1_runtime_desired_count do_runtime_desired_count = var.do_runtime_desired_count d1_test_hooks_enabled = var.d1_test_hooks_enabled - do_test_hooks_enabled = var.do_test_hooks_enabled scheduler_desired_count = var.scheduler_desired_count workflows_desired_count = var.workflows_desired_count gateway_cpu = var.gateway_cpu diff --git a/terraform/modules/compute/do_runtime_service.tf b/terraform/modules/compute/do_runtime_service.tf index 47664c2..ef5fee2 100644 --- a/terraform/modules/compute/do_runtime_service.tf +++ b/terraform/modules/compute/do_runtime_service.tf @@ -79,9 +79,7 @@ resource "aws_ecs_task_definition" "do_runtime" { { name = "DO_TASK_CONTAINER_NAME", value = "do-runtime" }, { name = "DO_OWNER_TTL_SECONDS", value = "120" }, { name = "D1_QUERY_TIMEOUT_MS", value = "30000" }, - ], var.do_test_hooks_enabled ? [ - { name = "DO_TEST_HOOKS", value = "1" }, - ] : [], local.r2_s3_env) + ], local.r2_s3_env) secrets = concat([ { name = "R2_S3_ACCESS_KEY_ID", valueFrom = "${var.runtime_r2_secret_arn}:access_key_id::" }, @@ -116,10 +114,6 @@ resource "aws_ecs_task_definition" "do_runtime" { error_message = "do_runtime_container_memory must leave task-level memory reservation for the redis-proxy sidecar and additional task headroom." } - precondition { - condition = !var.do_test_hooks_enabled || can(regex("(^|-)test($|-)", var.name)) - error_message = "do_test_hooks_enabled may only be enabled for test-named compute stacks." - } } } diff --git a/terraform/modules/compute/variables.tf b/terraform/modules/compute/variables.tf index 86e5046..3feae48 100644 --- a/terraform/modules/compute/variables.tf +++ b/terraform/modules/compute/variables.tf @@ -3,6 +3,13 @@ variable "region" { type = string } variable "vpc_id" { type = string } variable "private_subnet_ids" { type = list(string) } +variable "private_subnet_availability_zone_ids" { + type = set(string) + validation { + condition = length(var.private_subnet_availability_zone_ids) == length(var.private_subnet_ids) + error_message = "Each private subnet must belong to a different Availability Zone so EFS creates at most one mount target per AZ." + } +} variable "alb_https_listener_arn" { type = string } variable "alb_security_group_id" { type = string } @@ -44,7 +51,6 @@ variable "runtime_desired_count" { type = number } variable "d1_runtime_desired_count" { type = number } variable "do_runtime_desired_count" { type = number } variable "d1_test_hooks_enabled" { type = bool } -variable "do_test_hooks_enabled" { type = bool } variable "scheduler_desired_count" { type = number } variable "workflows_desired_count" { type = number } variable "gateway_cpu" { type = number } diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index 38fb77f..53be769 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -21,7 +21,6 @@ alb_https_listener_arn = "arn:aws:elasticloadbalancing:ap-east-1:123456789012:li alb_security_group_id = "sg-xxxxxxxxxxxxxxxxx" d1_test_hooks_enabled = false -do_test_hooks_enabled = false # Stateless gateway/user-runtime/system-runtime keep at least one on-demand # Fargate task and place overflow according to these weights. Stateful runtimes, diff --git a/terraform/variables.tf b/terraform/variables.tf index 6213a71..2a976f0 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -120,10 +120,13 @@ variable "vpc_id" { variable "private_subnet_ids" { type = list(string) - description = "Private subnet ids across at least three AZs for ECS tasks, EFS mount targets, and ElastiCache." + description = "Private subnet ids across at least two AZs for ECS tasks, EFS mount targets, and ElastiCache." validation { - condition = length(var.private_subnet_ids) >= 3 - error_message = "Provide at least three private subnet ids across different AZs." + condition = ( + length(var.private_subnet_ids) >= 2 && + length(var.private_subnet_ids) == length(distinct(var.private_subnet_ids)) + ) + error_message = "Provide at least two unique private subnet ids." } } @@ -165,12 +168,6 @@ variable "d1_test_hooks_enabled" { description = "Enable D1 internal test hooks in d1-runtime. Keep false outside disposable/test environments." } -variable "do_test_hooks_enabled" { - type = bool - default = false - description = "Enable DO internal test hooks in do-runtime. Keep false outside disposable/test environments." -} - variable "gateway_cpu" { type = number default = 512 diff --git a/tests/fixtures/version-tags.json b/tests/fixtures/version-tags.json new file mode 100644 index 0000000..e673116 --- /dev/null +++ b/tests/fixtures/version-tags.json @@ -0,0 +1,16 @@ +{ + "cases": [ + { "tag": "v1", "parsed": 1 }, + { "tag": "v42", "parsed": 42 }, + { "tag": "v9007199254740991", "parsed": 9007199254740991 }, + { "tag": "", "parsed": null }, + { "tag": "v", "parsed": null }, + { "tag": "v0", "parsed": null }, + { "tag": "v01", "parsed": null }, + { "tag": "V1", "parsed": null }, + { "tag": "v1a", "parsed": null }, + { "tag": "v9007199254740992", "parsed": null }, + { "tag": "v18446744073709551615", "parsed": null }, + { "tag": "v999999999999999999999999999999", "parsed": null } + ] +} diff --git a/tests/helpers/control-shared-stub.js b/tests/helpers/control-shared-stub.js index d310fec..e54a047 100644 --- a/tests/helpers/control-shared-stub.js +++ b/tests/helpers/control-shared-stub.js @@ -11,10 +11,10 @@ import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); -const SHARED_ENV_URL = repositoryFileUrl("shared/env.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_OPTIMISTIC_RETRY_URL = repositoryFileUrl("shared/optimistic-retry.js"); const CONTROL_ERRORS_URL = freshRepositoryModuleDataUrl("control/errors.js", [ [/from "shared-respond"/g, `from ${JSON.stringify(SHARED_RESPOND_URL)}`], ]); @@ -22,13 +22,10 @@ const CONTROL_WORKFLOWS_CLIENT_URL = freshRepositoryModuleDataUrl("control/workf [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], [/from "control-errors"/g, `from ${JSON.stringify(CONTROL_ERRORS_URL)}`], ]); -const SHARED_OWNER_LEASE_URL = freshRepositoryModuleDataUrl("shared/owner-lease.js", [ - [/from "shared-env"/g, `from ${JSON.stringify(SHARED_ENV_URL)}`], -]); const CONTROL_OPTIMISTIC_REDIS_URL = sharedRedisStubUrl(); const CONTROL_OPTIMISTIC_URL = freshRepositoryModuleDataUrl("control/optimistic.js", [ [/from "shared-redis"/g, `from ${JSON.stringify(CONTROL_OPTIMISTIC_REDIS_URL)}`], - [/from "shared-owner-lease"/g, `from ${JSON.stringify(SHARED_OWNER_LEASE_URL)}`], + [/from "shared-optimistic-retry"/g, `from ${JSON.stringify(SHARED_OPTIMISTIC_RETRY_URL)}`], ]); const CONTROL_JSON_BODY_URL = freshRepositoryModuleDataUrl("control/json-body.js", [ [/from "shared-bounded-body"/g, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)}`], diff --git a/tests/helpers/load-d1-owner-client.js b/tests/helpers/load-d1-owner-client.js index d622f5f..82070bb 100644 --- a/tests/helpers/load-d1-owner-client.js +++ b/tests/helpers/load-d1-owner-client.js @@ -42,6 +42,8 @@ const d1OwnerClientModule = await importOwnerClientModule("d1-runtime/owner-clie "d1-runtime-owner-registry": ownerRegistryUrl, "shared-internal-auth": ownerHarness.internalAuthUrl, "shared-owner-forwarder": ownerHarness.ownerForwarderUrl, + "shared-owner-endpoint": ownerHarness.ownerEndpointUrl, + "shared-owner-lease": ownerHarness.ownerLeaseUrl, "d1-runtime-state": ownerHarness.stateUrl, }); diff --git a/tests/helpers/load-d1-owner-registry.js b/tests/helpers/load-d1-owner-registry.js index 610de4c..928d687 100644 --- a/tests/helpers/load-d1-owner-registry.js +++ b/tests/helpers/load-d1-owner-registry.js @@ -48,7 +48,7 @@ export function resetD1OwnerRegistryTestState() { D1_OWNER_REGISTRY_TEST_STATE.forgottenStorageSizes = []; D1_OWNER_REGISTRY_TEST_STATE.pendingQueries = 0; D1_OWNER_REGISTRY_TEST_STATE.draining = false; - D1_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-a", endpoint: "task-a:8787" }; + D1_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-a", endpoint: "d1-runtime-a:8787" }; D1_OWNER_REGISTRY_TEST_STATE.onWatchExecFailure = null; D1_OWNER_REGISTRY_TEST_STATE.logEntries = []; D1_OWNER_REGISTRY_TEST_STATE.metricIncrements = []; @@ -173,6 +173,7 @@ const src = applyModuleReplacements(readRepositoryFile("d1-runtime/owner-registr [/from "shared-redis";/, `from ${JSON.stringify(redisUrl)};`], [/from "shared-env";/, `from ${JSON.stringify(SHARED_ENV_URL)};`], [/from "shared-errors";/, `from ${JSON.stringify(repositoryFileUrl("shared/errors.js"))};`], + [/from "shared-owner-endpoint";/, `from ${JSON.stringify(repositoryFileUrl("shared/owner-endpoint.js"))};`], [/from "shared-owner-lease";/, `from ${JSON.stringify(SHARED_OWNER_LEASE_URL)};`], [/from "shared-owner-protocol";/, `from ${JSON.stringify(SHARED_OWNER_PROTOCOL_URL)};`], [/from "shared-redis-client";/, `from ${JSON.stringify(redisClientUrl)};`], diff --git a/tests/helpers/load-d1-protocol.js b/tests/helpers/load-d1-protocol.js index 11b7c34..c149ac2 100644 --- a/tests/helpers/load-d1-protocol.js +++ b/tests/helpers/load-d1-protocol.js @@ -5,6 +5,7 @@ const DATA_FIELD_URL = repositoryFileUrl("shared/d1-data-field.js"); const FNV_URL = repositoryFileUrl("shared/fnv1a32.js"); const BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const NS_PATTERN_URL = repositoryFileUrl("shared/ns-pattern.js"); export function d1ProtocolDataUrl() { const queryWireUrl = d1QueryWireDataUrl(); @@ -15,6 +16,7 @@ export function d1ProtocolDataUrl() { `import { fnv1a32CodeUnits } from ${JSON.stringify(FNV_URL)};`], [/from "shared-bounded-body";/g, `from ${JSON.stringify(BOUNDED_BODY_URL)};`], [/from "shared-errors";/g, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], + [/from "shared-ns-pattern";/g, `from ${JSON.stringify(NS_PATTERN_URL)};`], [/from "shared-d1-query-wire";/g, `from ${JSON.stringify(queryWireUrl)};`], ]); } diff --git a/tests/helpers/load-do-owner-client.js b/tests/helpers/load-do-owner-client.js index 5ae6266..b4f3885 100644 --- a/tests/helpers/load-do-owner-client.js +++ b/tests/helpers/load-do-owner-client.js @@ -14,6 +14,7 @@ const doOwnerClientModule = await importOwnerClientModule("do-runtime/owner-clie "do-runtime-protocol": protocolUrl, "shared-internal-auth": ownerHarness.internalAuthUrl, "shared-owner-forwarder": ownerHarness.ownerForwarderUrl, + "shared-owner-lease": ownerHarness.ownerLeaseUrl, "do-runtime-state": ownerHarness.stateUrl, }); diff --git a/tests/helpers/load-do-owner-registry.js b/tests/helpers/load-do-owner-registry.js index 1683246..3e38344 100644 --- a/tests/helpers/load-do-owner-registry.js +++ b/tests/helpers/load-do-owner-registry.js @@ -43,7 +43,7 @@ export const DO_OWNER_REGISTRY_TEST_STATE = { redisState, store: redisState.strings, ownedScopes: new Map(), - taskIdentity: { taskId: "task-a", endpoint: "task-a:8788" }, + taskIdentity: { taskId: "task-a", endpoint: "do-runtime-a:8788" }, watchedKeys: redisState.watched, metricIncrements: [], logEntries: [], @@ -61,7 +61,7 @@ export function resetDoOwnerRegistryTestState() { const testState = DO_OWNER_REGISTRY_TEST_STATE; resetFakeRedisState(testState.redisState); testState.ownedScopes.clear(); - testState.taskIdentity = { taskId: "task-a", endpoint: "task-a:8788" }; + testState.taskIdentity = { taskId: "task-a", endpoint: "do-runtime-a:8788" }; testState.metricIncrements = []; testState.logEntries = []; testState.redisTimeMs = Date.now(); @@ -129,6 +129,8 @@ const src = applyModuleReplacements(readRepositoryFile("do-runtime/owner-registr [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], [/from "do-runtime-state";/, `from ${JSON.stringify(stateUrl)};`], [/from "shared-errors";/, `from ${JSON.stringify(repositoryFileUrl("shared/errors.js"))};`], + [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], + [/from "shared-owner-endpoint";/, `from ${JSON.stringify(repositoryFileUrl("shared/owner-endpoint.js"))};`], ]); const registry = await import(moduleDataUrl(src)); @@ -140,6 +142,7 @@ export const { ownerGenerationKeyOf, ownerLeaseGuardMs, ownerKeyOf, + parseOwner, renewOwnedScopes, releaseOwner, resolveDoOwner, diff --git a/tests/helpers/load-do-protocol.js b/tests/helpers/load-do-protocol.js index 1ef16ee..dccfe68 100644 --- a/tests/helpers/load-do-protocol.js +++ b/tests/helpers/load-do-protocol.js @@ -8,8 +8,9 @@ const SHARED_FNV_URL = repositoryFileUrl("shared/fnv1a32.js"); const SHARED_WORKER_ID_URL = repositoryFileUrl("shared/worker-id.js"); const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); +const SHARED_NS_PATTERN_URL = repositoryFileUrl("shared/ns-pattern.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); -const SHARED_WORKERD_COMPAT_FLAGS_URL = repositoryFileUrl("shared/workerd-compat-flags.js"); +const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const DO_WIRE_GRAMMAR_URL = repositoryFileUrl("do-runtime/protocol/wire-grammar.js"); const DO_ERRORS_URL = repositoryModuleDataUrl("do-runtime/protocol/errors.js", [ [/from "shared-respond";/g, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], @@ -27,9 +28,10 @@ export function doProtocolDataUrl() { "do-runtime-protocol-errors": DO_ERRORS_URL, "do-runtime-protocol-identity": DO_IDENTITY_URL, "shared-worker-id": SHARED_WORKER_ID_URL, - "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, "shared-bounded-body": SHARED_BOUNDED_BODY_URL, "shared-internal-auth": SHARED_INTERNAL_AUTH_URL, + "shared-ns-pattern": SHARED_NS_PATTERN_URL, + "shared-version": SHARED_VERSION_URL, }), ]); } diff --git a/tests/helpers/load-owner-client-harness.js b/tests/helpers/load-owner-client-harness.js index 1e6c64b..a38a5ba 100644 --- a/tests/helpers/load-owner-client-harness.js +++ b/tests/helpers/load-owner-client-harness.js @@ -4,6 +4,7 @@ import { moduleDataUrl, repositoryFileUrl, repositoryModuleDataUrl, + sharedModuleDataUrl, } from "./load-shared-module.js"; import { sharedInternalAuthUrl } from "./runtime-proxy-stub.js"; @@ -41,9 +42,12 @@ export function log(level, event, fields) { `); const internalAuthUrl = sharedInternalAuthUrl(); const errorsUrl = repositoryFileUrl("shared/errors.js"); + const ownerEndpointUrl = repositoryFileUrl("shared/owner-endpoint.js"); + const ownerLeaseUrl = sharedModuleDataUrl("shared/owner-lease.js"); const ownerForwarderUrl = repositoryModuleDataUrl("shared/owner-forwarder.js", [ [/from "shared-internal-auth";/, `from ${JSON.stringify(internalAuthUrl)};`], [/from "shared-errors";/, `from ${JSON.stringify(errorsUrl)};`], + [/from "shared-owner-endpoint";/, `from ${JSON.stringify(ownerEndpointUrl)};`], ]); return { @@ -52,6 +56,8 @@ export function log(level, event, fields) { errorsUrl, internalAuthUrl, ownerForwarderUrl, + ownerEndpointUrl, + ownerLeaseUrl, reset() { state.fetches = []; state.logs = []; diff --git a/tests/helpers/load-shared-module.js b/tests/helpers/load-shared-module.js index f72082f..8e44de8 100644 --- a/tests/helpers/load-shared-module.js +++ b/tests/helpers/load-shared-module.js @@ -11,7 +11,11 @@ const REPO_ROOT = path.resolve(__dirname, "../.."); const SHARED_ENV_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/env.js")).href; const SHARED_ERRORS_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/errors.js")).href; const SHARED_BASE64_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/base64.js")).href; +const SHARED_VERSION_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/version.js")).href; const SHARED_HEX_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/hex.js")).href; +const SHARED_OPTIMISTIC_RETRY_URL = pathToFileURL( + path.resolve(REPO_ROOT, "shared/optimistic-retry.js") +).href; const SHARED_WORKERD_COMPAT_FLAGS_URL = pathToFileURL( path.resolve(REPO_ROOT, "shared/workerd-compat-flags.js") ).href; @@ -89,6 +93,7 @@ export function repositoryModuleDataUrl(relativePath, replacements = []) { export function runtimeLibModuleDataUrl() { return repositoryModuleDataUrl("runtime/lib.js", importSpecifierReplacements({ "shared-base64": SHARED_BASE64_URL, + "shared-version": SHARED_VERSION_URL, "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, })); } @@ -125,6 +130,7 @@ export async function importRepositoryModuleFresh(relativePath, replacements = [ export function sharedModuleDataUrl(relativePath) { const src = applyModuleReplacements(readRepositoryFile(relativePath), [ [/from "shared-env";?/g, `from ${JSON.stringify(SHARED_ENV_URL)};`], + [/from "shared-optimistic-retry";?/g, `from ${JSON.stringify(SHARED_OPTIMISTIC_RETRY_URL)};`], [/from "\.\/errors\.js";?/g, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "\.\/hex\.js";?/g, `from ${JSON.stringify(SHARED_HEX_URL)};`], ]); diff --git a/tests/helpers/runtime-injection-sources.js b/tests/helpers/runtime-injection-sources.js index 633c8ad..d6f2519 100644 --- a/tests/helpers/runtime-injection-sources.js +++ b/tests/helpers/runtime-injection-sources.js @@ -15,7 +15,7 @@ const REAL_RUNTIME_INJECTION_SOURCE_PATHS = Object.freeze({ r2UtilsSource: "runtime/r2-utils.js", doClientSource: "runtime/do-client.js", doTransportSource: "runtime/_wdl-do-transport.js", - ownerEndpointSource: "runtime/_wdl-owner-endpoint.js", + ownerEndpointSource: "shared/owner-endpoint.js", ownerHintCacheSource: "runtime/_wdl-owner-hint-cache.js", requestIdSource: "runtime/_wdl-request-id.js", workflowsClientSource: "runtime/workflows-client.js", diff --git a/tests/helpers/style-contract-scanner.js b/tests/helpers/style-contract-scanner.js index e2c7400..1fb0d0b 100644 --- a/tests/helpers/style-contract-scanner.js +++ b/tests/helpers/style-contract-scanner.js @@ -224,13 +224,47 @@ function findClosingDelimiter(source, openIndex, open, close) { return -1; } +/** @param {string} source */ +export function stripCapnpComments(source) { + let out = ""; + let quoted = false; + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + if (quoted) { + out += char; + if (char === "\\") { + index += 1; + out += source[index] ?? ""; + } else if (char === '"') { + quoted = false; + } + continue; + } + if (char === '"') { + quoted = true; + out += char; + continue; + } + if (char !== "#") { + out += char; + continue; + } + while (index < source.length && source[index] !== "\n") { + out += " "; + index += 1; + } + if (index < source.length) out += "\n"; + } + return out; +} + /** * Extracts a Cap'n Proto const struct while ignoring comments and formatting. * @param {string} source * @param {string} name */ export function extractCapnpConstBlock(source, name) { - const clean = source.replace(/#.*$/gm, ""); + const clean = stripCapnpComments(source); const declaration = new RegExp(`const\\s+${RegExp.escape(name)}\\s+:[^=]+=`).exec(clean); assert.ok(declaration, `Cap'n Proto const ${name} must be present`); const start = clean.indexOf("(", declaration.index + declaration[0].length); diff --git a/tests/integration/durable-objects-alarms.test.js b/tests/integration/durable-objects-alarms.test.js index 76e3313..7ef561f 100644 --- a/tests/integration/durable-objects-alarms.test.js +++ b/tests/integration/durable-objects-alarms.test.js @@ -62,6 +62,54 @@ const DO_BLOCKING_ALARM_WORKER = readFileSync( "utf8" ); +const DO_ACCESSOR_ALARM_WORKER = ` +const instances = new WeakMap(); + +export class AccessorAlarm { + #ctx; + + constructor(ctx) { + this.#ctx = ctx; + instances.set(this, true); + } + + #assertReceiver() { + if (!instances.has(this)) throw new TypeError("invalid AccessorAlarm receiver"); + } + + get fetch() { + this.#assertReceiver(); + return async (request) => { + const url = new URL(request.url); + if (url.pathname === "/schedule") { + await this.#ctx.storage.setAlarm(Date.now() - 1000); + return Response.json({ pending: true }); + } + return Response.json({ + alarms: (await this.#ctx.storage.get("alarms")) ?? 0, + pending: await this.#ctx.storage.getAlarm(), + }); + }; + } + + get alarm() { + this.#assertReceiver(); + return async () => { + const alarms = (await this.#ctx.storage.get("alarms")) ?? 0; + await this.#ctx.storage.put("alarms", alarms + 1); + }; + } +} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + const id = env.ACCESSORS.idFromName(url.searchParams.get("name") || "main"); + return await env.ACCESSORS.get(id).fetch("https://do.internal" + url.pathname); + }, +}; +`; + /** * @param {string} ns * @param {string} worker @@ -252,6 +300,34 @@ test("do-runtime shim supports storage alarms on SQLite-backed Durable Objects", assert.equal(spoofJson.pending, null); }); +test("do-runtime alarm wrapper preserves accessor handler receivers through host proxies", async () => { + const ns = uniqueNs("do-alarm-accessor"); + await deployAndPromote(ns, "alarms", { + mainModule: "worker.js", + modules: { "worker.js": DO_ACCESSOR_ALARM_WORKER }, + bindings: { + ACCESSORS: { type: "do", className: "AccessorAlarm" }, + }, + }); + + const scheduled = await gatewayFetch(ns, "/alarms/schedule?name=alice"); + const scheduledText = await scheduled.text(); + assert.equal(scheduled.status, 200, scheduledText); + assert.deepEqual(responseJson({ body: scheduledText }), { pending: true }); + + const status = await waitForJson( + "accessor-backed DO alarm", + async () => { + const response = await gatewayFetch(ns, "/alarms/status?name=alice"); + const body = await response.text(); + assert.equal(response.status, 200, body); + return responseJson({ body }); + }, + (json) => json.alarms === 1 && json.pending === null + ); + assert.deepEqual(status, { alarms: 1, pending: null }); +}); + test("scheduler replicas: Workflows DO alarm tick delivers a due alarm once", async () => { const ns = uniqueNs("do-alarm-replica"); await deployAndPromote(ns, "alarms", { diff --git a/tests/integration/durable-objects-ownership.test.js b/tests/integration/durable-objects-ownership.test.js index 88f1bfd..cbff8f6 100644 --- a/tests/integration/durable-objects-ownership.test.js +++ b/tests/integration/durable-objects-ownership.test.js @@ -384,8 +384,8 @@ test("do-runtime replicas forward a sharded object owner scope instead of splitt worker: "counter", doStorageId, className: "Counter", - taskId: "stale-task", - endpoint: "stale-task:8788", + taskId: "missing-owner", + endpoint: "do-runtime-missing:8788", generation: 42, leaseExpiresAt: Date.now() - 1000, }); diff --git a/tests/integration/helpers/durable-objects.js b/tests/integration/helpers/durable-objects.js index c6fada1..6a6a0cb 100644 --- a/tests/integration/helpers/durable-objects.js +++ b/tests/integration/helpers/durable-objects.js @@ -1,6 +1,7 @@ import { readFileSync } from "node:fs"; import assert from "node:assert/strict"; import { fnv1a32CodeUnits } from "../../../shared/fnv1a32.js"; +import { doStorageIdKey } from "../../../shared/version.js"; import { doAlarmJobIdForStorage } from "../../helpers/do-alarm-job-id.js"; import { serviceInternalPost, serviceInternalPostAsync } from "./internal-http.js"; import { @@ -147,7 +148,7 @@ export function redisAddDoAlarmByWorker(ns, worker, jobId) { /** @param {string} ns @param {string} worker */ export function redisGetDoStorageId(ns, worker) { - return redisGetRaw(`worker:do-storage:${ns}:${worker}`) ?? ""; + return redisGetRaw(doStorageIdKey(ns, worker)) ?? ""; } /** diff --git a/tests/integration/helpers/misc.js b/tests/integration/helpers/misc.js index 98bcf7a..3707819 100644 --- a/tests/integration/helpers/misc.js +++ b/tests/integration/helpers/misc.js @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { bundleKey } from "../../../shared/version.js"; import { redisHGetJson } from "./redis.js"; @@ -13,6 +14,6 @@ export function cronId(cron, timezone) { * @param {string} version */ export function readMeta(ns, name, version) { - const key = `worker:${ns}:${name}:v:${version.slice(1)}`; + const key = bundleKey(ns, name, version); return redisHGetJson(key, "__meta__", { label: `${key} __meta__` }); } diff --git a/tests/unit/control-d1-runtime-client.test.js b/tests/unit/control-d1-runtime-client.test.js index 0df3d67..976878c 100644 --- a/tests/unit/control-d1-runtime-client.test.js +++ b/tests/unit/control-d1-runtime-client.test.js @@ -29,7 +29,7 @@ const { "shared-d1-query-wire": d1QueryWireDataUrl(), "shared-respond": repositoryFileUrl("shared/respond.js"), "shared-internal-auth": sharedInternalAuthUrl(), - "runtime-owner-endpoint": repositoryFileUrl("runtime/_wdl-owner-endpoint.js"), + "shared-owner-endpoint": repositoryFileUrl("shared/owner-endpoint.js"), })); /** @template {Record} T @param {T} env */ @@ -87,30 +87,28 @@ test("control D1 runtime client query sends request timeout signal", async () => }); }); -test("control D1 runtime client preserves zero owner generation hints", async () => { - const result = await d1RuntimeQuery(withInternalAuthEnv({ - D1_BACKEND: { - async fetch() { - return d1Response( - { success: true, results: [] }, - { - headers: { - "x-wdl-d1-owner-task-id": "task-a", - "x-wdl-d1-owner-endpoint": "10.0.0.9:8787", - "x-wdl-d1-owner-generation": "0", - }, - } - ); +test("control D1 runtime client rejects non-positive and unsafe owner generation hints", async () => { + for (const rawGeneration of ["0", "9007199254740992"]) { + const result = await d1RuntimeQuery(withInternalAuthEnv({ + D1_BACKEND: { + async fetch() { + return d1Response( + { success: true, results: [] }, + { + headers: { + "x-wdl-d1-owner-task-id": "task-a", + "x-wdl-d1-owner-endpoint": "10.0.0.9:8787", + "x-wdl-d1-owner-generation": rawGeneration, + }, + } + ); + }, }, - }, - }), "tenant-a", "db1", "all", [{ sql: "select 1", params: [] }]); + }), "tenant-a", "db1", "all", [{ sql: "select 1", params: [] }]); - assert.equal(result.ok, true); - assert.deepEqual(result.owner, { - taskId: "task-a", - endpoint: "10.0.0.9:8787", - generation: 0, - }); + assert.equal(result.ok, true); + assert.equal(result.owner, null, rawGeneration); + } }); test("control D1 runtime client rejects fractional owner generation hints", async () => { diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index 020fe6e..ec1baa2 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -18,6 +18,7 @@ const runtimeEnvBuildUrl = repositoryModuleDataUrl( "runtime/load/env-build.js", importSpecifierReplacements({ "shared-ns-pattern": repositoryFileUrl("shared/ns-pattern.js"), + "shared-version": repositoryFileUrl("shared/version.js"), }) ); const { diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 4f17b2d..d375ad7 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -23,6 +23,7 @@ const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const SHARED_SECRET_KEYS_URL = repositoryFileUrl("shared/secret-keys.js"); const RUNTIME_ENV_BUILD_URL = repositoryModuleDataUrl("runtime/load/env-build.js", [ [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], + [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], ]); const { libUrl: PRODUCTION_CONTROL_LIB_URL } = await compileControlGraph(); const env = { diff --git a/tests/unit/control-shared.test.js b/tests/unit/control-shared.test.js index 7afdcdc..8d31d82 100644 --- a/tests/unit/control-shared.test.js +++ b/tests/unit/control-shared.test.js @@ -29,7 +29,6 @@ const controlR2Url = moduleDataUrl(`export function makeR2AdminClient() { return const sharedAuthTokenUrl = moduleDataUrl(`export function extractToken() { return null; }`); const sharedAuthRolesUrl = moduleDataUrl(`export function validatePrincipalShape() { return false; }`); const sharedBoundedBodyUrl = repositoryFileUrl("shared/bounded-body.js"); -const sharedEnvUrl = repositoryFileUrl("shared/env.js"); const sharedErrorsUrl = repositoryFileUrl("shared/errors.js"); const sharedRandomIdUrl = repositoryFileUrl("shared/random-id.js"); const sharedRedisLockUrl = moduleDataUrl(applyModuleReplacements( @@ -41,6 +40,7 @@ const sharedRespondUrl = moduleDataUrl(readRepositoryFile("shared/respond.js")); const controlLibUrl = moduleDataUrl(`export function deleteLockKey(ns, worker) { return \`worker-delete-lock:\${ns}:\${worker}\`; }`); const sharedS3CleanupLifecycleUrl = repositoryFileUrl("shared/s3-cleanup-lifecycle.js"); const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const sharedOptimisticRetryUrl = repositoryFileUrl("shared/optimistic-retry.js"); const controlErrorsUrl = moduleDataUrl(applyModuleReplacements( readRepositoryFile("control/errors.js"), [[/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`]] @@ -53,15 +53,11 @@ const controlWorkflowsClientUrl = moduleDataUrl(applyModuleReplacements( ] )); const { postWorkflowsInternalRequest } = await import(controlWorkflowsClientUrl); -const sharedOwnerLeaseUrl = moduleDataUrl(applyModuleReplacements( - readRepositoryFile("shared/owner-lease.js"), - [[/from "shared-env"/g, `from ${JSON.stringify(sharedEnvUrl)}`]] -)); const controlOptimisticUrl = moduleDataUrl(applyModuleReplacements( readRepositoryFile("control/optimistic.js"), [ [/from "shared-redis"/g, `from ${JSON.stringify(sharedRedisUrl)}`], - [/from "shared-owner-lease"/g, `from ${JSON.stringify(sharedOwnerLeaseUrl)}`], + [/from "shared-optimistic-retry"/g, `from ${JSON.stringify(sharedOptimisticRetryUrl)}`], ] )); const controlJsonBodyUrl = moduleDataUrl(applyModuleReplacements( diff --git a/tests/unit/d1-owner-client.test.js b/tests/unit/d1-owner-client.test.js index d2c8f91..4c80f50 100644 --- a/tests/unit/d1-owner-client.test.js +++ b/tests/unit/d1-owner-client.test.js @@ -86,6 +86,18 @@ test("D1 owner client classifies non-error forwarded HTTP status as ok", async ( assert.equal(D1_OWNER_CLIENT_TEST_STATE.logs.at(-1).level, "info"); }); +test("D1 owner client rejects invalid endpoints before authenticated forwarding", async () => { + await assert.rejects( + () => mod.forwardToOwner(query(), env(), { ...owner(), endpoint: "8.8.8.8:8787" }, "req-invalid", 0), + (err) => { + assert.equal(/** @type {{ status?: unknown }} */ (err).status, 503); + assert.equal(/** @type {{ code?: unknown }} */ (err).code, "owner-endpoint-invalid"); + return true; + } + ); + assert.equal(D1_OWNER_CLIENT_TEST_STATE.fetches.length, 0); +}); + test("D1 owner client maps post-forward transport failures to result-unknown", async () => { restoreFetch(); restoreFetch = installMockFetch(makeRecordingFetch(D1_OWNER_CLIENT_TEST_STATE.fetches, { diff --git a/tests/unit/d1-owner-registry-regressions.test.js b/tests/unit/d1-owner-registry-regressions.test.js index c54d0f6..322c795 100644 --- a/tests/unit/d1-owner-registry-regressions.test.js +++ b/tests/unit/d1-owner-registry-regressions.test.js @@ -8,6 +8,7 @@ import { assertCurrentOwnerWithLeaseBudget, drainOwnedDbs, drainConcurrency, + normalizeDatabases, ownerGenerationKeyOf, ownerKeyOf, rebalanceDatabase, @@ -19,6 +20,26 @@ import { beforeEach(resetD1OwnerRegistryTestState); +test("D1 owner registry: records stored under another database scope fail closed", async () => { + const [requested, misplaced] = normalizeDatabases([ + { namespace: "tenant-a", databaseId: "db-a" }, + { namespace: "tenant-b", databaseId: "db-b" }, + ]); + D1_TEST_STATE.registryStore.set(ownerKeyOf(requested.dbKey), JSON.stringify({ + ...misplaced, + taskId: "task-b", + endpoint: "d1-runtime-b:8787", + generation: 3, + leaseExpiresAt: Date.now() + 60_000, + })); + + await assert.rejects( + () => resolveDbOwner({ REDIS_ADDR: "redis:6379" }, requested), + /D1 owner record is invalid/ + ); + assert.deepEqual(D1_TEST_STATE.setCommands, []); +}); + test("D1 owner registry: same-task reclaim repairs a stale generation counter without advancing owner generation", async () => { const identity = { namespace: "tenant-a", databaseId: "db1", dbKey: "tenant-a:db1", slot: 7 }; const ownerKey = ownerKeyOf(identity.dbKey); @@ -26,7 +47,7 @@ test("D1 owner registry: same-task reclaim repairs a stale generation counter wi D1_TEST_STATE.registryStore.set(ownerKey, JSON.stringify({ ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 11, leaseExpiresAt: Date.now() + 60_000, })); @@ -81,7 +102,7 @@ test("D1 owner registry: same-task hot path does not refresh the owner lease", a const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 11, leaseExpiresAt: Date.now() + 60_000, }; @@ -107,7 +128,7 @@ test("D1 owner registry: expired same-task owner is reclaimed before local dispa const expiredOwner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 11, leaseExpiresAt: redisNow - 1, }; @@ -137,7 +158,7 @@ test("D1 owner registry: expired same-task owner is reclaimed before local dispa test("D1 owner registry: lost owner state cannot validate an old owner from another task", async () => { const identity = { namespace: "tenant-a", databaseId: "db1", dbKey: "tenant-a:db1", slot: 7 }; - D1_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "task-b:8787" }; + D1_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "d1-runtime-b:8787" }; const owner = await resolveDbOwner({ REDIS_ADDR: "redis:6379" }, identity); @@ -160,13 +181,13 @@ test("D1 owner registry: lost owner state cannot validate an old owner from anot test("D1 owner registry: same task reclaims generation one after owner state loss", async () => { const identity = { namespace: "tenant-a", databaseId: "db1", dbKey: "tenant-a:db1", slot: 7 }; - D1_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "task-b:8787" }; + D1_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "d1-runtime-b:8787" }; const owner = await resolveDbOwner({ REDIS_ADDR: "redis:6379" }, identity); assert.equal(owner.taskId, "task-b"); assert.equal(owner.generation, 1); - assert.equal(owner.endpoint, "task-b:8787"); + assert.equal(owner.endpoint, "d1-runtime-b:8787"); assert.deepEqual(await assertCurrentOwner({ REDIS_ADDR: "redis:6379" }, owner), owner); }); @@ -176,7 +197,7 @@ test("D1 owner registry: current-owner check rejects expired matching leases", a const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow - 1, }; @@ -199,7 +220,7 @@ test("D1 owner registry: current-owner check renews leases inside the guard wind const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow + 250, }; @@ -224,7 +245,7 @@ test("D1 owner registry: current-owner budget result carries Redis-time remainin const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow + 5000, }; @@ -246,7 +267,7 @@ test("D1 owner registry: current-owner check rejects renewals that still miss th const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow + 250, }; @@ -274,7 +295,7 @@ test("D1 owner registry: post-renew guard budget uses the renew Redis time", asy const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow + 250, }; @@ -300,7 +321,7 @@ test("D1 owner registry: current-owner check uses Redis server time as lease aut const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt, }; @@ -324,7 +345,7 @@ test("D1 owner registry: stale generation repair failure is best-effort", async const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 11, leaseExpiresAt: Date.now() + 60_000, }; @@ -352,7 +373,7 @@ test("D1 owner registry: claim race falls back to the winner owner", async () => const winner = { ...identity, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; @@ -381,7 +402,7 @@ test("D1 owner registry: claim race waits briefly for a winner owner", async () const winner = { ...identity, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; @@ -411,7 +432,7 @@ test("D1 owner registry: observed remote owner avoids repeated registry reads", const owner = { ...identity, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; @@ -438,7 +459,7 @@ test("D1 owner registry: observed local owner avoids repeated registry reads", a const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; @@ -462,14 +483,14 @@ test("D1 owner registry: forced refresh bypasses observed owner cache", async () const firstOwner = { ...identity, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; const nextOwner = { ...identity, taskId: "task-c", - endpoint: "task-c:8787", + endpoint: "d1-runtime-c:8787", generation: 5, leaseExpiresAt: Date.now() + 60_000, }; @@ -494,7 +515,7 @@ test("D1 owner registry: takeover bumps generation monotonically even when the s dbKey: "tenant-a:db1", slot: 7, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 11, leaseExpiresAt: redisNow - 1, }; @@ -520,7 +541,7 @@ test("D1 owner registry: drain waits for in-flight queries before releasing owne dbKey: "tenant-a:db1", slot: 7, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: Date.now() + 60_000, }; @@ -560,7 +581,7 @@ test("D1 owner registry: drain releases idle databases even when one actor times dbKey: `tenant-a:${databaseId}`, slot: idx, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: Date.now() + 60_000, })); @@ -614,7 +635,7 @@ test("D1 owner registry: takeover retries WatchError before surfacing an ownersh dbKey: "tenant-a:db1", slot: 7, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: redisNow - 1, }; @@ -641,7 +662,7 @@ test("D1 owner registry: rebalance retries WatchError before moving ownership", const currentOwner = { ...database, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 6, leaseExpiresAt: Date.now() + 60_000, }; @@ -655,7 +676,7 @@ test("D1 owner registry: rebalance retries WatchError before moving ownership", const result = await rebalanceDatabase( { REDIS_ADDR: "redis:6379" }, database, - { taskId: "task-b", endpoint: "task-b:8787" } + { taskId: "task-b", endpoint: "d1-runtime-b:8787" } ); assert.equal(result.outcome, "moved"); @@ -674,7 +695,7 @@ test("D1 owner registry: rebalance to the current owner is a no-op", async () => const currentOwner = { ...database, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 6, leaseExpiresAt: Date.now() + 60_000, }; @@ -687,7 +708,7 @@ test("D1 owner registry: rebalance to the current owner is a no-op", async () => const result = await rebalanceDatabase( { REDIS_ADDR: "redis:6379" }, database, - { taskId: "task-a", endpoint: "task-a:8787" } + { taskId: "task-a", endpoint: "d1-runtime-a:8787" } ); assert.equal(result.outcome, "unchanged"); @@ -709,7 +730,7 @@ test("D1 owner registry: rebalance to same task with a different endpoint refres const currentOwner = { ...database, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 6, leaseExpiresAt: Date.now() + 60_000, }; @@ -720,12 +741,12 @@ test("D1 owner registry: rebalance to same task with a different endpoint refres const result = await rebalanceDatabase( { REDIS_ADDR: "redis:6379" }, database, - { taskId: "task-a", endpoint: "task-a-new:8787" } + { taskId: "task-a", endpoint: "d1-runtime-a-new:8787" } ); assert.equal(result.outcome, "moved"); assert.equal(result.owner.taskId, "task-a"); - assert.equal(result.owner.endpoint, "task-a-new:8787"); + assert.equal(result.owner.endpoint, "d1-runtime-a-new:8787"); assert.equal(result.owner.generation, 7); assert.equal(D1_TEST_STATE.ownedDbs.has(database.dbKey), false); assert.deepEqual(D1_TEST_STATE.forgottenStorageSizes, [database.dbKey]); @@ -740,7 +761,7 @@ test("D1 owner registry: renew uses bounded concurrency instead of strict serial dbKey: `tenant-a:db${idx}`, slot: idx, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 1, leaseExpiresAt: Date.now() + 60_000, }; diff --git a/tests/unit/d1-owner-registry.test.js b/tests/unit/d1-owner-registry.test.js index 600a39e..aa38848 100644 --- a/tests/unit/d1-owner-registry.test.js +++ b/tests/unit/d1-owner-registry.test.js @@ -49,18 +49,26 @@ test("D1 owner registry: owner keys encode database keys safely", () => { }); test("D1 owner registry: parseOwner accepts Redis strings and bulk bytes", () => { + const [identity] = normalizeDatabases([{ namespace: "tenant-a", databaseId: "d1_main" }]); const owner = { - namespace: "tenant-a", - databaseId: "d1_main", - dbKey: "tenant-a:d1_main", + ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: Date.now() + 1000, }; - assert.deepEqual(parseOwner(JSON.stringify(owner)), owner); - assert.deepEqual(parseOwner(new TextEncoder().encode(JSON.stringify(owner))), owner); - assert.equal(parseOwner(null), null); + assert.deepEqual(parseOwner(JSON.stringify(owner), identity.dbKey), owner); + assert.deepEqual(parseOwner(new TextEncoder().encode(JSON.stringify(owner)), identity.dbKey), owner); + assert.equal(parseOwner(null, identity.dbKey), null); + assert.throws( + () => parseOwner(JSON.stringify({ ...owner, endpoint: "8.8.8.8:8787" }), identity.dbKey), + /D1 owner record is invalid/ + ); + assert.throws( + () => parseOwner(JSON.stringify({ ...owner, dbKey: "tenant-b:db-b" }), identity.dbKey), + /D1 owner record is invalid/ + ); + assert.throws(() => parseOwner("{not-json", identity.dbKey), /D1 owner record is invalid/); }); test("D1 owner registry: rebalance request normalization validates shape", () => { @@ -76,7 +84,15 @@ test("D1 owner registry: rebalance request normalization validates shape", () => assert.throws(() => normalizeDatabases([]), /databases must be a non-empty array/); assert.throws( () => normalizeDatabases([{ namespace: "tenant-a", databaseId: "db:bad" }]), - /databaseId must not contain ':'/ + /databaseId is invalid/ + ); + assert.throws( + () => normalizeDatabases([{ namespace: "admin", databaseId: "main" }]), + /namespace is invalid/ ); assert.throws(() => normalizeTarget({ taskId: "task-b" }), /target.endpoint is required/); + assert.throws( + () => normalizeTarget({ taskId: "task-b", endpoint: "8.8.8.8:8787" }), + /target.endpoint is invalid/ + ); }); diff --git a/tests/unit/d1-protocol.test.js b/tests/unit/d1-protocol.test.js index f60e2b7..a3c4bea 100644 --- a/tests/unit/d1-protocol.test.js +++ b/tests/unit/d1-protocol.test.js @@ -56,6 +56,12 @@ test("D1 protocol: db key includes namespace and database id", () => { assert.notEqual(dbKeyOf("tenant-a", "main"), dbKeyOf("tenant-b", "main")); }); +test("D1 protocol: db keys reject non-canonical runtime namespaces and database ids", () => { + assert.throws(() => dbKeyOf("admin", "main"), /namespace is invalid/); + assert.throws(() => dbKeyOf("tenant-a", "db/child"), /databaseId is invalid/); + assert.throws(() => dbKeyOf("tenant-a", "a".repeat(129)), /databaseId is invalid/); +}); + test("D1 protocol: slot hash is stable and bounded", () => { const slot = slotOf("tenant-a", "main", 128); assert.equal(slot, slotOf("tenant-a", "main", 128)); @@ -386,7 +392,12 @@ test("D1 protocol: invalid request shape throws protocol error", () => { }); test("D1 protocol: classifies user-facing errors by category", () => { - for (const code of ["owner-lease-too-short", "lease-budget-exhausted"]) { + for (const code of [ + "owner-record-invalid", + "owner-endpoint-invalid", + "owner-lease-too-short", + "lease-budget-exhausted", + ]) { assert.deepEqual( classifyD1Error(new D1ProtocolError(503, code, "lease budget low")), { diff --git a/tests/unit/d1-router.test.js b/tests/unit/d1-router.test.js index e9e8a59..b076945 100644 --- a/tests/unit/d1-router.test.js +++ b/tests/unit/d1-router.test.js @@ -7,7 +7,7 @@ import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.j const taskIdentityUrl = moduleDataUrl(` -export async function resolveTaskIdentity() { return { taskId: "task-a", endpoint: "task-a:8787" }; } +export async function resolveTaskIdentity() { return { taskId: "task-a", endpoint: "d1-runtime-a:8787" }; } `); const timeoutUrl = moduleDataUrl(` export function createD1QueryDeadline() { @@ -20,7 +20,7 @@ export async function ownerLeaseExpiredByRedisTime(_env, owner) { } export async function resolveDbOwner(env, query, options) { return /** @type {any} */ (globalThis).__d1RouterResolveDbOwner?.(env, query, options) || - { taskId: "task-a", endpoint: "task-a:8787", generation: 1 }; + { taskId: "task-a", endpoint: "d1-runtime-a:8787", generation: 1 }; } export async function takeoverExpiredOwner(env, owner) { return /** @type {any} */ (globalThis).__d1RouterTakeoverExpiredOwner?.(env, owner) || owner; @@ -144,14 +144,14 @@ test("D1 router uses takeover owner even after refresh is disabled", async () => const oldOwner = { dbKey: query.dbKey, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 7, leaseExpiresAt: Date.now() - 1_000, }; const takeoverOwner = { dbKey: query.dbKey, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 8, leaseExpiresAt: Date.now() + 60_000, }; @@ -206,7 +206,7 @@ test("D1 router owner-not-ready errors do not expose owner task identity", async const owner = { dbKey: query.dbKey, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 7, leaseExpiresAt: Date.now() + 60_000, }; diff --git a/tests/unit/d1-task-identity.test.js b/tests/unit/d1-task-identity.test.js index f254edb..eef4487 100644 --- a/tests/unit/d1-task-identity.test.js +++ b/tests/unit/d1-task-identity.test.js @@ -6,6 +6,7 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, + repositoryFileUrl, sharedModuleDataUrl, } from "../helpers/load-shared-module.js"; @@ -13,11 +14,16 @@ const { D1ProtocolError } = await loadD1Protocol(); const PROTOCOL_URL = d1ProtocolDataUrl(); const SHARED_TASK_IDENTITY_URL = sharedModuleDataUrl("shared/task-identity.js"); +const SHARED_OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); const src = applyModuleReplacements(readRepositoryFile("d1-runtime/task-identity.js"), [ [ /import \{ createTaskIdentityResolver \} from "shared-task-identity";/, `import { createTaskIdentityResolver } from ${JSON.stringify(SHARED_TASK_IDENTITY_URL)};`, ], + [ + /import \{ validOwnerEndpointForService \} from "shared-owner-endpoint";/, + `import { validOwnerEndpointForService } from ${JSON.stringify(SHARED_OWNER_ENDPOINT_URL)};`, + ], [ /import \{ D1ProtocolError \} from "d1-runtime-protocol";/, `import { D1ProtocolError } from ${JSON.stringify(PROTOCOL_URL)};`, @@ -68,6 +74,13 @@ test("D1 task identity: env task id and endpoint win", async () => { }); }); +test("D1 task identity: public owner endpoints are rejected", () => { + assert.throws( + () => taskIdentityFromEnv({ D1_TASK_ID: "task-a", D1_TASK_ENDPOINT: "8.8.8.8:8787" }), + (err) => isProtocolError(err, 503, "task-identity-unavailable") + ); +}); + test("D1 task identity: partial env identity is rejected", () => { assert.throws( () => taskIdentityFromEnv({ D1_TASK_ID: "task-only" }), @@ -85,11 +98,11 @@ test("D1 task identity: ECS metadata yields task arn and private IPv4 endpoint", Networks: [{ IPv4Addresses: ["10.0.42.17"] }], }, ], - }, { D1_TASK_PORT: "9797" }); + }); assert.deepEqual(identity, { taskId: "arn:aws:ecs:us-east-1:123456789012:task/cluster/abc", - endpoint: "10.0.42.17:9797", + endpoint: "10.0.42.17:8787", source: "ecs-metadata", }); }); diff --git a/tests/unit/do-alarm-shim.test.js b/tests/unit/do-alarm-shim.test.js index 02f9f5e..439cb91 100644 --- a/tests/unit/do-alarm-shim.test.js +++ b/tests/unit/do-alarm-shim.test.js @@ -2,14 +2,304 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { DO_ALARM_SHIM_SOURCE } from "../../do-runtime/alarm-shim-source.js"; import { makeDoAlarmBinding, makeDoAlarmStorage } from "../helpers/do-alarm-shim-fixture.js"; -import { moduleDataUrl } from "../helpers/load-shared-module.js"; +import { applyModuleReplacements, moduleDataUrl } from "../helpers/load-shared-module.js"; +import { withMockedGlobal, withMockedProperty } from "../helpers/mock-global.js"; import { assertJsonResponse } from "../helpers/response-json.js"; -const shimSource = DO_ALARM_SHIM_SOURCE.replace( - "function wrapStorage", - "export function wrapStorage" -); -const { wrapDurableObjectClass, wrapStorage } = await import(moduleDataUrl(shimSource)); +const shimSource = applyModuleReplacements(DO_ALARM_SHIM_SOURCE, [ + ["function wrapStorage", "export function wrapStorage"], + ["function formatWrappedError", "export function formatWrappedError"], +]); +const { formatWrappedError, wrapDurableObjectClass, wrapStorage } = await import(moduleDataUrl(shimSource)); + +test("DO alarm shim: error formatting cannot replace the original failure", () => { + const hostile = new Error("original"); + Object.defineProperties(hostile, { + name: { get() { throw new Error("name getter failed"); } }, + message: { get() { throw new Error("message getter failed"); } }, + code: { get() { throw new Error("code getter failed"); } }, + }); + assert.deepEqual(formatWrappedError(hostile), { + error_name: "Error", + error_message: "Unknown error", + }); + + const proxy = new Proxy({}, { + get() { throw new Error("proxy getter failed"); }, + getPrototypeOf() { throw new Error("prototype trap failed"); }, + }); + assert.deepEqual(formatWrappedError(proxy), { + error_name: "Error", + error_message: "Unknown error", + }); +}); + +test("DO alarm shim: repair logging cannot replace the stored alarm result", async () => { + /** @param {(read: () => Promise) => Promise} callback */ + async function readWithBrokenLogDependency(callback) { + const { storage } = makeDoAlarmStorage({ + scheduled_time: 1234, + retry_count: 0, + in_flight: 0, + token: "sqlite-token", + }); + const alarmBinding = makeDoAlarmBinding([]); + alarmBinding.setAlarmIndex = async () => { + throw new Error("backend unavailable"); + }; + const wrapped = wrapStorage(storage, alarmBinding, "Room", "alice"); + await callback(async () => assert.equal(await wrapped.getAlarm(), 1234)); + } + + await readWithBrokenLogDependency((read) => + withMockedGlobal("Date", new Proxy(Date, { + construct() { + throw new Error("tenant Date"); + }, + }), read)); + await readWithBrokenLogDependency((read) => + withMockedProperty(JSON, "stringify", () => { + throw new Error("tenant stringify"); + }, read)); + await readWithBrokenLogDependency((read) => + withMockedProperty(console, "log", () => { + throw new Error("tenant console"); + }, read)); +}); + +test("DO alarm shim: internal alarm dispatch ignores tenant-patched request intrinsics", async () => { + /** @type {unknown[][]} */ + const calls = []; + const { storage, state } = makeDoAlarmStorage({ + scheduled_time: Date.now() - 1000, + retry_count: 0, + in_flight: 0, + token: "captured-intrinsics-token", + }); + class AlarmCounter { + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + this.alarms = 0; + } + async alarm() { + this.alarms += 1; + } + async fetch() { + return new Response("tenant fetch"); + } + } + const Wrapped = wrapDurableObjectClass(AlarmCounter, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding(calls) } + ); + const request = new Request("https://do.internal/__wdl_alarm", { + method: "POST", + headers: { "x-wdl-do-internal-alarm": "1" }, + body: JSON.stringify({ token: "captured-intrinsics-token", retryCount: 0 }), + }); + + await withMockedProperty(Headers.prototype, "get", () => null, async () => { + await withMockedProperty(Request.prototype, "json", async () => { + throw new Error("tenant Request.json"); + }, async () => { + await withMockedProperty(Response, "json", () => { + throw new Error("tenant Response.json"); + }, async () => { + const response = await instance.fetch(request); + await assertJsonResponse(response, 200, { ok: true }); + }); + }); + }); + + assert.equal(instance.alarms, 1); + assert.equal(state.row, null); +}); + +test("DO alarm shim: class-field fetch cannot intercept internal alarm dispatch", async () => { + const { storage, state } = makeDoAlarmStorage({ + scheduled_time: Date.now() - 1000, + retry_count: 0, + in_flight: 0, + token: "class-field-fetch-token", + }); + class ClassFieldFetch { + alarms = 0; + fetchCalls = 0; + ["fetch"] = async () => { + this.fetchCalls += 1; + return new Response("tenant fetch"); + }; + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + async alarm() { + this.alarms += 1; + } + } + const Wrapped = wrapDurableObjectClass(ClassFieldFetch, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + const alarmResponse = await instance.fetch(new Request("https://do.internal/__wdl_alarm", { + method: "POST", + headers: { "x-wdl-do-internal-alarm": "1" }, + body: JSON.stringify({ token: "class-field-fetch-token", retryCount: 0 }), + })); + await assertJsonResponse(alarmResponse, 200, { ok: true }); + assert.equal(instance.alarms, 1); + assert.equal(instance.fetchCalls, 0); + assert.equal(state.row, null); + + assert.equal(await (await instance.fetch(new Request("https://do.internal/tenant"))).text(), "tenant fetch"); + assert.equal(instance.fetchCalls, 1); +}); + +test("DO alarm shim: class-field alarm executes before its row is cleared", async () => { + const { storage, state } = makeDoAlarmStorage({ + scheduled_time: Date.now() - 1000, + retry_count: 2, + in_flight: 0, + token: "class-field-alarm-token", + }); + class ClassFieldAlarm { + alarms = 0; + /** @type {number | null} */ + retryCount = null; + /** @param {{ retryCount: number }} info */ + alarm = async (info) => { + this.alarms += 1; + this.retryCount = info.retryCount; + }; + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + } + const Wrapped = wrapDurableObjectClass(ClassFieldAlarm, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + + const response = await instance.fetch(new Request("https://do.internal/__wdl_alarm", { + method: "POST", + headers: { "x-wdl-do-internal-alarm": "1" }, + body: JSON.stringify({ token: "class-field-alarm-token", retryCount: 2 }), + })); + + await assertJsonResponse(response, 200, { ok: true }); + assert.equal(instance.alarms, 1); + assert.equal(instance.retryCount, 2); + assert.equal(state.row, null); +}); + +test("DO alarm shim: own accessors retain their instance receiver", async () => { + const { storage, state } = makeDoAlarmStorage({ + scheduled_time: Date.now() - 1000, + retry_count: 0, + in_flight: 0, + token: "accessor-alarm-token", + }); + class AccessorHandlers { + #alarms = 0; + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + Object.defineProperties(this, { + fetch: { + configurable: true, + get: () => async () => new Response(String(this.#alarms)), + }, + alarm: { + configurable: true, + get: () => async () => { + this.#alarms += 1; + }, + }, + }); + } + } + const Wrapped = wrapDurableObjectClass(AccessorHandlers, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + + const alarmResponse = await instance.fetch(new Request("https://do.internal/__wdl_alarm", { + method: "POST", + headers: { "x-wdl-do-internal-alarm": "1" }, + body: JSON.stringify({ token: "accessor-alarm-token", retryCount: 0 }), + })); + await assertJsonResponse(alarmResponse, 200, { ok: true }); + assert.equal(state.row, null); + assert.equal(await (await instance.fetch(new Request("https://do.internal/status"))).text(), "1"); +}); + +test("DO alarm shim: alarm getter remains lazy until internal alarm dispatch", async () => { + class LazyAlarmGetter { + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + async fetch() { + return new Response("tenant fetch"); + } + get alarm() { + throw new Error("alarm getter must remain lazy"); + } + } + const { storage } = makeDoAlarmStorage(); + const Wrapped = wrapDurableObjectClass(LazyAlarmGetter, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + + const response = await instance.fetch(new Request("https://do.internal/tenant")); + assert.equal(await response.text(), "tenant fetch"); +}); + +test("DO alarm shim: storage facade ignores tenant-patched proxy intrinsics", async () => { + const { storage } = makeDoAlarmStorage({ + scheduled_time: 1234, + retry_count: 0, + in_flight: 0, + token: "captured-proxy-token", + }); + const alarmBinding = makeDoAlarmBinding([]); + const hostileProxy = new Proxy(Proxy, { + construct() { + throw new Error("tenant Proxy"); + }, + }); + + await withMockedGlobal("Proxy", hostileProxy, async () => { + await withMockedProperty(Reflect, "get", () => { + throw new Error("tenant Reflect.get"); + }, async () => { + await withMockedProperty(Reflect, "apply", () => { + throw new Error("tenant Reflect.apply"); + }, async () => { + class AlarmReader { + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + } + const Wrapped = wrapDurableObjectClass(AlarmReader, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: alarmBinding } + ); + assert.equal(await instance.ctx.storage.getAlarm(), 1234); + }); + }); + }); +}); test("DO alarm shim: transaction setAlarm then deleteAlarm flushes only the final delete", async () => { /** @type {unknown[][]} */ diff --git a/tests/unit/do-owner-client.test.js b/tests/unit/do-owner-client.test.js index 951c04c..09f8e3f 100644 --- a/tests/unit/do-owner-client.test.js +++ b/tests/unit/do-owner-client.test.js @@ -103,6 +103,14 @@ test("DO owner client forwards invoke requests with owner fence and hop headers" assert.equal(DO_OWNER_CLIENT_TEST_STATE.logs.at(-1).fields.path, "/internal/do/storage/delete"); }); +test("DO owner client rejects invalid endpoints before authenticated forwarding", async () => { + await assert.rejects( + mod.forwardToOwner(invoke(), env(), { ...owner(), endpoint: "8.8.8.8:8788" }, "req-invalid", 0), + (err) => doErrorMatches(err, { status: 503, code: "owner_unavailable" }) + ); + assert.equal(DO_OWNER_CLIENT_TEST_STATE.fetches.length, 0); +}); + test("DO owner client maps exhausted forward hops to a stable 503 code", async () => { await assert.rejects( mod.forwardToOwner(invoke(), env(), owner(), "req-2", 2), diff --git a/tests/unit/do-owner-registry.test.js b/tests/unit/do-owner-registry.test.js index dd21a5d..331e89b 100644 --- a/tests/unit/do-owner-registry.test.js +++ b/tests/unit/do-owner-registry.test.js @@ -10,6 +10,7 @@ import { ownerGenerationKeyOf, ownerLeaseGuardMs, ownerKeyOf, + parseOwner, renewOwnedScopes, releaseOwner, resetDoOwnerRegistryTestState, @@ -60,13 +61,37 @@ function ownerRecord(overrides = {}) { worker: "chat", doStorageId: DO_STORAGE_ID, taskId: "task-a", - endpoint: "task-a:8788", + endpoint: "do-runtime-a:8788", generation: 7, leaseExpiresAt: Date.now() + LEASE_DURATION_MS, ...overrides, }; } +test("DO owner registry rejects malformed persisted owner endpoints", () => { + const owner = ownerRecord(); + assert.deepEqual(parseOwner(JSON.stringify(owner), OWNER_KEY), owner); + assert.equal(parseOwner(null, OWNER_KEY), null); + assert.throws( + () => parseOwner(JSON.stringify(ownerRecord({ endpoint: "8.8.8.8:8788" })), OWNER_KEY), + /DO owner record is invalid/ + ); + assert.throws( + () => parseOwner(JSON.stringify(ownerRecord({ ownerKey: `${DO_STORAGE_ID}:Room:shard1` })), OWNER_KEY), + /DO owner record is invalid/ + ); + for (const invalidScope of [ + { ns: "tenant:other" }, + { worker: "chat:other" }, + ]) { + assert.throws( + () => parseOwner(JSON.stringify(ownerRecord(invalidScope)), OWNER_KEY), + /DO owner record is invalid/ + ); + } + assert.throws(() => parseOwner("{not-json", OWNER_KEY), /DO owner record is invalid/); +}); + test("DO owner registry: keys encode owner scope and generation separately", () => { assert.equal(ownerKeyOf(OWNER_KEY), "do:owner:scope:do_0123456789abcdef0123456789abcdef%3ARoom%3Ashard0"); assert.equal( @@ -75,6 +100,43 @@ test("DO owner registry: keys encode owner scope and generation separately", () ); }); +test("DO owner registry: records stored under another owner scope fail closed", async () => { + const misplacedStorageId = "do_fedcba9876543210fedcba9876543210"; + const misplacedOwnerKey = `${misplacedStorageId}:Room:shard1`; + DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(OWNER_KEY), JSON.stringify(ownerRecord({ + ownerKey: misplacedOwnerKey, + hostId: misplacedOwnerKey, + doStorageId: misplacedStorageId, + }))); + + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + /DO owner record is invalid/ + ); + assert.deepEqual(doOwnerRegistryWriteCommands(), []); +}); + +test("DO owner registry: records for another bundle scope fail closed before pointer lookup", async () => { + const misplacedPointerKey = "worker:do-storage:other:worker"; + DO_OWNER_REGISTRY_TEST_STATE.store.set(misplacedPointerKey, DO_STORAGE_ID); + DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(OWNER_KEY), JSON.stringify(ownerRecord({ + ns: "other", + worker: "worker", + }))); + + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + /DO owner record is invalid/ + ); + assert.equal( + DO_OWNER_REGISTRY_TEST_STATE.redisState.commands.some((command) => ( + command[0] === "get" && command[1] === misplacedPointerKey + )), + false + ); + assert.deepEqual(doOwnerRegistryWriteCommands(), []); +}); + test("DO owner registry: lease guard config bounds values and permits test disable", () => { assert.equal(ownerLeaseGuardMs({}), 1000); assert.equal(ownerLeaseGuardMs({ DO_OWNER_LEASE_GUARD_MS: "0" }), 0); @@ -126,7 +188,7 @@ test("DO owner registry: claim retries WatchError before surfacing owner", async test("DO owner registry: lost owner state cannot validate an old owner from another task", async () => { setStoragePointer(); - DO_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "task-b:8788" }; + DO_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "do-runtime-b:8788" }; const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); @@ -144,13 +206,13 @@ test("DO owner registry: lost owner state cannot validate an old owner from anot test("DO owner registry: same task reclaims generation one after owner state loss", async () => { setStoragePointer(); - DO_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "task-b:8788" }; + DO_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "do-runtime-b:8788" }; const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); assert.equal(owner.taskId, "task-b"); assert.equal(owner.generation, 1); - assert.equal(owner.endpoint, "task-b:8788"); + assert.equal(owner.endpoint, "do-runtime-b:8788"); assert.deepEqual(await assertCurrentOwner({ REDIS_ADDR: "redis:6379" }, owner), owner); }); @@ -199,7 +261,7 @@ test("DO owner registry: draining task returns a healthy remote owner", async () ownerKey, hostId: ownerKey, taskId: "task-b", - endpoint: "task-b:8788", + endpoint: "do-runtime-b:8788", }); setStoragePointer(); DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(ownerKey), JSON.stringify(owner)); @@ -234,7 +296,7 @@ test("DO owner registry: expired takeover bumps generation monotonically", async ownerKey, hostId: ownerKey, taskId: "task-b", - endpoint: "task-b:8788", + endpoint: "do-runtime-b:8788", generation: 11, leaseExpiresAt: redisNow - EXPIRED_LEASE_AGE_MS, }); @@ -481,7 +543,7 @@ test("DO owner registry: renewOwnedScopes forgets owners lost to another task", const remoteOwner = { ...owner, taskId: "task-b", - endpoint: "task-b:8788", + endpoint: "do-runtime-b:8788", generation: 9, }; DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(ownerKey), JSON.stringify(remoteOwner)); @@ -510,7 +572,7 @@ test("DO owner registry: renewOwnedScopes logs lease loss with in-flight dispatc const remoteOwner = { ...owner, taskId: "task-b", - endpoint: "task-b:8788", + endpoint: "do-runtime-b:8788", generation: 9, }; DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(ownerKey), JSON.stringify(remoteOwner)); diff --git a/tests/unit/do-runtime-index.test.js b/tests/unit/do-runtime-index.test.js index ee10c55..3f3c71f 100644 --- a/tests/unit/do-runtime-index.test.js +++ b/tests/unit/do-runtime-index.test.js @@ -6,6 +6,7 @@ import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; import { importSpecifierReplacements, moduleDataUrl, + readRepositoryJson, readRepositoryModuleSource, repositoryFileUrl, repositoryModuleDataUrl, @@ -29,8 +30,10 @@ function stub(src) { const protocolUrl = doProtocolDataUrl(); const sharedRespondUrl = repositoryFileUrl("shared/respond.js"); const sharedEnvUrl = repositoryFileUrl("shared/env.js"); +const sharedOptimisticRetryUrl = repositoryFileUrl("shared/optimistic-retry.js"); const sharedOwnerLeaseUrl = repositoryModuleDataUrl("shared/owner-lease.js", [ [/from "shared-env";/, `from ${JSON.stringify(sharedEnvUrl)};`], + [/from "shared-optimistic-retry";/, `from ${JSON.stringify(sharedOptimisticRetryUrl)};`], ]); const httpUrl = stub(readRepositoryModuleSource("do-runtime/http.js", importSpecifierReplacements({ "shared-respond": sharedRespondUrl, @@ -46,9 +49,11 @@ export function createHttpRequestScope({ request }) { } `); const workerIdUrl = repositoryFileUrl("shared/worker-id.js"); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const actorUrl = stub(`export class WdlDoHostActor {}`); const alarmDispatchUrl = stub(readRepositoryModuleSource("do-runtime/alarm-dispatch.js", importSpecifierReplacements({ "shared-worker-id": workerIdUrl, + "shared-version": sharedVersionUrl, "do-runtime-protocol": protocolUrl, "do-runtime-http": httpUrl, }))); @@ -124,6 +129,7 @@ const IMPORT_STUBS = { "shared-owner-lease": sharedOwnerLeaseUrl, "shared-respond": sharedRespondUrl, "shared-request-scope": requestScopeUrl, + "shared-version": sharedVersionUrl, "shared-worker-id": workerIdUrl, "do-runtime-actor": actorUrl, "do-runtime-alarm-dispatch": alarmDispatchUrl, @@ -151,6 +157,7 @@ const src = readRepositoryModuleSource("do-runtime/index.js", importSpecifierRep const { default: app } = await import(stub(src)); const { DO_INVOKE_CONTENT_TYPE, DoRuntimeError, encodeDoInvokeRequest } = await import(protocolUrl); const doState = await import(stateUrl); +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); beforeEach(() => { /** @type {any} */ (globalThis).__doIndexHostResponse = null; @@ -250,6 +257,29 @@ test("do-runtime alarm dispatch endpoint invokes the local alarm shim path", asy assert.equal(fetchCall.input.url, "https://do-runtime.internal/invoke"); }); +test("do-runtime alarm dispatch rejects non-canonical versions from the shared fixture", async () => { + for (const { tag, parsed } of versionFixture.cases) { + if (parsed != null) continue; + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/alarms/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + ns: "tenant", + worker: "alarms", + version: tag, + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + objectName: "alice", + retryCount: 0, + token: "row-token", + }), + }), env()); + assert.equal(response.status, 400, tag); + assert.match((await jsonBody(response)).message, /version/, tag); + } + assert.deepEqual(/** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches), []); +}); + test("do-runtime alarm dispatch endpoint maps failed local dispatch to retryable 503", async () => { /** @type {any} */ (globalThis).__doIndexHostResponse = Response.json({ error: "alarm_failed", @@ -488,6 +518,26 @@ test("do-runtime storage-delete-worker classifies invalid members as 207 partial }); }); +test("do-runtime storage-delete-worker rejects non-canonical versions before dispatch", async () => { + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/storage/delete-worker", { + method: "POST", + body: JSON.stringify({ + ns: "tenant", + worker: "chat", + version: "v9007199254740992", + doStorageId: "do_0123456789abcdef0123456789abcdef", + members: ["Room:room-a:0"], + }), + }), env()); + + assert.equal(response.status, 400); + assert.deepEqual(await jsonBody(response), { + error: "invalid_request", + message: "version is invalid", + }); + assert.deepEqual(/** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches), []); +}); + test("do-runtime storage-delete-worker preserves stable member error codes from host responses", async () => { /** @type {any} */ (globalThis).__doIndexHostResponse = Response.json({ error: "stale_owner_generation", diff --git a/tests/unit/do-runtime-load.test.js b/tests/unit/do-runtime-load.test.js index db630f4..a6b19e0 100644 --- a/tests/unit/do-runtime-load.test.js +++ b/tests/unit/do-runtime-load.test.js @@ -254,10 +254,16 @@ test("DO runtime load: applies the host-binding wrapper before the alarm wrapper assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /function deleteAllSqlStorage/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /export function wrapDurableObjectClass/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /delete out\[ALARMS_BINDING\];/); + assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /const objectDefineProperty = Object\.defineProperty;/); + assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /return new NativeProxy\(storage,/); assert.doesNotMatch(loaded.modules["_wdl-do-alarm-shim.js"], /delete out\.__WDL_HOST_BINDINGS_WRAPPED__/); const wrapper = loaded.modules["_wdl-do-runtime-wrapper.js"]; assert.match(wrapper, /import \* as user from "\.\/_wdl-wrapper\.js";/); assert.match(wrapper, /import \{ wrapDurableObjectClass \} from "\.\/_wdl-do-alarm-shim\.js";/); + assert.ok( + wrapper.indexOf('from "./_wdl-do-alarm-shim.js";') < + wrapper.indexOf('import * as user from "./_wdl-wrapper.js";') + ); assert.match(wrapper, /export class Room extends wrapDurableObjectClass\(user\.Room, "Room"\)/); }); diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 35da117..899a952 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -3,6 +3,7 @@ import { test } from "node:test"; import { decodeDoEnvelope } from "../helpers/do-envelope.js"; import { loadDoProtocol } from "../helpers/load-do-protocol.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { assertJsonResponse } from "../helpers/response-json.js"; import { dispatchDoInvokeWithHintCache, @@ -35,6 +36,7 @@ const { readLocalActorInvokeRequest, readJsonBody, } = await loadDoProtocol(); +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); test("DO ownership errors stay aligned with runtime hint and retry handling", async () => { const ownershipCodes = new Set(Object.values(DO_OWNERSHIP_CODE)); @@ -141,19 +143,6 @@ function withRequestBody(invoke, bodyBytes) { }; } -const INLINE_BODY = { - ...BASE_BODY, - hostId: CHAT_ROOM_HOST_ID, - workerId: "tenant:chat:1", - workerCode: { - compatibilityDate: "2026-04-24", - mainModule: "worker.js", - modules: { - "worker.js": "export class ChatRoom {}", - }, - }, -}; - test("normalizes do invoke request", () => { const invoke = normalizeDoInvokeRequest(BASE_BODY); @@ -174,20 +163,48 @@ test("normalizes do invoke request", () => { assert.equal(buildFacetName(invoke), "ChatRoom:room-a"); }); -test("normalizes do invoke request for reserved namespaces", () => { - const invoke = normalizeDoInvokeRequest({ - ...BASE_BODY, - ns: "__system__", - }); +test("DO invoke version ingress matches the shared JS/Rust fixture", () => { + for (const { tag, parsed } of versionFixture.cases) { + if (parsed == null) { + assert.throws( + () => normalizeDoInvokeRequest({ ...BASE_BODY, version: tag }), + /version/, + tag + ); + } else { + const invoke = normalizeDoInvokeRequest({ ...BASE_BODY, version: tag }); + assert.ok("version" in invoke, tag); + assert.equal(invoke.version, tag, tag); + } + } +}); - assert.equal(invoke.workerId, "__system__:chat:v1"); - assert.deepEqual(invoke.props, { - ns: "__system__", - worker: "chat", - version: "v1", - doStorageId: DO_STORAGE_ID, - className: "ChatRoom", - }); +test("normalizes do invoke requests for runtime-load reserved namespaces", () => { + for (const ns of ["__system__", "__platform__"]) { + const invoke = normalizeDoInvokeRequest({ + ...BASE_BODY, + ns, + }); + + assert.equal(invoke.workerId, `${ns}:chat:v1`); + assert.deepEqual(invoke.props, { + ns, + worker: "chat", + version: "v1", + doStorageId: DO_STORAGE_ID, + className: "ChatRoom", + }); + } +}); + +test("DO invoke namespace ingress uses the canonical runtime-load grammar", () => { + for (const ns of ["-tenant", "tenant-", "a".repeat(64), "admin", "__community__"]) { + assert.throws( + () => normalizeDoInvokeRequest({ ...BASE_BODY, ns }), + /ns is not valid/, + ns + ); + } }); test("normalizes do invoke request for mixed-case worker names", () => { @@ -337,29 +354,16 @@ test("do-runtime RPC JSON validation uses captured intrinsics", async () => { }); }); -test("normalizes inline workerCode only for test hooks", () => { - assert.throws( - () => normalizeDoInvokeRequest(INLINE_BODY), - /workerCode is only accepted when DO_TEST_HOOKS=1/ - ); - const invoke = normalizeDoInvokeRequest(INLINE_BODY, { allowInlineWorkerCode: true }); - assert.equal(invoke.hostId, CHAT_ROOM_HOST_ID); - assert.equal("workerCode" in invoke ? "allowExperimental" in invoke.workerCode : undefined, false); -}); - -test("rejects experimental workerd compatibility flags in inline workerCode", () => { +test("rejects inline workerCode", () => { assert.throws( () => normalizeDoInvokeRequest({ - ...INLINE_BODY, + ...BASE_BODY, workerCode: { - ...INLINE_BODY.workerCode, - compatibilityFlags: ["nodejs_compat", "unsafe_module"], + mainModule: "worker.js", + modules: { "worker.js": "export class ChatRoom {}" }, }, - }, { allowInlineWorkerCode: true }), - (err) => err instanceof DoRuntimeError && - err.status === 400 && - err.code === "experimental_compat_flag_unsupported" && - /"unsafe_module"/.test(err.message) + }), + /workerCode is not accepted/ ); }); @@ -550,13 +554,15 @@ test("rejects invalid do alarm retry counts", () => { }); test("rejects invalid owner fence generation", () => { - assert.throws( - () => normalizeDoInvokeRequest({ - ...BASE_BODY, - owner: { ownerKey: CHAT_ROOM_HOST_ID, taskId: "task-a", generation: -1 }, - }), - /owner\.generation must be a non-negative integer/ - ); + for (const generation of [-1, 0, 9007199254740992]) { + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + owner: { ownerKey: CHAT_ROOM_HOST_ID, taskId: "task-a", generation }, + }), + /owner\.generation must be a positive safe integer/ + ); + } }); test("rejects invalid class names", () => { @@ -577,36 +583,25 @@ test("rejects bare numeric versions", () => { ); }); -test("rejects missing main module", () => { +test("rejects malformed, out-of-range, and mismatched host ids", () => { assert.throws( - () => normalizeDoInvokeRequest({ - ...BASE_BODY, - hostId: CHAT_ROOM_HOST_ID, - workerId: "tenant:chat:1", - workerCode: { - mainModule: "missing.js", - modules: { "worker.js": "export default {};" }, - }, - }, { allowInlineWorkerCode: true }), - /workerCode\.mainModule must reference a module/ - ); -}); - -test("rejects malformed inline test-hook host ids", () => { - assert.throws( - () => normalizeDoInvokeRequest({ - ...INLINE_BODY, - hostId: "tenant-worker", - }, { allowInlineWorkerCode: true }), + () => normalizeDoInvokeRequest({ ...BASE_BODY, hostId: "tenant-worker" }), /hostId is not valid/ ); assert.throws( () => normalizeDoInvokeRequest({ - ...INLINE_BODY, + ...BASE_BODY, hostId: `${DO_STORAGE_ID}:ChatRoom:shard16`, - }, { allowInlineWorkerCode: true }), + }), /hostId shard is not valid/ ); + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + hostId: `${DO_STORAGE_ID}:ChatRoom:shard012`, + }), + /hostId is not valid/ + ); assert.throws( () => normalizeDoInvokeRequest({ ...BASE_BODY, @@ -616,6 +611,15 @@ test("rejects malformed inline test-hook host ids", () => { ); }); +test("generated host ids honor the aggregate protocol size limit", () => { + const maxHostId = hostIdForShard("a".repeat(496), "ChatRoom", 0); + assert.equal(Buffer.byteLength(maxHostId), 512); + assert.throws( + () => hostIdForShard("a".repeat(497), "ChatRoom", 0), + /hostId is too large/ + ); +}); + test("shards object names using UTF-8 bytes", () => { assert.equal(hostIdForObject(DO_STORAGE_ID, "ChatRoom", "中文"), `${DO_STORAGE_ID}:ChatRoom:shard5`); assert.equal(hostIdForObject(DO_STORAGE_ID, "ChatRoom", "🚀"), `${DO_STORAGE_ID}:ChatRoom:shard10`); @@ -733,29 +737,6 @@ test("rejects invalid header names and control characters", () => { ); }); -test("rejects inline workerCode module limits", () => { - const tooManyModules = Object.fromEntries( - Array.from({ length: 129 }, (_, index) => [`module-${index}.js`, "export default {};"]) - ); - assert.throws( - () => normalizeDoInvokeRequest({ - ...INLINE_BODY, - workerCode: { ...INLINE_BODY.workerCode, modules: tooManyModules }, - }, { allowInlineWorkerCode: true }), - /workerCode\.modules has too many modules/ - ); - assert.throws( - () => normalizeDoInvokeRequest({ - ...INLINE_BODY, - workerCode: { - ...INLINE_BODY.workerCode, - modules: { "worker.js": "x".repeat(1024 * 1024 + 1) }, - }, - }, { allowInlineWorkerCode: true }), - /workerCode\.modules\.worker\.js is too large/ - ); -}); - test("maps invalid JSON bodies to protocol errors", async () => { await assert.rejects( readJsonBody(new Request("https://demo.workers.example/", { method: "POST", body: "{" })), diff --git a/tests/unit/do-task-identity.test.js b/tests/unit/do-task-identity.test.js index 716ba57..abfca0d 100644 --- a/tests/unit/do-task-identity.test.js +++ b/tests/unit/do-task-identity.test.js @@ -5,17 +5,23 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, + repositoryFileUrl, sharedModuleDataUrl, } from "../helpers/load-shared-module.js"; import { doProtocolDataUrl } from "../helpers/load-do-protocol.js"; const PROTOCOL_URL = doProtocolDataUrl(); const SHARED_TASK_IDENTITY_URL = sharedModuleDataUrl("shared/task-identity.js"); +const SHARED_OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); const src = applyModuleReplacements(readRepositoryFile("do-runtime/task-identity.js"), [ [ /import \{ createTaskIdentityResolver \} from "shared-task-identity";/, `import { createTaskIdentityResolver } from ${JSON.stringify(SHARED_TASK_IDENTITY_URL)};`, ], + [ + /import \{ validOwnerEndpointForService \} from "shared-owner-endpoint";/, + `import { validOwnerEndpointForService } from ${JSON.stringify(SHARED_OWNER_ENDPOINT_URL)};`, + ], [ /import \{ DoRuntimeError \} from "do-runtime-protocol";/, `import { DoRuntimeError } from ${JSON.stringify(PROTOCOL_URL)};`, @@ -65,6 +71,13 @@ test("DO task identity: env task id and endpoint win", async () => { }); }); +test("DO task identity: public owner endpoints are rejected", () => { + assert.throws( + () => taskIdentityFromEnv({ DO_TASK_ID: "task-a", DO_TASK_ENDPOINT: "8.8.8.8:8788" }), + (err) => isRuntimeError(err, 503, "task_identity_unavailable") + ); +}); + test("DO task identity: partial env identity is rejected", () => { assert.throws( () => taskIdentityFromEnv({ DO_TASK_ID: "task-only" }), @@ -81,11 +94,11 @@ test("DO task identity: ECS metadata yields task arn and private IPv4 endpoint", Networks: [{ IPv4Addresses: ["10.0.42.17"] }], }, ], - }, { DO_TASK_PORT: "9798" }); + }); assert.deepEqual(identity, { taskId: "arn:aws:ecs:us-east-1:123456789012:task/cluster/abc", - endpoint: "10.0.42.17:9798", + endpoint: "10.0.42.17:8788", source: "ecs-metadata", }); }); diff --git a/tests/unit/owner-lease.test.js b/tests/unit/owner-lease.test.js index c14c34f..a472c34 100644 --- a/tests/unit/owner-lease.test.js +++ b/tests/unit/owner-lease.test.js @@ -31,7 +31,9 @@ test("owner lease helpers parse Redis owner records", () => { assert.equal(parseOwnerRecord(null), null); assert.equal(parseOwnerRecord("{not-json"), null); assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "missing-generation" })), null); + assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "zero", generation: 0 })), null); assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "fractional", generation: 1.5 })), null); + assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "unsafe", generation: Number.MAX_SAFE_INTEGER + 1 })), null); assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "bad-lease", generation: 1, leaseExpiresAt: "soon" })), null); }); @@ -68,6 +70,8 @@ test("owner lease helpers derive monotonic owner generations from Redis counters values.set("fractional", "3.5"); values.set("bad", "not-a-number"); values.set("negative", "-1"); + values.set("near-max", String(Number.MAX_SAFE_INTEGER - 1)); + values.set("max", String(Number.MAX_SAFE_INTEGER)); values.set("huge", "9007199254740992"); const session = { /** @param {string} key */ @@ -97,6 +101,16 @@ test("owner lease helpers derive monotonic owner generations from Redis counters assert.equal(await nextOwnerGeneration(session, "normal", 3), 8); assert.equal(await nextOwnerGeneration(session, "normal", 10), 11); assert.equal(await nextOwnerGeneration(session, "missing", 0), 1); + assert.equal(await nextOwnerGeneration(session, "near-max", 0), Number.MAX_SAFE_INTEGER); + assert.equal(await currentOwnerGenerationCounter(session, "max"), Number.MAX_SAFE_INTEGER); + await assert.rejects( + () => nextOwnerGeneration(session, "max", 0), + /Owner generation counter is exhausted: max/ + ); + await assert.rejects( + () => nextOwnerGeneration(session, "missing", Number.MAX_SAFE_INTEGER), + /Owner generation counter is exhausted: missing/ + ); await assert.rejects( () => nextOwnerGeneration(session, "bad", 0), /Owner generation counter is corrupt: bad/ diff --git a/tests/unit/runtime-bindings-do.test.js b/tests/unit/runtime-bindings-do.test.js index 14712a0..bf2f0be 100644 --- a/tests/unit/runtime-bindings-do.test.js +++ b/tests/unit/runtime-bindings-do.test.js @@ -124,14 +124,19 @@ test("DO fetch requestSpec uses captured header mutation intrinsics", async () = assert.equal(headers.get("x-request-id"), "scope-rid"); }); -test("DO owner hint parser requires integer owner generation", () => { - assert.deepEqual(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 0 }))), { +test("DO owner hint parser requires positive safe-integer owner generation", () => { + assert.deepEqual(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 3 }))), { ownerKey: "do_0123456789abcdef0123456789abcdef:Room:shard0", taskId: "do-runtime-a", endpoint: "do-runtime-a:8788", - generation: 0, + generation: 3, }); + assert.equal(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 0 }))), null); assert.equal(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 1.5 }))), null); + assert.equal( + ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 9007199254740992 }))), + null + ); const missingGeneration = new Headers(doOwnerHintHeaders({})); missingGeneration.delete("x-wdl-do-owner-generation"); assert.equal(ownerHintFromHeaders(missingGeneration), null); diff --git a/tests/unit/runtime-d1-stub.test.js b/tests/unit/runtime-d1-stub.test.js index 55db260..d81a138 100644 --- a/tests/unit/runtime-d1-stub.test.js +++ b/tests/unit/runtime-d1-stub.test.js @@ -12,8 +12,10 @@ import { } from "../helpers/load-shared-module.js"; import { runtimeProxyBindingStubUrl, sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; -const OWNER_ENDPOINT_URL = repositoryFileUrl("runtime/_wdl-owner-endpoint.js"); +const OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); const STALE_OWNER_HINT_ERRORS = [ + "owner-record-invalid", + "owner-endpoint-invalid", "task-draining", "owner-lease-expired", "owner-lease-too-short", @@ -94,7 +96,7 @@ const stubSourceReplacements = [ ], [/import \{ createOwnerHintCache \} from "runtime-owner-hint-cache";/g, ownerHintCacheSource], [ - /import \{ validOwnerEndpointForService \} from "runtime-owner-endpoint";/g, + /import \{ validOwnerEndpointForService \} from "shared-owner-endpoint";/g, `import { validOwnerEndpointForService } from ${JSON.stringify(OWNER_ENDPOINT_URL)};`, ], [/from "runtime-bindings-proxy";/g, `from ${JSON.stringify(PROXY_BINDING_URL)};`], @@ -304,80 +306,31 @@ test("D1 runtime stub: learned owner hints survive new binding instances", async }); }); -test("D1 runtime stub: owner hints preserve zero and reject fractional generations", async () => { - const { db, makeDb, calls } = makeRuntimeDb(() => d1Response({ - success: true, - results: [{ via: "router" }], - }, { - headers: { - "x-wdl-d1-owner-task-id": "task-a", - "x-wdl-d1-owner-endpoint": "d1-runtime-a:8787", - "x-wdl-d1-owner-generation": "0", - }, - })); - await db.query("all", [{ sql: "select 1", params: [] }]); - - /** @type {string[]} */ - const directCalls = []; - await withMockedFetch(async (/** @type {any} */ url) => { - directCalls.push(String(url)); - return d1Response({ success: true, results: [{ via: "owner" }] }); - }, async () => { - await makeDb().query("all", [{ sql: "select 2", params: [] }]); - }); - assert.equal(calls.length, 1); - assert.deepEqual(directCalls, ["http://d1-runtime-a:8787/internal/d1/query"]); - - clearD1OwnerHintsForTest(); - const bad = makeRuntimeDb(() => d1Response({ - success: true, - results: [], - }, { - headers: { - "x-wdl-d1-owner-task-id": "task-b", - "x-wdl-d1-owner-endpoint": "d1-runtime-b:8787", - "x-wdl-d1-owner-generation": "1.5", - }, - })); - await bad.db.query("all", [{ sql: "select 3", params: [] }]); - let unexpectedFractionalFetchCalled = false; - await withMockedFetch(async () => { - unexpectedFractionalFetchCalled = true; - return d1Response({ success: true, results: [] }); - }, async () => { - await bad.makeDb().query("all", [{ sql: "select 4", params: [] }]); - }); - assert.equal( - unexpectedFractionalFetchCalled, - false, - "expected no direct fetch for fractional generation, but fetch was called" - ); - assert.equal(bad.calls.length, 2); +test("D1 runtime stub rejects non-canonical owner generations", async () => { + for (const rawGeneration of ["0", "1.5", "9007199254740992", null]) { + clearD1OwnerHintsForTest(); + const fixture = makeRuntimeDb(() => d1Response({ + success: true, + results: [{ via: "router" }], + }, { + headers: { + "x-wdl-d1-owner-task-id": "task-a", + "x-wdl-d1-owner-endpoint": "d1-runtime-a:8787", + ...(rawGeneration == null ? {} : { "x-wdl-d1-owner-generation": rawGeneration }), + }, + })); + await fixture.db.query("all", [{ sql: "select 1", params: [] }]); - clearD1OwnerHintsForTest(); - const missing = makeRuntimeDb(() => d1Response({ - success: true, - results: [], - }, { - headers: { - "x-wdl-d1-owner-task-id": "task-c", - "x-wdl-d1-owner-endpoint": "d1-runtime-c:8787", - }, - })); - await missing.db.query("all", [{ sql: "select 5", params: [] }]); - let unexpectedMissingGenerationFetchCalled = false; - await withMockedFetch(async () => { - unexpectedMissingGenerationFetchCalled = true; - return d1Response({ success: true, results: [] }); - }, async () => { - await missing.makeDb().query("all", [{ sql: "select 6", params: [] }]); - }); - assert.equal( - unexpectedMissingGenerationFetchCalled, - false, - "expected no direct fetch without owner generation, but fetch was called" - ); - assert.equal(missing.calls.length, 2); + let unexpectedDirectFetch = false; + await withMockedFetch(async () => { + unexpectedDirectFetch = true; + return d1Response({ success: true, results: [{ via: "owner" }] }); + }, async () => { + await fixture.makeDb().query("all", [{ sql: "select 2", params: [] }]); + }); + assert.equal(unexpectedDirectFetch, false, String(rawGeneration)); + assert.equal(fixture.calls.length, 2, String(rawGeneration)); + } }); test("D1 runtime stub: accepts Kubernetes headless owner endpoints", async () => { diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index afa408d..ffae137 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -14,6 +14,7 @@ import { rpcInvokeBody, } from "../../runtime/_wdl-do-transport.js"; import { decodeDoEnvelope } from "../helpers/do-envelope.js"; +import { loadDoProtocol } from "../helpers/load-do-protocol.js"; import { doOwnerHintHeaders, doOwnerHintResponse, @@ -26,6 +27,8 @@ import { } from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; +const { normalizeDoInvokeRequest } = await loadDoProtocol(); + test.afterEach(() => { clearDoOwnerHintsForTest(); }); @@ -112,6 +115,27 @@ test("DurableObjectNamespace metadata host proxy handles normal fetch while webs assert.equal(backendCalls[0].url, "http://do-runtime/internal/do/connect"); }); +test("DurableObjectNamespace classifies metadata with a captured Object.hasOwn", async () => { + await withMockedProperty(Object, "hasOwn", () => { + throw new Error("tenant Object.hasOwn was called"); + }, async () => { + const namespace = new DurableObjectNamespace({ + ns: "tenant", + worker: "worker", + version: "v1", + doStorageId: "storage", + className: "Room", + hostProxy: { + async fetchObject() { + return new Response("ok"); + }, + }, + }); + const response = await namespace.get(namespace.idFromName("room")).fetch("https://example.test/"); + assert.equal(await response.text(), "ok"); + }); +}); + test("DurableObjectNamespace direct backend keeps binding/backend in private fields", async () => { /** @type {any[]} */ const calls = []; @@ -395,6 +419,60 @@ test("DO RPC validation uses captured JSON and method intrinsics", async () => { }); }); +test("DO RPC method bounds match server normalization", () => { + const props = { + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }; + for (const { method, valid } of [ + { method: "save", valid: true }, + { method: "m".repeat(256), valid: true }, + { method: "m".repeat(257), valid: false }, + ]) { + const metadata = { + ...props, + objectName: "room-a", + kind: "rpc", + rpc: { method, args: [] }, + }; + if (!valid) { + assert.throws(() => rpcInvokeBody(props, "room-a", method, []), /rpc\.method is not valid/); + assert.throws(() => normalizeDoInvokeRequest(metadata), /rpc\.method is too large/); + continue; + } + const envelope = rpcInvokeBody(props, "room-a", method, []); + const invoke = normalizeDoInvokeRequest(decodeDoEnvelope(envelope).metadata); + assert.equal("rpc" in invoke ? invoke.rpc.method : null, method); + } +}); + +test("DO RPC snapshots tenant arguments once before sizing and encoding", () => { + const props = { + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }; + let reads = 0; + const argument = {}; + Object.defineProperty(argument, "payload", { + enumerable: true, + get() { + reads += 1; + return reads === 1 ? "stable" : "x".repeat(1_200_000); + }, + }); + + const envelope = rpcInvokeBody(props, "room-a", "save", [argument]); + const { metadata } = decodeDoEnvelope(envelope); + assert.equal(reads, 1); + assert.equal(/** @type {any} */ (metadata).rpc.args[0].payload, "stable"); +}); + test("DurableObjectNamespace RPC rejects oversized args before transport", async () => { const backend = { async fetch() { @@ -1532,8 +1610,21 @@ test("DO owner hint parsing keeps the validated endpoint with patched String", a assert.equal(hint?.endpoint, "do-runtime-a:8788"); }); +test("DO owner hint generation parsing requires a positive safe integer with captured intrinsics", async () => { + const headers = new Headers(doOwnerHintHeaders({ endpoint: "do-runtime-a:8788" })); + await withMockedProperty(Number, "isSafeInteger", () => false, () => { + assert.equal(ownerHintFromHeaders(headers)?.generation, 3); + }); + for (const generation of ["0", "not-an-integer", "9007199254740992"]) { + headers.set("x-wdl-do-owner-generation", generation); + await withMockedProperty(Number, "isSafeInteger", () => true, () => { + assert.equal(ownerHintFromHeaders(headers), null, generation); + }); + } +}); + test("DurableObjectNamespace direct backend rejects unsafe IPv4 owner hint endpoints", async () => { - for (const endpoint of ["0.0.0.0:8788", "127.0.0.1:8788", "169.254.169.254:8788", "224.0.0.1:8788"]) { + for (const endpoint of ["0.0.0.0:8788", "8.8.8.8:8788", "127.0.0.1:8788", "169.254.169.254:8788", "224.0.0.1:8788"]) { /** @type {any[]} */ const ownerCalls = []; const backend = { @@ -1744,6 +1835,23 @@ test("DurableObjectNamespace direct backend rejects oversized declared request b ); }); +test("DO request body bounds use captured Number intrinsics", async () => { + const request = new Request("https://demo.workers.example/send", { + method: "POST", + headers: { "content-length": String(1024 * 1024 + 1) }, + body: "x", + }); + await withMockedProperty(globalThis, "Number", /** @type {NumberConstructor} */ (() => 0), async () => { + await assert.rejects(fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", request, null), /Durable Object fetch body exceeds/); + }); +}); + test("DO requestSpec rejects streaming bodies as soon as they cross the cap", async () => { let pulls = 0; const body = new ReadableStream({ diff --git a/tests/unit/runtime-do-owner-network.test.js b/tests/unit/runtime-do-owner-network.test.js index 65b7b4a..e0426c7 100644 --- a/tests/unit/runtime-do-owner-network.test.js +++ b/tests/unit/runtime-do-owner-network.test.js @@ -13,14 +13,14 @@ import { } from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; -import { validOwnerEndpointForService } from "../../runtime/_wdl-owner-endpoint.js"; +import { validOwnerEndpointForService } from "../../shared/owner-endpoint.js"; -const OWNER_ENDPOINT_URL = repositoryFileUrl("runtime/_wdl-owner-endpoint.js"); +const OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; async function loadWorker() { const source = applyModuleReplacements(readRepositoryFile("runtime/do-owner-network.js"), [ - [/from "runtime-owner-endpoint";/g, `from ${JSON.stringify(OWNER_ENDPOINT_URL)};`], + [/from "shared-owner-endpoint";/g, `from ${JSON.stringify(OWNER_ENDPOINT_URL)};`], [/from "shared-internal-auth";/g, `from ${JSON.stringify(sharedInternalAuthUrl())};`], ]); return await import(moduleDataUrl(source)); @@ -64,6 +64,7 @@ test("do owner network rejects invalid owner endpoints before forwarding", async "http://d1-runtime-a:8787/internal/do/invoke", "http://do-runtime-a:8787/internal/do/invoke", "http://169.254.169.254:8788/internal/do/invoke", + "http://8.8.8.8:8788/internal/do/invoke", "http://evil.test:8788/internal/do/invoke", ]) { const response = await worker.fetch(new Request(url)); diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index 61f7bd9..6ff3c8e 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { parseBase64Json } from "../helpers/json-payload.js"; -import { runtimeLibModuleDataUrl } from "../helpers/load-shared-module.js"; +import { readRepositoryJson, runtimeLibModuleDataUrl } from "../helpers/load-shared-module.js"; const { base64ToBytes, @@ -19,6 +19,7 @@ const { normalizeScheduledDispatchBody, } = await import(runtimeLibModuleDataUrl()); const sharedBase64 = await import("../../shared/base64.js"); +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); const enc = new TextEncoder(); @@ -193,6 +194,19 @@ test("normalizeWorkflowNotifyBody validates route worker identity grammar", () = ); }); +test("workflow dispatch version ingress matches the shared JS/Rust fixture", () => { + for (const { tag, parsed } of versionFixture.cases) { + const body = workflowBody({ frozenVersion: tag }); + if (parsed == null) { + assert.throws(() => normalizeWorkflowRunBody(body), /frozenVersion/, tag); + assert.throws(() => normalizeWorkflowNotifyBody(body), /frozenVersion/, tag); + } else { + assert.equal(normalizeWorkflowRunBody(body).frozenVersion, tag, tag); + assert.equal(normalizeWorkflowNotifyBody(body).frozenVersion, tag, tag); + } + } +}); + test("decodeQueuedDispatchMessages decodes runtime queue wire messages", () => { const messages = decodeQueuedDispatchMessages([ { diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index d056b2a..49bffdf 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -6,6 +6,7 @@ import { pathToFileURL } from "node:url"; import { importRepositoryModule, importSpecifierReplacements, + readRepositoryJson, readRepositoryModuleSource, repositoryFileUrl, repositoryModuleDataUrl, @@ -33,8 +34,12 @@ const SHARED_REDIS_URL = sharedRedisStubUrl(); const { libUrl: CONTROL_LIB_URL } = await compileControlGraph(); const RUNTIME_ENV_BUILD_URL = repositoryModuleDataUrl( "runtime/load/env-build.js", - importSpecifierReplacements({ "shared-ns-pattern": SHARED_NS_PATTERN_URL }) + importSpecifierReplacements({ + "shared-ns-pattern": SHARED_NS_PATTERN_URL, + "shared-version": SHARED_VERSION_URL, + }) ); +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); const FETCH_STUB = { fetch() {} }; const { estimatedWorkerLoaderEnv } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ "control-lib": CONTROL_LIB_URL, @@ -104,6 +109,7 @@ const LOAD_TEST_RUNTIME_DIR = path.join(LOAD_TEST_DIR, "runtime"); const LOAD_TEST_SUBMODULE_DIR = path.join(LOAD_TEST_RUNTIME_DIR, "load"); const ENV_BUILD_SOURCE = readRepositoryModuleSource("runtime/load/env-build.js", importSpecifierReplacements({ "shared-ns-pattern": SHARED_NS_PATTERN_URL, + "shared-version": SHARED_VERSION_URL, })); mkdirSync(LOAD_TEST_SUBMODULE_DIR, { recursive: true }); writeFileSync(path.join(LOAD_TEST_RUNTIME_DIR, "load.js"), src); @@ -757,7 +763,36 @@ test("buildWorkerEnv: D1 binding requires databaseId", () => { "https://assets.example", makeCtx() ), - /D1 binding but missing databaseId/ + /D1 binding but has invalid databaseId/ + ); +}); + +test("buildWorkerEnv: D1 bindings revalidate namespace and database id grammar", () => { + assert.throws( + () => buildWorkerEnv( + { bindings: { DB: { type: "d1", databaseId: "db/child" } } }, + {}, + {}, + "demo", + "app", + "v1", + "https://assets.example", + makeCtx() + ), + /invalid databaseId/ + ); + assert.throws( + () => buildWorkerEnv( + { bindings: { DB: { type: "d1", databaseId: "main" } } }, + {}, + {}, + "admin", + "app", + "v1", + "https://assets.example", + makeCtx() + ), + /invalid namespace/ ); }); @@ -795,6 +830,26 @@ test("buildWorkerEnv: service binding requires service and version", () => { ); }); +test("buildWorkerEnv: stored service binding versions match the shared fixture", () => { + for (const { tag, parsed } of versionFixture.cases) { + if (parsed != null || tag === "") continue; + assert.throws( + () => buildWorkerEnv( + { bindings: { AUTH: { type: "service", service: "auth", version: tag } } }, + {}, + {}, + "demo", + "app", + "v1", + "https://assets.example", + makeCtx() + ), + /service binding but has invalid version/, + tag + ); + } +}); + test("buildWorkerEnv: stored service binding cannot target runtime-reserved entrypoints", () => { assert.throws( () => diff --git a/tests/unit/runtime-loaded-facade-surface.test.js b/tests/unit/runtime-loaded-facade-surface.test.js index 2531c00..c2dc119 100644 --- a/tests/unit/runtime-loaded-facade-surface.test.js +++ b/tests/unit/runtime-loaded-facade-surface.test.js @@ -1,6 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; const requestIdFromOptionsStub = `const requestIdFromOptions = (options) => { if (!options || typeof options !== "object") return null; @@ -78,3 +79,20 @@ test("loaded R2 facade does not expose the runtime RPC stub handle", () => { "resumeMultipartUpload", ]); }); + +test("loaded D1 and R2 facades use WeakMap intrinsics captured before tenant evaluation", async () => { + const fail = () => { + throw new Error("tenant WeakMap method was called"); + }; + await withMockedProperty(WeakMap.prototype, "get", fail, () => + withMockedProperty(WeakMap.prototype, "has", fail, () => + withMockedProperty(WeakMap.prototype, "set", fail, async () => { + const db = new d1Facade.D1Database({ query: async () => ({ results: [] }) }); + assert.deepEqual(await db.prepare("SELECT 1").all(), { results: [] }); + + const bucket = new r2Facade.R2Bucket({ head: async () => null }); + assert.equal(await bucket.head("key"), null); + }) + ) + ); +}); diff --git a/tests/unit/runtime-service-binding.test.js b/tests/unit/runtime-service-binding.test.js index 0fe4fd6..99004f1 100644 --- a/tests/unit/runtime-service-binding.test.js +++ b/tests/unit/runtime-service-binding.test.js @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { importRepositoryModule } from "../helpers/load-shared-module.js"; +import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; import { CLOUDFLARE_WORKERS_URL } from "../helpers/mocks/cloudflare-workers.js"; import { readJsonResponse } from "../helpers/response-json.js"; @@ -13,6 +13,10 @@ const { ServiceBinding } = await importRepositoryModule("runtime/bindings/servic /import \{ INTERNAL_AUTH_HEADER \} from "shared-internal-auth";/, `const INTERNAL_AUTH_HEADER = ${JSON.stringify(TEST_INTERNAL_AUTH_HEADER)};` ], + [ + /import \{ sanitizeRequestId \} from "shared-observability";/, + `import { sanitizeRequestId } from ${JSON.stringify(repositoryFileUrl("shared/observability.js"))};` + ], [ /import \{ getLoadedWorkerStub \} from "runtime-load";/, `const getLoadedWorkerStub = (options) => { @@ -143,3 +147,19 @@ test("ServiceBinding fetch cold-load propagates request id when present", async assert.equal(options.length, 1); assert.equal(options[0].requestId, requestId); }); + +test("ServiceBinding fetch canonicalizes request ids without minting replacements", async () => { + resetLoaderCallbackOptions(); + const binding = makeServiceBinding(); + + const canonical = await readJsonResponse(await binding.fetch(new Request("https://worker.workers.example/rpc", { + headers: { "x-request-id": " rid-first , rid-second" }, + })), 200); + assert.equal(canonical.requestId, "rid-first"); + + const invalid = await readJsonResponse(await binding.fetch(new Request("https://worker.workers.example/rpc", { + headers: { "x-request-id": "bad\\id" }, + })), 200); + assert.equal(invalid.requestId, null); + assert.equal(loaderCallbackOptions().at(-1).requestId, null); +}); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index cb4022b..b157f13 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -263,13 +263,6 @@ test("secret Redis key construction uses the shared JS owner", () => { ); }); -test("do-runtime worker-name grammar matches shared control grammar", () => { - assert.equal( - extractRegex("do-runtime/protocol/wire-grammar.js", "WORKER_NAME_RE"), - extractRegex("shared/ns-pattern.js", "WORKER_NAME_RE") - ); -}); - test("runtime workflow instance id grammar matches shared control grammar", () => { assert.equal( extractRegex("runtime/workflows-client.js", "WORKFLOW_INSTANCE_ID_RE"), @@ -291,7 +284,6 @@ test("runtime workflow dispatch identity grammar mirrors shared grammar", () => extractRegex("runtime/lib.js", "WORKER_NAME_RE"), extractRegex("shared/ns-pattern.js", "WORKER_NAME_RE") ); - assert.equal(extractRegex("runtime/lib.js", "VERSION_RE"), "/^v[1-9][0-9]*$/"); }); test("D1 object field setters stay shared across wire and transport codecs", () => { @@ -1160,10 +1152,12 @@ test("redis-proxy HTTP observability matches platform request strategy", () => { assert.match(lib, /struct ResponseError/); }); -test("embedded DO alarm shim follows shared log stream routing", () => { +test("embedded DO alarm shim keeps best-effort log routing no-throw", () => { const source = withoutLineComments(readRepoFile("do-runtime/alarm-shim-source.js")); + const logger = /function logStructured\([^]*?\n}\n\nfunction ensureAlarmTable/.exec(source)?.[0] || ""; assert.equal(/console\.warn\(/.test(source), false); - assert.match(source, /if \(level === "error"\) console\.error\(line\);\n {2}else console\.log\(line\);/); + assert.match(logger, /if \(level === "error"\) console\.error\(line\);\s+else console\.log\(line\);/); + assert.match(logger, /try \{[^]*} catch \{/); }); test("active docs and sources do not point at note paths", () => { @@ -1207,42 +1201,48 @@ test("active bilingual docs stay paired", () => { assert.deepEqual(offenders.toSorted(), []); }); -test("active bilingual docs keep normative protocol tokens aligned", () => { - const files = activeBilingualDocFiles(); - const fileSet = new Set(files); - const tokenPatterns = [ - /\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b/g, - /\bx-[a-z0-9]+(?:-[a-z0-9]+)+\b/g, - /:\d{2,5}\b/g, - /\bDB\s*[0-9]+\b/gi, +test("owning bilingual docs retain selected cross-tier protocol literals", () => { + // Explicitly pin only high-value literals in their owning docs. Do not return + // to scanning every uppercase token, example port, or incidental DB mention. + /** @param {string} value */ + const codeSpan = (value) => `\`${value}\``; + /** @type {Array<{ files: string[], literals: string[] }>} */ + const contracts = [ + { + files: ["docs/modules/infra.md", "docs/modules/infra.zh.md"], + literals: [ + "WDL_INTERNAL_AUTH_TOKEN", + "x-wdl-internal-auth", + ":8088", + ":8787", + ":8788", + ":9120", + ], + }, + { + files: ["docs/redis-key-layout.md", "docs/redis-key-layout.zh.md"], + literals: ["DB 0", "DB 1", "DB 2"], + }, + { + files: ["docs/modules/control-auth.md", "docs/modules/control-auth.zh.md"], + literals: ["X-Admin-Token"], + }, ]; - const tokens = (/** @type {string} */ file) => { - const found = new Set(); - const source = readRepoFile(file); - for (const pattern of tokenPatterns) { - for (const match of source.matchAll(pattern)) { - found.add(match[0].toUpperCase().replace(/\s+/g, "")); - } - } - return found; - }; + for (const suffix of [".v2", "+v2", "~v2", "-v2"]) { + assert.equal(codeSpan(`x-wdl-internal-auth${suffix}`).includes(codeSpan("x-wdl-internal-auth")), false); + } + assert.equal(codeSpan(":8787abc").includes(codeSpan(":8787")), false); + assert.equal(codeSpan("DB 01").includes(codeSpan("DB 0")), false); const offenders = []; - - for (const english of files.filter((file) => file.endsWith(".md") && !file.endsWith(".zh.md"))) { - if (INTENTIONAL_ONE_LANGUAGE_ACTIVE_DOCS.has(english)) continue; - const chinese = english.replace(/\.md$/, ".zh.md"); - if (!fileSet.has(chinese)) continue; - const englishTokens = tokens(english); - const chineseTokens = tokens(chinese); - for (const token of englishTokens) { - if (!chineseTokens.has(token)) offenders.push(`${chinese}: missing ${token}`); - } - for (const token of chineseTokens) { - if (!englishTokens.has(token)) offenders.push(`${english}: missing ${token}`); + for (const { files, literals } of contracts) { + for (const file of files) { + const source = readRepoFile(file); + for (const literal of literals) { + if (!source.includes(codeSpan(literal))) offenders.push(`${file}: missing ${literal}`); + } } } - - assert.deepEqual(offenders.toSorted(), []); + assert.deepEqual(offenders, []); }); test("protocol and contributor contracts stay discoverable from active docs", () => { @@ -1380,7 +1380,8 @@ test("do-runtime operational logs avoid known camelCase field drift", () => { test("DO alarm tokens stay generated inside the storage shim", () => { const source = withoutLineComments(readRepoFile("do-runtime/alarm-shim-source.js")); - assert.match(source, /function alarmToken\(\)\s*\{\s*return crypto\.randomUUID\(\);\s*\}/); + assert.match(source, /const cryptoRandomUUID = crypto\.randomUUID;/); + assert.match(source, /function alarmToken\(\)\s*\{\s*return reflectApply\(cryptoRandomUUID, nativeCrypto, \[\]\);\s*\}/); assert.match(source, /const token = alarmToken\(\);/); assert.match(source, /alarmBinding\.setAlarmIndex\(\{\s*className,\s*objectName,\s*scheduledTime: alarmTime,\s*retryCount: 0,\s*token,/); }); @@ -1563,7 +1564,7 @@ test("published Compose overrides cover every locally built image service", () = } }); -test("Kubernetes runtime families share the exact redis-proxy sidecar spec", () => { +test("Kubernetes runtime families pin the redis-proxy functional contract", () => { const files = [ "deploy/kubernetes/base/user-runtime.yaml", "deploy/kubernetes/base/system-runtime.yaml", @@ -1584,8 +1585,32 @@ test("Kubernetes runtime families share the exact redis-proxy sidecar spec", () return sidecar; }); - assert.deepEqual(sidecars[1], sidecars[0], `${files[1]} redis-proxy sidecar drifted`); - assert.deepEqual(sidecars[2], sidecars[0], `${files[2]} redis-proxy sidecar drifted`); + for (let index = 0; index < sidecars.length; index += 1) { + const sidecar = sidecars[index]; + assert.equal(sidecar.image, "docker.io/getwdl/wdl-rust:latest", `${files[index]} image`); + assert.deepEqual(sidecar.command, ["/redis-proxy"], `${files[index]} command`); + assert.deepEqual( + /** @type {Array> | undefined} */ (sidecar.ports) + ?.find((port) => port.name === "redis-proxy"), + { name: "redis-proxy", containerPort: 7070 }, + `${files[index]} port` + ); + assert.deepEqual(sidecar.envFrom, [ + { configMapRef: { name: "wdl-config" } }, + { secretRef: { name: "wdl-secrets" } }, + ], `${files[index]} envFrom`); + for (const probe of ["readinessProbe", "livenessProbe"]) { + assert.deepEqual(sidecar[probe], { + exec: { command: ["/redis-proxy", "healthcheck"] }, + }, `${files[index]} ${probe}`); + } + const resources = /** @type {{ requests?: Record, limits?: Record } | undefined} */ ( + sidecar.resources + ); + assert.ok(resources?.requests?.cpu, `${files[index]} resource request cpu`); + assert.ok(resources?.requests?.memory, `${files[index]} resource request memory`); + assert.ok(resources?.limits?.memory, `${files[index]} resource limit memory`); + } }); test("Kubernetes NetworkPolicies pin the per-component ingress matrix", () => { @@ -2227,26 +2252,11 @@ test("D1 and DO workerd env tunables are exposed through capnp bindings", () => "DO_OWNER_LEASE_GUARD_MS", "DO_RENEW_CONCURRENCY", "DO_DRAIN_IN_FLIGHT_TIMEOUT_MS", - "DO_TEST_HOOKS", ]) { assert.equal(exposed(doRuntime, name), true, `${name} must reach do-runtime workerd env`); } }); -test("Terraform gates DO test hooks like D1 test hooks", () => { - const rootVars = readRepoFile("terraform/variables.tf"); - const moduleVars = readRepoFile("terraform/modules/compute/variables.tf"); - const main = readRepoFile("terraform/main.tf"); - const doService = readRepoFile("terraform/modules/compute/do_runtime_service.tf"); - - assert.match(rootVars, /variable "do_test_hooks_enabled"/); - assert.match(moduleVars, /variable "do_test_hooks_enabled"/); - assert.match(main, /do_test_hooks_enabled\s+=\s+var\.do_test_hooks_enabled/); - assert.match(doService, /var\.do_test_hooks_enabled \? \[/); - assert.match(doService, /\{ name = "DO_TEST_HOOKS", value = "1" \}/); - assert.match(doService, /!var\.do_test_hooks_enabled \|\| can\(regex\("\(\^\|-\)test\(\$\|-\)", var\.name\)\)/); -}); - test("D1 and DO workerd containers keep explicit memory ceilings", () => { const rootVars = readRepoFile("terraform/variables.tf"); const moduleVars = readRepoFile("terraform/modules/compute/variables.tf"); @@ -2438,30 +2448,6 @@ test("Terraform ECS services use Fargate-only launch contracts", () => { } }); -test("DO RPC JSON-data validators stay aligned across client and server", () => { - // Match jsonDataError() through the following public checker declaration. - // The checker may be preceded by JSDoc and may be exported assertJsonRpcArgs() - // or local requireJsonData(). - const jsonDataErrorWithChecker = - /function jsonDataError\([^]*?\n}\n\n(?:\/\*\*[^]*?\*\/\n)?(?:export function assertJsonRpcArgs|function requireJsonData)/; - const trailingCheckerMarker = - /\n\n(?:\/\*\*[^]*?\*\/\n)?(?:export function assertJsonRpcArgs|function requireJsonData)$/; - const extract = (/** @type {string} */ file) => { - const source = readRepoFile(file); - const match = source.match(jsonDataErrorWithChecker); - assert.ok(match, `${file} must define jsonDataError next to its public checker`); - return match[0] - .replace(trailingCheckerMarker, "") - .replace(/\s+/g, " ") - .trim(); - }; - - assert.equal( - extract("runtime/_wdl-do-transport.js"), - extract("do-runtime/protocol.js") - ); -}); - test("DO invoke request-size caps stay aligned across client and server", () => { const runtime = readRepoFile("runtime/_wdl-do-transport.js"); const protocol = readRepoFile("do-runtime/protocol.js"); @@ -2555,32 +2541,40 @@ test("DO owner hints are trusted only from do-runtime headers", () => { assert.match(doRuntime, /doPlatformErrorResponse\(err\)/); }); -test("owner endpoint validation lives in neutral runtime helper", () => { - const endpoint = readRepoFile("runtime/_wdl-owner-endpoint.js"); +test("owner endpoint validation lives in a shared contract owner", () => { + const endpoint = readRepoFile("shared/owner-endpoint.js"); + const adapter = readRepoFile("runtime/_wdl-owner-endpoint.js"); const doTransport = readRepoFile("runtime/_wdl-do-transport.js"); const d1Binding = readRepoFile("runtime/bindings/d1.js"); const controlD1RuntimeClient = readRepoFile("control/d1-runtime-client.js"); const userConfig = readRepoFile("runtime/config-user.capnp"); const systemConfig = readRepoFile("runtime/config-system.capnp"); + const d1Config = readRepoFile("d1-runtime/config.capnp"); const doConfig = readRepoFile("do-runtime/config.capnp"); + const taskIdentity = readRepoFile("shared/task-identity.js"); const tsconfig = readRepoFile("tsconfig.json"); assert.match(endpoint, /export function validOwnerEndpointForService/); assert.match(endpoint, /"d1-runtime": \/\^d1-runtime/); assert.match(endpoint, /"do-runtime": \/\^do-runtime/); + assert.match(endpoint, /function acceptablePrivateIpv4/); + assert.match(adapter, /from "\.\.\/shared\/owner-endpoint\.js"/); assert.match(doTransport, /from "\.\/_wdl-owner-endpoint\.js"/); - assert.match(d1Binding, /from "runtime-owner-endpoint"/); - assert.match(controlD1RuntimeClient, /from "runtime-owner-endpoint"/); + assert.match(d1Binding, /from "shared-owner-endpoint"/); + assert.match(controlD1RuntimeClient, /from "shared-owner-endpoint"/); assert.match(controlD1RuntimeClient, /validOwnerEndpointForService\(owner\.endpoint, 8787, "d1-runtime"\)/); - assert.match(tsconfig, /"runtime-owner-endpoint": \["runtime\/_wdl-owner-endpoint\.js"\]/); + assert.match(tsconfig, /"shared-\*": \["shared\/\*\.js"\]/); + assert.doesNotMatch(taskIdentity, /_TASK_PORT/); + assert.doesNotMatch(d1Config, /D1_TASK_PORT/); + assert.doesNotMatch(doConfig, /DO_TASK_PORT/); for (const config of [userConfig, systemConfig, doConfig]) { - assert.match(config, /name = "runtime-owner-endpoint"/); + assert.match(config, /name = "shared-owner-endpoint"/); assert.match(config, /name = "_wdl-owner-endpoint\.js"/); assert.match(config, /name = "runtime-owner-endpoint-source"/); } for (const config of [userConfig, systemConfig]) { const doOwnerNetwork = /const doOwnerNetworkWorker[\s\S]*?\n\);/.exec(config)?.[0] || ""; - assert.match(doOwnerNetwork, /name = "runtime-owner-endpoint"/); + assert.match(doOwnerNetwork, /name = "shared-owner-endpoint"/); } }); diff --git a/tests/unit/test-helper-style-contracts.test.js b/tests/unit/test-helper-style-contracts.test.js index 458244a..05168d3 100644 --- a/tests/unit/test-helper-style-contracts.test.js +++ b/tests/unit/test-helper-style-contracts.test.js @@ -4,10 +4,26 @@ import { readdirSync } from "node:fs"; import { readRepoFile, repoPath, withoutLineComments } from "../helpers/source-scan.js"; import { + extractCapnpConstBlock, scannedTestFiles, + stripCapnpComments, withoutStringAndTemplateLiterals, } from "../helpers/style-contract-scanner.js"; +test("Cap'n Proto comment stripping preserves hashes inside strings", () => { + const source = ` +const demo :Config = ( + url = "https://example.test/path#fragment", + nested = (value = "escaped \\"# hash"), # trailing comment +); # final comment +`; + const stripped = stripCapnpComments(source); + assert.match(stripped, /path#fragment/); + assert.match(stripped, /escaped \\"# hash/); + assert.doesNotMatch(stripped, /trailing comment|final comment/); + assert.match(extractCapnpConstBlock(source, "demo"), /path#fragment/); +}); + test("test files use moduleDataUrl/repositoryFileUrl instead of raw fs+URL boilerplate", () => { // Skips: // test-helper-style-contracts.test.js — holds tripwire literals as error-message text. diff --git a/tests/unit/version.test.js b/tests/unit/version.test.js index ac481dd..63b3bc4 100644 --- a/tests/unit/version.test.js +++ b/tests/unit/version.test.js @@ -1,5 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { DECLARED_HOSTS_KEY, HOST_DECLARATIONS_SCAN_PATTERN, @@ -15,6 +16,8 @@ import { parseVersion, } from "../../shared/version.js"; +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); + test("formatVersion: integer → v", () => { assert.equal(formatVersion(1), "v1"); assert.equal(formatVersion(42), "v42"); @@ -45,6 +48,12 @@ test("parseVersion: returns null for malformed", () => { assert.equal(parseVersion(1), null); }); +test("parseVersion matches the shared JS/Rust version fixture", () => { + for (const { tag, parsed } of versionFixture.cases) { + assert.equal(parseVersion(tag), parsed, tag); + } +}); + test("bundleKey: composes worker:::v:", () => { assert.equal(bundleKey("demo", "hello", "v1"), "worker:demo:hello:v:1"); assert.equal(bundleKey("demo", "hello", "v42"), "worker:demo:hello:v:42"); diff --git a/tsconfig.json b/tsconfig.json index 8447adc..b461fab 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,7 +22,6 @@ "runtime-dispatch-workflow-*": ["runtime/dispatch/workflow-*.js"], "runtime-load-*": ["runtime/load/*.js"], "runtime-do-transport": ["runtime/_wdl-do-transport.js"], - "runtime-owner-endpoint": ["runtime/_wdl-owner-endpoint.js"], "runtime-owner-hint-cache": ["runtime/_wdl-owner-hint-cache.js"], "runtime-state": ["runtime/runtime.js"], "runtime-*": ["runtime/*.js"], From a26da61a65d4f32f861e9da3e292d51c3a60c76f Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Tue, 14 Jul 2026 18:42:36 +0800 Subject: [PATCH 08/23] Document final runtime and Terraform contracts - Document structural DO RPC serialization and its 1 MiB boundary. - Record Terraform subnet validation as a plan-time upgrade impact. Signed-off-by: Lu Zhang --- CHANGELOG.md | 1 + docs/modules/durable-objects.md | 6 +++++- docs/modules/durable-objects.zh.md | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 057f695..f616be2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, worker-version, scheduler-projection, and workflow-limit fixtures. - Normalized `PLATFORM_DOMAIN` across routing tiers, stopped `/whoami` from advertising the internal `workers.local` fallback as a public namespace URL, and aligned Compose/Kubernetes/test wiring with the consolidated owners. - Removed the unreachable DO inline worker-code test hook and its Terraform switch; do-runtime invoke paths now accept only canonical persisted bundle identities and reject non-canonical or oversized host ids. +- Added Terraform plan-time validation for application subnets: every supplied subnet must belong to the configured VPC, use an RFC1918 or `100.64.0.0/10` IPv4 CIDR, and occupy a distinct Availability Zone. Existing unsupported subnet selections now fail during planning. ## wdl.20260701.1 - 2026-07-08 diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 7fd9626..d91a975 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -65,7 +65,11 @@ Tenant-originated DO fetch bodies are capped at 1 MiB in the runtime facade. The rejects an oversized `Content-Length` before reading, and streamed bodies are read incrementally so the cap is enforced before buffering the full body. DO RPC method names use the JavaScript identifier grammar and are capped at 256 ASCII -bytes by both the runtime client and do-runtime protocol reader. +bytes by both the runtime client and do-runtime protocol reader. RPC arguments are +structural JSON data capped at 1 MiB: finite numbers, strings, booleans, null, dense +arrays, and plain objects are accepted. Serialization does not invoke `toJSON()` hooks; +sparse arrays, circular structures, non-plain objects, and non-JSON values fail before +dispatch. DO invoke envelopes identify persisted bundles by canonical namespace, worker, version, and storage id. They do not accept inline worker source. DO host ids are capped at 512 UTF-8 bytes and use canonical `shardN` suffixes without diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 474709e..36ee9c4 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -35,7 +35,7 @@ Storage cleanup endpoint 是 native facet storage cleanup 和 worker storage cle DO protocol error 使用 `{ error, message, details? }`。不同于 admin HTTP 的 flat additive error shape，DO protocol detail 会嵌套在 `details` 下，因为消费者是 runtime/DO client protocol，不是通用 admin JSON parser。未知 internal exception 仍会降级成安全的 `internal_error` / `Internal error` message。Storage delete-worker 在 partial batch result 时可能返回 HTTP 207 和 `{ ok:false, deleted, errors }`；这是 result envelope，不是 generic JSON error envelope。 Tenant-originated DO fetch body 在 runtime facade 中限制为 1 MiB。Facade 会在读取前拒绝超限的 `Content-Length`，streamed body 会增量读取，因此 limit 会在完整 buffering 前生效。 -DO RPC method name 使用 JavaScript identifier grammar，并由 runtime client 和 do-runtime protocol reader 同时限制为最多 256 ASCII bytes。 +DO RPC method name 使用 JavaScript identifier grammar，并由 runtime client 和 do-runtime protocol reader 同时限制为最多 256 ASCII bytes。RPC 参数是最多 1 MiB 的 structural JSON data：接受 finite number、string、boolean、null、dense array 和 plain object。序列化不会调用 `toJSON()` hook；sparse array、circular structure、non-plain object 和 non-JSON value 会在 dispatch 前失败。 DO invoke envelope 通过 canonical namespace、worker、version 和 storage id 标识 persisted bundle，不接受 inline worker source。 DO host id 最多 512 UTF-8 bytes，并使用不带前导零的 canonical `shardN` suffix。 From c8820e9f3edb8ee1d780056aba1c76f0214bbd63 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Tue, 14 Jul 2026 19:46:57 +0800 Subject: [PATCH 09/23] Harden owner hint caching and dependency checks - Capture Map intrinsics so cached owner metadata cannot be leaked or forged. - Reject empty service target metadata during promote and watched deploy commit. - Clarify alarm handler snapshot semantics and optimistic retry ownership. Signed-off-by: Lu Zhang --- control/handlers/deploy.js | 2 +- control/routing.js | 2 +- do-runtime/alarm-shim-source.js | 4 +- docs/modules/control-auth.md | 2 +- docs/modules/control-auth.zh.md | 2 +- runtime/_wdl-owner-hint-cache.js | 79 +++++++++++++++---- tests/unit/control-deploy-watch.test.js | 100 +++++++++++++++++------- tests/unit/control-routing.test.js | 25 ++++++ tests/unit/runtime-do-client.test.js | 96 +++++++++++++++++++++++ 9 files changed, 262 insertions(+), 50 deletions(-) diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 5b6d883..43e9c97 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -1079,7 +1079,7 @@ async function validateOutgoingRefsForCommit(iso, { outgoingRefs }) { }); } const targetMeta = targetMetas[index]; - if (typeof targetMeta !== "string") { + if (typeof targetMeta !== "string" || targetMeta.length === 0) { throw new DeployAbort(409, "target_drift", { target: { ns: ref.targetNs, diff --git a/control/routing.js b/control/routing.js index 99f5c1e..c8bd7c9 100644 --- a/control/routing.js +++ b/control/routing.js @@ -297,7 +297,7 @@ async function assertOutgoingDependenciesPresent(iso, targetLabel, outgoingRefs) bundleKey(ref.targetNs, ref.targetWorker, ref.targetVersion), "__meta__" ); - if (targetMetaRaw == null) { + if (typeof targetMetaRaw !== "string" || targetMetaRaw.length === 0) { throw new RoutingError( 409, "service_binding_dependency_missing", diff --git a/do-runtime/alarm-shim-source.js b/do-runtime/alarm-shim-source.js index 30d6c47..7023a7d 100644 --- a/do-runtime/alarm-shim-source.js +++ b/do-runtime/alarm-shim-source.js @@ -503,8 +503,8 @@ export function wrapDurableObjectClass(Base, className) { // only the alarm binding here so __WDL_HOST_BINDINGS_WRAPPED can survive // through the two-layer wrapper contract. super(constructorCtx, withoutInternalEnv(env)); - // Resolve through the host wrapper so prototype methods, class fields, - // and accessors retain the real instance as their receiver. + // Resolve once through the host wrapper after construction so prototype + // methods, class fields, and accessors retain the real instance receiver. const tenantFetch = reflectGet(this, "fetch", this); const wrappedCtx = wrapCtx(constructorCtx, alarmBinding, className); try { diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index 2a43b60..27df9c2 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -282,7 +282,7 @@ Auth-specific contract: `ControlAbort` where commit cleanup requires a distinct catch boundary. - `control/json-body.js` owns bounded Control JSON parsing and its `400`/`413` wire mapping. `control/optimistic.js` binds strict `WatchError` recognition and Redis - sessions to the retry loop owned by `shared/owner-lease.js`. + sessions to the retry loop owned by `shared/optimistic-retry.js`. - `control/lib.js::parseBundleMeta()` is the single parser for persisted bundle `__meta__`. It requires a JSON object and accepts an error factory so routing, workflows, delete, deploy, and env-budget paths retain their own catch boundaries. diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index e105529..1664c74 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -138,7 +138,7 @@ Auth 子合同： - Control JSON error 形状是 `{ error, message }`；details 只能 additive。 - Details 可以增加字段，但不能覆盖 `error`、`message` 或 legacy `reason`。Auth reject reason 是 `error` machine code；日志可以把 `reason` 作为诊断上下文。 - `control/errors.js::ControlAbort` 是 Control tier 内的基础 coded abort contract。Routing 和 Auth 保留各自 boundary-specific error class；Deploy 可以在 commit cleanup 需要独立 catch boundary 时 subclass `ControlAbort`。 -- `control/json-body.js` 持有 bounded Control JSON parsing 及其 `400`/`413` wire mapping；`control/optimistic.js` 把 strict `WatchError` recognition 和 Redis session 绑定到 `shared/owner-lease.js` 持有的 retry loop。 +- `control/json-body.js` 持有 bounded Control JSON parsing 及其 `400`/`413` wire mapping；`control/optimistic.js` 把 strict `WatchError` recognition 和 Redis session 绑定到 `shared/optimistic-retry.js` 持有的 retry loop。 - `control/lib.js::parseBundleMeta()` 是 persisted bundle `__meta__` 的唯一 parser。它要求值是 JSON object，并通过 error factory 让 routing、workflows、delete、deploy 和 env-budget 保留各自的 catch boundary。Bundle 缺失的语义仍由具体 use site 持有：当 projection 变更、唯一性证明、lifecycle cleanup、workflow view 或 environment budget 必须消费 metadata 才能产生正确结果时，只要权威 route 或 index 仍指向该 bundle，缺失就 fail closed。Deploy discovery/link preflight 不把缺失归类为 `corrupt_meta`；watched commit 会以 `target_drift` 拒绝缺失的 pinned service target。 - Delegated issue 的 409 reason 有不同 retry 语义：`delegated_issue_busy` 表示 issuer/template lock 清除后可重试；`active_quota_exceeded` 在已有 delegated credential 过期或 revoke 前不应重试；`namespace_collision` 表示 Auth 已耗尽 configured candidate retry budget。 - Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志；除非 endpoint 明确拥有某个 diagnostic response field，否则不进入客户端 body。 diff --git a/runtime/_wdl-owner-hint-cache.js b/runtime/_wdl-owner-hint-cache.js index 33310ce..1595583 100644 --- a/runtime/_wdl-owner-hint-cache.js +++ b/runtime/_wdl-owner-hint-cache.js @@ -1,3 +1,52 @@ +// This module is evaluated before tenant code. Keep cache state inaccessible +// after tenant top-level evaluation mutates the shared isolate realm. +const IntrinsicMap = Map; +const IntrinsicNumber = Number; +const intrinsicReflectApply = Reflect.apply; +const intrinsicMapClear = Map.prototype.clear; +const intrinsicMapDelete = Map.prototype.delete; +const intrinsicMapGet = Map.prototype.get; +const intrinsicMapKeys = Map.prototype.keys; +const intrinsicMapSet = Map.prototype.set; +const intrinsicMapSizeGet = /** @type {(this: Map) => number} */ ( + Object.getOwnPropertyDescriptor(Map.prototype, "size")?.get +); +const intrinsicNumberIsInteger = Number.isInteger; +const intrinsicMapIteratorNext = Object.getPrototypeOf( + intrinsicReflectApply(intrinsicMapKeys, new IntrinsicMap(), []) +).next; + +/** @param {Map} map */ +function mapSize(map) { + return intrinsicReflectApply(intrinsicMapSizeGet, map, []); +} + +/** @param {Map} map @param {unknown} key */ +function mapGet(map, key) { + return intrinsicReflectApply(intrinsicMapGet, map, [key]); +} + +/** @param {Map} map @param {unknown} key @param {unknown} value */ +function mapSet(map, key, value) { + intrinsicReflectApply(intrinsicMapSet, map, [key, value]); +} + +/** @param {Map} map @param {unknown} key */ +function mapDelete(map, key) { + return intrinsicReflectApply(intrinsicMapDelete, map, [key]); +} + +/** @param {Map} map */ +function mapClear(map) { + intrinsicReflectApply(intrinsicMapClear, map, []); +} + +/** @param {Map} map */ +function firstMapKey(map) { + const iterator = intrinsicReflectApply(intrinsicMapKeys, map, []); + return intrinsicReflectApply(intrinsicMapIteratorNext, iterator, []).value; +} + /** * @param {{ * defaultMaxEntries?: number, @@ -9,14 +58,16 @@ export function createOwnerHintCache({ keyFor = (value) => value, } = {}) { /** @type {Map} */ - const entries = new Map(); + const entries = new IntrinsicMap(); /** @type {number | null} */ let maxEntriesForTest = null; function maxEntries() { const override = maxEntriesForTest; - return typeof override === "number" && Number.isInteger(override) && override > 0 - ? Number(override) + return typeof override === "number" && + intrinsicReflectApply(intrinsicNumberIsInteger, IntrinsicNumber, [override]) && + override > 0 + ? override : defaultMaxEntries; } @@ -27,22 +78,22 @@ export function createOwnerHintCache({ } function trim() { - while (entries.size > maxEntries()) { - const oldestKey = entries.keys().next().value; + while (mapSize(entries) > maxEntries()) { + const oldestKey = firstMapKey(entries); if (oldestKey === undefined) break; - entries.delete(oldestKey); + mapDelete(entries, oldestKey); } } return { clearForTest() { - entries.clear(); + mapClear(entries); maxEntriesForTest = null; }, /** @param {number | null} maxEntriesValue */ setMaxEntriesForTest(maxEntriesValue) { - entries.clear(); + mapClear(entries); maxEntriesForTest = maxEntriesValue; }, @@ -50,10 +101,10 @@ export function createOwnerHintCache({ get(value) { const key = keyOf(value); if (key == null) return null; - const hint = entries.get(key); + const hint = mapGet(entries, key); if (!hint) return null; - entries.delete(key); - entries.set(key, hint); + mapDelete(entries, key); + mapSet(entries, key, hint); return hint; }, @@ -64,8 +115,8 @@ export function createOwnerHintCache({ set(value, hint) { const key = keyOf(value); if (key == null) return false; - entries.delete(key); - entries.set(key, hint); + mapDelete(entries, key); + mapSet(entries, key, hint); trim(); return true; }, @@ -74,7 +125,7 @@ export function createOwnerHintCache({ delete(value) { const key = keyOf(value); if (key == null) return false; - return entries.delete(key); + return mapDelete(entries, key); }, }; } diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 5d1046e..5fd228e 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -1463,6 +1463,40 @@ test("commitWithWatch reads workflow defs with own-property discipline", async ( ]); }); +/** @param {any} redis */ +function platformBindingCommitArgs(redis) { + return { + redis, + ns: "tenant-a", + name: "caller", + version: "v1", + prepared: { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "esm" } }, + bindings: { + PLATFORM: { + type: "service", + ns: "__platform__", + service: "platformApi", + version: "v1", + entrypoint: "Api", + }, + }, + }, + normalized: [["worker.js", "export default {}"]], + }, + outgoingRefs: [{ + binding: "PLATFORM", + targetNs: "__platform__", + targetWorker: "platformApi", + targetVersion: "v1", + }], + d1Refs: [], + controlEnv: {}, + }; +} + test("commitWithWatch rejects platform binding target drift before commit", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ @@ -1486,36 +1520,7 @@ test("commitWithWatch rejects platform binding target drift before commit", asyn }; await assert.rejects( - () => commitWithWatch({ - redis, - ns: "tenant-a", - name: "caller", - version: "v1", - prepared: { - meta: { - mainModule: "worker.js", - modules: { "worker.js": { type: "esm" } }, - bindings: { - PLATFORM: { - type: "service", - ns: "__platform__", - service: "platformApi", - version: "v1", - entrypoint: "Api", - }, - }, - }, - normalized: [["worker.js", "export default {}"]], - }, - outgoingRefs: [{ - binding: "PLATFORM", - targetNs: "__platform__", - targetWorker: "platformApi", - targetVersion: "v1", - }], - d1Refs: [], - controlEnv: {}, - }), + () => commitWithWatch(platformBindingCommitArgs(redis)), (err) => { const deployErr = /** @type {any} */ (err); assert.equal(deployErr.code, "target_drift"); @@ -1532,3 +1537,38 @@ test("commitWithWatch rejects platform binding target drift before commit", asyn assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("worker:__platform__:platformApi:v:1")); assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("worker-delete-lock:__platform__:platformApi")); }); + +test("commitWithWatch rejects empty platform binding target metadata before commit", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ + ["routes:__platform__", { platformApi: "v1" }], + ["worker:__platform__:platformApi:v:1", { "__meta__": "" }], + ]); + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta = null; + /** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys = []; + /** @type {any} */ (globalThis).__controlDeployTestState.execFailures = 0; + + const redis = { + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + await assert.rejects( + () => commitWithWatch(platformBindingCommitArgs(redis)), + (err) => { + const deployErr = /** @type {any} */ (err); + assert.equal(deployErr.code, "target_drift"); + assert.equal(deployErr.details.target.ns, "__platform__"); + assert.equal(deployErr.details.target.worker, "platformApi"); + assert.equal(deployErr.details.target.version, "v1"); + assert.equal(deployErr.details.target.reason, "bundle_missing"); + return true; + } + ); + + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta, null); + assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("worker:__platform__:platformApi:v:1")); + assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("worker-delete-lock:__platform__:platformApi")); +}); diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index 9200c67..abb972b 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -292,6 +292,31 @@ test("promoteWithRoutes watches and rejects missing service-binding target bundl assert.equal(redis.state.hashes.has(productionBundleKey("demo", "worker", "v2")), false); }); +test("promoteWithRoutes rejects empty service-binding target metadata", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { + bindings: { + TARGET: { + type: "service", + ns: "other", + service: "api", + version: "v3", + }, + }, + }); + redis.state.hashes.set(productionBundleKey("other", "api", "v3"), { __meta__: "" }); + + await assert.rejects( + promoteWithRoutes(redis, "demo", "worker", "v1"), + (err) => { + const shaped = assertRoutingErrorShape(err, 409, "service_binding_dependency_missing"); + assert.equal(shaped.details.broken_dependency.targetNs, "other"); + return true; + } + ); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, undefined); +}); + test("promoteWithRoutes batches service-binding dependency watches", async () => { const redis = makeRedis(); seedBundle(redis, "v1", { diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index ffae137..2c92d48 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -1714,6 +1714,102 @@ test("DurableObjectNamespace direct backend reuses learned owner hints", async ( assert.equal(ownerCalls[1].url, "http://do-runtime-a:8788/internal/do/invoke"); }); +function hostileOwnerHintCacheFixture() { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + const backend = { + fetch: makeRecordingFetch(routerCalls, { response: doOwnerHintResponse() }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { response: new Response("owner-ok") }), + }; + const props = { + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }; + return { + ns: new DurableObjectNamespace(props, { backend, ownerNetwork }), + ownerCalls, + props, + routerCalls, + }; +} + +test("DurableObjectNamespace owner hint cache ignores tenant-patched Map#get", async () => { + const { ns, ownerCalls, props, routerCalls } = hostileOwnerHintCacheFixture(); + const objectName = "room-hostile-map-get"; + const cacheKey = `${props.doStorageId}:${props.className}:${objectName}`; + const originalMapGet = Map.prototype.get; + let hostileGetCalls = 0; + + const response = await withMockedProperty( + Map.prototype, + "get", + /** @this {Map} */ + function hostileMapGet(key) { + if (key === cacheKey) { + hostileGetCalls += 1; + return { + ownerKey: "forged-owner", + taskId: "forged-task", + endpoint: "10.0.0.5:8788", + generation: 1, + }; + } + return Reflect.apply(originalMapGet, this, [key]); + }, + () => ns.get(ns.idFromName(objectName)).fetch("https://demo.workers.example/send") + ); + + assert.equal(await response.text(), "owner-ok"); + assert.equal(hostileGetCalls, 0); + assert.equal(routerCalls.length, 1); + assert.equal(ownerCalls.length, 1); + assert.equal(ownerCalls[0].url, "http://do-runtime-a:8788/internal/do/invoke"); +}); + +test("DurableObjectNamespace owner hint cache hides hints from tenant-patched Map methods", async () => { + const { ns, ownerCalls, props, routerCalls } = hostileOwnerHintCacheFixture(); + const objectName = "room-hostile-map-set"; + const cacheKey = `${props.doStorageId}:${props.className}:${objectName}`; + const originalMapDelete = Map.prototype.delete; + const originalMapSet = Map.prototype.set; + let hostileDeleteCalls = 0; + let leakedHint = null; + + const response = await withMockedProperty( + Map.prototype, + "delete", + /** @this {Map} */ + function hostileMapDelete(key) { + if (key === cacheKey) hostileDeleteCalls += 1; + return Reflect.apply(originalMapDelete, this, [key]); + }, + () => withMockedProperty( + Map.prototype, + "set", + /** @this {Map} */ + function hostileMapSet(key, value) { + if (key === cacheKey) leakedHint = value; + return Reflect.apply(originalMapSet, this, [key, value]); + }, + () => ns.get(ns.idFromName(objectName)).fetch("https://demo.workers.example/send") + ) + ); + + assert.equal(await response.text(), "owner-ok"); + assert.equal(hostileDeleteCalls, 0); + assert.equal(leakedHint, null); + assert.equal(routerCalls.length, 1); + assert.equal(ownerCalls.length, 1); +}); + test("DurableObjectNamespace RPC reuses learned owner hints", async () => { /** @type {any[]} */ const routerCalls = []; From dee044c145e095bfb4d9b57de656daadf414c31c Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Tue, 14 Jul 2026 20:46:38 +0800 Subject: [PATCH 10/23] Retry routing reads after active-version drift - Recheck watched routes when dependent metadata disappears before EXEC. - Retry promote, platform alias, and bump races while preserving stable corruption failures. - Cover supported active-version transitions and projection cleanup. Signed-off-by: Lu Zhang --- control/routing.js | 19 ++++ tests/helpers/load-control-routing.js | 3 + tests/unit/control-routing.test.js | 131 ++++++++++++++++++++++++++ 3 files changed, 153 insertions(+) diff --git a/control/routing.js b/control/routing.js index c8bd7c9..d1c3297 100644 --- a/control/routing.js +++ b/control/routing.js @@ -39,6 +39,7 @@ import { diffCrons, nextFireMs, slotMsFor } from "control-cron-index"; import { isReservedNs, ROUTES_ALLOWED_RESERVED_NS } from "shared-ns-pattern"; import { PLATFORM_TIER_RESERVED_NS } from "shared-auth-roles"; import { queueConsumerKey } from "shared-queue-keys"; +import { WatchError } from "shared-redis"; import { computeAffectedHosts, computeNsHostDeltas, @@ -155,9 +156,18 @@ function parseCronMeta(raw, logContext) { async function readMeta(iso, ns, workerName, version) { if (!version) return null; const raw = await iso.hGet(bundleKey(ns, workerName, version), "__meta__"); + if (raw == null) await retryIfRouteChanged(iso, ns, workerName, version); return routingBundleMeta(ns, workerName, version, raw); } +/** @param {RedisIso} iso @param {string} ns @param {string} workerName @param {string} version */ +async function retryIfRouteChanged(iso, ns, workerName, version) { + // A watched route can move after its snapshot but before the dependent + // metadata read. Retry that race; a stable route still fails as corruption. + const currentVersion = await iso.hGet(routesKey(ns), workerName); + if (currentVersion !== version) throw new WatchError(); +} + /** * @param {string} ns * @param {string} workerName @@ -429,6 +439,14 @@ async function assertPlatformAsAvailable(iso, ns, workerName, newExports) { for (let i = 0; i < routeEntries.length; i += 1) { const [otherWorker, otherVersion] = routeEntries[i]; const rawMeta = rawMetas[i]; + if (rawMeta == null) { + await retryIfRouteChanged( + iso, + otherNs, + otherWorker, + /** @type {string} */ (otherVersion) + ); + } const otherMeta = routingBundleMeta( otherNs, otherWorker, @@ -718,6 +736,7 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) const srcMetaRaw = await iso.hGet(srcKey, "__meta__"); if (srcMetaRaw == null) { + await retryIfRouteChanged(iso, ns, workerName, currentVersion); throw new RoutingError( 500, "bundle_copy_failed", diff --git a/tests/helpers/load-control-routing.js b/tests/helpers/load-control-routing.js index 66b5f3d..ac4aaf0 100644 --- a/tests/helpers/load-control-routing.js +++ b/tests/helpers/load-control-routing.js @@ -6,6 +6,7 @@ import { } from "./load-shared-module.js"; import { controlSharedStubUrl } from "./control-shared-stub.js"; import { compileControlGraph } from "./load-control-lib.js"; +import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; const { sharedNsUrl, @@ -22,6 +23,7 @@ const { } = await compileControlGraph(); const controlSharedUrl = controlSharedStubUrl(); +const sharedRedisUrl = sharedRedisStubUrl(); export const CONTROL_ROUTING_TEST_URL = moduleDataUrl(applyModuleReplacements(readRepositoryFile("control/routing.js"), [ ...importSpecifierReplacements({ @@ -36,6 +38,7 @@ export const CONTROL_ROUTING_TEST_URL = moduleDataUrl(applyModuleReplacements(re "shared-ns-pattern": sharedNsUrl, "shared-auth-roles": sharedAuthRolesUrl, "shared-queue-keys": sharedQueueKeysUrl, + "shared-redis": sharedRedisUrl, "control-routing-route-plan": routePlanUrl, }), ])); diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index abb972b..3779b91 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -189,6 +189,98 @@ test("promoteWithRoutes rejects a platform route whose active metadata is missin assert.equal(redis.state.hashes.get("routes:__platform__")?.api, undefined); }); +test("promoteWithRoutes retries from the new active when the prior bundle is deleted", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { routes: [], queueConsumers: [] }); + seedBundle(redis, "v2", { routes: [], queueConsumers: [] }); + seedBundle(redis, "v3", { routes: [], queueConsumers: [consumerWithoutOptions] }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + + const originalSession = redis.session.bind(redis); + let attempts = 0; + redis.session = async (fn) => { + attempts += 1; + return await originalSession(async (iso) => { + if (attempts !== 1) return await fn(iso); + const originalHGet = iso.hGet.bind(iso); + let raced = false; + return await fn({ + ...iso, + async hGet(key, field) { + const value = await originalHGet(key, field); + if (!raced && key === "routes:demo" && field === "worker") { + raced = true; + redis.state.hashes.set("routes:demo", { worker: "v3" }); + redis.state.hashes.set("queue-consumer:demo:jobs", { + worker: "worker", + version: "v3", + max_batch_size: "5", + max_batch_timeout_ms: "2000", + max_retries: "3", + }); + redis.state.sets.set( + "queue:index:consumers", + new Set(["queue-consumer:demo:jobs"]) + ); + redis.state.hashes.delete(productionBundleKey("demo", "worker", "v1")); + } + return value; + }, + }); + }); + }; + + await promoteWithRoutes(redis, "demo", "worker", "v2"); + + assert.equal(attempts, 2); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v2"); + assert.equal(redis.state.hashes.has("queue-consumer:demo:jobs"), false); + assert.equal( + redis.state.sets.get("queue:index:consumers")?.has("queue-consumer:demo:jobs") ?? false, + false + ); +}); + +test("promoteWithRoutes retries when a platform route is deleted during as validation", async () => { + const redis = makeRedis(); + redis.state.hashes.set(productionBundleKey("__platform__", "api", "v1"), { + __meta__: JSON.stringify({ exports: [{ entrypoint: "default", as: "demo" }] }), + }); + redis.state.hashes.set(productionBundleKey("__platform__", "other", "v2"), { + __meta__: JSON.stringify({ exports: [{ entrypoint: "default", as: "other" }] }), + }); + redis.state.hashes.set("routes:__platform__", { other: "v2" }); + + const originalSession = redis.session.bind(redis); + let attempts = 0; + redis.session = async (fn) => { + attempts += 1; + return await originalSession(async (iso) => { + if (attempts !== 1) return await fn(iso); + const originalHGetMany = iso.hGetMany.bind(iso); + let raced = false; + return await fn({ + ...iso, + async hGetMany(pairs) { + if (!raced && pairs.some(([key]) => + key === productionBundleKey("__platform__", "other", "v2") + )) { + raced = true; + delete redis.state.hashes.get("routes:__platform__")?.other; + redis.state.hashes.delete(productionBundleKey("__platform__", "other", "v2")); + } + return await originalHGetMany(pairs); + }, + }); + }); + }; + + await promoteWithRoutes(redis, "__platform__", "api", "v1"); + + assert.equal(attempts, 2); + assert.equal(redis.state.hashes.get("routes:__platform__")?.api, "v1"); +}); + test("promoteWithRoutes rejects non-object candidate bundle metadata", async () => { const redis = makeRedis(); redis.state.hashes.set(productionBundleKey("demo", "worker", "v1"), { @@ -374,6 +466,45 @@ test("bumpActiveAndPromote also rewrites full queue consumer projection", async }); }); +test("bumpActiveAndPromote retries when active changes before source metadata read", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", {}); + seedBundle(redis, "v3", {}); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.strings.set("worker:demo:worker:next_version", "3"); + + const originalSession = redis.session.bind(redis); + let attempts = 0; + redis.session = async (fn) => { + attempts += 1; + return await originalSession(async (iso) => { + if (attempts !== 1) return await fn(iso); + const originalHGet = iso.hGet.bind(iso); + let raced = false; + return await fn({ + ...iso, + async hGet(key, field) { + const value = await originalHGet(key, field); + if (!raced && key === "routes:demo" && field === "worker") { + raced = true; + redis.state.hashes.set("routes:demo", { worker: "v3" }); + redis.state.hashes.delete(productionBundleKey("demo", "worker", "v1")); + } + return value; + }, + }); + }); + }; + + const result = await bumpActiveAndPromote(redis, "demo", "worker"); + + assert.equal(attempts, 2); + assert.equal(result.previousVersion, "v3"); + assert.equal(result.version, "v4"); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v4"); + assert.ok(redis.state.hashes.has(productionBundleKey("demo", "worker", "v4"))); +}); + test("promoteWithRoutes rejects missing D1 dependency databases", async () => { const redis = makeRedis(); seedBundle(redis, "v1", { From efba416dd8200e7822af792916f749f1a064a1a3 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Wed, 15 Jul 2026 19:33:19 +0800 Subject: [PATCH 11/23] Harden lifecycle and runtime contracts Fence delete and DO ownership transitions, propagate invocation-local request context, bound diagnostics and response bodies, align queue and version contracts, update SigV4, and pin shutdown windows across deployment targets. Signed-off-by: Lu Zhang --- CHANGELOG.md | 2 + control/errors.js | 138 ++- control/handlers/delete.js | 247 +++-- control/handlers/deploy.js | 42 +- control/handlers/hosts.js | 6 + control/handlers/ns-secrets.js | 52 +- control/handlers/promote.js | 7 +- control/handlers/secret-put.js | 17 +- control/handlers/versions.js | 44 +- control/handlers/worker-secrets.js | 69 +- control/handlers/workflows.js | 22 +- control/lib.js | 13 +- control/package.json | 2 +- control/shared.js | 37 +- deploy/kubernetes/base/scheduler.yaml | 1 + deploy/kubernetes/base/workflows.yaml | 1 + do-runtime/actor.js | 37 +- do-runtime/alarm-shim-source.js | 49 +- do-runtime/load-code-budget.js | 14 +- do-runtime/owner-registry.js | 44 +- do-runtime/protocol.js | 26 +- docker-compose.yml | 2 + docs/modules/control-auth.md | 20 +- docs/modules/control-auth.zh.md | 6 +- docs/modules/durable-objects.md | 19 +- docs/modules/durable-objects.zh.md | 5 +- docs/modules/infra.md | 5 + docs/modules/infra.zh.md | 1 + docs/modules/log-tail-observability.md | 5 + docs/modules/log-tail-observability.zh.md | 1 + docs/modules/queues-cron.md | 8 +- docs/modules/queues-cron.zh.md | 4 +- docs/modules/runtime.md | 17 +- docs/modules/runtime.zh.md | 5 +- docs/redis-key-layout.md | 4 +- docs/redis-key-layout.zh.md | 2 +- docs/source-map.md | 2 +- docs/source-map.zh.md | 2 +- package-lock.json | 8 +- runtime/_wdl-do-transport.js | 29 + runtime/load/code-budget.js | 15 + runtime/load/wrapper-generate.js | 178 ++-- runtime/r2-client.js | 17 +- rust/common/src/queue_keys.rs | 12 +- rust/common/src/version.rs | 17 +- rust/scheduler/src/queue/registry.rs | 144 +-- rust/scheduler/src/runtime_client.rs | 52 +- rust/workflows/src/api/active_export.rs | 4 +- shared/bounded-body.js | 6 +- shared/queue-keys.js | 13 +- shared/respond.js | 2 +- shared/vendor/aws-sigv4.js | 907 ++++++++++++------ shared/version.js | 35 + .../modules/compute/scheduler_service.tf | 2 +- .../modules/compute/workflows_service.tf | 2 +- test-workers/do-rpc/src/index.js | 34 + tests/fixtures/queue-key-parse.json | 3 +- tests/helpers/control-shared-stub.js | 6 +- tests/helpers/load-do-host-actor.js | 9 + tests/helpers/load-do-owner-registry.js | 4 + tests/integration/delete-api.test.js | 18 +- .../integration/durable-objects-core.test.js | 46 + tests/integration/helpers/stack.js | 2 +- tests/unit/aws-sigv4.test.js | 104 +- tests/unit/bounded-body.test.js | 28 + tests/unit/control-delete-handler.test.js | 462 ++++++++- tests/unit/control-deploy-watch.test.js | 218 ++++- tests/unit/control-handlers-workflows.test.js | 114 ++- tests/unit/control-lib.test.js | 28 + .../control-secret-envelope-handlers.test.js | 88 +- tests/unit/control-shared.test.js | 120 ++- tests/unit/do-alarm-shim.test.js | 94 +- tests/unit/do-owner-registry.test.js | 66 +- tests/unit/do-runtime-actor.test.js | 75 +- tests/unit/do-runtime-load.test.js | 49 +- tests/unit/do-runtime-protocol.test.js | 25 +- tests/unit/respond.test.js | 31 + tests/unit/runtime-do-client.test.js | 278 +++++- tests/unit/runtime-load.test.js | 102 +- tests/unit/runtime-r2-client.test.js | 79 ++ tests/unit/runtime-wrapper-generate.test.js | 178 +++- tests/unit/style-contracts.test.js | 14 +- tests/unit/version.test.js | 27 + 83 files changed, 3788 insertions(+), 935 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f616be2..33205ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ - Normalized `PLATFORM_DOMAIN` across routing tiers, stopped `/whoami` from advertising the internal `workers.local` fallback as a public namespace URL, and aligned Compose/Kubernetes/test wiring with the consolidated owners. - Removed the unreachable DO inline worker-code test hook and its Terraform switch; do-runtime invoke paths now accept only canonical persisted bundle identities and reject non-canonical or oversized host ids. - Added Terraform plan-time validation for application subnets: every supplied subnet must belong to the configured VPC, use an RFC1918 or `100.64.0.0/10` IPv4 CIDR, and occupy a distinct Availability Zone. Existing unsupported subnet selections now fail during planning. +- Aligned scheduler and Workflows Compose, Kubernetes, and ECS stop windows with their 25-second application drain so rollout and scale-in do not terminate in-flight work before the configured drain completes. +- Upgraded `@wdl-dev/aws-sigv4` to 3.0.1; WDL's S3-only URL-input paths preserve their existing signatures while adopting stable request snapshots, fail-closed redirects, lowercase region validation, and non-blocking response cleanup. ## wdl.20260701.1 - 2026-07-08 diff --git a/control/errors.js b/control/errors.js index ab1a1d8..12d1ba3 100644 --- a/control/errors.js +++ b/control/errors.js @@ -1,4 +1,5 @@ import { jsonError } from "shared-respond"; +import { errorMessage } from "shared-errors"; // Control owns this base abort contract. Routing and auth remain separate; // deploy subclasses it where commit cleanup needs its own catch boundary. @@ -8,6 +9,52 @@ function isRecord(value) { return value !== null && typeof value === "object" && !Array.isArray(value); } +const SERVER_DIAGNOSTIC_DETAIL_KEYS = new Set([ + "cause", + "detail", + "error_detail", + "stack", + "stage", +]); +const CODED_ERROR_LOG_CONTEXT_RESERVED_KEYS = new Set([ + ...SERVER_DIAGNOSTIC_DETAIL_KEYS, + "code", + "error_message", + "message", + "metadata_version", + "reason", + "status", + "version", +]); +const CONTROL_LOG_STRING_MAX_CHARS = 2048; +const CONTROL_LOG_TRUNCATION_SUFFIX = "..."; + +/** @param {string} value */ +function boundedControlLogString(value) { + if (value.length <= CONTROL_LOG_STRING_MAX_CHARS) return value; + const prefixLength = CONTROL_LOG_STRING_MAX_CHARS - CONTROL_LOG_TRUNCATION_SUFFIX.length; + return `${value.slice(0, prefixLength)}${CONTROL_LOG_TRUNCATION_SUFFIX}`; +} + +/** @param {unknown} value */ +function codedErrorLogContext(value) { + if (!isRecord(value)) return {}; + return Object.fromEntries( + Object.entries(value).filter(([key]) => !CODED_ERROR_LOG_CONTEXT_RESERVED_KEYS.has(key)) + ); +} + +/** + * @param {number} status + * @param {Record} details + */ +function publicErrorDetails(status, details) { + if (status < 500) return details; + return Object.fromEntries( + Object.entries(details).filter(([key]) => !SERVER_DIAGNOSTIC_DETAIL_KEYS.has(key)) + ); +} + export class ControlAbort extends Error { /** * @param {number} status @@ -22,6 +69,55 @@ export class ControlAbort extends Error { } } +/** + * Return the bounded diagnostics that a server-side coded error may add to a + * structured log after those fields have been removed from the wire response. + * + * @param {ControlAbort} err + * @returns {Record} + */ +export function controlAbortLogDetails(err) { + if (err.status < 500 || !isRecord(err.details)) return {}; + /** @type {Record} */ + const out = {}; + if (typeof err.details.version === "string") { + out.metadata_version = boundedControlLogString(err.details.version); + } + if (typeof err.details.stage === "string") { + out.stage = boundedControlLogString(err.details.stage); + } + const detail = typeof err.details.error_detail === "string" + ? err.details.error_detail + : err.details.detail; + if (typeof detail === "string") { + out.error_detail = boundedControlLogString(detail); + } + return out; +} + +/** + * @param {unknown} err + * @param {string} [fallbackCode] + * @param {{ errorDetail?: string, context?: Record }} [diagnostics] + * @returns {{ status: number, reason: string, error_message: string, [key: string]: unknown }} + */ +export function codedErrorLogFields(err, fallbackCode = "internal_error", diagnostics = {}) { + const record = isRecord(err) ? err : {}; + const status = typeof record.status === "number" ? record.status : 500; + const reason = typeof record.code === "string" && record.code ? record.code : fallbackCode; + const message = typeof record.message === "string" ? record.message : errorMessage(err); + return { + ...codedErrorLogContext(diagnostics.context), + status, + reason: boundedControlLogString(reason), + error_message: boundedControlLogString(message), + ...(err instanceof ControlAbort ? controlAbortLogDetails(err) : {}), + ...(typeof diagnostics.errorDetail === "string" + ? { error_detail: boundedControlLogString(diagnostics.errorDetail) } + : {}), + }; +} + // Domain errors flow through jsonError() so details cannot shadow the // top-level wire contract. /** @@ -35,17 +131,51 @@ export function codedErrorResponse(err, fallbackCode = "internal_error", extraDe const status = typeof record.status === "number" ? record.status : 500; const code = typeof record.code === "string" && record.code ? record.code : fallbackCode; const recordMessage = typeof record.message === "string" && record.message ? record.message : undefined; - const message = err instanceof ControlAbort - ? (typeof err.details.message === "string" && err.details.message) || err.message || code - : recordMessage || (typeof details.message === "string" && details.message) || code; + let message = "Internal error"; + if (status < 500) { + message = err instanceof ControlAbort + ? (typeof err.details.message === "string" && err.details.message) || err.message || code + : recordMessage || (typeof details.message === "string" && details.message) || code; + } return jsonError( status, code, message, - { ...details, ...extraDetails }, + publicErrorDetails(status, { ...details, ...extraDetails }), ); } +/** + * Log a secret-envelope provider failure internally while returning only the + * stable coded 503 contract to the caller. + * + * @param {{ + * err: import("shared-secret-envelope").SecretEnvelopeError, + * log: (level: string, event: string, fields: Record) => void, + * event: string, + * fields: Record, + * responseDetails?: Record, + * }} args + */ +export function secretEnvelopeErrorResponse({ + err, + log, + event, + fields, + responseDetails = {}, +}) { + const diagnostic = errorMessage(err); + log("error", event, { + ...fields, + ...codedErrorLogFields( + { status: 503, code: err.code, message: diagnostic }, + err.code, + { errorDetail: diagnostic } + ), + }); + return codedErrorResponse({ status: 503, code: err.code }, err.code, responseDetails); +} + /** * @param {ControlAbort} err * @param {Record} [extraDetails] diff --git a/control/handlers/delete.js b/control/handlers/delete.js index bb9b550..1af3512 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -7,16 +7,15 @@ import { jsonResponse, jsonError, requireControlLog, requireControlRedis, errMessage, - acquireDeleteLock, releaseDeleteLock, + acquireDeleteLock, releaseDeleteLock, renewDeleteLock, assertWorkflowDeleteAllowed, cleanupDoAlarmsForWorker, buildS3CleanupTaskId, recordCleanupIntentOrWarn, - ControlAbort, controlAbortResponse, + ControlAbort, codedErrorLogFields, controlAbortResponse, withOptimisticRetries, ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL, PATTERNS_CHANNEL, } from "control-shared"; import { workerVersionsKey, referrersKey, - deleteLockKey, doObjectRegistryKey, doOwnerScopeScanPatternForStorage, doStorageIdKey, @@ -25,7 +24,14 @@ import { bundleAssetPrefix, parseBundleMeta, } from "control-lib"; -import { bundleKey, hostsKey, patternsKey, routesKey } from "shared-version"; +import { + WHOLE_DELETE_LOCK_KIND, + bundleKey, + deleteLockKey, + hostsKey, + patternsKey, + routesKey, +} from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; import { decodeBulk, WatchError } from "shared-redis"; import { queueConsumerScanPrefix } from "shared-queue-keys"; @@ -91,6 +97,14 @@ function throwWholeDeleteContention(ns, name) { }); } +/** @param {string} ns @param {string} name @returns {never} */ +function throwDeleteLockExpired(ns, name) { + throw new WholeDeleteError(409, "deleting", { + namespace: ns, name, + message: "worker delete lock expired; retry the request", + }); +} + // Raised when collection or an under-WATCH check observes state that // changed across reads; callers retry from a fresh snapshot. class DriftSignal extends Error { @@ -112,7 +126,7 @@ export async function handle({ request, env: _env, url, ns, name, principal, req // client's pipe closes cleanly. await request.text().catch(() => {}); - const lockToken = await acquireDeleteLock(redis, ns, name); + const lockToken = await acquireDeleteLock(redis, ns, name, WHOLE_DELETE_LOCK_KIND); if (!lockToken) { log("warn", "worker_delete_rejected", { request_id: requestId, @@ -128,16 +142,15 @@ export async function handle({ request, env: _env, url, ns, name, principal, req let outcome; try { outcome = await executeWholeDelete({ - redis, ns, name, principal, requestId, log, + redis, ns, name, principal, requestId, log, lockToken, }); } catch (err) { if (err instanceof ControlAbort) { - log("warn", "worker_delete_rejected", { + log(err.status >= 500 ? "error" : "warn", "worker_delete_rejected", { request_id: requestId, namespace: ns, worker: name, - status: err.status, - reason: err.code, + ...codedErrorLogFields(err), }); return controlAbortResponse(err); } @@ -216,7 +229,11 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { let workflowBlocker = null; try { collected = await withOptimisticRetries( - async () => await collectDeleteInputs({ redis, ns, name }), + async () => { + const snapshot = await collectDeleteInputs({ redis, ns, name }); + await assertDryRunActiveVersionRetained(redis, snapshot); + return snapshot; + }, { attempts: MAX_DELETE_ATTEMPTS, isRetryableError: (err) => err instanceof DriftSignal, @@ -226,15 +243,21 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { await assertWorkflowDeleteAllowed({ ns, worker: name }); } catch (err) { if (err instanceof WholeDeleteError) { - log("warn", "worker_dry_run_rejected", { + log(err.status >= 500 ? "error" : "warn", "worker_dry_run_rejected", { request_id: requestId, namespace: ns, worker: name, - status: err.status, reason: err.code, + ...codedErrorLogFields(err), }); return controlAbortResponse(err, { dryRun: true }); } if (err instanceof ControlAbort && err.code === "workflow_instances_active") { workflowBlocker = err; } else if (err instanceof ControlAbort) { + log(err.status >= 500 ? "error" : "warn", "worker_dry_run_rejected", { + request_id: requestId, + namespace: ns, + worker: name, + ...codedErrorLogFields(err), + }); return controlAbortResponse(err, { dryRun: true }); } else { throw err; @@ -298,12 +321,13 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { } /** - * @param {{ redis: RedisClient, ns: string, name: string, principal?: AccessPrincipal | null, requestId: string, log: ControlLogger }} args + * @param {{ redis: RedisClient, ns: string, name: string, principal?: AccessPrincipal | null, requestId: string, log: ControlLogger, lockToken: string }} args * @returns {Promise} */ -async function executeWholeDelete({ redis, ns, name, principal, requestId, log }) { +async function executeWholeDelete({ redis, ns, name, principal, requestId, log, lockToken }) { return await withOptimisticRetries(async () => { const collected = await collectDeleteInputs({ redis, ns, name }); + assertActiveVersionRetained(collected); let workflowBlocker = null; try { await assertWorkflowDeleteAllowed({ ns, worker: name, allowCleanup: true }); @@ -323,7 +347,10 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log } throw workflowBlocker; } if (!hasWorkerLifecycle) { - await deleteResidualDoRedis(redis, collected); + if (!await renewDeleteLock(redis, ns, name, lockToken)) { + throwDeleteLockExpired(ns, name); + } + await deleteResidualDoRedis(redis, collected, lockToken); await cleanupDoAlarmsOrWarn({ ns, worker: name, doStorageId: collected.doStorageId, requestId, log, }); @@ -355,9 +382,12 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log } blockers, }); } + if (!await renewDeleteLock(redis, ns, name, lockToken)) { + throwDeleteLockExpired(ns, name); + } const result = await runSessionEXEC({ - redis, ns, name, principal, requestId, collected, + redis, ns, name, principal, requestId, collected, lockToken, }); await cleanupDoAlarmsOrWarn({ ns, worker: name, doStorageId: collected.doStorageId, requestId, log, @@ -371,6 +401,33 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log } }); } +/** @param {DeleteInputs} collected */ +function assertActiveVersionRetained(collected) { + if (!collected.activeVersion || collected.retainedVersions.includes(collected.activeVersion)) return; + throw new WholeDeleteError(409, "projection_drift", { + namespace: collected.ns, + name: collected.name, + active_version: collected.activeVersion, + reason: "active_not_in_worker_versions", + }); +} + +/** @param {RedisClient} redis @param {DeleteInputs} collected */ +async function assertDryRunActiveVersionRetained(redis, collected) { + if (!collected.activeVersion || collected.retainedVersions.includes(collected.activeVersion)) return; + const [retainedVersions, activeVersion] = await Promise.all([ + redis.zRange(workerVersionsKey(collected.ns, collected.name), 0, -1), + redis.hGet(routesKey(collected.ns), collected.name), + ]); + if ( + activeVersion !== collected.activeVersion || + !arraysShallowEqual(retainedVersions, collected.retainedVersions) + ) { + throw new DriftSignal("active version or retained versions changed during dry-run"); + } + assertActiveVersionRetained(collected); +} + /** @param {DeleteInputs} collected */ function describeDoStorageRetention(collected) { return { @@ -386,15 +443,14 @@ function describeDoStorageRetention(collected) { */ async function scanDoOwnerKeys(redis, doStorageId) { if (!doStorageId) return []; - /** @type {string[]} */ - const keys = []; + const keys = new Set(); let cursor = "0"; do { const [next, found] = await redis.scan(cursor, doOwnerScopeScanPatternForStorage(doStorageId), 100); - keys.push(...found); + for (const key of found) keys.add(key); cursor = next; } while (cursor !== "0"); - return keys; + return [...keys]; } /** @@ -403,36 +459,73 @@ async function scanDoOwnerKeys(redis, doStorageId) { * @param {string} name */ async function scanQueueConsumerKeysForWorker(redis, ns, name) { - /** @type {string[]} */ - const queueConsumerKeys = []; + const queueConsumerKeys = new Set(); const prefix = queueConsumerScanPrefix(ns); let cursor = "0"; do { - const [next, keys] = await redis.scan(cursor, `${prefix}*`, 100); + const [next, found] = await redis.scan(cursor, `${prefix}*`, 100); + const keys = [...new Set(found)]; const workers = typeof redis.hGetMany === "function" ? await redis.hGetMany(keys.map((k) => [k, "worker"])) : await mapConcurrent(keys, DELETE_COLLECT_READ_CONCURRENCY, (k) => redis.hGet(k, "worker") ); for (let i = 0; i < keys.length; i += 1) { - if (workers[i] === name) queueConsumerKeys.push(keys[i]); + if (workers[i] === name) queueConsumerKeys.add(keys[i]); } cursor = next; } while (cursor !== "0"); - return queueConsumerKeys; + return [...queueConsumerKeys]; } /** * @param {RedisClient} redis * @param {DeleteInputs} collected + * @param {string} lockToken */ -async function deleteResidualDoRedis(redis, collected) { - if (collected.doOwnerKeys.length) { - await redis.del(...collected.doOwnerKeys); - } - if (collected.doStorageId) { - await redis.del(doStorageIdKey(collected.ns, collected.name)); - } +async function deleteResidualDoRedis(redis, collected, lockToken) { + await redis.session(async (iso) => { + const lockKey = deleteLockKey(collected.ns, collected.name); + const storageKey = doStorageIdKey(collected.ns, collected.name); + await iso.watch( + lockKey, + routesKey(collected.ns), + workerVersionsKey(collected.ns, collected.name), + workerSecretsKey(collected.ns, collected.name), + storageKey, + ...collected.doOwnerKeys, + ); + + if (await iso.get(lockKey) !== lockToken) { + await iso.unwatch(); + throwDeleteLockExpired(collected.ns, collected.name); + } + const activeVersion = await iso.hGet(routesKey(collected.ns), collected.name); + const retainedVersions = await iso.zRange(workerVersionsKey(collected.ns, collected.name), 0, -1); + const hasWorkerSecrets = (await iso.exists(workerSecretsKey(collected.ns, collected.name))) > 0; + if (activeVersion || retainedVersions.length > 0 || hasWorkerSecrets) { + await iso.unwatch(); + throw new DriftSignal("worker lifecycle appeared during residual cleanup"); + } + if (await iso.get(storageKey) !== collected.doStorageId) { + await iso.unwatch(); + throw new DriftSignal("DO storage pointer changed during residual cleanup"); + } + const currentOwnerKeys = await scanDoOwnerKeys(iso, collected.doStorageId); + if (!arraysShallowEqual(currentOwnerKeys.toSorted(), collected.doOwnerKeys.toSorted())) { + await iso.unwatch(); + throw new DriftSignal("DO owner keys changed during residual cleanup"); + } + + if (!collected.doStorageId && collected.doOwnerKeys.length === 0) { + await iso.unwatch(); + return; + } + const multi = iso.multi(); + if (collected.doOwnerKeys.length) multi.del(...collected.doOwnerKeys); + if (collected.doStorageId) multi.del(storageKey); + await multi.exec(); + }); } /** @@ -543,23 +636,13 @@ async function collectDeleteInputs({ redis, ns, name }) { // pattern slot on it — SREMing because we just happen to own a slot // there would strand those other workers from their declared host. const hostsInActiveRoutes = [...new Set(activeRoutes.map((r) => r.host))]; - const hostsLosingNsOwnership = []; const hostPatternReads = await redis.hGetAllMany(hostsInActiveRoutes.map((host) => patternsKey(host))); - for (let i = 0; i < hostsInActiveRoutes.length; i += 1) { - const h = hostsInActiveRoutes[i]; - const patternsOnHost = hostPatternReads[i]; - const ourSlots = new Set( - activeRoutes.filter((r) => r.host === h).map((r) => r.slot) - ); - let someoneElseFromNs = false; - for (const [slot, raw] of Object.entries(patternsOnHost)) { - if (ourSlots.has(slot)) continue; - if (typeof raw !== "string") continue; - const parsed = decodePatternProjection(raw); - if (parsed && parsed.ns === ns) { someoneElseFromNs = true; break; } - } - if (!someoneElseFromNs) hostsLosingNsOwnership.push(h); - } + const hostsLosingNsOwnership = findHostsLosingNsOwnership( + ns, + activeRoutes, + hostsInActiveRoutes, + hostPatternReads, + ); // Drives the EXEC-time channel choice. Empty namespaces need routes:flush // because routes:invalidate would re-add the freshly SREM'd ns to @@ -659,6 +742,33 @@ function arraysShallowEqual(a, b) { return true; } +/** + * @param {string} ns + * @param {RouteSlot[]} activeRoutes + * @param {string[]} hosts + * @param {Array>} patternRecords + */ +function findHostsLosingNsOwnership(ns, activeRoutes, hosts, patternRecords) { + const losing = []; + for (let i = 0; i < hosts.length; i += 1) { + const host = hosts[i]; + const ourSlots = new Set( + activeRoutes.filter((route) => route.host === host).map((route) => route.slot) + ); + let siblingOwnsHost = false; + for (const [slot, raw] of Object.entries(patternRecords[i] || {})) { + if (ourSlots.has(slot) || typeof raw !== "string") continue; + const projection = decodePatternProjection(raw); + if (projection?.ns === ns) { + siblingOwnsHost = true; + break; + } + } + if (!siblingOwnsHost) losing.push(host); + } + return losing; +} + /** * @template T,U * @param {T[]} items @@ -676,10 +786,10 @@ async function mapConcurrent(items, concurrency, fn) { } /** - * @param {{ redis: RedisClient, ns: string, name: string, principal?: AccessPrincipal | null, requestId: string, collected: DeleteInputs }} args + * @param {{ redis: RedisClient, ns: string, name: string, principal?: AccessPrincipal | null, requestId: string, collected: DeleteInputs, lockToken: string }} args * @returns {Promise>} */ -async function runSessionEXEC({ redis, ns, name, principal, requestId, collected }) { +async function runSessionEXEC({ redis, ns, name, principal, requestId, collected, lockToken }) { return await redis.session(async (iso) => { const watchKeys = [ routesKey(ns), @@ -699,6 +809,11 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected ]; await iso.watch(...watchKeys); + if (await iso.get(deleteLockKey(ns, name)) !== lockToken) { + await iso.unwatch(); + throwDeleteLockExpired(ns, name); + } + // Drift guards — normally unreachable for sanctioned control writers once // the delete lock is held. Keep whole-worker delete fail-closed if // control-owned projections no longer match the collected snapshot. @@ -707,11 +822,30 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected await iso.unwatch(); throw new DriftSignal("worker_versions changed during delete"); } - const curActive = (await iso.hGet(routesKey(ns), name)) || null; + const curRoutes = await iso.hGetAll(routesKey(ns)); + const curActive = curRoutes[name] || null; if (curActive !== collected.activeVersion) { await iso.unwatch(); throw new DriftSignal("active version changed during delete"); } + if (curActive && !curVersions.includes(curActive)) { + await iso.unwatch(); + throw new WholeDeleteError(409, "projection_drift", { + namespace: ns, name, + active_version: curActive, + reason: "active_not_in_worker_versions", + }); + } + const currentNamespaceStillActive = Object.keys(curRoutes).some((worker) => worker !== name); + const currentPatternRecords = await iso.hGetAllMany( + collected.affectedHosts.map((host) => patternsKey(host)) + ); + const currentHostsLosingNsOwnership = findHostsLosingNsOwnership( + ns, + collected.activeRoutes, + collected.affectedHosts, + currentPatternRecords, + ); const curHasSecrets = (await iso.exists(workerSecretsKey(ns, name))) > 0; if (curHasSecrets !== collected.hasWorkerSecrets) { await iso.unwatch(); @@ -773,12 +907,17 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected } } + const commitInputs = { + ...collected, + hostsLosingNsOwnership: currentHostsLosingNsOwnership, + namespaceStillActive: currentNamespaceStillActive, + }; const { cleanupTaskId, cleanupIntent, dedupedPrefixes } = - buildWorkerDeleteCleanup(collected, requestId, buildS3CleanupTaskId); + buildWorkerDeleteCleanup(commitInputs, requestId, buildS3CleanupTaskId); const multi = iso.multi(); stageWorkerDelete(multi, { - collected, + collected: commitInputs, channels: { routes: ROUTES_CHANNEL, routesFlush: ROUTES_FLUSH_CHANNEL, @@ -794,7 +933,7 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected retainedVersions: collected.retainedVersions, affectedHosts: collected.affectedHosts, queueConsumersRemoved: collected.queueConsumerKeys.length, - namespaceStillActive: collected.namespaceStillActive, + namespaceStillActive: currentNamespaceStillActive, cleanupTaskId, cleanupIntent, dedupedPrefixes, diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 43e9c97..d0386ec 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -5,7 +5,8 @@ import { getControlS3, runOptimistic, stageBundleCommit, buildS3CleanupTaskId, recordS3CleanupIntent, - ControlAbort, controlAbortResponse, codedErrorResponse, + ControlAbort, controlAbortResponse, codedErrorLogFields, codedErrorResponse, + secretEnvelopeErrorResponse, } from "control-shared"; import { PLATFORM_TIER_RESERVED_NS } from "shared-auth-roles"; import { @@ -65,6 +66,13 @@ const DEPLOY_ASSET_UPLOAD_CONCURRENCY = 8; // pre-commit shape rejections. class DeployAbort extends ControlAbort {} +/** @param {Record} details */ +function deployAbortLogContext(details) { + const { databaseId, ...context } = details; + if (databaseId !== undefined) context.database_id = databaseId; + return context; +} + /** * @typedef {import("control-shared").ControlLogger} ControlLogger * @typedef {import("shared-redis").RedisClient} RedisClient @@ -524,7 +532,7 @@ async function uploadDeployAssets({ ) ); if (firstFailure) { - const { assetPath, key, err } = firstFailure; + const { key, err } = firstFailure; log("error", "asset_upload_failed", { request_id: requestId, namespace: ns, @@ -542,7 +550,7 @@ async function uploadDeployAssets({ response: jsonError( 502, "asset_upload_failed", - `Asset upload failed for ${assetPath}: ${errMessage(err)}`, + "Asset upload failed", { ...(warnings.length ? { warnings } : {}) } ), }; @@ -715,20 +723,33 @@ async function commitPreparedDeploy({ } const warningDetails = warnings.length ? { warnings } : {}; if (err instanceof DeployAbort) { - log("warn", "deploy_rejected", { + log(err.status >= 500 ? "error" : "warn", "deploy_rejected", { request_id: requestId, namespace: ns, worker: name, version, - status: err.status, - reason: err.code, - ...err.details, + ...codedErrorLogFields(err, err.code, { context: deployAbortLogContext(err.details) }), }); return { response: controlAbortResponse(err, warningDetails) }; } if (err instanceof WorkerEnvBudgetError) return { response: codedErrorResponse(err, err.code, warningDetails) }; if (err instanceof WorkerCodeBudgetError) return { response: codedErrorResponse(err, err.code, warningDetails) }; - if (err instanceof SecretEnvelopeError) return { response: jsonError(503, err.code, err.message, warningDetails) }; + if (err instanceof SecretEnvelopeError) { + return { + response: secretEnvelopeErrorResponse({ + err, + log, + event: "deploy_rejected", + fields: { + request_id: requestId, + namespace: ns, + worker: name, + version, + }, + responseDetails: warningDetails, + }), + }; + } throw err; } return { commitDurationMs: Date.now() - commitStartedAt }; @@ -758,8 +779,7 @@ export async function handle({ request, env, ns, name, requestId }) { request_id: requestId, namespace: ns, worker: name, - status: err.status, - reason: err.code, + ...codedErrorLogFields(err, "link_error"), }); return codedErrorResponse(err, "link_error"); } @@ -884,7 +904,7 @@ async function scheduleDeployAbortCleanup({ warnings.push({ kind: "assets_cleanup_task_failed", prefix, - reason: errMessage(err), + reason: "cleanup_task_write_failed", }); } } diff --git a/control/handlers/hosts.js b/control/handlers/hosts.js index 84d38f1..a96f6d5 100644 --- a/control/handlers/hosts.js +++ b/control/handlers/hosts.js @@ -2,6 +2,7 @@ import { jsonResponse, jsonError, readJsonBody, + codedErrorLogFields, codedErrorResponse, requireControlLog, requireControlRedis, @@ -39,6 +40,11 @@ export async function handle({ request, env, method, nsName, requestId }) { return jsonResponse(200, { namespace: nsName, hosts }); } catch (err) { if (err instanceof RoutingError) { + log(err.status >= 500 ? "error" : "warn", "hosts_reconcile_rejected", { + request_id: requestId, + namespace: nsName, + ...codedErrorLogFields(err, "routing_error"), + }); return codedErrorResponse(err, "routing_error"); } throw err; diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 3ab6310..9144088 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -5,10 +5,12 @@ import { requireControlLog, requireControlRedis, stringEnv, + codedErrorLogFields, codedErrorResponse, runOptimistic, ControlAbort, controlAbortResponse, + secretEnvelopeErrorResponse, } from "control-shared"; import { invalidSecretMutationKeyResponse, @@ -36,21 +38,40 @@ class NamespaceSecretAbort extends ControlAbort {} * @param {{ log: import("control-shared").ControlLogger, requestId: string, nsName: string, secretKey: string, method: string }} context */ function namespaceSecretMutationErrorResponse(err, { log, requestId, nsName, secretKey, method }) { - if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); + if (err instanceof NamespaceSecretAbort) { + log(err.status >= 500 ? "error" : "warn", "ns_secret_mutation_rejected", { + request_id: requestId, + namespace: nsName, + key: secretKey, + method, + ...codedErrorLogFields(err), + }); + return controlAbortResponse(err); + } if (err instanceof BundleMetaError) { log("error", "ns_secret_mutation_rejected", { request_id: requestId, namespace: nsName, key: secretKey, method, - status: err.status, - reason: err.code, - error_detail: errMessage(err.cause), + ...codedErrorLogFields(err, err.code, { errorDetail: errMessage(err.cause) }), }); return codedErrorResponse(err, err.code); } if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); - if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + if (err instanceof SecretEnvelopeError) { + return secretEnvelopeErrorResponse({ + err, + log, + event: "ns_secret_mutation_rejected", + fields: { + request_id: requestId, + namespace: nsName, + key: secretKey, + method, + }, + }); + } return null; } @@ -199,12 +220,21 @@ export async function handle({ request, env, method, nsName, secretKey, requestI if (method === "PUT" && secretKey !== undefined) { const invalidKey = invalidSecretMutationKeyResponse(secretKey); if (invalidKey) return invalidKey; - const put = await readEncryptedSecretPutValue({ - request, - env, - hashKey: nsSecretHashKey, - fieldName: secretKey, - }); + let put; + try { + put = await readEncryptedSecretPutValue({ + request, + env, + hashKey: nsSecretHashKey, + fieldName: secretKey, + }); + } catch (err) { + const response = namespaceSecretMutationErrorResponse(err, { + log, requestId, nsName, secretKey, method, + }); + if (response) return response; + throw err; + } if ("response" in put) return put.response; try { await mutateNamespaceSecret({ diff --git a/control/handlers/promote.js b/control/handlers/promote.js index 1177437..050a04a 100644 --- a/control/handlers/promote.js +++ b/control/handlers/promote.js @@ -2,7 +2,7 @@ import { jsonResponse, jsonError, readJsonBody, - formatError, + codedErrorLogFields, codedErrorResponse, requireControlLog, requireControlRedis, @@ -47,13 +47,12 @@ export async function handle({ request, env, ns, name, requestId }) { }); } catch (err) { if (err instanceof RoutingError) { - log("warn", "worker_promote_rejected", { + log(err.status >= 500 ? "error" : "warn", "worker_promote_rejected", { request_id: requestId, namespace: ns, worker: name, version: body.version, - status: err.status, - ...formatError(err), + ...codedErrorLogFields(err, "routing_error"), ...(err.details.attempts ? { attempts: err.details.attempts } : {}), }); return codedErrorResponse(err, "routing_error"); diff --git a/control/handlers/secret-put.js b/control/handlers/secret-put.js index 548db58..62947a2 100644 --- a/control/handlers/secret-put.js +++ b/control/handlers/secret-put.js @@ -5,7 +5,7 @@ import { stringEnv, } from "control-shared"; import { validateSecretKey } from "control-lib"; -import { encryptSecretValue, SecretEnvelopeError } from "shared-secret-envelope"; +import { encryptSecretValue } from "shared-secret-envelope"; const SECRET_PUT_JSON_BODY_MAX_BYTES = 128 * 1024; @@ -44,15 +44,8 @@ export async function readEncryptedSecretPutValue({ request, env, hashKey, field if (Buffer.byteLength(body.value, "utf8") > 64 * 1024) { return { response: jsonError(400, "invalid_request", "secret value too large (max 64 KiB utf-8)") }; } - try { - return { - plaintext: body.value, - encrypted: await encryptSecretValue(body.value, { env: stringEnv(env), hashKey, fieldName }), - }; - } catch (err) { - if (err instanceof SecretEnvelopeError) { - return { response: jsonError(503, err.code, err.message) }; - } - throw err; - } + return { + plaintext: body.value, + encrypted: await encryptSecretValue(body.value, { env: stringEnv(env), hashKey, fieldName }), + }; } diff --git a/control/handlers/versions.js b/control/handlers/versions.js index eb0eded..7eac8cc 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -1,9 +1,9 @@ import { jsonResponse, jsonError, - acquireDeleteLock, releaseDeleteLock, + acquireDeleteLock, releaseDeleteLock, renewDeleteLock, assertWorkflowDeleteAllowed, buildS3CleanupTaskId, recordCleanupIntentOrWarn, - ControlAbort, controlAbortResponse, + ControlAbort, codedErrorLogFields, controlAbortResponse, requireControlLog, requireControlRedis, runOptimistic, @@ -22,7 +22,13 @@ import { stageWorkerVersionIndexDelete, stageWorkerVersionIndexRemove, } from "control-lifecycle-indexes"; -import { parseVersion, bundleKey, routesKey } from "shared-version"; +import { + VERSION_DELETE_LOCK_KIND, + bundleKey, + deleteLockKey, + parseVersion, + routesKey, +} from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; const MAX_DELETE_ATTEMPTS = 5; @@ -69,7 +75,7 @@ async function handleDelete({ ns, name, version, principal, requestId }) { return jsonError(400, "invalid_version", `Version must be "v", got ${JSON.stringify(version)}`); } - const lockToken = await acquireDeleteLock(redis, ns, name); + const lockToken = await acquireDeleteLock(redis, ns, name, VERSION_DELETE_LOCK_KIND); if (!lockToken) { log("warn", "version_delete_rejected", { request_id: requestId, @@ -86,18 +92,23 @@ async function handleDelete({ ns, name, version, principal, requestId }) { let result; try { await assertWorkflowDeleteAllowed({ ns, worker: name, version, allowCleanup: true }); + if (!await renewDeleteLock(redis, ns, name, lockToken)) { + throw new VersionDeleteError(409, "deleting", { + namespace: ns, name, version, + message: "worker delete lock expired; retry the request", + }); + } result = await executeVersionDelete({ - redis, ns, name, version, principal, requestId, + redis, ns, name, version, principal, requestId, lockToken, }); } catch (err) { if (err instanceof ControlAbort) { - log("warn", "version_delete_rejected", { + log(err.status >= 500 ? "error" : "warn", "version_delete_rejected", { request_id: requestId, namespace: ns, worker: name, version, - status: err.status, - reason: err.code, + ...codedErrorLogFields(err), }); return controlAbortResponse(err); } @@ -138,9 +149,9 @@ async function handleDelete({ ns, name, version, principal, requestId }) { } /** - * @param {{ redis: RedisClient, ns: string, name: string, version: string, principal?: AccessPrincipal | null, requestId: string }} args + * @param {{ redis: RedisClient, ns: string, name: string, version: string, principal?: AccessPrincipal | null, requestId: string, lockToken: string }} args */ -async function executeVersionDelete({ redis, ns, name, version, principal, requestId }) { +async function executeVersionDelete({ redis, ns, name, version, principal, requestId, lockToken }) { return await runOptimistic(redis, { attempts: MAX_DELETE_ATTEMPTS, onExhausted: () => { @@ -151,6 +162,7 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque }, }, async (iso) => { await iso.watch( + deleteLockKey(ns, name), routesKey(ns), workerVersionsKey(ns, name), workerSecretsKey(ns, name), @@ -158,6 +170,14 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque referrersKey(ns, name, version), ); + if (await iso.get(deleteLockKey(ns, name)) !== lockToken) { + await iso.unwatch(); + throw new VersionDeleteError(409, "deleting", { + namespace: ns, name, version, + message: "worker delete lock expired; retry the request", + }); + } + const currentActive = await iso.hGet(routesKey(ns), name); if (currentActive === version) { throw new VersionDeleteError(409, "active_version", { @@ -184,8 +204,10 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque ns, worker: name, version, - makeError: () => new VersionDeleteError(500, "corrupt_meta", { + makeError: ({ reason }) => new VersionDeleteError(500, "corrupt_meta", { namespace: ns, name, version, + stage: "target_meta_parse", + detail: reason, }), }); diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index b074877..bc726e3 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -8,7 +8,9 @@ import { requireControlRedis, runOptimistic, stringEnv, + codedErrorLogFields, codedErrorResponse, + secretEnvelopeErrorResponse, } from "control-shared"; import { deleteLockKey, @@ -68,18 +70,43 @@ function secretMutationContentionAbort() { * }} args */ function rejectSecretMutation({ abort, requestId, ns, name, key, method, log }) { - log("warn", "secret_mutation_rejected", { + log(abort.status >= 500 ? "error" : "warn", "secret_mutation_rejected", { request_id: requestId, namespace: ns, worker: name, key, method, - status: abort.status, - reason: abort.code, + ...codedErrorLogFields(abort), }); return controlAbortResponse(abort); } +/** + * @param {{ + * err: SecretEnvelopeError, + * requestId: string, + * ns: string, + * name: string, + * key: string, + * method: string, + * log: (level: string, event: string, fields: Record) => void, + * }} args + */ +function rejectSecretEnvelopeMutation({ err, requestId, ns, name, key, method, log }) { + return secretEnvelopeErrorResponse({ + err, + log, + event: "secret_mutation_rejected", + fields: { + request_id: requestId, + namespace: ns, + worker: name, + key, + method, + }, + }); +} + /** * @param {{ * request: Request, @@ -110,12 +137,20 @@ export async function handle({ request, env, method, ns, name, subPath, requestI let storedValue = null; let putPlaintext = null; if (method === "PUT") { - const put = await readEncryptedSecretPutValue({ - request, - env, - hashKey: secretsKey, - fieldName: key, - }); + let put; + try { + put = await readEncryptedSecretPutValue({ + request, + env, + hashKey: secretsKey, + fieldName: key, + }); + } catch (err) { + if (err instanceof SecretEnvelopeError) { + return rejectSecretEnvelopeMutation({ err, requestId, ns, name, key, method, log }); + } + throw err; + } if ("response" in put) return put.response; storedValue = put.encrypted; putPlaintext = put.plaintext; @@ -217,9 +252,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI worker: name, key, method, - status: err.status, - reason: err.code, - error_detail: errMessage(err.cause), + ...codedErrorLogFields(err, err.code, { errorDetail: errMessage(err.cause) }), }); return codedErrorResponse(err, err.code); } @@ -242,9 +275,19 @@ export async function handle({ request, env, method, ns, name, subPath, requestI }); return rejectSecretMutation({ abort, requestId, ns, name, key, method, log }); } + log(err.status >= 500 ? "error" : "warn", "secret_mutation_rejected", { + request_id: requestId, + namespace: ns, + worker: name, + key, + method, + ...codedErrorLogFields(err, "routing_error"), + }); return codedErrorResponse(err, err.code); } - if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + if (err instanceof SecretEnvelopeError) { + return rejectSecretEnvelopeMutation({ err, requestId, ns, name, key, method, log }); + } throw err; } } diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index 5d98dfb..a5d8e59 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -8,6 +8,7 @@ import { } from "control-lib"; import { ControlAbort, + codedErrorLogFields, controlAbortResponse, errMessage, jsonError, @@ -50,6 +51,18 @@ export async function handle({ method, url, ns, subPath, requestId }) { return await handleInner({ method, url, ns, subPath, requestId }); } catch (err) { if (err instanceof ControlAbort) { + if (err.status >= 500) { + const detailWorker = typeof err.details?.worker === "string" + ? err.details.worker + : subPath[0]; + requireControlLog()("error", "workflow_request_rejected", { + request_id: requestId, + namespace: ns, + ...(detailWorker ? { worker: detailWorker } : {}), + ...(subPath[1] ? { workflow: subPath[1] } : {}), + ...codedErrorLogFields(err), + }); + } return controlAbortResponse(err); } throw err; @@ -313,7 +326,14 @@ function workflowBundleMeta(ns, worker, version, raw) { ns, worker, version, - makeError: ({ message }) => new ControlAbort(500, "corrupt_meta", { message }), + makeError: ({ message, reason }) => new ControlAbort(500, "corrupt_meta", { + message, + namespace: ns, + worker, + version, + stage: "bundle_meta_parse", + detail: reason, + }), }); } diff --git a/control/lib.js b/control/lib.js index 3c2be48..a3b8b5f 100644 --- a/control/lib.js +++ b/control/lib.js @@ -18,8 +18,8 @@ import { } from "shared-ns-pattern"; import { PLATFORM_TIER_RESERVED_NS, ROLES } from "shared-auth-roles"; import { errorMessage } from "shared-errors"; -import { doStorageIdKey, routesKey, workerVersionsKey } from "shared-version"; -export { doStorageIdKey, routesKey, workerVersionsKey }; +import { deleteLockKey, doStorageIdKey, routesKey, workerVersionsKey } from "shared-version"; +export { deleteLockKey, doStorageIdKey, routesKey, workerVersionsKey }; export { validateModulePath }; export { WORKER_NAME_RE }; export { WORKFLOW_NAME_RE }; @@ -73,8 +73,8 @@ export function parseBundleMeta(raw, { ns, worker, version, makeError }) { let parsed; try { parsed = JSON.parse(raw); - } catch (err) { - fail(err); + } catch { + fail(new SyntaxError("__meta__ is not valid JSON")); } if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { fail(new TypeError("__meta__ must be a JSON object")); @@ -339,11 +339,6 @@ export function referrersKey(ns, worker, version) { return `worker-version-referrers:${ns}:${worker}:${version}`; } -/** @param {string} ns @param {string} worker */ -export function deleteLockKey(ns, worker) { - return `worker-delete-lock:${ns}:${worker}`; -} - /** @param {string} ns */ export function workersIndexKey(ns) { return `workers:${ns}`; diff --git a/control/package.json b/control/package.json index 70731c9..6e46eb2 100644 --- a/control/package.json +++ b/control/package.json @@ -5,7 +5,7 @@ "author": "Sean Consulting OÜ", "type": "module", "dependencies": { - "@wdl-dev/aws-sigv4": "^2.0.0", + "@wdl-dev/aws-sigv4": "^3.0.1", "croner": "^10.0.1" } } diff --git a/control/shared.js b/control/shared.js index 9f0ccde..deb567b 100644 --- a/control/shared.js +++ b/control/shared.js @@ -11,11 +11,12 @@ import { internalErrorResponse, jsonError, jsonResponse, sanitizeJsonErrorDetail import { withInternalAuth } from "shared-internal-auth"; import { errorMessage } from "shared-errors"; import { randomHex } from "shared-random-id"; -import { deleteLockKey } from "control-lib"; import { DECLARED_HOSTS_KEY, HOST_DECLARATIONS_SCAN_PATTERN, HOSTS_SCAN_PATTERN, + deleteLockKey, + formatDeleteLockToken, hostDeclarationsKey, namespaceFromHostsKey, } from "shared-version"; @@ -36,8 +37,11 @@ import { } from "control-workflows-client"; import { ControlAbort, + controlAbortLogDetails, + codedErrorLogFields, codedErrorResponse, controlAbortResponse, + secretEnvelopeErrorResponse, } from "control-errors"; import { runOptimistic, withOptimisticRetries } from "control-optimistic"; import { @@ -48,6 +52,7 @@ import { acquireTokenLock, createTokenLock, releaseTokenLock, + renewTokenLock, } from "shared-redis-lock"; export const ROUTES_CHANNEL = "routes:invalidate"; @@ -89,7 +94,14 @@ let s3Initialized = false; let r2Initialized = false; export { formatError, internalErrorResponse, jsonError, jsonResponse }; -export { ControlAbort, codedErrorResponse, controlAbortResponse }; +export { + ControlAbort, + controlAbortLogDetails, + codedErrorLogFields, + codedErrorResponse, + controlAbortResponse, + secretEnvelopeErrorResponse, +}; export { runOptimistic, withOptimisticRetries }; export { DEFAULT_JSON_BODY_MAX_BYTES, readJsonBody }; @@ -424,21 +436,38 @@ export function buildS3CleanupTaskId() { } // Normal paths release this advisory lock in finally; the TTL only bounds -// leaks after a crash. The token fences release, not the protected operation. +// leaks after a crash. Delete transactions also verify this token under their +// final WATCH so an expired request cannot commit under a replacement holder. const DELETE_LOCK_TTL_SECONDS = 30; /** * @param {RedisClient} redis * @param {string} ns * @param {string} worker + * @param {"whole" | "version"} kind */ -export async function acquireDeleteLock(redis, ns, worker) { +export async function acquireDeleteLock(redis, ns, worker, kind) { const key = deleteLockKey(ns, worker); const lock = createTokenLock(key); + lock.token = formatDeleteLockToken(kind, lock.token); return await acquireTokenLock(redis, lock, { ttlSeconds: DELETE_LOCK_TTL_SECONDS }) ? lock.token : null; } +/** + * @param {RedisClient} redis + * @param {string} ns + * @param {string} worker + * @param {string} token + */ +export async function renewDeleteLock(redis, ns, worker, token) { + return await renewTokenLock( + redis, + { key: deleteLockKey(ns, worker), token }, + DELETE_LOCK_TTL_SECONDS + ); +} + // Compare-and-delete so we don't free a token a TTL-expired-then-reacquired // lock holder is depending on. /** diff --git a/deploy/kubernetes/base/scheduler.yaml b/deploy/kubernetes/base/scheduler.yaml index 2bd97e9..b8dc59d 100644 --- a/deploy/kubernetes/base/scheduler.yaml +++ b/deploy/kubernetes/base/scheduler.yaml @@ -19,6 +19,7 @@ spec: app.kubernetes.io/name: wdl app.kubernetes.io/component: scheduler spec: + terminationGracePeriodSeconds: 30 containers: - name: scheduler image: docker.io/getwdl/wdl-rust:latest diff --git a/deploy/kubernetes/base/workflows.yaml b/deploy/kubernetes/base/workflows.yaml index 955e3f9..5927d84 100644 --- a/deploy/kubernetes/base/workflows.yaml +++ b/deploy/kubernetes/base/workflows.yaml @@ -33,6 +33,7 @@ spec: app.kubernetes.io/name: wdl app.kubernetes.io/component: workflows spec: + terminationGracePeriodSeconds: 30 containers: - name: workflows image: docker.io/getwdl/wdl-rust:latest diff --git a/do-runtime/actor.js b/do-runtime/actor.js index ffb64b7..8f56371 100644 --- a/do-runtime/actor.js +++ b/do-runtime/actor.js @@ -5,6 +5,7 @@ import { buildAlarmRequest, buildFacetName, buildForwardRequest, + buildRpcRequest, DO_OWNERSHIP_ERROR_CONTROL_HEADER, DO_OWNERSHIP_CODE, doPlatformErrorResponse, @@ -25,7 +26,6 @@ import { metrics, SERVICE, } from "do-runtime-state"; -import { errorMessage } from "shared-errors"; import { formatError } from "shared-observability"; import { rebuildResponseWithHeaders } from "shared-respond"; @@ -156,10 +156,10 @@ export class WdlDoHostActor extends DurableObject { id: invoke.objectName, })); if (invoke.kind === "alarm") { - return await facet.fetch(buildAlarmRequest(invoke.alarm)); + return await facet.fetch(buildAlarmRequest(invoke.alarm, requestId)); } if (invoke.kind === "rpc") { - return await dispatchRpc(facet, invoke.rpc); + return await dispatchRpc(facet, invoke.rpc, requestId); } return await facet.fetch(buildForwardRequest(invoke.request)); }); @@ -177,8 +177,8 @@ export class WdlDoHostActor extends DurableObject { throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); } try { - const { owner, leaseRemainingMs } = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); await this.rememberObject(invoke); + const { owner, leaseRemainingMs } = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); return withoutOwnershipErrorControlHeader( await this.dispatchWithLeaseBudget(invoke, owner, leaseRemainingMs, run) ); @@ -267,31 +267,8 @@ function withoutOwnershipErrorControlHeader(response) { /** * @param {DoFacet} facet * @param {{ method: string, args: unknown[] }} rpc + * @param {string | null} requestId */ -async function dispatchRpc(facet, rpc) { - // Facets are workerd JSRPC stubs; reading a method returns a forwarder - // already bound to that stub, unlike ordinary unbound JavaScript methods. - const methods = /** @type {Record} */ (/** @type {unknown} */ (facet)); - const fn = methods[rpc.method]; - if (typeof fn !== "function") { - return Response.json({ - error: "do_rpc_method_not_found", - message: `Durable Object RPC method ${rpc.method} was not found`, - }, { status: 404 }); - } - try { - return Response.json({ ok: true, result: await fn(...rpc.args) }); - } catch (err) { - const errorObject = err && typeof err === "object" ? err : {}; - // The stack captured from fn(...) is tenant method execution, not - // do-runtime framework internals, so it belongs on the tenant RPC boundary. - // Runtime failures still go through the platform error mapper, which - // intentionally hides internal stack traces and infrastructure details. - return Response.json({ - error: "do_rpc_error", - name: "name" in errorObject && typeof errorObject.name === "string" ? errorObject.name : "Error", - message: errorMessage(err), - ...("stack" in errorObject && typeof errorObject.stack === "string" && errorObject.stack ? { stack: errorObject.stack } : {}), - }, { status: 500 }); - } +export async function dispatchRpc(facet, rpc, requestId) { + return await facet.fetch(buildRpcRequest(rpc, requestId)); } diff --git a/do-runtime/alarm-shim-source.js b/do-runtime/alarm-shim-source.js index 7023a7d..9a78989 100644 --- a/do-runtime/alarm-shim-source.js +++ b/do-runtime/alarm-shim-source.js @@ -1,5 +1,6 @@ export const DO_ALARM_SHIM_SOURCE = ` const ALARM_HEADER = "x-wdl-do-internal-alarm"; +const RPC_HEADER = "x-wdl-do-internal-rpc"; const ALARMS_BINDING = "__WDL_DO_ALARMS__"; const ALARM_TABLE = "_wdl_do_alarms"; @@ -14,11 +15,13 @@ const NativeResponse = Response; const NativeString = String; const nativeCrypto = crypto; const arrayAt = Array.prototype.at; +const arrayIsArray = Array.isArray; const arrayPush = Array.prototype.push; const cryptoRandomUUID = crypto.randomUUID; const dateGetTime = Date.prototype.getTime; const dateNow = Date.now; const headersGet = Headers.prototype.get; +const mapForEach = Map.prototype.forEach; const mathTrunc = Math.trunc; const numberIsFinite = Number.isFinite; const numberIsInteger = Number.isInteger; @@ -139,9 +142,11 @@ function ensureAlarmTable(storage) { function readAlarmRow(storage) { ensureAlarmTable(storage); - return [...storage.sql.exec( + const result = storage.sql.exec( "SELECT scheduled_time, retry_count, in_flight, token FROM " + ALARM_TABLE + " WHERE id = 1" - )][0] || null; + ); + const rows = arrayIsArray(result) ? result : [...result]; + return rows[0] || null; } function writeAlarmRow(storage, row) { @@ -333,21 +338,26 @@ function sqlObjectDropStatement(row) { async function deleteAllKvStorage(storage) { const entries = await storage.list(); - const keys = [...entries.keys()]; + const keys = []; + reflectApply(mapForEach, entries, [(_value, key) => { + reflectApply(arrayPush, keys, [key]); + }]); if (keys.length) await storage.delete(keys); } function deleteAllSqlStorage(storage, deleteAlarm) { - const rows = [...storage.sql.exec( + const result = storage.sql.exec( "SELECT type, name FROM sqlite_master " + "WHERE type IN ('trigger', 'view', 'table', 'index') " + "ORDER BY CASE type WHEN 'trigger' THEN 0 WHEN 'view' THEN 1 WHEN 'table' THEN 2 ELSE 3 END" - )]; + ); + const rows = arrayIsArray(result) ? result : [...result]; // workerd permits this connection-level PRAGMA; integration coverage pins it // because database-level PRAGMA writes are not part of the public DO SQL API. storage.sql.exec("PRAGMA foreign_keys = OFF"); try { - for (const row of rows) { + for (let i = 0; i < rows.length; i += 1) { + const row = rows[i]; if (NativeString(row.name) === ALARM_TABLE && !deleteAlarm) continue; const statement = sqlObjectDropStatement(row); if (statement) storage.sql.exec(statement); @@ -420,8 +430,8 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = throw new TypeError("deleteAll() cannot be used inside transactionSync(); use transaction()"); } return (async () => { - const [options, ...rest] = args; - if (rest.length) throw new TypeError("deleteAll() accepts at most one options argument"); + if (args.length > 1) throw new TypeError("deleteAll() accepts at most one options argument"); + const options = args[0]; const deleteAlarm = options?.deleteAlarm !== false; const alarmRow = readAlarmRow(alarmStorage); const preservedAlarm = deleteAlarm ? null : alarmRow; @@ -536,6 +546,29 @@ export function wrapDurableObjectClass(Base, className) { completeStorageAlarm(this.ctx.storage, claim.token); return reflectApply(responseJson, NativeResponse, [{ ok: true }]); } + if (reflectApply(headersGet, headers, [RPC_HEADER]) === "1") { + const rpc = await reflectApply(requestJson, request, []); + try { + const tenantMethod = reflectGet(this, rpc.method, this); + if (typeof tenantMethod !== "function") { + return reflectApply(responseJson, NativeResponse, [{ + error: "do_rpc_method_not_found", + message: "Durable Object RPC method " + rpc.method + " was not found", + }, { status: 404 }]); + } + const result = await reflectApply(tenantMethod, this, rpc.args); + return reflectApply(responseJson, NativeResponse, [{ ok: true, result }]); + } catch (err) { + const formatted = formatWrappedError(err); + const stack = safeErrorString(safeErrorField(err, "stack")); + return reflectApply(responseJson, NativeResponse, [{ + error: "do_rpc_error", + name: formatted.error_name, + message: formatted.error_message, + ...(stack ? { stack } : {}), + }, { status: 500 }]); + } + } if (typeof tenantFetch !== "function") { return new NativeResponse("Durable Object class has no fetch handler", { status: 500 }); } diff --git a/do-runtime/load-code-budget.js b/do-runtime/load-code-budget.js index be0f421..ff272bc 100644 --- a/do-runtime/load-code-budget.js +++ b/do-runtime/load-code-budget.js @@ -31,13 +31,19 @@ export function hasDoRuntimeInjectedModules(meta) { function generateDoRuntimeWrapperModule(userMainSpecifier, classNames) { const userMain = JSON.stringify(`./${userMainSpecifier}`); const alarmShim = JSON.stringify(`./${DO_ALARM_SHIM_MODULE}`); - const wrappedClasses = classNames.map((name) => ` -export class ${name} extends wrapDurableObjectClass(user.${name}, ${JSON.stringify(name)}) {} + const wrappedClasses = classNames.map((name, index) => ` +const __WdlWrappedDurableObject${index}__ = ({ + [${JSON.stringify(name)}]: class extends __WdlWrapDurableObjectClass__( + __WdlUserModule__.${name}, + ${JSON.stringify(name)} + ) {}, +})[${JSON.stringify(name)}]; +export { __WdlWrappedDurableObject${index}__ as ${name} }; `).join(""); return ` -import { wrapDurableObjectClass } from ${alarmShim}; +import { wrapDurableObjectClass as __WdlWrapDurableObjectClass__ } from ${alarmShim}; -import * as user from ${userMain}; +import * as __WdlUserModule__ from ${userMain}; export * from ${userMain}; ${wrappedClasses} diff --git a/do-runtime/owner-registry.js b/do-runtime/owner-registry.js index b7be7c0..a055683 100644 --- a/do-runtime/owner-registry.js +++ b/do-runtime/owner-registry.js @@ -42,7 +42,12 @@ import { stageOwnerRelease, stageOwnerRenew, } from "shared-owner-protocol"; -import { doStorageIdKey } from "shared-version"; +import { + VERSION_DELETE_LOCK_KIND, + deleteLockKey, + doStorageIdKey, + parseDeleteLockKind, +} from "shared-version"; const DEFAULT_OWNER_TTL_SECONDS = 120; const DEFAULT_RENEW_CONCURRENCY = 8; @@ -248,7 +253,7 @@ async function readOwnerWithTimeFromClient(client, ownerKey) { /** * @param {RedisLike} session - * @param {DoOwner | null | undefined} owner + * @param {Pick | null | undefined} owner */ async function ownerStoragePointerCurrent(session, owner) { if (!owner?.doStorageId || !isValidBundleScope(owner)) return false; @@ -343,16 +348,31 @@ export async function resolveDoOwner(env, invoke) { const ownerKey = buildOwnerKey(scope); const key = ownerKeyOf(ownerKey); const generationKey = ownerGenerationKeyOf(ownerKey); + const workerDeleteLockKey = deleteLockKey(scope.ns, scope.worker); + const storagePointerKey = doStorageIdKey(scope.ns, scope.worker); return await withWatchRetries(async () => client.session(async (session) => { - await session.watch(key, generationKey); + await session.watch(key, generationKey, workerDeleteLockKey); const { owner: current, nowMs } = await readOwnerRecordWithRedisTime( session, key, (raw) => parseOwner(raw, ownerKey, scope) ); + const deleteLockToken = decodeBulk(await session.get(workerDeleteLockKey)); + if ( + deleteLockToken != null && + parseDeleteLockKind(deleteLockToken) !== VERSION_DELETE_LOCK_KIND + ) { + await session.unwatch(); + forgetOwnedScope(ownerKey); + throw new DoRuntimeError( + 503, + DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, + `DO scope ${ownerKey} worker is being deleted` + ); + } if (current && !ownerLeaseExpired(current, nowMs)) { - await session.watch(doStorageIdKey(scope.ns, scope.worker)); + await session.watch(storagePointerKey); if (!await ownerStoragePointerCurrent(session, current)) { await session.unwatch(); forgetOwnedScope(ownerKey); @@ -390,6 +410,16 @@ export async function resolveDoOwner(env, invoke) { if (isDraining()) { throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); } + await session.watch(storagePointerKey); + if (!await ownerStoragePointerCurrent(session, scope)) { + await session.unwatch(); + forgetOwnedScope(ownerKey); + throw new DoRuntimeError( + 503, + DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, + `DO scope ${ownerKey} no longer matches active worker storage` + ); + } const generation = await nextOwnerGeneration(session, generationKey, current?.generation); const owner = ownerRecordFor(env, invoke, localTask, generation, nowMs); await stageOwnerClaim(session.multi(), { ownerKey: key, generationKey }, owner, ownerTtlSeconds(env)).exec(); @@ -475,9 +505,9 @@ export async function renewOwner(env, owner) { const client = redisClient(env); const key = ownerKeyOf(owner.ownerKey); return await withWatchRetries(async () => client.session(async (session) => { - // Redis WATCH observes explicit writes/deletes, not passive TTL expiry. - // Renewal is therefore a freshness optimization; generation fencing remains - // the authority if a peer claims the scope in the tiny expiry window. + // Valkey expiration deletes invalidate WATCH just like explicit writes. + // Generation fencing remains authoritative if a peer claims the scope after + // the lease expires and before this renewal commits. await session.watch(key); const { owner: current, nowMs } = await readOwnerWithTimeFromClient(session, owner.ownerKey); if (!current || !ownerFenceMatches(current, owner)) { diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index aba3603..3132e5f 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -72,6 +72,8 @@ const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); const ALARM_INTERNAL_URL = "https://do.internal/__wdl_alarm"; const ALARM_INTERNAL_HEADER = "x-wdl-do-internal-alarm"; +const RPC_INTERNAL_URL = "https://do.internal/__wdl_rpc"; +const RPC_INTERNAL_HEADER = "x-wdl-do-internal-rpc"; const CONNECT_HEADERS = { ns: "x-wdl-do-ns", worker: "x-wdl-do-worker", @@ -94,10 +96,12 @@ const OWNER_HINT_PROTOCOL_HEADERS = [ ]; const CONNECT_INTERNAL_HEADER_NAMES = new Set([ INTERNAL_AUTH_HEADER, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, ...Object.values(CONNECT_HEADERS), ...OWNER_HINT_PROTOCOL_HEADERS, "x-wdl-do-forwarded", "x-wdl-do-hop-count", + RPC_INTERNAL_HEADER, ]); const LOCAL_ACTOR_ENVELOPE_HEADER = "x-wdl-do-local-envelope"; const LOCAL_ACTOR_ENVELOPE_MARKER = "binary"; @@ -596,6 +600,7 @@ export function buildForwardRequest(spec) { // User-controlled DO fetches must not be able to spoof internal Workflows // alarm delivery; only the alarm builder may attach this. request.headers.delete(ALARM_INTERNAL_HEADER); + request.headers.delete(RPC_INTERNAL_HEADER); request.headers.delete(INTERNAL_AUTH_HEADER); return request; } @@ -756,14 +761,31 @@ export function encodeDoInvokeRequest(invoke) { }); } -/** @param {JsonRecord} alarm */ -export function buildAlarmRequest(alarm) { +/** @param {JsonRecord} alarm @param {string | null} requestId */ +export function buildAlarmRequest(alarm, requestId) { return new Request(ALARM_INTERNAL_URL, { method: "POST", headers: { "content-type": "application/json", [ALARM_INTERNAL_HEADER]: "1", + ...(requestId ? { "x-request-id": requestId } : {}), }, body: JSON.stringify(alarm), }); } + +/** + * @param {RpcInfo} rpc + * @param {string | null} requestId + */ +export function buildRpcRequest(rpc, requestId) { + return new Request(RPC_INTERNAL_URL, { + method: "POST", + headers: { + "content-type": "application/json", + [RPC_INTERNAL_HEADER]: "1", + ...(requestId ? { "x-request-id": requestId } : {}), + }, + body: JSON.stringify({ method: rpc.method, args: rpc.args }), + }); +} diff --git a/docker-compose.yml b/docker-compose.yml index aafd9bd..357ae31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -315,6 +315,7 @@ services: scheduler: <<: *rust-sidecar-image command: ["/scheduler"] + stop_grace_period: 30s environment: <<: *internal-auth-env REDIS_URL: redis://redis:6379 @@ -359,6 +360,7 @@ services: workflows: <<: *rust-sidecar-image command: ["/workflows"] + stop_grace_period: 30s environment: <<: *internal-auth-env REDIS_URL: redis://redis:6379 diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index 27df9c2..a3e14d7 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -119,12 +119,15 @@ Control lifecycle operations are split so each critical transition has one autho active routes, retained versions, service refs, D1 refs, workflow lifecycle checks, queue/cron projections, and delete locks before committing Redis lifecycle deletion. S3 object cleanup is enqueued only after Redis commit succeeds. -- Worker delete lock values and `s3cleanup:` task ids are server-generated random - values. `x-request-id` is diagnostic only and may be reused by clients or retries; it - must not become a lock token or cleanup-task id. Delete, D1 migration, and delegated - issue locks use the token-fenced primitive in `shared/redis-lock.js`; release is - token-scoped and best-effort because TTL expiry bounds a leaked advisory lock and a - release error must not replace the operation's real result. +- Worker delete lock values are a `whole:` or `version:` operation kind followed by a + server-generated random token; `s3cleanup:` task ids are also server-generated. + `x-request-id` is diagnostic only and may be reused by clients or retries; it must not + become a lock token or cleanup-task id. Delete, D1 migration, and delegated issue locks + use the token-fenced primitive in `shared/redis-lock.js`; release is token-scoped and + best-effort because TTL expiry bounds a leaked advisory lock and a release error must + not replace the operation's real result. Before a final lifecycle mutation, version + and whole-worker delete refresh the token once, then verify it inside the final WATCH + snapshot so an expired request cannot commit under a replacement holder's lock. - Auth is not a middleware convention. `parseControlRoute()` assigns an action, control sends that action and namespace to auth, and auth evaluates the stored token record against `shared/auth-roles.js`. Dispatcher code should not infer permissions from URL @@ -194,7 +197,7 @@ Key families: | `ns-hosts:` | Set | Control | Active host reverse index for a namespace. | Promote/reconcile maintains SADD/SREM deltas in the same EXEC. | | `patterns:` | Hash | Control | Pattern-host route slots; values are compact `v2` tab-separated projections. | Reconcile/promote updates and publishes pattern invalidation. | | `worker-version-referrers:::` | Set | Control | Rebuildable service-binding referrer index. | Blocks version delete while callers reference the version. | -| `worker-delete-lock::` | String EX | Control | Per-worker delete critical-section lock. | Expires automatically; execute delete releases by completion. | +| `worker-delete-lock::` | String EX | Control | Per-worker delete critical-section lock; value is `whole:` or `version:`. | Expires automatically; execute delete releases by completion. Only `whole` fences new DO ownership. | | `secrets:`, `secrets::` | Hash | Control | Namespace and worker secret stores; values are `WDL-ENC:` envelopes. Control encrypts writes; redis-proxy decrypts only during `/runtime/load`. | Deleted by secret lifecycle or worker delete. | | `queue:__system__:worker-delete-s3-cleanup:s` | DB 1 Stream | Control/s3-cleanup worker | Best-effort post-commit object cleanup queue; logical queue name is `worker-delete-s3-cleanup`. | Enqueued only after Redis delete commit succeeds; enqueue failure returns `cleanup_queue_failed` warning. | | `auth:token:` | Hash | Auth | Authoritative token record. | Revoke/expiry delete active record and write tombstone fields. | @@ -298,7 +301,8 @@ Auth-specific contract: candidate retry budget. - Control 5xx responses use generic/safe messages. Internal exception text, auth Redis diagnostics, backend messages, and provider errors belong in logs unless the endpoint - explicitly owns a diagnostic response field. + explicitly owns a diagnostic response field. Structured coded-error diagnostic strings + are truncated to 2,048 characters before log emission. - Deploy returns `worker_code_invalid` when final WorkerCode would collide with injected WDL runtime/do-runtime reserved module names or lacks required bundle metadata, and `worker_code_too_large` when final WorkerCode, including runtime/do-runtime-injected diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 1664c74..68b809e 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -70,7 +70,7 @@ Control lifecycle 操作会拆开处理，确保每个关键 transition 只有 - Promote 是唯一 active-route flip。它会 WATCH 本次候选需要的 delete lock、bundle metadata、D1 ref、service-binding target ref、queue consumer key、host declaration 和 pattern key。EXEC 在一个受审计的 transition 中更新 active route、host 反向索引、cron/queue projection、lifecycle index 和 invalidation publication。 - Secret update/delete 在 worker 有 active route 时，会把 secret-store mutation stage 到 `bumpActiveAndPromote()` 内部，让 budget check、secret hash write、bundle copy 和 route flip 共享同一个 WATCH/MULTI transaction。如果没有 active route，则会在直接写 secret hash 前检查 retained versions 的预算；只有没有 retained versions 的 secret-only worker 会把第一次 load-time budget check 交给 deploy。Secret PUT 会先校验 plaintext 大小和形状，在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope，并在重试间复用同一个 envelope。Envelope JSON 使用固定的 `v,alg,kid,edek,iv,ct,tag` 字段顺序，各 base64 字段也必须是 canonical shape；Control 与 redis-proxy 都拒绝 non-canonical persisted forms，使 malformed direct Redis writes fail closed。Runtime 因此看到新的 immutable version id，而不是原地可变的 secret。Secret DELETE 会先从 env 估算输入里移除目标字段，再解密剩余 secret hash，所以删除损坏的目标字段仍可成功；但剩余 namespace 或 worker secret 只要损坏就 fail closed，direct Redis repair 不是受支持的一致性路径。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata；如果并发 metadata 变化持续使视图失效，control 返回 `namespace_secret_mutation_contention`。 - Version delete 和 whole-worker delete 都 fail-closed。提交 Redis lifecycle deletion 前，会先收集 active route、retained version、service ref、D1 ref、workflow lifecycle check、queue/cron projection 和 delete lock blocker。S3 object cleanup 只在 Redis commit 成功后 enqueue。 -- Worker delete lock value 和 `s3cleanup:` task id 必须由服务端生成随机值。`x-request-id` 只用于诊断，客户端或重试可能复用它；不能把它用作 lock token 或 cleanup-task id。Delete、D1 migration 和 delegated issue lock 使用 `shared/redis-lock.js` 的 token-fenced primitive；release 按 token 限定且是 best-effort，因为 TTL expiry 会限制 advisory lock 泄漏时间，release error 不能覆盖操作的真实结果。 +- Worker delete lock value 由 `whole:` 或 `version:` 操作类型加服务端生成的随机 token 组成；`s3cleanup:` task id 也由服务端生成。`x-request-id` 只用于诊断，客户端或重试可能复用它；不能把它用作 lock token 或 cleanup-task id。Delete、D1 migration 和 delegated issue lock 使用 `shared/redis-lock.js` 的 token-fenced primitive；release 按 token 限定且是 best-effort，因为 TTL expiry 会限制 advisory lock 泄漏时间，release error 不能覆盖操作的真实结果。Version delete 和 whole-worker delete 会在最终 lifecycle mutation 前续租一次，并在最终 WATCH snapshot 内验证 token，因此过期请求不能借替代 holder 的 lock 提交。 - Auth 不是一层约定俗成的 middleware。`parseControlRoute()` 分配 action，control 把 action 和 namespace 发给 auth，auth 用存储的 token record 对照 `shared/auth-roles.js` 评估权限。Dispatcher 代码不应自己从 URL prefix 推断权限。 - Delegated token issue 刻意不放进 direct `auth.token.*` lifecycle。`POST /auth/delegated-tokens` 使用 action `auth.delegated_token.issue`，因此 `/auth/tokens` issue/list/revoke 仍保持 strict ops-only；`token-issuer` credential 只能请求 Auth 按一个已配置 template materialize token。 - `/whoami` 是唯一 namespace-less non-ops action。它对任何有效 token 开放，因为只报告当前 token 自己的 principal、token id、request id 和 public diagnostics。`platformVersion` 是由 bundled workerd date version 派生的 WDL version，使用 `wdl.` namespace，例如 `workerd` `1.20260531.1` 会变成 `wdl.20260531.1`；它不是 project release tag，后者的末尾 counter 可以在同一个 workerd date 上为 WDL-only patch 递增。raw workerd version 不返回。`minCliVersion` 是最低支持的下游 CLI 版本。`urls.control` 是到达 control 的 public origin；当 ingress 提供单个 `x-forwarded-proto` 且值为 `http` 或 `https` 时，`/whoami` 会用该协议生成 `urls.control` 和 `urls.namespace`，否则回退到 control 看到的 request URL 协议。`urls.namespace` 只在 caller 是 tenant namespace token 且显式配置了 public hostname 形态的 `PLATFORM_DOMAIN` 时返回；Control 不会把内部 `workers.local` fallback 伪装成 public URL。`urls.assets` 只在 `ASSETS_CDN_BASE` 是安全的绝对 `http`/`https` URL 时返回，并且会去掉 query 和 fragment。它不能扩展成 token list、token lookup 或携带 secret 的 diagnostics。 @@ -110,7 +110,7 @@ Key families： | `ns-hosts:` | Set | Control | namespace 的 active host 反向索引。 | promote/reconcile 在同一个 EXEC 内维护 SADD/SREM delta。 | | `patterns:` | Hash | Control | pattern-host route slots；value 是紧凑 `v2` tab-separated projection。 | reconcile/promote 更新并 publish pattern invalidation。 | | `worker-version-referrers:::` | Set | Control | 可重建 service-binding referrer index。 | caller 仍引用该 version 时阻止 version delete。 | -| `worker-delete-lock::` | String EX | Control | per-worker delete critical-section lock。 | 自动过期；execute delete 完成后释放。 | +| `worker-delete-lock::` | String EX | Control | per-worker delete critical-section lock；value 是 `whole:` 或 `version:`。 | 自动过期；execute delete 完成后释放；只有 `whole` 会阻止新的 DO ownership。 | | `secrets:`、`secrets::` | Hash | Control | namespace 和 worker secret store；value 是 `WDL-ENC:` envelope。Control 写入前加密；redis-proxy 只在 `/runtime/load` 时解密。 | secret lifecycle 或 worker delete 删除。 | | `queue:__system__:worker-delete-s3-cleanup:s` | DB 1 Stream | Control / s3-cleanup worker | best-effort post-commit object cleanup queue；逻辑队列名是 `worker-delete-s3-cleanup`。 | 只在 Redis delete commit 成功后 enqueue；enqueue 失败返回 `cleanup_queue_failed` warning。 | | `auth:token:` | Hash | Auth | token record 权威记录。 | revoke/expiry 删除 active record 并写 tombstone 字段。 | @@ -141,7 +141,7 @@ Auth 子合同： - `control/json-body.js` 持有 bounded Control JSON parsing 及其 `400`/`413` wire mapping；`control/optimistic.js` 把 strict `WatchError` recognition 和 Redis session 绑定到 `shared/optimistic-retry.js` 持有的 retry loop。 - `control/lib.js::parseBundleMeta()` 是 persisted bundle `__meta__` 的唯一 parser。它要求值是 JSON object，并通过 error factory 让 routing、workflows、delete、deploy 和 env-budget 保留各自的 catch boundary。Bundle 缺失的语义仍由具体 use site 持有：当 projection 变更、唯一性证明、lifecycle cleanup、workflow view 或 environment budget 必须消费 metadata 才能产生正确结果时，只要权威 route 或 index 仍指向该 bundle，缺失就 fail closed。Deploy discovery/link preflight 不把缺失归类为 `corrupt_meta`；watched commit 会以 `target_drift` 拒绝缺失的 pinned service target。 - Delegated issue 的 409 reason 有不同 retry 语义：`delegated_issue_busy` 表示 issuer/template lock 清除后可重试；`active_quota_exceeded` 在已有 delegated credential 过期或 revoke 前不应重试；`namespace_collision` 表示 Auth 已耗尽 configured candidate retry budget。 -- Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志；除非 endpoint 明确拥有某个 diagnostic response field，否则不进入客户端 body。 +- Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志；除非 endpoint 明确拥有某个 diagnostic response field，否则不进入客户端 body。Structured coded-error diagnostic string 在写入日志前截断为最多 2,048 个字符。 - Deploy 在最终 WorkerCode 会碰撞 WDL 注入的 runtime/do-runtime 保留模块名或缺少必要 bundle metadata 时返回 `worker_code_invalid`，在最终 WorkerCode（包含 runtime/do-runtime 注入模块和生成的 workflow keys）超过 workerd 64 MiB dynamic code limit 时返回 `worker_code_too_large`。Deploy 和 secret mutation 在估算的 `workerLoader` env 超过 WDL headroomed 1 MiB budget 时返回 `worker_env_too_large`。`worker_env_too_large` details 包含 `namespace`、可选 `worker`、`env_bytes`、`max_env_bytes`、`upstream_max_env_bytes` 和 `headroom_bytes`。Deploy 阶段的逐版本检查还会包含 `version`。Secret mutation 重新估算既有 version 时，details 还会包含 `source_version` 和 `estimated_version`。`source_version` 是 operator 应检查、删除或 redeploy 的已存版本；`estimated_version` 是本次预算估算使用的 tag，在 worker-secret bump 路径中就是已分配的精确 bump version。 - Control 不直接调用 gateway。它写 Redis 并 publish invalidation message。 - Control 在进入 Redis mutation loop 前加密 secret PUT value。加密/provider 失败会返回 control error，不写 plaintext fallback。 diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index d91a975..2dc883f 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -70,6 +70,10 @@ structural JSON data capped at 1 MiB: finite numbers, strings, booleans, null, d arrays, and plain objects are accepted. Serialization does not invoke `toJSON()` hooks; sparse arrays, circular structures, non-plain objects, and non-JSON values fail before dispatch. +do-runtime invokes tenant alarm and RPC methods through private fetch dispatches +intercepted by the generated wrapper. Those requests carry the outer request id so the +host facade establishes invocation-local context without adding platform metadata to +the tenant argument list. DO invoke envelopes identify persisted bundles by canonical namespace, worker, version, and storage id. They do not accept inline worker source. DO host ids are capped at 512 UTF-8 bytes and use canonical `shardN` suffixes without @@ -186,8 +190,12 @@ when the logical worker is gone or now points at a different `doStorageId`. Owner resolution is the single-writer protocol: 1. do-runtime derives an owner scope from `doStorageId`, class name, and shard. -2. It WATCHes the owner record, generation key, and active worker storage pointer before - claiming or renewing. +2. Owner resolution WATCHes the owner record, generation key, worker delete lock, and + active worker storage pointer. A `whole` delete lock rejects ownership; a `version` + lock remains part of the watched snapshot but does not interrupt active storage. The + WATCH prevents a claim from committing after whole-worker delete starts. Renewal + separately WATCHes the owner record and active storage pointer; its generation fence + is carried by the owner record rather than a second generation-key read. 3. If a live owner exists on another task, the router returns that owner or an owner-hint header; the runtime facade may retry directly, but the owner task still rechecks the fence. @@ -307,8 +315,11 @@ recovery after the initial 101. fenced to the deleted `doStorageId`, so a same-name redeploy with a new storage id is not swept by the old delete. If best-effort cleanup fails, a far-future residual alarm job can remain in DB 2 until it becomes due; it then self-discards because the storage - pointer is gone. - `do:objects:` remains a tombstone for future platform cleanup. + pointer is gone. First owner claim watches the same per-worker delete lock and storage + pointer; only the `whole` lock kind rejects ownership, so deleting an inactive version + does not interrupt the active worker. A whole-worker delete therefore cannot miss + owner/generation state created after its final owner scan. `do:objects:` + remains a tombstone for future platform cleanup. - DO object registry writes are best-effort. Dispatch continues if the registry write fails, so the tombstone set may be incomplete; future cleanup must tolerate missing members and treat the active storage pointer plus owner/alarm state as the stronger diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 36ee9c4..dc92528 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -36,6 +36,7 @@ Storage cleanup endpoint 是 native facet storage cleanup 和 worker storage cle DO protocol error 使用 `{ error, message, details? }`。不同于 admin HTTP 的 flat additive error shape，DO protocol detail 会嵌套在 `details` 下，因为消费者是 runtime/DO client protocol，不是通用 admin JSON parser。未知 internal exception 仍会降级成安全的 `internal_error` / `Internal error` message。Storage delete-worker 在 partial batch result 时可能返回 HTTP 207 和 `{ ok:false, deleted, errors }`；这是 result envelope，不是 generic JSON error envelope。 Tenant-originated DO fetch body 在 runtime facade 中限制为 1 MiB。Facade 会在读取前拒绝超限的 `Content-Length`，streamed body 会增量读取，因此 limit 会在完整 buffering 前生效。 DO RPC method name 使用 JavaScript identifier grammar，并由 runtime client 和 do-runtime protocol reader 同时限制为最多 256 ASCII bytes。RPC 参数是最多 1 MiB 的 structural JSON data：接受 finite number、string、boolean、null、dense array 和 plain object。序列化不会调用 `toJSON()` hook；sparse array、circular structure、non-plain object 和 non-JSON value 会在 dispatch 前失败。 +do-runtime 会通过 generated wrapper 截获的私有 fetch dispatch 调用 tenant alarm 和 RPC method；这些 request 携带外层 request id，使 host facade 建立 invocation-local context，且不会把平台 metadata 加入 tenant argument list。 DO invoke envelope 通过 canonical namespace、worker、version 和 storage id 标识 persisted bundle，不接受 inline worker source。 DO host id 最多 512 UTF-8 bytes，并使用不带前导零的 canonical `shardN` suffix。 @@ -88,7 +89,7 @@ workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespa Owner resolution 是单写入协议： 1. do-runtime 从 `doStorageId`、class name 和 shard 派生 owner scope。 -2. Claim 或 renew 前 WATCH owner record、generation key 和 active worker storage pointer。 +2. Owner resolution 会 WATCH owner record、generation key、worker delete lock 和 active worker storage pointer。`whole` delete lock 会拒绝 ownership；`version` lock 仍属于 WATCH snapshot，但不会中断 active storage。这个 WATCH 会阻止 claim 在 whole-worker delete 开始后提交。Renew 会单独 WATCH owner record 和 active storage pointer；generation fence 已包含在 owner record 中，不需要再次读取 generation key。 3. 如果另一个 task 持有 live owner，router 返回该 owner 或 owner-hint header；runtime facade 可以直连重试，但 owner task 仍会重新检查 fence。 4. 如果 owner 缺失或过期，claimant 在一个 Redis transaction 中递增 monotonic generation counter，并写入带 TTL 的 owner record。 5. Local dispatch 使用 native facet 前检查 `taskId`、`generation`、lease expiry、active `doStorageId` 和剩余 lease budget。stale generation、expired lease 或 storage pointer 改变都会 fail closed。剩余 lease 小于 `DO_OWNER_LEASE_GUARD_MS`（默认 `1000`）时，owner 会先尝试 same-task、same-generation CAS renew；如果 renew 失败，才 fail closed。这个 guard 缩窄 takeover window，但不是 per-SQL-call 或 SQLite commit-time fence。 @@ -146,7 +147,7 @@ do-runtime 围绕 owner resolution、dispatch、alarm execution、drain、renew ## 已知约束和非目标 - 当前 lifecycle 不会在 worker delete 时物理清除 native facet SQLite storage。 -- Whole-worker delete 会删除 active `worker:do-storage::` pointer，并在 delete commit 后请求 Workflows 删除 internal DO alarm jobs；pointer 消失后，旧 facet 的 late `setAlarm()` 写入会被忽略。Cleanup 会 fence 到被删除的 `doStorageId`，因此同名 worker 以新 storage id redeploy 后不会被旧 delete 扫掉。如果 best-effort cleanup 失败，远期残留 alarm job 可能留在 DB 2 直到到期；随后 dispatch 会因为 storage pointer 已消失而自清理。`do:objects:` 会作为未来 platform cleanup 的 tombstone 保留。 +- Whole-worker delete 会删除 active `worker:do-storage::` pointer，并在 delete commit 后请求 Workflows 删除 internal DO alarm jobs；pointer 消失后，旧 facet 的 late `setAlarm()` 写入会被忽略。Cleanup 会 fence 到被删除的 `doStorageId`，因此同名 worker 以新 storage id redeploy 后不会被旧 delete 扫掉。如果 best-effort cleanup 失败，远期残留 alarm job 可能留在 DB 2 直到到期；随后 dispatch 会因为 storage pointer 已消失而自清理。First owner claim 会 WATCH 同一把 per-worker delete lock 和 storage pointer；只有 `whole` lock kind 会拒绝 ownership，因此删除 inactive version 不会中断 active worker，而 whole-worker delete 也不会漏掉最终 owner scan 之后创建的 owner/generation state。`do:objects:` 会作为未来 platform cleanup 的 tombstone 保留。 - DO object registry 写入是 best-effort。Registry 写失败时 dispatch 会继续，因此 tombstone set 可能不完整；未来 cleanup 必须容忍缺失 member，并把 active storage pointer、owner/alarm state 当成更强的 lifecycle signal。 - Gateway-held WebSocket recovery 只对 client connection continuity 做 best-effort。Backend DO facet 在初始 `101` 之后不会逐消息 re-fence；owner handoff 安全依赖 reconnect/rebind 行为，以及创建 backend facet 前运行的 owner-side dispatch fence。Gateway 重置 backend reconnect epoch 时，旧 epoch 下排队的 client message 可能被丢弃，且没有逐帧 ack/nack。 - Owner-hinted WebSocket direct retry 失败不会 fall back 到 router，因为最终 101 必须来自 owner endpoint。 diff --git a/docs/modules/infra.md b/docs/modules/infra.md index 30f970b..a8afb97 100644 --- a/docs/modules/infra.md +++ b/docs/modules/infra.md @@ -130,6 +130,11 @@ Stateful storage: multi-replica safe, but rollout can still pause scheduling because ECS uses stop-before-start replacement. - Workflows is a separate Rust service. +- By default, Scheduler and Workflows drain in-flight work for up to 25 seconds. + Compose, Kubernetes, and Terraform ECS pin 30-second stop windows so the platform + does not terminate either process before its default application drain closes. + Deployments that override `SCHEDULER_SHUTDOWN_DRAIN_MS` or + `WORKFLOWS_SHUTDOWN_DRAIN_MS` should review the corresponding stop window. - Gateway, user-runtime, and system-runtime can be horizontally replicated behind the environment's load balancing and service discovery layer. Local route caches, loaded isolates, and owner hints are optimizations, not authority. diff --git a/docs/modules/infra.zh.md b/docs/modules/infra.zh.md index 4cd55e0..1cd3a8b 100644 --- a/docs/modules/infra.zh.md +++ b/docs/modules/infra.zh.md @@ -80,6 +80,7 @@ Stateful storage： - Scheduler 部署默认 1 个副本；当前 dispatch 路径具备多副本安全性，但 ECS rollout 仍使用 stop-before-start replacement，部署期间可能短暂停止调度。 - Workflows 是独立 Rust service。 +- Scheduler 和 Workflows 默认最多 drain 25 秒 in-flight work；Compose、Kubernetes 和 Terraform ECS 均固定 30 秒 stop window，避免平台在默认应用 drain 结束前终止进程。部署如覆盖 `SCHEDULER_SHUTDOWN_DRAIN_MS` 或 `WORKFLOWS_SHUTDOWN_DRAIN_MS`，应同步评估对应的 stop window。 - Gateway、user-runtime 和 system-runtime 可以放在环境的 load balancing / service discovery 层后面水平扩展。本地 route cache、已加载 isolate 和 owner hint 都是优化，不是 authority。 - D1/DO 使用 owner lease、monotonic generation fence 和本地 drain/renew。超过 1 个 task 时需要稳定的 per-replica storage identity，并确保 supervisor drain/renew 只走私有本地入口，不能通过 service alias 打到其他 replica。 - 普通 D1/DO task 丢失会退回到 lease expiry，再由其他 replica takeover；graceful rollout 应优先走 supervisor drain，在 child workerd process 退出前释放 ownership。 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index 6850cb6..2aaa85b 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -145,6 +145,11 @@ Common rules: `request_id` on `request_complete`; Rust sidecars sanitize inbound ids and log them where request middleware owns completion. Control's Redis `PUBLISH` path logs the id locally but does not put it in the pub/sub payload. +- Loaded-worker host facades keep request ids in invocation-local ALS across native + Promise and `async` continuations. Private do-runtime fetch, alarm, and RPC dispatches + carry the id in `x-request-id`. Custom thenables are not a supported propagation + boundary; the wrapper does not inspect or replace tenant return values to extend it. + Nested handler calls without a new id inherit the current ALS context. - Logs use snake_case fields. Only `level=error` is emitted on stderr; debug/info/warn JSON log lines go to stdout so log routing stays identical across JS, Rust, and embedded workerd shims. diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index 62ae958..46913e5 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -92,6 +92,7 @@ WDL 在 JS workerd tier 和 Rust 服务中使用同一套可观测性策略： 通用规则： - `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker 和 D1 传播。缺失的 inbound id 会在入口生成；header adapter 只考虑第一个逗号分隔 token，包含 visible ASCII 以外字节、引号、反斜杠或超过 128 字节的 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 +- Loaded-worker host facade 通过 invocation-local ALS 在 native Promise 和 `async` continuation 中保留 request id；do-runtime 的私有 fetch、alarm 和 RPC dispatch 会在 `x-request-id` 中携带该 id。Custom thenable 不属于受支持的传播边界；wrapper 不会为延长 context 而检查或替换 tenant 返回值。没有新 id 的嵌套 handler 调用会继承当前 ALS context。 - 日志字段使用 snake_case。只有 `level=error` 写入 stderr；debug/info/warn JSON log line 都写入 stdout，这样 JS、Rust 和 embedded workerd shim 的日志路由保持一致。 - 内部运维日志，包括 system-worker cleanup 日志和防御性的 Redis callback warning，也使用同一套单行 JSON envelope：`ts`、`service`、`level`、`event`，再加 snake_case 字段。JS 服务使用 `shared/observability.js`；Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`；无法转成文本的 throwable 降级为 `Unknown error`，不能反向覆盖原始失败。不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 - 产品 API response body 默认使用 camelCase，除非 endpoint 明确记录不同 wire contract。 diff --git a/docs/modules/queues-cron.md b/docs/modules/queues-cron.md index e32d6ef..8198901 100644 --- a/docs/modules/queues-cron.md +++ b/docs/modules/queues-cron.md @@ -40,8 +40,9 @@ Scheduler owns runtime delivery: route namespace, worker name, and version with the canonical `wdl-rust-common` grammar. Invalid cron refs are removed before slot advance or runtime dispatch, and cron sweep does not recreate refs from malformed worker metadata. Queue consumer - projections remain trusted Control-owned internal state; scheduler does not add a - second worker identity/version validator for unsupported direct Redis writes. + dispatch identities are also revalidated before use. A noncanonical identity makes + the projection unusable and follows absent-consumer handling, so queued backlog may + move to the orphan stream. - Scheduler defaults to one replica in deployment, and current dispatch paths are multi-replica safe. Extra replicas improve runtime concurrency but do not imply zero-gap deployment: production rollout may still use stop-before-start semantics and @@ -143,7 +144,8 @@ Queue dispatch is stream-driven rather than wall-clock driven: failures that cannot become valid by retrying the same bytes, such as queue-message decode failures or invalid queue dispatch bodies, go directly to DLQ without consuming the retry budget. Aggregate request-body-too-large responses are split - and retried with smaller batches first. + and retried with smaller batches first. Scheduler bounds each runtime response body + to 1 MiB before parsing it. 5. The delayed loop wakes from `queue-delayed-wake` and from wall-clock sleeps until the next due delayed member. Each due member first takes a `queue-delayed-claim:*` lease sized to `SCHEDULER_FIRE_TIMEOUT_MS + 5000ms`; the winner moves it back to the main diff --git a/docs/modules/queues-cron.zh.md b/docs/modules/queues-cron.zh.md index 71df338..0962a04 100644 --- a/docs/modules/queues-cron.zh.md +++ b/docs/modules/queues-cron.zh.md @@ -23,7 +23,7 @@ Scheduler 负责实际投递： - Cron 代码在 `rust/scheduler/src/cron/`。 - Queue registry、consume、delayed delivery、DLQ 和 orphan 处理在 `rust/scheduler/src/queue/`。 -- Scheduler 在构造 cron runtime worker id 前会用 canonical `wdl-rust-common` grammar 重新校验 projection 的 route namespace、worker name 和 version。非法 cron ref 会在 slot advance 或 runtime dispatch 前移除，cron sweep 也不会从 malformed worker metadata 重新播种 ref。Queue consumer projection 仍是受信任的 Control-owned internal state；scheduler 不为不受支持的 direct Redis write 增加第二套 worker identity/version validator。 +- Scheduler 在构造 cron runtime worker id 前会用 canonical `wdl-rust-common` grammar 重新校验 projection 的 route namespace、worker name 和 version。非法 cron ref 会在 slot advance 或 runtime dispatch 前移除，cron sweep 也不会从 malformed worker metadata 重新播种 ref。Queue consumer 的 dispatch identity 也会在使用前重新校验；noncanonical identity 会使 projection 不可用，并按 consumer absent 处理，因此 backlog 可能进入 orphan stream。 - 部署上 scheduler 默认 1 个副本，当前 dispatch 路径具备多副本安全性。增加副本可以提高运行时并发，但不等于部署零中断：生产 rollout 仍可采用 stop-before-start 语义，并短暂暂停调度。 ## 接口 @@ -74,7 +74,7 @@ Queue dispatch 是 stream-driven，不是 wall-clock driven： 1. Producer 把 message envelope 写入 DB 1 stream；当 `delivery_delay` 或 retry delay 非零时，先写入 delayed ZSET。 2. Scheduler reconcile `queue:index:*` discovery set，为 live stream 创建固定的 `wdl-scheduler` consumer group，并维护一个内存中的 known delayed queue 集合。空 queue index 可以从权威 hash、stream 和 delayed ZSET 一次性 backfill；之后 projection 由 writer 维护，stale-index cleanup 由 reconcile 负责。 3. Consume loop 使用 `XREADGROUP` 读取 main stream，并按 `max_batch_size` 组 batch 投递，且 hard cap 为 `100`。每次 read 会用当前 active stream set 的 consumer batch-size snapshot 限制 `COUNT`，避免一次 poll 把超过当前 consumer 单批 dispatch 能力的 entries 放进 PEL。PEL reap 在 consumer 仍存在时使用同一 per-consumer cap；consumer 已消失的 orphan movement 仍可按 hard cap 分页。Consume 和 PEL reap 可以在 queue semaphore 下按 stream 并行 dispatch。每条 dispatch path 在向 runtime 发送 message 前都会重读该 stream 的权威 `queue-consumer` hash 并刷新内存 registry，因此 consumer promote 后，一旦 message 被选中，就不需要等下一次 reconcile tick 才使用新版本。当前模型里 `max_batch_timeout_ms` 不是 batching wait window。 -4. Runtime 返回 queue outcome envelope。Scheduler 解析 explicit `ack`、explicit `retry`、batch retry 和 implicit ack，然后用 Redis pipeline 执行 `XACK`/`XDEL`、delayed retry `ZADD`、DLQ append 或 immediate retry write。不会因为重试同一批 bytes 变好的平台失败，例如 queue message decode failure 或非法 queue dispatch body，会直接进 DLQ，不消耗 retry budget。聚合 request-body-too-large 会先拆成更小 batch 重试。 +4. Runtime 返回 queue outcome envelope。Scheduler 解析 explicit `ack`、explicit `retry`、batch retry 和 implicit ack，然后用 Redis pipeline 执行 `XACK`/`XDEL`、delayed retry `ZADD`、DLQ append 或 immediate retry write。不会因为重试同一批 bytes 变好的平台失败，例如 queue message decode failure 或非法 queue dispatch body，会直接进 DLQ，不消耗 retry budget。聚合 request-body-too-large 会先拆成更小 batch 重试；scheduler 在解析前把每个 runtime response body 限制为 1 MiB。 5. Delayed loop 由 `queue-delayed-wake` 和下一条 due member 的 wall-clock sleep 唤醒。每个到期 member 会先取得 `queue-delayed-claim:*` lease，TTL 为 `SCHEDULER_FIRE_TIMEOUT_MS + 5000ms`；抢到的副本把它移回 main stream，或在 consumer 已消失时移入 orphan stream。 6. Orphan / Pending-Entry cleanup 是诊断和保护机制。它防止 consumer 删除或 scheduler crash 路径静默丢消息，但 main queue stream 仍是 durable backlog，并且故意不 trim。Delayed ZSET 和 orphan stream-tail migration 受 `QUEUE_SWEEP_BATCH_SIZE` 分页控制，默认 `100`。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 8ad7263..296a209 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -205,11 +205,20 @@ opaque cold-load failures under the current stock binary. workflow dispatches do not. - Wrapper generation hides raw env from unwrapped entrypoints whenever privileged internal Fetchers are injected. Its host-wrapper runtime evaluates before tenant - modules and captures every intrinsic used to decide handler or env wrapping and to - settle request-context cleanup, so tenant top-level prototype mutation cannot bypass - that boundary or retain stale request context. + modules and captures the intrinsics used to decide handler or env wrapping, so tenant + top-level prototype mutation cannot bypass the env boundary. +- Host facade request ids use an invocation-local `AsyncLocalStorage` store. Runtime + ensures that the loaded worker has ALS support: an existing `nodejs_compat` setup is + left unchanged; otherwise a standalone `no_nodejs_als` is replaced by the narrow + `nodejs_als` flag. Concurrent calls on a persistent Durable Object instance therefore + cannot overwrite one shared context. The wrapper does not inspect, mutate, or replace + tenant return values. Native Promise and `async` continuations retain context; custom + thenables are not a supported request-id propagation boundary. A nested handler call + without a new request id inherits the current ALS context instead of clearing it. - Request context wrappers swap facade objects into env and propagate request id where - that event class can carry it. + that event class can carry it. do-runtime alarm and RPC calls enter through private + fetch dispatches carrying the outer request id, so the generated wrapper establishes + invocation-local context without adding platform metadata to tenant method arguments. - An uncaught tenant `fetch()` exception maps to a platform `502 runtime_error` response with the request id. Exception details are emitted to structured logs/live tail and are not copied into the client body. Tail formatting cannot replace the diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index aeb04f7..1a3b3d3 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -82,8 +82,9 @@ Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materializ - workerLoader cache 没有 LRU。Runtime 给每个 loaded worker 注入 `__WdlAbort__`，并在 active-version cold-load 时 evict sibling historical versions。 - Service-binding cold load 会记录 loaded version，但不 evict sibling，因为 service binding 可能有意指向 frozen historical version。 - Internal active-version scheduled/queue dispatch 会 opt into sibling eviction；frozen workflow dispatch 不会。 -- 只要注入 privileged internal Fetcher，wrapper generation 就会避免 raw env 暴露给未包装 entrypoint。Host-wrapper runtime 会在 tenant module 前执行，并捕获所有参与 handler/env wrapping 决策和 request-context cleanup 的 intrinsic，因此 tenant 顶层 prototype mutation 不能绕过这条边界或遗留 stale request context。 -- Request context wrapper 会把 facade object 换进 env，并在事件类型允许时传播 request id。 +- 只要注入 privileged internal Fetcher，wrapper generation 就会避免 raw env 暴露给未包装 entrypoint。Host-wrapper runtime 会在 tenant module 前执行，并捕获参与 handler/env wrapping 决策的 intrinsic，因此 tenant 顶层 prototype mutation 不能绕过 env 边界。 +- Host facade request id 使用 invocation-local `AsyncLocalStorage` store。已有 `nodejs_compat` 配置会保持不变；否则 Runtime 会把 standalone `no_nodejs_als` 替换为范围更窄的 `nodejs_als` flag。持久 Durable Object 实例上的并发调用不会再覆盖同一个共享 context，wrapper 也不会检查、修改或替换 tenant 返回值。Native Promise 和 `async` continuation 会保留 context；custom thenable 不属于受支持的 request-id propagation boundary。没有新 request id 的嵌套 handler 调用会继承当前 ALS context，而不是把它清空。 +- Request context wrapper 会把 facade object 换进 env，并在事件类型允许时传播 request id。do-runtime alarm 和 RPC 通过携带外层 request id 的私有 fetch dispatch 进入，使 generated wrapper 建立 invocation-local context，而不会把平台 metadata 加入 tenant method argument。 - Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error` response，并带 request id。异常细节输出到结构化日志/live tail，不复制进客户端 body；throwable 自身无法转成字符串时，tail formatting 也不能反向覆盖原始异常。 - Internal scheduled、queue 和 workflow dispatch route 使用 result envelope 表达 handler outcome。Tenant handler error 是 scheduler/workflow 协议中的 outcome state，不是 generic platform transport error。 - Runtime 没有 route-cache invalidation protocol。`workerLoader` cache key 是不可变 worker id，因此 promote 后的新 version 是新 key，会自然 cold-load。 diff --git a/docs/redis-key-layout.md b/docs/redis-key-layout.md index d86c0b8..322aea6 100644 --- a/docs/redis-key-layout.md +++ b/docs/redis-key-layout.md @@ -36,7 +36,9 @@ workers: Set, worker names with worker-owned lifecycle st worker:::next_version String, monotonic version counter, survives delete worker-versions:: ZSET, score=int version, member="v" worker:::v: Hash, bundle bytes plus __meta__ -worker-delete-lock:: String EX 30, per-worker delete critical-section lock +worker-delete-lock:: String EX 30, per-worker delete critical-section lock; + value is whole: or version:; DO first-owner + claim WATCHes it and only whole fences ownership worker-version-referrers::: Set, canonical JSON version-pinned caller refs hosts: Set, declared operator host intent diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index 51350c3..34b36c6 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -21,7 +21,7 @@ workers: Set, 有 worker-owned lifecycle state 的 worker worker:::next_version String, 单调 version counter，delete 后保留 worker-versions:: ZSET, score=int version, member="v" worker:::v: Hash, bundle bytes + __meta__ -worker-delete-lock:: String EX 30, 每个 worker 的 delete critical-section lock +worker-delete-lock:: String EX 30, 每个 worker 的 delete critical-section lock；value 是 whole: 或 version:；DO first-owner claim 会 WATCH，且只有 whole 阻止 ownership worker-version-referrers::: Set, canonical JSON 的 version-pinned caller ref hosts: Set, operator 声明的 host intent diff --git a/docs/source-map.md b/docs/source-map.md index 720e073..9486814 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -71,7 +71,7 @@ are outside this map unless they own runtime or deployable service behavior. | `shared/respond.js` | Shared HTTP response, JSON error, Prometheus text, best-effort response body discard, and `x-request-id` echo helpers. | | `shared/bounded-body.js` | Shared bounded byte-stream and request-body readers; each tier maps limit errors to its own contract. | | `shared/ns-pattern.js` | Platform-domain normalization plus namespace, worker, binding, queue, KV/D1/R2 id, module path, reserved object-key, and reserved namespace grammars. | -| `shared/version.js` | Worker version formatting plus worker and route-plane Redis key helpers. | +| `shared/version.js` | Worker version formatting plus worker, route-plane, and lifecycle Redis key helpers. | | `shared/workerd-compat-flags.js` | Pinned mirror of upstream workerd experimental compatibility enable flags used to reject tenant metadata before cold-load. | | `shared/queue-keys.js` | JavaScript queue key helpers used by tests and cross-tier key-shape checks. | | `shared/route-projection.js` | Compact pattern-route projection encoding shared by control writers, delete checks, and gateway readers. | diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index 7b9208b..4d0b4b4 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -68,7 +68,7 @@ | `shared/respond.js` | 共享 HTTP response、JSON error、Prometheus text、best-effort response body discard 和 `x-request-id` echo helpers。 | | `shared/bounded-body.js` | 共享 bounded byte-stream 和 request-body readers；各 tier 自己把 limit error 映射为对应 contract。 | | `shared/ns-pattern.js` | Platform-domain normalization，以及 namespace、worker、binding、queue、KV/D1/R2 id、module path、reserved object-key 和 reserved namespace grammars。 | -| `shared/version.js` | Worker version formatting，以及 worker 和 route-plane Redis key helpers。 | +| `shared/version.js` | Worker version formatting，以及 worker、route-plane 和 lifecycle Redis key helpers。 | | `shared/workerd-compat-flags.js` | 上游 workerd experimental compatibility enable flags 的 pinned mirror，用于在 cold-load 前拒绝 tenant metadata。 | | `shared/queue-keys.js` | JavaScript queue key helpers，供 tests 和 cross-tier key-shape checks 使用。 | | `shared/route-projection.js` | Control writer、delete check 和 gateway reader 共用的紧凑 pattern-route projection encoding。 | diff --git a/package-lock.json b/package-lock.json index ce26d41..4e696e7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "license": "Apache-2.0", "author": "Sean Consulting OÜ", "dependencies": { - "@wdl-dev/aws-sigv4": "^2.0.0", + "@wdl-dev/aws-sigv4": "^3.0.1", "croner": "^10.0.1" } }, @@ -1549,9 +1549,9 @@ } }, "node_modules/@wdl-dev/aws-sigv4": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@wdl-dev/aws-sigv4/-/aws-sigv4-2.0.0.tgz", - "integrity": "sha512-7Fh4/i8OaRs/P4V4sf0z7gFEsvc1joHTQK6thc4VWwmie6OqI6UpJ7wYz8gPfFzBIQYfmtD6WwwcdeybPHotkg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@wdl-dev/aws-sigv4/-/aws-sigv4-3.0.1.tgz", + "integrity": "sha512-XXdWT8OQt7TeJd8MQ9QXQJGFpMcnE3HS89mGvdi0Oe1Zx7y78mUK27gTYZXkMlbhsAsZsfCeg2XdDN1j2x/ggA==", "license": "Apache-2.0", "engines": { "node": ">=24" diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index 364005c..e51df47 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -86,6 +86,7 @@ const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; const intrinsicObjectSetPrototypeOf = Object.setPrototypeOf; const intrinsicPromiseThen = Promise.prototype.then; +const intrinsicReadableStreamCancel = ReadableStream.prototype.cancel; const intrinsicReadableStreamGetReader = ReadableStream.prototype.getReader; const intrinsicReadableStreamReaderCancel = ReadableStreamDefaultReader.prototype.cancel; const intrinsicReadableStreamReaderRead = ReadableStreamDefaultReader.prototype.read; @@ -329,6 +330,20 @@ function cancelStreamReader(reader) { } } +/** @param {Response} response */ +function cancelResponseBody(response) { + // This source is injected into loaded workers, so keep cleanup local rather + // than adding shared-respond as another injected facade module. + try { + const body = responseBody(response); + if (!body) return; + const promise = intrinsicReflectApply(intrinsicReadableStreamCancel, body, []); + intrinsicReflectApply(intrinsicPromiseThen, promise, [undefined, () => {}]); + } catch { + // Best-effort cleanup only; the replacement response owns behavior. + } +} + /** @param {Response} response */ function responseBody(response) { return intrinsicReflectApply(intrinsicResponseBodyGet, response, []); @@ -800,8 +815,10 @@ function validOwnerEndpoint(endpoint) { export async function followOwnerHint(response, ownerFetch, pathname, init) { const hint = await ownerHintFromResponse(response); if (!hint || typeof ownerFetch !== "function") return response; + cancelResponseBody(response); const direct = await ownerFetch(ownerRequestUrl(hint, pathname), init); if (ownerEndpointUnavailableResponse(direct)) { + cancelResponseBody(direct); throw new Error(`DO owner endpoint returned ${responseStatus(direct)}`); } return direct; @@ -846,10 +863,13 @@ export async function dispatchDoRequestWithOwnerHint({ try { const direct = await ownerFetch(ownerRequestUrl(cachedHint, ownerPath), init); if (await ownerHintFromResponse(direct)) { + cancelResponseBody(direct); clearHint(); } else if (await staleCachedResponse(direct)) { + cancelResponseBody(direct); clearHint(); } else if (ownerEndpointUnavailableResponse(direct)) { + cancelResponseBody(direct); clearHint(); if (replayOwnerUnavailable) { return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); @@ -883,6 +903,14 @@ export async function dispatchDoRequestWithOwnerHint({ rememberHint(hinted); try { const direct = await followOwnerHint(routed, ownerFetch, ownerPath, init); + if (await ownerHintFromResponse(direct)) { + cancelResponseBody(direct); + clearHint(); + if (replayOwnerUnavailable) { + return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); + } + return ownerUnavailableResponse(); + } const learned = ownerHintFromHeaders(responseHeaders(direct)); if (learned) rememberHint(learned); return direct; @@ -925,6 +953,7 @@ async function dispatchDoWithHintCache({ replayOwnerUnavailable, }); if (retryOwnerRace && await retryableOwnerRaceResponse(response)) { + cancelResponseBody(response); cache.delete(hintKey); response = await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); } diff --git a/runtime/load/code-budget.js b/runtime/load/code-budget.js index 39a9c84..b92f2b1 100644 --- a/runtime/load/code-budget.js +++ b/runtime/load/code-budget.js @@ -21,6 +21,9 @@ import { export const WORKER_LOADER_CODE_MAX_BYTES = 64 * 1024 * 1024; const D1_DATA_FIELD_MODULE_NAME = "_wdl-d1-data-field.js"; +const NODEJS_ALS_COMPATIBILITY_FLAG = "nodejs_als"; +const NODEJS_COMPATIBILITY_FLAG = "nodejs_compat"; +const NO_NODEJS_ALS_COMPATIBILITY_FLAG = "no_nodejs_als"; const utf8Decoder = new TextDecoder(); /** @@ -332,6 +335,18 @@ export function injectRuntimeModulesForHostBindings( for (const [name, source] of runtimeInjectedModuleSources(originalMain, meta, runtimeSources, plan)) { workerCode.modules[name] = source; } + if (plan.needsHostBindingWrapper) { + const flags = Array.isArray(workerCode.compatibilityFlags) + ? /** @type {string[]} */ (workerCode.compatibilityFlags) + : []; + if (!flags.includes(NODEJS_COMPATIBILITY_FLAG)) { + const normalized = flags.filter((flag) => flag !== NO_NODEJS_ALS_COMPATIBILITY_FLAG); + if (!normalized.includes(NODEJS_ALS_COMPATIBILITY_FLAG)) { + normalized.push(NODEJS_ALS_COMPATIBILITY_FLAG); + } + workerCode.compatibilityFlags = normalized; + } + } workerCode.mainModule = "_wdl-wrapper.js"; return workerCode; } diff --git a/runtime/load/wrapper-generate.js b/runtime/load/wrapper-generate.js index 8b79fcf..1e58ded 100644 --- a/runtime/load/wrapper-generate.js +++ b/runtime/load/wrapper-generate.js @@ -12,21 +12,36 @@ const DEFAULT_EXPORT_CLASS_TEST_SOURCE = "/^\\s*class\\b/.test(source)"; export const HOST_BINDING_RUNTIME_MODULE_NAME = "_wdl-host-wrapper-runtime.js"; export const HOST_BINDING_RUNTIME_SOURCE = ` +import { AsyncLocalStorage } from "node:async_hooks"; + const IntrinsicObject = Object; -const IntrinsicPromise = Promise; const IntrinsicProxy = Proxy; const IntrinsicReflect = Reflect; const IntrinsicSymbol = Symbol; +const intrinsicAsyncLocalStorageGetStore = AsyncLocalStorage.prototype.getStore; +const intrinsicAsyncLocalStorageRun = AsyncLocalStorage.prototype.run; const intrinsicArrayForEach = Array.prototype.forEach; const intrinsicFunctionToString = Function.prototype.toString; const intrinsicObjectDefineProperty = Object.defineProperty; const intrinsicObjectEntries = Object.entries; const intrinsicObjectKeys = Object.keys; -const intrinsicPromiseResolve = Promise.resolve; -const intrinsicPromiseThen = Promise.prototype.then; const intrinsicReflectApply = Reflect.apply; const intrinsicReflectGet = Reflect.get; const intrinsicRegExpTest = RegExp.prototype.test; +const requestContext = new AsyncLocalStorage(); + +intrinsicReflectApply(intrinsicObjectDefineProperty, IntrinsicObject, [ + requestContext, + "getStore", + { + configurable: false, + enumerable: false, + value() { + return intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); + }, + writable: false, + }, +]); export function applyFunction(fn, receiver, args) { return intrinsicReflectApply(fn, receiver, args); @@ -69,19 +84,21 @@ export function regexpTest(regexp, value) { return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); } -export function settleWithFinally(value, callback) { - const promise = intrinsicReflectApply(intrinsicPromiseResolve, IntrinsicPromise, [value]); - return intrinsicReflectApply(intrinsicPromiseThen, promise, [ - (result) => { - callback(); - return result; - }, - (error) => { - callback(); - throw error; - }, +export function currentRequestId() { + const requestId = intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); + return typeof requestId === "string" && requestId ? requestId : null; +} + +export function runWithRequestContext(requestId, callback) { + if (typeof requestId !== "string" || !requestId) { + return intrinsicReflectApply(callback, undefined, []); + } + return intrinsicReflectApply(intrinsicAsyncLocalStorageRun, requestContext, [ + requestId, + callback, ]); } + `; /** @param {string} userMainSpecifier */ @@ -142,44 +159,34 @@ export function generateHostBindingWrapperModule(userMainSpecifier, d1Bindings, const starExport = hidesRawEnvExports ? "// Internal Fetcher bindings are present; only wrapped entrypoints are re-exported." : `export * from ${userMain};`; - const namedEntrypoints = entrypointNames.map((/** @type {string} */ name) => ` -export class ${name} extends user.${name} { - constructor(ctx, env) { - const requestContext = createRequestContext(); - super(ctx, wrapEnv(env, requestContext)); - return wrapClassInstance(this, requestContext); - } -} + const namedEntrypoints = entrypointNames.map((/** @type {string} */ name, index) => ` +const __WdlWrappedEntrypoint${index}__ = ({ + [${JSON.stringify(name)}]: class extends __WdlUserModule__.${name} { + constructor(ctx, env) { + super(ctx, wrapEnv(env)); + return wrapClassInstance(this); + } + }, +})[${JSON.stringify(name)}]; +export { __WdlWrappedEntrypoint${index}__ as ${name} }; `).join(""); return ` import { WorkerEntrypoint, abortIsolate } from "cloudflare:workers"; -import { - applyFunction, - createPrivateSymbol, - createProxy, - defineProperty, - forEachArray, - forEachObjectEntry, - functionSource, - objectKeys, - reflectGet, - regexpTest, - settleWithFinally, -} from "./${HOST_BINDING_RUNTIME_MODULE_NAME}"; +import * as __WdlHostRuntime__ from "./${HOST_BINDING_RUNTIME_MODULE_NAME}"; ${d1Import} ${r2Import} ${doImport} ${workflowImport} -import * as user from ${userMain}; -// Local wrapper subclasses intentionally shadow same-name user exports, so -// declared entrypoints get facade-aware env even when star exports are present. +import * as __WdlUserModule__ from ${userMain}; +// Explicit aliases replace same-name star exports without declaring tenant +// entrypoint names in the generated module's local binding scope. ${starExport} ${WDL_ABORT_SHIM_SOURCE} export class __WdlWorkflowNotify__ extends WorkerEntrypoint { - async fetch(request) { - return await notifyWorkflowCallback(request, wrapEnv(this.env, requestIdFromEventArg(request))); + fetch(request) { + return withRequestContext(request, () => notifyWorkflowCallback(request, wrapEnv(this.env))); } } @@ -190,7 +197,7 @@ const WORKFLOW_BINDINGS = ${workflowBindingJson}; const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; -const HOST_BINDINGS_WRAPPED = createPrivateSymbol("wdl.host-bindings-wrapped"); +const HOST_BINDINGS_WRAPPED = __WdlHostRuntime__.createPrivateSymbol("wdl.host-bindings-wrapped"); const INTERNAL_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; function requestIdFromEventArg(arg) { @@ -198,59 +205,35 @@ function requestIdFromEventArg(arg) { return arg.headers.get("x-request-id"); } -function createRequestContext(requestId = null) { - // workerd constructs a fresh WorkerEntrypoint instance per call; covered by - // service-bindings-rpc.test.js. Do not share this object across instances. - return { requestId }; +function requestIdOptions() { + return { requestIdProvider: __WdlHostRuntime__.currentRequestId }; } -function requestIdOptions(requestIdOrContext) { - if (requestIdOrContext && typeof requestIdOrContext === "object") { - return { requestIdProvider: () => requestIdOrContext.requestId }; - } - return { requestId: requestIdOrContext }; +function doOptions(backend, ownerNetwork) { + return { ...requestIdOptions(), backend, ownerNetwork }; } -function doOptions(requestIdOrContext, backend, ownerNetwork) { - return { ...requestIdOptions(requestIdOrContext), backend, ownerNetwork }; +function workflowOptions(backend) { + return { ...requestIdOptions(), backend }; } -function workflowOptions(requestIdOrContext, backend) { - return { ...requestIdOptions(requestIdOrContext), backend }; +function withRequestContext(arg, fn) { + return __WdlHostRuntime__.runWithRequestContext(requestIdFromEventArg(arg), fn); } -function withRequestContext(context, arg, fn) { - const previous = context.requestId; - const requestId = requestIdFromEventArg(arg); - if (requestId) context.requestId = requestId; - try { - const result = fn(); - if (result && typeof result.then === "function") { - return settleWithFinally(result, () => { - context.requestId = previous; - }); - } - context.requestId = previous; - return result; - } catch (err) { - context.requestId = previous; - throw err; - } -} - -function wrapClassInstance(instance, requestContext) { - return createProxy(instance, { +function wrapClassInstance(instance) { + return __WdlHostRuntime__.createProxy(instance, { get(target, prop) { - const value = reflectGet(target, prop, target); + const value = __WdlHostRuntime__.reflectGet(target, prop, target); if (typeof value !== "function") return value; return function(...args) { - return withRequestContext(requestContext, args[0], () => applyFunction(value, target, args)); + return withRequestContext(args[0], () => __WdlHostRuntime__.applyFunction(value, target, args)); }; }, }); } -function wrapEnv(env, requestIdOrContext = null) { +function wrapEnv(env) { // Idempotence is a contract, not an optimization: WorkerEntrypoint methods // and default handlers may re-enter with an env already wrapped by this // module. A symbol marker cannot be forged by tenant vars/secrets. @@ -262,26 +245,26 @@ function wrapEnv(env, requestIdOrContext = null) { delete out[DO_BACKEND_BINDING]; delete out[DO_OWNER_NETWORK_BINDING]; delete out[WORKFLOWS_BACKEND_BINDING]; - forEachArray(objectKeys(out), (name) => { - if (regexpTest(INTERNAL_BINDING_RE, name)) delete out[name]; + __WdlHostRuntime__.forEachArray(__WdlHostRuntime__.objectKeys(out), (name) => { + if (__WdlHostRuntime__.regexpTest(INTERNAL_BINDING_RE, name)) delete out[name]; }); - forEachArray(D1_BINDINGS, (name) => { - if (out[name] !== undefined) out[name] = new D1Database(out[name], requestIdOptions(requestIdOrContext)); + __WdlHostRuntime__.forEachArray(D1_BINDINGS, (name) => { + if (out[name] !== undefined) out[name] = new D1Database(out[name], requestIdOptions()); }); - forEachArray(R2_BINDINGS, (name) => { - if (out[name] !== undefined) out[name] = new R2Bucket(out[name], requestIdOptions(requestIdOrContext)); + __WdlHostRuntime__.forEachArray(R2_BINDINGS, (name) => { + if (out[name] !== undefined) out[name] = new R2Bucket(out[name], requestIdOptions()); }); - forEachArray(DO_BINDINGS, (name) => { + __WdlHostRuntime__.forEachArray(DO_BINDINGS, (name) => { if (out[name] !== undefined) { - out[name] = new DurableObjectNamespace(out[name], doOptions(requestIdOrContext, doBackend, doOwnerNetwork)); + out[name] = new DurableObjectNamespace(out[name], doOptions(doBackend, doOwnerNetwork)); } }); - forEachObjectEntry(WORKFLOW_BINDINGS, (name, metadata) => { + __WdlHostRuntime__.forEachObjectEntry(WORKFLOW_BINDINGS, (name, metadata) => { if (out[name] !== undefined) { - out[name] = new Workflow(out[name] || metadata, workflowOptions(requestIdOrContext, workflowsBackend)); + out[name] = new Workflow(out[name] || metadata, workflowOptions(workflowsBackend)); } }); - defineProperty(out, HOST_BINDINGS_WRAPPED, { value: true }); + __WdlHostRuntime__.defineProperty(out, HOST_BINDINGS_WRAPPED, { value: true }); return out; } @@ -331,13 +314,13 @@ async function notifyWorkflowCallback(request, env) { function wrapHandler(owner, fn) { return function(arg1, env, ctx) { - return applyFunction(fn, owner, [arg1, wrapEnv(env, requestIdFromEventArg(arg1)), ctx]); + return withRequestContext(arg1, () => __WdlHostRuntime__.applyFunction(fn, owner, [arg1, wrapEnv(env), ctx])); }; } const HOST_WRAPPED_HANDLER_KEYS = ["fetch", "scheduled", "queue", "tail"]; -const raw = user.default; +const raw = __WdlUserModule__.default; let wrappedDefault = raw; if (raw && typeof raw === "object") { @@ -345,7 +328,7 @@ if (raw && typeof raw === "object") { const wrapDefaultFunctionKey = (key) => { const fn = raw[key]; if (typeof fn === "function") { - defineProperty(wrappedDefault, key, { + __WdlHostRuntime__.defineProperty(wrappedDefault, key, { value: wrapHandler(raw, fn), writable: true, enumerable: true, @@ -353,17 +336,16 @@ if (raw && typeof raw === "object") { }); } }; - forEachArray(HOST_WRAPPED_HANDLER_KEYS, (key) => { + __WdlHostRuntime__.forEachArray(HOST_WRAPPED_HANDLER_KEYS, (key) => { wrapDefaultFunctionKey(key); }); } else if (typeof raw === "function") { - const source = functionSource(raw); - if (regexpTest(/^\\s*class\\b/, source)) { + const source = __WdlHostRuntime__.functionSource(raw); + if (__WdlHostRuntime__.regexpTest(/^\\s*class\\b/, source)) { wrappedDefault = class extends raw { constructor(ctx, env) { - const requestContext = createRequestContext(); - super(ctx, wrapEnv(env, requestContext)); - return wrapClassInstance(this, requestContext); + super(ctx, wrapEnv(env)); + return wrapClassInstance(this); } }; } else { diff --git a/runtime/r2-client.js b/runtime/r2-client.js index d3b6a64..5de68be 100644 --- a/runtime/r2-client.js +++ b/runtime/r2-client.js @@ -46,6 +46,13 @@ function bytesToArrayBuffer(bytes) { ); } +/** @param {ReadableStreamDefaultReader} reader @param {unknown} reason */ +function cancelReaderBestEffort(reader, reason) { + try { + void reader.cancel(reason).catch(() => {}); + } catch {} +} + /** @param {ReadableStream} stream @param {string} operation */ async function readStreamWithLimit(stream, operation) { const reader = stream.getReader(); @@ -59,9 +66,10 @@ async function readStreamWithLimit(stream, operation) { const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); total += chunk.byteLength; if (total > R2_OBJECT_MAX_BUFFER_BYTES) { - await reader.cancel( + cancelReaderBestEffort( + reader, `R2 ${operation}: object exceeds ${R2_OBJECT_MAX_BUFFER_BYTES} byte limit` - ).catch(() => {}); + ); assertR2BufferSize(total, operation); } chunks.push(chunk); @@ -100,9 +108,10 @@ function cappedReadableStream(stream, operation) { const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); total += chunk.byteLength; if (total > R2_OBJECT_MAX_BUFFER_BYTES) { - await reader.cancel( + cancelReaderBestEffort( + reader, `R2 ${operation}: object exceeds ${R2_OBJECT_MAX_BUFFER_BYTES} byte limit` - ).catch(() => {}); + ); try { reader.releaseLock(); } catch {} try { assertR2BufferSize(total, operation); diff --git a/rust/common/src/queue_keys.rs b/rust/common/src/queue_keys.rs index d9d5b30..8c95c67 100644 --- a/rust/common/src/queue_keys.rs +++ b/rust/common/src/queue_keys.rs @@ -1,6 +1,6 @@ //! Queue Redis key helpers shared by producer and scheduler services. -use crate::identity::is_valid_runtime_load_ns; +use crate::identity::{is_valid_route_ns, is_valid_runtime_load_ns}; pub const QUEUE_CONSUMER_INDEX_KEY: &str = "queue:index:consumers"; pub const QUEUE_STREAM_INDEX_KEY: &str = "queue:index:streams"; @@ -48,22 +48,22 @@ pub fn queue_consumer_key(ns: &str, queue: &str) -> String { pub fn parse_stream_key(key: &str) -> Option<(String, String)> { let rest = key.strip_prefix("queue:")?; let rest = rest.strip_suffix(":s")?; - split_queue_key_rest(rest) + split_queue_key_rest(rest, is_valid_runtime_load_ns) } pub fn parse_delayed_key(key: &str) -> Option<(String, String)> { let rest = key.strip_prefix("queue-delayed:")?; - split_queue_key_rest(rest) + split_queue_key_rest(rest, is_valid_runtime_load_ns) } pub fn parse_consumer_key(key: &str) -> Option<(String, String)> { let rest = key.strip_prefix("queue-consumer:")?; - split_queue_key_rest(rest) + split_queue_key_rest(rest, is_valid_route_ns) } -fn split_queue_key_rest(rest: &str) -> Option<(String, String)> { +fn split_queue_key_rest(rest: &str, is_valid_ns: fn(&str) -> bool) -> Option<(String, String)> { let (ns, queue) = rest.split_once(':')?; - if !is_valid_runtime_load_ns(ns) || !is_valid_queue_name(queue) { + if !is_valid_ns(ns) || !is_valid_queue_name(queue) { return None; } Some((ns.to_string(), queue.to_string())) diff --git a/rust/common/src/version.rs b/rust/common/src/version.rs index 72b6a9b..3d32901 100644 --- a/rust/common/src/version.rs +++ b/rust/common/src/version.rs @@ -6,10 +6,10 @@ //! must use the same grammar so malformed Redis state fails closed instead of //! silently normalizing to another worker version. //! -//! `routes_key` / `worker_versions_key` / `do_storage_id_key` mirror -//! `shared/version.js`'s `routesKey` / `workerVersionsKey` / -//! `doStorageIdKey`. Control owns these keys; Rust readers must build them -//! here so a future key-grammar change updates JS and Rust together. +//! `routes_key` / `worker_versions_key` / `worker_delete_lock_key` / +//! `do_storage_id_key` mirror `shared/version.js`'s lifecycle key builders. +//! Control owns these keys; Rust readers must build them here so a future +//! key-grammar change updates JS and Rust together. use std::fmt; @@ -56,6 +56,11 @@ pub fn worker_versions_key(ns: &str, worker: &str) -> String { format!("worker-versions:{ns}:{worker}") } +/// Worker-scoped Control mutation lock shared with lifecycle readers. +pub fn worker_delete_lock_key(ns: &str, worker: &str) -> String { + format!("worker-delete-lock:{ns}:{worker}") +} + /// Logical Worker -> Durable Object storage pointer. Control owns writes; DO /// runtime and workflows read it for owner/storage fencing. pub fn do_storage_id_key(ns: &str, worker: &str) -> String { @@ -113,6 +118,10 @@ mod tests { worker_versions_key("tenant", "worker"), "worker-versions:tenant:worker" ); + assert_eq!( + worker_delete_lock_key("tenant", "worker"), + "worker-delete-lock:tenant:worker" + ); assert_eq!( do_storage_id_key("tenant", "worker"), "worker:do-storage:tenant:worker" diff --git a/rust/scheduler/src/queue/registry.rs b/rust/scheduler/src/queue/registry.rs index b19a870..94db856 100644 --- a/rust/scheduler/src/queue/registry.rs +++ b/rust/scheduler/src/queue/registry.rs @@ -3,9 +3,13 @@ use std::future::Future; use std::sync::Arc; use redis::{AsyncCommands, Value}; +use wdl_rust_common::identity::{is_valid_route_ns, is_valid_worker_name}; +use wdl_rust_common::queue_keys::is_valid_queue_name; +use wdl_rust_common::version::parse_version_tag; use crate::{ - AppState, CONSUMER_GROUP, QueueState, SchedulerResult, indexed_data_keys, indexed_keys, + AppState, CONSUMER_GROUP, MAX_BATCH_SIZE_CAP, QueueState, SchedulerResult, indexed_data_keys, + indexed_keys, }; use super::{ @@ -16,9 +20,9 @@ use super::{ const QUEUE_RECONCILE_CONSUMER_HASH_BATCH_SIZE: usize = 128; -pub(crate) fn finite_or(value: Option<&String>, fallback: i64) -> i64 { +fn finite_or(value: Option<&String>, fallback: i64) -> i64 { value - .and_then(|v| v.parse::().ok()) + .and_then(|value| value.parse::().ok()) .unwrap_or(fallback) } @@ -27,15 +31,25 @@ pub(crate) fn hydrate_consumer( queue: &str, hash: &HashMap, ) -> Option { - let worker = hash.get("worker")?.clone(); - let version = hash.get("version")?.clone(); - let max_batch_size = finite_or(hash.get("max_batch_size"), 10).max(1) as usize; + if !is_valid_route_ns(ns) || !is_valid_queue_name(queue) { + return None; + } + let worker = hash + .get("worker") + .filter(|value| is_valid_worker_name(value)) + .cloned()?; + let version = hash + .get("version") + .filter(|value| parse_version_tag(value).is_ok()) + .cloned()?; + let max_batch_size = + finite_or(hash.get("max_batch_size"), 10).clamp(1, MAX_BATCH_SIZE_CAP as i64) as usize; let max_batch_timeout_ms = finite_or(hash.get("max_batch_timeout_ms"), 5000); let max_retries = finite_or(hash.get("max_retries"), 3); let retry_delay_secs = finite_or(hash.get("retry_delay_secs"), 0).max(0); let dead_letter_queue = hash .get("dead_letter_queue") - .filter(|v| !v.is_empty()) + .filter(|value| !value.is_empty()) .cloned(); Some(Consumer { ns: ns.to_string(), @@ -226,10 +240,7 @@ where let hash = load_hash(queue_consumer_key(ns, queue)).await?; let consumer = hydrate_consumer(ns, queue, &hash); write_resolved_consumer(queues, stream_key, consumer.as_ref()).await; - let Some(consumer) = consumer else { - return Ok(None); - }; - Ok(Some(consumer)) + Ok(consumer) } async fn write_resolved_consumer( @@ -246,8 +257,10 @@ async fn write_resolved_consumer( } else { queues.registry.write().await.remove(stream_key) }; - if previous.as_ref() != consumer { + let registry_changed = previous.as_ref() != consumer; + if registry_changed { refresh_consumer_streams_for(queues).await; + queues.delayed_changed.notify_one(); } } @@ -264,6 +277,16 @@ mod tests { .collect() } + fn valid_consumer_hash(worker: &str, version: &str) -> HashMap { + str_map(&[ + ("worker", worker), + ("version", version), + ("max_batch_size", "10"), + ("max_batch_timeout_ms", "5000"), + ("max_retries", "3"), + ]) + } + #[test] fn queue_index_lookup_plan_covers_backfill_and_stale_branches() { assert_eq!( @@ -307,7 +330,7 @@ mod tests { } #[test] - fn hydrate_consumer_preserves_defaults_and_explicit_zeroes() { + fn hydrate_consumer_preserves_defaults_and_runtime_batch_cap() { let defaults = hydrate_consumer( "demo", "jobs", @@ -347,12 +370,14 @@ mod tests { &str_map(&[ ("worker", "w"), ("version", "v3"), + ("max_batch_size", "1000"), ("max_retries", "0"), ("max_batch_timeout_ms", "0"), ("retry_delay_secs", "0"), ]), ) .unwrap(); + assert_eq!(zeroes.max_batch_size, MAX_BATCH_SIZE_CAP); assert_eq!(zeroes.max_retries, 0); assert_eq!(zeroes.max_batch_timeout_ms, 0); assert_eq!(zeroes.retry_delay_secs, 0); @@ -377,36 +402,23 @@ mod tests { } #[test] - fn hydrate_consumer_falls_back_for_invalid_or_incomplete_hashes() { - let invalid = hydrate_consumer( - "demo", - "jobs", - &str_map(&[ - ("worker", "w"), - ("version", "v1"), - ("max_batch_size", "bogus"), - ("max_retries", "nan"), - ("retry_delay_secs", "bogus"), - ]), - ) - .unwrap(); - assert_eq!(invalid.max_batch_size, 10); - assert_eq!(invalid.max_retries, 3); - assert_eq!(invalid.retry_delay_secs, 0); - + fn hydrate_consumer_rejects_noncanonical_dispatch_identity() { + assert!(hydrate_consumer("demo", "jobs", &HashMap::new()).is_none()); assert!(hydrate_consumer("demo", "jobs", &str_map(&[("version", "v1")])).is_none()); assert!(hydrate_consumer("demo", "jobs", &str_map(&[("worker", "w")])).is_none()); + assert!( + hydrate_consumer("__platform__", "jobs", &valid_consumer_hash("w", "v1")).is_none() + ); + assert!( + hydrate_consumer("demo", "jobs", &valid_consumer_hash("bad:worker", "v1")).is_none() + ); + assert!(hydrate_consumer("demo", "jobs", &valid_consumer_hash("w", "v01")).is_none()); } #[test] fn sorted_consumer_streams_returns_stable_key_snapshot() { let mut registry = HashMap::new(); - let jobs = hydrate_consumer( - "demo", - "jobs", - &str_map(&[("worker", "w"), ("version", "v1")]), - ) - .unwrap(); + let jobs = hydrate_consumer("demo", "jobs", &valid_consumer_hash("w", "v1")).unwrap(); registry.insert(queue_stream_key("demo", "z"), jobs.clone()); registry.insert(queue_stream_key("demo", "a"), jobs); @@ -421,12 +433,8 @@ mod tests { async fn resolve_consumer_loads_authoritative_hash_and_refreshes_registry() { let queues = QueueState::default(); let stream_key = queue_stream_key("demo", "jobs"); - let stale = hydrate_consumer( - "demo", - "jobs", - &str_map(&[("worker", "old-worker"), ("version", "v1")]), - ) - .unwrap(); + let stale = + hydrate_consumer("demo", "jobs", &valid_consumer_hash("old-worker", "v1")).unwrap(); queues .registry .write() @@ -439,6 +447,8 @@ mod tests { ("worker", "fresh-worker"), ("version", "v2"), ("max_batch_size", "4"), + ("max_batch_timeout_ms", "5000"), + ("max_retries", "3"), ]); let consumer = resolve_consumer_with_hash_loader(&queues, &stream_key, "demo", "jobs", { let loaded_keys = loaded_keys.clone(); @@ -477,31 +487,35 @@ mod tests { } #[tokio::test] - async fn resolve_consumer_removes_stale_registry_when_authoritative_hash_is_missing() { - let queues = QueueState::default(); - let stream_key = queue_stream_key("demo", "jobs"); - let stale = hydrate_consumer( - "demo", - "jobs", - &str_map(&[("worker", "old-worker"), ("version", "v1")]), - ) - .unwrap(); - queues - .registry - .write() - .await - .insert(stream_key.clone(), stale); - refresh_consumer_streams_for(&queues).await; - - let consumer = - resolve_consumer_with_hash_loader(&queues, &stream_key, "demo", "jobs", |_| async { - Ok::<_, redis::RedisError>(HashMap::new()) - }) + async fn resolve_consumer_treats_missing_or_invalid_identity_as_absent() { + for authoritative in [ + HashMap::new(), + str_map(&[("worker", "worker"), ("version", "invalid")]), + ] { + let queues = QueueState::default(); + let stream_key = queue_stream_key("demo", "jobs"); + let stale = + hydrate_consumer("demo", "jobs", &valid_consumer_hash("old-worker", "v1")).unwrap(); + queues + .registry + .write() + .await + .insert(stream_key.clone(), stale); + refresh_consumer_streams_for(&queues).await; + + let consumer = resolve_consumer_with_hash_loader( + &queues, + &stream_key, + "demo", + "jobs", + |_| async { Ok::<_, redis::RedisError>(authoritative) }, + ) .await .unwrap(); - assert_eq!(consumer, None); - assert!(!queues.registry.read().await.contains_key(&stream_key)); - assert!(queues.consumer_streams.read().await.is_empty()); + assert_eq!(consumer, None); + assert!(!queues.registry.read().await.contains_key(&stream_key)); + assert!(queues.consumer_streams.read().await.is_empty()); + } } } diff --git a/rust/scheduler/src/runtime_client.rs b/rust/scheduler/src/runtime_client.rs index 1df6716..5f313f1 100644 --- a/rust/scheduler/src/runtime_client.rs +++ b/rust/scheduler/src/runtime_client.rs @@ -5,6 +5,8 @@ use serde_json::Value as JsonValue; use crate::{AppState, Config}; use wdl_rust_common::internal_auth::INTERNAL_AUTH_HEADER; +const MAX_RUNTIME_RESPONSE_BYTES: usize = 1024 * 1024; + pub(crate) struct RuntimeResponse { pub(crate) status: Option, pub(crate) json: Option, @@ -70,7 +72,17 @@ pub(crate) async fn post_runtime( }; }; let status = response.status().as_u16(); - let text = response.text().await.unwrap_or_default(); + let text = match read_runtime_response(response).await { + Ok(text) => text, + Err(error) => { + return RuntimeResponse { + status: Some(status), + json: None, + text: None, + error: Some(error), + }; + } + }; let parsed = serde_json::from_str::(&text).ok(); RuntimeResponse { status: Some(status), @@ -80,6 +92,32 @@ pub(crate) async fn post_runtime( } } +async fn read_runtime_response(mut response: reqwest::Response) -> Result { + if response + .content_length() + .is_some_and(|length| length > MAX_RUNTIME_RESPONSE_BYTES as u64) + { + return Err("runtime response body exceeds 1048576 bytes".to_string()); + } + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|err| format!("failed to read runtime response body: {err}"))? + { + append_runtime_response_chunk(&mut body, &chunk)?; + } + String::from_utf8(body).map_err(|_| "runtime response body is not valid UTF-8".to_string()) +} + +fn append_runtime_response_chunk(body: &mut Vec, chunk: &[u8]) -> Result<(), String> { + if body.len().saturating_add(chunk.len()) > MAX_RUNTIME_RESPONSE_BYTES { + return Err("runtime response body exceeds 1048576 bytes".to_string()); + } + body.extend_from_slice(chunk); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -115,4 +153,16 @@ mod tests { "error" ); } + + #[test] + fn runtime_response_body_accepts_the_limit_and_rejects_the_next_byte() { + let mut body = Vec::new(); + let exact_limit = vec![b'a'; MAX_RUNTIME_RESPONSE_BYTES]; + append_runtime_response_chunk(&mut body, &exact_limit).unwrap(); + assert_eq!(body.len(), MAX_RUNTIME_RESPONSE_BYTES); + assert_eq!( + append_runtime_response_chunk(&mut body, b"x"), + Err("runtime response body exceeds 1048576 bytes".to_string()) + ); + } } diff --git a/rust/workflows/src/api/active_export.rs b/rust/workflows/src/api/active_export.rs index 0236343..700e4a0 100644 --- a/rust/workflows/src/api/active_export.rs +++ b/rust/workflows/src/api/active_export.rs @@ -1,5 +1,5 @@ use serde_json::Value as JsonValue; -use wdl_rust_common::version::routes_key; +use wdl_rust_common::version::{routes_key, worker_delete_lock_key}; use crate::{AppState, WorkflowError, WorkflowResult}; @@ -190,7 +190,7 @@ pub(super) async fn ensure_worker_not_deleting( ns: &str, worker: &str, ) -> WorkflowResult<()> { - let lock_key = format!("worker-delete-lock:{ns}:{worker}"); + let lock_key = worker_delete_lock_key(ns, worker); let lock_exists: i64 = state .control_redis .with_conn(async |mut conn| { diff --git a/shared/bounded-body.js b/shared/bounded-body.js index 80d2218..112ca12 100644 --- a/shared/bounded-body.js +++ b/shared/bounded-body.js @@ -32,7 +32,11 @@ export async function readBoundedStreamBytes( total += chunk.byteLength; if (total > maxBytes) { const error = overflowError(); - await reader.cancel(error).catch(() => {}); + try { + void reader.cancel(error).catch(() => {}); + } catch { + // Cancellation is best-effort; the size error must not wait on it. + } throw error; } chunks.push(chunk); diff --git a/shared/queue-keys.js b/shared/queue-keys.js index ae9b395..1c7ea3d 100644 --- a/shared/queue-keys.js +++ b/shared/queue-keys.js @@ -2,7 +2,7 @@ // corrupt parseStreamKey's anchor, so inline construction is a drift // risk — every tier imports from here. -import { isValidRuntimeLoadNs, QUEUE_NAME_RE } from "./ns-pattern.js"; +import { isValidRouteNs, isValidRuntimeLoadNs, QUEUE_NAME_RE } from "./ns-pattern.js"; /** * @typedef {{ ns: string, queue: string }} QueueKeyParts @@ -30,14 +30,15 @@ export function queueConsumerScanPrefix(ns) { return `queue-consumer:${ns}:`; } /** * @param {string} rest + * @param {(ns: string) => boolean} isValidNs * @returns {QueueKeyParts | null} */ -function parseQueueKeyRest(rest) { +function parseQueueKeyRest(rest, isValidNs) { const separator = rest.indexOf(":"); if (separator <= 0) return null; const ns = rest.slice(0, separator); const queue = rest.slice(separator + 1); - if (!isValidRuntimeLoadNs(ns) || !QUEUE_NAME_RE.test(queue)) return null; + if (!isValidNs(ns) || !QUEUE_NAME_RE.test(queue)) return null; return { ns, queue }; } @@ -47,7 +48,7 @@ function parseQueueKeyRest(rest) { */ export function parseStreamKey(key) { if (!key.startsWith("queue:") || !key.endsWith(":s")) return null; - return parseQueueKeyRest(key.slice("queue:".length, -":s".length)); + return parseQueueKeyRest(key.slice("queue:".length, -":s".length), isValidRuntimeLoadNs); } /** @@ -56,7 +57,7 @@ export function parseStreamKey(key) { */ export function parseDelayedKey(key) { if (!key.startsWith("queue-delayed:")) return null; - return parseQueueKeyRest(key.slice("queue-delayed:".length)); + return parseQueueKeyRest(key.slice("queue-delayed:".length), isValidRuntimeLoadNs); } /** @@ -65,5 +66,5 @@ export function parseDelayedKey(key) { */ export function parseConsumerKey(key) { if (!key.startsWith("queue-consumer:")) return null; - return parseQueueKeyRest(key.slice("queue-consumer:".length)); + return parseQueueKeyRest(key.slice("queue-consumer:".length), isValidRouteNs); } diff --git a/shared/respond.js b/shared/respond.js index 9a06428..b4aaabb 100644 --- a/shared/respond.js +++ b/shared/respond.js @@ -34,7 +34,7 @@ export function echoResponseWithRequestId(response, requestId) { */ export async function discardResponseBody(response) { try { - await response.body?.cancel(); + void response.body?.cancel().catch(() => {}); } catch { // Best-effort cleanup only; the caller's status/error path owns behavior. } diff --git a/shared/vendor/aws-sigv4.js b/shared/vendor/aws-sigv4.js index 6dde753..a654ee5 100644 --- a/shared/vendor/aws-sigv4.js +++ b/shared/vendor/aws-sigv4.js @@ -31,31 +31,57 @@ var DEFAULT_UNSIGNABLE_HEADERS = /* @__PURE__ */ new Set([ "user-agent", "x-amzn-trace-id" ]); -var RFC3986_EXTRA_ESCAPE_RE = /[!'()*]/g; -var LOWER_HEX = "0123456789abcdef"; var AWS_ALGORITHM = "AWS4-HMAC-SHA256"; var AWS_REQUEST = "aws4_request"; var UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; -var CONTROL_CHAR_RE = /[\u0000-\u001f\u007f]/u; -var WHITESPACE_RE = /\s/u; -var AUTH_PARAM_SEPARATOR_RE = /[,=;]/u; -var HTTP_METHOD_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/u; -var ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$/u; -var SIGNING_DATE_ERROR = "signingDate must be a valid Date, ISO-8601 string, or YYYYMMDDTHHMMSSZ string"; -var IDEMPOTENT_METHODS = /* @__PURE__ */ new Set(["GET", "HEAD", "OPTIONS", "PUT", "DELETE"]); +var LOWER_HEX = "0123456789abcdef"; +var inFlightSigningKeys = /* @__PURE__ */ new WeakMap(); async function signatureHex(options) { - const secretAccessKeyHash = options.secretAccessKeyHash ?? await sha256Hex(options.secretAccessKey); - const cacheKey = ["sigv4", secretAccessKeyHash, options.date, options.region, options.service].join(","); - let signingKey = options.cache?.get(cacheKey); - if (!signingKey) { - const kDate = await hmac(`AWS4${options.secretAccessKey}`, options.date); - const kRegion = await hmac(kDate, options.region); - const kService = await hmac(kRegion, options.service); - signingKey = await hmac(kService, AWS_REQUEST); - options.cache?.set(cacheKey, signingKey); + let signingKey; + if (options.cache === void 0) { + signingKey = await deriveSigningKey(options); + } else { + const secretAccessKeyHash = options.secretAccessKeyHash ?? await sha256Hex(options.secretAccessKey); + const cacheKey = ["sigv4", secretAccessKeyHash, options.date, options.region, options.service].join(","); + const cachedSigningKey = options.cache.get(cacheKey); + signingKey = isSigningKeyCacheMiss(cachedSigningKey) ? await deriveCachedSigningKey(options, options.cache, cacheKey) : cachedSigningKey; } return hex(await hmac(signingKey, options.stringToSign)); } +function isSigningKeyCacheMiss(value) { + return value === void 0 || value === null; +} +async function deriveCachedSigningKey(options, cache, cacheKey) { + let byKey = inFlightSigningKeys.get(cache); + if (byKey === void 0) { + byKey = /* @__PURE__ */ new Map(); + inFlightSigningKeys.set(cache, byKey); + } + const existing = byKey.get(cacheKey); + if (existing !== void 0) { + return existing; + } + const derivation = (async () => { + try { + const signingKey = await deriveSigningKey(options); + cache.set(cacheKey, signingKey); + return signingKey; + } finally { + byKey.delete(cacheKey); + if (byKey.size === 0) { + inFlightSigningKeys.delete(cache); + } + } + })(); + byKey.set(cacheKey, derivation); + return derivation; +} +async function deriveSigningKey(options) { + const kDate = await hmac(`AWS4${options.secretAccessKey}`, options.date); + const kRegion = await hmac(kDate, options.region); + const kService = await hmac(kRegion, options.service); + return hmac(kService, AWS_REQUEST); +} async function sha256Hex(value) { const bytes = cryptoBufferSource(value); return hex(await crypto.subtle.digest("SHA-256", bytes)); @@ -88,7 +114,30 @@ function hex(input) { function hexNibble(value) { return LOWER_HEX.charAt(value); } -async function prepareBody(body, headers, materialize) { +async function prepareSigningBody(body, headers, options) { + if (options.unsignedPayload && !headers.has(AMZ_CONTENT_SHA256_HEADER)) { + headers.set(AMZ_CONTENT_SHA256_HEADER, UNSIGNED_PAYLOAD); + } + const hasBody = body !== null && body !== void 0; + const computePayloadHash = !headers.has(AMZ_CONTENT_SHA256_HEADER); + const materialize = options.replay && shouldMaterializeBodyForReplay(body) || computePayloadHash && hasBody; + const prepared = await prepareBody(body, headers, materialize, options.signal); + let payloadHash = headers.get(AMZ_CONTENT_SHA256_HEADER); + if (payloadHash === null) { + if (prepared.bytes === void 0) { + throw new Error("body bytes must be materialized before payload hashing"); + } + payloadHash = await sha256Hex(prepared.bytes); + options.signal?.throwIfAborted(); + if (hasBody || options.service === "s3") { + headers.set(AMZ_CONTENT_SHA256_HEADER, payloadHash); + } + } + return { body: prepared.body, payloadHash }; +} +async function prepareBody(body, headers, materialize, signal) { + signal?.throwIfAborted(); + assertSupportedBody(body); if (body === null || body === void 0) { return { body, bytes: new Uint8Array() }; } @@ -99,20 +148,29 @@ async function prepareBody(body, headers, materialize) { if (contentType) { headers.set(CONTENT_TYPE_HEADER, contentType); } - const bytes2 = new Uint8Array(await request.arrayBuffer()); + const bytes2 = signal && request.body ? await readStreamBytes(request.body, signal) : new Uint8Array(await request.arrayBuffer()); + signal?.throwIfAborted(); return { body: bytes2, bytes: bytes2 }; } setGeneratedContentType(body, headers); - if (!materialize) { - return { body, bytes: new Uint8Array() }; - } if (body instanceof ReadableStream) { - const bytes2 = new Uint8Array(await new Response(body).arrayBuffer()); + assertReadableStreamUsable(body); + if (!materialize) { + return { body }; + } + const bytes2 = await readStreamBytes(body, signal); return { body: bytes2, bytes: bytes2 }; } - const bytes = await bodyBytes(body); + if (!materialize && (typeof body === "string" || body instanceof Blob)) { + return { body }; + } + const bytes = await bodyBytes(body, signal); + signal?.throwIfAborted(); return { body: stableMaterializedBody(body, bytes), bytes }; } +function shouldMaterializeBodyForReplay(body) { + return body !== void 0 && body !== null && typeof body !== "string" && !(body instanceof Blob); +} function rejectManualFormDataContentType(headers) { if (headers.has(CONTENT_TYPE_HEADER)) { throw new TypeError("FormData content-type must be generated by the runtime"); @@ -126,44 +184,107 @@ function setGeneratedContentType(body, headers) { headers.set(CONTENT_TYPE_HEADER, "text/plain;charset=UTF-8"); } else if (body instanceof URLSearchParams) { headers.set(CONTENT_TYPE_HEADER, "application/x-www-form-urlencoded;charset=UTF-8"); - } else if (body instanceof Blob && body.type) { - headers.set(CONTENT_TYPE_HEADER, body.type); + } else if (body instanceof Blob) { + if (body.type) { + headers.set(CONTENT_TYPE_HEADER, body.type); + } } } -async function bodyBytes(body) { +async function bodyBytes(body, signal) { if (typeof body === "string") { return textEncoder.encode(body); } if (body instanceof Uint8Array) { - return body; + return new Uint8Array(body); } if (body instanceof ArrayBuffer) { - return new Uint8Array(body); + return new Uint8Array(body).slice(); } if (ArrayBuffer.isView(body)) { - return new Uint8Array(body.buffer, body.byteOffset, body.byteLength); + return new Uint8Array(body.buffer, body.byteOffset, body.byteLength).slice(); } if (body instanceof URLSearchParams) { return textEncoder.encode(body.toString()); } if (body instanceof Blob) { + if (signal) { + return readStreamBytes(body.stream(), signal); + } return new Uint8Array(await body.arrayBuffer()); } throw new TypeError("body must be a string, Blob, URLSearchParams, ArrayBuffer, or ArrayBufferView"); } -function shouldHashPayload(body, headers, unsignedPayload) { - return body !== void 0 && body !== null && !unsignedPayload && !headers.has(AMZ_CONTENT_SHA256_HEADER); +function assertReadableStreamUsable(body) { + try { + void new Response(body); + } catch (error) { + if (error instanceof TypeError) { + throw new TypeError("ReadableStream body must not be disturbed or locked", { cause: error }); + } + throw error; + } +} +async function readStreamBytes(stream, signal) { + const reader = stream.getReader(); + const chunks = []; + let byteLength = 0; + const onAbort = () => { + if (signal) { + void reader.cancel(signal.reason).catch(() => void 0); + } + }; + signal?.addEventListener("abort", onAbort, { once: true }); + try { + for (; ; ) { + signal?.throwIfAborted(); + const result = await reader.read(); + signal?.throwIfAborted(); + if (result.done) { + break; + } + const value = result.value; + if (!(value instanceof Uint8Array)) { + throw new TypeError("ReadableStream body must yield Uint8Array chunks"); + } + const chunk = new Uint8Array(value); + chunks.push(chunk); + byteLength += chunk.byteLength; + } + } catch (err) { + if (signal?.aborted) { + throw signal.reason; + } + void reader.cancel(err).catch(() => void 0); + throw err; + } finally { + signal?.removeEventListener("abort", onAbort); + reader.releaseLock(); + } + const bytes = new Uint8Array(byteLength); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +} +function isAsyncIterable(value) { + return typeof value[Symbol.asyncIterator] === "function"; } -async function prepareHashedBody(body, headers, unsignedPayload, materialize = false) { - const hashPayload = shouldHashPayload(body, headers, unsignedPayload); - const preparedBody = await prepareBody(body, headers, materialize || hashPayload); - if (hashPayload) { - headers.set(AMZ_CONTENT_SHA256_HEADER, await sha256Hex(preparedBody.bytes)); +function assertSupportedBody(body) { + if (body === null || body === void 0) { + return; + } + if (typeof body === "string" || body instanceof Blob || body instanceof FormData || body instanceof ReadableStream || body instanceof URLSearchParams || body instanceof ArrayBuffer || ArrayBuffer.isView(body)) { + return; } - return preparedBody; + if (isAsyncIterable(body)) { + throw new TypeError("async iterable bodies are not supported; use a ReadableStream"); + } + throw new TypeError("body must be a string, Blob, URLSearchParams, ArrayBuffer, or ArrayBufferView"); } function stableMaterializedBody(body, bytes) { - if (typeof body === "string" || body instanceof Uint8Array) { + if (typeof body === "string") { return body; } if (bytes.buffer instanceof ArrayBuffer) { @@ -191,6 +312,7 @@ function canonicalHeaderBlock(url, headers, options) { }; } function validateSignedHeaderValues(headers, options) { + rejectMandatoryHeaderExclusions(headers, options); const overwrittenHeaderNames = new Set((options.overwrittenHeaderNames || []).map((value) => value.toLowerCase())); for (const header of signedHeaderNames(headers, options)) { if (header !== HOST_HEADER && !overwrittenHeaderNames.has(header)) { @@ -204,7 +326,7 @@ function signerOverwrittenHeaderNames(hasSessionToken) { function signedHeaderNames(headers, options) { const userUnsignable = new Set((options.unsignableHeaders || []).map((value) => value.toLowerCase())); return [.../* @__PURE__ */ new Set([HOST_HEADER, ...headers.keys()])].filter((header) => header !== AUTHORIZATION_HEADER).filter((header) => { - if (MANDATORY_SIGNED_HEADERS.has(header)) { + if (isMandatorySignedHeader(header, options.service)) { return true; } if (userUnsignable.has(header)) { @@ -213,6 +335,18 @@ function signedHeaderNames(headers, options) { return options.signAllHeaders || !DEFAULT_UNSIGNABLE_HEADERS.has(header); }).sort(); } +function rejectMandatoryHeaderExclusions(headers, options) { + for (const value of options.unsignableHeaders || []) { + const header = value.toLowerCase(); + const isPresentDynamicHeader = headers.has(header) && (header.startsWith("x-amz-") || options.service === "s3" && header === "content-md5"); + if (MANDATORY_SIGNED_HEADERS.has(header) || isPresentDynamicHeader) { + throw new TypeError(`unsignableHeaders must not include mandatory signed header ${header}`); + } + } +} +function isMandatorySignedHeader(header, service) { + return MANDATORY_SIGNED_HEADERS.has(header) || header.startsWith("x-amz-") || service === "s3" && header === "content-md5"; +} function canonicalHeaderValue(value, name) { rejectNonPrintableAsciiHeaderValue(value, name); return value.trim().replace(/\s+/gu, " "); @@ -220,6 +354,8 @@ function canonicalHeaderValue(value, name) { function rejectNonPrintableAsciiHeaderValue(value, name) { rejectNonPrintableAscii(value, `${name} header value must contain only printable ASCII characters`); } +var ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$/u; +var SIGNING_DATE_ERROR = "signingDate must be a valid Date, ISO-8601 string, or YYYYMMDDTHHMMSSZ string"; function optionalAmzDate(value) { if (value === void 0 || value === null) { return void 0; @@ -240,25 +376,15 @@ function formatAmzDate(value) { throw new TypeError(SIGNING_DATE_ERROR); } const date = typeof value === "string" ? new Date(value) : value; - if (!(date instanceof Date)) { - throw new TypeError(SIGNING_DATE_ERROR); - } - if (Number.isNaN(dateTimeValue(date))) { + if (!(date instanceof Date) || Number.isNaN(date.getTime())) { throw new TypeError(SIGNING_DATE_ERROR); } - const amzDate = Date.prototype.toISOString.call(date).replace(/[:-]|\.\d{3}/g, ""); + const amzDate = date.toISOString().replace(/[:-]|\.\d{3}/g, ""); if (!/^\d{8}T\d{6}Z$/u.test(amzDate) || !isValidCompactAmzDate(amzDate)) { throw new TypeError(SIGNING_DATE_ERROR); } return amzDate; } -function dateTimeValue(date) { - try { - return Date.prototype.getTime.call(date); - } catch { - throw new TypeError(SIGNING_DATE_ERROR); - } -} function isValidIsoDate(value) { const match = ISO_DATE_RE.exec(value); if (!match) { @@ -283,6 +409,10 @@ function isValidDateParts(year, month, day, hour, minute, second) { date.setUTCFullYear(year); return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day && date.getUTCHours() === hour && date.getUTCMinutes() === minute && date.getUTCSeconds() === second; } +var AUTH_PARAM_SEPARATOR_RE = /[,=;]/u; +var CONTROL_CHAR_RE = /[\u0000-\u001f\u007f]/u; +var HEADER_NAME_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/u; +var WHITESPACE_RE = /\s/u; var CLIENT_SIGNING_OPTION_KEYS = /* @__PURE__ */ new Set([ "service", "region", @@ -293,6 +423,13 @@ var CLIENT_SIGNING_OPTION_KEYS = /* @__PURE__ */ new Set([ "doubleUrlEncode" ]); var UNSIGNABLE_HEADER_SNAPSHOTS = /* @__PURE__ */ new WeakMap(); +function snapshotSignAwsRequestOptions(value) { + requireOptionsObject(value, "signAwsRequest options are required"); + const snapshot = { ...value }; + validateCredentialOptions(snapshot, "signAwsRequest options are required"); + snapshot.unsignableHeaders = snapshotUnsignableHeaders(value, snapshot.unsignableHeaders, "unsignableHeaders"); + return snapshot; +} function normalizeClientSigningOptions(options) { if (options === void 0) { return {}; @@ -300,28 +437,21 @@ function normalizeClientSigningOptions(options) { if (options === null || typeof options !== "object") { throw new TypeError("init.signing must be an object"); } - for (const key of Object.keys(options)) { + const source = { ...options }; + for (const key of Object.keys(source)) { if (!CLIENT_SIGNING_OPTION_KEYS.has(key)) { throw new TypeError(`${signingOptionDisplayName(key)} cannot override client credentials or transport options`); } } - const record = options; - const normalized = { ...record }; - const service = optionalCredentialComponent(ownNonNullOption(record, "service"), "init.signing.service"); - setOrDelete(normalized, "service", service); - const region = optionalCredentialComponent(ownNonNullOption(record, "region"), "init.signing.region"); - setOrDelete(normalized, "region", region); - const signingDate = optionalAmzDate(ownNonNullOption(record, "signingDate")); - setOrDelete(normalized, "signingDate", signingDate); - const unsignedPayload = optionalBoolean(ownOption(record, "unsignedPayload"), "init.signing.unsignedPayload"); - const signAllHeaders = optionalBoolean(ownOption(record, "signAllHeaders"), "init.signing.signAllHeaders"); - const doubleUrlEncode = optionalBoolean(ownOption(record, "doubleUrlEncode"), "init.signing.doubleUrlEncode"); - const unsignableHeaders = snapshotUnsignableHeaders(options, ownOption(record, "unsignableHeaders"), "init.signing.unsignableHeaders"); - setOrDelete(normalized, "unsignedPayload", unsignedPayload); - setOrDelete(normalized, "signAllHeaders", signAllHeaders); - setOrDelete(normalized, "unsignableHeaders", unsignableHeaders); - setOrDelete(normalized, "doubleUrlEncode", doubleUrlEncode); - return normalized; + return { + service: optionalCredentialComponent(nullAsUndefined(source.service), "init.signing.service"), + region: optionalCredentialComponent(nullAsUndefined(source.region), "init.signing.region"), + signingDate: optionalAmzDate(nullAsUndefined(source.signingDate)), + unsignedPayload: optionalBoolean(source.unsignedPayload, "init.signing.unsignedPayload"), + signAllHeaders: optionalBoolean(source.signAllHeaders, "init.signing.signAllHeaders"), + unsignableHeaders: snapshotUnsignableHeaders(options, source.unsignableHeaders, "init.signing.unsignableHeaders"), + doubleUrlEncode: optionalBoolean(source.doubleUrlEncode, "init.signing.doubleUrlEncode") + }; } function signingOptionDisplayName(key) { if (key.length === 0) { @@ -340,8 +470,8 @@ function validateCredentialOptions(options, message) { const record = options; requireCredentialComponent(record.accessKeyId, "accessKeyId"); requireSecretAccessKey(record.secretAccessKey); - requireCredentialComponent(record.service, "service"); - requireCredentialComponent(record.region, "region"); + requireLowercaseCredentialComponent(record.service, "service"); + requireLowercaseCredentialComponent(record.region, "region"); if (record.sessionToken !== void 0) { validateSessionToken(record.sessionToken); } @@ -365,10 +495,19 @@ function requireCredentialComponent(value, name) { function requireSecretAccessKey(value) { requireString(value, "secretAccessKey"); rejectControlChars(value, "secretAccessKey"); + if (!value.isWellFormed()) { + throw new TypeError("secretAccessKey must contain well-formed UTF-16"); + } +} +function requireLowercaseCredentialComponent(value, name) { + requireCredentialComponent(value, name); + if (value !== value.toLowerCase()) { + throw new TypeError(`${name} must be lowercase`); + } } function requireNonNegativeInteger(value, name) { - if (!Number.isInteger(value) || value < 0) { - throw new TypeError(`${name} must be a non-negative integer`); + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError(`${name} must be a non-negative integer within the safe integer range`); } return value; } @@ -390,6 +529,9 @@ function optionalBoolean(value, name) { function resolveUnsignedPayload(explicit, service) { return explicit ?? service === "s3"; } +function resolveDoubleUrlEncode(explicit, service) { + return explicit ?? service !== "s3"; +} function normalizeUnsignableHeaders(value, name) { if (value === void 0) { return void 0; @@ -401,6 +543,9 @@ function normalizeUnsignableHeaders(value, name) { if (typeof header !== "string" || header.length === 0) { throw new TypeError(`${name} must contain only non-empty strings`); } + if (!HEADER_NAME_RE.test(header)) { + throw new TypeError(`${name} must contain only valid header names`); + } return header; }); } @@ -408,14 +553,11 @@ function snapshotUnsignableHeaders(owner, source, name) { if (source === void 0) { return void 0; } - if (!isIterable(source)) { - return normalizeUnsignableHeaders(source, name); - } - if (!isOneShotIterable(source)) { + if (!isIterable(source) || !isOneShotIterable(source)) { return normalizeUnsignableHeaders(source, name); } const cached = UNSIGNABLE_HEADER_SNAPSHOTS.get(owner); - if (cached && cached.source === source) { + if (cached?.source === source) { if (cached.result.ok) { return cached.result.value; } @@ -474,32 +616,21 @@ function rejectSurroundingWhitespace(value, name) { throw new TypeError(`${name} must not contain leading or trailing whitespace`); } } -function ownNonNullOption(record, name) { - const value = ownOption(record, name); - return value === null ? void 0 : value; -} -function ownOption(record, name) { - return Object.hasOwn(record, name) ? record[name] : void 0; -} function optionalCredentialComponent(value, name) { if (value === void 0) { return void 0; } - requireCredentialComponent(value, name); + requireLowercaseCredentialComponent(value, name); return value; } -function setOrDelete(record, key, value) { - if (value === void 0) { - Reflect.deleteProperty(record, key); - } else { - record[key] = value; - } +function nullAsUndefined(value) { + return value === null ? void 0 : value; } function isOneShotIterable(value) { return Object.is(value[Symbol.iterator](), value); } function isIterable(value) { - return value !== null && typeof value !== "string" && typeof value[Symbol.iterator] === "function"; + return value !== null && (typeof value === "object" || typeof value === "function") && typeof value[Symbol.iterator] === "function"; } function isSigningCache(value) { if (value === null || typeof value !== "object" || value instanceof WeakMap) { @@ -508,10 +639,11 @@ function isSigningCache(value) { const candidate = value; return typeof candidate.get === "function" && typeof candidate.set === "function"; } +var RFC3986_EXTRA_ESCAPE_RE = /[!'()*]/g; function parseRequestUrl(input) { const raw = String(input); - if (typeof input === "string" && /\s/u.test(raw)) { - throw new TypeError("url must not contain unescaped whitespace"); + if (typeof input === "string" && /[\s\u0000-\u001f\u007f]/u.test(raw)) { + throw new TypeError("url must not contain unescaped whitespace or control characters"); } if (typeof input === "string" && raw.includes("\\")) { throw new TypeError("url must not contain backslashes"); @@ -576,7 +708,7 @@ function canonicalQuery(search) { const separator = part.indexOf("="); const key = separator === -1 ? part : part.slice(0, separator); const value = separator === -1 ? "" : part.slice(separator + 1); - return [canonicalQueryComponent(key), canonicalQueryComponent(value)]; + return [canonicalUriComponent(key), canonicalUriComponent(value)]; }).sort(([ak, av], [bk, bv]) => compareCodepoint(ak, bk) || compareCodepoint(av, bv)).map(([key, value]) => `${key}=${value}`).join("&"); } function hasDotPathSegment(pathname) { @@ -592,9 +724,6 @@ function stripUrlFragment(value) { function hasMalformedPercentEncoding(value) { return /%(?![0-9A-Fa-f]{2})/u.test(value); } -function canonicalQueryComponent(value) { - return canonicalUriComponent(value); -} function compareCodepoint(left, right) { if (left < right) { return -1; @@ -642,7 +771,30 @@ function canonicalSingleEncodedPathname(pathname) { return out; } function canonicalDoubleEncodedPathname(pathname) { - return strictEncode(pathname).replace(/%2F/gu, "/"); + return strictEncode(wireEncodedPathname(pathname)).replace(/%2F/gu, "/"); +} +function wireEncodedPathname(pathname) { + let out = ""; + for (let index = 0; index < pathname.length; ) { + const char = pathname[index]; + if (char === "/" || char === "%" && isHexPair(pathname, index + 1)) { + const width = char === "/" ? 1 : 3; + out += pathname.slice(index, index + width); + index += width; + continue; + } + const codePoint = pathname.codePointAt(index); + if (codePoint === void 0) { + break; + } + const charValue = String.fromCodePoint(codePoint); + out += shouldWhatwgEncodePathCodePoint(codePoint) ? strictEncode(charValue) : charValue; + index += charValue.length; + } + return out; +} +function shouldWhatwgEncodePathCodePoint(codePoint) { + return codePoint > 126 || codePoint === 34 || codePoint === 60 || codePoint === 62 || codePoint === 94 || codePoint === 96 || codePoint === 123 || codePoint === 125; } function collapsePathSlashes(pathname) { return pathname.replace(/\/+/gu, "/"); @@ -674,9 +826,63 @@ function isHexPair(value, index) { function isUnreservedByte(byte) { return byte >= 65 && byte <= 90 || byte >= 97 && byte <= 122 || byte >= 48 && byte <= 57 || byte === 45 || byte === 46 || byte === 95 || byte === 126; } +var HTTP_METHOD_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/u; +var IDEMPOTENT_METHODS = /* @__PURE__ */ new Set(["GET", "HEAD", "OPTIONS", "PUT", "DELETE"]); +function resolveClientSignRequest(input, init) { + return resolveClientRequest(input, init).request; +} +function resolveClientFetchRequest(input, init) { + const { request, inheritedRedirect, explicitRedirect } = resolveClientRequest(input, init); + const redirectPolicy = resolveFetchRedirectPolicy(inheritedRedirect, explicitRedirect); + request.init.redirect = "manual"; + return { ...request, redirectPolicy }; +} +function resolveClientRequest(input, init) { + const inputSnapshot = input instanceof Request ? snapshotRequestInput(input) : void 0; + const requestUrl = parseRequestUrl(inputSnapshot?.url ?? input); + const initSnapshot = snapshotRequestInit(init); + const requestInit = inputSnapshot ? mergeDefinedRequestInit(inputSnapshot.init, initSnapshot) : initSnapshot; + const signal = requestInit.signal ?? void 0; + signal?.throwIfAborted(); + rejectNoCorsMode(requestInit.mode); + const headers = inputSnapshot === void 0 ? new Headers(requestInit.headers) : mergeHeaders(inputSnapshot.headers, requestInit.headers); + let body = requestInit.body; + let method = requestInit.method; + if (inputSnapshot !== void 0) { + rejectUsedRequestBody(inputSnapshot.bodyUsed, body); + if ((body === void 0 || body === null) && inputSnapshot.body) { + body = inputSnapshot.body; + } + if (method === void 0) { + method = inputSnapshot.method; + } + } + const normalizedMethod = normalizeMethod(method === void 0 ? defaultMethod(body) : method); + rejectFetchForbiddenMethod(normalizedMethod); + rejectRequestBodyForGetHead(normalizedMethod, body); + requestInit.method = normalizedMethod; + requestInit.headers = headers; + if (body === void 0) { + delete requestInit.body; + } else { + requestInit.body = body; + } + return { + request: { + requestUrl, + init: requestInit, + method: normalizedMethod, + headers, + body, + signal + }, + inheritedRedirect: inputSnapshot?.init.redirect, + explicitRedirect: initSnapshot.redirect + }; +} function mergeHeaders(base, override) { const headers = new Headers(base); - if (override) { + if (override !== void 0) { new Headers(override).forEach((value, name) => { headers.set(name, value); }); @@ -692,12 +898,26 @@ function mergeDefinedRequestInit(base, override) { } return out; } +function snapshotRequestInit(value) { + if (value !== void 0 && value !== null && typeof value !== "object") { + throw new TypeError("init must be an object"); + } + if (value === void 0 || value === null) { + return {}; + } + return { ...value }; +} function rejectEmptyHeader(headers, name) { if (headers.get(name) === "") { throw new TypeError(`${name} must not be empty`); } } -function requestInitFromRequest(request) { +function snapshotRequestInput(request) { + const url = request.url; + const method = request.method; + const headers = new Headers(request.headers); + const body = request.body; + const bodyUsed = request.bodyUsed; const init = { cache: request.cache, credentials: request.credentials, @@ -712,24 +932,21 @@ function requestInitFromRequest(request) { }; const duplex = request.duplex; if (duplex === "half") { - return { ...init, duplex }; - } - return init; -} -function requestBodyForInput(input, init) { - if (init.body !== void 0) { - return init.body; + return { + url, + method, + headers, + body, + bodyUsed, + init: { ...init, duplex } + }; } - return input instanceof Request ? input.body : void 0; + return { url, method, headers, body, bodyUsed, init }; } -function methodForRequest(input, init) { - if (init.method !== void 0) { - return normalizeMethod(init.method); +function rejectUsedRequestBody(bodyUsed, override) { + if ((override === void 0 || override === null) && bodyUsed) { + throw new TypeError("Request body has already been used"); } - if (input instanceof Request) { - return normalizeMethod(input.method); - } - return defaultMethod(init.body); } function defaultMethod(body) { return hasRequestBody(body) ? "POST" : "GET"; @@ -740,6 +957,11 @@ function normalizeMethod(method) { } return method.toUpperCase(); } +function rejectFetchForbiddenMethod(method) { + if (method === "CONNECT" || method === "TRACE" || method === "TRACK") { + throw new TypeError(`SigV4Client cannot use Fetch-forbidden method ${method}`); + } +} function isIdempotentMethod(method) { return IDEMPOTENT_METHODS.has(method); } @@ -751,9 +973,6 @@ function rejectRequestBodyForGetHead(method, body) { function hasRequestBody(body) { return body !== void 0 && body !== null; } -function assertRequestCanRepresentSignedUrl(url, service) { - assertParsedRequestCanRepresentSignedUrl(parseRequestUrl(url).pathname, service); -} function assertParsedRequestCanRepresentSignedUrl(pathname, service) { if (hasDotPathSegment(pathname)) { throw new TypeError(`SigV4Client cannot represent ${service} URLs with dot segments; use signAwsRequest`); @@ -770,18 +989,92 @@ function requestInitForSignedRequest(base, signed) { } return out; } +function createSignedRequest(url, init, expectedHeaders) { + let request; + try { + request = new Request(url, init); + } catch (err) { + const duplex = init.duplex; + if (!(err instanceof TypeError) || !(init.body instanceof ReadableStream) || duplex !== void 0) { + throw err; + } + request = new Request(url, { + ...init, + duplex: "half" + }); + } + assertSignedRequestHeadersPreserved(expectedHeaders, request.headers); + return request; +} +function rejectNoCorsMode(mode) { + if (mode === "no-cors") { + throw new TypeError('SigV4Client cannot sign requests with mode "no-cors" because required headers may be removed'); + } +} +function validateRequestBeforeTransport(request) { + try { + request.signal.throwIfAborted(); + rejectNoCorsMode(request.mode); + if (request.redirect !== "manual") { + throw new TypeError('SigV4Client.fetch signed Request must use redirect: "manual"'); + } + return request.signal; + } catch (err) { + cancelRequestBody(request, err); + throw err; + } +} +function isRedirectResponse(response) { + return response.type === "opaqueredirect" || response.redirected || response.status === 301 || response.status === 302 || response.status === 303 || response.status === 307 || response.status === 308; +} +function assertSignedRequestHeadersPreserved(expected, actual) { + const authorization = expected.get(AUTHORIZATION_HEADER); + if (authorization === null || actual.get(AUTHORIZATION_HEADER) !== authorization) { + throw new TypeError("runtime removed or rewrote the authorization header after signing"); + } + const signedHeaders = /(?:^|,\s*)SignedHeaders=([^,\s]+)/u.exec(authorization)?.[1]; + if (!signedHeaders) { + throw new Error("generated SigV4 authorization is missing SignedHeaders"); + } + for (const name of signedHeaders.split(";")) { + if (name === HOST_HEADER) { + continue; + } + if (actual.get(name) !== expected.get(name)) { + throw new TypeError(`runtime removed or rewrote the signed ${name} header`); + } + } +} function bindFetch(fetchFn) { return Object.is(fetchFn, globalThis.fetch) ? fetchFn.bind(globalThis) : fetchFn; } -function isAbortError(err, request) { - return request.signal.aborted || err instanceof DOMException && err.name === "AbortError"; +function resolveFetchRedirectPolicy(inherited, explicit) { + if (explicit !== void 0) { + if (explicit === "follow") { + throw new TypeError('SigV4Client.fetch does not allow redirect: "follow"; redirected requests must be re-signed'); + } + if (explicit !== "error" && explicit !== "manual") { + throw new TypeError('redirect must be "error" or "manual"'); + } + return explicit; + } + return inherited === "manual" ? "manual" : "error"; } -function sleep(ms, signal) { - if (!signal) { - return new Promise((resolve) => setTimeout(resolve, ms)); +function cancelRequestBody(request, reason) { + try { + const cancellation = request.body?.cancel(reason); + if (cancellation !== void 0) { + void cancellation.catch(() => void 0); + } + } catch { } +} +function isAbortError(err) { + return err instanceof DOMException && err.name === "AbortError"; +} +function sleep(ms, signal) { if (signal.aborted) { - return Promise.reject(abortReason(signal)); + return Promise.reject(signal.reason); } return new Promise((resolve, reject) => { let timeout; @@ -790,7 +1083,7 @@ function sleep(ms, signal) { clearTimeout(timeout); } signal.removeEventListener("abort", onAbort); - reject(abortReason(signal)); + reject(signal.reason); }; signal.addEventListener("abort", onAbort, { once: true }); timeout = setTimeout(() => { @@ -799,55 +1092,61 @@ function sleep(ms, signal) { }, ms); }); } -function abortReason(signal) { - return signal.reason ?? new DOMException("aborted", "AbortError"); -} -async function cancelResponseBody(response) { +function cancelResponseBody(response) { try { - await response.body?.cancel(); + const cancellation = response.body?.cancel(); + if (cancellation) { + void cancellation.catch(() => void 0); + } } catch { } } async function signAwsRequest(options) { - return signAwsRequestInternal(options); -} -async function signAwsRequestInternal(options, secretAccessKeyHash, parsedRequestUrl) { + const snapshot = snapshotSignAwsRequestOptions(options); + const signal = snapshot.signal ?? void 0; + signal?.throwIfAborted(); + const signed = await signAwsRequestInternal(snapshot); + signal?.throwIfAborted(); + return signed; +} +async function signAwsRequestInternal(options, secretAccessKeyHash, parsedRequestUrl, reusablePreparedBody) { validateCredentialOptions(options, "signAwsRequest options are required"); const cache = requireSigningCache(options.cache, "cache"); requireDefinedOption(options.url, "url"); const requestUrl = parsedRequestUrl ?? parseRequestUrl(options.url); const url = requestUrl.url; const method = normalizeMethod(options.method === void 0 ? defaultMethod(options.body) : options.method); - const headers = new Headers(options.headers || {}); + const headers = new Headers(options.headers); rejectEmptyHeader(headers, AMZ_CONTENT_SHA256_HEADER); const unsignedPayload = resolveUnsignedPayload(optionalBoolean(options.unsignedPayload, "unsignedPayload"), options.service); const signAllHeaders = optionalBoolean(options.signAllHeaders, "signAllHeaders"); - const unsignableHeaders = snapshotUnsignableHeaders(options, options.unsignableHeaders, "unsignableHeaders"); - const doubleUrlEncode = optionalBoolean(options.doubleUrlEncode, "doubleUrlEncode") ?? false; - if (unsignedPayload && !headers.has(AMZ_CONTENT_SHA256_HEADER)) { - headers.set(AMZ_CONTENT_SHA256_HEADER, UNSIGNED_PAYLOAD); - } + const unsignableHeaders = normalizeUnsignableHeaders(options.unsignableHeaders, "unsignableHeaders"); + const doubleUrlEncode = resolveDoubleUrlEncode(optionalBoolean(options.doubleUrlEncode, "doubleUrlEncode"), options.service); const explicitAmzDate = optionalAmzDate(options.signingDate); headers.set(HOST_HEADER, url.host); if (options.sessionToken) { headers.set(AMZ_SECURITY_TOKEN_HEADER, options.sessionToken); } validateSignedHeaderValues(headers, { + service: options.service, signAllHeaders, unsignableHeaders, overwrittenHeaderNames: signerOverwrittenHeaderNames(options.sessionToken !== void 0) }); const canonicalPath = canonicalPathname(requestUrl.pathname, options.service, doubleUrlEncode); - const preparedBody = await prepareHashedBody(options.body, headers, unsignedPayload); + const preparedBody = reusablePreparedBody ?? await prepareSigningBody(options.body, headers, { + service: options.service, + unsignedPayload, + replay: false, + signal: options.signal ?? void 0 + }); const amzDate = explicitAmzDate ?? formatAmzDate(/* @__PURE__ */ new Date()); const date = amzDate.slice(0, 8); const credentialScope = `${date}/${options.region}/${options.service}/${AWS_REQUEST}`; headers.set(AMZ_DATE_HEADER, amzDate); - const canonicalPayloadHash = await canonicalPayloadHashValue(headers, preparedBody.bytes); - if (options.service === "s3" && !headers.has(AMZ_CONTENT_SHA256_HEADER)) { - headers.set(AMZ_CONTENT_SHA256_HEADER, canonicalPayloadHash); - } + const canonicalPayloadHash = preparedBody.payloadHash; const { canonicalHeaders, signedHeaders } = canonicalHeaderBlock(url, headers, { + service: options.service, signAllHeaders, unsignableHeaders }); @@ -861,9 +1160,10 @@ async function signAwsRequestInternal(options, secretAccessKeyHash, parsedReques canonicalPayloadHash ].join("\n"); const stringToSign = [AWS_ALGORITHM, amzDate, credentialScope, await sha256Hex(canonicalRequest)].join("\n"); + const resolvedSecretAccessKeyHash = typeof secretAccessKeyHash === "function" ? await secretAccessKeyHash() : secretAccessKeyHash; const signature = await signatureHex({ secretAccessKey: options.secretAccessKey, - secretAccessKeyHash, + secretAccessKeyHash: resolvedSecretAccessKeyHash, date, region: options.region, service: options.service, @@ -882,209 +1182,184 @@ async function signAwsRequestInternal(options, secretAccessKeyHash, parsedReques body: preparedBody.body }; } -async function canonicalPayloadHashValue(headers, body) { - const explicit = headers.get(AMZ_CONTENT_SHA256_HEADER); - if (explicit) { - return explicit; - } - return sha256Hex(body); -} var MAX_RETRY_DELAY_MS = 2147483647; var SigV4Client = class { - accessKeyId; - secretAccessKey; - secretAccessKeyHash; - sessionToken; - service; - region; - cache; - retries; - initialRetryDelayMs; - maxRetryDelayMs; - unsignedPayload; - signAllHeaders; - unsignableHeaders; - doubleUrlEncode; - fetchFn; + #accessKeyId; + #secretAccessKey; + #secretAccessKeyHash; + #sessionToken; + #service; + #region; + #cache; + #retries; + #initialRetryDelayMs; + #maxRetryDelayMs; + #unsignedPayload; + #signAllHeaders; + #unsignableHeaders; + #doubleUrlEncode; + #fetchFn; constructor(options) { validateCredentialOptions(options, "SigV4Client options are required"); - this.accessKeyId = options.accessKeyId; - this.secretAccessKey = options.secretAccessKey; - this.sessionToken = options.sessionToken; - this.service = options.service; - this.region = options.region; - this.cache = requireSigningCache(options.cache, "cache") ?? /* @__PURE__ */ new Map(); - this.retries = requireNonNegativeInteger(options.retries ?? 0, "retries"); - this.initialRetryDelayMs = requireNonNegativeFiniteNumber(options.initialRetryDelayMs ?? 50, "initialRetryDelayMs"); - this.maxRetryDelayMs = requireNonNegativeFiniteNumber(options.maxRetryDelayMs ?? 5e3, "maxRetryDelayMs"); - this.unsignedPayload = optionalBoolean(options.unsignedPayload, "unsignedPayload"); - this.signAllHeaders = optionalBoolean(options.signAllHeaders, "signAllHeaders"); - this.unsignableHeaders = normalizeUnsignableHeaders(options.unsignableHeaders, "unsignableHeaders"); - this.doubleUrlEncode = optionalBoolean(options.doubleUrlEncode, "doubleUrlEncode"); + this.#accessKeyId = options.accessKeyId; + this.#secretAccessKey = options.secretAccessKey; + this.#sessionToken = options.sessionToken; + this.#service = options.service; + this.#region = options.region; + this.#cache = requireSigningCache(options.cache, "cache") ?? /* @__PURE__ */ new Map(); + this.#retries = requireNonNegativeInteger(options.retries === void 0 ? 0 : options.retries, "retries"); + this.#initialRetryDelayMs = requireNonNegativeFiniteNumber(options.initialRetryDelayMs === void 0 ? 50 : options.initialRetryDelayMs, "initialRetryDelayMs"); + this.#maxRetryDelayMs = requireNonNegativeFiniteNumber(options.maxRetryDelayMs === void 0 ? 5e3 : options.maxRetryDelayMs, "maxRetryDelayMs"); + this.#unsignedPayload = optionalBoolean(options.unsignedPayload, "unsignedPayload"); + this.#signAllHeaders = optionalBoolean(options.signAllHeaders, "signAllHeaders"); + this.#unsignableHeaders = normalizeUnsignableHeaders(options.unsignableHeaders, "unsignableHeaders"); + this.#doubleUrlEncode = optionalBoolean(options.doubleUrlEncode, "doubleUrlEncode"); const fetchFn = options.fetch === void 0 ? globalThis.fetch : options.fetch; if (typeof fetchFn !== "function") { throw new TypeError(options.fetch === void 0 ? "fetch is not available" : "fetch must be a function"); } - this.fetchFn = bindFetch(fetchFn); - } - async sign(input, init = {}) { - const requestInit = input instanceof Request ? mergeDefinedRequestInit(requestInitFromRequest(input), init) : { ...init }; - const signingOptions = normalizeClientSigningOptions(requestInit.signing); - delete requestInit.signing; - let url; - let method = requestInit.method; - let headers = requestInit.headers; - let body = requestInit.body; - if (input instanceof Request) { - url = input.url; - if (method === void 0) { - method = input.method; - } - headers = mergeHeaders(input.headers, headers); - if (body === void 0 && input.body) { - body = input.clone().body; - } - } else { - url = input; - } - const normalizedMethod = normalizeMethod(method === void 0 ? defaultMethod(body) : method); - rejectRequestBodyForGetHead(normalizedMethod, body); - const service = signingOptions.service ?? this.service; - const requestUrl = parseRequestUrl(url); - assertParsedRequestCanRepresentSignedUrl(requestUrl.pathname, service); - const signed = await signAwsRequestInternal({ - accessKeyId: this.accessKeyId, - secretAccessKey: this.secretAccessKey, - sessionToken: this.sessionToken, - service, - region: signingOptions.region ?? this.region, - cache: this.cache, - unsignedPayload: signingOptions.unsignedPayload ?? this.unsignedPayload, - signAllHeaders: signingOptions.signAllHeaders ?? this.signAllHeaders, - unsignableHeaders: signingOptions.unsignableHeaders ?? this.unsignableHeaders, - doubleUrlEncode: signingOptions.doubleUrlEncode ?? this.doubleUrlEncode, - signingDate: signingOptions.signingDate, - method: normalizedMethod, - url, - headers, - body - }, await this.getSecretAccessKeyHash(), requestUrl); - assertRequestCanRepresentSignedUrl(signed.url, service); - const signedInit = requestInitForSignedRequest(requestInit, signed); - try { - return new Request(signed.url, signedInit); - } catch (err) { - if (err instanceof TypeError) { - return new Request(signed.url, { - ...signedInit, - duplex: "half" - }); - } - throw err; - } + this.#fetchFn = bindFetch(fetchFn); } - async fetch(input, init = {}) { - const requestInit = { - ...init, - signing: normalizeClientSigningOptions(init.signing) - }; - const method = methodForRequest(input, requestInit); - rejectRequestBodyForGetHead(method, requestBodyForInput(input, requestInit)); - const service = requestInit.signing.service ?? this.service; - assertRequestCanRepresentSignedUrl(input instanceof Request ? input.url : input, service); - const replayBody = this.retries > 0 && isIdempotentMethod(method); - const retryInit = await reusableRequestInitForInput(input, requestInit, { - defaultService: this.service, - defaultUnsignedPayload: this.unsignedPayload, - defaultSignAllHeaders: this.signAllHeaders, - defaultUnsignableHeaders: this.unsignableHeaders, - hasClientSessionToken: this.sessionToken !== void 0, - replayBody - }); - for (let attempt = 0; attempt <= this.retries; attempt += 1) { - const fetchFn = this.fetchFn; - const request = await this.sign(input, retryInit); + async sign(input, init) { + const request = resolveClientSignRequest(input, init); + const signing = this.#resolveSigningOptions(request.init.signing); + delete request.init.signing; + assertParsedRequestCanRepresentSignedUrl(request.requestUrl.pathname, signing.service); + return this.#signResolvedRequest(request, signing); + } + async #signResolvedRequest(request, signing, preparedBody) { + request.signal?.throwIfAborted(); + const signed = await signAwsRequestInternal({ + accessKeyId: this.#accessKeyId, + secretAccessKey: this.#secretAccessKey, + sessionToken: this.#sessionToken, + service: signing.service, + region: signing.region, + cache: this.#cache, + unsignedPayload: signing.unsignedPayload, + signAllHeaders: signing.signAllHeaders, + unsignableHeaders: signing.unsignableHeaders, + doubleUrlEncode: signing.doubleUrlEncode, + signingDate: signing.signingDate, + signal: request.signal, + method: request.method, + url: request.requestUrl.href, + headers: request.headers, + body: request.body + }, () => this.#getSecretAccessKeyHash(), request.requestUrl, preparedBody); + request.signal?.throwIfAborted(); + const signedInit = requestInitForSignedRequest(request.init, signed); + return createSignedRequest(signed.url, signedInit, signed.headers); + } + async fetch(input, init) { + const request = resolveClientFetchRequest(input, init); + const signing = this.#resolveSigningOptions(request.init.signing); + delete request.init.signing; + assertParsedRequestCanRepresentSignedUrl(request.requestUrl.pathname, signing.service); + const retryableMethod = isIdempotentMethod(request.method); + const prepared = await prepareFetchRequest(request, signing, this.#sessionToken !== void 0, this.#retries > 0 && retryableMethod); + const fetchFn = this.#fetchFn; + for (let attempt = 0; attempt <= this.#retries; attempt += 1) { + const signedRequest = await this.#signResolvedRequest(prepared.request, signing, prepared.preparedBody); + const attemptSignal = validateRequestBeforeTransport(signedRequest); let response; try { - response = await fetchFn(request); + response = await fetchFn(signedRequest); } catch (err) { - if (attempt === this.retries || !isIdempotentMethod(request.method) || isAbortError(err, request)) { + attemptSignal.throwIfAborted(); + if (attempt === this.#retries || !retryableMethod || isAbortError(err)) { throw err; } - await sleep(Math.random() * this.retryDelayMs(attempt), request.signal); + await sleep(Math.random() * this.#retryDelayMs(attempt), attemptSignal); continue; } - const retryableResponse = isIdempotentMethod(request.method) && (response.status >= 500 || response.status === 429); - if (attempt === this.retries || !retryableResponse) { - return response; + if (attemptSignal.aborted) { + cancelResponseBody(response); + attemptSignal.throwIfAborted(); } - await cancelResponseBody(response); - if (request.signal.aborted) { - throw abortReason(request.signal); + if (prepared.request.redirectPolicy === "manual" && response.redirected) { + cancelResponseBody(response); + attemptSignal.throwIfAborted(); + throw new TypeError('SigV4Client.fetch custom transport followed a redirect despite redirect: "manual"'); } - await sleep(Math.random() * this.retryDelayMs(attempt), request.signal); + if (prepared.request.redirectPolicy === "error" && isRedirectResponse(response)) { + cancelResponseBody(response); + attemptSignal.throwIfAborted(); + throw new TypeError("SigV4Client.fetch received a redirect response; redirect targets must be re-signed"); + } + const retryableResponse = retryableMethod && (response.status >= 500 || response.status === 429); + if (attempt === this.#retries || !retryableResponse) { + return response; + } + cancelResponseBody(response); + attemptSignal.throwIfAborted(); + await sleep(Math.random() * this.#retryDelayMs(attempt), attemptSignal); } throw new Error("unreachable retry loop exit"); } - getSecretAccessKeyHash() { - if (this.secretAccessKeyHash !== void 0) { - return this.secretAccessKeyHash; + #resolveSigningOptions(value) { + const options = normalizeClientSigningOptions(value); + const service = options.service ?? this.#service; + return { + service, + region: options.region ?? this.#region, + unsignedPayload: resolveUnsignedPayload(options.unsignedPayload ?? this.#unsignedPayload, service), + signAllHeaders: options.signAllHeaders ?? this.#signAllHeaders, + unsignableHeaders: options.unsignableHeaders ?? this.#unsignableHeaders, + doubleUrlEncode: resolveDoubleUrlEncode(options.doubleUrlEncode ?? this.#doubleUrlEncode, service), + signingDate: options.signingDate + }; + } + #getSecretAccessKeyHash() { + if (this.#secretAccessKeyHash !== void 0) { + return this.#secretAccessKeyHash; } - const hash = sha256Hex(this.secretAccessKey).catch((err) => { - if (this.secretAccessKeyHash === hash) { - this.secretAccessKeyHash = void 0; + const hash = sha256Hex(this.#secretAccessKey).catch((err) => { + if (this.#secretAccessKeyHash === hash) { + this.#secretAccessKeyHash = void 0; } throw err; }); - this.secretAccessKeyHash = hash; + this.#secretAccessKeyHash = hash; return hash; } - retryDelayMs(attempt) { - return Math.min(MAX_RETRY_DELAY_MS, this.maxRetryDelayMs, this.initialRetryDelayMs * 2 ** attempt); + #retryDelayMs(attempt) { + return Math.min(MAX_RETRY_DELAY_MS, this.#maxRetryDelayMs, this.#initialRetryDelayMs * 2 ** attempt); } }; -async function reusableRequestInitForInput(input, init, options) { - if (!(input instanceof Request)) { - return reusableRequestInit(init, options); - } - const inputInit = { - ...init, - method: init.method ?? input.method, - headers: mergeHeaders(input.headers, init.headers) - }; - if (init.body === void 0 && input.body) { - inputInit.body = input.clone().body; - } - return reusableRequestInit(inputInit, options); -} -async function reusableRequestInit(init, options) { - const headers = new Headers(init.headers || {}); +async function prepareFetchRequest(request, signing, hasSessionToken, replay) { + const headers = new Headers(request.headers); rejectEmptyHeader(headers, AMZ_CONTENT_SHA256_HEADER); - const service = init.signing?.service ?? options.defaultService; - const unsignedPayload = resolveUnsignedPayload(init.signing?.unsignedPayload ?? options.defaultUnsignedPayload, service); - const unsignableHeaders = init.signing?.unsignableHeaders ?? options.defaultUnsignableHeaders; validateSignedHeaderValues(headers, { - signAllHeaders: init.signing?.signAllHeaders ?? options.defaultSignAllHeaders, - unsignableHeaders, - overwrittenHeaderNames: signerOverwrittenHeaderNames(options.hasClientSessionToken) + service: signing.service, + signAllHeaders: signing.signAllHeaders, + unsignableHeaders: signing.unsignableHeaders, + overwrittenHeaderNames: signerOverwrittenHeaderNames(hasSessionToken) + }); + const preparedBody = await prepareSigningBody(request.body, headers, { + service: signing.service, + unsignedPayload: signing.unsignedPayload, + replay, + signal: request.signal }); - const materializeBody = options.replayBody && (init.body instanceof FormData || init.body instanceof ReadableStream); - const hashPayload = shouldHashPayload(init.body, headers, unsignedPayload); - if (!materializeBody && !hashPayload) { - return { - ...init, - headers - }; - } - const body = await prepareHashedBody(init.body, headers, unsignedPayload, materializeBody); const out = { - ...init, + ...request.init, headers }; - if (body.body !== void 0) { - out.body = body.body; + if (preparedBody.body === void 0) { + delete out.body; + } else { + out.body = preparedBody.body; } - return out; + return { + request: { + ...request, + init: out, + headers, + body: preparedBody.body + }, + preparedBody + }; } export { SigV4Client, diff --git a/shared/version.js b/shared/version.js index 725d536..8723aad 100644 --- a/shared/version.js +++ b/shared/version.js @@ -105,3 +105,38 @@ export function workerVersionsKey(ns, worker) { export function doStorageIdKey(ns, worker) { return `worker:do-storage:${ns}:${worker}`; } + +// Per-worker lifecycle lock. Control owns acquisition/release; other tiers may +// WATCH it when creating state that whole-worker delete must discover. +export const WHOLE_DELETE_LOCK_KIND = "whole"; +export const VERSION_DELETE_LOCK_KIND = "version"; + +/** @param {string} ns @param {string} worker */ +export function deleteLockKey(ns, worker) { + return `worker-delete-lock:${ns}:${worker}`; +} + +/** + * @param {"whole" | "version"} kind + * @param {string} token + */ +export function formatDeleteLockToken(kind, token) { + if ( + (kind !== WHOLE_DELETE_LOCK_KIND && kind !== VERSION_DELETE_LOCK_KIND) || + typeof token !== "string" || !token + ) { + throw new TypeError("invalid worker delete lock token"); + } + return `${kind}:${token}`; +} + +/** @param {unknown} value @returns {"whole" | "version" | null} */ +export function parseDeleteLockKind(value) { + if (typeof value !== "string") return null; + const separator = value.indexOf(":"); + if (separator <= 0 || separator === value.length - 1) return null; + const kind = value.slice(0, separator); + return kind === WHOLE_DELETE_LOCK_KIND || kind === VERSION_DELETE_LOCK_KIND + ? kind + : null; +} diff --git a/terraform/modules/compute/scheduler_service.tf b/terraform/modules/compute/scheduler_service.tf index 6656660..4d9ec98 100644 --- a/terraform/modules/compute/scheduler_service.tf +++ b/terraform/modules/compute/scheduler_service.tf @@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "scheduler" { image = var.rust_image essential = true command = ["/scheduler"] - stopTimeout = 20 + stopTimeout = 30 environment = [ { name = "REDIS_URL", value = "redis://${local.redis_addr}" }, diff --git a/terraform/modules/compute/workflows_service.tf b/terraform/modules/compute/workflows_service.tf index 0ad489c..c1192e9 100644 --- a/terraform/modules/compute/workflows_service.tf +++ b/terraform/modules/compute/workflows_service.tf @@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "workflows" { image = var.rust_image essential = true command = ["/workflows"] - stopTimeout = 20 + stopTimeout = 30 portMappings = [{ name = "workflows-http" diff --git a/test-workers/do-rpc/src/index.js b/test-workers/do-rpc/src/index.js index 8d24117..1d8d2e1 100644 --- a/test-workers/do-rpc/src/index.js +++ b/test-workers/do-rpc/src/index.js @@ -44,6 +44,30 @@ export class Room extends DurableObject { result: await target.addMessage(text, { role: "peer" }), }; } + + async forwardRequestId(targetName) { + const target = this.env.ROOM.get(this.env.ROOM.idFromName(targetName)); + const response = await target.fetch(new Request("https://do.internal/request-id")); + return response.text(); + } + + async nestedForwardRequestId(targetName) { + const response = await this.fetch(new Request( + `https://do.internal/forward-request-id?to=${encodeURIComponent(targetName)}` + )); + return response.text(); + } + + fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/forward-request-id") { + const target = this.env.ROOM.get(this.env.ROOM.idFromName( + url.searchParams.get("to") || "peer" + )); + return target.fetch(new Request("https://do.internal/request-id")); + } + return new Response(request.headers.get("x-request-id") || ""); + } } export default { @@ -85,6 +109,16 @@ export default { "forwarded" )); } + if (url.pathname === "/request-id") { + return Response.json({ + requestId: await stub.forwardRequestId(url.searchParams.get("to") || "peer"), + }); + } + if (url.pathname === "/nested-request-id") { + return Response.json({ + requestId: await stub.nestedForwardRequestId(url.searchParams.get("to") || "peer"), + }); + } return Response.json(await stub.addMessage("hello", { role: "user" })); }, }; diff --git a/tests/fixtures/queue-key-parse.json b/tests/fixtures/queue-key-parse.json index 5a85d1b..175bb3f 100644 --- a/tests/fixtures/queue-key-parse.json +++ b/tests/fixtures/queue-key-parse.json @@ -24,7 +24,8 @@ ], "consumer": [ { "key": "queue-consumer:demo:jobs", "parsed": { "ns": "demo", "queue": "jobs" } }, - { "key": "queue-consumer:__platform__:jobs-1", "parsed": { "ns": "__platform__", "queue": "jobs-1" } }, + { "key": "queue-consumer:__system__:jobs-1", "parsed": { "ns": "__system__", "queue": "jobs-1" } }, + { "key": "queue-consumer:__platform__:jobs-1", "parsed": null }, { "key": "queue-consumer:demo", "parsed": null }, { "key": "queue-consumer:", "parsed": null }, { "key": "queue-consumer:admin:jobs", "parsed": null }, diff --git a/tests/helpers/control-shared-stub.js b/tests/helpers/control-shared-stub.js index e54a047..1ec1b2f 100644 --- a/tests/helpers/control-shared-stub.js +++ b/tests/helpers/control-shared-stub.js @@ -16,6 +16,7 @@ const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); const SHARED_OPTIMISTIC_RETRY_URL = repositoryFileUrl("shared/optimistic-retry.js"); const CONTROL_ERRORS_URL = freshRepositoryModuleDataUrl("control/errors.js", [ + [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], [/from "shared-respond"/g, `from ${JSON.stringify(SHARED_RESPOND_URL)}`], ]); const CONTROL_WORKFLOWS_CLIENT_URL = freshRepositoryModuleDataUrl("control/workflows-client.js", [ @@ -35,7 +36,7 @@ const CONTROL_JSON_BODY_URL = freshRepositoryModuleDataUrl("control/json-body.js const CONTROL_SHARED_BASE = ` import { jsonError, jsonResponse, sanitizeJsonErrorDetails } from ${JSON.stringify(SHARED_RESPOND_URL)}; import { createPostWorkflowsInternal } from ${JSON.stringify(CONTROL_WORKFLOWS_CLIENT_URL)}; -import { ControlAbort, codedErrorResponse, controlAbortResponse } from ${JSON.stringify(CONTROL_ERRORS_URL)}; +import { ControlAbort, controlAbortLogDetails, codedErrorLogFields, codedErrorResponse, controlAbortResponse, secretEnvelopeErrorResponse } from ${JSON.stringify(CONTROL_ERRORS_URL)}; import { runOptimistic, withOptimisticRetries } from ${JSON.stringify(CONTROL_OPTIMISTIC_URL)}; import { readJsonBody } from ${JSON.stringify(CONTROL_JSON_BODY_URL)}; import { errorMessage as errMessage } from ${JSON.stringify(SHARED_ERRORS_URL)}; @@ -43,8 +44,11 @@ import { randomHex } from ${JSON.stringify(SHARED_RANDOM_ID_URL)}; import { formatError } from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}; export { ControlAbort, + controlAbortLogDetails, + codedErrorLogFields, codedErrorResponse, controlAbortResponse, + secretEnvelopeErrorResponse, errMessage, formatError, jsonError, diff --git a/tests/helpers/load-do-host-actor.js b/tests/helpers/load-do-host-actor.js index 105990f..7d56a3e 100644 --- a/tests/helpers/load-do-host-actor.js +++ b/tests/helpers/load-do-host-actor.js @@ -19,6 +19,8 @@ import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; * assertCalls: number, * forgottenOwners: string[], * registryError?: unknown, + * registryWait?: Promise, + * registryWaitStarted?: (() => void), * abortReject?: ((reason: unknown) => void), * }} DoHostActorHarnessState */ @@ -52,6 +54,10 @@ export function objectRegistryMember(invoke) { return [invoke.className, invoke.objectName].join(":"); } export async function rememberDoObject(env, invoke) { + if (globalThis.__doActorTestState.registryWait) { + globalThis.__doActorTestState.registryWaitStarted?.(); + await globalThis.__doActorTestState.registryWait; + } if (globalThis.__doActorTestState.registryError) throw globalThis.__doActorTestState.registryError; globalThis.__doActorTestState.remembered.push({ env, invoke }); } @@ -62,6 +68,7 @@ export { DO_OWNERSHIP_CODE, DO_OWNERSHIP_ERROR_CONTROL_HEADER, DoRuntimeError, + buildRpcRequest, doPlatformErrorResponse, } from ${JSON.stringify(productionProtocolUrl)}; export function buildFacetName(invoke) { @@ -149,6 +156,8 @@ export function resetDoHostActorHarness() { state.assertCalls = 0; state.forgottenOwners = []; delete state.registryError; + delete state.registryWait; + delete state.registryWaitStarted; delete state.abortReject; } diff --git a/tests/helpers/load-do-owner-registry.js b/tests/helpers/load-do-owner-registry.js index 3e38344..f472313 100644 --- a/tests/helpers/load-do-owner-registry.js +++ b/tests/helpers/load-do-owner-registry.js @@ -35,6 +35,7 @@ const redisState = createFakeRedisState(); * @property {Array<{ level: string, event: string, fields: Record }>} logEntries * @property {number} redisTimeMs * @property {number[]} redisTimeSequence + * @property {(() => void) | null} onExecFailure * @property {boolean} draining * @property {number} inFlightDispatches */ @@ -49,6 +50,7 @@ export const DO_OWNER_REGISTRY_TEST_STATE = { logEntries: [], redisTimeMs: Date.now(), redisTimeSequence: [], + onExecFailure: null, draining: false, inFlightDispatches: 0, }; @@ -66,6 +68,7 @@ export function resetDoOwnerRegistryTestState() { testState.logEntries = []; testState.redisTimeMs = Date.now(); testState.redisTimeSequence = []; + testState.onExecFailure = null; testState.draining = false; testState.inFlightDispatches = 0; } @@ -114,6 +117,7 @@ export function createRedisClient() { if (Array.isArray(sequence) && sequence.length) return sequence.shift(); return testState().redisTimeMs; }, + onExecFailure: () => testState().onExecFailure?.(), }); } `); diff --git a/tests/integration/delete-api.test.js b/tests/integration/delete-api.test.js index cc2cfc2..d6d041c 100644 --- a/tests/integration/delete-api.test.js +++ b/tests/integration/delete-api.test.js @@ -445,7 +445,9 @@ test("POST /delete fail-closes when active route metadata is malformed", async ( assert.equal(r.status, 500); const body = await responseJson(r); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.stage, "active_meta_routes"); + assert.equal(body.message, "Internal error"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); assert.ok(redisHGetAll("patterns:bad-route.workers.example")["/api/*"]); assert.equal(redisSIsMember(`ns-hosts:${ns}`, "bad-route.workers.example"), true); @@ -656,9 +658,9 @@ test("POST /delete on a worker whose active bundle has corrupt __meta__ → 500 const body = await responseJson(r); assert.equal(body.error, "corrupt_meta"); assert.equal(body.version, "v1"); - // v1 is both active AND the only retained; which parse path trips - // first is an implementation detail, but it must be one of them. - assert.match(body.stage, /meta_parse$/); + assert.equal(body.message, "Internal error"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); assert.equal(redisHGet(`routes:${ns}`, "api"), preActive); assert.deepEqual(redisSMembers(`workers:${ns}`), preWorkers); @@ -680,7 +682,9 @@ test("POST /delete on a worker whose retained (non-active) bundle has corrupt __ const body = await responseJson(r); assert.equal(body.error, "corrupt_meta"); assert.equal(body.version, "v1"); - assert.equal(body.stage, "retained_meta_parse"); + assert.equal(body.message, "Internal error"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); // Active pointer must survive — no partial tear-down. assert.equal(redisHGet(`routes:${ns}`, "api"), "v2"); @@ -726,8 +730,10 @@ test("DELETE /versions/ with corrupt sibling meta during shared-prefix scan assert.equal(r.status, 500); const body = await responseJson(r); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.stage, "sibling_meta_parse"); assert.equal(body.version, "v2"); + assert.equal(body.message, "Internal error"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); assert.equal(redisExists(`worker:${ns}:w:v:1`), true); assert.equal(queueCleanupIntents().length, 0); diff --git a/tests/integration/durable-objects-core.test.js b/tests/integration/durable-objects-core.test.js index c7c0ee1..7f9cad9 100644 --- a/tests/integration/durable-objects-core.test.js +++ b/tests/integration/durable-objects-core.test.js @@ -14,6 +14,7 @@ import { DO_RPC_WORKER, DO_WORKER, } from "./helpers/durable-objects.js"; +import { redisDel, redisSetEx } from "./helpers/redis.js"; setupIntegrationSuite(); @@ -63,6 +64,33 @@ test("worker Durable Object binding routes through do-runtime and preserves obje }); }); +test("version-delete lock does not interrupt active Durable Object traffic", async () => { + const ns = uniqueNs("do-version-delete"); + await deployAndPromote(ns, "counter", { + mainModule: "worker.js", + modules: { "worker.js": DO_WORKER }, + bindings: { + COUNTER: { type: "do", className: "Counter" }, + }, + }); + + const lockKey = `worker-delete-lock:${ns}:counter`; + redisSetEx(lockKey, "version:integration-token", 30); + try { + const response = await gatewayFetch(ns, "/counter?name=alice"); + const text = await response.text(); + assert.equal(response.status, 200, text); + assert.deepEqual(responseJson({ body: text }), { + objectId: "alice", + memory: 1, + storage: 1, + body: "from-worker", + }); + } finally { + redisDel(lockKey); + } +}); + test("tenant top-level intrinsic patches cannot bypass host env wrapping", async () => { const ns = uniqueNs("do-intrinsic-guard"); await deployAndPromote(ns, "guard", { @@ -130,6 +158,7 @@ test("Durable Object RPC dispatches class methods with structured args", async ( await deployAndPromote(ns, "rooms", { mainModule: "worker.js", modules: { "worker.js": DO_RPC_WORKER }, + compatibilityFlags: ["no_nodejs_als"], bindings: { ROOM: { type: "do", className: "Room" }, }, @@ -205,4 +234,21 @@ test("Durable Object RPC dispatches class methods with structured args", async ( meta: { role: "peer" }, }, }); + + const requestId = "rid-do-rpc-context"; + const context = await gatewayFetch(ns, "/rooms/request-id?name=alice&to=bob", { + headers: { "x-request-id": requestId }, + }); + const contextText = await context.text(); + assert.equal(context.status, 200, contextText); + assert.deepEqual(responseJson({ body: contextText }), { requestId }); + + const nestedContext = await gatewayFetch( + ns, + "/rooms/nested-request-id?name=alice&to=bob", + { headers: { "x-request-id": requestId } } + ); + const nestedContextText = await nestedContext.text(); + assert.equal(nestedContext.status, 200, nestedContextText); + assert.deepEqual(responseJson({ body: nestedContextText }), { requestId }); }); diff --git a/tests/integration/helpers/stack.js b/tests/integration/helpers/stack.js index 520bc4c..0c5e438 100644 --- a/tests/integration/helpers/stack.js +++ b/tests/integration/helpers/stack.js @@ -182,7 +182,7 @@ export async function waitForScheduler() { redisHSet(consumerKey, { worker: "probe", version: "v1", - max_batch_size: "1000", + max_batch_size: "100", max_batch_timeout_ms: "1000", max_retries: "0", retry_delay_secs: "0", diff --git a/tests/unit/aws-sigv4.test.js b/tests/unit/aws-sigv4.test.js index 94b93df..88dbc45 100644 --- a/tests/unit/aws-sigv4.test.js +++ b/tests/unit/aws-sigv4.test.js @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import { SigV4Client as InstalledSigV4Client } from "@wdl-dev/aws-sigv4"; import { SigV4Client } from "../../shared/vendor/aws-sigv4.js"; const ACCESS_KEY_ID = "AKIDEXAMPLE"; @@ -9,6 +10,22 @@ const SESSION_TOKEN = "session-token-example"; const FIXED_DATETIME = "20260616T010203Z"; const S3_ENDPOINT = "https://s3.us-east-1.amazonaws.com"; +/** + * @typedef {{ + * fetch(input: Request | string | URL, init?: import("@wdl-dev/aws-sigv4").SigV4RequestInit): Promise, + * }} TestSigV4Client + * @typedef {(options: import("@wdl-dev/aws-sigv4").SigV4ClientOptions) => TestSigV4Client} TestSigV4ClientFactory + */ + +/** @type {TestSigV4ClientFactory} */ +const createInstalledClient = (options) => new InstalledSigV4Client(options); +/** @type {TestSigV4ClientFactory} */ +const createVendoredClient = (options) => new SigV4Client(options); +const SIGNER_IMPLEMENTATIONS = [ + { name: "installed package", createClient: createInstalledClient }, + { name: "vendored bundle", createClient: createVendoredClient }, +]; + // Fixed vectors are kept inline so vendored signer changes cannot silently // redefine expected output. const FIXTURES = [ @@ -70,32 +87,46 @@ const FIXTURES = [ }, ]; -test("aws-sigv4 S3 signing matches fixed SigV4 golden vectors", async () => { - for (const fixture of FIXTURES) { - const client = new SigV4Client({ - accessKeyId: ACCESS_KEY_ID, - secretAccessKey: SECRET_ACCESS_KEY, - sessionToken: fixture.sessionToken, - service: "s3", - region: "us-east-1", - cache: new Map(), - retries: 0, - }); - const signed = await client.sign(fixture.url, { - ...fixture.init, - signing: { signingDate: FIXED_DATETIME }, - }); - assert.equal(signed.url, fixture.expectedUrl, fixture.name); - assert.equal(signed.headers.get("authorization"), fixture.expectedAuthorization, fixture.name); - assert.equal(signed.headers.get("x-amz-date"), FIXED_DATETIME, fixture.name); - assert.equal(signed.headers.get("x-amz-content-sha256"), fixture.expectedContentSha256, fixture.name); - assert.equal(signed.headers.get("x-amz-security-token"), fixture.expectedSecurityToken ?? null, fixture.name); +test("aws-sigv4 fetch emits fixed S3 SigV4 golden vectors", async () => { + for (const implementation of SIGNER_IMPLEMENTATIONS) { + for (const fixture of FIXTURES) { + const label = `${implementation.name}: ${fixture.name}`; + /** @type {Request[]} */ + const requests = []; + const client = implementation.createClient({ + accessKeyId: ACCESS_KEY_ID, + secretAccessKey: SECRET_ACCESS_KEY, + sessionToken: fixture.sessionToken, + service: "s3", + region: "us-east-1", + cache: new Map(), + retries: 0, + fetch: async (input) => { + requests.push(/** @type {Request} */ (input)); + return new Response(null, { status: 200 }); + }, + }); + const response = await client.fetch(fixture.url, /** @type {import("@wdl-dev/aws-sigv4").SigV4RequestInit} */ ({ + ...fixture.init, + signing: { signingDate: FIXED_DATETIME }, + })); + assert.equal(response.status, 200, label); + assert.equal(requests.length, 1, label); + const signed = requests[0]; + assert.equal(signed.url, fixture.expectedUrl, label); + assert.equal(signed.method, fixture.init.method ?? "GET", label); + assert.equal(await signed.clone().text(), fixture.init.body ?? "", label); + assert.equal(signed.headers.get("authorization"), fixture.expectedAuthorization, label); + assert.equal(signed.headers.get("x-amz-date"), FIXED_DATETIME, label); + assert.equal(signed.headers.get("x-amz-content-sha256"), fixture.expectedContentSha256, label); + assert.equal(signed.headers.get("x-amz-security-token"), fixture.expectedSecurityToken ?? null, label); + } } }); -/** @param {typeof globalThis.fetch} fetch */ -function testClient(fetch) { - return new SigV4Client({ +/** @param {typeof globalThis.fetch} fetch @param {TestSigV4ClientFactory} [createClient] */ +function testClient(fetch, createClient = createVendoredClient) { + return createClient({ accessKeyId: ACCESS_KEY_ID, secretAccessKey: SECRET_ACCESS_KEY, service: "s3", @@ -122,7 +153,7 @@ test("aws-sigv4 fetch retries idempotent transient responses", async () => { url: request.url, method: request.method, authorization: request.headers.get("authorization"), - body: await request.clone().text(), + body: await request.text(), }); return new Response(null, { status: calls.length === 1 ? 500 : 200 }); }); @@ -145,6 +176,31 @@ test("aws-sigv4 fetch retries idempotent transient responses", async () => { } }); +test("aws-sigv4 retry does not wait for response cancellation", { timeout: 2_000 }, async () => { + for (const implementation of SIGNER_IMPLEMENTATIONS) { + let calls = 0; + let cancellations = 0; + const client = testClient(async () => { + calls += 1; + if (calls === 2) return new Response("ok"); + return new Response(new ReadableStream({ + cancel() { + cancellations += 1; + return new Promise(() => {}); + }, + }), { status: 500 }); + }, implementation.createClient); + + const response = await client.fetch(`${S3_ENDPOINT}/wdl-r2/retry-cancel.txt`, { + signing: { signingDate: FIXED_DATETIME }, + }); + + assert.equal(await response.text(), "ok", implementation.name); + assert.equal(calls, 2, implementation.name); + assert.equal(cancellations, 1, implementation.name); + } +}); + test("aws-sigv4 fetch does not retry POST transient responses", async () => { /** @type {Request[]} */ const calls = []; diff --git a/tests/unit/bounded-body.test.js b/tests/unit/bounded-body.test.js index e51d6b6..155bcc9 100644 --- a/tests/unit/bounded-body.test.js +++ b/tests/unit/bounded-body.test.js @@ -77,3 +77,31 @@ test("readBoundedStreamBytes supports caller-owned overflow errors", async () => ); assert.equal(cancelled, true); }); + +test("readBoundedStreamBytes rejects without waiting for stream cancellation", async () => { + let cancelled = false; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2, 3])); + }, + cancel() { + cancelled = true; + return new Promise(() => {}); + }, + }); + let timeout; + try { + await assert.rejects( + Promise.race([ + readBoundedStreamBytes(stream, 2, () => new TypeError("custom limit")), + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error("body reader waited for stream cancellation")), 1000); + }), + ]), + /custom limit/ + ); + } finally { + clearTimeout(timeout); + } + assert.equal(cancelled, true); +}); diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index 339ed08..54139b2 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -20,7 +20,11 @@ const controlSharedExtraSource = ` export const ROUTES_CHANNEL = "routes:invalidate"; export const ROUTES_FLUSH_CHANNEL = "routes:flush"; export const PATTERNS_CHANNEL = "patterns:invalidate"; -export async function acquireDeleteLock() { return "lock-token"; } +export async function acquireDeleteLock(_redis, _ns, _worker, kind) { + /** @type {any} */ (globalThis).__deleteHandlerState.lockKinds.push(kind); + return "lock-token"; +} +export async function renewDeleteLock() { return true; } export async function assertWorkflowDeleteAllowed(args) { /** @type {any} */ (globalThis).__deleteHandlerState.workflowChecks.push(args); if (/** @type {any} */ (globalThis).__deleteHandlerWorkflowBlocker) { @@ -48,7 +52,11 @@ export async function recordS3CleanupIntent(intent) { const versionsControlSharedExtraSource = ` export function hasCompleteBundle() { return true; } -export async function acquireDeleteLock() { return "lock-token"; } +export async function acquireDeleteLock(_redis, _ns, _worker, kind) { + /** @type {any} */ (globalThis).__versionDeleteHandlerState.lockKinds.push(kind); + return "lock-token"; +} +export async function renewDeleteLock() { return true; } export async function assertWorkflowDeleteAllowed(args) { /** @type {any} */ (globalThis).__versionDeleteHandlerState.workflowChecks.push(args); } @@ -87,6 +95,7 @@ const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const sharedSecretKeysUrl = repositoryFileUrl("shared/secret-keys.js"); const sharedRedisUrl = sharedRedisStubUrl(); +const { WatchError: TestWatchError } = await import(sharedRedisUrl); const sharedQueueKeysUrl = moduleDataUrl(` export const QUEUE_CONSUMER_INDEX_KEY = "queue:index:consumers"; @@ -144,14 +153,22 @@ const { handle: handleVersions } = await importControlHandler("control/handlers/ * assetPrefix?: string | null, * hasWorkerSecrets?: boolean, * doStorageId?: string | null, + * doStorageIdDuringExec?: string | null, * doOwnerKeys?: string[], * doOwnerKeysDuringExec?: string[], * queueConsumerKeys?: string[], + * queueConsumerKeysDuringExec?: string[], * queueConsumerWorker?: string | null, * queueConsumerWorkerDuringExec?: string | null, * referrersByVersion?: Record, * retainedVersions?: string[], * bundleMetaRaw?: string | null, + * lockTokenDuringExec?: string | null, + * replaceLockAfterRead?: boolean, + * siblingVersion?: string | null, + * siblingVersionBeforeFirstWatch?: string | null, + * patternRecords?: Record>, + * patternRecordsDuringExec?: Record>, * }} [opts] */ function resetDeleteHandlerState({ @@ -159,14 +176,22 @@ function resetDeleteHandlerState({ assetPrefix = "assets/demo/api/v1/", hasWorkerSecrets = false, doStorageId = "do_old", + doStorageIdDuringExec = doStorageId, doOwnerKeys = [], doOwnerKeysDuringExec = doOwnerKeys, queueConsumerKeys = ["queue-consumer:demo:jobs"], + queueConsumerKeysDuringExec = queueConsumerKeys, queueConsumerWorker = "api", queueConsumerWorkerDuringExec = queueConsumerWorker, referrersByVersion = {}, retainedVersions = ["v1"], bundleMetaRaw, + lockTokenDuringExec = "lock-token", + replaceLockAfterRead = false, + siblingVersion = null, + siblingVersionBeforeFirstWatch = null, + patternRecords = {}, + patternRecordsDuringExec = patternRecords, } = {}) { const referrers = /** @type {Record} */ (referrersByVersion); const meta = bundleMetaRaw === undefined @@ -180,6 +205,19 @@ function resetDeleteHandlerState({ const multiCalls = []; /** @type {unknown[][]} */ const commands = []; + /** @type {string[]} */ + let watchedKeys = []; + /** @type {string[][]} */ + const watchBatches = []; + let watchedLockToken = lockTokenDuringExec; + let currentLockToken = lockTokenDuringExec; + let lockReplaced = false; + /** @type {Record} */ + const routeVersions = { + ...(activeVersion ? { api: activeVersion } : {}), + ...(siblingVersion ? { workerB: siblingVersion } : {}), + }; + let siblingChanged = false; /** @param {string} key */ function versionReferrers(key) { const match = /^worker-version-referrers:demo:api:(v[1-9][0-9]*)$/.exec(key); @@ -187,8 +225,34 @@ function resetDeleteHandlerState({ return []; } const session = { - async watch() {}, - async unwatch() {}, + /** @param {string[]} keys */ + async watch(...keys) { + if ( + siblingVersionBeforeFirstWatch && + !siblingChanged && + keys.includes("routes:demo") + ) { + routeVersions.workerB = siblingVersionBeforeFirstWatch; + siblingChanged = true; + } + watchedKeys = keys; + watchBatches.push(keys); + watchedLockToken = currentLockToken; + }, + async unwatch() { watchedKeys = []; }, + /** @param {string} key */ + async get(key) { + if (key === "worker-delete-lock:demo:api") { + const token = currentLockToken; + if (replaceLockAfterRead && !lockReplaced) { + currentLockToken = "replacement-token"; + lockReplaced = true; + } + return token; + } + if (key === "worker:do-storage:demo:api") return doStorageIdDuringExec; + return null; + }, /** @param {string} key */ async zRange(key) { if (key === "worker-versions:demo:api") return retainedVersions; @@ -196,7 +260,8 @@ function resetDeleteHandlerState({ }, /** @param {string} key @param {string} field */ async hGet(key, field) { - if (key === "routes:demo" && field === "api") return activeVersion; + if (key === "routes:demo") return routeVersions[field] || null; + if (key.startsWith("patterns:")) return patternRecordsDuringExec[key]?.[field] || null; if (queueConsumerKeys.includes(key) && field === "worker") { return queueConsumerWorkerDuringExec; } @@ -218,19 +283,23 @@ function resetDeleteHandlerState({ /** @param {string[]} keys */ async hGetAllMany(keys) { commands.push(["SESSION_HGETALLMANY", keys]); - return keys.map(() => ({})); + return keys.map((key) => patternRecordsDuringExec[key] || {}); }, /** @param {string} _cursor @param {string} pattern */ async scan(_cursor, pattern) { if (pattern === "do:owner:scope:do_old%3A*") return ["0", doOwnerKeysDuringExec]; if (pattern === "queue-consumer:demo:*" && queueConsumerWorkerDuringExec) { - return ["0", queueConsumerKeys]; + return ["0", queueConsumerKeysDuringExec]; } return ["0", []]; }, /** @param {unknown[]} args */ async del(...args) { multiCalls.push(["DEL", ...args]); }, - async hGetAll() { return {}; }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "routes:demo") return { ...routeVersions }; + return {}; + }, multi() { return { /** @param {unknown[]} args */ @@ -243,7 +312,15 @@ function resetDeleteHandlerState({ zRem(...args) { multiCalls.push(["ZREM", ...args]); return this; }, /** @param {unknown[]} args */ publish(...args) { multiCalls.push(["PUBLISH", ...args]); return this; }, - async exec() { multiCalls.push(["EXEC"]); }, + async exec() { + if ( + watchedKeys.includes("worker-delete-lock:demo:api") && + currentLockToken !== watchedLockToken + ) { + throw new TestWatchError(); + } + multiCalls.push(["EXEC"]); + }, }; }, }; @@ -268,7 +345,7 @@ function resetDeleteHandlerState({ }, /** @param {string} key @param {string} field */ async hGet(key, field) { - if (key === "routes:demo" && field === "api") return activeVersion; + if (key === "routes:demo") return routeVersions[field] || null; if (/^worker:demo:api:v:\d+$/.test(key) && field === "__meta__") return meta; if (queueConsumerKeys.includes(key) && field === "worker") return queueConsumerWorker; return null; @@ -284,7 +361,7 @@ function resetDeleteHandlerState({ async sMembers(key) { return versionReferrers(key); }, /** @param {string} key */ async hGetAll(key) { - if (key === "routes:demo" && activeVersion) return { api: activeVersion }; + if (key === "routes:demo") return { ...routeVersions }; return {}; }, /** @param {string[]} keys */ @@ -292,7 +369,7 @@ function resetDeleteHandlerState({ commands.push(["CLIENT_HGETALLMANY", keys]); return keys.map((key) => { if (key === "routes:demo" && activeVersion) return { api: activeVersion }; - return {}; + return patternRecords[key] || {}; }); }, /** @param {string} key */ @@ -309,21 +386,25 @@ function resetDeleteHandlerState({ cleanupIntents: [], doAlarmCleanups: [], logs: [], + lockKinds: [], multiCalls, releaseCalls: 0, redis, metrics: { increment() {}, observe() {} }, service: "control", workflowChecks: [], + watchBatches, + get watchedKeys() { return watchedKeys; }, }); } -/** @param {{ assetPrefix?: string | null, bundleMetaRaw?: string | null, retainedVersions?: string[], siblingMetaRaw?: string | null }} [opts] */ +/** @param {{ assetPrefix?: string | null, bundleMetaRaw?: string | null, retainedVersions?: string[], siblingMetaRaw?: string | null, lockTokenDuringExec?: string | null }} [opts] */ function resetVersionDeleteHandlerState({ assetPrefix = "assets/demo/api/v1/", bundleMetaRaw, retainedVersions = ["v1"], siblingMetaRaw = null, + lockTokenDuringExec = "lock-token", } = {}) { const meta = bundleMetaRaw === undefined ? JSON.stringify({ @@ -336,6 +417,11 @@ function resetVersionDeleteHandlerState({ const session = { async watch() {}, async unwatch() {}, + /** @param {string} key */ + async get(key) { + if (key === "worker-delete-lock:demo:api") return lockTokenDuringExec; + return null; + }, /** @param {string} key @param {string} field */ async hGet(key, field) { if (key === "routes:demo" && field === "api") return null; @@ -369,6 +455,7 @@ function resetVersionDeleteHandlerState({ return installControlHandlerState("__versionDeleteHandlerState", { cleanupIntents: [], logs: [], + lockKinds: [], multiCalls, releaseCalls: 0, redis, @@ -451,6 +538,7 @@ test("worker delete reports cleanup_queue_failed when data-plane cleanup enqueue const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); + assert.deepEqual(testState.lockKinds, ["whole"]); assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api", allowCleanup: true }]); assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); assert.equal(body.assets.queueHint, "failed"); @@ -515,8 +603,16 @@ test("worker delete classifies non-object retained bundle metadata as corrupt", const body = await readJsonResponse(response, 500); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.stage, "retained_meta_parse"); - assert.equal(body.detail, "__meta__ must be a JSON object"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); + const rejection = /** @type {any} */ (testState.logs.find((/** @type {any} */ entry) => + entry.event === "worker_delete_rejected" + )); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.metadata_version, "v1"); + assert.equal(rejection.fields.stage, "retained_meta_parse"); + assert.equal(rejection.fields.error_detail, "__meta__ must be a JSON object"); assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); assert.equal(testState.releaseCalls, 1); }); @@ -538,11 +634,37 @@ test("worker delete fails closed when indexed active bundle metadata is missing" const body = await readJsonResponse(response, 500); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.stage, "active_meta_parse"); + assert.equal(body.stage, undefined); assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); assert.equal(testState.releaseCalls, 1); }); +test("worker delete dry-run logs redacted metadata diagnostics", async () => { + const testState = resetDeleteHandlerState({ bundleMetaRaw: "[]" }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-dry-run-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); + const rejection = /** @type {any} */ (testState.logs.find((/** @type {any} */ entry) => + entry.event === "worker_dry_run_rejected" + )); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.metadata_version, "v1"); + assert.equal(rejection.fields.stage, "retained_meta_parse"); + assert.equal(rejection.fields.error_detail, "__meta__ must be a JSON object"); +}); + for (const { label, bundleMetaRaw } of [ { label: "missing", bundleMetaRaw: null }, { label: "empty", bundleMetaRaw: "" }, @@ -564,7 +686,8 @@ for (const { label, bundleMetaRaw } of [ const body = await readJsonResponse(response, 500); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.stage, "retained_meta_parse"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); assert.equal(testState.releaseCalls, 1); }); @@ -602,6 +725,7 @@ test("worker delete collects retained version metadata without serial round trip const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); + assert.deepEqual(testState.lockKinds, ["whole"]); assert.deepEqual(body.versionsDeleted, ["v1", "v2", "v3"]); assert.ok( maxActiveBundleMetaReads > 1, @@ -661,6 +785,240 @@ test("worker delete retries instead of committing when queue consumer keys drift assert.deepEqual(testState.doAlarmCleanups, []); }); +test("worker delete deduplicates Redis SCAN results before drift checks and counts", async () => { + const ownerKey = "do:owner:scope:do_old%3ARoom%3Ashard0"; + const queueKey = "queue-consumer:demo:jobs"; + const testState = resetDeleteHandlerState({ + assetPrefix: null, + doOwnerKeys: [ownerKey, ownerKey], + doOwnerKeysDuringExec: [ownerKey], + queueConsumerKeys: [queueKey, queueKey], + queueConsumerKeysDuringExec: [queueKey], + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-scan-duplicates", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(body.queueConsumersRemoved, 1); + assert.deepEqual( + testState.commands.find((/** @type {unknown[]} */ call) => call[0] === "CLIENT_HGETMANY"), + ["CLIENT_HGETMANY", [[queueKey, "worker"]]], + ); + assert.equal( + (testState.watchBatches.at(-1) || []).filter((key) => key === ownerKey).length, + 1, + ); +}); + +test("worker delete uses the under-WATCH sibling route snapshot", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + siblingVersionBeforeFirstWatch: "v2", + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-sibling-promote-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(body.namespaceStillActive, true); + assert.equal(testState.watchBatches.length, 1); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => + call[0] === "SREM" && call[1] === "namespaces" && call[2] === "demo" + ), false); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => + call[0] === "PUBLISH" && call[1] === "routes:flush" + ), false); + assert.ok(testState.multiCalls.some((/** @type {any} */ call) => + call[0] === "PUBLISH" && call[1] === "routes:invalidate" && call[2] === "demo" + )); +}); + +test("worker delete ignores an unrelated sibling version change before WATCH", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + siblingVersion: "v2", + siblingVersionBeforeFirstWatch: "v3", + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-sibling-version-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(body.namespaceStillActive, true); + assert.equal(testState.watchBatches.length, 1); +}); + +test("worker delete recomputes sibling host ownership under WATCH", async () => { + const host = "app.example"; + const testState = resetDeleteHandlerState({ + assetPrefix: null, + bundleMetaRaw: JSON.stringify({ + bindings: {}, + routes: [{ host, slot: "target-slot" }], + }), + siblingVersionBeforeFirstWatch: "v2", + patternRecordsDuringExec: { + [`patterns:${host}`]: { + "sibling-slot": "v2\tdemo\tworkerB\tv2\texact\t/other", + }, + }, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-sibling-host-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.namespaceStillActive, true); + assert.equal(testState.watchBatches.length, 1); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => + call[0] === "SREM" && call[1] === "ns-hosts:demo" && call.includes(host) + ), false); +}); + +test("worker delete cannot commit after its delete lock token is replaced", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + lockTokenDuringExec: "replacement-token", + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-lock-replaced", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.deepEqual(testState.doAlarmCleanups, []); +}); + +test("worker delete WATCH rejects a lock replacement after the token read", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + replaceLockAfterRead: true, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-lock-replaced-after-read", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.ok(testState.watchBatches.some((/** @type {string[]} */ keys) => + keys.includes("worker-delete-lock:demo:api") + )); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); +}); + +test("worker delete rejects an active version missing from the retained index", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + retainedVersions: [], + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-active-index-drift", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "projection_drift"); + assert.equal(body.active_version, "v1"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); +}); + +test("worker delete dry-run rejects an active version missing from the retained index", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + retainedVersions: [], + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-dry-run-active-index-drift", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "projection_drift"); + assert.equal(body.active_version, "v1"); + assert.equal(body.dryRun, true); + assert.equal(testState.releaseCalls, 0); +}); + +test("worker delete dry-run retries an active/index snapshot split by a secret bump", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: "v2", + assetPrefix: null, + retainedVersions: ["v1", "v2"], + }); + let versionReads = 0; + testState.redis.zRange = async (/** @type {string} */ key) => { + assert.equal(key, "worker-versions:demo:api"); + versionReads += 1; + return versionReads === 1 ? ["v1"] : ["v1", "v2"]; + }; + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-dry-run-secret-bump-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.dryRun, true); + assert.deepEqual(body.versionsDeleted, ["v1", "v2"]); + assert.equal(versionReads, 3); + assert.equal(testState.releaseCalls, 0); +}); + test("worker delete dry-run includes workflow lifecycle blockers", async () => { const testState = resetDeleteHandlerState(); /** @type {any} */ (globalThis).__deleteHandlerWorkflowBlocker = { @@ -904,7 +1262,34 @@ test("worker delete retry compensates DO alarm cleanup when stale storage pointe const body = await readJsonResponse(response, 200); assert.equal(body.deleted, false); assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); - assert.deepEqual(testState.multiCalls, [["DEL", "worker:do-storage:demo:api"]]); + assert.deepEqual(testState.multiCalls, [ + ["DEL", "worker:do-storage:demo:api"], + ["EXEC"], + ]); +}); + +test("worker delete noop cannot clean residual state after its lock is replaced", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + assetPrefix: null, + queueConsumerWorker: null, + retainedVersions: [], + lockTokenDuringExec: "replacement-token", + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-noop-lock-replaced", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.deepEqual(testState.multiCalls, []); + assert.deepEqual(testState.doAlarmCleanups, []); }); test("worker delete noop skips DO alarm cleanup when old storage id is absent", async () => { @@ -972,6 +1357,7 @@ test("version delete reports cleanup_queue_failed when data-plane cleanup enqueu const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); + assert.deepEqual(testState.lockKinds, ["version"]); assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api", @@ -1002,6 +1388,26 @@ test("version delete reports cleanup_queue_failed when data-plane cleanup enqueu )); }); +test("version delete cannot commit after its delete lock token is replaced", async () => { + const testState = resetVersionDeleteHandlerState({ + lockTokenDuringExec: "replacement-token", + }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: "rid-version-delete-lock-replaced", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.deepEqual(testState.cleanupIntents, []); +}); + test("version delete classifies non-object bundle metadata as corrupt", async () => { const testState = resetVersionDeleteHandlerState({ bundleMetaRaw: "null" }); @@ -1019,6 +1425,14 @@ test("version delete classifies non-object bundle metadata as corrupt", async () assert.equal(body.namespace, "demo"); assert.equal(body.name, "api"); assert.equal(body.version, "v1"); + const rejection = /** @type {any} */ (testState.logs.find((/** @type {any} */ entry) => + entry.event === "version_delete_rejected" + )); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.metadata_version, "v1"); + assert.equal(rejection.fields.stage, "target_meta_parse"); + assert.equal(rejection.fields.error_detail, "__meta__ must be a JSON object"); assert.deepEqual(testState.cleanupIntents, []); assert.equal(testState.releaseCalls, 1); }); @@ -1065,7 +1479,17 @@ for (const [label, siblingMetaRaw] of [ const body = await readJsonResponse(response, 500); assert.equal(body.error, "corrupt_meta"); assert.equal(body.version, "v2"); - assert.equal(body.stage, "sibling_meta_parse"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); + const rejection = /** @type {any} */ (testState.logs.find((/** @type {any} */ entry) => + entry.event === "version_delete_rejected" + )); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.version, "v1"); + assert.equal(rejection.fields.metadata_version, "v2"); + assert.equal(rejection.fields.stage, "sibling_meta_parse"); + assert.equal(typeof rejection.fields.error_detail, "string"); assert.deepEqual(testState.cleanupIntents, []); assert.equal(testState.releaseCalls, 1); }); diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 5fd228e..9b155fa 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -42,6 +42,7 @@ const CONTROL_DEPLOY_TEST_STATE = { watchedKeys: null, envBudgetError: false, envBudgetCalls: [], + secretEnvelopeError: null, redis: null, logs: [], metrics: { increment() {}, observe() {} }, @@ -70,6 +71,7 @@ function resetControlDeployTestState() { CONTROL_DEPLOY_TEST_STATE.watchedKeys = null; CONTROL_DEPLOY_TEST_STATE.envBudgetError = false; CONTROL_DEPLOY_TEST_STATE.envBudgetCalls = []; + CONTROL_DEPLOY_TEST_STATE.secretEnvelopeError = null; CONTROL_DEPLOY_TEST_STATE.logs = []; CONTROL_DEPLOY_TEST_STATE.metrics = { increment() {}, observe() {} }; CONTROL_DEPLOY_TEST_STATE.service = "control"; @@ -232,7 +234,17 @@ export async function resolveDatabaseRefFrom(session, ns, databaseRef) { } `); +const secretEnvelopeUrl = moduleDataUrl(` +export class SecretEnvelopeError extends Error { + constructor(code, message) { + super(message); + this.code = code; + } +} +`); + const controlEnvBudgetUrl = moduleDataUrl(` +import { SecretEnvelopeError } from ${JSON.stringify(secretEnvelopeUrl)}; export class WorkerEnvBudgetError extends Error { constructor(message, details = {}) { super(message); @@ -253,15 +265,12 @@ export function assertWorkerLoaderUserEnvBudget() { } return 0; } -export async function decryptSecretHash() { return {}; } -`); - -const secretEnvelopeUrl = moduleDataUrl(` -export class SecretEnvelopeError extends Error { - constructor(code, message) { - super(message); - this.code = code; +export async function decryptSecretHash() { + const error = /** @type {any} */ (globalThis).__controlDeployTestState.secretEnvelopeError; + if (error) { + throw new SecretEnvelopeError(error.code, error.message); } + return {}; } `); @@ -702,7 +711,11 @@ test("deploy handler classifies empty service target metadata before commit", as const body = await readJsonResponse(response, 500); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.message, "Corrupt __meta__ for other/api/v1"); + assert.equal(body.message, "Internal error"); + assert.ok(CONTROL_DEPLOY_TEST_STATE.logs.some((/** @type {any} */ entry) => ( + entry.event === "deploy_rejected" && + entry.fields.error_message === "Corrupt __meta__ for other/api/v1" + ))); }); test("deploy handler fails closed instead of hiding empty platform export metadata", async () => { @@ -730,7 +743,7 @@ test("deploy handler fails closed instead of hiding empty platform export metada const body = await readJsonResponse(response, 500); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.message, "Corrupt __meta__ for __platform__/auth/v1"); + assert.equal(body.message, "Internal error"); }); test("deploy handler schedules cleanup when the first asset upload fails", async () => { @@ -784,7 +797,10 @@ test("deploy handler schedules cleanup when the first asset upload fails", async requestId: "rid-asset-upload-fail", }); - assert.equal(response.status, 502); + const body = await readJsonResponse(response, 502); + assert.equal(body.error, "asset_upload_failed"); + assert.equal(body.message, "Asset upload failed"); + assert.doesNotMatch(JSON.stringify(body), /simulated upload failure/); assert.deepEqual(/** @type {any} */ (globalThis).__controlDeployTestState.putAssetCalls[0].slice(0, 2), [ {}, "assets/demo/style.css", @@ -962,6 +978,186 @@ test("deploy handler preserves warnings on commit env-budget rejection", async ( } }); +test("deploy handler filters diagnostic aliases from rejection logs", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.execFailures = 10; + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + async hKeys() { + return []; + }, + async hGetAll() { + return {}; + }, + async hGet() { + return null; + }, + async incr() { + return 1; + }, + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/contention/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "contention", + requestId: "rid-deploy-contention", + }); + + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "deploy_contention"); + const rejection = CONTROL_DEPLOY_TEST_STATE.logs.find((/** @type {any} */ entry) => + entry.event === "deploy_rejected" + ); + assert.equal(rejection.fields.version, "v1"); + assert.equal(rejection.fields.error_message, "exhausted 5 retries; retry later"); + assert.equal(Object.hasOwn(rejection.fields, "message"), false); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + } +}); + +test("deploy handler keeps D1 ids camelCase on the wire and snake_case in logs", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + async hKeys() { + return []; + }, + async hGetAll() { + return {}; + }, + async hGet() { + return null; + }, + async incr() { + return 2; + }, + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/d1-missing/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + bindings: { + DB: { type: "d1", databaseId: "missing-db" }, + }, + }), + }), + env: {}, + ns: "tenant-a", + name: "d1-missing", + requestId: "rid-deploy-d1-missing", + }); + + const body = await readJsonResponse(response, 404); + assert.equal(body.error, "d1_database_not_found"); + assert.equal(body.databaseId, "missing-db"); + const rejection = CONTROL_DEPLOY_TEST_STATE.logs.find((/** @type {any} */ entry) => + entry.event === "deploy_rejected" + ); + assert.equal(rejection.fields.database_id, "missing-db"); + assert.equal(Object.hasOwn(rejection.fields, "databaseId"), false); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + } +}); + +test("deploy handler hides secret provider diagnostics and logs them", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "module" } }, + }, + normalized: [["worker.js", "export default {}"]], + }; + /** @type {any} */ (globalThis).__controlDeployTestState.secretEnvelopeError = { + code: "secret_encryption_unconfigured", + message: "SECRET_ENVELOPE_LOCAL_KEY_B64 must decode to 32 bytes", + }; + + installPlatformAuthWarningFixture({ + async incr() { + return 8; + }, + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }); + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/secret-error/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "secret-error", + requestId: "rid-secret-provider", + }); + + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "secret_encryption_unconfigured"); + assert.equal(body.message, "Internal error"); + assert.equal(JSON.stringify(body).includes("SECRET_ENVELOPE_LOCAL_KEY_B64"), false); + assert.deepEqual(body.warnings, [{ + binding: "AUTH", + platform: "auth", + missingCallerSecrets: ["API_TOKEN"], + }]); + assert.deepEqual( + CONTROL_DEPLOY_TEST_STATE.logs.find((/** @type {any} */ entry) => + entry.event === "deploy_rejected" + ), + { + level: "error", + event: "deploy_rejected", + fields: { + request_id: "rid-secret-provider", + namespace: "tenant-a", + worker: "secret-error", + version: "v8", + status: 503, + reason: "secret_encryption_unconfigured", + error_message: "SECRET_ENVELOPE_LOCAL_KEY_B64 must decode to 32 bytes", + error_detail: "SECRET_ENVELOPE_LOCAL_KEY_B64 must decode to 32 bytes", + }, + }, + ); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; + /** @type {any} */ (globalThis).__controlDeployTestState.parsedPlatformBindings = null; + /** @type {any} */ (globalThis).__controlDeployTestState.secretEnvelopeError = null; + } +}); + test("deploy handler rejects runtime reserved module collisions before version allocation", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); diff --git a/tests/unit/control-handlers-workflows.test.js b/tests/unit/control-handlers-workflows.test.js index 8221b81..ce4a5b4 100644 --- a/tests/unit/control-handlers-workflows.test.js +++ b/tests/unit/control-handlers-workflows.test.js @@ -96,7 +96,7 @@ test("workflows handler lists active workflow definitions from bundle metadata", for (const [label, rawMeta] of [ ["missing", null], ["empty", ""], - ["malformed", "{bad"], + ["malformed", "SECRET_TOKEN_ABC"], ["non-object", "[]"], ]) { test(`workflows handler fails closed on ${label} active bundle metadata`, async () => { @@ -115,9 +115,30 @@ for (const [label, rawMeta] of [ }); await assertJsonResponse(response, 500, { + namespace: "demo", + worker: "api", + version: "v2", error: "corrupt_meta", - message: "Corrupt __meta__ for demo/api/v2", + message: "Internal error", }); + const rejection = state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_request_rejected" + ); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.request_id, `rid-${label}-meta`); + assert.equal(rejection.fields.namespace, "demo"); + assert.equal(rejection.fields.worker, "api"); + assert.equal(rejection.fields.status, 500); + assert.equal(rejection.fields.reason, "corrupt_meta"); + assert.equal(rejection.fields.error_message, "Corrupt __meta__ for demo/api/v2"); + assert.equal(rejection.fields.metadata_version, "v2"); + assert.equal(rejection.fields.stage, "bundle_meta_parse"); + assert.equal(typeof rejection.fields.error_detail, "string"); + if (label === "malformed") { + assert.equal(rejection.fields.error_detail, "__meta__ is not valid JSON"); + assert.equal(JSON.stringify(state.logs).includes(String(rawMeta)), false); + } }); } @@ -167,19 +188,32 @@ test("workflows handler preserves metadata-unavailable error shape for batched r await assertJsonResponse(response, 500, { error: "workflow_metadata_unavailable", - message: "Workflow metadata is unavailable", + message: "Internal error", namespace: "demo", worker_count: 2, }); - assert.deepEqual(state.logs, [{ - level: "error", - event: "workflow_metadata_unavailable", - fields: { - namespace: "demo", - worker_count: 2, - error_message: "redis unavailable", + assert.deepEqual(state.logs, [ + { + level: "error", + event: "workflow_metadata_unavailable", + fields: { + namespace: "demo", + worker_count: 2, + error_message: "redis unavailable", + }, }, - }]); + { + level: "error", + event: "workflow_request_rejected", + fields: { + request_id: "rid-meta-fail", + namespace: "demo", + status: 500, + reason: "workflow_metadata_unavailable", + error_message: "Workflow metadata is unavailable", + }, + }, + ]); }); test("workflows handler wraps workflow definition batch read failures", async () => { @@ -204,19 +238,32 @@ test("workflows handler wraps workflow definition batch read failures", async () await assertJsonResponse(response, 500, { error: "workflow_metadata_unavailable", - message: "Workflow metadata is unavailable", + message: "Internal error", namespace: "demo", worker_count: 2, }); - assert.deepEqual(state.logs, [{ - level: "error", - event: "workflow_metadata_unavailable", - fields: { - namespace: "demo", - worker_count: 2, - error_message: "defs unavailable", + assert.deepEqual(state.logs, [ + { + level: "error", + event: "workflow_metadata_unavailable", + fields: { + namespace: "demo", + worker_count: 2, + error_message: "defs unavailable", + }, }, - }]); + { + level: "error", + event: "workflow_request_rejected", + fields: { + request_id: "rid-defs-fail", + namespace: "demo", + status: 500, + reason: "workflow_metadata_unavailable", + error_message: "Workflow metadata is unavailable", + }, + }, + ]); }); test("workflows handler resolves retired workflow definitions from wf:defs", async () => { @@ -365,12 +412,25 @@ test("workflows handler fails closed when workflows backend is unavailable", asy await assertJsonResponse(response, 503, { error: "workflow_internal_dispatch_failed", - message: "Workflow backend is unavailable", + message: "Internal error", }); assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], ]); + assert.deepEqual(state.logs, [{ + level: "error", + event: "workflow_request_rejected", + fields: { + request_id: "rid-down", + namespace: "demo", + worker: "api", + workflow: "orders", + status: 503, + reason: "workflow_internal_dispatch_failed", + error_message: "Workflow backend is unavailable", + }, + }]); }); test("workflows handler hides backend 5xx messages but logs diagnostics", async () => { @@ -399,9 +459,11 @@ test("workflows handler hides backend 5xx messages but logs diagnostics", async await assertJsonResponse(response, 500, { upstream_status: 500, error: "redis_error", - message: "Workflow backend request failed", + message: "Internal error", }); - assert.deepEqual(state.logs.at(-1), { + assert.deepEqual(state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_backend_error" + ), { level: "error", event: "workflow_backend_error", fields: { @@ -432,9 +494,11 @@ test("workflows handler hides backend fetch exceptions but logs diagnostics", as await assertJsonResponse(response, 503, { error: "workflow_internal_dispatch_failed", - message: "Workflow backend request failed", + message: "Internal error", }); - assert.deepEqual(state.logs.at(-1), { + assert.deepEqual(state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_backend_request_failed" + ), { level: "error", event: "workflow_backend_request_failed", fields: { diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 81beece..e5f9f02 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -178,6 +178,34 @@ test("parseBundleMeta delegates error construction without changing parse contex assert.ok(captured.cause instanceof TypeError); }); +test("parseBundleMeta does not retain malformed persisted input in diagnostics", () => { + const marker = "SECRET_TOKEN_ABC"; + /** @type {{ reason: string, cause: unknown } | null} */ + let failure = null; + const expected = new Error("routing-owned error"); + + assert.throws( + () => parseBundleMeta(marker, { + ns: "demo", + worker: "api", + version: "v3", + makeError(/** @type {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown }} */ value) { + failure = value; + return expected; + }, + }), + (err) => err === expected + ); + + assert.ok(failure); + const captured = /** @type {{ reason: string, cause: unknown }} */ ( + /** @type {unknown} */ (failure) + ); + assert.equal(captured.reason, "__meta__ is not valid JSON"); + assert.ok(captured.cause instanceof SyntaxError); + assert.equal(JSON.stringify({ reason: captured.reason }).includes(marker), false); +}); + test("encodeReferrerMember: key order is canonical (alphabetical) regardless of caller shape", () => { const a = encodeReferrerMember({ binding: "AUTH", callerNs: "demo", callerWorker: "api", callerVersion: "v3", diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index d375ad7..5b855bd 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -273,6 +273,44 @@ test("namespace secret PUT stores an envelope instead of plaintext", async () => }); }); +test("namespace secret PUT hides envelope configuration details and logs them", async () => { + const logStart = namespaceSecretState.logs.length; + await withNamespaceSecretRedis(namespaceSecretState, () => {}, async (redis) => { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env: {}, + method: "PUT", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-unconfigured", + }); + + const body = await readJsonResponse(response, 503); + assert.deepEqual(body, { + error: "secret_encryption_unconfigured", + message: "Internal error", + }); + assert.deepEqual(namespaceSecretState.logs.slice(logStart), [{ + level: "error", + event: "ns_secret_mutation_rejected", + fields: { + request_id: "rid-secret-unconfigured", + namespace: "demo", + key: "TOKEN", + method: "PUT", + status: 503, + reason: "secret_encryption_unconfigured", + error_message: "SECRET_ENVELOPE_KID must be a canonical local provider kid", + error_detail: "SECRET_ENVELOPE_KID must be a canonical local provider kid", + }, + }]); + assert.equal(redis.ops.length, 0); + }); +}); + test("namespace secret mutation rejects invalid keys through shared validator", async () => { await withNamespaceSecretRedis(namespaceSecretState, () => {}, async (redis) => { const response = await handle({ @@ -426,6 +464,7 @@ test("namespace secret PUT returns corrupt_meta for invalid retained bundle meta method: "PUT", status: 500, reason: "corrupt_meta", + error_message: "Corrupt __meta__ for demo/api/v1", error_detail: "__meta__ must be a JSON object", }, }); @@ -705,6 +744,47 @@ const workerSrc = applyModuleReplacements(readRepositoryFile("control/handlers/w ]); const { handle: workerHandle } = await import(moduleDataUrl(workerSrc)); +test("worker secret PUT hides envelope configuration details and logs them", async () => { + const state = workerSecretState; + const logStart = state.logs.length; + await withWorkerSecretRedis(state, () => {}, async (redis) => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env: {}, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-unconfigured", + }); + + const body = await readJsonResponse(response, 503); + assert.deepEqual(body, { + error: "secret_encryption_unconfigured", + message: "Internal error", + }); + assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ + level: "error", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-unconfigured", + namespace: "demo", + worker: "api", + key: "TOKEN", + method: "PUT", + status: 503, + reason: "secret_encryption_unconfigured", + error_message: "SECRET_ENVELOPE_KID must be a canonical local provider kid", + error_detail: "SECRET_ENVELOPE_KID must be a canonical local provider kid", + }, + }]); + assert.equal(redis.ops.length, 0); + }); +}); + test("worker secret PUT encrypts before WATCH retries and reuses the envelope", async () => { const state = workerSecretState; await withWorkerSecretRedis(state, (redis) => { @@ -773,7 +853,7 @@ test("worker secret PUT returns corrupt_meta for invalid active bundle metadata" const body = await readJsonResponse(response, 500); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.message, "Corrupt __meta__ for demo/api/v1"); + assert.equal(body.message, "Internal error"); assert.equal(body.detail, undefined); assert.deepEqual(redisHSetAttempts(redis, "secrets:demo:api"), []); }); @@ -816,6 +896,7 @@ test("worker secret PUT logs retained metadata diagnostics without exposing them method: "PUT", status: 500, reason: "corrupt_meta", + error_message: "Corrupt __meta__ for demo/api/v1", error_detail: "__meta__ must be a JSON object", }, }); @@ -910,6 +991,7 @@ test("worker secret PUT maps active bump delete-lock errors to deleting", async method: "PUT", status: 409, reason: "deleting", + error_message: "deleting", }, }]); }); @@ -938,7 +1020,7 @@ test("worker secret PUT maps active bump contention to secret mutation contentio assert.equal(body.error, "secret_mutation_contention"); assert.equal(redis.ops.some((op) => op[0] === "hSet" && op[1] === "secrets:demo:api"), false); assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ - level: "warn", + level: "error", event: "secret_mutation_rejected", fields: { request_id: "rid-worker-secret-bump-contention", @@ -948,6 +1030,7 @@ test("worker secret PUT maps active bump contention to secret mutation contentio method: "PUT", status: 503, reason: "secret_mutation_contention", + error_message: "active version changed during secret mutation; retry later", }, }]); }); @@ -984,6 +1067,7 @@ test("worker secret DELETE precheck logs direct mutation aborts through the shar method: "DELETE", status: 409, reason: "deleting", + error_message: "deleting", }, }]); }); diff --git a/tests/unit/control-shared.test.js b/tests/unit/control-shared.test.js index 8d31d82..b5a9019 100644 --- a/tests/unit/control-shared.test.js +++ b/tests/unit/control-shared.test.js @@ -43,7 +43,10 @@ const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const sharedOptimisticRetryUrl = repositoryFileUrl("shared/optimistic-retry.js"); const controlErrorsUrl = moduleDataUrl(applyModuleReplacements( readRepositoryFile("control/errors.js"), - [[/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`]] + [ + [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], + [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], + ] )); const controlWorkflowsClientUrl = moduleDataUrl(applyModuleReplacements( readRepositoryFile("control/workflows-client.js"), @@ -97,8 +100,10 @@ const rewritten = applyModuleReplacements(readRepositoryFile("control/shared.js" const { authErrorBody, authPolicyResponse, + acquireDeleteLock, assertWorkflowDeleteAllowed, cleanupDoAlarmsForWorker, + codedErrorLogFields, codedErrorResponse, ControlAbort, controlAbortResponse, @@ -107,6 +112,7 @@ const { rebuildDeclaredHostIndexes, releaseDeleteLock, runOptimistic, + secretEnvelopeErrorResponse, state, } = await import(moduleDataUrl(rewritten)); @@ -745,7 +751,7 @@ test("codedErrorResponse preserves semantic status/code with a fallback code", a }); }); -test("codedErrorResponse falls back to detail message for non-ControlAbort coded errors", async () => { +test("codedErrorResponse hides diagnostic messages on coded server errors", async () => { const err = { status: 503, code: "workflow_backend_invalid_response", @@ -759,10 +765,95 @@ test("codedErrorResponse falls back to detail message for non-ControlAbort coded await assertJsonResponse(res, 503, { upstreamStatus: 200, error: "workflow_backend_invalid_response", - message: "Workflow backend returned a malformed response", + message: "Internal error", + }); +}); + +test("codedErrorResponse keeps safe server context but strips diagnostic detail fields", async () => { + const err = new ControlAbort(500, "corrupt_meta", { + namespace: "demo", + worker: "api", + stage: "retained_meta_parse", + detail: "Unexpected token near secret bytes", + error_detail: "provider diagnostic", + }); + const res = controlAbortResponse(err); + + await assertJsonResponse(res, 500, { + namespace: "demo", + worker: "api", + error: "corrupt_meta", + message: "Internal error", }); }); +test("codedErrorLogFields preserves bounded server diagnostics at error level callers", () => { + const err = new ControlAbort(500, "corrupt_meta", { + message: "Corrupt __meta__ for demo/api/v2", + version: "v2", + stage: "bundle_meta_parse", + detail: "__meta__ is not valid JSON", + }); + + assert.deepEqual(codedErrorLogFields(err), { + status: 500, + reason: "corrupt_meta", + error_message: "Corrupt __meta__ for demo/api/v2", + metadata_version: "v2", + stage: "bundle_meta_parse", + error_detail: "__meta__ is not valid JSON", + }); +}); + +test("codedErrorLogFields bounds structured diagnostic strings", () => { + const longValue = "x".repeat(4096); + const err = new ControlAbort(500, longValue, { + message: longValue, + version: longValue, + stage: longValue, + detail: longValue, + }); + + const fields = codedErrorLogFields(err, err.code, { + context: { ...err.details, safe_context: "kept" }, + }); + assert.equal(fields.safe_context, "kept"); + for (const alias of ["message", "detail", "version"]) { + assert.equal(Object.hasOwn(fields, alias), false, alias); + } + for (const key of ["reason", "error_message", "metadata_version", "stage", "error_detail"]) { + assert.equal(/** @type {string} */ (fields[key]).length, 2048, key); + assert.match(/** @type {string} */ (fields[key]), /\.\.\.$/, key); + } +}); + +test("secretEnvelopeErrorResponse bounds the final structured log diagnostics", async () => { + const longValue = "x".repeat(4096); + /** @type {Array<{ level: string, event: string, fields: Record }>} */ + const logs = []; + const err = Object.assign(new Error(longValue), { code: "secret_provider_error" }); + + const response = secretEnvelopeErrorResponse({ + err: /** @type {any} */ (err), + log(/** @type {string} */ level, /** @type {string} */ event, /** @type {Record} */ fields) { + logs.push({ level, event, fields }); + }, + event: "secret_mutation_rejected", + fields: { request_id: "rid-long-diagnostic" }, + }); + + await assertJsonResponse(response, 503, { + error: "secret_provider_error", + message: "Internal error", + }); + assert.equal(logs.length, 1); + const fields = logs[0].fields; + assert.equal(/** @type {string} */ (fields.error_message).length, 2048); + assert.equal(/** @type {string} */ (fields.error_detail).length, 2048); + assert.match(/** @type {string} */ (fields.error_message), /\.\.\.$/); + assert.match(/** @type {string} */ (fields.error_detail), /\.\.\.$/); +}); + test("codedErrorResponse strips only top-level wire-reserved detail fields", async () => { const err = Object.assign(new Error("secret nope"), { status: 403, @@ -873,6 +964,25 @@ test("readJsonBody: streamed body over maxBytes fails while reading", async () = }); }); +test("acquireDeleteLock stores a kind-prefixed random token", async () => { + /** @type {unknown[][]} */ + const calls = []; + const token = await acquireDeleteLock({ + /** @param {unknown[]} args */ + async set(...args) { + calls.push(args); + return "OK"; + }, + }, "demo", "api", "version"); + + assert.match(token || "", /^version:[0-9a-f]{32}$/); + assert.deepEqual(calls, [[ + "worker-delete-lock:demo:api", + token, + { nx: true, ttl: 30 }, + ]]); +}); + test("releaseDeleteLock uses token-scoped DELIFEQ", async (t) => { restoreControlSharedStateAfter(t); /** @type {unknown[][]} */ @@ -887,8 +997,8 @@ test("releaseDeleteLock uses token-scoped DELIFEQ", async (t) => { calls.push(args); return 1; }, - }, "demo", "api", "token-a", "rid-delete"); + }, "demo", "api", "whole:token-a", "rid-delete"); - assert.deepEqual(calls, [["worker-delete-lock:demo:api", "token-a"]]); + assert.deepEqual(calls, [["worker-delete-lock:demo:api", "whole:token-a"]]); assert.deepEqual(logs, []); }); diff --git a/tests/unit/do-alarm-shim.test.js b/tests/unit/do-alarm-shim.test.js index 439cb91..7140409 100644 --- a/tests/unit/do-alarm-shim.test.js +++ b/tests/unit/do-alarm-shim.test.js @@ -1,6 +1,8 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { DO_ALARM_SHIM_SOURCE } from "../../do-runtime/alarm-shim-source.js"; +import { + DO_ALARM_SHIM_SOURCE, +} from "../../do-runtime/alarm-shim-source.js"; import { makeDoAlarmBinding, makeDoAlarmStorage } from "../helpers/do-alarm-shim-fixture.js"; import { applyModuleReplacements, moduleDataUrl } from "../helpers/load-shared-module.js"; import { withMockedGlobal, withMockedProperty } from "../helpers/mock-global.js"; @@ -34,6 +36,41 @@ test("DO alarm shim: error formatting cannot replace the original failure", () = }); }); +test("DO alarm shim: internal RPC dispatch invokes tenant methods without adding arguments", async () => { + const { storage } = makeDoAlarmStorage(); + class RpcRoom { + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + async inspect(/** @type {unknown} */ value) { + await Promise.resolve(); + return { value }; + } + } + const Wrapped = wrapDurableObjectClass(RpcRoom, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + /** @param {string} method */ + const request = (method) => new Request("https://do.internal/__wdl_rpc", { + method: "POST", + headers: { "x-wdl-do-internal-rpc": "1", "x-request-id": "rid-rpc" }, + body: JSON.stringify({ method, args: ["value"] }), + }); + await assertJsonResponse(await instance.fetch(request("inspect")), 200, { + ok: true, + result: { + value: "value", + }, + }); + await assertJsonResponse(await instance.fetch(request("missing")), 404, { + error: "do_rpc_method_not_found", + message: "Durable Object RPC method missing was not found", + }); +}); + test("DO alarm shim: repair logging cannot replace the stored alarm result", async () => { /** @param {(read: () => Promise) => Promise} callback */ async function readWithBrokenLogDependency(callback) { @@ -791,3 +828,58 @@ test("DO alarm shim: deleteAll deleteAlarm false preserves alarm row without bac assert.deepEqual(state.row, { ...initial, last_error: null }); assert.equal(kv.size, 0); }); + +test("DO alarm shim: deleteAll ignores tenant-patched array iteration", async () => { + /** @type {unknown[][]} */ + const calls = []; + /** @type {string[]} */ + const dropped = []; + const initial = { + scheduled_time: 1234, + retry_count: 0, + in_flight: 0, + token: "preserve-token", + }; + const { storage, state } = makeDoAlarmStorage(initial); + const originalExec = storage.sql.exec; + /** @param {string} statement @param {...unknown} params */ + storage.sql.exec = function exec(statement, ...params) { + if (statement.startsWith("SELECT type, name FROM sqlite_master")) { + return [ + { type: "table", name: "tenant_table" }, + { type: "table", name: "_wdl_do_alarms" }, + ]; + } + if (statement === 'DROP TABLE IF EXISTS "tenant_table"') { + dropped.push(statement); + return []; + } + return Reflect.apply(originalExec, this, [statement, ...params]); + }; + const wrapped = wrapStorage(storage, makeDoAlarmBinding(calls), "Room", "alice"); + const originalIterator = Array.prototype[Symbol.iterator]; + + await withMockedProperty(Map.prototype, "keys", () => { + throw new Error("tenant Map.keys"); + }, () => withMockedProperty( + Array.prototype, + Symbol.iterator, + /** @this {any[]} */ function hostileIterator() { + const first = this[0]; + if ( + (this.length === 1 && first?.deleteAlarm === false) || + (first && typeof first === "object" && "type" in first) + ) { + return Reflect.apply(originalIterator, [], []); + } + return Reflect.apply(originalIterator, this, []); + }, + async () => { + await wrapped.deleteAll({ deleteAlarm: false }); + } + )); + + assert.deepEqual(dropped, ['DROP TABLE IF EXISTS "tenant_table"']); + assert.deepEqual(calls, []); + assert.deepEqual(state.row, { ...initial, last_error: null }); +}); diff --git a/tests/unit/do-owner-registry.test.js b/tests/unit/do-owner-registry.test.js index 331e89b..c9b9406 100644 --- a/tests/unit/do-owner-registry.test.js +++ b/tests/unit/do-owner-registry.test.js @@ -34,6 +34,7 @@ const INVALID_RENEW_TIME_BASES = [1234.5, NaN, Infinity, -Infinity, -1]; const DO_STORAGE_ID = "do_0123456789abcdef0123456789abcdef"; const OWNER_KEY = `${DO_STORAGE_ID}:Room:shard0`; const STORAGE_POINTER_KEY = "worker:do-storage:tenant:chat"; +const DELETE_LOCK_KEY = "worker-delete-lock:tenant:chat"; beforeEach(resetDoOwnerRegistryTestState); @@ -145,6 +146,7 @@ test("DO owner registry: lease guard config bounds values and permits test disab }); test("DO owner registry: claim writes owner generation counter", async () => { + setStoragePointer(); const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); assert.equal(owner.taskId, "task-a"); @@ -176,6 +178,7 @@ test("DO owner registry: corrupt generation counter fails closed", async () => { }); test("DO owner registry: claim retries WatchError before surfacing owner", async () => { + setStoragePointer(); DO_OWNER_REGISTRY_TEST_STATE.redisState.execFailures = 1; const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); @@ -183,7 +186,67 @@ test("DO owner registry: claim retries WatchError before surfacing owner", async assert.equal(owner.taskId, "task-a"); assert.equal(owner.generation, 1); assert.equal(DO_OWNER_REGISTRY_TEST_STATE.redisState.execFailures, 0); - assert.equal(DO_OWNER_REGISTRY_TEST_STATE.redisState.watchBatches.length, 2); + assert.equal( + DO_OWNER_REGISTRY_TEST_STATE.redisState.watchBatches.filter((keys) => + keys.includes(ownerKeyOf(OWNER_KEY)) + ).length, + 2 + ); +}); + +test("DO owner registry: first claim requires the active worker storage pointer", async () => { + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + (err) => { + assert.equal(/** @type {{ code?: unknown }} */ (err).code, "stale_owner_storage"); + return true; + } + ); + assert.deepEqual(doOwnerRegistryWriteCommands(), []); +}); + +test("DO owner registry: first claim refuses a worker under whole-delete lock", async () => { + setStoragePointer(); + DO_OWNER_REGISTRY_TEST_STATE.store.set(DELETE_LOCK_KEY, "whole:delete-token"); + + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + (err) => { + assert.equal(/** @type {{ code?: unknown }} */ (err).code, "stale_owner_storage"); + return true; + } + ); + assert.ok(DO_OWNER_REGISTRY_TEST_STATE.watchedKeys.includes(DELETE_LOCK_KEY)); + assert.deepEqual(doOwnerRegistryWriteCommands(), []); +}); + +test("DO owner registry: version delete lock does not interrupt active storage", async () => { + setStoragePointer(); + DO_OWNER_REGISTRY_TEST_STATE.store.set(DELETE_LOCK_KEY, "version:delete-token"); + + const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); + + assert.equal(owner.taskId, "task-a"); + assert.equal(owner.generation, 1); + assert.ok(DO_OWNER_REGISTRY_TEST_STATE.watchedKeys.includes(DELETE_LOCK_KEY)); +}); + +test("DO owner registry: delete lock appearing before claim commit aborts the claim", async () => { + setStoragePointer(); + DO_OWNER_REGISTRY_TEST_STATE.redisState.execFailures = 1; + DO_OWNER_REGISTRY_TEST_STATE.onExecFailure = () => { + DO_OWNER_REGISTRY_TEST_STATE.store.set(DELETE_LOCK_KEY, "whole:delete-token"); + }; + + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + (err) => { + assert.equal(/** @type {{ code?: unknown }} */ (err).code, "stale_owner_storage"); + return true; + } + ); + assert.equal(DO_OWNER_REGISTRY_TEST_STATE.store.has(ownerKeyOf(OWNER_KEY)), false); + assert.equal(DO_OWNER_REGISTRY_TEST_STATE.store.has(ownerGenerationKeyOf(OWNER_KEY)), false); }); test("DO owner registry: lost owner state cannot validate an old owner from another task", async () => { @@ -300,6 +363,7 @@ test("DO owner registry: expired takeover bumps generation monotonically", async generation: 11, leaseExpiresAt: redisNow - EXPIRED_LEASE_AGE_MS, }); + setStoragePointer(); DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(ownerKey), JSON.stringify(staleOwner)); DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerGenerationKeyOf(ownerKey), "7"); diff --git a/tests/unit/do-runtime-actor.test.js b/tests/unit/do-runtime-actor.test.js index 241f2d1..5f441fa 100644 --- a/tests/unit/do-runtime-actor.test.js +++ b/tests/unit/do-runtime-actor.test.js @@ -5,9 +5,10 @@ import { loadDoHostActor, resetDoHostActorHarness, } from "../helpers/load-do-host-actor.js"; +import { assertJsonResponse } from "../helpers/response-json.js"; import { delay } from "../helpers/timing.js"; -const { WdlDoHostActor } = await loadDoHostActor(); +const { WdlDoHostActor, dispatchRpc } = await loadDoHostActor(); const harness = doHostActorHarnessState(); beforeEach(() => { @@ -39,6 +40,43 @@ function invoke(overrides = {}) { }; } +test("DO host actor: RPC dispatch passes request id through the internal wrapper boundary", async () => { + /** @type {Array<{ url: string, header: string | null, requestId: string | null, body: unknown }>} */ + const calls = []; + const facet = { + async fetch(/** @type {Request} */ request) { + const body = await request.json(); + calls.push({ + url: request.url, + header: request.headers.get("x-wdl-do-internal-rpc"), + requestId: request.headers.get("x-request-id"), + body, + }); + return Response.json({ ok: true, result: body }); + }, + }; + + const response = await dispatchRpc( + facet, + { method: "inspect", args: ["value"] }, + "rid-rpc" + ); + + await assertJsonResponse(response, 200, { + ok: true, + result: { + method: "inspect", + args: ["value"], + }, + }); + assert.deepEqual(calls, [{ + url: "https://do.internal/__wdl_rpc", + header: "1", + requestId: "rid-rpc", + body: { method: "inspect", args: ["value"] }, + }]); +}); + test("DO host actor: lease budget aborts a facet when the owner fence stops renewing", async () => { const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); const owner = { @@ -122,6 +160,41 @@ test("DO host actor: expired initial lease aborts before tenant dispatch", async assert.equal(harness.logs.at(-1).fields.reason, "expired"); }); +test("DO host actor: registry delay completes before the owner fence snapshot", async () => { + const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); + const registryStarted = Promise.withResolvers(); + const releaseRegistry = Promise.withResolvers(); + const owner = { + ownerKey: "do_0123456789abcdef0123456789abcdef:Room:shard0", + taskId: "task-a", + generation: 7, + leaseExpiresAt: Date.now() - 1, + }; + harness.assertResponses = [owner]; + harness.registryWait = releaseRegistry.promise; + harness.registryWaitStarted = () => registryStarted.resolve(undefined); + let ran = false; + + const dispatch = host.dispatchWithFence(invoke({ + doStorageId: "do_0123456789abcdef0123456789abcdef", + }), () => { + ran = true; + return new Response("should not run"); + }); + const first = await Promise.race([ + registryStarted.promise.then(() => "registry"), + dispatch.then(() => "dispatch-resolved", () => "dispatch-rejected"), + ]); + + assert.equal(first, "registry"); + assert.equal(harness.assertCalls, 0); + releaseRegistry.resolve(undefined); + await assert.rejects(dispatch, /owner lease has expired/); + + assert.equal(ran, false); + assert.equal(harness.assertCalls, 1); +}); + test("DO host actor: lease budget reschedules when renew extended the owner fence", async () => { const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); const owner = { diff --git a/tests/unit/do-runtime-load.test.js b/tests/unit/do-runtime-load.test.js index a6b19e0..e95965c 100644 --- a/tests/unit/do-runtime-load.test.js +++ b/tests/unit/do-runtime-load.test.js @@ -11,6 +11,10 @@ import { installMockFetch, withMockedFetch } from "../helpers/mock-fetch.js"; import { withMockedGlobal } from "../helpers/mock-global.js"; import { installConsoleMethodCapture } from "../helpers/output-capture.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; +import { + DO_RUNTIME_RESERVED_MODULE, + doRuntimeInjectedModuleSources, +} from "../../do-runtime/load-code-budget.js"; /** @type {any} */ (globalThis).__doLoadFetches = []; /** @type {any} */ (globalThis).__doLoadResponses = []; @@ -221,7 +225,9 @@ test("DO runtime load: abort timeout is retried inside the load budget", async ( test("DO runtime load: applies the host-binding wrapper before the alarm wrapper", async () => { /** @type {any} */ (globalThis).__doRuntimeHostWrap = (/** @type {any} */ workerCode) => { - workerCode.modules["_wdl-wrapper.js"] = "export class Room {}; export default {};"; + workerCode.modules["_wdl-wrapper.js"] = [ + "export class Room {}; export default {};", + ].join("\n"); workerCode.mainModule = "_wdl-wrapper.js"; }; /** @type {any} */ (globalThis).__doLoadResponses.push(jsonLoadResponse(200, { @@ -248,7 +254,7 @@ test("DO runtime load: applies the host-binding wrapper before the alarm wrapper assert.equal(loaded.mainModule, "_wdl-do-runtime-wrapper.js"); assert.deepEqual(loaded.compatibilityFlags, ["nodejs_compat", "delete_all_preserves_alarm"]); - assert.equal(loaded.modules["_wdl-wrapper.js"], "export class Room {}; export default {};"); + assert.doesNotMatch(loaded.modules["_wdl-wrapper.js"], /__wdlRunWithRequestContext/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /function withoutInternalEnv/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /function deleteAllKvStorage/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /function deleteAllSqlStorage/); @@ -258,13 +264,44 @@ test("DO runtime load: applies the host-binding wrapper before the alarm wrapper assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /return new NativeProxy\(storage,/); assert.doesNotMatch(loaded.modules["_wdl-do-alarm-shim.js"], /delete out\.__WDL_HOST_BINDINGS_WRAPPED__/); const wrapper = loaded.modules["_wdl-do-runtime-wrapper.js"]; - assert.match(wrapper, /import \* as user from "\.\/_wdl-wrapper\.js";/); - assert.match(wrapper, /import \{ wrapDurableObjectClass \} from "\.\/_wdl-do-alarm-shim\.js";/); + assert.match(wrapper, /import \* as __WdlUserModule__ from "\.\/_wdl-wrapper\.js";/); + assert.match(wrapper, /import \{ wrapDurableObjectClass as __WdlWrapDurableObjectClass__ \} from "\.\/_wdl-do-alarm-shim\.js";/); assert.ok( wrapper.indexOf('from "./_wdl-do-alarm-shim.js";') < - wrapper.indexOf('import * as user from "./_wdl-wrapper.js";') + wrapper.indexOf('import * as __WdlUserModule__ from "./_wdl-wrapper.js";') ); - assert.match(wrapper, /export class Room extends wrapDurableObjectClass\(user\.Room, "Room"\)/); + assert.match(wrapper, /__WdlUserModule__\.Room,\s+"Room"/); + assert.doesNotMatch(wrapper, /__wdlRunWithRequestContext/); +}); + +test("DO runtime wrapper aliases legal class names without declaration collisions", async () => { + const classNames = ["user", "wrapDurableObjectClass"]; + const injections = Object.fromEntries(doRuntimeInjectedModuleSources("worker.js", { + bindings: { + USER: { type: "do", className: classNames[0] }, + WRAPPER: { type: "do", className: classNames[1] }, + }, + }, "export function wrapDurableObjectClass() {}")); + const userUrl = moduleDataUrl(` + export class user {} + export class wrapDurableObjectClass {} + `); + const shimUrl = moduleDataUrl(` + export function wrapDurableObjectClass(Base) { + return class extends Base {}; + } + `); + const source = applyModuleReplacements(injections[DO_RUNTIME_RESERVED_MODULE], [ + ['from "./_wdl-do-alarm-shim.js"', `from ${JSON.stringify(shimUrl)}`], + [/from "\.\/worker\.js"/g, `from ${JSON.stringify(userUrl)}`], + ]); + const wrapped = await import(moduleDataUrl(source)); + const userModule = await import(userUrl); + + for (const name of classNames) { + assert.equal(wrapped[name].name, name); + assert.ok(wrapped[name].prototype instanceof userModule[name]); + } }); test("DO runtime load: rejects reserved alarm shim module collisions", async () => { diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 899a952..76ee4bc 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -24,6 +24,7 @@ const { buildFacetName, buildAlarmRequest, buildForwardRequest, + buildRpcRequest, encodeDoInvokeRequest, buildLocalActorRequest, doErrorResponse, @@ -255,9 +256,10 @@ test("normalizes do alarm invoke request", async () => { assert.deepEqual(invoke.alarm, { retryCount: 2, isRetry: true, token: "alarm-token" }); assert.equal("request" in invoke ? invoke.request : undefined, undefined); - const request = buildAlarmRequest(invoke.alarm); + const request = buildAlarmRequest(invoke.alarm, "rid-alarm"); assert.equal(request.method, "POST"); assert.equal(request.headers.get("x-wdl-do-internal-alarm"), "1"); + assert.equal(request.headers.get("x-request-id"), "rid-alarm"); assert.deepEqual(await request.json(), { retryCount: 2, isRetry: true, token: "alarm-token" }); }); @@ -280,6 +282,22 @@ test("normalizes do rpc invoke request", () => { }); }); +test("builds private rpc dispatch requests", async () => { + const request = buildRpcRequest({ + method: "addMessage", + args: ["hello", { role: "user" }], + }, "rid-rpc"); + + assert.equal(request.method, "POST"); + assert.equal(request.url, "https://do.internal/__wdl_rpc"); + assert.equal(request.headers.get("x-wdl-do-internal-rpc"), "1"); + assert.equal(request.headers.get("x-request-id"), "rid-rpc"); + assert.deepEqual(await request.json(), { + method: "addMessage", + args: ["hello", { role: "user" }], + }); +}); + test("rejects invalid do rpc shapes", () => { assert.throws( () => normalizeDoInvokeRequest({ @@ -375,6 +393,7 @@ test("builds forwarded Request for user durable object fetch", async () => { headers: { ...BASE_BODY.request.headers, "x-wdl-do-internal-alarm": "1", + "x-wdl-do-internal-rpc": "1", "x-wdl-internal-auth": "platform-token", }, }, @@ -386,6 +405,7 @@ test("builds forwarded Request for user durable object fetch", async () => { assert.equal(request.url, "https://demo.workers.example/messages"); assert.equal(request.headers.get("content-type"), "text/plain"); assert.equal(request.headers.get("x-wdl-do-internal-alarm"), null); + assert.equal(request.headers.get("x-wdl-do-internal-rpc"), null); assert.equal(request.headers.get("x-wdl-internal-auth"), null); assert.equal(await request.text(), "hello"); }); @@ -402,6 +422,7 @@ test("normalizes DO connect without exposing internal auth to tenant code", () = "x-wdl-do-object-name": "room-a", "x-wdl-do-request-url": "https://demo.workers.example/ws", "x-wdl-internal-auth": "platform-token", + "x-wdl-do-ownership-error": "stale_owner_generation", "x-tenant-visible": "ok", }, })); @@ -519,6 +540,7 @@ test("normalizes do websocket connect request from internal headers", () => { "x-wdl-do-owner-endpoint": "do-runtime-a:8788", "x-wdl-do-owner-generation": "4", "x-wdl-do-owner-hint": "1", + "x-wdl-do-ownership-error": "stale_owner_generation", }, }); const invoke = normalizeDoConnectRequest(request); @@ -538,6 +560,7 @@ test("normalizes do websocket connect request from internal headers", () => { assert.equal(invoke.request.headers.some(([name]) => name === "x-wdl-do-owner-generation"), false); assert.equal(invoke.request.headers.some(([name]) => name === "x-wdl-do-owner-endpoint"), false); assert.equal(invoke.request.headers.some(([name]) => name === "x-wdl-do-owner-hint"), false); + assert.equal(invoke.request.headers.some(([name]) => name === "x-wdl-do-ownership-error"), false); assert.ok(invoke.request.headers.some(([name, value]) => name === "upgrade" && value === "websocket")); assert.ok(invoke.request.headers.some(([name, value]) => name === "x-request-id" && value === "rid-1")); }); diff --git a/tests/unit/respond.test.js b/tests/unit/respond.test.js index 384f4b2..042c731 100644 --- a/tests/unit/respond.test.js +++ b/tests/unit/respond.test.js @@ -17,6 +17,22 @@ import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.j const OBSERVABILITY_CONTRACT = readRepositoryJson("tests/fixtures/observability-contract.json"); +/** @param {Promise} promise */ +async function settlesWithinTestWindow(promise) { + /** @type {ReturnType | undefined} */ + let timeout; + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { + timeout = setTimeout(() => resolve(false), 100); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + test("discardResponseBody cancels response bodies", async () => { let cancelled = false; const response = new Response(new ReadableStream({ @@ -40,6 +56,21 @@ test("discardResponseBody ignores cancellation failures", async () => { await discardResponseBody(response); }); +test("discardResponseBody does not wait for cancellation cleanup", async () => { + let cancelled = false; + const response = new Response(new ReadableStream({ + cancel() { + cancelled = true; + return new Promise(() => {}); + }, + })); + + const settled = await settlesWithinTestWindow(discardResponseBody(response)); + + assert.equal(settled, true); + assert.equal(cancelled, true); +}); + // Node's Response doesn't accept `webSocket` in init, so the 101 branch // is covered end-to-end by tests/integration/gateway-websocket.test.js. test("echoResponseWithRequestId preserves status/body/headers and stamps x-request-id", async () => { diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index 2c92d48..cdd65c6 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -38,6 +38,47 @@ function headerValue(headers, name) { return new Headers(headers).get(name); } +/** + * @template T + * @param {Promise} promise + * @param {string} message + */ +async function withTestTimeout(promise, message) { + /** @type {ReturnType | undefined} */ + let timeout; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), 1000); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + +/** @param {UnderlyingSourceCancelCallback} onCancel @param {ResponseInit} [init] */ +function cancellableResponse(onCancel, init = {}) { + return new Response(new ReadableStream({ + pull(controller) { + controller.error(new Error("discarded response body was read")); + }, + cancel: onCancel, + }, { highWaterMark: 0 }), init); +} + +/** + * @param {UnderlyingSourceCancelCallback} onCancel + * @param {Parameters[0]} [options] + */ +function cancellableDoOwnerHintResponse(onCancel, options = {}) { + return cancellableResponse(onCancel, { + status: 409, + headers: doOwnerHintHeaders(options), + }); +} + test("DurableObjectNamespace facade forwards fetch with object name and request id", async () => { /** @type {any[]} */ const calls = []; @@ -561,10 +602,14 @@ test("DurableObjectNamespace RPC retries stale owner generation once", async () test("DurableObjectNamespace RPC retries owner claim races once", async () => { /** @type {any[]} */ const calls = []; + let cancellations = 0; + let hostileCancelCalls = 0; const backend = { fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + ? cancellableResponse(() => { + cancellations += 1; + }, { status: 503, headers: doOwnershipErrorHeaders("owner_claim_raced"), }) @@ -579,12 +624,24 @@ test("DurableObjectNamespace RPC retries owner claim races once", async () => { binding: "ROOM", className: "Room", }, { backend }); + const streamCancel = ReadableStream.prototype.cancel; - const result = await Reflect.get(ns.get(ns.idFromName("room-a")), "addMessage")("hello"); + const result = await withMockedProperty( + ReadableStream.prototype, + "cancel", + /** @this {ReadableStream} */ + function hostileCancel(reason) { + hostileCancelCalls += 1; + return Reflect.apply(streamCancel, this, [reason]); + }, + () => Reflect.get(ns.get(ns.idFromName("room-a")), "addMessage")("hello") + ); assert.equal(result, "claimed"); assert.equal(calls.length, 2); assert.equal(calls[1].init.headers.get("x-wdl-do-accept-owner-hint"), null); + assert.equal(cancellations, 1); + assert.equal(hostileCancelCalls, 0); }); test("DurableObjectNamespace direct backend preserves binary request bodies", async () => { @@ -1401,6 +1458,92 @@ test("DurableObjectNamespace direct backend follows owner hints on the same requ assert.equal(ownerCalls[0].init.body, routerCalls[0].init.body); }); +test("DurableObjectNamespace replays a safe GET through the router after a second owner hint", async () => { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + /** @type {string[]} */ + const cancellations = []; + const backend = { + fetch: makeRecordingFetch(routerCalls, { + response: () => routerCalls.length === 1 + ? cancellableDoOwnerHintResponse(() => { + cancellations.push("router"); + }) + : new Response("router-ok"), + }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { + response: () => cancellableDoOwnerHintResponse( + () => { + cancellations.push("direct"); + }, + { + taskId: "do-runtime-b", + endpoint: "do-runtime-b:8788", + generation: 4, + }, + ), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend, ownerNetwork }); + + const response = await ns.get(ns.idFromName("room-consecutive-hint")) + .fetch("https://demo.workers.example/send"); + assert.equal(await response.text(), "router-ok"); + assert.equal(response.headers.get("x-wdl-do-owner-key"), null); + assert.equal(routerCalls.length, 2); + assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); + assert.equal(ownerCalls.length, 1); + assert.deepEqual(cancellations, ["router", "direct"]); +}); + +test("DurableObjectNamespace hides a second owner hint without replaying a POST", async () => { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + const backend = { + fetch: makeRecordingFetch(routerCalls, { response: doOwnerHintResponse() }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { response: doOwnerHintResponse({ + taskId: "do-runtime-b", + endpoint: "do-runtime-b:8788", + generation: 4, + }) }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend, ownerNetwork }); + + const response = await ns.get(ns.idFromName("room-consecutive-post")) + .fetch("https://demo.workers.example/send", { method: "POST", body: "hello" }); + const body = await readJsonResponse(response, 503); + + assert.deepEqual(body, { + error: "owner_unavailable", + message: "DO owner is unavailable; request outcome may be unknown", + }); + assert.equal(response.headers.get("x-wdl-do-owner-key"), null); + assert.equal(routerCalls.length, 1); + assert.equal(ownerCalls.length, 1); +}); + test("DurableObjectNamespace direct backend accepts Kubernetes headless owner endpoints", async () => { /** @type {any[]} */ const routerCalls = []; @@ -1714,6 +1857,50 @@ test("DurableObjectNamespace direct backend reuses learned owner hints", async ( assert.equal(ownerCalls[1].url, "http://do-runtime-a:8788/internal/do/invoke"); }); +test("DurableObjectNamespace cancels a cached-owner response before router rediscovery", async () => { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + let cancellations = 0; + const backend = { + fetch: makeRecordingFetch(routerCalls, { + response: () => routerCalls.length === 1 + ? doOwnerHintResponse() + : new Response("router-ok"), + }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { + response: () => ownerCalls.length === 1 + ? new Response("owner-ok", { headers: doOwnerHintHeaders() }) + : cancellableDoOwnerHintResponse(() => { + cancellations += 1; + }, { + taskId: "do-runtime-b", + endpoint: "do-runtime-b:8788", + generation: 4, + }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend, ownerNetwork }); + const id = ns.idFromName("room-cached-hint"); + + assert.equal(await ns.get(id).fetch("https://demo.workers.example/one").then((r) => r.text()), "owner-ok"); + assert.equal(await ns.get(id).fetch("https://demo.workers.example/two").then((r) => r.text()), "router-ok"); + + assert.equal(cancellations, 1); + assert.equal(routerCalls.length, 2); + assert.equal(ownerCalls.length, 2); +}); + function hostileOwnerHintCacheFixture() { /** @type {any[]} */ const routerCalls = []; @@ -1965,22 +2152,13 @@ test("DO requestSpec rejects streaming bodies as soon as they cross the cap", as body, duplex: "half", })); - /** @type {ReturnType | undefined} */ - let timeout; - - try { - await assert.rejects( - Promise.race([ - requestSpec(request, "rid-stream"), - new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error("requestSpec kept reading the oversized stream")), 1000); - }), - ]), - /Durable Object fetch body exceeds/ - ); - } finally { - if (timeout) clearTimeout(timeout); - } + await assert.rejects( + withTestTimeout( + requestSpec(request, "rid-stream"), + "requestSpec kept reading the oversized stream" + ), + /Durable Object fetch body exceeds/ + ); }); test("DO requestSpec rejects oversized streams without waiting for cancel", async () => { @@ -1997,31 +2175,23 @@ test("DO requestSpec rejects oversized streams without waiting for cancel", asyn body, duplex: "half", })); - /** @type {ReturnType | undefined} */ - let timeout; let hostileCatchCalls = 0; - try { - await withMockedProperty( - Promise.prototype, - "catch", - function hostileCatch() { - hostileCatchCalls += 1; - throw new Error("tenant Promise.catch must not run"); - }, - () => assert.rejects( - Promise.race([ - requestSpec(request, "rid-cancel"), - new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error("requestSpec waited for stream cancel")), 1000); - }), - ]), - /Durable Object fetch body exceeds/ - ) - ); - } finally { - if (timeout) clearTimeout(timeout); - } + await withMockedProperty( + Promise.prototype, + "catch", + function hostileCatch() { + hostileCatchCalls += 1; + throw new Error("tenant Promise.catch must not run"); + }, + () => assert.rejects( + withTestTimeout( + requestSpec(request, "rid-cancel"), + "requestSpec waited for stream cancel" + ), + /Durable Object fetch body exceeds/ + ) + ); assert.equal(hostileCatchCalls, 0); }); @@ -2212,6 +2382,7 @@ test("DurableObjectNamespace websocket path does not fall back to router after o test("DurableObjectNamespace POST fetch does not replay through router after direct owner failure", async () => { /** @type {any[]} */ const routerCalls = []; + let cancellations = 0; const backend = { fetch: makeRecordingFetch(routerCalls, { response: () => routerCalls.length === 1 @@ -2221,7 +2392,10 @@ test("DurableObjectNamespace POST fetch does not replay through router after dir }; const ownerNetwork = { async fetch() { - return new Response("upstream request timeout", { status: 504 }); + return cancellableResponse(() => { + cancellations += 1; + return Promise.reject(new Error("cancel failed")); + }, { status: 504 }); }, }; const ns = new DurableObjectNamespace({ @@ -2242,6 +2416,7 @@ test("DurableObjectNamespace POST fetch does not replay through router after dir assert.equal(body.error, "owner_unavailable"); assert.equal(routerCalls.length, 1); assert.equal(headerValue(routerCalls[0].init.headers, "x-wdl-do-accept-owner-hint"), "1"); + assert.equal(cancellations, 1); }); test("DurableObjectNamespace POST drops stale cached owner hints without replay after endpoint timeout", async () => { @@ -2249,6 +2424,7 @@ test("DurableObjectNamespace POST drops stale cached owner hints without replay const routerCalls = []; /** @type {any[]} */ const ownerCalls = []; + let cancellations = 0; const backend = { fetch: makeRecordingFetch(routerCalls, { response: () => routerCalls.length === 1 @@ -2262,7 +2438,10 @@ test("DurableObjectNamespace POST drops stale cached owner hints without replay ? new Response("owner-ok", { headers: doOwnerHintHeaders(), }) - : new Response("upstream request timeout", { status: 504 }), + : cancellableResponse(() => { + cancellations += 1; + return new Promise(() => {}); + }, { status: 504 }), }), }; const ns = new DurableObjectNamespace({ @@ -2276,11 +2455,16 @@ test("DurableObjectNamespace POST drops stale cached owner hints without replay const id = ns.idFromName("room-stale-cached-owner"); assert.equal(await ns.get(id).fetch("https://demo.workers.example/one", { method: "POST", body: "first" }).then((r) => r.text()), "owner-ok"); - const body = await readJsonResponse(await ns.get(id).fetch("https://demo.workers.example/two", { method: "POST", body: "second" }), 503); + const response = await withTestTimeout( + ns.get(id).fetch("https://demo.workers.example/two", { method: "POST", body: "second" }), + "owner dispatch waited for response cancellation" + ); + const body = await readJsonResponse(response, 503); assert.equal(body.error, "owner_unavailable"); assert.equal(ownerCalls.length, 2); assert.equal(routerCalls.length, 1); + assert.equal(cancellations, 1); }); test("DurableObjectNamespace replays safe GET after stale cached owner endpoint timeout", async () => { @@ -2327,6 +2511,7 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu const routerCalls = []; /** @type {any[]} */ const ownerCalls = []; + let cancellations = 0; const backend = { fetch: makeRecordingFetch(routerCalls, { response: () => routerCalls.length === 1 @@ -2338,7 +2523,9 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu fetch: makeRecordingFetch(ownerCalls, { response: () => ownerCalls.length === 1 ? new Response("owner-ok", { headers: doOwnerHintHeaders() }) - : Response.json({ error, message: "owner lease unavailable" }, { + : cancellableResponse(() => { + cancellations += 1; + }, { status: 503, headers: doOwnershipErrorHeaders(error), }), @@ -2360,6 +2547,7 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu assert.equal(ownerCalls.length, 2, error); assert.equal(routerCalls.length, 2, error); assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), "1", error); + assert.equal(cancellations, 1, error); } }); diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index 49bffdf..c34fcc2 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -1738,18 +1738,39 @@ test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserv assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-d1-client.js"], /class D1Database/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], /intrinsicArrayForEach/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as user from "\.\/src\/worker\.js";/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /Local wrapper subclasses intentionally shadow/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], /AsyncLocalStorage/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as __WdlHostRuntime__/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as __WdlUserModule__ from "\.\/src\/worker\.js";/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /Explicit aliases replace same-name star exports/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const D1_BINDINGS = \["DB"\];/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const R2_BINDINGS = \[\];/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const DO_BINDINGS = \[\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new D1Database\(out\[name\], requestIdOptions\(requestIdOrContext\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new D1Database\(out\[name\], requestIdOptions\(\)\)/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /requestIdFromEventArg/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /wrapClassInstance\(this, requestContext\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /wrapClassInstance\(this\)/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /forEachArray\(HOST_WRAPPED_HANDLER_KEYS/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class Api extends user\.Api/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class Admin extends user\.Admin/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.Api/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /__WdlWrappedEntrypoint\d+__ as Admin/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class default/); + assert.deepEqual(/** @type {any} */ (workerCode).compatibilityFlags, ["nodejs_als"]); +}); + +test("wrapWorkerCodeForHostBindings: normalizes standalone ALS compatibility flags", () => { + for (const [input, expected] of [ + [["no_nodejs_als"], ["nodejs_als"]], + [["streams_enable_constructors", "no_nodejs_als"], ["streams_enable_constructors", "nodejs_als"]], + [["nodejs_compat", "no_nodejs_als"], ["nodejs_compat", "no_nodejs_als"]], + ]) { + const workerCode = { + compatibilityFlags: input, + mainModule: "worker.js", + modules: { "worker.js": "export default {};" }, + }; + wrapWorkerCodeForHostBindings(workerCode, { + bindings: { DB: { type: "d1", databaseId: "main" } }, + }); + assert.deepEqual(workerCode.compatibilityFlags, expected); + } }); test("wrapWorkerCodeForHostBindings: default object wraps inherited and accessor handler keys", async () => { @@ -1818,14 +1839,23 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc constructor(ctx, env) { this.env = env; } dbConstructorName() { return this.env.DB?.constructor?.name; } rawDbVisible() { return this.env.DB?.raw === true; } + echo(value) { return value; } } + export class currentRequestId extends Api {} + export class runWithRequestContext extends Api {} + export class __wdlRunWithRequestContext extends Api {} export default {}; `, }, }; wrapWorkerCodeForHostBindings(workerCode, { bindings: { DB: { type: "d1", databaseId: "main" } }, - exports: [{ entrypoint: "Api", allowedCallers: ["*"] }], + exports: [ + { entrypoint: "Api", allowedCallers: ["*"] }, + { entrypoint: "currentRequestId", allowedCallers: ["*"] }, + { entrypoint: "runWithRequestContext", allowedCallers: ["*"] }, + { entrypoint: "__wdlRunWithRequestContext", allowedCallers: ["*"] }, + ], }); await withTempDir("wdl-d1-wrapper-", async (dir) => { @@ -1840,7 +1870,9 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc for (const [name, source] of Object.entries(workerCode.modules)) { const file = path.join(dir, name); const stubbed = name === "_wdl-wrapper.js" - ? source.replace(`from "cloudflare:workers"`, `from "./_cf_workers_stub.js"`) + ? source + .replace(`from "cloudflare:workers"`, `from "./_cf_workers_stub.js"`) + .replace("function withRequestContext", "export function withRequestContext") : source; writeFileSync(file, stubbed); } @@ -1856,6 +1888,48 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc }); assert.equal(instance.dbConstructorName(), "D1Database"); assert.equal(instance.rawDbVisible(), false); + for (const name of ["currentRequestId", "runWithRequestContext", "__wdlRunWithRequestContext"]) { + assert.ok(new wrapped[name]({}, { DB: { raw: true } }) instanceof user[name]); + } + + const request = new Request("https://demo.workers.example", { + headers: { "x-request-id": "rid-instance" }, + }); + const contextRuntime = await import(`file://${path.join(dir, "_wdl-host-wrapper-runtime.js")}`); + const firstStarted = Promise.withResolvers(); + const releaseFirst = Promise.withResolvers(); + const first = wrapped.withRequestContext(request, () => { + const result = (async () => { + firstStarted.resolve(undefined); + await releaseFirst.promise; + return contextRuntime.currentRequestId(); + })(); + return result; + }); + await firstStarted.promise; + + const secondRequest = new Request("https://demo.workers.example", { + headers: { "x-request-id": "rid-second" }, + }); + const second = wrapped.withRequestContext(secondRequest, async () => { + await Promise.resolve(); + return contextRuntime.currentRequestId(); + }); + assert.equal(await second, "rid-second"); + releaseFirst.resolve(undefined); + assert.equal(await first, "rid-instance"); + assert.equal(contextRuntime.currentRequestId(), null); + + let thenReads = 0; + const stateful = {}; + Object.defineProperty(stateful, "then", { + get() { + thenReads += 1; + return undefined; + }, + }); + assert.equal(instance.echo(stateful), stateful); + assert.equal(thenReads, 0); }); }); @@ -1998,7 +2072,7 @@ test("wrapWorkerCodeForHostBindings: injects local R2 facade for R2 bindings", ( assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-r2-client.js"], /this\._stub/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const R2_BINDINGS = \["BUCKET"\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new R2Bucket\(out\[name\], requestIdOptions\(requestIdOrContext\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new R2Bucket\(out\[name\], requestIdOptions\(\)\)/); }); test("wrapWorkerCodeForHostBindings: injects local DO facade for Durable Object bindings", () => { @@ -2017,9 +2091,9 @@ test("wrapWorkerCodeForHostBindings: injects local DO facade for Durable Object assert.match(/** @type {any} */ (workerCode.modules)["_wdl-owner-endpoint.js"], /validOwnerEndpointForService/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const DO_BINDINGS = \["ROOMS"\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new DurableObjectNamespace\(out\[name\], doOptions\(requestIdOrContext, doBackend, doOwnerNetwork\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new DurableObjectNamespace\(out\[name\], doOptions\(doBackend, doOwnerNetwork\)\)/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /internalAuthToken|__WDL_INTERNAL_AUTH_TOKEN__/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class Room extends user\.Room/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.Room/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export \* from/); }); @@ -2053,11 +2127,11 @@ test("wrapWorkerCodeForHostBindings: injects local Workflow facade and wraps wor assert.match(/** @type {any} */ (workerCode.modules)["_wdl-cloudflare-workflows.js"], /this\.name = "NonRetryableError"/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \{ Workflow \}/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const WORKFLOW_BINDINGS = \{"ORDERS":/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new Workflow\(out\[name\] \|\| metadata, workflowOptions\(requestIdOrContext, workflowsBackend\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new Workflow\(out\[name\] \|\| metadata, workflowOptions\(workflowsBackend\)\)/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /internalAuthToken|__WDL_INTERNAL_AUTH_TOKEN__/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /notifyWorkflowCallback/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /notifyWorkflowCallback\(request, wrapEnv\(this\.env, requestIdFromEventArg\(request\)\)\)/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class OrderWorkflow extends user\.OrderWorkflow/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /withRequestContext\(request, \(\) => notifyWorkflowCallback\(request, wrapEnv\(this\.env\)\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.OrderWorkflow/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export \* from "\.\/worker\.js";/); }); diff --git a/tests/unit/runtime-r2-client.test.js b/tests/unit/runtime-r2-client.test.js index 6199328..fa7e07a 100644 --- a/tests/unit/runtime-r2-client.test.js +++ b/tests/unit/runtime-r2-client.test.js @@ -3,6 +3,28 @@ import assert from "node:assert/strict"; import { R2Bucket, R2Object, R2ObjectBody } from "../../runtime/r2-client.js"; import { R2_OBJECT_MAX_BUFFER_BYTES } from "../../runtime/r2-utils.js"; +/** + * @param {Promise} promise + * @returns {Promise<{ status: "fulfilled", value: unknown } | { status: "rejected", reason: unknown } | { status: "pending" }>} + */ +async function settlementWithinTestWindow(promise) { + /** @type {ReturnType | undefined} */ + let timeout; + try { + return await Promise.race([ + promise.then( + (value) => ({ status: /** @type {const} */ ("fulfilled"), value }), + (reason) => ({ status: /** @type {const} */ ("rejected"), reason }), + ), + new Promise((resolve) => { + timeout = setTimeout(() => resolve({ status: /** @type {const} */ ("pending") }), 100); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + test("R2Bucket.list validates limit before host binding call", async () => { const bucket = new R2Bucket({ async list() { @@ -298,6 +320,33 @@ test("R2Bucket.put keeps single-chunk ReadableStream bytes without re-copying", assert.equal(observed, chunk); }); +test("R2Bucket.put rejects an oversized stream without waiting for cancel", async () => { + let cancelled = false; + let hostCalls = 0; + const bucket = new R2Bucket({ + async put() { + hostCalls += 1; + return null; + }, + }); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(R2_OBJECT_MAX_BUFFER_BYTES + 1)); + }, + cancel() { + cancelled = true; + return new Promise(() => {}); + }, + }); + + const outcome = await settlementWithinTestWindow(bucket.put("huge.bin", stream)); + + assert.equal(outcome.status, "rejected"); + assert.match(String("reason" in outcome ? outcome.reason : ""), /exceeds the 25 MiB WDL R2 limit/); + assert.equal(cancelled, true); + assert.equal(hostCalls, 0); +}); + test("R2Bucket.get returns R2Object when host binding returns no body", async () => { const bucket = new R2Bucket({ async get() { @@ -410,6 +459,36 @@ test("R2ObjectBody raw body stream enforces the object byte cap", async () => { ); }); +test("R2ObjectBody byte cap does not wait for underlying cancel", async () => { + let cancelled = false; + const obj = new R2ObjectBody({ + key: "huge.bin", + version: "", + size: R2_OBJECT_MAX_BUFFER_BYTES + 1, + etag: "abc", + httpEtag: '"abc"', + uploaded: Date.now(), + httpMetadata: {}, + customMetadata: {}, + checksums: {}, + storageClass: "Standard", + }, new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(R2_OBJECT_MAX_BUFFER_BYTES + 1)); + }, + cancel() { + cancelled = true; + return new Promise(() => {}); + }, + })); + + const outcome = await settlementWithinTestWindow(obj.body.getReader().read()); + + assert.equal(outcome.status, "rejected"); + assert.match(String("reason" in outcome ? outcome.reason : ""), /exceeds the 25 MiB WDL R2 limit/); + assert.equal(cancelled, true); +}); + test("R2Bucket multipart upload methods fail with WDL-specific errors", () => { const bucket = new R2Bucket({}); diff --git a/tests/unit/runtime-wrapper-generate.test.js b/tests/unit/runtime-wrapper-generate.test.js index e51e785..cf43cf1 100644 --- a/tests/unit/runtime-wrapper-generate.test.js +++ b/tests/unit/runtime-wrapper-generate.test.js @@ -1,13 +1,14 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { AsyncLocalStorage } from "node:async_hooks"; import { HOST_BINDING_RUNTIME_MODULE_NAME, HOST_BINDING_RUNTIME_SOURCE, generateAbortShimWrapperModule, generateHostBindingWrapperModule, } from "../../runtime/load/wrapper-generate.js"; -import { moduleDataUrl } from "../helpers/load-shared-module.js"; -import { withMockedProperty } from "../helpers/mock-global.js"; +import { applyModuleReplacements, moduleDataUrl } from "../helpers/load-shared-module.js"; +import { installMockProperty } from "../helpers/mock-global.js"; const hostBindingRuntime = await import(moduleDataUrl(HOST_BINDING_RUNTIME_SOURCE)); @@ -49,23 +50,26 @@ test("generated wrapper flavors preserve default-export class detection", () => assert.equal(abortOnly.split(sourceLine).length - 1, 1); assert.match(abortOnly, new RegExp(`if \\(!${RegExp.escape(classTest)}\\)`)); - assert.match(hostBindings, /const source = functionSource\(raw\);/); - assert.match(hostBindings, /if \(regexpTest\(\/\^\\s\*class\\b\/, source\)\)/); + assert.match(hostBindings, /const source = __WdlHostRuntime__\.functionSource\(raw\);/); + assert.match(hostBindings, /if \(__WdlHostRuntime__\.regexpTest\(\/\^\\s\*class\\b\/, source\)\)/); }); test("host wrapper runtime captures platform intrinsics before user module evaluation", () => { const source = generateHostBindingWrapperModule("worker.js", [], [], ["ROOM"], {}, []); assert.ok( source.indexOf(`from "./${HOST_BINDING_RUNTIME_MODULE_NAME}";`) < - source.indexOf('import * as user from "./worker.js";') + source.indexOf('import * as __WdlUserModule__ from "./worker.js";') ); + assert.match(source, /import \* as __WdlHostRuntime__/); + assert.doesNotMatch(source, /__wdlRunWithRequestContext/); for (const intrinsic of [ "Array.prototype.forEach", "Function.prototype.toString", "Object.defineProperty", "Object.entries", "Object.keys", - "Promise.prototype.then", + "AsyncLocalStorage.prototype.getStore", + "AsyncLocalStorage.prototype.run", "Reflect.apply", "Reflect.get", "RegExp.prototype.test", @@ -77,39 +81,133 @@ test("host wrapper runtime captures platform intrinsics before user module evalu assert.doesNotMatch(source, /Function\.prototype\.toString\.call/); }); -test("host wrapper cleanup ignores tenant-patched Promise.then", async () => { - let cleaned = false; - await withMockedProperty( - Promise.prototype, - "then", - () => { - throw new Error("tenant Promise.then must not run"); - }, - async () => { - const result = await hostBindingRuntime.settleWithFinally( - Promise.resolve("settled"), - () => { - cleaned = true; - } - ); - assert.equal(result, "settled"); - assert.equal(cleaned, true); - - cleaned = false; - const failure = new Error("rejected"); - let rejection; - try { - await hostBindingRuntime.settleWithFinally( - Promise.reject(failure), - () => { - cleaned = true; - } - ); - } catch (error) { - rejection = error; - } - assert.equal(rejection, failure); - assert.equal(cleaned, true); - } +test("host wrapper request context is invocation-local and preserves return identity", async () => { + const firstStarted = Promise.withResolvers(); + const releaseFirst = Promise.withResolvers(); + const firstResult = hostBindingRuntime.runWithRequestContext("rid-first", () => { + const result = (async () => { + assert.equal(hostBindingRuntime.currentRequestId(), "rid-first"); + firstStarted.resolve(undefined); + await releaseFirst.promise; + return hostBindingRuntime.currentRequestId(); + })(); + assert.equal(hostBindingRuntime.currentRequestId(), "rid-first"); + return result; + }); + + await firstStarted.promise; + assert.equal(hostBindingRuntime.currentRequestId(), null); + const secondResult = hostBindingRuntime.runWithRequestContext("rid-second", async () => { + await Promise.resolve(); + return hostBindingRuntime.currentRequestId(); + }); + assert.equal(await secondResult, "rid-second"); + releaseFirst.resolve(undefined); + assert.equal(await firstResult, "rid-first"); + assert.equal(hostBindingRuntime.currentRequestId(), null); + + const nativePromise = Promise.resolve("same"); + assert.equal( + hostBindingRuntime.runWithRequestContext("rid-identity", () => nativePromise), + nativePromise ); }); + +test("host wrapper request context inherits when a nested call has no new id", () => { + const nested = hostBindingRuntime.runWithRequestContext("rid-outer", () => + hostBindingRuntime.runWithRequestContext(null, () => hostBindingRuntime.currentRequestId())); + + assert.equal(nested, "rid-outer"); + assert.equal(hostBindingRuntime.currentRequestId(), null); +}); + +test("generated host wrappers alias legal entrypoint names without declaration collisions", async () => { + const entrypointNames = [ + "user", + "WorkerEntrypoint", + "abortIsolate", + "withRequestContext", + "wrapEnv", + "wrapClassInstance", + "D1Database", + "R2Bucket", + "DurableObjectNamespace", + "Workflow", + ]; + const userUrl = moduleDataUrl(` + ${entrypointNames.map((name) => `export class ${name} {}`).join("\n")} + export default {}; + `); + const cloudflareUrl = moduleDataUrl(` + export class WorkerEntrypoint {} + export function abortIsolate() {} + `); + const d1Url = moduleDataUrl("export class D1Database {}"); + const r2Url = moduleDataUrl("export class R2Bucket {}"); + const doUrl = moduleDataUrl("export class DurableObjectNamespace {}"); + const workflowUrl = moduleDataUrl("export class Workflow {}"); + const source = applyModuleReplacements( + generateHostBindingWrapperModule( + "worker.js", + ["DB"], + ["BUCKET"], + ["ROOM"], + { FLOW: { className: "Workflow" } }, + entrypointNames + ), + [ + ['from "cloudflare:workers"', `from ${JSON.stringify(cloudflareUrl)}`], + [`from "./${HOST_BINDING_RUNTIME_MODULE_NAME}"`, `from ${JSON.stringify(moduleDataUrl(HOST_BINDING_RUNTIME_SOURCE))}`], + ['from "./_wdl-d1-client.js"', `from ${JSON.stringify(d1Url)}`], + ['from "./_wdl-r2-client.js"', `from ${JSON.stringify(r2Url)}`], + ['from "./_wdl-do-client.js"', `from ${JSON.stringify(doUrl)}`], + ['from "./_wdl-workflows-client.js"', `from ${JSON.stringify(workflowUrl)}`], + ['from "./worker.js"', `from ${JSON.stringify(userUrl)}`], + ] + ); + const wrapped = await import(moduleDataUrl(source)); + const userModule = await import(userUrl); + + for (const name of entrypointNames) { + assert.equal(wrapped[name].name, name); + assert.ok(new wrapped[name]({}, {}) instanceof userModule[name]); + } +}); + +test("host wrapper context uses captured AsyncLocalStorage intrinsics", async () => { + const restoreRun = installMockProperty(AsyncLocalStorage.prototype, "run", () => { + throw new Error("live AsyncLocalStorage.run must not run"); + }); + const restoreGetStore = installMockProperty(AsyncLocalStorage.prototype, "getStore", () => { + throw new Error("live AsyncLocalStorage.getStore must not run"); + }); + let result; + try { + result = hostBindingRuntime.runWithRequestContext("rid-captured", async () => { + await Promise.resolve(); + return hostBindingRuntime.currentRequestId(); + }); + } finally { + restoreGetStore(); + restoreRun(); + } + + assert.equal(await result, "rid-captured"); +}); + +test("host wrapper does not inspect or replace tenant return values", () => { + let thenReads = 0; + const value = {}; + Object.defineProperty(value, "then", { + get() { + thenReads += 1; + return undefined; + }, + }); + + const result = hostBindingRuntime.runWithRequestContext("rid-object", () => value); + + assert.equal(result, value); + assert.equal(thenReads, 0); + assert.equal(hostBindingRuntime.currentRequestId(), null); +}); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index b157f13..239005f 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -394,13 +394,21 @@ test("secret Redis key literals stay aligned across JS and redis-proxy", () => { }); test("worker delete lock key stays aligned across control and workflows", () => { - const controlLib = withoutLineComments(readRepoFile("control/lib.js")); + const sharedVersion = withoutLineComments(readRepoFile("shared/version.js")); + const commonVersion = withoutLineComments(readRepoFile("rust/common/src/version.rs")); const controlShared = withoutLineComments(readRepoFile("control/shared.js")); const workflowsActiveExport = withoutLineComments(readRepoFile("rust/workflows/src/api/active_export.rs")); - assert.match(controlLib, /`worker-delete-lock:\$\{ns\}:\$\{worker\}`/); + assert.match(sharedVersion, /`worker-delete-lock:\$\{ns\}:\$\{worker\}`/); + assert.match(commonVersion, /format!\("worker-delete-lock:\{ns\}:\{worker\}"\)/); assert.match(controlShared, /\bdeleteLockKey\(ns, worker\)/); - assert.match(workflowsActiveExport, /format!\("worker-delete-lock:\{ns\}:\{worker\}"\)/); + assert.match(workflowsActiveExport, /\bworker_delete_lock_key\(ns, worker\)/); + + const rustOwner = "rust/common/src/version.rs"; + const offenders = rustFiles("rust").filter((file) => + file !== rustOwner && withoutLineComments(readRepoFile(file)).includes("worker-delete-lock:") + ); + assert.deepEqual(offenders, []); }); test("product success payloads use camelCase fields", () => { diff --git a/tests/unit/version.test.js b/tests/unit/version.test.js index 63b3bc4..0ca53b7 100644 --- a/tests/unit/version.test.js +++ b/tests/unit/version.test.js @@ -6,13 +6,19 @@ import { HOST_DECLARATIONS_SCAN_PATTERN, HOSTS_SCAN_PATTERN, NAMESPACES_KEY, + VERSION_DELETE_LOCK_KIND, + WHOLE_DELETE_LOCK_KIND, bundleKey, + deleteLockKey, + doStorageIdKey, + formatDeleteLockToken, formatVersion, hostDeclarationsKey, hostsKey, namespaceFromHostsKey, nextVersionKey, nsHostsKey, + parseDeleteLockKind, parseVersion, } from "../../shared/version.js"; @@ -81,3 +87,24 @@ test("route-plane registry key helpers compose and parse canonical keys", () => test("nextVersionKey composes the worker version counter key", () => { assert.equal(nextVersionKey("demo", "hello"), "worker:demo:hello:next_version"); }); + +test("worker lifecycle key helpers compose canonical keys", () => { + assert.equal(doStorageIdKey("demo", "hello"), "worker:do-storage:demo:hello"); + assert.equal(deleteLockKey("demo", "hello"), "worker-delete-lock:demo:hello"); +}); + +test("worker delete lock tokens carry the operation kind", () => { + assert.equal( + formatDeleteLockToken(WHOLE_DELETE_LOCK_KIND, "abc123"), + "whole:abc123" + ); + assert.equal( + formatDeleteLockToken(VERSION_DELETE_LOCK_KIND, "abc123"), + "version:abc123" + ); + assert.equal(parseDeleteLockKind("whole:abc123"), WHOLE_DELETE_LOCK_KIND); + assert.equal(parseDeleteLockKind("version:abc123"), VERSION_DELETE_LOCK_KIND); + assert.equal(parseDeleteLockKind("unknown:abc123"), null); + assert.equal(parseDeleteLockKind("whole:"), null); + assert.equal(parseDeleteLockKind("legacy-token"), null); +}); From 75ec56723734a0576600e456906c46f473defff1 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Wed, 15 Jul 2026 21:23:14 +0800 Subject: [PATCH 12/23] Optimize owner snapshots and stabilize workflow reads Pipeline owner, delete-lock, and Redis-time reads into one watched snapshot, align Redis test doubles, and retry workflow metadata reads without masking stable corrupt metadata during unrelated route churn. Signed-off-by: Lu Zhang --- control/handlers/workflows.js | 138 +++++++---- do-runtime/owner-registry.js | 6 +- shared/owner-protocol.js | 35 ++- shared/redis-session.js | 17 +- tests/helpers/mocks/fake-redis.js | 12 + tests/unit/control-handlers-workflows.test.js | 214 ++++++++++++++++++ tests/unit/do-owner-registry.test.js | 10 + tests/unit/fake-redis.test.js | 10 + tests/unit/owner-protocol.test.js | 20 ++ tests/unit/redis-session.test.js | 36 +++ 10 files changed, 444 insertions(+), 54 deletions(-) diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index a5d8e59..c9fa80c 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -20,6 +20,7 @@ import { import { bundleKey, routesKey } from "shared-version"; const LIFECYCLE_ACTIONS = new Set(["pause", "resume", "restart", "terminate"]); +const MAX_WORKFLOW_SNAPSHOT_ATTEMPTS = 2; /** * @typedef {import("shared-redis").RedisClient} RedisClient @@ -158,19 +159,50 @@ async function handleInner({ method, url, ns, subPath, requestId }) { * @param {string} ns */ async function listWorkflowDefinitions({ redis }, ns) { - const routes = await redis.hGetAll(routesKey(ns)); - /** @type {Array<[string, string]>} */ - const routeEntries = []; - for (const [worker, activeVersion] of Object.entries(routes)) { - if (typeof activeVersion === "string" && activeVersion) routeEntries.push([worker, activeVersion]); + for (let attempt = 0; attempt < MAX_WORKFLOW_SNAPSHOT_ATTEMPTS; attempt += 1) { + const routes = await redis.hGetAll(routesKey(ns)); + /** @type {Array<[string, string]>} */ + const routeEntries = []; + for (const [worker, activeVersion] of Object.entries(routes)) { + if (typeof activeVersion === "string" && activeVersion) routeEntries.push([worker, activeVersion]); + } + if (routeEntries.length === 0) return { namespace: ns, workflows: [] }; + const [metaRaws, defsRaws] = await readWorkflowListRaws({ redis }, ns, routeEntries); + const currentRoutes = await redis.hGetAll(routesKey(ns)); + /** @type {Array | undefined>} */ + const metas = new Array(routeEntries.length); + for (let index = 0; index < routeEntries.length; index += 1) { + const [worker, activeVersion] = routeEntries[index]; + if (currentRoutes[worker] === activeVersion) { + metas[index] = workflowBundleMeta(ns, worker, activeVersion, metaRaws[index]); + } + } + if (!sameRouteSnapshot(routes, currentRoutes)) continue; + return buildWorkflowDefinitionList( + ns, + routeEntries, + /** @type {Record[]} */ (metas), + defsRaws + ); } - if (routeEntries.length === 0) return { namespace: ns, workflows: [] }; - const [metaRaws, defsRaws] = await readWorkflowListRaws({ redis }, ns, routeEntries); + throw new ControlAbort(503, "workflow_metadata_contention", { + message: "Workflow metadata changed while it was being read", + namespace: ns, + }); +} + +/** + * @param {string} ns + * @param {Array<[string, string]>} routeEntries + * @param {Array>} metas + * @param {Array>} defsRaws + */ +function buildWorkflowDefinitionList(ns, routeEntries, metas, defsRaws) { /** @type {ListedWorkflowEntry[]} */ const workflows = []; for (let i = 0; i < routeEntries.length; i += 1) { const [worker, activeVersion] = routeEntries[i]; - const meta = workflowBundleMeta(ns, worker, activeVersion, metaRaws[i]); + const meta = metas[i]; const activeByName = new Map(); for (const workflow of workflowsFromMeta(meta)) { activeByName.set(workflow.name, workflow); @@ -206,6 +238,12 @@ async function listWorkflowDefinitions({ redis }, ns) { return { namespace: ns, workflows: sortedWorkflows }; } +/** @param {Record} left @param {Record} right */ +function sameRouteSnapshot(left, right) { + const keys = Object.keys(left); + return keys.length === Object.keys(right).length && keys.every((key) => right[key] === left[key]); +} + /** * @param {RedisDeps} deps * @param {string} ns @@ -224,15 +262,34 @@ async function resolveWorkflow({ redis }, ns, worker, workflowName) { message: `Invalid workflow name ${JSON.stringify(workflowName)}. Must match ${WORKFLOW_NAME_RE}.`, }); } - const activeVersion = await redis.hGet(routesKey(ns), worker); - if (!activeVersion) { - throw new ControlAbort(404, "worker_not_found", { - message: `Worker ${ns}/${worker} is not active`, - }); - } - const meta = await readBundleMeta({ redis }, ns, worker, activeVersion); - const workflow = workflowsFromMeta(meta).find((entry) => entry.name === workflowName); - if (workflow) { + for (let attempt = 0; attempt < MAX_WORKFLOW_SNAPSHOT_ATTEMPTS; attempt += 1) { + const active = await readActiveWorkflowMeta({ redis }, ns, worker); + if (!active) continue; + const { activeVersion, meta } = active; + const activeWorkflow = workflowsFromMeta(meta).find((entry) => entry.name === workflowName); + /** @type {WorkflowEntry | undefined} */ + let workflow = activeWorkflow; + if (!workflow) { + const defs = await readWorkflowDefs({ redis }, ns, worker); + const def = Object.hasOwn(defs, workflowName) + ? defs[workflowName] + : undefined; + if (def) { + workflow = { + name: workflowName, + binding: null, + className: def.className, + workflowKey: def.workflowKey, + retired: true, + }; + } + } + if (!activeWorkflow && await redis.hGet(routesKey(ns), worker) !== activeVersion) continue; + if (!workflow) { + throw new ControlAbort(404, "workflow_not_found", { + message: `Workflow ${ns}/${worker}/${workflowName} is not exported`, + }); + } return { workflow, request: { @@ -245,34 +302,28 @@ async function resolveWorkflow({ redis }, ns, worker, workflowName) { }, }; } + throw new ControlAbort(503, "workflow_metadata_contention", { + message: `Workflow metadata changed while ${ns}/${worker} was being read`, + namespace: ns, + worker, + }); +} - const defs = await readWorkflowDefs({ redis }, ns, worker); - const def = Object.hasOwn(defs, workflowName) - ? defs[workflowName] - : undefined; - if (!def) { - throw new ControlAbort(404, "workflow_not_found", { - message: `Workflow ${ns}/${worker}/${workflowName} is not exported`, +/** + * @param {RedisDeps} deps + * @param {string} ns + * @param {string} worker + * @returns {Promise<{ activeVersion: string, meta: Record } | null>} + */ +async function readActiveWorkflowMeta({ redis }, ns, worker) { + const activeVersion = await redis.hGet(routesKey(ns), worker); + if (!activeVersion) { + throw new ControlAbort(404, "worker_not_found", { + message: `Worker ${ns}/${worker} is not active`, }); } - const retiredWorkflow = { - name: workflowName, - binding: null, - className: def.className, - workflowKey: def.workflowKey, - retired: true, - }; - return { - workflow: retiredWorkflow, - request: { - ns, - worker, - frozenVersion: activeVersion, - workflowName: retiredWorkflow.name, - workflowKey: retiredWorkflow.workflowKey, - className: retiredWorkflow.className, - }, - }; + const meta = await readBundleMeta({ redis }, ns, worker, activeVersion); + return meta ? { activeVersion, meta } : null; } /** @@ -370,12 +421,13 @@ async function readWorkflowListRaws({ redis }, ns, routeEntries) { * @param {string} ns * @param {string} worker * @param {string} version - * @returns {Promise>} + * @returns {Promise | null>} */ async function readBundleMeta({ redis }, ns, worker, version) { let raw; try { raw = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); + if (raw == null && await redis.hGet(routesKey(ns), worker) !== version) return null; } catch (err) { requireControlLog()("error", "workflow_metadata_unavailable", { namespace: ns, diff --git a/do-runtime/owner-registry.js b/do-runtime/owner-registry.js index a055683..8a64fc5 100644 --- a/do-runtime/owner-registry.js +++ b/do-runtime/owner-registry.js @@ -38,6 +38,7 @@ import { ownerProtocolKeys, readOwnerRecord, readOwnerRecordWithRedisTime, + readOwnerSnapshotWithRedisTime, stageOwnerClaim, stageOwnerRelease, stageOwnerRenew, @@ -353,12 +354,13 @@ export async function resolveDoOwner(env, invoke) { return await withWatchRetries(async () => client.session(async (session) => { await session.watch(key, generationKey, workerDeleteLockKey); - const { owner: current, nowMs } = await readOwnerRecordWithRedisTime( + const { owner: current, relatedValues, nowMs } = await readOwnerSnapshotWithRedisTime( session, key, + [workerDeleteLockKey], (raw) => parseOwner(raw, ownerKey, scope) ); - const deleteLockToken = decodeBulk(await session.get(workerDeleteLockKey)); + const deleteLockToken = decodeBulk(relatedValues[0]); if ( deleteLockToken != null && parseDeleteLockKind(deleteLockToken) !== VERSION_DELETE_LOCK_KIND diff --git a/shared/owner-protocol.js b/shared/owner-protocol.js index 2f4acc9..afc35fd 100644 --- a/shared/owner-protocol.js +++ b/shared/owner-protocol.js @@ -13,6 +13,7 @@ import { * exec(): Promise, * }} OwnerWriteMulti * @typedef {{ value: string | Uint8Array | ArrayBuffer | null | undefined, nowMs: number }} OwnerReadWithTimeResult + * @typedef {{ values: Array, nowMs: number }} OwnerSnapshotReadWithTimeResult */ /** * @template T @@ -21,6 +22,20 @@ import { * @returns {T | null} */ +/** + * @template T + * @param {string | Uint8Array | ArrayBuffer | null | undefined} value + * @param {number} nowMs + * @param {OwnerParser} parseOwner + * @returns {{ owner: T | null, nowMs: number }} + */ +function parseTimedOwnerRead(value, nowMs, parseOwner) { + if (!Number.isSafeInteger(nowMs) || nowMs < 0) { + throw new Error("Redis server time is invalid"); + } + return { owner: parseOwner(value), nowMs }; +} + /** * @param {string} prefix * @param {string} scope @@ -68,12 +83,22 @@ export async function readOwnerRecord(client, ownerKey, parseOwner) { */ export async function readOwnerRecordWithRedisTime(client, ownerKey, parseOwner) { const result = await client.getWithTime(ownerKey); - if (!Number.isSafeInteger(result.nowMs) || result.nowMs < 0) { - throw new Error("Redis server time is invalid"); - } + return parseTimedOwnerRead(result.value, result.nowMs, parseOwner); +} + +/** + * @template T + * @param {{ getManyWithTime(keys: string[]): Promise }} client + * @param {string} ownerKey + * @param {string[]} relatedKeys + * @param {OwnerParser} parseOwner + * @returns {Promise<{ owner: T | null, relatedValues: Array, nowMs: number }>} + */ +export async function readOwnerSnapshotWithRedisTime(client, ownerKey, relatedKeys, parseOwner) { + const result = await client.getManyWithTime([ownerKey, ...relatedKeys]); return { - owner: parseOwner(result.value), - nowMs: result.nowMs, + ...parseTimedOwnerRead(result.values[0], result.nowMs, parseOwner), + relatedValues: result.values.slice(1), }; } diff --git a/shared/redis-session.js b/shared/redis-session.js index 3d5de2f..ce5b4ac 100644 --- a/shared/redis-session.js +++ b/shared/redis-session.js @@ -259,13 +259,22 @@ export class RedisSession { } /** @param {string} key */ async getWithTime(key) { - const [value, time] = await this._execPipeline("GET_TIME_PIPELINE", [ - ["GET", key], + const { values, nowMs } = await this.getManyWithTime([key]); + return { + value: values[0], + nowMs, + }; + } + /** @param {string[]} keys */ + async getManyWithTime(keys) { + if (keys.length === 0) throw new Error("getManyWithTime requires at least one key"); + const replies = await this._execPipeline("GET_TIME_PIPELINE", [ + ...keys.map((key) => /** @type {RedisCommand} */ (["GET", key])), ["TIME"], ]); return { - value: decodeBulk(value), - nowMs: decodeRedisTimeMs(time), + values: replies.slice(0, -1).map(decodeBulk), + nowMs: decodeRedisTimeMs(replies.at(-1)), }; } async time() { return decodeRedisTimeMs(await this._exec("TIME")); } diff --git a/tests/helpers/mocks/fake-redis.js b/tests/helpers/mocks/fake-redis.js index f7a7849..d150eed 100644 --- a/tests/helpers/mocks/fake-redis.js +++ b/tests/helpers/mocks/fake-redis.js @@ -314,6 +314,18 @@ export function createFakeRedisSession(state, options = {}) { async getWithTime(key) { return { value: await this.get(key), nowMs: await this.time() }; }, + /** @param {string[]} keys */ + async getManyWithTime(keys) { + if (keys.length === 0) throw new Error("getManyWithTime requires at least one key"); + state.commands.push(["getManyWithTime", [...keys]]); + return { + values: keys.map((key) => { + expireIfNeeded(state, key, options); + return state.strings.get(key) ?? null; + }), + nowMs: await this.time(), + }; + }, async time() { return currentFakeRedisTimeMs(state, options); }, diff --git a/tests/unit/control-handlers-workflows.test.js b/tests/unit/control-handlers-workflows.test.js index ce4a5b4..578a6a7 100644 --- a/tests/unit/control-handlers-workflows.test.js +++ b/tests/unit/control-handlers-workflows.test.js @@ -85,6 +85,7 @@ test("workflows handler lists active workflow definitions from bundle metadata", ["hGetAll", "routes:demo"], ["hGetMany", [["worker:demo:api:v:2", "__meta__"]]], ["hGetAllMany", ["wf:defs:demo:api"]], + ["hGetAll", "routes:demo"], ]); assert.deepEqual(state.logs, [{ level: "info", @@ -142,6 +143,219 @@ for (const [label, rawMeta] of [ }); } +test("workflows handler retries a list snapshot split by whole-worker delete", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + redis.hGetMany = async (/** @type {Array<[string, string]>} */ pairs) => { + redis.commands.push(["hGetMany", pairs]); + redis.hashes.set("routes:demo", {}); + redis.hashes.set("worker:demo:api:v:2", {}); + return pairs.map(() => null); + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-list-delete-race", + }); + + await assertJsonResponse(response, 200, { + namespace: "demo", + workflows: [], + }); + assert.deepEqual(state.logs, [{ + level: "info", + event: "workflows_listed", + fields: { request_id: "rid-list-delete-race", namespace: "demo", count: 0 }, + }]); +}); + +test("workflows handler returns worker_not_found when whole-delete wins metadata resolution", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + const originalHGet = redis.hGet.bind(redis); + redis.hGet = async (/** @type {string} */ key, /** @type {string} */ field) => { + if (key === "worker:demo:api:v:2" && field === "__meta__") { + redis.commands.push(["hGet", key, field]); + redis.hashes.set("routes:demo", {}); + redis.hashes.set("worker:demo:api:v:2", {}); + return null; + } + return await originalHGet(key, field); + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances/order-1"), + ns: "demo", + subPath: ["api", "orders", "instances", "order-1"], + requestId: "rid-resolve-delete-race", + }); + + await assertJsonResponse(response, 404, { + error: "worker_not_found", + message: "Worker demo/api is not active", + }); + assert.equal(redis.commands.some((/** @type {unknown[]} */ command) => command[0] === "fetch"), false); +}); + +test("workflows handler returns contention when workflow resolution never stabilizes", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + const originalHGet = redis.hGet.bind(redis); + const nextVersions = new Map([ + ["worker:demo:api:v:2", "v3"], + ["worker:demo:api:v:3", "v4"], + ]); + redis.hGet = async (/** @type {string} */ key, /** @type {string} */ field) => { + const nextVersion = field === "__meta__" ? nextVersions.get(key) : undefined; + if (nextVersion) { + redis.commands.push(["hGet", key, field]); + redis.hashes.set("routes:demo", { api: nextVersion }); + return null; + } + return await originalHGet(key, field); + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances/order-1"), + ns: "demo", + subPath: ["api", "orders", "instances", "order-1"], + requestId: "rid-resolve-contention", + }); + + await assertJsonResponse(response, 503, { + error: "workflow_metadata_contention", + message: "Internal error", + namespace: "demo", + worker: "api", + }); + assert.equal(redis.commands.filter((/** @type {unknown[]} */ command) => + command[0] === "hGet" && String(command[1]).startsWith("worker:demo:api:v:") + ).length, 2); + assert.equal(redis.commands.some((/** @type {unknown[]} */ command) => command[0] === "fetch"), false); +}); + +for (const [label, rawMeta] of [ + ["missing", null], + ["empty", ""], + ["malformed", "SECRET_TOKEN_ABC"], + ["non-object", "[]"], +]) { + test(`workflows handler reports stable ${label} metadata despite sibling route churn`, async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + const emptyMeta = JSON.stringify({ workflows: [] }); + redis.hashes.set("worker:demo:api:v:2", rawMeta == null ? {} : { "__meta__": rawMeta }); + redis.hashes.set("worker:demo:billing:v:1", { "__meta__": emptyMeta }); + redis.hashes.set("worker:demo:billing:v:2", { "__meta__": emptyMeta }); + const routeSnapshots = [ + { api: "v2", billing: "v1" }, + { api: "v2", billing: "v2" }, + { api: "v2", billing: "v2" }, + { api: "v2", billing: "v3" }, + ]; + redis.hGetAll = async (/** @type {string} */ key) => { + redis.commands.push(["hGetAll", key]); + if (key === "routes:demo") return routeSnapshots.shift() ?? routeSnapshots.at(-1) ?? {}; + return redis.hashes.get(key) ?? {}; + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: `rid-stable-${label}-with-sibling-churn`, + }); + + await assertJsonResponse(response, 500, { + namespace: "demo", + worker: "api", + version: "v2", + error: "corrupt_meta", + message: "Internal error", + }); + assert.equal(routeSnapshots.length, 2); + }); +} + +test("workflows handler returns contention when list routes never stabilize", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + const meta = redis.hashes.get("worker:demo:api:v:2")["__meta__"]; + redis.hashes.set("worker:demo:api:v:3", { "__meta__": meta }); + const routeSnapshots = [ + { api: "v2" }, + { api: "v3" }, + { api: "v3" }, + { api: "v4" }, + ]; + redis.hGetAll = async (/** @type {string} */ key) => { + redis.commands.push(["hGetAll", key]); + if (key === "routes:demo") return routeSnapshots.shift() ?? {}; + return redis.hashes.get(key) ?? {}; + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-list-contention", + }); + + await assertJsonResponse(response, 503, { + error: "workflow_metadata_contention", + message: "Internal error", + namespace: "demo", + }); + assert.equal(routeSnapshots.length, 0); +}); + +test("workflows handler does not combine active metadata with redeployed workflow defs", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + redis.hashes.set("worker:demo:api:v:2", { "__meta__": JSON.stringify({ workflows: [] }) }); + const originalHGetAll = redis.hGetAll.bind(redis); + let redeployed = false; + redis.hGetAll = async (/** @type {string} */ key) => { + if (key === "wf:defs:demo:api" && !redeployed) { + redeployed = true; + redis.hashes.set("routes:demo", { api: "v3" }); + redis.hashes.set("worker:demo:api:v:3", { "__meta__": JSON.stringify({ workflows: [] }) }); + redis.hashes.set(key, { + orders: JSON.stringify({ workflowKey: "wf_new", className: "NewOrderWorkflow" }), + }); + } + return await originalHGetAll(key); + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances/order-1"), + ns: "demo", + subPath: ["api", "orders", "instances", "order-1"], + requestId: "rid-redeploy-defs-race", + }); + + assert.equal(response.status, 200); + assert.deepEqual(redis.commands.at(-1), ["fetch", "http://workflows/internal/workflows/status", { + ns: "demo", + worker: "api", + frozenVersion: "v3", + workflowName: "orders", + workflowKey: "wf_new", + className: "NewOrderWorkflow", + instanceId: "order-1", + options: {}, + requestId: "rid-redeploy-defs-race", + }]); +}); + test("workflows handler lists empty namespaces without batch reads", async () => { const state = resetWorkflowsHandlerState(); state.redis.hashes.set("routes:demo", {}); diff --git a/tests/unit/do-owner-registry.test.js b/tests/unit/do-owner-registry.test.js index c9b9406..5ed2cec 100644 --- a/tests/unit/do-owner-registry.test.js +++ b/tests/unit/do-owner-registry.test.js @@ -217,6 +217,16 @@ test("DO owner registry: first claim refuses a worker under whole-delete lock", } ); assert.ok(DO_OWNER_REGISTRY_TEST_STATE.watchedKeys.includes(DELETE_LOCK_KEY)); + assert.deepEqual( + DO_OWNER_REGISTRY_TEST_STATE.redisState.commands.filter((command) => command[0] === "getManyWithTime"), + [["getManyWithTime", [ownerKeyOf(OWNER_KEY), DELETE_LOCK_KEY]]] + ); + assert.equal( + DO_OWNER_REGISTRY_TEST_STATE.redisState.commands.some((command) => ( + command[0] === "get" && command[1] === DELETE_LOCK_KEY + )), + false + ); assert.deepEqual(doOwnerRegistryWriteCommands(), []); }); diff --git a/tests/unit/fake-redis.test.js b/tests/unit/fake-redis.test.js index 4399466..2847117 100644 --- a/tests/unit/fake-redis.test.js +++ b/tests/unit/fake-redis.test.js @@ -108,6 +108,16 @@ test("fake redis session records single and batched hash reads", async () => { ]); }); +test("fake redis getManyWithTime matches the production empty-input contract", async () => { + const redis = createFakeRedis(); + await redis.session(async (session) => { + await assert.rejects( + session.getManyWithTime([]), + /getManyWithTime requires at least one key/ + ); + }); +}); + test("fake redis supports hash existence and key reads on clients and sessions", async () => { const redis = createFakeRedis(); redis.hashes.set("hash:a", { one: "1", two: "2" }); diff --git a/tests/unit/owner-protocol.test.js b/tests/unit/owner-protocol.test.js index c831fa2..80c305f 100644 --- a/tests/unit/owner-protocol.test.js +++ b/tests/unit/owner-protocol.test.js @@ -9,6 +9,7 @@ const { ownerProtocolKeys, readOwnerRecord, readOwnerRecordWithRedisTime, + readOwnerSnapshotWithRedisTime, stageOwnerClaim, stageOwnerRelease, stageOwnerRenew, @@ -68,6 +69,25 @@ test("owner protocol helpers reject invalid Redis time from combined owner reads ); }); +test("owner protocol helpers read related owner state in the timed snapshot", async () => { + const result = await readOwnerSnapshotWithRedisTime({ + /** @param {string[]} keys */ + async getManyWithTime(keys) { + assert.deepEqual(keys, ["owner-key", "delete-lock"]); + return { + values: ["raw-owner", "whole:token"], + nowMs: 1_700_000_000_456, + }; + }, + }, "owner-key", ["delete-lock"], (/** @type {string | Uint8Array | ArrayBuffer | null | undefined} */ raw) => ({ raw })); + + assert.deepEqual(result, { + owner: { raw: "raw-owner" }, + relatedValues: ["whole:token"], + nowMs: 1_700_000_000_456, + }); +}); + test("owner protocol helpers stage claim, renew, and release writes", () => { /** @type {Array} */ const commands = []; diff --git a/tests/unit/redis-session.test.js b/tests/unit/redis-session.test.js index 5435694..863fa61 100644 --- a/tests/unit/redis-session.test.js +++ b/tests/unit/redis-session.test.js @@ -331,6 +331,42 @@ test("RedisSession.getWithTime batches GET and TIME on its held socket", async ( ); }); +test("RedisSession.getManyWithTime batches related GETs and TIME on its held socket", async () => { + const socket = makeFakeSocket([ + bytes("$5\r\nowner\r\n$11\r\nwhole:token\r\n*2\r\n$10\r\n1700000000\r\n$6\r\n654321\r\n"), + ]); + const { connect, state } = scriptedConnect(socket); + const client = new RedisClient("x", { connect }); + + const result = await client.session((/** @type {any} */ s) => ( + s.getManyWithTime(["owner-key", "delete-lock"]) + )); + + assert.equal(state.count, 1); + assert.deepEqual(result, { + values: ["owner", "whole:token"], + nowMs: 1_700_000_000_654, + }); + assert.equal( + decode(socket._writes[0]), + "*2\r\n$3\r\nGET\r\n$9\r\nowner-key\r\n" + + "*2\r\n$3\r\nGET\r\n$11\r\ndelete-lock\r\n" + + "*1\r\n$4\r\nTIME\r\n" + ); +}); + +test("RedisSession.getManyWithTime rejects an empty key set before opening IO", async () => { + const session = new RedisSession("x", { + connect() { + throw new Error("unexpected connection"); + }, + }); + await assert.rejects( + session.getManyWithTime([]), + /getManyWithTime requires at least one key/ + ); +}); + test("RedisSession.hSet supports object form", async () => { const socket = makeFakeSocket([bytes(":2\r\n")]); const { connect } = scriptedConnect(socket); From e3d486c8955ef986c704a647ee03a09844fd36d9 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Thu, 16 Jul 2026 05:27:23 +0800 Subject: [PATCH 13/23] Consolidate runtime and lifecycle contracts Fence delegated issuance and deletion races Centralize DO alarm, ownership, and transport validation Align cross-tier fixtures, diagnostics, and request propagation Signed-off-by: Lu Zhang --- auth/index.js | 55 +++- auth/runtime.js | 40 +-- control/handlers/delete.js | 6 +- control/handlers/versions.js | 4 +- control/handlers/workflows.js | 2 +- control/lib.js | 21 +- control/routing.js | 5 +- control/shared.js | 17 +- control/workflows-client.js | 12 +- do-runtime/alarm-dispatch.js | 78 ++---- do-runtime/alarm.js | 9 +- do-runtime/owner-registry.js | 6 +- do-runtime/protocol.js | 24 +- do-runtime/protocol/identity.js | 18 +- docs/modules/control-auth.md | 9 +- docs/modules/control-auth.zh.md | 2 +- docs/modules/durable-objects.md | 3 + docs/modules/durable-objects.zh.md | 1 + docs/modules/log-tail-observability.md | 2 +- docs/modules/log-tail-observability.zh.md | 2 +- docs/modules/workflows.md | 11 +- docs/modules/workflows.zh.md | 3 +- docs/source-map.md | 4 +- docs/source-map.zh.md | 4 +- gateway/runtime.js | 20 +- runtime/_wdl-do-transport.js | 191 +++++-------- runtime/bindings/r2.js | 4 +- runtime/config-system.capnp | 1 + runtime/do-client.js | 27 +- runtime/lib.js | 11 +- runtime/workflows-client.js | 8 +- rust/workflows/src/api/create.rs | 6 +- rust/workflows/src/api/do_alarms/model.rs | 264 +++++++++++++++--- rust/workflows/src/api/do_alarms/mutations.rs | 6 +- rust/workflows/src/api/limits.rs | 5 +- shared/redis-lock.js | 16 +- shared/redis-session.js | 5 + shared/version.js | 12 + tests/fixtures/do-alarm-identity.json | 142 ++++++++++ tests/fixtures/workflow-limits.json | 1 + tests/helpers/control-shared-stub.js | 7 +- tests/helpers/load-auth-index.js | 62 +++- tests/helpers/load-runtime-r2-binding.js | 2 + tests/helpers/load-shared-module.js | 2 + tests/helpers/mocks/fake-redis.js | 4 +- tests/unit/auth-index.test.js | 94 ++++++- tests/unit/control-delete-handler.test.js | 50 +++- tests/unit/control-shared-stub.test.js | 23 ++ tests/unit/control-shared.test.js | 18 +- tests/unit/do-alarm-client.test.js | 16 ++ tests/unit/do-runtime-index.test.js | 75 +++++ tests/unit/do-runtime-protocol.test.js | 77 ++++- tests/unit/redis-lock.test.js | 31 -- tests/unit/redis-session.test.js | 11 + tests/unit/runtime-do-client.test.js | 94 ++++++- tests/unit/runtime-workflows-client.test.js | 16 ++ tests/unit/style-contracts.test.js | 44 --- tests/unit/version.test.js | 13 + 58 files changed, 1224 insertions(+), 472 deletions(-) create mode 100644 tests/fixtures/do-alarm-identity.json diff --git a/auth/index.js b/auth/index.js index f9cde88..a9caaa3 100644 --- a/auth/index.js +++ b/auth/index.js @@ -148,16 +148,26 @@ async function releaseDelegatedIssueLock(session, lock, runtime, requestId) { /** * @param {import("shared-redis").RedisSession} session - * @param {string} key - * @param {string} token + * @param {{ key: string, token: string }} lock + * @param {ReturnType} runtime + * @param {string} issuerTokenId + * @param {string} templateId */ -async function watchDelegatedIssueLockHeld(session, key, token) { - await session.watch(key); - const current = await session.get(key); - if (current !== token) { +async function watchDelegatedIssuePrerequisites( + session, + lock, + runtime, + issuerTokenId, + templateId, +) { + await session.watch(lock.key, tokenKey(issuerTokenId)); + const current = await session.get(lock.key); + if (current !== lock.token) { throw new AuthPolicyError(409, "delegated_issue_busy", "delegated issue lock expired before the token could be issued; retry"); } + const issuer = await runtime.readRecord(session, issuerTokenId); + validateIssuerForDelegatedIssue(issuer, templateId); } /** @param {number} lockAttemptStartedAtMs */ @@ -477,20 +487,23 @@ export default class Auth extends WorkerEntrypoint { const lockKey = delegatedIssueLockKey(issuerTokenId, templateId); const issueLock = createTokenLock(lockKey); const lockAttemptStartedAtMs = Date.now(); + let locked; try { - const locked = await acquireTokenLock(session, issueLock, { + locked = await acquireTokenLock(session, issueLock, { ttlSeconds: DELEGATED_ISSUE_LOCK_TTL_SECONDS, - transactional: true, }); - if (!locked) { - throw new AuthPolicyError(409, "delegated_issue_busy", - "delegated issue is already in progress"); - } } catch (err) { - await releaseDelegatedIssueLock(session, issueLock, runtime, requestId); + // A failed SET reply has an unknown server-side outcome and leaves this + // RESP session untrustworthy. The lock TTL is the recovery path. return runtime.rethrowPolicy(requestId, "delegated_issue", err); } + if (!locked) { + return runtime.rethrowPolicy(requestId, "delegated_issue", + new AuthPolicyError(409, "delegated_issue_busy", + "delegated issue is already in progress")); + } + let canReleaseIssueLock = true; try { const nowMs = Date.now(); const tokenRecords = await scanTokenRecords(runtime, session); @@ -505,7 +518,13 @@ export default class Auth extends WorkerEntrypoint { } const namespaceChoice = await chooseDelegatedNamespace(session, tokenRecords, template); assertDelegatedIssueLockLocalBudget(lockAttemptStartedAtMs); - await watchDelegatedIssueLockHeld(session, issueLock.key, issueLock.token); + await watchDelegatedIssuePrerequisites( + session, + issueLock, + runtime, + issuerTokenId, + templateId, + ); const tokenId = generateTokenId(); const { ns, label } = namespaceChoice; const expiresAt = new Date(nowMs + template.ttlSeconds * 1000).toISOString(); @@ -534,6 +553,10 @@ export default class Auth extends WorkerEntrypoint { throw new AuthPolicyError(409, "delegated_issue_busy", "delegated issue lock changed before the token could be issued; retry"); } + // Any other EXEC failure has an unknown transaction outcome. Keep + // the lock and let its TTL bound recovery rather than retrying an + // issuance that may already have committed. + canReleaseIssueLock = false; throw err; } runtime.recordLifecycleOk("delegated_issue", requestId, { @@ -557,7 +580,9 @@ export default class Auth extends WorkerEntrypoint { } catch (err) { return runtime.rethrowPolicy(requestId, "delegated_issue", err); } finally { - await releaseDelegatedIssueLock(session, issueLock, runtime, requestId); + if (canReleaseIssueLock) { + await releaseDelegatedIssueLock(session, issueLock, runtime, requestId); + } } }); } diff --git a/auth/runtime.js b/auth/runtime.js index 0fe10e8..d71ce21 100644 --- a/auth/runtime.js +++ b/auth/runtime.js @@ -19,6 +19,7 @@ import { import { actionCategory, } from "shared-auth-roles"; +import { withOptimisticRetries } from "shared-optimistic-retry"; /** * @typedef {import("shared-redis").RedisSession} RedisSession @@ -256,28 +257,31 @@ async function ensureBootstrapHash(redis, requestId, logAuth, desiredHash) { return true; }; - for (let attempt = 0; attempt < 5; attempt++) { - try { + await withOptimisticRetries( + async () => { if (typeof redis.session === "function") await redis.session(rotateBootstrap); else await rotateBootstrap(/** @type {RedisAuthSession} */ (redis)); - bootstrapMissingLogged = false; - verifyBootstrapHash = desiredHash; - logAuth("info", "auth_bootstrap", requestId, { - outcome: "ok", - token_id: BOOTSTRAP_TOKEN_ID, - rotated: rotatedFromExisting, - }); - return; - } catch (err) { - if (err instanceof WatchError) continue; - throw err; + return true; + }, + { + attempts: 5, + isRetryableError: (err) => err instanceof WatchError, + onExhausted: () => { + logAuth("error", "auth_bootstrap", requestId, { + outcome: "error", + reason: "bootstrap_watch_exhausted", + }); + throw new Error("bootstrap upsert WATCH retry exhausted"); + }, } - } - logAuth("error", "auth_bootstrap", requestId, { - outcome: "error", - reason: "bootstrap_watch_exhausted", + ); + bootstrapMissingLogged = false; + verifyBootstrapHash = desiredHash; + logAuth("info", "auth_bootstrap", requestId, { + outcome: "ok", + token_id: BOOTSTRAP_TOKEN_ID, + rotated: rotatedFromExisting, }); - throw new Error("bootstrap upsert WATCH retry exhausted"); } /** @param {AuthLogger} logAuth @param {string | null} requestId @param {string} command @param {unknown} err @returns {never} */ diff --git a/control/handlers/delete.js b/control/handlers/delete.js index 1af3512..ef4786e 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -240,7 +240,7 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { onExhausted: () => throwWholeDeleteContention(ns, name), } ); - await assertWorkflowDeleteAllowed({ ns, worker: name }); + await assertWorkflowDeleteAllowed({ ns, worker: name, requestId }); } catch (err) { if (err instanceof WholeDeleteError) { log(err.status >= 500 ? "error" : "warn", "worker_dry_run_rejected", { @@ -330,7 +330,7 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log, assertActiveVersionRetained(collected); let workflowBlocker = null; try { - await assertWorkflowDeleteAllowed({ ns, worker: name, allowCleanup: true }); + await assertWorkflowDeleteAllowed({ ns, worker: name, allowCleanup: true, requestId }); } catch (err) { if (err instanceof ControlAbort && err.code === "workflow_instances_active") { workflowBlocker = err; @@ -534,7 +534,7 @@ async function deleteResidualDoRedis(redis, collected, lockToken) { async function cleanupDoAlarmsOrWarn({ ns, worker, doStorageId, requestId, log }) { if (!doStorageId) return; try { - await cleanupDoAlarmsForWorker({ ns, worker, doStorageId }); + await cleanupDoAlarmsForWorker({ ns, worker, doStorageId, requestId }); } catch (err) { log("warn", "worker_do_alarm_cleanup_failed", { request_id: requestId, diff --git a/control/handlers/versions.js b/control/handlers/versions.js index 7eac8cc..7073ef9 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -91,7 +91,9 @@ async function handleDelete({ ns, name, version, principal, requestId }) { try { let result; try { - await assertWorkflowDeleteAllowed({ ns, worker: name, version, allowCleanup: true }); + await assertWorkflowDeleteAllowed({ + ns, worker: name, version, allowCleanup: true, requestId, + }); if (!await renewDeleteLock(redis, ns, name, lockToken)) { throw new VersionDeleteError(409, "deleting", { namespace: ns, name, version, diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index c9fa80c..8ee32d6 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -483,9 +483,9 @@ async function callWorkflowsRust(endpoint, body) { const { response, body: parsed } = await postWorkflowsInternal({ endpoint: `workflows/${endpoint}`, body, + requestId: body.requestId || null, logEvent: "workflow_backend_request_failed", logFields: { - request_id: body.requestId || null, endpoint, }, timeoutMs: null, diff --git a/control/lib.js b/control/lib.js index a3b8b5f..88e63db 100644 --- a/control/lib.js +++ b/control/lib.js @@ -18,8 +18,20 @@ import { } from "shared-ns-pattern"; import { PLATFORM_TIER_RESERVED_NS, ROLES } from "shared-auth-roles"; import { errorMessage } from "shared-errors"; -import { deleteLockKey, doStorageIdKey, routesKey, workerVersionsKey } from "shared-version"; -export { deleteLockKey, doStorageIdKey, routesKey, workerVersionsKey }; +import { + deleteLockKey, + doOwnerScopeScanPatternForStorage, + doStorageIdKey, + routesKey, + workerVersionsKey, +} from "shared-version"; +export { + deleteLockKey, + doOwnerScopeScanPatternForStorage, + doStorageIdKey, + routesKey, + workerVersionsKey, +}; export { validateModulePath }; export { WORKER_NAME_RE }; export { WORKFLOW_NAME_RE }; @@ -379,11 +391,6 @@ export function d1DatabaseTombstonesKey(ns) { return `d1:database-tombstones:${ns}`; } -/** @param {string} storageId */ -export function doOwnerScopeScanPatternForStorage(storageId) { - return `do:owner:scope:${encodeURIComponent(`${storageId}:`)}*`; -} - /** @param {string} storageId */ export function doObjectRegistryKey(storageId) { return `do:objects:${encodeURIComponent(storageId)}`; diff --git a/control/routing.js b/control/routing.js index d1c3297..58dd436 100644 --- a/control/routing.js +++ b/control/routing.js @@ -24,6 +24,8 @@ import { decodePatternProjection, encodePatternProjection } from "shared-route-p import { DECLARED_HOSTS_KEY, NAMESPACES_KEY, + PATTERNS_CHANNEL, + ROUTES_CHANNEL, bundleKey, formatVersion, hostDeclarationsKey, @@ -49,9 +51,6 @@ import { } from "control-routing-route-plan"; const MAX_ATTEMPTS = 5; -const PATTERNS_CHANNEL = "patterns:invalidate"; -const ROUTES_CHANNEL = "routes:invalidate"; - /** * @typedef {import("shared-route-projection").PatternProjection} PatternProjection * @typedef {Pick & { host: string, slot: string }} RoutePattern diff --git a/control/shared.js b/control/shared.js index deb567b..7b45265 100644 --- a/control/shared.js +++ b/control/shared.js @@ -15,6 +15,9 @@ import { DECLARED_HOSTS_KEY, HOST_DECLARATIONS_SCAN_PATTERN, HOSTS_SCAN_PATTERN, + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + ROUTES_FLUSH_CHANNEL, deleteLockKey, formatDeleteLockToken, hostDeclarationsKey, @@ -55,9 +58,7 @@ import { renewTokenLock, } from "shared-redis-lock"; -export const ROUTES_CHANNEL = "routes:invalidate"; -export const ROUTES_FLUSH_CHANNEL = "routes:flush"; -export const PATTERNS_CHANNEL = "patterns:invalidate"; +export { PATTERNS_CHANNEL, ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL }; /** * @typedef {import("shared-observability").RedisCommandEvent} RedisCommandEvent @@ -542,9 +543,9 @@ export async function recordCleanupIntentOrWarn({ } /** - * @param {{ ns: string, worker: string, version?: string, allowCleanup?: boolean }} args + * @param {{ ns: string, worker: string, version?: string, allowCleanup?: boolean, requestId?: string | null }} args */ -export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefined, allowCleanup = false }) { +export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefined, allowCleanup = false, requestId = null }) { const context = { namespace: ns, worker, @@ -558,6 +559,7 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi ...(version ? { version } : {}), ...(allowCleanup ? { allowCleanup: true } : {}), }, + requestId, logEvent: "workflow_lifecycle_check_failed", logFields: context, errorDetails: context, @@ -595,13 +597,14 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi } /** - * @param {{ ns: string, worker: string, doStorageId: string }} args + * @param {{ ns: string, worker: string, doStorageId: string, requestId?: string | null }} args */ -export async function cleanupDoAlarmsForWorker({ ns, worker, doStorageId }) { +export async function cleanupDoAlarmsForWorker({ ns, worker, doStorageId, requestId = null }) { const context = { namespace: ns, worker }; const { response, body } = await postWorkflowsInternal({ endpoint: "workflows/do-alarms/cleanup-worker", body: { ns, worker, doStorageId }, + requestId, logEvent: "workflow_do_alarm_cleanup_failed", logFields: context, errorDetails: context, diff --git a/control/workflows-client.js b/control/workflows-client.js index b4dcaf8..717ab50 100644 --- a/control/workflows-client.js +++ b/control/workflows-client.js @@ -15,6 +15,7 @@ export const WORKFLOWS_INTERNAL_TIMEOUT_MS = 5_000; * headers: () => HeadersInit, * endpoint: string, * body: unknown, + * requestId?: string | null, * log?: WorkflowClientLogger | null, * logEvent: string, * logFields?: Record, @@ -28,6 +29,7 @@ export async function postWorkflowsInternalRequest({ headers, endpoint, body, + requestId = null, log = null, logEvent, logFields = {}, @@ -39,9 +41,13 @@ export async function postWorkflowsInternalRequest({ } try { + const requestHeaders = new Headers(headers()); + if (typeof requestId === "string" && requestId) { + requestHeaders.set("x-request-id", requestId); + } const response = await workflows.fetch(`http://workflows/internal/${endpoint}`, { method: "POST", - headers: headers(), + headers: requestHeaders, body: JSON.stringify(body), ...(timeoutMs === null ? {} : { signal: AbortSignal.timeout(timeoutMs) }), }); @@ -52,6 +58,7 @@ export async function postWorkflowsInternalRequest({ } catch (err) { log?.("error", logEvent, { ...logFields, + ...(typeof requestId === "string" && requestId ? { request_id: requestId } : {}), error_message: errorMessage(err), }); throw makeError("request_failed"); @@ -70,6 +77,7 @@ export function createPostWorkflowsInternal({ getWorkflows, headers, getLog = () * @param {{ * endpoint: string, * body: unknown, + * requestId?: string | null, * logEvent: string, * logFields?: Record, * errorDetails?: Record, @@ -81,6 +89,7 @@ export function createPostWorkflowsInternal({ getWorkflows, headers, getLog = () return async function postWorkflowsInternal({ endpoint, body, + requestId = null, logEvent, logFields = {}, errorDetails = {}, @@ -93,6 +102,7 @@ export function createPostWorkflowsInternal({ getWorkflows, headers, getLog = () headers, endpoint, body, + requestId, log: getLog(), logEvent, logFields, diff --git a/do-runtime/alarm-dispatch.js b/do-runtime/alarm-dispatch.js index 355e4e2..0beadfa 100644 --- a/do-runtime/alarm-dispatch.js +++ b/do-runtime/alarm-dispatch.js @@ -1,6 +1,4 @@ -import { formatWorkerId } from "shared-worker-id"; -import { parseVersion } from "shared-version"; -import { DoRuntimeError, hostIdForObject, nonEmptyAlarmString, readJsonBody } from "do-runtime-protocol"; +import { DoRuntimeError, normalizeDoInvokeRequest, readJsonBody } from "do-runtime-protocol"; import { json } from "do-runtime-http"; /** @@ -9,35 +7,6 @@ import { json } from "do-runtime-http"; * @typedef {(env: DoEnv, invoke: DoInvoke, requestId?: string | null) => Promise} AlarmDispatcher */ -/** - * @param {unknown} value - * @param {string} field - */ -function requiredAlarmString(value, field) { - const text = nonEmptyAlarmString(value); - if (text === null) { - throw new DoRuntimeError(400, "invalid_do_alarm_dispatch", `DO alarm ${field} must be a non-empty string`); - } - return text; -} - -/** @param {unknown} value */ -function requiredAlarmVersion(value) { - const version = requiredAlarmString(value, "version"); - if (parseVersion(version) == null) { - throw new DoRuntimeError(400, "invalid_do_alarm_dispatch", "DO alarm version is invalid"); - } - return version; -} - -/** @param {unknown} value */ -function alarmRetryCount(value) { - if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { - throw new DoRuntimeError(400, "invalid_do_alarm_dispatch", "DO alarm retryCount must be a non-negative integer"); - } - return value; -} - /** * @param {Request} request * @param {DoEnv} env @@ -49,38 +18,25 @@ export async function handleAlarmDispatch(request, env, dispatchInvoke, requestI const record = body && typeof body === "object" && !Array.isArray(body) ? /** @type {Record} */ (body) : {}; - const ns = requiredAlarmString(record.ns, "ns"); - const worker = requiredAlarmString(record.worker, "worker"); - const version = requiredAlarmVersion(record.version); - const doStorageId = requiredAlarmString(record.doStorageId, "doStorageId"); - const className = requiredAlarmString(record.className, "className"); - const objectName = requiredAlarmString(record.objectName, "objectName"); - const retryCount = alarmRetryCount(record.retryCount); - const token = requiredAlarmString(record.token, "token"); - /** @type {DoInvoke} */ - const invoke = { + if (record.retryCount == null) { + throw new DoRuntimeError(400, "invalid_request", "alarm.retryCount is required"); + } + const invoke = normalizeDoInvokeRequest({ kind: "alarm", - ns, - worker, - version, - doStorageId, - workerId: formatWorkerId({ namespace: ns, worker, version }), - hostId: hostIdForObject(doStorageId, className, objectName), - className, - objectName, - props: { - ns, - worker, - version, - doStorageId, - className, - }, + ns: record.ns, + worker: record.worker, + version: record.version, + doStorageId: record.doStorageId, + className: record.className, + objectName: record.objectName, alarm: { - retryCount, - isRetry: retryCount > 0, - token, + retryCount: record.retryCount, + token: record.token, }, - }; + }); + if (invoke.kind !== "alarm" || invoke.alarm.token == null) { + throw new DoRuntimeError(400, "invalid_request", "alarm.token is required"); + } const response = await dispatchInvoke(env, invoke, requestId); const text = await response.text(); if (!response.ok) { diff --git a/do-runtime/alarm.js b/do-runtime/alarm.js index ee77e6c..4672a65 100644 --- a/do-runtime/alarm.js +++ b/do-runtime/alarm.js @@ -1,4 +1,8 @@ -import { DoRuntimeError, nonEmptyAlarmString } from "do-runtime-protocol"; +import { + DoRuntimeError, + isWellFormedUnicodeString, + nonEmptyAlarmString, +} from "do-runtime-protocol"; import { errorMessage } from "shared-errors"; import { withInternalAuth } from "shared-internal-auth"; @@ -38,6 +42,9 @@ function requiredString(value, field) { if (text === null) { throw new TypeError(`DO alarm ${field} must be a non-empty string`); } + if (!isWellFormedUnicodeString(text)) { + throw new TypeError(`DO alarm ${field} must contain well-formed Unicode`); + } return text; } diff --git a/do-runtime/owner-registry.js b/do-runtime/owner-registry.js index 8a64fc5..c87d3b2 100644 --- a/do-runtime/owner-registry.js +++ b/do-runtime/owner-registry.js @@ -44,6 +44,7 @@ import { stageOwnerRenew, } from "shared-owner-protocol"; import { + DO_OWNER_SCOPE_PREFIX, VERSION_DELETE_LOCK_KIND, deleteLockKey, doStorageIdKey, @@ -54,7 +55,6 @@ const DEFAULT_OWNER_TTL_SECONDS = 120; const DEFAULT_RENEW_CONCURRENCY = 8; const MAX_RENEW_CONCURRENCY = 64; const DEFAULT_OWNER_LEASE_GUARD_MS = 1_000; -const OWNER_PREFIX = "do:owner:scope:"; const OWNER_CLAIM_RETRIES = 3; const OWNER_RENEW_FRACTION = 0.5; const DO_OWNER_PORT = 8788; @@ -94,12 +94,12 @@ export function renewConcurrency(env) { /** @param {string} ownerKey */ export function ownerKeyOf(ownerKey) { - return ownerProtocolKeys(OWNER_PREFIX, ownerKey).ownerKey; + return ownerProtocolKeys(DO_OWNER_SCOPE_PREFIX, ownerKey).ownerKey; } /** @param {string} ownerKey */ export function ownerGenerationKeyOf(ownerKey) { - return ownerProtocolKeys(OWNER_PREFIX, ownerKey).generationKey; + return ownerProtocolKeys(DO_OWNER_SCOPE_PREFIX, ownerKey).generationKey; } /** @param {unknown} value @returns {value is BundleScope & Record} */ diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index 3132e5f..c117df0 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -8,7 +8,11 @@ import { MAX_ID_BYTES, } from "do-runtime-protocol-wire-grammar"; import { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; -import { hostIdForObject, shardForObjectName } from "do-runtime-protocol-identity"; +import { + hostIdForObject, + isWellFormedUnicodeString, + shardForObjectName, +} from "do-runtime-protocol-identity"; import { formatWorkerId } from "shared-worker-id"; import { BodyTooLargeError, @@ -21,7 +25,12 @@ import { parseVersion } from "shared-version"; export { DO_HOST_SHARD_COUNT } from "do-runtime-protocol-wire-grammar"; export { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; -export { hostIdForObject, hostIdForShard, shardForObjectName } from "do-runtime-protocol-identity"; +export { + hostIdForObject, + hostIdForShard, + isWellFormedUnicodeString, + shardForObjectName, +} from "do-runtime-protocol-identity"; export const DO_OWNERSHIP_CODE = Object.freeze({ OWNER_CLAIM_RACED: "owner_claim_raced", @@ -200,6 +209,9 @@ function requireString(value, field, { maxBytes = MAX_ID_BYTES, pattern = null } if (typeof value !== "string" || value.length === 0) { throw new DoRuntimeError(400, "invalid_request", `${field} must be a non-empty string`); } + if (!isWellFormedUnicodeString(value)) { + throw new DoRuntimeError(400, "invalid_request", `${field} must contain well-formed Unicode`); + } if (byteLength(value) > maxBytes) { throw new DoRuntimeError(400, "invalid_request", `${field} is too large`); } @@ -395,8 +407,12 @@ function normalizeInvokeKind(value) { */ function normalizeAlarmInfo(value) { const input = value == null ? {} : requireRecord(value, "alarm"); - const retryCount = input.retryCount == null ? 0 : Number(input.retryCount); - if (!Number.isInteger(retryCount) || retryCount < 0) { + const retryCount = input.retryCount == null ? 0 : input.retryCount; + if ( + typeof retryCount !== "number" || + !Number.isInteger(retryCount) || + retryCount < 0 + ) { throw new DoRuntimeError(400, "invalid_request", "alarm.retryCount must be a non-negative integer"); } const token = input.token == null ? undefined : requireString(input.token, "alarm.token"); diff --git a/do-runtime/protocol/identity.js b/do-runtime/protocol/identity.js index 722e883..353b63d 100644 --- a/do-runtime/protocol/identity.js +++ b/do-runtime/protocol/identity.js @@ -9,6 +9,14 @@ import { DoRuntimeError } from "do-runtime-protocol-errors"; import { fnv1a32Utf8 } from "shared-fnv1a32"; const utf8Encoder = new TextEncoder(); +const intrinsicReflectApply = Reflect.apply; +const intrinsicStringIsWellFormed = String.prototype.isWellFormed; + +/** @param {unknown} value */ +export function isWellFormedUnicodeString(value) { + return typeof value === "string" && + intrinsicReflectApply(intrinsicStringIsWellFormed, value, []); +} /** @param {string} value */ function byteLength(value) { @@ -18,12 +26,15 @@ function byteLength(value) { /** * @param {unknown} value * @param {string} field - * @param {RegExp} pattern + * @param {RegExp | null} pattern */ function requireIdentityString(value, field, pattern) { if (typeof value !== "string" || value.length === 0) { throw new DoRuntimeError(400, "invalid_request", `${field} must be a non-empty string`); } + if (!isWellFormedUnicodeString(value)) { + throw new DoRuntimeError(400, "invalid_request", `${field} must contain well-formed Unicode`); + } if (byteLength(value) > MAX_ID_BYTES) { throw new DoRuntimeError(400, "invalid_request", `${field} is too large`); } @@ -33,7 +44,7 @@ function requireIdentityString(value, field, pattern) { throw new DoRuntimeError(400, "invalid_request", `${field} must not contain control characters`); } } - if (!pattern.test(value)) { + if (pattern && !pattern.test(value)) { throw new DoRuntimeError(400, "invalid_request", `${field} is not valid`); } } @@ -43,11 +54,12 @@ function requireIdentityString(value, field, pattern) { * @param {number} [shardCount] */ export function shardForObjectName(objectName, shardCount = DO_HOST_SHARD_COUNT) { + requireIdentityString(objectName, "objectName", null); const count = Number(shardCount); if (!Number.isInteger(count) || count <= 0) { throw new DoRuntimeError(500, "invalid_shard_count", "DO host shard count must be a positive integer"); } - return fnv1a32Utf8(String(objectName)) % count; + return fnv1a32Utf8(objectName) % count; } /** diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index a3e14d7..d7d73d3 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -240,10 +240,11 @@ Auth-specific contract: non-diagnostic action is `auth.delegated_token.issue`. A delegated issue request accepts only `{ template }`; caller-provided `kind`, `ns`, `label`, `expiresAt`, or template allowlist fields are - rejected. Auth re-reads the issuer token record, verifies it is active and has the - template allowlist entry, applies the code-defined Auth template registry, then - writes a target token with `created_by`, `issue_template`, and - `issue_template_version` metadata. Active quota is a live credential capacity guard + rejected. Auth re-reads the issuer token record under the final issue-lock WATCH, + verifies it is active and has the template allowlist entry, applies the code-defined + Auth template registry, then writes a target token with `created_by`, + `issue_template`, and `issue_template_version` metadata. Active quota is a live + credential capacity guard based on `created_by + issue_template + expires_at`; it is not an environment quota. Generated namespaces are ordinary tenant namespace strings embedded in the issued token record. Auth serializes issue for each issuer/template with diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 68b809e..bccc26f 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -127,7 +127,7 @@ Auth 子合同： - Reserved namespace check 同时存在于 route/auth red line 和 role scope check；两者不能互相替代。 - `issue` 可以签发 `ns`、`platform`、`platform-observer` 和 `ops-observer` token。`token-issuer` 也可由 ops direct issue，但必须提供 camelCase `issueTemplates`，且其中每个 id 都要指向已有 delegated issue template。Auth 把这个 allowlist 存成 Redis `issue_templates` JSON array string，并拒绝 public `issue_templates` 输入，避免 API shape 和 storage shape 混用。`ops` 只能来自 bootstrap。Token plaintext 只生成一次、展示一次，并且只以 SHA-256 hash 形式存储。 - 内置 delegated issue template 包括用于 workshop pool 的 `wdl-chat-ns-pool`，以及用于 hosted CLI live integration 短期 namespace 的 `wdl-cli-integration`。 -- `token-issuer` 是 unbound role；除 `/whoami` self-introspection 外，它唯一 non-diagnostic action 是 `auth.delegated_token.issue`。Delegated issue 请求只接受 `{ template }`；caller-provided `kind`、`ns`、`label`、`expiresAt` 或 template allowlist 字段都会被拒。Auth 会重新读取 issuer token record，确认它仍 active 且 allowlist 包含该 template，再应用 Auth 代码内定义的 template registry，并写入带 `created_by`、`issue_template`、`issue_template_version` metadata 的目标 token。Active quota 基于 `created_by + issue_template + expires_at` 计算，是 live credential capacity guard，不是 environment quota。生成的 namespace 是写入 token record 的普通 tenant namespace 字符串。Auth 会先用 `auth:delegated-issue-lock::` 串行化同一 issuer/template 的发放和 quota 计数，然后拒绝已经出现在 control 维护的 `namespaces` set 或任意已扫描 token record stored `ns` 中的 generated namespace candidate，不区分 token kind、issuer、template、是否过期或是否撤销。Quota 计算 fail-closed：同一 issuer/template 的 delegated token record 损坏会阻止新的 delegated issue，直到 storage contract violation 被修复。`GET /auth/tokens` 是 operator repair surface；单条 malformed `issue_templates` 会作为 invalid entry 列出，不会拖垮整个 list。V1 的 delegated namespace collision check 是 best-effort：unbound/full-plane token 仍能执行不会留下 auth-visible `ns` token record 的 namespace-scoped 写入，而 `namespaces` 只表示 active worker。常规 delegated namespace workflow 应使用 namespace-bound credential 做 namespace-scoped 写；未来的持久 namespace fact index 才是这个 residual risk 的完整修复。 +- `token-issuer` 是 unbound role；除 `/whoami` self-introspection 外，它唯一 non-diagnostic action 是 `auth.delegated_token.issue`。Delegated issue 请求只接受 `{ template }`；caller-provided `kind`、`ns`、`label`、`expiresAt` 或 template allowlist 字段都会被拒。Auth 会在最终 issue-lock WATCH 内重新读取 issuer token record，确认它仍 active 且 allowlist 包含该 template，再应用 Auth 代码内定义的 template registry，并写入带 `created_by`、`issue_template`、`issue_template_version` metadata 的目标 token。Active quota 基于 `created_by + issue_template + expires_at` 计算，是 live credential capacity guard，不是 environment quota。生成的 namespace 是写入 token record 的普通 tenant namespace 字符串。Auth 会先用 `auth:delegated-issue-lock::` 串行化同一 issuer/template 的发放和 quota 计数，然后拒绝已经出现在 control 维护的 `namespaces` set 或任意已扫描 token record stored `ns` 中的 generated namespace candidate，不区分 token kind、issuer、template、是否过期或是否撤销。Quota 计算 fail-closed：同一 issuer/template 的 delegated token record 损坏会阻止新的 delegated issue，直到 storage contract violation 被修复。`GET /auth/tokens` 是 operator repair surface；单条 malformed `issue_templates` 会作为 invalid entry 列出，不会拖垮整个 list。V1 的 delegated namespace collision check 是 best-effort：unbound/full-plane token 仍能执行不会留下 auth-visible `ns` token record 的 namespace-scoped 写入，而 `namespaces` 只表示 active worker。常规 delegated namespace workflow 应使用 namespace-bound credential 做 namespace-scoped 写；未来的持久 namespace fact index 才是这个 residual risk 的完整修复。 - `expiresAt` 必须是严格 ISO-8601 UTC、毫秒精度，并通过真实日历 round trip 校验。过期 token 在过期后第一次 verify 时 lazy collect：删除 active hash index，并写入 `expired_at`；主动 revoke 使用 `revoked_at`。 - 保留 token id `bootstrap` 是 infra-managed ops token。`BOOTSTRAP_TOKEN` 在 auth cold start 时 upsert，`revoke("bootstrap")` 被禁止，轮换方式是更新环境变量并 redeploy。Verify 会缓存已 ensure 的 bootstrap hash，但 Redis 丢失后可以重新 ensure，因此 bootstrap token 能恢复被 flush 的 auth store。 diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 2dc883f..4cb2d30 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -76,6 +76,9 @@ host facade establishes invocation-local context without adding platform metadat the tenant argument list. DO invoke envelopes identify persisted bundles by canonical namespace, worker, version, and storage id. They do not accept inline worker source. +Tenant-facing DO object names and ids must be well-formed Unicode strings. Lone UTF-16 +surrogates are rejected by `idFromName()` / `idFromString()` before hashing or dispatch; +do-runtime alarm ingress and Workflows revalidation enforce the same boundary. DO host ids are capped at 512 UTF-8 bytes and use canonical `shardN` suffixes without leading zeroes. diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index dc92528..c0a1a66 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -38,6 +38,7 @@ Tenant-originated DO fetch body 在 runtime facade 中限制为 1 MiB。Facade DO RPC method name 使用 JavaScript identifier grammar，并由 runtime client 和 do-runtime protocol reader 同时限制为最多 256 ASCII bytes。RPC 参数是最多 1 MiB 的 structural JSON data：接受 finite number、string、boolean、null、dense array 和 plain object。序列化不会调用 `toJSON()` hook；sparse array、circular structure、non-plain object 和 non-JSON value 会在 dispatch 前失败。 do-runtime 会通过 generated wrapper 截获的私有 fetch dispatch 调用 tenant alarm 和 RPC method；这些 request 携带外层 request id，使 host facade 建立 invocation-local context，且不会把平台 metadata 加入 tenant argument list。 DO invoke envelope 通过 canonical namespace、worker、version 和 storage id 标识 persisted bundle，不接受 inline worker source。 +Tenant-facing DO object name/id 必须是 well-formed Unicode string。`idFromName()` / `idFromString()` 会在 hash 或 dispatch 前拒绝 lone UTF-16 surrogate；do-runtime alarm ingress 和 Workflows revalidation 执行相同边界。 DO host id 最多 512 UTF-8 bytes，并使用不带前导零的 canonical `shardN` suffix。 ## Redis / Storage 合同 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index 2aaa85b..d6a63a7 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -137,7 +137,7 @@ Central owners: Common rules: -- `x-request-id` propagates across gateway, control/runtime, loaded workers, and D1 +- `x-request-id` propagates across gateway, control/runtime, loaded workers, D1, and Workflows where possible. Missing inbound ids are minted at ingress. Header adapters consider only the first comma-delimited token; ids containing bytes outside visible ASCII, quotes, backslashes, or more than 128 bytes are treated as absent. JS entrypoints using diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index 46913e5..e81f423 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -91,7 +91,7 @@ WDL 在 JS workerd tier 和 Rust 服务中使用同一套可观测性策略： 通用规则： -- `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker 和 D1 传播。缺失的 inbound id 会在入口生成；header adapter 只考虑第一个逗号分隔 token，包含 visible ASCII 以外字节、引号、反斜杠或超过 128 字节的 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 +- `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker、D1 和 Workflows 传播。缺失的 inbound id 会在入口生成；header adapter 只考虑第一个逗号分隔 token，包含 visible ASCII 以外字节、引号、反斜杠或超过 128 字节的 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 - Loaded-worker host facade 通过 invocation-local ALS 在 native Promise 和 `async` continuation 中保留 request id；do-runtime 的私有 fetch、alarm 和 RPC dispatch 会在 `x-request-id` 中携带该 id。Custom thenable 不属于受支持的传播边界；wrapper 不会为延长 context 而检查或替换 tenant 返回值。没有新 id 的嵌套 handler 调用会继承当前 ALS context。 - 日志字段使用 snake_case。只有 `level=error` 写入 stderr；debug/info/warn JSON log line 都写入 stdout，这样 JS、Rust 和 embedded workerd shim 的日志路由保持一致。 - 内部运维日志，包括 system-worker cleanup 日志和防御性的 Redis callback warning，也使用同一套单行 JSON envelope：`ts`、`service`、`level`、`event`，再加 snake_case 字段。JS 服务使用 `shared/observability.js`；Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`；无法转成文本的 throwable 降级为 `Unknown error`，不能反向覆盖原始失败。不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 diff --git a/docs/modules/workflows.md b/docs/modules/workflows.md index 7dcc4e9..bcc9616 100644 --- a/docs/modules/workflows.md +++ b/docs/modules/workflows.md @@ -133,9 +133,12 @@ Key families: - Scheduler also wakes Workflows-owned internal DO alarm jobs through the same `/internal/workflows/tick` endpoint; scheduler never reads or writes DO alarm state directly. -- Workflows rejects non-canonical worker versions before persisting DO alarm jobs, - revalidates persisted alarm identity before dispatch, and validates an active route - version before using it as a retarget. These checks use the `wdl-rust-common` owners. +- Workflows rejects non-canonical DO alarm identity before persisting jobs, revalidates + persisted alarm identity before dispatch, and validates an active route + version before using it as a retarget. Namespace, worker, and version checks reuse + `wdl-rust-common`; do-runtime protocol grammar and identity helpers own the canonical + alarm-specific fields and aggregate 512-byte DO host-id contract. Workflows mirrors + and revalidates that contract before persistence and dispatch. Runtime run dispatch and progress callbacks share one system-vs-user runtime endpoint selector inside the workflows crate. - 32 scheduling shards partition ready/due work. @@ -159,6 +162,8 @@ Key families: (`step.sleep`, `step.sleepUntil`, `step.waitForEvent`) remain exclusive and must not overlap another in-flight step because they suspend the whole workflow run. - Termination is an explicit non-success terminal outcome and uses error retention. +- `Workflow.createBatch()` accepts at most 100 entries per call. Runtime prevalidation + and Rust admission share this pinned limit. - A single workflow result is capped at 1 MiB and a runtime-to-workflows backend JSON request at 2 MiB. Runtime prevalidation and the Rust backend share the pinned `workflow_payload_too_large` contract. The per-instance aggregate payload cap is diff --git a/docs/modules/workflows.zh.md b/docs/modules/workflows.zh.md index f5403ba..3b2f1aa 100644 --- a/docs/modules/workflows.zh.md +++ b/docs/modules/workflows.zh.md @@ -83,13 +83,14 @@ Key families： - Instance 冻结创建时的 worker version/class identity。 - Scheduler 只负责唤醒 workflows；admission、fairness、shard tick、ready/due movement 和 runtime dispatch 都由 workflows 负责。 - Scheduler 也通过同一个 `/internal/workflows/tick` endpoint 唤醒 Workflows-owned internal DO alarm jobs；scheduler 不直接读写 DO alarm state。 -- Workflows 在持久化 DO alarm job 前拒绝 non-canonical worker version，在 dispatch 前重新校验持久化 alarm identity，并在把 active route 用作 retarget 前校验其 version。这些检查共用 `wdl-rust-common` owner。Runtime run dispatch 与 progress callback 在 workflows crate 内共用同一个 system-vs-user runtime endpoint selector。 +- Workflows 在持久化 DO alarm job 前拒绝 non-canonical alarm identity，在 dispatch 前重新校验持久化 alarm identity，并在把 active route 用作 retarget 前校验其 version。其中 namespace、worker、version 校验复用 `wdl-rust-common`；do-runtime protocol grammar 与 identity helper 拥有 canonical alarm-specific field 和 aggregate 512-byte DO host-id 合同，Workflows 在持久化和 dispatch 前镜像并重新校验该合同。Runtime run dispatch 与 progress callback 在 workflows crate 内共用同一个 system-vs-user runtime endpoint selector。 - 32 个 scheduling shards 划分 ready/due work。 - Ready token 是去重 hint；instance hash state 是权威状态。 - Execution commit 同时用 `generation`、`runToken`、active instance status 和未过期 run lease fence。Step commit/register 接受同一 run 的 `running` 或 `waiting` 状态，因此一个并行 sibling 进入 retry/wait 后，另一个 sibling 仍可完成；completed runtime terminal 要求 `running`，failed runtime terminal 在 run lease 仍有效时也可以关闭由非法未 await suspending step 造成的同一 run `waiting` 状态。如果 lease 已过期，workflows 只恢复 ready hint，让下一次 claim 在新 lease 下 replay。Lifecycle commit 只用 generation fence，并在同一个 Lua commit 内 rotate `generation`。 - Runtime replay cache 只是 advisory。DB 2 step state 是权威。 - Runtime 可以并发发起多个 `step.do`，常见形式是 `Promise.all`；每次调用按用户代码调用顺序分配 deterministic ordinal，从当前已完成 step frontier 记录 DAG dependencies，并在 run fence 下独立 commit。`step.do` callback 不能启动另一个 workflow step，即使在 callback 的 `await` 之后也不允许；并行 sibling promise 应在 run body 中、callback 代码进入 in-flight 之前创建。如果 run 在已启动 step settle 前返回，会按 invalid run 失败，所以用户代码必须 await 并发 step promise。Suspending operation（`step.sleep`、`step.sleepUntil`、`step.waitForEvent`）仍保持互斥，不能和其它 in-flight step 重叠，因为它们会 suspend 整个 workflow run。 - Termination 是显式 non-success terminal outcome，使用 error retention。 +- `Workflow.createBatch()` 每次调用最多接受 100 项；Runtime prevalidation 与 Rust admission 共享这项 pinned limit。 - 单个 workflow result 的上限是 1 MiB，runtime-to-workflows backend JSON request 的上限是 2 MiB。Runtime prevalidation 和 Rust backend 共享 pinned `workflow_payload_too_large` contract。每个 instance 的 aggregate payload cap 是 16 MiB。Step/event 超 cap 写入会让请求失败；runtime terminal result 超 cap 会在同一事务内把 instance 转成 failed。 - Workflows 语义 request cap 使用 `request_too_large`；它不同于 control/runtime 协议中的 HTTP-body parser `request_body_too_large`。除此之外，HTTP 边界上的 workflow error 使用平台 `{ error, message }` envelope。Client-facing proxy 应把 workflows 5xx 当作 backend/platform failure，不应依赖 response body 中的 raw backend diagnostic message。 diff --git a/docs/source-map.md b/docs/source-map.md index 9486814..ba64d53 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -59,7 +59,7 @@ are outside this map unless they own runtime or deployable service behavior. |---|---| | `shared/redis.js`, `shared/redis-*.js` | Public Redis import surface plus split RESP codec, per-call client, WATCH/MULTI session, and subscriber loop modules. Runtime hot paths prefer the Rust redis-proxy sidecar. | | `shared/redis-lock.js` | Token-fenced Redis lock creation, acquire, renewal, and best-effort token-scoped release shared by Control and Auth. | -| `shared/optimistic-retry.js` | Generic bounded optimistic retry loop used by Control and the D1/DO owner-lease adapters. | +| `shared/optimistic-retry.js` | Generic bounded optimistic retry loop used by Control, Auth, and the D1/DO owner-lease adapters. | | `shared/owner-endpoint.js`, `shared/owner-lease.js`, `shared/owner-protocol.js`, `shared/owner-forwarder.js` | Shared owner endpoint grammar, lease parsing, generation counters, key derivation, fence matching, staged Redis owner writes, and authenticated forwarding mechanics used by Control and the D1/DO runtimes. | | `shared/auth-roles.js` | Role table, principal validation, reserved namespace policy, and auth action capabilities. | | `shared/auth-token.js` | Shared `x-admin-token` sanitizer used by control and auth. | @@ -71,7 +71,7 @@ are outside this map unless they own runtime or deployable service behavior. | `shared/respond.js` | Shared HTTP response, JSON error, Prometheus text, best-effort response body discard, and `x-request-id` echo helpers. | | `shared/bounded-body.js` | Shared bounded byte-stream and request-body readers; each tier maps limit errors to its own contract. | | `shared/ns-pattern.js` | Platform-domain normalization plus namespace, worker, binding, queue, KV/D1/R2 id, module path, reserved object-key, and reserved namespace grammars. | -| `shared/version.js` | Worker version formatting plus worker, route-plane, and lifecycle Redis key helpers. | +| `shared/version.js` | Worker version formatting plus worker, route-plane, lifecycle, DO owner-scope key, and route-invalidation channel helpers. | | `shared/workerd-compat-flags.js` | Pinned mirror of upstream workerd experimental compatibility enable flags used to reject tenant metadata before cold-load. | | `shared/queue-keys.js` | JavaScript queue key helpers used by tests and cross-tier key-shape checks. | | `shared/route-projection.js` | Compact pattern-route projection encoding shared by control writers, delete checks, and gateway readers. | diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index 4d0b4b4..8f5b743 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -56,7 +56,7 @@ |---|---| | `shared/redis.js`、`shared/redis-*.js` | 公共 Redis import surface，以及拆分后的 RESP codec、per-call client、WATCH/MULTI session 和 subscriber loop modules。Runtime hot path 优先使用 Rust redis-proxy sidecar。 | | `shared/redis-lock.js` | Control 和 Auth 共用的 token-fenced Redis lock creation、acquire、renewal 和 best-effort token-scoped release。 | -| `shared/optimistic-retry.js` | Control 和 D1/DO owner-lease adapter 共用的通用有界 optimistic retry loop。 | +| `shared/optimistic-retry.js` | Control、Auth 和 D1/DO owner-lease adapter 共用的通用有界 optimistic retry loop。 | | `shared/owner-endpoint.js`、`shared/owner-lease.js`、`shared/owner-protocol.js`、`shared/owner-forwarder.js` | Control 与 D1/DO runtimes 共用的 owner endpoint grammar、owner lease parsing、generation counters、key derivation、fence matching、staged Redis owner writes 和 authenticated forwarding mechanics。 | | `shared/auth-roles.js` | Role table、principal validation、reserved namespace policy 和 auth action capabilities。 | | `shared/auth-token.js` | Control 和 auth 共用的 `x-admin-token` sanitizer。 | @@ -68,7 +68,7 @@ | `shared/respond.js` | 共享 HTTP response、JSON error、Prometheus text、best-effort response body discard 和 `x-request-id` echo helpers。 | | `shared/bounded-body.js` | 共享 bounded byte-stream 和 request-body readers；各 tier 自己把 limit error 映射为对应 contract。 | | `shared/ns-pattern.js` | Platform-domain normalization，以及 namespace、worker、binding、queue、KV/D1/R2 id、module path、reserved object-key 和 reserved namespace grammars。 | -| `shared/version.js` | Worker version formatting，以及 worker、route-plane 和 lifecycle Redis key helpers。 | +| `shared/version.js` | Worker version formatting，以及 worker、route-plane、lifecycle、DO owner-scope key 与 route invalidation channel helpers。 | | `shared/workerd-compat-flags.js` | 上游 workerd experimental compatibility enable flags 的 pinned mirror，用于在 cold-load 前拒绝 tenant metadata。 | | `shared/queue-keys.js` | JavaScript queue key helpers，供 tests 和 cross-tier key-shape checks 使用。 | | `shared/route-projection.js` | Control writer、delete check 和 gateway reader 共用的紧凑 pattern-route projection encoding。 | diff --git a/gateway/runtime.js b/gateway/runtime.js index 1699fdc..7804994 100644 --- a/gateway/runtime.js +++ b/gateway/runtime.js @@ -11,7 +11,15 @@ import { recordRedisCommand, } from "shared-observability"; import { isValidRouteNs } from "shared-ns-pattern"; -import { DECLARED_HOSTS_KEY, NAMESPACES_KEY, patternsKey, routesKey } from "shared-version"; +import { + DECLARED_HOSTS_KEY, + NAMESPACES_KEY, + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + ROUTES_FLUSH_CHANNEL, + patternsKey, + routesKey, +} from "shared-version"; import { isCanonicalPatternHost, sortPatterns } from "gateway-lib"; /** @type {Set | null} */ @@ -147,7 +155,7 @@ export function ensureGatewaySubscriber(redisAddr) { if (subscriber) return null; subscriber = new RedisSubscriber( redisAddr, - ["routes:invalidate", "routes:flush", "patterns:invalidate"], + [ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL, PATTERNS_CHANNEL], { onConnect: () => { subscriberConnected = 1; @@ -171,7 +179,7 @@ export function ensureGatewaySubscriber(redisAddr) { }, onMessage: (channel, payload) => { const value = utf8Decoder.decode(payload); - if (channel === "patterns:invalidate") { + if (channel === PATTERNS_CHANNEL) { if (value === "*") { clearPatternState(); } else if (isCanonicalPatternHost(value)) { @@ -191,14 +199,14 @@ export function ensureGatewaySubscriber(redisAddr) { log("info", "patterns_invalidated", { host: value }); return; } - if (channel === "routes:flush") { + if (channel === ROUTES_FLUSH_CHANNEL) { clearRouteState(); metrics.increment("subscriber_invalidations", { service: "gateway", scope: "all" }); log("info", "routes_invalidated_all", {}); return; } - // routes:invalidate owns routeCache + knownNs only; the patterns cache - // is invalidated exclusively via patterns:invalidate so channel + // The route channel owns routeCache + knownNs only; the patterns cache + // is invalidated exclusively via the pattern channel so channel // semantics stay orthogonal. if (!isValidRouteNs(value)) { log("warn", "routes_invalidation_ignored", { diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index e51df47..43abff1 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -186,7 +186,7 @@ export function doOwnerHintCacheKey(props, objectName) { } /** @param {Response} response */ -export async function staleDoOwnerHintResponse(response) { +export function staleDoOwnerHintResponse(response) { if (responseStatus(response) < 400) return false; const code = responseHeader(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER); return code !== null && setHas(OWNER_HINT_STALE_CODES, code); @@ -660,7 +660,7 @@ export async function rpcResultFromResponse(response) { } /** @param {Response} response */ -export async function retryableOwnerRaceResponse(response) { +export function retryableOwnerRaceResponse(response) { if (responseStatus(response) !== 503) return false; const code = responseHeader(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER); return code !== null && setHas(OWNER_RACE_RETRY_CODES, code); @@ -748,7 +748,7 @@ export function connectHeaders(props, objectName, request, requestId) { * @param {DoOwnerHint} owner * @param {string} pathname */ -export function ownerRequestUrl(owner, pathname) { +function ownerRequestUrl(owner, pathname) { if (!validOwnerEndpoint(owner.endpoint)) { throw new Error("Invalid DO owner endpoint"); } @@ -792,7 +792,7 @@ export function ownerHintFromHeaders(headers) { } /** @param {Response} response */ -export async function ownerHintFromResponse(response) { +function ownerHintFromResponse(response) { if (responseStatus(response) !== 409) return null; // Only do-runtime-authored headers are trusted. Tenant DO code controls the // response body and may intentionally return a do_owner_hint-shaped 409. @@ -806,25 +806,7 @@ function validOwnerEndpoint(endpoint) { return validOwnerEndpointForService(endpoint, 8788, "do-runtime"); } -/** - * @param {Response} response - * @param {DoFetch | typeof fetch | null | undefined} ownerFetch - * @param {string} pathname - * @param {RequestInit} init - */ -export async function followOwnerHint(response, ownerFetch, pathname, init) { - const hint = await ownerHintFromResponse(response); - if (!hint || typeof ownerFetch !== "function") return response; - cancelResponseBody(response); - const direct = await ownerFetch(ownerRequestUrl(hint, pathname), init); - if (ownerEndpointUnavailableResponse(direct)) { - cancelResponseBody(direct); - throw new Error(`DO owner endpoint returned ${responseStatus(direct)}`); - } - return direct; -} - -export function ownerUnavailableResponse() { +function ownerUnavailableResponse() { return Response.json( { error: "owner_unavailable", message: "DO owner is unavailable; request outcome may be unknown" }, { status: 503 } @@ -832,132 +814,97 @@ export function ownerUnavailableResponse() { } /** - * @param {{ - * routerFetch: DoFetch, - * routerUrl: string, - * ownerFetch: DoFetch | typeof fetch | null | undefined, - * ownerPath: string, - * init: RequestInit, - * cachedHint?: DoOwnerHint | null, - * rememberHint?: (hint: DoOwnerHint) => void, - * clearHint?: () => void, - * staleCachedResponse?: (response: Response) => Promise, - * bypassOwnerHintResponse?: (response: Response) => Promise, - * replayOwnerUnavailable?: boolean, - * }} options + * @param {DoOwnerHintDispatchOptions} options + * @param {boolean} retryOwnerRace */ -export async function dispatchDoRequestWithOwnerHint({ +async function dispatchDoWithHintCache({ routerFetch, routerUrl, ownerFetch, ownerPath, init, - cachedHint = null, - rememberHint = (_hint) => {}, - clearHint = () => {}, - staleCachedResponse = async (_response) => false, - bypassOwnerHintResponse = async (_response) => false, + cache, + hintKey, replayOwnerUnavailable = false, -}) { +}, retryOwnerRace) { + /** @param {DoOwnerHint} hint */ + const rememberHint = (hint) => cache.set(hintKey, hint); + const clearHint = () => cache.delete(hintKey); + const replayOrUnavailable = async () => replayOwnerUnavailable + ? await routerFetch(routerUrl, withoutOwnerHintOptIn(init)) + : ownerUnavailableResponse(); + /** @param {Response} response */ + const finish = async (response) => { + let result = response; + if (retryOwnerRace && retryableOwnerRaceResponse(result)) { + cancelResponseBody(result); + clearHint(); + result = await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); + } + return stripOwnerHintHeaders(result); + }; + + const cachedHint = /** @type {DoOwnerHint | null} */ (cache.get(hintKey)); if (cachedHint && typeof ownerFetch === "function") { + /** @type {Response} */ + let direct; try { - const direct = await ownerFetch(ownerRequestUrl(cachedHint, ownerPath), init); - if (await ownerHintFromResponse(direct)) { - cancelResponseBody(direct); - clearHint(); - } else if (await staleCachedResponse(direct)) { - cancelResponseBody(direct); - clearHint(); - } else if (ownerEndpointUnavailableResponse(direct)) { - cancelResponseBody(direct); - clearHint(); - if (replayOwnerUnavailable) { - return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); - } - return ownerUnavailableResponse(); - } else { - const learned = ownerHintFromHeaders(responseHeaders(direct)); - if (learned) rememberHint(learned); - return direct; - } + direct = await ownerFetch(ownerRequestUrl(cachedHint, ownerPath), init); } catch { clearHint(); - if (replayOwnerUnavailable) { - return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); - } - return ownerUnavailableResponse(); + return await finish(await replayOrUnavailable()); + } + if (ownerHintFromResponse(direct) || staleDoOwnerHintResponse(direct)) { + cancelResponseBody(direct); + clearHint(); + } else if (ownerEndpointUnavailableResponse(direct)) { + cancelResponseBody(direct); + clearHint(); + return await finish(await replayOrUnavailable()); + } else { + const learned = ownerHintFromHeaders(responseHeaders(direct)); + if (learned) rememberHint(learned); + return await finish(direct); } } const routed = await routerFetch(routerUrl, init); - if (await bypassOwnerHintResponse(routed)) return routed; - const hinted = await ownerHintFromResponse(routed); + if (retryOwnerRace && retryableOwnerRaceResponse(routed)) { + return await finish(routed); + } + + const hinted = ownerHintFromResponse(routed); if (!hinted || typeof ownerFetch !== "function") { if (hinted) rememberHint(hinted); else { const learned = ownerHintFromHeaders(responseHeaders(routed)); if (learned) rememberHint(learned); } - return routed; + return await finish(routed); } + rememberHint(hinted); + cancelResponseBody(routed); + /** @type {Response} */ + let direct; try { - const direct = await followOwnerHint(routed, ownerFetch, ownerPath, init); - if (await ownerHintFromResponse(direct)) { + direct = await ownerFetch(ownerRequestUrl(hinted, ownerPath), init); + if (ownerEndpointUnavailableResponse(direct)) { cancelResponseBody(direct); - clearHint(); - if (replayOwnerUnavailable) { - return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); - } - return ownerUnavailableResponse(); + throw new Error(`DO owner endpoint returned ${responseStatus(direct)}`); } - const learned = ownerHintFromHeaders(responseHeaders(direct)); - if (learned) rememberHint(learned); - return direct; } catch { clearHint(); - if (replayOwnerUnavailable) { - return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); - } - return ownerUnavailableResponse(); + return await finish(await replayOrUnavailable()); } -} - -/** - * @param {DoOwnerHintDispatchOptions} options - * @param {boolean} retryOwnerRace - */ -async function dispatchDoWithHintCache({ - routerFetch, - routerUrl, - ownerFetch, - ownerPath, - init, - cache, - hintKey, - replayOwnerUnavailable = false, -}, retryOwnerRace) { - let response = await dispatchDoRequestWithOwnerHint({ - routerFetch, - routerUrl, - ownerFetch, - ownerPath, - init, - cachedHint: /** @type {DoOwnerHint | null} */ (cache.get(hintKey)), - rememberHint: (hint) => cache.set(hintKey, hint), - clearHint: () => cache.delete(hintKey), - staleCachedResponse: staleDoOwnerHintResponse, - bypassOwnerHintResponse: retryOwnerRace - ? retryableOwnerRaceResponse - : async (_response) => false, - replayOwnerUnavailable, - }); - if (retryOwnerRace && await retryableOwnerRaceResponse(response)) { - cancelResponseBody(response); - cache.delete(hintKey); - response = await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); + if (ownerHintFromResponse(direct)) { + cancelResponseBody(direct); + clearHint(); + return await finish(await replayOrUnavailable()); } - return stripOwnerHintHeaders(response); + const learned = ownerHintFromHeaders(responseHeaders(direct)); + if (learned) rememberHint(learned); + return await finish(direct); } /** @param {DoOwnerHintDispatchOptions} options */ @@ -974,13 +921,13 @@ export async function dispatchDoConnectWithHintCache(options) { } /** @param {Response} response */ -export function ownerEndpointUnavailableResponse(response) { +function ownerEndpointUnavailableResponse(response) { return setHas(OWNER_ENDPOINT_UNAVAILABLE_STATUSES, responseStatus(response)) && !ownerHintFromHeaders(responseHeaders(response)); } /** @param {RequestInit} init */ -export function withoutOwnerHintOptIn(init) { +function withoutOwnerHintOptIn(init) { const headers = init.headers instanceof IntrinsicHeaders ? copyHeaders(init.headers) : new IntrinsicHeaders(init.headers || {}); @@ -989,7 +936,7 @@ export function withoutOwnerHintOptIn(init) { } /** @param {Response} response */ -export function stripOwnerHintHeaders(response) { +function stripOwnerHintHeaders(response) { const headers = copyHeaders(responseHeaders(response)); forEachArray(DO_OWNER_HINT_STRIP_HEADERS, (name) => { headerDelete(headers, name); diff --git a/runtime/bindings/r2.js b/runtime/bindings/r2.js index 5cd2ce6..98e1d3f 100644 --- a/runtime/bindings/r2.js +++ b/runtime/bindings/r2.js @@ -3,6 +3,7 @@ import { SigV4Client } from "@wdl-dev/aws-sigv4"; import { recordBindingOperation } from "runtime-metrics"; import { serviceNameFromEnv } from "runtime-bindings-proxy"; import { discardResponseBody } from "shared-respond"; +import { bytesToBase64 } from "shared-base64"; import { S3_TRANSIENT_RETRIES, fetchRetryableS3Post } from "shared-s3-retry"; import { assertR2BufferSize, @@ -49,8 +50,7 @@ function r2Binding(bucket) { /** @param {BufferSource} bytes */ async function sha256Base64(bytes) { const digest = await crypto.subtle.digest("SHA-256", bytes); - // SHA-256 digests are fixed 32-byte values, so spreading here is bounded. - return btoa(String.fromCharCode(...new Uint8Array(digest))); + return bytesToBase64(new Uint8Array(digest)); } /** diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 48c6006..69274af 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -319,6 +319,7 @@ const authWorker :Workerd.Worker = ( (name = "shared-hex", esModule = embed "../shared/hex.js"), (name = "shared-random-id", esModule = embed "../shared/random-id.js"), (name = "shared-redis-lock", esModule = embed "../shared/redis-lock.js"), + (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), ], compatibilityDate = "2026-04-24", compatibilityFlags = ["nodejs_compat"], diff --git a/runtime/do-client.js b/runtime/do-client.js index 2d5e6cb..dad1d33 100644 --- a/runtime/do-client.js +++ b/runtime/do-client.js @@ -17,12 +17,29 @@ import { requestIdFromOptions } from "./_wdl-request-id.js"; const ownerHintCache = createOwnerHintCache(); const intrinsicObjectHasOwn = Object.hasOwn; const intrinsicReflectApply = Reflect.apply; +const intrinsicStringIsWellFormed = String.prototype.isWellFormed; /** @param {object} object @param {PropertyKey} key */ function objectHasOwn(object, key) { return intrinsicReflectApply(intrinsicObjectHasOwn, undefined, [object, key]); } +/** @param {string} value */ +function isWellFormedUnicodeString(value) { + return intrinsicReflectApply(intrinsicStringIsWellFormed, value, []); +} + +/** @param {unknown} value @param {string} method */ +function requireObjectIdString(value, method) { + if (typeof value !== "string" || !value) { + throw new TypeError(`DurableObjectNamespace.${method}() requires a non-empty string`); + } + if (!isWellFormedUnicodeString(value)) { + throw new TypeError(`DurableObjectNamespace.${method}() requires well-formed Unicode`); + } + return value; +} + /** * @typedef {{ fetch(url: string, init?: RequestInit): Promise }} DoBackend * @typedef {{ @@ -276,18 +293,12 @@ export class DurableObjectNamespace { /** @param {string} name */ idFromName(name) { - if (typeof name !== "string" || !name) { - throw new TypeError("DurableObjectNamespace.idFromName() requires a non-empty string"); - } - return new DurableObjectId(name); + return new DurableObjectId(requireObjectIdString(name, "idFromName")); } /** @param {string} value */ idFromString(value) { - if (typeof value !== "string" || !value) { - throw new TypeError("DurableObjectNamespace.idFromString() requires a non-empty string"); - } - return new DurableObjectId(value); + return new DurableObjectId(requireObjectIdString(value, "idFromString")); } newUniqueId() { diff --git a/runtime/lib.js b/runtime/lib.js index 28a6a95..59974bb 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -1,6 +1,7 @@ // Pure helpers for the runtime worker. import { base64ToBytes, bytesToBase64 } from "shared-base64"; +import { WORKER_NAME_RE, isValidRouteNs } from "shared-ns-pattern"; import { parseVersion } from "shared-version"; import { isWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; @@ -96,7 +97,7 @@ export function workerQueueAttempts(internalAttempts) { return Number.isFinite(n) && n >= 0 ? Math.floor(n) + 1 : 1; } -// Mirror of control/lib.js#deepFreeze — bundle meta is immutable on the +// Mirror of control/bundle.js#deepFreeze — bundle meta is immutable on the // wire, so the post-parse object should be too. /** @template T @param {T} obj @returns {T} */ function deepFreeze(obj) { @@ -109,10 +110,6 @@ function deepFreeze(obj) { const DEFAULT_COMPATIBILITY_DATE = "2026-04-24"; const ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE = "2026-04-21"; const ENHANCED_ERROR_SERIALIZATION_FLAG = "enhanced_error_serialization"; -const ROUTE_NS_RE = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?|__system__)$/; -const RESERVED_TENANT_NS = new Set(["admin"]); -const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,254}$/; - // Loaded workers may declare an older compatibilityDate than the platform // workers. Keep enhanced error serialization as a floor only before the date // where workerd made it the default; newer dates reject the explicit flag. @@ -298,8 +295,8 @@ function requiredPatternString(body, field, pattern, description) { /** @param {Record | null} body */ function requiredRouteNs(body) { - const ns = requiredPatternString(body, "ns", ROUTE_NS_RE, "a route namespace"); - if (RESERVED_TENANT_NS.has(ns)) { + const ns = requiredString(body, "ns"); + if (!isValidRouteNs(ns)) { throw new Error("ns must be a route namespace"); } return ns; diff --git a/runtime/workflows-client.js b/runtime/workflows-client.js index 8c17494..8315ea1 100644 --- a/runtime/workflows-client.js +++ b/runtime/workflows-client.js @@ -199,6 +199,7 @@ export class Workflow { throw new Error("Workflow backend is not configured"); } const metadata = this.#metadata; + const requestId = requestIdFromOptions(this.#options); const body = { ...fields, ns: metadata.ns, @@ -207,11 +208,14 @@ export class Workflow { workflowName: metadata.name, workflowKey: metadata.workflowKey, className: metadata.className, - requestId: requestIdFromOptions(this.#options), + requestId, }; const response = await this.#backend.fetch(`${WORKFLOWS_BASE_URL}/${endpoint}`, { method: "POST", - headers: { "content-type": "application/json" }, + headers: { + "content-type": "application/json", + ...(requestId ? { "x-request-id": requestId } : {}), + }, body: JSON.stringify(body), }); return await readWorkflowResponse(response); diff --git a/rust/workflows/src/api/create.rs b/rust/workflows/src/api/create.rs index 1aa05b2..b574dc5 100644 --- a/rust/workflows/src/api/create.rs +++ b/rust/workflows/src/api/create.rs @@ -240,9 +240,9 @@ pub(crate) async fn create_batch( )); } if req.entries.len() > MAX_CREATE_BATCH_SIZE { - return Err(WorkflowError::request_too_large( - "createBatch exceeds 100 instance limit", - )); + return Err(WorkflowError::request_too_large(format!( + "createBatch exceeds {MAX_CREATE_BATCH_SIZE} instance limit" + ))); } let req = request_with_active_version(state, req).await?; validate_identity(&req)?; diff --git a/rust/workflows/src/api/do_alarms/model.rs b/rust/workflows/src/api/do_alarms/model.rs index 1b611f0..df180ce 100644 --- a/rust/workflows/src/api/do_alarms/model.rs +++ b/rust/workflows/src/api/do_alarms/model.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; +use wdl_rust_common::hash::fnv1a32; use wdl_rust_common::identity::{is_valid_runtime_load_ns, is_valid_worker_name}; use wdl_rust_common::version::parse_version_tag; @@ -85,30 +86,19 @@ pub(crate) struct DoAlarmTickCounters { pub(crate) skipped: usize, } -pub(super) fn validate_non_empty(value: &str, field: &'static str) -> WorkflowResult<()> { - if value.is_empty() { - return Err(WorkflowError::invalid_request(format!( - "{field} is required" - ))); - } - Ok(()) -} - pub(super) fn validate_set_request(req: &DoAlarmSetRequest) -> WorkflowResult<()> { - // These identity fields arrive from the private do-runtime alarm shim. They are - // hashed into job ids before touching Redis key names, so non-empty validation - // preserves the internal contract without imposing tenant namespace/name grammar - // on Durable Object class names or object names. - validate_non_empty(&req.ns, "ns")?; - validate_non_empty(&req.worker, "worker")?; - validate_non_empty(&req.version, "version")?; - if parse_version_tag(&req.version).is_err() { - return Err(WorkflowError::invalid_request("version is invalid")); - } - validate_non_empty(&req.do_storage_id, "doStorageId")?; - validate_non_empty(&req.class_name, "className")?; - validate_non_empty(&req.object_name, "objectName")?; - validate_non_empty(&req.token, "token")?; + // The job id is hashed, but the by-worker index embeds ns/worker directly. + // Validate the complete dispatch identity before either key family is built. + validate_protocol_request_field(&req.ns, "ns", is_valid_runtime_load_ns)?; + validate_protocol_request_field(&req.worker, "worker", is_valid_worker_name)?; + validate_protocol_request_field(&req.version, "version", |value| { + parse_version_tag(value).is_ok() + })?; + validate_protocol_request_field(&req.do_storage_id, "doStorageId", is_storage_id)?; + validate_protocol_request_field(&req.class_name, "className", is_class_name)?; + validate_opaque_request_field(&req.object_name, "objectName")?; + validate_opaque_request_field(&req.token, "token")?; + validate_alarm_host_id(&req.do_storage_id, &req.class_name, &req.object_name)?; if req.scheduled_time <= 0 { return Err(WorkflowError::invalid_request( "scheduledTime must be a positive Unix millisecond timestamp", @@ -118,12 +108,20 @@ pub(super) fn validate_set_request(req: &DoAlarmSetRequest) -> WorkflowResult<() } pub(super) fn validate_delete_request(req: &DoAlarmDeleteRequest) -> WorkflowResult<()> { - validate_non_empty(&req.ns, "ns")?; - validate_non_empty(&req.worker, "worker")?; - validate_non_empty(&req.do_storage_id, "doStorageId")?; - validate_non_empty(&req.class_name, "className")?; - validate_non_empty(&req.object_name, "objectName")?; - validate_non_empty(&req.token, "token")?; + validate_protocol_request_field(&req.ns, "ns", is_valid_runtime_load_ns)?; + validate_protocol_request_field(&req.worker, "worker", is_valid_worker_name)?; + validate_protocol_request_field(&req.do_storage_id, "doStorageId", is_storage_id)?; + validate_protocol_request_field(&req.class_name, "className", is_class_name)?; + validate_opaque_request_field(&req.object_name, "objectName")?; + validate_opaque_request_field(&req.token, "token")?; + validate_alarm_host_id(&req.do_storage_id, &req.class_name, &req.object_name)?; + Ok(()) +} + +pub(super) fn validate_cleanup_request(req: &DoAlarmCleanupRequest) -> WorkflowResult<()> { + validate_protocol_request_field(&req.ns, "ns", is_valid_runtime_load_ns)?; + validate_protocol_request_field(&req.worker, "worker", is_valid_worker_name)?; + validate_protocol_request_field(&req.do_storage_id, "doStorageId", is_storage_id)?; Ok(()) } @@ -152,6 +150,23 @@ pub(super) fn map_hgetall(flat: Vec) -> HashMap { } const MAX_PROTOCOL_STRING_BYTES: usize = 512; +const DO_HOST_SHARD_COUNT: u32 = 16; + +fn alarm_host_id_len(do_storage_id: &str, class_name: &str, object_name: &str) -> usize { + let shard = fnv1a32(object_name.as_bytes()) % DO_HOST_SHARD_COUNT; + do_storage_id.len() + class_name.len() + ":".len() + ":shard".len() + shard.to_string().len() +} + +fn validate_alarm_host_id( + do_storage_id: &str, + class_name: &str, + object_name: &str, +) -> WorkflowResult<()> { + if alarm_host_id_len(do_storage_id, class_name, object_name) > MAX_PROTOCOL_STRING_BYTES { + return Err(WorkflowError::invalid_request("hostId is invalid")); + } + Ok(()) +} fn required_field<'a>( state: &'a HashMap, @@ -172,6 +187,30 @@ fn has_control_char(value: &str) -> bool { value.bytes().any(|byte| byte < 0x20 || byte == 0x7f) } +fn is_protocol_string(value: &str, valid: impl Fn(&str) -> bool) -> bool { + !value.is_empty() + && value.len() <= MAX_PROTOCOL_STRING_BYTES + && !has_control_char(value) + && valid(value) +} + +fn validate_protocol_request_field( + value: &str, + field: &'static str, + valid: impl Fn(&str) -> bool, +) -> WorkflowResult<()> { + if !is_protocol_string(value, valid) { + return Err(WorkflowError::invalid_request(format!( + "{field} is invalid" + ))); + } + Ok(()) +} + +fn validate_opaque_request_field(value: &str, field: &'static str) -> WorkflowResult<()> { + validate_protocol_request_field(value, field, |_| true) +} + fn is_class_name(value: &str) -> bool { let mut chars = value.chars(); let Some(first) = chars.next() else { @@ -193,18 +232,18 @@ fn protocol_string_field<'a>( valid: impl Fn(&str) -> bool, ) -> WorkflowResult<&'a str> { let value = required_field(state, field)?; - if value.len() > MAX_PROTOCOL_STRING_BYTES || has_control_char(value) || !valid(value) { + if !is_protocol_string(value, valid) { return Err(invalid_job_field(field)); } Ok(value) } -fn object_name_field<'a>( +fn opaque_string_field<'a>( state: &'a HashMap, field: &'static str, ) -> WorkflowResult<&'a str> { let value = required_field(state, field)?; - if value.len() > MAX_PROTOCOL_STRING_BYTES || has_control_char(value) { + if !is_protocol_string(value, |_| true) { return Err(invalid_job_field(field)); } Ok(value) @@ -236,6 +275,12 @@ pub(super) fn job_from_state( state: HashMap, ) -> WorkflowResult { parse_positive_i64_field(&state, "dueAtMs")?; + let do_storage_id = protocol_string_field(&state, "doStorageId", is_storage_id)?; + let class_name = protocol_string_field(&state, "className", is_class_name)?; + let object_name = opaque_string_field(&state, "objectName")?; + if alarm_host_id_len(do_storage_id, class_name, object_name) > MAX_PROTOCOL_STRING_BYTES { + return Err(invalid_job_field("hostId")); + } Ok(DoAlarmJob { job_id, ns: protocol_string_field(&state, "ns", is_valid_runtime_load_ns)?.to_string(), @@ -244,11 +289,11 @@ pub(super) fn job_from_state( parse_version_tag(value).is_ok() })? .to_string(), - do_storage_id: protocol_string_field(&state, "doStorageId", is_storage_id)?.to_string(), - class_name: protocol_string_field(&state, "className", is_class_name)?.to_string(), - object_name: object_name_field(&state, "objectName")?.to_string(), - row_token: required_field(&state, "rowToken")?.to_string(), - run_token: required_field(&state, "runToken")?.to_string(), + do_storage_id: do_storage_id.to_string(), + class_name: class_name.to_string(), + object_name: object_name.to_string(), + row_token: opaque_string_field(&state, "rowToken")?.to_string(), + run_token: opaque_string_field(&state, "runToken")?.to_string(), retry_count: parse_u64_field(&state, "retryCount")?, }) } @@ -271,6 +316,31 @@ mod tests { } } + fn valid_delete_request() -> DoAlarmDeleteRequest { + DoAlarmDeleteRequest { + ns: "demo".to_string(), + worker: "alarms".to_string(), + do_storage_id: "do_abc".to_string(), + class_name: "AlarmCounter".to_string(), + object_name: "alpha".to_string(), + token: "row-token".to_string(), + } + } + + fn fixture_string(value: &serde_json::Value) -> String { + if let Some(value) = value.as_str() { + return value.to_string(); + } + value["repeat"] + .as_str() + .expect("fixture repeat is a string") + .repeat( + value["count"] + .as_u64() + .expect("fixture count is an integer") as usize, + ) + } + fn valid_job_state() -> HashMap { HashMap::from([ ("ns".to_string(), "demo".to_string()), @@ -286,6 +356,126 @@ mod tests { ]) } + #[test] + fn do_alarm_mutation_ingress_rejects_noncanonical_identity() { + for (field, value) in [ + ("ns", "Demo"), + ("worker", "bad:worker"), + ("version", "v01"), + ("doStorageId", "DO_ABC"), + ("className", "bad-name"), + ("objectName", "bad\nname"), + ("token", "bad\ntoken"), + ] { + let mut req = valid_set_request(); + match field { + "ns" => req.ns = value.to_string(), + "worker" => req.worker = value.to_string(), + "version" => req.version = value.to_string(), + "doStorageId" => req.do_storage_id = value.to_string(), + "className" => req.class_name = value.to_string(), + "objectName" => req.object_name = value.to_string(), + "token" => req.token = value.to_string(), + _ => unreachable!(), + } + let err = validate_set_request(&req).expect_err("invalid set request"); + assert_eq!(err.code, "invalid_request"); + assert!(err.message.contains(field), "{}", err.message); + } + + let delete = DoAlarmDeleteRequest { + ns: "bad:ns".to_string(), + worker: "alarms".to_string(), + do_storage_id: "do_abc".to_string(), + class_name: "AlarmCounter".to_string(), + object_name: "alpha".to_string(), + token: "row-token".to_string(), + }; + assert!(validate_delete_request(&delete).is_err()); + + let cleanup = DoAlarmCleanupRequest { + ns: "demo".to_string(), + worker: "alarms".to_string(), + do_storage_id: "bad/storage".to_string(), + }; + assert!(validate_cleanup_request(&cleanup).is_err()); + } + + #[test] + fn do_alarm_protocol_fields_match_cross_language_fixture() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../../tests/fixtures/do-alarm-identity.json" + )) + .expect("DO alarm identity fixture parses"); + assert_eq!( + fixture["maxBytes"].as_u64(), + Some(MAX_PROTOCOL_STRING_BYTES as u64) + ); + assert_eq!( + fixture["hostShardCount"].as_u64(), + Some(DO_HOST_SHARD_COUNT as u64) + ); + for case in fixture["cases"] + .as_array() + .expect("DO alarm identity fixture cases is an array") + { + let do_storage_id = fixture_string(&case["doStorageId"]); + let class_name = fixture_string(&case["className"]); + let object_name = fixture_string(&case["objectName"]); + let token = fixture_string(&case["token"]); + let mut set = valid_set_request(); + set.do_storage_id = do_storage_id.clone(); + set.class_name = class_name.clone(); + set.object_name = object_name.clone(); + set.token = token.clone(); + let mut delete = valid_delete_request(); + delete.do_storage_id = do_storage_id; + delete.class_name = class_name; + delete.object_name = object_name; + delete.token = token; + let valid = case["valid"].as_bool().expect("valid is a boolean"); + assert_eq!( + validate_set_request(&set).is_ok(), + valid, + "set:{}", + case["name"].as_str().expect("name is a string") + ); + assert_eq!( + validate_delete_request(&delete).is_ok(), + valid, + "delete:{}", + case["name"].as_str().expect("name is a string") + ); + let mut state = valid_job_state(); + state.insert("doStorageId".to_string(), set.do_storage_id.clone()); + state.insert("className".to_string(), set.class_name.clone()); + state.insert("objectName".to_string(), set.object_name.clone()); + state.insert("rowToken".to_string(), set.token.clone()); + assert_eq!( + job_from_state("job".to_string(), state).is_ok(), + valid, + "persisted:{}", + case["name"].as_str().expect("name is a string") + ); + } + } + + #[test] + fn do_alarm_json_rejects_unpaired_surrogate_object_names() { + let request = r#"{ + "ns":"demo", + "worker":"alarms", + "version":"v1", + "doStorageId":"do_abc", + "className":"AlarmCounter", + "objectName":"\ud800", + "scheduledTime":123456789, + "retryCount":0, + "token":"row-token" + }"#; + assert!(serde_json::from_str::(request).is_err()); + } + #[test] fn do_alarm_job_state_requires_valid_due_time() { for due_at_ms in [None, Some(""), Some("bad"), Some("0"), Some("-1")] { diff --git a/rust/workflows/src/api/do_alarms/mutations.rs b/rust/workflows/src/api/do_alarms/mutations.rs index 2509541..56af2d8 100644 --- a/rust/workflows/src/api/do_alarms/mutations.rs +++ b/rust/workflows/src/api/do_alarms/mutations.rs @@ -8,7 +8,7 @@ use crate::{AppState, LogLevel, WorkflowResult, do_alarm_by_worker_key, log}; use super::super::eval_script; use super::model::{ DoAlarmCleanupRequest, DoAlarmDeleteRequest, DoAlarmMutationResponse, DoAlarmSetRequest, - job_keys_for_identity, validate_delete_request, validate_non_empty, validate_set_request, + job_keys_for_identity, validate_cleanup_request, validate_delete_request, validate_set_request, }; use super::scripts::{ CLEANUP_DO_ALARM_FOR_STORAGE_SCRIPT, DELETE_DO_ALARM_SCRIPT, SET_DO_ALARM_SCRIPT, @@ -177,9 +177,7 @@ pub(crate) async fn cleanup_do_alarms_for_worker( app: &AppState, req: DoAlarmCleanupRequest, ) -> WorkflowResult { - validate_non_empty(&req.ns, "ns")?; - validate_non_empty(&req.worker, "worker")?; - validate_non_empty(&req.do_storage_id, "doStorageId")?; + validate_cleanup_request(&req)?; let by_worker = do_alarm_by_worker_key(&req.ns, &req.worker); let snapshot_key = format!("{by_worker}:cleanup-snapshot:{}", random_hex_64()); let mut cursor = 0_u64; diff --git a/rust/workflows/src/api/limits.rs b/rust/workflows/src/api/limits.rs index 83d3b90..caff1e0 100644 --- a/rust/workflows/src/api/limits.rs +++ b/rust/workflows/src/api/limits.rs @@ -17,7 +17,8 @@ mod tests { use serde::Deserialize; use super::{ - MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_RESULT_BYTES, WORKFLOW_PAYLOAD_TOO_LARGE_CODE, + MAX_CREATE_BATCH_SIZE, MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_RESULT_BYTES, + WORKFLOW_PAYLOAD_TOO_LARGE_CODE, }; #[derive(Deserialize)] @@ -25,6 +26,7 @@ mod tests { struct WorkflowLimitsContract { result_bytes_max: usize, backend_request_bytes_max: usize, + create_batch_max: usize, payload_too_large_code: String, } @@ -40,6 +42,7 @@ mod tests { contract.backend_request_bytes_max, MAX_WORKFLOW_JSON_BODY_BYTES ); + assert_eq!(contract.create_batch_max, MAX_CREATE_BATCH_SIZE); assert_eq!( contract.payload_too_large_code, WORKFLOW_PAYLOAD_TOO_LARGE_CODE diff --git a/shared/redis-lock.js b/shared/redis-lock.js index 358c25d..091a31a 100644 --- a/shared/redis-lock.js +++ b/shared/redis-lock.js @@ -3,7 +3,6 @@ import { randomHex } from "shared-random-id"; /** * @typedef {{ key: string, token: string }} TokenLock * @typedef {{ set(key: string, value: string, options: { nx?: boolean, ttl?: number, ifeq?: string }): Promise }} TokenLockSetClient - * @typedef {{ multi(): { set(key: string, value: string, options: { nx: true, ttl: number }): { exec(): Promise } } }} TokenLockTransactionalClient * @typedef {{ delIfEq(key: string, value: string): Promise }} TokenLockReleaseClient */ @@ -13,20 +12,11 @@ export function createTokenLock(key) { } /** - * @param {TokenLockSetClient | TokenLockTransactionalClient} client + * @param {TokenLockSetClient} client * @param {TokenLock} lock - * @param {{ ttlSeconds: number, transactional?: boolean }} options + * @param {{ ttlSeconds: number }} options */ -export async function acquireTokenLock(client, lock, { ttlSeconds, transactional = false }) { - if (transactional) { - if (!("multi" in client) || typeof client.multi !== "function") { - throw new TypeError("transactional token lock requires multi()"); - } - const reply = await client.multi() - .set(lock.key, lock.token, { nx: true, ttl: ttlSeconds }) - .exec(); - return Array.isArray(reply) && reply[0] === "OK"; - } +export async function acquireTokenLock(client, lock, { ttlSeconds }) { if (!("set" in client) || typeof client.set !== "function") { throw new TypeError("token lock acquire requires set()"); } diff --git a/shared/redis-session.js b/shared/redis-session.js index ce5b4ac..91f201f 100644 --- a/shared/redis-session.js +++ b/shared/redis-session.js @@ -249,6 +249,11 @@ export class RedisSession { async incr(key) { return /** @type {number} */ (await this._exec("INCR", key)); } /** @param {string} key */ async get(key) { return decodeBulk(await this._exec("GET", key)); } + /** @param {string} key @param {RedisArg} value @param {RedisSetOptions} [opts] */ + async set(key, value, opts = {}) { + const reply = await this._exec(...buildSetArgs(key, value, opts)); + return reply === null ? null : decodeBulk(reply); + } /** @param {string[]} keys */ async getMany(keys) { const replies = /** @type {unknown[]} */ (await this._execPipeline( diff --git a/shared/version.js b/shared/version.js index 8723aad..4b34de9 100644 --- a/shared/version.js +++ b/shared/version.js @@ -67,6 +67,9 @@ export function patternsKey(host) { export const NAMESPACES_KEY = "namespaces"; export const DECLARED_HOSTS_KEY = "declared-hosts"; +export const ROUTES_CHANNEL = "routes:invalidate"; +export const ROUTES_FLUSH_CHANNEL = "routes:flush"; +export const PATTERNS_CHANNEL = "patterns:invalidate"; const HOSTS_PREFIX = "hosts:"; const NS_HOSTS_PREFIX = "ns-hosts:"; const HOST_DECLARATIONS_PREFIX = "host-declarations:"; @@ -106,6 +109,15 @@ export function doStorageIdKey(ns, worker) { return `worker:do-storage:${ns}:${worker}`; } +// DO runtime owns the records; Control uses the storage-scoped pattern during +// whole-worker cleanup. Keep both sides on the same encoded Redis prefix. +export const DO_OWNER_SCOPE_PREFIX = "do:owner:scope:"; + +/** @param {string} storageId */ +export function doOwnerScopeScanPatternForStorage(storageId) { + return `${DO_OWNER_SCOPE_PREFIX}${encodeURIComponent(`${storageId}:`)}*`; +} + // Per-worker lifecycle lock. Control owns acquisition/release; other tiers may // WATCH it when creating state that whole-worker delete must discover. export const WHOLE_DELETE_LOCK_KIND = "whole"; diff --git a/tests/fixtures/do-alarm-identity.json b/tests/fixtures/do-alarm-identity.json new file mode 100644 index 0000000..c846749 --- /dev/null +++ b/tests/fixtures/do-alarm-identity.json @@ -0,0 +1,142 @@ +{ + "maxBytes": 512, + "hostShardCount": 16, + "cases": [ + { + "name": "canonical", + "doStorageId": "do_abc-123", + "className": "AlarmCounter", + "objectName": "alpha", + "token": "row-token", + "valid": true + }, + { + "name": "empty-object-name", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "", + "token": "row-token", + "valid": false + }, + { + "name": "empty-token", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "alpha", + "token": "", + "valid": false + }, + { + "name": "storage-id-grammar", + "doStorageId": "bad/storage", + "className": "AlarmCounter", + "objectName": "alpha", + "token": "row-token", + "valid": false + }, + { + "name": "class-name-grammar", + "doStorageId": "do_abc", + "className": "1Alarm", + "objectName": "alpha", + "token": "row-token", + "valid": false + }, + { + "name": "one-digit-shard-aggregate-max", + "doStorageId": { "repeat": "a", "count": 496 }, + "className": "ChatRoom", + "objectName": "zero", + "token": "row-token", + "valid": true + }, + { + "name": "one-digit-shard-aggregate-over-limit", + "doStorageId": { "repeat": "a", "count": 497 }, + "className": "ChatRoom", + "objectName": "zero", + "token": "row-token", + "valid": false + }, + { + "name": "two-digit-shard-aggregate-max", + "doStorageId": { "repeat": "a", "count": 495 }, + "className": "ChatRoom", + "objectName": "alpha", + "token": "row-token", + "valid": true + }, + { + "name": "two-digit-shard-aggregate-over-limit", + "doStorageId": { "repeat": "a", "count": 496 }, + "className": "ChatRoom", + "objectName": "alpha", + "token": "row-token", + "valid": false + }, + { + "name": "object-name-control-character", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "bad\u000bname", + "token": "row-token", + "valid": false + }, + { + "name": "object-name-supplementary-scalar", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "\ud83d\ude00", + "token": "row-token", + "valid": true + }, + { + "name": "supplementary-scalar-two-digit-shard-over-limit", + "doStorageId": { "repeat": "a", "count": 496 }, + "className": "ChatRoom", + "objectName": "\ud83d\ude80", + "token": "row-token", + "valid": false + }, + { + "name": "object-name-utf8-max", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": { "repeat": "\u00e9", "count": 256 }, + "token": "row-token", + "valid": true + }, + { + "name": "object-name-utf8-over-limit", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": { "repeat": "\u00e9", "count": 257 }, + "token": "row-token", + "valid": false + }, + { + "name": "token-control-character", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "alpha", + "token": "bad\u000btoken", + "valid": false + }, + { + "name": "token-utf8-max", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "alpha", + "token": { "repeat": "\u00e9", "count": 256 }, + "valid": true + }, + { + "name": "token-utf8-over-limit", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "alpha", + "token": { "repeat": "\u00e9", "count": 257 }, + "valid": false + } + ] +} diff --git a/tests/fixtures/workflow-limits.json b/tests/fixtures/workflow-limits.json index 29e1509..804b6bf 100644 --- a/tests/fixtures/workflow-limits.json +++ b/tests/fixtures/workflow-limits.json @@ -1,5 +1,6 @@ { "resultBytesMax": 1048576, "backendRequestBytesMax": 2097152, + "createBatchMax": 100, "payloadTooLargeCode": "workflow_payload_too_large" } diff --git a/tests/helpers/control-shared-stub.js b/tests/helpers/control-shared-stub.js index 1ec1b2f..2563f66 100644 --- a/tests/helpers/control-shared-stub.js +++ b/tests/helpers/control-shared-stub.js @@ -12,6 +12,7 @@ import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); const SHARED_OPTIMISTIC_RETRY_URL = repositoryFileUrl("shared/optimistic-retry.js"); @@ -40,6 +41,7 @@ import { ControlAbort, controlAbortLogDetails, codedErrorLogFields, codedErrorRe import { runOptimistic, withOptimisticRetries } from ${JSON.stringify(CONTROL_OPTIMISTIC_URL)}; import { readJsonBody } from ${JSON.stringify(CONTROL_JSON_BODY_URL)}; import { errorMessage as errMessage } from ${JSON.stringify(SHARED_ERRORS_URL)}; +import { withInternalAuth } from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)}; import { randomHex } from ${JSON.stringify(SHARED_RANDOM_ID_URL)}; import { formatError } from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}; export { @@ -89,10 +91,7 @@ export function getControlR2() { return state.r2; } export function controlInternalJsonHeaders() { - const token = state.env?.WDL_INTERNAL_AUTH_TOKEN; - return token - ? { "content-type": "application/json", "x-wdl-internal-auth": token } - : { "content-type": "application/json" }; + return withInternalAuth({ "content-type": "application/json" }, state.env); } export const postWorkflowsInternal = createPostWorkflowsInternal({ getWorkflows: () => state.workflows, diff --git a/tests/helpers/load-auth-index.js b/tests/helpers/load-auth-index.js index afb3eeb..0bd5220 100644 --- a/tests/helpers/load-auth-index.js +++ b/tests/helpers/load-auth-index.js @@ -44,7 +44,16 @@ function bumpKeyVersion(key) { s.keyVersions.set(key, (s.keyVersions.get(key) || 0) + 1); } +function expireStringIfNeeded(key) { + const s = ensureState(); + const expiresAt = s.expirations.get(key); + if (expiresAt == null || expiresAt > Date.now()) return; + s.expirations.delete(key); + if (s.strings.delete(key)) bumpKeyVersion(key); +} + function keyVersion(key) { + expireStringIfNeeded(key); const s = ensureState(); return s.keyVersions.get(key) || 0; } @@ -56,6 +65,7 @@ export class RedisClient { } async get(key) { const s = ensureState(); + expireStringIfNeeded(key); if (s.getThrows && s.getThrows.has(key)) { const err = new Error("forced get throw on " + key); recordCommand("GET", false, err.message); @@ -91,13 +101,16 @@ export class RedisClient { } async del(key) { const s = ensureState(); + expireStringIfNeeded(key); recordCommand("DEL", true); const had = s.strings.delete(key) || s.hashes.delete(key); + s.expirations.delete(key); if (had) bumpKeyVersion(key); return had ? 1 : 0; } async delIfEq(key, value) { const s = ensureState(); + expireStringIfNeeded(key); if (s.delIfEqThrows && s.delIfEqThrows.has(key)) { const err = new Error("forced delIfEq throw on " + key); recordCommand("DELIFEQ", false, err.message); @@ -106,9 +119,33 @@ export class RedisClient { recordCommand("DELIFEQ", true); if (s.strings.get(key) !== value) return 0; s.strings.delete(key); + s.expirations.delete(key); bumpKeyVersion(key); return 1; } + async set(key, value, options = {}) { + const s = ensureState(); + expireStringIfNeeded(key); + if (options.nx && s.strings.has(key)) { + recordCommand("SET", true); + return null; + } + s.strings.set(key, value); + if (typeof options.ttl === "number") { + s.expirations.set(key, Date.now() + options.ttl * 1000); + } else { + s.expirations.delete(key); + } + bumpKeyVersion(key); + try { + if (s.afterSetAppliedBeforeReply) s.afterSetAppliedBeforeReply(key, value, options); + } catch (err) { + recordCommand("SET", false, err instanceof Error ? err.message : String(err)); + throw err; + } + recordCommand("SET", true); + return "OK"; + } async scan(_cursor, pattern, _count) { const s = ensureState(); recordCommand("SCAN", true); @@ -138,7 +175,9 @@ export class RedisClient { for (const cmd of cmds) { const [op, key, ...rest] = cmd; if (op === "DEL") { + expireStringIfNeeded(key); const had = s.strings.delete(key) || s.hashes.delete(key); + s.expirations.delete(key); if (had) bumpKeyVersion(key); out.push(had ? 1 : 0); recordCommand("DEL", true); @@ -152,11 +191,19 @@ export class RedisClient { out.push(rest.length / 2); recordCommand("HSET", true); } else if (op === "SET") { - const options = rest.slice(1).map((arg) => String(arg).toUpperCase()); - if (options.includes("NX") && s.strings.has(key)) { + expireStringIfNeeded(key); + const options = rest.slice(1).map((arg) => String(arg)); + const upperOptions = options.map((arg) => arg.toUpperCase()); + if (upperOptions.includes("NX") && s.strings.has(key)) { out.push(null); } else { s.strings.set(key, rest[0]); + const exIndex = upperOptions.indexOf("EX"); + if (exIndex >= 0) { + s.expirations.set(key, Date.now() + Number(options[exIndex + 1]) * 1000); + } else { + s.expirations.delete(key); + } bumpKeyVersion(key); out.push("OK"); } @@ -165,6 +212,12 @@ export class RedisClient { throw new Error("multiExec mock: unsupported " + op); } } + try { + if (s.afterMultiExecAppliedBeforeReply) s.afterMultiExecAppliedBeforeReply(cmds); + } catch (err) { + recordCommand("MULTI_EXEC", false, err instanceof Error ? err.message : String(err)); + throw err; + } return out; } async session(fn) { @@ -179,6 +232,7 @@ export class RedisClient { async get(key) { return self.get(key); }, async del(key) { return self.del(key); }, async delIfEq(key, value) { return self.delIfEq(key, value); }, + async set(key, value, options) { return self.set(key, value, options); }, async hGet(key, field) { return self.hGet(key, field); }, async hGetAll(key) { return self.hGetAll(key); }, async hGetAllMany(keys) { return self.hGetAllMany(keys); }, @@ -312,6 +366,7 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel [/from "shared-observability"/g, `from ${JSON.stringify(sharedObservabilityUrl)}`], [/from "auth-lib"/g, `from ${JSON.stringify(authLibUrl)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], + [/from "shared-optimistic-retry"/g, `from ${JSON.stringify(repositoryFileUrl("shared/optimistic-retry.js"))}`], ]); const indexUrl = freshRepositoryModuleDataUrl("auth/index.js", [ @@ -334,6 +389,7 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel export function resetAuthMockState() { /** @type {any} */ (globalThis).__authMockState = { strings: new Map(), + expirations: new Map(), hashes: new Map(), sets: new Map(), getThrows: new Set(), @@ -341,6 +397,8 @@ export function resetAuthMockState() { delIfEqThrows: new Set(), scanPages: [], keyVersions: new Map(), + afterSetAppliedBeforeReply: null, + afterMultiExecAppliedBeforeReply: null, beforeMultiExec: null, onCommand: null, commands: [], diff --git a/tests/helpers/load-runtime-r2-binding.js b/tests/helpers/load-runtime-r2-binding.js index face7a1..090add4 100644 --- a/tests/helpers/load-runtime-r2-binding.js +++ b/tests/helpers/load-runtime-r2-binding.js @@ -13,6 +13,7 @@ const R2_UTILS_URL = repositoryFileUrl("runtime/r2-utils.js"); const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); const SHARED_S3_XML_URL = repositoryFileUrl("shared/s3-xml.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_BASE64_URL = repositoryFileUrl("shared/base64.js"); const SHARED_S3_RETRY_URL = repositoryFileUrl("shared/s3-retry.js"); const R2_METADATA_URL = repositoryModuleDataUrl("runtime/bindings/r2/metadata.js", [ [/from "runtime-r2-utils";/, `from ${JSON.stringify(R2_UTILS_URL)};`], @@ -52,6 +53,7 @@ const mod = await importRepositoryModule("runtime/bindings/r2.js", [ [/from "runtime-bindings-r2-metadata";/, `from ${JSON.stringify(R2_METADATA_URL)};`], [/from "runtime-bindings-r2-xml";/, `from ${JSON.stringify(R2_XML_URL)};`], [/from "shared-respond";/, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], + [/from "shared-base64";/, `from ${JSON.stringify(SHARED_BASE64_URL)};`], [/from "shared-s3-retry";/, `from ${JSON.stringify(SHARED_S3_RETRY_URL)};`], ]); diff --git a/tests/helpers/load-shared-module.js b/tests/helpers/load-shared-module.js index 8e44de8..66a8914 100644 --- a/tests/helpers/load-shared-module.js +++ b/tests/helpers/load-shared-module.js @@ -11,6 +11,7 @@ const REPO_ROOT = path.resolve(__dirname, "../.."); const SHARED_ENV_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/env.js")).href; const SHARED_ERRORS_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/errors.js")).href; const SHARED_BASE64_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/base64.js")).href; +const SHARED_NS_PATTERN_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/ns-pattern.js")).href; const SHARED_VERSION_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/version.js")).href; const SHARED_HEX_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/hex.js")).href; const SHARED_OPTIMISTIC_RETRY_URL = pathToFileURL( @@ -93,6 +94,7 @@ export function repositoryModuleDataUrl(relativePath, replacements = []) { export function runtimeLibModuleDataUrl() { return repositoryModuleDataUrl("runtime/lib.js", importSpecifierReplacements({ "shared-base64": SHARED_BASE64_URL, + "shared-ns-pattern": SHARED_NS_PATTERN_URL, "shared-version": SHARED_VERSION_URL, "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, })); diff --git a/tests/helpers/mocks/fake-redis.js b/tests/helpers/mocks/fake-redis.js index d150eed..bab6d4d 100644 --- a/tests/helpers/mocks/fake-redis.js +++ b/tests/helpers/mocks/fake-redis.js @@ -47,8 +47,8 @@ export class FakeRedisWatchError extends Error { export function sharedRedisStubUrl(extraSource = "") { return moduleDataUrl(` import { FakeRedisWatchError as WatchError } from ${JSON.stringify(import.meta.url)}; -export { WatchError }; -export { decodeBulk } from ${JSON.stringify(SHARED_REDIS_RESP_URL)}; +import { decodeBulk } from ${JSON.stringify(SHARED_REDIS_RESP_URL)}; +export { WatchError, decodeBulk }; ${extraSource} `); } diff --git a/tests/unit/auth-index.test.js b/tests/unit/auth-index.test.js index 27b5a55..45e92e7 100644 --- a/tests/unit/auth-index.test.js +++ b/tests/unit/auth-index.test.js @@ -826,7 +826,7 @@ test("delegatedIssue: local lock budget is measured from lock acquisition", asyn }); }); -test("delegatedIssue: releases issue lock when SET reply is lost after apply", async () => { +test("delegatedIssue: leaves an uncertain applied lock for TTL expiry after a lost SET reply", async () => { const { auth } = await freshAuth(); const issuer = await auth.issue({ kind: "token-issuer", @@ -836,22 +836,66 @@ test("delegatedIssue: releases issue lock when SET reply is lost after apply", a const state = authMockState(); const lockKey = delegatedIssueLockKey(issuer.tokenId); let lostReplyInjected = false; - state.beforeMultiExec = (/** @type {unknown[][]} */ cmds) => { - if (lostReplyInjected) return; - const lockSet = cmds.find((cmd) => cmd[0] === "SET" && cmd[1] === lockKey); - if (!lockSet) return; + let nowMs = 1_750_000_000_000; + state.afterSetAppliedBeforeReply = (/** @type {string} */ key) => { + if (lostReplyInjected || key !== lockKey) return; lostReplyInjected = true; - state.strings.set(lockKey, lockSet[2]); - state.keyVersions.set(lockKey, (state.keyVersions.get(lockKey) || 0) + 1); throw new Error("lost delegated issue lock SET reply"); }; + await withMockedProperty(Date, "now", () => nowMs, async () => { + await assert.rejects( + () => auth.delegatedIssue({ issuerTokenId: issuer.tokenId, template: "wdl-chat-ns-pool" }), + /lost delegated issue lock SET reply/ + ); + assert.equal(lostReplyInjected, true); + assert.equal(state.strings.has(lockKey), true); + assert.equal(state.expirations.get(lockKey), nowMs + 30_000); + assert.equal(state.commands.at(-1)?.command, "SET"); + assert.equal(state.commands.at(-1)?.ok, false); + assert.equal(state.commands.some((/** @type {any} */ event) => event.command === "DELIFEQ"), false); + + nowMs += 29_999; + await assert.rejects( + () => auth.delegatedIssue({ issuerTokenId: issuer.tokenId, template: "wdl-chat-ns-pool" }), + (err) => /** @type {any} */ (err).reason === "delegated_issue_busy" + ); + + nowMs += 2; + const issued = await auth.delegatedIssue({ + issuerTokenId: issuer.tokenId, + template: "wdl-chat-ns-pool", + }); + assert.equal(issued.kind, "ns"); + assert.equal(state.strings.has(lockKey), false); + assert.equal(state.expirations.has(lockKey), false); + }); +}); + +test("delegatedIssue: leaves its lock for TTL expiry when the final EXEC reply is lost", async () => { + const { auth } = await freshAuth(); + const issuer = await auth.issue({ + kind: "token-issuer", + issueTemplates: ["wdl-chat-ns-pool"], + issuerTokenId: "bootstrap", + }); + const state = authMockState(); + const lockKey = delegatedIssueLockKey(issuer.tokenId); + state.afterMultiExecAppliedBeforeReply = () => { + throw new Error("lost delegated issue EXEC reply"); + }; + await assert.rejects( () => auth.delegatedIssue({ issuerTokenId: issuer.tokenId, template: "wdl-chat-ns-pool" }), - /lost delegated issue lock SET reply/ + /lost delegated issue EXEC reply/ ); - assert.equal(lostReplyInjected, true); - assert.equal(state.strings.has(lockKey), false); + const committed = [...state.hashes.values()].filter((/** @type {any} */ record) => + record.created_by === issuer.tokenId && record.issue_template === "wdl-chat-ns-pool"); + assert.equal(committed.length, 1); + assert.equal(state.strings.has(lockKey), true); + assert.equal(state.commands.at(-1)?.command, "MULTI_EXEC"); + assert.equal(state.commands.at(-1)?.ok, false); + assert.equal(state.commands.some((/** @type {any} */ event) => event.command === "DELIFEQ"), false); }); test("delegatedIssue: issue lock release failure warns without failing issued token", async () => { @@ -922,6 +966,36 @@ test("delegatedIssue: issue lock WATCH prevents writing if the lock changes befo ); }); +test("delegatedIssue: issuer revoke invalidates the final token write", async () => { + const { auth } = await freshAuth(); + const issuer = await auth.issue({ + kind: "token-issuer", + issueTemplates: ["wdl-chat-ns-pool"], + issuerTokenId: "bootstrap", + }); + const state = authMockState(); + const issuerKey = `auth:token:${issuer.tokenId}`; + let revoked = false; + state.beforeMultiExec = (/** @type {unknown[][]} */ cmds) => { + if (revoked || !cmds.some((cmd) => cmd[0] === "HSET" && cmd[1] !== issuerKey)) return; + revoked = true; + state.hashes.get(issuerKey).revoked_at = "2026-07-16T00:00:00.000Z"; + state.keyVersions.set(issuerKey, (state.keyVersions.get(issuerKey) || 0) + 1); + }; + + await assert.rejects( + () => auth.delegatedIssue({ issuerTokenId: issuer.tokenId, template: "wdl-chat-ns-pool" }), + (err) => /** @type {any} */ (err).reason === "delegated_issue_busy" + ); + assert.equal(revoked, true); + assert.equal(state.hashes.get(issuerKey).revoked_at, "2026-07-16T00:00:00.000Z"); + assert.equal( + [...state.hashes.values()].filter((/** @type {any} */ record) => + record.created_by === issuer.tokenId && record.issue_template === "wdl-chat-ns-pool").length, + 0 + ); +}); + test("delegatedIssue: namespace collision retries against namespaces set", async () => { const { auth } = await freshAuth({ delegatedTemplatePatch: TINY_RANDOM_TEMPLATE }); const issuer = await auth.issue({ diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index 54139b2..acdcd84 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -15,11 +15,14 @@ import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; const { libUrl: productionControlLibUrl, lifecycleIndexesUrl } = await compileControlGraph(); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const controlSharedExtraSource = ` -export const ROUTES_CHANNEL = "routes:invalidate"; -export const ROUTES_FLUSH_CHANNEL = "routes:flush"; -export const PATTERNS_CHANNEL = "patterns:invalidate"; +export { + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + ROUTES_FLUSH_CHANNEL, +} from ${JSON.stringify(sharedVersionUrl)}; export async function acquireDeleteLock(_redis, _ns, _worker, kind) { /** @type {any} */ (globalThis).__deleteHandlerState.lockKinds.push(kind); return "lock-token"; @@ -90,8 +93,6 @@ export function extractOutgoingRefs() { return []; } export function formatReferrerBlocker(members) { return { referrers: members }; } `); -const sharedVersionUrl = repositoryFileUrl("shared/version.js"); - const sharedSecretKeysUrl = repositoryFileUrl("shared/secret-keys.js"); const sharedRedisUrl = sharedRedisStubUrl(); @@ -539,8 +540,12 @@ test("worker delete reports cleanup_queue_failed when data-plane cleanup enqueue const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); assert.deepEqual(testState.lockKinds, ["whole"]); - assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api", allowCleanup: true }]); - assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); + assert.deepEqual(testState.workflowChecks, [{ + ns: "demo", worker: "api", allowCleanup: true, requestId: "rid-delete", + }]); + assert.deepEqual(testState.doAlarmCleanups, [{ + ns: "demo", worker: "api", doStorageId: "do_old", requestId: "rid-delete", + }]); assert.equal(body.assets.queueHint, "failed"); assert.deepEqual(body.assets.warnings, [{ code: "cleanup_queue_failed", @@ -1043,7 +1048,9 @@ test("worker delete dry-run includes workflow lifecycle blockers", async () => { delete /** @type {any} */ (globalThis).__deleteHandlerWorkflowBlocker; const body = await readJsonResponse(response, 200); assert.equal(body.deleted, false); - assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api" }]); + assert.deepEqual(testState.workflowChecks, [{ + ns: "demo", worker: "api", requestId: "rid-dry-run-workflows", + }]); assert.deepEqual(body.workflowBlocker, { error: "workflow_instances_active", message: "demo/api has active workflow instances", @@ -1126,6 +1133,7 @@ test("worker delete reports workflow and version referrer blockers together", as ns: "demo", worker: "api", allowCleanup: true, + requestId: "rid-delete-both-blockers", }]); assert.deepEqual(testState.doAlarmCleanups, []); assert.equal(testState.releaseCalls, 1); @@ -1165,6 +1173,7 @@ test("worker delete keeps DO alarm jobs when version referrers block deletion", ns: "demo", worker: "api", allowCleanup: true, + requestId: "rid-delete-version-blocker", }]); assert.deepEqual(testState.doAlarmCleanups, []); assert.equal(testState.multiCalls.length, 0); @@ -1186,7 +1195,12 @@ test("worker delete logs post-commit DO alarm cleanup failure without rolling ba const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); - assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); + assert.deepEqual(testState.doAlarmCleanups, [{ + ns: "demo", + worker: "api", + doStorageId: "do_old", + requestId: "rid-delete-alarm-cleanup-failed", + }]); assert.ok(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC")); assert.ok(testState.logs.some((/** @type {any} */ entry) => entry.level === "warn" && @@ -1233,6 +1247,7 @@ test("worker delete keeps workflow blocker even when worker lifecycle is already ns: "demo", worker: "api", allowCleanup: true, + requestId: "rid-delete-workflow-no-lifecycle", }]); assert.equal(testState.releaseCalls, 1); assert.deepEqual(testState.cleanupIntents, []); @@ -1261,7 +1276,12 @@ test("worker delete retry compensates DO alarm cleanup when stale storage pointe const body = await readJsonResponse(response, 200); assert.equal(body.deleted, false); - assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); + assert.deepEqual(testState.doAlarmCleanups, [{ + ns: "demo", + worker: "api", + doStorageId: "do_old", + requestId: "rid-delete-noop-alarm-cleanup", + }]); assert.deepEqual(testState.multiCalls, [ ["DEL", "worker:do-storage:demo:api"], ["EXEC"], @@ -1330,8 +1350,12 @@ test("worker delete reports queueHint none when no content cleanup is needed", a const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); - assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api", allowCleanup: true }]); - assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); + assert.deepEqual(testState.workflowChecks, [{ + ns: "demo", worker: "api", allowCleanup: true, requestId: "rid-delete-no-cleanup", + }]); + assert.deepEqual(testState.doAlarmCleanups, [{ + ns: "demo", worker: "api", doStorageId: "do_old", requestId: "rid-delete-no-cleanup", + }]); assert.equal(body.assets.queueHint, "none"); assert.deepEqual(body.assets.warnings, []); assert.deepEqual(testState.cleanupIntents, []); @@ -1363,6 +1387,7 @@ test("version delete reports cleanup_queue_failed when data-plane cleanup enqueu worker: "api", version: "v1", allowCleanup: true, + requestId: "rid-version-delete", }]); assert.equal(body.assets.queueHint, "failed"); assert.deepEqual(body.assets.warnings, [{ @@ -1514,6 +1539,7 @@ test("version delete reports queueHint none when no content cleanup is needed", worker: "api", version: "v1", allowCleanup: true, + requestId: "rid-version-delete-no-cleanup", }]); assert.equal(body.assets.queueHint, "none"); assert.deepEqual(body.assets.warnings, []); diff --git a/tests/unit/control-shared-stub.test.js b/tests/unit/control-shared-stub.test.js index 3255b12..ead44cf 100644 --- a/tests/unit/control-shared-stub.test.js +++ b/tests/unit/control-shared-stub.test.js @@ -38,3 +38,26 @@ test("control shared stub enforces streamed request limits through shared bounde await readJsonResponse(result.response, 413); assert.equal(canceled, true); }); + +test("control shared stub matches production internal-auth header validation", async () => { + const missing = await import(controlSharedStubUrl("export const state = { env: {} };")); + assert.throws( + () => missing.controlInternalJsonHeaders(), + /WDL_INTERNAL_AUTH_TOKEN must be configured/, + ); + + const invalid = await import(controlSharedStubUrl(` + export const state = { env: { WDL_INTERNAL_AUTH_TOKEN: "bad token" } }; + `)); + assert.throws( + () => invalid.controlInternalJsonHeaders(), + /visible ASCII without whitespace or commas/, + ); + + const valid = await import(controlSharedStubUrl(` + export const state = { env: { WDL_INTERNAL_AUTH_TOKEN: "test-token" } }; + `)); + const headers = valid.controlInternalJsonHeaders(); + assert.equal(headers.get("content-type"), "application/json"); + assert.equal(headers.get("x-wdl-internal-auth"), "test-token"); +}); diff --git a/tests/unit/control-shared.test.js b/tests/unit/control-shared.test.js index b5a9019..a005ce3 100644 --- a/tests/unit/control-shared.test.js +++ b/tests/unit/control-shared.test.js @@ -602,7 +602,12 @@ test("assertWorkflowDeleteAllowed hides transport diagnostics from response deta }; await assert.rejects( - () => assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", version: "v2" }), + () => assertWorkflowDeleteAllowed({ + ns: "demo", + worker: "api", + version: "v2", + requestId: "rid-lifecycle-failure", + }), (err) => { assert.ok(err instanceof ControlAbort); const abort = /** @type {InstanceType} */ (err); @@ -620,6 +625,7 @@ test("assertWorkflowDeleteAllowed hides transport diagnostics from response deta namespace: "demo", worker: "api", version: "v2", + request_id: "rid-lifecycle-failure", error_message: "connect ECONNREFUSED workflows", }, }); @@ -688,11 +694,13 @@ test("shared workflows calls preserve endpoint-specific timeout behavior", async assert.equal(new Headers(init?.headers).get("x-wdl-internal-auth"), TEST_INTERNAL_AUTH_TOKEN); const endpoint = String(url); if (endpoint.endsWith("/lifecycle/check-delete")) { + assert.equal(new Headers(init?.headers).get("x-request-id"), "rid-lifecycle"); assert.equal(init?.signal, undefined); requestBodies.lifecycle = parseJsonObjectRequestBody(init, "workflow lifecycle request body"); return Response.json({ allowed: true }); } assert.ok(init?.signal instanceof AbortSignal); + assert.equal(new Headers(init?.headers).get("x-request-id"), "rid-cleanup"); fetchSignals.push(init.signal); requestBodies.cleanup = parseJsonObjectRequestBody(init, "DO alarm cleanup request body"); return Response.json({ ok: true }); @@ -700,8 +708,12 @@ test("shared workflows calls preserve endpoint-specific timeout behavior", async }; try { - await assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", version: "v2" }); - await cleanupDoAlarmsForWorker({ ns: "demo", worker: "api", doStorageId: "do_old" }); + await assertWorkflowDeleteAllowed({ + ns: "demo", worker: "api", version: "v2", requestId: "rid-lifecycle", + }); + await cleanupDoAlarmsForWorker({ + ns: "demo", worker: "api", doStorageId: "do_old", requestId: "rid-cleanup", + }); } finally { for (const handle of timeoutHandles) clearTimeout(handle); restoreTimeout(); diff --git a/tests/unit/do-alarm-client.test.js b/tests/unit/do-alarm-client.test.js index 34eebf3..8a7cd81 100644 --- a/tests/unit/do-alarm-client.test.js +++ b/tests/unit/do-alarm-client.test.js @@ -22,6 +22,9 @@ export class DoRuntimeError extends Error { export function nonEmptyAlarmString(value) { return typeof value === "string" && value.length > 0 ? value : null; } +export function isWellFormedUnicodeString(value) { + return typeof value === "string" && value.isWellFormed(); +} `); const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; @@ -194,5 +197,18 @@ test("alarm input validation remains local before backend calls", async () => { ), /retryCount must be a non-negative integer/, ); + await assert.rejects( + () => mod.setAlarmIndex( + alarmEnv({ WORKFLOWS_BACKEND: workflowsBackend() }), + props, + { + className: "Room", + objectName: "\ud800", + scheduledTime: 123456, + token: "row-token", + }, + ), + /objectName must contain well-formed Unicode/, + ); assert.equal(calls.length, 0); }); diff --git a/tests/unit/do-runtime-index.test.js b/tests/unit/do-runtime-index.test.js index 3f3c71f..d5a6b11 100644 --- a/tests/unit/do-runtime-index.test.js +++ b/tests/unit/do-runtime-index.test.js @@ -3,6 +3,7 @@ import { beforeEach, test } from "node:test"; import { doProtocolDataUrl } from "../helpers/load-do-protocol.js"; import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; +import { readJsonResponse } from "../helpers/response-json.js"; import { importSpecifierReplacements, moduleDataUrl, @@ -257,6 +258,80 @@ test("do-runtime alarm dispatch endpoint invokes the local alarm shim path", asy assert.equal(fetchCall.input.url, "https://do-runtime.internal/invoke"); }); +test("do-runtime alarm dispatch delegates object identity validation", async () => { + const hostFetches = /** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches); + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/alarms/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + ns: "tenant", + worker: "alarms", + version: "v7", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + objectName: "\ud800", + retryCount: 0, + token: "row-token", + }), + }), env()); + + const body = await readJsonResponse(response, 400, "unpaired surrogate objectName"); + assert.equal(body.error, "invalid_request"); + assert.equal(hostFetches.length, 0); +}); + +test("do-runtime alarm dispatch requires retryCount and token before dispatch", async () => { + const hostFetches = /** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches); + const validBody = { + ns: "tenant", + worker: "alarms", + version: "v7", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + objectName: "alice", + retryCount: 0, + token: "row-token", + }; + for (const { label, field, value } of [ + { label: "missing retryCount", field: "retryCount", value: undefined }, + { label: "null retryCount", field: "retryCount", value: null }, + { label: "missing token", field: "token", value: undefined }, + { label: "null token", field: "token", value: null }, + ]) { + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/alarms/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ...validBody, [field]: value }), + }), env()); + + const body = await readJsonResponse(response, 400, label); + assert.equal(body.error, "invalid_request", label); + assert.equal(hostFetches.length, 0, label); + } +}); + +test("do-runtime alarm dispatch delegates retryCount validation before dispatch", async () => { + const hostFetches = /** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches); + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/alarms/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + ns: "tenant", + worker: "alarms", + version: "v7", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + objectName: "alice", + retryCount: "2", + token: "row-token", + }), + }), env()); + + const body = await readJsonResponse(response, 400, "non-number retryCount"); + assert.equal(body.error, "invalid_request"); + assert.equal(hostFetches.length, 0); +}); + test("do-runtime alarm dispatch rejects non-canonical versions from the shared fixture", async () => { for (const { tag, parsed } of versionFixture.cases) { if (parsed != null) continue; diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 76ee4bc..ebc7b29 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -10,6 +10,7 @@ import { retryableOwnerRaceResponse, staleDoOwnerHintResponse, } from "../../runtime/_wdl-do-transport.js"; +import { DO_HOST_SHARD_COUNT, MAX_ID_BYTES } from "../../do-runtime/protocol/wire-grammar.js"; import { doOwnerHintHeaders, doOwnershipErrorHeaders, @@ -38,8 +39,11 @@ const { readJsonBody, } = await loadDoProtocol(); const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); +const doAlarmIdentityFixture = /** @type {any} */ ( + readRepositoryJson("tests/fixtures/do-alarm-identity.json") +); -test("DO ownership errors stay aligned with runtime hint and retry handling", async () => { +test("DO ownership errors stay aligned with runtime hint and retry handling", () => { const ownershipCodes = new Set(Object.values(DO_OWNERSHIP_CODE)); /** @type {Set} */ const ownerRaceRetryCodes = new Set([ @@ -51,16 +55,16 @@ test("DO ownership errors stay aligned with runtime hint and retry handling", as ]); for (const code of ownershipCodes) { const untrusted = Response.json({ error: code }, { status: 503 }); - assert.equal(await staleDoOwnerHintResponse(untrusted), false, code); - assert.equal(await retryableOwnerRaceResponse(untrusted), false, code); + assert.equal(staleDoOwnerHintResponse(untrusted), false, code); + assert.equal(retryableOwnerRaceResponse(untrusted), false, code); const response = Response.json({ error: code }, { status: 503, headers: doOwnershipErrorHeaders(code), }); - assert.equal(await staleDoOwnerHintResponse(response), true, code); + assert.equal(staleDoOwnerHintResponse(response), true, code); assert.equal( - await retryableOwnerRaceResponse(response), + retryableOwnerRaceResponse(response), ownerRaceRetryCodes.has(code), code ); @@ -566,14 +570,17 @@ test("normalizes do websocket connect request from internal headers", () => { }); test("rejects invalid do alarm retry counts", () => { - assert.throws( - () => normalizeDoInvokeRequest({ - ...BASE_BODY, - kind: "alarm", - alarm: { retryCount: -1 }, - }), - /alarm\.retryCount must be a non-negative integer/ - ); + for (const retryCount of [-1, 1.5, "2", true, false, "", [], [2]]) { + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + kind: "alarm", + alarm: { retryCount }, + }), + /alarm\.retryCount must be a non-negative integer/, + `retryCount=${JSON.stringify(retryCount)}` + ); + } }); test("rejects invalid owner fence generation", () => { @@ -599,6 +606,37 @@ test("rejects invalid class names", () => { ); }); +/** @param {string | { repeat: string, count: number }} value */ +function fixtureString(value) { + return typeof value === "string" ? value : value.repeat.repeat(value.count); +} + +test("DO alarm ingress matches the cross-language identity fixture", () => { + assert.equal(doAlarmIdentityFixture.maxBytes, MAX_ID_BYTES); + assert.equal(doAlarmIdentityFixture.hostShardCount, DO_HOST_SHARD_COUNT); + for (const entry of doAlarmIdentityFixture.cases) { + const input = { + ...BASE_BODY, + kind: "alarm", + doStorageId: fixtureString(entry.doStorageId), + className: fixtureString(entry.className), + objectName: fixtureString(entry.objectName), + alarm: { retryCount: 0, token: fixtureString(entry.token) }, + }; + if (entry.valid) { + assert.doesNotThrow(() => normalizeDoInvokeRequest(input), entry.name); + } else { + assert.throws( + () => normalizeDoInvokeRequest(input), + (err) => err instanceof DoRuntimeError && + err.status === 400 && + err.code === "invalid_request", + entry.name + ); + } + } +}); + test("rejects bare numeric versions", () => { assert.throws( () => normalizeDoInvokeRequest({ ...BASE_BODY, version: "1" }), @@ -760,6 +798,19 @@ test("rejects invalid header names and control characters", () => { ); }); +test("rejects unpaired surrogates in DO object identities", () => { + for (const objectName of ["\ud800", "\udc00"]) { + assert.throws( + () => normalizeDoInvokeRequest({ ...BASE_BODY, objectName }), + /objectName must contain well-formed Unicode/, + ); + assert.throws( + () => hostIdForObject(BASE_BODY.doStorageId, BASE_BODY.className, objectName), + /objectName must contain well-formed Unicode/, + ); + } +}); + test("maps invalid JSON bodies to protocol errors", async () => { await assert.rejects( readJsonBody(new Request("https://demo.workers.example/", { method: "POST", body: "{" })), diff --git a/tests/unit/redis-lock.test.js b/tests/unit/redis-lock.test.js index 185c826..b6abdde 100644 --- a/tests/unit/redis-lock.test.js +++ b/tests/unit/redis-lock.test.js @@ -37,37 +37,6 @@ test("token lock owner acquires and renews with the canonical Redis options", as ]); }); -test("token lock owner supports session-local transactional acquisition", async () => { - /** @type {unknown[][]} */ - const calls = []; - const multi = { - /** @param {string} key @param {string} token @param {Record} options */ - set(key, token, options) { - calls.push(["set", key, token, options]); - return this; - }, - async exec() { - calls.push(["exec"]); - return ["OK"]; - }, - }; - const client = { - async set() { throw new Error("direct SET must not run"); }, - async delIfEq() { return 1; }, - multi() { return multi; }, - }; - const lock = { key: "lock:auth", token: "token" }; - - assert.equal(await acquireTokenLock(client, lock, { - ttlSeconds: 30, - transactional: true, - }), true); - assert.deepEqual(calls, [ - ["set", "lock:auth", "token", { nx: true, ttl: 30 }], - ["exec"], - ]); -}); - test("token lock release is token-scoped and never masks the primary outcome", async () => { /** @type {unknown[][]} */ const seen = []; diff --git a/tests/unit/redis-session.test.js b/tests/unit/redis-session.test.js index 863fa61..b08d3a0 100644 --- a/tests/unit/redis-session.test.js +++ b/tests/unit/redis-session.test.js @@ -379,6 +379,17 @@ test("RedisSession.hSet supports object form", async () => { ); }); +test("RedisSession.set supports atomic lock options on the held socket", async () => { + const socket = makeFakeSocket([bytes("+OK\r\n")]); + const { connect } = scriptedConnect(socket); + const session = new RedisSession("x", { connect }); + await session.open(); + + assert.equal(await session.set("lock", "token", { nx: true, ttl: 30 }), "OK"); + assert.match(decode(socket._writes[0]), /SET\r\n.*lock\r\n.*token\r\n.*EX\r\n.*30\r\n.*NX\r\n/s); + await session.close(); +}); + test("RedisSession.sAdd/sRem accept scalar or array", async () => { const socket = makeFakeSocket([bytes(":1\r\n:2\r\n")]); const { connect } = scriptedConnect(socket); diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index cdd65c6..bb461bd 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -8,6 +8,7 @@ import { } from "../../runtime/do-client.js"; import { MAX_DO_REQUEST_HEADER_BYTES, + dispatchDoInvokeWithHintCache, fetchInvokeInit, ownerHintFromHeaders, requestSpec, @@ -1389,6 +1390,80 @@ test("DurableObjectNamespace direct backend ignores hints attached to owner-race assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); }); +test("DO owner-race router retry failures do not trigger another replay", async (t) => { + await t.test("fresh owner response", async () => { + let routerCalls = 0; + let ownerCalls = 0; + const ownerMetadata = new Headers(doOwnerHintHeaders()); + ownerMetadata.delete("x-wdl-do-owner-hint"); + + await assert.rejects( + dispatchDoInvokeWithHintCache({ + routerFetch: async () => { + routerCalls += 1; + if (routerCalls === 1) return doOwnerHintResponse(); + if (routerCalls === 2) throw new Error("owner-race router retry failed"); + return new Response("unexpected replay"); + }, + routerUrl: "http://do-runtime/internal/do/invoke", + ownerFetch: async () => { + ownerCalls += 1; + return Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced", ownerMetadata), + }); + }, + ownerPath: "/internal/do/invoke", + init: { method: "GET" }, + cache: new Map(), + hintKey: "room-a", + replayOwnerUnavailable: true, + }), + /owner-race router retry failed/ + ); + assert.equal(ownerCalls, 1); + assert.equal(routerCalls, 2); + }); + + await t.test("cached endpoint fallback", async () => { + const hint = ownerHintFromHeaders(doOwnerHintResponse().headers); + assert.ok(hint); + const cache = new Map(); + cache.set("room-a", hint); + let routerCalls = 0; + let ownerCalls = 0; + + await assert.rejects( + dispatchDoInvokeWithHintCache({ + routerFetch: async () => { + routerCalls += 1; + if (routerCalls === 1) { + return Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }); + } + if (routerCalls === 2) throw new Error("owner-race router retry failed"); + return new Response("unexpected replay"); + }, + routerUrl: "http://do-runtime/internal/do/invoke", + ownerFetch: async () => { + ownerCalls += 1; + return new Response("owner timeout", { status: 504 }); + }, + ownerPath: "/internal/do/invoke", + init: { method: "GET" }, + cache, + hintKey: "room-a", + replayOwnerUnavailable: true, + }), + /owner-race router retry failed/ + ); + assert.equal(ownerCalls, 1); + assert.equal(routerCalls, 2); + }); +}); + test("DurableObjectNamespace direct backend retries owner lease budget errors once", async () => { for (const error of ["owner_lease_expired", "owner_lease_too_short"]) { /** @type {any[]} */ @@ -2477,7 +2552,12 @@ test("DurableObjectNamespace replays safe GET after stale cached owner endpoint fetch: makeRecordingFetch(routerCalls, { response: () => routerCalls.length === 1 ? doOwnerHintResponse({ endpoint, generation: 11 }) - : new Response("router-ok"), + : routerCalls.length === 2 + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) + : new Response("router-ok"), }), }; const ownerNetwork = { @@ -2501,8 +2581,9 @@ test("DurableObjectNamespace replays safe GET after stale cached owner endpoint assert.equal(await ns.get(id).fetch("https://demo.workers.example/two").then((r) => r.text()), "router-ok"); assert.equal(ownerCalls.length, 2); - assert.equal(routerCalls.length, 2); + assert.equal(routerCalls.length, 3); assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); + assert.equal(headerValue(routerCalls[2].init.headers, "x-wdl-do-accept-owner-hint"), null); }); test("DurableObjectNamespace drops stale cached owner hints after owner lease budget errors", async () => { @@ -2551,8 +2632,15 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu } }); -test("DurableObjectNamespace facade rejects foreign ids", () => { +test("DurableObjectNamespace facade rejects foreign ids", async () => { const ns = new DurableObjectNamespace({ fetchObject() {} }); assert.throws(() => ns.idFromName(""), /requires a non-empty string/); + for (const value of ["\ud800", "\udc00"]) { + assert.throws(() => ns.idFromName(value), /requires well-formed Unicode/); + assert.throws(() => ns.idFromString(value), /requires well-formed Unicode/); + } + await withMockedProperty(String.prototype, "isWellFormed", () => true, () => { + assert.throws(() => ns.idFromName("\ud800"), /requires well-formed Unicode/); + }); assert.throws(() => ns.get({ name: "room-a" }), /requires an id returned by this namespace/); }); diff --git a/tests/unit/runtime-workflows-client.test.js b/tests/unit/runtime-workflows-client.test.js index af4ceca..099511f 100644 --- a/tests/unit/runtime-workflows-client.test.js +++ b/tests/unit/runtime-workflows-client.test.js @@ -2,8 +2,13 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { Workflow } from "../../runtime/workflows-client.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; +const workflowLimits = /** @type {{ createBatchMax: number }} */ ( + readRepositoryJson("tests/fixtures/workflow-limits.json") +); + /** * @param {Record} [runtimeOptions] * @param {Record} [metadataOverrides] @@ -65,6 +70,7 @@ test("Workflow.create preserves runtime metadata from params override", async () assert.equal(fetchCalls, 1); assert.equal(capturedUrl, "http://workflows/internal/workflows/create"); assert.equal(capturedInit?.method, "POST"); + assert.equal(new Headers(capturedInit?.headers).get("x-request-id"), "runtime-request"); assert.equal(typeof capturedInit?.body, "string"); assert.deepEqual(capturedRequestBody, { ns: "tenant-a", @@ -229,6 +235,16 @@ test("Workflow.create supports omitted requestId in context", async () => { assert.equal(capturedBody.requestId, null); }); +test("Workflow.createBatch limit matches the cross-language fixture", async () => { + const workflow = createWorkflowForTest(); + await assert.rejects( + () => workflow.createBatch(Array.from({ length: workflowLimits.createBatchMax + 1 }, (_, index) => ({ + id: `inst-${index}`, + }))), + new RegExp(`exceeds ${workflowLimits.createBatchMax} item limit`), + ); +}); + test("Workflow.createBatch accepts backend-skipped ids", async () => { /** @type {Record | undefined} */ let capturedBody; diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 239005f..d98a346 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -270,22 +270,6 @@ test("runtime workflow instance id grammar matches shared control grammar", () = ); }); -test("runtime workflow dispatch identity grammar mirrors shared grammar", () => { - const shared = readRepoFile("shared/ns-pattern.js"); - assert.equal( - extractRegex("runtime/lib.js", "ROUTE_NS_RE"), - `/^(?:${extractStringConst(shared, "NS_PATTERN")}|__system__)$/` - ); - assert.deepEqual( - extractStringSetConst(readRepoFile("runtime/lib.js"), "RESERVED_TENANT_NS"), - extractStringSetConst(shared, "RESERVED_TENANT_NS") - ); - assert.equal( - extractRegex("runtime/lib.js", "WORKER_NAME_RE"), - extractRegex("shared/ns-pattern.js", "WORKER_NAME_RE") - ); -}); - test("D1 object field setters stay shared across wire and transport codecs", () => { assert.match(readRepoFile("shared/d1-data-field.js"), /export function setDataField\(/); for (const file of ["shared/d1-query-wire.js", "shared/d1-transport.js"]) { @@ -911,24 +895,6 @@ test("Redis command metric allow-list covers shared Redis wrappers", () => { ); }); -test("route invalidation channel literals stay aligned across control and gateway", () => { - const shared = withoutLineComments(readRepoFile("control/shared.js")); - const routing = withoutLineComments(readRepoFile("control/routing.js")); - const gateway = withoutLineComments(readRepoFile("gateway/runtime.js")); - const routesChannel = extractStringConst(shared, "ROUTES_CHANNEL"); - const routesFlushChannel = extractStringConst(shared, "ROUTES_FLUSH_CHANNEL"); - const patternsChannel = extractStringConst(shared, "PATTERNS_CHANNEL"); - - assert.equal(extractStringConst(routing, "ROUTES_CHANNEL"), routesChannel); - assert.equal(extractStringConst(routing, "PATTERNS_CHANNEL"), patternsChannel); - assert.match( - gateway, - new RegExp(`\\[\\s*"${RegExp.escape(routesChannel)}",\\s*"${RegExp.escape(routesFlushChannel)}",\\s*"${RegExp.escape(patternsChannel)}"\\s*\\]`) - ); - assert.match(gateway, new RegExp(`channel === "${RegExp.escape(routesFlushChannel)}"`)); - assert.match(gateway, new RegExp(`channel === "${RegExp.escape(patternsChannel)}"`)); -}); - test("declared-host Redis key literals stay aligned across control, gateway, and docs", () => { const version = withoutLineComments(readRepoFile("shared/version.js")); const shared = withoutLineComments(readRepoFile("control/shared.js")); @@ -2520,16 +2486,6 @@ test("internal binary protocol content-types stay centralized in transport const test("DO owner hints are trusted only from do-runtime headers", () => { const runtime = readRepoFile("runtime/_wdl-do-transport.js"); const doRuntime = readRepoFile("do-runtime/index.js"); - const match = runtime.match( - /export async function ownerHintFromResponse\(response\) \{[^]*?\n}\n\n(?:\/\*\*[^]*?\*\/\n)?export async function followOwnerHint/ - ); - assert.ok(match, "runtime DO transport must keep ownerHintFromResponse next to followOwnerHint"); - const ownerHintFromResponse = match[0]; - - assert.match(ownerHintFromResponse, /const headers = responseHeaders\(response\)/); - assert.match(ownerHintFromResponse, /ownerHintFromHeaders\(headers\)/); - assert.doesNotMatch(ownerHintFromResponse, /response\.clone\(\)\.json\(\)/); - assert.doesNotMatch(ownerHintFromResponse, /\.json\(\)/); const wrapper = doRuntime.match(/function withOwnerHintHeaders\(response, owner\) \{[^]*?\n}\n\n/)?.[0] || ""; assert.match(wrapper, /headers\.delete\(DO_OWNER_HINT_CONTROL_HEADER\)/); diff --git a/tests/unit/version.test.js b/tests/unit/version.test.js index 0ca53b7..8db377f 100644 --- a/tests/unit/version.test.js +++ b/tests/unit/version.test.js @@ -3,13 +3,18 @@ import assert from "node:assert/strict"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { DECLARED_HOSTS_KEY, + DO_OWNER_SCOPE_PREFIX, HOST_DECLARATIONS_SCAN_PATTERN, HOSTS_SCAN_PATTERN, NAMESPACES_KEY, + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + ROUTES_FLUSH_CHANNEL, VERSION_DELETE_LOCK_KIND, WHOLE_DELETE_LOCK_KIND, bundleKey, deleteLockKey, + doOwnerScopeScanPatternForStorage, doStorageIdKey, formatDeleteLockToken, formatVersion, @@ -82,6 +87,9 @@ test("route-plane registry key helpers compose and parse canonical keys", () => assert.equal(namespaceFromHostsKey("routes:demo"), ""); assert.equal(nsHostsKey("demo"), "ns-hosts:demo"); assert.equal(hostDeclarationsKey("app.example"), "host-declarations:app.example"); + assert.equal(ROUTES_CHANNEL, "routes:invalidate"); + assert.equal(ROUTES_FLUSH_CHANNEL, "routes:flush"); + assert.equal(PATTERNS_CHANNEL, "patterns:invalidate"); }); test("nextVersionKey composes the worker version counter key", () => { @@ -90,6 +98,11 @@ test("nextVersionKey composes the worker version counter key", () => { test("worker lifecycle key helpers compose canonical keys", () => { assert.equal(doStorageIdKey("demo", "hello"), "worker:do-storage:demo:hello"); + assert.equal(DO_OWNER_SCOPE_PREFIX, "do:owner:scope:"); + assert.equal( + doOwnerScopeScanPatternForStorage("do_abc"), + "do:owner:scope:do_abc%3A*" + ); assert.equal(deleteLockKey("demo", "hello"), "worker-delete-lock:demo:hello"); }); From 77533a2dd7a593d377497d9ba144d7cf93b0df26 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Thu, 16 Jul 2026 21:08:04 +0800 Subject: [PATCH 14/23] Fence workflow restarts and tighten worker contracts Fence version deletion across workflow restart handoff Preserve definitions-only workers and fail closed on malformed projections Make request context outermost-only and enforce forward-only worker bounds Return subnet validation to Terraform and AWS provider boundaries Signed-off-by: Lu Zhang --- CHANGELOG.md | 6 +- control/bindings.js | 10 +- control/bundle.js | 40 +++- control/handlers/delete-plan.js | 1 + control/handlers/delete.js | 46 +++- control/handlers/deploy.js | 64 ++++-- control/handlers/versions.js | 10 +- control/handlers/worker-secrets.js | 5 +- control/handlers/workers.js | 8 +- control/handlers/workflows.js | 127 ++++++++--- control/routing.js | 39 +++- do-runtime/config.capnp | 1 + docs/compatibility.md | 2 +- docs/compatibility.zh.md | 2 +- docs/modules/cli.md | 11 +- docs/modules/cli.zh.md | 6 +- docs/modules/control-auth.md | 2 +- docs/modules/control-auth.zh.md | 2 +- docs/modules/durable-objects.md | 7 +- docs/modules/durable-objects.zh.md | 4 +- docs/modules/gateway.md | 7 +- docs/modules/gateway.zh.md | 2 +- docs/modules/infra.md | 6 - docs/modules/infra.zh.md | 4 +- docs/modules/log-tail-observability.md | 5 +- docs/modules/log-tail-observability.zh.md | 2 +- docs/modules/runtime.md | 20 +- docs/modules/runtime.zh.md | 4 +- docs/modules/workflows.md | 15 +- docs/modules/workflows.zh.md | 6 +- docs/redis-key-layout.md | 16 +- docs/redis-key-layout.zh.md | 7 +- docs/source-map.md | 4 +- docs/source-map.zh.md | 4 +- runtime/_wdl-do-transport.js | 21 +- runtime/_wdl-request-id.js | 50 ++++- runtime/config-system.capnp | 1 + runtime/config-user.capnp | 1 + runtime/do-client.js | 18 +- runtime/lib.js | 32 ++- runtime/load/wrapper-generate.js | 65 +++--- rust/workflows/src/api.rs | 7 +- rust/workflows/src/api/active_export.rs | 45 +--- rust/workflows/src/api/lifecycle/cleanup.rs | 31 ++- rust/workflows/src/api/lifecycle/restart.rs | 72 +++++- rust/workflows/src/api/pending_restart.rs | 212 ++++++++++++++++++ rust/workflows/src/keys.rs | 4 + rust/workflows/src/tests.rs | 12 +- shared/ns-pattern.js | 32 ++- shared/workerd-compat-flags.js | 16 +- terraform/README.md | 10 - terraform/foundation/main.tf | 37 +-- terraform/main.tf | 48 +--- terraform/modules/compute/variables.tf | 7 - terraform/variables.tf | 9 +- test-workers/do-rpc/src/index.js | 21 +- tests/helpers/runtime-injection-sources.js | 2 +- .../integration/durable-objects-core.test.js | 9 + tests/unit/control-delete-handler.test.js | 127 ++++++++++- tests/unit/control-deploy-watch.test.js | 54 +++++ tests/unit/control-handlers-workers.test.js | 20 +- tests/unit/control-handlers-workflows.test.js | 123 ++++++++-- tests/unit/control-lib.test.js | 76 ++++++- tests/unit/control-routing.test.js | 90 ++++++++ .../control-secret-envelope-handlers.test.js | 38 +++- tests/unit/observability.test.js | 27 +++ tests/unit/runtime-bindings-do.test.js | 8 +- tests/unit/runtime-do-client.test.js | 19 +- tests/unit/runtime-lib.test.js | 47 +++- tests/unit/runtime-load.test.js | 39 +--- tests/unit/runtime-wrapper-generate.test.js | 122 +++++++--- tests/unit/style-contracts.test.js | 18 ++ 72 files changed, 1608 insertions(+), 457 deletions(-) create mode 100644 rust/workflows/src/api/pending_restart.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 33205ca..1aff099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,14 @@ - Updated maintenance dependencies and image baselines: workerd runtime images now use distroless `base-debian13`, local and Kubernetes Valkey use `valkey/valkey:9.1-alpine`, and root development tooling moved to current patch/minor releases. - Consolidated duplicated Control request/error/retry paths, Redis locks and key grammars, queue and cron projections, S3 retry policy, observability helpers, and Rust sidecar contracts into canonical owners with shared behavioral and cross-language fixture coverage. -- Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and D1/R2/DO facades resist tenant prototype mutation, owner records and forwarding targets require service-specific private endpoints, and DO ownership retries rely on private do-runtime markers without replaying unknown-outcome non-idempotent requests. -- Tightened forward-only input contracts: internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. +- Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and D1/R2/DO facades resist tenant prototype mutation, owner records and forwarding targets require service-specific private endpoints, ordinary nested DO calls cannot override the parent request id through request headers or public facade options, and DO ownership retries rely on private do-runtime markers without replaying unknown-outcome non-idempotent requests. +- Tightened forward-only input contracts: dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01` and error-serialization flags that conflict with the effective date or WDL's required enhanced serialization, internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, DO class names are capped so every shard fits the host-id limit, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. - Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, worker-version, scheduler-projection, and workflow-limit fixtures. - Normalized `PLATFORM_DOMAIN` across routing tiers, stopped `/whoami` from advertising the internal `workers.local` fallback as a public namespace URL, and aligned Compose/Kubernetes/test wiring with the consolidated owners. - Removed the unreachable DO inline worker-code test hook and its Terraform switch; do-runtime invoke paths now accept only canonical persisted bundle identities and reject non-canonical or oversized host ids. -- Added Terraform plan-time validation for application subnets: every supplied subnet must belong to the configured VPC, use an RFC1918 or `100.64.0.0/10` IPv4 CIDR, and occupy a distinct Availability Zone. Existing unsupported subnet selections now fail during planning. - Aligned scheduler and Workflows Compose, Kubernetes, and ECS stop windows with their 25-second application drain so rollout and scale-in do not terminate in-flight work before the configured drain completes. - Upgraded `@wdl-dev/aws-sigv4` to 3.0.1; WDL's S3-only URL-input paths preserve their existing signatures while adopting stable request snapshots, fail-closed redirects, lowercase region validation, and non-blocking response cleanup. +- Closed workflow restart/version-delete races, made malformed workflow and pattern projections fail closed, and ensured whole-worker delete removes orphan workflow definitions. ## wdl.20260701.1 - 2026-07-08 diff --git a/control/bindings.js b/control/bindings.js index 493e568..fc34ad3 100644 --- a/control/bindings.js +++ b/control/bindings.js @@ -13,6 +13,7 @@ import { RESERVED_OBJECT_KEYS, RESERVED_TENANT_NS, WDL_RESERVED_ENTRYPOINT_RE, + MAX_DO_CLASS_NAME_BYTES, isValidJsIdentifier, isValidJsClassDeclarationName, } from "shared-ns-pattern"; @@ -142,9 +143,14 @@ const BINDING_VALIDATORS = Object.assign(Object.create(null), /** @type {Record< }, do(b, name) { const className = b.className; - if (typeof className !== "string" || !isValidJsClassDeclarationName(className)) { + if ( + typeof className !== "string" || + !isValidJsClassDeclarationName(className) || + className.length > MAX_DO_CLASS_NAME_BYTES + ) { throw new Error( - `bindings.${name}: do className must be a valid JS class declaration name, got ${JSON.stringify(className)}` + `bindings.${name}: do className must be a valid JS class declaration name of at most ` + + `${MAX_DO_CLASS_NAME_BYTES} bytes, got ${JSON.stringify(className)}` ); } assertNotRuntimeReservedEntrypoint(`bindings.${name}`, className); diff --git a/control/bundle.js b/control/bundle.js index d2d19b0..a502e7e 100644 --- a/control/bundle.js +++ b/control/bundle.js @@ -12,7 +12,14 @@ import { isValidJsClassDeclarationName, validateModulePath, } from "shared-ns-pattern"; -import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; +import { + DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE, + ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE, + ENHANCED_ERROR_SERIALIZATION_FLAG, + LEGACY_ERROR_SERIALIZATION_FLAG, + MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, + firstWorkerdExperimentalCompatFlag, +} from "shared-workerd-compat-flags"; import { normalizeBindings, validateBindings } from "control-bindings"; import { parseWorkerdDependencyVersion } from "control-lib"; import PACKAGE_JSON_SOURCE from "wdl-package-json-source"; @@ -79,6 +86,11 @@ export function validateCompatibilityDate(value, today = new Date()) { ) { throw new Error(`compatibilityDate must be a real calendar date, got ${JSON.stringify(value)}`); } + if (value < MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE) { + throw new Error( + `compatibilityDate ${value} is older than WDL supports (${MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE})` + ); + } const todayUtc = utcDateString(today); if (value > todayUtc) { throw new Error(`compatibilityDate ${value} must not be later than today UTC (${todayUtc})`); @@ -141,8 +153,8 @@ export function normalizeModule(value) { // like { compatibilityFlags: "nodejs_compat" } or { flag: true } drops // the user's declaration and leaves only the platform floor — looks // healthy, isn't. -/** @param {unknown} flags */ -function validateCompatibilityFlags(flags) { +/** @param {unknown} flags @param {string} compatibilityDate */ +function validateCompatibilityFlags(flags, compatibilityDate) { if (flags === undefined || flags === null) return; if (!Array.isArray(flags)) { throw new Error( @@ -164,6 +176,23 @@ function validateCompatibilityFlags(flags) { `compatibilityFlags contains experimental workerd flag ${JSON.stringify(experimentalFlag)}, which WDL does not support for tenant workers` ); } + if (flags.includes(LEGACY_ERROR_SERIALIZATION_FLAG)) { + throw new BundleConfigError( + 400, + "compatibility_flag_unsupported", + `${JSON.stringify(LEGACY_ERROR_SERIALIZATION_FLAG)} is not supported because WDL requires enhanced error serialization` + ); + } + if ( + compatibilityDate >= ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE && + flags.includes(ENHANCED_ERROR_SERIALIZATION_FLAG) + ) { + throw new BundleConfigError( + 400, + "compatibility_flag_unsupported", + `${JSON.stringify(ENHANCED_ERROR_SERIALIZATION_FLAG)} became the workerd default on ${ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE} and must not be specified for compatibilityDate ${compatibilityDate}` + ); + } } /** @param {unknown} workflows */ @@ -287,7 +316,10 @@ export function prepareBundle(mainModule, rawModules, extras = {}) { } validateBindings(extras.bindings); const compatibilityDate = validateCompatibilityDate(extras.compatibilityDate); - validateCompatibilityFlags(extras.compatibilityFlags); + validateCompatibilityFlags( + extras.compatibilityFlags, + compatibilityDate ?? DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE + ); const vars = normalizeVars(extras.vars); // Null-proto: any path slipping past validateModulePath can't take // the __proto__ setter path on assignment. diff --git a/control/handlers/delete-plan.js b/control/handlers/delete-plan.js index eb1583f..183aca5 100644 --- a/control/handlers/delete-plan.js +++ b/control/handlers/delete-plan.js @@ -35,6 +35,7 @@ import { workerSecretsKey } from "shared-secret-keys"; * hostsLosingNsOwnership: string[], * namespaceStillActive: boolean, * hasWorkerSecrets: boolean, + * hasWorkflowDefs: boolean, * }} DeleteInputs * @typedef {import("control-lifecycle-indexes").RedisMulti} RedisMulti * @typedef {{ routes: string, routesFlush: string, patterns: string }} DeleteChannels diff --git a/control/handlers/delete.js b/control/handlers/delete.js index ef4786e..e0ff591 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -19,6 +19,7 @@ import { doObjectRegistryKey, doOwnerScopeScanPatternForStorage, doStorageIdKey, + workflowDefsKey, extractD1Refs, extractOutgoingRefs, formatReferrerBlocker, bundleAssetPrefix, @@ -70,6 +71,7 @@ const DELETE_COLLECT_READ_CONCURRENCY = 16; * hostsLosingNsOwnership: string[], * namespaceStillActive: boolean, * hasWorkerSecrets: boolean, + * hasWorkflowDefs: boolean, * }} DeleteInputs * @typedef {{ retained: boolean, objects: number, doStorageId?: string }} DoStorageRetention * @typedef {{ taskId: string, prefixes: string[], source: Record, nowMs?: number }} CleanupIntent @@ -89,6 +91,19 @@ const DELETE_COLLECT_READ_CONCURRENCY = 16; class WholeDeleteError extends ControlAbort {} +/** @param {unknown} raw @param {string} host @param {string} slot */ +function requirePatternProjection(raw, host, slot) { + const projection = decodePatternProjection(raw); + if (!projection) { + throw new WholeDeleteError(500, "corrupt_pattern_projection", { + host, + slot, + stage: "pattern_projection_parse", + }); + } + return projection; +} + /** @param {string} ns @param {string} name @returns {never} */ function throwWholeDeleteContention(ns, name) { throw new WholeDeleteError(503, "whole_delete_contention", { @@ -273,7 +288,8 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { const noop = !collected.activeVersion && collected.retainedVersions.length === 0 && - !collected.hasWorkerSecrets; + !collected.hasWorkerSecrets && + !collected.hasWorkflowDefs; log("info", "worker_delete_dry_run", { request_id: requestId, @@ -296,6 +312,7 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { queueConsumersRemoved: collected.queueConsumerKeys.length, namespaceStillActive: collected.namespaceStillActive, hasWorkerSecrets: collected.hasWorkerSecrets, + hasWorkflowDefs: collected.hasWorkflowDefs, durableObjects: { storageRetention: describeDoStorageRetention(collected), }, @@ -342,7 +359,8 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log, const hasWorkerLifecycle = collected.activeVersion || collected.retainedVersions.length > 0 || - collected.hasWorkerSecrets; + collected.hasWorkerSecrets || + collected.hasWorkflowDefs; if (workflowBlocker && !hasWorkerLifecycle) { throw workflowBlocker; } @@ -492,6 +510,7 @@ async function deleteResidualDoRedis(redis, collected, lockToken) { routesKey(collected.ns), workerVersionsKey(collected.ns, collected.name), workerSecretsKey(collected.ns, collected.name), + workflowDefsKey(collected.ns, collected.name), storageKey, ...collected.doOwnerKeys, ); @@ -503,7 +522,8 @@ async function deleteResidualDoRedis(redis, collected, lockToken) { const activeVersion = await iso.hGet(routesKey(collected.ns), collected.name); const retainedVersions = await iso.zRange(workerVersionsKey(collected.ns, collected.name), 0, -1); const hasWorkerSecrets = (await iso.exists(workerSecretsKey(collected.ns, collected.name))) > 0; - if (activeVersion || retainedVersions.length > 0 || hasWorkerSecrets) { + const hasWorkflowDefs = (await iso.exists(workflowDefsKey(collected.ns, collected.name))) > 0; + if (activeVersion || retainedVersions.length > 0 || hasWorkerSecrets || hasWorkflowDefs) { await iso.unwatch(); throw new DriftSignal("worker lifecycle appeared during residual cleanup"); } @@ -651,6 +671,7 @@ async function collectDeleteInputs({ redis, ns, name }) { const namespaceStillActive = otherActive.length > 0; const hasWorkerSecrets = (await redis.exists(workerSecretsKey(ns, name))) > 0; + const hasWorkflowDefs = (await redis.exists(workflowDefsKey(ns, name))) > 0; return { ns, @@ -671,6 +692,7 @@ async function collectDeleteInputs({ redis, ns, name }) { hostsLosingNsOwnership, namespaceStillActive, hasWorkerSecrets, + hasWorkflowDefs, }; } @@ -757,9 +779,9 @@ function findHostsLosingNsOwnership(ns, activeRoutes, hosts, patternRecords) { ); let siblingOwnsHost = false; for (const [slot, raw] of Object.entries(patternRecords[i] || {})) { - if (ourSlots.has(slot) || typeof raw !== "string") continue; - const projection = decodePatternProjection(raw); - if (projection?.ns === ns) { + if (ourSlots.has(slot)) continue; + const projection = requirePatternProjection(raw, host, slot); + if (projection.ns === ns) { siblingOwnsHost = true; break; } @@ -798,6 +820,7 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected doStorageIdKey(ns, name), workerVersionsKey(ns, name), workerSecretsKey(ns, name), + workflowDefsKey(ns, name), ...(collected.doObjectRegistry ? [collected.doObjectRegistry] : []), ...collected.doOwnerKeys, ...collected.affectedHosts.map((h) => patternsKey(h)), @@ -851,6 +874,11 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected await iso.unwatch(); throw new DriftSignal("secrets presence changed during delete"); } + const curHasWorkflowDefs = (await iso.exists(workflowDefsKey(ns, name))) > 0; + if (curHasWorkflowDefs !== collected.hasWorkflowDefs) { + await iso.unwatch(); + throw new DriftSignal("workflow definitions presence changed during delete"); + } if (collected.doObjectRegistry) { const curDoObjectMembers = await iso.sMembers(collected.doObjectRegistry); if (!arraysShallowEqual(curDoObjectMembers.toSorted(), collected.doObjectMembers.toSorted())) { @@ -887,9 +915,9 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected for (const r of collected.activeRoutes) { const held = await iso.hGet(patternsKey(r.host), r.slot); - if (!held) continue; - const parsed = decodePatternProjection(held); - if (parsed && (parsed.ns !== ns || parsed.worker !== name)) { + if (held == null) continue; + const parsed = requirePatternProjection(held, r.host, r.slot); + if (parsed.ns !== ns || parsed.worker !== name) { await iso.unwatch(); throw new WholeDeleteError(409, "projection_drift", { host: r.host, slot: r.slot, owner: parsed, diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index d0386ec..a9d103b 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -42,7 +42,12 @@ import { } from "control-topology"; import { formatVersion, parseVersion, bundleKey, nextVersionKey, routesKey } from "shared-version"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; -import { isReservedNs, isValidRouteNs, ROUTES_ALLOWED_RESERVED_NS } from "shared-ns-pattern"; +import { + isReservedNs, + isValidRouteNs, + ROUTES_ALLOWED_RESERVED_NS, + WORKFLOW_KEY_RE, +} from "shared-ns-pattern"; import { putAsset, inferContentType } from "control-s3"; import { generateAssetsToken, assetsPrefixFor } from "shared-assets-token"; import { resolveDatabaseRefFrom } from "control-d1-store"; @@ -1126,26 +1131,16 @@ async function materializeCommittedMetadata(iso, { ns, name, prepared, resolvedD const workflowDefUpdates = []; if (Array.isArray(committedMeta.workflows) && committedMeta.workflows.length) { const defsKey = workflowDefsKey(ns, name); - const rawDefs = await iso.hMGet(defsKey, committedMeta.workflows.map((workflow) => workflow.name)); - for (const [index, workflow] of committedMeta.workflows.entries()) { - const rawDef = rawDefs[index]; + const existingDefRaws = await iso.hMGet( + defsKey, + committedMeta.workflows.map((workflow) => workflow.name) + ); + for (let index = 0; index < committedMeta.workflows.length; index += 1) { + const workflow = committedMeta.workflows[index]; + const existingDef = parsePersistedWorkflowDef(workflow.name, existingDefRaws[index]); let workflowKey; - if (rawDef) { - /** @type {JsonObject} */ - let parsed; - try { - parsed = /** @type {JsonObject} */ (JSON.parse(rawDef)); - } catch { - throw new DeployAbort(500, "workflow_definition_corrupt", { - workflow: workflow.name, - }); - } - if (typeof parsed?.workflowKey !== "string" || !/^wf_[0-9a-f]{32}$/.test(parsed.workflowKey)) { - throw new DeployAbort(500, "workflow_definition_corrupt", { - workflow: workflow.name, - }); - } - workflowKey = parsed.workflowKey; + if (existingDef) { + workflowKey = existingDef.workflowKey; } else { workflowKey = newWorkflowKey(); } @@ -1175,6 +1170,35 @@ async function materializeCommittedMetadata(iso, { ns, name, prepared, resolvedD return { committedMeta, workflowDefUpdates, doStorageId }; } +/** + * @param {string} workflowName + * @param {string | null | undefined} rawDef + * @returns {{ workflowKey: string } | null} + */ +function parsePersistedWorkflowDef(workflowName, rawDef) { + if (rawDef == null) return null; + /** @type {JsonObject | null} */ + let parsed; + try { + parsed = typeof rawDef === "string" + ? /** @type {JsonObject} */ (JSON.parse(rawDef)) + : null; + } catch { + parsed = null; + } + if ( + !parsed || + Array.isArray(parsed) || + typeof parsed.workflowKey !== "string" || + !WORKFLOW_KEY_RE.test(parsed.workflowKey) + ) { + throw new DeployAbort(500, "workflow_definition_corrupt", { + workflow: workflowName, + }); + } + return { workflowKey: parsed.workflowKey }; +} + /** * @param {DeployCommitMulti} multi * @param {{ diff --git a/control/handlers/versions.js b/control/handlers/versions.js index 7073ef9..0c61ea2 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -14,6 +14,7 @@ import { extractOutgoingRefs, formatReferrerBlocker, bundleAssetPrefix, parseBundleMeta, + workflowDefsKey, } from "control-lib"; import { stageD1ReferrerRemovals, @@ -168,6 +169,7 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque routesKey(ns), workerVersionsKey(ns, name), workerSecretsKey(ns, name), + workflowDefsKey(ns, name), bundleKey(ns, name, version), referrersKey(ns, name, version), ); @@ -252,6 +254,7 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque } const hasWorkerSecrets = (await iso.exists(workerSecretsKey(ns, name))) > 0; + const hasWorkflowDefs = (await iso.exists(workflowDefsKey(ns, name))) > 0; const lastRetainedNoActive = currentVersions.length === 1 && currentVersions[0] === version && @@ -271,11 +274,10 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque databaseIdFor: (ref) => /** @type {string} */ (ref.databaseId), }); if (lastRetainedNoActive) { - // Single-version DELETE must never touch worker secrets — that's - // whole-delete's job. A worker with secrets but no bundles / - // active projection stays in workers: as a secret-only entry. + // Single-version DELETE leaves non-version lifecycle state for + // whole-delete. Keep those workers discoverable through workers:. stageWorkerVersionIndexDelete(multi, ns, name); - if (!hasWorkerSecrets) { + if (!hasWorkerSecrets && !hasWorkflowDefs) { stageWorkerHidden(multi, ns, name); } } diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index bc726e3..a9434ed 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -15,6 +15,7 @@ import { import { deleteLockKey, routesKey, + workflowDefsKey, workerVersionsKey, } from "control-lib"; import { @@ -491,6 +492,7 @@ async function stageWorkerSecretForBump({ async function mutateSecretWithoutActive({ redis, ns, name, key, method, value, plaintext = null, controlEnv }) { const secretsKey = workerSecretsKey(ns, name); const nsSecretHashKey = nsSecretsKey(ns); + const defsKey = workflowDefsKey(ns, name); return await runOptimistic(redis, { attempts: MAX_SECRET_ATTEMPTS, onExhausted: () => { @@ -506,6 +508,7 @@ async function mutateSecretWithoutActive({ redis, ns, name, key, method, value, routesKey(ns), workerVersionsKey(ns, name), ]; + if (method === "DELETE") watches.push(defsKey); await iso.watch(...watches); const callerLock = await iso.get(deleteLockKey(ns, name)); @@ -527,7 +530,7 @@ async function mutateSecretWithoutActive({ redis, ns, name, key, method, value, } if (hkeys.length === 1 && hkeys[0] === key) { if (retainedVersions.length === 0) { - removeFromWorkersIndex = true; + removeFromWorkersIndex = (await iso.exists(defsKey)) === 0; } } } diff --git a/control/handlers/workers.js b/control/handlers/workers.js index 6cde730..8ed485b 100644 --- a/control/handlers/workers.js +++ b/control/handlers/workers.js @@ -1,5 +1,5 @@ import { jsonResponse, jsonError, requireControlLog, requireControlRedis } from "control-shared"; -import { routesKey, workersIndexKey, workerVersionsKey } from "control-lib"; +import { routesKey, workflowDefsKey, workersIndexKey, workerVersionsKey } from "control-lib"; import { workerSecretsKey } from "shared-secret-keys"; /** @param {{ method: string, nsName: string, requestId: string }} args */ @@ -18,8 +18,11 @@ export async function handle({ method, nsName, requestId }) { const versionKeys = names.map((name) => workerVersionsKey(nsName, name)); const secretKeys = names.map((name) => workerSecretsKey(nsName, name)); + const definitionKeys = names.map((name) => workflowDefsKey(nsName, name)); const versionsByWorker = await session.zRangeMany(versionKeys, 0, -1); - const secretFlags = await session.existsMany(secretKeys); + const stateFlags = await session.existsMany([...secretKeys, ...definitionKeys]); + const secretFlags = stateFlags.slice(0, names.length); + const definitionFlags = stateFlags.slice(names.length); return names.map((name, idx) => { const activeVersion = routesHash[name] || null; @@ -30,6 +33,7 @@ export async function handle({ method, nsName, requestId }) { versions, versionCount: versions.length, hasSecrets: Boolean(secretFlags[idx]), + hasWorkflowDefs: Boolean(definitionFlags[idx]), }; }); }); diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index 8ee32d6..dc5e034 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -18,6 +18,11 @@ import { requireControlRedis, } from "control-shared"; import { bundleKey, routesKey } from "shared-version"; +import { + BINDING_NAME_RE, + WORKFLOW_KEY_RE, + isValidJsClassDeclarationName, +} from "shared-ns-pattern"; const LIFECYCLE_ACTIONS = new Set(["pause", "resume", "restart", "terminate"]); const MAX_WORKFLOW_SNAPSHOT_ATTEMPTS = 2; @@ -216,7 +221,7 @@ function buildWorkflowDefinitionList(ns, routeEntries, metas, defsRaws) { workflowKey: workflow.workflowKey, }); } - const defs = parseWorkflowDefs(defsRaws[i]); + const defs = parseWorkflowDefs(defsRaws[i], { ns, worker }); for (const [name, def] of Object.entries(defs)) { if (activeByName.has(name)) continue; workflows.push({ @@ -270,10 +275,7 @@ async function resolveWorkflow({ redis }, ns, worker, workflowName) { /** @type {WorkflowEntry | undefined} */ let workflow = activeWorkflow; if (!workflow) { - const defs = await readWorkflowDefs({ redis }, ns, worker); - const def = Object.hasOwn(defs, workflowName) - ? defs[workflowName] - : undefined; + const def = await readWorkflowDef({ redis }, ns, worker, workflowName); if (def) { workflow = { name: workflowName, @@ -284,7 +286,7 @@ async function resolveWorkflow({ redis }, ns, worker, workflowName) { }; } } - if (!activeWorkflow && await redis.hGet(routesKey(ns), worker) !== activeVersion) continue; + if (await redis.hGet(routesKey(ns), worker) !== activeVersion) continue; if (!workflow) { throw new ControlAbort(404, "workflow_not_found", { message: `Workflow ${ns}/${worker}/${workflowName} is not exported`, @@ -330,37 +332,62 @@ async function readActiveWorkflowMeta({ redis }, ns, worker) { * @param {RedisDeps} deps * @param {string} ns * @param {string} worker - * @returns {Promise} + * @param {string} workflowName + * @returns {Promise} */ -async function readWorkflowDefs({ redis }, ns, worker) { - return parseWorkflowDefs(await redis.hGetAll(workflowDefsKey(ns, worker))); +async function readWorkflowDef({ redis }, ns, worker, workflowName) { + const raw = await redis.hGet(workflowDefsKey(ns, worker), workflowName); + return parseWorkflowDef(workflowName, raw, { ns, worker }); +} + +/** + * @param {string} name + * @param {string | null | undefined} value + * @param {{ ns: string, worker: string }} context + * @returns {RetiredWorkflowDef | null} + */ +function parseWorkflowDef(name, value, context) { + if (value == null) return null; + let parsed; + try { + parsed = typeof value === "string" ? JSON.parse(value) : null; + } catch { + parsed = null; + } + if ( + !isValidWorkflowName(name) || + !parsed || + typeof parsed !== "object" || + Array.isArray(parsed) || + typeof parsed.workflowKey !== "string" || + !WORKFLOW_KEY_RE.test(parsed.workflowKey) || + !isValidJsClassDeclarationName(parsed.className) + ) { + throw new ControlAbort(500, "corrupt_meta", { + message: `Corrupt workflow definition for ${context.ns}/${context.worker}`, + namespace: context.ns, + worker: context.worker, + stage: "workflow_defs_parse", + }); + } + return { + workflowKey: parsed.workflowKey, + className: parsed.className, + }; } /** * @param {Record} raw + * @param {{ ns: string, worker: string }} context * @returns {RetiredWorkflowDefs} */ -function parseWorkflowDefs(raw) { +function parseWorkflowDefs(raw, context) { /** @type {RetiredWorkflowDefs} */ const defs = Object.create(null); for (const [name, value] of Object.entries(raw || {})) { - if (typeof value !== "string") continue; - try { - const parsed = JSON.parse(value); - if ( - parsed && - typeof parsed.workflowKey === "string" && - typeof parsed.className === "string" - ) { - defs[name] = { - workflowKey: parsed.workflowKey, - className: parsed.className, - }; - } - } catch { - // Corrupt retired definitions are ignored here; execution paths still - // fail closed inside workflows when they validate a concrete def. - } + const parsed = parseWorkflowDef(name, value, context); + if (!parsed) continue; + defs[name] = parsed; } return defs; } @@ -373,7 +400,7 @@ function parseWorkflowDefs(raw) { * @returns {Record} */ function workflowBundleMeta(ns, worker, version, raw) { - return parseBundleMeta(raw, { + const meta = parseBundleMeta(raw, { ns, worker, version, @@ -386,6 +413,40 @@ function workflowBundleMeta(ns, worker, version, raw) { detail: reason, }), }); + const workflows = meta.workflows; + if (workflows !== undefined) { + if (!Array.isArray(workflows)) { + throw corruptWorkflowEntries(ns, worker, version); + } + const names = new Set(); + const bindings = new Set(); + const workflowKeys = new Set(); + for (const entry of workflows) { + if ( + !isActiveWorkflowMeta(entry) || + names.has(entry.name) || + bindings.has(entry.binding) || + workflowKeys.has(entry.workflowKey) + ) { + throw corruptWorkflowEntries(ns, worker, version); + } + names.add(entry.name); + bindings.add(entry.binding); + workflowKeys.add(entry.workflowKey); + } + } + return meta; +} + +/** @param {string} ns @param {string} worker @param {string} version */ +function corruptWorkflowEntries(ns, worker, version) { + return new ControlAbort(500, "corrupt_meta", { + message: `Corrupt workflow metadata for ${ns}/${worker}/${version}`, + namespace: ns, + worker, + version, + stage: "workflow_entries_parse", + }); } /** @@ -454,7 +515,7 @@ function workflowsFromMeta(meta) { meta && typeof meta === "object" ? meta : null ); if (!record || !Array.isArray(record.workflows)) return []; - return record.workflows.filter(isActiveWorkflowMeta); + return /** @type {ActiveWorkflowMeta[]} */ (record.workflows); } /** @@ -467,10 +528,10 @@ function isActiveWorkflowMeta(entry) { ); return Boolean( record && - typeof record.name === "string" && - typeof record.binding === "string" && - typeof record.className === "string" && - typeof record.workflowKey === "string", + isValidWorkflowName(record.name) && + typeof record.binding === "string" && BINDING_NAME_RE.test(record.binding) && + isValidJsClassDeclarationName(record.className) && + typeof record.workflowKey === "string" && WORKFLOW_KEY_RE.test(record.workflowKey), ); } diff --git a/control/routing.js b/control/routing.js index 58dd436..383af48 100644 --- a/control/routing.js +++ b/control/routing.js @@ -71,7 +71,7 @@ const MAX_ATTEMPTS = 5; * @typedef {{ routes?: RoutePattern[], crons?: CronSpec[], queueConsumers?: QueueConsumer[], exports?: ExportSpec[], bindings?: unknown }} BundleMeta * @typedef {Record>} HostState * @typedef {import("shared-redis").RedisMulti} RedisMulti - * @typedef {{ watch: (...keys: string[]) => Promise, unwatch: () => Promise, hGet: (key: string, field: string) => Promise, hGetMany: (pairs: Array<[string, string]>) => Promise>, hGetAll: (key: string) => Promise>, get: (key: string) => Promise, exists: (key: string) => Promise, sMIsMember: (key: string, ...members: string[]) => Promise, sMembers: (key: string) => Promise, zRange: (key: string, start: number, stop: number) => Promise, copy: (src: string, dst: string, options?: Record) => Promise, multi: () => RedisMulti }} RedisIso + * @typedef {{ watch: (...keys: string[]) => Promise, unwatch: () => Promise, hGet: (key: string, field: string) => Promise, hGetMany: (pairs: Array<[string, string]>) => Promise>, hGetAll: (key: string) => Promise>, hGetAllMany: (keys: string[]) => Promise>>, get: (key: string) => Promise, exists: (key: string) => Promise, sMIsMember: (key: string, ...members: string[]) => Promise, sMembers: (key: string) => Promise, zRange: (key: string, start: number, stop: number) => Promise, copy: (src: string, dst: string, options?: Record) => Promise, multi: () => RedisMulti }} RedisIso * @typedef {{ hGet: (key: string, field: string) => Promise, incr: (key: string) => Promise, session: (fn: (iso: RedisIso) => Promise) => Promise }} RedisClient */ @@ -87,6 +87,20 @@ export class RoutingError extends Error { } } +/** @param {unknown} raw @param {string} host @param {string} slot */ +function requirePatternProjection(raw, host, slot) { + const projection = decodePatternProjection(raw); + if (!projection) { + throw new RoutingError( + 500, + "corrupt_pattern_projection", + `Pattern projection for ${host}${slot} is corrupt`, + { host, slot } + ); + } + return projection; +} + /** @param {Array} keys @returns {string[]} */ function uniqueKeys(keys) { const unique = new Set(); @@ -373,16 +387,20 @@ async function readHostStateAndAssertRouteConflicts(iso, ns, workerName, newRout const hostState = {}; /** @type {Map }>} */ const analysisByHost = new Map(); - for (const h of affectedHosts) { - const state = await iso.hGetAll(patternsKey(h)); + const hosts = [...affectedHosts]; + const states = hosts.length > 0 + ? await iso.hGetAllMany(hosts.map((host) => patternsKey(host))) + : []; + for (let index = 0; index < hosts.length; index += 1) { + const h = hosts[index]; + const state = states[index]; hostState[h] = state; /** @type {{ slot: string, held: PatternProjection } | null} */ let foreign = null; /** @type {Map} */ const slots = new Map(); for (const [slot, raw] of Object.entries(state)) { - const held = decodePatternProjection(raw); - if (!held) continue; + const held = requirePatternProjection(raw, h, slot); slots.set(slot, held); if (!foreign && held.ns && held.ns !== ns) foreign = { slot, held }; } @@ -755,6 +773,13 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) const affectedHosts = new Set(); for (const r of routes) affectedHosts.add(r.host); await watchKeys(iso, [...affectedHosts].map((h) => patternsKey(h))); + await readHostStateAndAssertRouteConflicts( + iso, + ns, + workerName, + routes, + affectedHosts + ); // Scheduler builds x-worker-id from cron __meta__.version. const cronKey = cronWorkerKey(ns, workerName); @@ -875,8 +900,8 @@ export async function reconcileHosts(redis, ns, body, platformDomain) { if (!reverseFlags[i]) continue; const entries = await iso.hGetAll(patternsKey(h)); for (const [slot, raw] of Object.entries(entries)) { - const parsed = decodePatternProjection(raw); - if (parsed && parsed.ns === ns) { + const parsed = requirePatternProjection(raw, h, slot); + if (parsed.ns === ns) { throw new RoutingError( 409, "host_in_use", diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index 0006c91..50d5f82 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -63,6 +63,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "../runtime/bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "../runtime/bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "../runtime/_wdl-do-transport.js"), + (name = "_wdl-request-id.js", esModule = embed "../runtime/_wdl-request-id.js"), (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), # Injected DO transport uses a relative module name; host modules use the # shared bare name. Both resolve to the same shared contract owner. diff --git a/docs/compatibility.md b/docs/compatibility.md index 6693d11..8db661c 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -31,7 +31,7 @@ Each row separates four compatibility claims: |---|---|---|---|---|---| | ES module Workers and `fetch()` | Supported | Module evaluation, request dispatch, `Response`/`Request`, service binding JSRPC machinery. | Dynamic `workerLoader`, immutable version ids, wrapper-generated `env`, gateway routing, request logs, and public/private outbound separation. | An uncaught tenant `fetch()` exception maps to platform `502 runtime_error`; exception detail goes to structured logs/live tail, not the client body. | WDL does not emulate every compatibility-date behavior change; workerd is configured with the platform's enabled flags. | | WebSocket upgrade | Supported | WebSocket API and 101 response handling inside workerd. | Gateway `GatewayWsHolder` Durable Object holds public sockets and forwards to runtime/do-runtime so long-lived 101s do not live on ordinary gateway request IoContexts. | WDL optimizes for preserving long-lived gateway-held sockets, while Cloudflare can rely on its global edge session model. | Gateway rolling still drops physical client sockets; application-level resume is not implemented. | -| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Control validates that `compatibility_date` is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd, and rejects upstream `$experimental` enable flags such as `experimental` / `unsafe_module` before deploy. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; tenant workers cannot opt into upstream experimental-only flags. | +| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01`; Control also validates that the value is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd, rejects upstream `$experimental` enable flags such as `experimental` / `unsafe_module`, rejects `legacy_error_serialization`, and rejects explicit `enhanced_error_serialization` once that behavior is implied by the effective date. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; tenant workers cannot opt into upstream experimental-only flags or disable WDL's required enhanced error serialization. | | `nodejs_compat` | Partial | workerd-provided compatibility when the runtime service has the flag enabled. | CLI carries compatibility flags into metadata. | WDL exposes the workerd-enabled compatibility surface rather than a separate Node.js runtime. | This is not a full Node.js platform contract beyond workerd's enabled surface. | | Python Workers modules | Not supported | workerd has an experimental Python Workers path. | Control rejects `py` module manifests with `python_workers_unsupported`; runtime and do-runtime also fail closed for retained metadata containing `py` modules. | WDL keeps tenant bundles JavaScript/WebAssembly/data only and does not permit cold-load-time Pyodide bootstrap. | Python Workers and mixed JS/Python bundles are unsupported. | diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index dc7f5de..b350476 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -24,7 +24,7 @@ |---|---|---|---|---|---| | ES module Workers 和 `fetch()` | Supported | Module evaluation、request dispatch、`Response`/`Request`、service binding JSRPC 机制。 | Dynamic `workerLoader`、immutable version id、wrapper 生成 `env`、gateway routing、request logs、public/private outbound 隔离。 | Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error`；异常细节进入结构化日志/live tail，不进客户端 body。 | WDL 不模拟每个 compatibility date 的历史行为；workerd 按平台启用的 flags 运行。 | | WebSocket upgrade | Supported | workerd 内的 WebSocket API 和 101 response 处理。 | Gateway `GatewayWsHolder` Durable Object 承载公网 socket 并转发到 runtime/do-runtime，避免长生命周期 101 留在普通 gateway request IoContext。 | WDL 优先保留 gateway-held 长连接；Cloudflare 依赖自己的全球 edge session 模型。 | Gateway rolling 仍会断开物理 client socket；应用级 resume 未实现。 | -| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | CLI/control 保存 bundle metadata。Control 会校验 `compatibility_date` 是真实的 `YYYY-MM-DD` 日期，且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期；上游 `$experimental` enable flags（例如 `experimental` / `unsafe_module`）会在 deploy 前被拒绝。 | WDL 把 compatibility metadata 当成部署期平台 metadata，不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为；tenant worker 不能 opt into 上游 experimental-only flags。 | +| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | Dynamic Worker 会拒绝早于 `2026-04-01` 的显式 `compatibility_date`；Control 还会校验该值是真实的 `YYYY-MM-DD` 日期，且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期；上游 `$experimental` enable flags（例如 `experimental` / `unsafe_module`）、`legacy_error_serialization`，以及 effective date 已隐式启用该行为时显式声明的 `enhanced_error_serialization` 会在 deploy 前被拒绝。 | WDL 把 compatibility metadata 当成部署期平台 metadata，不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为；tenant worker 不能 opt into 上游 experimental-only flags，也不能禁用 WDL 要求的 enhanced error serialization。 | | `nodejs_compat` | Partial | runtime service 启用 flag 后由 workerd 提供的兼容 surface。 | CLI 把 compatibility flags 带入 metadata。 | WDL 暴露 workerd 已启用的兼容 surface，而不是单独 Node.js runtime。 | 不承诺超出 workerd 已启用 surface 的完整 Node.js 平台能力。 | | Python Workers modules | Not supported | workerd 有 experimental Python Workers 路径。 | Control 用 `python_workers_unsupported` 拒绝 `py` module manifest；runtime 和 do-runtime 对 retained metadata 中的 `py` module 也 fail closed。 | WDL 保持 tenant bundle 只包含 JavaScript/WebAssembly/data，不允许 cold-load 时触发 Pyodide bootstrap。 | 不支持 Python Workers 和 JS/Python 混合 bundle。 | diff --git a/docs/modules/cli.md b/docs/modules/cli.md index 63a4dd9..cacf5d0 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -149,7 +149,7 @@ Supported config surfaces: | Field | WDL behavior | |---|---| -| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit; final WorkerCode, including runtime/do-runtime-injected modules and generated workflow keys, must fit workerd's 64 MiB `workerLoader` code limit. | +| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects supplied `compatibility_date` values earlier than `2026-04-01`, as well as malformed, future, or bundled-workerd-unsupported values, before commit; final WorkerCode, including runtime/do-runtime-injected modules and generated workflow keys, must fit workerd's 64 MiB `workerLoader` code limit. | | `[vars]` | String, number, and boolean values are accepted and stringified into `env`; vars, namespace/worker secrets, and runtime-injected binding/workflow env values must fit WDL's headroomed workerd 1 MiB `workerLoader` env budget. | | `[[kv_namespaces]]` | `id` is a platform-local KV namespace id, not a Cloudflare UUID. | | `[[r2_buckets]]` | `binding` plus `bucket_name` become a namespace-scoped virtual R2 bucket under the platform S3 bucket. | @@ -172,10 +172,11 @@ would imply platform behavior WDL does not implement. The downstream CLI may expose these surfaces with its own command shape, but the platform-side behavior is fixed: -- Worker listing reads active versions, retained versions, and secret-only entries from - control. -- Worker deletion hard-deletes routes, retained versions, worker secrets, queue - consumers, and crons, then stages asset cleanup after the Redis commit. +- Worker listing reads active versions, retained versions, secret-only entries, and + workflow-definitions-only entries from control. +- Worker deletion hard-deletes routes, retained versions, worker secrets, workflow + definitions, queue consumers, and crons, then stages asset cleanup after the Redis + commit. - Version deletion hard-deletes one retained non-active version. - Secret mutation requires an explicit worker scope or namespace scope. Namespace-wide writes must not be accidental. A submitted empty string is a set secret, not unset. diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index 5d9257f..ebc1f6c 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -91,7 +91,7 @@ WDL 遵循 Wrangler selected-env 继承规则： | 字段 | WDL 行为 | |---|---| -| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`；包含 runtime/do-runtime 注入模块和生成 workflow keys 后的最终 WorkerCode 必须落在 workerd 64 MiB `workerLoader` code limit 内。 | +| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会拒绝早于 `2026-04-01` 的显式 `compatibility_date`，并在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的值；包含 runtime/do-runtime 注入模块和生成 workflow keys 后的最终 WorkerCode 必须落在 workerd 64 MiB `workerLoader` code limit 内。 | | `[vars]` | 接受 string、number、boolean，并 stringified 进 `env`；vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 必须落在 WDL 留有 headroom 的 workerd 1 MiB `workerLoader` env budget 内。 | | `[[kv_namespaces]]` | `id` 是 platform-local KV namespace id，不是 Cloudflare UUID。 | | `[[r2_buckets]]` | `binding` 加 `bucket_name` 映射为平台 S3 bucket 下的 namespace-scoped virtual R2 bucket。 | @@ -111,8 +111,8 @@ WDL 遵循 Wrangler selected-env 继承规则： 下游 CLI 可以用自己的命令形状暴露这些 surface，但平台侧行为是固定的： -- Worker list 从 control 读取 active version、retained version 和 secret-only entry。 -- Worker deletion hard-delete route、retained version、worker secret、queue consumer 和 cron，并在 Redis commit 后 stage asset cleanup。 +- Worker list 从 control 读取 active version、retained version、secret-only entry 和 workflow-definitions-only entry。 +- Worker deletion hard-delete route、retained version、worker secret、workflow definition、queue consumer 和 cron，并在 Redis commit 后 stage asset cleanup。 - Version deletion hard-delete 一个 retained non-active version。 - Secret mutation 必须显式选择 worker scope 或 namespace scope，避免误写 namespace-wide secret。提交的空字符串是已设置 secret，不是 unset。 - D1 命令管理 namespace D1 database 和 forward-only migration file。Migration filename 是 migration id；已 apply 的文件不应 rename 或编辑。 diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index d7d73d3..e69ae0e 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -50,7 +50,7 @@ Worker lifecycle: | Method | Path | Contract | |---|---|---| -| `GET` | `/ns//workers` | Lists workers with namespace-owned state, including deploy-only, active, and secret-only workers. | +| `GET` | `/ns//workers` | Lists workers with namespace-owned state, including deploy-only, active, secret-only, and workflow-definitions-only workers. Each result reports `hasSecrets` and `hasWorkflowDefs`. | | `GET` | `/ns//worker//versions` | Lists retained versions and active status. | | `POST` | `/ns//worker//deploy` | Creates a new immutable version from shorthand code or full module manifest; routes, crons, queue consumers, service refs, platform refs, assets, vars, bindings, and `exports` are version metadata. Python modules and upstream experimental compatibility flags are rejected before commit. | | `POST` | `/ns//worker//promote` | Promotes `{"version":"vN"}` through the WATCH/MULTI routing path. Host declaration failures are 403; live pattern conflicts are 409; exhausted transaction contention is 503. | diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index bccc26f..4ec47e7 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -34,7 +34,7 @@ Worker lifecycle： | Method | Path | 合同 | |---|---|---| -| `GET` | `/ns//workers` | 列出有 namespace-owned state 的 worker，包括 deploy-only、active 和 secret-only worker。 | +| `GET` | `/ns//workers` | 列出有 namespace-owned state 的 worker，包括 deploy-only、active、secret-only 和 workflow-definitions-only worker；每一项都会返回 `hasSecrets` 和 `hasWorkflowDefs`。 | | `GET` | `/ns//worker//versions` | 列出 retained versions 和 active status。 | | `POST` | `/ns//worker//deploy` | 从 shorthand code 或完整 module manifest 创建新的 immutable version；routes、crons、queue consumers、service refs、platform refs、assets、vars、bindings 和 `exports` 都是 version metadata。Python modules 和上游 experimental compatibility flags 会在 commit 前被拒绝。 | | `POST` | `/ns//worker//promote` | 通过 WATCH/MULTI routing path promote `{"version":"vN"}`。Host declaration 失败是 403；live pattern conflict 是 409；transaction contention 耗尽是 503。 | diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 4cb2d30..f49dfda 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -73,14 +73,17 @@ dispatch. do-runtime invokes tenant alarm and RPC methods through private fetch dispatches intercepted by the generated wrapper. Those requests carry the outer request id so the host facade establishes invocation-local context without adding platform metadata to -the tenant argument list. +the tenant argument list. Nested DO fetch/connect requests discard tenant-supplied +`x-request-id` values and, when the ambient parent context is available, propagate its +sanitized request id. Request ids remain untrusted diagnostic metadata. DO invoke envelopes identify persisted bundles by canonical namespace, worker, version, and storage id. They do not accept inline worker source. Tenant-facing DO object names and ids must be well-formed Unicode strings. Lone UTF-16 surrogates are rejected by `idFromName()` / `idFromString()` before hashing or dispatch; do-runtime alarm ingress and Workflows revalidation enforce the same boundary. DO host ids are capped at 512 UTF-8 bytes and use canonical `shardN` suffixes without -leading zeroes. +leading zeroes. DO binding class names use the ASCII JavaScript class-name grammar and +are capped at 468 bytes at deploy, so every shard suffix fits the aggregate host-id cap. ## Redis / Storage Contracts diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index c0a1a66..d0a542b 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -36,10 +36,10 @@ Storage cleanup endpoint 是 native facet storage cleanup 和 worker storage cle DO protocol error 使用 `{ error, message, details? }`。不同于 admin HTTP 的 flat additive error shape，DO protocol detail 会嵌套在 `details` 下，因为消费者是 runtime/DO client protocol，不是通用 admin JSON parser。未知 internal exception 仍会降级成安全的 `internal_error` / `Internal error` message。Storage delete-worker 在 partial batch result 时可能返回 HTTP 207 和 `{ ok:false, deleted, errors }`；这是 result envelope，不是 generic JSON error envelope。 Tenant-originated DO fetch body 在 runtime facade 中限制为 1 MiB。Facade 会在读取前拒绝超限的 `Content-Length`，streamed body 会增量读取，因此 limit 会在完整 buffering 前生效。 DO RPC method name 使用 JavaScript identifier grammar，并由 runtime client 和 do-runtime protocol reader 同时限制为最多 256 ASCII bytes。RPC 参数是最多 1 MiB 的 structural JSON data：接受 finite number、string、boolean、null、dense array 和 plain object。序列化不会调用 `toJSON()` hook；sparse array、circular structure、non-plain object 和 non-JSON value 会在 dispatch 前失败。 -do-runtime 会通过 generated wrapper 截获的私有 fetch dispatch 调用 tenant alarm 和 RPC method；这些 request 携带外层 request id，使 host facade 建立 invocation-local context，且不会把平台 metadata 加入 tenant argument list。 +do-runtime 会通过 generated wrapper 截获的私有 fetch dispatch 调用 tenant alarm 和 RPC method；这些 request 携带外层 request id，使 host facade 建立 invocation-local context，且不会把平台 metadata 加入 tenant argument list。嵌套 DO fetch/connect request 会丢弃 tenant 提供的 `x-request-id`，并在 ambient parent context 可用时传播其净化后的 request id；request id 仍是不可信的诊断 metadata。 DO invoke envelope 通过 canonical namespace、worker、version 和 storage id 标识 persisted bundle，不接受 inline worker source。 Tenant-facing DO object name/id 必须是 well-formed Unicode string。`idFromName()` / `idFromString()` 会在 hash 或 dispatch 前拒绝 lone UTF-16 surrogate；do-runtime alarm ingress 和 Workflows revalidation 执行相同边界。 -DO host id 最多 512 UTF-8 bytes，并使用不带前导零的 canonical `shardN` suffix。 +DO host id 最多 512 UTF-8 bytes，并使用不带前导零的 canonical `shardN` suffix。DO binding class name 使用 ASCII JavaScript class-name grammar，并在 deploy 时限制为最多 468 bytes，确保所有 shard suffix 都能满足 aggregate host-id 上限。 ## Redis / Storage 合同 diff --git a/docs/modules/gateway.md b/docs/modules/gateway.md index f9f977a..a22b02b 100644 --- a/docs/modules/gateway.md +++ b/docs/modules/gateway.md @@ -34,9 +34,10 @@ responses before they cross back onto the public socket. The `ADMIN_HOST` branch is infrastructure traffic, not a loaded-worker request. It does not set `x-worker-id` or `x-worker-prefix`. `PLATFORM_DOMAIN` and -`ADMIN_HOST` are environment-configurable. All tiers normalize `PLATFORM_DOMAIN` as a -plain lowercase hostname without trailing dots and default it to `workers.local`; -admin-host short-circuiting remains unset by default. +`ADMIN_HOST` are environment-configurable. All tiers normalize `PLATFORM_DOMAIN` as an +ALB-compatible ASCII DNS hostname of at most 126 bytes, with an alphabetic final label, +and default it to `workers.local`; one trailing dot is removed and the result is +lowercased. Admin-host short-circuiting remains unset by default. ## Interfaces diff --git a/docs/modules/gateway.zh.md b/docs/modules/gateway.zh.md index 8d581df..f4e20e2 100644 --- a/docs/modules/gateway.zh.md +++ b/docs/modules/gateway.zh.md @@ -18,7 +18,7 @@ Gateway 有三条 dispatch 分支： 转发前，gateway 会清除客户端提供的 `x-worker-id`、`x-worker-prefix` 和所有 `x-wdl-*` header。同一套内部 header 策略也会在 WebSocket upgrade response 回到公开 socket 前进行过滤。 -`ADMIN_HOST` 分支是 infrastructure traffic，不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置。所有 tier 都会把 `PLATFORM_DOMAIN` normalize 成不带尾点的纯小写 hostname，并默认使用 `workers.local`；admin-host short-circuit 默认仍未设置。 +`ADMIN_HOST` 分支是 infrastructure traffic，不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置。所有 tier 都会把 `PLATFORM_DOMAIN` normalize 成最多 126 bytes、final label 仅包含字母的 ALB-compatible ASCII DNS hostname；单个尾点会被移除，结果转成小写，并默认使用 `workers.local`。Admin-host short-circuit 默认仍未设置。 ## 接口 diff --git a/docs/modules/infra.md b/docs/modules/infra.md index a8afb97..708da28 100644 --- a/docs/modules/infra.md +++ b/docs/modules/infra.md @@ -148,12 +148,6 @@ Stateful storage: changes should document which services can use `FARGATE_SPOT`; stateful runtimes and singleton control loops should stay on on-demand Fargate unless their interruption semantics are re-reviewed. -- The application Terraform root accepts at least two existing subnets. Every subnet - must belong to the configured VPC, use an RFC1918 or `100.64.0.0/10` IPv4 CIDR, and - occupy a distinct Availability Zone because each D1/DO EFS file system creates one - mount target per supplied subnet. Foundation keeps a three-AZ default. The subnet - `map_public_ip_on_launch` flag is not used as a routing classifier; ECS services - explicitly disable public task IP assignment. - In addition to the Fargate task memory limit, D1 and DO workerd containers set explicit container memory hard limits. DO also reserves memory for its local redis-proxy sidecar. diff --git a/docs/modules/infra.zh.md b/docs/modules/infra.zh.md index 1cd3a8b..5a961c7 100644 --- a/docs/modules/infra.zh.md +++ b/docs/modules/infra.zh.md @@ -35,8 +35,7 @@ Local compose 是开发便利环境，不是 production delivery contract。它 ## 接口 -- 公开入口通过 ALB/gateway。Terraform-managed ALB ingress 可以在同一个 gateway - service 上承载 admin host、platform wildcard、canonical site host 和额外 exact public host。 +- 公开入口通过 ALB/gateway。Terraform-managed ALB ingress 可以在同一个 gateway service 上承载 admin host、platform wildcard、canonical site host 和额外 exact public host。 - Admin-host ingress 通过 gateway 的 `ADMIN_HOST` 分支进入 system-runtime control。 - Local Compose private service hop 使用 `envoy/envoy.yaml` 和编译进 `dist/workerd-configs` 的 `*-local.capnp` workerd config。 - ECS Service Connect 覆盖 runtime、D1、DO 和 workflows service target。 @@ -85,7 +84,6 @@ Stateful storage： - D1/DO 使用 owner lease、monotonic generation fence 和本地 drain/renew。超过 1 个 task 时需要稳定的 per-replica storage identity，并确保 supervisor drain/renew 只走私有本地入口，不能通过 service alias 打到其他 replica。 - 普通 D1/DO task 丢失会退回到 lease expiry，再由其他 replica takeover；graceful rollout 应优先走 supervisor drain，在 child workerd process 退出前释放 ownership。 - Terraform 在 ECS Fargate 上运行这套应用 stack 的服务。修改 capacity policy 时应记录哪些服务可以使用 `FARGATE_SPOT`；stateful runtime 和 singleton control loop 除非重新评估 interruption 语义，否则应保持 on-demand Fargate。 -- Application Terraform root 接受至少两个既有 subnet。每个 subnet 都必须属于指定 VPC、使用 RFC1918 或 `100.64.0.0/10` IPv4 CIDR，并且位于不同 Availability Zone，因为 D1/DO 的每个 EFS file system 都会为每个传入 subnet 创建一个 mount target。Foundation 仍默认创建三个 AZ。Subnet 的 `map_public_ip_on_launch` flag 不用于判断 routing 类型；ECS service 会显式关闭 task public IP assignment。 - 除了 Fargate task memory limit，D1 和 DO 的 workerd container 还会设置显式 container memory hard limit。DO 还会给本地 redis-proxy sidecar 保留内存。 - Terraform Fargate service 在服务能容忍 overlapping capacity 时应使用 rolling replacement。D1/DO 使用 sequential replacement；scheduler 作为 singleton control loop 保持 stop-before-start。D1/DO 和 scheduler 都关闭 Availability Zone rebalancing，让 replacement 遵循各自显式 deployment strategy。 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index d6a63a7..e60b3ef 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -149,7 +149,10 @@ Common rules: Promise and `async` continuations. Private do-runtime fetch, alarm, and RPC dispatches carry the id in `x-request-id`. Custom thenables are not a supported propagation boundary; the wrapper does not inspect or replace tenant return values to extend it. - Nested handler calls without a new id inherit the current ALS context. + Ordinary nested handler calls without a new id inherit the current ALS context. + Tenant code can deliberately switch the isolate's ambient ALS frame, so this + propagation is best-effort diagnostic correlation rather than an authorization or + integrity boundary. - Logs use snake_case fields. Only `level=error` is emitted on stderr; debug/info/warn JSON log lines go to stdout so log routing stays identical across JS, Rust, and embedded workerd shims. diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index e81f423..2976de9 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -92,7 +92,7 @@ WDL 在 JS workerd tier 和 Rust 服务中使用同一套可观测性策略： 通用规则： - `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker、D1 和 Workflows 传播。缺失的 inbound id 会在入口生成；header adapter 只考虑第一个逗号分隔 token，包含 visible ASCII 以外字节、引号、反斜杠或超过 128 字节的 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 -- Loaded-worker host facade 通过 invocation-local ALS 在 native Promise 和 `async` continuation 中保留 request id；do-runtime 的私有 fetch、alarm 和 RPC dispatch 会在 `x-request-id` 中携带该 id。Custom thenable 不属于受支持的传播边界；wrapper 不会为延长 context 而检查或替换 tenant 返回值。没有新 id 的嵌套 handler 调用会继承当前 ALS context。 +- Loaded-worker host facade 通过 invocation-local ALS 在 native Promise 和 `async` continuation 中保留 request id；do-runtime 的私有 fetch、alarm 和 RPC dispatch 会在 `x-request-id` 中携带该 id。Custom thenable 不属于受支持的传播边界；wrapper 不会为延长 context 而检查或替换 tenant 返回值。普通的无新 id 嵌套 handler 调用会继承当前 ALS context；tenant code 也可以主动切换 isolate 的 ambient ALS frame，因此这只是 best-effort 的诊断关联，不是授权或完整性边界。 - 日志字段使用 snake_case。只有 `level=error` 写入 stderr；debug/info/warn JSON log line 都写入 stdout，这样 JS、Rust 和 embedded workerd shim 的日志路由保持一致。 - 内部运维日志，包括 system-worker cleanup 日志和防御性的 Redis callback warning，也使用同一套单行 JSON envelope：`ts`、`service`、`level`、`event`，再加 snake_case 字段。JS 服务使用 `shared/observability.js`；Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`；无法转成文本的 throwable 降级为 `Unknown error`，不能反向覆盖原始失败。不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 - 产品 API response body 默认使用 camelCase，除非 endpoint 明确记录不同 wire contract。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 296a209..92c921e 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -191,9 +191,12 @@ Data-plane bindings use their own storage: Runtime must treat Redis bundle metadata as control-authored, but still revalidates reserved runtime entrypoint and binding names when materializing older stored metadata. -It also fails closed if older metadata contains Python module entries or upstream -experimental compatibility flags, because those would otherwise reach workerd as -opaque cold-load failures under the current stock binary. +It also fails closed if older metadata has a `compatibilityDate` before `2026-04-01`, +contains Python module entries or upstream experimental compatibility flags, disables +WDL's required enhanced error serialization, or explicitly enables that behavior after +it became the workerd default. The date floor and enhanced-serialization requirement +are WDL forward-only policy; the remaining checks prevent unsupported metadata from +reaching the current stock workerd binary as an opaque cold-load failure. ## Ownership / Concurrency / Failure Semantics @@ -213,8 +216,15 @@ opaque cold-load failures under the current stock binary. `nodejs_als` flag. Concurrent calls on a persistent Durable Object instance therefore cannot overwrite one shared context. The wrapper does not inspect, mutate, or replace tenant return values. Native Promise and `async` continuations retain context; custom - thenables are not a supported request-id propagation boundary. A nested handler call - without a new request id inherits the current ALS context instead of clearing it. + thenables are not a supported request-id propagation boundary. Only the outermost + generated-wrapper invocation establishes the ALS store, including a store whose + request id is null, so ordinary nested wrapper calls inherit it instead of replacing + it with a second id. Tenant code shares the isolate's ambient ALS machinery and can + deliberately bind or restore another ambient frame, so loaded-worker propagation is + best-effort against tenant interference, not a security boundary. Request ids are + untrusted diagnostic metadata and must never authorize, fence, or deduplicate work. + Request-id syntax is owned by the injected canonical request-id module rather than + copied into the wrapper runtime. - Request context wrappers swap facade objects into env and propagate request id where that event class can carry it. do-runtime alarm and RPC calls enter through private fetch dispatches carrying the outer request id, so the generated wrapper establishes diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 1a3b3d3..c97ebe5 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -75,7 +75,7 @@ Runtime 通过 `redis-proxy` 从 DB 0 读取不可变 bundle 和 metadata。Data - D1 和 DO binding 调用专门 runtime service。 - R2/ASSETS 使用 S3-compatible object storage。 -Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。旧 metadata 如果包含 Python module entry 或上游 experimental compatibility flag，也会 fail closed；否则这些形态会在当前 stock workerd binary 下变成 opaque cold-load failure。 +Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。旧 metadata 如果包含早于 `2026-04-01` 的 `compatibilityDate`、Python module entry、上游 experimental compatibility flag、禁用 WDL 要求的 enhanced error serialization，或在该行为已成为 workerd 默认值后仍显式启用对应 flag，也会 fail closed。日期下限与 enhanced serialization 要求是 WDL 的 forward-only policy；其余检查避免 unsupported metadata 直到当前 stock workerd cold-load 才以 opaque error 失败。 ## Ownership / 并发 / 失败语义 @@ -83,7 +83,7 @@ Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materializ - Service-binding cold load 会记录 loaded version，但不 evict sibling，因为 service binding 可能有意指向 frozen historical version。 - Internal active-version scheduled/queue dispatch 会 opt into sibling eviction；frozen workflow dispatch 不会。 - 只要注入 privileged internal Fetcher，wrapper generation 就会避免 raw env 暴露给未包装 entrypoint。Host-wrapper runtime 会在 tenant module 前执行，并捕获参与 handler/env wrapping 决策的 intrinsic，因此 tenant 顶层 prototype mutation 不能绕过 env 边界。 -- Host facade request id 使用 invocation-local `AsyncLocalStorage` store。已有 `nodejs_compat` 配置会保持不变；否则 Runtime 会把 standalone `no_nodejs_als` 替换为范围更窄的 `nodejs_als` flag。持久 Durable Object 实例上的并发调用不会再覆盖同一个共享 context，wrapper 也不会检查、修改或替换 tenant 返回值。Native Promise 和 `async` continuation 会保留 context；custom thenable 不属于受支持的 request-id propagation boundary。没有新 request id 的嵌套 handler 调用会继承当前 ALS context，而不是把它清空。 +- Host facade request id 使用 invocation-local `AsyncLocalStorage` store。已有 `nodejs_compat` 配置会保持不变；否则 Runtime 会把 standalone `no_nodejs_als` 替换为范围更窄的 `nodejs_als` flag。持久 Durable Object 实例上的并发调用不会再覆盖同一个共享 context，wrapper 也不会检查、修改或替换 tenant 返回值。Native Promise 和 `async` continuation 会保留 context；custom thenable 不属于受支持的 request-id propagation boundary。只有最外层 generated-wrapper invocation 会建立 ALS store，即使该 store 的 request id 为 null，因此普通嵌套 wrapper 调用会继承它而不是写入第二个 id。Tenant code 与 host 共用 isolate 的 ambient ALS 机制，可以主动 bind 或恢复另一个 ambient frame，因此 loaded-worker propagation 面对 tenant 干预时只是 best-effort，不是安全边界。Request id 是不可信的诊断 metadata，不能用于授权、fence 或去重。Request-id 语法由 injected canonical request-id module 拥有，不在 wrapper runtime 内复制。 - Request context wrapper 会把 facade object 换进 env，并在事件类型允许时传播 request id。do-runtime alarm 和 RPC 通过携带外层 request id 的私有 fetch dispatch 进入，使 generated wrapper 建立 invocation-local context，而不会把平台 metadata 加入 tenant method argument。 - Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error` response，并带 request id。异常细节输出到结构化日志/live tail，不复制进客户端 body；throwable 自身无法转成字符串时，tail formatting 也不能反向覆盖原始异常。 - Internal scheduled、queue 和 workflow dispatch route 使用 result envelope 表达 handler outcome。Tenant handler error 是 scheduler/workflow 协议中的 outcome state，不是 generic platform transport error。 diff --git a/docs/modules/workflows.md b/docs/modules/workflows.md index bcc9616..2d07fc1 100644 --- a/docs/modules/workflows.md +++ b/docs/modules/workflows.md @@ -82,7 +82,9 @@ Internal: Workflows exclusively owns Valkey DB 2 for instance execution state. Control owns `wf:defs::` in DB 0 for deploy-time workflow key allocation and stable -identity. +identity. The hash retains retired names until whole-worker delete. Definition listing +enumerates that retired history for currently active workers; deploy and single-workflow +status/lifecycle paths read only the names they need. Key concepts: @@ -117,6 +119,7 @@ Key families: | `wf:by-worker::` | Set | workflows | Instance discovery by worker. | Used by list/delete checks; entries are removed by retention/delete cleanup. | | `wf:by-workflow:::` | ZSET | workflows | Per-workflow instance list index ordered for bounded pagination. | Retention/delete cleanup removes the sorted-set member. | | `wf:by-version:::` | Set | workflows | Frozen-version referrer index. | Blocks version delete while live instances reference the version. | +| `wf:pending-version:::` | ZSET | workflows | Short-lived restart target-version blockers, scored by expiry time. | Version-delete checks active members; restart renews only a live member and atomically validates it before creating the durable `wf:by-version` referrer. Members expire after 30 seconds, and the ZSET has a refreshed 60-second key TTL for physical cleanup. | | `wf:retention` | ZSET | workflows | Terminal retention due index. | Retention tick deletes expired terminal instances. | | `wf:internal:do-alarm:{}:state` | Hash | workflows | Authoritative backend job state for one Durable Object SQLite alarm row. | Successful delivery, retry exhaustion, explicit delete, and worker cleanup remove the job. | | `wf:internal:do-alarm:due:` | ZSET | workflows | DO alarm due index. Score is due timestamp in milliseconds. | Tick promotion moves eligible jobs to ready. | @@ -128,6 +131,12 @@ Key families: - Workflows are same-worker only in V2. - Instances freeze the worker version/class identity they were created with. +- Control fails closed on malformed active workflow entries and malformed `wf:defs` + records encountered by an operation; management paths return `corrupt_meta`, while + deploy returns `workflow_definition_corrupt` when reusing a damaged historical + definition. Damaged authoritative metadata is not exposed as a normal missing or + retired workflow. Normal deploy and single-workflow paths do not scan unrelated + historical definitions. - Scheduler only wakes workflows; workflows owns admission, fairness, shard ticks, ready/due movement, and runtime dispatch. - Scheduler also wakes Workflows-owned internal DO alarm jobs through the same @@ -190,6 +199,10 @@ canonicalizes against the current active route before writing DB 2, so new durab business processes start on the active version. Existing instances replay against their stored `frozenVersion`; promotion does not change their code. Worker-version delete is blocked by `wf:by-version` while non-expired instances still reference the version. +Before restart revalidates the active export, it publishes a short-lived target-version +blocker. Its final DB 2 transition atomically creates the durable referrer and removes +that blocker, so version delete cannot pass between active-version resolution and the +restart commit. Runtime validates every dispatched `frozenVersion` with the same positive JavaScript-safe-integer version parser used by bundle keys; malformed persisted tags fail before worker loading. diff --git a/docs/modules/workflows.zh.md b/docs/modules/workflows.zh.md index 3b2f1aa..70418c7 100644 --- a/docs/modules/workflows.zh.md +++ b/docs/modules/workflows.zh.md @@ -44,7 +44,7 @@ Control / CLI： ## Redis / Storage 合同 -Workflows 独占 Valkey DB 2 作为 instance execution state。Control 在 DB 0 拥有 `wf:defs::`，用于 deploy-time workflow key allocation 和稳定 identity。 +Workflows 独占 Valkey DB 2 作为 instance execution state。Control 在 DB 0 拥有 `wf:defs::`，用于 deploy-time workflow key allocation 和稳定 identity。该 Hash 会保留 retired name 直到 whole-worker delete；definition list 只会为当前 active worker 枚举这段 retired history，而 deploy 和单个 workflow 的 status/lifecycle 路径只读取自己需要的 name。 关键概念： @@ -70,6 +70,7 @@ Key families： | `wf:by-worker::` | Set | workflows | 按 worker 发现 instance 的索引。 | list/delete check 使用；retention/delete cleanup 移除 entry。 | | `wf:by-workflow:::` | ZSET | workflows | 按 workflow key 分页列 instance 的有序索引。 | retention/delete cleanup 删除 sorted-set member。 | | `wf:by-version:::` | Set | workflows | frozen-version referrer index。 | live instance 仍引用该 version 时阻止 version delete。 | +| `wf:pending-version:::` | ZSET | workflows | 按过期时间计分的短期 restart target-version blocker。 | Version-delete 检查 active member；restart 只续租未过期 member，并在创建持久 `wf:by-version` referrer 前原子复核。Member 30 秒后过期，ZSET key 使用随写入刷新的 60 秒 TTL 做物理回收。 | | `wf:retention` | ZSET | workflows | terminal retention due index。 | Retention tick 删除过期 terminal instance。 | | `wf:internal:do-alarm:{}:state` | Hash | workflows | 单个 Durable Object SQLite alarm row 的 backend job 权威状态。 | 成功 delivery、retry 耗尽、显式 delete 和 worker cleanup 会移除 job。 | | `wf:internal:do-alarm:due:` | ZSET | workflows | DO alarm due index。score 是 due timestamp milliseconds。 | Tick promotion 把到期 job 移到 ready。 | @@ -81,6 +82,7 @@ Key families： - V2 workflow 只支持 same-worker。 - Instance 冻结创建时的 worker version/class identity。 +- Control 会对当前操作实际读到的 malformed active workflow entry 和 malformed `wf:defs` record fail closed；管理路径返回 `corrupt_meta`，deploy 在复用损坏的历史 definition 时返回 `workflow_definition_corrupt`。损坏的权威 metadata 不会被暴露为正常的 missing 或 retired workflow。正常 deploy 和单个 workflow 路径不会扫描无关的历史 definition。 - Scheduler 只负责唤醒 workflows；admission、fairness、shard tick、ready/due movement 和 runtime dispatch 都由 workflows 负责。 - Scheduler 也通过同一个 `/internal/workflows/tick` endpoint 唤醒 Workflows-owned internal DO alarm jobs；scheduler 不直接读写 DO alarm state。 - Workflows 在持久化 DO alarm job 前拒绝 non-canonical alarm identity，在 dispatch 前重新校验持久化 alarm identity，并在把 active route 用作 retarget 前校验其 version。其中 namespace、worker、version 校验复用 `wdl-rust-common`；do-runtime protocol grammar 与 identity helper 拥有 canonical alarm-specific field 和 aggregate 512-byte DO host-id 合同，Workflows 在持久化和 dispatch 前镜像并重新校验该合同。Runtime run dispatch 与 progress callback 在 workflows crate 内共用同一个 system-vs-user runtime endpoint selector。 @@ -99,7 +101,7 @@ Workflow execution 使用两条 channel： 1. Loaded worker 通过 reserved `__WDL_WORKFLOWS_BACKEND__` Fetcher binding 调 workflows。Runtime 从 bundle metadata 附加 identity；workflows 不信任 tenant body 中的 namespace、worker、version、workflow key、class 或 instance identity。 2. workflows 把已 claim 的 run dispatch 回 runtime `:8088` 上的 `/internal/workflows/run`。Runtime 加载 frozen worker version 并调用 `className.run(event, stepFacade)`。 -Create/restart 与 replay 的 version pinning 不同。新的 `create()` 或 `restart()` 写 DB 2 前会按当前 active route canonicalize，因此新的 durable business process 从 active version 开始。已有 instance 使用自己存的 `frozenVersion` replay；promotion 不会改变它的代码。只要 non-expired instance 仍引用某个 version，`wf:by-version` 就会阻止 worker-version delete。Runtime 会用 bundle key 共用的正 JavaScript-safe-integer version parser 校验每个 dispatch 的 `frozenVersion`；malformed persisted tag 会在加载 worker 前失败。 +Create/restart 与 replay 的 version pinning 不同。新的 `create()` 或 `restart()` 写 DB 2 前会按当前 active route canonicalize，因此新的 durable business process 从 active version 开始。已有 instance 使用自己存的 `frozenVersion` replay；promotion 不会改变它的代码。只要 non-expired instance 仍引用某个 version，`wf:by-version` 就会阻止 worker-version delete。Restart 在重新校验 active export 前写入一个短期 target-version blocker，最终 DB 2 transition 会原子地建立持久 `wf:by-version` referrer 并删除该 blocker，因此 version delete 不能从 active-version resolution 和 restart commit 之间穿过。Runtime 会用 bundle key 共用的正 JavaScript-safe-integer version parser 校验每个 dispatch 的 `frozenVersion`；malformed persisted tag 会在加载 worker 前失败。 Scheduling 是 hint-based，但状态权威在 instance hash： diff --git a/docs/redis-key-layout.md b/docs/redis-key-layout.md index 322aea6..8313e4d 100644 --- a/docs/redis-key-layout.md +++ b/docs/redis-key-layout.md @@ -14,8 +14,8 @@ WDL uses a deliberate logical split: - **`DB 1`, data plane:** KV hash buckets, queue streams, delayed queues, orphan streams, and live log-tail streams. - **`DB 2`, workflows:** `wf:schema_version`, instance state, step records/summaries, - ready/due shards, events and event-type indexes, payload refs, retention indexes, and - run leases. + ready/due shards, events and event-type indexes, payload refs, retention indexes, + restart target-version blockers, and run leases. Local compose, Kubernetes, and Terraform enable this split. Rust services and the Rust `redis-proxy` use `DATA_REDIS_URL` / `DATA_REDIS_DB` to select the @@ -74,8 +74,8 @@ scheduled version is no longer retained. A key-grammar change must update the JS helper, the Rust helper, and every reader together. `workers:` means the worker has worker-owned lifecycle state: retained bundle, -active projection, or worker-level secrets. Secret-only workers are intentionally listed -and whole-deletable. +active projection, worker-level secrets, or workflow definitions. Secret-only and +definitions-only workers are intentionally listed and whole-deletable. ## Route And Host Projection @@ -84,7 +84,8 @@ then reads `patterns:` and uses the slot value's embedded `version` to con `x-worker-id` without consulting `routes:`. Pattern slot values are compact `v2\t\t\t\t\t` records encoded by `shared/route-projection.js`, not JSON. Promote updates both projections in the same -Redis transaction. +Redis transaction. Control mutation and delete paths fail closed on a nonempty slot that +cannot be decoded; they do not treat an unknown owner as an empty slot. `hosts:` is operator intent: the namespace is allowed to use those hosts. `declared-hosts` is a gateway gate for hosts declared by at least one namespace. @@ -164,6 +165,11 @@ Cross-cutting constraints: - Workflows owns DB 2 instance state. `wf:ready:cursor` is the internal ready-shard fairness cursor. Control owns only DB 0 `wf:defs:*`; other tiers must not write DB 2 directly. +- `wf:pending-version:::` is a Workflows-owned, 30-second restart + blocker. Version-delete checks it with `wf:by-version`; renewals cannot resurrect an + expired member, and the successful-restart DB 2 script revalidates the lease before + replacing it with the durable version referrer. The ZSET key has a refreshed + 60-second TTL so abandoned marker keys are physically reclaimed. - Workflows also owns internal DB 2 `wf:internal:do-alarm:*` jobs for Durable Object alarm backend scheduling. do-runtime writes alarms through the workflows HTTP API instead of writing those keys directly. `wf:internal:do-alarm:ready:cursor` is the diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index 34b36c6..2efe666 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -8,7 +8,7 @@ WDL 使用明确的逻辑切分： - **`DB 0`，控制面：**bundle、routes/patterns、auth、D1/DO owner state、cron config、queue-consumer config、lifecycle metadata，以及 workflow definition（`wf:defs:*`）。 - **`DB 1`，数据面：**KV hash bucket、queue stream、delayed queue、orphan stream 和 live log-tail stream。 -- **`DB 2`，workflows：**`wf:schema_version`、instance state、step record/summary、ready/due shard、event 和 event-type index、payload ref、retention index、run lease。 +- **`DB 2`，workflows：**`wf:schema_version`、instance state、step record/summary、ready/due shard、event 和 event-type index、payload ref、retention index、restart target-version blocker、run lease。 Local compose、Kubernetes 和 Terraform 都启用这个切分。Rust service 和 Rust `redis-proxy` 使用 `DATA_REDIS_URL` / `DATA_REDIS_DB` 选择 data-plane Redis connection/database；嵌入的 JS control/log-tail 路径使用 `DATA_REDIS_ADDR` 加 `DATA_REDIS_DB`，因为它们的 RESP client 接收 host:port address。未设置这些 data-plane 变量的部署会把数据面 key 留在 control Redis connection/database，直到显式 opt in。Workflows 不同：未设置 `WORKFLOWS_REDIS_URL` 时 workflows service 仍默认使用 DB 2；只有显式设置 `WORKFLOWS_REDIS_DB=0` 时才使用 DB 0。 @@ -43,11 +43,11 @@ secrets:: Hash, worker-level WDL-ENC envelope `routes:` 和 `worker-versions::` 只能通过 `shared/version.js#routesKey` / `#workerVersionsKey`（以及它们的 Rust 镜像 `rust/common/src/version.rs#routes_key` / `#worker_versions_key`）构造。Control 是唯一 writer；sanctioned reader 是 gateway（route resolution）和 workflows。workflows 有两条读取路径：workflow create / verify 时的 active-export resolution，以及 fired alarm 的 scheduled version 已不再 retained 时的 internal DO alarm retarget。改 key 语法时必须同时更新 JS helper、Rust helper 和所有 reader。 -`workers:` 表示这个 worker 有 worker-owned lifecycle state：retained bundle、active projection 或 worker-level secrets。Secret-only worker 会被有意列出，并可以 whole-delete。 +`workers:` 表示这个 worker 有 worker-owned lifecycle state：retained bundle、active projection、worker-level secrets 或 workflow definitions。Secret-only 和 definitions-only worker 会被有意列出，并可以 whole-delete。 ## Route 和 Host Projection -Subdomain routing 读取 `routes:`。Pattern routing 先检查 `declared-hosts`，再读取 `patterns:`，并使用 slot value 中嵌入的 `version` 构造 `x-worker-id`，不再查 `routes:`。Pattern slot value 是由 `shared/route-projection.js` 编码的紧凑 `v2\t\t\t\t\t` record，不再是 JSON。Promote 在同一个 Redis transaction 中更新两套 projection。 +Subdomain routing 读取 `routes:`。Pattern routing 先检查 `declared-hosts`，再读取 `patterns:`，并使用 slot value 中嵌入的 `version` 构造 `x-worker-id`，不再查 `routes:`。Pattern slot value 是由 `shared/route-projection.js` 编码的紧凑 `v2\t\t\t\t\t` record，不再是 JSON。Promote 在同一个 Redis transaction 中更新两套 projection。Control mutation 和 delete 路径遇到无法 decode 的非空 slot 时会 fail closed，不会把未知 owner 当成空槽。 `hosts:` 是 operator intent：这个 namespace 被允许使用这些 host。`declared-hosts` 是 gateway 对“至少被一个 namespace 声明过的 host”的 gate。`host-declarations:` 记录声明该 host 的 namespace，因此一个 namespace 移除声明时，不会在另一个 namespace 仍声明该 host 的情况下清掉全局 gate。`POST /reload` 会先从 `hosts:` 重建这两个声明索引，再发布 gateway cache invalidation；这给 operator-managed host declaration 提供显式 repair/backfill 路径。`ns-hosts:` 是 active reverse index：这个 namespace 当前在这些 host 上拥有至少一个 slot。`hosts:` 应是 superset。Host reconcile 会先用 `ns-hosts:` 做 fast path，再扫描 `patterns:`。 @@ -97,4 +97,5 @@ Routes、crons、queue consumers、bindings、vars、exports、workflow definiti - Queue main stream 不做 trim，因为 at-least-once delivery 是合同。DLQ、orphan、log-tail 这类诊断 stream 可以使用有界 approximate trim。 - Secret hash value 在 steady state 下是 `WDL-ENC:` envelope。`/runtime/load` 没有 plaintext fallback。 - Workflows 拥有 DB 2 instance state。`wf:ready:cursor` 是内部 ready-shard 公平性 cursor。Control 只拥有 DB 0 的 `wf:defs:*`；其他 tier 不应直接写 DB 2。 +- `wf:pending-version:::` 是 Workflows-owned、30 秒的 restart blocker。Version-delete 会将它与 `wf:by-version` 一起检查；续租不能复活已过期 member，restart 成功的 DB 2 script 会在创建持久 version referrer 前再次验证 lease。ZSET key 使用随写入刷新的 60 秒 TTL，确保遗留 marker key 会被物理回收。 - Workflows 还拥有 DB 2 中的 internal `wf:internal:do-alarm:*` jobs，用于 Durable Object alarm backend scheduling。do-runtime 通过 workflows HTTP API 写 alarm，而不是直接写这些 key。`wf:internal:do-alarm:ready:cursor` 是内部 ready-shard 公平性 cursor，不是租户状态。 diff --git a/docs/source-map.md b/docs/source-map.md index ba64d53..a95880a 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -72,7 +72,7 @@ are outside this map unless they own runtime or deployable service behavior. | `shared/bounded-body.js` | Shared bounded byte-stream and request-body readers; each tier maps limit errors to its own contract. | | `shared/ns-pattern.js` | Platform-domain normalization plus namespace, worker, binding, queue, KV/D1/R2 id, module path, reserved object-key, and reserved namespace grammars. | | `shared/version.js` | Worker version formatting plus worker, route-plane, lifecycle, DO owner-scope key, and route-invalidation channel helpers. | -| `shared/workerd-compat-flags.js` | Pinned mirror of upstream workerd experimental compatibility enable flags used to reject tenant metadata before cold-load. | +| `shared/workerd-compat-flags.js` | Pinned upstream mirror of experimental enable flags plus WDL-owned dynamic-worker date and error-serialization policy. | | `shared/queue-keys.js` | JavaScript queue key helpers used by tests and cross-tier key-shape checks. | | `shared/route-projection.js` | Compact pattern-route projection encoding shared by control writers, delete checks, and gateway readers. | | `shared/d1-*.js`, `shared/sql-splitter.js` | D1 parameter, data-field, transport, timeout, query-wire, and SQL splitting utilities shared by runtime, d1-runtime, control, and tests. | @@ -112,7 +112,7 @@ are outside this map unless they own runtime or deployable service behavior. | `examples/` | Manual demos and reference projects. Tests should not silently depend on them unless the fixture graduates to `test-workers/`. | | `scripts/run-integration-tests.js` | Integration worker-pool runner. | | `scripts/compile-workerd-configs.js` | Compiles workerd Cap'n Proto configs into `dist/workerd-configs/*.bin`. | -| `scripts/extract-workerd-experimental-compat-flags.mjs` | Pin-bump flag extractor. | +| `scripts/extract-workerd-experimental-compat-flags.mjs` | Pin-bump experimental-flag extractor. | ## Infrastructure diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index 8f5b743..6a7d92e 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -69,7 +69,7 @@ | `shared/bounded-body.js` | 共享 bounded byte-stream 和 request-body readers；各 tier 自己把 limit error 映射为对应 contract。 | | `shared/ns-pattern.js` | Platform-domain normalization，以及 namespace、worker、binding、queue、KV/D1/R2 id、module path、reserved object-key 和 reserved namespace grammars。 | | `shared/version.js` | Worker version formatting，以及 worker、route-plane、lifecycle、DO owner-scope key 与 route invalidation channel helpers。 | -| `shared/workerd-compat-flags.js` | 上游 workerd experimental compatibility enable flags 的 pinned mirror，用于在 cold-load 前拒绝 tenant metadata。 | +| `shared/workerd-compat-flags.js` | 上游 workerd experimental enable flags 的 pinned mirror，以及 WDL-owned dynamic-worker 日期和 error-serialization policy。 | | `shared/queue-keys.js` | JavaScript queue key helpers，供 tests 和 cross-tier key-shape checks 使用。 | | `shared/route-projection.js` | Control writer、delete check 和 gateway reader 共用的紧凑 pattern-route projection encoding。 | | `shared/d1-*.js`、`shared/sql-splitter.js` | Runtime、d1-runtime、control 和 tests 共用的 D1 parameter、data-field、transport、timeout、query-wire 和 SQL splitting utilities。 | @@ -109,7 +109,7 @@ | `examples/` | 手工 demo 和 reference projects。测试不应悄悄依赖它们，除非 fixture 明确迁入 `test-workers/`。 | | `scripts/run-integration-tests.js` | Integration worker-pool runner。 | | `scripts/compile-workerd-configs.js` | 把 workerd Cap'n Proto configs 编译成 `dist/workerd-configs/*.bin`。 | -| `scripts/extract-workerd-experimental-compat-flags.mjs` | pin bump flag 提取脚本。 | +| `scripts/extract-workerd-experimental-compat-flags.mjs` | pin bump experimental flag 提取脚本。 | ## Infrastructure diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index 43abff1..1a74d10 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -1,4 +1,5 @@ import { validOwnerEndpointForService } from "./_wdl-owner-endpoint.js"; +import { sanitizeRequestId } from "./_wdl-request-id.js"; export const DO_INVOKE_URL = "http://do-runtime/internal/do/invoke"; export const DO_CONNECT_URL = "http://do-runtime/internal/do/connect"; @@ -75,7 +76,6 @@ const intrinsicHeadersAppend = Headers.prototype.append; const intrinsicHeadersDelete = Headers.prototype.delete; const intrinsicHeadersForEach = Headers.prototype.forEach; const intrinsicHeadersGet = Headers.prototype.get; -const intrinsicHeadersHas = Headers.prototype.has; const intrinsicHeadersSet = Headers.prototype.set; const intrinsicJsonStringify = JSON.stringify; const intrinsicNumberIsFinite = Number.isFinite; @@ -86,6 +86,7 @@ const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; const intrinsicObjectSetPrototypeOf = Object.setPrototypeOf; const intrinsicPromiseThen = Promise.prototype.then; +const intrinsicResponseJson = Response.json; const intrinsicReadableStreamCancel = ReadableStream.prototype.cancel; const intrinsicReadableStreamGetReader = ReadableStream.prototype.getReader; const intrinsicReadableStreamReaderCancel = ReadableStreamDefaultReader.prototype.cancel; @@ -233,10 +234,6 @@ function headerValue(headers, name) { } /** @param {Headers} headers @param {string} name */ -function headerHas(headers, name) { - return intrinsicReflectApply(intrinsicHeadersHas, headers, [name]); -} - /** @param {Headers} headers @param {string} name @param {string} value */ function headerSet(headers, name, value) { intrinsicReflectApply(intrinsicHeadersSet, headers, [name, value]); @@ -677,9 +674,9 @@ export async function requestSpec(request, requestId) { forEachArray(DO_FETCH_STRIP_HEADERS, (name) => { headerDelete(forwardedHeaders, name); }); - if (requestId && !headerHas(forwardedHeaders, "x-request-id")) { - headerSet(forwardedHeaders, "x-request-id", requestId); - } + headerDelete(forwardedHeaders, "x-request-id"); + const canonicalRequestId = sanitizeRequestId(requestId); + if (canonicalRequestId) headerSet(forwardedHeaders, "x-request-id", canonicalRequestId); const method = stringToUpperCase(requestMethod(forwarded)); const headers = headerEntries(forwardedHeaders); enforceRequestHeadersBudget(headers); @@ -739,7 +736,9 @@ export function connectHeaders(props, objectName, request, requestId) { headerSet(headers, DO_CONNECT_HEADERS.className, props.className); headerSet(headers, DO_CONNECT_HEADERS.objectName, objectName); headerSet(headers, DO_CONNECT_HEADERS.requestUrl, requestUrl(request)); - if (requestId && !headerHas(headers, "x-request-id")) headerSet(headers, "x-request-id", requestId); + headerDelete(headers, "x-request-id"); + const canonicalRequestId = sanitizeRequestId(requestId); + if (canonicalRequestId) headerSet(headers, "x-request-id", canonicalRequestId); headerSet(headers, DO_ACCEPT_OWNER_HINT_HEADER, "1"); return headers; } @@ -807,10 +806,10 @@ function validOwnerEndpoint(endpoint) { } function ownerUnavailableResponse() { - return Response.json( + return intrinsicReflectApply(intrinsicResponseJson, IntrinsicResponse, [ { error: "owner_unavailable", message: "DO owner is unavailable; request outcome may be unknown" }, { status: 503 } - ); + ]); } /** diff --git a/runtime/_wdl-request-id.js b/runtime/_wdl-request-id.js index 564bd03..80735d3 100644 --- a/runtime/_wdl-request-id.js +++ b/runtime/_wdl-request-id.js @@ -1,3 +1,37 @@ +const intrinsicArrayIsArray = Array.isArray; +const intrinsicObjectHasOwn = Object.hasOwn; +const intrinsicReflectApply = Reflect.apply; +const intrinsicStringCharCodeAt = String.prototype.charCodeAt; +const intrinsicStringIndexOf = String.prototype.indexOf; +const intrinsicStringSlice = String.prototype.slice; + +/** @param {string} value */ +function trimAsciiWhitespace(value) { + let start = 0; + let end = value.length; + const isWhitespace = (/** @type {number} */ code) => + code === 0x20 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d; + while (start < end && isWhitespace(intrinsicReflectApply(intrinsicStringCharCodeAt, value, [start]))) start += 1; + while (end > start && isWhitespace(intrinsicReflectApply(intrinsicStringCharCodeAt, value, [end - 1]))) end -= 1; + return intrinsicReflectApply(intrinsicStringSlice, value, [start, end]); +} + +/** @param {unknown} raw @returns {string | null} */ +export function sanitizeRequestId(raw) { + if (intrinsicReflectApply(intrinsicArrayIsArray, undefined, [raw])) { + raw = /** @type {unknown[]} */ (raw)[0]; + } + if (typeof raw !== "string") return null; + const comma = intrinsicReflectApply(intrinsicStringIndexOf, raw, [","]); + const first = trimAsciiWhitespace(comma === -1 ? raw : intrinsicReflectApply(intrinsicStringSlice, raw, [0, comma])); + if (!first || first.length > 128) return null; + for (let i = 0; i < first.length; i++) { + const code = intrinsicReflectApply(intrinsicStringCharCodeAt, first, [i]); + if (code < 0x21 || code > 0x7e || code === 0x22 || code === 0x5c) return null; + } + return first; +} + /** * Resolve the request id options shared by loaded-isolate host facades. * @@ -5,17 +39,19 @@ * a stable env wrapper and swaps the current request id through the provider. * * @param {unknown} options - * @param {{ providerFirst?: boolean }} [resolution] * @returns {string | null} */ -export function requestIdFromOptions(options, { providerFirst = true } = {}) { +export function requestIdFromOptions(options) { if (!options || typeof options !== "object") return null; const record = /** @type {{ requestIdProvider?: unknown, requestId?: unknown }} */ (options); const fromProvider = () => { - if (typeof record.requestIdProvider !== "function") return null; - const id = record.requestIdProvider(); - return typeof id === "string" && id ? id : null; + if (!intrinsicReflectApply(intrinsicObjectHasOwn, undefined, [record, "requestIdProvider"])) return null; + const provider = record.requestIdProvider; + if (typeof provider !== "function") return null; + return sanitizeRequestId(intrinsicReflectApply(provider, undefined, [])); }; - const fromValue = () => typeof record.requestId === "string" && record.requestId ? record.requestId : null; - return providerFirst ? fromProvider() || fromValue() : fromValue() || fromProvider(); + const fromValue = () => intrinsicReflectApply(intrinsicObjectHasOwn, undefined, [record, "requestId"]) + ? sanitizeRequestId(record.requestId) + : null; + return fromProvider() || fromValue(); } diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 69274af..782bc8e 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -66,6 +66,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "_wdl-do-transport.js"), + (name = "_wdl-request-id.js", esModule = embed "_wdl-request-id.js"), (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), # Injected DO transport uses a relative module name; host modules use the # shared bare name. Both resolve to the same shared contract owner. diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index ea34afc..666afce 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -65,6 +65,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "_wdl-do-transport.js"), + (name = "_wdl-request-id.js", esModule = embed "_wdl-request-id.js"), (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), # Injected DO transport uses a relative module name; host modules use the # shared bare name. Both resolve to the same shared contract owner. diff --git a/runtime/do-client.js b/runtime/do-client.js index dad1d33..8ef3198 100644 --- a/runtime/do-client.js +++ b/runtime/do-client.js @@ -100,7 +100,7 @@ class DurableObjectStub { const real = Reflect.get(target, prop, receiver); if (real !== undefined) return real; /** @param {...unknown} args */ - const method = (...args) => target.namespace.rpcObject(target.id.name, prop, args, target.namespace.requestId()); + const method = (...args) => target.namespace.rpcObject(target.id.name, prop, args); return method; }, }); @@ -109,7 +109,7 @@ class DurableObjectStub { /** @param {RequestInfo | URL} input @param {RequestInit} [init] */ async fetch(input, init = undefined) { const request = new Request(input, init); - return await this.namespace.fetchObject(this.id.name, request, this.namespace.requestId()); + return await this.namespace.fetchObject(this.id.name, request); } } @@ -195,12 +195,13 @@ export class DurableObjectNamespace { } } - requestId() { - return requestIdFromOptions(this.#requestIdOptions, { providerFirst: false }); + #currentRequestId() { + return requestIdFromOptions(this.#requestIdOptions); } - /** @param {string} objectName @param {Request} request @param {string | null} [requestId] */ - async fetchObject(objectName, request, requestId = null) { + /** @param {string} objectName @param {Request} request */ + async fetchObject(objectName, request) { + const requestId = this.#currentRequestId(); if (this.#bindingFetchObject && !isWebSocketUpgrade(request)) { return await this.#bindingFetchObject(objectName, request, requestId); } @@ -237,8 +238,9 @@ export class DurableObjectNamespace { ); } - /** @param {string} objectName @param {string} method @param {unknown[]} args @param {string | null} [requestId] */ - async rpcObject(objectName, method, args, requestId = null) { + /** @param {string} objectName @param {string} method @param {unknown[]} args */ + async rpcObject(objectName, method, args) { + const requestId = this.#currentRequestId(); if (this.#bindingRpcObject) { return await this.#bindingRpcObject(objectName, method, args, requestId); } diff --git a/runtime/lib.js b/runtime/lib.js index 59974bb..56f290c 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -3,7 +3,14 @@ import { base64ToBytes, bytesToBase64 } from "shared-base64"; import { WORKER_NAME_RE, isValidRouteNs } from "shared-ns-pattern"; import { parseVersion } from "shared-version"; -import { isWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; +import { + DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE, + ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE, + ENHANCED_ERROR_SERIALIZATION_FLAG, + LEGACY_ERROR_SERIALIZATION_FLAG, + MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, + isWorkerdExperimentalCompatFlag, +} from "shared-workerd-compat-flags"; const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); @@ -107,9 +114,6 @@ function deepFreeze(obj) { return Object.freeze(obj); } -const DEFAULT_COMPATIBILITY_DATE = "2026-04-24"; -const ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE = "2026-04-21"; -const ENHANCED_ERROR_SERIALIZATION_FLAG = "enhanced_error_serialization"; // Loaded workers may declare an older compatibilityDate than the platform // workers. Keep enhanced error serialization as a floor only before the date // where workerd made it the default; newer dates reject the explicit flag. @@ -148,6 +152,19 @@ function mergeCompatFlags(userFlags, compatibilityDate) { `meta.compatibilityFlags contains experimental workerd flag ${JSON.stringify(f)}, which WDL does not support for tenant workers` ); } + if (f === LEGACY_ERROR_SERIALIZATION_FLAG) { + throw new Error( + `meta.compatibilityFlags contains unsupported flag ${JSON.stringify(f)}; WDL requires enhanced error serialization` + ); + } + if ( + f === ENHANCED_ERROR_SERIALIZATION_FLAG && + compatibilityDate >= ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE + ) { + throw new Error( + `meta.compatibilityFlags contains ${JSON.stringify(f)}, which became the workerd default on ${ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE} and must not be specified for compatibilityDate ${compatibilityDate}` + ); + } out.push(f); } for (const f of platformFloorCompatFlags(compatibilityDate)) { @@ -222,7 +239,12 @@ export function bundleToWorkerCode(hash) { // methods cannot shadow RPC method names on bindings. const compatibilityDate = typeof meta.compatibilityDate === "string" && meta.compatibilityDate ? meta.compatibilityDate - : DEFAULT_COMPATIBILITY_DATE; + : DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE; + if (compatibilityDate < MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE) { + throw new Error( + `meta.compatibilityDate ${compatibilityDate} is older than WDL supports (${MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE})` + ); + } return { compatibilityDate, diff --git a/runtime/load/wrapper-generate.js b/runtime/load/wrapper-generate.js index 1e58ded..0f4f2e3 100644 --- a/runtime/load/wrapper-generate.js +++ b/runtime/load/wrapper-generate.js @@ -13,6 +13,7 @@ const DEFAULT_EXPORT_CLASS_TEST_SOURCE = "/^\\s*class\\b/.test(source)"; export const HOST_BINDING_RUNTIME_MODULE_NAME = "_wdl-host-wrapper-runtime.js"; export const HOST_BINDING_RUNTIME_SOURCE = ` import { AsyncLocalStorage } from "node:async_hooks"; +import { sanitizeRequestId } from "./_wdl-request-id.js"; const IntrinsicObject = Object; const IntrinsicProxy = Proxy; @@ -22,26 +23,14 @@ const intrinsicAsyncLocalStorageGetStore = AsyncLocalStorage.prototype.getStore; const intrinsicAsyncLocalStorageRun = AsyncLocalStorage.prototype.run; const intrinsicArrayForEach = Array.prototype.forEach; const intrinsicFunctionToString = Function.prototype.toString; +const intrinsicHeadersGet = Headers.prototype.get; const intrinsicObjectDefineProperty = Object.defineProperty; const intrinsicObjectEntries = Object.entries; const intrinsicObjectKeys = Object.keys; const intrinsicReflectApply = Reflect.apply; const intrinsicReflectGet = Reflect.get; const intrinsicRegExpTest = RegExp.prototype.test; -const requestContext = new AsyncLocalStorage(); - -intrinsicReflectApply(intrinsicObjectDefineProperty, IntrinsicObject, [ - requestContext, - "getStore", - { - configurable: false, - enumerable: false, - value() { - return intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); - }, - writable: false, - }, -]); +const intrinsicRequestHeadersGetter = Object.getOwnPropertyDescriptor(Request.prototype, "headers").get; export function applyFunction(fn, receiver, args) { return intrinsicReflectApply(fn, receiver, args); @@ -84,19 +73,37 @@ export function regexpTest(regexp, value) { return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); } -export function currentRequestId() { - const requestId = intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); - return typeof requestId === "string" && requestId ? requestId : null; +export function requestIdFromEventArg(arg) { + if (!arg || typeof arg !== "object") return null; + try { + const headers = intrinsicReflectApply(intrinsicRequestHeadersGetter, arg, []); + const requestId = intrinsicReflectApply(intrinsicHeadersGet, headers, ["x-request-id"]); + return sanitizeRequestId(requestId); + } catch { + return null; + } } -export function runWithRequestContext(requestId, callback) { - if (typeof requestId !== "string" || !requestId) { - return intrinsicReflectApply(callback, undefined, []); - } - return intrinsicReflectApply(intrinsicAsyncLocalStorageRun, requestContext, [ - requestId, - callback, - ]); +export function createRequestContext() { + const requestContext = new AsyncLocalStorage(); + return { + currentRequestId() { + const store = intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); + const requestId = store?.requestId; + return typeof requestId === "string" && requestId ? requestId : null; + }, + runWithRequestContext(requestId, callback) { + const activeStore = intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); + if (activeStore !== undefined) { + return intrinsicReflectApply(callback, undefined, []); + } + const canonicalRequestId = sanitizeRequestId(requestId); + return intrinsicReflectApply(intrinsicAsyncLocalStorageRun, requestContext, [ + { requestId: canonicalRequestId }, + callback, + ]); + }, + }; } `; @@ -199,14 +206,14 @@ const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; const HOST_BINDINGS_WRAPPED = __WdlHostRuntime__.createPrivateSymbol("wdl.host-bindings-wrapped"); const INTERNAL_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; +const REQUEST_CONTEXT = __WdlHostRuntime__.createRequestContext(); function requestIdFromEventArg(arg) { - if (!arg || !arg.headers || typeof arg.headers.get !== "function") return null; - return arg.headers.get("x-request-id"); + return __WdlHostRuntime__.requestIdFromEventArg(arg); } function requestIdOptions() { - return { requestIdProvider: __WdlHostRuntime__.currentRequestId }; + return { requestIdProvider: REQUEST_CONTEXT.currentRequestId }; } function doOptions(backend, ownerNetwork) { @@ -218,7 +225,7 @@ function workflowOptions(backend) { } function withRequestContext(arg, fn) { - return __WdlHostRuntime__.runWithRequestContext(requestIdFromEventArg(arg), fn); + return REQUEST_CONTEXT.runWithRequestContext(requestIdFromEventArg(arg), fn); } function wrapClassInstance(instance) { diff --git a/rust/workflows/src/api.rs b/rust/workflows/src/api.rs index 5f154ae..8c8e247 100644 --- a/rust/workflows/src/api.rs +++ b/rust/workflows/src/api.rs @@ -16,6 +16,7 @@ mod limits; mod model; mod payload; mod pending_create; +mod pending_restart; mod progress; mod redis_script; mod retention; @@ -25,7 +26,7 @@ mod status; mod tick; use active_export::{ ensure_worker_not_deleting, request_with_active_version, verify_active_workflow_current, - verify_active_workflow_export, verify_workflow_def, + verify_workflow_def, }; pub(crate) use create::{create_batch, create_instance}; pub(crate) use do_alarms::{ @@ -72,6 +73,10 @@ use pending_create::{ pending_create_cleanup_from_state, pending_create_expired, pending_create_token, public_state_or_empty, wait_for_public_create_state, }; +use pending_restart::{ + active_pending_restart_blockers, create_pending_restart, pending_restart_marker, + remove_pending_restart, renew_pending_restart, +}; use progress::{ spawn_progress_from_identity, spawn_progress_from_request, spawn_progress_from_step, }; diff --git a/rust/workflows/src/api/active_export.rs b/rust/workflows/src/api/active_export.rs index 700e4a0..7da1295 100644 --- a/rust/workflows/src/api/active_export.rs +++ b/rust/workflows/src/api/active_export.rs @@ -124,47 +124,6 @@ pub(super) async fn verify_workflow_def_values( Ok(()) } -pub(super) async fn verify_active_workflow_export( - state: &AppState, - req: &WorkflowRequest, -) -> WorkflowResult<()> { - let key = bundle_key(&req.ns, &req.worker, &req.frozen_version)?; - let raw: Option = state - .control_redis - .with_conn(async |mut conn| { - redis::cmd("HGET") - .arg(key) - .arg("__meta__") - .query_async(&mut conn) - .await - }) - .await?; - let raw = raw.ok_or_else(|| { - WorkflowError::not_exported("Workflow is not exported by the active worker version") - })?; - let meta: JsonValue = serde_json::from_str(&raw).map_err(|err| { - WorkflowError::invalid_state(format!("Active worker metadata is corrupt: {err}")) - })?; - let workflows = meta - .get("workflows") - .and_then(JsonValue::as_array) - .ok_or_else(|| { - WorkflowError::not_exported("Workflow is not exported by the active worker version") - })?; - let exported = workflows.iter().any(|entry| { - entry.get("name").and_then(JsonValue::as_str) == Some(req.workflow_name.as_str()) - && entry.get("workflowKey").and_then(JsonValue::as_str) - == Some(req.workflow_key.as_str()) - && entry.get("className").and_then(JsonValue::as_str) == Some(req.class_name.as_str()) - }); - if !exported { - return Err(WorkflowError::not_exported( - "Workflow is not exported by the active worker version", - )); - } - Ok(()) -} - pub(super) async fn verify_active_workflow_current( state: &AppState, req: &WorkflowRequest, @@ -179,7 +138,7 @@ pub(super) async fn verify_active_workflow_current( .await?; if active.version != req.frozen_version || active.class_name != req.class_name { return Err(WorkflowError::not_exported( - "Workflow active worker export changed during create", + "Workflow active worker export changed during mutation", )); } Ok(()) @@ -202,7 +161,7 @@ pub(super) async fn ensure_worker_not_deleting( .await?; if lock_exists > 0 { return Err(WorkflowError::deleting( - "Worker is being deleted; workflow creation is blocked", + "Worker is being deleted; workflow mutation is blocked", )); } Ok(()) diff --git a/rust/workflows/src/api/lifecycle/cleanup.rs b/rust/workflows/src/api/lifecycle/cleanup.rs index f0cc240..db78df2 100644 --- a/rust/workflows/src/api/lifecycle/cleanup.rs +++ b/rust/workflows/src/api/lifecycle/cleanup.rs @@ -4,9 +4,9 @@ use crate::{AppState, WorkflowResult, by_version_key, by_worker_key}; use super::super::{ LIFECYCLE_BLOCKER_LIMIT, LifecycleBlocker, LifecycleCheckRequest, LifecycleCheckResponse, - PendingCreateCleanup, cleanup_pending_create_identity, is_pending_create, - parse_workflow_referrer_member, pending_create_cleanup_from_state, pending_create_expired, - read_state_by_id, require_non_empty, + PendingCreateCleanup, active_pending_restart_blockers, cleanup_pending_create_identity, + is_pending_create, parse_workflow_referrer_member, pending_create_cleanup_from_state, + pending_create_expired, read_state_by_id, require_non_empty, }; enum LifecycleMemberState { @@ -55,8 +55,22 @@ pub(crate) async fn check_delete_lifecycle( Some(version) => by_version_key(&req.ns, &req.worker, version), None => by_worker_key(&req.ns, &req.worker), }; + let (pending_count, mut blockers) = match &req.version { + Some(version) => { + active_pending_restart_blockers( + state, + &req.ns, + &req.worker, + version, + req.allow_cleanup, + LIFECYCLE_BLOCKER_LIMIT, + ) + .await? + } + None => (0, Vec::new()), + }; let count_key = key.clone(); - let count: usize = state + let referrer_count: usize = state .redis .with_conn(async |mut conn| { redis::cmd("SCARD") @@ -65,6 +79,7 @@ pub(crate) async fn check_delete_lifecycle( .await }) .await?; + let count = referrer_count.saturating_add(pending_count); if count == 0 { return Ok(LifecycleCheckResponse { allowed: true, @@ -72,8 +87,14 @@ pub(crate) async fn check_delete_lifecycle( blockers: Vec::new(), }); } + if blockers.len() >= LIFECYCLE_BLOCKER_LIMIT { + return Ok(LifecycleCheckResponse { + allowed: false, + count, + blockers, + }); + } let mut cursor = 0_u64; - let mut blockers = Vec::new(); let mut expired_pending = Vec::new(); loop { let scan_key = key.clone(); diff --git a/rust/workflows/src/api/lifecycle/restart.rs b/rust/workflows/src/api/lifecycle/restart.rs index 841f5cb..16845eb 100644 --- a/rust/workflows/src/api/lifecycle/restart.rs +++ b/rust/workflows/src/api/lifecycle/restart.rs @@ -6,16 +6,23 @@ use crate::{ }; use super::super::{ - InstanceResponse, InstanceRouteKeys, WorkflowRequest, eval_script, identity_from_state, - instance_id, payload_bytes_arg, read_public_state, request_with_active_version, - validate_identity, verify_active_workflow_export, verify_workflow_def, - workflow_referrer_member, + InstanceResponse, InstanceRouteKeys, WorkflowRequest, create_pending_restart, + ensure_worker_not_deleting, eval_script, identity_from_state, instance_id, payload_bytes_arg, + pending_restart_marker, read_public_state, remove_pending_restart, renew_pending_restart, + request_with_active_version, validate_identity, verify_active_workflow_current, + verify_workflow_def, workflow_referrer_member, }; use super::common::{ TransitionSignal, next_generation, stale_transition_response, successful_transition_response, }; const RESTART_SCRIPT: &str = r#" +local marker_score = redis.call("ZSCORE", KEYS[16], ARGV[12]) +local marker_time = redis.call("TIME") +local marker_now = tonumber(marker_time[1]) * 1000 + math.floor(tonumber(marker_time[2]) / 1000) +if not marker_score or tonumber(marker_score) <= marker_now then + return -1 +end local status = redis.call("HGET", KEYS[1], "status") if not status then return 0 @@ -36,6 +43,7 @@ redis.call("HDEL", KEYS[1], "outputRef", "errorRef", "errorCode", "errorMessage" redis.call("HDEL", KEYS[1], "runToken", "runLeaseExpiresAtMs", "waitingEventIndexPrefix") redis.call("SADD", KEYS[7], ARGV[6]) redis.call("SADD", KEYS[13], ARGV[10]) +redis.call("ZREM", KEYS[16], ARGV[12]) redis.call("ZREM", KEYS[8], ARGV[6]) redis.call("SREM", KEYS[9], ARGV[7]) redis.call("SADD", KEYS[10], ARGV[7]) @@ -45,14 +53,14 @@ redis.call("ZADD", KEYS[14], ARGV[3], ARGV[11]) return 1 "#; +const RESTART_LEASE_EXPIRED: i64 = -1; + pub(crate) async fn restart_instance( state: &AppState, req: WorkflowRequest, ) -> WorkflowResult { let req = request_with_active_version(state, req).await?; validate_identity(&req)?; - verify_active_workflow_export(state, &req).await?; - verify_workflow_def(state, &req).await?; let id = instance_id(&req)?.to_string(); let mut existing = read_public_state(state, &req).await?; if existing.is_empty() { @@ -84,6 +92,28 @@ pub(crate) async fn restart_instance( }) .await? .ok_or_else(|| WorkflowError::payload_missing("Workflow params payload is missing"))?; + ensure_worker_not_deleting(state, &req.ns, &req.worker).await?; + let pending_restart = pending_restart_marker(state, &req, &id); + create_pending_restart(state, &pending_restart).await?; + if let Err(err) = verify_active_workflow_current(state, &req).await { + remove_pending_restart(state, &pending_restart).await?; + return Err(err); + } + if let Err(err) = verify_workflow_def(state, &req).await { + remove_pending_restart(state, &pending_restart).await?; + return Err(err); + } + // Refresh immediately before the final delete-lock check. A version delete + // that starts on either side of this check sees the lock or this blocker. + if !renew_pending_restart(state, &pending_restart).await? { + return Err(WorkflowError::conflict( + "Workflow restart target lease expired; retry restart", + )); + } + if let Err(err) = ensure_worker_not_deleting(state, &req.ns, &req.worker).await { + remove_pending_restart(state, &pending_restart).await?; + return Err(err); + } let shard = keys.shard(); let ready = keys.ready(); let due = keys.due(); @@ -102,7 +132,7 @@ pub(crate) async fn restart_instance( let stored_payloads_key = payloads_key.clone(); let request_version = req.frozen_version.clone(); let params_bytes = payload_bytes_arg(¶ms_json); - let updated: i64 = eval_script( + let updated: i64 = match eval_script( state, RESTART_SCRIPT, &[ @@ -121,6 +151,7 @@ pub(crate) async fn restart_instance( ready_active_key(), &by_workflow, &event_index_key, + &pending_restart.key, ], &[ &expected_generation, @@ -134,10 +165,24 @@ pub(crate) async fn restart_instance( ¶ms_bytes, &shard_arg, &identity.instance_id, + &pending_restart.member, ], ) - .await?; + .await + { + Ok(updated) => updated, + Err(err) => { + remove_pending_restart(state, &pending_restart).await?; + return Err(err); + } + }; + if updated == RESTART_LEASE_EXPIRED { + return Err(WorkflowError::conflict( + "Workflow restart target lease expired; retry restart", + )); + } if updated != 1 { + remove_pending_restart(state, &pending_restart).await?; return stale_transition_response(state, &identity, &id).await; } existing.insert("status".to_string(), "queued".to_string()); @@ -189,10 +234,21 @@ mod tests { #[test] fn restart_updates_class_name_for_active_version() { + assert!( + RESTART_SCRIPT + .contains(r#"local marker_score = redis.call("ZSCORE", KEYS[16], ARGV[12])"#) + ); + assert!(RESTART_SCRIPT.contains(r#"local marker_time = redis.call("TIME")"#)); assert!(RESTART_SCRIPT.contains(r#""className", ARGV[5]"#)); assert!(RESTART_SCRIPT.contains(r#"redis.call("SADD", KEYS[7], ARGV[6])"#)); assert!(RESTART_SCRIPT.contains(r#"redis.call("SADD", KEYS[13], ARGV[10])"#)); + assert!(RESTART_SCRIPT.contains(r#"redis.call("ZREM", KEYS[16], ARGV[12])"#)); assert!(RESTART_SCRIPT.contains(r#"redis.call("ZADD", KEYS[14], ARGV[3], ARGV[11])"#)); + let lease_check = RESTART_SCRIPT.find("if not marker_score").unwrap(); + let first_write = RESTART_SCRIPT + .find(r#"redis.call("DEL", KEYS[2])"#) + .unwrap(); + assert!(lease_check < first_write); } #[test] diff --git a/rust/workflows/src/api/pending_restart.rs b/rust/workflows/src/api/pending_restart.rs new file mode 100644 index 0000000..1600e8e --- /dev/null +++ b/rust/workflows/src/api/pending_restart.rs @@ -0,0 +1,212 @@ +use wdl_rust_common::time::random_hex_64; + +use crate::{AppState, WorkflowError, WorkflowResult, pending_version_key}; + +use super::{LifecycleBlocker, WorkflowRequest, eval_script}; + +const PENDING_RESTART_TTL_MS: i64 = 30_000; +const PENDING_RESTART_KEY_TTL_MS: i64 = 60_000; + +const CREATE_PENDING_RESTART_SCRIPT: &str = r#" +local time = redis.call("TIME") +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now) +redis.call("ZADD", KEYS[1], now + tonumber(ARGV[2]), ARGV[1]) +redis.call("PEXPIRE", KEYS[1], ARGV[3]) +return 1 +"#; + +const RENEW_PENDING_RESTART_SCRIPT: &str = r#" +local time = redis.call("TIME") +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now) +local current = redis.call("ZSCORE", KEYS[1], ARGV[1]) +if not current then + return 0 +end +redis.call("ZADD", KEYS[1], now + tonumber(ARGV[2]), ARGV[1]) +redis.call("PEXPIRE", KEYS[1], ARGV[3]) +return 1 +"#; + +const READ_PENDING_RESTART_SCRIPT: &str = r#" +local time = redis.call("TIME") +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +if ARGV[1] == "1" then + redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now) +end +local active_min = "(" .. tostring(now) +local count = redis.call("ZCOUNT", KEYS[1], active_min, "+inf") +local members = redis.call("ZRANGEBYSCORE", KEYS[1], active_min, "+inf", "LIMIT", 0, ARGV[2]) +return { count, members } +"#; + +pub(super) struct PendingRestartMarker { + pub(super) key: String, + pub(super) member: String, +} + +pub(super) fn pending_restart_marker( + state: &AppState, + req: &WorkflowRequest, + instance_id: &str, +) -> PendingRestartMarker { + PendingRestartMarker { + key: pending_version_key(&req.ns, &req.worker, &req.frozen_version), + member: format!( + "{}\t{}\t{}:{}", + req.workflow_key, + instance_id, + state.instance_id, + random_hex_64() + ), + } +} + +pub(super) async fn create_pending_restart( + state: &AppState, + marker: &PendingRestartMarker, +) -> WorkflowResult<()> { + let marker_ttl = PENDING_RESTART_TTL_MS.to_string(); + let key_ttl = PENDING_RESTART_KEY_TTL_MS.to_string(); + let _: i64 = eval_script( + state, + CREATE_PENDING_RESTART_SCRIPT, + &[&marker.key], + &[&marker.member, &marker_ttl, &key_ttl], + ) + .await?; + Ok(()) +} + +pub(super) async fn renew_pending_restart( + state: &AppState, + marker: &PendingRestartMarker, +) -> WorkflowResult { + let marker_ttl = PENDING_RESTART_TTL_MS.to_string(); + let key_ttl = PENDING_RESTART_KEY_TTL_MS.to_string(); + let renewed: i64 = eval_script( + state, + RENEW_PENDING_RESTART_SCRIPT, + &[&marker.key], + &[&marker.member, &marker_ttl, &key_ttl], + ) + .await?; + Ok(renewed == 1) +} + +pub(super) async fn remove_pending_restart( + state: &AppState, + marker: &PendingRestartMarker, +) -> WorkflowResult<()> { + let key = marker.key.clone(); + let member = marker.member.clone(); + state + .redis + .with_conn(async move |mut conn| { + redis::cmd("ZREM") + .arg(key) + .arg(member) + .query_async::<()>(&mut conn) + .await + }) + .await?; + Ok(()) +} + +pub(super) async fn active_pending_restart_blockers( + state: &AppState, + ns: &str, + worker: &str, + version: &str, + allow_cleanup: bool, + limit: usize, +) -> WorkflowResult<(usize, Vec)> { + let key = pending_version_key(ns, worker, version); + let cleanup = if allow_cleanup { "1" } else { "0" }; + let limit = limit.to_string(); + let (count, members): (usize, Vec) = eval_script( + state, + READ_PENDING_RESTART_SCRIPT, + &[&key], + &[cleanup, &limit], + ) + .await?; + let blockers = members + .into_iter() + .map(|member| parse_pending_restart_member(&member)) + .collect::>>()?; + Ok((count, blockers)) +} + +fn parse_pending_restart_member(member: &str) -> WorkflowResult { + let mut parts = member.split('\t'); + let workflow_key = parts.next().unwrap_or_default(); + let instance_id = parts.next().unwrap_or_default(); + let token = parts.next().unwrap_or_default(); + if workflow_key.is_empty() + || instance_id.is_empty() + || token.is_empty() + || parts.next().is_some() + { + return Err(WorkflowError::invalid_state( + "Pending workflow restart blocker is corrupt", + )); + } + Ok(LifecycleBlocker { + workflow_key: workflow_key.to_string(), + instance_id: instance_id.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pending_restart_member_preserves_blocker_identity() { + let blocker = parse_pending_restart_member("wf_key\tinstance-1\ttask:token").unwrap(); + assert_eq!(blocker.workflow_key, "wf_key"); + assert_eq!(blocker.instance_id, "instance-1"); + assert_eq!(PENDING_RESTART_TTL_MS, 30_000); + assert_eq!(PENDING_RESTART_KEY_TTL_MS, 60_000); + } + + #[test] + fn pending_restart_member_rejects_malformed_state() { + for member in [ + "", + "wf_key", + "wf_key\tinstance-1", + "wf\tinstance\ttoken\textra", + ] { + assert!(parse_pending_restart_member(member).is_err()); + } + } + + #[test] + fn pending_restart_scripts_use_redis_time_and_do_not_resurrect_expired_members() { + assert!(CREATE_PENDING_RESTART_SCRIPT.contains(r#"redis.call("TIME")"#)); + assert!( + CREATE_PENDING_RESTART_SCRIPT.contains(r#"redis.call("PEXPIRE", KEYS[1], ARGV[3])"#) + ); + assert!( + RENEW_PENDING_RESTART_SCRIPT + .contains(r#"local current = redis.call("ZSCORE", KEYS[1], ARGV[1])"#) + ); + assert!(RENEW_PENDING_RESTART_SCRIPT.contains("if not current then")); + assert!(READ_PENDING_RESTART_SCRIPT.contains(r#"redis.call("TIME")"#)); + assert!(READ_PENDING_RESTART_SCRIPT.contains("return { count, members }")); + assert!(CREATE_PENDING_RESTART_SCRIPT.contains("ZREMRANGEBYSCORE")); + assert!(RENEW_PENDING_RESTART_SCRIPT.contains("ZREMRANGEBYSCORE")); + let cleanup = RENEW_PENDING_RESTART_SCRIPT + .find("ZREMRANGEBYSCORE") + .unwrap(); + let expiry_check = RENEW_PENDING_RESTART_SCRIPT.find("if not current").unwrap(); + let write = RENEW_PENDING_RESTART_SCRIPT + .find(r#"redis.call("ZADD""#) + .unwrap(); + assert!(cleanup < expiry_check); + assert!(expiry_check < write); + } +} diff --git a/rust/workflows/src/keys.rs b/rust/workflows/src/keys.rs index faf7dc1..9738af4 100644 --- a/rust/workflows/src/keys.rs +++ b/rust/workflows/src/keys.rs @@ -162,6 +162,10 @@ pub(crate) fn by_version_key(ns: &str, worker: &str, version: &str) -> String { format!("wf:by-version:{ns}:{worker}:{version}") } +pub(crate) fn pending_version_key(ns: &str, worker: &str, version: &str) -> String { + format!("wf:pending-version:{ns}:{worker}:{version}") +} + pub(crate) fn retention_key() -> &'static str { "wf:retention" } diff --git a/rust/workflows/src/tests.rs b/rust/workflows/src/tests.rs index f0881f7..79e1d24 100644 --- a/rust/workflows/src/tests.rs +++ b/rust/workflows/src/tests.rs @@ -4,8 +4,8 @@ use super::{ }; use crate::api::{canonical_json, retry_due_at_ms, retry_policy}; use crate::{ - InstanceKeys, config_from_env, due_key, ready_active_key, ready_key, schema_version_key, - validate_instance_id_value, + InstanceKeys, config_from_env, due_key, pending_version_key, ready_active_key, ready_key, + schema_version_key, validate_instance_id_value, }; use wdl_rust_common::test_env::with_temp_envs; // Source-contract tests below intentionally couple to these module paths. If @@ -287,6 +287,14 @@ fn workflow_ready_key_uses_fixed_shards() { assert_eq!(ready_key(31), "wf:ready:31"); } +#[test] +fn pending_restart_key_is_scoped_to_the_target_version() { + assert_eq!( + pending_version_key("demo", "orders", "v3"), + "wf:pending-version:demo:orders:v3" + ); +} + #[test] fn workflow_schema_version_key_is_global_db2_marker() { assert_eq!(schema_version_key(), "wf:schema_version"); diff --git a/shared/ns-pattern.js b/shared/ns-pattern.js index c13bbba..ac954e8 100644 --- a/shared/ns-pattern.js +++ b/shared/ns-pattern.js @@ -6,13 +6,31 @@ // the tier that routes traffic. export const NS_PATTERN = "[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?"; export const DEFAULT_PLATFORM_DOMAIN = "workers.local"; +export const MAX_PLATFORM_DOMAIN_BYTES = 126; +const DNS_LABEL_RE = /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$/; +const DNS_FINAL_LABEL_RE = /^[A-Za-z]+$/; + +/** @param {string} value */ +function isAscii(value) { + for (let index = 0; index < value.length; index += 1) { + if (value.charCodeAt(index) > 0x7f) return false; + } + return true; +} /** @param {unknown} value */ export function configuredHostname(value) { if (typeof value !== "string" || !value.trim()) return null; - const host = value.trim().replace(/\.+$/, "").toLowerCase(); - if (!host || host.includes("/") || host.includes(":") || /\s/.test(host)) return null; - return host; + const trimmed = value.trim(); + const rawHost = trimmed.endsWith(".") ? trimmed.slice(0, -1) : trimmed; + if (!isAscii(rawHost)) return null; + if (rawHost.length === 0 || rawHost.length > MAX_PLATFORM_DOMAIN_BYTES) return null; + const labels = rawHost.split("."); + if ( + labels.some((label) => !DNS_LABEL_RE.test(label)) || + !DNS_FINAL_LABEL_RE.test(labels.at(-1) || "") + ) return null; + return rawHost.toLowerCase(); } /** @param {Record} env */ @@ -20,7 +38,7 @@ export function platformDomainFromEnv(env) { const raw = env.PLATFORM_DOMAIN; if (raw == null || raw === "" || typeof raw !== "string") return DEFAULT_PLATFORM_DOMAIN; const domain = configuredHostname(raw); - if (!domain) throw new TypeError("PLATFORM_DOMAIN must be a plain hostname"); + if (!domain) throw new TypeError("PLATFORM_DOMAIN must be an ALB-compatible ASCII DNS hostname"); return domain; } @@ -114,6 +132,7 @@ export const WORKFLOW_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; // Workflow instance ids are embedded into DB2 keys and tab-delimited scheduler // tokens. Keep them URL/key friendly and delimiter-free. export const WORKFLOW_INSTANCE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +export const WORKFLOW_KEY_RE = /^wf_[0-9a-f]{32}$/; // `:` in an id would corrupt `queue:::s` parsing; camelCase // would split one logical queue across two log-field entries. @@ -129,6 +148,11 @@ export const BINDING_NAME_RE = /^[A-Za-z_$][A-Za-z0-9_$]{0,63}$/; // while binding names add a platform cap for env/log hygiene. export const JS_IDENTIFIER_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/; +// DO host ids are `<35-byte storage id>::shard15` and may not exceed +// 512 UTF-8 bytes. Class declarations use ASCII grammar, so 468 characters is +// the deploy-time bound that keeps every shard addressable. +export const MAX_DO_CLASS_NAME_BYTES = 468; + /** * @param {unknown} value * @returns {boolean} diff --git a/shared/workerd-compat-flags.js b/shared/workerd-compat-flags.js index 33e72b9..b06e252 100644 --- a/shared/workerd-compat-flags.js +++ b/shared/workerd-compat-flags.js @@ -1,13 +1,25 @@ /* - * Mirrors workerd v1.20260701.1 src/workerd/io/compatibility-date.capnp. - * Regenerate on every workerd pin bump from an upstream workerd source checkout: + * The experimental flags mirror workerd v1.20260701.1 + * src/workerd/io/compatibility-date.capnp. Refresh them on every workerd pin + * bump from an upstream source checkout: * * node scripts/extract-workerd-experimental-compat-flags.mjs \ * /path/to/workerd/src/workerd/io/compatibility-date.capnp + * + * The dynamic-worker minimum/default dates and required error-serialization + * behavior below are WDL policy rather than upstream mirror data. */ export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION = "1.20260701.1"; +// WDL supports one forward-only dynamic-worker compatibility surface. Static +// platform workers keep their independently pinned workerd service dates. +export const MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE = "2026-04-01"; +export const DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE = "2026-04-24"; +export const ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE = "2026-04-21"; +export const ENHANCED_ERROR_SERIALIZATION_FLAG = "enhanced_error_serialization"; +export const LEGACY_ERROR_SERIALIZATION_FLAG = "legacy_error_serialization"; + export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS = Object.freeze([ "allow_insecure_inefficient_logged_eval", "allow_irrevocable_stub_storage", diff --git a/terraform/README.md b/terraform/README.md index 477ee6d..259d41f 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -147,16 +147,6 @@ assets_cdn_domain = "" assets_cdn_acm_certificate_arn = "" ``` -The application stack accepts at least two pre-existing subnet ids. It validates -that each subnet belongs to `vpc_id`, uses an RFC1918 or `100.64.0.0/10` IPv4 CIDR, -and occupies a different Availability Zone so each D1/DO EFS file system creates -at most one mount target per AZ. The subnet `map_public_ip_on_launch` flag is not a -private-routing proof; ECS services explicitly disable public task IP assignment. -Foundation keeps its three-AZ default and applies the same address contract to the -VPC and every subnet before provisioning. Unsupported ranges or duplicate-AZ -subnets fail during planning rather than after tasks start claiming owner records -or creating EFS mount targets. - Initialize the application root with its own `terraform/backend.hcl` and apply: ```sh diff --git a/terraform/foundation/main.tf b/terraform/foundation/main.tf index ab3e5c8..359ed22 100644 --- a/terraform/foundation/main.tf +++ b/terraform/foundation/main.tf @@ -3,33 +3,9 @@ data "aws_availability_zones" "available" { } locals { - azs = slice(data.aws_availability_zones.available.names, 0, 3) - public_subnets = { for idx, cidr in var.public_subnet_cidrs : idx => cidr } - private_subnets = { for idx, cidr in var.private_subnet_cidrs : idx => cidr } - network_cidrs = concat([var.vpc_cidr], var.public_subnet_cidrs, var.private_subnet_cidrs) - private_cidr_contract = { - for cidr in local.network_cidrs : cidr => try( - ( - tonumber(split(".", cidrhost(cidr, 0))[0]) == 10 && - tonumber(split("/", cidr)[1]) >= 8 - ) || ( - tonumber(split(".", cidrhost(cidr, 0))[0]) == 100 && - tonumber(split(".", cidrhost(cidr, 0))[1]) >= 64 && - tonumber(split(".", cidrhost(cidr, 0))[1]) <= 127 && - tonumber(split("/", cidr)[1]) >= 10 - ) || ( - tonumber(split(".", cidrhost(cidr, 0))[0]) == 172 && - tonumber(split(".", cidrhost(cidr, 0))[1]) >= 16 && - tonumber(split(".", cidrhost(cidr, 0))[1]) <= 31 && - tonumber(split("/", cidr)[1]) >= 12 - ) || ( - tonumber(split(".", cidrhost(cidr, 0))[0]) == 192 && - tonumber(split(".", cidrhost(cidr, 0))[1]) == 168 && - tonumber(split("/", cidr)[1]) >= 16 - ), - false, - ) - } + azs = slice(data.aws_availability_zones.available.names, 0, 3) + public_subnets = { for idx, cidr in var.public_subnet_cidrs : idx => cidr } + private_subnets = { for idx, cidr in var.private_subnet_cidrs : idx => cidr } assets_cdn_enabled = var.assets_cdn_domain != "" site_host_enabled = var.site_host != "" site_www_host = local.site_host_enabled ? "www.${var.site_host}" : "" @@ -50,13 +26,6 @@ resource "aws_vpc" "this" { enable_dns_hostnames = true enable_dns_support = true - lifecycle { - precondition { - condition = alltrue(values(local.private_cidr_contract)) - error_message = "vpc_cidr and subnet CIDRs must be IPv4 ranges within RFC1918 or 100.64.0.0/10." - } - } - tags = { Name = "${var.name_prefix}-vpc" } diff --git a/terraform/main.tf b/terraform/main.tf index f1e4f26..b3e5651 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -2,43 +2,6 @@ locals { name = var.name_prefix } -data "aws_subnet" "private_contract" { - for_each = toset(var.private_subnet_ids) - - id = each.value - - lifecycle { - postcondition { - condition = self.vpc_id == var.vpc_id - error_message = "Private subnet ${self.id} must belong to vpc_id ${var.vpc_id}." - } - postcondition { - condition = try( - ( - tonumber(split(".", cidrhost(self.cidr_block, 0))[0]) == 10 && - tonumber(split("/", self.cidr_block)[1]) >= 8 - ) || ( - tonumber(split(".", cidrhost(self.cidr_block, 0))[0]) == 100 && - tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) >= 64 && - tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) <= 127 && - tonumber(split("/", self.cidr_block)[1]) >= 10 - ) || ( - tonumber(split(".", cidrhost(self.cidr_block, 0))[0]) == 172 && - tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) >= 16 && - tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) <= 31 && - tonumber(split("/", self.cidr_block)[1]) >= 12 - ) || ( - tonumber(split(".", cidrhost(self.cidr_block, 0))[0]) == 192 && - tonumber(split(".", cidrhost(self.cidr_block, 0))[1]) == 168 && - tonumber(split("/", self.cidr_block)[1]) >= 16 - ), - false, - ) - error_message = "Private subnet ${self.id} must use an IPv4 CIDR within RFC1918 or 100.64.0.0/10." - } - } -} - module "data" { source = "./modules/data" @@ -52,13 +15,10 @@ module "data" { module "compute" { source = "./modules/compute" - name = local.name - region = var.region - vpc_id = var.vpc_id - private_subnet_ids = var.private_subnet_ids - private_subnet_availability_zone_ids = toset([ - for subnet in data.aws_subnet.private_contract : subnet.availability_zone_id - ]) + name = local.name + region = var.region + vpc_id = var.vpc_id + private_subnet_ids = var.private_subnet_ids alb_https_listener_arn = var.alb_https_listener_arn alb_security_group_id = var.alb_security_group_id platform_domain = var.platform_domain diff --git a/terraform/modules/compute/variables.tf b/terraform/modules/compute/variables.tf index 3feae48..7ff69f4 100644 --- a/terraform/modules/compute/variables.tf +++ b/terraform/modules/compute/variables.tf @@ -3,13 +3,6 @@ variable "region" { type = string } variable "vpc_id" { type = string } variable "private_subnet_ids" { type = list(string) } -variable "private_subnet_availability_zone_ids" { - type = set(string) - validation { - condition = length(var.private_subnet_availability_zone_ids) == length(var.private_subnet_ids) - error_message = "Each private subnet must belong to a different Availability Zone so EFS creates at most one mount target per AZ." - } -} variable "alb_https_listener_arn" { type = string } variable "alb_security_group_id" { type = string } diff --git a/terraform/variables.tf b/terraform/variables.tf index 2a976f0..c619a2e 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -120,13 +120,10 @@ variable "vpc_id" { variable "private_subnet_ids" { type = list(string) - description = "Private subnet ids across at least two AZs for ECS tasks, EFS mount targets, and ElastiCache." + description = "Private subnet ids across at least three AZs for ECS tasks, EFS mount targets, and ElastiCache." validation { - condition = ( - length(var.private_subnet_ids) >= 2 && - length(var.private_subnet_ids) == length(distinct(var.private_subnet_ids)) - ) - error_message = "Provide at least two unique private subnet ids." + condition = length(var.private_subnet_ids) >= 3 + error_message = "Provide at least three private subnet ids across different AZs." } } diff --git a/test-workers/do-rpc/src/index.js b/test-workers/do-rpc/src/index.js index 1d8d2e1..5019f5e 100644 --- a/test-workers/do-rpc/src/index.js +++ b/test-workers/do-rpc/src/index.js @@ -1,4 +1,6 @@ import { DurableObject } from "cloudflare:workers"; +import * as hostWrapperRuntime from "./_wdl-host-wrapper-runtime.js"; +import wrappedWorker from "./_wdl-wrapper.js"; export class Room extends DurableObject { constructor(ctx, env) { @@ -46,9 +48,13 @@ export class Room extends DurableObject { } async forwardRequestId(targetName) { + this.env.ROOM.requestId = () => "tenant-rid"; const target = this.env.ROOM.get(this.env.ROOM.idFromName(targetName)); - const response = await target.fetch(new Request("https://do.internal/request-id")); - return response.text(); + const tenantContext = hostWrapperRuntime.createRequestContext(); + return tenantContext.runWithRequestContext("tenant-rid", async () => { + const response = await target.fetch(new Request("https://do.internal/request-id")); + return response.text(); + }); } async nestedForwardRequestId(targetName) { @@ -73,6 +79,12 @@ export class Room extends DurableObject { export default { async fetch(request, env) { const url = new URL(request.url); + if (url.pathname === "/reenter-request-id" && url.searchParams.get("nested") !== "1") { + url.searchParams.set("nested", "1"); + return wrappedWorker.fetch(new Request(url, { + headers: { "x-request-id": "tenant-rid" }, + }), env, {}); + } const id = env.ROOM.idFromName(url.searchParams.get("name") || "main"); const stub = env.ROOM.get(id); if (url.pathname === "/fail") { @@ -114,6 +126,11 @@ export default { requestId: await stub.forwardRequestId(url.searchParams.get("to") || "peer"), }); } + if (url.pathname === "/reenter-request-id") { + return Response.json({ + requestId: await stub.forwardRequestId(url.searchParams.get("to") || "peer"), + }); + } if (url.pathname === "/nested-request-id") { return Response.json({ requestId: await stub.nestedForwardRequestId(url.searchParams.get("to") || "peer"), diff --git a/tests/helpers/runtime-injection-sources.js b/tests/helpers/runtime-injection-sources.js index d6f2519..6b095e2 100644 --- a/tests/helpers/runtime-injection-sources.js +++ b/tests/helpers/runtime-injection-sources.js @@ -61,7 +61,7 @@ export const STUB_RUNTIME_INJECTION_SOURCES = Object.freeze({ ownerHintCacheSource: "export function createOwnerHintCache() { return {}; }", requestIdSource: - "export function requestIdFromOptions() { return null; }", + "export function requestIdFromOptions() { return null; } export function sanitizeRequestId() { return null; }", workflowsClientSource: "export class Workflow { constructor(metadata) { this.metadata = metadata; } }", }); diff --git a/tests/integration/durable-objects-core.test.js b/tests/integration/durable-objects-core.test.js index 7f9cad9..ce8e75c 100644 --- a/tests/integration/durable-objects-core.test.js +++ b/tests/integration/durable-objects-core.test.js @@ -251,4 +251,13 @@ test("Durable Object RPC dispatches class methods with structured args", async ( const nestedContextText = await nestedContext.text(); assert.equal(nestedContext.status, 200, nestedContextText); assert.deepEqual(responseJson({ body: nestedContextText }), { requestId }); + + const reenteredContext = await gatewayFetch( + ns, + "/rooms/reenter-request-id?name=alice&to=bob", + { headers: { "x-request-id": requestId } } + ); + const reenteredContextText = await reenteredContext.text(); + assert.equal(reenteredContext.status, 200, reenteredContextText); + assert.deepEqual(responseJson({ body: reenteredContextText }), { requestId }); }); diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index acdcd84..cb4cd9d 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -153,6 +153,7 @@ const { handle: handleVersions } = await importControlHandler("control/handlers/ * activeVersion?: string | null, * assetPrefix?: string | null, * hasWorkerSecrets?: boolean, + * hasWorkflowDefs?: boolean, * doStorageId?: string | null, * doStorageIdDuringExec?: string | null, * doOwnerKeys?: string[], @@ -176,6 +177,7 @@ function resetDeleteHandlerState({ activeVersion = "v1", assetPrefix = "assets/demo/api/v1/", hasWorkerSecrets = false, + hasWorkflowDefs = false, doStorageId = "do_old", doStorageIdDuringExec = doStorageId, doOwnerKeys = [], @@ -277,7 +279,9 @@ function resetDeleteHandlerState({ }, /** @param {string} key */ async exists(key) { - return key === "secrets:demo:api" && hasWorkerSecrets ? 1 : 0; + if (key === "secrets:demo:api") return hasWorkerSecrets ? 1 : 0; + if (key === "wf:defs:demo:api") return hasWorkflowDefs ? 1 : 0; + return 0; }, /** @param {string} key */ async sMembers(key) { return versionReferrers(key); }, @@ -375,7 +379,9 @@ function resetDeleteHandlerState({ }, /** @param {string} key */ async exists(key) { - return key === "secrets:demo:api" && hasWorkerSecrets ? 1 : 0; + if (key === "secrets:demo:api") return hasWorkerSecrets ? 1 : 0; + if (key === "wf:defs:demo:api") return hasWorkflowDefs ? 1 : 0; + return 0; }, /** @param {unknown[]} args */ async del(...args) { multiCalls.push(["DEL", ...args]); }, @@ -399,13 +405,14 @@ function resetDeleteHandlerState({ }); } -/** @param {{ assetPrefix?: string | null, bundleMetaRaw?: string | null, retainedVersions?: string[], siblingMetaRaw?: string | null, lockTokenDuringExec?: string | null }} [opts] */ +/** @param {{ assetPrefix?: string | null, bundleMetaRaw?: string | null, retainedVersions?: string[], siblingMetaRaw?: string | null, lockTokenDuringExec?: string | null, hasWorkflowDefs?: boolean }} [opts] */ function resetVersionDeleteHandlerState({ assetPrefix = "assets/demo/api/v1/", bundleMetaRaw, retainedVersions = ["v1"], siblingMetaRaw = null, lockTokenDuringExec = "lock-token", + hasWorkflowDefs = false, } = {}) { const meta = bundleMetaRaw === undefined ? JSON.stringify({ @@ -415,8 +422,11 @@ function resetVersionDeleteHandlerState({ : bundleMetaRaw; /** @type {unknown[][]} */ const multiCalls = []; + /** @type {string[][]} */ + const watchBatches = []; const session = { - async watch() {}, + /** @param {string[]} keys */ + async watch(...keys) { watchBatches.push(keys); }, async unwatch() {}, /** @param {string} key */ async get(key) { @@ -436,7 +446,10 @@ function resetVersionDeleteHandlerState({ return []; }, async sMembers() { return []; }, - async exists() { return 0; }, + /** @param {string} key */ + async exists(key) { + return key === "wf:defs:demo:api" && hasWorkflowDefs ? 1 : 0; + }, multi() { return { /** @param {unknown[]} args */ @@ -462,6 +475,7 @@ function resetVersionDeleteHandlerState({ redis, metrics: { increment() {}, observe() {} }, service: "control", + watchBatches, workflowChecks: [], }); } @@ -908,6 +922,34 @@ test("worker delete recomputes sibling host ownership under WATCH", async () => ), false); }); +test("worker delete fails closed on malformed held pattern projections", async () => { + const host = "app.example"; + const slot = "target-slot"; + const testState = resetDeleteHandlerState({ + assetPrefix: null, + bundleMetaRaw: JSON.stringify({ + bindings: {}, + routes: [{ host, slot }], + }), + patternRecordsDuringExec: { + [`patterns:${host}`]: { [slot]: "not-a-projection" }, + }, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-corrupt-pattern", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_pattern_projection"); + assert.equal(testState.multiCalls.some((call) => call[0] === "EXEC"), false); +}); + test("worker delete cannot commit after its delete lock token is replaced", async () => { const testState = resetDeleteHandlerState({ assetPrefix: null, @@ -1336,6 +1378,59 @@ test("worker delete noop skips DO alarm cleanup when old storage id is absent", assert.equal(testState.multiCalls.length, 0); }); +test("worker delete removes orphaned workflow definitions without active versions", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + assetPrefix: null, + doStorageId: null, + queueConsumerWorker: null, + retainedVersions: [], + hasWorkflowDefs: true, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-orphan-workflow-defs", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.ok(testState.watchBatches.some((keys) => keys.includes("wf:defs:demo:api"))); + assert.ok(testState.multiCalls.some((call) => + call[0] === "DEL" && call.includes("wf:defs:demo:api") + )); +}); + +test("worker delete dry-run reports orphaned workflow definitions", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + assetPrefix: null, + doStorageId: null, + queueConsumerWorker: null, + retainedVersions: [], + hasWorkflowDefs: true, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-dry-run-orphan-workflow-defs", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(body.hasWorkflowDefs, true); + assert.deepEqual(body.versionsDeleted, []); + assert.equal(testState.multiCalls.length, 0); +}); + test("worker delete reports queueHint none when no content cleanup is needed", async () => { const testState = resetDeleteHandlerState({ assetPrefix: null }); @@ -1433,6 +1528,28 @@ test("version delete cannot commit after its delete lock token is replaced", asy assert.deepEqual(testState.cleanupIntents, []); }); +test("version delete keeps a definitions-only worker discoverable", async () => { + const testState = resetVersionDeleteHandlerState({ + assetPrefix: null, + hasWorkflowDefs: true, + }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: "rid-version-delete-workflow-defs", + }); + + await readJsonResponse(response, 200); + assert.ok(testState.watchBatches.some((keys) => keys.includes("wf:defs:demo:api"))); + assert.equal(testState.multiCalls.some((call) => + call[0] === "SREM" && call[1] === "workers:demo" && call[2] === "api" + ), false); +}); + test("version delete classifies non-object bundle metadata as corrupt", async () => { const testState = resetVersionDeleteHandlerState({ bundleMetaRaw: "null" }); diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 9b155fa..19c5e0b 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -1540,6 +1540,60 @@ test("commitWithWatch assigns stable workflow keys into bundle meta and wf:defs" ]); }); +test("commitWithWatch reads only workflow definitions declared by the new bundle", async () => { + const duplicateKey = "wf_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + /** @type {unknown[][]} */ + const workflowDefReads = []; + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ + ["wf:defs:tenant-a:orders", { + orders: JSON.stringify({ workflowKey: duplicateKey, className: "OrderWorkflow" }), + historical: "not-json", + }], + ]); + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta = null; + /** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys = []; + /** @type {any} */ (globalThis).__controlDeployTestState.execFailures = 0; + + const redis = { + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + const session = makeSession(); + const hMGet = session.hMGet.bind(session); + session.hMGet = async (key, fields) => { + workflowDefReads.push([key, [...fields]]); + return await hMGet(key, fields); + }; + return await fn(session); + }, + }; + + await commitWithWatch({ + redis, + ns: "tenant-a", + name: "orders", + version: "v1", + prepared: { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "esm" } }, + workflows: [ + { name: "orders", binding: "ORDERS", className: "OrderWorkflow" }, + ], + }, + normalized: [["worker.js", "export default {}"]], + }, + outgoingRefs: [], + d1Refs: [], + controlEnv: {}, + }); + assert.equal( + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta.workflows[0].workflowKey, + duplicateKey + ); + assert.deepEqual(workflowDefReads, [["wf:defs:tenant-a:orders", ["orders"]]]); +}); + test("commitWithWatch checks code budget after workflow keys are materialized", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ diff --git a/tests/unit/control-handlers-workers.test.js b/tests/unit/control-handlers-workers.test.js index 66af90b..f6b02cf 100644 --- a/tests/unit/control-handlers-workers.test.js +++ b/tests/unit/control-handlers-workers.test.js @@ -38,7 +38,7 @@ const { handle } = await import(moduleDataUrl(src)); function resetWorkersHandlerState() { const redis = createFakeRedis(); - redis.sets.set("workers:demo", new Set(["beta", "alpha"])); + redis.sets.set("workers:demo", new Set(["gamma", "beta", "alpha"])); redis.hashes.set("routes:demo", { alpha: "v2" }); redis.zsets.set("worker-versions:demo:alpha", new Map([ ["v1", 1], @@ -46,6 +46,7 @@ function resetWorkersHandlerState() { ])); redis.zsets.set("worker-versions:demo:beta", new Map([["v1", 1]])); redis.hashes.set("secrets:demo:beta", { TOKEN: "WDL-ENC:test" }); + redis.hashes.set("wf:defs:demo:gamma", { flow: "{}" }); let sessions = 0; const session = redis.session.bind(redis); redis.session = async (fn) => { @@ -72,6 +73,7 @@ test("workers handler lists namespace state through one Redis session", async () versions: ["v1", "v2"], versionCount: 2, hasSecrets: false, + hasWorkflowDefs: false, }, { name: "beta", @@ -79,6 +81,15 @@ test("workers handler lists namespace state through one Redis session", async () versions: ["v1"], versionCount: 1, hasSecrets: true, + hasWorkflowDefs: false, + }, + { + name: "gamma", + activeVersion: null, + versions: [], + versionCount: 0, + hasSecrets: false, + hasWorkflowDefs: true, }, ], }); @@ -88,15 +99,20 @@ test("workers handler lists namespace state through one Redis session", async () ["zRangeMany", [ "worker-versions:demo:alpha", "worker-versions:demo:beta", + "worker-versions:demo:gamma", ], 0, -1], ["existsMany", [ "secrets:demo:alpha", "secrets:demo:beta", + "secrets:demo:gamma", + "wf:defs:demo:alpha", + "wf:defs:demo:beta", + "wf:defs:demo:gamma", ]], ]); assert.deepEqual(state.logs, [{ level: "info", event: "workers_listed", - fields: { request_id: "rid-workers", namespace: "demo", count: 2 }, + fields: { request_id: "rid-workers", namespace: "demo", count: 3 }, }]); }); diff --git a/tests/unit/control-handlers-workflows.test.js b/tests/unit/control-handlers-workflows.test.js index 578a6a7..cdd6ef8 100644 --- a/tests/unit/control-handlers-workflows.test.js +++ b/tests/unit/control-handlers-workflows.test.js @@ -10,19 +10,24 @@ import { } from "../helpers/load-shared-module.js"; import { compileControlGraph } from "../helpers/load-control-lib.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; -import { assertJsonResponse } from "../helpers/response-json.js"; +import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; const WORKFLOWS_HANDLER_GLOBAL = "__workflowsHandlerState"; +const ACTIVE_WORKFLOW_KEY = `wf_${"1".repeat(32)}`; +const NEW_WORKFLOW_KEY = `wf_${"2".repeat(32)}`; +const RETIRED_WORKFLOW_KEY = `wf_${"3".repeat(32)}`; const { libUrl: productionControlLibUrl } = await compileControlGraph(); const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const sharedNsPatternUrl = repositoryFileUrl("shared/ns-pattern.js"); const { handle } = await importControlHandler("control/handlers/workflows.js", { globalName: WORKFLOWS_HANDLER_GLOBAL, replacements: { "control-lib": productionControlLibUrl, "shared-internal-auth": sharedInternalAuthUrl(), + "shared-ns-pattern": sharedNsPatternUrl, "shared-version": sharedVersionUrl, }, }); @@ -33,7 +38,7 @@ function resetWorkflowsHandlerState() { name: "orders", binding: "ORDERS", className: "OrderWorkflow", - workflowKey: "wf_1234", + workflowKey: ACTIVE_WORKFLOW_KEY, }], }); const state = createControlHandlerState({ @@ -78,7 +83,7 @@ test("workflows handler lists active workflow definitions from bundle metadata", name: "orders", binding: "ORDERS", className: "OrderWorkflow", - workflowKey: "wf_1234", + workflowKey: ACTIVE_WORKFLOW_KEY, }], }); assert.deepEqual(state.redis.commands, [ @@ -143,6 +148,92 @@ for (const [label, rawMeta] of [ }); } +test("workflows handler fails closed on malformed active workflow entries", async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set("worker:demo:api:v:2", { + "__meta__": JSON.stringify({ + workflows: [{ + name: "orders", + binding: "ORDERS", + className: "OrderWorkflow", + }], + }), + }); + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-malformed-workflow-entry", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, undefined); + const rejection = state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_request_rejected" + ); + assert.equal(rejection.fields.stage, "workflow_entries_parse"); +}); + +test("workflows handler rejects active workflows that share a workflow key", async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set("worker:demo:api:v:2", { + "__meta__": JSON.stringify({ + workflows: [ + { + name: "orders", + binding: "ORDERS", + className: "OrderWorkflow", + workflowKey: ACTIVE_WORKFLOW_KEY, + }, + { + name: "billing", + binding: "BILLING", + className: "BillingWorkflow", + workflowKey: ACTIVE_WORKFLOW_KEY, + }, + ], + }), + }); + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-duplicate-active-workflow-key", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + const rejection = state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_request_rejected" + ); + assert.equal(rejection.fields.stage, "workflow_entries_parse"); +}); + +test("workflows handler fails closed on malformed persisted workflow definitions", async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set("wf:defs:demo:api", { retired: "not-json" }); + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-malformed-workflow-def", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + const rejection = state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_request_rejected" + ); + assert.equal(rejection.fields.stage, "workflow_defs_parse"); +}); + test("workflows handler retries a list snapshot split by whole-worker delete", async () => { const state = resetWorkflowsHandlerState(); const redis = /** @type {any} */ (state.redis); @@ -320,18 +411,18 @@ test("workflows handler does not combine active metadata with redeployed workflo const state = resetWorkflowsHandlerState(); const redis = /** @type {any} */ (state.redis); redis.hashes.set("worker:demo:api:v:2", { "__meta__": JSON.stringify({ workflows: [] }) }); - const originalHGetAll = redis.hGetAll.bind(redis); + const originalHGet = redis.hGet.bind(redis); let redeployed = false; - redis.hGetAll = async (/** @type {string} */ key) => { - if (key === "wf:defs:demo:api" && !redeployed) { + redis.hGet = async (/** @type {string} */ key, /** @type {string} */ field) => { + if (key === "wf:defs:demo:api" && field === "orders" && !redeployed) { redeployed = true; redis.hashes.set("routes:demo", { api: "v3" }); redis.hashes.set("worker:demo:api:v:3", { "__meta__": JSON.stringify({ workflows: [] }) }); redis.hashes.set(key, { - orders: JSON.stringify({ workflowKey: "wf_new", className: "NewOrderWorkflow" }), + orders: JSON.stringify({ workflowKey: NEW_WORKFLOW_KEY, className: "NewOrderWorkflow" }), }); } - return await originalHGetAll(key); + return await originalHGet(key, field); }; const response = await handle({ @@ -348,7 +439,7 @@ test("workflows handler does not combine active metadata with redeployed workflo worker: "api", frozenVersion: "v3", workflowName: "orders", - workflowKey: "wf_new", + workflowKey: NEW_WORKFLOW_KEY, className: "NewOrderWorkflow", instanceId: "order-1", options: {}, @@ -486,7 +577,7 @@ test("workflows handler resolves retired workflow definitions from wf:defs", asy state.redis.hashes.set("worker:demo:api:v:3", { "__meta__": JSON.stringify({ workflows: [] }) }); state.redis.hashes.set("wf:defs:demo:api", { orders: JSON.stringify({ - workflowKey: "wf_retired", + workflowKey: RETIRED_WORKFLOW_KEY, className: "OldOrderWorkflow", }), }); @@ -505,7 +596,7 @@ test("workflows handler resolves retired workflow definitions from wf:defs", asy worker: "api", frozenVersion: "v3", workflowName: "orders", - workflowKey: "wf_retired", + workflowKey: RETIRED_WORKFLOW_KEY, className: "OldOrderWorkflow", instanceId: "order-1", requestId: "rid-retired", @@ -518,7 +609,7 @@ test("workflows handler rejects restart for retired workflow definitions", async state.redis.hashes.set("worker:demo:api:v:3", { "__meta__": JSON.stringify({ workflows: [] }) }); state.redis.hashes.set("wf:defs:demo:api", { orders: JSON.stringify({ - workflowKey: "wf_retired", + workflowKey: RETIRED_WORKFLOW_KEY, className: "OldOrderWorkflow", }), }); @@ -574,12 +665,13 @@ test("workflows handler resolves active workflow identity before lifecycle proxy assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], + ["hGet", "routes:demo", "api"], ["fetch", "http://workflows/internal/workflows/resume", { ns: "demo", worker: "api", frozenVersion: "v2", workflowName: "orders", - workflowKey: "wf_1234", + workflowKey: ACTIVE_WORKFLOW_KEY, className: "OrderWorkflow", instanceId: "order-1", requestId: "rid-resume", @@ -604,7 +696,7 @@ test("workflows handler forwards status includeSteps options", async () => { worker: "api", frozenVersion: "v2", workflowName: "orders", - workflowKey: "wf_1234", + workflowKey: ACTIVE_WORKFLOW_KEY, className: "OrderWorkflow", instanceId: "order-1", options: { includeSteps: true, stepLimit: 10 }, @@ -631,6 +723,7 @@ test("workflows handler fails closed when workflows backend is unavailable", asy assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], + ["hGet", "routes:demo", "api"], ]); assert.deepEqual(state.logs, [{ level: "error", @@ -741,6 +834,7 @@ test("workflows handler rejects invalid status query options before backend disp assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], + ["hGet", "routes:demo", "api"], ]); }); @@ -762,5 +856,6 @@ test("workflows handler rejects snake_case status query options", async () => { assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], + ["hGet", "routes:demo", "api"], ]); }); diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index e5f9f02..2558b1b 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -4,6 +4,7 @@ import { loadControlLib } from "../helpers/load-control-lib.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { RESERVED_NS } from "../../shared/ns-pattern.js"; import { + MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, WORKERD_EXPERIMENTAL_COMPAT_FLAGS, WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION, } from "../../shared/workerd-compat-flags.js"; @@ -625,6 +626,17 @@ test("prepareBundle: durable object binding requires className-shaped class", () ); assert.equal(meta.bindings.ROOMS.type, "do"); assert.equal(meta.bindings.ROOMS.className, "Room"); + + const maxClassName = `R${"o".repeat(467)}`; + assert.doesNotThrow(() => prepareBundle("w.js", { "w.js": "x" }, { + bindings: { ROOMS: { type: "do", className: maxClassName } }, + })); + assert.throws( + () => prepareBundle("w.js", { "w.js": "x" }, { + bindings: { ROOMS: { type: "do", className: `${maxClassName}m` } }, + }), + /at most 468 bytes/ + ); }); test("prepareBundle: queue binding requires queue-name-shaped 'id'", () => { @@ -838,6 +850,42 @@ test("prepareBundle: experimental workerd compatibility flags are rejected", () ); }); +test("prepareBundle: error serialization flags match the effective compatibility date", () => { + assert.doesNotThrow(() => prepareBundle( + "w.js", + { "w.js": "x" }, + { + compatibilityDate: "2026-04-20", + compatibilityFlags: ["enhanced_error_serialization"], + } + )); + + /** @type {[string | undefined, string][]} */ + const cases = [ + ["2026-04-20", "legacy_error_serialization"], + ["2026-04-21", "legacy_error_serialization"], + ["2026-04-21", "enhanced_error_serialization"], + [undefined, "enhanced_error_serialization"], + ]; + for (const [compatibilityDate, flag] of cases) { + assert.throws( + () => prepareBundle( + "w.js", + { "w.js": "x" }, + { compatibilityDate, compatibilityFlags: [flag] } + ), + (err) => { + if (!(err instanceof Error)) return false; + const coded = /** @type {Error & { code?: unknown, status?: unknown }} */ (err); + return coded.code === "compatibility_flag_unsupported" && + coded.status === 400 && + coded.message.includes(flag); + }, + `${flag} should be rejected for ${compatibilityDate ?? "the default date"}` + ); + } +}); + test("prepareBundle: compatibilityDate validates shape before commit", () => { assert.equal( prepareBundle("w.js", { "w.js": "x" }, { compatibilityDate: "2026-04-24" }).meta.compatibilityDate, @@ -866,6 +914,14 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( validateCompatibilityDate("2026-06-20", new Date("2026-06-30T00:00:00Z")), "2026-06-20" ); + assert.equal( + validateCompatibilityDate(MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, new Date("2026-06-30T00:00:00Z")), + MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE + ); + assert.throws( + () => validateCompatibilityDate("2026-03-31", new Date("2026-06-30T00:00:00Z")), + /older than WDL supports/ + ); assert.throws( () => validateCompatibilityDate("2026-06-15", new Date("2026-06-14T00:00:00Z")), /must not be later than today UTC/ @@ -2075,6 +2131,10 @@ test("projectAccessPrincipal returns only the public principal shape", () => { test("configuredHostname accepts plain hosts and rejects URL injection shapes", () => { assert.equal(configuredHostname(" Workers.Local. "), "workers.local"); assert.equal(configuredHostname("workers.local."), "workers.local"); + assert.equal( + configuredHostname(`${"a".repeat(63)}.${"b".repeat(58)}.com`), + `${"a".repeat(63)}.${"b".repeat(58)}.com` + ); for (const bad of [ "", " ", @@ -2082,6 +2142,20 @@ test("configuredHostname accepts plain hosts and rejects URL injection shapes", "workers.local/path", "workers.local:8080", "workers local", + "workers.example#oops", + "workers.example?query", + "workers@example", + "workers..example", + ".workers.example", + "workers.example..", + "-workers.example", + "workers-.example", + "wörkers.example", + "K.example", + "workers.123", + `${"a".repeat(64)}.example`, + `${"a".repeat(63)}.${"b".repeat(59)}.com`, + `${"a".repeat(250)}.com`, null, ]) { assert.equal(configuredHostname(bad), null, `expected ${JSON.stringify(bad)} rejected`); @@ -2093,7 +2167,7 @@ test("platformDomainFromEnv normalizes configured values and owns the default", assert.equal(platformDomainFromEnv({ PLATFORM_DOMAIN: " Workers.Example. " }), "workers.example"); assert.throws( () => platformDomainFromEnv({ PLATFORM_DOMAIN: "workers.example:8443" }), - /plain hostname/ + /ALB-compatible ASCII DNS hostname/ ); }); diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index 3779b91..7c94e9e 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -466,6 +466,38 @@ test("bumpActiveAndPromote also rewrites full queue consumer projection", async }); }); +test("bumpActiveAndPromote batches pattern projection reads across hosts", async () => { + const redis = makeRedis(); + const routes = ["api.example", "admin.example"].map((host) => ({ + host, + slot: "/*", + kind: "prefix", + value: "/", + })); + seedBundle(redis, "v1", { routes }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.sets.set("hosts:demo", new Set(routes.map((route) => route.host))); + for (const route of routes) { + redis.state.hashes.set(`patterns:${route.host}`, { + [route.slot]: patternProjection("demo", "worker", "v1", route.kind, route.value), + }); + } + redis.state.strings.set("worker:demo:worker:next_version", "1"); + + await bumpActiveAndPromote(redis, "demo", "worker"); + + assert.deepEqual( + redis.state.commands.filter(([command]) => command === "hGetAllMany"), + [["hGetAllMany", ["patterns:api.example", "patterns:admin.example"]]] + ); + assert.equal( + redis.state.commands.some(([command, key]) => + command === "hGetAll" && typeof key === "string" && key.startsWith("patterns:") + ), + false + ); +}); + test("bumpActiveAndPromote retries when active changes before source metadata read", async () => { const redis = makeRedis(); seedBundle(redis, "v1", {}); @@ -644,6 +676,40 @@ test("bumpActiveAndPromote rejects active routes that no longer declare their ho ); }); +/** @type {Array<[string, string, number, string]>} */ +const bumpProjectionFailureCases = [ + ["malformed", "not-a-projection", 500, "corrupt_pattern_projection"], + ["foreign", patternProjection("other", "site", "v7", "prefix", "/"), 409, "route_conflict"], +]; +for (const [label, held, expectedStatus, expectedCode] of bumpProjectionFailureCases) { + test(`bumpActiveAndPromote fails closed on ${label} held pattern projections`, async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { + routes: [{ + host: "app.workers.example", + slot: "/*", + kind: "prefix", + value: "/", + }], + }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.hashes.set("patterns:app.workers.example", { "/*": held }); + redis.state.sets.set("hosts:demo", new Set(["app.workers.example"])); + redis.state.strings.set("worker:demo:worker:next_version", "1"); + + await assert.rejects( + bumpActiveAndPromote(redis, "demo", "worker"), + (err) => { + assertRoutingErrorShape(err, expectedStatus, expectedCode); + return true; + } + ); + + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v1"); + assert.equal(redis.state.hashes.has(productionBundleKey("demo", "worker", "v2")), false); + }); +} + test("bumpActiveAndPromote rejects malformed cron metadata", async () => { const redis = makeRedis(); /** @type {Array<{ level: string, event: string, fields: any }>} */ @@ -762,6 +828,30 @@ test("promoteWithRoutes rejects a custom host already owned by another namespace assert.equal(redis.state.hashes.get("routes:demo")?.worker, undefined); }); +test("promoteWithRoutes fails closed on malformed occupied pattern projections", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { + routes: [{ + host: "app.workers.example", + slot: "/*", + kind: "prefix", + value: "/", + }], + }); + redis.state.sets.set("hosts:demo", new Set(["app.workers.example"])); + redis.state.hashes.set("patterns:app.workers.example", { "/*": "not-a-projection" }); + + await assert.rejects( + promoteWithRoutes(redis, "demo", "worker", "v1"), + (err) => { + const shaped = assertRoutingErrorShape(err, 500, "corrupt_pattern_projection"); + assert.deepEqual(shaped.details, { host: "app.workers.example", slot: "/*" }); + return true; + } + ); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, undefined); +}); + test("reconcileHosts stages declared host indexes and pattern invalidation", async () => { const redis = makeRedis(); diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 5b855bd..8a8dee3 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -719,13 +719,16 @@ function workerSecretRejectionLogsSince(start) { const workerControlSharedUrl = controlSharedHarnessUrl(WORKER_SECRET_STATE_GLOBAL); const workerLibStubUrl = moduleDataUrl(` ${validateSecretKeyStubSource} +export { workflowDefsKey } from ${JSON.stringify(PRODUCTION_CONTROL_LIB_URL)}; export const deleteLockKey = (ns, worker) => \`worker-delete-lock:\${ns}:\${worker}\`; export const workerVersionsKey = (ns, worker) => \`worker-versions:\${ns}:\${worker}\`; export const routesKey = (ns) => \`routes:\${ns}\`; export const workersIndexKey = (ns) => \`workers:\${ns}\`; `); const lifecycleStubUrl = moduleDataUrl(` -export function stageWorkerHidden() {} +export function stageWorkerHidden(multi, ns, name) { + multi.sRem(\`workers:\${ns}\`, name); +} export function stageWorkerVisible(multi, ns, name) { multi.sAdd(\`workers:\${ns}\`, name); } @@ -958,6 +961,39 @@ test("worker secret PUT active precheck reads the shared fake Redis state", asyn }); }); +test("worker secret DELETE keeps a definitions-only worker discoverable", async () => { + const state = workerSecretState; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + await withWorkerSecretRedis(state, (redis) => { + redis.hashes.delete("routes:demo"); + redis.hashes.set("secrets:demo:api", { TOKEN: encrypted }); + redis.hashes.set("wf:defs:demo:api", { flow: "{}" }); + redis.sets.set("workers:demo", new Set(["api"])); + }, async (redis) => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-definitions-only", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(redis.hashes.has("secrets:demo:api"), false); + assert.equal(redis.sets.get("workers:demo")?.has("api"), true); + assert.equal(redis.watched.includes("wf:defs:demo:api"), true); + }); +}); + test("worker secret PUT maps active bump delete-lock errors to deleting", async () => { const state = workerSecretState; const logStart = state.logs.length; diff --git a/tests/unit/observability.test.js b/tests/unit/observability.test.js index 19e73b2..c586f4e 100644 --- a/tests/unit/observability.test.js +++ b/tests/unit/observability.test.js @@ -17,11 +17,38 @@ import { parseStdoutJson } from "../helpers/json-payload.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; import { withCapturedConsole } from "../helpers/output-capture.js"; +import { withMockedPropertyDescriptor } from "../helpers/mock-global.js"; +import { + requestIdFromOptions, + sanitizeRequestId as sanitizeRuntimeRequestId, +} from "../../runtime/_wdl-request-id.js"; const REQUEST_ID_FIXTURES = readRepositoryJson("tests/fixtures/request-id-sanitizer.json"); const OBSERVABILITY_CONTRACT = readRepositoryJson("tests/fixtures/observability-contract.json"); const OBSERVABILITY_NOOP = await import(OBSERVABILITY_NOOP_URL); +test("loaded-runtime request-id sanitizer matches the shared contract", () => { + for (const fixture of REQUEST_ID_FIXTURES) { + assert.equal(sanitizeRuntimeRequestId(fixture.raw), fixture.sanitized); + } +}); + +test("loaded-runtime request-id options ignore inherited values", async () => { + await withMockedPropertyDescriptor(/** @type {any} */ (Object.prototype), "requestId", { + configurable: true, + value: "inherited-value", + }, async () => { + await withMockedPropertyDescriptor(/** @type {any} */ (Object.prototype), "requestIdProvider", { + configurable: true, + value: () => "inherited-provider", + }, async () => { + assert.equal(requestIdFromOptions({}), null); + assert.equal(requestIdFromOptions({ requestId: "own-value" }), "own-value"); + assert.equal(requestIdFromOptions({ requestIdProvider: () => "own-provider" }), "own-provider"); + }); + }); +}); + test("generateRequestId produces 16 lowercase hex chars", () => { for (let i = 0; i < 20; i++) { const id = generateRequestId(); diff --git a/tests/unit/runtime-bindings-do.test.js b/tests/unit/runtime-bindings-do.test.js index bf2f0be..9615c71 100644 --- a/tests/unit/runtime-bindings-do.test.js +++ b/tests/unit/runtime-bindings-do.test.js @@ -42,7 +42,7 @@ function bindingWithBackend(backend) { }); } -test("DO connect headers strip tenant protocol hop and owner hint headers", () => { +test("DO connect headers strip tenant routing headers and preserve the scope request id", () => { const request = new Request("https://tenant.workers.example/ws", { headers: { "x-wdl-do-hop-count": "99", @@ -62,12 +62,12 @@ test("DO connect headers strip tenant protocol hop and owner hint headers", () = assert.equal(headers.get("x-wdl-do-hop-count"), null); assert.equal(headers.get("x-wdl-do-owner-key"), null); assert.equal(headers.get("x-wdl-do-owner-generation"), null); - assert.equal(headers.get("x-request-id"), "tenant-rid"); + assert.equal(headers.get("x-request-id"), "scope-rid"); assert.equal(headers.get("x-wdl-do-ns"), "tenant"); assert.equal(headers.get("x-wdl-do-object-name"), "room-a"); }); -test("DO fetch requestSpec strips tenant internal owner routing headers", async () => { +test("DO fetch requestSpec strips tenant routing headers and preserves the scope request id", async () => { const { spec } = await requestSpec(new Request("https://tenant.workers.example/send", { method: "POST", headers: { @@ -88,7 +88,7 @@ test("DO fetch requestSpec strips tenant internal owner routing headers", async assert.equal(headers.get("x-wdl-do-ownership-error"), null); assert.equal(headers.get("x-wdl-do-owner-key"), null); assert.equal(headers.get("x-wdl-do-owner-generation"), null); - assert.equal(headers.get("x-request-id"), "tenant-rid"); + assert.equal(headers.get("x-request-id"), "scope-rid"); }); test("DO fetch requestSpec uses captured header mutation intrinsics", async () => { diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index bb461bd..723aca8 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -80,7 +80,7 @@ function cancellableDoOwnerHintResponse(onCancel, options = {}) { }); } -test("DurableObjectNamespace facade forwards fetch with object name and request id", async () => { +test("DurableObjectNamespace fetch uses its private request-id provider", async () => { /** @type {any[]} */ const calls = []; const ns = new DurableObjectNamespace({ @@ -89,6 +89,7 @@ test("DurableObjectNamespace facade forwards fetch with object name and request return new Response("ok", { status: 201 }); }, }, { requestIdProvider: () => "rid-1" }); + Reflect.set(ns, "requestId", () => "tenant-rid"); const id = ns.idFromName("room-a"); const response = await ns.get(id).fetch("https://demo.workers.example/chat", { @@ -199,6 +200,7 @@ test("DurableObjectNamespace direct backend keeps binding/backend in private fie const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { method: "POST", + headers: { "x-request-id": "tenant-forged" }, body: "hello", }); @@ -215,6 +217,8 @@ test("DurableObjectNamespace direct backend keeps binding/backend in private fie assert.equal(body.doStorageId, "do_0123456789abcdef0123456789abcdef"); assert.equal(body.className, "Room"); assert.equal(body.objectName, "room-a"); + const forwardedRequest = /** @type {{ headers: HeadersInit }} */ (body.request); + assert.equal(new Headers(forwardedRequest.headers).get("x-request-id"), "rid-1"); assert.equal(Buffer.from(bodyBytes).toString("utf8"), "hello"); }); @@ -317,7 +321,7 @@ test("DurableObjectNamespace fetch rejects oversized invoke envelopes before tra ); }); -test("DurableObjectNamespace stub forwards arbitrary RPC methods", async () => { +test("DurableObjectNamespace RPC uses its private request-id provider", async () => { /** @type {any[]} */ const calls = []; const backend = { @@ -331,6 +335,7 @@ test("DurableObjectNamespace stub forwards arbitrary RPC methods", async () => { binding: "ROOM", className: "Room", }, { backend, requestIdProvider: () => "rid-rpc" }); + Reflect.set(ns, "requestId", () => "tenant-rid"); const stub = ns.get(ns.idFromName("room-a")); const result = await Reflect.get(stub, "addMessage")("hi", { role: "user" }); @@ -2290,6 +2295,7 @@ test("DurableObjectNamespace facade uses direct upgrade path for websockets", as Connection: "Upgrade", Upgrade: "websocket", "Sec-WebSocket-Key": "abc", + "x-request-id": "tenant-forged", }, }); @@ -2441,17 +2447,22 @@ test("DurableObjectNamespace websocket path does not fall back to router after o className: "Room", }, { backend, ownerNetwork }); - const response = await ns.get(ns.idFromName("room-hint-ws-fail")).fetch("https://demo.workers.example/ws", { + let liveResponseJsonCalls = 0; + const response = await withMockedProperty(Response, "json", () => { + liveResponseJsonCalls += 1; + return new Response("forged", { status: 200 }); + }, () => ns.get(ns.idFromName("room-hint-ws-fail")).fetch("https://demo.workers.example/ws", { headers: { Connection: "Upgrade", Upgrade: "websocket", "Sec-WebSocket-Key": "abc", }, - }); + })); const body = await readJsonResponse(response, 503); assert.equal(body.error, "owner_unavailable"); assert.equal(routerCalls.length, 1); + assert.equal(liveResponseJsonCalls, 0); }); test("DurableObjectNamespace POST fetch does not replay through router after direct owner failure", async () => { diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index 6ff3c8e..e180753 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -329,13 +329,29 @@ test("bundleToWorkerCode: uses meta.compatibilityDate when set", () => { mkBundle( { mainModule: "w.js", - compatibilityDate: "2024-01-01", + compatibilityDate: "2026-04-01", modules: { "w.js": { type: "module" } }, }, { "w.js": enc.encode("x") } ) ); - assert.equal(code.compatibilityDate, "2024-01-01"); + assert.equal(code.compatibilityDate, "2026-04-01"); +}); + +test("bundleToWorkerCode: rejects compatibilityDate below the WDL dynamic-worker floor", () => { + assert.throws( + () => bundleToWorkerCode( + mkBundle( + { + mainModule: "w.js", + compatibilityDate: "2026-03-31", + modules: { "w.js": { type: "module" } }, + }, + { "w.js": enc.encode("x") } + ) + ), + /meta\.compatibilityDate 2026-03-31 is older than WDL supports \(2026-04-01\)/ + ); }); test("bundleToWorkerCode: compatibilityFlags merge user-declared with old-date platform floor, meta is frozen", () => { @@ -401,6 +417,33 @@ test("bundleToWorkerCode: compatibilityFlags already includes floor → no dup", assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization", "nodejs_compat"]); }); +test("bundleToWorkerCode: incompatible error serialization flags fail closed", () => { + /** @type {[string | undefined, string, RegExp][]} */ + const cases = [ + ["2026-04-20", "legacy_error_serialization", /requires enhanced error serialization/], + ["2026-04-21", "legacy_error_serialization", /requires enhanced error serialization/], + ["2026-04-21", "enhanced_error_serialization", /became the workerd default/], + [undefined, "enhanced_error_serialization", /became the workerd default/], + ]; + for (const [compatibilityDate, flag, message] of cases) { + assert.throws( + () => bundleToWorkerCode( + mkBundle( + { + mainModule: "w.js", + ...(compatibilityDate ? { compatibilityDate } : {}), + compatibilityFlags: [flag], + modules: { "w.js": { type: "module" } }, + }, + { "w.js": enc.encode("x") } + ) + ), + message, + `${flag} should be rejected for ${compatibilityDate ?? "the default date"}` + ); + } +}); + test("bundleToWorkerCode: throws (doesn't silently drop) on malformed compatibilityFlags in bundle bytes", () => { assert.throws( () => bundleToWorkerCode( diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index c34fcc2..9ee0e13 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -1758,7 +1758,7 @@ test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserv test("wrapWorkerCodeForHostBindings: normalizes standalone ALS compatibility flags", () => { for (const [input, expected] of [ [["no_nodejs_als"], ["nodejs_als"]], - [["streams_enable_constructors", "no_nodejs_als"], ["streams_enable_constructors", "nodejs_als"]], + [["no_global_navigator", "no_nodejs_als"], ["no_global_navigator", "nodejs_als"]], [["nodejs_compat", "no_nodejs_als"], ["nodejs_compat", "no_nodejs_als"]], ]) { const workerCode = { @@ -1835,6 +1835,13 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc mainModule: "worker.js", modules: { "worker.js": ` + import * as hostRuntime from "./_wdl-host-wrapper-runtime.js"; + export function hostRuntimeContextExports() { + return { + currentRequestId: typeof hostRuntime.currentRequestId, + runWithRequestContext: typeof hostRuntime.runWithRequestContext, + }; + } export class Api { constructor(ctx, env) { this.env = env; } dbConstructorName() { return this.env.DB?.constructor?.name; } @@ -1872,7 +1879,6 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc const stubbed = name === "_wdl-wrapper.js" ? source .replace(`from "cloudflare:workers"`, `from "./_cf_workers_stub.js"`) - .replace("function withRequestContext", "export function withRequestContext") : source; writeFileSync(file, stubbed); } @@ -1892,33 +1898,10 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc assert.ok(new wrapped[name]({}, { DB: { raw: true } }) instanceof user[name]); } - const request = new Request("https://demo.workers.example", { - headers: { "x-request-id": "rid-instance" }, - }); - const contextRuntime = await import(`file://${path.join(dir, "_wdl-host-wrapper-runtime.js")}`); - const firstStarted = Promise.withResolvers(); - const releaseFirst = Promise.withResolvers(); - const first = wrapped.withRequestContext(request, () => { - const result = (async () => { - firstStarted.resolve(undefined); - await releaseFirst.promise; - return contextRuntime.currentRequestId(); - })(); - return result; - }); - await firstStarted.promise; - - const secondRequest = new Request("https://demo.workers.example", { - headers: { "x-request-id": "rid-second" }, - }); - const second = wrapped.withRequestContext(secondRequest, async () => { - await Promise.resolve(); - return contextRuntime.currentRequestId(); + assert.deepEqual(wrapped.hostRuntimeContextExports(), { + currentRequestId: "undefined", + runWithRequestContext: "undefined", }); - assert.equal(await second, "rid-second"); - releaseFirst.resolve(undefined); - assert.equal(await first, "rid-instance"); - assert.equal(contextRuntime.currentRequestId(), null); let thenReads = 0; const stateful = {}; diff --git a/tests/unit/runtime-wrapper-generate.test.js b/tests/unit/runtime-wrapper-generate.test.js index cf43cf1..902a697 100644 --- a/tests/unit/runtime-wrapper-generate.test.js +++ b/tests/unit/runtime-wrapper-generate.test.js @@ -7,10 +7,25 @@ import { generateAbortShimWrapperModule, generateHostBindingWrapperModule, } from "../../runtime/load/wrapper-generate.js"; -import { applyModuleReplacements, moduleDataUrl } from "../helpers/load-shared-module.js"; -import { installMockProperty } from "../helpers/mock-global.js"; +import { + applyModuleReplacements, + moduleDataUrl, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; +import { + installMockProperty, + withMockedProperty, + withMockedPropertyDescriptor, +} from "../helpers/mock-global.js"; -const hostBindingRuntime = await import(moduleDataUrl(HOST_BINDING_RUNTIME_SOURCE)); +const HOST_BINDING_RUNTIME_TEST_SOURCE = applyModuleReplacements(HOST_BINDING_RUNTIME_SOURCE, [ + [ + 'from "./_wdl-request-id.js"', + `from ${JSON.stringify(repositoryFileUrl("runtime/_wdl-request-id.js"))}`, + ], +]); +const hostBindingRuntime = await import(moduleDataUrl(HOST_BINDING_RUNTIME_TEST_SOURCE)); +const hostRequestContext = hostBindingRuntime.createRequestContext(); function generatedWrappers() { return { @@ -65,6 +80,7 @@ test("host wrapper runtime captures platform intrinsics before user module evalu for (const intrinsic of [ "Array.prototype.forEach", "Function.prototype.toString", + "Headers.prototype.get", "Object.defineProperty", "Object.entries", "Object.keys", @@ -73,9 +89,12 @@ test("host wrapper runtime captures platform intrinsics before user module evalu "Reflect.apply", "Reflect.get", "RegExp.prototype.test", + 'Object.getOwnPropertyDescriptor(Request.prototype, "headers")', ]) { assert.match(HOST_BINDING_RUNTIME_SOURCE, new RegExp(RegExp.escape(intrinsic))); } + assert.match(HOST_BINDING_RUNTIME_SOURCE, /import \{ sanitizeRequestId \} from "\.\/_wdl-request-id\.js"/); + assert.doesNotMatch(HOST_BINDING_RUNTIME_SOURCE, /\benterWith\b/); assert.doesNotMatch(source, /Object\.(?:defineProperty|entries|keys)\(/); assert.doesNotMatch(source, /for \(const .* of /); assert.doesNotMatch(source, /Function\.prototype\.toString\.call/); @@ -84,41 +103,83 @@ test("host wrapper runtime captures platform intrinsics before user module evalu test("host wrapper request context is invocation-local and preserves return identity", async () => { const firstStarted = Promise.withResolvers(); const releaseFirst = Promise.withResolvers(); - const firstResult = hostBindingRuntime.runWithRequestContext("rid-first", () => { + const firstResult = hostRequestContext.runWithRequestContext("rid-first", () => { const result = (async () => { - assert.equal(hostBindingRuntime.currentRequestId(), "rid-first"); + assert.equal(hostRequestContext.currentRequestId(), "rid-first"); firstStarted.resolve(undefined); await releaseFirst.promise; - return hostBindingRuntime.currentRequestId(); + return hostRequestContext.currentRequestId(); })(); - assert.equal(hostBindingRuntime.currentRequestId(), "rid-first"); + assert.equal(hostRequestContext.currentRequestId(), "rid-first"); return result; }); await firstStarted.promise; - assert.equal(hostBindingRuntime.currentRequestId(), null); - const secondResult = hostBindingRuntime.runWithRequestContext("rid-second", async () => { + assert.equal(hostRequestContext.currentRequestId(), null); + const secondResult = hostRequestContext.runWithRequestContext("rid-second", async () => { await Promise.resolve(); - return hostBindingRuntime.currentRequestId(); + return hostRequestContext.currentRequestId(); }); assert.equal(await secondResult, "rid-second"); releaseFirst.resolve(undefined); assert.equal(await firstResult, "rid-first"); - assert.equal(hostBindingRuntime.currentRequestId(), null); + assert.equal(hostRequestContext.currentRequestId(), null); const nativePromise = Promise.resolve("same"); assert.equal( - hostBindingRuntime.runWithRequestContext("rid-identity", () => nativePromise), + hostRequestContext.runWithRequestContext("rid-identity", () => nativePromise), nativePromise ); }); -test("host wrapper request context inherits when a nested call has no new id", () => { - const nested = hostBindingRuntime.runWithRequestContext("rid-outer", () => - hostBindingRuntime.runWithRequestContext(null, () => hostBindingRuntime.currentRequestId())); +test("host wrapper request context is outermost-only", () => { + const withoutChildId = hostRequestContext.runWithRequestContext("rid-outer", () => + hostRequestContext.runWithRequestContext(null, () => hostRequestContext.currentRequestId())); + const withChildId = hostRequestContext.runWithRequestContext("rid-outer", () => + hostRequestContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); + const withoutParentId = hostRequestContext.runWithRequestContext(null, () => + hostRequestContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); + + assert.equal(withoutChildId, "rid-outer"); + assert.equal(withChildId, "rid-outer"); + assert.equal(withoutParentId, null); + assert.equal(hostRequestContext.currentRequestId(), null); +}); + +test("host wrapper request context rejects malformed child request ids", () => { + const inherited = hostRequestContext.runWithRequestContext("rid-outer", () => + hostRequestContext.runWithRequestContext("bad\\id", () => hostRequestContext.currentRequestId())); - assert.equal(nested, "rid-outer"); - assert.equal(hostBindingRuntime.currentRequestId(), null); + assert.equal(inherited, "rid-outer"); + assert.equal(hostRequestContext.currentRequestId(), null); +}); + +test("host wrapper runtime keeps independently created request-context stores separate", () => { + assert.equal(hostBindingRuntime.currentRequestId, undefined); + assert.equal(hostBindingRuntime.runWithRequestContext, undefined); + const tenantContext = hostBindingRuntime.createRequestContext(); + + const observed = hostRequestContext.runWithRequestContext("rid-parent", () => + tenantContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); + + assert.equal(observed, "rid-parent"); + assert.equal(hostRequestContext.currentRequestId(), null); +}); + +test("host wrapper request-id extraction ignores tenant-patched request intrinsics", async () => { + const request = new Request("https://worker.example", { + headers: { "x-request-id": "rid-real" }, + }); + await withMockedProperty(Headers.prototype, "get", () => "rid-forged", async () => { + await withMockedPropertyDescriptor(Request.prototype, "headers", { + configurable: true, + get() { + return new Headers({ "x-request-id": "rid-forged" }); + }, + }, async () => { + assert.equal(hostBindingRuntime.requestIdFromEventArg(request), "rid-real"); + }); + }); }); test("generated host wrappers alias legal entrypoint names without declaration collisions", async () => { @@ -157,7 +218,7 @@ test("generated host wrappers alias legal entrypoint names without declaration c ), [ ['from "cloudflare:workers"', `from ${JSON.stringify(cloudflareUrl)}`], - [`from "./${HOST_BINDING_RUNTIME_MODULE_NAME}"`, `from ${JSON.stringify(moduleDataUrl(HOST_BINDING_RUNTIME_SOURCE))}`], + [`from "./${HOST_BINDING_RUNTIME_MODULE_NAME}"`, `from ${JSON.stringify(moduleDataUrl(HOST_BINDING_RUNTIME_TEST_SOURCE))}`], ['from "./_wdl-d1-client.js"', `from ${JSON.stringify(d1Url)}`], ['from "./_wdl-r2-client.js"', `from ${JSON.stringify(r2Url)}`], ['from "./_wdl-do-client.js"', `from ${JSON.stringify(doUrl)}`], @@ -174,25 +235,28 @@ test("generated host wrappers alias legal entrypoint names without declaration c } }); -test("host wrapper context uses captured AsyncLocalStorage intrinsics", async () => { +test("host wrapper context uses captured AsyncLocalStorage intrinsics", () => { const restoreRun = installMockProperty(AsyncLocalStorage.prototype, "run", () => { throw new Error("live AsyncLocalStorage.run must not run"); }); - const restoreGetStore = installMockProperty(AsyncLocalStorage.prototype, "getStore", () => { - throw new Error("live AsyncLocalStorage.getStore must not run"); - }); let result; try { - result = hostBindingRuntime.runWithRequestContext("rid-captured", async () => { - await Promise.resolve(); - return hostBindingRuntime.currentRequestId(); + const requestContext = hostBindingRuntime.createRequestContext(); + result = requestContext.runWithRequestContext("rid-captured", () => { + const restoreGetStore = installMockProperty(AsyncLocalStorage.prototype, "getStore", () => { + throw new Error("live AsyncLocalStorage.getStore must not run"); + }); + try { + return requestContext.currentRequestId(); + } finally { + restoreGetStore(); + } }); } finally { - restoreGetStore(); restoreRun(); } - assert.equal(await result, "rid-captured"); + assert.equal(result, "rid-captured"); }); test("host wrapper does not inspect or replace tenant return values", () => { @@ -205,9 +269,9 @@ test("host wrapper does not inspect or replace tenant return values", () => { }, }); - const result = hostBindingRuntime.runWithRequestContext("rid-object", () => value); + const result = hostRequestContext.runWithRequestContext("rid-object", () => value); assert.equal(result, value); assert.equal(thenReads, 0); - assert.equal(hostBindingRuntime.currentRequestId(), null); + assert.equal(hostRequestContext.currentRequestId(), null); }); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index d98a346..8fe58c0 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -2027,6 +2027,7 @@ test("workflow instance state is owned by workflows DB2", () => { "wf:by-worker:", "wf:by-workflow:", "wf:by-version:", + "wf:pending-version:", "wf:retention", ]; const allowed = new Set([ @@ -2542,6 +2543,23 @@ test("owner endpoint validation lives in a shared contract owner", () => { } }); +test("DO transport relative dependencies are registered in host and loaded-worker module graphs", () => { + const codeBudget = readRepoFile("runtime/load/code-budget.js"); + assert.match(codeBudget, /\["_wdl-request-id\.js", sources\.requestIdSource\]/); + assert.match(codeBudget, /\["_wdl-do-transport\.js", sources\.doTransportSource\]/); + + for (const file of [ + "runtime/config-user.capnp", + "runtime/config-system.capnp", + "do-runtime/config.capnp", + ]) { + const source = readRepoFile(file); + assert.match(source, /name = "runtime-do-transport", esModule = embed/); + assert.match(source, /name = "_wdl-request-id\.js", esModule = embed/); + assert.match(source, /name = "runtime-request-id-source", text = embed/); + } +}); + test("worker-id helper carries its relative grammar dependencies in workerd configs", () => { for (const file of [ "gateway/config.capnp", From af9162bc16ecdd7d4901699e3aae1ce8804d9bee Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 02:38:35 +0800 Subject: [PATCH 15/23] Close remaining workflow and control contract gaps This commit has not received sufficient review and requires additional review before merge. Targeted tests and integration pass, but the final boundary needs another complete audit. Signed-off-by: Lu Zhang --- control/d1-lifecycle.js | 24 ++- control/d1-migrations.js | 51 ++++++ control/d1-runtime-client.js | 59 ++++++- control/handlers/delete-plan.js | 10 +- control/handlers/delete.js | 16 +- control/handlers/deploy.js | 14 +- control/handlers/hosts.js | 2 +- control/handlers/logs-tail.js | 3 +- control/handlers/promote.js | 2 +- control/handlers/versions.js | 3 +- control/handlers/worker-secrets.js | 8 +- control/handlers/workers.js | 3 +- control/handlers/workflows.js | 4 +- control/index.js | 3 +- control/lib.js | 22 --- control/lifecycle-indexes.js | 2 +- control/routing.js | 2 +- docs/modules/d1.md | 12 +- docs/modules/d1.zh.md | 4 +- runtime/dispatch.js | 67 ++++---- runtime/dispatch/workflow-step.js | 106 +++++++++---- runtime/runtime.js | 4 +- runtime/workflows-client.js | 62 ++++++-- rust/workflows/src/api/do_alarms/model.rs | 58 ++++++- shared/observability.js | 11 +- shared/request-scope.js | 7 +- shared/respond.js | 6 +- .../scheduler-projection-contract.json | 4 +- tests/helpers/load-control-lib.js | 2 +- tests/helpers/load-d1-protocol.js | 25 ++- tests/unit/control-cron-index.test.js | 16 +- tests/unit/control-d1-lifecycle.test.js | 101 ++++++++++-- tests/unit/control-d1-migrations.test.js | 114 +++++++++---- tests/unit/control-d1-runtime-client.test.js | 117 +++++++++++--- tests/unit/control-delete-handler.test.js | 35 +++- tests/unit/control-handlers-workers.test.js | 3 +- tests/unit/control-index.test.js | 3 +- tests/unit/control-lib.test.js | 28 ++-- tests/unit/control-lifecycle-indexes.test.js | 9 +- tests/unit/control-logs-tail.test.js | 5 +- .../control-secret-envelope-handlers.test.js | 3 - tests/unit/d1-runtime-index.test.js | 3 + tests/unit/observability.test.js | 14 ++ tests/unit/request-scope.test.js | 22 ++- tests/unit/runtime-dispatch-handlers.test.js | 84 +++++++++- tests/unit/runtime-dispatch-workflows.test.js | 101 ++++++++++++ tests/unit/runtime-eviction.test.js | 25 +++ tests/unit/runtime-workflows-client.test.js | 150 ++++++++++++++++++ tests/unit/version.test.js | 2 + 49 files changed, 1173 insertions(+), 258 deletions(-) diff --git a/control/d1-lifecycle.js b/control/d1-lifecycle.js index 4e9494e..3666d8b 100644 --- a/control/d1-lifecycle.js +++ b/control/d1-lifecycle.js @@ -16,6 +16,7 @@ import { } from "control-d1-model"; import { d1RuntimeFailure, + d1RuntimeFailureLogFields, d1RuntimeProbeOwner, d1RuntimePublicResult, d1RuntimeQuery, @@ -257,13 +258,22 @@ export async function createDatabase({ request, env, ns, requestId }) { reason: rollbackReason, }); } - log("warn", "d1_database_initialize_failed", { + log("error", "d1_database_initialize_failed", { request_id: requestId, namespace: ns, database_id: database.databaseId, - backend_status: initialized.status, + status: 503, + reason: "d1_database_initialize_failed", + ...d1RuntimeFailureLogFields(initialized), }); - return jsonResponse(503, d1RuntimeFailure("d1_database_initialize_failed", ns, database.databaseId, initialized)); + return jsonResponse(503, d1RuntimeFailure( + "d1_database_initialize_failed", + ns, + database.databaseId, + initialized, + {}, + { publicStatus: 503 } + )); } const readyAt = new Date().toISOString(); @@ -440,6 +450,14 @@ export async function executeDatabase({ request, env, ns, databaseId, requestId const result = await d1RuntimeQuery(env, ns, database.databaseId, mode, statements, requestId); if (!result.ok) { + log(result.status >= 500 ? "error" : "warn", "d1_database_execute_failed", { + request_id: requestId, + namespace: ns, + database_id: database.databaseId, + status: result.status, + reason: "d1_execute_failed", + ...d1RuntimeFailureLogFields(result), + }); return jsonResponse(result.status, d1RuntimeFailure("d1_execute_failed", ns, database.databaseId, result)); } diff --git a/control/d1-migrations.js b/control/d1-migrations.js index 958c88f..97310f3 100644 --- a/control/d1-migrations.js +++ b/control/d1-migrations.js @@ -23,6 +23,7 @@ import { } from "control-d1-model"; import { d1RuntimeFailure, + d1RuntimeFailureLogFields, d1RuntimePublicResult, d1RuntimeQuery, } from "control-d1-runtime-client"; @@ -51,6 +52,38 @@ function migrationLockKey(ns, databaseId) { const MIGRATION_LOCK_TTL_SECONDS = 600; const MIGRATIONS_TABLE_NAME = "_wdl_d1_migrations"; +/** + * @param {string} event + * @param {string} reason + * @param {string} requestId + * @param {string} ns + * @param {string} databaseId + * @param {{ body?: unknown, status: number }} result + * @param {Record} [extra] + */ +function logD1MigrationFailure(event, reason, requestId, ns, databaseId, result, extra = {}) { + requireControlLog()(result.status >= 500 ? "error" : "warn", event, { + request_id: requestId, + namespace: ns, + database_id: databaseId, + status: result.status, + reason, + ...extra, + ...d1RuntimeFailureLogFields(result), + }); +} + +/** + * @param {unknown[]} applied + * @param {unknown[]} skipped + */ +function migrationProgressLogFields(applied, skipped) { + return { + applied_count: applied.length, + skipped_count: skipped.length, + }; +} + /** * @param {unknown[]} applied * @param {unknown[]} skipped @@ -174,6 +207,10 @@ export async function listMigrations({ env, ns, databaseId, requestId }) { const database = checked.database; const applied = await readAppliedMigrationsIfTableExists(env, ns, database.databaseId, requestId); if (applied.ok === false) { + logD1MigrationFailure( + "d1_migrations_list_failed", "d1_migrations_list_failed", + requestId, ns, database.databaseId, applied + ); return jsonResponse(applied.status, d1RuntimeFailure("d1_migrations_list_failed", ns, database.databaseId, applied)); } return jsonResponse(200, { @@ -200,6 +237,10 @@ export async function migrationStatusEndpoint({ request, env, ns, databaseId, re } const applied = await readAppliedMigrationsIfTableExists(env, ns, database.databaseId, requestId); if (applied.ok === false) { + logD1MigrationFailure( + "d1_migrations_status_failed", "d1_migrations_status_failed", + requestId, ns, database.databaseId, applied + ); return jsonResponse(applied.status, d1RuntimeFailure("d1_migrations_status_failed", ns, database.databaseId, applied)); } const migrations = migrationStatus(localMigrations, applied.migrations); @@ -251,6 +292,11 @@ export async function applyMigrations({ request, env, ns, databaseId, requestId try { const existing = await readAppliedMigrations(env, ns, database.databaseId, requestId); if (existing.ok === false) { + logD1MigrationFailure( + "d1_migrations_apply_failed", "d1_migrations_apply_failed", + requestId, ns, database.databaseId, existing, + migrationProgressLogFields(appliedNow, skipped) + ); return jsonResponse(existing.status, d1RuntimeFailure( "d1_migrations_apply_failed", ns, @@ -314,6 +360,11 @@ export async function applyMigrations({ request, env, ns, databaseId, requestId }, ], requestId); if (!result.ok) { + logD1MigrationFailure( + "d1_migration_apply_failed", "d1_migration_apply_failed", + requestId, ns, database.databaseId, result, + { migration_id: migration.id, ...migrationProgressLogFields(appliedNow, skipped) } + ); return jsonResponse(result.status, d1RuntimeFailure( "d1_migration_apply_failed", ns, diff --git a/control/d1-runtime-client.js b/control/d1-runtime-client.js index 0ff5d88..78b7459 100644 --- a/control/d1-runtime-client.js +++ b/control/d1-runtime-client.js @@ -81,6 +81,9 @@ export async function d1RuntimeQuery(env, ns, databaseId, mode, statements, requ body = { error: "invalid_d1_runtime_response", message: err instanceof Error ? err.message : "invalid d1-runtime response", + category: "invalid-response", + retryable: false, + causeCode: "binary-decode-failed", }; } } else { @@ -88,6 +91,9 @@ export async function d1RuntimeQuery(env, ns, databaseId, mode, statements, requ body = { error: "invalid_d1_runtime_response", message: `unsupported d1-runtime response content-type ${contentType || "none"}`, + category: "invalid-response", + retryable: false, + causeCode: "unsupported-content-type", }; } const owner = { @@ -248,7 +254,36 @@ function normalizeD1PublicResult(value) { * @returns {Record} */ function d1RuntimeFailureExtra(extra) { - return /** @type {Record} */ (sanitizeJsonErrorDetails(extra) || {}); + const sanitized = /** @type {Record} */ (sanitizeJsonErrorDetails(extra) || {}); + for (const key of ["upstreamCode", "upstreamCategory", "upstreamRetryable", "upstreamStatus", "detail"]) { + delete sanitized[key]; + } + return sanitized; +} + +/** @param {unknown} value @param {string} fallback */ +function d1RuntimeLogToken(value, fallback) { + return typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(value) + ? value + : fallback; +} + +/** + * Stable diagnostics for Control logs. Raw backend messages stay in the D1 + * runtime's own request log and are omitted from redacted 5xx responses. + * + * @param {{ body?: unknown, status?: unknown } | null | undefined} result + */ +export function d1RuntimeFailureLogFields(result) { + const body = /** @type {Record} */ (Object(result?.body || {})); + const causeCode = d1RuntimeLogToken(body.causeCode, ""); + return { + ...(typeof result?.status === "number" ? { upstream_status: result.status } : {}), + upstream_code: d1RuntimeLogToken(body.error, "d1-runtime-error"), + upstream_category: d1RuntimeLogToken(body.category, "internal"), + upstream_retryable: body.retryable === true, + ...(causeCode ? { upstream_cause_code: causeCode } : {}), + }; } /** @@ -257,15 +292,31 @@ function d1RuntimeFailureExtra(extra) { * @param {string} databaseId * @param {{ body?: unknown, status?: unknown } | null | undefined} result * @param {Record} [extra] + * @param {{ publicStatus?: number }} [options] */ -export function d1RuntimeFailure(error, ns, databaseId, result, extra = {}) { +export function d1RuntimeFailure(error, ns, databaseId, result, extra = {}, options = {}) { const body = /** @type {Record} */ (Object(result?.body || {})); - const upstreamCode = typeof body.error === "string" ? body.error : "d1-runtime-error"; - return { + const status = typeof result?.status === "number" ? result.status : 500; + const publicStatus = typeof options.publicStatus === "number" ? options.publicStatus : status; + const publicFields = { ...d1RuntimeFailureExtra(extra), error, namespace: ns, databaseId, + }; + if (publicStatus < 400 || publicStatus >= 500) { + return { + ...publicFields, + message: "Internal error", + upstreamCode: d1RuntimeLogToken(body.error, "d1-runtime-error"), + upstreamCategory: d1RuntimeLogToken(body.category, "internal"), + upstreamRetryable: body.retryable === true, + ...(typeof result?.status === "number" ? { upstreamStatus: result.status } : {}), + }; + } + const upstreamCode = typeof body.error === "string" ? body.error : "d1-runtime-error"; + return { + ...publicFields, message: typeof body.message === "string" ? body.message : typeof body.error === "string" diff --git a/control/handlers/delete-plan.js b/control/handlers/delete-plan.js index 183aca5..5764ef0 100644 --- a/control/handlers/delete-plan.js +++ b/control/handlers/delete-plan.js @@ -1,6 +1,5 @@ import { referrersKey, - doStorageIdKey, workflowDefsKey, } from "control-lib"; import { @@ -12,7 +11,14 @@ import { stageWorkerHidden, stageWorkerVersionIndexRemove, } from "control-lifecycle-indexes"; -import { NAMESPACES_KEY, bundleKey, nsHostsKey, patternsKey, routesKey } from "shared-version"; +import { + NAMESPACES_KEY, + bundleKey, + doStorageIdKey, + nsHostsKey, + patternsKey, + routesKey, +} from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; /** diff --git a/control/handlers/delete.js b/control/handlers/delete.js index e0ff591..2313343 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -15,10 +15,8 @@ import { ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL, PATTERNS_CHANNEL, } from "control-shared"; import { - workerVersionsKey, referrersKey, + referrersKey, doObjectRegistryKey, - doOwnerScopeScanPatternForStorage, - doStorageIdKey, workflowDefsKey, extractD1Refs, extractOutgoingRefs, formatReferrerBlocker, @@ -29,14 +27,18 @@ import { WHOLE_DELETE_LOCK_KIND, bundleKey, deleteLockKey, + doOwnerScopeScanPatternForStorage, + doStorageIdKey, hostsKey, patternsKey, routesKey, + workerVersionsKey, } from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; import { decodeBulk, WatchError } from "shared-redis"; import { queueConsumerScanPrefix } from "shared-queue-keys"; import { decodePatternProjection } from "shared-route-projection"; +import { discardResponseBody } from "shared-respond"; import { buildWorkerDeleteCleanup, stageWorkerDelete } from "control-handlers-delete-plan"; const MAX_DELETE_ATTEMPTS = 5; @@ -133,14 +135,14 @@ export async function handle({ request, env: _env, url, ns, name, principal, req const log = requireControlLog(); const dryRun = url.searchParams.get("dry_run") === "1"; + // Flags live in the query. Cancel an unexpected body without buffering it or + // allowing a non-settling cancellation to block either delete path. + await discardResponseBody(request); + if (dryRun) { return handleDryRun({ redis, ns, name, principal, requestId, log }); } - // POST body is ignored (flags go in the query), but drain it so the - // client's pipe closes cleanly. - await request.text().catch(() => {}); - const lockToken = await acquireDeleteLock(redis, ns, name, WHOLE_DELETE_LOCK_KIND); if (!lockToken) { log("warn", "worker_delete_rejected", { diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index a9d103b..21d757c 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -14,11 +14,8 @@ import { d1DatabaseNameKey, extractD1Refs, extractOutgoingRefs, - deleteLockKey, - doStorageIdKey, parseBundleMeta, workflowDefsKey, - platformDomainFromEnv, } from "control-lib"; import { stageD1ReferrerAdds, @@ -40,11 +37,20 @@ import { parseCronList, parseQueueConsumers, } from "control-topology"; -import { formatVersion, parseVersion, bundleKey, nextVersionKey, routesKey } from "shared-version"; +import { + bundleKey, + deleteLockKey, + doStorageIdKey, + formatVersion, + nextVersionKey, + parseVersion, + routesKey, +} from "shared-version"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; import { isReservedNs, isValidRouteNs, + platformDomainFromEnv, ROUTES_ALLOWED_RESERVED_NS, WORKFLOW_KEY_RE, } from "shared-ns-pattern"; diff --git a/control/handlers/hosts.js b/control/handlers/hosts.js index a96f6d5..e4f3360 100644 --- a/control/handlers/hosts.js +++ b/control/handlers/hosts.js @@ -8,7 +8,7 @@ import { requireControlRedis, } from "control-shared"; import { reconcileHosts, RoutingError } from "control-routing"; -import { platformDomainFromEnv } from "control-lib"; +import { platformDomainFromEnv } from "shared-ns-pattern"; import { hostsKey } from "shared-version"; /** diff --git a/control/handlers/logs-tail.js b/control/handlers/logs-tail.js index fb95ae2..366633c 100644 --- a/control/handlers/logs-tail.js +++ b/control/handlers/logs-tail.js @@ -6,9 +6,8 @@ import { RedisSession, redisDbFromEnv } from "shared-redis"; import { envValueOr } from "shared-env"; import { PLATFORM_TIER_RESERVED_NS } from "shared-auth-roles"; -import { isReservedNs } from "shared-ns-pattern"; +import { isReservedNs, WORKER_NAME_RE } from "shared-ns-pattern"; import { - WORKER_NAME_RE, compareStreamIds, isValidResumeId, isValidWorkerName, diff --git a/control/handlers/promote.js b/control/handlers/promote.js index 050a04a..ca3bcbd 100644 --- a/control/handlers/promote.js +++ b/control/handlers/promote.js @@ -9,7 +9,7 @@ import { } from "control-shared"; import { parseVersion } from "shared-version"; import { promoteWithRoutes, RoutingError } from "control-routing"; -import { platformDomainFromEnv } from "control-lib"; +import { platformDomainFromEnv } from "shared-ns-pattern"; /** * @param {{ request: Request, env: Record, ns: string, name: string, requestId: string }} args diff --git a/control/handlers/versions.js b/control/handlers/versions.js index 0c61ea2..d684647 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -9,7 +9,7 @@ import { runOptimistic, } from "control-shared"; import { - workerVersionsKey, referrersKey, + referrersKey, extractD1Refs, extractOutgoingRefs, formatReferrerBlocker, bundleAssetPrefix, @@ -29,6 +29,7 @@ import { deleteLockKey, parseVersion, routesKey, + workerVersionsKey, } from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index a9434ed..36a351a 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -12,12 +12,7 @@ import { codedErrorResponse, secretEnvelopeErrorResponse, } from "control-shared"; -import { - deleteLockKey, - routesKey, - workflowDefsKey, - workerVersionsKey, -} from "control-lib"; +import { workflowDefsKey } from "control-lib"; import { invalidSecretMutationKeyResponse, readEncryptedSecretPutValue, @@ -33,6 +28,7 @@ import { } from "control-env-budget"; import { SecretEnvelopeError } from "shared-secret-envelope"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; +import { deleteLockKey, routesKey, workerVersionsKey } from "shared-version"; const MAX_SECRET_ATTEMPTS = 5; diff --git a/control/handlers/workers.js b/control/handlers/workers.js index 8ed485b..3513a38 100644 --- a/control/handlers/workers.js +++ b/control/handlers/workers.js @@ -1,6 +1,7 @@ import { jsonResponse, jsonError, requireControlLog, requireControlRedis } from "control-shared"; -import { routesKey, workflowDefsKey, workersIndexKey, workerVersionsKey } from "control-lib"; +import { workflowDefsKey, workersIndexKey } from "control-lib"; import { workerSecretsKey } from "shared-secret-keys"; +import { routesKey, workerVersionsKey } from "shared-version"; /** @param {{ method: string, nsName: string, requestId: string }} args */ export async function handle({ method, nsName, requestId }) { diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index dc5e034..45951bb 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -1,6 +1,4 @@ import { - WORKER_NAME_RE, - WORKFLOW_NAME_RE, isValidWorkerName, isValidWorkflowName, parseBundleMeta, @@ -20,6 +18,8 @@ import { import { bundleKey, routesKey } from "shared-version"; import { BINDING_NAME_RE, + WORKER_NAME_RE, + WORKFLOW_NAME_RE, WORKFLOW_KEY_RE, isValidJsClassDeclarationName, } from "shared-ns-pattern"; diff --git a/control/index.js b/control/index.js index 73b8890..8330883 100644 --- a/control/index.js +++ b/control/index.js @@ -7,14 +7,13 @@ import { NS_RE, parseControlRoute, - configuredHostname, configuredPublicUrl, platformVersionFromPackageJson, projectAccessPrincipal, isAdminAcceptableNs, isValidWorkerName, - WORKER_NAME_RE, } from "control-lib"; +import { configuredHostname, WORKER_NAME_RE } from "shared-ns-pattern"; import { ensureInit, authorizeControlRequest, diff --git a/control/lib.js b/control/lib.js index 88e63db..fcdcc8f 100644 --- a/control/lib.js +++ b/control/lib.js @@ -12,31 +12,9 @@ import { QUEUE_NAME_RE, WDL_RESERVED_BINDING_RE, KV_ID_RE, - configuredHostname, - platformDomainFromEnv, - validateModulePath, } from "shared-ns-pattern"; import { PLATFORM_TIER_RESERVED_NS, ROLES } from "shared-auth-roles"; import { errorMessage } from "shared-errors"; -import { - deleteLockKey, - doOwnerScopeScanPatternForStorage, - doStorageIdKey, - routesKey, - workerVersionsKey, -} from "shared-version"; -export { - deleteLockKey, - doOwnerScopeScanPatternForStorage, - doStorageIdKey, - routesKey, - workerVersionsKey, -}; -export { validateModulePath }; -export { WORKER_NAME_RE }; -export { WORKFLOW_NAME_RE }; -export { configuredHostname, platformDomainFromEnv }; - export const NS_RE = new RegExp(`^${NS_PATTERN}$`); export const MAX_QUEUE_DELAY_SECONDS = 86_400; const WORKERD_DEPENDENCY_VERSION_RE = /^1\.(\d{4})(\d{2})(\d{2})\.(\d+)$/; diff --git a/control/lifecycle-indexes.js b/control/lifecycle-indexes.js index 1610493..c055781 100644 --- a/control/lifecycle-indexes.js +++ b/control/lifecycle-indexes.js @@ -2,10 +2,10 @@ import { d1DatabaseReferrersKey, encodeReferrerMember, referrersKey, - workerVersionsKey, workersIndexKey, } from "control-lib"; import { QUEUE_CONSUMER_INDEX_KEY, queueConsumerKey } from "shared-queue-keys"; +import { workerVersionsKey } from "shared-version"; export const CRON_WORKER_INDEX_KEY = "cron:index:workers"; diff --git a/control/routing.js b/control/routing.js index 383af48..8f4240e 100644 --- a/control/routing.js +++ b/control/routing.js @@ -5,7 +5,6 @@ import { runOptimistic } from "control-shared"; import { d1DatabaseKey, - deleteLockKey, extractD1Refs, extractOutgoingRefs, parseBundleMeta, } from "control-lib"; @@ -27,6 +26,7 @@ import { PATTERNS_CHANNEL, ROUTES_CHANNEL, bundleKey, + deleteLockKey, formatVersion, hostDeclarationsKey, hostsKey, diff --git a/docs/modules/d1.md b/docs/modules/d1.md index 7a441c0..28859ae 100644 --- a/docs/modules/d1.md +++ b/docs/modules/d1.md @@ -55,7 +55,11 @@ freezing of a binding to a physical `databaseId`. `snake_case` vocabulary unless the D1 protocol itself changes. - Control D1 lifecycle APIs remain admin APIs and use the control `{ error, message, ...details }` contract. d1-runtime internal query/classified errors are a separate - D1 protocol surface. + D1 protocol surface. When the outward Control status is 5xx, Control replaces the + backend message with `Internal error`, omits raw detail, and preserves only bounded + `upstreamCode`, `upstreamCategory`, `upstreamRetryable`, and `upstreamStatus` + classifier fields. Outward 4xx SQL and migration failures retain their existing D1 + diagnostic details. ## Redis / Storage Contracts @@ -182,6 +186,12 @@ accounting. D1-compatible query responses may still carry `meta.size_after`; the the current local owner and is cleared when that owner is lost or released. Forwarded router responses must not be scraped into this gauge. +Control's D1 runtime failure events for database initialization, query execution, and +migration operations include request and database identity plus bounded +`upstream_status`, `upstream_code`, `upstream_category`, `upstream_retryable`, and, +when available, `upstream_cause_code` fields. Control does not copy raw backend messages +or response detail into these events. + The supervisor emits `drain_timeout_near_owner_ttl` when configured drain timeout is at least half the owner TTL. The warning carries `drain_timeout_ms`, `owner_ttl_ms`, and `renew_interval_ms` so operators can see whether shutdown drain may run too close to diff --git a/docs/modules/d1.zh.md b/docs/modules/d1.zh.md index f46fc04..a6bc4e4 100644 --- a/docs/modules/d1.zh.md +++ b/docs/modules/d1.zh.md @@ -29,7 +29,7 @@ workerd 给 WDL 提供了托管 SQLite-backed service 所需的 isolate 和 loca - Multi-statement `exec()` request 会在一个 actor-local SQLite transaction 中执行。如果后续 statement 失败，同一 request 中此前已执行的 statement 会回滚。 - D1 query/facade 失败使用 D1-specific payload 和 tenant `D1_ERROR` object，而不是 generic platform/admin HTTP envelope。D1 payload 会按需携带 `success:false`、`error`、`message`、`category`、`retryable`、`statementIndex` 和 `causeCode` 等字段。 - D1 error code 使用 D1 compatibility vocabulary，包括 `limit-exceeded`、`sql-error` 这类 `hyphen-case` code。除非 D1 protocol 本身变化，否则不要把它们改写成平台 `snake_case` vocabulary。 -- Control D1 lifecycle API 仍是 admin API，使用 control `{ error, message, ...details }` 合同。d1-runtime internal query/classified error 是独立的 D1 protocol surface。 +- Control D1 lifecycle API 仍是 admin API，使用 control `{ error, message, ...details }` 合同。d1-runtime internal query/classified error 是独立的 D1 protocol surface。Control 对外状态为 5xx 时会把 backend message 替换为 `Internal error`、移除 raw detail，只保留有界的 `upstreamCode`、`upstreamCategory`、`upstreamRetryable` 和 `upstreamStatus` classifier fields；对外 4xx SQL 与 migration failure 继续保留既有 D1 diagnostic detail。 ## Redis / Storage 合同 @@ -97,6 +97,8 @@ Key families： d1-runtime 记录 owner 变化、routing、query failure、drain/renew 和 test-hook 操作。Storage size metrics 是 owner-local observed state，不是权威 quota accounting。D1-compatible query response 仍可携带 `meta.size_after`；`wdl_d1_storage_size_bytes` 只来自当前 local owner 成功记录的 sample，并在 owner 丢失或释放时清除。不要把 forwarded router response scrape 进这个 gauge。 +Control 的 database initialization、query execution 和 migration operation runtime failure event 会包含 request/database identity，以及有界的 `upstream_status`、`upstream_code`、`upstream_category`、`upstream_retryable` 和可用时的 `upstream_cause_code` 字段；Control 不会把原始 backend message 或 response detail 复制进这些 event。 + 当配置的 drain timeout 大于等于 owner TTL 的一半时，supervisor 会输出 `drain_timeout_near_owner_ttl`。该 warning 携带 `drain_timeout_ms`、`owner_ttl_ms` 和 `renew_interval_ms`，方便 operator 判断 shutdown drain 是否过于接近 lease expiry。 ## 部署 / Rollout 注意事项 diff --git a/runtime/dispatch.js b/runtime/dispatch.js index 97217b4..f873b37 100644 --- a/runtime/dispatch.js +++ b/runtime/dispatch.js @@ -6,7 +6,7 @@ import { internalErrorResponse, jsonError, jsonResponse } from "shared-respond"; import { BodyTooLargeError, readBoundedText } from "shared-bounded-body"; import { withInternalAuthEntries } from "shared-internal-auth"; -import { errorMessage as sharedErrorMessage } from "shared-errors"; +import { errorMessage } from "shared-errors"; import { decodeQueuedDispatchMessages, normalizeWorkflowNotifyBody, @@ -27,7 +27,8 @@ import { } from "runtime-dispatch-workflow-json"; export { _resetWorkflowReplayCacheForTest } from "runtime-dispatch-workflow-replay-cache"; import { - createStepFacade, + createStepController, + isWorkflowSuspensionSignal, isWorkflowSuspended, workflowError, } from "runtime-dispatch-workflow-step"; @@ -72,11 +73,6 @@ function workflowBackendForStep(env) { }; } -/** @param {unknown} err */ -function errorMessage(err) { - return sharedErrorMessage(err || ""); -} - /** @param {unknown} result */ function queueDispatchResult(result) { const record = result && typeof result === "object" @@ -199,17 +195,16 @@ export async function handleWorkflowRunDispatch({ run, stub, scope, env, ctx, id let step = null; try { const entry = stub.getEntrypoint(run.className); - step = createStepFacade(run, workflowBackendForStep(env), scope.requestId); + step = createStepController(run, workflowBackendForStep(env), scope.requestId); if (typeof entry.run !== "function") { throw workflowStepError("workflow_invalid_step", `workflow class ${run.className} does not expose run()`); } - const output = await entry.run(run.event, step); + const output = await entry.run(run.event, step.facade); if (step.hasInFlightSteps()) { step.closeForRunReturn(); throw workflowStepError("workflow_invalid_step", "workflow run returned while workflow steps were still in flight"); } - const terminalStepFailure = step.terminalStepFailure(); - if (terminalStepFailure) throw terminalStepFailure; + if (step.hasTerminalStepFailure()) throw step.terminalStepFailure(); if (step.isSuspended()) { throw workflowStepError("workflow_invalid_step", "workflow run returned after a step suspension was registered"); } @@ -236,10 +231,12 @@ export async function handleWorkflowRunDispatch({ run, stub, scope, env, ctx, id return scope.respond(response); } catch (err) { let caught = err; - if (step?.hasInFlightSteps() && !isWorkflowSuspended(caught)) { + const caughtSuspended = Boolean( + step?.isSuspended() && isWorkflowSuspensionSignal(caught) + ); + if (step?.hasInFlightSteps() && !caughtSuspended) { step.closeForRunReturn(); } - const caughtSuspended = isWorkflowSuspended(caught); if (step?.hasInFlightSteps() && caughtSuspended) { const settled = await step.waitForInFlightSteps(); const terminalFailure = settled.find((result) => ( @@ -247,11 +244,16 @@ export async function handleWorkflowRunDispatch({ run, stub, scope, env, ctx, id )); if (terminalFailure?.status === "rejected") caught = terminalFailure.reason; } - if (isWorkflowSuspended(caught)) { - const terminalStepFailure = step?.terminalStepFailure(); - if (terminalStepFailure) caught = terminalStepFailure; + let terminalStepError = null; + if (step?.hasTerminalStepFailure()) { + caught = step.terminalStepFailure(); + terminalStepError = step.terminalStepError(); } - if (step?.isSuspended() && isWorkflowSuspended(caught)) { + if ( + !terminalStepError && + step?.isSuspended() && + isWorkflowSuspensionSignal(caught) + ) { const durationMs = Date.now() - startedAt; emitRuntimeTailEvent({ env, ctx, identity, @@ -269,9 +271,9 @@ export async function handleWorkflowRunDispatch({ run, stub, scope, env, ctx, id duration_ms: durationMs, })); } - scope.markError(caught); const durationMs = Date.now() - startedAt; - const error = workflowError(caught); + const error = terminalStepError ?? workflowError(caught); + scope.markError(caught); emitRuntimeTailEvent({ env, ctx, identity, event: "worker_workflow", @@ -357,13 +359,17 @@ export async function handleScheduledDispatch({ request, stub, scope, env, ctx, ? /** @type {Record} */ (scheduledResult) : {}; if (scheduledRecord.outcome === "exception") { + const message = typeof scheduledRecord.error === "string" + ? scheduledRecord.error + : "scheduled handler threw"; + scope.markError(message); const durationMs = tail.finish({ outcome: "error", - error: typeof scheduledRecord.error === "string" ? scheduledRecord.error : "scheduled handler threw", + error: message, }); return scope.respond(jsonResponse(200, { outcome: "error", - error: typeof scheduledRecord.error === "string" ? scheduledRecord.error : "scheduled handler threw", + error: message, duration_ms: durationMs, })); } @@ -376,13 +382,14 @@ export async function handleScheduledDispatch({ request, stub, scope, env, ctx, })); } catch (err) { scope.markError(err); + const message = errorMessage(err); const durationMs = tail.finish({ outcome: "error", - error: errorMessage(err), + error: message, }); return scope.respond(jsonResponse(200, { outcome: "error", - error: errorMessage(err), + error: message, duration_ms: durationMs, })); } @@ -424,23 +431,29 @@ export async function handleQueuedDispatch({ request, stub, scope, env, ctx, ide throw new Error("worker does not expose queue()"); } const resp = await entry.queue(queued.queueName, decoded); + const result = queueDispatchResult(resp); + const handlerFailed = result.outcome === "exception"; + const message = "queue handler threw"; + if (handlerFailed) scope.markError(message); const durationMs = tail.finish({ - outcome: "ok", + outcome: handlerFailed ? "error" : "ok", + ...(handlerFailed ? { error: message } : {}), }); return scope.respond(jsonResponse(200, { outcome: "ok", - result: queueDispatchResult(resp), + result, duration_ms: durationMs, })); } catch (err) { scope.markError(err); + const message = errorMessage(err); const durationMs = tail.finish({ outcome: "error", - error: errorMessage(err), + error: message, }); return scope.respond(jsonResponse(200, { outcome: "error", - error: errorMessage(err), + error: message, duration_ms: durationMs, })); } diff --git a/runtime/dispatch/workflow-step.js b/runtime/dispatch/workflow-step.js index d549517..cd6561f 100644 --- a/runtime/dispatch/workflow-step.js +++ b/runtime/dispatch/workflow-step.js @@ -30,25 +30,42 @@ import { * @typedef {import("runtime-dispatch-workflow-replay-cache").WorkflowReplayCache} WorkflowReplayCache */ -/** @param {unknown} err */ -export function workflowError(err) { - const error = /** @type {{ name?: unknown, message?: unknown } | null} */ ( - err && typeof err === "object" ? err : null - ); - return { - name: typeof error?.name === "string" && error.name ? error.name : "Error", - message: String( - typeof error?.message === "string" && error.message ? error.message : err || "Workflow run failed" - ), - }; +/** @param {Record} record @param {string} field */ +function readFailureField(record, field) { + try { + return { readable: true, value: record[field] }; + } catch { + return { readable: false, value: undefined }; + } } /** @param {unknown} err */ -function isNonRetryableError(err) { - const error = /** @type {{ name?: unknown } | null} */ ( - err && typeof err === "object" ? err : null - ); - return error?.name === "NonRetryableError" || error?.name === "workflow_invalid_step"; +export function workflowError(err) { + const record = err !== null && (typeof err === "object" || typeof err === "function") + ? /** @type {Record} */ (err) + : null; + const nameField = record + ? readFailureField(record, "name") + : { readable: true, value: undefined }; + const messageField = record + ? readFailureField(record, "message") + : { readable: true, value: undefined }; + const name = typeof nameField.value === "string" && nameField.value + ? nameField.value + : "Error"; + let message = "Unknown error"; + if (messageField.readable && typeof messageField.value === "string") { + message = messageField.value; + } else if (!record && err == null) { + message = "Workflow run failed"; + } else if (!record) { + try { + message = String(err); + } catch { + // Keep the stable fallback. + } + } + return { name, message }; } /** @@ -87,11 +104,19 @@ class WorkflowSuspended extends Error { /** @param {unknown} err */ export function isWorkflowSuspended(err) { - return err instanceof WorkflowSuspended || ( - err !== null && - typeof err === "object" && - /** @type {{ name?: unknown }} */ (err).name === "WorkflowSuspended" - ); + try { + return err instanceof WorkflowSuspended; + } catch { + return false; + } +} + +/** @param {unknown} err */ +export function isWorkflowSuspensionSignal(err) { + if (isWorkflowSuspended(err)) return true; + if (err === null || (typeof err !== "object" && typeof err !== "function")) return false; + const name = readFailureField(/** @type {Record} */ (err), "name"); + return name.readable && name.value === "WorkflowSuspended"; } /** @param {unknown} value */ @@ -191,7 +216,7 @@ async function fetchReplayStepPage(backend, run, cache, ordinal, requestId) { * @param {WorkflowBackend | null | undefined} backend * @param {string | null} [requestId] */ -export function createStepFacade(run, backend, requestId = null) { +export function createStepController(run, backend, requestId = null) { let ordinal = 0; let startedSteps = 0; let suspendingStepInFlight = false; @@ -200,6 +225,9 @@ export function createStepFacade(run, backend, requestId = null) { let stepCallbackDepth = 0; /** @type {unknown} */ let terminalStepFailure = null; + /** @type {{ name: string, message: string } | null} */ + let terminalStepError = null; + let hasTerminalStepFailure = false; const activeStepPromises = new Set(); const nameCounts = new Map(); const dependencyFrontier = new Set(); @@ -209,6 +237,14 @@ export function createStepFacade(run, backend, requestId = null) { /** @type {Promise | null} */ let replayFetchPromise = null; + /** @param {unknown} reason @param {{ name: string, message: string } | null} [error] */ + const rememberTerminalStepFailure = (reason, error = null) => { + if (hasTerminalStepFailure) return; + terminalStepFailure = reason; + terminalStepError = error ?? workflowError(reason); + hasTerminalStepFailure = true; + }; + /** @param {number[]} left @param {number[]} right */ const sameDependencies = (left, right) => ( left.length === right.length && left.every((value, index) => value === right[index]) @@ -249,7 +285,7 @@ export function createStepFacade(run, backend, requestId = null) { if (stepCallbackDepth > 0) { throw workflowStepError("workflow_invalid_step", "workflow steps cannot start while another step.do callback is in flight"); } - if (terminalStepFailure) throw terminalStepFailure; + if (hasTerminalStepFailure) throw terminalStepFailure; if (options.suspending) { if (activeStepCount > 0) { throw workflowStepError("workflow_invalid_step", "workflow suspending steps must execute when no other step is in flight"); @@ -362,7 +398,7 @@ export function createStepFacade(run, backend, requestId = null) { try { return await promise; } catch (reason) { - if (!isWorkflowSuspended(reason)) terminalStepFailure ??= reason; + if (!isWorkflowSuspended(reason)) rememberTerminalStepFailure(reason); throw reason; } finally { activeStepRecords.delete(record); @@ -481,7 +517,7 @@ export function createStepFacade(run, backend, requestId = null) { return null; }; - return { + const facade = { /** * @param {string} name * @param {unknown | (() => unknown | Promise)} configOrCallback @@ -504,7 +540,7 @@ export function createStepFacade(run, backend, requestId = null) { identity = nextStepIdentity(name, config); assertCanStartStep({ stepDo: true, stepDependencies: identity.dependencies }); } catch (err) { - if (!isWorkflowSuspended(err)) terminalStepFailure ??= err; + if (!isWorkflowSuspended(err)) rememberTerminalStepFailure(err); const rejected = Promise.reject(err); rejected.catch(() => {}); return rejected; @@ -553,11 +589,12 @@ export function createStepFacade(run, backend, requestId = null) { assertRunStillOpen(); } catch (err) { if (runReturned) throw err; + const error = workflowError(err); const committed = await workflowBackendCall(backend, "commit-step-error", { ...identity, attempt, - error: workflowError(err), - nonRetryable: isNonRetryableError(err), + error, + nonRetryable: error.name === "NonRetryableError" || error.name === "workflow_invalid_step", }, requestId); if (committed?.state === "waiting") { cacheStep(identity, { status: "waiting", attempt, dueAtMs: committed.dueAtMs ?? null }); @@ -565,8 +602,9 @@ export function createStepFacade(run, backend, requestId = null) { throw new WorkflowSuspended(); } if (committed?.state === "failed") { - cacheStep(identity, { status: "failed", attempt, error: workflowError(err) }); + cacheStep(identity, { status: "failed", attempt, error }); } + rememberTerminalStepFailure(err, error); throw err; } await workflowBackendCall(backend, "commit-step-success", { @@ -654,12 +692,22 @@ export function createStepFacade(run, backend, requestId = null) { }); })()); }, + }; + + return { + facade, isSuspended() { return suspended; }, terminalStepFailure() { return terminalStepFailure; }, + terminalStepError() { + return terminalStepError; + }, + hasTerminalStepFailure() { + return hasTerminalStepFailure; + }, hasInFlightSteps() { return activeUnsettledStepCount() > 0; }, diff --git a/runtime/runtime.js b/runtime/runtime.js index 29147cf..f316948 100644 --- a/runtime/runtime.js +++ b/runtime/runtime.js @@ -149,7 +149,7 @@ export async function abortLoadedWorker({ env, workerId }) { await stub.getEntrypoint("__WdlAbort__").abort("wdl-evict"); return { aborted: false, reason: "no_internal_error" }; } catch (err) { - const msg = errorMessage(err || ""); + const msg = errorMessage(err); if (msg.startsWith(ABORT_SUCCESS_PREFIX)) { forgetLoadedWorker(workerId); return { aborted: true }; @@ -202,7 +202,7 @@ export async function evictSiblings({ env, workerId, log: logger = null }) { worker_id: old, triggered_by: workerId, reason: result.reason, - ...(result.error ? formatError(result.error) : {}), + ...(result.reason === "unexpected" ? formatError(result.error) : {}), }); } } diff --git a/runtime/workflows-client.js b/runtime/workflows-client.js index 8315ea1..6f26061 100644 --- a/runtime/workflows-client.js +++ b/runtime/workflows-client.js @@ -3,6 +3,33 @@ import { requestIdFromOptions } from "./_wdl-request-id.js"; const WORKFLOWS_BASE_URL = "http://workflows/internal/workflows"; const MAX_CREATE_BATCH = 100; const WORKFLOW_INSTANCE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +const intrinsicJsonStringify = JSON.stringify; +const intrinsicObjectAssign = Object.assign; +const intrinsicObjectCreate = Object.create; +const intrinsicReflectApply = Reflect.apply; + +/** @param {unknown} value */ +function stringifyJson(value) { + return intrinsicReflectApply(intrinsicJsonStringify, undefined, [value]); +} + +/** + * @param {Record} fields + * @param {WorkflowMetadata} metadata + * @param {string | null} requestId + */ +function workflowRequestBody(fields, metadata, requestId) { + const body = /** @type {Record} */ (intrinsicObjectCreate(null)); + intrinsicObjectAssign(body, fields); + body.ns = metadata.ns; + body.worker = metadata.worker; + body.frozenVersion = metadata.version; + body.workflowName = metadata.name; + body.workflowKey = metadata.workflowKey; + body.className = metadata.className; + body.requestId = requestId; + return body; +} /** * @typedef {{ fetch(url: string, init: RequestInit): Promise }} WorkflowBackend @@ -144,7 +171,11 @@ export class Workflow { retention: opts.retention ?? null, callback: opts.callback ?? null, }); - return this.#instance(body.id === undefined || body.id === null ? id : validateInstanceId(body.id)); + const responseId = validateInstanceId(body.id); + if (responseId !== id) { + throw new Error(`Workflow create response id mismatch: expected ${id}, got ${responseId}`); + } + return this.#instance(responseId); } /** @param {unknown} options */ @@ -168,15 +199,19 @@ export class Workflow { if (!Array.isArray(body.instances)) { throw new Error("Workflow createBatch response must include instances"); } - const instances = body.instances; const expectedIds = new Set(entries.map((entry) => entry.instanceId)); - return instances.map((entry, index) => { + const returnedIds = new Set(); + return body.instances.map((entry, index) => { const instance = ensureObject(entry, "Workflow createBatch response entry"); const id = validateCreateBatchResponseId(instance.id, index); if (!expectedIds.has(id)) { const expected = [...expectedIds].join(", "); throw new Error(`Workflow createBatch response id mismatch: unexpected ${id}; expected one of ${expected}`); } + if (returnedIds.has(id)) { + throw new Error(`Workflow createBatch response contains duplicate id ${id}`); + } + returnedIds.add(id); return this.#instance(id); }); } @@ -185,12 +220,16 @@ export class Workflow { async get(id) { validateInstanceId(id); const body = await this.#call("get", { instanceId: id }); - return this.#instance(body.id === undefined || body.id === null ? id : validateInstanceId(body.id)); + const responseId = validateInstanceId(body.id); + if (responseId !== id) { + throw new Error(`Workflow get response id mismatch: expected ${id}, got ${responseId}`); + } + return this.#instance(responseId); } /** @param {string} id */ #instance(id) { - return new WorkflowInstance(id, this.#call.bind(this)); + return new WorkflowInstance(id, (endpoint, fields) => this.#call(endpoint, fields)); } /** @param {string} endpoint @param {Record} [fields] */ @@ -200,23 +239,14 @@ export class Workflow { } const metadata = this.#metadata; const requestId = requestIdFromOptions(this.#options); - const body = { - ...fields, - ns: metadata.ns, - worker: metadata.worker, - frozenVersion: metadata.version, - workflowName: metadata.name, - workflowKey: metadata.workflowKey, - className: metadata.className, - requestId, - }; + const body = workflowRequestBody(fields, metadata, requestId); const response = await this.#backend.fetch(`${WORKFLOWS_BASE_URL}/${endpoint}`, { method: "POST", headers: { "content-type": "application/json", ...(requestId ? { "x-request-id": requestId } : {}), }, - body: JSON.stringify(body), + body: stringifyJson(body), }); return await readWorkflowResponse(response); } diff --git a/rust/workflows/src/api/do_alarms/model.rs b/rust/workflows/src/api/do_alarms/model.rs index df180ce..09f0bfa 100644 --- a/rust/workflows/src/api/do_alarms/model.rs +++ b/rust/workflows/src/api/do_alarms/model.rs @@ -275,20 +275,25 @@ pub(super) fn job_from_state( state: HashMap, ) -> WorkflowResult { parse_positive_i64_field(&state, "dueAtMs")?; + let ns = protocol_string_field(&state, "ns", is_valid_runtime_load_ns)?; + let worker = protocol_string_field(&state, "worker", is_valid_worker_name)?; + let version = protocol_string_field(&state, "scheduledVersion", |value| { + parse_version_tag(value).is_ok() + })?; let do_storage_id = protocol_string_field(&state, "doStorageId", is_storage_id)?; let class_name = protocol_string_field(&state, "className", is_class_name)?; let object_name = opaque_string_field(&state, "objectName")?; if alarm_host_id_len(do_storage_id, class_name, object_name) > MAX_PROTOCOL_STRING_BYTES { return Err(invalid_job_field("hostId")); } + if job_id != do_alarm_job_id(ns, worker, do_storage_id, class_name, object_name) { + return Err(invalid_job_field("jobId")); + } Ok(DoAlarmJob { job_id, - ns: protocol_string_field(&state, "ns", is_valid_runtime_load_ns)?.to_string(), - worker: protocol_string_field(&state, "worker", is_valid_worker_name)?.to_string(), - version: protocol_string_field(&state, "scheduledVersion", |value| { - parse_version_tag(value).is_ok() - })? - .to_string(), + ns: ns.to_string(), + worker: worker.to_string(), + version: version.to_string(), do_storage_id: do_storage_id.to_string(), class_name: class_name.to_string(), object_name: object_name.to_string(), @@ -356,6 +361,25 @@ mod tests { ]) } + fn job_id_for_state(state: &HashMap) -> String { + do_alarm_job_id( + state.get("ns").map(String::as_str).unwrap_or_default(), + state.get("worker").map(String::as_str).unwrap_or_default(), + state + .get("doStorageId") + .map(String::as_str) + .unwrap_or_default(), + state + .get("className") + .map(String::as_str) + .unwrap_or_default(), + state + .get("objectName") + .map(String::as_str) + .unwrap_or_default(), + ) + } + #[test] fn do_alarm_mutation_ingress_rejects_noncanonical_identity() { for (field, value) in [ @@ -451,8 +475,9 @@ mod tests { state.insert("className".to_string(), set.class_name.clone()); state.insert("objectName".to_string(), set.object_name.clone()); state.insert("rowToken".to_string(), set.token.clone()); + let job_id = job_id_for_state(&state); assert_eq!( - job_from_state("job".to_string(), state).is_ok(), + job_from_state(job_id, state).is_ok(), valid, "persisted:{}", case["name"].as_str().expect("name is a string") @@ -460,6 +485,25 @@ mod tests { } } + #[test] + fn do_alarm_job_state_requires_canonical_job_id() { + let state = valid_job_state(); + let canonical = job_id_for_state(&state); + assert!(job_from_state(canonical, state.clone()).is_ok()); + + let mut other_state = state.clone(); + other_state.insert("objectName".to_string(), "other-object".to_string()); + let other_canonical = job_id_for_state(&other_state); + assert_ne!(other_canonical, job_id_for_state(&state)); + + let err = match job_from_state(other_canonical, state) { + Ok(_) => panic!("mis-keyed DO alarm state must fail closed"), + Err(err) => err, + }; + assert_eq!(err.code, "workflow_invalid_state"); + assert!(err.message.contains("jobId")); + } + #[test] fn do_alarm_json_rejects_unpaired_surrogate_object_names() { let request = r#"{ diff --git a/shared/observability.js b/shared/observability.js index 1993d31..a1fa9eb 100644 --- a/shared/observability.js +++ b/shared/observability.js @@ -29,6 +29,7 @@ const CARDINALITY_WARN_LIMIT = 100; * status: number, * startedAt: number, * error?: unknown, + * hasError?: boolean, * extras?: Record | null, * probeRoutes?: string[], * }} RequestCompleteOptions @@ -200,10 +201,12 @@ export function recordRequestComplete({ status, startedAt, error = null, + hasError = false, extras = null, probeRoutes = DEFAULT_PROBE_ROUTES, }) { const durationMs = Date.now() - startedAt; + const requestHasError = hasError || error !== null; const requestLabels = { service, route }; const statusLabel = String(status); if (metrics) { @@ -211,16 +214,16 @@ export function recordRequestComplete({ metrics.observe("request_duration_ms", requestLabels, durationMs); if (status >= 500) metrics.increment("request_errors", { ...requestLabels, status: statusLabel }); } - if (!probeRoutes.includes(route) || error || status >= 500) { + if (!probeRoutes.includes(route) || requestHasError || status >= 500) { const pruned = pruneExtras(extras); - log(error || status >= 500 ? "error" : "info", "request_complete", { + log(requestHasError || status >= 500 ? "error" : "info", "request_complete", { request_id: requestId, method, route, status, duration_ms: durationMs, ...(pruned || {}), - ...(error ? formatError(error) : {}), + ...(requestHasError ? formatError(error) : {}), }); } } @@ -230,7 +233,7 @@ export function recordRequestComplete({ * @returns {Record} */ export function formatError(err) { - if (!err) return { error_message: "Unknown error" }; + if (err == null) return { error_message: "Unknown error" }; if (isErrorObject(err)) { /** @type {Record} */ const out = { diff --git a/shared/request-scope.js b/shared/request-scope.js index 25ebe56..0cf369a 100644 --- a/shared/request-scope.js +++ b/shared/request-scope.js @@ -35,6 +35,7 @@ export function createHttpRequestScope({ let status = 500; /** @type {unknown} */ let requestError = null; + let hasRequestError = false; return { requestId, @@ -46,11 +47,10 @@ export function createHttpRequestScope({ currentRoute = nextRoute; }, - /** - * @param {unknown} err - */ + /** @param {unknown} err */ markError(err) { requestError = err; + hasRequestError = true; }, /** @@ -77,6 +77,7 @@ export function createHttpRequestScope({ status, startedAt, error: requestError, + hasError: hasRequestError, extras: requestExtras, probeRoutes, }); diff --git a/shared/respond.js b/shared/respond.js index b4aaabb..a4aac49 100644 --- a/shared/respond.js +++ b/shared/respond.js @@ -29,12 +29,12 @@ export function echoResponseWithRequestId(response, requestId) { } /** - * @param {Response} response + * @param {{ body: ReadableStream | null }} message * @returns {Promise} */ -export async function discardResponseBody(response) { +export async function discardResponseBody(message) { try { - void response.body?.cancel().catch(() => {}); + void message.body?.cancel().catch(() => {}); } catch { // Best-effort cleanup only; the caller's status/error path owns behavior. } diff --git a/tests/fixtures/scheduler-projection-contract.json b/tests/fixtures/scheduler-projection-contract.json index ff84136..d72f61e 100644 --- a/tests/fixtures/scheduler-projection-contract.json +++ b/tests/fixtures/scheduler-projection-contract.json @@ -7,9 +7,9 @@ "slotMs": 1778856600000, "slotKey": "cron-slot:1778856600000", "slotExpireAt": 1778857200, - "cronId": "cron-1", + "cronId": "1cb037f29d", "gen": 7, - "reference": "tenant:worker:cron-1:7", + "reference": "tenant:worker:1cb037f29d:7", "meta": { "version": "v3", "seq": 7, diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index 5e937b2..2aaf91c 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -42,7 +42,6 @@ export async function compileControlGraph(opts = {}) { [/from "shared-ns-pattern"/g, `from ${JSON.stringify(SHARED_NS_URL)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], - [/from "shared-version"/g, `from ${JSON.stringify(SHARED_VERSION_URL)}`], ]); const bindingsUrl = freshRepositoryModuleDataUrl("control/bindings.js", [ @@ -62,6 +61,7 @@ export async function compileControlGraph(opts = {}) { const lifecycleIndexesUrl = freshRepositoryModuleDataUrl("control/lifecycle-indexes.js", [ [/from "control-lib"/g, `from ${JSON.stringify(libUrl)}`], [/from "shared-queue-keys"/g, `from ${JSON.stringify(SHARED_QUEUE_KEYS_URL)}`], + [/from "shared-version"/g, `from ${JSON.stringify(SHARED_VERSION_URL)}`], ]); const cronIndexUrl = freshRepositoryModuleDataUrl("control/cron-index.js", [ diff --git a/tests/helpers/load-d1-protocol.js b/tests/helpers/load-d1-protocol.js index c149ac2..331d5ea 100644 --- a/tests/helpers/load-d1-protocol.js +++ b/tests/helpers/load-d1-protocol.js @@ -1,4 +1,8 @@ -import { repositoryFileUrl, repositoryModuleDataUrl } from "./load-shared-module.js"; +import { + repositoryFileUrl, + repositoryModuleDataUrl, + sharedModuleDataUrl, +} from "./load-shared-module.js"; const PARAMS_URL = repositoryFileUrl("shared/d1-params.js"); const DATA_FIELD_URL = repositoryFileUrl("shared/d1-data-field.js"); @@ -6,6 +10,10 @@ const FNV_URL = repositoryFileUrl("shared/fnv1a32.js"); const BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const NS_PATTERN_URL = repositoryFileUrl("shared/ns-pattern.js"); +const D1_TIMEOUT_URL = sharedModuleDataUrl("shared/d1-timeout.js"); +const RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); +const OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); export function d1ProtocolDataUrl() { const queryWireUrl = d1QueryWireDataUrl(); @@ -34,6 +42,17 @@ export function d1TransportDataUrl() { ]); } +export function controlD1RuntimeClientDataUrl() { + return repositoryModuleDataUrl("control/d1-runtime-client.js", [ + [/from "shared-d1-timeout";/g, `from ${JSON.stringify(D1_TIMEOUT_URL)};`], + [/from "shared-d1-transport";/g, `from ${JSON.stringify(d1TransportDataUrl())};`], + [/from "shared-d1-query-wire";/g, `from ${JSON.stringify(d1QueryWireDataUrl())};`], + [/from "shared-respond";/g, `from ${JSON.stringify(RESPOND_URL)};`], + [/from "shared-internal-auth";/g, `from ${JSON.stringify(INTERNAL_AUTH_URL)};`], + [/from "shared-owner-endpoint";/g, `from ${JSON.stringify(OWNER_ENDPOINT_URL)};`], + ]); +} + export async function loadD1Protocol() { return await import(d1ProtocolDataUrl()); } @@ -41,3 +60,7 @@ export async function loadD1Protocol() { export async function loadD1QueryWire() { return await import(d1QueryWireDataUrl()); } + +export async function loadControlD1RuntimeClient() { + return await import(controlD1RuntimeClientDataUrl()); +} diff --git a/tests/unit/control-cron-index.test.js b/tests/unit/control-cron-index.test.js index 40d763d..1e403e4 100644 --- a/tests/unit/control-cron-index.test.js +++ b/tests/unit/control-cron-index.test.js @@ -7,7 +7,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModule } from "../helpers/load-shared-module.js"; +import { importRepositoryModule, readRepositoryJson } from "../helpers/load-shared-module.js"; + +const schedulerProjectionContract = /** @type {{ cron: { cronId: string, entry: { cron: string, timezone: string } } }} */ ( + readRepositoryJson("tests/fixtures/scheduler-projection-contract.json") +); const { cronId, diffCrons } = await importRepositoryModule("control/cron-index.js", [ [/export \{[^}]+\} from "shared-cron-time";/, "/* re-export stubbed for node test */"], @@ -18,6 +22,16 @@ test("cronId: stable + 10 hex chars", () => { assert.match(cronId("*/5 * * * *", "UTC"), /^[0-9a-f]{10}$/); }); +test("cronId matches the cross-language scheduler projection fixture", () => { + assert.equal( + cronId( + schedulerProjectionContract.cron.entry.cron, + schedulerProjectionContract.cron.entry.timezone + ), + schedulerProjectionContract.cron.cronId + ); +}); + test("cronId: differs on cron or timezone", () => { assert.notEqual(cronId("*/5 * * * *", "UTC"), cronId("*/6 * * * *", "UTC")); assert.notEqual(cronId("*/5 * * * *", "UTC"), cronId("*/5 * * * *", "Asia/Shanghai")); diff --git a/tests/unit/control-d1-lifecycle.test.js b/tests/unit/control-d1-lifecycle.test.js index 5455419..fccf94f 100644 --- a/tests/unit/control-d1-lifecycle.test.js +++ b/tests/unit/control-d1-lifecycle.test.js @@ -1,6 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; +import { controlD1RuntimeClientDataUrl } from "../helpers/load-d1-protocol.js"; import { applyModuleReplacements, moduleDataUrl, readRepositoryFile } from "../helpers/load-shared-module.js"; import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; @@ -29,7 +30,9 @@ export function validateDatabaseName() {} export function isReadyDatabase(database) { return database?.state === "ready"; } `); +const productionBackendUrl = controlD1RuntimeClientDataUrl(); const backendUrl = moduleDataUrl(` +export { d1RuntimeFailure, d1RuntimeFailureLogFields } from ${JSON.stringify(productionBackendUrl)}; export async function d1RuntimeQuery() { /** @type {any} */ (globalThis).__d1RuntimeQueryArgs = arguments; return /** @type {any} */ (globalThis).__d1RuntimeResult || { @@ -51,17 +54,6 @@ export async function d1RuntimeProbeOwner(env, ns, databaseId, requestId) { /** @type {any} */ (globalThis).__d1ProbeOwnerArgs = { env, ns, databaseId, requestId }; return /** @type {any} */ (globalThis).__d1ProbeResult || { ok: true, status: 200, body: { owner: null }, owner: null }; } -export function d1RuntimeFailure(error, ns, databaseId, result) { - return { - error, - namespace: ns, - databaseId, - message: result.body.message, - upstreamCode: result.body.error, - upstreamCategory: result.body.category, - upstreamRetryable: result.body.retryable, - }; -} export function d1RuntimePublicResult(value) { /** @type {any} */ (globalThis).__d1RuntimePublicResultArgs = arguments; return value; @@ -168,7 +160,17 @@ test("listDatabases batches metadata reads and returns ready databases", async ( }); test("createDatabase: backend initialization failure rolls back provisional metadata", async () => { - /** @type {any} */ (globalThis).__d1RuntimeResult = null; + /** @type {any} */ (globalThis).__d1LifecycleLogs = []; + /** @type {any} */ (globalThis).__d1RuntimeResult = { + ok: false, + status: 400, + body: { + error: "sql-error", + message: "private initialization SQL", + category: "sql", + retryable: false, + }, + }; /** @type {any} */ (globalThis).__d1CommitResult = null; /** @type {any} */ (globalThis).__d1ReleasedOwnerArgs = null; /** @type {any} */ (globalThis).__d1LifecycleCommitted = false; @@ -185,10 +187,85 @@ test("createDatabase: backend initialization failure rolls back provisional meta const body = await readJsonResponse(response, 503); assert.equal(body.error, "d1_database_initialize_failed"); + assert.equal(body.message, "Internal error"); assert.equal(body.databaseId, "d1_test"); + assert.equal(body.upstreamCode, "sql-error"); + assert.equal(body.upstreamCategory, "sql"); + assert.equal(body.upstreamRetryable, false); + assert.equal(body.upstreamStatus, 400); + assert.equal(body.detail, undefined); assert.equal(/** @type {any} */ (globalThis).__d1LifecycleCommitted, true); assert.equal(/** @type {any} */ (globalThis).__d1ProvisionalRolledBack, true); assert.equal(/** @type {any} */ (globalThis).__d1ReleasedOwnerArgs, null); + assert.deepEqual(/** @type {any} */ (globalThis).__d1LifecycleLogs.at(-1), { + level: "error", + event: "d1_database_initialize_failed", + fields: { + request_id: "rid-create-fail", + namespace: "demo", + database_id: "d1_test", + status: 503, + reason: "d1_database_initialize_failed", + upstream_status: 400, + upstream_code: "sql-error", + upstream_category: "sql", + upstream_retryable: false, + }, + }); +}); + +test("executeDatabase redacts D1 runtime 5xx diagnostics and logs stable fields", async () => { + /** @type {any} */ (globalThis).__d1LifecycleLogs = []; + /** @type {any} */ (globalThis).__d1RequireResult = { + database: { databaseId: "d1_test", databaseName: "main", state: "ready", createdAt: "now", updatedAt: "now" }, + }; + /** @type {any} */ (globalThis).__d1RuntimeResult = { + ok: false, + status: 503, + body: { + error: "owner-record-invalid", + message: "private owner diagnostic", + category: "ownership", + retryable: true, + }, + }; + + const response = await executeDatabase({ + request: new Request("http://control/ns/demo/d1/databases/main/query", { + method: "POST", + body: JSON.stringify({ sql: "select 1" }), + }), + env: {}, + ns: "demo", + databaseId: "main", + requestId: "rid-execute-fail", + }); + + await assertJsonResponse(response, 503, { + error: "d1_execute_failed", + namespace: "demo", + databaseId: "d1_test", + message: "Internal error", + upstreamCode: "owner-record-invalid", + upstreamCategory: "ownership", + upstreamRetryable: true, + upstreamStatus: 503, + }); + assert.deepEqual(/** @type {any} */ (globalThis).__d1LifecycleLogs.at(-1), { + level: "error", + event: "d1_database_execute_failed", + fields: { + request_id: "rid-execute-fail", + namespace: "demo", + database_id: "d1_test", + status: 503, + reason: "d1_execute_failed", + upstream_status: 503, + upstream_code: "owner-record-invalid", + upstream_category: "ownership", + upstream_retryable: true, + }, + }); }); test("createDatabase: ready flip failure releases initialized owner and rolls back provisional metadata", async () => { diff --git a/tests/unit/control-d1-migrations.test.js b/tests/unit/control-d1-migrations.test.js index c112382..f902fee 100644 --- a/tests/unit/control-d1-migrations.test.js +++ b/tests/unit/control-d1-migrations.test.js @@ -1,6 +1,7 @@ import { afterEach, test } from "node:test"; import assert from "node:assert/strict"; import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; +import { controlD1RuntimeClientDataUrl } from "../helpers/load-d1-protocol.js"; import { applyModuleReplacements, moduleDataUrl, @@ -68,22 +69,9 @@ export function splitSqlStatements(sql) { } `); +const productionBackendUrl = controlD1RuntimeClientDataUrl(); const backendUrl = moduleDataUrl(` -export function d1RuntimeFailure(error, ns, databaseId, result, extra = {}) { - const body = result?.body || {}; - return { - error, - namespace: ns, - databaseId, - message: body.message || body.error || "D1 runtime request failed", - upstreamCode: body.error || "d1-runtime-error", - upstreamCategory: body.category || "internal", - upstreamRetryable: body.retryable === true, - upstreamStatus: result?.status, - detail: body, - ...extra, - }; -} +export { d1RuntimeFailure, d1RuntimeFailureLogFields } from ${JSON.stringify(productionBackendUrl)}; export async function d1RuntimeQuery() { const args = Array.from(arguments); if (!Array.isArray(/** @type {any} */ (globalThis).__d1MigrationTestState.runtimeCalls)) { @@ -301,19 +289,42 @@ test("applyMigrations includes empty progress when reading existing migrations f const body = await readJsonResponse(response, 503); assert.equal(body.error, "d1_migrations_apply_failed"); + assert.equal(body.message, "Internal error"); + assert.equal(body.upstreamCode, "d1-runtime-unavailable"); + assert.equal(body.upstreamCategory, "internal"); + assert.equal(body.upstreamRetryable, false); assert.deepEqual(body.applied, []); assert.deepEqual(body.skipped, []); - assert.deepEqual(/** @type {any} */ (globalThis).__d1MigrationTestState.logs, [{ - level: "warn", - event: "d1_migration_lock_release_failed", - fields: { - request_id: "rid-migration-read-failure", - namespace: "demo", - database_id: "d1_main", - error_name: "Error", - error_message: "migration lock release unavailable", + assert.deepEqual(/** @type {any} */ (globalThis).__d1MigrationTestState.logs, [ + { + level: "error", + event: "d1_migrations_apply_failed", + fields: { + request_id: "rid-migration-read-failure", + namespace: "demo", + database_id: "d1_main", + status: 503, + reason: "d1_migrations_apply_failed", + applied_count: 0, + skipped_count: 0, + upstream_status: 503, + upstream_code: "d1-runtime-unavailable", + upstream_category: "internal", + upstream_retryable: false, + }, }, - }]); + { + level: "warn", + event: "d1_migration_lock_release_failed", + fields: { + request_id: "rid-migration-read-failure", + namespace: "demo", + database_id: "d1_main", + error_name: "Error", + error_message: "migration lock release unavailable", + }, + }, + ]); } finally { /** @type {any} */ (globalThis).__d1MigrationTestState.redis = null; /** @type {any} */ (globalThis).__d1MigrationTestState.runtimeResult = null; @@ -330,17 +341,32 @@ test("applyMigrations includes completed progress when a later migration fails", /** @type {any} */ (globalThis).__d1MigrationTestState.lockToken = token; return "OK"; } - return setCalls === 2 ? "OK" : null; + return "OK"; }, }; /** @type {any} */ (globalThis).__d1MigrationTestState.splitStatements = (/** @type {string} */ sql) => [ { sql, params: [] }, ]; - /** @type {any} */ (globalThis).__d1MigrationTestState.runtimeResult = () => ({ - ok: true, - status: 200, - body: { success: true, results: [] }, - }); + /** @type {any} */ (globalThis).__d1MigrationTestState.runtimeResult = (/** @type {any[]} */ ...args) => { + const statements = /** @type {Array<{ sql?: string }>} */ (args[4] || []); + if (statements.some((statement) => statement.sql?.startsWith("alter table demo"))) { + return { + ok: false, + status: 503, + body: { + error: "result-unknown", + message: "private commit diagnostic", + category: "result-unknown", + retryable: false, + }, + }; + } + return { + ok: true, + status: 200, + body: { success: true, results: [] }, + }; + }; try { const response = await applyMigrations({ @@ -359,14 +385,36 @@ test("applyMigrations includes completed progress when a later migration fails", requestId: "rid-migration-partial", }); - const body = await readJsonResponse(response, 409); - assert.equal(body.error, "d1_migrations_apply_lock_lost"); + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "d1_migration_apply_failed"); + assert.equal(body.message, "Internal error"); assert.equal(body.migrationId, "0002_next.sql"); + assert.equal(body.upstreamCode, "result-unknown"); + assert.equal(body.upstreamCategory, "result-unknown"); + assert.equal(body.upstreamRetryable, false); assert.deepEqual( body.applied.map((/** @type {{ id: string }} */ migration) => migration.id), ["0001_init.sql"] ); assert.deepEqual(body.skipped, []); + assert.deepEqual(/** @type {any} */ (globalThis).__d1MigrationTestState.logs.at(-1), { + level: "error", + event: "d1_migration_apply_failed", + fields: { + request_id: "rid-migration-partial", + namespace: "demo", + database_id: "d1_main", + status: 503, + reason: "d1_migration_apply_failed", + migration_id: "0002_next.sql", + applied_count: 1, + skipped_count: 0, + upstream_status: 503, + upstream_code: "result-unknown", + upstream_category: "result-unknown", + upstream_retryable: false, + }, + }); } finally { /** @type {any} */ (globalThis).__d1MigrationTestState.redis = null; /** @type {any} */ (globalThis).__d1MigrationTestState.lockToken = null; diff --git a/tests/unit/control-d1-runtime-client.test.js b/tests/unit/control-d1-runtime-client.test.js index 976878c..eb9c836 100644 --- a/tests/unit/control-d1-runtime-client.test.js +++ b/tests/unit/control-d1-runtime-client.test.js @@ -1,15 +1,8 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { loadD1QueryWire, d1QueryWireDataUrl, d1TransportDataUrl } from "../helpers/load-d1-protocol.js"; -import { - importRepositoryModule, - importSpecifierReplacements, - repositoryFileUrl, - sharedModuleDataUrl, -} from "../helpers/load-shared-module.js"; +import { loadControlD1RuntimeClient, loadD1QueryWire } from "../helpers/load-d1-protocol.js"; import { withRecordingFetch } from "../helpers/mock-fetch.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; -import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; const { @@ -20,17 +13,11 @@ const { } = await loadD1QueryWire(); const { d1RuntimeFailure, + d1RuntimeFailureLogFields, d1RuntimePublicResult, d1RuntimeQuery, d1RuntimeReleaseOwner, -} = await importRepositoryModule("control/d1-runtime-client.js", importSpecifierReplacements({ - "shared-d1-timeout": sharedModuleDataUrl("shared/d1-timeout.js"), - "shared-d1-transport": d1TransportDataUrl(), - "shared-d1-query-wire": d1QueryWireDataUrl(), - "shared-respond": repositoryFileUrl("shared/respond.js"), - "shared-internal-auth": sharedInternalAuthUrl(), - "shared-owner-endpoint": repositoryFileUrl("shared/owner-endpoint.js"), -})); +} = await loadControlD1RuntimeClient(); /** @template {Record} T @param {T} env */ function withInternalAuthEnv(env) { @@ -186,7 +173,17 @@ test("control D1 runtime client query treats unsupported response content-type a assert.equal(result.status, 502); assert.equal(result.body.error, "invalid_d1_runtime_response"); assert.match(result.body.message, /unsupported d1-runtime response content-type/); + assert.equal(result.body.category, "invalid-response"); + assert.equal(result.body.retryable, false); + assert.equal(result.body.causeCode, "unsupported-content-type"); assert.equal(result.owner, null); + assert.deepEqual(d1RuntimeFailureLogFields(result), { + upstream_status: 502, + upstream_code: "invalid_d1_runtime_response", + upstream_category: "invalid-response", + upstream_retryable: false, + upstream_cause_code: "unsupported-content-type", + }); }); test("control D1 runtime client query treats undecodable binary response as upstream failure", async () => { @@ -209,6 +206,9 @@ test("control D1 runtime client query treats undecodable binary response as upst assert.equal(result.ok, false); assert.equal(result.status, 502); assert.equal(result.body.error, "invalid_d1_runtime_response"); + assert.equal(result.body.category, "invalid-response"); + assert.equal(result.body.retryable, false); + assert.equal(result.body.causeCode, "binary-decode-failed"); assert.equal(result.owner, null); }); @@ -313,13 +313,13 @@ test("control D1 runtime client public result preserves raw row/column payloads" ); }); -test("control D1 runtime failure keeps extra fields additive only", () => { +test("control D1 runtime failure redacts 5xx diagnostics and keeps safe extra fields", () => { const body = d1RuntimeFailure("d1_execute_failed", "demo", "db1", { status: 502, body: { - error: "sql-error", - message: "near syntax", - category: "sql", + error: "result-unknown", + message: "commit reply was lost", + category: "result-unknown", retryable: false, }, }, { @@ -333,6 +333,81 @@ test("control D1 runtime failure keeps extra fields additive only", () => { assert.deepEqual(body, { diagnostic: "kept", + error: "d1_execute_failed", + namespace: "demo", + databaseId: "db1", + message: "Internal error", + upstreamCode: "result-unknown", + upstreamCategory: "result-unknown", + upstreamRetryable: false, + upstreamStatus: 502, + }); + assert.deepEqual(d1RuntimeFailureLogFields({ + status: 502, + body: { + error: "result-unknown", + message: "commit reply was lost", + category: "result-unknown", + retryable: false, + }, + }), { + upstream_status: 502, + upstream_code: "result-unknown", + upstream_category: "result-unknown", + upstream_retryable: false, + }); +}); + +test("control D1 runtime failure log fields reject unbounded or non-token diagnostics", () => { + for (const invalid of [" leading-space", "line\nbreak", "x".repeat(129)]) { + assert.deepEqual(d1RuntimeFailureLogFields({ + status: 502, + body: { + error: invalid, + category: invalid, + causeCode: invalid, + retryable: false, + }, + }), { + upstream_status: 502, + upstream_code: "d1-runtime-error", + upstream_category: "internal", + upstream_retryable: false, + }); + } +}); + +test("control D1 runtime failure uses the outward status to redact upstream 4xx details", () => { + assert.deepEqual(d1RuntimeFailure("d1_database_initialize_failed", "demo", "db1", { + status: 400, + body: { + error: "sql-error", + message: "private initialization SQL", + category: "sql", + retryable: false, + }, + }, {}, { publicStatus: 503 }), { + error: "d1_database_initialize_failed", + namespace: "demo", + databaseId: "db1", + message: "Internal error", + upstreamCode: "sql-error", + upstreamCategory: "sql", + upstreamRetryable: false, + upstreamStatus: 400, + }); +}); + +test("control D1 runtime failure preserves client-facing 4xx diagnostics", () => { + assert.deepEqual(d1RuntimeFailure("d1_execute_failed", "demo", "db1", { + status: 400, + body: { + error: "sql-error", + message: "near syntax", + category: "sql", + retryable: false, + }, + }), { error: "d1_execute_failed", namespace: "demo", databaseId: "db1", @@ -340,7 +415,7 @@ test("control D1 runtime failure keeps extra fields additive only", () => { upstreamCode: "sql-error", upstreamCategory: "sql", upstreamRetryable: false, - upstreamStatus: 502, + upstreamStatus: 400, detail: { error: "sql-error", message: "near syntax", diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index cb4cd9d..c3c9bca 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -77,14 +77,10 @@ const controlLibUrl = moduleDataUrl(` export { bundleAssetPrefix, d1DatabaseReferrersKey, - deleteLockKey, doObjectRegistryKey, - doOwnerScopeScanPatternForStorage, - doStorageIdKey, encodeReferrerMember, parseBundleMeta, referrersKey, - workerVersionsKey, workersIndexKey, workflowDefsKey, } from ${JSON.stringify(productionControlLibUrl)}; @@ -132,6 +128,7 @@ const { handle } = await importControlHandler("control/handlers/delete.js", { "shared-redis": sharedRedisUrl, "shared-queue-keys": sharedQueueKeysUrl, "shared-route-projection": sharedRouteProjectionUrl, + "shared-respond": repositoryFileUrl("shared/respond.js"), "control-handlers-delete-plan": deletePlanUrl, }, }); @@ -608,6 +605,36 @@ test("worker delete reports cleanup_queue_failed when data-plane cleanup enqueue )); }); +test("worker delete paths cancel ignored request bodies without awaiting cancellation", { timeout: 1000 }, async () => { + for (const suffix of ["", "?dry_run=1"]) { + resetDeleteHandlerState(); + let cancelCalls = 0; + const stream = new ReadableStream({ + cancel() { + cancelCalls += 1; + return new Promise(() => {}); + }, + }); + const request = new Request(`http://control/ns/demo/worker/api/delete${suffix}`, /** @type {RequestInit} */ ({ + method: "POST", + body: stream, + duplex: "half", + })); + + const response = await handle({ + request, + url: new URL(request.url), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-body", + }); + + assert.equal(response.status, 200, suffix || "execute"); + assert.equal(cancelCalls, 1, suffix || "execute"); + } +}); + test("worker delete classifies non-object retained bundle metadata as corrupt", async () => { const testState = resetDeleteHandlerState({ bundleMetaRaw: "[]" }); diff --git a/tests/unit/control-handlers-workers.test.js b/tests/unit/control-handlers-workers.test.js index f6b02cf..05b4796 100644 --- a/tests/unit/control-handlers-workers.test.js +++ b/tests/unit/control-handlers-workers.test.js @@ -24,7 +24,7 @@ const workersHandlerState = installControlHandlerState( }) ); const controlSharedUrl = controlSharedHarnessUrl(WORKERS_HANDLER_STATE_GLOBAL); -const { libUrl: controlLibUrl } = await compileControlGraph(); +const { libUrl: controlLibUrl, sharedVersionUrl } = await compileControlGraph(); const sharedSecretKeysUrl = repositoryFileUrl("shared/secret-keys.js"); @@ -32,6 +32,7 @@ const src = applyModuleReplacements(readRepositoryFile("control/handlers/workers [/from "control-shared";/, `from ${JSON.stringify(controlSharedUrl)};`], [/from "control-lib";/, `from ${JSON.stringify(controlLibUrl)};`], [/from "shared-secret-keys";/, `from ${JSON.stringify(sharedSecretKeysUrl)};`], + [/from "shared-version";/, `from ${JSON.stringify(sharedVersionUrl)};`], ]); const { handle } = await import(moduleDataUrl(src)); diff --git a/tests/unit/control-index.test.js b/tests/unit/control-index.test.js index 457e405..ebf1584 100644 --- a/tests/unit/control-index.test.js +++ b/tests/unit/control-index.test.js @@ -13,8 +13,6 @@ import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.j const sharedNsPatternUrl = repositoryFileUrl("shared/ns-pattern.js"); const controlLibUrl = moduleDataUrl(` export const NS_RE = /^[a-z0-9-]+$/; -export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; -export { configuredHostname } from ${JSON.stringify(sharedNsPatternUrl)}; export function configuredPublicUrl() { return null; } export function platformVersionFromPackageJson() { return "wdl.test"; } export function projectAccessPrincipal(principal) { return principal || null; } @@ -75,6 +73,7 @@ export async function handle(args) { const controlIndex = (await importRepositoryModule("control/index.js", importSpecifierReplacements({ "control-lib": controlLibUrl, "control-shared": controlSharedUrl, + "shared-ns-pattern": sharedNsPatternUrl, "shared-request-scope": requestScopeUrl, "wdl-package-json-source": packageJsonUrl, "control-handlers-reload": handlerUrl, diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 2558b1b..9186b55 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -2,7 +2,12 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { loadControlLib } from "../helpers/load-control-lib.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; -import { RESERVED_NS } from "../../shared/ns-pattern.js"; +import { + RESERVED_NS, + configuredHostname, + platformDomainFromEnv, + validateModulePath, +} from "../../shared/ns-pattern.js"; import { MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, WORKERD_EXPERIMENTAL_COMPAT_FLAGS, @@ -27,8 +32,6 @@ const { PLATFORM_TIER_RESERVED_NS } = sharedAuthRoles; const { NS_RE, parseControlRoute, - configuredHostname, - platformDomainFromEnv, configuredPublicUrl, parseWorkerdDependencyVersion, platformVersionFromPackageJson, @@ -39,7 +42,6 @@ const { parseR2DispatchPath, isAdminAcceptableNs, validateSecretKey, - validateModulePath, encodeReferrerMember, formatD1ReferrerBlockers, formatReferrerBlocker, @@ -52,11 +54,7 @@ const { d1DatabaseTombstoneKey, d1DatabaseTombstonesKey, doObjectRegistryKey, - doOwnerScopeScanPatternForStorage, - doStorageIdKey, referrersKey, - deleteLockKey, - workerVersionsKey, workersIndexKey, } = controlLib; const { @@ -281,15 +279,11 @@ test("extractD1Refs: yields d1 database refs only", () => { test("control Redis key helpers match canonical schema", () => { assert.equal(referrersKey("demo", "api", "v3"), "worker-version-referrers:demo:api:v3"); - assert.equal(deleteLockKey("demo", "api"), "worker-delete-lock:demo:api"); - assert.equal(workerVersionsKey("demo", "api"), "worker-versions:demo:api"); assert.equal(workersIndexKey("demo"), "workers:demo"); assert.equal(d1DatabaseNameKey("demo", "main"), "d1:database-name:demo:main"); assert.equal(d1DatabaseReferrersKey("demo", "d1_main"), "d1:database-referrers:demo:d1_main"); assert.equal(d1DatabaseTombstoneKey("demo", "d1_main"), "d1:database-tombstone:demo:d1_main"); assert.equal(d1DatabaseTombstonesKey("demo"), "d1:database-tombstones:demo"); - assert.equal(doStorageIdKey("demo", "api"), "worker:do-storage:demo:api"); - assert.equal(doOwnerScopeScanPatternForStorage("do_abc"), "do:owner:scope:do_abc%3A*"); assert.equal(doObjectRegistryKey("do_abc"), "do:objects:do_abc"); }); @@ -1303,11 +1297,6 @@ test("normalizeHost: lowercases and strips :port", () => { assert.equal(normalizeHost(" api.workers.example "), "api.workers.example"); }); -test("normalizeHost: idempotent", () => { - const once = normalizeHost("API.Workers.example:443"); - assert.equal(normalizeHost(once), once); -}); - test("normalizeHost: rejects empty / non-string / invalid shapes", () => { assert.throws(() => normalizeHost(""), /must not be empty/); assert.throws(() => normalizeHost(" "), /must not be empty/); @@ -1318,6 +1307,11 @@ test("normalizeHost: rejects empty / non-string / invalid shapes", () => { assert.throws(() => normalizeHost("has\ttab"), /invalid host/); }); +test("normalizeHost: idempotent", () => { + const once = normalizeHost("API.Workers.example:443"); + assert.equal(normalizeHost(once), once); +}); + test("normalizeHost: strips trailing FQDN dot(s)", () => { assert.equal(normalizeHost("workers.example."), "workers.example"); assert.equal(normalizeHost("workers.example..."), "workers.example"); diff --git a/tests/unit/control-lifecycle-indexes.test.js b/tests/unit/control-lifecycle-indexes.test.js index 0a57c4b..00eceea 100644 --- a/tests/unit/control-lifecycle-indexes.test.js +++ b/tests/unit/control-lifecycle-indexes.test.js @@ -1,10 +1,15 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { importRepositoryModule, readRepositoryJson } from "../helpers/load-shared-module.js"; +import { + importRepositoryModule, + readRepositoryJson, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; const SCHEDULER_PROJECTION_CONTRACT = readRepositoryJson( "tests/fixtures/scheduler-projection-contract.json" ); +const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const { cronEntryJson, @@ -27,11 +32,11 @@ const { const encodeReferrerMember = ({ callerNs, callerWorker, callerVersion, binding }) => [callerNs, callerWorker, callerVersion, binding].join("\\t"); const referrersKey = (ns, worker, version) => \`referrers:\${ns}:\${worker}:\${version}\`; -const workerVersionsKey = (ns, worker) => \`worker-versions:\${ns}:\${worker}\`; const workersIndexKey = (ns) => \`workers:\${ns}\`;`], [/import \{ QUEUE_CONSUMER_INDEX_KEY, queueConsumerKey \} from "shared-queue-keys";/, `const QUEUE_CONSUMER_INDEX_KEY = "queue-consumer-index"; const queueConsumerKey = (ns, queue) => \`queue-consumer:\${ns}:\${queue}\`;`], + [/from "shared-version"/, `from ${JSON.stringify(SHARED_VERSION_URL)}`], ]); function recordMulti() { diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index b110279..108e030 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -59,9 +59,8 @@ function loadLogsTailHandler(options = {}) { `from ${JSON.stringify(SHARED_NS_PATTERN_URL)};`, ], [ - /import \{\n {2}WORKER_NAME_RE,\n {2}compareStreamIds,\n {2}isValidResumeId,\n {2}isValidWorkerName,\n\} from "control-lib";/, - `const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; - const compareStreamIds = () => 0; + /import \{\n {2}compareStreamIds,\n {2}isValidResumeId,\n {2}isValidWorkerName,\n\} from "control-lib";/, + `const compareStreamIds = () => 0; const isValidResumeId = () => true; const isValidWorkerName = (name) => WORKER_NAME_RE.test(name);`, ], diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 8a8dee3..ee1d19d 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -720,9 +720,6 @@ const workerControlSharedUrl = controlSharedHarnessUrl(WORKER_SECRET_STATE_GLOBA const workerLibStubUrl = moduleDataUrl(` ${validateSecretKeyStubSource} export { workflowDefsKey } from ${JSON.stringify(PRODUCTION_CONTROL_LIB_URL)}; -export const deleteLockKey = (ns, worker) => \`worker-delete-lock:\${ns}:\${worker}\`; -export const workerVersionsKey = (ns, worker) => \`worker-versions:\${ns}:\${worker}\`; -export const routesKey = (ns) => \`routes:\${ns}\`; export const workersIndexKey = (ns) => \`workers:\${ns}\`; `); const lifecycleStubUrl = moduleDataUrl(` diff --git a/tests/unit/d1-runtime-index.test.js b/tests/unit/d1-runtime-index.test.js index 3f169d1..4078edc 100644 --- a/tests/unit/d1-runtime-index.test.js +++ b/tests/unit/d1-runtime-index.test.js @@ -122,10 +122,12 @@ export function createHttpRequestScope({ request, service, metrics, log, route, const startedAt = Date.now(); let status = 500; let requestError = null; + let hasRequestError = false; return { requestId, markError(err) { requestError = err; + hasRequestError = true; return err; }, respond(response) { @@ -143,6 +145,7 @@ export function createHttpRequestScope({ request, service, metrics, log, route, status, startedAt, error: requestError, + hasError: hasRequestError, probeRoutes, }); }, diff --git a/tests/unit/observability.test.js b/tests/unit/observability.test.js index c586f4e..14b33e1 100644 --- a/tests/unit/observability.test.js +++ b/tests/unit/observability.test.js @@ -838,6 +838,20 @@ test("recordRequestComplete: error argument flows into log via formatError", () assert.equal(entries[0].fields.error_message, "bad"); }); +test("recordRequestComplete: falsey thrown values remain errors", () => { + for (const [error, message] of [[0, "0"], [false, "false"], [null, "Unknown error"], [undefined, "Unknown error"]]) { + const { log, entries } = captureLog(); + recordRequestComplete({ + service: "runtime", metrics: null, log, + method: "POST", requestId: "r", route: "worker_scheduled", status: 200, + startedAt: Date.now(), error, hasError: true, + }); + assert.equal(entries.length, 1); + assert.equal(entries[0].level, "error"); + assert.equal(entries[0].fields.error_message, message); + } +}); + test("setLogLevel ignores unknown names (keeps previous level)", () => { setLogLevel("warn"); try { diff --git a/tests/unit/request-scope.test.js b/tests/unit/request-scope.test.js index cea637b..d9e8ece 100644 --- a/tests/unit/request-scope.test.js +++ b/tests/unit/request-scope.test.js @@ -57,7 +57,7 @@ test("createHttpRequestScope echoes request id and records final request state", extras: () => ({ namespace: "tenant-a" }), }); scope.setRoute("worker_fetch"); - const err = new Error("boom"); + const err = false; scope.markError(err); const response = scope.respond(Response.json({ ok: false }, { status: 502 })); scope.complete(); @@ -68,5 +68,25 @@ test("createHttpRequestScope echoes request id and records final request state", assert.equal(observability.calls.complete[0].route, "worker_fetch"); assert.equal(observability.calls.complete[0].status, 502); assert.equal(observability.calls.complete[0].error, err); + assert.equal(observability.calls.complete[0].hasError, true); assert.deepEqual(observability.calls.complete[0].extras, { namespace: "tenant-a" }); }); + +test("createHttpRequestScope records nullish thrown values as errors", () => { + for (const err of [null, undefined]) { + observability.calls.complete.length = 0; + const scope = createHttpRequestScope({ + request: new Request("http://runtime.test/path"), + service: "runtime", + log() {}, + route: "worker_fetch", + }); + + scope.markError(err); + scope.respond(new Response(null, { status: 502 })); + scope.complete(); + + assert.equal(observability.calls.complete[0].hasError, true); + assert.equal(observability.calls.complete[0].error, err); + } +}); diff --git a/tests/unit/runtime-dispatch-handlers.test.js b/tests/unit/runtime-dispatch-handlers.test.js index 59a4c77..26932b5 100644 --- a/tests/unit/runtime-dispatch-handlers.test.js +++ b/tests/unit/runtime-dispatch-handlers.test.js @@ -317,6 +317,26 @@ test("handleScheduledDispatch records handler errors without turning them into H assert.equal(body.error, "scheduled failed"); }); +test("handleScheduledDispatch preserves falsey thrown values", async () => { + for (const thrown of [0, false]) { + const scope = makeScope(); + const res = await handleScheduledDispatch({ + request: jsonRequest({ scheduledTime: 123, cron: "* * * * *" }), + scope, + stub: makeStub({ + async scheduled() { + throw thrown; + }, + }), + }); + + const body = await readJsonResponse(res, 200, String(thrown)); + assert.deepEqual(scope.errors, [thrown]); + assert.equal(body.outcome, "error"); + assert.equal(body.error, String(thrown)); + } +}); + test("handleScheduledDispatch maps service_binding_extra_handlers exception outcome", async () => { const scope = makeScope(); const res = await handleScheduledDispatch({ @@ -330,7 +350,7 @@ test("handleScheduledDispatch maps service_binding_extra_handlers exception outc }); const body = await readJsonResponse(res, 200); - assert.deepEqual(scope.errors, []); + assert.deepEqual(scope.errors, ["thrown by handler"]); assert.equal(body.outcome, "error"); assert.equal(body.error, "thrown by handler"); }); @@ -379,6 +399,68 @@ test("handleQueuedDispatch decodes queue messages before JSRPC dispatch", async assert.equal(calls[0].messages[0].attempts, 3); }); +test("handleQueuedDispatch records service_binding_extra_handlers exception outcome", async () => { + const scope = makeScope(); + const response = await handleQueuedDispatch({ + request: jsonRequest({ + queue: "jobs", + messages: [{ + id: "m1", + first_seen_ms: "123", + attempts: "0", + body_b64: btoa("payload"), + content_type: "text", + }], + }), + scope, + stub: makeStub({ + async queue() { + return { + outcome: "exception", + ackAll: false, + explicitAcks: [], + retryMessages: [], + retryBatch: { retry: false }, + }; + }, + }), + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.outcome, "ok"); + assert.equal(body.result.outcome, "exception"); + assert.deepEqual(scope.errors, ["queue handler threw"]); +}); + +test("handleQueuedDispatch preserves falsey thrown values", async () => { + for (const thrown of [0, false]) { + const scope = makeScope(); + const res = await handleQueuedDispatch({ + request: jsonRequest({ + queue: "jobs", + messages: [{ + id: "m1", + first_seen_ms: "123", + attempts: "0", + body_b64: btoa("payload"), + content_type: "text", + }], + }), + scope, + stub: makeStub({ + async queue() { + throw thrown; + }, + }), + }); + + const body = await readJsonResponse(res, 200, String(thrown)); + assert.deepEqual(scope.errors, [thrown]); + assert.equal(body.outcome, "error"); + assert.equal(body.error, String(thrown)); + } +}); + test("handleQueuedDispatch maps invalid base64 to a permanent decode failure", async () => { let dispatchCalls = 0; const scope = makeScope(); diff --git a/tests/unit/runtime-dispatch-workflows.test.js b/tests/unit/runtime-dispatch-workflows.test.js index fe2d9f5..632765c 100644 --- a/tests/unit/runtime-dispatch-workflows.test.js +++ b/tests/unit/runtime-dispatch-workflows.test.js @@ -240,6 +240,8 @@ test("handleWorkflowRunDispatch invokes named workflow run with step.do facade", scope.requestId = "rid-workflow"; /** @type {any[]} */ const calls = []; + /** @type {string[]} */ + let stepSurface = []; const backend = makeWorkflowBackend(async (url) => { if (url.endsWith("/claim-step")) return Response.json({ state: "run", attempt: 2 }); if (url.endsWith("/commit-step-success")) return Response.json({ state: "complete" }); @@ -266,6 +268,7 @@ test("handleWorkflowRunDispatch invokes named workflow run with step.do facade", OrderWorkflow: { async run(/** @type {any} */ event, /** @type {any} */ step) { calls.push(event); + stepSurface = Object.keys(step).toSorted(); return await step.do("charge", async (/** @type {{ attempt: number }} */ { attempt }) => ({ charged: event.payload.orderId, attempt, @@ -286,6 +289,7 @@ test("handleWorkflowRunDispatch invokes named workflow run with step.do facade", }); assert.equal(typeof body.duration_ms, "number"); assert.deepEqual(calls, [{ payload: { orderId: 123 } }]); + assert.deepEqual(stepSurface, ["do", "sleep", "sleepUntil", "waitForEvent"]); assert.deepEqual(backend.calls.map((c) => c.url), [ "http://workflows/internal/workflows/replay-steps", "http://workflows/internal/workflows/claim-step", @@ -1955,6 +1959,50 @@ test("workflow internal error codes cannot be forged with Error.name", async () }); }); +test("handleWorkflowRunDispatch does not trust a forged WorkflowSuspended name", async () => { + const scope = makeScope(); + const forged = { name: "WorkflowSuspended", message: "forged suspension" }; + const backend = makeWorkflowBackend(async (url) => { + if (url.endsWith("/claim-step")) return Response.json({ state: "run" }); + if (url.endsWith("/commit-step-error")) return Response.json({ state: "failed" }); + return Response.json({ error: "unexpected", message: "unexpected backend call" }, { status: 500 }); + }); + const response = await handleWorkflowRunDispatch({ + run: { + ns: "demo", + worker: "shop", + frozenVersion: "v1", + workflowName: "orders", + workflowKey: "wf_abc", + className: "OrderWorkflow", + instanceId: "inst-forged-suspension", + generation: 1, + runToken: "run-1", + event: { payload: {} }, + }, + scope, + env: workflowEnv(backend), + stub: makeStub({ + entrypoints: { + OrderWorkflow: { + async run(/** @type {any} */ _event, /** @type {any} */ step) { + try { + await step.do("forged", () => Promise.reject(forged)); + } catch { + return "tenant recovered"; + } + }, + }, + }, + }), + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.outcome, "failed"); + assert.deepEqual(body.error, forged); + assert.deepEqual(scope.errors, [forged]); +}); + test("handleWorkflowRunDispatch rejects new durable steps after swallowed terminal step.do failure", async () => { const scope = makeScope(); const backend = makeWorkflowBackend(async (url) => { @@ -2264,6 +2312,59 @@ test("handleWorkflowRunDispatch marks NonRetryableError step failures terminal", assert.equal(scope.errors.length, 1); }); +test("handleWorkflowRunDispatch keeps falsey step failures terminal after tenant catch", async () => { + for (const [thrown, expectedMessage] of [ + [false, "false"], + [0, "0"], + ["", ""], + [null, "Workflow run failed"], + [undefined, "Workflow run failed"], + ]) { + _resetWorkflowReplayCacheForTest(); + const scope = makeScope(); + const backend = makeWorkflowBackend(async (url) => { + if (url.endsWith("/claim-step")) return Response.json({ state: "run", attempt: 1 }); + if (url.endsWith("/commit-step-error")) return Response.json({ state: "failed" }); + return Response.json({ error: "unexpected", message: "unexpected backend call" }, { status: 500 }); + }); + const res = await handleWorkflowRunDispatch({ + run: { + ns: "demo", + worker: "shop", + frozenVersion: "v1", + workflowName: "orders", + workflowKey: "wf_abc", + className: "OrderWorkflow", + instanceId: "inst-1", + generation: 1, + runToken: "run-1", + event: { payload: { orderId: 123 } }, + }, + scope, + env: workflowEnv(backend), + stub: makeStub({ + entrypoints: { + OrderWorkflow: { + async run(/** @type {any} */ _event, /** @type {any} */ step) { + try { + await step.do("caught", () => Promise.reject(thrown)); + } catch { + return "tenant recovered"; + } + return "unreachable"; + }, + }, + }, + }), + }); + + const body = await readJsonResponse(res, 200, String(thrown)); + assert.equal(body.outcome, "failed"); + assert.equal(body.error.message, expectedMessage); + assert.deepEqual(scope.errors, [thrown]); + } +}); + test("handleWorkflowRunDispatch suspends on step.waitForEvent", async () => { const scope = makeScope(); const backend = makeWorkflowBackend(async (url) => { diff --git a/tests/unit/runtime-eviction.test.js b/tests/unit/runtime-eviction.test.js index 6db61d4..fb2acfc 100644 --- a/tests/unit/runtime-eviction.test.js +++ b/tests/unit/runtime-eviction.test.js @@ -73,6 +73,7 @@ function makeAbortStub({ behavior }) { if (behavior === "internal-error-other") throw new Error("internal error; abort signal aborted"); if (behavior === "internal-error-suffix") throw new Error("preamble: internal error; reference = abc"); if (behavior === "other-error") throw new Error("kaboom"); + if (behavior === "false-error") return Promise.reject(false); // no-op: returns cleanly }, }; @@ -187,6 +188,30 @@ test("abortLoadedWorker surfaces unexpected errors as { aborted: false, reason: assert.equal(loadedWorkerCount(), 1); }); +test("evictSiblings preserves falsey unexpected errors in diagnostics", async () => { + _resetLoadedWorkersForTest(); + recordLoadedWorker("alpha:web:v1"); + recordLoadedWorker("alpha:web:v2"); + const { log, records } = captureLogs(); + + await evictSiblings({ + env: makeAbortStub({ behavior: "false-error" }), + workerId: "alpha:web:v2", + log, + }); + + assert.deepEqual(records.at(-1), { + level: "warn", + event: "evict_skipped", + fields: { + worker_id: "alpha:web:v1", + triggered_by: "alpha:web:v2", + reason: "unexpected", + error_message: "false", + }, + }); +}); + test("evictSiblings aborts every same-ns/same-name sibling and keeps current loaded", async () => { _resetLoadedWorkersForTest(); recordLoadedWorker("alpha:web:v1"); diff --git a/tests/unit/runtime-workflows-client.test.js b/tests/unit/runtime-workflows-client.test.js index 099511f..39818eb 100644 --- a/tests/unit/runtime-workflows-client.test.js +++ b/tests/unit/runtime-workflows-client.test.js @@ -35,6 +35,38 @@ test("Workflow facade does not expose private backend caller", () => { assert.equal(typeof workflow.create, "function"); }); +test("Workflow instances do not expose the private caller through patched Function.prototype.bind", async () => { + /** @type {string[]} */ + const endpoints = []; + const workflow = createWorkflowForTest({ + backend: { + /** @param {string} url */ + async fetch(url) { + endpoints.push(new URL(url).pathname); + return Response.json({ id: "inst-1", status: "running" }); + }, + }, + }); + const originalBind = Function.prototype.bind; + let privateBindCalls = 0; + Function.prototype.bind = function (/** @type {any[]} */ ...args) { + if (this.name === "#call") privateBindCalls += 1; + return Reflect.apply(originalBind, this, args); + }; + try { + const instance = await workflow.get("inst-1"); + await instance.status(); + } finally { + Function.prototype.bind = originalBind; + } + + assert.equal(privateBindCalls, 0); + assert.deepEqual(endpoints, [ + "/internal/workflows/get", + "/internal/workflows/status", + ]); +}); + test("Workflow.create preserves runtime metadata from params override", async () => { /** @type {Record | undefined} */ let capturedRequestBody; @@ -92,6 +124,88 @@ test("Workflow.create preserves runtime metadata from params override", async () }); }); +test("Workflow request identity ignores tenant-patched JSON.stringify", async () => { + let capturedBody = ""; + const workflow = createWorkflowForTest({ + backend: { + /** @param {string} _url @param {RequestInit} init */ + async fetch(_url, init) { + capturedBody = String(init.body); + return new Response('{"id":"inst-1"}', { + headers: { "content-type": "application/json" }, + }); + }, + }, + }); + const originalStringify = JSON.stringify; + let patchedCalls = 0; + JSON.stringify = () => { + patchedCalls += 1; + return '{"ns":"victim","worker":"victim-worker"}'; + }; + try { + await workflow.create({ id: "inst-1" }); + } finally { + JSON.stringify = originalStringify; + } + + assert.equal(patchedCalls, 0); + assert.deepEqual(JSON.parse(capturedBody), { + instanceId: "inst-1", + params: null, + retention: null, + callback: null, + ns: "tenant-a", + worker: "shop", + frozenVersion: "v7", + workflowName: "orders", + workflowKey: "wf_0123456789abcdef0123456789abcdef", + className: "OrderWorkflow", + requestId: null, + }); +}); + +test("Workflow request identity ignores inherited Object.prototype.toJSON", async () => { + let capturedBody = ""; + const workflow = createWorkflowForTest({ + backend: { + /** @param {string} _url @param {RequestInit} init */ + async fetch(_url, init) { + capturedBody = String(init.body); + return new Response('{"id":"inst-2"}', { + headers: { "content-type": "application/json" }, + }); + }, + }, + }); + const previousDescriptor = Object.getOwnPropertyDescriptor(Object.prototype, "toJSON"); + let inheritedCalls = 0; + Object.defineProperty(Object.prototype, "toJSON", { + configurable: true, + value() { + inheritedCalls += 1; + return { ns: "victim", worker: "victim-worker" }; + }, + }); + try { + await workflow.create({ id: "inst-2" }); + } finally { + if (previousDescriptor) { + Object.defineProperty(Object.prototype, "toJSON", previousDescriptor); + } else { + delete /** @type {{ toJSON?: unknown }} */ (Object.prototype).toJSON; + } + } + + assert.equal(inheritedCalls, 0); + const body = JSON.parse(capturedBody); + assert.equal(body.ns, "tenant-a"); + assert.equal(body.worker, "shop"); + assert.equal(body.frozenVersion, "v7"); + assert.equal(body.workflowKey, "wf_0123456789abcdef0123456789abcdef"); + assert.equal(body.className, "OrderWorkflow"); +}); + test("Workflow.create forwards explicit non-null retention", async () => { /** @type {Record | undefined} */ let capturedBody; @@ -198,6 +312,42 @@ test("Workflow.createBatch accepts valid instances array", async () => { assert.deepEqual(instances.map((instance) => instance.id), ["inst-1", "inst-2"]); }); +test("Workflow.createBatch rejects duplicate backend response ids", async () => { + const workflow = createWorkflowForTest({ + backend: { + async fetch() { + return Response.json({ + instances: [{ id: "inst-1" }, { id: "inst-1" }], + }); + }, + }, + }); + + await assert.rejects( + () => workflow.createBatch([{ id: "inst-1" }, { id: "inst-2" }]), + /Workflow createBatch response contains duplicate id inst-1/ + ); +}); + +test("Workflow create and get require matching response ids", async () => { + for (const endpoint of ["create", "get"]) { + const workflow = createWorkflowForTest({ + backend: { + async fetch() { + return Response.json({ id: "inst-other" }); + }, + }, + }); + + await assert.rejects( + () => endpoint === "create" + ? workflow.create({ id: "inst-1" }) + : workflow.get("inst-1"), + new RegExp(`Workflow ${endpoint} response id mismatch`) + ); + } +}); + test("Workflow.create rejects non-object success responses", async () => { const workflow = createWorkflowForTest({ backend: { diff --git a/tests/unit/version.test.js b/tests/unit/version.test.js index 8db377f..dbb8804 100644 --- a/tests/unit/version.test.js +++ b/tests/unit/version.test.js @@ -25,6 +25,7 @@ import { nsHostsKey, parseDeleteLockKind, parseVersion, + workerVersionsKey, } from "../../shared/version.js"; const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); @@ -97,6 +98,7 @@ test("nextVersionKey composes the worker version counter key", () => { }); test("worker lifecycle key helpers compose canonical keys", () => { + assert.equal(workerVersionsKey("demo", "hello"), "worker-versions:demo:hello"); assert.equal(doStorageIdKey("demo", "hello"), "worker:do-storage:demo:hello"); assert.equal(DO_OWNER_SCOPE_PREFIX, "do:owner:scope:"); assert.equal( From 0c84c89eea1ca2febc8c1330a7b98ceca1f60404 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 05:36:04 +0800 Subject: [PATCH 16/23] Reduce redundant defensive contract layers Trust authenticated in-tree RPC outputs while retaining validation at tenant, persisted-state, credential, and external boundaries. Simplify compatibility, workflow, DO, scheduler, Redis, and style-contract machinery while preserving behavior-level owner-hint and effective Compose profile guards. Signed-off-by: Lu Zhang --- CHANGELOG.md | 4 +- CLAUDE.md | 6 + control/bundle.js | 22 +- control/d1-runtime-client.js | 4 +- control/routing.js | 16 +- do-runtime/protocol.js | 55 +-- do-runtime/protocol/identity.js | 5 +- docs/compatibility.md | 2 +- docs/compatibility.zh.md | 2 +- docs/modules/d1.md | 10 +- docs/modules/d1.zh.md | 4 +- docs/modules/durable-objects.md | 14 +- docs/modules/durable-objects.zh.md | 4 +- docs/modules/log-tail-observability.md | 15 +- docs/modules/log-tail-observability.zh.md | 2 +- docs/modules/queues-cron.md | 3 +- docs/modules/queues-cron.zh.md | 2 +- docs/modules/runtime.md | 34 +- docs/modules/runtime.zh.md | 6 +- docs/modules/workflows.md | 2 +- docs/modules/workflows.zh.md | 2 +- docs/redis-key-layout.md | 8 +- docs/redis-key-layout.zh.md | 2 +- runtime/_wdl-do-transport.js | 142 +++---- runtime/dispatch/workflow-step.js | 46 +-- runtime/lib.js | 10 +- runtime/load/code-budget.js | 15 - runtime/load/wrapper-generate.js | 114 +++--- runtime/workflows-client.js | 16 - rust/redis-proxy/src/kv.rs | 129 ++----- rust/scheduler/src/runtime_client.rs | 44 +-- rust/workflows/src/api.rs | 2 +- rust/workflows/src/api/lifecycle/restart.rs | 13 +- rust/workflows/src/api/pending_restart.rs | 55 --- test-workers/do-rpc/src/index.js | 20 +- tests/helpers/style-contract-scanner.js | 141 +------ .../integration/durable-objects-core.test.js | 8 - tests/unit/control-d1-runtime-client.test.js | 19 - tests/unit/control-lib.test.js | 38 +- tests/unit/control-routing.test.js | 55 ++- tests/unit/do-runtime-index.test.js | 13 +- tests/unit/do-runtime-protocol.test.js | 24 -- tests/unit/runtime-dispatch-workflows.test.js | 29 +- tests/unit/runtime-do-client.test.js | 237 ++++-------- tests/unit/runtime-lib.test.js | 19 +- tests/unit/runtime-load.test.js | 39 +- tests/unit/runtime-workflows-client.test.js | 59 --- tests/unit/runtime-wrapper-generate.test.js | 155 +------- tests/unit/style-contracts.test.js | 346 ++---------------- .../unit/test-helper-style-contracts.test.js | 16 - 50 files changed, 484 insertions(+), 1544 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aff099..1d28195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ - Updated maintenance dependencies and image baselines: workerd runtime images now use distroless `base-debian13`, local and Kubernetes Valkey use `valkey/valkey:9.1-alpine`, and root development tooling moved to current patch/minor releases. - Consolidated duplicated Control request/error/retry paths, Redis locks and key grammars, queue and cron projections, S3 retry policy, observability helpers, and Rust sidecar contracts into canonical owners with shared behavioral and cross-language fixture coverage. -- Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and D1/R2/DO facades resist tenant prototype mutation, owner records and forwarding targets require service-specific private endpoints, ordinary nested DO calls cannot override the parent request id through request headers or public facade options, and DO ownership retries rely on private do-runtime markers without replaying unknown-outcome non-idempotent requests. -- Tightened forward-only input contracts: dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01` and error-serialization flags that conflict with the effective date or WDL's required enhanced serialization, internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, DO class names are capped so every shard fits the host-id limit, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. +- Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and D1/R2/DO facades resist tenant prototype mutation at capability boundaries, owner records and forwarding targets require service-specific private endpoints, generated wrappers propagate diagnostic request ids on a best-effort basis, and DO ownership retries rely on private do-runtime markers without replaying unknown-outcome non-idempotent requests. +- Tightened forward-only input contracts: dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01`, upstream experimental flags, and flags that disable WDL's required enhanced error serialization; internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, DO class names are capped so every shard fits the host-id limit, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. - Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, worker-version, scheduler-projection, and workflow-limit fixtures. - Normalized `PLATFORM_DOMAIN` across routing tiers, stopped `/whoami` from advertising the internal `workers.local` fallback as a public namespace URL, and aligned Compose/Kubernetes/test wiring with the consolidated owners. - Removed the unreachable DO inline worker-code test hook and its Terraform switch; do-runtime invoke paths now accept only canonical persisted bundle identities and reject non-canonical or oversized host ids. diff --git a/CLAUDE.md b/CLAUDE.md index 7e7bc84..9fae187 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -114,6 +114,12 @@ shared crate `wdl-rust-common`. convergence, not a new abstraction. - Do not add fake old/new protocol fallback for in-tree tiers that ship together unless there is a real external rollout requirement. +- After authentication, in-tree tiers that ship together may trust ephemeral RPC output + owned by the peer; do not repeat full semantic validation at every hop. Keep strict + validation at tenant inputs, persisted-state readers, credential attachment points, + and external protocol boundaries. Enforce each invariant at one owning layer, and do + not pre-implement a rejection that workerd or the backing platform already performs + unless WDL has an independent product policy or must prevent an earlier side effect. - If you change a route/body protocol, roll the side that accepts the new shape before the side that sends it. - If you change Redis ownership, update writers, readers, docs, and style-contract diff --git a/control/bundle.js b/control/bundle.js index a502e7e..8fdd6ce 100644 --- a/control/bundle.js +++ b/control/bundle.js @@ -13,9 +13,6 @@ import { validateModulePath, } from "shared-ns-pattern"; import { - DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE, - ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE, - ENHANCED_ERROR_SERIALIZATION_FLAG, LEGACY_ERROR_SERIALIZATION_FLAG, MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, firstWorkerdExperimentalCompatFlag, @@ -153,8 +150,8 @@ export function normalizeModule(value) { // like { compatibilityFlags: "nodejs_compat" } or { flag: true } drops // the user's declaration and leaves only the platform floor — looks // healthy, isn't. -/** @param {unknown} flags @param {string} compatibilityDate */ -function validateCompatibilityFlags(flags, compatibilityDate) { +/** @param {unknown} flags */ +function validateCompatibilityFlags(flags) { if (flags === undefined || flags === null) return; if (!Array.isArray(flags)) { throw new Error( @@ -183,16 +180,6 @@ function validateCompatibilityFlags(flags, compatibilityDate) { `${JSON.stringify(LEGACY_ERROR_SERIALIZATION_FLAG)} is not supported because WDL requires enhanced error serialization` ); } - if ( - compatibilityDate >= ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE && - flags.includes(ENHANCED_ERROR_SERIALIZATION_FLAG) - ) { - throw new BundleConfigError( - 400, - "compatibility_flag_unsupported", - `${JSON.stringify(ENHANCED_ERROR_SERIALIZATION_FLAG)} became the workerd default on ${ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE} and must not be specified for compatibilityDate ${compatibilityDate}` - ); - } } /** @param {unknown} workflows */ @@ -316,10 +303,7 @@ export function prepareBundle(mainModule, rawModules, extras = {}) { } validateBindings(extras.bindings); const compatibilityDate = validateCompatibilityDate(extras.compatibilityDate); - validateCompatibilityFlags( - extras.compatibilityFlags, - compatibilityDate ?? DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE - ); + validateCompatibilityFlags(extras.compatibilityFlags); const vars = normalizeVars(extras.vars); // Null-proto: any path slipping past validateModulePath can't take // the __proto__ setter path on assignment. diff --git a/control/d1-runtime-client.js b/control/d1-runtime-client.js index 78b7459..5869287 100644 --- a/control/d1-runtime-client.js +++ b/control/d1-runtime-client.js @@ -263,9 +263,7 @@ function d1RuntimeFailureExtra(extra) { /** @param {unknown} value @param {string} fallback */ function d1RuntimeLogToken(value, fallback) { - return typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(value) - ? value - : fallback; + return typeof value === "string" && value ? value : fallback; } /** diff --git a/control/routing.js b/control/routing.js index 8f4240e..ea16b37 100644 --- a/control/routing.js +++ b/control/routing.js @@ -773,13 +773,15 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) const affectedHosts = new Set(); for (const r of routes) affectedHosts.add(r.host); await watchKeys(iso, [...affectedHosts].map((h) => patternsKey(h))); - await readHostStateAndAssertRouteConflicts( - iso, - ns, - workerName, - routes, - affectedHosts - ); + const hosts = [...affectedHosts]; + const patternStates = hosts.length > 0 + ? await iso.hGetAllMany(hosts.map((host) => patternsKey(host))) + : []; + for (let index = 0; index < hosts.length; index += 1) { + for (const [slot, raw] of Object.entries(patternStates[index])) { + requirePatternProjection(raw, hosts[index], slot); + } + } // Scheduler builds x-worker-id from cron __meta__.version. const cronKey = cronWorkerKey(ns, workerName); diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index c117df0..1baf83a 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -64,19 +64,6 @@ const MAX_INVOKE_ENVELOPE_BYTES = 2 * 1024 * 1024; const MAX_REQUEST_HEADER_COUNT = 128; const MAX_REQUEST_HEADER_BYTES = 64 * 1024; export const DO_INVOKE_CONTENT_TYPE = "application/vnd.wdl.do-invoke"; -const IntrinsicArray = Array; -const IntrinsicNumber = Number; -const IntrinsicObject = Object; -const IntrinsicWeakSet = WeakSet; -const intrinsicReflectApply = Reflect.apply; -const intrinsicArrayIsArray = Array.isArray; -const intrinsicNumberIsFinite = Number.isFinite; -const intrinsicObjectEntries = Object.entries; -const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; -const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; -const intrinsicWeakSetAdd = WeakSet.prototype.add; -const intrinsicWeakSetDelete = WeakSet.prototype.delete; -const intrinsicWeakSetHas = WeakSet.prototype.has; const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); const ALARM_INTERNAL_URL = "https://do.internal/__wdl_alarm"; @@ -245,47 +232,35 @@ function requireJsonSerializedSize(value, field, maxBytes) { * @param {WeakSet} [seen] * @returns {string | null} */ -function jsonDataError(value, field, seen = new IntrinsicWeakSet()) { +function jsonDataError(value, field, seen = new WeakSet()) { if (value === null) return null; const type = typeof value; if (type === "string" || type === "boolean") return null; - if (type === "number") { - return intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [value]) - ? null - : `${field} must be a finite number`; - } + if (type === "number") return Number.isFinite(value) ? null : `${field} must be a finite number`; if (type === "bigint" || type === "function" || type === "symbol" || type === "undefined") { return `${field} must be JSON data`; } if (type !== "object") return `${field} must be JSON data`; const objectValue = /** @type {Record | unknown[]} */ (value); - if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { - return `${field} must not be circular`; - } - intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); - if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { - const arrayValue = /** @type {unknown[]} */ (objectValue); - for (let i = 0; i < arrayValue.length; i++) { - if (!(i in arrayValue)) return `${field} must not be sparse`; - const error = jsonDataError(arrayValue[i], `${field}[${i}]`, seen); + if (seen.has(objectValue)) return `${field} must not be circular`; + seen.add(objectValue); + if (Array.isArray(objectValue)) { + for (let i = 0; i < objectValue.length; i++) { + if (!(i in objectValue)) return `${field} must not be sparse`; + const error = jsonDataError(objectValue[i], `${field}[${i}]`, seen); if (error) return error; } - intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); + seen.delete(objectValue); return null; } - const proto = intrinsicReflectApply(intrinsicObjectGetPrototypeOf, IntrinsicObject, [objectValue]); - if (proto !== IntrinsicObject.prototype && proto !== null) return `${field} must be a plain JSON object`; - const symbols = intrinsicReflectApply(intrinsicObjectGetOwnPropertySymbols, IntrinsicObject, [objectValue]); - if (symbols.length > 0) return `${field} must not contain symbol keys`; - const entries = /** @type {[string, unknown][]} */ ( - intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) - ); - for (let i = 0; i < entries.length; i++) { - const entry = entries[i]; - const error = jsonDataError(entry[1], `${field}.${entry[0]}`, seen); + const proto = Object.getPrototypeOf(objectValue); + if (proto !== Object.prototype && proto !== null) return `${field} must be a plain JSON object`; + if (Object.getOwnPropertySymbols(objectValue).length > 0) return `${field} must not contain symbol keys`; + for (const [key, entry] of Object.entries(objectValue)) { + const error = jsonDataError(entry, `${field}.${key}`, seen); if (error) return error; } - intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); + seen.delete(objectValue); return null; } diff --git a/do-runtime/protocol/identity.js b/do-runtime/protocol/identity.js index 353b63d..02f7096 100644 --- a/do-runtime/protocol/identity.js +++ b/do-runtime/protocol/identity.js @@ -9,13 +9,10 @@ import { DoRuntimeError } from "do-runtime-protocol-errors"; import { fnv1a32Utf8 } from "shared-fnv1a32"; const utf8Encoder = new TextEncoder(); -const intrinsicReflectApply = Reflect.apply; -const intrinsicStringIsWellFormed = String.prototype.isWellFormed; /** @param {unknown} value */ export function isWellFormedUnicodeString(value) { - return typeof value === "string" && - intrinsicReflectApply(intrinsicStringIsWellFormed, value, []); + return typeof value === "string" && value.isWellFormed(); } /** @param {string} value */ diff --git a/docs/compatibility.md b/docs/compatibility.md index 8db661c..079b642 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -31,7 +31,7 @@ Each row separates four compatibility claims: |---|---|---|---|---|---| | ES module Workers and `fetch()` | Supported | Module evaluation, request dispatch, `Response`/`Request`, service binding JSRPC machinery. | Dynamic `workerLoader`, immutable version ids, wrapper-generated `env`, gateway routing, request logs, and public/private outbound separation. | An uncaught tenant `fetch()` exception maps to platform `502 runtime_error`; exception detail goes to structured logs/live tail, not the client body. | WDL does not emulate every compatibility-date behavior change; workerd is configured with the platform's enabled flags. | | WebSocket upgrade | Supported | WebSocket API and 101 response handling inside workerd. | Gateway `GatewayWsHolder` Durable Object holds public sockets and forwards to runtime/do-runtime so long-lived 101s do not live on ordinary gateway request IoContexts. | WDL optimizes for preserving long-lived gateway-held sockets, while Cloudflare can rely on its global edge session model. | Gateway rolling still drops physical client sockets; application-level resume is not implemented. | -| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01`; Control also validates that the value is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd, rejects upstream `$experimental` enable flags such as `experimental` / `unsafe_module`, rejects `legacy_error_serialization`, and rejects explicit `enhanced_error_serialization` once that behavior is implied by the effective date. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; tenant workers cannot opt into upstream experimental-only flags or disable WDL's required enhanced error serialization. | +| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01`; Control also validates that the value is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd, rejects upstream `$experimental` enable flags such as `experimental` / `unsafe_module`, and rejects `legacy_error_serialization`. Workerd owns validation of redundant or otherwise incompatible upstream flags at cold load. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; tenant workers cannot opt into upstream experimental-only flags or disable WDL's required enhanced error serialization. | | `nodejs_compat` | Partial | workerd-provided compatibility when the runtime service has the flag enabled. | CLI carries compatibility flags into metadata. | WDL exposes the workerd-enabled compatibility surface rather than a separate Node.js runtime. | This is not a full Node.js platform contract beyond workerd's enabled surface. | | Python Workers modules | Not supported | workerd has an experimental Python Workers path. | Control rejects `py` module manifests with `python_workers_unsupported`; runtime and do-runtime also fail closed for retained metadata containing `py` modules. | WDL keeps tenant bundles JavaScript/WebAssembly/data only and does not permit cold-load-time Pyodide bootstrap. | Python Workers and mixed JS/Python bundles are unsupported. | diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index b350476..a9d41ec 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -24,7 +24,7 @@ |---|---|---|---|---|---| | ES module Workers 和 `fetch()` | Supported | Module evaluation、request dispatch、`Response`/`Request`、service binding JSRPC 机制。 | Dynamic `workerLoader`、immutable version id、wrapper 生成 `env`、gateway routing、request logs、public/private outbound 隔离。 | Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error`；异常细节进入结构化日志/live tail，不进客户端 body。 | WDL 不模拟每个 compatibility date 的历史行为；workerd 按平台启用的 flags 运行。 | | WebSocket upgrade | Supported | workerd 内的 WebSocket API 和 101 response 处理。 | Gateway `GatewayWsHolder` Durable Object 承载公网 socket 并转发到 runtime/do-runtime，避免长生命周期 101 留在普通 gateway request IoContext。 | WDL 优先保留 gateway-held 长连接；Cloudflare 依赖自己的全球 edge session 模型。 | Gateway rolling 仍会断开物理 client socket；应用级 resume 未实现。 | -| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | Dynamic Worker 会拒绝早于 `2026-04-01` 的显式 `compatibility_date`；Control 还会校验该值是真实的 `YYYY-MM-DD` 日期，且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期；上游 `$experimental` enable flags（例如 `experimental` / `unsafe_module`）、`legacy_error_serialization`，以及 effective date 已隐式启用该行为时显式声明的 `enhanced_error_serialization` 会在 deploy 前被拒绝。 | WDL 把 compatibility metadata 当成部署期平台 metadata，不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为；tenant worker 不能 opt into 上游 experimental-only flags，也不能禁用 WDL 要求的 enhanced error serialization。 | +| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | Dynamic Worker 会拒绝早于 `2026-04-01` 的显式 `compatibility_date`；Control 还会校验该值是真实的 `YYYY-MM-DD` 日期，且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期，并拒绝上游 `$experimental` enable flags（例如 `experimental` / `unsafe_module`）及 `legacy_error_serialization`。冗余或其他不兼容的上游 flag 由 workerd 在 cold load 时校验。 | WDL 把 compatibility metadata 当成部署期平台 metadata，不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为；tenant worker 不能 opt into 上游 experimental-only flags，也不能禁用 WDL 要求的 enhanced error serialization。 | | `nodejs_compat` | Partial | runtime service 启用 flag 后由 workerd 提供的兼容 surface。 | CLI 把 compatibility flags 带入 metadata。 | WDL 暴露 workerd 已启用的兼容 surface，而不是单独 Node.js runtime。 | 不承诺超出 workerd 已启用 surface 的完整 Node.js 平台能力。 | | Python Workers modules | Not supported | workerd 有 experimental Python Workers 路径。 | Control 用 `python_workers_unsupported` 拒绝 `py` module manifest；runtime 和 do-runtime 对 retained metadata 中的 `py` module 也 fail closed。 | WDL 保持 tenant bundle 只包含 JavaScript/WebAssembly/data，不允许 cold-load 时触发 Pyodide bootstrap。 | 不支持 Python Workers 和 JS/Python 混合 bundle。 | diff --git a/docs/modules/d1.md b/docs/modules/d1.md index 28859ae..a8c6349 100644 --- a/docs/modules/d1.md +++ b/docs/modules/d1.md @@ -56,10 +56,10 @@ freezing of a binding to a physical `databaseId`. - Control D1 lifecycle APIs remain admin APIs and use the control `{ error, message, ...details }` contract. d1-runtime internal query/classified errors are a separate D1 protocol surface. When the outward Control status is 5xx, Control replaces the - backend message with `Internal error`, omits raw detail, and preserves only bounded - `upstreamCode`, `upstreamCategory`, `upstreamRetryable`, and `upstreamStatus` - classifier fields. Outward 4xx SQL and migration failures retain their existing D1 - diagnostic details. + backend message with `Internal error`, omits raw detail, and preserves only the + machine classifier fields `upstreamCode`, `upstreamCategory`, `upstreamRetryable`, + and `upstreamStatus`. Outward 4xx SQL and migration failures retain their existing + D1 diagnostic details. ## Redis / Storage Contracts @@ -189,7 +189,7 @@ router responses must not be scraped into this gauge. Control's D1 runtime failure events for database initialization, query execution, and migration operations include request and database identity plus bounded `upstream_status`, `upstream_code`, `upstream_category`, `upstream_retryable`, and, -when available, `upstream_cause_code` fields. Control does not copy raw backend messages +when available, `upstream_cause_code` machine fields. Control does not copy raw backend messages or response detail into these events. The supervisor emits `drain_timeout_near_owner_ttl` when configured drain timeout is at diff --git a/docs/modules/d1.zh.md b/docs/modules/d1.zh.md index a6bc4e4..ce6d111 100644 --- a/docs/modules/d1.zh.md +++ b/docs/modules/d1.zh.md @@ -29,7 +29,7 @@ workerd 给 WDL 提供了托管 SQLite-backed service 所需的 isolate 和 loca - Multi-statement `exec()` request 会在一个 actor-local SQLite transaction 中执行。如果后续 statement 失败，同一 request 中此前已执行的 statement 会回滚。 - D1 query/facade 失败使用 D1-specific payload 和 tenant `D1_ERROR` object，而不是 generic platform/admin HTTP envelope。D1 payload 会按需携带 `success:false`、`error`、`message`、`category`、`retryable`、`statementIndex` 和 `causeCode` 等字段。 - D1 error code 使用 D1 compatibility vocabulary，包括 `limit-exceeded`、`sql-error` 这类 `hyphen-case` code。除非 D1 protocol 本身变化，否则不要把它们改写成平台 `snake_case` vocabulary。 -- Control D1 lifecycle API 仍是 admin API，使用 control `{ error, message, ...details }` 合同。d1-runtime internal query/classified error 是独立的 D1 protocol surface。Control 对外状态为 5xx 时会把 backend message 替换为 `Internal error`、移除 raw detail，只保留有界的 `upstreamCode`、`upstreamCategory`、`upstreamRetryable` 和 `upstreamStatus` classifier fields；对外 4xx SQL 与 migration failure 继续保留既有 D1 diagnostic detail。 +- Control D1 lifecycle API 仍是 admin API，使用 control `{ error, message, ...details }` 合同。d1-runtime internal query/classified error 是独立的 D1 protocol surface。Control 对外状态为 5xx 时会把 backend message 替换为 `Internal error`、移除 raw detail，只保留 `upstreamCode`、`upstreamCategory`、`upstreamRetryable` 和 `upstreamStatus` 机器分类字段；对外 4xx SQL 与 migration failure 继续保留既有 D1 diagnostic detail。 ## Redis / Storage 合同 @@ -97,7 +97,7 @@ Key families： d1-runtime 记录 owner 变化、routing、query failure、drain/renew 和 test-hook 操作。Storage size metrics 是 owner-local observed state，不是权威 quota accounting。D1-compatible query response 仍可携带 `meta.size_after`；`wdl_d1_storage_size_bytes` 只来自当前 local owner 成功记录的 sample，并在 owner 丢失或释放时清除。不要把 forwarded router response scrape 进这个 gauge。 -Control 的 database initialization、query execution 和 migration operation runtime failure event 会包含 request/database identity，以及有界的 `upstream_status`、`upstream_code`、`upstream_category`、`upstream_retryable` 和可用时的 `upstream_cause_code` 字段；Control 不会把原始 backend message 或 response detail 复制进这些 event。 +Control 的 database initialization、query execution 和 migration operation runtime failure event 会包含 request/database identity，以及 `upstream_status`、`upstream_code`、`upstream_category`、`upstream_retryable` 和可用时的 `upstream_cause_code` machine fields；Control 不会把原始 backend message 或 response detail 复制进这些 event。 当配置的 drain timeout 大于等于 owner TTL 的一半时，supervisor 会输出 `drain_timeout_near_owner_ttl`。该 warning 携带 `drain_timeout_ms`、`owner_ttl_ms` 和 `renew_interval_ms`，方便 operator 判断 shutdown drain 是否过于接近 lease expiry。 diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index f49dfda..69d13a5 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -64,18 +64,20 @@ partial batch result; that is a result envelope, not a generic JSON error envelo Tenant-originated DO fetch bodies are capped at 1 MiB in the runtime facade. The facade rejects an oversized `Content-Length` before reading, and streamed bodies are read incrementally so the cap is enforced before buffering the full body. -DO RPC method names use the JavaScript identifier grammar and are capped at 256 ASCII -bytes by both the runtime client and do-runtime protocol reader. RPC arguments are +DO RPC method names use the JavaScript identifier grammar. The do-runtime protocol +reader caps them at 256 ASCII bytes. RPC arguments are structural JSON data capped at 1 MiB: finite numbers, strings, booleans, null, dense arrays, and plain objects are accepted. Serialization does not invoke `toJSON()` hooks; sparse arrays, circular structures, non-plain objects, and non-JSON values fail before dispatch. do-runtime invokes tenant alarm and RPC methods through private fetch dispatches intercepted by the generated wrapper. Those requests carry the outer request id so the -host facade establishes invocation-local context without adding platform metadata to -the tenant argument list. Nested DO fetch/connect requests discard tenant-supplied -`x-request-id` values and, when the ambient parent context is available, propagate its -sanitized request id. Request ids remain untrusted diagnostic metadata. +host facade can propagate it without adding platform metadata to the tenant argument +list. Persistent class instances use a small mutable diagnostic context, so concurrent +or re-entrant calls may observe another invocation's id. Nested DO fetch/connect +requests discard tenant-supplied `x-request-id` values and propagate the sanitized +context id when available. Request ids remain best-effort, untrusted diagnostic +metadata. DO invoke envelopes identify persisted bundles by canonical namespace, worker, version, and storage id. They do not accept inline worker source. Tenant-facing DO object names and ids must be well-formed Unicode strings. Lone UTF-16 diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index d0a542b..de51894 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -35,8 +35,8 @@ Storage cleanup endpoint 是 native facet storage cleanup 和 worker storage cle DO protocol error 使用 `{ error, message, details? }`。不同于 admin HTTP 的 flat additive error shape，DO protocol detail 会嵌套在 `details` 下，因为消费者是 runtime/DO client protocol，不是通用 admin JSON parser。未知 internal exception 仍会降级成安全的 `internal_error` / `Internal error` message。Storage delete-worker 在 partial batch result 时可能返回 HTTP 207 和 `{ ok:false, deleted, errors }`；这是 result envelope，不是 generic JSON error envelope。 Tenant-originated DO fetch body 在 runtime facade 中限制为 1 MiB。Facade 会在读取前拒绝超限的 `Content-Length`，streamed body 会增量读取，因此 limit 会在完整 buffering 前生效。 -DO RPC method name 使用 JavaScript identifier grammar，并由 runtime client 和 do-runtime protocol reader 同时限制为最多 256 ASCII bytes。RPC 参数是最多 1 MiB 的 structural JSON data：接受 finite number、string、boolean、null、dense array 和 plain object。序列化不会调用 `toJSON()` hook；sparse array、circular structure、non-plain object 和 non-JSON value 会在 dispatch 前失败。 -do-runtime 会通过 generated wrapper 截获的私有 fetch dispatch 调用 tenant alarm 和 RPC method；这些 request 携带外层 request id，使 host facade 建立 invocation-local context，且不会把平台 metadata 加入 tenant argument list。嵌套 DO fetch/connect request 会丢弃 tenant 提供的 `x-request-id`，并在 ambient parent context 可用时传播其净化后的 request id；request id 仍是不可信的诊断 metadata。 +DO RPC method name 使用 JavaScript identifier grammar，do-runtime protocol reader 将其限制为最多 256 ASCII bytes。RPC 参数是最多 1 MiB 的 structural JSON data：接受 finite number、string、boolean、null、dense array 和 plain object。序列化不会调用 `toJSON()` hook；sparse array、circular structure、non-plain object 和 non-JSON value 会在 dispatch 前失败。 +do-runtime 会通过 generated wrapper 截获的私有 fetch dispatch 调用 tenant alarm 和 RPC method；这些 request 携带外层 request id，使 host facade 可以传播该 id，且不会把平台 metadata 加入 tenant argument list。持久化 class instance 使用一个小型可变诊断 context，因此 concurrent 或 re-entrant call 可能观察到另一次 invocation 的 id。嵌套 DO fetch/connect request 会丢弃 tenant 提供的 `x-request-id`，并在 context id 可用时传播其净化值；request id 仍是 best-effort、不可信的诊断 metadata。 DO invoke envelope 通过 canonical namespace、worker、version 和 storage id 标识 persisted bundle，不接受 inline worker source。 Tenant-facing DO object name/id 必须是 well-formed Unicode string。`idFromName()` / `idFromString()` 会在 hash 或 dispatch 前拒绝 lone UTF-16 surrogate；do-runtime alarm ingress 和 Workflows revalidation 执行相同边界。 DO host id 最多 512 UTF-8 bytes，并使用不带前导零的 canonical `shardN` suffix。DO binding class name 使用 ASCII JavaScript class-name grammar，并在 deploy 时限制为最多 468 bytes，确保所有 shard suffix 都能满足 aggregate host-id 上限。 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index e60b3ef..0db3d18 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -145,14 +145,13 @@ Common rules: `request_id` on `request_complete`; Rust sidecars sanitize inbound ids and log them where request middleware owns completion. Control's Redis `PUBLISH` path logs the id locally but does not put it in the pub/sub payload. -- Loaded-worker host facades keep request ids in invocation-local ALS across native - Promise and `async` continuations. Private do-runtime fetch, alarm, and RPC dispatches - carry the id in `x-request-id`. Custom thenables are not a supported propagation - boundary; the wrapper does not inspect or replace tenant return values to extend it. - Ordinary nested handler calls without a new id inherit the current ALS context. - Tenant code can deliberately switch the isolate's ambient ALS frame, so this - propagation is best-effort diagnostic correlation rather than an authorization or - integrity boundary. +- Loaded-worker host facades receive request ids directly for ordinary handlers. + Persistent class entrypoints keep a small mutable diagnostic context until the + wrapped result settles; the wrapper inspects callable `then` values and may return a + normalized Promise while arranging cleanup. Concurrent or re-entrant class calls may + therefore observe another invocation's id. Private do-runtime fetch, alarm, and RPC + dispatches carry the available id in `x-request-id`. Propagation is best-effort + diagnostic correlation rather than an authorization or integrity boundary. - Logs use snake_case fields. Only `level=error` is emitted on stderr; debug/info/warn JSON log lines go to stdout so log routing stays identical across JS, Rust, and embedded workerd shims. diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index 2976de9..5c8e1bf 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -92,7 +92,7 @@ WDL 在 JS workerd tier 和 Rust 服务中使用同一套可观测性策略： 通用规则： - `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker、D1 和 Workflows 传播。缺失的 inbound id 会在入口生成；header adapter 只考虑第一个逗号分隔 token，包含 visible ASCII 以外字节、引号、反斜杠或超过 128 字节的 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id，并在 `request_complete` 日志中记录 `request_id`；Rust sidecar 会 sanitize inbound id，并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id，不把它放进 pub/sub payload。 -- Loaded-worker host facade 通过 invocation-local ALS 在 native Promise 和 `async` continuation 中保留 request id；do-runtime 的私有 fetch、alarm 和 RPC dispatch 会在 `x-request-id` 中携带该 id。Custom thenable 不属于受支持的传播边界；wrapper 不会为延长 context 而检查或替换 tenant 返回值。普通的无新 id 嵌套 handler 调用会继承当前 ALS context；tenant code 也可以主动切换 isolate 的 ambient ALS frame，因此这只是 best-effort 的诊断关联，不是授权或完整性边界。 +- 普通 loaded-worker handler 的 host facade 会直接获得 request id。持久化 class entrypoint 在 wrapped result settle 前使用一个小型可变诊断 context；wrapper 会检查 callable `then` 并可能返回 normalized Promise 以安排 cleanup，因此 concurrent 或 re-entrant class call 可能观察到另一次 invocation 的 id。do-runtime 的私有 fetch、alarm 和 RPC dispatch 会在 `x-request-id` 中携带可用 id。这只是 best-effort 的诊断关联，不是授权或完整性边界。 - 日志字段使用 snake_case。只有 `level=error` 写入 stderr；debug/info/warn JSON log line 都写入 stdout，这样 JS、Rust 和 embedded workerd shim 的日志路由保持一致。 - 内部运维日志，包括 system-worker cleanup 日志和防御性的 Redis callback warning，也使用同一套单行 JSON envelope：`ts`、`service`、`level`、`event`，再加 snake_case 字段。JS 服务使用 `shared/observability.js`；Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`；无法转成文本的 throwable 降级为 `Unknown error`，不能反向覆盖原始失败。不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 - 产品 API response body 默认使用 camelCase，除非 endpoint 明确记录不同 wire contract。 diff --git a/docs/modules/queues-cron.md b/docs/modules/queues-cron.md index 8198901..85a85c6 100644 --- a/docs/modules/queues-cron.md +++ b/docs/modules/queues-cron.md @@ -144,8 +144,7 @@ Queue dispatch is stream-driven rather than wall-clock driven: failures that cannot become valid by retrying the same bytes, such as queue-message decode failures or invalid queue dispatch bodies, go directly to DLQ without consuming the retry budget. Aggregate request-body-too-large responses are split - and retried with smaller batches first. Scheduler bounds each runtime response body - to 1 MiB before parsing it. + and retried with smaller batches first. 5. The delayed loop wakes from `queue-delayed-wake` and from wall-clock sleeps until the next due delayed member. Each due member first takes a `queue-delayed-claim:*` lease sized to `SCHEDULER_FIRE_TIMEOUT_MS + 5000ms`; the winner moves it back to the main diff --git a/docs/modules/queues-cron.zh.md b/docs/modules/queues-cron.zh.md index 0962a04..3cdcfb0 100644 --- a/docs/modules/queues-cron.zh.md +++ b/docs/modules/queues-cron.zh.md @@ -74,7 +74,7 @@ Queue dispatch 是 stream-driven，不是 wall-clock driven： 1. Producer 把 message envelope 写入 DB 1 stream；当 `delivery_delay` 或 retry delay 非零时，先写入 delayed ZSET。 2. Scheduler reconcile `queue:index:*` discovery set，为 live stream 创建固定的 `wdl-scheduler` consumer group，并维护一个内存中的 known delayed queue 集合。空 queue index 可以从权威 hash、stream 和 delayed ZSET 一次性 backfill；之后 projection 由 writer 维护，stale-index cleanup 由 reconcile 负责。 3. Consume loop 使用 `XREADGROUP` 读取 main stream，并按 `max_batch_size` 组 batch 投递，且 hard cap 为 `100`。每次 read 会用当前 active stream set 的 consumer batch-size snapshot 限制 `COUNT`，避免一次 poll 把超过当前 consumer 单批 dispatch 能力的 entries 放进 PEL。PEL reap 在 consumer 仍存在时使用同一 per-consumer cap；consumer 已消失的 orphan movement 仍可按 hard cap 分页。Consume 和 PEL reap 可以在 queue semaphore 下按 stream 并行 dispatch。每条 dispatch path 在向 runtime 发送 message 前都会重读该 stream 的权威 `queue-consumer` hash 并刷新内存 registry，因此 consumer promote 后，一旦 message 被选中，就不需要等下一次 reconcile tick 才使用新版本。当前模型里 `max_batch_timeout_ms` 不是 batching wait window。 -4. Runtime 返回 queue outcome envelope。Scheduler 解析 explicit `ack`、explicit `retry`、batch retry 和 implicit ack，然后用 Redis pipeline 执行 `XACK`/`XDEL`、delayed retry `ZADD`、DLQ append 或 immediate retry write。不会因为重试同一批 bytes 变好的平台失败，例如 queue message decode failure 或非法 queue dispatch body，会直接进 DLQ，不消耗 retry budget。聚合 request-body-too-large 会先拆成更小 batch 重试；scheduler 在解析前把每个 runtime response body 限制为 1 MiB。 +4. Runtime 返回 queue outcome envelope。Scheduler 解析 explicit `ack`、explicit `retry`、batch retry 和 implicit ack，然后用 Redis pipeline 执行 `XACK`/`XDEL`、delayed retry `ZADD`、DLQ append 或 immediate retry write。不会因为重试同一批 bytes 变好的平台失败，例如 queue message decode failure 或非法 queue dispatch body，会直接进 DLQ，不消耗 retry budget。聚合 request-body-too-large 会先拆成更小 batch 重试。 5. Delayed loop 由 `queue-delayed-wake` 和下一条 due member 的 wall-clock sleep 唤醒。每个到期 member 会先取得 `queue-delayed-claim:*` lease，TTL 为 `SCHEDULER_FIRE_TIMEOUT_MS + 5000ms`；抢到的副本把它移回 main stream，或在 consumer 已消失时移入 orphan stream。 6. Orphan / Pending-Entry cleanup 是诊断和保护机制。它防止 consumer 删除或 scheduler crash 路径静默丢消息，但 main queue stream 仍是 durable backlog，并且故意不 trim。Delayed ZSET 和 orphan stream-tail migration 受 `QUEUE_SWEEP_BATCH_SIZE` 分页控制，默认 `100`。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 92c921e..c4fabdb 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -193,10 +193,10 @@ Runtime must treat Redis bundle metadata as control-authored, but still revalida reserved runtime entrypoint and binding names when materializing older stored metadata. It also fails closed if older metadata has a `compatibilityDate` before `2026-04-01`, contains Python module entries or upstream experimental compatibility flags, disables -WDL's required enhanced error serialization, or explicitly enables that behavior after -it became the workerd default. The date floor and enhanced-serialization requirement -are WDL forward-only policy; the remaining checks prevent unsupported metadata from -reaching the current stock workerd binary as an opaque cold-load failure. +WDL's required enhanced error serialization, or violates another WDL-owned metadata +contract. The date floor and enhanced-serialization requirement are WDL forward-only +policy. Workerd remains responsible for rejecting redundant or otherwise incompatible +upstream flags during cold load. ## Ownership / Concurrency / Failure Semantics @@ -210,25 +210,17 @@ reaching the current stock workerd binary as an opaque cold-load failure. internal Fetchers are injected. Its host-wrapper runtime evaluates before tenant modules and captures the intrinsics used to decide handler or env wrapping, so tenant top-level prototype mutation cannot bypass the env boundary. -- Host facade request ids use an invocation-local `AsyncLocalStorage` store. Runtime - ensures that the loaded worker has ALS support: an existing `nodejs_compat` setup is - left unchanged; otherwise a standalone `no_nodejs_als` is replaced by the narrow - `nodejs_als` flag. Concurrent calls on a persistent Durable Object instance therefore - cannot overwrite one shared context. The wrapper does not inspect, mutate, or replace - tenant return values. Native Promise and `async` continuations retain context; custom - thenables are not a supported request-id propagation boundary. Only the outermost - generated-wrapper invocation establishes the ALS store, including a store whose - request id is null, so ordinary nested wrapper calls inherit it instead of replacing - it with a second id. Tenant code shares the isolate's ambient ALS machinery and can - deliberately bind or restore another ambient frame, so loaded-worker propagation is - best-effort against tenant interference, not a security boundary. Request ids are - untrusted diagnostic metadata and must never authorize, fence, or deduplicate work. - Request-id syntax is owned by the injected canonical request-id module rather than - copied into the wrapper runtime. +- Generated wrappers pass request ids directly to per-request facade objects. Persistent + class instances keep a small mutable diagnostic context that is refreshed when a + wrapped handler starts; concurrent or deliberately re-entrant calls may therefore + observe another invocation's id. Propagation is best-effort observability, not a + security or correctness boundary, and Runtime does not rewrite tenant compatibility + flags to support it. Request ids must never authorize, fence, or deduplicate work. + Request-id syntax remains owned by the injected canonical request-id module. - Request context wrappers swap facade objects into env and propagate request id where that event class can carry it. do-runtime alarm and RPC calls enter through private - fetch dispatches carrying the outer request id, so the generated wrapper establishes - invocation-local context without adding platform metadata to tenant method arguments. + fetch dispatches carrying the outer request id, without adding platform metadata to + tenant method arguments. - An uncaught tenant `fetch()` exception maps to a platform `502 runtime_error` response with the request id. Exception details are emitted to structured logs/live tail and are not copied into the client body. Tail formatting cannot replace the diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index c97ebe5..7c982db 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -75,7 +75,7 @@ Runtime 通过 `redis-proxy` 从 DB 0 读取不可变 bundle 和 metadata。Data - D1 和 DO binding 调用专门 runtime service。 - R2/ASSETS 使用 S3-compatible object storage。 -Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。旧 metadata 如果包含早于 `2026-04-01` 的 `compatibilityDate`、Python module entry、上游 experimental compatibility flag、禁用 WDL 要求的 enhanced error serialization，或在该行为已成为 workerd 默认值后仍显式启用对应 flag，也会 fail closed。日期下限与 enhanced serialization 要求是 WDL 的 forward-only policy；其余检查避免 unsupported metadata 直到当前 stock workerd cold-load 才以 opaque error 失败。 +Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。旧 metadata 如果包含早于 `2026-04-01` 的 `compatibilityDate`、Python module entry、上游 experimental compatibility flag、禁用 WDL 要求的 enhanced error serialization，或违反其他由 WDL 拥有的 metadata contract，也会 fail closed。日期下限与 enhanced serialization 要求是 WDL 的 forward-only policy；冗余或其他不兼容的上游 flag 仍由 workerd 在 cold load 时拒绝。 ## Ownership / 并发 / 失败语义 @@ -83,8 +83,8 @@ Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materializ - Service-binding cold load 会记录 loaded version，但不 evict sibling，因为 service binding 可能有意指向 frozen historical version。 - Internal active-version scheduled/queue dispatch 会 opt into sibling eviction；frozen workflow dispatch 不会。 - 只要注入 privileged internal Fetcher，wrapper generation 就会避免 raw env 暴露给未包装 entrypoint。Host-wrapper runtime 会在 tenant module 前执行，并捕获参与 handler/env wrapping 决策的 intrinsic，因此 tenant 顶层 prototype mutation 不能绕过 env 边界。 -- Host facade request id 使用 invocation-local `AsyncLocalStorage` store。已有 `nodejs_compat` 配置会保持不变；否则 Runtime 会把 standalone `no_nodejs_als` 替换为范围更窄的 `nodejs_als` flag。持久 Durable Object 实例上的并发调用不会再覆盖同一个共享 context，wrapper 也不会检查、修改或替换 tenant 返回值。Native Promise 和 `async` continuation 会保留 context；custom thenable 不属于受支持的 request-id propagation boundary。只有最外层 generated-wrapper invocation 会建立 ALS store，即使该 store 的 request id 为 null，因此普通嵌套 wrapper 调用会继承它而不是写入第二个 id。Tenant code 与 host 共用 isolate 的 ambient ALS 机制，可以主动 bind 或恢复另一个 ambient frame，因此 loaded-worker propagation 面对 tenant 干预时只是 best-effort，不是安全边界。Request id 是不可信的诊断 metadata，不能用于授权、fence 或去重。Request-id 语法由 injected canonical request-id module 拥有，不在 wrapper runtime 内复制。 -- Request context wrapper 会把 facade object 换进 env，并在事件类型允许时传播 request id。do-runtime alarm 和 RPC 通过携带外层 request id 的私有 fetch dispatch 进入，使 generated wrapper 建立 invocation-local context，而不会把平台 metadata 加入 tenant method argument。 +- Generated wrapper 会把 request id 直接传给每次请求创建的 facade。持久 class instance 使用一个小型 mutable diagnostic context，并在 wrapped handler 开始时刷新；并发或有意重入的调用因此可能观察到另一次 invocation 的 id。该传播只属于 best-effort observability，不是安全或正确性边界，Runtime 也不会为它重写 tenant compatibility flags。Request id 不能用于授权、fence 或去重；其语法仍由 injected canonical request-id module 拥有。 +- Request context wrapper 会把 facade object 换进 env，并在事件类型允许时传播 request id。do-runtime alarm 和 RPC 通过携带外层 request id 的私有 fetch dispatch 进入，不会把平台 metadata 加入 tenant method argument。 - Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error` response，并带 request id。异常细节输出到结构化日志/live tail，不复制进客户端 body；throwable 自身无法转成字符串时，tail formatting 也不能反向覆盖原始异常。 - Internal scheduled、queue 和 workflow dispatch route 使用 result envelope 表达 handler outcome。Tenant handler error 是 scheduler/workflow 协议中的 outcome state，不是 generic platform transport error。 - Runtime 没有 route-cache invalidation protocol。`workerLoader` cache key 是不可变 worker id，因此 promote 后的新 version 是新 key，会自然 cold-load。 diff --git a/docs/modules/workflows.md b/docs/modules/workflows.md index 2d07fc1..e7c03ce 100644 --- a/docs/modules/workflows.md +++ b/docs/modules/workflows.md @@ -119,7 +119,7 @@ Key families: | `wf:by-worker::` | Set | workflows | Instance discovery by worker. | Used by list/delete checks; entries are removed by retention/delete cleanup. | | `wf:by-workflow:::` | ZSET | workflows | Per-workflow instance list index ordered for bounded pagination. | Retention/delete cleanup removes the sorted-set member. | | `wf:by-version:::` | Set | workflows | Frozen-version referrer index. | Blocks version delete while live instances reference the version. | -| `wf:pending-version:::` | ZSET | workflows | Short-lived restart target-version blockers, scored by expiry time. | Version-delete checks active members; restart renews only a live member and atomically validates it before creating the durable `wf:by-version` referrer. Members expire after 30 seconds, and the ZSET has a refreshed 60-second key TTL for physical cleanup. | +| `wf:pending-version:::` | ZSET | workflows | Short-lived restart target-version blockers, scored by expiry time. | Version-delete checks active members; restart atomically validates its marker before creating the durable `wf:by-version` referrer. Members expire after 30 seconds, and the ZSET has a 60-second key TTL for physical cleanup. | | `wf:retention` | ZSET | workflows | Terminal retention due index. | Retention tick deletes expired terminal instances. | | `wf:internal:do-alarm:{}:state` | Hash | workflows | Authoritative backend job state for one Durable Object SQLite alarm row. | Successful delivery, retry exhaustion, explicit delete, and worker cleanup remove the job. | | `wf:internal:do-alarm:due:` | ZSET | workflows | DO alarm due index. Score is due timestamp in milliseconds. | Tick promotion moves eligible jobs to ready. | diff --git a/docs/modules/workflows.zh.md b/docs/modules/workflows.zh.md index 70418c7..53de3b8 100644 --- a/docs/modules/workflows.zh.md +++ b/docs/modules/workflows.zh.md @@ -70,7 +70,7 @@ Key families： | `wf:by-worker::` | Set | workflows | 按 worker 发现 instance 的索引。 | list/delete check 使用；retention/delete cleanup 移除 entry。 | | `wf:by-workflow:::` | ZSET | workflows | 按 workflow key 分页列 instance 的有序索引。 | retention/delete cleanup 删除 sorted-set member。 | | `wf:by-version:::` | Set | workflows | frozen-version referrer index。 | live instance 仍引用该 version 时阻止 version delete。 | -| `wf:pending-version:::` | ZSET | workflows | 按过期时间计分的短期 restart target-version blocker。 | Version-delete 检查 active member；restart 只续租未过期 member，并在创建持久 `wf:by-version` referrer 前原子复核。Member 30 秒后过期，ZSET key 使用随写入刷新的 60 秒 TTL 做物理回收。 | +| `wf:pending-version:::` | ZSET | workflows | 按过期时间计分的短期 restart target-version blocker。 | Version-delete 检查 active member；restart 在创建持久 `wf:by-version` referrer 前原子复核自己的 marker。Member 30 秒后过期，ZSET key 使用 60 秒 TTL 做物理回收。 | | `wf:retention` | ZSET | workflows | terminal retention due index。 | Retention tick 删除过期 terminal instance。 | | `wf:internal:do-alarm:{}:state` | Hash | workflows | 单个 Durable Object SQLite alarm row 的 backend job 权威状态。 | 成功 delivery、retry 耗尽、显式 delete 和 worker cleanup 会移除 job。 | | `wf:internal:do-alarm:due:` | ZSET | workflows | DO alarm due index。score 是 due timestamp milliseconds。 | Tick promotion 把到期 job 移到 ready。 | diff --git a/docs/redis-key-layout.md b/docs/redis-key-layout.md index 8313e4d..31e72a5 100644 --- a/docs/redis-key-layout.md +++ b/docs/redis-key-layout.md @@ -166,10 +166,10 @@ Cross-cutting constraints: fairness cursor. Control owns only DB 0 `wf:defs:*`; other tiers must not write DB 2 directly. - `wf:pending-version:::` is a Workflows-owned, 30-second restart - blocker. Version-delete checks it with `wf:by-version`; renewals cannot resurrect an - expired member, and the successful-restart DB 2 script revalidates the lease before - replacing it with the durable version referrer. The ZSET key has a refreshed - 60-second TTL so abandoned marker keys are physically reclaimed. + blocker. Version-delete checks it with `wf:by-version`, and the successful-restart + DB 2 script atomically revalidates the initial marker before replacing it with the + durable version referrer. The ZSET key has a refreshed 60-second TTL so abandoned + marker keys are physically reclaimed. - Workflows also owns internal DB 2 `wf:internal:do-alarm:*` jobs for Durable Object alarm backend scheduling. do-runtime writes alarms through the workflows HTTP API instead of writing those keys directly. `wf:internal:do-alarm:ready:cursor` is the diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index 2efe666..00e4678 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -97,5 +97,5 @@ Routes、crons、queue consumers、bindings、vars、exports、workflow definiti - Queue main stream 不做 trim，因为 at-least-once delivery 是合同。DLQ、orphan、log-tail 这类诊断 stream 可以使用有界 approximate trim。 - Secret hash value 在 steady state 下是 `WDL-ENC:` envelope。`/runtime/load` 没有 plaintext fallback。 - Workflows 拥有 DB 2 instance state。`wf:ready:cursor` 是内部 ready-shard 公平性 cursor。Control 只拥有 DB 0 的 `wf:defs:*`；其他 tier 不应直接写 DB 2。 -- `wf:pending-version:::` 是 Workflows-owned、30 秒的 restart blocker。Version-delete 会将它与 `wf:by-version` 一起检查；续租不能复活已过期 member，restart 成功的 DB 2 script 会在创建持久 version referrer 前再次验证 lease。ZSET key 使用随写入刷新的 60 秒 TTL，确保遗留 marker key 会被物理回收。 +- `wf:pending-version:::` 是 Workflows-owned、30 秒的 restart blocker。Version-delete 会将它与 `wf:by-version` 一起检查；restart 成功的 DB 2 script 会在创建持久 version referrer 前原子复验初始 marker。ZSET key 使用随写入刷新的 60 秒 TTL，确保遗留 marker key 会被物理回收。 - Workflows 还拥有 DB 2 中的 internal `wf:internal:do-alarm:*` jobs，用于 Durable Object alarm backend scheduling。do-runtime 通过 workflows HTTP API 写 alarm，而不是直接写这些 key。`wf:internal:do-alarm:ready:cursor` 是内部 ready-shard 公平性 cursor，不是租户状态。 diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index 1a74d10..d400b8c 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -8,7 +8,6 @@ export const MAX_DO_REQUEST_BODY_BYTES = 1024 * 1024; export const MAX_DO_INVOKE_ENVELOPE_BYTES = 2 * 1024 * 1024; export const MAX_DO_REQUEST_HEADER_COUNT = 128; export const MAX_DO_REQUEST_HEADER_BYTES = 64 * 1024; -const MAX_DO_RPC_METHOD_BYTES = 256; export const DO_ACCEPT_OWNER_HINT_HEADER = "x-wdl-do-accept-owner-hint"; export const DO_OWNER_HINT_CONTROL_HEADER = "x-wdl-do-owner-hint"; export const DO_OWNERSHIP_ERROR_CONTROL_HEADER = "x-wdl-do-ownership-error"; @@ -68,9 +67,7 @@ const IntrinsicResponse = Response; const IntrinsicUint8Array = Uint8Array; const IntrinsicWeakSet = WeakSet; const intrinsicReflectApply = Reflect.apply; -const intrinsicArrayForEach = Array.prototype.forEach; const intrinsicArrayIsArray = Array.isArray; -const intrinsicArrayPush = Array.prototype.push; const intrinsicDataViewSetUint32 = DataView.prototype.setUint32; const intrinsicHeadersAppend = Headers.prototype.append; const intrinsicHeadersDelete = Headers.prototype.delete; @@ -85,14 +82,10 @@ const intrinsicObjectEntries = Object.entries; const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; const intrinsicObjectSetPrototypeOf = Object.setPrototypeOf; -const intrinsicPromiseThen = Promise.prototype.then; const intrinsicResponseJson = Response.json; -const intrinsicReadableStreamCancel = ReadableStream.prototype.cancel; const intrinsicReadableStreamGetReader = ReadableStream.prototype.getReader; -const intrinsicReadableStreamReaderCancel = ReadableStreamDefaultReader.prototype.cancel; const intrinsicReadableStreamReaderRead = ReadableStreamDefaultReader.prototype.read; const intrinsicReadableStreamReaderReleaseLock = ReadableStreamDefaultReader.prototype.releaseLock; -const intrinsicRegExpTest = RegExp.prototype.test; const intrinsicSetHas = Set.prototype.has; const intrinsicStringToLowerCase = String.prototype.toLowerCase; const intrinsicStringToUpperCase = String.prototype.toUpperCase; @@ -193,16 +186,6 @@ export function staleDoOwnerHintResponse(response) { return code !== null && setHas(OWNER_HINT_STALE_CODES, code); } -/** @template T @param {T[]} values @param {(value: T) => void} callback */ -function forEachArray(values, callback) { - intrinsicReflectApply(intrinsicArrayForEach, values, [callback]); -} - -/** @template T @param {T[]} values @param {T} value */ -function pushArray(values, value) { - intrinsicReflectApply(intrinsicArrayPush, values, [value]); -} - /** @param {Uint8Array} target @param {Uint8Array} source @param {number} offset */ function setBytes(target, source, offset) { intrinsicReflectApply(intrinsicUint8ArraySet, target, [source, offset]); @@ -255,7 +238,9 @@ function headerEntries(headers) { const out = []; intrinsicReflectApply(intrinsicHeadersForEach, headers, [ /** @param {string} value @param {string} name */ - (value, name) => pushArray(out, [name, value]), + (value, name) => { + out[out.length] = [name, value]; + }, ]); return out; } @@ -275,11 +260,6 @@ function stringToUpperCase(value) { return intrinsicReflectApply(intrinsicStringToUpperCase, value, []); } -/** @param {RegExp} regexp @param {string} value */ -function regexpTest(regexp, value) { - return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); -} - /** @param {Request} request */ function requestHeaders(request) { return intrinsicReflectApply(intrinsicRequestHeadersGet, request, []); @@ -320,8 +300,7 @@ function releaseStreamReader(reader) { /** @param {ReadableStreamDefaultReader} reader */ function cancelStreamReader(reader) { try { - const promise = intrinsicReflectApply(intrinsicReadableStreamReaderCancel, reader, []); - intrinsicReflectApply(intrinsicPromiseThen, promise, [undefined, () => {}]); + reader.cancel().catch(() => {}); } catch { // Cancellation is best-effort after the bounded reader has already rejected. } @@ -334,8 +313,7 @@ function cancelResponseBody(response) { try { const body = responseBody(response); if (!body) return; - const promise = intrinsicReflectApply(intrinsicReadableStreamCancel, body, []); - intrinsicReflectApply(intrinsicPromiseThen, promise, [undefined, () => {}]); + body.cancel().catch(() => {}); } catch { // Best-effort cleanup only; the replacement response owns behavior. } @@ -384,38 +362,7 @@ function numberValue(value) { /** @param {unknown} value */ function stringifyJson(value) { - return intrinsicReflectApply(intrinsicJsonStringify, IntrinsicJSON, [cloneJsonData(value)]); -} - -/** @param {unknown} value @param {WeakSet} [seen] */ -function cloneJsonData(value, seen = new IntrinsicWeakSet()) { - if (value === null || typeof value !== "object") return value; - const objectValue = /** @type {object} */ (value); - if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { - throw new TypeError("Cannot serialize circular JSON data"); - } - intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); - try { - if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { - const source = /** @type {unknown[]} */ (objectValue); - const out = new IntrinsicArray(source.length); - intrinsicReflectApply(intrinsicObjectSetPrototypeOf, IntrinsicObject, [out, null]); - for (let i = 0; i < source.length; i++) out[i] = cloneJsonData(source[i], seen); - return out; - } - const out = /** @type {Record} */ ( - intrinsicReflectApply(intrinsicObjectCreate, IntrinsicObject, [null]) - ); - const entries = /** @type {[string, unknown][]} */ ( - intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) - ); - forEachArray(entries, (entry) => { - out[entry[0]] = cloneJsonData(entry[1], seen); - }); - return out; - } finally { - intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); - } + return intrinsicReflectApply(intrinsicJsonStringify, IntrinsicJSON, [value]); } /** @@ -475,7 +422,7 @@ async function readRequestBodyBytes(request) { cancelStreamReader(reader); throw new TypeError(`Durable Object fetch body exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } - pushArray(chunks, value); + chunks[chunks.length] = value; } } finally { releaseStreamReader(reader); @@ -483,10 +430,11 @@ async function readRequestBodyBytes(request) { const body = new IntrinsicUint8Array(total); let offset = 0; - forEachArray(chunks, (chunk) => { + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; setBytes(body, chunk, offset); offset += byteArrayLength(chunk); - }); + } return body; } @@ -553,8 +501,7 @@ function cloneJsonRpcData(value, field, seen = new IntrinsicWeakSet()) { export function assertRpcMethod(method) { if ( typeof method !== "string" || - byteLength(method) > MAX_DO_RPC_METHOD_BYTES || - !regexpTest(/^[A-Za-z_$][A-Za-z0-9_$]*$/, method) + !/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(method) ) { throw new TypeError("rpc.method is not valid"); } @@ -579,16 +526,23 @@ export function rpcInvokeBody(props, objectName, method, args) { if (byteLength(stringifyJson(stableArgs)) > MAX_DO_REQUEST_BODY_BYTES) { throw new TypeError(`rpc.args exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } - return encodeDoInvokeEnvelope({ - ns: props.ns, - worker: props.worker, - version: props.version, - doStorageId: props.doStorageId, - className: props.className, - objectName, - kind: "rpc", - rpc: { method, args: stableArgs }, - }); + const rpc = /** @type {Record} */ ( + intrinsicReflectApply(intrinsicObjectCreate, IntrinsicObject, [null]) + ); + rpc.method = method; + rpc.args = stableArgs; + const metadata = /** @type {Record} */ ( + intrinsicReflectApply(intrinsicObjectCreate, IntrinsicObject, [null]) + ); + metadata.ns = props.ns; + metadata.worker = props.worker; + metadata.version = props.version; + metadata.doStorageId = props.doStorageId; + metadata.className = props.className; + metadata.objectName = objectName; + metadata.kind = "rpc"; + metadata.rpc = rpc; + return encodeDoInvokeEnvelope(metadata); } /** @@ -612,18 +566,19 @@ function binaryInvokeHeaders(requestId) { */ export async function fetchInvokeInit(props, objectName, request, requestId) { const { spec, bodyBytes } = await requestSpec(request, requestId); + const metadata = /** @type {Record} */ (cloneJsonRpcData({ + ns: props.ns, + worker: props.worker, + version: props.version, + doStorageId: props.doStorageId, + className: props.className, + objectName, + request: spec, + }, "invoke")); return { method: "POST", headers: binaryInvokeHeaders(requestId), - body: /** @type {BodyInit} */ (encodeDoInvokeEnvelope({ - ns: props.ns, - worker: props.worker, - version: props.version, - doStorageId: props.doStorageId, - className: props.className, - objectName, - request: spec, - }, bodyBytes)), + body: /** @type {BodyInit} */ (encodeDoInvokeEnvelope(metadata, bodyBytes)), }; } @@ -671,9 +626,9 @@ export function retryableOwnerRaceResponse(response) { export async function requestSpec(request, requestId) { const forwarded = new IntrinsicRequest(request); const forwardedHeaders = requestHeaders(forwarded); - forEachArray(DO_FETCH_STRIP_HEADERS, (name) => { - headerDelete(forwardedHeaders, name); - }); + for (let i = 0; i < DO_FETCH_STRIP_HEADERS.length; i++) { + headerDelete(forwardedHeaders, DO_FETCH_STRIP_HEADERS[i]); + } headerDelete(forwardedHeaders, "x-request-id"); const canonicalRequestId = sanitizeRequestId(requestId); if (canonicalRequestId) headerSet(forwardedHeaders, "x-request-id", canonicalRequestId); @@ -699,14 +654,15 @@ function enforceRequestHeadersBudget(headers) { throw new TypeError(`Durable Object fetch headers exceed ${MAX_DO_REQUEST_HEADER_COUNT} entries`); } let total = 0; - forEachArray(headers, (entry) => { + for (let i = 0; i < headers.length; i++) { + const entry = headers[i]; const name = entry[0]; const value = entry[1]; total += byteLength(name) + byteLength(value); if (total > MAX_DO_REQUEST_HEADER_BYTES) { throw new TypeError(`Durable Object fetch headers exceed ${MAX_DO_REQUEST_HEADER_BYTES} bytes`); } - }); + } } /** @param {Request} request */ @@ -728,7 +684,9 @@ export function replayOwnerUnavailableForFetch(request) { */ export function connectHeaders(props, objectName, request, requestId) { const headers = copyHeaders(requestHeaders(request)); - forEachArray(DO_CONNECT_STRIP_HEADERS, (name) => headerDelete(headers, name)); + for (let i = 0; i < DO_CONNECT_STRIP_HEADERS.length; i++) { + headerDelete(headers, DO_CONNECT_STRIP_HEADERS[i]); + } headerSet(headers, DO_CONNECT_HEADERS.ns, props.ns); headerSet(headers, DO_CONNECT_HEADERS.worker, props.worker); headerSet(headers, DO_CONNECT_HEADERS.version, props.version); @@ -937,9 +895,9 @@ function withoutOwnerHintOptIn(init) { /** @param {Response} response */ function stripOwnerHintHeaders(response) { const headers = copyHeaders(responseHeaders(response)); - forEachArray(DO_OWNER_HINT_STRIP_HEADERS, (name) => { - headerDelete(headers, name); - }); + for (let i = 0; i < DO_OWNER_HINT_STRIP_HEADERS.length; i++) { + headerDelete(headers, DO_OWNER_HINT_STRIP_HEADERS[i]); + } const status = responseStatus(response); const init = /** @type {ResponseInit & { webSocket?: WebSocket }} */ ({ status, diff --git a/runtime/dispatch/workflow-step.js b/runtime/dispatch/workflow-step.js index cd6561f..1f5ee51 100644 --- a/runtime/dispatch/workflow-step.js +++ b/runtime/dispatch/workflow-step.js @@ -30,42 +30,36 @@ import { * @typedef {import("runtime-dispatch-workflow-replay-cache").WorkflowReplayCache} WorkflowReplayCache */ -/** @param {Record} record @param {string} field */ -function readFailureField(record, field) { +/** @param {unknown} value @param {"name" | "message"} field */ +function readThrowableField(value, field) { + if (value === null || (typeof value !== "object" && typeof value !== "function")) { + return undefined; + } try { - return { readable: true, value: record[field] }; + return /** @type {Record} */ (value)[field]; } catch { - return { readable: false, value: undefined }; + return undefined; } } /** @param {unknown} err */ export function workflowError(err) { - const record = err !== null && (typeof err === "object" || typeof err === "function") - ? /** @type {Record} */ (err) - : null; - const nameField = record - ? readFailureField(record, "name") - : { readable: true, value: undefined }; - const messageField = record - ? readFailureField(record, "message") - : { readable: true, value: undefined }; - const name = typeof nameField.value === "string" && nameField.value - ? nameField.value - : "Error"; - let message = "Unknown error"; - if (messageField.readable && typeof messageField.value === "string") { - message = messageField.value; - } else if (!record && err == null) { - message = "Workflow run failed"; - } else if (!record) { + const rawName = readThrowableField(err, "name"); + const rawMessage = readThrowableField(err, "message"); + let message = "Workflow run failed"; + if (typeof rawMessage === "string") { + message = rawMessage; + } else if (err != null) { try { message = String(err); } catch { - // Keep the stable fallback. + // Some valid thrown values have no usable primitive conversion. } } - return { name, message }; + return { + name: typeof rawName === "string" && rawName ? rawName : "Error", + message, + }; } /** @@ -114,9 +108,7 @@ export function isWorkflowSuspended(err) { /** @param {unknown} err */ export function isWorkflowSuspensionSignal(err) { if (isWorkflowSuspended(err)) return true; - if (err === null || (typeof err !== "object" && typeof err !== "function")) return false; - const name = readFailureField(/** @type {Record} */ (err), "name"); - return name.readable && name.value === "WorkflowSuspended"; + return readThrowableField(err, "name") === "WorkflowSuspended"; } /** @param {unknown} value */ diff --git a/runtime/lib.js b/runtime/lib.js index 56f290c..6453968 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -116,7 +116,7 @@ function deepFreeze(obj) { // Loaded workers may declare an older compatibilityDate than the platform // workers. Keep enhanced error serialization as a floor only before the date -// where workerd made it the default; newer dates reject the explicit flag. +// where workerd made it the default; workerd owns later flag compatibility. /** @param {string} compatibilityDate */ function platformFloorCompatFlags(compatibilityDate) { const out = []; @@ -157,14 +157,6 @@ function mergeCompatFlags(userFlags, compatibilityDate) { `meta.compatibilityFlags contains unsupported flag ${JSON.stringify(f)}; WDL requires enhanced error serialization` ); } - if ( - f === ENHANCED_ERROR_SERIALIZATION_FLAG && - compatibilityDate >= ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE - ) { - throw new Error( - `meta.compatibilityFlags contains ${JSON.stringify(f)}, which became the workerd default on ${ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE} and must not be specified for compatibilityDate ${compatibilityDate}` - ); - } out.push(f); } for (const f of platformFloorCompatFlags(compatibilityDate)) { diff --git a/runtime/load/code-budget.js b/runtime/load/code-budget.js index b92f2b1..39a9c84 100644 --- a/runtime/load/code-budget.js +++ b/runtime/load/code-budget.js @@ -21,9 +21,6 @@ import { export const WORKER_LOADER_CODE_MAX_BYTES = 64 * 1024 * 1024; const D1_DATA_FIELD_MODULE_NAME = "_wdl-d1-data-field.js"; -const NODEJS_ALS_COMPATIBILITY_FLAG = "nodejs_als"; -const NODEJS_COMPATIBILITY_FLAG = "nodejs_compat"; -const NO_NODEJS_ALS_COMPATIBILITY_FLAG = "no_nodejs_als"; const utf8Decoder = new TextDecoder(); /** @@ -335,18 +332,6 @@ export function injectRuntimeModulesForHostBindings( for (const [name, source] of runtimeInjectedModuleSources(originalMain, meta, runtimeSources, plan)) { workerCode.modules[name] = source; } - if (plan.needsHostBindingWrapper) { - const flags = Array.isArray(workerCode.compatibilityFlags) - ? /** @type {string[]} */ (workerCode.compatibilityFlags) - : []; - if (!flags.includes(NODEJS_COMPATIBILITY_FLAG)) { - const normalized = flags.filter((flag) => flag !== NO_NODEJS_ALS_COMPATIBILITY_FLAG); - if (!normalized.includes(NODEJS_ALS_COMPATIBILITY_FLAG)) { - normalized.push(NODEJS_ALS_COMPATIBILITY_FLAG); - } - workerCode.compatibilityFlags = normalized; - } - } workerCode.mainModule = "_wdl-wrapper.js"; return workerCode; } diff --git a/runtime/load/wrapper-generate.js b/runtime/load/wrapper-generate.js index 0f4f2e3..f897559 100644 --- a/runtime/load/wrapper-generate.js +++ b/runtime/load/wrapper-generate.js @@ -12,25 +12,23 @@ const DEFAULT_EXPORT_CLASS_TEST_SOURCE = "/^\\s*class\\b/.test(source)"; export const HOST_BINDING_RUNTIME_MODULE_NAME = "_wdl-host-wrapper-runtime.js"; export const HOST_BINDING_RUNTIME_SOURCE = ` -import { AsyncLocalStorage } from "node:async_hooks"; import { sanitizeRequestId } from "./_wdl-request-id.js"; const IntrinsicObject = Object; +const IntrinsicPromise = Promise; const IntrinsicProxy = Proxy; const IntrinsicReflect = Reflect; const IntrinsicSymbol = Symbol; -const intrinsicAsyncLocalStorageGetStore = AsyncLocalStorage.prototype.getStore; -const intrinsicAsyncLocalStorageRun = AsyncLocalStorage.prototype.run; const intrinsicArrayForEach = Array.prototype.forEach; const intrinsicFunctionToString = Function.prototype.toString; -const intrinsicHeadersGet = Headers.prototype.get; const intrinsicObjectDefineProperty = Object.defineProperty; const intrinsicObjectEntries = Object.entries; const intrinsicObjectKeys = Object.keys; +const intrinsicPromiseResolve = Promise.resolve; +const intrinsicPromiseThen = Promise.prototype.then; const intrinsicReflectApply = Reflect.apply; const intrinsicReflectGet = Reflect.get; const intrinsicRegExpTest = RegExp.prototype.test; -const intrinsicRequestHeadersGetter = Object.getOwnPropertyDescriptor(Request.prototype, "headers").get; export function applyFunction(fn, receiver, args) { return intrinsicReflectApply(fn, receiver, args); @@ -76,34 +74,27 @@ export function regexpTest(regexp, value) { export function requestIdFromEventArg(arg) { if (!arg || typeof arg !== "object") return null; try { - const headers = intrinsicReflectApply(intrinsicRequestHeadersGetter, arg, []); - const requestId = intrinsicReflectApply(intrinsicHeadersGet, headers, ["x-request-id"]); - return sanitizeRequestId(requestId); + const headers = arg.headers; + return headers && typeof headers.get === "function" + ? sanitizeRequestId(headers.get("x-request-id")) + : null; } catch { return null; } } -export function createRequestContext() { - const requestContext = new AsyncLocalStorage(); - return { - currentRequestId() { - const store = intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); - const requestId = store?.requestId; - return typeof requestId === "string" && requestId ? requestId : null; +export function settleWithFinally(value, callback) { + const promise = intrinsicReflectApply(intrinsicPromiseResolve, IntrinsicPromise, [value]); + return intrinsicReflectApply(intrinsicPromiseThen, promise, [ + (result) => { + callback(); + return result; }, - runWithRequestContext(requestId, callback) { - const activeStore = intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); - if (activeStore !== undefined) { - return intrinsicReflectApply(callback, undefined, []); - } - const canonicalRequestId = sanitizeRequestId(requestId); - return intrinsicReflectApply(intrinsicAsyncLocalStorageRun, requestContext, [ - { requestId: canonicalRequestId }, - callback, - ]); + (error) => { + callback(); + throw error; }, - }; + ]); } `; @@ -170,8 +161,9 @@ export function generateHostBindingWrapperModule(userMainSpecifier, d1Bindings, const __WdlWrappedEntrypoint${index}__ = ({ [${JSON.stringify(name)}]: class extends __WdlUserModule__.${name} { constructor(ctx, env) { - super(ctx, wrapEnv(env)); - return wrapClassInstance(this); + const requestContext = createRequestContext(); + super(ctx, wrapEnv(env, requestContext)); + return wrapClassInstance(this, requestContext); } }, })[${JSON.stringify(name)}]; @@ -193,7 +185,7 @@ ${WDL_ABORT_SHIM_SOURCE} export class __WdlWorkflowNotify__ extends WorkerEntrypoint { fetch(request) { - return withRequestContext(request, () => notifyWorkflowCallback(request, wrapEnv(this.env))); + return notifyWorkflowCallback(request, wrapEnv(this.env, requestIdFromEventArg(request))); } } @@ -206,41 +198,62 @@ const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; const HOST_BINDINGS_WRAPPED = __WdlHostRuntime__.createPrivateSymbol("wdl.host-bindings-wrapped"); const INTERNAL_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; -const REQUEST_CONTEXT = __WdlHostRuntime__.createRequestContext(); function requestIdFromEventArg(arg) { return __WdlHostRuntime__.requestIdFromEventArg(arg); } -function requestIdOptions() { - return { requestIdProvider: REQUEST_CONTEXT.currentRequestId }; +function createRequestContext(requestId = null) { + return { requestId }; +} + +function requestIdOptions(requestIdOrContext) { + return requestIdOrContext && typeof requestIdOrContext === "object" + ? { requestIdProvider: () => requestIdOrContext.requestId } + : { requestId: requestIdOrContext }; } -function doOptions(backend, ownerNetwork) { - return { ...requestIdOptions(), backend, ownerNetwork }; +function doOptions(requestIdOrContext, backend, ownerNetwork) { + return { ...requestIdOptions(requestIdOrContext), backend, ownerNetwork }; } -function workflowOptions(backend) { - return { ...requestIdOptions(), backend }; +function workflowOptions(requestIdOrContext, backend) { + return { ...requestIdOptions(requestIdOrContext), backend }; } -function withRequestContext(arg, fn) { - return REQUEST_CONTEXT.runWithRequestContext(requestIdFromEventArg(arg), fn); +function withRequestContext(context, arg, fn) { + const previous = context.requestId; + const requestId = requestIdFromEventArg(arg); + if (requestId) context.requestId = requestId; + try { + const result = fn(); + if (result && typeof result.then === "function") { + return __WdlHostRuntime__.settleWithFinally(result, () => { + context.requestId = previous; + }); + } + context.requestId = previous; + return result; + } catch (err) { + context.requestId = previous; + throw err; + } } -function wrapClassInstance(instance) { +function wrapClassInstance(instance, requestContext) { return __WdlHostRuntime__.createProxy(instance, { get(target, prop) { const value = __WdlHostRuntime__.reflectGet(target, prop, target); if (typeof value !== "function") return value; return function(...args) { - return withRequestContext(args[0], () => __WdlHostRuntime__.applyFunction(value, target, args)); + return withRequestContext(requestContext, args[0], () => + __WdlHostRuntime__.applyFunction(value, target, args)); }; }, }); } -function wrapEnv(env) { +function wrapEnv(env, requestIdOrContext = null) { // Idempotence is a contract, not an optimization: WorkerEntrypoint methods // and default handlers may re-enter with an env already wrapped by this // module. A symbol marker cannot be forged by tenant vars/secrets. @@ -256,19 +269,19 @@ function wrapEnv(env) { if (__WdlHostRuntime__.regexpTest(INTERNAL_BINDING_RE, name)) delete out[name]; }); __WdlHostRuntime__.forEachArray(D1_BINDINGS, (name) => { - if (out[name] !== undefined) out[name] = new D1Database(out[name], requestIdOptions()); + if (out[name] !== undefined) out[name] = new D1Database(out[name], requestIdOptions(requestIdOrContext)); }); __WdlHostRuntime__.forEachArray(R2_BINDINGS, (name) => { - if (out[name] !== undefined) out[name] = new R2Bucket(out[name], requestIdOptions()); + if (out[name] !== undefined) out[name] = new R2Bucket(out[name], requestIdOptions(requestIdOrContext)); }); __WdlHostRuntime__.forEachArray(DO_BINDINGS, (name) => { if (out[name] !== undefined) { - out[name] = new DurableObjectNamespace(out[name], doOptions(doBackend, doOwnerNetwork)); + out[name] = new DurableObjectNamespace(out[name], doOptions(requestIdOrContext, doBackend, doOwnerNetwork)); } }); __WdlHostRuntime__.forEachObjectEntry(WORKFLOW_BINDINGS, (name, metadata) => { if (out[name] !== undefined) { - out[name] = new Workflow(out[name] || metadata, workflowOptions(workflowsBackend)); + out[name] = new Workflow(out[name] || metadata, workflowOptions(requestIdOrContext, workflowsBackend)); } }); __WdlHostRuntime__.defineProperty(out, HOST_BINDINGS_WRAPPED, { value: true }); @@ -321,7 +334,11 @@ async function notifyWorkflowCallback(request, env) { function wrapHandler(owner, fn) { return function(arg1, env, ctx) { - return withRequestContext(arg1, () => __WdlHostRuntime__.applyFunction(fn, owner, [arg1, wrapEnv(env), ctx])); + return __WdlHostRuntime__.applyFunction(fn, owner, [ + arg1, + wrapEnv(env, requestIdFromEventArg(arg1)), + ctx, + ]); }; } @@ -351,8 +368,9 @@ if (raw && typeof raw === "object") { if (__WdlHostRuntime__.regexpTest(/^\\s*class\\b/, source)) { wrappedDefault = class extends raw { constructor(ctx, env) { - super(ctx, wrapEnv(env)); - return wrapClassInstance(this); + const requestContext = createRequestContext(); + super(ctx, wrapEnv(env, requestContext)); + return wrapClassInstance(this, requestContext); } }; } else { diff --git a/runtime/workflows-client.js b/runtime/workflows-client.js index 6f26061..a4768ae 100644 --- a/runtime/workflows-client.js +++ b/runtime/workflows-client.js @@ -172,9 +172,6 @@ export class Workflow { callback: opts.callback ?? null, }); const responseId = validateInstanceId(body.id); - if (responseId !== id) { - throw new Error(`Workflow create response id mismatch: expected ${id}, got ${responseId}`); - } return this.#instance(responseId); } @@ -199,19 +196,9 @@ export class Workflow { if (!Array.isArray(body.instances)) { throw new Error("Workflow createBatch response must include instances"); } - const expectedIds = new Set(entries.map((entry) => entry.instanceId)); - const returnedIds = new Set(); return body.instances.map((entry, index) => { const instance = ensureObject(entry, "Workflow createBatch response entry"); const id = validateCreateBatchResponseId(instance.id, index); - if (!expectedIds.has(id)) { - const expected = [...expectedIds].join(", "); - throw new Error(`Workflow createBatch response id mismatch: unexpected ${id}; expected one of ${expected}`); - } - if (returnedIds.has(id)) { - throw new Error(`Workflow createBatch response contains duplicate id ${id}`); - } - returnedIds.add(id); return this.#instance(id); }); } @@ -221,9 +208,6 @@ export class Workflow { validateInstanceId(id); const body = await this.#call("get", { instanceId: id }); const responseId = validateInstanceId(body.id); - if (responseId !== id) { - throw new Error(`Workflow get response id mismatch: expected ${id}, got ${responseId}`); - } return this.#instance(responseId); } diff --git a/rust/redis-proxy/src/kv.rs b/rust/redis-proxy/src/kv.rs index 052df02..bee405c 100644 --- a/rust/redis-proxy/src/kv.rs +++ b/rust/redis-proxy/src/kv.rs @@ -310,63 +310,22 @@ impl RawByteBudget { } } -struct GroupedHmgetRequest { - redis_key: String, +fn apply_grouped_hmget_response( + output: &mut [Option>], + budget: &mut RawByteBudget, indices: Vec, - fields: Vec, - remaining: usize, -} - -struct GroupedFieldRead { - pending: std::vec::IntoIter, - output: Vec>>, -} - -impl GroupedFieldRead { - fn new(commands: Vec, output_len: usize) -> Self { - Self { - pending: commands.into_iter(), - output: vec![None; output_len], - } - } - - fn next_request(&mut self, budget: &RawByteBudget) -> AppResult> { - let Some((redis_key, indices, fields)) = self.pending.next() else { - return Ok(None); - }; - if indices.len() != fields.len() || indices.iter().any(|index| *index >= self.output.len()) - { - return Err(AppError::internal_error("invalid grouped KV HMGET plan")); - } - Ok(Some(GroupedHmgetRequest { - redis_key, - indices, - fields, - remaining: budget.remaining()?, - })) - } - - fn apply_response( - &mut self, - budget: &mut RawByteBudget, - request: GroupedHmgetRequest, - preflight_bytes: usize, - values: Vec>>, - ) -> AppResult<()> { - if values.len() != request.indices.len() { - return Err(AppError::internal_error("invalid grouped KV HMGET reply")); - } - budget.record_preflight(preflight_bytes)?; - for (index, value) in request.indices.into_iter().zip(values) { - budget.record_actual(value.as_deref())?; - self.output[index] = value; - } - Ok(()) + preflight_bytes: usize, + values: Vec>>, +) -> AppResult<()> { + if values.len() != indices.len() { + return Err(AppError::internal_error("invalid grouped KV HMGET reply")); } - - fn finish(self) -> Vec>> { - self.output + budget.record_preflight(preflight_bytes)?; + for (index, value) in indices.into_iter().zip(values) { + budget.record_actual(value.as_deref())?; + output[index] = value; } + Ok(()) } async fn load_grouped_fields_with_raw_byte_budget( @@ -377,18 +336,17 @@ async fn load_grouped_fields_with_raw_byte_budget( ) -> AppResult>>> { // The preflight total limits transfer between groups; the separate actual // total rechecks returned payloads before callers encode a response. - let mut read = GroupedFieldRead::new(commands, output_len); - while let Some(request) = read.next_request(budget)? { - let (preflight_bytes, values) = query_hmget_with_raw_byte_budget( - conn, - &request.redis_key, - request.remaining, - &request.fields, - ) - .await?; - read.apply_response(budget, request, preflight_bytes, values)?; + let mut output = vec![None; output_len]; + for (redis_key, indices, fields) in commands { + if indices.len() != fields.len() || indices.iter().any(|index| *index >= output.len()) { + return Err(AppError::internal_error("invalid grouped KV HMGET plan")); + } + let (preflight_bytes, values) = + query_hmget_with_raw_byte_budget(conn, &redis_key, budget.remaining()?, &fields) + .await?; + apply_grouped_hmget_response(&mut output, budget, indices, preflight_bytes, values)?; } - Ok(read.finish()) + Ok(output) } pub(crate) fn kv_put_pipeline( @@ -950,40 +908,27 @@ mod tests { } #[test] - fn grouped_budgeted_reads_decrement_budget_and_restore_input_order() { - let commands = vec![ - ( - "hash-a".to_string(), - vec![2, 0], - vec!["v:charlie".to_string(), "v:alpha".to_string()], - ), - ("hash-b".to_string(), vec![1], vec!["v:bravo".to_string()]), - ]; - let mut read = GroupedFieldRead::new(commands, 3); + fn grouped_budgeted_response_restores_input_order() { + let mut output = vec![None; 3]; let mut budget = RawByteBudget::default(); - - let first = read.next_request(&budget).unwrap().unwrap(); - assert_eq!(first.redis_key, "hash-a"); - assert_eq!(first.remaining, KV_BATCH_RAW_BYTES_MAX); - assert_eq!(first.fields, ["v:charlie", "v:alpha"]); - read.apply_response( + apply_grouped_hmget_response( + &mut output, &mut budget, - first, + vec![2, 0], 1, vec![Some(b"ccc".to_vec()), Some(b"aa".to_vec())], ) .unwrap(); - - let second = read.next_request(&budget).unwrap().unwrap(); - assert_eq!(second.redis_key, "hash-b"); - assert_eq!(second.remaining, KV_BATCH_RAW_BYTES_MAX - 1); - assert_eq!(second.fields, ["v:bravo"]); - read.apply_response(&mut budget, second, 2, vec![Some(b"bbbb".to_vec())]) - .unwrap(); - - assert!(read.next_request(&budget).unwrap().is_none()); + apply_grouped_hmget_response( + &mut output, + &mut budget, + vec![1], + 2, + vec![Some(b"bbbb".to_vec())], + ) + .unwrap(); assert_eq!( - read.finish(), + output, vec![ Some(b"aa".to_vec()), Some(b"bbbb".to_vec()), diff --git a/rust/scheduler/src/runtime_client.rs b/rust/scheduler/src/runtime_client.rs index 5f313f1..4998a76 100644 --- a/rust/scheduler/src/runtime_client.rs +++ b/rust/scheduler/src/runtime_client.rs @@ -5,8 +5,6 @@ use serde_json::Value as JsonValue; use crate::{AppState, Config}; use wdl_rust_common::internal_auth::INTERNAL_AUTH_HEADER; -const MAX_RUNTIME_RESPONSE_BYTES: usize = 1024 * 1024; - pub(crate) struct RuntimeResponse { pub(crate) status: Option, pub(crate) json: Option, @@ -72,14 +70,14 @@ pub(crate) async fn post_runtime( }; }; let status = response.status().as_u16(); - let text = match read_runtime_response(response).await { + let text = match response.text().await { Ok(text) => text, Err(error) => { return RuntimeResponse { status: Some(status), json: None, text: None, - error: Some(error), + error: Some(format!("failed to read runtime response body: {error}")), }; } }; @@ -92,32 +90,6 @@ pub(crate) async fn post_runtime( } } -async fn read_runtime_response(mut response: reqwest::Response) -> Result { - if response - .content_length() - .is_some_and(|length| length > MAX_RUNTIME_RESPONSE_BYTES as u64) - { - return Err("runtime response body exceeds 1048576 bytes".to_string()); - } - let mut body = Vec::new(); - while let Some(chunk) = response - .chunk() - .await - .map_err(|err| format!("failed to read runtime response body: {err}"))? - { - append_runtime_response_chunk(&mut body, &chunk)?; - } - String::from_utf8(body).map_err(|_| "runtime response body is not valid UTF-8".to_string()) -} - -fn append_runtime_response_chunk(body: &mut Vec, chunk: &[u8]) -> Result<(), String> { - if body.len().saturating_add(chunk.len()) > MAX_RUNTIME_RESPONSE_BYTES { - return Err("runtime response body exceeds 1048576 bytes".to_string()); - } - body.extend_from_slice(chunk); - Ok(()) -} - #[cfg(test)] mod tests { use super::*; @@ -153,16 +125,4 @@ mod tests { "error" ); } - - #[test] - fn runtime_response_body_accepts_the_limit_and_rejects_the_next_byte() { - let mut body = Vec::new(); - let exact_limit = vec![b'a'; MAX_RUNTIME_RESPONSE_BYTES]; - append_runtime_response_chunk(&mut body, &exact_limit).unwrap(); - assert_eq!(body.len(), MAX_RUNTIME_RESPONSE_BYTES); - assert_eq!( - append_runtime_response_chunk(&mut body, b"x"), - Err("runtime response body exceeds 1048576 bytes".to_string()) - ); - } } diff --git a/rust/workflows/src/api.rs b/rust/workflows/src/api.rs index 8c8e247..fc015c9 100644 --- a/rust/workflows/src/api.rs +++ b/rust/workflows/src/api.rs @@ -75,7 +75,7 @@ use pending_create::{ }; use pending_restart::{ active_pending_restart_blockers, create_pending_restart, pending_restart_marker, - remove_pending_restart, renew_pending_restart, + remove_pending_restart, }; use progress::{ spawn_progress_from_identity, spawn_progress_from_request, spawn_progress_from_step, diff --git a/rust/workflows/src/api/lifecycle/restart.rs b/rust/workflows/src/api/lifecycle/restart.rs index 16845eb..075528d 100644 --- a/rust/workflows/src/api/lifecycle/restart.rs +++ b/rust/workflows/src/api/lifecycle/restart.rs @@ -8,9 +8,9 @@ use crate::{ use super::super::{ InstanceResponse, InstanceRouteKeys, WorkflowRequest, create_pending_restart, ensure_worker_not_deleting, eval_script, identity_from_state, instance_id, payload_bytes_arg, - pending_restart_marker, read_public_state, remove_pending_restart, renew_pending_restart, - request_with_active_version, validate_identity, verify_active_workflow_current, - verify_workflow_def, workflow_referrer_member, + pending_restart_marker, read_public_state, remove_pending_restart, request_with_active_version, + validate_identity, verify_active_workflow_current, verify_workflow_def, + workflow_referrer_member, }; use super::common::{ TransitionSignal, next_generation, stale_transition_response, successful_transition_response, @@ -103,13 +103,6 @@ pub(crate) async fn restart_instance( remove_pending_restart(state, &pending_restart).await?; return Err(err); } - // Refresh immediately before the final delete-lock check. A version delete - // that starts on either side of this check sees the lock or this blocker. - if !renew_pending_restart(state, &pending_restart).await? { - return Err(WorkflowError::conflict( - "Workflow restart target lease expired; retry restart", - )); - } if let Err(err) = ensure_worker_not_deleting(state, &req.ns, &req.worker).await { remove_pending_restart(state, &pending_restart).await?; return Err(err); diff --git a/rust/workflows/src/api/pending_restart.rs b/rust/workflows/src/api/pending_restart.rs index 1600e8e..b32d36d 100644 --- a/rust/workflows/src/api/pending_restart.rs +++ b/rust/workflows/src/api/pending_restart.rs @@ -16,19 +16,6 @@ redis.call("PEXPIRE", KEYS[1], ARGV[3]) return 1 "#; -const RENEW_PENDING_RESTART_SCRIPT: &str = r#" -local time = redis.call("TIME") -local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) -redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now) -local current = redis.call("ZSCORE", KEYS[1], ARGV[1]) -if not current then - return 0 -end -redis.call("ZADD", KEYS[1], now + tonumber(ARGV[2]), ARGV[1]) -redis.call("PEXPIRE", KEYS[1], ARGV[3]) -return 1 -"#; - const READ_PENDING_RESTART_SCRIPT: &str = r#" local time = redis.call("TIME") local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) @@ -79,22 +66,6 @@ pub(super) async fn create_pending_restart( Ok(()) } -pub(super) async fn renew_pending_restart( - state: &AppState, - marker: &PendingRestartMarker, -) -> WorkflowResult { - let marker_ttl = PENDING_RESTART_TTL_MS.to_string(); - let key_ttl = PENDING_RESTART_KEY_TTL_MS.to_string(); - let renewed: i64 = eval_script( - state, - RENEW_PENDING_RESTART_SCRIPT, - &[&marker.key], - &[&marker.member, &marker_ttl, &key_ttl], - ) - .await?; - Ok(renewed == 1) -} - pub(super) async fn remove_pending_restart( state: &AppState, marker: &PendingRestartMarker, @@ -183,30 +154,4 @@ mod tests { assert!(parse_pending_restart_member(member).is_err()); } } - - #[test] - fn pending_restart_scripts_use_redis_time_and_do_not_resurrect_expired_members() { - assert!(CREATE_PENDING_RESTART_SCRIPT.contains(r#"redis.call("TIME")"#)); - assert!( - CREATE_PENDING_RESTART_SCRIPT.contains(r#"redis.call("PEXPIRE", KEYS[1], ARGV[3])"#) - ); - assert!( - RENEW_PENDING_RESTART_SCRIPT - .contains(r#"local current = redis.call("ZSCORE", KEYS[1], ARGV[1])"#) - ); - assert!(RENEW_PENDING_RESTART_SCRIPT.contains("if not current then")); - assert!(READ_PENDING_RESTART_SCRIPT.contains(r#"redis.call("TIME")"#)); - assert!(READ_PENDING_RESTART_SCRIPT.contains("return { count, members }")); - assert!(CREATE_PENDING_RESTART_SCRIPT.contains("ZREMRANGEBYSCORE")); - assert!(RENEW_PENDING_RESTART_SCRIPT.contains("ZREMRANGEBYSCORE")); - let cleanup = RENEW_PENDING_RESTART_SCRIPT - .find("ZREMRANGEBYSCORE") - .unwrap(); - let expiry_check = RENEW_PENDING_RESTART_SCRIPT.find("if not current").unwrap(); - let write = RENEW_PENDING_RESTART_SCRIPT - .find(r#"redis.call("ZADD""#) - .unwrap(); - assert!(cleanup < expiry_check); - assert!(expiry_check < write); - } } diff --git a/test-workers/do-rpc/src/index.js b/test-workers/do-rpc/src/index.js index 5019f5e..3a24f6b 100644 --- a/test-workers/do-rpc/src/index.js +++ b/test-workers/do-rpc/src/index.js @@ -1,6 +1,4 @@ import { DurableObject } from "cloudflare:workers"; -import * as hostWrapperRuntime from "./_wdl-host-wrapper-runtime.js"; -import wrappedWorker from "./_wdl-wrapper.js"; export class Room extends DurableObject { constructor(ctx, env) { @@ -50,11 +48,8 @@ export class Room extends DurableObject { async forwardRequestId(targetName) { this.env.ROOM.requestId = () => "tenant-rid"; const target = this.env.ROOM.get(this.env.ROOM.idFromName(targetName)); - const tenantContext = hostWrapperRuntime.createRequestContext(); - return tenantContext.runWithRequestContext("tenant-rid", async () => { - const response = await target.fetch(new Request("https://do.internal/request-id")); - return response.text(); - }); + const response = await target.fetch(new Request("https://do.internal/request-id")); + return response.text(); } async nestedForwardRequestId(targetName) { @@ -79,12 +74,6 @@ export class Room extends DurableObject { export default { async fetch(request, env) { const url = new URL(request.url); - if (url.pathname === "/reenter-request-id" && url.searchParams.get("nested") !== "1") { - url.searchParams.set("nested", "1"); - return wrappedWorker.fetch(new Request(url, { - headers: { "x-request-id": "tenant-rid" }, - }), env, {}); - } const id = env.ROOM.idFromName(url.searchParams.get("name") || "main"); const stub = env.ROOM.get(id); if (url.pathname === "/fail") { @@ -126,11 +115,6 @@ export default { requestId: await stub.forwardRequestId(url.searchParams.get("to") || "peer"), }); } - if (url.pathname === "/reenter-request-id") { - return Response.json({ - requestId: await stub.forwardRequestId(url.searchParams.get("to") || "peer"), - }); - } if (url.pathname === "/nested-request-id") { return Response.json({ requestId: await stub.nestedForwardRequestId(url.searchParams.get("to") || "peer"), diff --git a/tests/helpers/style-contract-scanner.js b/tests/helpers/style-contract-scanner.js index 1fb0d0b..5513da7 100644 --- a/tests/helpers/style-contract-scanner.js +++ b/tests/helpers/style-contract-scanner.js @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { readdirSync } from "node:fs"; import path from "node:path"; -import { isAlias, isSeq, parseAllDocuments, parseDocument, visit } from "yaml"; +import { parseAllDocuments } from "yaml"; import { readRepoFile, repoPath } from "./source-scan.js"; @@ -53,49 +53,15 @@ export function testSourceFiles(dir) { return out; } -/** - * @param {string} file - * @param {{ merge?: boolean }} [options] - */ -export function yamlDocument(file, options = {}) { - const document = parseDocument(readRepoFile(file), { merge: options.merge === true }); - assert.equal(document.errors.length, 0, `${file} must parse as YAML`); - return document; -} - /** @param {string} file */ export function yamlDocuments(file) { - const documents = parseAllDocuments(readRepoFile(file)); + const documents = parseAllDocuments(readRepoFile(file), { merge: true }); for (const document of documents) { assert.equal(document.errors.length, 0, `${file} must parse as YAML`); } return documents; } -/** - * @param {import("yaml").Document} document - * @param {(string | number)[]} pathParts - * @returns {string[]} - */ -export function yamlMergeAliases(document, pathParts) { - const merge = document.getIn([...pathParts, "<<"], true); - if (isAlias(merge)) return [merge.source]; - assert.ok(isSeq(merge), `YAML path ${pathParts.join(".")} must contain a merge alias`); - assert.ok(merge.items.every(isAlias), `YAML path ${pathParts.join(".")} merge entries must be aliases`); - return merge.items.map((item) => /** @type {import("yaml").Alias} */ (item).source); -} - -/** @param {import("yaml").Document} document @param {string} source */ -export function yamlAliasCount(document, source) { - let count = 0; - visit(document, { - Alias(_key, alias) { - if (alias.source === source) count += 1; - }, - }); - return count; -} - /** @param {Set} [exempt] */ export function scannedTestFiles(exempt = new Set()) { const scanned = [ @@ -197,106 +163,11 @@ export function extractRegex(file, name) { } /** - * @param {string} source - * @param {number} openIndex - * @param {string} open - * @param {string} close + * @param {string} service + * @param {string} anchor */ -function findClosingDelimiter(source, openIndex, open, close) { - let depth = 0; - let quote = ""; - for (let index = openIndex; index < source.length; index += 1) { - const char = source[index]; - if (quote) { - if (char === "\\") index += 1; - else if (char === quote) quote = ""; - continue; - } - if (char === '"') { - quote = char; - } else if (char === open) { - depth += 1; - } else if (char === close) { - depth -= 1; - if (depth === 0) return index; - } - } - return -1; -} - -/** @param {string} source */ -export function stripCapnpComments(source) { - let out = ""; - let quoted = false; - for (let index = 0; index < source.length; index += 1) { - const char = source[index]; - if (quoted) { - out += char; - if (char === "\\") { - index += 1; - out += source[index] ?? ""; - } else if (char === '"') { - quoted = false; - } - continue; - } - if (char === '"') { - quoted = true; - out += char; - continue; - } - if (char !== "#") { - out += char; - continue; - } - while (index < source.length && source[index] !== "\n") { - out += " "; - index += 1; - } - if (index < source.length) out += "\n"; - } - return out; -} - -/** - * Extracts a Cap'n Proto const struct while ignoring comments and formatting. - * @param {string} source - * @param {string} name - */ -export function extractCapnpConstBlock(source, name) { - const clean = stripCapnpComments(source); - const declaration = new RegExp(`const\\s+${RegExp.escape(name)}\\s+:[^=]+=`).exec(clean); - assert.ok(declaration, `Cap'n Proto const ${name} must be present`); - const start = clean.indexOf("(", declaration.index + declaration[0].length); - assert.notEqual(start, -1, `Cap'n Proto const ${name} must contain a struct value`); - const end = findClosingDelimiter(clean, start, "(", ")"); - assert.notEqual(end, -1, `Cap'n Proto const ${name} must have balanced parentheses`); - return clean.slice(start, end + 1); -} - -/** - * Returns normalized top-level tuple entries from a list field in a Cap'n Proto struct. - * @param {string} block - * @param {string} field - */ -export function extractCapnpListEntries(block, field) { - const assignment = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*\\[`).exec(block); - assert.ok(assignment, `Cap'n Proto field ${field} must be present`); - const listStart = block.indexOf("[", assignment.index); - const listEnd = findClosingDelimiter(block, listStart, "[", "]"); - assert.notEqual(listEnd, -1, `Cap'n Proto field ${field} must have balanced brackets`); - - const entries = []; - let index = listStart + 1; - while (index < listEnd) { - const start = block.indexOf("(", index); - if (start === -1 || start >= listEnd) break; - const end = findClosingDelimiter(block, start, "(", ")"); - assert.notEqual(end, -1, `Cap'n Proto field ${field} entry must have balanced parentheses`); - entries.push(block.slice(start, end + 1).replace(/\s+/g, " ").trim()); - index = end + 1; - } - return entries; +export function serviceAnchorRegex(service, anchor) { + return new RegExp(`\\n ${RegExp.escape(service)}:\\n <<: \\*${RegExp.escape(anchor)}\\b`); } /** diff --git a/tests/integration/durable-objects-core.test.js b/tests/integration/durable-objects-core.test.js index ce8e75c..6f16245 100644 --- a/tests/integration/durable-objects-core.test.js +++ b/tests/integration/durable-objects-core.test.js @@ -252,12 +252,4 @@ test("Durable Object RPC dispatches class methods with structured args", async ( assert.equal(nestedContext.status, 200, nestedContextText); assert.deepEqual(responseJson({ body: nestedContextText }), { requestId }); - const reenteredContext = await gatewayFetch( - ns, - "/rooms/reenter-request-id?name=alice&to=bob", - { headers: { "x-request-id": requestId } } - ); - const reenteredContextText = await reenteredContext.text(); - assert.equal(reenteredContext.status, 200, reenteredContextText); - assert.deepEqual(responseJson({ body: reenteredContextText }), { requestId }); }); diff --git a/tests/unit/control-d1-runtime-client.test.js b/tests/unit/control-d1-runtime-client.test.js index eb9c836..d348e2c 100644 --- a/tests/unit/control-d1-runtime-client.test.js +++ b/tests/unit/control-d1-runtime-client.test.js @@ -358,25 +358,6 @@ test("control D1 runtime failure redacts 5xx diagnostics and keeps safe extra fi }); }); -test("control D1 runtime failure log fields reject unbounded or non-token diagnostics", () => { - for (const invalid of [" leading-space", "line\nbreak", "x".repeat(129)]) { - assert.deepEqual(d1RuntimeFailureLogFields({ - status: 502, - body: { - error: invalid, - category: invalid, - causeCode: invalid, - retryable: false, - }, - }), { - upstream_status: 502, - upstream_code: "d1-runtime-error", - upstream_category: "internal", - upstream_retryable: false, - }); - } -}); - test("control D1 runtime failure uses the outward status to redact upstream 4xx details", () => { assert.deepEqual(d1RuntimeFailure("d1_database_initialize_failed", "demo", "db1", { status: 400, diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 9186b55..feae2c3 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -844,42 +844,38 @@ test("prepareBundle: experimental workerd compatibility flags are rejected", () ); }); -test("prepareBundle: error serialization flags match the effective compatibility date", () => { - assert.doesNotThrow(() => prepareBundle( - "w.js", - { "w.js": "x" }, - { - compatibilityDate: "2026-04-20", - compatibilityFlags: ["enhanced_error_serialization"], - } - )); - - /** @type {[string | undefined, string][]} */ - const cases = [ - ["2026-04-20", "legacy_error_serialization"], - ["2026-04-21", "legacy_error_serialization"], - ["2026-04-21", "enhanced_error_serialization"], - [undefined, "enhanced_error_serialization"], - ]; - for (const [compatibilityDate, flag] of cases) { +test("prepareBundle: rejects legacy error serialization", () => { + for (const compatibilityDate of ["2026-04-20", "2026-04-21", undefined]) { assert.throws( () => prepareBundle( "w.js", { "w.js": "x" }, - { compatibilityDate, compatibilityFlags: [flag] } + { compatibilityDate, compatibilityFlags: ["legacy_error_serialization"] } ), (err) => { if (!(err instanceof Error)) return false; const coded = /** @type {Error & { code?: unknown, status?: unknown }} */ (err); return coded.code === "compatibility_flag_unsupported" && coded.status === 400 && - coded.message.includes(flag); + coded.message.includes("legacy_error_serialization"); }, - `${flag} should be rejected for ${compatibilityDate ?? "the default date"}` + `legacy_error_serialization should be rejected for ${compatibilityDate ?? "the default date"}` ); } }); +test("prepareBundle: leaves redundant positive flags to workerd", () => { + const { meta } = prepareBundle( + "w.js", + { "w.js": "x" }, + { + compatibilityDate: "2026-04-21", + compatibilityFlags: ["enhanced_error_serialization"], + } + ); + assert.deepEqual(meta.compatibilityFlags, ["enhanced_error_serialization"]); +}); + test("prepareBundle: compatibilityDate validates shape before commit", () => { assert.equal( prepareBundle("w.js", { "w.js": "x" }, { compatibilityDate: "2026-04-24" }).meta.compatibilityDate, diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index 7c94e9e..05d0ac1 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -676,39 +676,32 @@ test("bumpActiveAndPromote rejects active routes that no longer declare their ho ); }); -/** @type {Array<[string, string, number, string]>} */ -const bumpProjectionFailureCases = [ - ["malformed", "not-a-projection", 500, "corrupt_pattern_projection"], - ["foreign", patternProjection("other", "site", "v7", "prefix", "/"), 409, "route_conflict"], -]; -for (const [label, held, expectedStatus, expectedCode] of bumpProjectionFailureCases) { - test(`bumpActiveAndPromote fails closed on ${label} held pattern projections`, async () => { - const redis = makeRedis(); - seedBundle(redis, "v1", { - routes: [{ - host: "app.workers.example", - slot: "/*", - kind: "prefix", - value: "/", - }], - }); - redis.state.hashes.set("routes:demo", { worker: "v1" }); - redis.state.hashes.set("patterns:app.workers.example", { "/*": held }); - redis.state.sets.set("hosts:demo", new Set(["app.workers.example"])); - redis.state.strings.set("worker:demo:worker:next_version", "1"); +test("bumpActiveAndPromote fails closed on malformed held pattern projections", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { + routes: [{ + host: "app.workers.example", + slot: "/*", + kind: "prefix", + value: "/", + }], + }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.hashes.set("patterns:app.workers.example", { "/*": "not-a-projection" }); + redis.state.sets.set("hosts:demo", new Set(["app.workers.example"])); + redis.state.strings.set("worker:demo:worker:next_version", "1"); - await assert.rejects( - bumpActiveAndPromote(redis, "demo", "worker"), - (err) => { - assertRoutingErrorShape(err, expectedStatus, expectedCode); - return true; - } - ); + await assert.rejects( + bumpActiveAndPromote(redis, "demo", "worker"), + (err) => { + assertRoutingErrorShape(err, 500, "corrupt_pattern_projection"); + return true; + } + ); - assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v1"); - assert.equal(redis.state.hashes.has(productionBundleKey("demo", "worker", "v2")), false); - }); -} + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v1"); + assert.equal(redis.state.hashes.has(productionBundleKey("demo", "worker", "v2")), false); +}); test("bumpActiveAndPromote rejects malformed cron metadata", async () => { const redis = makeRedis(); diff --git a/tests/unit/do-runtime-index.test.js b/tests/unit/do-runtime-index.test.js index d5a6b11..751da06 100644 --- a/tests/unit/do-runtime-index.test.js +++ b/tests/unit/do-runtime-index.test.js @@ -547,8 +547,17 @@ test("do-runtime returns owner hints instead of forwarding when caller accepts t }); }); -test("do-runtime connect local owner responses carry owner hint headers", async () => { - /** @type {any} */ (globalThis).__doIndexHostResponse = new Response("maintenance", { status: 503 }); +test("do-runtime connect strips tenant owner hint control markers", async () => { + /** @type {any} */ (globalThis).__doIndexHostResponse = new Response("maintenance", { + status: 503, + headers: { + "x-wdl-do-owner-hint": "1", + "x-wdl-do-owner-key": "tenant-forged-owner", + "x-wdl-do-owner-task-id": "tenant-forged-task", + "x-wdl-do-owner-endpoint": "do-runtime-forged:8788", + "x-wdl-do-owner-generation": "99", + }, + }); const response = await app.fetch(internalRequest("https://do-runtime/internal/do/connect", { method: "GET", diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index ebc7b29..bc300a1 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -15,7 +15,6 @@ import { doOwnerHintHeaders, doOwnershipErrorHeaders, } from "../helpers/do-owner-hint.js"; -import { withMockedProperty } from "../helpers/mock-global.js"; const { DoRuntimeError, @@ -353,29 +352,6 @@ test("rejects invalid do rpc shapes", () => { ); }); -test("do-runtime RPC JSON validation uses captured intrinsics", async () => { - await withMockedProperty(Object, "entries", () => [], () => { - assert.throws( - () => normalizeDoInvokeRequest({ - ...BASE_BODY, - kind: "rpc", - rpc: { method: "addMessage", args: [{ fn() {} }] }, - }), - (error) => error instanceof Error && error.message === "rpc.args[0].fn must be JSON data" - ); - }); - await withMockedProperty(Number, "isFinite", () => true, () => { - assert.throws( - () => normalizeDoInvokeRequest({ - ...BASE_BODY, - kind: "rpc", - rpc: { method: "addMessage", args: [Number.NaN] }, - }), - (error) => error instanceof Error && error.message === "rpc.args[0] must be a finite number" - ); - }); -}); - test("rejects inline workerCode", () => { assert.throws( () => normalizeDoInvokeRequest({ diff --git a/tests/unit/runtime-dispatch-workflows.test.js b/tests/unit/runtime-dispatch-workflows.test.js index 632765c..850c7c8 100644 --- a/tests/unit/runtime-dispatch-workflows.test.js +++ b/tests/unit/runtime-dispatch-workflows.test.js @@ -1902,8 +1902,16 @@ test("handleWorkflowRunDispatch commits failed step.do errors", async () => { assert.equal(scope.errors.length, 1); }); -test("handleWorkflowRunDispatch rejects swallowed terminal step.do failures", async () => { +test("handleWorkflowRunDispatch commits hostile terminal step.do failures", async () => { const scope = makeScope(); + const throwable = new Proxy(Object.create(null), { + get() { + throw new Error("throwable field trap"); + }, + getPrototypeOf() { + throw new Error("throwable prototype trap"); + }, + }); const backend = makeWorkflowBackend(async (url) => { if (url.endsWith("/claim-step")) return Response.json({ state: "run" }); if (url.endsWith("/commit-step-error")) return Response.json({ state: "failed" }); @@ -1930,7 +1938,7 @@ test("handleWorkflowRunDispatch rejects swallowed terminal step.do failures", as async run(/** @type {any} */ _event, /** @type {any} */ step) { try { await step.do("charge", async () => { - throw new TypeError("card declined"); + throw throwable; }); } catch { return "swallowed"; @@ -1944,12 +1952,25 @@ test("handleWorkflowRunDispatch rejects swallowed terminal step.do failures", as const body = await readJsonResponse(res, 200); assert.equal(body.outcome, "failed"); assert.deepEqual(body.error, { - name: "TypeError", - message: "card declined", + name: "Error", + message: "Workflow run failed", }); + assert.deepEqual(backend.calls[2].body.error, body.error); assert.equal(scope.errors.length, 1); }); +test("workflowError falls back when throwable conversion throws", () => { + const throwable = { + toString() { + throw new Error("conversion failed"); + }, + }; + assert.deepEqual(workflowError(throwable), { + name: "Error", + message: "Workflow run failed", + }); +}); + test("workflow internal error codes cannot be forged with Error.name", async () => { const forged = new Error("not internal"); forged.name = "workflow_invalid_step"; diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index 723aca8..18918ff 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -10,7 +10,9 @@ import { MAX_DO_REQUEST_HEADER_BYTES, dispatchDoInvokeWithHintCache, fetchInvokeInit, + isWebSocketUpgrade, ownerHintFromHeaders, + replayOwnerUnavailableForFetch, requestSpec, rpcInvokeBody, } from "../../runtime/_wdl-do-transport.js"; @@ -437,7 +439,7 @@ test("DurableObjectNamespace RPC rejects invalid and reserved methods before tra await assert.rejects(Reflect.get(stub, "alarm")(), /rpc\.method is reserved/); }); -test("DO RPC validation uses captured JSON and method intrinsics", async () => { +test("DO RPC validation uses captured JSON intrinsics", async () => { const props = { ns: "tenant", worker: "chat", @@ -458,15 +460,9 @@ test("DO RPC validation uses captured JSON and method intrinsics", async () => { (error) => error instanceof TypeError && error.message === "rpc.args[0] must be a finite number" ); }); - await withMockedProperty(RegExp.prototype, "test", () => true, () => { - assert.throws( - () => rpcInvokeBody(props, "room-a", "not-valid-method", []), - (error) => error instanceof TypeError && error.message === "rpc.method is not valid" - ); - }); }); -test("DO RPC method bounds match server normalization", () => { +test("do-runtime owns the DO RPC method byte limit", () => { const props = { ns: "tenant", worker: "chat", @@ -485,13 +481,15 @@ test("DO RPC method bounds match server normalization", () => { kind: "rpc", rpc: { method, args: [] }, }; + const envelope = rpcInvokeBody(props, "room-a", method, []); if (!valid) { - assert.throws(() => rpcInvokeBody(props, "room-a", method, []), /rpc\.method is not valid/); - assert.throws(() => normalizeDoInvokeRequest(metadata), /rpc\.method is too large/); + assert.throws( + () => normalizeDoInvokeRequest(decodeDoEnvelope(envelope).metadata), + /rpc\.method is too large/ + ); continue; } - const envelope = rpcInvokeBody(props, "room-a", method, []); - const invoke = normalizeDoInvokeRequest(decodeDoEnvelope(envelope).metadata); + const invoke = normalizeDoInvokeRequest(metadata); assert.equal("rpc" in invoke ? invoke.rpc.method : null, method); } }); @@ -609,7 +607,6 @@ test("DurableObjectNamespace RPC retries owner claim races once", async () => { /** @type {any[]} */ const calls = []; let cancellations = 0; - let hostileCancelCalls = 0; const backend = { fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 @@ -630,24 +627,12 @@ test("DurableObjectNamespace RPC retries owner claim races once", async () => { binding: "ROOM", className: "Room", }, { backend }); - const streamCancel = ReadableStream.prototype.cancel; - - const result = await withMockedProperty( - ReadableStream.prototype, - "cancel", - /** @this {ReadableStream} */ - function hostileCancel(reason) { - hostileCancelCalls += 1; - return Reflect.apply(streamCancel, this, [reason]); - }, - () => Reflect.get(ns.get(ns.idFromName("room-a")), "addMessage")("hello") - ); + const result = await Reflect.get(ns.get(ns.idFromName("room-a")), "addMessage")("hello"); assert.equal(result, "claimed"); assert.equal(calls.length, 2); assert.equal(calls[1].init.headers.get("x-wdl-do-accept-owner-hint"), null); assert.equal(cancellations, 1); - assert.equal(hostileCancelCalls, 0); }); test("DurableObjectNamespace direct backend preserves binary request bodies", async () => { @@ -865,54 +850,6 @@ test("DurableObjectNamespace retry policy uses the captured Request method gette assert.equal(/** @type {any} */ (metadata.request).method, "POST"); }); -test("DurableObjectNamespace retry policy uses captured string case intrinsics", async () => { - /** @type {any[]} */ - const calls = []; - const backend = { - fetch: makeRecordingFetch(calls, { - response: Response.json({ error: "owner_unavailable", message: "outcome unknown" }, { - status: 503, - headers: doOwnershipErrorHeaders("owner_unavailable"), - }), - }), - }; - const ns = new DurableObjectNamespace({ - ns: "tenant", - worker: "chat", - version: "v1", - doStorageId: "do_0123456789abcdef0123456789abcdef", - binding: "ROOM", - className: "Room", - }, { backend }); - let upperReads = 0; - - const response = await withMockedProperty( - String.prototype, - "toUpperCase", - function mockedToUpperCase() { - upperReads += 1; - return upperReads === 1 ? "POST" : "GET"; - }, - () => withMockedProperty( - String.prototype, - "toLowerCase", - function mockedToLowerCase() { - return "websocket"; - }, - () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { - method: "POST", - body: "side effect", - }) - ) - ); - - assert.equal(response.status, 503); - assert.equal(calls.length, 1); - assert.equal(upperReads, 0); - const { metadata } = decodeDoEnvelope(calls[0].init.body); - assert.equal(/** @type {any} */ (metadata.request).method, "POST"); -}); - test("DurableObjectNamespace strips owner metadata with patched Headers methods", async () => { const backend = { fetch: makeRecordingFetch([], { @@ -960,39 +897,6 @@ test("DurableObjectNamespace strips owner metadata with patched Headers methods" assert.equal(response.headers.get("x-wdl-do-ownership-error"), null); }); -test("DO requestSpec header budget uses captured array iteration", async () => { - const iteratorSymbol = Symbol.iterator; - const arrayIterator = Array.prototype[Symbol.iterator]; - const request = new Request("https://demo.workers.example/send", { - headers: { "x-oversized": "a".repeat(MAX_DO_REQUEST_HEADER_BYTES + 1) }, - }); - - await withMockedProperty( - /** @type {any} */ (Array.prototype), - iteratorSymbol, - /** @this {unknown[]} */ - function targetedIterator() { - const first = this[0]; - if ( - (Array.isArray(first) && first[0] === "x-oversized") || - first === "x-oversized" - ) { - return { - next: () => ({ done: true, value: undefined }), - [iteratorSymbol]() { return this; }, - }; - } - return Reflect.apply(arrayIterator, this, []); - }, - async () => { - await assert.rejects( - () => requestSpec(request, null), - /fetch headers exceed 65536 bytes/ - ); - } - ); -}); - test("DO requestSpec header budget uses captured TextEncoder.encode", async () => { const textEncode = TextEncoder.prototype.encode; let hostileEncodeCalls = 0; @@ -1021,6 +925,54 @@ test("DO requestSpec header budget uses captured TextEncoder.encode", async () = assert.equal(hostileEncodeCalls, 0); }); +test("DO request method and upgrade decisions use captured string normalization", async () => { + const post = new Request("https://example.com/objects", { + method: "POST", + body: "payload", + }); + const websocket = new Request("https://example.com/socket", { + headers: { Upgrade: "WebSocket" }, + }); + const ordinary = new Request("https://example.com/fetch"); + const originalToLowerCase = String.prototype.toLowerCase; + const originalToUpperCase = String.prototype.toUpperCase; + /** @type {{ method: string, body: Uint8Array | null, replay: boolean, websocket: boolean, ordinary: boolean } | undefined} */ + let observed; + + await withMockedProperty( + String.prototype, + "toUpperCase", + /** @this {string} */ function hostileToUpperCase() { + const normalized = Reflect.apply(originalToUpperCase, this, []); + return normalized === "POST" ? "GET" : normalized; + }, + () => withMockedProperty( + String.prototype, + "toLowerCase", + /** @this {string} */ function hostileToLowerCase() { + const normalized = Reflect.apply(originalToLowerCase, this, []); + return normalized === "" ? "websocket" : normalized; + }, + async () => { + const { spec, bodyBytes } = await requestSpec(post, null); + observed = { + method: spec.method, + body: bodyBytes, + replay: replayOwnerUnavailableForFetch(post), + websocket: isWebSocketUpgrade(websocket), + ordinary: isWebSocketUpgrade(ordinary), + }; + }, + ), + ); + + assert.equal(observed?.method, "POST"); + assert.equal(new TextDecoder().decode(observed?.body ?? undefined), "payload"); + assert.equal(observed?.replay, false); + assert.equal(observed?.websocket, true); + assert.equal(observed?.ordinary, false); +}); + function chunkedBodyRequest() { return new Request("https://demo.workers.example/send", /** @type {RequestInit} */ (/** @type {unknown} */ ({ method: "POST", @@ -1035,54 +987,6 @@ function chunkedBodyRequest() { }))); } -test("DO requestSpec body collection uses captured Array.push", async () => { - const arrayPush = Array.prototype.push; - let hostilePushCalls = 0; - - const { bodyBytes } = await withMockedProperty( - Array.prototype, - "push", - /** @this {unknown[]} */ - function targetedPush(value) { - if (value instanceof Uint8Array) { - hostilePushCalls += 1; - return this.length; - } - return Reflect.apply(arrayPush, this, [value]); - }, - () => requestSpec(chunkedBodyRequest(), null) - ); - - assert.equal(hostilePushCalls, 0); - assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); -}); - -test("DO requestSpec body copying uses captured array iteration", async () => { - const iteratorSymbol = Symbol.iterator; - const arrayIterator = Array.prototype[Symbol.iterator]; - let hostileIteratorCalls = 0; - - const { bodyBytes } = await withMockedProperty( - /** @type {any} */ (Array.prototype), - iteratorSymbol, - /** @this {unknown[]} */ - function targetedIterator() { - if (this[0] instanceof Uint8Array) { - hostileIteratorCalls += 1; - return { - next: () => ({ done: true, value: undefined }), - [iteratorSymbol]() { return this; }, - }; - } - return Reflect.apply(arrayIterator, this, []); - }, - () => requestSpec(chunkedBodyRequest(), null) - ); - - assert.equal(hostileIteratorCalls, 0); - assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); -}); - test("DO requestSpec body copying uses captured Uint8Array.set", async () => { const uint8ArraySet = Uint8Array.prototype.set; let hostileSetCalls = 0; @@ -2255,24 +2159,13 @@ test("DO requestSpec rejects oversized streams without waiting for cancel", asyn body, duplex: "half", })); - let hostileCatchCalls = 0; - - await withMockedProperty( - Promise.prototype, - "catch", - function hostileCatch() { - hostileCatchCalls += 1; - throw new Error("tenant Promise.catch must not run"); - }, - () => assert.rejects( - withTestTimeout( - requestSpec(request, "rid-cancel"), - "requestSpec waited for stream cancel" - ), - /Durable Object fetch body exceeds/ - ) + await assert.rejects( + withTestTimeout( + requestSpec(request, "rid-cancel"), + "requestSpec waited for stream cancel" + ), + /Durable Object fetch body exceeds/ ); - assert.equal(hostileCatchCalls, 0); }); test("DurableObjectNamespace facade uses direct upgrade path for websockets", async () => { diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index e180753..24affa8 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -417,13 +417,11 @@ test("bundleToWorkerCode: compatibilityFlags already includes floor → no dup", assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization", "nodejs_compat"]); }); -test("bundleToWorkerCode: incompatible error serialization flags fail closed", () => { +test("bundleToWorkerCode: legacy error serialization fails closed", () => { /** @type {[string | undefined, string, RegExp][]} */ const cases = [ ["2026-04-20", "legacy_error_serialization", /requires enhanced error serialization/], ["2026-04-21", "legacy_error_serialization", /requires enhanced error serialization/], - ["2026-04-21", "enhanced_error_serialization", /became the workerd default/], - [undefined, "enhanced_error_serialization", /became the workerd default/], ]; for (const [compatibilityDate, flag, message] of cases) { assert.throws( @@ -444,6 +442,21 @@ test("bundleToWorkerCode: incompatible error serialization flags fail closed", ( } }); +test("bundleToWorkerCode: leaves redundant positive flags to workerd", () => { + const code = bundleToWorkerCode( + mkBundle( + { + mainModule: "w.js", + compatibilityDate: "2026-04-21", + compatibilityFlags: ["enhanced_error_serialization"], + modules: { "w.js": { type: "module" } }, + }, + { "w.js": enc.encode("x") } + ) + ); + assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization"]); +}); + test("bundleToWorkerCode: throws (doesn't silently drop) on malformed compatibilityFlags in bundle bytes", () => { assert.throws( () => bundleToWorkerCode( diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index 9ee0e13..f6c36f1 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -1738,28 +1738,28 @@ test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserv assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-d1-client.js"], /class D1Database/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], /intrinsicArrayForEach/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], /AsyncLocalStorage/); + assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], /AsyncLocalStorage/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as __WdlHostRuntime__/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as __WdlUserModule__ from "\.\/src\/worker\.js";/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /Explicit aliases replace same-name star exports/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const D1_BINDINGS = \["DB"\];/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const R2_BINDINGS = \[\];/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const DO_BINDINGS = \[\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new D1Database\(out\[name\], requestIdOptions\(\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new D1Database\(out\[name\], requestIdOptions\(requestIdOrContext\)\)/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /requestIdFromEventArg/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /wrapClassInstance\(this\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /wrapClassInstance\(this, requestContext\)/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /forEachArray\(HOST_WRAPPED_HANDLER_KEYS/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.Api/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /__WdlWrappedEntrypoint\d+__ as Admin/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class default/); - assert.deepEqual(/** @type {any} */ (workerCode).compatibilityFlags, ["nodejs_als"]); + assert.equal(/** @type {any} */ (workerCode).compatibilityFlags, undefined); }); -test("wrapWorkerCodeForHostBindings: normalizes standalone ALS compatibility flags", () => { - for (const [input, expected] of [ - [["no_nodejs_als"], ["nodejs_als"]], - [["no_global_navigator", "no_nodejs_als"], ["no_global_navigator", "nodejs_als"]], - [["nodejs_compat", "no_nodejs_als"], ["nodejs_compat", "no_nodejs_als"]], +test("wrapWorkerCodeForHostBindings preserves compatibility flags", () => { + for (const input of [ + ["no_nodejs_als"], + ["no_global_navigator", "no_nodejs_als"], + ["nodejs_compat", "no_nodejs_als"], ]) { const workerCode = { compatibilityFlags: input, @@ -1769,7 +1769,7 @@ test("wrapWorkerCodeForHostBindings: normalizes standalone ALS compatibility fla wrapWorkerCodeForHostBindings(workerCode, { bindings: { DB: { type: "d1", databaseId: "main" } }, }); - assert.deepEqual(workerCode.compatibilityFlags, expected); + assert.deepEqual(workerCode.compatibilityFlags, input); } }); @@ -1902,17 +1902,6 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc currentRequestId: "undefined", runWithRequestContext: "undefined", }); - - let thenReads = 0; - const stateful = {}; - Object.defineProperty(stateful, "then", { - get() { - thenReads += 1; - return undefined; - }, - }); - assert.equal(instance.echo(stateful), stateful); - assert.equal(thenReads, 0); }); }); @@ -2055,7 +2044,7 @@ test("wrapWorkerCodeForHostBindings: injects local R2 facade for R2 bindings", ( assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-r2-client.js"], /this\._stub/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const R2_BINDINGS = \["BUCKET"\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new R2Bucket\(out\[name\], requestIdOptions\(\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new R2Bucket\(out\[name\], requestIdOptions\(requestIdOrContext\)\)/); }); test("wrapWorkerCodeForHostBindings: injects local DO facade for Durable Object bindings", () => { @@ -2074,7 +2063,7 @@ test("wrapWorkerCodeForHostBindings: injects local DO facade for Durable Object assert.match(/** @type {any} */ (workerCode.modules)["_wdl-owner-endpoint.js"], /validOwnerEndpointForService/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const DO_BINDINGS = \["ROOMS"\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new DurableObjectNamespace\(out\[name\], doOptions\(doBackend, doOwnerNetwork\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new DurableObjectNamespace\(out\[name\], doOptions\(requestIdOrContext, doBackend, doOwnerNetwork\)\)/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /internalAuthToken|__WDL_INTERNAL_AUTH_TOKEN__/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.Room/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export \* from/); @@ -2110,10 +2099,10 @@ test("wrapWorkerCodeForHostBindings: injects local Workflow facade and wraps wor assert.match(/** @type {any} */ (workerCode.modules)["_wdl-cloudflare-workflows.js"], /this\.name = "NonRetryableError"/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \{ Workflow \}/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const WORKFLOW_BINDINGS = \{"ORDERS":/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new Workflow\(out\[name\] \|\| metadata, workflowOptions\(workflowsBackend\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new Workflow\(out\[name\] \|\| metadata, workflowOptions\(requestIdOrContext, workflowsBackend\)\)/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /internalAuthToken|__WDL_INTERNAL_AUTH_TOKEN__/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /notifyWorkflowCallback/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /withRequestContext\(request, \(\) => notifyWorkflowCallback\(request, wrapEnv\(this\.env\)\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /notifyWorkflowCallback\(request, wrapEnv\(this\.env, requestIdFromEventArg\(request\)\)\)/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.OrderWorkflow/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export \* from "\.\/worker\.js";/); }); diff --git a/tests/unit/runtime-workflows-client.test.js b/tests/unit/runtime-workflows-client.test.js index 39818eb..07c1bd5 100644 --- a/tests/unit/runtime-workflows-client.test.js +++ b/tests/unit/runtime-workflows-client.test.js @@ -312,42 +312,6 @@ test("Workflow.createBatch accepts valid instances array", async () => { assert.deepEqual(instances.map((instance) => instance.id), ["inst-1", "inst-2"]); }); -test("Workflow.createBatch rejects duplicate backend response ids", async () => { - const workflow = createWorkflowForTest({ - backend: { - async fetch() { - return Response.json({ - instances: [{ id: "inst-1" }, { id: "inst-1" }], - }); - }, - }, - }); - - await assert.rejects( - () => workflow.createBatch([{ id: "inst-1" }, { id: "inst-2" }]), - /Workflow createBatch response contains duplicate id inst-1/ - ); -}); - -test("Workflow create and get require matching response ids", async () => { - for (const endpoint of ["create", "get"]) { - const workflow = createWorkflowForTest({ - backend: { - async fetch() { - return Response.json({ id: "inst-other" }); - }, - }, - }); - - await assert.rejects( - () => endpoint === "create" - ? workflow.create({ id: "inst-1" }) - : workflow.get("inst-1"), - new RegExp(`Workflow ${endpoint} response id mismatch`) - ); - } -}); - test("Workflow.create rejects non-object success responses", async () => { const workflow = createWorkflowForTest({ backend: { @@ -417,26 +381,3 @@ test("Workflow.createBatch accepts backend-skipped ids", async () => { const instanceIds = instances.map((instance) => instance.id); assert.deepEqual(instanceIds, ["inst-1"]); }); - -test("Workflow.createBatch rejects unexpected backend ids", async () => { - const workflow = createWorkflowForTest({ - backend: { - async fetch() { - return Response.json({ instances: [{ id: "inst-1" }, { id: "inst-3" }] }); - }, - }, - }); - - await assert.rejects( - () => workflow.createBatch([{ id: "inst-1" }, { id: "inst-2" }, { id: "inst-4" }]), - /** @param {unknown} error */ - (error) => { - assert.ok(error instanceof Error); - assert.match(error.message, /Workflow createBatch response id mismatch/); - assert.match(error.message, /inst-3/); - assert.match(error.message, /inst-2/); - assert.match(error.message, /inst-4/); - return true; - } - ); -}); diff --git a/tests/unit/runtime-wrapper-generate.test.js b/tests/unit/runtime-wrapper-generate.test.js index 902a697..b573f83 100644 --- a/tests/unit/runtime-wrapper-generate.test.js +++ b/tests/unit/runtime-wrapper-generate.test.js @@ -1,6 +1,5 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { AsyncLocalStorage } from "node:async_hooks"; import { HOST_BINDING_RUNTIME_MODULE_NAME, HOST_BINDING_RUNTIME_SOURCE, @@ -12,11 +11,6 @@ import { moduleDataUrl, repositoryFileUrl, } from "../helpers/load-shared-module.js"; -import { - installMockProperty, - withMockedProperty, - withMockedPropertyDescriptor, -} from "../helpers/mock-global.js"; const HOST_BINDING_RUNTIME_TEST_SOURCE = applyModuleReplacements(HOST_BINDING_RUNTIME_SOURCE, [ [ @@ -24,8 +18,6 @@ const HOST_BINDING_RUNTIME_TEST_SOURCE = applyModuleReplacements(HOST_BINDING_RU `from ${JSON.stringify(repositoryFileUrl("runtime/_wdl-request-id.js"))}`, ], ]); -const hostBindingRuntime = await import(moduleDataUrl(HOST_BINDING_RUNTIME_TEST_SOURCE)); -const hostRequestContext = hostBindingRuntime.createRequestContext(); function generatedWrappers() { return { @@ -69,117 +61,15 @@ test("generated wrapper flavors preserve default-export class detection", () => assert.match(hostBindings, /if \(__WdlHostRuntime__\.regexpTest\(\/\^\\s\*class\\b\/, source\)\)/); }); -test("host wrapper runtime captures platform intrinsics before user module evaluation", () => { +test("host wrapper runtime evaluates before the tenant module", () => { const source = generateHostBindingWrapperModule("worker.js", [], [], ["ROOM"], {}, []); assert.ok( source.indexOf(`from "./${HOST_BINDING_RUNTIME_MODULE_NAME}";`) < source.indexOf('import * as __WdlUserModule__ from "./worker.js";') ); assert.match(source, /import \* as __WdlHostRuntime__/); - assert.doesNotMatch(source, /__wdlRunWithRequestContext/); - for (const intrinsic of [ - "Array.prototype.forEach", - "Function.prototype.toString", - "Headers.prototype.get", - "Object.defineProperty", - "Object.entries", - "Object.keys", - "AsyncLocalStorage.prototype.getStore", - "AsyncLocalStorage.prototype.run", - "Reflect.apply", - "Reflect.get", - "RegExp.prototype.test", - 'Object.getOwnPropertyDescriptor(Request.prototype, "headers")', - ]) { - assert.match(HOST_BINDING_RUNTIME_SOURCE, new RegExp(RegExp.escape(intrinsic))); - } assert.match(HOST_BINDING_RUNTIME_SOURCE, /import \{ sanitizeRequestId \} from "\.\/_wdl-request-id\.js"/); - assert.doesNotMatch(HOST_BINDING_RUNTIME_SOURCE, /\benterWith\b/); - assert.doesNotMatch(source, /Object\.(?:defineProperty|entries|keys)\(/); - assert.doesNotMatch(source, /for \(const .* of /); - assert.doesNotMatch(source, /Function\.prototype\.toString\.call/); -}); - -test("host wrapper request context is invocation-local and preserves return identity", async () => { - const firstStarted = Promise.withResolvers(); - const releaseFirst = Promise.withResolvers(); - const firstResult = hostRequestContext.runWithRequestContext("rid-first", () => { - const result = (async () => { - assert.equal(hostRequestContext.currentRequestId(), "rid-first"); - firstStarted.resolve(undefined); - await releaseFirst.promise; - return hostRequestContext.currentRequestId(); - })(); - assert.equal(hostRequestContext.currentRequestId(), "rid-first"); - return result; - }); - - await firstStarted.promise; - assert.equal(hostRequestContext.currentRequestId(), null); - const secondResult = hostRequestContext.runWithRequestContext("rid-second", async () => { - await Promise.resolve(); - return hostRequestContext.currentRequestId(); - }); - assert.equal(await secondResult, "rid-second"); - releaseFirst.resolve(undefined); - assert.equal(await firstResult, "rid-first"); - assert.equal(hostRequestContext.currentRequestId(), null); - - const nativePromise = Promise.resolve("same"); - assert.equal( - hostRequestContext.runWithRequestContext("rid-identity", () => nativePromise), - nativePromise - ); -}); - -test("host wrapper request context is outermost-only", () => { - const withoutChildId = hostRequestContext.runWithRequestContext("rid-outer", () => - hostRequestContext.runWithRequestContext(null, () => hostRequestContext.currentRequestId())); - const withChildId = hostRequestContext.runWithRequestContext("rid-outer", () => - hostRequestContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); - const withoutParentId = hostRequestContext.runWithRequestContext(null, () => - hostRequestContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); - - assert.equal(withoutChildId, "rid-outer"); - assert.equal(withChildId, "rid-outer"); - assert.equal(withoutParentId, null); - assert.equal(hostRequestContext.currentRequestId(), null); -}); - -test("host wrapper request context rejects malformed child request ids", () => { - const inherited = hostRequestContext.runWithRequestContext("rid-outer", () => - hostRequestContext.runWithRequestContext("bad\\id", () => hostRequestContext.currentRequestId())); - - assert.equal(inherited, "rid-outer"); - assert.equal(hostRequestContext.currentRequestId(), null); -}); - -test("host wrapper runtime keeps independently created request-context stores separate", () => { - assert.equal(hostBindingRuntime.currentRequestId, undefined); - assert.equal(hostBindingRuntime.runWithRequestContext, undefined); - const tenantContext = hostBindingRuntime.createRequestContext(); - - const observed = hostRequestContext.runWithRequestContext("rid-parent", () => - tenantContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); - - assert.equal(observed, "rid-parent"); - assert.equal(hostRequestContext.currentRequestId(), null); -}); - -test("host wrapper request-id extraction ignores tenant-patched request intrinsics", async () => { - const request = new Request("https://worker.example", { - headers: { "x-request-id": "rid-real" }, - }); - await withMockedProperty(Headers.prototype, "get", () => "rid-forged", async () => { - await withMockedPropertyDescriptor(Request.prototype, "headers", { - configurable: true, - get() { - return new Headers({ "x-request-id": "rid-forged" }); - }, - }, async () => { - assert.equal(hostBindingRuntime.requestIdFromEventArg(request), "rid-real"); - }); - }); + assert.doesNotMatch(HOST_BINDING_RUNTIME_SOURCE, /AsyncLocalStorage|node:async_hooks/); }); test("generated host wrappers alias legal entrypoint names without declaration collisions", async () => { @@ -234,44 +124,3 @@ test("generated host wrappers alias legal entrypoint names without declaration c assert.ok(new wrapped[name]({}, {}) instanceof userModule[name]); } }); - -test("host wrapper context uses captured AsyncLocalStorage intrinsics", () => { - const restoreRun = installMockProperty(AsyncLocalStorage.prototype, "run", () => { - throw new Error("live AsyncLocalStorage.run must not run"); - }); - let result; - try { - const requestContext = hostBindingRuntime.createRequestContext(); - result = requestContext.runWithRequestContext("rid-captured", () => { - const restoreGetStore = installMockProperty(AsyncLocalStorage.prototype, "getStore", () => { - throw new Error("live AsyncLocalStorage.getStore must not run"); - }); - try { - return requestContext.currentRequestId(); - } finally { - restoreGetStore(); - } - }); - } finally { - restoreRun(); - } - - assert.equal(result, "rid-captured"); -}); - -test("host wrapper does not inspect or replace tenant return values", () => { - let thenReads = 0; - const value = {}; - Object.defineProperty(value, "then", { - get() { - thenReads += 1; - return undefined; - }, - }); - - const result = hostRequestContext.runWithRequestContext("rid-object", () => value); - - assert.equal(result, value); - assert.equal(thenReads, 0); - assert.equal(hostRequestContext.currentRequestId(), null); -}); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 8fe58c0..6a1d7b9 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -4,8 +4,6 @@ import path from "node:path"; import { extractAssignedConstant, extractBraceBlock, - extractCapnpConstBlock, - extractCapnpListEntries, extractExportedStringConst, extractRegex, extractStringConst, @@ -14,11 +12,9 @@ import { markdownFiles, objectJsonPayloads, scannedTestFiles, + serviceAnchorRegex, withoutStringAndTemplateLiterals, - yamlAliasCount, - yamlDocument, yamlDocuments, - yamlMergeAliases, } from "../helpers/style-contract-scanner.js"; import { jsFiles, readRepoFile, rustFiles, sourceFiles, withoutLineComments } from "../helpers/source-scan.js"; @@ -1175,50 +1171,6 @@ test("active bilingual docs stay paired", () => { assert.deepEqual(offenders.toSorted(), []); }); -test("owning bilingual docs retain selected cross-tier protocol literals", () => { - // Explicitly pin only high-value literals in their owning docs. Do not return - // to scanning every uppercase token, example port, or incidental DB mention. - /** @param {string} value */ - const codeSpan = (value) => `\`${value}\``; - /** @type {Array<{ files: string[], literals: string[] }>} */ - const contracts = [ - { - files: ["docs/modules/infra.md", "docs/modules/infra.zh.md"], - literals: [ - "WDL_INTERNAL_AUTH_TOKEN", - "x-wdl-internal-auth", - ":8088", - ":8787", - ":8788", - ":9120", - ], - }, - { - files: ["docs/redis-key-layout.md", "docs/redis-key-layout.zh.md"], - literals: ["DB 0", "DB 1", "DB 2"], - }, - { - files: ["docs/modules/control-auth.md", "docs/modules/control-auth.zh.md"], - literals: ["X-Admin-Token"], - }, - ]; - for (const suffix of [".v2", "+v2", "~v2", "-v2"]) { - assert.equal(codeSpan(`x-wdl-internal-auth${suffix}`).includes(codeSpan("x-wdl-internal-auth")), false); - } - assert.equal(codeSpan(":8787abc").includes(codeSpan(":8787")), false); - assert.equal(codeSpan("DB 01").includes(codeSpan("DB 0")), false); - const offenders = []; - for (const { files, literals } of contracts) { - for (const file of files) { - const source = readRepoFile(file); - for (const literal of literals) { - if (!source.includes(codeSpan(literal))) offenders.push(`${file}: missing ${literal}`); - } - } - } - assert.deepEqual(offenders, []); -}); - test("protocol and contributor contracts stay discoverable from active docs", () => { const links = [ ["CLAUDE.md", "docs/protocol-contracts.md"], @@ -1456,85 +1408,46 @@ test("owner forward metrics classify non-error HTTP statuses consistently", () = }); test("local compose services inherit shared image anchors", () => { - const document = yamlDocument("docker-compose.yml"); - const merged = /** @type {{ services: Record }} */ ( - yamlDocument("docker-compose.yml", { merge: true }).toJS() - ); - assert.deepEqual(yamlMergeAliases(document, ["x-redis-proxy-service"]), ["rust-sidecar-image"]); - assert.deepEqual(yamlMergeAliases(document, ["x-d1-runtime-service"]), ["workerd-image"]); - assert.deepEqual(yamlMergeAliases(document, ["x-do-runtime-service"]), ["workerd-image"]); - assert.deepEqual( - yamlMergeAliases(document, ["x-redis-proxy-service", "environment"]), - ["internal-auth-env", "secret-envelope-env"] - ); - assert.deepEqual( - yamlMergeAliases(document, ["services", "system-runtime", "environment"]), - ["internal-auth-env", "secret-envelope-env"] - ); - assert.equal(yamlAliasCount(document, "secret-envelope-env"), 2); - - /** @type {Map} */ - const serviceAnchors = new Map(); + const source = readRepoFile("docker-compose.yml"); + const offenders = []; for (const service of ["redis-proxy-user", "redis-proxy-system", "redis-proxy-do"]) { - serviceAnchors.set(service, "redis-proxy-service"); - } - for (const service of ["scheduler", "workflows"]) serviceAnchors.set(service, "rust-sidecar-image"); - for (const service of ["user-runtime", "system-runtime", "gateway"]) { - serviceAnchors.set(service, "workerd-image"); - } - for (const service of ["d1-runtime", "d1-runtime-a", "d1-runtime-b", "d1-runtime-c"]) { - serviceAnchors.set(service, "d1-runtime-service"); - } - for (const service of ["do-runtime", "do-runtime-a", "do-runtime-b", "do-runtime-c"]) { - serviceAnchors.set(service, "do-runtime-service"); + if (!serviceAnchorRegex(service, "redis-proxy-service").test(source)) { + offenders.push(service); + } } - for (const [service, anchor] of serviceAnchors) { - assert.deepEqual(yamlMergeAliases(document, ["services", service]), [anchor], service); + if (!serviceAnchorRegex("scheduler", "rust-sidecar-image").test(source)) { + offenders.push("scheduler"); } - for (const family of ["d1", "do"]) { - assert.equal(merged.services[`${family}-runtime`].profiles, undefined); - for (const suffix of ["a", "b", "c"]) { - assert.deepEqual(merged.services[`${family}-runtime-${suffix}`].profiles, [`${family}-multi`]); + for (const service of ["user-runtime", "system-runtime", "gateway"]) { + if (!serviceAnchorRegex(service, "workerd-image").test(source)) { + offenders.push(service); } } + assert.deepEqual(offenders, []); }); -test("published Compose overrides cover every locally built image service", () => { - const local = /** @type {{ - * services: Record, - * }} */ (yamlDocument("docker-compose.yml", { merge: true }).toJS()); - const publishedDocument = yamlDocument("docker-compose.images.yml"); - const published = /** @type {{ - * services: Record, - * }} */ (yamlDocument("docker-compose.images.yml", { merge: true }).toJS()); - const familyByDockerfile = new Map([ - ["Dockerfile.rust", { alias: "rust-published", image: "docker.io/getwdl/wdl-rust:latest" }], - ["Dockerfile.workerd", { alias: "workerd-published", image: "docker.io/getwdl/wdl-workerd:latest" }], - ]); - for (const family of familyByDockerfile.values()) { - const build = /** @type {{ tag?: string, value?: unknown } | undefined} */ ( - publishedDocument.getIn([`x-${family.alias}`, "build"], true) - ); - assert.equal(build?.tag, "!reset", `${family.alias} must reset inherited build config`); - assert.equal(build?.value, "null", `${family.alias} build reset must use null`); - } - const expected = new Map(); - for (const [service, config] of Object.entries(local.services)) { - if (!config.build) continue; - assert.equal(config.build.context, ".", `${service} build context`); - const family = familyByDockerfile.get(config.build.dockerfile || ""); - assert.ok(family, `${service} must use a recognized shared Dockerfile`); - expected.set(service, family); - } - - assert.deepEqual(Object.keys(published.services).toSorted(), [...expected.keys()].toSorted()); - for (const [service, family] of expected) { - assert.deepEqual(yamlMergeAliases(publishedDocument, ["services", service]), [family.alias], service); - assert.deepEqual(published.services[service], { - image: family.image, - pull_policy: "always", - build: "null", - }); +test("local compose runtime profiles keep base services enabled by default", () => { + const [document] = yamlDocuments("docker-compose.yml"); + const compose = /** @type {{ services?: Record }} */ ( + document.toJS() + ); + assert.ok(compose.services); + for (const family of ["d1", "do"]) { + const baseName = `${family}-runtime`; + const base = compose.services[baseName]; + assert.ok(base, `${baseName} must exist`); + assert.deepEqual(base.entrypoint, [`${family}-supervisor`], `${baseName} effective entrypoint`); + assert.equal(base.profiles, undefined); + for (const suffix of ["a", "b", "c"]) { + const variantName = `${family}-runtime-${suffix}`; + const variant = compose.services[variantName]; + assert.ok(variant, `${variantName} must exist`); + assert.deepEqual( + variant.profiles, + [`${family}-multi`], + variantName + ); + } } }); @@ -1807,126 +1720,6 @@ test("local compose routes private HTTP hops through Envoy only", () => { assert.ok((envoy.match(/preserve_external_request_id: true/g) || []).length >= 5); }); -test("user and system runtime worker contracts differ only at privilege fields", () => { - const user = readRepoFile("runtime/config-user.capnp"); - const system = readRepoFile("runtime/config-system.capnp"); - - const entryMap = (/** @type {string[]} */ entries) => Object.fromEntries( - entries.map((entry) => { - const name = /\(name\s*=\s*"([^"]+)"/.exec(entry); - assert.ok(name, `Cap'n Proto tuple must have a name: ${entry}`); - return [name[1], entry]; - }).toSorted(([left], [right]) => left.localeCompare(right)) - ); - const stringField = (/** @type {string} */ block, /** @type {string} */ field) => { - const match = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*"([^"]+)"`).exec(block); - assert.ok(match, `Cap'n Proto field ${field} must be present`); - return match[1]; - }; - const listField = (/** @type {string} */ block, /** @type {string} */ field) => { - const match = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*(\\[[^\\]]*\\])`).exec(block); - assert.ok(match, `Cap'n Proto field ${field} must be present`); - return match[1].replace(/\s+/g, ""); - }; - const withoutEntries = (/** @type {Record} */ entries, /** @type {string[]} */ names) => Object.fromEntries( - Object.entries(entries).filter(([name]) => !names.includes(name)) - ); - - const userLoader = extractCapnpConstBlock(user, "loaderWorker"); - const systemLoader = extractCapnpConstBlock(system, "loaderWorker"); - assert.deepEqual( - entryMap(extractCapnpListEntries(userLoader, "modules")), - entryMap(extractCapnpListEntries(systemLoader, "modules")) - ); - assert.equal(stringField(userLoader, "compatibilityDate"), stringField(systemLoader, "compatibilityDate")); - assert.equal(listField(userLoader, "compatibilityFlags"), listField(systemLoader, "compatibilityFlags")); - - const userBindings = entryMap(extractCapnpListEntries(userLoader, "bindings")); - const systemBindings = entryMap(extractCapnpListEntries(systemLoader, "bindings")); - assert.deepEqual( - withoutEntries(userBindings, ["SERVICE_NAME", "PUBLIC_NETWORK"]), - withoutEntries(systemBindings, ["SERVICE_NAME", "PUBLIC_NETWORK"]) - ); - assert.match(userBindings.SERVICE_NAME, /text = "user-runtime"/); - assert.match(systemBindings.SERVICE_NAME, /text = "system-runtime"/); - assert.match(userBindings.PUBLIC_NETWORK, /service = "public-network"/); - assert.match(systemBindings.PUBLIC_NETWORK, /service = "network"/); - assert.equal(stringField(userLoader, "globalOutbound"), "internal-network"); - assert.equal(stringField(systemLoader, "globalOutbound"), "network"); - - const userOwner = extractCapnpConstBlock(user, "doOwnerNetworkWorker"); - const systemOwner = extractCapnpConstBlock(system, "doOwnerNetworkWorker"); - assert.deepEqual( - entryMap(extractCapnpListEntries(userOwner, "modules")), - entryMap(extractCapnpListEntries(systemOwner, "modules")) - ); - assert.deepEqual( - entryMap(extractCapnpListEntries(userOwner, "bindings")), - entryMap(extractCapnpListEntries(systemOwner, "bindings")) - ); - assert.equal(stringField(userOwner, "compatibilityDate"), stringField(systemOwner, "compatibilityDate")); - assert.equal(stringField(userOwner, "globalOutbound"), "internal-network"); - assert.equal(stringField(systemOwner, "globalOutbound"), "network"); - - const userTail = extractCapnpConstBlock(user, "tailWorker"); - const systemTail = extractCapnpConstBlock(system, "tailWorker"); - assert.deepEqual( - entryMap(extractCapnpListEntries(userTail, "modules")), - entryMap(extractCapnpListEntries(systemTail, "modules")) - ); - const userTailBindings = entryMap(extractCapnpListEntries(userTail, "bindings")); - const systemTailBindings = entryMap(extractCapnpListEntries(systemTail, "bindings")); - assert.deepEqual( - withoutEntries(userTailBindings, ["SERVICE_NAME"]), - withoutEntries(systemTailBindings, ["SERVICE_NAME"]) - ); - assert.match(userTailBindings.SERVICE_NAME, /text = "user-runtime-tail"/); - assert.match(systemTailBindings.SERVICE_NAME, /text = "system-runtime-tail"/); - assert.equal(stringField(userTail, "compatibilityDate"), stringField(systemTail, "compatibilityDate")); - assert.equal(stringField(userTail, "globalOutbound"), "internal-network"); - assert.equal(stringField(systemTail, "globalOutbound"), "network"); -}); - -test("local workerd configs preserve base service and socket topology", () => { - const pairs = [ - ["gateway/config.capnp", "gateway/config-local.capnp"], - ["runtime/config-user.capnp", "runtime/config-user-local.capnp"], - ["runtime/config-system.capnp", "runtime/config-system-local.capnp"], - ["do-runtime/config.capnp", "do-runtime/config-local.capnp"], - ]; - const entryMap = (/** @type {string[]} */ entries) => Object.fromEntries( - entries.map((entry) => { - const name = /\(name\s*=\s*"([^"]+)"/.exec(entry); - assert.ok(name, `Cap'n Proto tuple must have a name: ${entry}`); - return [name[1], entry]; - }).toSorted(([left], [right]) => left.localeCompare(right)) - ); - const normalizeBaseReference = (/** @type {string} */ entry) => entry.replace(/\.Base\./g, "."); - - for (const [baseFile, localFile] of pairs) { - const base = extractCapnpConstBlock(readRepoFile(baseFile), "config"); - const local = extractCapnpConstBlock(readRepoFile(localFile), "config"); - const baseServices = entryMap(extractCapnpListEntries(base, "services")); - const localServices = entryMap(extractCapnpListEntries(local, "services")); - assert.deepEqual(Object.keys(localServices), Object.keys(baseServices), `${localFile} service names`); - - for (const [name, baseService] of Object.entries(baseServices)) { - const localService = localServices[name]; - if (baseService.includes("external =")) { - assert.match(localService, /external = \(address = "[^"]+", http = \(\)\)/, `${localFile}:${name}`); - } else { - assert.equal(normalizeBaseReference(localService), baseService, `${localFile}:${name}`); - } - } - - assert.deepEqual( - entryMap(extractCapnpListEntries(local, "sockets")), - entryMap(extractCapnpListEntries(base, "sockets")), - `${localFile} sockets` - ); - } -}); - test("S3 query encoding stays aligned between shared and injected runtime helpers", () => { const extract = (/** @type {string} */ file) => { const source = readRepoFile(file); @@ -2484,28 +2277,6 @@ test("internal binary protocol content-types stay centralized in transport const assert.equal(redisRuntimeLoadContentType, runtimeLoadContentType); }); -test("DO owner hints are trusted only from do-runtime headers", () => { - const runtime = readRepoFile("runtime/_wdl-do-transport.js"); - const doRuntime = readRepoFile("do-runtime/index.js"); - - const wrapper = doRuntime.match(/function withOwnerHintHeaders\(response, owner\) \{[^]*?\n}\n\n/)?.[0] || ""; - assert.match(wrapper, /headers\.delete\(DO_OWNER_HINT_CONTROL_HEADER\)/); - assert.match(wrapper, /ownerHintHeaders\(owner\)/); - assert.doesNotMatch(wrapper, /headers\.set\(DO_OWNER_HINT_CONTROL_HEADER/); - - const ownerHintHeadersFn = runtime.match(/export function ownerHintHeaders\(owner, \{ control = false \} = \{\}\) \{[^]*?\n}\n\n/)?.[0] || ""; - assert.match(ownerHintHeadersFn, /\[DO_OWNER_HINT_HEADERS\.ownerKey\]/); - assert.match(ownerHintHeadersFn, /\[DO_OWNER_HINT_HEADERS\.taskId\]/); - assert.match(ownerHintHeadersFn, /\[DO_OWNER_HINT_HEADERS\.endpoint\]/); - assert.match(ownerHintHeadersFn, /\[DO_OWNER_HINT_HEADERS\.generation\]/); - assert.match(ownerHintHeadersFn, /if \(control\) headers\[DO_OWNER_HINT_CONTROL_HEADER\] = "1";/); - - assert.doesNotMatch(runtime, /export const OWNER_(?:HINT_STALE|RACE_RETRY)_CODES/); - assert.match(runtime, /const code = responseHeader\(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER\)/); - assert.match(runtime, /setHas\(OWNER_RACE_RETRY_CODES, code\)/); - assert.match(doRuntime, /doPlatformErrorResponse\(err\)/); -}); - test("owner endpoint validation lives in a shared contract owner", () => { const endpoint = readRepoFile("shared/owner-endpoint.js"); const adapter = readRepoFile("runtime/_wdl-owner-endpoint.js"); @@ -2585,7 +2356,6 @@ test("queue delay cap literals stay aligned across standalone tiers", () => { test("Docker images build from pinned public base images", () => { const workerdDockerfile = withoutLineComments(readRepoFile("Dockerfile.workerd")); const rustDockerfile = withoutLineComments(readRepoFile("Dockerfile.rust")); - const cargoWorkspace = withoutLineComments(readRepoFile("rust/Cargo.toml")); const integrationEnvironment = withoutLineComments(readRepoFile("scripts/integration-environment.js")); assert.match(workerdDockerfile, /FROM rust:1-alpine AS supervisor-build/); @@ -2602,54 +2372,6 @@ test("Docker images build from pinned public base images", () => { assert.doesNotMatch(workerdDockerfile, /\bwget\b/); assert.doesNotMatch(workerdDockerfile, /^FROM\s+\S+:latest\b/m); assert.match(integrationEnvironment, /DOCKER_COMPOSE_BUILD_ARGS = \["compose", "build", "gateway", "workflows"\]/); - - const workspaceMembersMatch = /\bmembers\s*=\s*\[([\s\S]*?)\]/.exec(cargoWorkspace); - assert.ok(workspaceMembersMatch, "rust/Cargo.toml must declare workspace members"); - const workspaceMembers = [...workspaceMembersMatch[1].matchAll(/"([^"]+)"/g)] - .map((match) => match[1]) - .toSorted(); - const copiedManifests = (/** @type {string} */ source) => { - const copies = [...source.matchAll(/^COPY rust\/([^/\s]+)\/Cargo\.toml \.\/([^/\s]+)\/$/gm)]; - for (const copy of copies) { - assert.equal(copy[2], copy[1], `Cargo manifest destination must preserve ${copy[1]}`); - } - return copies.map((copy) => copy[1]).toSorted(); - }; - assert.deepEqual(copiedManifests(rustDockerfile), workspaceMembers); - assert.deepEqual(copiedManifests(workerdDockerfile), workspaceMembers); - - const imageLabels = (/** @type {string} */ source) => Object.fromEntries( - [...source.matchAll(/org\.opencontainers\.image\.([a-z]+)="([^"]*)"/g)] - .map((match) => [match[1], match[2]]) - ); - const rustLabels = imageLabels(rustDockerfile); - const workerdLabels = imageLabels(workerdDockerfile); - const sharedLabelKeys = [ - "source", - "documentation", - "vendor", - "authors", - "licenses", - "version", - "revision", - "created", - ]; - assert.deepEqual( - Object.fromEntries(sharedLabelKeys.map((key) => [key, rustLabels[key]])), - Object.fromEntries(sharedLabelKeys.map((key) => [key, workerdLabels[key]])) - ); - assert.notEqual(rustLabels.title, workerdLabels.title); - assert.notEqual(rustLabels.description, workerdLabels.description); - - for (const source of [rustDockerfile, workerdDockerfile]) { - for (const arg of ["WDL_VERSION", "VCS_REF", "BUILD_DATE"]) { - assert.equal( - [...source.matchAll(new RegExp(`^ARG ${arg}(?:=[^\\n]+)?$`, "gm"))].length, - 2, - `${arg} must be declared globally and in the final image stage` - ); - } - } }); test("workerd deploy configs use the supervisor-owned health check binary", () => { diff --git a/tests/unit/test-helper-style-contracts.test.js b/tests/unit/test-helper-style-contracts.test.js index 05168d3..458244a 100644 --- a/tests/unit/test-helper-style-contracts.test.js +++ b/tests/unit/test-helper-style-contracts.test.js @@ -4,26 +4,10 @@ import { readdirSync } from "node:fs"; import { readRepoFile, repoPath, withoutLineComments } from "../helpers/source-scan.js"; import { - extractCapnpConstBlock, scannedTestFiles, - stripCapnpComments, withoutStringAndTemplateLiterals, } from "../helpers/style-contract-scanner.js"; -test("Cap'n Proto comment stripping preserves hashes inside strings", () => { - const source = ` -const demo :Config = ( - url = "https://example.test/path#fragment", - nested = (value = "escaped \\"# hash"), # trailing comment -); # final comment -`; - const stripped = stripCapnpComments(source); - assert.match(stripped, /path#fragment/); - assert.match(stripped, /escaped \\"# hash/); - assert.doesNotMatch(stripped, /trailing comment|final comment/); - assert.match(extractCapnpConstBlock(source, "demo"), /path#fragment/); -}); - test("test files use moduleDataUrl/repositoryFileUrl instead of raw fs+URL boilerplate", () => { // Skips: // test-helper-style-contracts.test.js — holds tripwire literals as error-message text. From a668d8d8030ec1597d4066f7c74919582d07230e Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 06:03:03 +0800 Subject: [PATCH 17/23] Document final-state refactor contracts Remove branch-history wording from active docs, clarify final runtime and D1 contracts, and keep the Compose profile guard focused on effective behavior. Signed-off-by: Lu Zhang --- CHANGELOG.md | 2 +- CLAUDE.md | 4 ++++ docs/modules/control-auth.md | 9 ++++----- docs/modules/control-auth.zh.md | 2 +- docs/modules/d1.md | 10 +++++----- docs/modules/d1.zh.md | 2 +- docs/modules/durable-objects.md | 2 +- docs/modules/durable-objects.zh.md | 2 +- docs/modules/queues-cron.md | 6 +++--- docs/modules/queues-cron.zh.md | 2 +- docs/modules/runtime.md | 4 ++-- docs/modules/runtime.zh.md | 6 +++--- docs/project-standards.md | 7 +++++++ docs/project-standards.zh.md | 2 ++ docs/redis-key-layout.zh.md | 2 +- tests/unit/style-contracts.test.js | 3 +-- 16 files changed, 38 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d28195..4817825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and D1/R2/DO facades resist tenant prototype mutation at capability boundaries, owner records and forwarding targets require service-specific private endpoints, generated wrappers propagate diagnostic request ids on a best-effort basis, and DO ownership retries rely on private do-runtime markers without replaying unknown-outcome non-idempotent requests. - Tightened forward-only input contracts: dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01`, upstream experimental flags, and flags that disable WDL's required enhanced error serialization; internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, DO class names are capped so every shard fits the host-id limit, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. - Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, worker-version, scheduler-projection, and workflow-limit fixtures. -- Normalized `PLATFORM_DOMAIN` across routing tiers, stopped `/whoami` from advertising the internal `workers.local` fallback as a public namespace URL, and aligned Compose/Kubernetes/test wiring with the consolidated owners. +- Normalized `PLATFORM_DOMAIN` across routing tiers, made the published-image Compose overlay pull-only, and limited Kubernetes user-runtime loader ingress to gateway. - Removed the unreachable DO inline worker-code test hook and its Terraform switch; do-runtime invoke paths now accept only canonical persisted bundle identities and reject non-canonical or oversized host ids. - Aligned scheduler and Workflows Compose, Kubernetes, and ECS stop windows with their 25-second application drain so rollout and scale-in do not terminate in-flight work before the configured drain completes. - Upgraded `@wdl-dev/aws-sigv4` to 3.0.1; WDL's S3-only URL-input paths preserve their existing signatures while adopting stable request snapshots, fail-closed redirects, lowercase region validation, and non-blocking response cleanup. diff --git a/CLAUDE.md b/CLAUDE.md index 9fae187..2c3f4af 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -114,6 +114,10 @@ shared crate `wdl-rust-common`. convergence, not a new abstraction. - Do not add fake old/new protocol fallback for in-tree tiers that ship together unless there is a real external rollout requirement. +- Write active repository docs as final-state contracts, not as a history of branch + iterations or rejected approaches. Tests should protect observable behavior and + durable boundaries; do not pin a refactor's implementation shape unless that shape is + itself the contract. - After authentication, in-tree tiers that ship together may trust ephemeral RPC output owned by the peer; do not repeat full semantic validation at every hop. Keep strict validation at tenant inputs, persisted-state readers, credential attachment points, diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index e69ae0e..dd4b480 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -333,11 +333,10 @@ Auth-specific contract: scheduler dispatch open for minutes. - All Control-to-Workflows internal POSTs use the canonical transport in `control/workflows-client.js`. Callers retain endpoint-specific timeout, non-2xx, and - response-body interpretation. Workflow management calls and the - unbounded-by-namespace lifecycle delete scan intentionally have no client-side - timeout, matching their pre-consolidation behavior, while DO-alarm cleanup explicitly - retains its existing five-second bound. Workflow lifecycle blockers fail closed on - service errors. + response-body interpretation. Workflow management calls and the lifecycle delete scan + have no client-side timeout because the scan is unbounded by namespace size. DO-alarm + cleanup uses a five-second timeout. Workflow lifecycle blockers fail closed on service + errors. - AUTH JSRPC errors or Redis explosions are control-plane failures and map to 503 fail-closed behavior, not tenant-visible authorization fallbacks. diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 4ec47e7..936ea13 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -147,7 +147,7 @@ Auth 子合同： - Control 在进入 Redis mutation loop 前加密 secret PUT value。加密/provider 失败会返回 control error，不写 plaintext fallback。 - Worker delete 先 commit Redis lifecycle state；异步 S3 cleanup enqueue 是 best-effort，失败时返回 warning。 - `s3-cleanup` system worker 会把 cleanup task 持久化在 D1；row 存在后由 cron replay 负责重试。S3 失败使用分钟级 exponential backoff，最高 30 分钟；大前缀 cleanup 每完成一个 S3 List/Delete page 就 checkpoint continuation token，因此正常分页进度不会消耗 failure attempts，也不会在 scheduler timeout 后从头开始。每次 run 只处理一页，所以超大 prefix 会跨多个 cron 或 queue dispatch 排空，而不是让单次 scheduler dispatch 持续数分钟。 -- 所有 Control-to-Workflows internal POST 都使用 `control/workflows-client.js` 的 canonical transport。Caller 继续持有 endpoint-specific timeout、non-2xx 和 response-body interpretation。Workflow 管理调用和按 namespace 大小无上界的 lifecycle delete scan 都刻意不设 client-side timeout，保持 transport 收敛前的语义；DO-alarm cleanup 则显式保留原有五秒边界。Workflow lifecycle blocker 在服务错误时 fail closed。 +- 所有 Control-to-Workflows internal POST 都使用 `control/workflows-client.js` 的 canonical transport。Caller 继续持有 endpoint-specific timeout、non-2xx 和 response-body interpretation。Workflow 管理调用和 lifecycle delete scan 不设 client-side timeout，因为 scan 的规模不受 namespace 上限约束；DO-alarm cleanup 使用五秒 timeout。Workflow lifecycle blocker 在服务错误时 fail closed。 - AUTH JSRPC 错误或 Redis 爆炸属于控制面失败，映射为 503 fail closed，而不是 tenant-visible authorization fallback。 ## 安全边界 diff --git a/docs/modules/d1.md b/docs/modules/d1.md index a8c6349..ba9259a 100644 --- a/docs/modules/d1.md +++ b/docs/modules/d1.md @@ -58,8 +58,8 @@ freezing of a binding to a physical `databaseId`. D1 protocol surface. When the outward Control status is 5xx, Control replaces the backend message with `Internal error`, omits raw detail, and preserves only the machine classifier fields `upstreamCode`, `upstreamCategory`, `upstreamRetryable`, - and `upstreamStatus`. Outward 4xx SQL and migration failures retain their existing - D1 diagnostic details. + and `upstreamStatus`. Outward 4xx SQL and migration failures expose D1 diagnostic + details. ## Redis / Storage Contracts @@ -187,10 +187,10 @@ the current local owner and is cleared when that owner is lost or released. Forw router responses must not be scraped into this gauge. Control's D1 runtime failure events for database initialization, query execution, and -migration operations include request and database identity plus bounded +migration operations include request and database identity plus the machine fields `upstream_status`, `upstream_code`, `upstream_category`, `upstream_retryable`, and, -when available, `upstream_cause_code` machine fields. Control does not copy raw backend messages -or response detail into these events. +when available, `upstream_cause_code`. Control does not copy raw backend messages or +response detail into these events. The supervisor emits `drain_timeout_near_owner_ttl` when configured drain timeout is at least half the owner TTL. The warning carries `drain_timeout_ms`, `owner_ttl_ms`, and diff --git a/docs/modules/d1.zh.md b/docs/modules/d1.zh.md index ce6d111..01cf9df 100644 --- a/docs/modules/d1.zh.md +++ b/docs/modules/d1.zh.md @@ -29,7 +29,7 @@ workerd 给 WDL 提供了托管 SQLite-backed service 所需的 isolate 和 loca - Multi-statement `exec()` request 会在一个 actor-local SQLite transaction 中执行。如果后续 statement 失败，同一 request 中此前已执行的 statement 会回滚。 - D1 query/facade 失败使用 D1-specific payload 和 tenant `D1_ERROR` object，而不是 generic platform/admin HTTP envelope。D1 payload 会按需携带 `success:false`、`error`、`message`、`category`、`retryable`、`statementIndex` 和 `causeCode` 等字段。 - D1 error code 使用 D1 compatibility vocabulary，包括 `limit-exceeded`、`sql-error` 这类 `hyphen-case` code。除非 D1 protocol 本身变化，否则不要把它们改写成平台 `snake_case` vocabulary。 -- Control D1 lifecycle API 仍是 admin API，使用 control `{ error, message, ...details }` 合同。d1-runtime internal query/classified error 是独立的 D1 protocol surface。Control 对外状态为 5xx 时会把 backend message 替换为 `Internal error`、移除 raw detail，只保留 `upstreamCode`、`upstreamCategory`、`upstreamRetryable` 和 `upstreamStatus` 机器分类字段；对外 4xx SQL 与 migration failure 继续保留既有 D1 diagnostic detail。 +- Control D1 lifecycle API 是 admin API，使用 control `{ error, message, ...details }` 合同。d1-runtime internal query/classified error 是独立的 D1 protocol surface。Control 对外状态为 5xx 时会把 backend message 替换为 `Internal error`、移除 raw detail，只保留 `upstreamCode`、`upstreamCategory`、`upstreamRetryable` 和 `upstreamStatus` 机器分类字段；对外 4xx SQL 与 migration failure 会暴露 D1 diagnostic detail。 ## Redis / Storage 合同 diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 69d13a5..0c19af3 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -170,7 +170,7 @@ when the logical worker is gone or now points at a different `doStorageId`. backend reconnect budget and client-message buffering without a code rebuild. - Alarm delivery is at-least-once. Scheduler wakes Workflows; Workflows promotes due internal alarm jobs to ready, claims one job under a DB 2 run token, and calls - do-runtime `/internal/do/alarms/dispatch`. do-runtime still constructs the native + do-runtime `/internal/do/alarms/dispatch`. do-runtime constructs a native `DoInvoke{kind:"alarm"}` request and uses the normal owner router/fence path. - Alarm mutation, retarget, dispatch, and whole-worker storage cleanup accept only the canonical positive JavaScript-safe-integer worker version grammar. Invalid internal or diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index de51894..26b007f 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -81,7 +81,7 @@ workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespa - Ordinary fetch/RPC 只有在明确的 stale-owner 或 owner-race response 同时携带 do-runtime 私有 ownership-error control header 时才会回退到 router；tenant response body 不能触发重放。Direct owner transport failure，或 direct owner hop 返回 502/503/504 且没有 fresh owner-hint header 时，会清除 cached hint。安全的 `GET`/`HEAD` request 可以通过 router 重放以重新发现 owner；非幂等 method 和 RPC 会返回 `owner_unavailable`，不会重放，因为 owner 可能已经应用了该请求。 - Shared runtime transport 统一持有 host binding 与 injected facade 的 owner-hint cache wiring、invoke race retry 和 response-header stripping。Connect wrapper 刻意不包含 invoke-only router fallback，以保留 owner-established WebSocket upgrade 语义。 - `WEBSOCKET_RECONNECT_DELAYS_MS` 和 `WEBSOCKET_MAX_BUFFERED_MESSAGES` 可以在不改代码的情况下调整 gateway backend reconnect budget 和 client-message buffer cap。 -- Alarm delivery 是 at-least-once。Scheduler 唤醒 Workflows；Workflows 把到期 internal alarm job promote 到 ready，在 DB 2 run token 下 claim，然后调用 do-runtime `/internal/do/alarms/dispatch`。do-runtime 仍然构造原来的 `DoInvoke{kind:"alarm"}` 请求，并走正常 owner router/fence 路径。 +- Alarm delivery 是 at-least-once。Scheduler 唤醒 Workflows；Workflows 把到期 internal alarm job promote 到 ready，在 DB 2 run token 下 claim，然后调用 do-runtime `/internal/do/alarms/dispatch`。do-runtime 构造 `DoInvoke{kind:"alarm"}` 请求，并走正常 owner router/fence 路径。 - Alarm mutation、retarget、dispatch 和 whole-worker storage cleanup 只接受 canonical positive JavaScript-safe-integer worker version grammar。非法 internal 或 persisted version 会在写入 job 或尝试 worker invoke 前失败。 - Alarm due time 是传给 `setAlarm()` 的 Unix millisecond timestamp。Workflows 和 do-runtime 都用各自本地 wall clock 判断这些 timestamp；如果 backend ready hint 在 SQLite alarm row 对 do-runtime 来说尚未到期时抵达，do-runtime 会 ignore 这次 dispatch，但不清 row，让 backend due-index repair 路径之后继续投递。这是 alarm compatibility 边界，不属于 Redis-time owner lease fence。 - Failed alarm 使用 `WORKFLOWS_DO_ALARM_RETRY_DELAY_MS`、`WORKFLOWS_DO_ALARM_RETRY_MAX_DELAY_MS` 和 `WORKFLOWS_DO_ALARM_RETRY_JITTER` 的 exponential backoff 和 jitter 重试，最多到 `WORKFLOWS_DO_ALARM_RETRY_MAX_TRIES`（默认 `6`），之后 discard 并增加 `do_alarm_dispatches{outcome="discarded"}`。 diff --git a/docs/modules/queues-cron.md b/docs/modules/queues-cron.md index 85a85c6..81e94c9 100644 --- a/docs/modules/queues-cron.md +++ b/docs/modules/queues-cron.md @@ -333,6 +333,6 @@ Representative test anchors: - Main queue streams are intentionally untrimmed; backlog is an operational signal. - Scheduler multi-replica safety is covered by integration tests for cron due-ref claiming, cron sweep recovery, queue reconcile plus consumer-group delivery, delayed - queue promotion, PEL reap, and workflow ticks. Durable Object alarms are now driven - by the Workflows service, not scheduler. Any new scheduler dispatch path must still - be audited for its own Redis lease/fence semantics before assuming replica safety. + queue promotion, PEL reap, and workflow ticks. The Workflows service drives Durable + Object alarms; scheduler does not. Any new scheduler dispatch path must be audited for + its own Redis lease/fence semantics before assuming replica safety. diff --git a/docs/modules/queues-cron.zh.md b/docs/modules/queues-cron.zh.md index 3cdcfb0..69c3da9 100644 --- a/docs/modules/queues-cron.zh.md +++ b/docs/modules/queues-cron.zh.md @@ -211,4 +211,4 @@ Runtime tail logs 也会为 loaded worker 执行输出 `worker_scheduled` 和 `w - Queue `contentType = "v8"` 被拒绝。 - Queue consumer `max_concurrency` 被拒绝。 - Main queue stream 故意不 trim；backlog 是运维信号。 -- 集成测试覆盖了 scheduler 多副本下的 cron due-ref claim、cron sweep recovery、queue reconcile 加 consumer-group delivery、delayed queue promotion、PEL reap 和 workflow tick。Durable Object alarm 现在由 Workflows service 驱动，不再由 scheduler tick。新增 scheduler dispatch 路径前仍必须单独审计它自己的 Redis lease / fence 语义，不能默认具备副本安全性。 +- 集成测试覆盖了 scheduler 多副本下的 cron due-ref claim、cron sweep recovery、queue reconcile 加 consumer-group delivery、delayed queue promotion、PEL reap 和 workflow tick。Durable Object alarm 由 Workflows service 驱动，scheduler 不负责该路径。新增 scheduler dispatch 路径前必须单独审计它自己的 Redis lease / fence 语义，不能默认具备副本安全性。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index c4fabdb..094a466 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -261,7 +261,7 @@ when a matching active tail session exists. - Runtime must roll before scheduler/workflows if they depend on a new `:8088` internal path or dispatch body. - Runtime does not enable workerd's broad `experimental` flag for loaded workers. - Historical-version eviction still injects `__WdlAbort__`, but `abortIsolate()` is + Historical-version eviction injects `__WdlAbort__`, but `abortIsolate()` is available without that flag in the bundled workerd baseline. - Removing the broad loaded-worker `experimental` flag intentionally removes access to non-GA experimental-only tenant surfaces, such as irrevocable long-term stub storage. @@ -275,7 +275,7 @@ when a matching active tail session exists. workerd fail later with a mixed JS/Python bundle error. - The runtime workerd processes still run with process-level `--experimental` because upstream workerd 2026-07-01 continues to gate `workerLoader` bindings on that switch. - Do not re-add the `experimental` compatibility flag or `allowExperimental` to loaded + Do not add the `experimental` compatibility flag or `allowExperimental` to loaded WorkerCode unless another upstream API explicitly requires it. - Upstream workerd 2026-07-01 caps dynamic worker code at 64 MiB and serialized dynamic env at 1 MiB. Control estimates final WorkerCode before version allocation and again diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 7c982db..3a6ae91 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -106,11 +106,11 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - 修改 bundle metadata、wrapper generation 或 binding shape 时，runtime 和 control 应一起滚。 - 如果 scheduler/workflows 依赖新的 `:8088` internal path 或 dispatch body，runtime 必须先滚。 -- Runtime 不再为 loaded worker 开启 workerd 的宽泛 `experimental` flag。Historical-version eviction 仍然注入 `__WdlAbort__`，但当前 bundled workerd baseline 中 `abortIsolate()` 已经不需要该 flag。 -- 移除 loaded worker 的宽泛 `experimental` flag 会有意收紧 tenant 对非 GA experimental-only surface 的访问，例如不可撤销的长期 stub storage。不要为了兼容绕过而重新打开它，除非先完成明确的功能设计。 +- Runtime 不为 loaded worker 开启 workerd 的宽泛 `experimental` flag。Historical-version eviction 会注入 `__WdlAbort__`；当前 bundled workerd baseline 的 `abortIsolate()` 不需要该 flag。 +- Loaded worker 不能访问非 GA experimental-only surface，例如不可撤销的长期 stub storage。除非先完成明确的功能设计，否则不要为兼容绕过打开宽泛 `experimental` flag。 - Control 会在 deploy 时拒绝上游 `$experimental` compatibility enable flags，runtime 也会拒绝仍带有这些 flags 的 retained metadata。`no_*` 这类 disable-style flag 不属于这个 mirror，除非上游把对应 enable flag 本身标为 experimental。 - Python Workers modules 不受支持。Control 会拒绝新的 `py` module manifest，runtime/do-runtime 会拒绝 retained metadata 中的 `py` module，而不是让 workerd 之后抛 mixed JS/Python bundle error。 -- Runtime workerd 进程仍需要进程级 `--experimental`，因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`，除非新的上游 API 明确需要。 +- Runtime workerd 进程需要进程级 `--experimental`，因为上游 workerd 2026-07-01 用它 gate `workerLoader` binding。不要给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`，除非新的上游 API 明确需要。 - 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前和 commit metadata materialization 之后估算最终 WorkerCode，包含 runtime/do-runtime 注入的 wrapper/client modules、workflow import rewrite 和生成的 workflow keys；vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会在 Redis WATCH commit 和 secret mutation 路径里按 headroomed `workerLoader` env budget 做权威检查。Deploy 和 namespace-secret mutation 使用它们实际会加载的 version；worker-secret mutation 还会在 WATCH/COPY bump 事务内重新检查 forced bump，确保 routing flip 前覆盖已分配的 bump version。估算以 JSON bytes 为基底，并对非 Latin-1 字符串补上 V8 two-byte string overhead，因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 - 当前 stock workerd 中，客户端在 async `ReadableStream` response body 中途断开时，不一定调用 stream source 的 `cancel()`。Tenant streaming/SSE worker 应使用自己的 heartbeat、timeout 或应用层 close path，不要把 disconnect-driven `cancel()` 当成唯一资源清理信号。 - workerd 升级仍可能改变默认或 compatibility-flagged runtime surface；升级时要审 exposed surface，而不只审 loader/abort path。 diff --git a/docs/project-standards.md b/docs/project-standards.md index 08260bc..da97500 100644 --- a/docs/project-standards.md +++ b/docs/project-standards.md @@ -178,6 +178,13 @@ Refactors should reduce real complexity: duplicated policy, duplicated contract grammar, fake compatibility paths, stale stubs, or review burden. A large commit is acceptable when it is one coherent direction and its tests cover the touched contract. +Active repository documentation describes the final deployable state. Branch history, +superseded approaches, and review decisions belong in the pull request or changelog, not +in normative module docs. Tests should assert observable behavior, negative correctness +cases, and durable wire, storage, security, or deployment contracts. Do not pin helper +choice, source layout, or another refactor mechanism unless that structure is itself a +supported contract. + Use the smallest check that protects the changed behavior, then widen when the change crosses a module or language boundary. Cross-language changes usually need: diff --git a/docs/project-standards.zh.md b/docs/project-standards.zh.md index 0236881..329f070 100644 --- a/docs/project-standards.zh.md +++ b/docs/project-standards.zh.md @@ -100,6 +100,8 @@ Metrics label 变化、log field 重命名和 request-id propagation 变化都 重构应减少真实复杂度：重复 policy、重复 contract grammar、伪兼容路径、陈旧 stub 或 review 负担。只要方向内聚且测试覆盖被触碰的合同，大提交也可以接受。 +仓库中的 active 文档只描述最终可部署状态。分支过程、已放弃方案和 review 决策应留在 PR 或 changelog，不应写入 normative module doc。测试应断言可观察行为、正确性负例，以及持久 wire、storage、security 或 deployment 合同；除非某种结构本身就是受支持合同，否则不要固定 helper 选择、源码布局或其他重构手段。 + 先运行能保护改动行为的最小检查；当改动跨模块或跨语言时再扩大。跨语言改动通常需要： - workerd tier 的 JS type/unit/style 检查。 diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index 00e4678..33ff541 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -47,7 +47,7 @@ secrets:: Hash, worker-level WDL-ENC envelope ## Route 和 Host Projection -Subdomain routing 读取 `routes:`。Pattern routing 先检查 `declared-hosts`，再读取 `patterns:`，并使用 slot value 中嵌入的 `version` 构造 `x-worker-id`，不再查 `routes:`。Pattern slot value 是由 `shared/route-projection.js` 编码的紧凑 `v2\t\t\t\t\t` record，不再是 JSON。Promote 在同一个 Redis transaction 中更新两套 projection。Control mutation 和 delete 路径遇到无法 decode 的非空 slot 时会 fail closed，不会把未知 owner 当成空槽。 +Subdomain routing 读取 `routes:`。Pattern routing 先检查 `declared-hosts`，再读取 `patterns:`，并直接使用 slot value 中嵌入的 `version` 构造 `x-worker-id`。Pattern slot value 是由 `shared/route-projection.js` 编码的紧凑 `v2\t\t\t\t\t` record。Promote 在同一个 Redis transaction 中更新两套 projection。Control mutation 和 delete 路径遇到无法 decode 的非空 slot 时会 fail closed，不会把未知 owner 当成空槽。 `hosts:` 是 operator intent：这个 namespace 被允许使用这些 host。`declared-hosts` 是 gateway 对“至少被一个 namespace 声明过的 host”的 gate。`host-declarations:` 记录声明该 host 的 namespace，因此一个 namespace 移除声明时，不会在另一个 namespace 仍声明该 host 的情况下清掉全局 gate。`POST /reload` 会先从 `hosts:` 重建这两个声明索引，再发布 gateway cache invalidation；这给 operator-managed host declaration 提供显式 repair/backfill 路径。`ns-hosts:` 是 active reverse index：这个 namespace 当前在这些 host 上拥有至少一个 slot。`hosts:` 应是 superset。Host reconcile 会先用 `ns-hosts:` 做 fast path，再扫描 `patterns:`。 diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 6a1d7b9..d3a5508 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1428,7 +1428,7 @@ test("local compose services inherit shared image anchors", () => { test("local compose runtime profiles keep base services enabled by default", () => { const [document] = yamlDocuments("docker-compose.yml"); - const compose = /** @type {{ services?: Record }} */ ( + const compose = /** @type {{ services?: Record }} */ ( document.toJS() ); assert.ok(compose.services); @@ -1436,7 +1436,6 @@ test("local compose runtime profiles keep base services enabled by default", () const baseName = `${family}-runtime`; const base = compose.services[baseName]; assert.ok(base, `${baseName} must exist`); - assert.deepEqual(base.entrypoint, [`${family}-supervisor`], `${baseName} effective entrypoint`); assert.equal(base.profiles, undefined); for (const suffix of ["a", "b", "c"]) { const variantName = `${family}-runtime-${suffix}`; From e8b2c6159376f9ac46733c8f0e8649ed073e10ed Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 06:12:47 +0800 Subject: [PATCH 18/23] Fence DO registry writes with owner validation Validate ownership before registry side effects and revalidate after registry I/O so lease handoff cannot reach tenant dispatch; add race coverage and repair strict typing in the Compose profile contract test. Signed-off-by: Lu Zhang --- do-runtime/actor.js | 19 ++++++++---- tests/unit/do-runtime-actor.test.js | 45 ++++++++++++++++++++++++----- tests/unit/style-contracts.test.js | 16 ++++++---- 3 files changed, 61 insertions(+), 19 deletions(-) diff --git a/do-runtime/actor.js b/do-runtime/actor.js index 8f56371..9fb4764 100644 --- a/do-runtime/actor.js +++ b/do-runtime/actor.js @@ -93,11 +93,14 @@ export class WdlDoHostActor extends DurableObject { return facetName; } - /** @param {DoInvoke} invoke */ + /** + * @param {DoInvoke} invoke + * @returns {Promise} whether registry I/O was attempted + */ async rememberObject(invoke) { - if (!("doStorageId" in invoke) || typeof invoke.doStorageId !== "string") return; + if (!("doStorageId" in invoke) || typeof invoke.doStorageId !== "string") return false; const member = objectRegistryMember(invoke); - if (this.registeredObjectMembers.has(member)) return; + if (this.registeredObjectMembers.has(member)) return false; try { await rememberDoObject(this.env, invoke); } catch (err) { @@ -107,10 +110,11 @@ export class WdlDoHostActor extends DurableObject { worker_id: workerId, ...formatError(err), }); - return; + return true; } this.registeredObjectMembers.add(member); metrics.setGauge("do_host_actor_object_registry_size", { service: SERVICE }, this.registeredObjectMembers.size); + return true; } /** @param {Request} request */ @@ -177,8 +181,11 @@ export class WdlDoHostActor extends DurableObject { throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); } try { - await this.rememberObject(invoke); - const { owner, leaseRemainingMs } = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); + let fenced = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); + if (await this.rememberObject(invoke)) { + fenced = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); + } + const { owner, leaseRemainingMs } = fenced; return withoutOwnershipErrorControlHeader( await this.dispatchWithLeaseBudget(invoke, owner, leaseRemainingMs, run) ); diff --git a/tests/unit/do-runtime-actor.test.js b/tests/unit/do-runtime-actor.test.js index 5f441fa..b424767 100644 --- a/tests/unit/do-runtime-actor.test.js +++ b/tests/unit/do-runtime-actor.test.js @@ -160,7 +160,32 @@ test("DO host actor: expired initial lease aborts before tenant dispatch", async assert.equal(harness.logs.at(-1).fields.reason, "expired"); }); -test("DO host actor: registry delay completes before the owner fence snapshot", async () => { +test("DO host actor: stale initial owner fence does not write the object registry", async () => { + const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); + const stale = Object.assign(new Error("owner generation is stale"), { + status: 503, + code: "stale_owner_generation", + }); + harness.assertResponses = [stale]; + harness.registryWait = Promise.resolve(); + let registryStarted = false; + harness.registryWaitStarted = () => { + registryStarted = true; + }; + + await assert.rejects( + host.dispatchWithFence(invoke({ + doStorageId: "do_0123456789abcdef0123456789abcdef", + }), () => new Response("should not run")), + /owner generation is stale/ + ); + + assert.equal(harness.assertCalls, 1); + assert.equal(registryStarted, false); + assert.deepEqual(harness.remembered, []); +}); + +test("DO host actor: registry delay is re-fenced before tenant dispatch", async () => { const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); const registryStarted = Promise.withResolvers(); const releaseRegistry = Promise.withResolvers(); @@ -168,9 +193,13 @@ test("DO host actor: registry delay completes before the owner fence snapshot", ownerKey: "do_0123456789abcdef0123456789abcdef:Room:shard0", taskId: "task-a", generation: 7, - leaseExpiresAt: Date.now() - 1, + leaseExpiresAt: Date.now() + 60_000, }; - harness.assertResponses = [owner]; + const stale = Object.assign(new Error("owner generation is stale"), { + status: 503, + code: "stale_owner_generation", + }); + harness.assertResponses = [owner, stale]; harness.registryWait = releaseRegistry.promise; harness.registryWaitStarted = () => registryStarted.resolve(undefined); let ran = false; @@ -187,12 +216,13 @@ test("DO host actor: registry delay completes before the owner fence snapshot", ]); assert.equal(first, "registry"); - assert.equal(harness.assertCalls, 0); + assert.equal(harness.assertCalls, 1); releaseRegistry.resolve(undefined); - await assert.rejects(dispatch, /owner lease has expired/); + await assert.rejects(dispatch, /owner generation is stale/); assert.equal(ran, false); - assert.equal(harness.assertCalls, 1); + assert.equal(harness.assertCalls, 2); + assert.equal(harness.remembered.length, 1); }); test("DO host actor: lease budget reschedules when renew extended the owner fence", async () => { @@ -288,7 +318,7 @@ test("DO host actor: registry remember failure is best-effort and does not fail generation: 7, leaseExpiresAt: Date.now() + 60_000, }; - harness.assertResponses = [owner]; + harness.assertResponses = [owner, owner]; harness.registryError = new Error("redis unavailable"); const response = await host.dispatchWithFence(invoke({ @@ -297,6 +327,7 @@ test("DO host actor: registry remember failure is best-effort and does not fail }), () => Promise.resolve(new Response("ok"))); assert.equal(await response.text(), "ok"); + assert.equal(harness.assertCalls, 2); assert.equal(harness.inFlight, 0); assert.deepEqual(harness.remembered, []); assert.equal(harness.logs.at(-1).level, "warn"); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index d3a5508..60dd6ff 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1434,15 +1434,19 @@ test("local compose runtime profiles keep base services enabled by default", () assert.ok(compose.services); for (const family of ["d1", "do"]) { const baseName = `${family}-runtime`; - const base = compose.services[baseName]; - assert.ok(base, `${baseName} must exist`); - assert.equal(base.profiles, undefined); + const baseService = /** @type {{ profiles?: unknown } | undefined} */ ( + compose.services[baseName] + ); + assert.ok(baseService, `${baseName} must exist`); + assert.equal(baseService.profiles, undefined); for (const suffix of ["a", "b", "c"]) { const variantName = `${family}-runtime-${suffix}`; - const variant = compose.services[variantName]; - assert.ok(variant, `${variantName} must exist`); + const variantService = /** @type {{ profiles?: unknown } | undefined} */ ( + compose.services[variantName] + ); + assert.ok(variantService, `${variantName} must exist`); assert.deepEqual( - variant.profiles, + variantService.profiles, [`${family}-multi`], variantName ); From e7ec7c33b22ce79fa0c517571311b66b61820783 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 10:26:00 +0800 Subject: [PATCH 19/23] Finalize refactor contracts and integration scheduling Close trusted DO owner-race retry and response cleanup gaps. Consolidate remaining lifecycle and control contracts. Refresh four-shard integration ordering from the passing 54-file full run. Signed-off-by: Lu Zhang --- control/env-budget.js | 8 +- control/handlers/delete.js | 24 ++- control/handlers/ns-secrets.js | 1 - control/handlers/versions.js | 12 +- control/handlers/worker-secrets.js | 2 - control/lib.js | 11 ++ control/routing.js | 17 +- control/shared.js | 10 ++ do-runtime/load.js | 8 +- do-runtime/owner-registry.js | 6 - docs/modules/cli.md | 3 +- docs/modules/cli.zh.md | 2 +- docs/modules/control-auth.md | 5 +- docs/modules/control-auth.zh.md | 2 +- docs/modules/durable-objects.md | 35 ++-- docs/modules/durable-objects.zh.md | 9 +- docs/modules/gateway.md | 10 +- docs/modules/gateway.zh.md | 2 +- gateway/index.js | 4 +- runtime/_wdl-do-transport.js | 44 +++-- runtime/_wdl-owner-endpoint.js | 2 +- runtime/lib.js | 1 - runtime/load.js | 2 +- runtime/load/env-build.js | 11 +- runtime/r2-client.js | 10 +- rust/scheduler/src/cron/sweep.rs | 1 - rust/scheduler/src/queue/consume.rs | 11 +- rust/scheduler/src/queue/delivery/dispatch.rs | 6 +- rust/scheduler/src/queue/orphan.rs | 7 +- rust/workflows/src/api.rs | 4 +- rust/workflows/src/api/lifecycle/restart.rs | 38 ++-- scripts/integration-test-plan.js | 42 ++--- shared/owner-endpoint.js | 4 +- tests/helpers/do-owner-hint.js | 6 +- tests/helpers/load-auth-index.js | 4 + tests/helpers/load-control-lib.js | 1 + tests/unit/control-delete-handler.test.js | 17 ++ tests/unit/control-env-budget.test.js | 22 --- tests/unit/do-runtime-load.test.js | 3 + tests/unit/do-runtime-protocol.test.js | 3 + tests/unit/integration-test-plan.test.js | 2 +- tests/unit/runtime-do-client.test.js | 168 ++++++++++++------ tests/unit/runtime-lib.test.js | 8 +- tests/unit/runtime-load.test.js | 13 -- 44 files changed, 338 insertions(+), 263 deletions(-) diff --git a/control/env-budget.js b/control/env-budget.js index 5f11e13..b96d759 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -1,6 +1,6 @@ import { decryptSecretValue } from "shared-secret-envelope"; import { BundleMetaError, parseBundleMeta } from "control-lib"; -import { buildWorkerEnv } from "runtime-load-env-build"; +import { buildWorkerEnv, doAlarmBindingProps } from "runtime-load-env-build"; import { bundleKey } from "shared-version"; import { WatchError } from "shared-redis"; @@ -160,7 +160,7 @@ export function estimatedWorkerLoaderEnv({ if (typeof doAlarmStorageId === "string" && doAlarmStorageId) { estimated[DO_ALARMS_BINDING] = { __wdlBinding: "do-alarms", - props: { ns, worker, version, doStorageId: doAlarmStorageId }, + props: doAlarmBindingProps({ ns, worker, version, doStorageId: doAlarmStorageId }), }; } return estimated; @@ -330,7 +330,6 @@ export async function decryptMutatedSecretHashForBudget({ * nsSecrets?: Record | null, * workerSecrets?: Record | null, * assetsCdnBase?: string | null, - * retryMissingVersions?: boolean, * }} args */ export async function assertWorkerVersionsUserEnvBudget({ @@ -342,7 +341,6 @@ export async function assertWorkerVersionsUserEnvBudget({ nsSecrets = null, workerSecrets = null, assetsCdnBase = ESTIMATED_ASSETS_CDN_BASE, - retryMissingVersions = false, }) { const checks = [ ...[...versions] @@ -369,7 +367,7 @@ export async function assertWorkerVersionsUserEnvBudget({ // whose command protocol is single-flight even though secret decryption is not. for (const { sourceVersion, estimatedVersion } of uniqueChecks) { const rawMeta = await redis.hGet(bundleKey(ns, worker, sourceVersion), "__meta__"); - if (typeof rawMeta !== "string" && retryMissingVersions) throw new WatchError(); + if (typeof rawMeta !== "string") throw new WatchError(); const meta = parseBundleMeta(rawMeta, { ns, worker, diff --git a/control/handlers/delete.js b/control/handlers/delete.js index 2313343..5e95761 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -7,7 +7,7 @@ import { jsonResponse, jsonError, requireControlLog, requireControlRedis, errMessage, - acquireDeleteLock, releaseDeleteLock, renewDeleteLock, + acquireDeleteLock, releaseDeleteLock, renewDeleteLock, deleteLockExpiredDetails, assertWorkflowDeleteAllowed, cleanupDoAlarmsForWorker, buildS3CleanupTaskId, recordCleanupIntentOrWarn, ControlAbort, codedErrorLogFields, controlAbortResponse, @@ -22,6 +22,7 @@ import { formatReferrerBlocker, bundleAssetPrefix, parseBundleMeta, + parsePatternProjection, } from "control-lib"; import { WHOLE_DELETE_LOCK_KIND, @@ -37,7 +38,6 @@ import { import { workerSecretsKey } from "shared-secret-keys"; import { decodeBulk, WatchError } from "shared-redis"; import { queueConsumerScanPrefix } from "shared-queue-keys"; -import { decodePatternProjection } from "shared-route-projection"; import { discardResponseBody } from "shared-respond"; import { buildWorkerDeleteCleanup, stageWorkerDelete } from "control-handlers-delete-plan"; @@ -95,15 +95,14 @@ class WholeDeleteError extends ControlAbort {} /** @param {unknown} raw @param {string} host @param {string} slot */ function requirePatternProjection(raw, host, slot) { - const projection = decodePatternProjection(raw); - if (!projection) { - throw new WholeDeleteError(500, "corrupt_pattern_projection", { - host, - slot, + return parsePatternProjection(raw, { + host, + slot, + makeError: (details) => new WholeDeleteError(500, "corrupt_pattern_projection", { + ...details, stage: "pattern_projection_parse", - }); - } - return projection; + }), + }); } /** @param {string} ns @param {string} name @returns {never} */ @@ -116,10 +115,7 @@ function throwWholeDeleteContention(ns, name) { /** @param {string} ns @param {string} name @returns {never} */ function throwDeleteLockExpired(ns, name) { - throw new WholeDeleteError(409, "deleting", { - namespace: ns, name, - message: "worker delete lock expired; retry the request", - }); + throw new WholeDeleteError(409, "deleting", deleteLockExpiredDetails(ns, name)); } // Raised when collection or an under-WATCH check observes state that diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 9144088..9828d71 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -126,7 +126,6 @@ async function validateNamespaceSecretBudget({ nsSecrets, workerSecrets, assetsCdnBase: controlEnv.ASSETS_CDN_BASE, - retryMissingVersions: true, }); } } diff --git a/control/handlers/versions.js b/control/handlers/versions.js index d684647..e5500a3 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -1,6 +1,6 @@ import { jsonResponse, jsonError, - acquireDeleteLock, releaseDeleteLock, renewDeleteLock, + acquireDeleteLock, releaseDeleteLock, renewDeleteLock, deleteLockExpiredDetails, assertWorkflowDeleteAllowed, buildS3CleanupTaskId, recordCleanupIntentOrWarn, ControlAbort, codedErrorLogFields, controlAbortResponse, @@ -97,10 +97,7 @@ async function handleDelete({ ns, name, version, principal, requestId }) { ns, worker: name, version, allowCleanup: true, requestId, }); if (!await renewDeleteLock(redis, ns, name, lockToken)) { - throw new VersionDeleteError(409, "deleting", { - namespace: ns, name, version, - message: "worker delete lock expired; retry the request", - }); + throw new VersionDeleteError(409, "deleting", deleteLockExpiredDetails(ns, name, version)); } result = await executeVersionDelete({ redis, ns, name, version, principal, requestId, lockToken, @@ -177,10 +174,7 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque if (await iso.get(deleteLockKey(ns, name)) !== lockToken) { await iso.unwatch(); - throw new VersionDeleteError(409, "deleting", { - namespace: ns, name, version, - message: "worker delete lock expired; retry the request", - }); + throw new VersionDeleteError(409, "deleting", deleteLockExpiredDetails(ns, name, version)); } const currentActive = await iso.hGet(routesKey(ns), name); diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 36a351a..782f94a 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -457,7 +457,6 @@ async function stageWorkerSecretForBump({ nsSecrets, workerSecrets, assetsCdnBase: controlEnv.ASSETS_CDN_BASE, - retryMissingVersions: true, }); stageWorkerSecretMutation(multi, { @@ -551,7 +550,6 @@ async function mutateSecretWithoutActive({ redis, ns, name, key, method, value, nsSecrets, workerSecrets, assetsCdnBase: controlEnv.ASSETS_CDN_BASE, - retryMissingVersions: true, }); const multi = iso.multi(); diff --git a/control/lib.js b/control/lib.js index fcdcc8f..5f83227 100644 --- a/control/lib.js +++ b/control/lib.js @@ -13,6 +13,7 @@ import { WDL_RESERVED_BINDING_RE, KV_ID_RE, } from "shared-ns-pattern"; +import { decodePatternProjection } from "shared-route-projection"; import { PLATFORM_TIER_RESERVED_NS, ROLES } from "shared-auth-roles"; import { errorMessage } from "shared-errors"; export const NS_RE = new RegExp(`^${NS_PATTERN}$`); @@ -72,6 +73,16 @@ export function parseBundleMeta(raw, { ns, worker, version, makeError }) { return /** @type {Record} */ (parsed); } +/** + * @param {unknown} raw + * @param {{ host: string, slot: string, makeError: (details: { host: string, slot: string }) => Error }} options + */ +export function parsePatternProjection(raw, { host, slot, makeError }) { + const projection = decodePatternProjection(raw); + if (!projection) throw makeError({ host, slot }); + return projection; +} + /** * @param {Record} meta * @returns {string | null} diff --git a/control/routing.js b/control/routing.js index ea16b37..6792c49 100644 --- a/control/routing.js +++ b/control/routing.js @@ -7,6 +7,7 @@ import { d1DatabaseKey, extractD1Refs, extractOutgoingRefs, parseBundleMeta, + parsePatternProjection, } from "control-lib"; import { cronWorkerKey, @@ -19,7 +20,7 @@ import { stageWorkerVersionIndexUpsert, } from "control-lifecycle-indexes"; import { parseHostList } from "control-topology"; -import { decodePatternProjection, encodePatternProjection } from "shared-route-projection"; +import { encodePatternProjection } from "shared-route-projection"; import { DECLARED_HOSTS_KEY, NAMESPACES_KEY, @@ -89,16 +90,16 @@ export class RoutingError extends Error { /** @param {unknown} raw @param {string} host @param {string} slot */ function requirePatternProjection(raw, host, slot) { - const projection = decodePatternProjection(raw); - if (!projection) { - throw new RoutingError( + return parsePatternProjection(raw, { + host, + slot, + makeError: (details) => new RoutingError( 500, "corrupt_pattern_projection", `Pattern projection for ${host}${slot} is corrupt`, - { host, slot } - ); - } - return projection; + details + ), + }); } /** @param {Array} keys @returns {string[]} */ diff --git a/control/shared.js b/control/shared.js index 7b45265..bcc1e59 100644 --- a/control/shared.js +++ b/control/shared.js @@ -469,6 +469,16 @@ export async function renewDeleteLock(redis, ns, worker, token) { ); } +/** @param {string} ns @param {string} worker @param {string} [version] */ +export function deleteLockExpiredDetails(ns, worker, version) { + return { + namespace: ns, + name: worker, + ...(version === undefined ? {} : { version }), + message: "worker delete lock expired; retry the request", + }; +} + // Compare-and-delete so we don't free a token a TTL-expired-then-reacquired // lock holder is depending on. /** diff --git a/do-runtime/load.js b/do-runtime/load.js index a86fca7..4fcfa6f 100644 --- a/do-runtime/load.js +++ b/do-runtime/load.js @@ -1,6 +1,7 @@ import { bundleToWorkerCode } from "runtime-lib"; import { buildWorkerEnv, + doAlarmBindingProps, decodeRuntimeLoadPayload, internalAuthBackend, runtimeLoadContentTypeMatches, @@ -201,12 +202,7 @@ export function buildDoEnv(meta, nsSecrets, workerSecrets, env, ctx, invoke) { } ); out.__WDL_DO_ALARMS__ = ctx.exports.DoAlarmBinding({ - props: { - ns: invoke.ns, - worker: invoke.worker, - version: invoke.version, - doStorageId: invoke.doStorageId, - }, + props: doAlarmBindingProps(invoke), }); return out; } diff --git a/do-runtime/owner-registry.js b/do-runtime/owner-registry.js index c87d3b2..80c2eaf 100644 --- a/do-runtime/owner-registry.js +++ b/do-runtime/owner-registry.js @@ -191,9 +191,6 @@ export function parseOwner(raw, expectedOwnerKey, expectedBundleScope = null) { * @returns {DoOwner} */ function ownerRecordFor(env, invoke, localTask, generation, nowMs) { - if (!validOwnerEndpointForService(localTask.endpoint, DO_OWNER_PORT, "do-runtime")) { - throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner endpoint is invalid"); - } const ttl = ownerTtlSeconds(env); const scope = requireInvokeScope(invoke); return { @@ -218,9 +215,6 @@ function ownerRecordFor(env, invoke, localTask, generation, nowMs) { * @returns {DoOwner} */ function renewedOwnerRecordFor(env, current, localTask, nowMs) { - if (!validOwnerEndpointForService(localTask.endpoint, DO_OWNER_PORT, "do-runtime")) { - throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner endpoint is invalid"); - } const ttl = ownerTtlSeconds(env); return { ...current, diff --git a/docs/modules/cli.md b/docs/modules/cli.md index cacf5d0..5e124f9 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -83,7 +83,8 @@ CLI output may display: - `minCliVersion`: the minimum downstream CLI version supported by this platform build. - `urls.control`: the control origin that the request actually reached. - `urls.namespace`: the tenant namespace origin, returned only for namespace tokens when - the platform explicitly configures a public `PLATFORM_DOMAIN`. + the platform explicitly configures `PLATFORM_DOMAIN`; Control does not validate that + the configured hostname is publicly reachable. - `urls.assets`: the configured public assets base URL, returned only when the control plane has a safe absolute `http`/`https` `ASSETS_CDN_BASE`; query and fragment are stripped before returning the hint. diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index ebc1f6c..baebfbd 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -51,7 +51,7 @@ CLI 可以展示： - `platformVersion`：control 返回的 WDL platform version。Canonical derivation 记录在 `control-auth.zh.md` 的 `/whoami` 段；CLI 应直接展示这个值，不应从 package metadata 自行重建。 - `minCliVersion`：当前 platform build 支持的最低下游 CLI 版本。 - `urls.control`：请求实际到达的 control origin。 -- `urls.namespace`：tenant namespace origin，只在 caller 是 namespace token 且 platform 显式配置 public `PLATFORM_DOMAIN` 时返回。 +- `urls.namespace`：tenant namespace origin，只在 caller 是 namespace token 且 platform 显式配置 `PLATFORM_DOMAIN` 时返回；Control 不校验该 hostname 是否公网可达。 - `urls.assets`：配置的 public assets base URL，只在 control plane 有安全的绝对 `http`/`https` `ASSETS_CDN_BASE` 时返回；返回前会去掉 query 和 fragment。 CLI 应把这些字段当作 diagnostics 和 user-facing guidance 的默认值，而不是替代用户显式配置。如果 `minCliVersion` 大于当前 CLI 版本，CLI 应在执行 mutating command 前 warning 或 fail。可选 URL hint 缺失时，应展示为 unavailable，不应自行猜测。 diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index dd4b480..3459161 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -147,8 +147,9 @@ Control lifecycle operations are split so each critical transition has one autho single `x-forwarded-proto` value of `http` or `https`, `/whoami` uses that protocol for `urls.control` and `urls.namespace`; otherwise it falls back to the request URL protocol seen by control. `urls.namespace` is returned only for tenant namespace - tokens when `PLATFORM_DOMAIN` is explicitly configured as a public hostname; Control - does not advertise the internal `workers.local` fallback as a public URL. + tokens when `PLATFORM_DOMAIN` is explicitly configured. The gate is grammar-only — + any valid DNS hostname passes and publicness is the operator's responsibility; + Control never advertises the unset `workers.local` fallback as a public URL. `urls.assets` is returned only when `ASSETS_CDN_BASE` is a safe absolute `http`/`https` URL, with query and fragment stripped. The endpoint must not grow into token list, token lookup, or secret-bearing diagnostics. diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 936ea13..203d2bb 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -73,7 +73,7 @@ Control lifecycle 操作会拆开处理，确保每个关键 transition 只有 - Worker delete lock value 由 `whole:` 或 `version:` 操作类型加服务端生成的随机 token 组成；`s3cleanup:` task id 也由服务端生成。`x-request-id` 只用于诊断，客户端或重试可能复用它；不能把它用作 lock token 或 cleanup-task id。Delete、D1 migration 和 delegated issue lock 使用 `shared/redis-lock.js` 的 token-fenced primitive；release 按 token 限定且是 best-effort，因为 TTL expiry 会限制 advisory lock 泄漏时间，release error 不能覆盖操作的真实结果。Version delete 和 whole-worker delete 会在最终 lifecycle mutation 前续租一次，并在最终 WATCH snapshot 内验证 token，因此过期请求不能借替代 holder 的 lock 提交。 - Auth 不是一层约定俗成的 middleware。`parseControlRoute()` 分配 action，control 把 action 和 namespace 发给 auth，auth 用存储的 token record 对照 `shared/auth-roles.js` 评估权限。Dispatcher 代码不应自己从 URL prefix 推断权限。 - Delegated token issue 刻意不放进 direct `auth.token.*` lifecycle。`POST /auth/delegated-tokens` 使用 action `auth.delegated_token.issue`，因此 `/auth/tokens` issue/list/revoke 仍保持 strict ops-only；`token-issuer` credential 只能请求 Auth 按一个已配置 template materialize token。 -- `/whoami` 是唯一 namespace-less non-ops action。它对任何有效 token 开放，因为只报告当前 token 自己的 principal、token id、request id 和 public diagnostics。`platformVersion` 是由 bundled workerd date version 派生的 WDL version，使用 `wdl.` namespace，例如 `workerd` `1.20260531.1` 会变成 `wdl.20260531.1`；它不是 project release tag，后者的末尾 counter 可以在同一个 workerd date 上为 WDL-only patch 递增。raw workerd version 不返回。`minCliVersion` 是最低支持的下游 CLI 版本。`urls.control` 是到达 control 的 public origin；当 ingress 提供单个 `x-forwarded-proto` 且值为 `http` 或 `https` 时，`/whoami` 会用该协议生成 `urls.control` 和 `urls.namespace`，否则回退到 control 看到的 request URL 协议。`urls.namespace` 只在 caller 是 tenant namespace token 且显式配置了 public hostname 形态的 `PLATFORM_DOMAIN` 时返回；Control 不会把内部 `workers.local` fallback 伪装成 public URL。`urls.assets` 只在 `ASSETS_CDN_BASE` 是安全的绝对 `http`/`https` URL 时返回，并且会去掉 query 和 fragment。它不能扩展成 token list、token lookup 或携带 secret 的 diagnostics。 +- `/whoami` 是唯一 namespace-less non-ops action。它对任何有效 token 开放，因为只报告当前 token 自己的 principal、token id、request id 和 public diagnostics。`platformVersion` 是由 bundled workerd date version 派生的 WDL version，使用 `wdl.` namespace，例如 `workerd` `1.20260531.1` 会变成 `wdl.20260531.1`；它不是 project release tag，后者的末尾 counter 可以在同一个 workerd date 上为 WDL-only patch 递增。raw workerd version 不返回。`minCliVersion` 是最低支持的下游 CLI 版本。`urls.control` 是到达 control 的 public origin；当 ingress 提供单个 `x-forwarded-proto` 且值为 `http` 或 `https` 时，`/whoami` 会用该协议生成 `urls.control` 和 `urls.namespace`，否则回退到 control 看到的 request URL 协议。`urls.namespace` 只在 caller 是 tenant namespace token 且显式配置了 `PLATFORM_DOMAIN` 时返回；该门槛只校验 DNS 语法，任何语法合法的 hostname 都会通过，是否公网可达由运维负责。Control 绝不把未配置时的 `workers.local` fallback 伪装成 public URL。`urls.assets` 只在 `ASSETS_CDN_BASE` 是安全的绝对 `http`/`https` URL 时返回，并且会去掉 query 和 fragment。它不能扩展成 token list、token lookup 或携带 secret 的 diagnostics。 - 只有 method、path length 和 verb 精确匹配 authorized shape 时，route shape 才带 `action`。缺 action 是有意行为：非 ops 会触发 auth unknown-action red line，ops 仍可到 dispatcher/handler 的 path 和 method validation，而不是被 auth 拦下。`/reload` 这类已知 top-level route 的 wrong-method case 会在 ops auth 通过后返回 `405 method_not_allowed`。 ## Redis / Storage 合同 diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 0c19af3..6069ef0 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -64,12 +64,13 @@ partial batch result; that is a result envelope, not a generic JSON error envelo Tenant-originated DO fetch bodies are capped at 1 MiB in the runtime facade. The facade rejects an oversized `Content-Length` before reading, and streamed bodies are read incrementally so the cap is enforced before buffering the full body. + DO RPC method names use the JavaScript identifier grammar. The do-runtime protocol -reader caps them at 256 ASCII bytes. RPC arguments are -structural JSON data capped at 1 MiB: finite numbers, strings, booleans, null, dense -arrays, and plain objects are accepted. Serialization does not invoke `toJSON()` hooks; -sparse arrays, circular structures, non-plain objects, and non-JSON values fail before -dispatch. +reader caps them at 256 ASCII bytes. RPC arguments are structural JSON data capped at +1 MiB: finite numbers, strings, booleans, null, dense arrays, and plain objects are +accepted. Serialization does not invoke `toJSON()` hooks; sparse arrays, circular +structures, non-plain objects, and non-JSON values fail before dispatch. + do-runtime invokes tenant alarm and RPC methods through private fetch dispatches intercepted by the generated wrapper. Those requests carry the outer request id so the host facade can propagate it without adding platform metadata to the tenant argument @@ -78,11 +79,14 @@ or re-entrant calls may observe another invocation's id. Nested DO fetch/connect requests discard tenant-supplied `x-request-id` values and propagate the sanitized context id when available. Request ids remain best-effort, untrusted diagnostic metadata. + DO invoke envelopes identify persisted bundles by canonical namespace, worker, version, and storage id. They do not accept inline worker source. + Tenant-facing DO object names and ids must be well-formed Unicode strings. Lone UTF-16 surrogates are rejected by `idFromName()` / `idFromString()` before hashing or dispatch; do-runtime alarm ingress and Workflows revalidation enforce the same boundary. + DO host ids are capped at 512 UTF-8 bytes and use canonical `shardN` suffixes without leading zeroes. DO binding class names use the ASCII JavaScript class-name grammar and are capped at 468 bytes at deploy, so every shard suffix fits the aggregate host-id cap. @@ -155,13 +159,13 @@ when the logical worker is gone or now points at a different `doStorageId`. disconnect as the primary safety mechanism. Client messages queued under an older backend reconnect epoch may be discarded without per-frame ack/nack when the gateway resets that epoch. -- Ordinary fetch/RPC can fall back through the router after explicit stale-owner or - owner-race responses carrying do-runtime's private ownership-error control header. - Tenant response bodies cannot opt into replay. A direct owner transport failure, or a - 502/503/504 without a fresh owner-hint header, evicts the cached hint. Safe `GET`/`HEAD` - requests may replay through the router to rediscover the owner; non-idempotent methods - and RPC return `owner_unavailable` without replay because the owner may already have - applied the request. +- Ordinary fetch/RPC can perform one router rediscovery after a trusted owner-hint or an + explicit pre-dispatch stale-owner/owner-race response carrying do-runtime's private + ownership-error control header, including for non-idempotent methods and RPC. Tenant + response bodies cannot opt into replay. An unmarked direct owner transport failure, or + a 502/503/504 without either trusted marker, evicts the cached hint. Safe `GET`/`HEAD` + requests may replay through the router; non-idempotent methods and RPC return + `owner_unavailable` because the owner may already have applied the request. - The shared runtime transport owns owner-hint cache wiring, invoke race retry, and response-header stripping for both the host binding and injected facade. Its connect wrapper deliberately omits the invoke-only router fallback required to preserve @@ -340,10 +344,11 @@ recovery after the initial 101. that epoch. - Owner-hinted WebSocket direct retry failures do not fall back to the router, because the final 101 must come from the owner endpoint. -- Owner-hinted ordinary fetch/RPC direct failures only fall back to the router for safe +- Unmarked ordinary fetch/RPC direct failures only fall back to the router for safe `GET`/`HEAD` requests. Non-idempotent methods and RPC return `owner_unavailable` when - the outcome may be unknown. Explicit stale-owner and owner-race responses remain - retryable because they prove the owner-side fence rejected the request. + the outcome may be unknown. A trusted owner-hint or explicit stale-owner/owner-race + response remains retryable once for every method because it proves dispatch did not + reach tenant code. - Renamed/deleted migrations are deferred. - Long handlers still need user-level care; lease-budget watchdogs protect platform ownership and narrow failover races, not every storage call or the final SQLite diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 26b007f..3ae06b4 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -35,10 +35,15 @@ Storage cleanup endpoint 是 native facet storage cleanup 和 worker storage cle DO protocol error 使用 `{ error, message, details? }`。不同于 admin HTTP 的 flat additive error shape，DO protocol detail 会嵌套在 `details` 下，因为消费者是 runtime/DO client protocol，不是通用 admin JSON parser。未知 internal exception 仍会降级成安全的 `internal_error` / `Internal error` message。Storage delete-worker 在 partial batch result 时可能返回 HTTP 207 和 `{ ok:false, deleted, errors }`；这是 result envelope，不是 generic JSON error envelope。 Tenant-originated DO fetch body 在 runtime facade 中限制为 1 MiB。Facade 会在读取前拒绝超限的 `Content-Length`，streamed body 会增量读取，因此 limit 会在完整 buffering 前生效。 + DO RPC method name 使用 JavaScript identifier grammar，do-runtime protocol reader 将其限制为最多 256 ASCII bytes。RPC 参数是最多 1 MiB 的 structural JSON data：接受 finite number、string、boolean、null、dense array 和 plain object。序列化不会调用 `toJSON()` hook；sparse array、circular structure、non-plain object 和 non-JSON value 会在 dispatch 前失败。 + do-runtime 会通过 generated wrapper 截获的私有 fetch dispatch 调用 tenant alarm 和 RPC method；这些 request 携带外层 request id，使 host facade 可以传播该 id，且不会把平台 metadata 加入 tenant argument list。持久化 class instance 使用一个小型可变诊断 context，因此 concurrent 或 re-entrant call 可能观察到另一次 invocation 的 id。嵌套 DO fetch/connect request 会丢弃 tenant 提供的 `x-request-id`，并在 context id 可用时传播其净化值；request id 仍是 best-effort、不可信的诊断 metadata。 + DO invoke envelope 通过 canonical namespace、worker、version 和 storage id 标识 persisted bundle，不接受 inline worker source。 + Tenant-facing DO object name/id 必须是 well-formed Unicode string。`idFromName()` / `idFromString()` 会在 hash 或 dispatch 前拒绝 lone UTF-16 surrogate；do-runtime alarm ingress 和 Workflows revalidation 执行相同边界。 + DO host id 最多 512 UTF-8 bytes，并使用不带前导零的 canonical `shardN` suffix。DO binding class name 使用 ASCII JavaScript class-name grammar，并在 deploy 时限制为最多 468 bytes，确保所有 shard suffix 都能满足 aggregate host-id 上限。 ## Redis / Storage 合同 @@ -78,7 +83,7 @@ workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespa - Whole-worker delete 后 redeploy 会分配新的 `doStorageId`；旧 native storage tombstone 给后续 cleanup，而不是立即物理删除。 - WebSocket upgrade 必须在 owner endpoint 上完成。Owner-hinted WebSocket direct retry 不能 fall back 到 router-established 101。 - WDL 会尽量让 client-facing WebSocket 由 gateway 持有并保持连接，包括 user-runtime 或 do-runtime restart 后的 backend reconnect。这个连接连续性目标强于 Cloudflare shutdown 行为；Cloudflare 可以终止 WebSocket 让新 Durable Object instance 接管。当前 backend facet 仍属于 owner scope：初始 `101` 之后，WebSocket message / close event 不会在每一帧重新校验 Redis owner generation。未来增强应在尽量保持 client 连接的同时 rebinding 或拒绝 stale backend owner facet，而不是把 client disconnect 作为主要安全机制。Gateway 重置 backend reconnect epoch 时，旧 epoch 下排队的 client message 可能被丢弃，且没有逐帧 ack/nack。 -- Ordinary fetch/RPC 只有在明确的 stale-owner 或 owner-race response 同时携带 do-runtime 私有 ownership-error control header 时才会回退到 router；tenant response body 不能触发重放。Direct owner transport failure，或 direct owner hop 返回 502/503/504 且没有 fresh owner-hint header 时，会清除 cached hint。安全的 `GET`/`HEAD` request 可以通过 router 重放以重新发现 owner；非幂等 method 和 RPC 会返回 `owner_unavailable`，不会重放，因为 owner 可能已经应用了该请求。 +- Ordinary fetch/RPC 在收到可信 owner-hint，或携带 do-runtime 私有 ownership-error control header 的明确 pre-dispatch stale-owner/owner-race response 后，可以进行一次 router rediscovery；这也适用于非幂等 method 和 RPC。Tenant response body 不能触发重放。无可信标记的 direct owner transport failure，或不带这两类可信标记的 502/503/504，会清除 cached hint。安全的 `GET`/`HEAD` request 可以通过 router 重放；非幂等 method 和 RPC 会返回 `owner_unavailable`，因为 owner 可能已经应用了该请求。 - Shared runtime transport 统一持有 host binding 与 injected facade 的 owner-hint cache wiring、invoke race retry 和 response-header stripping。Connect wrapper 刻意不包含 invoke-only router fallback，以保留 owner-established WebSocket upgrade 语义。 - `WEBSOCKET_RECONNECT_DELAYS_MS` 和 `WEBSOCKET_MAX_BUFFERED_MESSAGES` 可以在不改代码的情况下调整 gateway backend reconnect budget 和 client-message buffer cap。 - Alarm delivery 是 at-least-once。Scheduler 唤醒 Workflows；Workflows 把到期 internal alarm job promote 到 ready，在 DB 2 run token 下 claim，然后调用 do-runtime `/internal/do/alarms/dispatch`。do-runtime 构造 `DoInvoke{kind:"alarm"}` 请求，并走正常 owner router/fence 路径。 @@ -152,6 +157,6 @@ do-runtime 围绕 owner resolution、dispatch、alarm execution、drain、renew - DO object registry 写入是 best-effort。Registry 写失败时 dispatch 会继续，因此 tombstone set 可能不完整；未来 cleanup 必须容忍缺失 member，并把 active storage pointer、owner/alarm state 当成更强的 lifecycle signal。 - Gateway-held WebSocket recovery 只对 client connection continuity 做 best-effort。Backend DO facet 在初始 `101` 之后不会逐消息 re-fence；owner handoff 安全依赖 reconnect/rebind 行为，以及创建 backend facet 前运行的 owner-side dispatch fence。Gateway 重置 backend reconnect epoch 时，旧 epoch 下排队的 client message 可能被丢弃，且没有逐帧 ack/nack。 - Owner-hinted WebSocket direct retry 失败不会 fall back 到 router，因为最终 101 必须来自 owner endpoint。 -- Owner-hinted ordinary fetch/RPC direct failure 只有安全的 `GET`/`HEAD` request 会 fall back 到 router。非幂等 method 和 RPC 在 outcome 可能未知时返回 `owner_unavailable`。明确的 stale-owner 和 owner-race response 仍可重试，因为它们证明 owner-side fence 拒绝了该请求。 +- 无可信标记的 ordinary fetch/RPC direct failure 只有安全的 `GET`/`HEAD` request 会 fall back 到 router。非幂等 method 和 RPC 在 outcome 可能未知时返回 `owner_unavailable`。可信 owner-hint 或明确的 stale-owner/owner-race response 对所有 method 都可重试一次，因为它们证明 dispatch 未进入 tenant code。 - Renamed/deleted migrations 延后。 - 长 handler 仍需要用户自己注意；lease-budget watchdog 保护平台 ownership 并缩窄 failover race，不 fence 每一次 storage call 或最终 SQLite commit point。 diff --git a/docs/modules/gateway.md b/docs/modules/gateway.md index a22b02b..df2ea23 100644 --- a/docs/modules/gateway.md +++ b/docs/modules/gateway.md @@ -34,10 +34,12 @@ responses before they cross back onto the public socket. The `ADMIN_HOST` branch is infrastructure traffic, not a loaded-worker request. It does not set `x-worker-id` or `x-worker-prefix`. `PLATFORM_DOMAIN` and -`ADMIN_HOST` are environment-configurable. All tiers normalize `PLATFORM_DOMAIN` as an -ALB-compatible ASCII DNS hostname of at most 126 bytes, with an alphabetic final label, -and default it to `workers.local`; one trailing dot is removed and the result is -lowercased. Admin-host short-circuiting remains unset by default. +`ADMIN_HOST` are environment-configurable. Routing tiers normalize `PLATFORM_DOMAIN` +as an ALB-compatible ASCII DNS hostname of at most 126 bytes, with an alphabetic final +label, and default it to `workers.local`; one trailing dot is removed and the result is +lowercased. Control's `/whoami` consumes only an explicitly configured value and never +advertises the `workers.local` default. Admin-host short-circuiting remains unset by +default. ## Interfaces diff --git a/docs/modules/gateway.zh.md b/docs/modules/gateway.zh.md index f4e20e2..6e6353a 100644 --- a/docs/modules/gateway.zh.md +++ b/docs/modules/gateway.zh.md @@ -18,7 +18,7 @@ Gateway 有三条 dispatch 分支： 转发前，gateway 会清除客户端提供的 `x-worker-id`、`x-worker-prefix` 和所有 `x-wdl-*` header。同一套内部 header 策略也会在 WebSocket upgrade response 回到公开 socket 前进行过滤。 -`ADMIN_HOST` 分支是 infrastructure traffic，不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置。所有 tier 都会把 `PLATFORM_DOMAIN` normalize 成最多 126 bytes、final label 仅包含字母的 ALB-compatible ASCII DNS hostname；单个尾点会被移除，结果转成小写，并默认使用 `workers.local`。Admin-host short-circuit 默认仍未设置。 +`ADMIN_HOST` 分支是 infrastructure traffic，不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置。各 routing tier 会把 `PLATFORM_DOMAIN` normalize 成最多 126 bytes、final label 仅包含字母的 ALB-compatible ASCII DNS hostname；单个尾点会被移除，结果转成小写，并默认使用 `workers.local`。Control 的 `/whoami` 只使用显式配置的值，绝不广告 `workers.local` 默认值。Admin-host short-circuit 默认仍未设置。 ## 接口 diff --git a/gateway/index.js b/gateway/index.js index cc89f53..1d15439 100644 --- a/gateway/index.js +++ b/gateway/index.js @@ -91,7 +91,6 @@ export default { }); try { - const platformDomain = platformDomainFromEnv(env); const subscriberStart = ensureGatewaySubscriber(env.REDIS_ADDR); if (subscriberStart) ctx.waitUntil(subscriberStart); @@ -110,6 +109,9 @@ export default { return scope.respond(prometheusResponse(metrics)); } + // After health/metrics so a malformed PLATFORM_DOMAIN cannot 502 probes. + const platformDomain = platformDomainFromEnv(env); + // Admin host short-circuit runs before any ns / Redis lookup so // control stays reachable even mid-FLUSHALL / Redis outage. const normalizedHost = normalizeRequestHost(url.hostname).toLowerCase(); diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index d400b8c..72acce6 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -1,4 +1,4 @@ -import { validOwnerEndpointForService } from "./_wdl-owner-endpoint.js"; +import { prototypeGetter, validOwnerEndpointForService } from "./_wdl-owner-endpoint.js"; import { sanitizeRequestId } from "./_wdl-request-id.js"; export const DO_INVOKE_URL = "http://do-runtime/internal/do/invoke"; @@ -40,7 +40,10 @@ const OWNER_RACE_RETRY_CODES = new Set([ "owner_claim_raced", "owner_fence_missing", "owner_lease_expired", + "stale_owner_storage", "owner_lease_too_short", + "owner_renew_raced", + "task_draining", ]); const OWNER_HINT_STALE_CODES = new Set([ "stale_owner_generation", @@ -82,8 +85,11 @@ const intrinsicObjectEntries = Object.entries; const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; const intrinsicObjectSetPrototypeOf = Object.setPrototypeOf; +const intrinsicPromiseThen = Promise.prototype.then; const intrinsicResponseJson = Response.json; +const intrinsicReadableStreamCancel = ReadableStream.prototype.cancel; const intrinsicReadableStreamGetReader = ReadableStream.prototype.getReader; +const intrinsicReadableStreamReaderCancel = ReadableStreamDefaultReader.prototype.cancel; const intrinsicReadableStreamReaderRead = ReadableStreamDefaultReader.prototype.read; const intrinsicReadableStreamReaderReleaseLock = ReadableStreamDefaultReader.prototype.releaseLock; const intrinsicSetHas = Set.prototype.has; @@ -161,19 +167,6 @@ const RPC_RESERVED_METHODS = new Set(["fetch", "alarm"]); * }} DoOwnerHintDispatchOptions */ -// workerd places some WebIDL mixin getters on a parent prototype. Resolve -// them once before tenant module evaluation rather than reading patched getters. -/** @param {object | null} prototype @param {string} name */ -function prototypeGetter(prototype, name) { - let current = prototype; - while (current) { - const getter = Object.getOwnPropertyDescriptor(current, name)?.get; - if (getter) return getter; - current = Object.getPrototypeOf(current); - } - return undefined; -} - /** @param {DoBindingProps} props @param {string} objectName */ export function doOwnerHintCacheKey(props, objectName) { return `${props.doStorageId}:${props.className}:${objectName}`; @@ -216,7 +209,6 @@ function headerValue(headers, name) { return intrinsicReflectApply(intrinsicHeadersGet, headers, [name]); } -/** @param {Headers} headers @param {string} name */ /** @param {Headers} headers @param {string} name @param {string} value */ function headerSet(headers, name, value) { intrinsicReflectApply(intrinsicHeadersSet, headers, [name, value]); @@ -300,7 +292,8 @@ function releaseStreamReader(reader) { /** @param {ReadableStreamDefaultReader} reader */ function cancelStreamReader(reader) { try { - reader.cancel().catch(() => {}); + const cancellation = intrinsicReflectApply(intrinsicReadableStreamReaderCancel, reader, []); + intrinsicReflectApply(intrinsicPromiseThen, cancellation, [undefined, () => {}]); } catch { // Cancellation is best-effort after the bounded reader has already rejected. } @@ -313,7 +306,8 @@ function cancelResponseBody(response) { try { const body = responseBody(response); if (!body) return; - body.cancel().catch(() => {}); + const cancellation = intrinsicReflectApply(intrinsicReadableStreamCancel, body, []); + intrinsicReflectApply(intrinsicPromiseThen, cancellation, [undefined, () => {}]); } catch { // Best-effort cleanup only; the replacement response owns behavior. } @@ -618,6 +612,11 @@ export function retryableOwnerRaceResponse(response) { return code !== null && setHas(OWNER_RACE_RETRY_CODES, code); } +/** @param {Response} response */ +function retryableDirectOwnerResponse(response) { + return ownerHintFromResponse(response) !== null || retryableOwnerRaceResponse(response); +} + /** * @param {Request} request * @param {string | null | undefined} requestId @@ -793,7 +792,7 @@ async function dispatchDoWithHintCache({ /** @param {Response} response */ const finish = async (response) => { let result = response; - if (retryOwnerRace && retryableOwnerRaceResponse(result)) { + if (retryOwnerRace && retryableDirectOwnerResponse(result)) { cancelResponseBody(result); clearHint(); result = await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); @@ -811,6 +810,9 @@ async function dispatchDoWithHintCache({ clearHint(); return await finish(await replayOrUnavailable()); } + if (retryOwnerRace && retryableDirectOwnerResponse(direct)) { + return await finish(direct); + } if (ownerHintFromResponse(direct) || staleDoOwnerHintResponse(direct)) { cancelResponseBody(direct); clearHint(); @@ -846,7 +848,8 @@ async function dispatchDoWithHintCache({ let direct; try { direct = await ownerFetch(ownerRequestUrl(hinted, ownerPath), init); - if (ownerEndpointUnavailableResponse(direct)) { + if (ownerEndpointUnavailableResponse(direct) && + !(retryOwnerRace && retryableDirectOwnerResponse(direct))) { cancelResponseBody(direct); throw new Error(`DO owner endpoint returned ${responseStatus(direct)}`); } @@ -854,6 +857,9 @@ async function dispatchDoWithHintCache({ clearHint(); return await finish(await replayOrUnavailable()); } + if (retryOwnerRace && retryableDirectOwnerResponse(direct)) { + return await finish(direct); + } if (ownerHintFromResponse(direct)) { cancelResponseBody(direct); clearHint(); diff --git a/runtime/_wdl-owner-endpoint.js b/runtime/_wdl-owner-endpoint.js index 94625ed..2ab0cfa 100644 --- a/runtime/_wdl-owner-endpoint.js +++ b/runtime/_wdl-owner-endpoint.js @@ -1,2 +1,2 @@ // Local adapter for the injected DO transport's relative module name. -export { validOwnerEndpointForService } from "../shared/owner-endpoint.js"; +export { prototypeGetter, validOwnerEndpointForService } from "../shared/owner-endpoint.js"; diff --git a/runtime/lib.js b/runtime/lib.js index 6453968..1333437 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -15,7 +15,6 @@ import { const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); -export { base64ToBytes, bytesToBase64 }; // Coerce a KV.put value into Uint8Array. ReadableStream is handled by the // caller (await body); here we stick to sync inputs. diff --git a/runtime/load.js b/runtime/load.js index 88904b0..4d2ca84 100644 --- a/runtime/load.js +++ b/runtime/load.js @@ -13,7 +13,7 @@ import { } from "runtime-load-code-budget"; import { RUNTIME_INJECTION_SOURCES } from "runtime-load-injection-sources"; import { buildWorkerEnv } from "runtime-load-env-build"; -export { buildWorkerEnv } from "runtime-load-env-build"; +export { buildWorkerEnv, doAlarmBindingProps } from "runtime-load-env-build"; const REDIS_PROXY_LOAD_TIMEOUT_MS = 8000; diff --git a/runtime/load/env-build.js b/runtime/load/env-build.js index c478f08..0d36d44 100644 --- a/runtime/load/env-build.js +++ b/runtime/load/env-build.js @@ -6,7 +6,6 @@ import { WDL_RESERVED_ENTRYPOINT_RE, isValidJsIdentifier, isValidJsClassDeclarationName, - isValidRuntimeLoadNs, } from "shared-ns-pattern"; import { parseVersion } from "shared-version"; @@ -91,12 +90,16 @@ function materializeQueueBinding({ name, spec, ns, ctx }) { }; } +// The estimated control-side copy of the do-runtime alarm binding env value +// must serialize like the real one; both build their props here. +/** @param {{ ns: string, worker: string, version: string, doStorageId: string }} identity */ +export function doAlarmBindingProps({ ns, worker, version, doStorageId }) { + return { ns, worker, version, doStorageId }; +} + /** @param {RuntimeBindingMaterializerArgs} args */ function materializeD1Binding({ name, spec, ns, ctx }) { const databaseId = spec.databaseId; - if (!isValidRuntimeLoadNs(ns)) { - throw new Error(`Binding "${name}" is a D1 binding but has invalid namespace`); - } if (typeof databaseId !== "string" || !D1_DATABASE_ID_RE.test(databaseId)) { throw new Error(`Binding "${name}" is a D1 binding but has invalid databaseId`); } diff --git a/runtime/r2-client.js b/runtime/r2-client.js index 5de68be..f7c85fd 100644 --- a/runtime/r2-client.js +++ b/runtime/r2-client.js @@ -132,15 +132,15 @@ function cappedReadableStream(stream, operation) { }); } -/** @param {Headers} headers @param {{ strictExpiry?: boolean }} [options] */ -function headersToHttpMetadata(headers, { strictExpiry = false } = {}) { +/** @param {Headers} headers */ +function headersToHttpMetadata(headers) { /** @type {AnyRecord} */ const out = {}; for (const [field, header] of R2_HTTP_METADATA_FIELDS) { out[field] = headers.get(header) || undefined; } - const cacheExpiry = r2CacheExpiryFromHeaders(headers, { canonical: strictExpiry }); - if (strictExpiry && headers.has("expires") && cacheExpiry === undefined) { + const cacheExpiry = r2CacheExpiryFromHeaders(headers, { canonical: true }); + if (headers.has("expires") && cacheExpiry === undefined) { throw new TypeError("R2 httpMetadata Expires header must be canonical IMF-fixdate"); } out.cacheExpiry = cacheExpiry; @@ -150,7 +150,7 @@ function headersToHttpMetadata(headers, { strictExpiry = false } = {}) { /** @param {unknown} input */ function normalizeHttpMetadata(input) { if (input == null) return undefined; - if (input instanceof Headers) return headersToHttpMetadata(input, { strictExpiry: true }); + if (input instanceof Headers) return headersToHttpMetadata(input); if (typeof input !== "object" || Array.isArray(input)) { throw new TypeError("R2 httpMetadata must be an object or Headers"); } diff --git a/rust/scheduler/src/cron/sweep.rs b/rust/scheduler/src/cron/sweep.rs index 84263ac..088bc23 100644 --- a/rust/scheduler/src/cron/sweep.rs +++ b/rust/scheduler/src/cron/sweep.rs @@ -207,7 +207,6 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { json!({ "workers": workers, "re_added": re_added, - "skipped": skipped, "entries_skipped": skipped, "workers_skipped": skipped_workers, "duration_ms": now_ms() - started, diff --git a/rust/scheduler/src/queue/consume.rs b/rust/scheduler/src/queue/consume.rs index bf0e28a..95dad76 100644 --- a/rust/scheduler/src/queue/consume.rs +++ b/rust/scheduler/src/queue/consume.rs @@ -5,8 +5,8 @@ use serde_json::json; use tokio::time::sleep; use crate::{ - AppState, CONSUMER_GROUP, LogLevel, MAX_BATCH_SIZE_CAP, SchedulerError, SchedulerResult, log, - now_ms, redis_fields_with_error, scheduler_fields_with_error, + AppState, CONSUMER_GROUP, LogLevel, SchedulerError, SchedulerResult, log, now_ms, + redis_fields_with_error, scheduler_fields_with_error, }; use super::{ @@ -195,7 +195,7 @@ pub(crate) fn queue_xread_count_from_consumers<'a>( ) -> usize { consumers .into_iter() - .filter_map(|consumer| consumer.map(|c| c.max_batch_size.clamp(1, MAX_BATCH_SIZE_CAP))) + .filter_map(|consumer| consumer.map(|c| c.max_batch_size)) .min() .unwrap_or(1) } @@ -254,11 +254,6 @@ mod tests { 2 ); assert_eq!(queue_xread_count_from_consumers([Some(&large)]), 25); - assert_eq!(queue_xread_count_from_consumers([Some(&consumer(0))]), 1); - assert_eq!( - queue_xread_count_from_consumers([Some(&consumer(MAX_BATCH_SIZE_CAP + 10))]), - MAX_BATCH_SIZE_CAP - ); assert_eq!(queue_xread_count_from_consumers([None]), 1); } } diff --git a/rust/scheduler/src/queue/delivery/dispatch.rs b/rust/scheduler/src/queue/delivery/dispatch.rs index d23c740..e867bbf 100644 --- a/rust/scheduler/src/queue/delivery/dispatch.rs +++ b/rust/scheduler/src/queue/delivery/dispatch.rs @@ -4,8 +4,8 @@ use redis::Value; use serde_json::json; use crate::{ - AppState, CONSUMER_GROUP, LogLevel, MAX_BATCH_SIZE_CAP, RuntimeResponse, SERVICE, - SchedulerError, SchedulerResult, log, now_ms, post_runtime, + AppState, CONSUMER_GROUP, LogLevel, RuntimeResponse, SERVICE, SchedulerError, SchedulerResult, + log, now_ms, post_runtime, }; use super::super::{Consumer, OutcomePlan, QueueMessage}; @@ -142,7 +142,7 @@ pub(crate) async fn dispatch_messages( consumer: &Consumer, kind: &str, ) -> SchedulerResult<()> { - let size = consumer.max_batch_size.clamp(1, MAX_BATCH_SIZE_CAP); + let size = consumer.max_batch_size; for (index, chunk) in messages.chunks(size).enumerate() { let mut pending = VecDeque::from([(index * size, 0_usize, chunk.to_vec())]); while let Some((offset, split_depth, batch)) = pending.pop_front() { diff --git a/rust/scheduler/src/queue/orphan.rs b/rust/scheduler/src/queue/orphan.rs index a9211a5..6043695 100644 --- a/rust/scheduler/src/queue/orphan.rs +++ b/rust/scheduler/src/queue/orphan.rs @@ -374,7 +374,7 @@ fn queue_pel_reap_idle_ms(configured_idle_ms: u64, consumer_present: bool) -> u6 fn queue_pel_claim_count(consumer: Option<&Consumer>) -> usize { consumer - .map(|consumer| consumer.max_batch_size.clamp(1, MAX_BATCH_SIZE_CAP)) + .map(|consumer| consumer.max_batch_size) .unwrap_or(MAX_BATCH_SIZE_CAP) } @@ -404,11 +404,6 @@ mod tests { #[test] fn pel_reap_claim_count_matches_consumer_batch_cap_when_consumer_exists() { assert_eq!(queue_pel_claim_count(Some(&consumer(2))), 2); - assert_eq!(queue_pel_claim_count(Some(&consumer(0))), 1); - assert_eq!( - queue_pel_claim_count(Some(&consumer(MAX_BATCH_SIZE_CAP + 10))), - MAX_BATCH_SIZE_CAP - ); assert_eq!(queue_pel_claim_count(None), MAX_BATCH_SIZE_CAP); } diff --git a/rust/workflows/src/api.rs b/rust/workflows/src/api.rs index fc015c9..b11207b 100644 --- a/rust/workflows/src/api.rs +++ b/rust/workflows/src/api.rs @@ -74,8 +74,8 @@ use pending_create::{ public_state_or_empty, wait_for_public_create_state, }; use pending_restart::{ - active_pending_restart_blockers, create_pending_restart, pending_restart_marker, - remove_pending_restart, + PendingRestartMarker, active_pending_restart_blockers, create_pending_restart, + pending_restart_marker, remove_pending_restart, }; use progress::{ spawn_progress_from_identity, spawn_progress_from_request, spawn_progress_from_step, diff --git a/rust/workflows/src/api/lifecycle/restart.rs b/rust/workflows/src/api/lifecycle/restart.rs index 075528d..cfbdfce 100644 --- a/rust/workflows/src/api/lifecycle/restart.rs +++ b/rust/workflows/src/api/lifecycle/restart.rs @@ -1,21 +1,35 @@ use wdl_rust_common::time::now_ms; use crate::{ - AppState, WorkflowError, WorkflowResult, by_version_key, by_worker_key, by_workflow_key, - ready_active_key, retention_key, + AppState, LogLevel, WorkflowError, WorkflowResult, by_version_key, by_worker_key, + by_workflow_key, log, ready_active_key, retention_key, workflow_error_fields, }; use super::super::{ - InstanceResponse, InstanceRouteKeys, WorkflowRequest, create_pending_restart, - ensure_worker_not_deleting, eval_script, identity_from_state, instance_id, payload_bytes_arg, - pending_restart_marker, read_public_state, remove_pending_restart, request_with_active_version, - validate_identity, verify_active_workflow_current, verify_workflow_def, - workflow_referrer_member, + InstanceResponse, InstanceRouteKeys, PendingRestartMarker, WorkflowRequest, + create_pending_restart, ensure_worker_not_deleting, eval_script, identity_from_state, + instance_id, payload_bytes_arg, pending_restart_marker, read_public_state, + remove_pending_restart, request_with_active_version, validate_identity, + verify_active_workflow_current, verify_workflow_def, workflow_referrer_member, }; + use super::common::{ TransitionSignal, next_generation, stale_transition_response, successful_transition_response, }; +// Marker cleanup is best-effort: the marker TTL expires a leaked member, and +// the caller's original rejection must stay the visible error. +async fn discard_pending_restart(state: &AppState, marker: &PendingRestartMarker) { + if let Err(err) = remove_pending_restart(state, marker).await { + log( + state, + LogLevel::Warn, + "pending_restart_cleanup_failed", + workflow_error_fields(&err), + ); + } +} + const RESTART_SCRIPT: &str = r#" local marker_score = redis.call("ZSCORE", KEYS[16], ARGV[12]) local marker_time = redis.call("TIME") @@ -96,15 +110,15 @@ pub(crate) async fn restart_instance( let pending_restart = pending_restart_marker(state, &req, &id); create_pending_restart(state, &pending_restart).await?; if let Err(err) = verify_active_workflow_current(state, &req).await { - remove_pending_restart(state, &pending_restart).await?; + discard_pending_restart(state, &pending_restart).await; return Err(err); } if let Err(err) = verify_workflow_def(state, &req).await { - remove_pending_restart(state, &pending_restart).await?; + discard_pending_restart(state, &pending_restart).await; return Err(err); } if let Err(err) = ensure_worker_not_deleting(state, &req.ns, &req.worker).await { - remove_pending_restart(state, &pending_restart).await?; + discard_pending_restart(state, &pending_restart).await; return Err(err); } let shard = keys.shard(); @@ -165,7 +179,7 @@ pub(crate) async fn restart_instance( { Ok(updated) => updated, Err(err) => { - remove_pending_restart(state, &pending_restart).await?; + discard_pending_restart(state, &pending_restart).await; return Err(err); } }; @@ -175,7 +189,7 @@ pub(crate) async fn restart_instance( )); } if updated != 1 { - remove_pending_restart(state, &pending_restart).await?; + discard_pending_restart(state, &pending_restart).await; return stale_transition_response(state, &identity, &id).await; } existing.insert("status".to_string(), "queued".to_string()); diff --git a/scripts/integration-test-plan.js b/scripts/integration-test-plan.js index fabd257..4a72834 100644 --- a/scripts/integration-test-plan.js +++ b/scripts/integration-test-plan.js @@ -5,7 +5,7 @@ const ROOT = path.resolve(import.meta.dirname, ".."); export const DEFAULT_INTEGRATION_DURATIONS_FILE = path.join(ROOT, ".integration-test-durations.json"); // Update when integration runs reveal new slow tests; ordering affects -// scheduling latency, not correctness. Last calibrated 2026-07-08 against +// scheduling latency, not correctness. Last calibrated 2026-07-17 against // .integration-test-durations.json (descending durationMs). The recorded // duration file, when present, takes precedence over this fallback list. export const SLOW_FIRST_FILES = [ @@ -13,8 +13,8 @@ export const SLOW_FIRST_FILES = [ "durable-objects-ownership.test.js", "d1-ownership-multi-runtime.test.js", "durable-objects-alarms.test.js", - "log-tail.test.js", "cron-triggers.test.js", + "log-tail.test.js", "durable-objects-websocket.test.js", "workflows-runtime-retention.test.js", "queues-orphan-and-control.test.js", @@ -25,44 +25,44 @@ export const SLOW_FIRST_FILES = [ "delete-api.test.js", "queues-delivery.test.js", "queues-batch-and-isolation.test.js", - "secrets.test.js", - "durable-objects-storage.test.js", + "gateway.test.js", "d1-binding.test.js", "auth-platform.test.js", - "admin-api.test.js", + "durable-objects-storage.test.js", + "secrets.test.js", "kv-binding.test.js", + "admin-api.test.js", "scheduler-shutdown-drain.test.js", "auth-worker.test.js", "workflows-runtime-pausing.test.js", - "service-bindings-rpc.test.js", - "gateway.test.js", - "d1-lifecycle.test.js", - "http-features.test.js", - "d1-storage-localdisk.test.js", "workflows-runtime-core.test.js", "r2-cli-binding.test.js", + "service-bindings-rpc.test.js", + "d1-storage-localdisk.test.js", "redis-conformance.test.js", "platform-bindings.test.js", - "delete-indexes.test.js", - "routing-gateway.test.js", - "observability.test.js", - "runtime-eviction.test.js", - "s3-cleanup.test.js", "cli-smoke.test.js", "cli-multi-env.test.js", + "http-features.test.js", + "s3-cleanup.test.js", + "d1-lifecycle.test.js", + "observability.test.js", + "delete-indexes.test.js", "network-boundary.test.js", - "queue-native-dispatch.test.js", - "routing-admin.test.js", - "route-demo.test.js", "service-bindings.test.js", - "pages-assets-demo.test.js", "assets-binding.test.js", + "pages-assets-demo.test.js", + "route-demo.test.js", + "routing-gateway.test.js", + "runtime-eviction.test.js", + "queue-native-dispatch.test.js", "durable-objects-core.test.js", - "workflows-metadata.test.js", + "routing-admin.test.js", "system-pool-auth.test.js", + "workflows-metadata.test.js", "worker-modules.test.js", - "workflows-durable-objects.test.js", "workflows-service.test.js", + "workflows-durable-objects.test.js", ]; export const CLI_INTEGRATION_MARKER = "@wdl-cli-integration"; diff --git a/shared/owner-endpoint.js b/shared/owner-endpoint.js index 97071b6..9d373f4 100644 --- a/shared/owner-endpoint.js +++ b/shared/owner-endpoint.js @@ -24,8 +24,10 @@ const intrinsicUrlPortGet = /** @type {(this: URL) => string} */ (prototypeGette const intrinsicUrlSearchGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "search")); const intrinsicUrlUsernameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "username")); +// workerd places some WebIDL mixin getters on a parent prototype. Resolve +// them once before tenant module evaluation rather than reading patched getters. /** @param {object | null} prototype @param {string} name */ -function prototypeGetter(prototype, name) { +export function prototypeGetter(prototype, name) { let current = prototype; while (current) { const getter = Object.getOwnPropertyDescriptor(current, name)?.get; diff --git a/tests/helpers/do-owner-hint.js b/tests/helpers/do-owner-hint.js index 368daf2..ba3155a 100644 --- a/tests/helpers/do-owner-hint.js +++ b/tests/helpers/do-owner-hint.js @@ -1,3 +1,7 @@ +import { loadDoProtocol } from "./load-do-protocol.js"; + +const { DO_OWNERSHIP_ERROR_CONTROL_HEADER } = await loadDoProtocol(); + /** * @param {{ * ownerKey?: string, @@ -34,7 +38,7 @@ export function doOwnerHintResponse(options = {}) { /** @param {string} code @param {HeadersInit | undefined} [headers] */ export function doOwnershipErrorHeaders(code, headers = undefined) { const out = new Headers(headers); - out.set("x-wdl-do-ownership-error", code); + out.set(DO_OWNERSHIP_ERROR_CONTROL_HEADER, code); return out; } diff --git a/tests/helpers/load-auth-index.js b/tests/helpers/load-auth-index.js index 0bd5220..9d2cc2f 100644 --- a/tests/helpers/load-auth-index.js +++ b/tests/helpers/load-auth-index.js @@ -130,6 +130,10 @@ export class RedisClient { recordCommand("SET", true); return null; } + if (options.ifeq != null && s.strings.get(key) !== options.ifeq) { + recordCommand("SET", true); + return null; + } s.strings.set(key, value); if (typeof options.ttl === "number") { s.expirations.set(key, Date.now() + options.ttl * 1000); diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index 2aaf91c..e9cca16 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -42,6 +42,7 @@ export async function compileControlGraph(opts = {}) { [/from "shared-ns-pattern"/g, `from ${JSON.stringify(SHARED_NS_URL)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], + [/from "shared-route-projection"/g, `from ${JSON.stringify(SHARED_ROUTE_PROJECTION_URL)}`], ]); const bindingsUrl = freshRepositoryModuleDataUrl("control/bindings.js", [ diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index c3c9bca..1d5ee12 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -28,6 +28,14 @@ export async function acquireDeleteLock(_redis, _ns, _worker, kind) { return "lock-token"; } export async function renewDeleteLock() { return true; } +export function deleteLockExpiredDetails(ns, worker, version) { + return { + namespace: ns, + name: worker, + ...(version === undefined ? {} : { version }), + message: "worker delete lock expired; retry the request", + }; +} export async function assertWorkflowDeleteAllowed(args) { /** @type {any} */ (globalThis).__deleteHandlerState.workflowChecks.push(args); if (/** @type {any} */ (globalThis).__deleteHandlerWorkflowBlocker) { @@ -60,6 +68,14 @@ export async function acquireDeleteLock(_redis, _ns, _worker, kind) { return "lock-token"; } export async function renewDeleteLock() { return true; } +export function deleteLockExpiredDetails(ns, worker, version) { + return { + namespace: ns, + name: worker, + ...(version === undefined ? {} : { version }), + message: "worker delete lock expired; retry the request", + }; +} export async function assertWorkflowDeleteAllowed(args) { /** @type {any} */ (globalThis).__versionDeleteHandlerState.workflowChecks.push(args); } @@ -80,6 +96,7 @@ export { doObjectRegistryKey, encodeReferrerMember, parseBundleMeta, + parsePatternProjection, referrersKey, workersIndexKey, workflowDefsKey, diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index ec1baa2..fe63c9c 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -519,27 +519,6 @@ test("worker env budget maps strict retained metadata failures to corrupt_meta", ); }); -test("worker env budget fails closed when retained bundle metadata is missing", async () => { - const redis = { - /** @param {string} key @param {string} field */ - async hGet(key, field) { - assert.equal(key, "worker:demo:api:v:1"); - assert.equal(field, "__meta__"); - return null; - }, - }; - - await assert.rejects( - () => assertWorkerVersionsUserEnvBudget({ - redis, - ns: "demo", - worker: "api", - versions: ["v1"], - }), - (err) => assertBundleMetaError(err, "__meta__ must be a JSON string") - ); -}); - test("worker env budget surfaces missing retained bundles as watch retry", async () => { const redis = { /** @param {string} key @param {string} field */ @@ -556,7 +535,6 @@ test("worker env budget surfaces missing retained bundles as watch retry", async ns: "demo", worker: "api", versions: ["v1"], - retryMissingVersions: true, }), (err) => err instanceof Error && err.name === "WatchError" ); diff --git a/tests/unit/do-runtime-load.test.js b/tests/unit/do-runtime-load.test.js index e95965c..e5eb987 100644 --- a/tests/unit/do-runtime-load.test.js +++ b/tests/unit/do-runtime-load.test.js @@ -25,6 +25,9 @@ export function bundleToWorkerCode(bundle) { return bundle; } `); const runtimeLoadUrl = moduleDataUrl(` export function buildWorkerEnv() { return {}; } +export function doAlarmBindingProps({ ns, worker, version, doStorageId }) { + return { ns, worker, version, doStorageId }; +} export function internalAuthBackend(ctx, env, binding) { return typeof ctx.exports.InternalAuthBackend === "function" ? ctx.exports.InternalAuthBackend({ props: { binding } }) diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index bc300a1..110000a 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -50,7 +50,10 @@ test("DO ownership errors stay aligned with runtime hint and retry handling", () DO_OWNERSHIP_CODE.OWNER_CLAIM_RACED, DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, DO_OWNERSHIP_CODE.OWNER_LEASE_EXPIRED, + DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, DO_OWNERSHIP_CODE.OWNER_LEASE_TOO_SHORT, + DO_OWNERSHIP_CODE.OWNER_RENEW_RACED, + DO_OWNERSHIP_CODE.TASK_DRAINING, ]); for (const code of ownershipCodes) { const untrusted = Response.json({ error: code }, { status: 503 }); diff --git a/tests/unit/integration-test-plan.test.js b/tests/unit/integration-test-plan.test.js index c33b8ef..ae0e86f 100644 --- a/tests/unit/integration-test-plan.test.js +++ b/tests/unit/integration-test-plan.test.js @@ -38,8 +38,8 @@ test("prioritizeDefaultFiles runs configured slow files first and preserves the "durable-objects-ownership.test.js", "d1-ownership-multi-runtime.test.js", "durable-objects-alarms.test.js", - "log-tail.test.js", "cron-triggers.test.js", + "log-tail.test.js", "durable-objects-websocket.test.js", "workflows-runtime-retention.test.js", "queues-orphan-and-control.test.js", diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index 18918ff..3daea62 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -475,12 +475,6 @@ test("do-runtime owns the DO RPC method byte limit", () => { { method: "m".repeat(256), valid: true }, { method: "m".repeat(257), valid: false }, ]) { - const metadata = { - ...props, - objectName: "room-a", - kind: "rpc", - rpc: { method, args: [] }, - }; const envelope = rpcInvokeBody(props, "room-a", method, []); if (!valid) { assert.throws( @@ -489,7 +483,7 @@ test("do-runtime owns the DO RPC method byte limit", () => { ); continue; } - const invoke = normalizeDoInvokeRequest(metadata); + const invoke = normalizeDoInvokeRequest(decodeDoEnvelope(envelope).metadata); assert.equal("rpc" in invoke ? invoke.rpc.method : null, method); } }); @@ -1300,39 +1294,44 @@ test("DurableObjectNamespace direct backend ignores hints attached to owner-race }); test("DO owner-race router retry failures do not trigger another replay", async (t) => { - await t.test("fresh owner response", async () => { - let routerCalls = 0; - let ownerCalls = 0; - const ownerMetadata = new Headers(doOwnerHintHeaders()); - ownerMetadata.delete("x-wdl-do-owner-hint"); + const ownerMetadata = new Headers(doOwnerHintHeaders()); + ownerMetadata.delete("x-wdl-do-owner-hint"); + for (const { name, code, headers } of [ + { name: "fresh owner response without metadata", code: "owner_claim_raced", headers: undefined }, + { name: "fresh owner renew race with metadata", code: "owner_renew_raced", headers: ownerMetadata }, + ]) { + await t.test(name, async () => { + let routerCalls = 0; + let ownerCalls = 0; - await assert.rejects( - dispatchDoInvokeWithHintCache({ - routerFetch: async () => { - routerCalls += 1; - if (routerCalls === 1) return doOwnerHintResponse(); - if (routerCalls === 2) throw new Error("owner-race router retry failed"); - return new Response("unexpected replay"); - }, - routerUrl: "http://do-runtime/internal/do/invoke", - ownerFetch: async () => { - ownerCalls += 1; - return Response.json({ error: "owner_claim_raced", message: "retry" }, { - status: 503, - headers: doOwnershipErrorHeaders("owner_claim_raced", ownerMetadata), - }); - }, - ownerPath: "/internal/do/invoke", - init: { method: "GET" }, - cache: new Map(), - hintKey: "room-a", - replayOwnerUnavailable: true, - }), - /owner-race router retry failed/ - ); - assert.equal(ownerCalls, 1); - assert.equal(routerCalls, 2); - }); + await assert.rejects( + dispatchDoInvokeWithHintCache({ + routerFetch: async () => { + routerCalls += 1; + if (routerCalls === 1) return doOwnerHintResponse(); + if (routerCalls === 2) throw new Error("owner-race router retry failed"); + return new Response("unexpected replay"); + }, + routerUrl: "http://do-runtime/internal/do/invoke", + ownerFetch: async () => { + ownerCalls += 1; + return Response.json({ error: code, message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders(code, headers), + }); + }, + ownerPath: "/internal/do/invoke", + init: { method: "POST" }, + cache: new Map(), + hintKey: "room-a", + replayOwnerUnavailable: false, + }), + /owner-race router retry failed/ + ); + assert.equal(ownerCalls, 1); + assert.equal(routerCalls, 2); + }); + } await t.test("cached endpoint fallback", async () => { const hint = ownerHintFromHeaders(doOwnerHintResponse().headers); @@ -1480,24 +1479,56 @@ test("DurableObjectNamespace replays a safe GET through the router after a secon binding: "ROOM", className: "Room", }, { backend, ownerNetwork }); + const streamCancel = ReadableStream.prototype.cancel; + const promiseThen = Promise.prototype.then; + let hostileCancelCalls = 0; + let hostileThenCalls = 0; + /** @type {ReadableStream | null} */ + let leakedStream = null; - const response = await ns.get(ns.idFromName("room-consecutive-hint")) - .fetch("https://demo.workers.example/send"); + const response = await withMockedProperty( + ReadableStream.prototype, + "cancel", + /** @this {ReadableStream} @param {unknown} reason */ + function hostileCancel(reason) { + hostileCancelCalls += 1; + leakedStream = this; + return Reflect.apply(streamCancel, this, [reason]); + }, + () => withMockedProperty( + Promise.prototype, + "then", + /** @this {Promise} @param {any} onFulfilled @param {any} onRejected */ + function hostileThen(onFulfilled, onRejected) { + hostileThenCalls += 1; + return Reflect.apply(promiseThen, this, [onFulfilled, onRejected]); + }, + () => ns.get(ns.idFromName("room-consecutive-hint")) + .fetch("https://demo.workers.example/send") + ) + ); assert.equal(await response.text(), "router-ok"); assert.equal(response.headers.get("x-wdl-do-owner-key"), null); assert.equal(routerCalls.length, 2); assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); assert.equal(ownerCalls.length, 1); assert.deepEqual(cancellations, ["router", "direct"]); + assert.equal(hostileCancelCalls, 0); + assert.equal(hostileThenCalls, 0); + assert.equal(leakedStream, null); }); -test("DurableObjectNamespace hides a second owner hint without replaying a POST", async () => { +test("DurableObjectNamespace replays a POST after a trusted second owner hint", async () => { /** @type {any[]} */ const routerCalls = []; /** @type {any[]} */ const ownerCalls = []; const backend = { - fetch: makeRecordingFetch(routerCalls, { response: doOwnerHintResponse() }), + fetch: makeRecordingFetch(routerCalls, { + response: () => routerCalls.length === 1 + ? doOwnerHintResponse() + : new Response("router-ok"), + }), }; const ownerNetwork = { fetch: makeRecordingFetch(ownerCalls, { response: doOwnerHintResponse({ @@ -1517,14 +1548,11 @@ test("DurableObjectNamespace hides a second owner hint without replaying a POST" const response = await ns.get(ns.idFromName("room-consecutive-post")) .fetch("https://demo.workers.example/send", { method: "POST", body: "hello" }); - const body = await readJsonResponse(response, 503); - assert.deepEqual(body, { - error: "owner_unavailable", - message: "DO owner is unavailable; request outcome may be unknown", - }); + assert.equal(await response.text(), "router-ok"); assert.equal(response.headers.get("x-wdl-do-owner-key"), null); - assert.equal(routerCalls.length, 1); + assert.equal(routerCalls.length, 2); + assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); assert.equal(ownerCalls.length, 1); }); @@ -2146,11 +2174,13 @@ test("DO requestSpec rejects streaming bodies as soon as they cross the cap", as }); test("DO requestSpec rejects oversized streams without waiting for cancel", async () => { + let cancellations = 0; const body = new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array(1024 * 1024 + 1)); }, cancel() { + cancellations += 1; return new Promise(() => {}); }, }); @@ -2159,13 +2189,39 @@ test("DO requestSpec rejects oversized streams without waiting for cancel", asyn body, duplex: "half", })); - await assert.rejects( - withTestTimeout( - requestSpec(request, "rid-cancel"), - "requestSpec waited for stream cancel" - ), - /Durable Object fetch body exceeds/ + const readerCancel = ReadableStreamDefaultReader.prototype.cancel; + const promiseCatch = Promise.prototype.catch; + let hostileCancelCalls = 0; + let hostileCatchCalls = 0; + + await withMockedProperty( + ReadableStreamDefaultReader.prototype, + "cancel", + /** @this {ReadableStreamDefaultReader} @param {unknown} reason */ + function hostileCancel(reason) { + hostileCancelCalls += 1; + return Reflect.apply(readerCancel, this, [reason]); + }, + () => withMockedProperty( + Promise.prototype, + "catch", + /** @this {Promise} @param {any} onRejected */ + function hostileCatch(onRejected) { + hostileCatchCalls += 1; + return Reflect.apply(promiseCatch, this, [onRejected]); + }, + () => assert.rejects( + withTestTimeout( + requestSpec(request, "rid-cancel"), + "requestSpec waited for stream cancel" + ), + /Durable Object fetch body exceeds/ + ) + ) ); + assert.equal(cancellations, 1); + assert.equal(hostileCancelCalls, 0); + assert.equal(hostileCatchCalls, 0); }); test("DurableObjectNamespace facade uses direct upgrade path for websockets", async () => { @@ -2531,7 +2587,7 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu assert.equal(ownerCalls.length, 2, error); assert.equal(routerCalls.length, 2, error); - assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), "1", error); + assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null, error); assert.equal(cancellations, 1, error); } }); diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index 24affa8..ded4604 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -4,8 +4,6 @@ import { parseBase64Json } from "../helpers/json-payload.js"; import { readRepositoryJson, runtimeLibModuleDataUrl } from "../helpers/load-shared-module.js"; const { - base64ToBytes, - bytesToBase64, toBytes, bundleToWorkerCode, buildAssetUrl, @@ -18,14 +16,12 @@ const { normalizeQueuedDispatchBody, normalizeScheduledDispatchBody, } = await import(runtimeLibModuleDataUrl()); -const sharedBase64 = await import("../../shared/base64.js"); +const { base64ToBytes, bytesToBase64 } = await import("../../shared/base64.js"); const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); const enc = new TextEncoder(); -test("runtime base64 exports use the shared codec owner", () => { - assert.equal(bytesToBase64, sharedBase64.bytesToBase64); - assert.equal(base64ToBytes, sharedBase64.base64ToBytes); +test("shared base64 codec round-trips large binary payloads", () => { const bytes = Uint8Array.from({ length: 70_000 }, (_, index) => index % 256); assert.deepEqual(base64ToBytes(bytesToBase64(bytes)), bytes); }); diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index f6c36f1..038dd24 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -781,19 +781,6 @@ test("buildWorkerEnv: D1 bindings revalidate namespace and database id grammar", ), /invalid databaseId/ ); - assert.throws( - () => buildWorkerEnv( - { bindings: { DB: { type: "d1", databaseId: "main" } } }, - {}, - {}, - "admin", - "app", - "v1", - "https://assets.example", - makeCtx() - ), - /invalid namespace/ - ); }); test("buildWorkerEnv: R2 binding requires bucketName", () => { From 318714b7dac02de103fd9a0a725ff8f4c8335fe9 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 10:39:11 +0800 Subject: [PATCH 20/23] Allow historical SigV4 fixtures in gitleaks Keep the secret scan narrow while recognizing the official AWS example vectors under both the former and current test filenames. Signed-off-by: Lu Zhang --- .gitleaks.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitleaks.toml b/.gitleaks.toml index 3e299a1..906ef65 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -16,7 +16,7 @@ regexes = [ description = "AWS SigV4 official example credentials used by parity fixtures." regexTarget = "match" paths = [ - '''^tests/unit/aws-sigv4\.test\.js$''' + '''^tests/unit/(?:aws4fetch-sigv4|aws-sigv4)\.test\.js$''' ] regexes = [ '''AKIDEXAMPLE''', @@ -28,7 +28,7 @@ condition = "AND" description = "Fixed AWS SigV4 signature fixture generated from official example credentials." regexTarget = "secret" paths = [ - '''^tests/unit/aws-sigv4\.test\.js$''' + '''^tests/unit/(?:aws4fetch-sigv4|aws-sigv4)\.test\.js$''' ] regexes = [ '''f54699d16ac946691025f331377c59814b85e1078bc70ea66925457e2549d0a7''' From 9f8601739f790899bb0ed211d4ffc610379d922f Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 12:17:09 +0800 Subject: [PATCH 21/23] fix: tighten internal response boundaries Restore bounded D1 5xx classifier fields, strip gateway-private headers at final HTTP egress, and pin the Kubernetes router/headless owner-endpoint topology. Signed-off-by: Lu Zhang --- control/d1-runtime-client.js | 16 +++++----- docs/modules/d1.md | 6 ++-- docs/modules/d1.zh.md | 2 +- docs/modules/gateway.md | 5 ++-- docs/modules/gateway.zh.md | 2 +- gateway/index.js | 1 + shared/request-scope.js | 4 ++- shared/respond.js | 4 ++- tests/integration/gateway.test.js | 28 ++++++++++++++++++ tests/unit/control-d1-runtime-client.test.js | 29 ++++++++++++++++++ tests/unit/request-scope.test.js | 13 ++++++-- tests/unit/respond.test.js | 18 ++++++++++++ tests/unit/style-contracts.test.js | 31 ++++++++++++++++++++ 13 files changed, 141 insertions(+), 18 deletions(-) diff --git a/control/d1-runtime-client.js b/control/d1-runtime-client.js index 5869287..af53639 100644 --- a/control/d1-runtime-client.js +++ b/control/d1-runtime-client.js @@ -261,9 +261,11 @@ function d1RuntimeFailureExtra(extra) { return sanitized; } +const D1_RUNTIME_CLASSIFIER_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; + /** @param {unknown} value @param {string} fallback */ -function d1RuntimeLogToken(value, fallback) { - return typeof value === "string" && value ? value : fallback; +function d1RuntimeClassifierToken(value, fallback) { + return typeof value === "string" && D1_RUNTIME_CLASSIFIER_RE.test(value) ? value : fallback; } /** @@ -274,11 +276,11 @@ function d1RuntimeLogToken(value, fallback) { */ export function d1RuntimeFailureLogFields(result) { const body = /** @type {Record} */ (Object(result?.body || {})); - const causeCode = d1RuntimeLogToken(body.causeCode, ""); + const causeCode = d1RuntimeClassifierToken(body.causeCode, ""); return { ...(typeof result?.status === "number" ? { upstream_status: result.status } : {}), - upstream_code: d1RuntimeLogToken(body.error, "d1-runtime-error"), - upstream_category: d1RuntimeLogToken(body.category, "internal"), + upstream_code: d1RuntimeClassifierToken(body.error, "d1-runtime-error"), + upstream_category: d1RuntimeClassifierToken(body.category, "internal"), upstream_retryable: body.retryable === true, ...(causeCode ? { upstream_cause_code: causeCode } : {}), }; @@ -306,8 +308,8 @@ export function d1RuntimeFailure(error, ns, databaseId, result, extra = {}, opti return { ...publicFields, message: "Internal error", - upstreamCode: d1RuntimeLogToken(body.error, "d1-runtime-error"), - upstreamCategory: d1RuntimeLogToken(body.category, "internal"), + upstreamCode: d1RuntimeClassifierToken(body.error, "d1-runtime-error"), + upstreamCategory: d1RuntimeClassifierToken(body.category, "internal"), upstreamRetryable: body.retryable === true, ...(typeof result?.status === "number" ? { upstreamStatus: result.status } : {}), }; diff --git a/docs/modules/d1.md b/docs/modules/d1.md index ba9259a..982e4f2 100644 --- a/docs/modules/d1.md +++ b/docs/modules/d1.md @@ -57,9 +57,9 @@ freezing of a binding to a physical `databaseId`. ...details }` contract. d1-runtime internal query/classified errors are a separate D1 protocol surface. When the outward Control status is 5xx, Control replaces the backend message with `Internal error`, omits raw detail, and preserves only the - machine classifier fields `upstreamCode`, `upstreamCategory`, `upstreamRetryable`, - and `upstreamStatus`. Outward 4xx SQL and migration failures expose D1 diagnostic - details. + bounded machine classifier fields `upstreamCode`, `upstreamCategory`, + `upstreamRetryable`, and `upstreamStatus`. Outward 4xx SQL and migration failures + expose D1 diagnostic details. ## Redis / Storage Contracts diff --git a/docs/modules/d1.zh.md b/docs/modules/d1.zh.md index 01cf9df..a9fb998 100644 --- a/docs/modules/d1.zh.md +++ b/docs/modules/d1.zh.md @@ -29,7 +29,7 @@ workerd 给 WDL 提供了托管 SQLite-backed service 所需的 isolate 和 loca - Multi-statement `exec()` request 会在一个 actor-local SQLite transaction 中执行。如果后续 statement 失败，同一 request 中此前已执行的 statement 会回滚。 - D1 query/facade 失败使用 D1-specific payload 和 tenant `D1_ERROR` object，而不是 generic platform/admin HTTP envelope。D1 payload 会按需携带 `success:false`、`error`、`message`、`category`、`retryable`、`statementIndex` 和 `causeCode` 等字段。 - D1 error code 使用 D1 compatibility vocabulary，包括 `limit-exceeded`、`sql-error` 这类 `hyphen-case` code。除非 D1 protocol 本身变化，否则不要把它们改写成平台 `snake_case` vocabulary。 -- Control D1 lifecycle API 是 admin API，使用 control `{ error, message, ...details }` 合同。d1-runtime internal query/classified error 是独立的 D1 protocol surface。Control 对外状态为 5xx 时会把 backend message 替换为 `Internal error`、移除 raw detail，只保留 `upstreamCode`、`upstreamCategory`、`upstreamRetryable` 和 `upstreamStatus` 机器分类字段；对外 4xx SQL 与 migration failure 会暴露 D1 diagnostic detail。 +- Control D1 lifecycle API 是 admin API，使用 control `{ error, message, ...details }` 合同。d1-runtime internal query/classified error 是独立的 D1 protocol surface。Control 对外状态为 5xx 时会把 backend message 替换为 `Internal error`、移除 raw detail，只保留 `upstreamCode`、`upstreamCategory`、`upstreamRetryable` 和 `upstreamStatus` 有界机器分类字段；对外 4xx SQL 与 migration failure 会暴露 D1 diagnostic detail。 ## Redis / Storage 合同 diff --git a/docs/modules/gateway.md b/docs/modules/gateway.md index df2ea23..ddfe5b0 100644 --- a/docs/modules/gateway.md +++ b/docs/modules/gateway.md @@ -29,8 +29,9 @@ The resolved `{ ns, worker, version }` becomes `x-worker-id: ::::` 和 `x-worker-prefix`。字面量 `__system__` route 进入 `RUNTIME_SYSTEM`；普通 tenant namespace 进入 `RUNTIME_USER`。 -转发前，gateway 会清除客户端提供的 `x-worker-id`、`x-worker-prefix` 和所有 `x-wdl-*` header。同一套内部 header 策略也会在 WebSocket upgrade response 回到公开 socket 前进行过滤。 +转发前，gateway 会清除客户端提供的 `x-worker-id`、`x-worker-prefix` 和所有 `x-wdl-*` header。同一套内部 header 策略也会过滤所有转发响应，包括失败和成功的 WebSocket upgrade，再将其返回公开 socket。 `ADMIN_HOST` 分支是 infrastructure traffic，不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置。各 routing tier 会把 `PLATFORM_DOMAIN` normalize 成最多 126 bytes、final label 仅包含字母的 ALB-compatible ASCII DNS hostname；单个尾点会被移除，结果转成小写，并默认使用 `workers.local`。Control 的 `/whoami` 只使用显式配置的值，绝不广告 `workers.local` 默认值。Admin-host short-circuit 默认仍未设置。 diff --git a/gateway/index.js b/gateway/index.js index 1d15439..6252b8a 100644 --- a/gateway/index.js +++ b/gateway/index.js @@ -88,6 +88,7 @@ export default { log, route: "worker_fetch", extras: () => ({ namespace, worker, version }), + responseHeaderFilter: deleteGatewayInternalHeaders, }); try { diff --git a/shared/request-scope.js b/shared/request-scope.js index 0cf369a..5a84584 100644 --- a/shared/request-scope.js +++ b/shared/request-scope.js @@ -18,6 +18,7 @@ import { echoResponseWithRequestId } from "shared-respond"; * route: string, * probeRoutes?: string[] | undefined, * extras?: Record | (() => Record) | null, + * responseHeaderFilter?: ((headers: Headers) => void) | undefined, * }} options */ export function createHttpRequestScope({ @@ -28,6 +29,7 @@ export function createHttpRequestScope({ route, probeRoutes = undefined, extras = null, + responseHeaderFilter = undefined, }) { const startedAt = Date.now(); const requestId = ensureRequestId(request.headers); @@ -59,7 +61,7 @@ export function createHttpRequestScope({ */ respond(response) { status = response.status; - return echoResponseWithRequestId(response, requestId); + return echoResponseWithRequestId(response, requestId, responseHeaderFilter); }, /** diff --git a/shared/respond.js b/shared/respond.js index a4aac49..8ec65cb 100644 --- a/shared/respond.js +++ b/shared/respond.js @@ -20,10 +20,12 @@ export function rebuildResponseWithHeaders(response, headers) { /** * @param {Response & { webSocket?: WebSocket | null }} response * @param {string} requestId + * @param {((headers: Headers) => void) | undefined} [filterHeaders] * @returns {Response} */ -export function echoResponseWithRequestId(response, requestId) { +export function echoResponseWithRequestId(response, requestId, filterHeaders = undefined) { const headers = new Headers(response.headers); + filterHeaders?.(headers); headers.set("x-request-id", requestId); return rebuildResponseWithHeaders(response, headers); } diff --git a/tests/integration/gateway.test.js b/tests/integration/gateway.test.js index e72044d..33c7477 100644 --- a/tests/integration/gateway.test.js +++ b/tests/integration/gateway.test.js @@ -194,6 +194,34 @@ test("gateway strips client-supplied internal forwarding headers", async () => { }); }); +test("gateway strips tenant response internal headers", async () => { + await deployAndPromote("gwns-response-headers", "echo", { + code: `export default { + fetch() { + return new Response("ok", { + headers: { + "x-worker-id": "forged:worker:v1", + "x-worker-prefix": "/forged", + "x-wdl-internal-auth": "secret", + "x-wdl-future-private": "hidden", + "x-public": "visible", + }, + }); + } + };`, + }); + + const res = await gatewayFetch("gwns-response-headers", "/echo"); + assert.equal(res.status, 200); + assert.equal(await res.text(), "ok"); + assert.equal(res.headers["x-worker-id"], undefined); + assert.equal(res.headers["x-worker-prefix"], undefined); + assert.equal(res.headers["x-wdl-internal-auth"], undefined); + assert.equal(res.headers["x-wdl-future-private"], undefined); + assert.equal(res.headers["x-public"], "visible"); + assert.ok(res.headers["x-request-id"]); +}); + test("promoting a new version replaces routed code (pub/sub invalidates cache)", async () => { await deployAndPromote("gwns4", "v", { code: "export default {fetch(){return new Response('one')}};", diff --git a/tests/unit/control-d1-runtime-client.test.js b/tests/unit/control-d1-runtime-client.test.js index d348e2c..2152497 100644 --- a/tests/unit/control-d1-runtime-client.test.js +++ b/tests/unit/control-d1-runtime-client.test.js @@ -358,6 +358,35 @@ test("control D1 runtime failure redacts 5xx diagnostics and keeps safe extra fi }); }); +test("control D1 runtime failure exposes only bounded machine classifiers", () => { + const result = { + status: 503, + body: { + error: "panic at /srv/d1-runtime/actor.rs:42", + category: `internal-${"x".repeat(128)}`, + causeCode: "backend\ntrace", + retryable: true, + }, + }; + + assert.deepEqual(d1RuntimeFailure("d1_execute_failed", "demo", "db1", result), { + error: "d1_execute_failed", + namespace: "demo", + databaseId: "db1", + message: "Internal error", + upstreamCode: "d1-runtime-error", + upstreamCategory: "internal", + upstreamRetryable: true, + upstreamStatus: 503, + }); + assert.deepEqual(d1RuntimeFailureLogFields(result), { + upstream_status: 503, + upstream_code: "d1-runtime-error", + upstream_category: "internal", + upstream_retryable: true, + }); +}); + test("control D1 runtime failure uses the outward status to redact upstream 4xx details", () => { assert.deepEqual(d1RuntimeFailure("d1_database_initialize_failed", "demo", "db1", { status: 400, diff --git a/tests/unit/request-scope.test.js b/tests/unit/request-scope.test.js index d9e8ece..15a2294 100644 --- a/tests/unit/request-scope.test.js +++ b/tests/unit/request-scope.test.js @@ -17,8 +17,9 @@ export function recordRequestComplete(fields) { `); const respondUrl = moduleDataUrl(` -export function echoResponseWithRequestId(response, requestId) { +export function echoResponseWithRequestId(response, requestId, filterHeaders) { const out = new Response(response.body, response); + filterHeaders?.(out.headers); out.headers.set("x-request-id", requestId); return out; } @@ -55,14 +56,22 @@ test("createHttpRequestScope echoes request id and records final request state", log, route: "initial", extras: () => ({ namespace: "tenant-a" }), + responseHeaderFilter(/** @type {Headers} */ headers) { + headers.delete("x-private"); + }, }); scope.setRoute("worker_fetch"); const err = false; scope.markError(err); - const response = scope.respond(Response.json({ ok: false }, { status: 502 })); + const response = scope.respond(Response.json({ ok: false }, { + status: 502, + headers: { "x-private": "hidden", "x-public": "visible" }, + })); scope.complete(); assert.equal(response.headers.get("x-request-id"), "rid-123"); + assert.equal(response.headers.get("x-private"), null); + assert.equal(response.headers.get("x-public"), "visible"); assert.equal(scope.requestId, "rid-123"); assert.equal(observability.calls.complete.length, 1); assert.equal(observability.calls.complete[0].route, "worker_fetch"); diff --git a/tests/unit/respond.test.js b/tests/unit/respond.test.js index 042c731..81c21f7 100644 --- a/tests/unit/respond.test.js +++ b/tests/unit/respond.test.js @@ -95,6 +95,24 @@ test("echoResponseWithRequestId overwrites a pre-existing x-request-id", () => { assert.equal(out.headers.get("x-request-id"), "ours-id"); }); +test("echoResponseWithRequestId filters copied headers before stamping the request id", () => { + const src = new Response(null, { + headers: { + "x-private": "hidden", + "x-public": "visible", + "x-request-id": "upstream-id", + }, + }); + const out = echoResponseWithRequestId(src, "ours-id", (headers) => { + headers.delete("x-private"); + headers.delete("x-request-id"); + }); + + assert.equal(out.headers.get("x-private"), null); + assert.equal(out.headers.get("x-public"), "visible"); + assert.equal(out.headers.get("x-request-id"), "ours-id"); +}); + test("rebuildResponseWithHeaders preserves response fields while replacing headers", async () => { const src = new Response("hello", { status: 202, diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 60dd6ff..77d0dcf 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1454,6 +1454,37 @@ test("local compose runtime profiles keep base services enabled by default", () } }); +test("Kubernetes stateful runtimes publish Pod-specific owner endpoints", () => { + const families = /** @type {Array<[string, number]>} */ ([["d1", 8787], ["do", 8788]]); + for (const [family, port] of families) { + const serviceName = `${family}-runtime`; + const headlessName = `${serviceName}-headless`; + const resources = /** @type {Array>} */ ( + yamlDocuments(`deploy/kubernetes/base/${serviceName}.yaml`).map((document) => document.toJS()) + ); + const router = resources.find((resource) => resource.kind === "Service" && resource.metadata?.name === serviceName); + const headless = resources.find((resource) => resource.kind === "Service" && resource.metadata?.name === headlessName); + const statefulSet = resources.find((resource) => resource.kind === "StatefulSet" && resource.metadata?.name === serviceName); + assert.ok(router, `${serviceName} router Service must exist`); + assert.notEqual(router.spec?.clusterIP, "None", `${serviceName} router must stay load-balanced`); + assert.equal(headless?.spec?.clusterIP, "None", `${headlessName} must stay headless`); + assert.equal(statefulSet?.spec?.serviceName, headlessName, `${serviceName} StatefulSet serviceName`); + + const container = statefulSet?.spec?.template?.spec?.containers + ?.find((/** @type {Record} */ entry) => entry.name === serviceName); + assert.ok(container, `${serviceName} container must exist`); + const env = Object.fromEntries(container.env.map( + (/** @type {Record} */ entry) => [entry.name, entry] + )); + assert.equal(env.POD_NAME?.valueFrom?.fieldRef?.fieldPath, "metadata.name"); + assert.equal(env[`${family.toUpperCase()}_TASK_ID`]?.valueFrom?.fieldRef?.fieldPath, "metadata.name"); + assert.equal( + env[`${family.toUpperCase()}_TASK_ENDPOINT`]?.value, + `$(POD_NAME).${headlessName}:${port}` + ); + } +}); + test("Kubernetes runtime families pin the redis-proxy functional contract", () => { const files = [ "deploy/kubernetes/base/user-runtime.yaml", From 8735ca660f404865d1b68ee3c4f7d1044c525ad2 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 19:42:51 +0800 Subject: [PATCH 22/23] refactor: consolidate worker contracts and test infrastructure Rename shared JS and Rust worker contract owners and centralize identity predicates. Reuse the canonical Control module graph and scoped global mocks across tests. Ensure the published Compose overlay clears service builds and document the reserved wrapper module. Signed-off-by: Lu Zhang --- CHANGELOG.md | 2 +- auth/index.js | 2 +- control/bindings.js | 6 +- control/env-budget.js | 2 +- control/handlers/delete-plan.js | 2 +- control/handlers/delete.js | 2 +- control/handlers/deploy.js | 2 +- control/handlers/hosts.js | 2 +- control/handlers/logs-tail.js | 8 +- control/handlers/ns-secrets.js | 2 +- control/handlers/promote.js | 2 +- control/handlers/versions.js | 2 +- control/handlers/worker-secrets.js | 2 +- control/handlers/workers.js | 2 +- control/handlers/workflows.js | 11 +- control/index.js | 3 +- control/lib.js | 30 +---- control/lifecycle-indexes.js | 2 +- control/routing.js | 2 +- control/shared.js | 2 +- control/topology.js | 4 +- do-runtime/config.capnp | 4 +- do-runtime/index.js | 2 +- do-runtime/owner-registry.js | 2 +- do-runtime/protocol.js | 2 +- docker-compose.images.yml | 18 ++- docs/redis-key-layout.md | 6 +- docs/redis-key-layout.zh.md | 4 +- docs/source-map.md | 4 +- docs/source-map.zh.md | 4 +- docs/testing.md | 20 +-- docs/testing.zh.md | 9 +- gateway/config.capnp | 4 +- gateway/dispatch.js | 2 +- gateway/runtime.js | 2 +- runtime/config-system.capnp | 8 +- runtime/config-user.capnp | 4 +- runtime/lib.js | 2 +- runtime/load/env-build.js | 2 +- rust/common/src/identity.rs | 4 +- rust/common/src/lib.rs | 4 +- .../src/{version.rs => worker_contract.rs} | 11 +- rust/redis-proxy/src/runtime.rs | 4 +- rust/scheduler/src/cron/reference.rs | 2 +- rust/scheduler/src/queue/registry.rs | 2 +- rust/workflows/src/api/active_export.rs | 2 +- rust/workflows/src/api/do_alarms/dispatch.rs | 2 +- rust/workflows/src/api/do_alarms/model.rs | 2 +- rust/workflows/src/api/do_alarms/mutations.rs | 2 +- rust/workflows/src/api/routing.rs | 2 +- shared/ns-pattern.js | 20 +++ shared/{version.js => worker-contract.js} | 18 ++- shared/worker-id.js | 2 +- tests/helpers/control-shared-stub.js | 36 ++---- tests/helpers/load-auth-index.js | 4 +- tests/helpers/load-control-lib.js | 6 +- tests/helpers/load-control-routing.js | 4 +- tests/helpers/load-control-shared.js | 116 ++++++++++++++++++ tests/helpers/load-do-owner-registry.js | 4 +- tests/helpers/load-do-protocol.js | 4 +- tests/helpers/load-shared-module.js | 4 +- tests/integration/helpers/durable-objects.js | 2 +- tests/integration/helpers/misc.js | 2 +- tests/unit/control-delete-handler.test.js | 10 +- tests/unit/control-deploy-watch.test.js | 4 +- tests/unit/control-env-budget.test.js | 6 +- tests/unit/control-handlers-workers.test.js | 4 +- tests/unit/control-handlers-workflows.test.js | 4 +- tests/unit/control-index.test.js | 1 - tests/unit/control-lifecycle-indexes.test.js | 4 +- tests/unit/control-logs-tail.test.js | 5 +- tests/unit/control-routing.test.js | 2 +- .../control-secret-envelope-handlers.test.js | 10 +- tests/unit/control-shared.test.js | 83 ++----------- tests/unit/do-runtime-index.test.js | 6 +- tests/unit/gateway-dispatch.test.js | 4 +- tests/unit/gateway-runtime.test.js | 2 +- tests/unit/ns-pattern.test.js | 21 ++++ tests/unit/runtime-load.test.js | 8 +- tests/unit/runtime-workflows-client.test.js | 34 ++--- tests/unit/style-contracts.test.js | 93 +++++++++++--- .../unit/test-helper-style-contracts.test.js | 5 +- ...ersion.test.js => worker-contract.test.js} | 2 +- 83 files changed, 436 insertions(+), 322 deletions(-) rename rust/common/src/{version.rs => worker_contract.rs} (90%) rename shared/{version.js => worker-contract.js} (90%) create mode 100644 tests/helpers/load-control-shared.js rename tests/unit/{version.test.js => worker-contract.test.js} (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4817825..cbb211b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - Updated maintenance dependencies and image baselines: workerd runtime images now use distroless `base-debian13`, local and Kubernetes Valkey use `valkey/valkey:9.1-alpine`, and root development tooling moved to current patch/minor releases. - Consolidated duplicated Control request/error/retry paths, Redis locks and key grammars, queue and cron projections, S3 retry policy, observability helpers, and Rust sidecar contracts into canonical owners with shared behavioral and cross-language fixture coverage. - Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and D1/R2/DO facades resist tenant prototype mutation at capability boundaries, owner records and forwarding targets require service-specific private endpoints, generated wrappers propagate diagnostic request ids on a best-effort basis, and DO ownership retries rely on private do-runtime markers without replaying unknown-outcome non-idempotent requests. -- Tightened forward-only input contracts: dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01`, upstream experimental flags, and flags that disable WDL's required enhanced error serialization; internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, DO class names are capped so every shard fits the host-id limit, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. +- Tightened forward-only input contracts: dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01`, upstream experimental flags, and flags that disable WDL's required enhanced error serialization; internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, runtime-reserved module-name collisions require redeployment under a non-reserved name, DO class names are capped so every shard fits the host-id limit, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. - Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, worker-version, scheduler-projection, and workflow-limit fixtures. - Normalized `PLATFORM_DOMAIN` across routing tiers, made the published-image Compose overlay pull-only, and limited Kubernetes user-runtime loader ingress to gateway. - Removed the unreachable DO inline worker-code test hook and its Terraform switch; do-runtime invoke paths now accept only canonical persisted bundle identities and reject non-canonical or oversized host ids. diff --git a/auth/index.js b/auth/index.js index a9caaa3..6eb49b1 100644 --- a/auth/index.js +++ b/auth/index.js @@ -42,7 +42,7 @@ import { createTokenLock, releaseTokenLock, } from "shared-redis-lock"; -import { NAMESPACES_KEY } from "shared-version"; +import { NAMESPACES_KEY } from "shared-worker-contract"; const utf8Decoder = new TextDecoder(); diff --git a/control/bindings.js b/control/bindings.js index fc34ad3..599b544 100644 --- a/control/bindings.js +++ b/control/bindings.js @@ -14,6 +14,9 @@ import { RESERVED_TENANT_NS, WDL_RESERVED_ENTRYPOINT_RE, MAX_DO_CLASS_NAME_BYTES, + isValidKvId, + isValidQueueName, + isValidWorkerName, isValidJsIdentifier, isValidJsClassDeclarationName, } from "shared-ns-pattern"; @@ -22,9 +25,6 @@ import { errorMessage } from "shared-errors"; import { NS_RE, isAdminAcceptableNs, - isValidKvId, - isValidQueueName, - isValidWorkerName, MAX_QUEUE_DELAY_SECONDS, } from "control-lib"; diff --git a/control/env-budget.js b/control/env-budget.js index b96d759..4991d48 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -1,7 +1,7 @@ import { decryptSecretValue } from "shared-secret-envelope"; import { BundleMetaError, parseBundleMeta } from "control-lib"; import { buildWorkerEnv, doAlarmBindingProps } from "runtime-load-env-build"; -import { bundleKey } from "shared-version"; +import { bundleKey } from "shared-worker-contract"; import { WatchError } from "shared-redis"; export { BundleMetaError }; diff --git a/control/handlers/delete-plan.js b/control/handlers/delete-plan.js index 5764ef0..8b4eb10 100644 --- a/control/handlers/delete-plan.js +++ b/control/handlers/delete-plan.js @@ -18,7 +18,7 @@ import { nsHostsKey, patternsKey, routesKey, -} from "shared-version"; +} from "shared-worker-contract"; import { workerSecretsKey } from "shared-secret-keys"; /** diff --git a/control/handlers/delete.js b/control/handlers/delete.js index 5e95761..675d263 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -34,7 +34,7 @@ import { patternsKey, routesKey, workerVersionsKey, -} from "shared-version"; +} from "shared-worker-contract"; import { workerSecretsKey } from "shared-secret-keys"; import { decodeBulk, WatchError } from "shared-redis"; import { queueConsumerScanPrefix } from "shared-queue-keys"; diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 21d757c..3a6d7c3 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -45,7 +45,7 @@ import { nextVersionKey, parseVersion, routesKey, -} from "shared-version"; +} from "shared-worker-contract"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; import { isReservedNs, diff --git a/control/handlers/hosts.js b/control/handlers/hosts.js index e4f3360..3ba628d 100644 --- a/control/handlers/hosts.js +++ b/control/handlers/hosts.js @@ -9,7 +9,7 @@ import { } from "control-shared"; import { reconcileHosts, RoutingError } from "control-routing"; import { platformDomainFromEnv } from "shared-ns-pattern"; -import { hostsKey } from "shared-version"; +import { hostsKey } from "shared-worker-contract"; /** * @param {{ request: Request, env: Record, method: string, nsName: string, requestId: string }} args diff --git a/control/handlers/logs-tail.js b/control/handlers/logs-tail.js index 366633c..fd65f27 100644 --- a/control/handlers/logs-tail.js +++ b/control/handlers/logs-tail.js @@ -6,12 +6,8 @@ import { RedisSession, redisDbFromEnv } from "shared-redis"; import { envValueOr } from "shared-env"; import { PLATFORM_TIER_RESERVED_NS } from "shared-auth-roles"; -import { isReservedNs, WORKER_NAME_RE } from "shared-ns-pattern"; -import { - compareStreamIds, - isValidResumeId, - isValidWorkerName, -} from "control-lib"; +import { isReservedNs, isValidWorkerName, WORKER_NAME_RE } from "shared-ns-pattern"; +import { compareStreamIds, isValidResumeId } from "control-lib"; import { controlTailRedis, errMessage, jsonError, requireControlLog } from "control-shared"; const TAIL_ACTIVATION_CHANNEL = "logs:tail:active"; diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 9828d71..3a25b1a 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -16,7 +16,7 @@ import { invalidSecretMutationKeyResponse, readEncryptedSecretPutValue, } from "control-handlers-secret-put"; -import { routesKey, workerVersionsKey } from "shared-version"; +import { routesKey, workerVersionsKey } from "shared-worker-contract"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; import { workersIndexKey } from "control-lib"; import { diff --git a/control/handlers/promote.js b/control/handlers/promote.js index ca3bcbd..69529fd 100644 --- a/control/handlers/promote.js +++ b/control/handlers/promote.js @@ -7,7 +7,7 @@ import { requireControlLog, requireControlRedis, } from "control-shared"; -import { parseVersion } from "shared-version"; +import { parseVersion } from "shared-worker-contract"; import { promoteWithRoutes, RoutingError } from "control-routing"; import { platformDomainFromEnv } from "shared-ns-pattern"; diff --git a/control/handlers/versions.js b/control/handlers/versions.js index e5500a3..4f6aaa2 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -30,7 +30,7 @@ import { parseVersion, routesKey, workerVersionsKey, -} from "shared-version"; +} from "shared-worker-contract"; import { workerSecretsKey } from "shared-secret-keys"; const MAX_DELETE_ATTEMPTS = 5; diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 782f94a..f055893 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -28,7 +28,7 @@ import { } from "control-env-budget"; import { SecretEnvelopeError } from "shared-secret-envelope"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; -import { deleteLockKey, routesKey, workerVersionsKey } from "shared-version"; +import { deleteLockKey, routesKey, workerVersionsKey } from "shared-worker-contract"; const MAX_SECRET_ATTEMPTS = 5; diff --git a/control/handlers/workers.js b/control/handlers/workers.js index 3513a38..b37685d 100644 --- a/control/handlers/workers.js +++ b/control/handlers/workers.js @@ -1,7 +1,7 @@ import { jsonResponse, jsonError, requireControlLog, requireControlRedis } from "control-shared"; import { workflowDefsKey, workersIndexKey } from "control-lib"; import { workerSecretsKey } from "shared-secret-keys"; -import { routesKey, workerVersionsKey } from "shared-version"; +import { routesKey, workerVersionsKey } from "shared-worker-contract"; /** @param {{ method: string, nsName: string, requestId: string }} args */ export async function handle({ method, nsName, requestId }) { diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index 45951bb..c271a00 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -1,9 +1,4 @@ -import { - isValidWorkerName, - isValidWorkflowName, - parseBundleMeta, - workflowDefsKey, -} from "control-lib"; +import { parseBundleMeta, workflowDefsKey } from "control-lib"; import { ControlAbort, codedErrorLogFields, @@ -15,12 +10,14 @@ import { requireControlLog, requireControlRedis, } from "control-shared"; -import { bundleKey, routesKey } from "shared-version"; +import { bundleKey, routesKey } from "shared-worker-contract"; import { BINDING_NAME_RE, WORKER_NAME_RE, WORKFLOW_NAME_RE, WORKFLOW_KEY_RE, + isValidWorkerName, + isValidWorkflowName, isValidJsClassDeclarationName, } from "shared-ns-pattern"; diff --git a/control/index.js b/control/index.js index 8330883..95084b3 100644 --- a/control/index.js +++ b/control/index.js @@ -11,9 +11,8 @@ import { platformVersionFromPackageJson, projectAccessPrincipal, isAdminAcceptableNs, - isValidWorkerName, } from "control-lib"; -import { configuredHostname, WORKER_NAME_RE } from "shared-ns-pattern"; +import { configuredHostname, isValidWorkerName, WORKER_NAME_RE } from "shared-ns-pattern"; import { ensureInit, authorizeControlRequest, diff --git a/control/lib.js b/control/lib.js index 5f83227..d07ca52 100644 --- a/control/lib.js +++ b/control/lib.js @@ -1,17 +1,13 @@ -// Name grammar predicates, lifecycle Redis key helpers, bundle metadata -// parsing, referrer-index encode/decode, URL → {gate, ns} classifier, -// and secret-key validation. Pure data-shaping only. +// Lifecycle Redis key helpers, bundle metadata parsing, referrer-index +// encode/decode, URL → {gate, ns} classifier, and secret-key validation. +// Pure data-shaping only. import { NS_PATTERN, RESERVED_OBJECT_KEYS, isReservedNs, RESERVED_TENANT_NS, - WORKER_NAME_RE, - WORKFLOW_NAME_RE, - QUEUE_NAME_RE, WDL_RESERVED_BINDING_RE, - KV_ID_RE, } from "shared-ns-pattern"; import { decodePatternProjection } from "shared-route-projection"; import { PLATFORM_TIER_RESERVED_NS, ROLES } from "shared-auth-roles"; @@ -94,26 +90,6 @@ export function bundleAssetPrefix(meta) { return typeof prefix === "string" ? prefix : null; } -/** @param {unknown} name */ -export function isValidWorkerName(name) { - return typeof name === "string" && WORKER_NAME_RE.test(name); -} - -/** @param {unknown} name */ -export function isValidWorkflowName(name) { - return typeof name === "string" && WORKFLOW_NAME_RE.test(name); -} - -/** @param {unknown} name */ -export function isValidQueueName(name) { - return typeof name === "string" && QUEUE_NAME_RE.test(name); -} - -/** @param {unknown} id */ -export function isValidKvId(id) { - return typeof id === "string" && KV_ID_RE.test(id); -} - // Relaxes NS_PATTERN to also accept reserved `____` ns (so JSRPC-only // reserved-ns flows can deploy / declare hosts), then re-tightens to // reject RESERVED_TENANT_NS (which NS_RE would otherwise pass). diff --git a/control/lifecycle-indexes.js b/control/lifecycle-indexes.js index c055781..a5fa293 100644 --- a/control/lifecycle-indexes.js +++ b/control/lifecycle-indexes.js @@ -5,7 +5,7 @@ import { workersIndexKey, } from "control-lib"; import { QUEUE_CONSUMER_INDEX_KEY, queueConsumerKey } from "shared-queue-keys"; -import { workerVersionsKey } from "shared-version"; +import { workerVersionsKey } from "shared-worker-contract"; export const CRON_WORKER_INDEX_KEY = "cron:index:workers"; diff --git a/control/routing.js b/control/routing.js index 6792c49..6878586 100644 --- a/control/routing.js +++ b/control/routing.js @@ -36,7 +36,7 @@ import { parseVersion, patternsKey, routesKey, -} from "shared-version"; +} from "shared-worker-contract"; import { errorMessage } from "shared-errors"; import { diffCrons, nextFireMs, slotMsFor } from "control-cron-index"; import { isReservedNs, ROUTES_ALLOWED_RESERVED_NS } from "shared-ns-pattern"; diff --git a/control/shared.js b/control/shared.js index bcc1e59..a08a99c 100644 --- a/control/shared.js +++ b/control/shared.js @@ -22,7 +22,7 @@ import { formatDeleteLockToken, hostDeclarationsKey, namespaceFromHostsKey, -} from "shared-version"; +} from "shared-worker-contract"; import { S3_CLEANUP_QUEUE_NAME, S3_CLEANUP_TASK_ID_PREFIX, diff --git a/control/topology.js b/control/topology.js index 6cc0b77..6487614 100644 --- a/control/topology.js +++ b/control/topology.js @@ -3,9 +3,9 @@ // Redis-facing writes that consume these live in control/routing.js. import { parseCron } from "shared-cron-time"; -import { QUEUE_NAME_RE } from "shared-ns-pattern"; +import { isValidQueueName, QUEUE_NAME_RE } from "shared-ns-pattern"; import { errorMessage } from "shared-errors"; -import { isValidQueueName, MAX_QUEUE_DELAY_SECONDS } from "control-lib"; +import { MAX_QUEUE_DELAY_SECONDS } from "control-lib"; /** * @typedef {import("shared-route-projection").PatternProjection} PatternProjection diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index 50d5f82..da355e9 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -91,7 +91,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "shared-d1-query-wire", esModule = embed "../shared/d1-query-wire.js"), (name = "shared-env", esModule = embed "../shared/env.js"), (name = "shared-fnv1a32", esModule = embed "../shared/fnv1a32.js"), - (name = "shared-version", esModule = embed "../shared/version.js"), + (name = "shared-worker-contract", esModule = embed "../shared/worker-contract.js"), (name = "shared-d1-timeout", esModule = embed "../shared/d1-timeout.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), @@ -120,7 +120,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "shared-worker-id", esModule = embed "../shared/worker-id.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), (name = "ns-pattern.js", esModule = embed "../shared/ns-pattern.js"), - (name = "version.js", esModule = embed "../shared/version.js"), + (name = "worker-contract.js", esModule = embed "../shared/worker-contract.js"), (name = "@wdl-dev/aws-sigv4", esModule = embed "../shared/vendor/aws-sigv4.js"), ], compatibilityDate = "2026-04-24", diff --git a/do-runtime/index.js b/do-runtime/index.js index 4828836..495458f 100644 --- a/do-runtime/index.js +++ b/do-runtime/index.js @@ -6,7 +6,7 @@ import { import { createHttpRequestScope } from "shared-request-scope"; import { discardResponseBody, prometheusResponse, rebuildResponseWithHeaders } from "shared-respond"; import { boundedPositiveIntEnv } from "shared-owner-lease"; -import { parseVersion } from "shared-version"; +import { parseVersion } from "shared-worker-contract"; import { formatWorkerId } from "shared-worker-id"; import { WdlDoHostActor } from "do-runtime-actor"; import { handleAlarmDispatch } from "do-runtime-alarm-dispatch"; diff --git a/do-runtime/owner-registry.js b/do-runtime/owner-registry.js index 80c2eaf..2631890 100644 --- a/do-runtime/owner-registry.js +++ b/do-runtime/owner-registry.js @@ -49,7 +49,7 @@ import { deleteLockKey, doStorageIdKey, parseDeleteLockKind, -} from "shared-version"; +} from "shared-worker-contract"; const DEFAULT_OWNER_TTL_SECONDS = 120; const DEFAULT_RENEW_CONCURRENCY = 8; diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index 1baf83a..e05ddb3 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -21,7 +21,7 @@ import { } from "shared-bounded-body"; import { INTERNAL_AUTH_HEADER } from "shared-internal-auth"; import { isValidRuntimeLoadNs, WORKER_NAME_RE } from "shared-ns-pattern"; -import { parseVersion } from "shared-version"; +import { parseVersion } from "shared-worker-contract"; export { DO_HOST_SHARD_COUNT } from "do-runtime-protocol-wire-grammar"; export { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; diff --git a/docker-compose.images.yml b/docker-compose.images.yml index 8b296a5..608f17c 100644 --- a/docker-compose.images.yml +++ b/docker-compose.images.yml @@ -1,43 +1,57 @@ x-rust-published: &rust-published image: docker.io/getwdl/wdl-rust:latest pull_policy: always - build: !reset null x-workerd-published: &workerd-published image: docker.io/getwdl/wdl-workerd:latest pull_policy: always - build: !reset null services: redis-proxy-user: <<: *rust-published + build: !reset null redis-proxy-system: <<: *rust-published + build: !reset null redis-proxy-do: <<: *rust-published + build: !reset null scheduler: <<: *rust-published + build: !reset null workflows: <<: *rust-published + build: !reset null user-runtime: <<: *workerd-published + build: !reset null system-runtime: <<: *workerd-published + build: !reset null gateway: <<: *workerd-published + build: !reset null d1-runtime: <<: *workerd-published + build: !reset null d1-runtime-a: <<: *workerd-published + build: !reset null d1-runtime-b: <<: *workerd-published + build: !reset null d1-runtime-c: <<: *workerd-published + build: !reset null do-runtime: <<: *workerd-published + build: !reset null do-runtime-a: <<: *workerd-published + build: !reset null do-runtime-b: <<: *workerd-published + build: !reset null do-runtime-c: <<: *workerd-published + build: !reset null diff --git a/docs/redis-key-layout.md b/docs/redis-key-layout.md index 31e72a5..6003a04 100644 --- a/docs/redis-key-layout.md +++ b/docs/redis-key-layout.md @@ -56,7 +56,7 @@ secrets:: Hash, worker-level WDL-ENC envelopes `worker:::v:` uses a positive JavaScript-safe integer version in the key, not the `"v"` tag. Test fixtures that seed Redis directly must use -`shared/version.js#bundleKey`. +`shared/worker-contract.js#bundleKey`. `namespaces` is an active worker gate. It is populated when a namespace has an active worker route and may be removed when the last active worker is deleted. @@ -65,8 +65,8 @@ in this set. Auth reads this set during delegated token issue only as a best-eff generated-namespace collision signal, not as a permanent namespace registry. `routes:` and `worker-versions::` are constructed only through -`shared/version.js#routesKey` / `#workerVersionsKey` (and their Rust mirror -`rust/common/src/version.rs#routes_key` / `#worker_versions_key`). Control is the +`shared/worker-contract.js#routesKey` / `#workerVersionsKey` (and their Rust mirror +`rust/common/src/worker_contract.rs#routes_key` / `#worker_versions_key`). Control is the sole writer; sanctioned readers are gateway and workflows. Gateway reads it for route resolution. Workflows reads it for active export resolution during workflow create / verify, and for internal DO alarm retargeting when a fired alarm's diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index 33ff541..fef76f8 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -37,11 +37,11 @@ secrets: Hash, namespace-level WDL-ENC envelope secrets:: Hash, worker-level WDL-ENC envelope ``` -`worker:::v:` 的 key 使用 JavaScript safe-integer 范围内的正整数 version，而不是 `"v"` tag。直接 seed Redis 的测试 fixture 必须使用 `shared/version.js#bundleKey`。 +`worker:::v:` 的 key 使用 JavaScript safe-integer 范围内的正整数 version，而不是 `"v"` tag。直接 seed Redis 的测试 fixture 必须使用 `shared/worker-contract.js#bundleKey`。 `namespaces` 是 active worker gate。有 active worker route 时会加入，最后一个 active worker 删除时可能移除。Namespace-level secrets 和 data-plane state 等资源可以比这个 set membership 活得更久。Auth 在 delegated token issue 时只把它作为 generated-namespace collision 的 best-effort 信号读取，而不是永久 namespace registry。 -`routes:` 和 `worker-versions::` 只能通过 `shared/version.js#routesKey` / `#workerVersionsKey`（以及它们的 Rust 镜像 `rust/common/src/version.rs#routes_key` / `#worker_versions_key`）构造。Control 是唯一 writer；sanctioned reader 是 gateway（route resolution）和 workflows。workflows 有两条读取路径：workflow create / verify 时的 active-export resolution，以及 fired alarm 的 scheduled version 已不再 retained 时的 internal DO alarm retarget。改 key 语法时必须同时更新 JS helper、Rust helper 和所有 reader。 +`routes:` 和 `worker-versions::` 只能通过 `shared/worker-contract.js#routesKey` / `#workerVersionsKey`（以及它们的 Rust 镜像 `rust/common/src/worker_contract.rs#routes_key` / `#worker_versions_key`）构造。Control 是唯一 writer；sanctioned reader 是 gateway（route resolution）和 workflows。workflows 有两条读取路径：workflow create / verify 时的 active-export resolution，以及 fired alarm 的 scheduled version 已不再 retained 时的 internal DO alarm retarget。改 key 语法时必须同时更新 JS helper、Rust helper 和所有 reader。 `workers:` 表示这个 worker 有 worker-owned lifecycle state：retained bundle、active projection、worker-level secrets 或 workflow definitions。Secret-only 和 definitions-only worker 会被有意列出，并可以 whole-delete。 diff --git a/docs/source-map.md b/docs/source-map.md index a95880a..c0ce0fd 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -71,7 +71,7 @@ are outside this map unless they own runtime or deployable service behavior. | `shared/respond.js` | Shared HTTP response, JSON error, Prometheus text, best-effort response body discard, and `x-request-id` echo helpers. | | `shared/bounded-body.js` | Shared bounded byte-stream and request-body readers; each tier maps limit errors to its own contract. | | `shared/ns-pattern.js` | Platform-domain normalization plus namespace, worker, binding, queue, KV/D1/R2 id, module path, reserved object-key, and reserved namespace grammars. | -| `shared/version.js` | Worker version formatting plus worker, route-plane, lifecycle, DO owner-scope key, and route-invalidation channel helpers. | +| `shared/worker-contract.js` | Worker version grammar plus worker, route-plane, lifecycle, DO owner-scope key, and route-invalidation channel helpers. | | `shared/workerd-compat-flags.js` | Pinned upstream mirror of experimental enable flags plus WDL-owned dynamic-worker date and error-serialization policy. | | `shared/queue-keys.js` | JavaScript queue key helpers used by tests and cross-tier key-shape checks. | | `shared/route-projection.js` | Compact pattern-route projection encoding shared by control writers, delete checks, and gateway readers. | @@ -101,7 +101,7 @@ are outside this map unless they own runtime or deployable service behavior. | `rust/scheduler/` | Cron, queue, delayed queue, orphan migration, and workflow tick scheduler. | | `rust/workflows/` | Workflows service, DB 2 state machine, and internal DO alarm backend jobs. | | `rust/supervisor/` | D1/DO supervisor binaries. | -| `rust/common/` | Shared Rust utilities such as time, logging, internal-auth matching, Redis connection primitives, and metrics primitives. | +| `rust/common/` | Shared Rust utilities such as worker-contract grammar and keys, time, logging, internal-auth matching, Redis connection primitives, and metrics primitives. | ## System Workers, Fixtures, And Examples diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index 6a7d92e..5d69fe1 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -68,7 +68,7 @@ | `shared/respond.js` | 共享 HTTP response、JSON error、Prometheus text、best-effort response body discard 和 `x-request-id` echo helpers。 | | `shared/bounded-body.js` | 共享 bounded byte-stream 和 request-body readers；各 tier 自己把 limit error 映射为对应 contract。 | | `shared/ns-pattern.js` | Platform-domain normalization，以及 namespace、worker、binding、queue、KV/D1/R2 id、module path、reserved object-key 和 reserved namespace grammars。 | -| `shared/version.js` | Worker version formatting，以及 worker、route-plane、lifecycle、DO owner-scope key 与 route invalidation channel helpers。 | +| `shared/worker-contract.js` | Worker version grammar，以及 worker、route-plane、lifecycle、DO owner-scope key 与 route invalidation channel helpers。 | | `shared/workerd-compat-flags.js` | 上游 workerd experimental enable flags 的 pinned mirror，以及 WDL-owned dynamic-worker 日期和 error-serialization policy。 | | `shared/queue-keys.js` | JavaScript queue key helpers，供 tests 和 cross-tier key-shape checks 使用。 | | `shared/route-projection.js` | Control writer、delete check 和 gateway reader 共用的紧凑 pattern-route projection encoding。 | @@ -98,7 +98,7 @@ | `rust/scheduler/` | Cron、queue、delayed queue、orphan migration 和 workflow tick scheduler。 | | `rust/workflows/` | Workflows service、DB 2 state machine 和 internal DO alarm backend jobs。 | | `rust/supervisor/` | D1/DO supervisor binaries。 | -| `rust/common/` | time、logging、internal-auth matching、Redis connection primitives 和 metrics primitives 等共享 Rust utilities。 | +| `rust/common/` | worker-contract grammar 与 keys、time、logging、internal-auth matching、Redis connection primitives 和 metrics primitives 等共享 Rust utilities。 | ## System Workers、Fixtures 和 Examples diff --git a/docs/testing.md b/docs/testing.md index 1e0d519..b8614fe 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -178,12 +178,13 @@ Prefer the narrow helper that matches the response or fixture source: | Integration workflow scenarios | Worker source and DB2 helpers from `tests/integration/helpers/workflows-scenarios.js` | Use for workflow demo source, workflow state keys, ready-shard helpers, and direct runtime replay helpers while keeping workflow assertions in the test file. | | Integration fetch worker source wrapper | `workerFetchCallerSource(...)` from `tests/integration/helpers/worker-source.js` | Use when several tests need the same `export default { async fetch(req, env) { try { ... } } }` caller shell while keeping the business body inline and readable. | | Unit control handler module graph | `createControlHandlerState(...)` / `importControlHandler(...)` from `tests/helpers/control-handler-harness.js` | Use for `control/handlers/*` tests that need `control-shared` state, logs, env, metrics, Redis, or backend service stubs. | +| Unit Control shared module graph | `compileControlSharedGraph(...)` / `compileControlSharedDependencies(...)` from `tests/helpers/load-control-shared.js` | Use for tests of `control/shared.js` and synthetic shared stubs instead of rebuilding its production-backed dependency graph. | | Unit runtime R2 binding module graph | `makeR2Bucket(...)` and fetch installers from `tests/helpers/load-runtime-r2-binding.js` | Use for `runtime/bindings/r2.js` host-surface tests instead of rebuilding the R2 module replacement graph in each file. | | Unit D1/DO owner-client module graph | `loadD1OwnerClient(...)` / `loadDoOwnerClient(...)` from `tests/helpers/load-d1-owner-client.js` and `tests/helpers/load-do-owner-client.js` | Use for owner forwarding client tests instead of rebuilding the state, protocol, internal-auth, and owner-forwarder replacement graph in each file. | | Unit auth entrypoint harness state | `authMockState(...)`, `authLogs(...)`, and `lastAuthLog(...)` from `tests/helpers/load-auth-index.js` | Tests should not read or write `globalThis.__authMockState` directly; the global is private storage for the harness' inline module mocks. | | Unit/integration Redis command parity | `redisConformanceCases` from `tests/helpers/redis-conformance-cases.js` | Add a shared case when fake Redis and the real integration Redis wrapper must agree on command semantics. Run `tests/unit/fake-redis.test.js` and the focused Redis conformance integration file. | | Mocked `fetch` call recording | `makeRecordingFetch(...)` / `withRecordingFetch(...)` from `tests/helpers/mock-fetch.js` | Use `capture` when the test needs a custom call record shape. | -| Temporary global or property replacement | `withMockedGlobal(...)` / `withMockedProperty(...)` from `tests/helpers/mock-global.js` | Use install-style helpers only when the file owns before/after cleanup. | +| Temporary global or property replacement | `withMockedGlobal(...)`, `withMockedProperty(...)`, and `withMockedPropertyDescriptor(...)` from `tests/helpers/mock-global.js` | Use install-style helpers only when the file owns before/after cleanup. | | Console or stream output capture | `withCapturedConsole(...)`, `installConsoleMethodCapture(...)`, or `installStreamWriteCapture(...)` from `tests/helpers/output-capture.js` | Keep direct `console.*` and `process.stderr/stdout.write` replacement out of test files. | | Simple sleeps or polling | `delay(...)` / `waitUntil(...)` from `tests/helpers/timing.js` or the integration `stack.js` re-export | Do not replace sleeps inside tenant worker source strings; those are fixture code under test. | | Temporary directories | `withTempDir(...)` from `tests/helpers/temp-dir.js` | Prefer scoped cleanup over hand-written `mkdtemp` / `rm` `finally` blocks. | @@ -201,7 +202,7 @@ Prefer the narrow helper that matches the response or fixture source: static-analysis utilities (`source-scan.js`), and `mocks/` for mocked Cloudflare runtime surfaces. `load-shared-module.js` owns repository module source loading, data-URL construction, and import-specifier rewrite helpers; - load import-free contract owners such as `shared/version.js` and + load import-free contract owners such as `shared/worker-contract.js` and `shared/ns-pattern.js` directly with `repositoryFileUrl(...)` instead of reimplementing their grammar in a stub. Use it instead of ad hoc `readFileSync(...).replace(...)` chains when a test @@ -218,10 +219,10 @@ Prefer the narrow helper that matches the response or fixture source: `control-handler-harness.js` owns the common `control-shared` harness for `control/handlers/*` unit tests. Use it for state/env/log/metrics/backend injection instead of declaring another file-local `control-shared` data-URL - stub when the handler under test follows the shared control entrypoint - shape. The harness stub imports pure response, bounded JSON body, coded abort, - and optimistic retry contracts from their production owners; keep only - state-bound wiring and test instrumentation local to the stub. + stub when the handler under test follows the shared control entrypoint shape. + `load-control-shared.js` owns the production-backed dependency graph used by + both the real `control/shared.js` test and that synthetic harness stub. Keep + only state-bound wiring and test instrumentation local to the stub. `load-runtime-r2-binding.js` owns the `runtime/bindings/r2.js` host binding module graph, including SigV4Client/fetch recording hooks. Extend that loader for additional R2 host tests instead of copying the replacement graph back @@ -250,8 +251,11 @@ Prefer the narrow helper that matches the response or fixture source: non-`__` `globalThis. = ...` assignments and direct `console.log/warn/error/info = ...` assignments outside those helpers. It also rejects direct assignments to common built-in/global property hooks used - by tests (`process.stderr.*`, `AbortSignal.*`, `Object.*`, - `Headers.prototype.*`, and `Array.prototype.*`). + by tests (`process.stderr.*`, `AbortSignal.*`, `Object.*`, `JSON.*`, + `Headers.prototype.*`, `Array.prototype.*`, and `Function.prototype.*`). + Built-in prototype descriptor mocks use + `withMockedPropertyDescriptor(...)` instead of direct + `Object.defineProperty(...)` save/restore blocks. Unit tests do not boot Docker; they import through these loaders. - `tests/integration/helpers/` is the integration-test helper home. Each concern lives in its own sub-module — `admin-http.js`, `cli.js`, diff --git a/docs/testing.zh.md b/docs/testing.zh.md index 373d98c..9fa95a2 100644 --- a/docs/testing.zh.md +++ b/docs/testing.zh.md @@ -150,12 +150,13 @@ npm install -g @wdl-dev/cli@1.4.0 | Integration workflow scenarios | `tests/integration/helpers/workflows-scenarios.js` 的 worker source 和 DB2 helpers | 用于 workflow demo source、workflow state keys、ready-shard helper 和直接 runtime replay helper；workflow 断言仍留在测试文件内。 | | Integration fetch worker source wrapper | `tests/integration/helpers/worker-source.js` 的 `workerFetchCallerSource(...)` | 多个测试需要相同 `export default { async fetch(req, env) { try { ... } } }` caller 外壳时使用；业务 body 仍留在测试内联可读。 | | Unit control handler module graph | `tests/helpers/control-handler-harness.js` 的 `createControlHandlerState(...)` / `importControlHandler(...)` | 用于需要 `control-shared` state、logs、env、metrics、Redis 或 backend service stub 的 `control/handlers/*` 测试。 | +| Unit Control shared module graph | `tests/helpers/load-control-shared.js` 的 `compileControlSharedGraph(...)` / `compileControlSharedDependencies(...)` | 用于测试 `control/shared.js` 和 synthetic shared stub，不重复构建其 production-backed dependency graph。 | | Unit runtime R2 binding module graph | `tests/helpers/load-runtime-r2-binding.js` 的 `makeR2Bucket(...)` 和 fetch installer | 用于 `runtime/bindings/r2.js` host-surface 测试，不在每个文件里重建 R2 module replacement graph。 | | Unit D1/DO owner-client module graph | `tests/helpers/load-d1-owner-client.js` 和 `tests/helpers/load-do-owner-client.js` 的 `loadD1OwnerClient(...)` / `loadDoOwnerClient(...)` | 用于 owner forwarding client 测试，不在每个文件里重建 state、protocol、internal-auth 和 owner-forwarder replacement graph。 | | Unit auth entrypoint harness state | `tests/helpers/load-auth-index.js` 的 `authMockState(...)`、`authLogs(...)` 和 `lastAuthLog(...)` | 测试不直接读写 `globalThis.__authMockState`；该 global 只是 harness 内联 module mock 的私有存储。 | | Unit/integration Redis command parity | `tests/helpers/redis-conformance-cases.js` 的 `redisConformanceCases` | fake Redis 和真实 integration Redis wrapper 必须对齐某个 command 语义时，加共享 case。运行 `tests/unit/fake-redis.test.js` 和聚焦 Redis conformance integration 文件。 | | 记录 mocked `fetch` 调用 | `tests/helpers/mock-fetch.js` 的 `makeRecordingFetch(...)` / `withRecordingFetch(...)` | 测试需要自定义 call record shape 时使用 `capture`。 | -| 临时替换 global 或 property | `tests/helpers/mock-global.js` 的 `withMockedGlobal(...)` / `withMockedProperty(...)` | 只有文件拥有 before/after cleanup 时才用 install-style helper。 | +| 临时替换 global 或 property | `tests/helpers/mock-global.js` 的 `withMockedGlobal(...)`、`withMockedProperty(...)` 和 `withMockedPropertyDescriptor(...)` | 只有文件拥有 before/after cleanup 时才用 install-style helper。 | | 捕获 console 或 stream 输出 | `tests/helpers/output-capture.js` 的 `withCapturedConsole(...)`、`installConsoleMethodCapture(...)` 或 `installStreamWriteCapture(...)` | 测试文件不要直接替换 `console.*` 或 `process.stderr/stdout.write`。 | | 简单 sleep 或轮询 | `tests/helpers/timing.js` 的 `delay(...)` / `waitUntil(...)`，或 integration `stack.js` re-export | 不要替换 tenant worker source string 里的 sleep；那是被测 fixture 代码。 | | 临时目录 | `tests/helpers/temp-dir.js` 的 `withTempDir(...)` | 优先用 scoped cleanup，不手写 `mkdtemp` / `rm` `finally` 块。 | @@ -164,12 +165,12 @@ npm install -g @wdl-dev/cli@1.4.0 - 被 JavaScript 和 Rust 两侧测试共同读取的 cross-language JSON fixture 放在 `tests/fixtures/`。JavaScript 测试用 `readRepositoryJson(...)`；Rust 测试用 `include_str!(...)` 读取同一个文件。这类 fixture 只 pin 测试合同和 drift guard，不新增 runtime shared owner。 - `tests/helpers/` 是 unit test helper 的归属： - 包含 `load-*.js` ESM data-URL loader 家族（每个被测 repo 模块一份 loader，例如 `load-auth-lib.js`、`load-control-lib.js`、`load-runtime-dispatch.js`）、共享 fixture（`runtime-dispatch-fixtures.js`、`control-shared-stub.js`）、静态分析工具（`source-scan.js`），以及 mock 化 Cloudflare runtime 表面的 `mocks/`。 - - `load-shared-module.js` 负责仓库模块源码读取、data-URL 构造和 import specifier rewrite helper；`shared/version.js`、`shared/ns-pattern.js` 这类无 import 的 contract owner 应通过 `repositoryFileUrl(...)` 直接加载，不在 stub 中重写其 grammar。测试需要重写 repo module import 时，用它代替手写 `readFileSync(...).replace(...)` 链。局部 source rewrite 使用 `readRepositoryFile(...)` 加 `applyModuleReplacements(...)`；需要读取 repo module source 并一次性应用 replacements 时用 `readRepositoryModuleSource(...)`；可复用 repo module 用 `repositoryModuleDataUrl(...)`；import-map 形态的 stub 用 `importSpecifierReplacements(...)`。这是被测试锁住的约定：`tests/unit/test-helper-style-contracts.test.js` 会拒绝共享 loader helper 之外新增的 source-producer `.replace(...)` module rewrite 链，包括直接链、变量中转形态，以及对 `applyModuleReplacements(...)` 产物的二次 rewrite。 - - `control-handler-harness.js` 负责 `control/handlers/*` unit tests 共用的 `control-shared` harness；handler 遵循 shared control entrypoint 形态时，用它注入 state/env/log/metrics/backend service 和 Redis，不再新增 file-local `control-shared` data-URL stub。Harness stub 从 production owner 导入纯 response、bounded JSON body、coded abort 和 optimistic retry contract；本地只保留 state-bound wiring 和 test instrumentation。 + - `load-shared-module.js` 负责仓库模块源码读取、data-URL 构造和 import specifier rewrite helper；`shared/worker-contract.js`、`shared/ns-pattern.js` 这类无 import 的 contract owner 应通过 `repositoryFileUrl(...)` 直接加载，不在 stub 中重写其 grammar。测试需要重写 repo module import 时，用它代替手写 `readFileSync(...).replace(...)` 链。局部 source rewrite 使用 `readRepositoryFile(...)` 加 `applyModuleReplacements(...)`；需要读取 repo module source 并一次性应用 replacements 时用 `readRepositoryModuleSource(...)`；可复用 repo module 用 `repositoryModuleDataUrl(...)`；import-map 形态的 stub 用 `importSpecifierReplacements(...)`。这是被测试锁住的约定：`tests/unit/test-helper-style-contracts.test.js` 会拒绝共享 loader helper 之外新增的 source-producer `.replace(...)` module rewrite 链，包括直接链、变量中转形态，以及对 `applyModuleReplacements(...)` 产物的二次 rewrite。 + - `control-handler-harness.js` 负责 `control/handlers/*` unit tests 共用的 `control-shared` harness；handler 遵循 shared control entrypoint 形态时，用它注入 state/env/log/metrics/backend service 和 Redis，不再新增 file-local `control-shared` data-URL stub。`load-control-shared.js` 负责真实 `control/shared.js` 测试和 synthetic harness stub 共用的 production-backed dependency graph；stub 本地只保留 state-bound wiring 和 test instrumentation。 - `load-runtime-r2-binding.js` 负责 `runtime/bindings/r2.js` host binding module graph，包括 SigV4Client/fetch 记录钩子；新增 R2 host 测试时扩展这个 loader，不把 replacement graph 复制回测试文件。`load-auth-index.js` 负责 auth entrypoint mock state accessor；测试使用 `authMockState(...)`、`authLogs(...)` 和 `lastAuthLog(...)`，不直接触碰 `globalThis.__authMockState`。 - `mocks/fake-redis.js` 负责 unit tests 可复用的 in-memory Redis session/multi 子集，并为读写、batch helper 和 multi ops 提供 command trace；多个测试需要同一 Redis command shape 时扩展它，不再新增 file-local mock。Data-URL `shared-redis` replacement 使用 `sharedRedisStubUrl(...)`，共享 fake `WatchError` constructor 和 production `decodeBulk` semantics；只追加 graph-specific state 或 client exports。 - `request-body.js` 负责 mock backend 捕获 `RequestInit.body` 时的 typed JSON request-body 解析；这类测试不要再直接手写 `JSON.parse(...)`。`do-envelope.js` 负责 Durable Object binary invoke envelope 的测试解码，不再新增 file-local `decodeDoEnvelope()`。style-contract 套件会拒绝新的直接 mock request-body `JSON.parse(...)` 和 file-local DO envelope decoder。 - - `mock-global.js` 负责临时替换 global 和 global property；单个 lexical async scope 用 `withMockedGlobal(...)` / `withMockedProperty(...)`，只有文件需要 before/after hook 所有权时才用 install-style helper。`mock-fetch.js` 是这套 helper 的 typed fetch wrapper。style-contract 套件会扫描仓库作者维护的 `.js` 测试，并拒绝这些 helper 之外新增的直接非 `__` `globalThis. = ...` 赋值、直接 `console.log/warn/error/info = ...` 赋值，以及测试常用 built-in/global property hook（`process.stderr.*`、`AbortSignal.*`、`Object.*`、`Headers.prototype.*`、`Array.prototype.*`）的直接赋值。 + - `mock-global.js` 负责临时替换 global 和 global property；单个 lexical async scope 用 `withMockedGlobal(...)` / `withMockedProperty(...)`，只有文件需要 before/after hook 所有权时才用 install-style helper。`mock-fetch.js` 是这套 helper 的 typed fetch wrapper。style-contract 套件会扫描仓库作者维护的 `.js` 测试，并拒绝这些 helper 之外新增的直接非 `__` `globalThis. = ...` 赋值、直接 `console.log/warn/error/info = ...` 赋值，以及测试常用 built-in/global property hook（`process.stderr.*`、`AbortSignal.*`、`Object.*`、`JSON.*`、`Headers.prototype.*`、`Array.prototype.*`、`Function.prototype.*`）的直接赋值。Built-in prototype descriptor mock 使用 `withMockedPropertyDescriptor(...)`，不手写 `Object.defineProperty(...)` save/restore。 - unit tests 不起 Docker，通过这些 loader 导入。 - `tests/integration/helpers/` 是 integration helper 的归属： - 每个关注点独占子模块：`admin-http.js`、`cli.js`、`compose.js`、`env.js`、`gateway-http.js`、`internal-http.js`、`http-response.js`、`websocket.js`、`stack.js`、`runtimes.js`、`redis.js`、`prometheus.js`、`misc.js`、`worker-source.js`，再加 tier 专用的 `d1-runtime.js` 和 `durable-objects.js`。`index.js` 是供 consumer 用的 barrel re-export。 diff --git a/gateway/config.capnp b/gateway/config.capnp index 1a7028e..fbb3193 100644 --- a/gateway/config.capnp +++ b/gateway/config.capnp @@ -40,7 +40,7 @@ const gatewayWorker :Workerd.Worker = ( (name = "gateway-websocket", esModule = embed "websocket.js"), (name = "shared-worker-id", esModule = embed "../shared/worker-id.js"), (name = "ns-pattern.js", esModule = embed "../shared/ns-pattern.js"), - (name = "version.js", esModule = embed "../shared/version.js"), + (name = "worker-contract.js", esModule = embed "../shared/worker-contract.js"), (name = "shared-redis", esModule = embed "../shared/redis.js"), (name = "shared-redis-command-client", esModule = embed "../shared/redis-command-client.js"), (name = "shared-redis-resp", esModule = embed "../shared/redis-resp.js"), @@ -49,7 +49,7 @@ const gatewayWorker :Workerd.Worker = ( (name = "hex.js", esModule = embed "../shared/hex.js"), (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-route-projection", esModule = embed "../shared/route-projection.js"), - (name = "shared-version", esModule = embed "../shared/version.js"), + (name = "shared-worker-contract", esModule = embed "../shared/worker-contract.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), diff --git a/gateway/dispatch.js b/gateway/dispatch.js index 95c93c6..0cc18b0 100644 --- a/gateway/dispatch.js +++ b/gateway/dispatch.js @@ -19,7 +19,7 @@ import { recordPatternMatchComparisons, recordRoutingLookup, } from "gateway-runtime"; -import { parseVersion } from "shared-version"; +import { parseVersion } from "shared-worker-contract"; /** * @typedef {{ diff --git a/gateway/runtime.js b/gateway/runtime.js index 7804994..813a268 100644 --- a/gateway/runtime.js +++ b/gateway/runtime.js @@ -19,7 +19,7 @@ import { ROUTES_FLUSH_CHANNEL, patternsKey, routesKey, -} from "shared-version"; +} from "shared-worker-contract"; import { isCanonicalPatternHost, sortPatterns } from "gateway-lib"; /** @type {Set | null} */ diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 782bc8e..5192930 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -93,7 +93,7 @@ const loaderWorker :Workerd.Worker = ( (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "shared-base64", esModule = embed "../shared/base64.js"), - (name = "shared-version", esModule = embed "../shared/version.js"), + (name = "shared-worker-contract", esModule = embed "../shared/worker-contract.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), @@ -107,7 +107,7 @@ const loaderWorker :Workerd.Worker = ( (name = "shared-worker-id", esModule = embed "../shared/worker-id.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), (name = "ns-pattern.js", esModule = embed "../shared/ns-pattern.js"), - (name = "version.js", esModule = embed "../shared/version.js"), + (name = "worker-contract.js", esModule = embed "../shared/worker-contract.js"), (name = "shared-d1-timeout", esModule = embed "../shared/d1-timeout.js"), (name = "@wdl-dev/aws-sigv4", esModule = embed "../shared/vendor/aws-sigv4.js"), ], @@ -223,7 +223,7 @@ const controlWorker :Workerd.Worker = ( (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-auth-token", esModule = embed "../shared/auth-token.js"), (name = "shared-auth-roles", esModule = embed "../shared/auth-roles.js"), - (name = "shared-version", esModule = embed "../shared/version.js"), + (name = "shared-worker-contract", esModule = embed "../shared/worker-contract.js"), (name = "shared-cron-time", esModule = embed "../shared/cron-time.js"), (name = "shared-assets-token", esModule = embed "../shared/assets-token.js"), (name = "shared-sql-splitter", esModule = embed "../shared/sql-splitter.js"), @@ -316,7 +316,7 @@ const authWorker :Workerd.Worker = ( (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-auth-token", esModule = embed "../shared/auth-token.js"), (name = "shared-auth-roles", esModule = embed "../shared/auth-roles.js"), - (name = "shared-version", esModule = embed "../shared/version.js"), + (name = "shared-worker-contract", esModule = embed "../shared/worker-contract.js"), (name = "shared-hex", esModule = embed "../shared/hex.js"), (name = "shared-random-id", esModule = embed "../shared/random-id.js"), (name = "shared-redis-lock", esModule = embed "../shared/redis-lock.js"), diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index 666afce..529f020 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -92,7 +92,7 @@ const loaderWorker :Workerd.Worker = ( (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "shared-base64", esModule = embed "../shared/base64.js"), - (name = "shared-version", esModule = embed "../shared/version.js"), + (name = "shared-worker-contract", esModule = embed "../shared/worker-contract.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), @@ -106,7 +106,7 @@ const loaderWorker :Workerd.Worker = ( (name = "shared-worker-id", esModule = embed "../shared/worker-id.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), (name = "ns-pattern.js", esModule = embed "../shared/ns-pattern.js"), - (name = "version.js", esModule = embed "../shared/version.js"), + (name = "worker-contract.js", esModule = embed "../shared/worker-contract.js"), (name = "shared-d1-timeout", esModule = embed "../shared/d1-timeout.js"), (name = "@wdl-dev/aws-sigv4", esModule = embed "../shared/vendor/aws-sigv4.js"), ], diff --git a/runtime/lib.js b/runtime/lib.js index 1333437..07c96db 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -2,7 +2,7 @@ import { base64ToBytes, bytesToBase64 } from "shared-base64"; import { WORKER_NAME_RE, isValidRouteNs } from "shared-ns-pattern"; -import { parseVersion } from "shared-version"; +import { parseVersion } from "shared-worker-contract"; import { DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE, ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE, diff --git a/runtime/load/env-build.js b/runtime/load/env-build.js index 0d36d44..c90ac6f 100644 --- a/runtime/load/env-build.js +++ b/runtime/load/env-build.js @@ -7,7 +7,7 @@ import { isValidJsIdentifier, isValidJsClassDeclarationName, } from "shared-ns-pattern"; -import { parseVersion } from "shared-version"; +import { parseVersion } from "shared-worker-contract"; const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; diff --git a/rust/common/src/identity.rs b/rust/common/src/identity.rs index 836c848..1a80c26 100644 --- a/rust/common/src/identity.rs +++ b/rust/common/src/identity.rs @@ -1,11 +1,11 @@ //! Runtime worker identity grammar shared by Rust services. //! //! JavaScript owns the canonical tenant namespace, route namespace, worker-name, -//! and version grammars in `shared/ns-pattern.js` and `shared/version.js`. Rust +//! and version grammars in `shared/ns-pattern.js` and `shared/worker-contract.js`. Rust //! services that sit on protocol boundaries mirror those rules here so internal //! requests cannot reach Redis key shapes that control would never write. -use crate::version::parse_version_tag; +use crate::worker_contract::parse_version_tag; pub fn is_valid_tenant_ns(ns: &str) -> bool { let bytes = ns.as_bytes(); diff --git a/rust/common/src/lib.rs b/rust/common/src/lib.rs index cd2c5b0..2e4f0ad 100644 --- a/rust/common/src/lib.rs +++ b/rust/common/src/lib.rs @@ -4,7 +4,7 @@ //! environment parsing, log-level parsing and common log-line formatting, HTTP //! health probes, shutdown/in-flight tracking, metric storage/formatting, FNV hashing, //! request-id sanitization, internal-auth constants/token matching, identity grammar, -//! version and queue-key helpers, Redis connection and EVAL command helpers, time +//! worker contract and queue-key helpers, Redis connection and EVAL command helpers, time //! helpers, structured error fields, test-only process-environment overrides, and //! UTF-8-safe text helpers. It should not own service protocols, Redis schemas, //! dispatch policy, or lifecycle behavior. @@ -26,7 +26,7 @@ pub mod shutdown; pub mod test_env; pub mod text; pub mod time; -pub mod version; +pub mod worker_contract; #[cfg(test)] pub(crate) mod test_fixtures { diff --git a/rust/common/src/version.rs b/rust/common/src/worker_contract.rs similarity index 90% rename from rust/common/src/version.rs rename to rust/common/src/worker_contract.rs index 3d32901..ac964ea 100644 --- a/rust/common/src/version.rs +++ b/rust/common/src/worker_contract.rs @@ -1,13 +1,12 @@ -//! Worker version and bundle-key helpers shared by Rust services. +//! Worker version grammar and control-plane key helpers shared by Rust services. //! //! JavaScript control code owns the canonical version tag grammar in -//! `shared/version.js`: `v[1-9][0-9]*`, bounded to JavaScript's safe-integer -//! range. Rust services that read bundle hashes -//! must use the same grammar so malformed Redis state fails closed instead of -//! silently normalizing to another worker version. +//! `shared/worker-contract.js`: `v[1-9][0-9]*`, bounded to JavaScript's safe-integer +//! range. Rust services that read bundle hashes must use the same grammar so malformed +//! Redis state fails closed instead of silently normalizing to another worker version. //! //! `routes_key` / `worker_versions_key` / `worker_delete_lock_key` / -//! `do_storage_id_key` mirror `shared/version.js`'s lifecycle key builders. +//! `do_storage_id_key` mirror `shared/worker-contract.js`'s lifecycle key builders. //! Control owns these keys; Rust readers must build them here so a future //! key-grammar change updates JS and Rust together. diff --git a/rust/redis-proxy/src/runtime.rs b/rust/redis-proxy/src/runtime.rs index e3b8e8f..3705497 100644 --- a/rust/redis-proxy/src/runtime.rs +++ b/rust/redis-proxy/src/runtime.rs @@ -6,7 +6,7 @@ use axum::response::{IntoResponse, Response}; use serde::Deserialize; use serde_json::json; use wdl_rust_common::identity::is_valid_runtime_worker_identity; -use wdl_rust_common::version::worker_bundle_key; +use wdl_rust_common::worker_contract::worker_bundle_key; use crate::{AppError, AppResult, AppState}; @@ -25,7 +25,7 @@ pub(crate) struct RuntimeLoadParams { } // Bundles are stored by integer ("worker:::v:42"), not the // `v` tag. Strip the prefix and validate so the Redis key shape -// matches how the control-plane writer (shared/version.js#bundleKey) +// matches how the control-plane writer (shared/worker-contract.js#bundleKey) // spells it; otherwise a cold load misses the hash entirely. pub(crate) fn bundle_key(ns: &str, worker: &str, version: &str) -> AppResult { if !is_valid_runtime_worker_identity(ns, worker, version) { diff --git a/rust/scheduler/src/cron/reference.rs b/rust/scheduler/src/cron/reference.rs index 3c2ad57..5e0ba1c 100644 --- a/rust/scheduler/src/cron/reference.rs +++ b/rust/scheduler/src/cron/reference.rs @@ -1,7 +1,7 @@ use serde::Deserialize; use wdl_rust_common::{ identity::{is_valid_route_ns, is_valid_worker_name}, - version::parse_version_tag, + worker_contract::parse_version_tag, }; const CRON_WORKER_KEY_PREFIX: &str = "crons:"; diff --git a/rust/scheduler/src/queue/registry.rs b/rust/scheduler/src/queue/registry.rs index 94db856..b1775b7 100644 --- a/rust/scheduler/src/queue/registry.rs +++ b/rust/scheduler/src/queue/registry.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use redis::{AsyncCommands, Value}; use wdl_rust_common::identity::{is_valid_route_ns, is_valid_worker_name}; use wdl_rust_common::queue_keys::is_valid_queue_name; -use wdl_rust_common::version::parse_version_tag; +use wdl_rust_common::worker_contract::parse_version_tag; use crate::{ AppState, CONSUMER_GROUP, MAX_BATCH_SIZE_CAP, QueueState, SchedulerResult, indexed_data_keys, diff --git a/rust/workflows/src/api/active_export.rs b/rust/workflows/src/api/active_export.rs index 7da1295..9fcf525 100644 --- a/rust/workflows/src/api/active_export.rs +++ b/rust/workflows/src/api/active_export.rs @@ -1,5 +1,5 @@ use serde_json::Value as JsonValue; -use wdl_rust_common::version::{routes_key, worker_delete_lock_key}; +use wdl_rust_common::worker_contract::{routes_key, worker_delete_lock_key}; use crate::{AppState, WorkflowError, WorkflowResult}; diff --git a/rust/workflows/src/api/do_alarms/dispatch.rs b/rust/workflows/src/api/do_alarms/dispatch.rs index fd4a908..94b7d94 100644 --- a/rust/workflows/src/api/do_alarms/dispatch.rs +++ b/rust/workflows/src/api/do_alarms/dispatch.rs @@ -4,7 +4,7 @@ use serde::Serialize; use serde_json::{Value as JsonValue, json}; use wdl_rust_common::internal_auth::INTERNAL_AUTH_HEADER; use wdl_rust_common::time::now_ms; -use wdl_rust_common::version::{ +use wdl_rust_common::worker_contract::{ do_storage_id_key, parse_version_tag, routes_key, worker_versions_key, }; diff --git a/rust/workflows/src/api/do_alarms/model.rs b/rust/workflows/src/api/do_alarms/model.rs index 09f0bfa..7a983b5 100644 --- a/rust/workflows/src/api/do_alarms/model.rs +++ b/rust/workflows/src/api/do_alarms/model.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use wdl_rust_common::hash::fnv1a32; use wdl_rust_common::identity::{is_valid_runtime_load_ns, is_valid_worker_name}; -use wdl_rust_common::version::parse_version_tag; +use wdl_rust_common::worker_contract::parse_version_tag; use crate::{ DoAlarmJobKeys, WorkflowError, WorkflowResult, do_alarm_by_worker_key, do_alarm_job_id, diff --git a/rust/workflows/src/api/do_alarms/mutations.rs b/rust/workflows/src/api/do_alarms/mutations.rs index 56af2d8..2b914be 100644 --- a/rust/workflows/src/api/do_alarms/mutations.rs +++ b/rust/workflows/src/api/do_alarms/mutations.rs @@ -1,7 +1,7 @@ use serde_json::json; use wdl_rust_common::redis_eval::append_eval_cmd; use wdl_rust_common::time::{now_ms, random_hex_64}; -use wdl_rust_common::version::do_storage_id_key; +use wdl_rust_common::worker_contract::do_storage_id_key; use crate::{AppState, LogLevel, WorkflowResult, do_alarm_by_worker_key, log}; diff --git a/rust/workflows/src/api/routing.rs b/rust/workflows/src/api/routing.rs index 1229ff2..92dc987 100644 --- a/rust/workflows/src/api/routing.rs +++ b/rust/workflows/src/api/routing.rs @@ -1,4 +1,4 @@ -use wdl_rust_common::{hash::fnv1a32, version::worker_bundle_key}; +use wdl_rust_common::{hash::fnv1a32, worker_contract::worker_bundle_key}; use crate::{InstanceKeys, WorkflowError, WorkflowResult, due_key, ready_key}; diff --git a/shared/ns-pattern.js b/shared/ns-pattern.js index ac954e8..4b717cf 100644 --- a/shared/ns-pattern.js +++ b/shared/ns-pattern.js @@ -124,11 +124,21 @@ export function isValidRuntimeLoadNs(ns) { // common Wrangler-style uppercase and underscore spellings. export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,254}$/; +/** @param {unknown} name */ +export function isValidWorkerName(name) { + return typeof name === "string" && WORKER_NAME_RE.test(name); +} + // Workflow names are class-adjacent user identifiers rather than Redis queue // ids, so they follow the worker-name family while keeping a smaller bound for // UI/API surfaces. export const WORKFLOW_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; +/** @param {unknown} name */ +export function isValidWorkflowName(name) { + return typeof name === "string" && WORKFLOW_NAME_RE.test(name); +} + // Workflow instance ids are embedded into DB2 keys and tab-delimited scheduler // tokens. Keep them URL/key friendly and delimiter-free. export const WORKFLOW_INSTANCE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; @@ -138,6 +148,11 @@ export const WORKFLOW_KEY_RE = /^wf_[0-9a-f]{32}$/; // would split one logical queue across two log-field entries. export const QUEUE_NAME_RE = /^[a-z0-9][a-z0-9-]{0,62}$/; +/** @param {unknown} name */ +export function isValidQueueName(name) { + return typeof name === "string" && QUEUE_NAME_RE.test(name); +} + // JS identifier — CF wrangler's "valid JavaScript variable name". // `__proto__` / `constructor` guard is separate in RESERVED_OBJECT_KEYS // so the regex stays CF-shaped. Cap 64 for log / metric label hygiene. @@ -235,6 +250,11 @@ export const WDL_RESERVED_ENTRYPOINT_RE = /^__Wdl[A-Za-z0-9_]*__$/; // key="bar") and (id="foo", key="v:bar"). export const KV_ID_RE = /^[a-z0-9][a-z0-9-]{0,62}$/; +/** @param {unknown} id */ +export function isValidKvId(id) { + return typeof id === "string" && KV_ID_RE.test(id); +} + // D1 API ids are stable control-plane references and may retain the // mixed-case/underscore forms accepted by the existing D1 model. export const D1_DATABASE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; diff --git a/shared/version.js b/shared/worker-contract.js similarity index 90% rename from shared/version.js rename to shared/worker-contract.js index 4b34de9..595b04f 100644 --- a/shared/version.js +++ b/shared/worker-contract.js @@ -1,7 +1,5 @@ -// Single point of truth for the `v` version tag shape used in -// `routes:` values and worker-bundle keys. Enforcing it here prevents -// silent format drift between the INCR path (new deploys) and the HGET -// path (promote / versions listing). +// Canonical JS contract for immutable worker versions and the control-plane +// Redis keys and channels shared across in-tree tiers. const VERSION_RE = /^v([1-9][0-9]*)$/; @@ -34,20 +32,20 @@ export function parseVersion(tag) { // another subkey even if someone ever writes Redis directly. /** * @param {string} ns - * @param {string} name + * @param {string} worker * @param {unknown} version * @returns {string} */ -export function bundleKey(ns, name, version) { +export function bundleKey(ns, worker, version) { const n = parseVersion(version); if (n == null) throw new Error(`invalid version tag ${JSON.stringify(version)}`); - return `worker:${ns}:${name}:v:${n}`; + return `worker:${ns}:${worker}:v:${n}`; } // Monotonic immutable-version allocator for one logical worker. -/** @param {string} ns @param {string} name */ -export function nextVersionKey(ns, name) { - return `worker:${ns}:${name}:next_version`; +/** @param {string} ns @param {string} worker */ +export function nextVersionKey(ns, worker) { + return `worker:${ns}:${worker}:next_version`; } // Active-route hash for a namespace: field=workerName, value=`v`. Control diff --git a/shared/worker-id.js b/shared/worker-id.js index c59e1c7..3a59579 100644 --- a/shared/worker-id.js +++ b/shared/worker-id.js @@ -1,5 +1,5 @@ import { isValidRouteNs, isValidRuntimeLoadNs, WORKER_NAME_RE } from "./ns-pattern.js"; -import { parseVersion } from "./version.js"; +import { parseVersion } from "./worker-contract.js"; /** @param {{ namespace: unknown, worker: unknown, version: unknown }} id */ export function formatWorkerId({ namespace, worker, version }) { diff --git a/tests/helpers/control-shared-stub.js b/tests/helpers/control-shared-stub.js index 2563f66..66fcb37 100644 --- a/tests/helpers/control-shared-stub.js +++ b/tests/helpers/control-shared-stub.js @@ -3,36 +3,22 @@ // helpers. import { - freshRepositoryModuleDataUrl, moduleDataUrl, - repositoryFileUrl, } from "./load-shared-module.js"; +import { compileControlSharedDependencies } from "./load-control-shared.js"; import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; -const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); -const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); -const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); -const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); -const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); -const SHARED_OPTIMISTIC_RETRY_URL = repositoryFileUrl("shared/optimistic-retry.js"); -const CONTROL_ERRORS_URL = freshRepositoryModuleDataUrl("control/errors.js", [ - [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], - [/from "shared-respond"/g, `from ${JSON.stringify(SHARED_RESPOND_URL)}`], -]); -const CONTROL_WORKFLOWS_CLIENT_URL = freshRepositoryModuleDataUrl("control/workflows-client.js", [ - [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], - [/from "control-errors"/g, `from ${JSON.stringify(CONTROL_ERRORS_URL)}`], -]); -const CONTROL_OPTIMISTIC_REDIS_URL = sharedRedisStubUrl(); -const CONTROL_OPTIMISTIC_URL = freshRepositoryModuleDataUrl("control/optimistic.js", [ - [/from "shared-redis"/g, `from ${JSON.stringify(CONTROL_OPTIMISTIC_REDIS_URL)}`], - [/from "shared-optimistic-retry"/g, `from ${JSON.stringify(SHARED_OPTIMISTIC_RETRY_URL)}`], -]); -const CONTROL_JSON_BODY_URL = freshRepositoryModuleDataUrl("control/json-body.js", [ - [/from "shared-bounded-body"/g, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)}`], - [/from "shared-respond"/g, `from ${JSON.stringify(SHARED_RESPOND_URL)}`], -]); +const { + sharedErrorsUrl: SHARED_ERRORS_URL, + sharedInternalAuthUrl: SHARED_INTERNAL_AUTH_URL, + sharedRandomIdUrl: SHARED_RANDOM_ID_URL, + sharedRespondUrl: SHARED_RESPOND_URL, + controlErrorsUrl: CONTROL_ERRORS_URL, + controlWorkflowsClientUrl: CONTROL_WORKFLOWS_CLIENT_URL, + controlOptimisticUrl: CONTROL_OPTIMISTIC_URL, + controlJsonBodyUrl: CONTROL_JSON_BODY_URL, +} = compileControlSharedDependencies({ sharedRedisUrl: sharedRedisStubUrl() }); const CONTROL_SHARED_BASE = ` import { jsonError, jsonResponse, sanitizeJsonErrorDetails } from ${JSON.stringify(SHARED_RESPOND_URL)}; diff --git a/tests/helpers/load-auth-index.js b/tests/helpers/load-auth-index.js index 9d2cc2f..7a71512 100644 --- a/tests/helpers/load-auth-index.js +++ b/tests/helpers/load-auth-index.js @@ -22,7 +22,7 @@ const SHARED_REDIS_LOCK_URL = freshRepositoryModuleDataUrl("shared/redis-lock.js [/from "shared-random-id"/g, `from ${JSON.stringify(SHARED_RANDOM_ID_URL)}`], ]); const SHARED_OBSERVABILITY_URL = repositoryFileUrl("shared/observability.js"); -const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); +const WORKER_CONTRACT_URL = repositoryFileUrl("shared/worker-contract.js"); const SHARED_REDIS_MOCK = ` function ensureState() { @@ -379,7 +379,7 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel [/from "auth-runtime"/g, `from ${JSON.stringify(authRuntimeUrl)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], [/from "shared-redis-lock"/g, `from ${JSON.stringify(SHARED_REDIS_LOCK_URL)}`], - [/from "shared-version"/g, `from ${JSON.stringify(SHARED_VERSION_URL)}`], + [/from "shared-worker-contract"/g, `from ${JSON.stringify(WORKER_CONTRACT_URL)}`], ]); const indexMod = await import(indexUrl); diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index e9cca16..05c1c63 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -16,7 +16,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const moduleRequire = createRequire(path.resolve(__dirname, "../../package.json")); const SHARED_NS_URL = repositoryFileUrl("shared/ns-pattern.js"); const SHARED_QUEUE_KEYS_URL = repositoryFileUrl("shared/queue-keys.js"); -const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); +const WORKER_CONTRACT_URL = repositoryFileUrl("shared/worker-contract.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const SHARED_ROUTE_PROJECTION_URL = repositoryFileUrl("shared/route-projection.js"); const SHARED_WORKERD_COMPAT_FLAGS_URL = repositoryFileUrl("shared/workerd-compat-flags.js"); @@ -62,7 +62,7 @@ export async function compileControlGraph(opts = {}) { const lifecycleIndexesUrl = freshRepositoryModuleDataUrl("control/lifecycle-indexes.js", [ [/from "control-lib"/g, `from ${JSON.stringify(libUrl)}`], [/from "shared-queue-keys"/g, `from ${JSON.stringify(SHARED_QUEUE_KEYS_URL)}`], - [/from "shared-version"/g, `from ${JSON.stringify(SHARED_VERSION_URL)}`], + [/from "shared-worker-contract"/g, `from ${JSON.stringify(WORKER_CONTRACT_URL)}`], ]); const cronIndexUrl = freshRepositoryModuleDataUrl("control/cron-index.js", [ @@ -91,7 +91,7 @@ export async function compileControlGraph(opts = {}) { sharedAuthRolesUrl, sharedAuthRoles, sharedQueueKeysUrl: SHARED_QUEUE_KEYS_URL, - sharedVersionUrl: SHARED_VERSION_URL, + workerContractUrl: WORKER_CONTRACT_URL, sharedErrorsUrl: SHARED_ERRORS_URL, sharedRouteProjectionUrl: SHARED_ROUTE_PROJECTION_URL, sharedCronTimeUrl, diff --git a/tests/helpers/load-control-routing.js b/tests/helpers/load-control-routing.js index ac4aaf0..b1b576a 100644 --- a/tests/helpers/load-control-routing.js +++ b/tests/helpers/load-control-routing.js @@ -12,7 +12,7 @@ const { sharedNsUrl, sharedAuthRolesUrl, sharedQueueKeysUrl, - sharedVersionUrl, + workerContractUrl, sharedErrorsUrl, sharedRouteProjectionUrl, libUrl: controlLibUrl, @@ -33,7 +33,7 @@ export const CONTROL_ROUTING_TEST_URL = moduleDataUrl(applyModuleReplacements(re "control-topology": topologyUrl, "shared-errors": sharedErrorsUrl, "shared-route-projection": sharedRouteProjectionUrl, - "shared-version": sharedVersionUrl, + "shared-worker-contract": workerContractUrl, "control-cron-index": cronIndexUrl, "shared-ns-pattern": sharedNsUrl, "shared-auth-roles": sharedAuthRolesUrl, diff --git a/tests/helpers/load-control-shared.js b/tests/helpers/load-control-shared.js new file mode 100644 index 0000000..f66ab7c --- /dev/null +++ b/tests/helpers/load-control-shared.js @@ -0,0 +1,116 @@ +import { + freshRepositoryModuleDataUrl, + importSpecifierReplacements, + repositoryFileUrl, +} from "./load-shared-module.js"; +import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; + +const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); +const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); +const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_OPTIMISTIC_RETRY_URL = repositoryFileUrl("shared/optimistic-retry.js"); +const SHARED_QUEUE_KEYS_URL = repositoryFileUrl("shared/queue-keys.js"); +const SHARED_S3_CLEANUP_LIFECYCLE_URL = repositoryFileUrl("shared/s3-cleanup-lifecycle.js"); +const WORKER_CONTRACT_URL = repositoryFileUrl("shared/worker-contract.js"); + +/** + * Compile the production-backed pure dependencies shared by the synthetic + * control-shared stub and the real control/shared.js module graph. + * + * @param {{ sharedRedisUrl: string }} options + */ +export function compileControlSharedDependencies({ sharedRedisUrl }) { + const controlErrorsUrl = freshRepositoryModuleDataUrl( + "control/errors.js", + importSpecifierReplacements({ + "shared-errors": SHARED_ERRORS_URL, + "shared-respond": SHARED_RESPOND_URL, + }), + ); + const controlWorkflowsClientUrl = freshRepositoryModuleDataUrl( + "control/workflows-client.js", + importSpecifierReplacements({ + "shared-errors": SHARED_ERRORS_URL, + "control-errors": controlErrorsUrl, + }), + ); + const controlOptimisticUrl = freshRepositoryModuleDataUrl( + "control/optimistic.js", + importSpecifierReplacements({ + "shared-redis": sharedRedisUrl, + "shared-optimistic-retry": SHARED_OPTIMISTIC_RETRY_URL, + }), + ); + const controlJsonBodyUrl = freshRepositoryModuleDataUrl( + "control/json-body.js", + importSpecifierReplacements({ + "shared-bounded-body": SHARED_BOUNDED_BODY_URL, + "shared-respond": SHARED_RESPOND_URL, + }), + ); + + return { + sharedErrorsUrl: SHARED_ERRORS_URL, + sharedInternalAuthUrl: SHARED_INTERNAL_AUTH_URL, + sharedRandomIdUrl: SHARED_RANDOM_ID_URL, + sharedRespondUrl: SHARED_RESPOND_URL, + controlErrorsUrl, + controlWorkflowsClientUrl, + controlOptimisticUrl, + controlJsonBodyUrl, + }; +} + +/** + * @param {{ + * sharedRedisUrl: string, + * controlS3Url: string, + * controlR2Url: string, + * sharedAuthTokenUrl: string, + * sharedAuthRolesUrl: string, + * sharedQueueKeysUrl?: string, + * }} options + */ +export function compileControlSharedGraph({ + sharedRedisUrl, + controlS3Url, + controlR2Url, + sharedAuthTokenUrl, + sharedAuthRolesUrl, + sharedQueueKeysUrl = SHARED_QUEUE_KEYS_URL, +}) { + const dependencies = compileControlSharedDependencies({ sharedRedisUrl }); + const sharedRedisLockUrl = freshRepositoryModuleDataUrl( + "shared/redis-lock.js", + importSpecifierReplacements({ + "shared-random-id": dependencies.sharedRandomIdUrl, + }), + ); + const controlSharedUrl = freshRepositoryModuleDataUrl( + "control/shared.js", + importSpecifierReplacements({ + "shared-redis": sharedRedisUrl, + "control-s3": controlS3Url, + "control-r2": controlR2Url, + "shared-auth-token": sharedAuthTokenUrl, + "shared-auth-roles": sharedAuthRolesUrl, + "shared-queue-keys": sharedQueueKeysUrl, + "shared-respond": dependencies.sharedRespondUrl, + "shared-internal-auth": dependencies.sharedInternalAuthUrl, + "shared-errors": dependencies.sharedErrorsUrl, + "shared-random-id": dependencies.sharedRandomIdUrl, + "shared-worker-contract": WORKER_CONTRACT_URL, + "shared-s3-cleanup-lifecycle": SHARED_S3_CLEANUP_LIFECYCLE_URL, + "shared-observability": OBSERVABILITY_NOOP_URL, + "control-workflows-client": dependencies.controlWorkflowsClientUrl, + "control-errors": dependencies.controlErrorsUrl, + "control-optimistic": dependencies.controlOptimisticUrl, + "control-json-body": dependencies.controlJsonBodyUrl, + "shared-redis-lock": sharedRedisLockUrl, + }), + ); + + return { ...dependencies, controlSharedUrl }; +} diff --git a/tests/helpers/load-do-owner-registry.js b/tests/helpers/load-do-owner-registry.js index f472313..c6a1531 100644 --- a/tests/helpers/load-do-owner-registry.js +++ b/tests/helpers/load-do-owner-registry.js @@ -17,7 +17,7 @@ const PROTOCOL_URL = doProtocolDataUrl(); const FAKE_REDIS_URL = repositoryFileUrl("tests/helpers/mocks/fake-redis.js"); const SHARED_ENV_URL = repositoryFileUrl("shared/env.js"); const SHARED_OWNER_LEASE_URL = sharedModuleDataUrl("shared/owner-lease.js"); -const SHARED_VERSION_URL = sharedModuleDataUrl("shared/version.js"); +const WORKER_CONTRACT_URL = sharedModuleDataUrl("shared/worker-contract.js"); const SHARED_OWNER_PROTOCOL_URL = repositoryModuleDataUrl("shared/owner-protocol.js", [ [/from "shared-owner-lease";/, `from ${JSON.stringify(SHARED_OWNER_LEASE_URL)};`], ]); @@ -130,7 +130,7 @@ const src = applyModuleReplacements(readRepositoryFile("do-runtime/owner-registr [/from "shared-env";/, `from ${JSON.stringify(SHARED_ENV_URL)};`], [/from "shared-owner-lease";/, `from ${JSON.stringify(SHARED_OWNER_LEASE_URL)};`], [/from "shared-owner-protocol";/, `from ${JSON.stringify(SHARED_OWNER_PROTOCOL_URL)};`], - [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(WORKER_CONTRACT_URL)};`], [/from "do-runtime-state";/, `from ${JSON.stringify(stateUrl)};`], [/from "shared-errors";/, `from ${JSON.stringify(repositoryFileUrl("shared/errors.js"))};`], [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], diff --git a/tests/helpers/load-do-protocol.js b/tests/helpers/load-do-protocol.js index dccfe68..a91d3cf 100644 --- a/tests/helpers/load-do-protocol.js +++ b/tests/helpers/load-do-protocol.js @@ -10,7 +10,7 @@ const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); const SHARED_NS_PATTERN_URL = repositoryFileUrl("shared/ns-pattern.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); -const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); +const WORKER_CONTRACT_URL = repositoryFileUrl("shared/worker-contract.js"); const DO_WIRE_GRAMMAR_URL = repositoryFileUrl("do-runtime/protocol/wire-grammar.js"); const DO_ERRORS_URL = repositoryModuleDataUrl("do-runtime/protocol/errors.js", [ [/from "shared-respond";/g, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], @@ -31,7 +31,7 @@ export function doProtocolDataUrl() { "shared-bounded-body": SHARED_BOUNDED_BODY_URL, "shared-internal-auth": SHARED_INTERNAL_AUTH_URL, "shared-ns-pattern": SHARED_NS_PATTERN_URL, - "shared-version": SHARED_VERSION_URL, + "shared-worker-contract": WORKER_CONTRACT_URL, }), ]); } diff --git a/tests/helpers/load-shared-module.js b/tests/helpers/load-shared-module.js index 66a8914..7b665d6 100644 --- a/tests/helpers/load-shared-module.js +++ b/tests/helpers/load-shared-module.js @@ -12,7 +12,7 @@ const SHARED_ENV_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/env.js")).h const SHARED_ERRORS_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/errors.js")).href; const SHARED_BASE64_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/base64.js")).href; const SHARED_NS_PATTERN_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/ns-pattern.js")).href; -const SHARED_VERSION_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/version.js")).href; +const WORKER_CONTRACT_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/worker-contract.js")).href; const SHARED_HEX_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/hex.js")).href; const SHARED_OPTIMISTIC_RETRY_URL = pathToFileURL( path.resolve(REPO_ROOT, "shared/optimistic-retry.js") @@ -95,7 +95,7 @@ export function runtimeLibModuleDataUrl() { return repositoryModuleDataUrl("runtime/lib.js", importSpecifierReplacements({ "shared-base64": SHARED_BASE64_URL, "shared-ns-pattern": SHARED_NS_PATTERN_URL, - "shared-version": SHARED_VERSION_URL, + "shared-worker-contract": WORKER_CONTRACT_URL, "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, })); } diff --git a/tests/integration/helpers/durable-objects.js b/tests/integration/helpers/durable-objects.js index 6a6a0cb..af2170a 100644 --- a/tests/integration/helpers/durable-objects.js +++ b/tests/integration/helpers/durable-objects.js @@ -1,7 +1,7 @@ import { readFileSync } from "node:fs"; import assert from "node:assert/strict"; import { fnv1a32CodeUnits } from "../../../shared/fnv1a32.js"; -import { doStorageIdKey } from "../../../shared/version.js"; +import { doStorageIdKey } from "../../../shared/worker-contract.js"; import { doAlarmJobIdForStorage } from "../../helpers/do-alarm-job-id.js"; import { serviceInternalPost, serviceInternalPostAsync } from "./internal-http.js"; import { diff --git a/tests/integration/helpers/misc.js b/tests/integration/helpers/misc.js index 3707819..5e76b42 100644 --- a/tests/integration/helpers/misc.js +++ b/tests/integration/helpers/misc.js @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { bundleKey } from "../../../shared/version.js"; +import { bundleKey } from "../../../shared/worker-contract.js"; import { redisHGetJson } from "./redis.js"; diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index 1d5ee12..a63dc81 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -15,14 +15,14 @@ import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; const { libUrl: productionControlLibUrl, lifecycleIndexesUrl } = await compileControlGraph(); -const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const workerContractUrl = repositoryFileUrl("shared/worker-contract.js"); const controlSharedExtraSource = ` export { PATTERNS_CHANNEL, ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL, -} from ${JSON.stringify(sharedVersionUrl)}; +} from ${JSON.stringify(workerContractUrl)}; export async function acquireDeleteLock(_redis, _ns, _worker, kind) { /** @type {any} */ (globalThis).__deleteHandlerState.lockKinds.push(kind); return "lock-token"; @@ -129,7 +129,7 @@ export function decodePatternProjection(raw) { const deletePlanSrc = applyModuleReplacements(readRepositoryFile("control/handlers/delete-plan.js"), [ [/from "control-lib";/, `from ${JSON.stringify(controlLibUrl)};`], [/from "control-lifecycle-indexes";/, `from ${JSON.stringify(lifecycleIndexesUrl)};`], - [/from "shared-version";/, `from ${JSON.stringify(sharedVersionUrl)};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(workerContractUrl)};`], [/from "shared-secret-keys";/, `from ${JSON.stringify(sharedSecretKeysUrl)};`], ]); const deletePlanUrl = moduleDataUrl(deletePlanSrc); @@ -140,7 +140,7 @@ const { handle } = await importControlHandler("control/handlers/delete.js", { replacements: { "control-lib": controlLibUrl, "control-lifecycle-indexes": lifecycleIndexesUrl, - "shared-version": sharedVersionUrl, + "shared-worker-contract": workerContractUrl, "shared-secret-keys": sharedSecretKeysUrl, "shared-redis": sharedRedisUrl, "shared-queue-keys": sharedQueueKeysUrl, @@ -156,7 +156,7 @@ const { handle: handleVersions } = await importControlHandler("control/handlers/ replacements: { "control-lib": controlLibUrl, "control-lifecycle-indexes": lifecycleIndexesUrl, - "shared-version": sharedVersionUrl, + "shared-worker-contract": workerContractUrl, "shared-secret-keys": sharedSecretKeysUrl, "shared-redis": sharedRedisUrl, }, diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 19c5e0b..b57451a 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -199,7 +199,7 @@ export function parseQueueConsumers() { } `); -const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const workerContractUrl = repositoryFileUrl("shared/worker-contract.js"); const sharedRedisUrl = sharedRedisStubUrl(); @@ -319,7 +319,7 @@ const { commitWithWatch, handle } = await importControlHandler("control/handlers "control-bundle": controlBundleUrl, "control-bindings": controlBindingsUrl, "control-topology": controlTopologyUrl, - "shared-version": sharedVersionUrl, + "shared-worker-contract": workerContractUrl, "shared-redis": sharedRedisUrl, "shared-ns-pattern": sharedNsUrl, "shared-auth-roles": sharedAuthRolesUrl, diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index fe63c9c..f48c076 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -12,13 +12,13 @@ import { encryptSecretValue } from "../../shared/secret-envelope.js"; const { libUrl: controlLibUrl } = await compileControlGraph(); const secretEnvelopeUrl = repositoryFileUrl("shared/secret-envelope.js"); -const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const workerContractUrl = repositoryFileUrl("shared/worker-contract.js"); const sharedRedisUrl = sharedRedisStubUrl(); const runtimeEnvBuildUrl = repositoryModuleDataUrl( "runtime/load/env-build.js", importSpecifierReplacements({ "shared-ns-pattern": repositoryFileUrl("shared/ns-pattern.js"), - "shared-version": repositoryFileUrl("shared/version.js"), + "shared-worker-contract": repositoryFileUrl("shared/worker-contract.js"), }) ); const { @@ -37,7 +37,7 @@ const { "control-lib": controlLibUrl, "runtime-load-env-build": runtimeEnvBuildUrl, "shared-secret-envelope": secretEnvelopeUrl, - "shared-version": sharedVersionUrl, + "shared-worker-contract": workerContractUrl, "shared-redis": sharedRedisUrl, })); diff --git a/tests/unit/control-handlers-workers.test.js b/tests/unit/control-handlers-workers.test.js index 05b4796..2f38f3e 100644 --- a/tests/unit/control-handlers-workers.test.js +++ b/tests/unit/control-handlers-workers.test.js @@ -24,7 +24,7 @@ const workersHandlerState = installControlHandlerState( }) ); const controlSharedUrl = controlSharedHarnessUrl(WORKERS_HANDLER_STATE_GLOBAL); -const { libUrl: controlLibUrl, sharedVersionUrl } = await compileControlGraph(); +const { libUrl: controlLibUrl, workerContractUrl } = await compileControlGraph(); const sharedSecretKeysUrl = repositoryFileUrl("shared/secret-keys.js"); @@ -32,7 +32,7 @@ const src = applyModuleReplacements(readRepositoryFile("control/handlers/workers [/from "control-shared";/, `from ${JSON.stringify(controlSharedUrl)};`], [/from "control-lib";/, `from ${JSON.stringify(controlLibUrl)};`], [/from "shared-secret-keys";/, `from ${JSON.stringify(sharedSecretKeysUrl)};`], - [/from "shared-version";/, `from ${JSON.stringify(sharedVersionUrl)};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(workerContractUrl)};`], ]); const { handle } = await import(moduleDataUrl(src)); diff --git a/tests/unit/control-handlers-workflows.test.js b/tests/unit/control-handlers-workflows.test.js index cdd6ef8..2816f8a 100644 --- a/tests/unit/control-handlers-workflows.test.js +++ b/tests/unit/control-handlers-workflows.test.js @@ -19,7 +19,7 @@ const ACTIVE_WORKFLOW_KEY = `wf_${"1".repeat(32)}`; const NEW_WORKFLOW_KEY = `wf_${"2".repeat(32)}`; const RETIRED_WORKFLOW_KEY = `wf_${"3".repeat(32)}`; const { libUrl: productionControlLibUrl } = await compileControlGraph(); -const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const workerContractUrl = repositoryFileUrl("shared/worker-contract.js"); const sharedNsPatternUrl = repositoryFileUrl("shared/ns-pattern.js"); const { handle } = await importControlHandler("control/handlers/workflows.js", { @@ -28,7 +28,7 @@ const { handle } = await importControlHandler("control/handlers/workflows.js", { "control-lib": productionControlLibUrl, "shared-internal-auth": sharedInternalAuthUrl(), "shared-ns-pattern": sharedNsPatternUrl, - "shared-version": sharedVersionUrl, + "shared-worker-contract": workerContractUrl, }, }); diff --git a/tests/unit/control-index.test.js b/tests/unit/control-index.test.js index ebf1584..beb2aca 100644 --- a/tests/unit/control-index.test.js +++ b/tests/unit/control-index.test.js @@ -17,7 +17,6 @@ export function configuredPublicUrl() { return null; } export function platformVersionFromPackageJson() { return "wdl.test"; } export function projectAccessPrincipal(principal) { return principal || null; } export function isAdminAcceptableNs(ns) { return typeof ns === "string" && !ns.includes("_"); } -export function isValidWorkerName() { return true; } function withAction(route, action) { return action ? { ...route, action } : route; } export function parseControlRoute(pathname, method) { if (pathname === "/reload") return withAction({ kind: "reload", scopeRoute: "reload" }, method === "POST" ? "system.reload" : null); diff --git a/tests/unit/control-lifecycle-indexes.test.js b/tests/unit/control-lifecycle-indexes.test.js index 00eceea..5940617 100644 --- a/tests/unit/control-lifecycle-indexes.test.js +++ b/tests/unit/control-lifecycle-indexes.test.js @@ -9,7 +9,7 @@ import { const SCHEDULER_PROJECTION_CONTRACT = readRepositoryJson( "tests/fixtures/scheduler-projection-contract.json" ); -const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); +const WORKER_CONTRACT_URL = repositoryFileUrl("shared/worker-contract.js"); const { cronEntryJson, @@ -36,7 +36,7 @@ const workersIndexKey = (ns) => \`workers:\${ns}\`;`], [/import \{ QUEUE_CONSUMER_INDEX_KEY, queueConsumerKey \} from "shared-queue-keys";/, `const QUEUE_CONSUMER_INDEX_KEY = "queue-consumer-index"; const queueConsumerKey = (ns, queue) => \`queue-consumer:\${ns}:\${queue}\`;`], - [/from "shared-version"/, `from ${JSON.stringify(SHARED_VERSION_URL)}`], + [/from "shared-worker-contract"/, `from ${JSON.stringify(WORKER_CONTRACT_URL)}`], ]); function recordMulti() { diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index 108e030..54940bb 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -59,10 +59,9 @@ function loadLogsTailHandler(options = {}) { `from ${JSON.stringify(SHARED_NS_PATTERN_URL)};`, ], [ - /import \{\n {2}compareStreamIds,\n {2}isValidResumeId,\n {2}isValidWorkerName,\n\} from "control-lib";/, + /import \{ compareStreamIds, isValidResumeId \} from "control-lib";/, `const compareStreamIds = () => 0; - const isValidResumeId = () => true; - const isValidWorkerName = (name) => WORKER_NAME_RE.test(name);`, + const isValidResumeId = () => true;`, ], [ /import \{ controlTailRedis, errMessage, jsonError, requireControlLog \} from "control-shared";/, diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index 05d0ac1..26b433f 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; import { loadControlRouting } from "../helpers/load-control-routing.js"; import { loadControlLib } from "../helpers/load-control-lib.js"; -import { bundleKey as productionBundleKey } from "../../shared/version.js"; +import { bundleKey as productionBundleKey } from "../../shared/worker-contract.js"; const { promoteWithRoutes, bumpActiveAndPromote, reconcileHosts } = await loadControlRouting(); diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index ee1d19d..7d1de61 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -19,11 +19,11 @@ import { compileControlGraph } from "../helpers/load-control-lib.js"; import { readJsonResponse } from "../helpers/response-json.js"; const SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); -const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); +const WORKER_CONTRACT_URL = repositoryFileUrl("shared/worker-contract.js"); const SHARED_SECRET_KEYS_URL = repositoryFileUrl("shared/secret-keys.js"); const RUNTIME_ENV_BUILD_URL = repositoryModuleDataUrl("runtime/load/env-build.js", [ [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], - [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(WORKER_CONTRACT_URL)};`], ]); const { libUrl: PRODUCTION_CONTROL_LIB_URL } = await compileControlGraph(); const env = { @@ -150,7 +150,7 @@ function envBudgetUrl() { [/from "control-lib";/, `from ${JSON.stringify(PRODUCTION_CONTROL_LIB_URL)};`], [/from "runtime-load-env-build";/, `from ${JSON.stringify(RUNTIME_ENV_BUILD_URL)};`], [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], - [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(WORKER_CONTRACT_URL)};`], [/from "shared-redis";/, `from ${JSON.stringify(sharedRedisUrl)};`], ]); return moduleDataUrl(source); @@ -172,7 +172,7 @@ const src = applyModuleReplacements(readRepositoryFile("control/handlers/ns-secr [/from "control-shared";/, `from ${JSON.stringify(controlSharedUrl)};`], [/from "control-lib";/, `from ${JSON.stringify(controlLibStubUrl)};`], [/from "control-handlers-secret-put";/, `from ${JSON.stringify(secretPutUrl(controlSharedUrl, controlLibStubUrl))};`], - [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(WORKER_CONTRACT_URL)};`], [/from "control-env-budget";/, `from ${JSON.stringify(envBudgetUrl())};`], [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], [/from "shared-secret-keys";/, `from ${JSON.stringify(SHARED_SECRET_KEYS_URL)};`], @@ -737,7 +737,7 @@ const workerSrc = applyModuleReplacements(readRepositoryFile("control/handlers/w [/from "control-handlers-secret-put";/, `from ${JSON.stringify(workerSecretPutUrl)};`], [/from "control-lifecycle-indexes";/, `from ${JSON.stringify(lifecycleStubUrl)};`], [/from "control-routing";/, `from ${JSON.stringify(CONTROL_ROUTING_TEST_URL)};`], - [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(WORKER_CONTRACT_URL)};`], [/from "control-env-budget";/, `from ${JSON.stringify(envBudgetUrl())};`], [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], [/from "shared-secret-keys";/, `from ${JSON.stringify(SHARED_SECRET_KEYS_URL)};`], diff --git a/tests/unit/control-shared.test.js b/tests/unit/control-shared.test.js index a005ce3..63d55d9 100644 --- a/tests/unit/control-shared.test.js +++ b/tests/unit/control-shared.test.js @@ -5,18 +5,12 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; -import { - applyModuleReplacements, - moduleDataUrl, - readRepositoryFile, - repositoryFileUrl, -} from "../helpers/load-shared-module.js"; +import { moduleDataUrl } from "../helpers/load-shared-module.js"; +import { compileControlSharedGraph } from "../helpers/load-control-shared.js"; import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { installMockProperty } from "../helpers/mock-global.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; import { assertJsonResponse } from "../helpers/response-json.js"; -import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; @@ -28,75 +22,20 @@ const controlS3Url = moduleDataUrl(`export function makeS3Client() { return null const controlR2Url = moduleDataUrl(`export function makeR2AdminClient() { return null; }`); const sharedAuthTokenUrl = moduleDataUrl(`export function extractToken() { return null; }`); const sharedAuthRolesUrl = moduleDataUrl(`export function validatePrincipalShape() { return false; }`); -const sharedBoundedBodyUrl = repositoryFileUrl("shared/bounded-body.js"); -const sharedErrorsUrl = repositoryFileUrl("shared/errors.js"); -const sharedRandomIdUrl = repositoryFileUrl("shared/random-id.js"); -const sharedRedisLockUrl = moduleDataUrl(applyModuleReplacements( - readRepositoryFile("shared/redis-lock.js"), - [[/from "shared-random-id"/g, `from ${JSON.stringify(sharedRandomIdUrl)}`]] -)); const sharedQueueKeysUrl = moduleDataUrl(`export function queueStreamKey() { return ""; }`); -const sharedRespondUrl = moduleDataUrl(readRepositoryFile("shared/respond.js")); -const controlLibUrl = moduleDataUrl(`export function deleteLockKey(ns, worker) { return \`worker-delete-lock:\${ns}:\${worker}\`; }`); -const sharedS3CleanupLifecycleUrl = repositoryFileUrl("shared/s3-cleanup-lifecycle.js"); -const sharedVersionUrl = repositoryFileUrl("shared/version.js"); -const sharedOptimisticRetryUrl = repositoryFileUrl("shared/optimistic-retry.js"); -const controlErrorsUrl = moduleDataUrl(applyModuleReplacements( - readRepositoryFile("control/errors.js"), - [ - [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], - [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], - ] -)); -const controlWorkflowsClientUrl = moduleDataUrl(applyModuleReplacements( - readRepositoryFile("control/workflows-client.js"), - [ - [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], - [/from "control-errors"/g, `from ${JSON.stringify(controlErrorsUrl)}`], - ] -)); +const { controlSharedUrl, controlWorkflowsClientUrl } = compileControlSharedGraph({ + sharedRedisUrl, + controlS3Url, + controlR2Url, + sharedAuthTokenUrl, + sharedAuthRolesUrl, + sharedQueueKeysUrl, +}); const { postWorkflowsInternalRequest } = await import(controlWorkflowsClientUrl); -const controlOptimisticUrl = moduleDataUrl(applyModuleReplacements( - readRepositoryFile("control/optimistic.js"), - [ - [/from "shared-redis"/g, `from ${JSON.stringify(sharedRedisUrl)}`], - [/from "shared-optimistic-retry"/g, `from ${JSON.stringify(sharedOptimisticRetryUrl)}`], - ] -)); -const controlJsonBodyUrl = moduleDataUrl(applyModuleReplacements( - readRepositoryFile("control/json-body.js"), - [ - [/from "shared-bounded-body"/g, `from ${JSON.stringify(sharedBoundedBodyUrl)}`], - [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], - ] -)); // Use the same stub constructor imported by control/shared.js so // runOptimistic's `instanceof WatchError` check observes the test error. const { WatchError: ControlSharedWatchError } = await import(sharedRedisUrl); -const rewritten = applyModuleReplacements(readRepositoryFile("control/shared.js"), [ - [/from "shared-redis"/g, `from ${JSON.stringify(sharedRedisUrl)}`], - [/from "control-s3"/g, `from ${JSON.stringify(controlS3Url)}`], - [/from "control-r2"/g, `from ${JSON.stringify(controlR2Url)}`], - [/from "shared-auth-token"/g, `from ${JSON.stringify(sharedAuthTokenUrl)}`], - [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], - [/from "shared-bounded-body"/g, `from ${JSON.stringify(sharedBoundedBodyUrl)}`], - [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], - [/from "shared-random-id"/g, `from ${JSON.stringify(sharedRandomIdUrl)}`], - [/from "shared-redis-lock"/g, `from ${JSON.stringify(sharedRedisLockUrl)}`], - [/from "shared-queue-keys"/g, `from ${JSON.stringify(sharedQueueKeysUrl)}`], - [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], - [/from "control-lib"/g, `from ${JSON.stringify(controlLibUrl)}`], - [/from "shared-s3-cleanup-lifecycle"/g, `from ${JSON.stringify(sharedS3CleanupLifecycleUrl)}`], - [/from "shared-version"/g, `from ${JSON.stringify(sharedVersionUrl)}`], - [/from "shared-internal-auth"/g, `from ${JSON.stringify(sharedInternalAuthUrl())}`], - [/from "shared-observability"/g, `from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}`], - [/from "control-workflows-client"/g, `from ${JSON.stringify(controlWorkflowsClientUrl)}`], - [/from "control-errors"/g, `from ${JSON.stringify(controlErrorsUrl)}`], - [/from "control-optimistic"/g, `from ${JSON.stringify(controlOptimisticUrl)}`], - [/from "control-json-body"/g, `from ${JSON.stringify(controlJsonBodyUrl)}`], -]); - const { authErrorBody, authPolicyResponse, @@ -114,7 +53,7 @@ const { runOptimistic, secretEnvelopeErrorResponse, state, -} = await import(moduleDataUrl(rewritten)); +} = await import(controlSharedUrl); /** * @param {import("node:test").TestContext} t diff --git a/tests/unit/do-runtime-index.test.js b/tests/unit/do-runtime-index.test.js index 751da06..9f92d55 100644 --- a/tests/unit/do-runtime-index.test.js +++ b/tests/unit/do-runtime-index.test.js @@ -50,11 +50,11 @@ export function createHttpRequestScope({ request }) { } `); const workerIdUrl = repositoryFileUrl("shared/worker-id.js"); -const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const workerContractUrl = repositoryFileUrl("shared/worker-contract.js"); const actorUrl = stub(`export class WdlDoHostActor {}`); const alarmDispatchUrl = stub(readRepositoryModuleSource("do-runtime/alarm-dispatch.js", importSpecifierReplacements({ "shared-worker-id": workerIdUrl, - "shared-version": sharedVersionUrl, + "shared-worker-contract": workerContractUrl, "do-runtime-protocol": protocolUrl, "do-runtime-http": httpUrl, }))); @@ -130,7 +130,7 @@ const IMPORT_STUBS = { "shared-owner-lease": sharedOwnerLeaseUrl, "shared-respond": sharedRespondUrl, "shared-request-scope": requestScopeUrl, - "shared-version": sharedVersionUrl, + "shared-worker-contract": workerContractUrl, "shared-worker-id": workerIdUrl, "do-runtime-actor": actorUrl, "do-runtime-alarm-dispatch": alarmDispatchUrl, diff --git a/tests/unit/gateway-dispatch.test.js b/tests/unit/gateway-dispatch.test.js index 2009fd3..7af4f46 100644 --- a/tests/unit/gateway-dispatch.test.js +++ b/tests/unit/gateway-dispatch.test.js @@ -9,7 +9,7 @@ import { const nsPatternUrl = repositoryModuleDataUrl("shared/ns-pattern.js"); const gatewayLibUrl = repositoryModuleDataUrl("gateway/lib.js"); -const versionUrl = repositoryModuleDataUrl("shared/version.js"); +const workerContractUrl = repositoryModuleDataUrl("shared/worker-contract.js"); const gatewayRuntimeUrl = moduleDataUrl(` function deps() { return globalThis.__gatewayDispatchTestDeps; @@ -43,7 +43,7 @@ const dispatchSource = applyModuleReplacements(readRepositoryFile("gateway/dispa [/from "shared-ns-pattern";/, `from ${JSON.stringify(nsPatternUrl)};`], [/from "gateway-lib";/, `from ${JSON.stringify(gatewayLibUrl)};`], [/from "gateway-runtime";/, `from ${JSON.stringify(gatewayRuntimeUrl)};`], - [/from "shared-version";/, `from ${JSON.stringify(versionUrl)};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(workerContractUrl)};`], ]); const { resolveGatewayDispatch } = await import(moduleDataUrl(dispatchSource)); diff --git a/tests/unit/gateway-runtime.test.js b/tests/unit/gateway-runtime.test.js index 50d866a..e6b86a3 100644 --- a/tests/unit/gateway-runtime.test.js +++ b/tests/unit/gateway-runtime.test.js @@ -28,7 +28,7 @@ const src = applyModuleReplacements(readRepositoryFile("gateway/runtime.js"), [ [/from "shared-observability";/, `from ${JSON.stringify(OBSERVABILITY_NOOP_URL)};`], [/from "shared-route-projection";/, `from ${JSON.stringify(routeProjectionUrl)};`], [/from "shared-ns-pattern";/, `from ${JSON.stringify(nsPatternUrl)};`], - [/from "shared-version";/, `from ${JSON.stringify(repositoryFileUrl("shared/version.js"))};`], + [/from "shared-worker-contract";/, `from ${JSON.stringify(repositoryFileUrl("shared/worker-contract.js"))};`], [/from "gateway-lib";/, `from ${JSON.stringify(gatewayLibUrl)};`], ]); diff --git a/tests/unit/ns-pattern.test.js b/tests/unit/ns-pattern.test.js index 9017bd2..5c7d1ce 100644 --- a/tests/unit/ns-pattern.test.js +++ b/tests/unit/ns-pattern.test.js @@ -10,11 +10,16 @@ import { PLATFORM_TIER_RESERVED_NS, ROUTES_ALLOWED_RESERVED_NS, WORKER_NAME_RE, + WORKFLOW_NAME_RE, QUEUE_NAME_RE, BINDING_NAME_RE, JS_IDENTIFIER_RE, WDL_RESERVED_BINDING_RE, KV_ID_RE, + isValidWorkerName, + isValidWorkflowName, + isValidQueueName, + isValidKvId, isValidJsIdentifier, } from "../../shared/ns-pattern.js"; @@ -147,6 +152,22 @@ test("KV_ID_RE blocks ':' — otherwise id='foo:v'+key='bar' aliases id='foo'+ke assert.ok(!KV_ID_RE.test("Foo")); }); +test("name and id predicates apply their canonical regex grammars", () => { + /** @type {Array<[(value: unknown) => boolean, string, string]>} */ + const cases = [ + [isValidWorkerName, "My_Worker-2", "-worker"], + [isValidWorkflowName, "Daily_Report", "workflow/name"], + [isValidQueueName, "orders-dlq", "Orders"], + [isValidKvId, "session-store", "session:store"], + ]; + for (const [predicate, valid, invalid] of cases) { + assert.equal(predicate(valid), true); + assert.equal(predicate(invalid), false); + assert.equal(predicate(null), false); + } + assert.equal(WORKFLOW_NAME_RE.test("Daily_Report"), true); +}); + test("BINDING_NAME_RE accepts JS-identifier binding names (CF-compatible)", () => { // CF wrangler requires "valid JavaScript variable name" — SCREAMING, // camelCase, and _private are all valid there and here. diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index 038dd24..3f74e8b 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -29,14 +29,14 @@ const SHARED_WORKER_ID_URL = repositoryFileUrl("shared/worker-id.js"); const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); const SHARED_SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); -const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); +const WORKER_CONTRACT_URL = repositoryFileUrl("shared/worker-contract.js"); const SHARED_REDIS_URL = sharedRedisStubUrl(); const { libUrl: CONTROL_LIB_URL } = await compileControlGraph(); const RUNTIME_ENV_BUILD_URL = repositoryModuleDataUrl( "runtime/load/env-build.js", importSpecifierReplacements({ "shared-ns-pattern": SHARED_NS_PATTERN_URL, - "shared-version": SHARED_VERSION_URL, + "shared-worker-contract": WORKER_CONTRACT_URL, }) ); const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); @@ -45,7 +45,7 @@ const { estimatedWorkerLoaderEnv } = await importRepositoryModule("control/env-b "control-lib": CONTROL_LIB_URL, "runtime-load-env-build": RUNTIME_ENV_BUILD_URL, "shared-secret-envelope": SHARED_SECRET_ENVELOPE_URL, - "shared-version": SHARED_VERSION_URL, + "shared-worker-contract": WORKER_CONTRACT_URL, "shared-redis": SHARED_REDIS_URL, })); const RUNTIME_LOAD_INJECTION_SOURCES_URL = stubRuntimeInjectionSourcesUrl(); @@ -109,7 +109,7 @@ const LOAD_TEST_RUNTIME_DIR = path.join(LOAD_TEST_DIR, "runtime"); const LOAD_TEST_SUBMODULE_DIR = path.join(LOAD_TEST_RUNTIME_DIR, "load"); const ENV_BUILD_SOURCE = readRepositoryModuleSource("runtime/load/env-build.js", importSpecifierReplacements({ "shared-ns-pattern": SHARED_NS_PATTERN_URL, - "shared-version": SHARED_VERSION_URL, + "shared-worker-contract": WORKER_CONTRACT_URL, })); mkdirSync(LOAD_TEST_SUBMODULE_DIR, { recursive: true }); writeFileSync(path.join(LOAD_TEST_RUNTIME_DIR, "load.js"), src); diff --git a/tests/unit/runtime-workflows-client.test.js b/tests/unit/runtime-workflows-client.test.js index 07c1bd5..0a31bfc 100644 --- a/tests/unit/runtime-workflows-client.test.js +++ b/tests/unit/runtime-workflows-client.test.js @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { Workflow } from "../../runtime/workflows-client.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; +import { withMockedProperty, withMockedPropertyDescriptor } from "../helpers/mock-global.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; const workflowLimits = /** @type {{ createBatchMax: number }} */ ( @@ -49,16 +50,13 @@ test("Workflow instances do not expose the private caller through patched Functi }); const originalBind = Function.prototype.bind; let privateBindCalls = 0; - Function.prototype.bind = function (/** @type {any[]} */ ...args) { + await withMockedProperty(Function.prototype, "bind", function (/** @type {any[]} */ ...args) { if (this.name === "#call") privateBindCalls += 1; return Reflect.apply(originalBind, this, args); - }; - try { + }, async () => { const instance = await workflow.get("inst-1"); await instance.status(); - } finally { - Function.prototype.bind = originalBind; - } + }); assert.equal(privateBindCalls, 0); assert.deepEqual(endpoints, [ @@ -137,17 +135,13 @@ test("Workflow request identity ignores tenant-patched JSON.stringify", async () }, }, }); - const originalStringify = JSON.stringify; let patchedCalls = 0; - JSON.stringify = () => { + await withMockedProperty(JSON, "stringify", () => { patchedCalls += 1; return '{"ns":"victim","worker":"victim-worker"}'; - }; - try { + }, async () => { await workflow.create({ id: "inst-1" }); - } finally { - JSON.stringify = originalStringify; - } + }); assert.equal(patchedCalls, 0); assert.deepEqual(JSON.parse(capturedBody), { @@ -178,24 +172,16 @@ test("Workflow request identity ignores inherited Object.prototype.toJSON", asyn }, }, }); - const previousDescriptor = Object.getOwnPropertyDescriptor(Object.prototype, "toJSON"); let inheritedCalls = 0; - Object.defineProperty(Object.prototype, "toJSON", { + await withMockedPropertyDescriptor(/** @type {any} */ (Object.prototype), "toJSON", { configurable: true, value() { inheritedCalls += 1; return { ns: "victim", worker: "victim-worker" }; }, - }); - try { + }, async () => { await workflow.create({ id: "inst-2" }); - } finally { - if (previousDescriptor) { - Object.defineProperty(Object.prototype, "toJSON", previousDescriptor); - } else { - delete /** @type {{ toJSON?: unknown }} */ (Object.prototype).toJSON; - } - } + }); assert.equal(inheritedCalls, 0); const body = JSON.parse(capturedBody); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 77d0dcf..dff3eca 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -239,6 +239,20 @@ test("shared primitive owners stay canonical", () => { assert.deepEqual(offenders, [], `shared primitive owners must stay canonical:\n${offenders.join("\n")}`); }); +test("name grammar predicates live with the shared regex owner", () => { + const owner = withoutLineComments(readRepoFile("shared/ns-pattern.js")); + const controlLib = withoutLineComments(readRepoFile("control/lib.js")); + for (const name of [ + "isValidWorkerName", + "isValidWorkflowName", + "isValidQueueName", + "isValidKvId", + ]) { + assert.match(owner, new RegExp(`export function ${name}\\(`), name); + assert.doesNotMatch(controlLib, new RegExp(`function ${name}\\(`), name); + } +}); + test("secret Redis key construction uses the shared JS owner", () => { const allowed = new Set(["shared/secret-keys.js"]); const offenders = []; @@ -294,8 +308,8 @@ test("control handlers do not bypass jsonError for literal error responses", () assert.deepEqual(offenders, []); }); -test("route/version registry keys go through shared/version.js helpers", () => { - // Only shared/version.js may hold the literal. The `${...}`-anchored regex +test("worker control-plane registry keys go through shared/worker-contract.js helpers", () => { + // Only shared/worker-contract.js may hold the literal. The `${...}`-anchored regex // matches key construction, so channel names ("routes:invalidate"/"flush") // and comments are skipped. const files = [ @@ -304,7 +318,7 @@ test("route/version registry keys go through shared/version.js helpers", () => { ...GATEWAY_FILES, ...DO_RUNTIME_FILES, ...RUNTIME_FILES, - ].filter((file) => file !== "shared/version.js"); + ].filter((file) => file !== "shared/worker-contract.js"); const offenders = []; for (const file of files) { const source = withoutLineComments(readRepoFile(file)); @@ -329,11 +343,11 @@ test("route/version registry keys go through shared/version.js helpers", () => { offenders.push(`${file}: inline host-declarations prefix — use hostDeclarationsKey(host)`); } } - // Rust services are production readers too; rust/common/version.rs holds the - // only literals (mirror of shared/version.js), every other crate must call + // Rust services are production readers too; rust/common/worker_contract.rs holds the + // only literals (mirror of shared/worker-contract.js), every other crate must call // routes_key()/worker_versions_key()/do_storage_id_key(). const rustOffenderFiles = rustFiles("rust").filter( - (file) => file !== "rust/common/src/version.rs" + (file) => file !== "rust/common/src/worker_contract.rs" ); for (const file of rustOffenderFiles) { const source = withoutLineComments(readRepoFile(file)); @@ -345,7 +359,7 @@ test("route/version registry keys go through shared/version.js helpers", () => { offenders.push(`${file}: inline worker:do-storage: — use do_storage_id_key(ns, worker)`); } } - assert.deepEqual(offenders, [], `route/version key literals must use shared helpers:\n${offenders.join("\n")}`); + assert.deepEqual(offenders, [], `worker contract key literals must use shared helpers:\n${offenders.join("\n")}`); }); test("platform domain configuration uses the shared normalized owner", () => { @@ -374,17 +388,19 @@ test("secret Redis key literals stay aligned across JS and redis-proxy", () => { }); test("worker delete lock key stays aligned across control and workflows", () => { - const sharedVersion = withoutLineComments(readRepoFile("shared/version.js")); - const commonVersion = withoutLineComments(readRepoFile("rust/common/src/version.rs")); + const workerContract = withoutLineComments(readRepoFile("shared/worker-contract.js")); + const commonWorkerContract = withoutLineComments( + readRepoFile("rust/common/src/worker_contract.rs"), + ); const controlShared = withoutLineComments(readRepoFile("control/shared.js")); const workflowsActiveExport = withoutLineComments(readRepoFile("rust/workflows/src/api/active_export.rs")); - assert.match(sharedVersion, /`worker-delete-lock:\$\{ns\}:\$\{worker\}`/); - assert.match(commonVersion, /format!\("worker-delete-lock:\{ns\}:\{worker\}"\)/); + assert.match(workerContract, /`worker-delete-lock:\$\{ns\}:\$\{worker\}`/); + assert.match(commonWorkerContract, /format!\("worker-delete-lock:\{ns\}:\{worker\}"\)/); assert.match(controlShared, /\bdeleteLockKey\(ns, worker\)/); assert.match(workflowsActiveExport, /\bworker_delete_lock_key\(ns, worker\)/); - const rustOwner = "rust/common/src/version.rs"; + const rustOwner = "rust/common/src/worker_contract.rs"; const offenders = rustFiles("rust").filter((file) => file !== rustOwner && withoutLineComments(readRepoFile(file)).includes("worker-delete-lock:") ); @@ -892,14 +908,14 @@ test("Redis command metric allow-list covers shared Redis wrappers", () => { }); test("declared-host Redis key literals stay aligned across control, gateway, and docs", () => { - const version = withoutLineComments(readRepoFile("shared/version.js")); + const workerContract = withoutLineComments(readRepoFile("shared/worker-contract.js")); const shared = withoutLineComments(readRepoFile("control/shared.js")); const routing = withoutLineComments(readRepoFile("control/routing.js")); const gateway = withoutLineComments(readRepoFile("gateway/runtime.js")); const layoutEn = readRepoFile("docs/redis-key-layout.md"); const layoutZh = readRepoFile("docs/redis-key-layout.zh.md"); - const declaredHostsKey = extractStringConst(version, "DECLARED_HOSTS_KEY"); - const hostDeclarationsPrefix = extractStringConst(version, "HOST_DECLARATIONS_PREFIX"); + const declaredHostsKey = extractStringConst(workerContract, "DECLARED_HOSTS_KEY"); + const hostDeclarationsPrefix = extractStringConst(workerContract, "HOST_DECLARATIONS_PREFIX"); assert.match(routing, new RegExp(`\\bDECLARED_HOSTS_KEY\\b`)); assert.match(routing, /\bhostDeclarationsKey\b/); @@ -1426,6 +1442,51 @@ test("local compose services inherit shared image anchors", () => { assert.deepEqual(offenders, []); }); +test("published compose services reset local builds on each service", () => { + const source = readRepoFile("docker-compose.images.yml"); + const serviceGroups = /** @type {Array<[string, string[]]>} */ ([ + ["rust-published", [ + "redis-proxy-user", + "redis-proxy-system", + "redis-proxy-do", + "scheduler", + "workflows", + ]], + ["workerd-published", [ + "user-runtime", + "system-runtime", + "gateway", + "d1-runtime", + "d1-runtime-a", + "d1-runtime-b", + "d1-runtime-c", + "do-runtime", + "do-runtime-a", + "do-runtime-b", + "do-runtime-c", + ]], + ]); + const servicesIndex = source.indexOf("services:"); + assert.notEqual(servicesIndex, -1); + const anchorSource = source.slice(0, servicesIndex); + assert.doesNotMatch(anchorSource, /build:\s*!reset\b/); + const expectedServices = serviceGroups.flatMap(([, services]) => services).toSorted(); + const actualServices = [...source.matchAll(/^ {2}([a-z0-9-]+):$/gm)] + .map((match) => match[1]) + .toSorted(); + assert.deepEqual(actualServices, expectedServices); + for (const [anchor, services] of serviceGroups) { + for (const service of services) { + const block = new RegExp( + String.raw`(?:^|\n) {2}${RegExp.escape(service)}:\n((?: {4}[^\n]*(?:\n|$))*)`, + ).exec(source)?.[1]; + assert.ok(block, `${service} must exist in the published-image overlay`); + assert.match(block, new RegExp(String.raw`^ {4}<<: \*${anchor}$`, "m"), service); + assert.match(block, /^ {4}build: !reset null$/m, service); + } + } +}); + test("local compose runtime profiles keep base services enabled by default", () => { const [document] = yamlDocuments("docker-compose.yml"); const compose = /** @type {{ services?: Record }} */ ( @@ -2375,7 +2436,7 @@ test("worker-id helper carries its relative grammar dependencies in workerd conf const source = readRepoFile(file); assert.match(source, /name = "shared-worker-id"/, file); assert.match(source, /name = "ns-pattern\.js"/, file); - assert.match(source, /name = "version\.js"/, file); + assert.match(source, /name = "worker-contract\.js"/, file); } }); diff --git a/tests/unit/test-helper-style-contracts.test.js b/tests/unit/test-helper-style-contracts.test.js index 458244a..67e7762 100644 --- a/tests/unit/test-helper-style-contracts.test.js +++ b/tests/unit/test-helper-style-contracts.test.js @@ -100,9 +100,12 @@ test("test global mocks go through mock-global helpers", () => { // Keep this allow-list in sync with the global-property mock targets named // in docs/testing.md. A fully generic `object.property =` matcher would // reject ordinary fixture setup assignments. - if (/\b(?:process\.stderr|AbortSignal|Object|Date|Math|Headers\.prototype|Array\.prototype)\.[A-Za-z_$][\w$]*\s*=(?!=)/.test(source)) { + if (/\b(?:process\.stderr|AbortSignal|Object|JSON|Date|Math|Headers\.prototype|Array\.prototype|Function\.prototype)\.[A-Za-z_$][\w$]*\s*=(?!=)/.test(source)) { offenders.push(`${file}: use withMockedProperty(...) for built-in global property mocks`); } + if (/\bObject\.defineProperty\(\s*(?:Object|Function|Array|Headers|Request|Response|Promise|RegExp)\.prototype\s*,/.test(source)) { + offenders.push(`${file}: use withMockedPropertyDescriptor(...) for built-in prototype descriptor mocks`); + } } assert.deepEqual(offenders, [], `test global mocks must use tests/helpers/mock-global.js:\n${offenders.join("\n")}`); }); diff --git a/tests/unit/version.test.js b/tests/unit/worker-contract.test.js similarity index 99% rename from tests/unit/version.test.js rename to tests/unit/worker-contract.test.js index dbb8804..e6aeb4e 100644 --- a/tests/unit/version.test.js +++ b/tests/unit/worker-contract.test.js @@ -26,7 +26,7 @@ import { parseDeleteLockKind, parseVersion, workerVersionsKey, -} from "../../shared/version.js"; +} from "../../shared/worker-contract.js"; const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); From 47394a8e7c27cd14fa6c4e92f08593243d4e3d7e Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 17 Jul 2026 19:59:02 +0800 Subject: [PATCH 23/23] fix: reject inherited slots in DO RPC arrays Use a captured own-property check before reading RPC array elements. Reject sparse arrays even when their prototype supplies numeric getters. Signed-off-by: Lu Zhang --- runtime/_wdl-do-transport.js | 5 ++++- tests/unit/runtime-do-client.test.js | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index 72acce6..0e53dae 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -84,6 +84,7 @@ const intrinsicObjectCreate = Object.create; const intrinsicObjectEntries = Object.entries; const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; +const intrinsicObjectHasOwn = Object.hasOwn; const intrinsicObjectSetPrototypeOf = Object.setPrototypeOf; const intrinsicPromiseThen = Promise.prototype.then; const intrinsicResponseJson = Response.json; @@ -464,7 +465,9 @@ function cloneJsonRpcData(value, field, seen = new IntrinsicWeakSet()) { const out = new IntrinsicArray(length); intrinsicReflectApply(intrinsicObjectSetPrototypeOf, IntrinsicObject, [out, null]); for (let i = 0; i < length; i++) { - if (!(i in arrayValue)) throw new TypeError(`${field} must not be sparse`); + if (!intrinsicReflectApply(intrinsicObjectHasOwn, undefined, [arrayValue, i])) { + throw new TypeError(`${field} must not be sparse`); + } out[i] = cloneJsonRpcData(arrayValue[i], `${field}[${i}]`, seen); } return out; diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index 3daea62..5695a52 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -410,11 +410,23 @@ test("DurableObjectNamespace RPC rejects non-JSON data before transport", async const stub = ns.get(ns.idFromName("room-a")); const circular = {}; circular.self = circular; + const sparse = new Array(1); + const sparsePrototype = Object.create(Array.prototype); + let inheritedSlotReads = 0; + Object.defineProperty(sparsePrototype, "0", { + get() { + inheritedSlotReads += 1; + return "inherited"; + }, + }); + Object.setPrototypeOf(sparse, sparsePrototype); await assert.rejects(Reflect.get(stub, "save")(new Map([["key", "value"]])), /plain JSON object/); await assert.rejects(Reflect.get(stub, "save")(new Headers()), /plain JSON object/); await assert.rejects(Reflect.get(stub, "save")({ fn() {} }), /rpc\.args\[0\]\.fn must be JSON data/); await assert.rejects(Reflect.get(stub, "save")([undefined]), /rpc\.args\[0\]\[0\] must be JSON data/); + await assert.rejects(Reflect.get(stub, "save")(sparse), /rpc\.args\[0\] must not be sparse/); + assert.equal(inheritedSlotReads, 0); await assert.rejects(Reflect.get(stub, "save")(Number.NaN), /finite number/); await assert.rejects(Reflect.get(stub, "save")(circular), /must not be circular/); }); @@ -460,6 +472,12 @@ test("DO RPC validation uses captured JSON intrinsics", async () => { (error) => error instanceof TypeError && error.message === "rpc.args[0] must be a finite number" ); }); + await withMockedProperty(Object, "hasOwn", () => true, () => { + assert.throws( + () => rpcInvokeBody(props, "room-a", "save", [new Array(1)]), + (error) => error instanceof TypeError && error.message === "rpc.args[0] must not be sparse" + ); + }); }); test("do-runtime owns the DO RPC method byte limit", () => {