diff --git a/.gitignore b/.gitignore index 4b7a93b15..c68e3334a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ coverage .nyc_output/ -node_modules/ +node_modules junit dist tmp diff --git a/docs/openapi/schemas.yaml b/docs/openapi/schemas.yaml index 1cb9eb9f8..819d5f16f 100644 --- a/docs/openapi/schemas.yaml +++ b/docs/openapi/schemas.yaml @@ -11123,10 +11123,20 @@ SerenityCreatePromptsRequest: minItems: 1 maxItems: 500 items: { $ref: '#/SerenityPromptInput' } + deferPublish: + type: boolean + default: false + description: > + When true, the created prompts are written to the upstream draft layer + but the project is NOT published as part of this request (the response + carries `published: false`). Used for chunked CSV imports, where the + client posts several chunks and issues a single publish after the last + one (serenity-docs#32). Omitted/false publishes once at the end of the + call as usual. SerenityCreatePromptsResponse: type: object - required: [created, skipped, failed] + required: [created, skipped, failed, published] properties: created: type: array @@ -11150,6 +11160,13 @@ SerenityCreatePromptsResponse: languageCode: { type: string } status: { type: integer } message: { type: string } + published: + type: boolean + description: > + Whether the project was published as part of this request. Always true + unless the request set `deferPublish: true`, in which case the prompts + were written as drafts and the caller must publish separately + (serenity-docs#32). SerenityUpdatePromptRequest: type: object diff --git a/docs/serenity.md b/docs/serenity.md index b05cddc44..09f1806f6 100644 --- a/docs/serenity.md +++ b/docs/serenity.md @@ -8,11 +8,12 @@ api-service exposes nine endpoints that front the Adobe-hosted Semrush AIO API a ## Environment configuration -The proxy is configured via a single env var that is **required at runtime** — Lambda fails at first request if it is unset. +`SEMRUSH_PROJECTS_BASE_URL` is **required at runtime** — Lambda fails at first request if it is unset. The server-side intent classifier (serenity-docs#32) adds one optional override. | Variable | Required | Source | Purpose | |---|---|---|---| | `SEMRUSH_PROJECTS_BASE_URL` | yes (no source default) | Vault `dx_mysticat//api-service`; locally `.env` | Upstream host for the Semrush AIO REST API. Must be `https://…`. Trailing slashes are stripped. Per-environment value so the production target can differ from the hackathon host without a code change. | +| `PROMPT_INTENT_CLASSIFICATION_DEPLOYMENT_NAME` | no (falls back to `AZURE_OPEN_AI_API_DEPLOYMENT_NAME`) | Vault `dx_mysticat//api-service`; locally `.env` | Classifier-scoped Azure OpenAI deployment (model) name for server-side prompt-intent classification (serenity-docs#32). Takes precedence over the shared `AZURE_OPEN_AI_API_DEPLOYMENT_NAME` other Azure consumers use (e.g. `org-detector`), so intent classification can target a different model without affecting them. Unset ⇒ shared deployment; behavior unchanged until explicitly configured. | ### Vault writes (dev / stage / prod) diff --git a/src/controllers/brands.js b/src/controllers/brands.js index c19258e79..1e8f140b6 100644 --- a/src/controllers/brands.js +++ b/src/controllers/brands.js @@ -64,6 +64,7 @@ import { import { listViewableResourceIds } from '../support/state-access-mapping-utils.js'; import { isFacsRebacResource } from '../routes/facs-capabilities.js'; import { provisionBrandSubworkspace, provisionBrandSubworkspaceBare, releaseProvisionedWorkspace } from '../support/serenity/brand-provisioning.js'; +import { computeWriteDeadline } from '../support/serenity/intent-classification.js'; import { ensureMarketSite } from '../support/serenity/site-linkage.js'; import { upsertMappingRow, linkSiteToLiveRows } from '../support/serenity/mapping-rows.js'; import { createSerenityTransport } from '../support/serenity/rest-transport.js'; @@ -1484,6 +1485,12 @@ function BrandsController(ctx, log, env) { const { spaceCatId } = context.params || {}; const brandData = context.data; + // One shared write-budget deadline for the whole request, computed at the + // true request entry (before auth/loadBrand/provisioning) so intent + // classification during provisioning budgets against the real request start + // rather than function-entry time deep in the call stack (serenity-docs#32). + const writeDeadline = computeWriteDeadline(); + // Hoisted above the try so the catch can run compensation: if a Semrush // sub-workspace was provisioned but the brand row failed to persist, the // catch releases the orphaned allocation (see below). @@ -1673,6 +1680,7 @@ function BrandsController(ctx, log, env) { // market's CI competitor list. Like URLs, they come from the create // payload (the brand row isn't written yet). competitors: brandData.competitors, + writeDeadline, }, log); provisionedWorkspaceId = provisioned.semrushSubWorkspaceId; provisionedInitialMarket = { diff --git a/src/controllers/serenity.js b/src/controllers/serenity.js index 1764c6dc3..e77f41872 100644 --- a/src/controllers/serenity.js +++ b/src/controllers/serenity.js @@ -68,6 +68,7 @@ import { isDynamicAllocationEnabled } from '../support/serenity/dynamic-allocati import { MAX_TOPICS_ON_CREATE } from '../support/serenity/brand-provisioning.js'; import { marketForGeoTargetId } from '../support/serenity/locations.js'; import { brandNeedles, classifyBrandedTag } from '../support/serenity/branded-classifier.js'; +import { computeWriteDeadline } from '../support/serenity/intent-classification.js'; import AccessControlUtil from '../support/access-control-util.js'; import { resolveBrandUuid } from '../support/prompts-storage.js'; import { @@ -521,6 +522,9 @@ function SerenityController(context, log, env) { } const transport = buildTransport(ctx, imsToken); const classifyPromptType = await buildPromptTypeClassifier(ctx, auth.brandUuid); + // serenity-docs#32: one shared write-budget deadline for classify + create + // + publish, computed once at request entry. + const writeDeadline = computeWriteDeadline(); const result = auth.mode === 'subworkspace' ? await handleCreatePromptsSubworkspace( transport, @@ -528,6 +532,8 @@ function SerenityController(context, log, env) { ctx.data || {}, log, classifyPromptType, + ctx.env, + writeDeadline, { dynamicAllocation: dynamicAllocationEnabled(ctx), parentWorkspaceId: auth.parentWorkspaceId ?? '', @@ -541,6 +547,8 @@ function SerenityController(context, log, env) { ctx.data || {}, log, classifyPromptType, + ctx.env, + writeDeadline, ); return createResponse(result, 200); } catch (e) { @@ -561,6 +569,7 @@ function SerenityController(context, log, env) { } const transport = buildTransport(ctx, imsToken); const classifyPromptType = await buildPromptTypeClassifier(ctx, auth.brandUuid); + const writeDeadline = computeWriteDeadline(); const result = auth.mode === 'subworkspace' ? await handleUpdatePromptSubworkspace( transport, @@ -569,6 +578,8 @@ function SerenityController(context, log, env) { ctx.data || {}, log, classifyPromptType, + ctx.env, + writeDeadline, ) : await handleUpdatePrompt( transport, @@ -579,6 +590,8 @@ function SerenityController(context, log, env) { ctx.data || {}, log, classifyPromptType, + ctx.env, + writeDeadline, ); return createResponse(result.body, result.status); } catch (e) { @@ -680,6 +693,10 @@ function SerenityController(context, log, env) { const createMarket = async (ctx) => { try { + // Shared write-budget deadline, computed once at request entry so intent + // classification during topic/prompt generation budgets against the true + // request start (serenity-docs#32). + const writeDeadline = computeWriteDeadline(); const imsToken = await resolveSemrushImsToken(ctx); const auth = await authorize(ctx); if (auth.error) { @@ -743,6 +760,8 @@ function SerenityController(context, log, env) { brandAliases, brandUrlSources, competitors, + env: ctx.env, + writeDeadline, // auth.brandUuid is an already-persisted brand row here (loadBrand // above), so the mapping-row upsert's FK to brands is satisfied — // see mapping-rows.js upsertMappingRow doc. @@ -1124,6 +1143,11 @@ function SerenityController(context, log, env) { */ const activate = async (ctx) => { try { + // Shared write-budget deadline, computed once at request entry so intent + // classification during per-market topic/prompt generation budgets against + // the true request start rather than per-market function entry + // (serenity-docs#32). + const writeDeadline = computeWriteDeadline(); const imsToken = await resolveSemrushImsToken(ctx); const auth = await authorize(ctx); if (auth.error) { @@ -1369,6 +1393,8 @@ function SerenityController(context, log, env) { brandAliases, brandUrlSources, competitors, + env: ctx.env, + writeDeadline, // `brand` was loaded via loadBrand above — an already-persisted // row, so the mapping-row upsert's FK to brands is satisfied. // Narrowed to the one model the mapping-row helpers touch — see diff --git a/src/support/intent-classifier.js b/src/support/intent-classifier.js index 51858e483..d1d5271dc 100644 --- a/src/support/intent-classifier.js +++ b/src/support/intent-classifier.js @@ -186,12 +186,14 @@ export function contentToString(content) { /** * Extracts a JSON object from a model response that may include code fences or - * surrounding prose, then returns its normalized `intent` (or null). + * surrounding prose, then returns the parsed object (or null on any parse + * failure). Shared by every category spec — none of this extraction logic is + * taxonomy-specific. * * @param {string} content - Raw model output - * @returns {string|null} + * @returns {object|null} */ -function parseIntent(content) { +function parseModelJson(content) { if (!hasText(content)) { return null; } @@ -208,20 +210,58 @@ function parseIntent(content) { [raw] = brace; } try { - const parsed = JSON.parse(raw); - // normalizeIntent lowercases, applies the legacy remap, and validates - // against the 6 canonical buckets (else null) — single source of truth. - return normalizeIntent(parsed?.intent); + return JSON.parse(raw); } catch { return null; } } /** - * Creates an intent classifier bound to the Azure OpenAI credentials in `env`. + * The native/DRS 6-bucket category spec — the classifier's default + * configuration, reproducing today's behavior byte-for-byte (it reuses the + * existing {@link SYSTEM_PROMPT} export as-is, so there is nothing new to keep + * in sync). `parseResult` discards `confidence`, matching the pre-existing + * `parseIntent` behavior. + * + * @typedef {object} CategorySpec + * @property {string} systemPrompt - the system prompt for this taxonomy. + * @property {(parsed: object) => (string|null)} parseResult - validates + + * normalizes the parsed JSON body into a canonical value, or null. + * @property {number} [invokeTimeoutMs] - per-call timeout override; falls back + * to {@link resolveInvokeTimeoutMs} (env-driven) when omitted. + */ + +/** @type {CategorySpec} */ +export const DRS_CATEGORY_SPEC = Object.freeze({ + systemPrompt: SYSTEM_PROMPT, + // normalizeIntent lowercases, applies the legacy remap, and validates + // against the 6 canonical buckets (else null) — single source of truth. + parseResult: (parsed) => normalizeIntent(parsed?.intent), +}); + +/** + * Resolves the Azure OpenAI deployment (model) name from env, allowing a + * classifier-scoped override (`PROMPT_INTENT_CLASSIFICATION_DEPLOYMENT_NAME`) + * to take precedence over the shared per-env deployment used by other Azure + * OpenAI consumers (e.g. `org-detector`). Falls back to the shared deployment + * when the override is unset, so existing behavior is unchanged until the + * override is explicitly configured. + * + * @param {object} env - Environment variables + * @returns {string|undefined} + */ +function resolveDeploymentName(env = {}) { + return env.PROMPT_INTENT_CLASSIFICATION_DEPLOYMENT_NAME + || env.AZURE_OPEN_AI_API_DEPLOYMENT_NAME; +} + +/** + * Creates an intent classifier bound to the Azure OpenAI credentials in `env`, + * for the given taxonomy (`categorySpec`). Defaults to {@link DRS_CATEGORY_SPEC} + * so existing single-arg callers are unaffected. * * Returns a function `classify(text) => Promise` that is always - * best-effort: it resolves to a canonical bucket on success, or `null` on any + * best-effort: it resolves to a canonical value on success, or `null` on any * failure or when text/credentials are missing. It NEVER rejects. * * Classification runs whenever Azure OpenAI is configured. When the Azure @@ -231,17 +271,19 @@ function parseIntent(content) { * @param {object} context - Helix universal context * @param {object} context.env - Environment variables (Azure OpenAI creds) * @param {object} [context.log] - Logger + * @param {CategorySpec} [categorySpec] - the taxonomy to classify into; + * defaults to the native 6-bucket DRS taxonomy. * @returns {((text: string) => Promise)|null} */ -export function createIntentClassifier(context = {}) { +export function createIntentClassifier(context = {}, categorySpec = DRS_CATEGORY_SPEC) { const { env = {}, log = console } = context; const { AZURE_OPEN_AI_API_KEY: azureOpenAIApiKey, AZURE_OPEN_AI_API_INSTANCE_NAME: azureOpenAIApiInstanceName, - AZURE_OPEN_AI_API_DEPLOYMENT_NAME: azureOpenAIApiDeploymentName, AZURE_OPEN_AI_API_VERSION: azureOpenAIApiVersion, } = env; + const azureOpenAIApiDeploymentName = resolveDeploymentName(env); if (!hasText(azureOpenAIApiKey) || !hasText(azureOpenAIApiInstanceName) @@ -266,7 +308,8 @@ export function createIntentClassifier(context = {}) { return null; } - const invokeTimeoutMs = resolveInvokeTimeoutMs(env); + const invokeTimeoutMs = categorySpec.invokeTimeoutMs ?? resolveInvokeTimeoutMs(env); + const { systemPrompt, parseResult } = categorySpec; return async function classify(text) { // Mirror DRS: skip empty / whitespace-only text (no LLM call, intent null). @@ -279,14 +322,15 @@ export function createIntentClassifier(context = {}) { // On timeout this rejects and falls through to the non-fatal catch below. const response = await withTimeout( model.invoke([ - { role: 'system', content: SYSTEM_PROMPT }, + { role: 'system', content: systemPrompt }, { role: 'user', content: trimmed.slice(0, MAX_PROMPT_CHARS) }, ]), invokeTimeoutMs, ); // content may be a string OR an array of content parts; coerce either way. const content = contentToString(response?.content); - return parseIntent(content); + const parsed = parseModelJson(content); + return parsed ? parseResult(parsed) : null; } catch (e) { log.warn(`Intent classification failed for prompt; persisting null: ${e.message}`); return null; diff --git a/src/support/serenity/brand-provisioning.js b/src/support/serenity/brand-provisioning.js index 44dd4f058..570c53c39 100644 --- a/src/support/serenity/brand-provisioning.js +++ b/src/support/serenity/brand-provisioning.js @@ -20,9 +20,7 @@ import { isSemrushTransportError } from './errors.js'; import { resolveWorkspaceId } from './workspace-resolver.js'; import { deleteAllProjects, releaseFullAllocation, ensureSubworkspace } from './workspace-lifecycle.js'; import { handleCreateMarketSubworkspace } from './handlers/markets-subworkspace.js'; - -// Re-exported for callers/tests that drive brand provisioning. The tag -// vocabularies themselves live in `prompt-tags.js` (single source of truth). +import { computeWriteDeadline } from './intent-classification.js'; // Brand-create generation policy (tunable). Keep the top N generated topics by // search volume; brand-topics returns up to 10 topics x up to 100 prompts each, @@ -81,6 +79,10 @@ export function initialMarketProjectName(market, languageCode) { * @param {object[]} [params.competitors] - the brand's competitors ("other * brands to track") tracked as region-filtered project benchmarks (domain-only). * Best-effort: a failed sync is logged and skipped, never aborts provisioning. + * @param {number} [params.writeDeadline] - shared request-write deadline (epoch + * ms), computed once at controller entry and threaded down so intent + * classification budgets against the true request start (serenity-docs#32); + * defaults to a fresh {@link computeWriteDeadline} for direct/test callers. * @param {object} [log] * @returns {Promise<{ * semrushSubWorkspaceId: string, @@ -103,7 +105,7 @@ export function initialMarketProjectName(market, languageCode) { export async function provisionBrandSubworkspace(context, { spaceCatId, brandId, brandName, market, languageCode, brandDomain, modelIds = [], brandAliases = [], brandUrlSources = null, competitors = [], - generateTopics = true, + generateTopics = true, writeDeadline = computeWriteDeadline(), }, log = console) { if (!hasText(brandName)) { throw new ErrorWithStatusCode('brandName is required for Semrush provisioning', 400); @@ -207,6 +209,8 @@ export async function provisionBrandSubworkspace(context, { generateTopics, topicCap: generateTopics ? MAX_TOPICS_ON_CREATE : 0, brandAliases, + env: context.env, + writeDeadline, brandUrlSources, competitors, // A project with neither models nor generated prompts would publish diff --git a/src/support/serenity/handlers/markets-subworkspace.js b/src/support/serenity/handlers/markets-subworkspace.js index bba5b64a9..4beca2167 100644 --- a/src/support/serenity/handlers/markets-subworkspace.js +++ b/src/support/serenity/handlers/markets-subworkspace.js @@ -41,9 +41,10 @@ import { withResourceLock } from '../resource-lock.js'; import { modelChangeUnits, releaseAiSurplus, PROJECT_BLOCK, PROMPT_BLOCK, } from '../resource-manager.js'; -import { DIMENSION, STANDARD_PROMPT_TAG_VALUES } from '../prompt-tags.js'; +import { DIMENSION, STANDARD_PROMPT_TAG_VALUES, INTENT_VALUE } from '../prompt-tags.js'; import { provisionDimensionTree } from '../tag-tree.js'; import { classifyBrandedTag, needlesFromNames } from '../branded-classifier.js'; +import { classifyPromptIntents, AI_GEN_CLASSIFY_MAX, computeWriteDeadline } from '../intent-classification.js'; import { collectBrandUrlEntries, attachBrandUrlsToProject, primaryDomainSet } from '../brand-urls.js'; import { resolveProjects } from '../resolve-projects.js'; import { buildReservedDomains, syncCompetitorBenchmarksForProject } from '../competitor-benchmarks.js'; @@ -271,9 +272,11 @@ function validateCreateBody(body) { * Generates topics + prompts for (domain, country) via the AI-SEO service * (transport.getBrandTopics) and attaches them to the project. Keeps the top * `topicCap` topics by search volume (0 = keep all) and tags every prompt with - * the standard closed-dimension values ({@link STANDARD_PROMPT_TAG_VALUES}) plus - * a branded / non-branded `type` value derived from `brandNames` (brand name + - * aliases). Returns the topic/prompt counts. A generation that yields nothing is + * the standard closed-dimension values ({@link STANDARD_PROMPT_TAG_VALUES}, minus + * its seeded `intent` default), plus a branded / non-branded `type` value derived + * from `brandNames` (brand name + aliases) and a per-prompt server-classified + * `intent` value (serenity-docs#32, replacing the seeded `Informational` + * default). Returns the topic/prompt counts. A generation that yields nothing is * a clean no-op (no upstream write). * * The generated topic name is NOT attached. Under the dimension-root model a @@ -283,9 +286,9 @@ function validateCreateBody(body) { * arrive uncategorized and are categorized later (adobe/serenity-docs#44). * * Writes are id-based: `createPromptsByIds` takes ONE shared `tag_ids` array per - * call, so the texts are partitioned by their resolved tag-id set — which, with - * topics gone, is exactly two groups (branded and non-branded). Identical text - * collapses to one entry per group. + * call, so the texts are partitioned by their resolved tag-id set — the (type, + * intent) pair, since topics are gone and everything else is constant. Identical + * text collapses to one entry per group. * * @param {object} transport - Serenity transport (Semrush proxy client). * @param {string} workspaceId - sub-workspace the project lives in. @@ -300,6 +303,9 @@ function validateCreateBody(body) { * @param {{ values: Map> }} options.provisioned - the * already-provisioned dimension tree. The caller provisions it unconditionally, * so re-resolving it here would read the whole taxonomy a second time per request. + * @param {object} [options.env] - environment (Azure OpenAI creds), for intent + * classification (serenity-docs#32). + * @param {number} [options.writeDeadline] - shared request-write deadline. * @param {object} log - logger. * @param {{ ensure: Function, retryOnQuota: Function }} headroom - the caller's headroom guard * (createHeadroomGuard) — @@ -310,7 +316,8 @@ function validateCreateBody(body) { * write loop, sized on the real prompt count now that it's known (`texts.size`), not an estimate. */ async function generateAndAttachPrompts(transport, workspaceId, projectId, { - domain, country, topicCap = 0, brandNames = [], provisioned, + domain, country, topicCap = 0, brandNames = [], provisioned, env, + writeDeadline = computeWriteDeadline(), }, log, headroom) { const raw = await transport.getBrandTopics(workspaceId, { domain, country }); let topics = []; @@ -350,37 +357,84 @@ async function generateAndAttachPrompts(transport, workspaceId, projectId, { // Resolve every tag id we are about to attach. `createPromptsByIds` is ATOMIC on // an unresolvable id (live 500s and creates nothing), so ids are never guessed. // `provisionDimensionTree` resolved every closed value or threw a 502, so the - // standard values and the whole `type` vocabulary are present here by construction. + // standard values and the whole `type`/`intent` vocabularies are present here by + // construction. const { values } = provisioned; - const standardIds = STANDARD_PROMPT_TAG_VALUES.map( - ({ dimension, name }) => /** @type {string} */ (values.get(dimension)?.get(name)), - ); + // The standard closed-dimension ids EXCEPT `intent`: intent is classified per + // prompt below (serenity-docs#32) and replaces the seeded `Informational` + // default, so it must not be double-attached from the standard set. + const standardIdsNonIntent = STANDARD_PROMPT_TAG_VALUES + .filter(({ dimension }) => dimension !== DIMENSION.INTENT) + .map(({ dimension, name }) => /** @type {string} */ (values.get(dimension)?.get(name))); const typeValues = /** @type {Map} */ (values.get(DIMENSION.TYPE)); + const intentValues = /** @type {Map} */ (values.get(DIMENSION.INTENT)); - /** @type {Map} bare type value → prompt texts */ - const byTypeValue = new Map(); - for (const text of texts) { - const value = classifyBrandedTag(text, needles); - const bucket = byTypeValue.get(value); + // Batch-classify intent ONCE for every generated text (capped at + // AI_GEN_CLASSIFY_MAX under the shared request deadline); anything unclassified + // falls back to the seeded `Informational` default. Every resolved value is in + // the `intent` vocabulary provisioned above, so its id is a Map lookup — no + // extra upstream call. + const allTexts = [...texts]; + // The cap is a designed budget bound, not an error: texts beyond it are not + // classified and take the seeded `Informational` default (see the partition + // below). Emit one line when it binds so a silently-defaulted tail is + // observable rather than invisible (serenity-docs#32 observability). + if (allTexts.length > AI_GEN_CLASSIFY_MAX) { + log?.info?.('generateAndAttachPrompts: AI-gen classify cap hit — tail defaults to Informational', { + workspaceId, + projectId, + total: allTexts.length, + classified: AI_GEN_CLASSIFY_MAX, + defaultedByCap: allTexts.length - AI_GEN_CLASSIFY_MAX, + }); + } + const intentByText = await classifyPromptIntents( + allTexts.slice(0, AI_GEN_CLASSIFY_MAX), + { + env, log, deadline: writeDeadline, writePath: 'ai-gen', workspaceId, + }, + ); + + // `createPromptsByIds` takes ONE shared `tag_ids` array per call, so partition + // the texts by their resolved (type, intent) id pair — the only two dimensions + // that vary per prompt. + /** @type {Map} */ + const byTagSet = new Map(); + for (const text of allTexts) { + const typeValue = classifyBrandedTag(text, needles); + const intentValue = intentByText.get(text) ?? INTENT_VALUE.INFORMATIONAL; + const key = `${typeValue}${intentValue}`; + const bucket = byTagSet.get(key); if (bucket) { - bucket.push(text); + bucket.items.push(text); } else { - byTypeValue.set(value, [text]); + // `branded` / `non-branded` are the classifier's only outputs and every + // classified/defaulted intent is a fixed vocabulary value — both are in the + // tree provisioned above. + const typeId = /** @type {string} */ (typeValues.get(typeValue)); + const intentId = /** @type {string} */ (intentValues.get(intentValue)); + byTagSet.set(key, { + items: [text], + tagIds: [...standardIdsNonIntent, intentId, typeId], + }); } } + // PROMPT metering seam (Rainer, live-verified LLMO-6190): front headroom sized + // on the real generated prompt count (`texts.size`) BEFORE the metered + // `createPromptsByIds` writes below — the disguised-quota 405 fires there, not + // at publish. No-op when the flag is OFF; NOT optional-chained, a caller that + // forgets to thread the guard must fail loud. await headroom.ensure({ prompts: texts.size }, { includeDrafted: true }); - for (const [value, items] of byTypeValue) { - // `branded` / `non-branded` are the classifier's only outputs and both are in - // the `type` vocabulary provisioned above. - const typeId = /** @type {string} */ (typeValues.get(value)); + for (const { items, tagIds } of byTagSet.values()) { // LLMO-6190 follow-up: the metered write can still 405 as a disguised metered-quota rejection // despite the `ensure` above (live-verified ~9s gateway write-enforcement lag after a JIT // top-up) — route through `headroom.retryOnQuota` (no-op passthrough when the flag is OFF). + // `tagIds` is precomputed per (type, intent) bucket above (standard + intent + type). // eslint-disable-next-line no-await-in-loop await headroom.retryOnQuota( - () => transport.createPromptsByIds(workspaceId, projectId, items, [...standardIds, typeId]), + () => transport.createPromptsByIds(workspaceId, projectId, items, tagIds), { callSite: 'createPromptsByIds' }, ); } @@ -449,6 +503,10 @@ async function generateAndAttachPrompts(transport, workspaceId, projectId, { * `ensureSubworkspace` is skipped. When false, byte-for-byte the pre-this-PR behavior. * (The units pool for JIT sizing is the positional `parentWorkspaceId` arg — the same id given * to `ensureSubworkspace` — so it is not duplicated in this options bag.) + * @param {object} [options.env] - environment (Azure OpenAI creds), threaded into + * intent classification when `generateTopics` is set (serenity-docs#32). + * @param {number} [options.writeDeadline] - shared request-write deadline; defaults + * to a fresh {@link computeWriteDeadline} for direct/test callers. */ export async function handleCreateMarketSubworkspace( transport, @@ -468,6 +526,8 @@ export async function handleCreateMarketSubworkspace( publishMode = 'require', dataAccess = null, dynamicAllocation = false, + env = null, + writeDeadline = computeWriteDeadline(), } = {}, ) { const errors = validateCreateBody(body); @@ -597,6 +657,8 @@ export async function handleCreateMarketSubworkspace( ...(Array.isArray(body.brandNames) ? body.brandNames : []), ...aliasNames, ], + env, + writeDeadline, }, log, headroom, diff --git a/src/support/serenity/handlers/prompts-subworkspace.js b/src/support/serenity/handlers/prompts-subworkspace.js index 5abda8700..d2cd16b40 100644 --- a/src/support/serenity/handlers/prompts-subworkspace.js +++ b/src/support/serenity/handlers/prompts-subworkspace.js @@ -23,6 +23,8 @@ import { normalizePromptInput, createOnePrompt, makePromptTagInjector, + makeIntentInjector, + validateDeferPublish, parseUpdatePromptBody, mapLimit, publishAffected, @@ -36,6 +38,7 @@ import { ORIGIN_VALUE } from '../prompt-tags.js'; import { resolveProject, buildSliceProjectMap, sliceKey } from '../subworkspace-projects.js'; import { redactUpstreamMessage } from '../rest-transport.js'; import { createHeadroomGuard } from '../dynamic-allocation-active.js'; +import { classifyPromptIntents } from '../intent-classification.js'; /** * Subworkspace-mode prompt handlers (serenity dual-mode, subworkspace path). Behaviourally @@ -127,6 +130,8 @@ export async function handleCreatePromptsSubworkspace( body, log, classifyPromptType, + env, + writeDeadline, { dynamicAllocation = false, parentWorkspaceId = '' } = {}, ) { const inputs = Array.isArray(body?.prompts) ? body.prompts : []; @@ -139,6 +144,7 @@ export async function handleCreatePromptsSubworkspace( 400, ); } + const deferPublish = validateDeferPublish(body); const projectsBySlice = await buildSliceProjectMap(transport, workspaceId, log); // CREATE: user-authenticated write → derived `origin` is `human` (see the @@ -150,6 +156,25 @@ export async function handleCreatePromptsSubworkspace( log, { originValue: ORIGIN_VALUE.HUMAN }, ); + // Unified layer (serenity-docs#32): batch-classify every distinct text ONCE + // under the shared request deadline, then thread the resolved map into each + // per-item injection below. + // Classify the TRIMMED text: `makeIntentInjector` looks up the map by + // `input.text`, which `normalizePromptInput` has already trimmed, so the + // classify key must be trimmed to match — otherwise a whitespace-padded prompt + // (common in CSV import) misses the map and silently defaults to Informational + // despite a real classification. + const intentByText = await classifyPromptIntents( + inputs.map((raw) => String(raw?.text || '').trim()), + { + env, + log, + deadline: writeDeadline, + writePath: deferPublish ? 'csv' : 'create', + workspaceId, + }, + ); + const injectComputedIntent = makeIntentInjector(transport, workspaceId, intentByText, log); // PROMPT metering seam (Rainer, live-verified LLMO-6190): the metered write is // `createPromptsByIds` (inside `createOnePrompt` below), NOT publish — a disguised-quota 405 @@ -185,9 +210,11 @@ export async function handleCreatePromptsSubworkspace( } const projectId = String(project.id); try { - // Unified layer: strip any caller-supplied type/origin + inject the - // computed type and the derived origin (`human`). - const typed = await injectComputedTags(projectId, input); + // Unified layer: strip caller-supplied type/origin/intent, then inject the + // computed type + derived origin (origin-dimension.md §3) and the classified + // intent (serenity-docs#32). The injectors act on disjoint dimensions. + let typed = await injectComputedTags(projectId, input); + typed = await injectComputedIntent(projectId, typed); // LLMO-6190 follow-up: the metered write itself can still 405 as a disguised metered-quota // rejection despite the pre-loop sizing above (the live-verified ~9s gateway // write-enforcement lag after a JIT top-up) — route it through `headroom.retryOnQuota` (a @@ -239,11 +266,18 @@ export async function handleCreatePromptsSubworkspace( invalidateTagCacheForProject(workspaceId, pid); } - // LLMO-6190 item 4: route each project's publish through `headroom.retryOnQuota` — a bounded - // top-up+retry if it STILL 405s as a disguised metered-quota rejection despite the pre-write - // sizing above (e.g. a raced concurrent write on the same child). `publishAffected` nests this - // per-project, so a surviving 405 after the retry still lands in `publishErrors` for that project - // rather than throwing. `headroom` is always defined (a genuine no-op when the flag is OFF). + if (deferPublish) { + log?.info?.('serenity create-prompts (subworkspace): deferPublish set — prompts written as draft, publish skipped', { + workspaceId, created: created.length, skipped: skipped.length, failed: failed.length, + }); + return { + created, skipped, failed, published: false, + }; + } + + // Route each project's publish through the headroom guard's retryOnQuota (LLMO-6190 item 4): + // a disguised metered-405 gets ONE bounded top-up+retry per project before being recorded as a + // failure. No-op passthrough when the flag is OFF (the guard's retryOnQuota is a plain call). const publishErrors = await publishAffected( transport, workspaceId, @@ -257,7 +291,9 @@ export async function handleCreatePromptsSubworkspace( failed.push({ text: '', status: 502, message: `publish: ${pubErr.message}` }); } - return { created, skipped, failed }; + return { + created, skipped, failed, published: true, + }; } /** @@ -276,6 +312,8 @@ export async function handleUpdatePromptSubworkspace( body, log, classifyPromptType, + env, + writeDeadline, ) { const parsedBody = parseUpdatePromptBody(body); if (!parsedBody.ok) { @@ -306,16 +344,24 @@ export async function handleUpdatePromptSubworkspace( } const projectId = String(project.id); - // Recompute the type tag from the NEW text BEFORE any upstream write (see - // the flat-mode twin): a classification failure aborts cleanly with the - // prompt completely untouched. - // No `originValue`: origin is never re-derived on edit (origin-dimension.md §3 - // item 3); the stored origin the caller echoes rides through untouched. See the - // flat-mode twin handleUpdatePrompt. + // Recompute the type AND intent tags from the NEW text BEFORE any upstream write + // (see the flat-mode twin handleUpdatePrompt): the unified layer must run before + // the rename so a classification failure aborts cleanly with the prompt untouched + // (serenity-docs#31, #32). No `originValue`: origin is never re-derived on edit + // (origin-dimension.md §3 item 3); the stored origin the caller echoes rides + // through the replace-mode tag write untouched. const injectComputedTags = makePromptTagInjector(transport, workspaceId, classifyPromptType, log); - const typed = await injectComputedTags(projectId, { + const intentByText = await classifyPromptIntents( + [nextText], + { + env, log, deadline: writeDeadline, writePath: 'edit', workspaceId, + }, + ); + const injectComputedIntent = makeIntentInjector(transport, workspaceId, intentByText, log); + let typed = await injectComputedTags(projectId, { text: nextText, geoTargetId, tagIds: nextTagIds, }); + typed = await injectComputedIntent(projectId, typed); try { await transport.renamePrompt(workspaceId, projectId, semrushPromptId, nextText); diff --git a/src/support/serenity/handlers/prompts.js b/src/support/serenity/handlers/prompts.js index e70b46752..e3ac67a7d 100644 --- a/src/support/serenity/handlers/prompts.js +++ b/src/support/serenity/handlers/prompts.js @@ -19,8 +19,9 @@ import { redactUpstreamMessage } from '../rest-transport.js'; import { ERROR_CODES, isUpstreamGone } from '../errors.js'; import { normalizeGeoTargetId, normalizeLanguageCode, isValidTagIdFormat } from '../validation.js'; import { invalidateTagCacheForProject } from './markets.js'; -import { resolveTypeValueInjection, resolveClosedValueInjection } from '../tag-tree.js'; -import { DIMENSION, ORIGIN_VALUE } from '../prompt-tags.js'; +import { resolveTypeValueInjection, resolveIntentValueInjection, resolveClosedValueInjection } from '../tag-tree.js'; +import { DIMENSION, ORIGIN_VALUE, INTENT_VALUE } from '../prompt-tags.js'; +import { classifyPromptIntents } from '../intent-classification.js'; // TWIN FILE: the slice→project orchestration here is paralleled by the // subworkspace-mode handlers in prompts-subworkspace.js. The duplication is @@ -49,6 +50,24 @@ export const BULK_CREATE_CONCURRENCY = 8; // of them. Defense-in-depth, not a correctness gate. export const BULK_PROMPTS_MAX_ITEMS = 500; +/** + * Validates the optional `deferPublish` body flag (serenity-docs#32 CSV-chunking). + * Present-but-non-boolean is a hard 400 (so a caller typo like `"yes"`/`1` is + * rejected at the write boundary rather than silently treated as "publish"); + * absent, `false`, or `true` are all accepted. Returns the resolved boolean + * (absent → false). + * + * @param {object} body - request body. + * @returns {boolean} whether the caller asked to skip the trailing publish. + */ +export function validateDeferPublish(body) { + const deferPublish = body?.deferPublish; + if (deferPublish !== undefined && typeof deferPublish !== 'boolean') { + throw new ErrorWithStatusCode('deferPublish must be a boolean', 400); + } + return deferPublish === true; +} + /** * Builds the prompt's tag list from the upstream item: one entry per tag, * carrying its id, bare name, parent id and root-first ancestry breadcrumb. @@ -480,6 +499,59 @@ export function makePromptTagInjector( }; } +/** + * Applies a pre-computed, per-request `intent` classification map to a prompt + * write (serenity-docs#32) — the structural analog of {@link makePromptTagInjector} + * for the `intent` closed dimension. Unlike `type`, the "compute the value" step + * is a `Map` lookup, not a per-item classify call: intent is batch-classified + * ONCE per request (see `classifyPromptIntents` in `../intent-classification.js`) + * because it is an LLM call, not a cheap pure function. A text missing from + * `intentByText` (e.g. beyond the AI-gen classify cap) falls back to + * `INTENT_VALUE.INFORMATIONAL`, the seeded standard value. + * + * Given the map, it returns `injectComputedIntent(projectId, input)` which: + * - STRIPS every caller-supplied tag id under the `intent` root (the client may + * never set the value), and + * - APPENDS the pre-resolved upstream id of the server-computed value. The + * atomic `createPromptsByIds` 500s on an unresolved id, so it is resolved + * BEFORE the write. + * + * Id-based resolution ({@link resolveIntentValueInjection}, two tag-tree reads + * per distinct `intent` value per project) is memoized for the request, mirroring + * {@link makePromptTagInjector}'s memoization. `resolveIntentValueInjection` resolves + * or throws, so the computed tag is always attached and never silently dropped. + * + * @param {object} transport - Serenity transport (Semrush proxy client). + * @param {string} semrushWorkspaceId + * @param {Map} intentByText - text -> bare `intent` value. + * @param {object} [log] + * @returns {(projectId: string, input: { text: string, geoTargetId: number, + * tagIds: string[] }) => + * Promise<{ text: string, geoTargetId: number, tagIds: string[] }>} + */ +export function makeIntentInjector(transport, semrushWorkspaceId, intentByText, log) { + /** @type {Map>} */ + const cache = new Map(); + return async function injectComputedIntent(projectId, input) { + const intentValue = intentByText.get(input.text) ?? INTENT_VALUE.INFORMATIONAL; + const key = `${projectId} ${intentValue}`; + let pending = cache.get(key); + if (!pending) { + pending = resolveIntentValueInjection( + transport, + semrushWorkspaceId, + projectId, + intentValue, + log, + ); + cache.set(key, pending); + } + const { computedId, intentTagIds } = await pending; + const stripped = input.tagIds.filter((id) => !intentTagIds.includes(id)); + return { ...input, tagIds: [...stripped, computedId] }; + }; +} + /** * Validates + normalizes a PATCH prompt body's `text` + `tagIds`, shared by * {@link handleUpdatePrompt} and its subworkspace twin. Tags are addressed by @@ -517,7 +589,18 @@ export function parseUpdatePromptBody(body) { }, }; } - const text = String(body.text); + // Mirror the create contract (`normalizePromptInput`): empty or whitespace-only + // text is rejected here rather than passed on to `renamePrompt`, where it would + // be classified and written as a blank prompt. `|| ''` also coerces a falsy + // non-string (`null`, `0`, `false`) to empty, matching create exactly. + const text = String(body.text || '').trim(); + if (!text) { + return { + ok: false, + status: 400, + body: { error: 'invalidRequest', message: 'text must be a non-empty string' }, + }; + } const tagIds = sanitizeTagIds(body.tagIds); if (tagIds.length === 0) { return { @@ -552,9 +635,14 @@ export async function mapLimit(items, limit, mapper) { /** * POST /serenity/prompts — bulk create. - * Each input must carry `(geoTargetId, languageCode, text, tags?)`. Inputs + * Each input must carry `(geoTargetId, languageCode, text, tagIds)`. Inputs * are grouped by slice; the matching BrandSemrushProject row resolves the - * upstream project; publish runs once per affected project at the end. + * upstream project; publish runs once per affected project at the end — + * unless `body.deferPublish` is true (serenity-docs#32 CSV-chunking), in + * which case the create is a draft-only write and the caller is responsible + * for triggering a publish itself (e.g. a normal, non-deferred call on the + * last chunk of an import, which publishes every project touched across the + * whole import since a single CSV import always targets one project). */ export async function handleCreatePrompts( transport, @@ -564,6 +652,8 @@ export async function handleCreatePrompts( body, log, classifyPromptType, + env, + writeDeadline, ) { const inputs = Array.isArray(body?.prompts) ? body.prompts : []; if (inputs.length === 0) { @@ -575,6 +665,7 @@ export async function handleCreatePrompts( 400, ); } + const deferPublish = validateDeferPublish(body); const projects = await dataAccess.BrandSemrushProject.allByBrandId(brandId); const projectsBySlice = new Map(); @@ -593,6 +684,25 @@ export async function handleCreatePrompts( log, { originValue: ORIGIN_VALUE.HUMAN }, ); + // Unified layer (serenity-docs#32): batch-classify every distinct text ONCE + // under the shared request deadline, then thread the resolved map into each + // per-item injection below (a per-item LLM call would be far too slow). + // Classify the TRIMMED text: `makeIntentInjector` looks up the map by + // `input.text`, which `normalizePromptInput` has already trimmed, so the + // classify key must be trimmed to match — otherwise a whitespace-padded prompt + // (common in CSV import) misses the map and silently defaults to Informational + // despite a real classification. + const intentByText = await classifyPromptIntents( + inputs.map((raw) => String(raw?.text || '').trim()), + { + env, + log, + deadline: writeDeadline, + writePath: deferPublish ? 'csv' : 'create', + workspaceId: semrushWorkspaceId, + }, + ); + const injectComputedIntent = makeIntentInjector(transport, semrushWorkspaceId, intentByText, log); const results = await mapLimit(inputs, BULK_CREATE_CONCURRENCY, async (raw) => { const { value: input, reason } = normalizePromptInput(raw); @@ -615,9 +725,12 @@ export async function handleCreatePrompts( } const projectId = project.getSemrushProjectId(); try { - // Unified layer: strip any caller-supplied type/origin + inject the - // computed type and the derived origin (`human`). - const typed = await injectComputedTags(projectId, input); + // Unified layer: strip caller-supplied type/origin/intent, then inject the + // computed type + derived origin (origin-dimension.md §3) and the + // classified intent (serenity-docs#32). The two injectors act on disjoint + // dimensions, so chaining composes cleanly. + let typed = await injectComputedTags(projectId, input); + typed = await injectComputedIntent(projectId, typed); const semrushPromptId = await createOnePrompt( transport, semrushWorkspaceId, @@ -670,6 +783,15 @@ export async function handleCreatePrompts( invalidateTagCacheForProject(semrushWorkspaceId, pid); } + if (deferPublish) { + log?.info?.('serenity create-prompts: deferPublish set — prompts written as draft, publish skipped', { + brandId, created: created.length, skipped: skipped.length, failed: failed.length, + }); + return { + created, skipped, failed, published: false, + }; + } + const publishErrors = await publishAffected( transport, semrushWorkspaceId, @@ -686,7 +808,9 @@ export async function handleCreatePrompts( }); } - return { created, skipped, failed }; + return { + created, skipped, failed, published: true, + }; } /** @@ -736,6 +860,8 @@ export async function handleUpdatePrompt( body, log, classifyPromptType, + env, + writeDeadline, ) { // `semrushPromptId` is validated as non-empty at the controller boundary // (serenity.js:259) before this handler is invoked over HTTP, so no @@ -773,22 +899,39 @@ export async function handleUpdatePrompt( } const projectId = project.getSemrushProjectId(); - // Recompute the type tag from the NEW text BEFORE any upstream write: the - // unified layer (tree read / on-demand tag create) resolves the computed - // value's id first, so a classification failure aborts cleanly with the - // prompt completely untouched. NO `originValue` is passed: `origin` is a fact - // about the row's creation, never re-derived on edit (origin-dimension.md §3 - // item 3). The prompt's stored origin id, echoed back by the caller, rides - // through the replace-mode tag write untouched. + // Recompute the type AND intent tags from the NEW text BEFORE the rename: the + // unified layer (tree read / on-demand tag create / LLM classify) must run + // before any upstream write, so a classification failure aborts cleanly with + // the old prompt still present (serenity-docs#31, #32). NO `originValue` is + // passed: `origin` is a fact about the row's creation, never re-derived on + // edit (origin-dimension.md §3 item 3) — the prompt's stored origin id, echoed + // back by the caller, rides through the replace-mode tag write untouched. + // + // This runs UNCONDITIONALLY, even when the PATCH does not change the text. The + // upstream provider has no GET-by-id and the handler is not sent the old text + // (the body is the full next state — see the docblock above), so it cannot + // know whether the text actually changed. `renamePrompt`'s `is_updated: false` + // reports a no-op only AFTER the rename, too late to gate a classify that has + // to run first for failure-safety. Skipping the reclassification would require + // the client to send the old text — a contract change deliberately out of + // scope here (keep the edit path a single straight line). const injectComputedTags = makePromptTagInjector( transport, semrushWorkspaceId, classifyPromptType, log, ); - const typed = await injectComputedTags(projectId, { + const intentByText = await classifyPromptIntents( + [nextText], + { + env, log, deadline: writeDeadline, writePath: 'edit', workspaceId: semrushWorkspaceId, + }, + ); + const injectComputedIntent = makeIntentInjector(transport, semrushWorkspaceId, intentByText, log); + let typed = await injectComputedTags(projectId, { text: nextText, geoTargetId, tagIds: nextTagIds, }); + typed = await injectComputedIntent(projectId, typed); try { await transport.renamePrompt(semrushWorkspaceId, projectId, semrushPromptId, nextText); diff --git a/src/support/serenity/intent-classification.js b/src/support/serenity/intent-classification.js new file mode 100644 index 000000000..31422999d --- /dev/null +++ b/src/support/serenity/intent-classification.js @@ -0,0 +1,352 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +// @ts-check + +import { createIntentClassifier, classifyIntents } from '../intent-classifier.js'; +import { emitMetric, resolveEnvironment } from '../metrics-emf.js'; +import { INTENT_VALUE } from './prompt-tags.js'; +import { + SERENITY_INTENT_CATEGORY_SPEC, + SERENITY_INTENT_VALUES, + inspectSerenityIntent, + PER_CALL_MS, +} from './intent-taxonomy.js'; + +/** + * CloudWatch namespace for Serenity write-path intent-classification metrics + * (serenity-docs#32 Observability). Mirrors the `Mysticat/Brands` convention in + * `src/controllers/brands.js`. + */ +const SERENITY_METRICS_NAMESPACE = 'Mysticat/Serenity'; + +// Cap on the number of soft-failure reasoning samples logged per request, so a +// low-confidence storm produces one bounded structured line, not a log flood. +const SOFT_FAILURE_LOG_CAP = 10; + +/** + * Nearest-rank percentile of a NON-EMPTY ascending-sorted numeric array. Used for + * per-call latency p50/p95 without pulling in a stats dep; the sole caller only + * invokes it once at least one call has been timed. + * + * @param {number[]} sortedAsc - non-empty values sorted ascending. + * @param {number} p - percentile in [0, 100]. + * @returns {number} + */ +function percentile(sortedAsc, p) { + const rank = Math.ceil((p / 100) * sortedAsc.length) - 1; + const idx = Math.min(sortedAsc.length - 1, Math.max(0, rank)); + return sortedAsc[idx]; +} + +/** + * Best-effort EMF emit for a single Serenity intent-classification metric. Wraps + * {@link emitMetric} (which already swallows its own errors) in an extra + * try/catch — belt-and-suspenders matching the `emitBrandDemotionBlocked` + * pattern in `brands.js` — so instrumentation can never throw into the classify + * path. + * + * @param {string} name - CloudWatch metric name. + * @param {number} value - metric value. + * @param {string} unit - CloudWatch unit ('Count', 'Milliseconds', ...). + * @param {object} dimensions - dimension key/value pairs (null values dropped). + * @param {object} env - environment (for {@link resolveEnvironment}). + */ +function emitIntentMetric(name, value, unit, dimensions, env) { + try { + emitMetric( + { + name, value, unit, dimensions, + }, + { environment: resolveEnvironment(env), namespace: SERENITY_METRICS_NAMESPACE }, + ); + } catch { + // best-effort: metric emission must never affect the classify path + } +} + +/** + * Serenity write-path request budget (serenity-docs#32) — a single shared + * deadline so classify + its budget-gated retry + create-fanout + publish + * together stay under the ~15s Fastly first-byte timeout the write already + * shares. `WRITE_SOFT_BUDGET_MS` leaves ~3s margin; `CREATE_PUBLISH_RESERVE_MS` + * is reserved for the write itself, so classification only gets the remainder. + */ +export const WRITE_SOFT_BUDGET_MS = 12000; +export const CREATE_PUBLISH_RESERVE_MS = 6000; + +// Classify concurrency. Deliberately a local constant rather than importing +// `BULK_CREATE_CONCURRENCY` from `./handlers/prompts.js` (which would create a +// circular import, since that module imports from here) — by design decision +// (serenity-docs#32) it currently mirrors that value. +const CLASSIFY_CONCURRENCY = 8; + +// AI-generation classify cap (serenity-docs#32, resolved 2026-07-09): 2 budget +// rounds (one call + one budget-gated retry) at CLASSIFY_CONCURRENCY each. +export const AI_GEN_CLASSIFY_MAX = CLASSIFY_CONCURRENCY * 2; + +/** + * Computes the request-scoped write deadline. Call once at controller entry, + * before any classify/create/publish work begins. + * + * @param {number} [now] - override for testing. + * @returns {number} epoch ms deadline. + */ +export function computeWriteDeadline(now = Date.now()) { + return now + WRITE_SOFT_BUDGET_MS; +} + +/** + * Remaining budget (ms) available for classification: whatever is left on the + * deadline, minus the reserve for create-fanout + publish. Never negative. + * + * @param {number} deadline - epoch ms deadline from {@link computeWriteDeadline}. + * @param {number} [now] - override for testing. + * @returns {number} + */ +function remainingClassifyBudget(deadline, now = Date.now()) { + return Math.max(0, (deadline - now) - CREATE_PUBLISH_RESERVE_MS); +} + +/** + * Batch-classifies prompt texts into bare `intent` values (e.g. `Task`) under + * the shared write-budget, applying the full fallback ladder from + * serenity-docs#32: + * + * 1. Hard skip-gate — if there's no room for even one call at entry, default + * everything immediately (no LLM calls attempted, counted `budget_skipped`). + * 2. Azure not configured — default everything (same in every environment; a + * `warn` + `prod_llm_unavailable` signal only in prod, `info` elsewhere). + * 3. First classify pass under the remaining budget. + * 4. One budget-gated retry for anything still unresolved — only if there's + * still room for a full per-call timeout; otherwise skip straight to default. + * 5. Terminal default `intent:Informational` for anything still unresolved. + * + * Every input text is guaranteed a value in the returned map — never missing — + * so callers never need a separate "no tag" branch. + * + * Emits best-effort observability (serenity-docs#32) — outcome counters, + * value distribution, per-call/batch latency, timeout/`budget_skipped` and + * `prod_llm_unavailable` — via EMF, dimensioned by `writePath` (and workspace on + * the outcome counters). All emission is best-effort and never throws into the + * classify path; the classify/retry/default decision logic is unchanged. + * + * @param {string[]} texts - prompt texts to classify (deduplicated internally). + * @param {object} options + * @param {object} [options.env] - environment (Azure OpenAI creds). + * @param {object} [options.log] - logger. + * @param {number} options.deadline - the request's write deadline (epoch ms). + * @param {string} [options.writePath] - which write-path this call serves, for + * the metric `WritePath` dimension: `'create' | 'edit' | 'csv' | 'ai-gen'`. + * @param {string} [options.workspaceId] - the Semrush workspace id, used as the + * per-customer `Workspace` dimension on the outcome counters so one catalog's + * degradation isn't hidden under a healthy aggregate. + * @returns {Promise>} text -> bare `intent` value. + */ +export async function classifyPromptIntents(texts, { + env, log = console, deadline, writePath = 'unknown', workspaceId, +}) { + // `env`/`log` may arrive explicitly `null` (some callers default optional + // options that way) — a default *parameter* only applies on `undefined`, so + // normalize here rather than relying on destructuring defaults. + const safeEnv = env || {}; + const safeLog = log || console; + const unique = [...new Set((texts || []).filter((t) => typeof t === 'string' && t.length > 0))]; + const result = new Map(); + if (unique.length === 0) { + return result; + } + + const counts = { + classified_ok: 0, + low_confidence: 0, + retry_attempted: 0, + retry_succeeded: 0, + defaulted: 0, + budget_skipped: 0, + }; + // Distribution of the emitted classified intent values (excludes `defaulted`, + // which is reported as its own bucket). Keyed by bare intent value. + const valueCounts = Object.create(null); + // Per-call LLM latencies (ms) and a heuristic per-call timeout tally. + const callDurations = []; + let callTimeouts = 0; + let batchStart = 0; + // Truncated soft-failure reasoning samples for one bounded structured log line. + const softFailures = []; + + const defaultAll = (list) => { + list.forEach((t) => result.set(t, INTENT_VALUE.INFORMATIONAL)); + counts.defaulted += list.length; + }; + + const resolve = (t, value, ok) => { + result.set(t, value); + counts[ok] += 1; + valueCounts[value] = (valueCounts[value] || 0) + 1; + }; + + // Emit all per-request EMF metrics from the current counters/timings. Called + // on every terminal path so the classified-vs-fallback ratio is always + // derivable. Best-effort: each emit is individually wrapped. + const emitObservability = () => { + const outcomeDims = { WritePath: writePath, Workspace: workspaceId }; + Object.entries(counts).forEach(([outcome, count]) => { + if (count > 0) { + emitIntentMetric('IntentOutcome', count, 'Count', { ...outcomeDims, Outcome: outcome }, safeEnv); + } + }); + SERENITY_INTENT_VALUES.forEach((word) => { + const c = valueCounts[word] || 0; + if (c > 0) { + emitIntentMetric('IntentValueDistribution', c, 'Count', { WritePath: writePath, Value: word }, safeEnv); + } + }); + if (counts.defaulted > 0) { + emitIntentMetric('IntentValueDistribution', counts.defaulted, 'Count', { WritePath: writePath, Value: 'defaulted' }, safeEnv); + } + if (callDurations.length > 0) { + const sorted = [...callDurations].sort((a, b) => a - b); + emitIntentMetric('PerCallLatencyP50Ms', percentile(sorted, 50), 'Milliseconds', { WritePath: writePath }, safeEnv); + emitIntentMetric('PerCallLatencyP95Ms', percentile(sorted, 95), 'Milliseconds', { WritePath: writePath }, safeEnv); + if (callTimeouts > 0) { + emitIntentMetric('PerCallTimeout', callTimeouts, 'Count', { WritePath: writePath }, safeEnv); + } + } + if (batchStart > 0) { + emitIntentMetric('ClassifyBatchDurationMs', Date.now() - batchStart, 'Milliseconds', { WritePath: writePath }, safeEnv); + } + }; + + const emitProdLlmUnavailable = (message) => { + emitIntentMetric('ProdLlmUnavailable', 1, 'Count', { WritePath: writePath }, safeEnv); + safeLog?.warn?.(`WARN: prod_llm_unavailable — ${message}`); + }; + + if (remainingClassifyBudget(deadline) < PER_CALL_MS) { + counts.budget_skipped = unique.length; + defaultAll(unique); + safeLog?.info?.('serenity intent classification: budget_skipped (no room at entry)', { count: unique.length }); + emitObservability(); + return result; + } + + // Per-request category spec: wraps the taxonomy's `inspectSerenityIntent` so + // the caller can count `low_confidence` soft-failures apart from other + // unresolved cases and capture their `reasoning`, while still returning the + // same `string|null` tag the retry/default ladder expects (behavior unchanged). + const observedSpec = { + systemPrompt: SERENITY_INTENT_CATEGORY_SPEC.systemPrompt, + invokeTimeoutMs: SERENITY_INTENT_CATEGORY_SPEC.invokeTimeoutMs, + parseResult: (parsed) => { + const { value, reason, reasoning } = inspectSerenityIntent(parsed); + if (reason !== 'ok') { + if (reason === 'low_confidence') { + counts.low_confidence += 1; + } + if (softFailures.length < SOFT_FAILURE_LOG_CAP) { + softFailures.push({ reason, reasoning: String(reasoning || '').slice(0, 200) }); + } + } + return value; + }, + }; + + const classify = createIntentClassifier( + { env: safeEnv, log: safeLog }, + observedSpec, + ); + if (typeof classify !== 'function') { + defaultAll(unique); + const message = 'serenity intent classification: Azure OpenAI is not configured; defaulting to Informational'; + if (resolveEnvironment(safeEnv) === 'prod') { + emitProdLlmUnavailable(message); + } else { + safeLog?.info?.(message); + } + emitObservability(); + return result; + } + + // Wrap the single-text classifier to record per-call latency (and a heuristic + // per-call timeout: a null result that took ~the full per-call budget). The + // classifier itself is best-effort and never rejects; the try/finally is + // defensive so timing is recorded even if that contract ever changes. + const timedClassify = async (text) => { + const t0 = Date.now(); + let r = null; + try { + r = await classify(text); + return r; + } finally { + const dt = Date.now() - t0; + callDurations.push(dt); + if (r === null && dt >= PER_CALL_MS) { + callTimeouts += 1; + } + } + }; + + batchStart = Date.now(); + const firstPass = await classifyIntents(timedClassify, unique, { + maxConcurrency: CLASSIFY_CONCURRENCY, + timeoutMs: remainingClassifyBudget(deadline), + }); + const stillUnresolved = []; + unique.forEach((t) => { + const value = firstPass.get(t); + if (value) { + resolve(t, value, 'classified_ok'); + } else { + stillUnresolved.push(t); + } + }); + + if (stillUnresolved.length > 0 && remainingClassifyBudget(deadline) >= PER_CALL_MS) { + counts.retry_attempted = stillUnresolved.length; + const retryPass = await classifyIntents(timedClassify, stillUnresolved, { + maxConcurrency: CLASSIFY_CONCURRENCY, + timeoutMs: remainingClassifyBudget(deadline), + }); + const stillUnresolvedAfterRetry = []; + stillUnresolved.forEach((t) => { + const value = retryPass.get(t); + if (value) { + resolve(t, value, 'retry_succeeded'); + } else { + stillUnresolvedAfterRetry.push(t); + } + }); + defaultAll(stillUnresolvedAfterRetry); + } else { + defaultAll(stillUnresolved); + } + + // Repeated-invoke-failure signal (serenity-docs#32): calls were actually made + // (the classifier was constructed) but not a single one resolved — the LLM is + // effectively unavailable, distinct from `budget_skipped` where no call runs. + if (callDurations.length > 0 + && counts.classified_ok + counts.retry_succeeded === 0 + && resolveEnvironment(safeEnv) === 'prod') { + emitProdLlmUnavailable('running on prod but the intent LLM returned no classifications for any prompt; prompts are being assigned the Informational default, not classified'); + } + + if (softFailures.length > 0) { + safeLog?.info?.('serenity intent classification soft failures', { + low_confidence: counts.low_confidence, + samples: softFailures, + }); + } + safeLog?.info?.('serenity intent classification summary', counts); + emitObservability(); + return result; +} diff --git a/src/support/serenity/intent-taxonomy.js b/src/support/serenity/intent-taxonomy.js new file mode 100644 index 000000000..ccb91bf6d --- /dev/null +++ b/src/support/serenity/intent-taxonomy.js @@ -0,0 +1,150 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +// @ts-check + +import { INTENT_VALUE } from './prompt-tags.js'; + +/** + * Serenity's 5-value intent taxonomy (serenity-docs#32) — the LLM-facing + * category spec consumed by `createIntentClassifier` from + * `../intent-classifier.js`. This is deliberately a SEPARATE taxonomy from the + * native/DRS 6-bucket one (`../intent.js`): `INTENT_REMAP` there collapses + * `navigational→informational` / `commercial→transactional`, so the DRS + * classifier structurally cannot emit two of these five values — the category + * set must be injected, not remapped. + */ + +// The 5 bare category words the model must emit verbatim (Capitalized, +// case-sensitive) — the `intent` closed-dimension vocabulary from +// `prompt-tags.js` (INTENT_VALUE). Under the dimension-root tag model these are +// bare values resolved to child tag ids under the `intent` root, not prefixed +// wire tags, so no taxonomy change is needed on the write path. +export const SERENITY_INTENT_VALUES = Object.freeze(Object.values(INTENT_VALUE)); + +// Validity floor, NOT a quality gate (calibration finding: the model is +// uniformly over-confident regardless of correctness — gpt-4.1 mean 0.959, +// gpt-4.1-mini mean 0.935 across 60 calibration prompts). Only catches a rare +// garbled / genuinely-uncertain output. Fixed code constant, not env-tunable. +export const PROMPT_INTENT_MIN_CONFIDENCE = 0.5; + +// Per-call timeout for the Serenity classifier — fixed by the shared +// request-budget design (serenity-docs#32), not env-driven like the native +// classifier's `PROMPT_INTENT_CLASSIFICATION_TIMEOUT_MS`. +export const PER_CALL_MS = 3000; + +// Validated 2026-07-07 against 60 real Lovesac prompts on the real prod +// `gpt-4.1` deployment (temperature 0, max_tokens 150, JSON output) — see the +// calibration report on serenity-docs#32. All five values fire correctly; the +// delegation→Task disambiguation rule below was validated 5/5 unanimous across +// gpt-4.1 / gpt-4.1-mini / gpt-4.1-nano / gpt-4o. +export const SERENITY_INTENT_SYSTEM_PROMPT = `You are a user intent classifier for AI assistant queries about a brand. Classify the given prompt into exactly one of 5 categories. + +The prompt may be in any language (English, Portuguese, German, Spanish, etc.). Classify based on meaning, not language. + +Categories: + +1. **Informational** - The user wants to KNOW, UNDERSTAND, or DISCOVER something — general lookups, definitions, explanations, or open-ended "what/how" questions that are not about buying or being personally advised. + Examples: "What is Adobe Firefly?", "How does a heat pump work?", "What's the history of this brand?" + +2. **Task** - The user wants the AI to RECOMMEND, DECIDE, or PICK something FOR THEM personally, OR wants step-by-step guidance to do something themselves. This is a personal ask ("recommend for me", "help me choose", "how do I set this up") — a self-directed research question about which is objectively best ("best X", "X vs Y") is NOT Task, it is Commercial. + Examples: "Can you recommend a couch for my apartment?", "What would you suggest for a small living room?", "How do I set up a glossary in my CMS?", "Help me choose between these two options." + Disambiguation: "Can you recommend/suggest X for me?" = Task (delegation — the AI decides). "What is the best X?" or "X vs Y" = Commercial (the user is researching, not delegating the decision). + +3. **Commercial** - The user is researching options themselves — comparing, ranking, or evaluating products/services, including "best X", "top X", "X vs Y", or general product research NOT phrased as a personal ask. + Examples: "Best AI workspace tool", "Top PDF editors 2024", "Figma vs Sketch for UI design", "Most durable couch brands", "Which is better, Notion or Confluence?" + +4. **Transactional** - The user wants to BUY, DOWNLOAD, SIGN UP, or take a direct commercial action. + Examples: "Adobe Creative Cloud pricing", "Free trial for Photoshop", "Where to buy this brand's products", "Download PDF editor free" + +5. **Navigational** - The user is trying to reach a specific known destination (a brand's site, page, app, or account), not researching or deciding. + Examples: "Adobe.com login", "Open Photoshop web", "Brand's official Instagram page" + +Decision rules: +- "Can you recommend/suggest X for me?", "What would you recommend?", "Help me choose" = **Task** (delegation — the AI decides for the user). "What is the best X?", "Best X for Y", "X vs Y" = **Commercial** (self-directed research), even though both start similarly. +- Step-by-step "how do I do X myself" = **Task**. +- Default to **Informational** if ambiguous. + +Output requirements (strict): Reply with ONLY valid JSON. Response is limited to ~150 tokens, so keep it short. +Output format: {"intent": "", "confidence": 0.0-1.0, "reasoning": ""} +Do not include markdown, code fences, or any text outside the JSON object.`; + +/** + * @typedef {object} SerenityIntentInspection + * @property {string|null} value - the bare `intent` value (e.g. `Task`) or null + * on any soft failure. + * @property {'ok'|'invalid_value'|'low_confidence'} reason - why the result is + * what it is: `ok` (usable value), `invalid_value` (garbled / unrecognized + * value), or `low_confidence` (recognized value below the validity floor). + * @property {number} confidence - the parsed confidence (NaN if unparseable). + * @property {string} reasoning - the model's `reasoning` field ('' if absent), + * surfaced so the caller can log it on soft failures without re-running. + */ + +/** + * Inspects a parsed model response against the Serenity taxonomy, distinguishing + * the soft-failure modes the {@link parseSerenityIntent} result collapses away — + * so the caller's observability layer can count `low_confidence` apart from + * other unresolved cases and log the `reasoning` for debuggability. The + * retry/default behavior is unchanged: any non-`ok` reason still yields a null + * `value`, which the caller folds into the same fallback ladder. + * + * @param {object} parsed - the parsed `{intent, confidence, reasoning}` body. + * @returns {SerenityIntentInspection} + */ +export function inspectSerenityIntent(parsed) { + const value = String(parsed?.intent ?? ''); + const confidence = Number(parsed?.confidence); + const reasoning = typeof parsed?.reasoning === 'string' ? parsed.reasoning : ''; + if (!(/** @type {readonly string[]} */ (SERENITY_INTENT_VALUES)).includes(value)) { + return { + value: null, reason: 'invalid_value', confidence, reasoning, + }; + } + if (!Number.isFinite(confidence) || confidence < PROMPT_INTENT_MIN_CONFIDENCE) { + return { + value: null, reason: 'low_confidence', confidence, reasoning, + }; + } + return { + value, reason: 'ok', confidence, reasoning, + }; +} + +/** + * Validates a parsed model response against the Serenity taxonomy: the value + * must be one of the 5 canonical Capitalized literals AND confidence must meet + * the validity floor. Returns the bare `intent` value (e.g. `Task`) on success — + * which the caller resolves to a child tag id under the `intent` root via + * {@link resolveIntentValueInjection} — or `null` on any validation failure + * (garbled output, an unrecognized value, or a below-floor confidence, all + * treated as the same "soft failure" by the caller's retry/default ladder). + * + * @param {object} parsed - the parsed `{intent, confidence, reasoning}` body. + * @returns {string|null} the bare `intent` value or null. + */ +export function parseSerenityIntent(parsed) { + return inspectSerenityIntent(parsed).value; +} + +/** + * The Serenity category spec passed to `createIntentClassifier` (see + * `../intent-classifier.js`) for every Serenity write-path classification. + * + * @type {{ systemPrompt: string, invokeTimeoutMs: number, + * parseResult: (parsed: object) => (string|null) }} + */ +export const SERENITY_INTENT_CATEGORY_SPEC = Object.freeze({ + systemPrompt: SERENITY_INTENT_SYSTEM_PROMPT, + parseResult: parseSerenityIntent, + invokeTimeoutMs: PER_CALL_MS, +}); diff --git a/src/support/serenity/tag-tree.js b/src/support/serenity/tag-tree.js index 5dba4f5e7..b649060ae 100644 --- a/src/support/serenity/tag-tree.js +++ b/src/support/serenity/tag-tree.js @@ -653,3 +653,38 @@ export async function resolveTypeValueInjection( ); return { computedId, typeTagIds: valueTagIds }; } + +/** + * Resolves the id-based injection of a server-computed `intent` value into a + * prompt write (serenity-docs#32). The exact structural analog of + * {@link resolveTypeValueInjection} for the `intent` closed dimension: returns + * the wanted value's id plus EVERY id under the `intent` root, so the caller can + * strip any caller-supplied `intent` tag id (the client must never set the value + * itself — it is server-classified). + * + * @param {object} transport - Serenity transport (Semrush proxy client). + * @param {string} semrushWorkspaceId + * @param {string} projectId + * @param {string} wantValue - the computed bare `intent` value (e.g. `Task`). + * @param {object} [log] - logger. + * @returns {Promise<{ computedId: string, intentTagIds: string[] }>} `computedId` + * is always resolved — {@link ensureChildren} throws rather than leave a hole, + * so a prompt can never be written with the server-computed `intent` tag missing. + */ +export async function resolveIntentValueInjection( + transport, + semrushWorkspaceId, + projectId, + wantValue, + log, +) { + const { computedId, valueTagIds } = await resolveClosedValueInjection( + transport, + semrushWorkspaceId, + projectId, + DIMENSION.INTENT, + wantValue, + log, + ); + return { computedId, intentTagIds: valueTagIds }; +} diff --git a/test/controllers/serenity.test.js b/test/controllers/serenity.test.js index c3d3be5d2..036beee4a 100644 --- a/test/controllers/serenity.test.js +++ b/test/controllers/serenity.test.js @@ -1422,13 +1422,19 @@ describe('SerenityController', () => { // so topic generation defaults off (today's behavior is unchanged). brandUuid // is already a persisted row (loadBrand), so dataAccess is threaded through // for the mapping-row upsert (mapping-rows.js). - expect(handlers.handleCreateMarketSubworkspace.firstCall.args[7]) + // writeDeadline is a request-scoped epoch-ms deadline (dynamic) — asserted + // as a number, then dropped before the deep-equal. + const { writeDeadline, ...marketOptions } = handlers.handleCreateMarketSubworkspace + .firstCall.args[7]; + expect(writeDeadline).to.be.a('number'); + expect(marketOptions) .to.deep.equal({ generateTopics: false, topicCap: 0, brandAliases: ['Acme Inc', 'ACME'], brandUrlSources: { urls: [], socialAccounts: [], earnedContent: [] }, competitors: [], + env: {}, dataAccess: { BrandSemrushProject: ctx.dataAccess.BrandSemrushProject }, // Dynamic-allocation kill-switch defaults OFF (env unset) — the guard is a no-op. dynamicAllocation: false, @@ -2122,12 +2128,20 @@ describe('SerenityController', () => { brandAliases: ['Acme Inc'], brandUrlSources: { urls: [], socialAccounts: [], earnedContent: [] }, competitors: [], + env: {}, dataAccess: { BrandSemrushProject: ctx.dataAccess.BrandSemrushProject }, dynamicAllocation: false, }; const { firstCall, secondCall } = handlers.handleCreateMarketSubworkspace; - expect(firstCall.args[7]).to.deep.equal(expectedOpts); - expect(secondCall.args[7]).to.deep.equal(expectedOpts); + // writeDeadline is computed ONCE at activate entry, so every market in the + // batch receives the SAME dynamic epoch-ms deadline — assert that, then + // drop it before comparing the rest of the options bag. + const { writeDeadline: dl1, ...opts1 } = firstCall.args[7]; + const { writeDeadline: dl2, ...opts2 } = secondCall.args[7]; + expect(dl1).to.be.a('number'); + expect(dl2).to.equal(dl1); + expect(opts1).to.deep.equal(expectedOpts); + expect(opts2).to.deep.equal(expectedOpts); // Org parent (JIT units pool) threaded positionally (arg index 2), not in the options bag. expect(firstCall.args[2]).to.equal(WORKSPACE); expect(secondCall.args[2]).to.equal(WORKSPACE); diff --git a/test/it/shared/tests/serenity.js b/test/it/shared/tests/serenity.js index 1b351d02c..5c0c60c89 100644 --- a/test/it/shared/tests/serenity.js +++ b/test/it/shared/tests/serenity.js @@ -556,13 +556,16 @@ export default function serenityTests( expect(created.status).to.equal(200); expect(created.body.created).to.have.lengthOf(1); expect(created.body.created[0].semrushPromptId).to.be.a('string').that.is.not.empty; - // The write path server-stamps two dimensions the caller may not set: a - // branded/non-branded `type:` tag (classified from the text) AND, on a - // user-authenticated create, the derived `origin:` tag (`human`) — - // origin-dimension.md §3 (WP-O2b). So the created prompt carries the two - // supplied tags plus one computed `type` tag plus one derived `origin` tag. + // The write path server-stamps THREE dimensions the caller may not set: a + // branded/non-branded `type:` tag (classified from the text), the derived + // `origin:` tag (`human`, on a user-authenticated create — origin-dimension.md + // §3 / WP-O2b), AND an `intent:` tag (serenity-docs#31, #32). Azure + // OpenAI is not configured in this IT environment, so intent deterministically + // defaults to `intent:Informational` (never null/omitted — see the fallback + // ladder). So the created prompt carries the two supplied tags plus the three + // computed ones. expect(created.body.created[0].tagIds).to.include.members([category.body.id, child.body.id]); - expect(created.body.created[0].tagIds).to.have.lengthOf(4); + expect(created.body.created[0].tagIds).to.have.lengthOf(5); expect(created.body.failed).to.deep.equal([]); // by_tags correlation: the id-based create embeds the tag ids, so filtering the prompt list diff --git a/test/openapi-contract/serenity-api.test.js b/test/openapi-contract/serenity-api.test.js index 57726b96a..d928f1f9a 100644 --- a/test/openapi-contract/serenity-api.test.js +++ b/test/openapi-contract/serenity-api.test.js @@ -129,6 +129,7 @@ const FIXTURES = { }], skipped: [], failed: [], + published: true, }, data: { prompts: [{ diff --git a/test/support/intent-classifier.test.js b/test/support/intent-classifier.test.js index 870570230..c0e6b304e 100644 --- a/test/support/intent-classifier.test.js +++ b/test/support/intent-classifier.test.js @@ -21,6 +21,8 @@ import yaml from 'js-yaml'; import { classifyIntents, contentToString, + DRS_CATEGORY_SPEC, + SYSTEM_PROMPT, } from '../../src/support/intent-classifier.js'; import { INTENT_VALUES } from '../../src/support/intent.js'; @@ -66,6 +68,13 @@ describe('intent-classifier', () => { afterEach(() => sinon.restore()); describe('createIntentClassifier', () => { + // Byte-for-byte parity guard for the native/DRS path: the 6-bucket refactor + // reuses SYSTEM_PROMPT as-is for DRS_CATEGORY_SPEC.systemPrompt, so the + // native prompt must be strictly the same string, not a copy that can drift. + it('reuses SYSTEM_PROMPT verbatim as the DRS category spec system prompt', () => { + expect(DRS_CATEGORY_SPEC.systemPrompt).to.equal(SYSTEM_PROMPT); + }); + it('returns null when Azure OpenAI is not configured', async () => { const { mod, ctorSpy } = await loadWithModel(() => ({ content: '{}' })); const classify = mod.createIntentClassifier({ env: {}, log }); diff --git a/test/support/serenity/brand-provisioning.test.js b/test/support/serenity/brand-provisioning.test.js index 710dabb93..aee1062c5 100644 --- a/test/support/serenity/brand-provisioning.test.js +++ b/test/support/serenity/brand-provisioning.test.js @@ -186,16 +186,21 @@ describe('provisionBrandSubworkspace', () => { expect(body.brandDomain).to.equal('acme.com'); expect(body.brandNames).to.deep.equal(['Acme']); // Brand-create attaches LLMs, generates+attaches prompts (each carrying the - // standard closed-dimension values + its branded/non-branded `type` value), - // then publishes best-effort. The dimension-root taxonomy is provisioned by - // createMarket itself, so it is not passed through here. - expect(options).to.deep.equal({ + // standard closed-dimension values, a branded/non-branded `type` value, and a + // server-classified `intent` value), then publishes best-effort. The + // dimension-root taxonomy is provisioned by createMarket itself, so it is not + // passed through here. writeDeadline is a request-scoped epoch-ms deadline + // (dynamic) — asserted as a number, then dropped before the deep-equal. + const { writeDeadline, ...restOptions } = options; + expect(writeDeadline).to.be.a('number'); + expect(restOptions).to.deep.equal({ modelIds: ['m-1', 'm-2'], generateTopics: true, topicCap: MAX_TOPICS_ON_CREATE, brandAliases: [], brandUrlSources: null, competitors: [], + env: { SEMRUSH_PROJECTS_BASE_URL: 'https://gw.example' }, publishMode: 'require', }); // The stub drives the sub-workspace title off the brand's name + id. @@ -204,6 +209,18 @@ describe('provisionBrandSubworkspace', () => { expect(brandStub.getSemrushSubWorkspaceId()).to.equal(undefined); }); + it('forwards a caller-supplied writeDeadline to the create handler (computed once at request entry, not defaulted here)', async () => { + const { provisionBrandSubworkspace } = await loadModule({ + resolveWorkspaceId, handleCreateMarketSubworkspace, + }); + // A deadline far in the future so it is unmistakably the passed-in value, + // not a fresh computeWriteDeadline() default (which would be ~now + 12s). + const writeDeadline = Date.now() + 999999; + await provisionBrandSubworkspace(buildContext(), { ...baseParams, writeDeadline }); + const options = handleCreateMarketSubworkspace.firstCall.args[7]; + expect(options.writeDeadline).to.equal(writeDeadline); + }); + it('resolves the IMS token via resolveSemrushImsToken and forwards it to createSerenityTransport (promise-token path)', async () => { const context = { ...buildContext(), diff --git a/test/support/serenity/dynamic-allocation-fronting.test.js b/test/support/serenity/dynamic-allocation-fronting.test.js index 59db8bab2..619963232 100644 --- a/test/support/serenity/dynamic-allocation-fronting.test.js +++ b/test/support/serenity/dynamic-allocation-fronting.test.js @@ -184,6 +184,8 @@ describe('dynamic-allocation fronting — create-prompts', () => { }, log, undefined, // classifyPromptType (tag-dimension path — not under test here) + undefined, // env + undefined, // writeDeadline { dynamicAllocation: true, parentWorkspaceId: MASTER }, ); expect(t.getWorkspaceResources).to.have.been.calledWith(WS); @@ -202,6 +204,8 @@ describe('dynamic-allocation fronting — create-prompts', () => { }, log, undefined, // classifyPromptType + undefined, // env + undefined, // writeDeadline { dynamicAllocation: false, parentWorkspaceId: MASTER }, ); expect(t.getWorkspaceResources).to.not.have.been.called; @@ -357,6 +361,8 @@ describe('dynamic-allocation — enforcement choke point', () => { }, log, undefined, // classifyPromptType + undefined, // env + undefined, // writeDeadline { dynamicAllocation: true, parentWorkspaceId: MASTER }, ), }, @@ -551,6 +557,8 @@ describe('dynamic-allocation fronting — retryOnQuota wiring', () => { }, log, undefined, + undefined, + undefined, { dynamicAllocation: true, parentWorkspaceId: MASTER }, ); // The retry succeeded, so the create is NOT recorded as a publish failure. @@ -699,7 +707,9 @@ describe('dynamic-allocation fronting — retryOnQuota wiring', () => { ], }, log, - undefined, + undefined, // classifyPromptType + undefined, // env + undefined, // writeDeadline { dynamicAllocation: true, parentWorkspaceId: MASTER }, ); expect(result.failed).to.deep.equal([]); @@ -741,7 +751,9 @@ describe('dynamic-allocation fronting — retryOnQuota wiring', () => { ], }, log, - undefined, + undefined, // classifyPromptType + undefined, // env + undefined, // writeDeadline { dynamicAllocation: true, parentWorkspaceId: MASTER }, ); expect(result.created).to.have.lengthOf(1); diff --git a/test/support/serenity/handlers/markets-subworkspace.test.js b/test/support/serenity/handlers/markets-subworkspace.test.js index 56d549c6a..59fbe4d85 100644 --- a/test/support/serenity/handlers/markets-subworkspace.test.js +++ b/test/support/serenity/handlers/markets-subworkspace.test.js @@ -1687,6 +1687,90 @@ describe('markets-subworkspace — defensive branch coverage', () => { expect(transport.createPromptsByIds).to.have.been.calledWithExactly(WS, 'new-proj', ['plain text'], [...STANDARD_IDS, TAG_IDS.typeNonBranded]); }); + // serenity-docs#32: a classified prompt gets its real intent id; a text absent + // from the classification map (e.g. beyond AI_GEN_CLASSIFY_MAX) falls back to + // the Informational default. Prompts are partitioned by their (type, intent) + // id pair, one upstream call per distinct pair. + it('generateAndAttachPrompts: applies the classified intent per prompt and defaults an unclassified one', async () => { + const SOURCE_AI_ID = TAG_IDS.originAi; + const handler = await esmock( + '../../../../src/support/serenity/handlers/markets-subworkspace.js', + { + '../../../../src/support/serenity/intent-classification.js': { + // 'buy now' classified Transactional; 'about it' omitted → default. + classifyPromptIntents: async () => new Map([['buy now', 'Transactional']]), + AI_GEN_CLASSIFY_MAX: 16, + computeWriteDeadline: () => Date.now() + 12000, + }, + }, + ); + const transport = makeTransport({ + getBrandTopics: sinon.stub().resolves([ + { topic: 'T', volume: 10, prompts: ['buy now', 'about it'] }, + ]), + }); + const res = await handler.handleCreateMarketSubworkspace( + transport, + makeBrand(), + PARENT, + { ...createBody, brandNames: ['Trail'] }, + log, + null, + null, + { generateTopics: true, publishMode: 'skip' }, + ); + expect(res.status).to.equal(201); + // Both are non-branded ('Trail' not mentioned); intent differs, so two calls. + expect(transport.createPromptsByIds).to.have.been.calledWithExactly(WS, 'new-proj', ['buy now'], [SOURCE_AI_ID, TAG_IDS.intentTransactional, TAG_IDS.typeNonBranded]); + expect(transport.createPromptsByIds).to.have.been.calledWithExactly(WS, 'new-proj', ['about it'], [SOURCE_AI_ID, TAG_IDS.intentInformational, TAG_IDS.typeNonBranded]); + }); + + // Boundary: at exactly AI_GEN_CLASSIFY_MAX + 1 texts, only the first MAX are + // sent to the classifier; the overflow text is never classified and takes the + // Informational default, and the cap-hit is logged (serenity-docs#32). + it('generateAndAttachPrompts: classifies only up to AI_GEN_CLASSIFY_MAX and logs the defaulted overflow', async () => { + const capLog = { info: sinon.spy(), error: sinon.spy(), warn: sinon.spy() }; + const classifySpy = sinon.stub().resolves(new Map([['p1', 'Transactional'], ['p2', 'Transactional']])); + const handler = await esmock( + '../../../../src/support/serenity/handlers/markets-subworkspace.js', + { + '../../../../src/support/serenity/intent-classification.js': { + classifyPromptIntents: classifySpy, + // Shrink the cap to 2 so a 3-text batch exercises the boundary. + AI_GEN_CLASSIFY_MAX: 2, + computeWriteDeadline: () => Date.now() + 12000, + }, + }, + ); + const transport = makeTransport({ + getBrandTopics: sinon.stub().resolves([ + { topic: 'T', volume: 10, prompts: ['p1', 'p2', 'p3'] }, + ]), + }); + const res = await handler.handleCreateMarketSubworkspace( + transport, + makeBrand(), + PARENT, + { ...createBody, brandNames: ['Trail'] }, + capLog, + null, + null, + { generateTopics: true, publishMode: 'skip' }, + ); + expect(res.status).to.equal(201); + // Only the first 2 texts were handed to the classifier — the slice bound. + expect(classifySpy).to.have.been.calledOnce; + expect(classifySpy.firstCall.args[0]).to.deep.equal(['p1', 'p2']); + // p1/p2 classified Transactional; p3 (beyond the cap) defaults Informational. + expect(transport.createPromptsByIds).to.have.been.calledWithExactly(WS, 'new-proj', ['p1', 'p2'], [...STANDARD_IDS.slice(0, 1), TAG_IDS.intentTransactional, TAG_IDS.typeNonBranded]); + expect(transport.createPromptsByIds).to.have.been.calledWithExactly(WS, 'new-proj', ['p3'], [...STANDARD_IDS.slice(0, 1), TAG_IDS.intentInformational, TAG_IDS.typeNonBranded]); + // The cap-hit is observable, not silent. + expect(capLog.info).to.have.been.calledWithMatch( + 'generateAndAttachPrompts: AI-gen classify cap hit — tail defaults to Informational', + sinon.match({ total: 3, classified: 2, defaultedByCap: 1 }), + ); + }); + // The two guards below both fire when the tag tree, freshly provisioned, still // does not contain a name we asked for. That is not hypothetical: upstream tag // writes land in the project's DRAFT layer while a default read serves the LIVE diff --git a/test/support/serenity/handlers/prompts-subworkspace.test.js b/test/support/serenity/handlers/prompts-subworkspace.test.js index 0f570c545..e35e864a7 100644 --- a/test/support/serenity/handlers/prompts-subworkspace.test.js +++ b/test/support/serenity/handlers/prompts-subworkspace.test.js @@ -22,6 +22,7 @@ import { handleBulkDeletePromptsSubworkspace, } from '../../../../src/support/serenity/handlers/prompts-subworkspace.js'; import { SerenityTransportError } from '../../../../src/support/serenity/rest-transport.js'; +import { ErrorWithStatusCode } from '../../../../src/support/utils.js'; import { TAG_IDS, makeListProjectTagsStub } from '../fixtures/tag-tree.js'; use(chaiAsPromised); @@ -155,9 +156,34 @@ describe('prompts-subworkspace handlers', () => { expect(result.created).to.have.length(1); expect(result.created[0]).to.include({ semrushPromptId: 'new-prompt', geoTargetId: 2840 }); // A create is a user-authenticated write: the derived `origin` (`human`) is - // stamped alongside the caller's tags (origin-dimension.md §3). - expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly(WS, 'p-us-en', ['p'], ['tag-1', TAG_IDS.originHuman]); + // stamped (origin-dimension.md §3) and intent defaults to Informational (Azure + // unconfigured, serenity-docs#32), alongside the caller's tag. + expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly(WS, 'p-us-en', ['p'], ['tag-1', TAG_IDS.originHuman, TAG_IDS.intentInformational]); expect(transport.publishProject).to.have.been.calledOnceWith(WS, 'p-us-en'); + expect(result.published).to.equal(true); + }); + + it('skips the trailing publish and reports published:false when deferPublish is true', async () => { + const transport = makeTransport(); + const result = await handleCreatePromptsSubworkspace(transport, WS, { + deferPublish: true, + prompts: [{ + text: 'p', tagIds: ['tag-1'], geoTargetId: 2840, languageCode: 'en', + }], + }, log); + expect(result.published).to.equal(false); + expect(result.created).to.have.length(1); + expect(transport.publishProject).to.not.have.been.called; + }); + + it('400s when deferPublish is present but not a boolean', async () => { + const transport = makeTransport(); + await expect(handleCreatePromptsSubworkspace(transport, WS, { + deferPublish: 'yes', + prompts: [{ + text: 'p', tagIds: ['tag-1'], geoTargetId: 2840, languageCode: 'en', + }], + }, log)).to.be.rejectedWith(ErrorWithStatusCode, /deferPublish must be a boolean/); }); it('dynamic-allocation ON: fronts headroom sized on the batch BEFORE the write, not just before publish (LLMO-6190, live-verified)', async () => { @@ -179,7 +205,9 @@ describe('prompts-subworkspace handlers', () => { prompts: [{ text: 'p', tagIds: ['tag-1'], geoTargetId: 2840, languageCode: 'en', }], - }, log, undefined, { dynamicAllocation: true, parentWorkspaceId: 'parent-ws' }); + }, log, undefined, undefined, undefined, { + dynamicAllocation: true, parentWorkspaceId: 'parent-ws', + }); expect(result.created).to.have.length(1); expect(transport.getWorkspaceResources).to.have.been.calledOnceWith(WS); expect(transport.getWorkspaceResources) @@ -213,12 +241,16 @@ describe('prompts-subworkspace handlers', () => { }, log, classifyByBrandMention); expect(result.created[0].tagIds).to.deep.equal([ TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, TAG_IDS.originHuman, + TAG_IDS.intentInformational, ]); expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly( WS, 'p-us-en', ['is Acme good?'], - [TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, TAG_IDS.originHuman], + [ + TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, TAG_IDS.originHuman, + TAG_IDS.intentInformational, + ], ); }); @@ -316,7 +348,7 @@ describe('prompts-subworkspace handlers', () => { expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( WS, 'p-us-en', - [{ id: 'old-id', references: ['tag-1'], replace: true }], + [{ id: 'old-id', references: ['tag-1', TAG_IDS.intentInformational], replace: true }], ); expect(transport.deletePromptsByIds).to.not.have.been.called; expect(transport.createPromptsByIds).to.not.have.been.called; @@ -375,13 +407,14 @@ describe('prompts-subworkspace handlers', () => { text: 'new', tagIds: ['tag-cat-1', '', undefined], geoTargetId: 2840, languageCode: 'en', }, log); expect(result.status).to.equal(200); + // The id is preserved — the edit is in place, never a re-create. expect(result.body.semrushPromptId).to.equal('old-id'); - expect(result.body.tagIds).to.deep.equal(['tag-cat-1']); + expect(result.body.tagIds).to.deep.equal(['tag-cat-1', TAG_IDS.intentInformational]); expect(transport.renamePrompt).to.have.been.calledOnceWithExactly(WS, 'p-us-en', 'old-id', 'new'); expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( WS, 'p-us-en', - [{ id: 'old-id', references: ['tag-cat-1'], replace: true }], + [{ id: 'old-id', references: ['tag-cat-1', TAG_IDS.intentInformational], replace: true }], ); }); @@ -412,14 +445,18 @@ describe('prompts-subworkspace handlers', () => { }, log, classifyByBrandMention); expect(result.status).to.equal(200); expect(result.body.tagIds).to.deep.equal([ - TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, + TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, TAG_IDS.intentInformational, ]); expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( WS, 'p-us-en', [{ id: 'old-id', - references: [TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded], + references: [ + TAG_IDS.categoryRunningShoes, + TAG_IDS.typeBranded, + TAG_IDS.intentInformational, + ], replace: true, }], ); @@ -742,6 +779,6 @@ describe('prompts-subworkspace — defensive branch coverage', () => { languageCode: 'en', }, log); expect(result.status).to.equal(200); - expect(result.body.tagIds).to.deep.equal(['keep']); + expect(result.body.tagIds).to.deep.equal(['keep', TAG_IDS.intentInformational]); }); }); diff --git a/test/support/serenity/handlers/prompts.test.js b/test/support/serenity/handlers/prompts.test.js index 3803c89ad..1928cf518 100644 --- a/test/support/serenity/handlers/prompts.test.js +++ b/test/support/serenity/handlers/prompts.test.js @@ -14,6 +14,7 @@ import { use, expect } from 'chai'; import chaiAsPromised from 'chai-as-promised'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; +import esmock from 'esmock'; import { handleListPrompts, @@ -21,6 +22,8 @@ import { handleUpdatePrompt, handleBulkDeletePrompts, makePromptTagInjector, + makeIntentInjector, + validateDeferPublish, } from '../../../../src/support/serenity/handlers/prompts.js'; import { ErrorWithStatusCode } from '../../../../src/support/utils.js'; import { SerenityTransportError } from '../../../../src/support/serenity/rest-transport.js'; @@ -576,9 +579,14 @@ describe('handlers/prompts.js — handleCreatePrompts', () => { geoTargetId: 2840, languageCode: 'en', text: 'hello', - tagIds: ['tag-cat-1', 'tag-child-1', TAG_IDS.originHuman], + tagIds: ['tag-cat-1', 'tag-child-1', TAG_IDS.originHuman, TAG_IDS.intentInformational], }); - expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en', ['hello'], ['tag-cat-1', 'tag-child-1', TAG_IDS.originHuman]); + expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly( + WORKSPACE, + 'proj-us-en', + ['hello'], + ['tag-cat-1', 'tag-child-1', TAG_IDS.originHuman, TAG_IDS.intentInformational], + ); expect(transport.publishProject).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en'); }); @@ -662,8 +670,8 @@ describe('handlers/prompts.js — handleCreatePrompts', () => { }, fakeLog()); expect(result.created[0].semrushPromptId).to.equal(''); - expect(result.created[0].tagIds).to.deep.equal(['keep', TAG_IDS.originHuman]); - expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en', ['hello'], ['keep', TAG_IDS.originHuman]); + expect(result.created[0].tagIds).to.deep.equal(['keep', TAG_IDS.originHuman, TAG_IDS.intentInformational]); + expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en', ['hello'], ['keep', TAG_IDS.originHuman, TAG_IDS.intentInformational]); }); it('returns empty semrushPromptId (not the string "undefined") when createPromptsByIds returns an item with no id', async () => { @@ -711,8 +719,8 @@ describe('handlers/prompts.js — handleCreatePrompts', () => { }], }, fakeLog()); - expect(result.created[0].tagIds).to.deep.equal(['keep', TAG_IDS.originHuman]); - expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en', ['hello'], ['keep', TAG_IDS.originHuman]); + expect(result.created[0].tagIds).to.deep.equal(['keep', TAG_IDS.originHuman, TAG_IDS.intentInformational]); + expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en', ['hello'], ['keep', TAG_IDS.originHuman, TAG_IDS.intentInformational]); }); it('caps a bulk-create tagIds array at MAX_TAG_IDS (50), mirroring the list-read query cap', async () => { @@ -736,10 +744,12 @@ describe('handlers/prompts.js — handleCreatePrompts', () => { }, fakeLog()); // The MAX_TAG_IDS cap bounds the CALLER's tags (50); the server-derived - // `origin` is injected on top, so the stored set is the 50 capped tags plus - // the derived origin id. - expect(result.created[0].tagIds).to.have.lengthOf(51); - expect(result.created[0].tagIds).to.deep.equal([...tooMany.slice(0, 50), TAG_IDS.originHuman]); + // `origin` (human) and classified `intent` (Informational) are injected on + // top, so the stored set is the 50 capped tags plus those two computed ids. + expect(result.created[0].tagIds).to.have.lengthOf(52); + expect(result.created[0].tagIds).to.deep.equal( + [...tooMany.slice(0, 50), TAG_IDS.originHuman, TAG_IDS.intentInformational], + ); }); it('skips a create row when tagIds sanitizes to empty (every entry malformed)', async () => { @@ -1006,6 +1016,49 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { expect(result.body.error).to.equal('invalidRequest'); }); + // Mirror of the create-path contract (normalizePromptInput trims + rejects + // empty): an edit must not slip an empty prompt past validation into the + // rename, where it would be classified and written blank. + it('400s when text is an empty string', async () => { + const dataAccess = makeDataAccess([]); + + const result = await handleUpdatePrompt( + {}, + dataAccess, + BRAND, + WORKSPACE, + 'sem-1', + { + geoTargetId: 2840, languageCode: 'en', text: '', tagIds: ['tag-1'], + }, + fakeLog(), + ); + + expect(result.status).to.equal(400); + expect(result.body.error).to.equal('invalidRequest'); + expect(result.body.message).to.match(/non-empty/); + }); + + it('400s when text is whitespace-only', async () => { + const dataAccess = makeDataAccess([]); + + const result = await handleUpdatePrompt( + {}, + dataAccess, + BRAND, + WORKSPACE, + 'sem-1', + { + geoTargetId: 2840, languageCode: 'en', text: ' ', tagIds: ['tag-1'], + }, + fakeLog(), + ); + + expect(result.status).to.equal(400); + expect(result.body.error).to.equal('invalidRequest'); + expect(result.body.message).to.match(/non-empty/); + }); + it('edits text+tagIds in place (rename + replace-mode tag write), id unchanged', async () => { const project = makeProject({ semrushProjectId: 'proj-us-en', geoTargetId: 2840, languageCode: 'en', @@ -1014,10 +1067,9 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { dataAccess.BrandSemrushProject.findBySlice.resolves(project); const transport = { + listProjectTags: makeListProjectTagsStub(), renamePrompt: sinon.stub().resolves({ id: 'sem-1', name: 'next', is_updated: true }), updatePromptTagsByIds: sinon.stub().resolves(null), - deletePromptsByIds: sinon.stub(), - createPromptsByIds: sinon.stub(), publishProject: sinon.stub().resolves(), }; @@ -1040,18 +1092,55 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { geoTargetId: 2840, languageCode: 'en', text: 'next', - tagIds: ['tag-cat-1'], + tagIds: ['tag-cat-1', TAG_IDS.intentInformational], }); expect(transport.renamePrompt).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en', 'sem-1', 'next'); + // Replace-mode tag write with the injector's full output (caller tag + intent). expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( WORKSPACE, 'proj-us-en', - [{ id: 'sem-1', references: ['tag-cat-1'], replace: true }], + [{ id: 'sem-1', references: ['tag-cat-1', TAG_IDS.intentInformational], replace: true }], ); - // Nothing is deleted or created anywhere on the edit path. - expect(transport.deletePromptsByIds).to.have.callCount(0); - expect(transport.createPromptsByIds).to.have.callCount(0); - expect(transport.publishProject).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en'); + }); + + // Guards the documented always-reclassify invariant: an unchanged-text edit + // comes back from rename as `is_updated: false`, but the handler still writes + // the (re-classified) tag set and publishes — it has no old text and upstream + // has no GET-by-id, so it cannot short-circuit. A future "optimization" that + // skips the tag write on is_updated:false would break this and fail here. + it('still writes tags + publishes when rename reports is_updated:false (no-op text)', async () => { + const project = makeProject({ + semrushProjectId: 'proj-us-en', geoTargetId: 2840, languageCode: 'en', + }); + const dataAccess = makeDataAccess([]); + dataAccess.BrandSemrushProject.findBySlice.resolves(project); + + const transport = { + listProjectTags: makeListProjectTagsStub(), + renamePrompt: sinon.stub().resolves({ id: 'sem-1', name: 'same', is_updated: false }), + updatePromptTagsByIds: sinon.stub().resolves(null), + publishProject: sinon.stub().resolves(), + }; + + const result = await handleUpdatePrompt( + transport, + dataAccess, + BRAND, + WORKSPACE, + 'sem-1', + { + geoTargetId: 2840, languageCode: 'en', text: 'same', tagIds: ['tag-cat-1'], + }, + fakeLog(), + ); + + expect(result.status).to.equal(200); + expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( + WORKSPACE, + 'proj-us-en', + [{ id: 'sem-1', references: ['tag-cat-1', TAG_IDS.intentInformational], replace: true }], + ); + expect(transport.publishProject).to.have.been.called; }); it('drops falsy tagIds entries on PATCH before the tag write', async () => { @@ -1062,6 +1151,7 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { dataAccess.BrandSemrushProject.findBySlice.resolves(project); const transport = { + listProjectTags: makeListProjectTagsStub(), renamePrompt: sinon.stub().resolves({ id: 'sem-1', name: 'next', is_updated: true }), updatePromptTagsByIds: sinon.stub().resolves(null), publishProject: sinon.stub().resolves(), @@ -1079,11 +1169,11 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { fakeLog(), ); - expect(result.body.tagIds).to.deep.equal(['keep']); + expect(result.body.tagIds).to.deep.equal(['keep', TAG_IDS.intentInformational]); expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( WORKSPACE, 'proj-us-en', - [{ id: 'sem-1', references: ['keep'], replace: true }], + [{ id: 'sem-1', references: ['keep', TAG_IDS.intentInformational], replace: true }], ); }); @@ -1095,6 +1185,7 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { dataAccess.BrandSemrushProject.findBySlice.resolves(project); const transport = { + listProjectTags: makeListProjectTagsStub(), renamePrompt: sinon.stub().resolves({ id: 'sem-1', name: 'next', is_updated: true }), updatePromptTagsByIds: sinon.stub().resolves(null), publishProject: sinon.stub().resolves(), @@ -1116,11 +1207,11 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { fakeLog(), ); - expect(result.body.tagIds).to.deep.equal(['keep']); + expect(result.body.tagIds).to.deep.equal(['keep', TAG_IDS.intentInformational]); expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( WORKSPACE, 'proj-us-en', - [{ id: 'sem-1', references: ['keep'], replace: true }], + [{ id: 'sem-1', references: ['keep', TAG_IDS.intentInformational], replace: true }], ); }); @@ -1190,7 +1281,8 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { dataAccess.BrandSemrushProject.findBySlice.resolves(project); const transport = { - // listPromptsByTags is not called from PATCH — wiring it as a + listProjectTags: makeListProjectTagsStub(), + // listPromptsByTags is no longer called from PATCH — wiring it as a // stub lets us assert callCount(0) so a regression that brings the // walk back fails this test. listPromptsByTags: sinon.stub(), @@ -1217,7 +1309,7 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { geoTargetId: 2840, languageCode: 'en', text: 'new text', - tagIds: ['tag-fresh'], + tagIds: ['tag-fresh', TAG_IDS.intentInformational], }); expect(transport.listPromptsByTags).to.have.callCount(0); expect(transport.renamePrompt).to.have.been.calledOnceWithExactly(WORKSPACE, 'proj-us-en', 'sem-1', 'new text'); @@ -1230,11 +1322,12 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { const dataAccess = makeDataAccess([]); dataAccess.BrandSemrushProject.findBySlice.resolves(project); - // The prompt-gone 404 is gated by isUpstreamGone which requires - // SerenityTransportError specifically. A generic Error with .status=404 - // must NOT trip the promptNotFound path. + // The prompt-gone 404 is gated by isUpstreamGone, which requires a + // SerenityTransportError. A rename that reports the prompt is gone maps to + // promptNotFound and never reaches the tag write. const err = new SerenityTransportError(404, 'not found'); const transport = { + listProjectTags: makeListProjectTagsStub(), renamePrompt: sinon.stub().rejects(err), updatePromptTagsByIds: sinon.stub(), }; @@ -1256,6 +1349,37 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { expect(transport.updatePromptTagsByIds).to.have.callCount(0); }); + // The negative of the case above: isUpstreamGone requires a + // SerenityTransportError, so a generic Error carrying .status=404 must NOT be + // treated as promptNotFound — it propagates like any other upstream failure. + it('generic Error with status 404 → throws (not promptNotFound)', async () => { + const project = makeProject({ + semrushProjectId: 'proj-us-en', geoTargetId: 2840, languageCode: 'en', + }); + const dataAccess = makeDataAccess([]); + dataAccess.BrandSemrushProject.findBySlice.resolves(project); + + const err = Object.assign(new Error('plain 404'), { status: 404 }); + const transport = { + listProjectTags: makeListProjectTagsStub(), + renamePrompt: sinon.stub().rejects(err), + updatePromptTagsByIds: sinon.stub(), + }; + + await expect(handleUpdatePrompt( + transport, + dataAccess, + BRAND, + WORKSPACE, + 'sem-1', + { + geoTargetId: 2840, languageCode: 'en', text: 'x', tagIds: ['tag-1'], + }, + fakeLog(), + )).to.be.rejectedWith(/plain 404/); + expect(transport.updatePromptTagsByIds).to.have.callCount(0); + }); + // The collision contract (serenity-docs#63 decision 2): a rename onto a // sibling prompt's exact text is refused upstream with 409 and NOTHING has // mutated — the handler propagates it untouched so the controller's mapError @@ -1269,9 +1393,10 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { const err = new SerenityTransportError(409, 'conflict'); const transport = { + listProjectTags: makeListProjectTagsStub(), renamePrompt: sinon.stub().rejects(err), - updatePromptTagsByIds: sinon.stub(), - publishProject: sinon.stub(), + updatePromptTagsByIds: sinon.stub().resolves(null), + publishProject: sinon.stub().resolves(), }; await expect(handleUpdatePrompt( @@ -1298,8 +1423,9 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { const err = Object.assign(new Error('upstream 503'), { status: 503 }); const transport = { + listProjectTags: makeListProjectTagsStub(), renamePrompt: sinon.stub().rejects(err), - updatePromptTagsByIds: sinon.stub(), + updatePromptTagsByIds: sinon.stub().resolves(null), }; await expect(handleUpdatePrompt( @@ -1328,6 +1454,7 @@ describe('handlers/prompts.js — handleUpdatePrompt', () => { const tagErr = Object.assign(new Error('tag write boom'), { status: 500 }); const transport = { + listProjectTags: makeListProjectTagsStub(), renamePrompt: sinon.stub().resolves({ id: 'sem-1', name: 'x', is_updated: true }), updatePromptTagsByIds: sinon.stub().rejects(tagErr), publishProject: sinon.stub().resolves(), @@ -1409,6 +1536,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { const dataAccess = makeDataAccess([project]); const err = new Error('opaque failure'); // no .status const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub().rejects(err), }; @@ -1439,6 +1567,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { // SerenityTransportError(404) — isUpstreamGone strict-match. const err = new SerenityTransportError(404, 'not found'); const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub().rejects(err), publishProject: sinon.stub().resolves(), }; @@ -1462,6 +1591,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { }); const dataAccess = makeDataAccess([project]); const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub().resolves(), publishProject: sinon.stub().resolves(), }; @@ -1485,6 +1615,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { }); const dataAccess = makeDataAccess([project]); const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub(), publishProject: sinon.stub(), }; @@ -1511,6 +1642,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { const dataAccess = makeDataAccess([project]); const err = Object.assign(new Error('upstream 503'), { status: 503 }); const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub().rejects(err), publishProject: sinon.stub().resolves(), }; @@ -1538,6 +1670,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { }); const dataAccess = makeDataAccess([project]); const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub().resolves(), publishProject: sinon.stub().rejects(new Error('publish boom')), }; @@ -1572,6 +1705,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { }); const dataAccess = makeDataAccess([projectA, projectB]); const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub(), publishProject: sinon.stub().resolves(), }; @@ -1607,6 +1741,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { }); const dataAccess = makeDataAccess([projectA, projectB]); const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub(), publishProject: sinon.stub().resolves(), }; @@ -1633,6 +1768,7 @@ describe('handlers/prompts.js — handleBulkDeletePrompts', () => { }); const dataAccess = makeDataAccess([project]); const transport = { + listProjectTags: makeListProjectTagsStub(), deletePromptsByIds: sinon.stub().resolves(), publishProject: sinon.stub().resolves(), }; @@ -1692,7 +1828,7 @@ describe('handlers/prompts.js — tag cache invalidation (Important #6)', () => }); // Step 1: populate cache via handleListTags with set A. - const transport = { listPromptsByTags }; + const transport = { listPromptsByTags, listProjectTags: makeListProjectTagsStub() }; await handleListTags(transport, dataAccess, BRAND, WORKSPACE, { geoTargetId: 2840, languageCode: 'en', }, fakeLog()); @@ -1849,12 +1985,16 @@ describe('handlers/prompts.js — unified type classification (serenity-docs#31) expect(result.created[0].tagIds).to.deep.equal([ TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, TAG_IDS.originHuman, + TAG_IDS.intentInformational, ]); expect(transport.createPromptsByIds).to.have.been.calledOnceWithExactly( WORKSPACE, 'proj-us-en', ['is Acme good?'], - [TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, TAG_IDS.originHuman], + [ + TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, TAG_IDS.originHuman, + TAG_IDS.intentInformational, + ], ); // The whole taxonomy already exists, so nothing is provisioned. expect(transport.createProjectTags).to.not.have.been.called; @@ -1881,6 +2021,7 @@ describe('handlers/prompts.js — unified type classification (serenity-docs#31) expect(result.created[0].tagIds).to.deep.equal([ TAG_IDS.categoryRunningShoes, TAG_IDS.typeNonBranded, TAG_IDS.originHuman, + TAG_IDS.intentInformational, ]); }); @@ -1920,6 +2061,7 @@ describe('handlers/prompts.js — unified type classification (serenity-docs#31) expect(result.created[0].tagIds).to.deep.equal([ decoyCategoryId, TAG_IDS.typeBranded, TAG_IDS.originHuman, + TAG_IDS.intentInformational, ]); }); @@ -1952,10 +2094,16 @@ describe('handlers/prompts.js — unified type classification (serenity-docs#31) expect(createProjectTags.firstCall.args[3]).to.deep.equal({}); expect(createProjectTags.secondCall.args[2]).to.deep.equal(['branded']); expect(createProjectTags.secondCall.args[3]).to.deep.equal({ parentId: 'created::type' }); + // origin injection (the second half of the unified type+origin injector) mints + // `human` beneath the freshly-created `origin` root... expect(createProjectTags.thirdCall.args[2]).to.deep.equal(['human']); expect(createProjectTags.thirdCall.args[3]).to.deep.equal({ parentId: 'created::origin' }); + // ...then intent injection mints the default `Informational` beneath `intent`. + expect(createProjectTags.getCall(3).args[2]).to.deep.equal(['Informational']); + expect(createProjectTags.getCall(3).args[3]).to.deep.equal({ parentId: 'created::intent' }); expect(result.created[0].tagIds).to.deep.equal([ 'tag-cat-1', 'created:created::type:branded', 'created:created::origin:human', + 'created:created::intent:Informational', ]); }); }); @@ -1980,7 +2128,7 @@ describe('handlers/prompts.js — unified type classification (serenity-docs#31) expect(result.status).to.equal(200); expect(result.body.tagIds).to.deep.equal([ - TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, + TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded, TAG_IDS.intentInformational, ]); // The injector's output is the full replacement set the tag write sends. expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( @@ -1988,7 +2136,11 @@ describe('handlers/prompts.js — unified type classification (serenity-docs#31) 'proj-us-en', [{ id: 'old-id', - references: [TAG_IDS.categoryRunningShoes, TAG_IDS.typeBranded], + references: [ + TAG_IDS.categoryRunningShoes, + TAG_IDS.typeBranded, + TAG_IDS.intentInformational, + ], replace: true, }], ); @@ -2056,6 +2208,209 @@ describe('handlers/prompts.js — unified type classification (serenity-docs#31) }); }); +describe('handlers/prompts.js — unified intent classification (serenity-docs#32)', () => { + it('resolves the classified intent to a tag id and strips a caller-supplied intent id', async () => { + const transport = { + listProjectTags: makeListProjectTagsStub(), + createProjectTags: sinon.stub(), + }; + const intentByText = new Map([['love it', 'Task']]); + const inject = makeIntentInjector(transport, WORKSPACE, intentByText, fakeLog()); + + // The caller supplies a stale `intent` id; it is stripped and replaced with + // the server-classified value's id. + const out = await inject('proj-1', { + text: 'love it', + geoTargetId: 2840, + tagIds: [TAG_IDS.categoryRunningShoes, TAG_IDS.intentInformational], + }); + + expect(out.tagIds).to.deep.equal([TAG_IDS.categoryRunningShoes, TAG_IDS.intentTask]); + expect(transport.createProjectTags).to.not.have.been.called; + }); + + it('falls back to the Informational default when a text is absent from the classification map', async () => { + const transport = { + listProjectTags: makeListProjectTagsStub(), + createProjectTags: sinon.stub(), + }; + // Empty map (e.g. a text beyond the AI-gen classify cap) → default. + const inject = makeIntentInjector(transport, WORKSPACE, new Map(), fakeLog()); + + const out = await inject('proj-1', { text: 'unclassified', geoTargetId: 2840, tagIds: ['x'] }); + + expect(out.tagIds).to.deep.equal(['x', TAG_IDS.intentInformational]); + }); + + it('resolves each (project, intent) once across a batch, re-resolving only on a new key', async () => { + const transport = { + listProjectTags: makeListProjectTagsStub(), + createProjectTags: sinon.stub(), + }; + const intentByText = new Map([['a', 'Task'], ['b', 'Task'], ['c', 'Commercial']]); + const inject = makeIntentInjector(transport, WORKSPACE, intentByText, fakeLog()); + + // Resolving one (project, intent) key reads two levels: the roots, then the + // children of the `intent` root. + const a = await inject('proj-1', { text: 'a', geoTargetId: 2840, tagIds: ['x'] }); + expect(transport.listProjectTags).to.have.callCount(2); + + // Same project + same computed intent => served from cache, no new reads. + const b = await inject('proj-1', { text: 'b', geoTargetId: 2840, tagIds: ['y'] }); + expect(transport.listProjectTags).to.have.callCount(2); + expect(a.tagIds).to.deep.equal(['x', TAG_IDS.intentTask]); + expect(b.tagIds).to.deep.equal(['y', TAG_IDS.intentTask]); + + // A different computed intent is a new cache key => one more resolution. + const c = await inject('proj-1', { text: 'c', geoTargetId: 2840, tagIds: ['z'] }); + expect(transport.listProjectTags).to.have.callCount(4); + expect(c.tagIds).to.deep.equal(['z', TAG_IDS.intentCommercial]); + }); + + // Strip-by-root, not strip-by-name (the intent twin of the `type` collision + // gate): a customer category may legitimately be named `Commercial` without + // being the `intent` value. Only ids beneath the `intent` root are the + // server's to overwrite, even when the computed intent shares that bare name. + it('leaves a same-named category tag alone while stripping the real intent value', async () => { + const decoyCategoryId = 'category-commercial-decoy'; + const levels = dimensionTreeLevels(); + levels[TAG_IDS.categoryRoot] = [ + ...levels[TAG_IDS.categoryRoot], + { + id: decoyCategoryId, + name: 'Commercial', + parent_id: TAG_IDS.categoryRoot, + children_count: 0, + path: [{ id: TAG_IDS.categoryRoot, name: 'category' }], + }, + ]; + const transport = { + listProjectTags: makeListProjectTagsStub(levels), + createProjectTags: sinon.stub(), + }; + // The server classifies this text's intent as `Commercial`. + const intentByText = new Map([['is Acme worth it?', 'Commercial']]); + const inject = makeIntentInjector(transport, WORKSPACE, intentByText, fakeLog()); + + // The caller supplies the decoy CATEGORY named `Commercial` plus a stale + // `intent` id; only the intent-root id is stripped and replaced. + const out = await inject('proj-1', { + text: 'is Acme worth it?', + geoTargetId: 2840, + tagIds: [decoyCategoryId, TAG_IDS.intentInformational], + }); + + // The `Commercial` CATEGORY survives; the intent-root id is stripped and the + // computed `Commercial` intent id injected. + expect(out.tagIds).to.deep.equal([decoyCategoryId, TAG_IDS.intentCommercial]); + expect(transport.createProjectTags).to.not.have.been.called; + }); +}); + +describe('handlers/prompts.js — deferPublish (serenity-docs#32 CSV-chunking)', () => { + const project = () => makeProject({ + semrushProjectId: 'proj-us-en', geoTargetId: 2840, languageCode: 'en', + }); + + it('validateDeferPublish resolves absent/false/true and rejects a non-boolean', () => { + expect(validateDeferPublish({})).to.equal(false); + expect(validateDeferPublish({ deferPublish: false })).to.equal(false); + expect(validateDeferPublish({ deferPublish: true })).to.equal(true); + expect(() => validateDeferPublish({ deferPublish: 'yes' })) + .to.throw(ErrorWithStatusCode, /deferPublish must be a boolean/); + }); + + it('skips the trailing publish and reports published:false when deferPublish is true', async () => { + const dataAccess = makeDataAccess([project()]); + const transport = { + listProjectTags: makeListProjectTagsStub(), + createPromptsByIds: sinon.stub().resolves({ + page: 1, total: 1, items: [{ id: 'new-sem-id', name: 'hi' }], + }), + publishProject: sinon.stub().resolves(), + }; + + const result = await handleCreatePrompts(transport, dataAccess, BRAND, WORKSPACE, { + deferPublish: true, + prompts: [{ + text: 'hi', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-cat-1'], + }], + }, fakeLog()); + + expect(result.published).to.equal(false); + expect(result.created).to.have.lengthOf(1); + expect(transport.publishProject).to.not.have.been.called; + }); + + it('publishes and reports published:true when deferPublish is absent', async () => { + const dataAccess = makeDataAccess([project()]); + const transport = { + listProjectTags: makeListProjectTagsStub(), + createPromptsByIds: sinon.stub().resolves({ + page: 1, total: 1, items: [{ id: 'new-sem-id', name: 'hi' }], + }), + publishProject: sinon.stub().resolves(), + }; + + const result = await handleCreatePrompts(transport, dataAccess, BRAND, WORKSPACE, { + prompts: [{ + text: 'hi', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-cat-1'], + }], + }, fakeLog()); + + expect(result.published).to.equal(true); + expect(transport.publishProject).to.have.been.calledOnce; + }); + + it('400s when deferPublish is present but not a boolean', async () => { + const dataAccess = makeDataAccess([project()]); + const transport = { listProjectTags: makeListProjectTagsStub() }; + await expect(handleCreatePrompts(transport, dataAccess, BRAND, WORKSPACE, { + deferPublish: 1, + prompts: [{ + text: 'hi', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-cat-1'], + }], + }, fakeLog())).to.be.rejectedWith(ErrorWithStatusCode, /deferPublish must be a boolean/); + }); + + it('reports published:false with a mixed created/failed batch (deferPublish, partial failure)', async () => { + const dataAccess = makeDataAccess([project()]); + // One prompt writes cleanly; the other 500s upstream. deferPublish is set, so + // neither the success nor the failure triggers a publish. + const transport = { + listProjectTags: makeListProjectTagsStub(), + createPromptsByIds: sinon.stub().callsFake(async (_ws, _pid, texts) => { + if (texts[0] === 'boom') { + throw Object.assign(new Error('upstream failure'), { status: 500 }); + } + return { page: 1, total: 1, items: [{ id: `new-${texts[0]}`, name: texts[0] }] }; + }), + publishProject: sinon.stub().resolves(), + }; + + const result = await handleCreatePrompts(transport, dataAccess, BRAND, WORKSPACE, { + deferPublish: true, + prompts: [ + { + text: 'ok', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-cat-1'], + }, + { + text: 'boom', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-cat-1'], + }, + ], + }, fakeLog()); + + expect(result.published).to.equal(false); + expect(result.created).to.have.lengthOf(1); + expect(result.created[0].text).to.equal('ok'); + expect(result.failed).to.have.lengthOf(1); + expect(result.failed[0].text).to.equal('boom'); + expect(result.failed[0].status).to.equal(500); + // deferPublish means no publish regardless of the partial failure. + expect(transport.publishProject).to.not.have.been.called; + }); +}); + // origin-dimension.md §3 (LLMO-6275): `origin` is derived from the write path, // never asserted by the user, and carries a create/update asymmetry — CREATE // injects the derived value; UPDATE never re-derives, preserving the stored one. @@ -2090,6 +2445,7 @@ describe('handlers/prompts.js — origin derivation (origin-dimension.md §3)', expect(result.created[0].tagIds).to.deep.equal([ TAG_IDS.categoryRunningShoes, TAG_IDS.typeNonBranded, TAG_IDS.originHuman, + TAG_IDS.intentInformational, ]); expect(result.created[0].tagIds).to.not.include(TAG_IDS.originAi); }); @@ -2132,6 +2488,7 @@ describe('handlers/prompts.js — origin derivation (origin-dimension.md §3)', // derived origin is injected. expect(result.created[0].tagIds).to.deep.equal([ decoyAiCategoryId, TAG_IDS.typeBranded, TAG_IDS.originHuman, + TAG_IDS.intentInformational, ]); }); @@ -2158,6 +2515,7 @@ describe('handlers/prompts.js — origin derivation (origin-dimension.md §3)', expect(result.status).to.equal(200); expect(result.body.tagIds).to.deep.equal([ TAG_IDS.categoryRunningShoes, TAG_IDS.originAi, TAG_IDS.typeBranded, + TAG_IDS.intentInformational, ]); expect(result.body.tagIds).to.not.include(TAG_IDS.originHuman); expect(transport.updatePromptTagsByIds).to.have.been.calledOnceWithExactly( @@ -2165,7 +2523,10 @@ describe('handlers/prompts.js — origin derivation (origin-dimension.md §3)', 'proj-us-en', [{ id: 'ai-prompt', - references: [TAG_IDS.categoryRunningShoes, TAG_IDS.originAi, TAG_IDS.typeBranded], + references: [ + TAG_IDS.categoryRunningShoes, TAG_IDS.originAi, TAG_IDS.typeBranded, + TAG_IDS.intentInformational, + ], replace: true, }], ); @@ -2216,3 +2577,50 @@ describe('handlers/prompts.js — origin derivation (origin-dimension.md §3)', }); }); }); + +// Regression (Alicia review, serenity-docs#32): the create paths must classify the +// TRIMMED text. `makeIntentInjector` looks up the classification map by `input.text`, +// which `normalizePromptInput` has already trimmed — so if the classify input is NOT +// trimmed, a whitespace-padded prompt misses the map and silently defaults to +// Informational despite a real classification. The fake classifier below keys its +// result by exactly the text it receives (like the real `classifyIntents`), so an +// un-trimmed classify input would reintroduce the miss and fail this test. +describe('handlers/prompts.js — intent classify/lookup key alignment (serenity-docs#32)', () => { + const project = () => makeProject({ + semrushProjectId: 'proj-us-en', geoTargetId: 2840, languageCode: 'en', + }); + + it('classifies the trimmed text so a whitespace-padded prompt keeps its real (non-default) intent', async () => { + const { handleCreatePrompts: handleCreate } = await esmock( + '../../../../src/support/serenity/handlers/prompts.js', + { + '../../../../src/support/serenity/intent-classification.js': { + classifyPromptIntents: async (texts) => new Map( + texts.map((t) => [t, /buy/i.test(t) ? 'Transactional' : 'Informational']), + ), + }, + }, + ); + + const dataAccess = makeDataAccess([project()]); + const transport = { + listProjectTags: makeListProjectTagsStub(), + createPromptsByIds: sinon.stub().resolves({ + page: 1, total: 1, items: [{ id: 'new-sem-id', name: 'buy now' }], + }), + publishProject: sinon.stub().resolves(), + }; + + const result = await handleCreate(transport, dataAccess, BRAND, WORKSPACE, { + prompts: [{ + text: ' buy now ', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-cat-1'], + }], + }, fakeLog(), classifyByBrandMention); + + // The padded prompt resolves to Transactional (its real classification), NOT the + // Informational default — proving the classify key was trimmed to match the + // injector's `input.text` lookup. + expect(result.created[0].tagIds).to.include(TAG_IDS.intentTransactional); + expect(result.created[0].tagIds).to.not.include(TAG_IDS.intentInformational); + }); +}); diff --git a/test/support/serenity/intent-classification.test.js b/test/support/serenity/intent-classification.test.js new file mode 100644 index 000000000..0f705055c --- /dev/null +++ b/test/support/serenity/intent-classification.test.js @@ -0,0 +1,455 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { expect } from 'chai'; +import sinon from 'sinon'; +import esmock from 'esmock'; + +import { CREATE_PUBLISH_RESERVE_MS } from '../../../src/support/serenity/intent-classification.js'; +import { PER_CALL_MS } from '../../../src/support/serenity/intent-taxonomy.js'; +import { classifyIntents as realClassifyIntents } from '../../../src/support/intent-classifier.js'; +import { resolveEnvironment as realResolveEnvironment } from '../../../src/support/metrics-emf.js'; + +const log = { + info: sinon.stub(), warn: sinon.stub(), error: sinon.stub(), debug: sinon.stub(), +}; + +// Shared EMF spy; a fresh reference per load so assertions target one call log. +let emitMetricSpy; + +function metricsMock() { + emitMetricSpy = sinon.spy(); + return { + '../../../src/support/metrics-emf.js': { + emitMetric: emitMetricSpy, + resolveEnvironment: realResolveEnvironment, + }, + }; +} + +// Collects the metrics emitted for a given name into {value, dimensions} rows. +function emittedMetric(name) { + return emitMetricSpy.getCalls() + .map((c) => c.args[0]) + .filter((m) => m.name === name); +} + +async function loadWithClassifier({ classify, classifyIntentsStub } = {}) { + return esmock('../../../src/support/serenity/intent-classification.js', { + '../../../src/support/intent-classifier.js': { + createIntentClassifier: sinon.stub().returns(classify), + classifyIntents: classifyIntentsStub || sinon.stub().resolves(new Map()), + }, + ...metricsMock(), + }); +} + +// Loads the module wired to the REAL batch runner (`classifyIntents`) so the +// per-call timing wrapper and the injected category spec's `parseResult` (which +// counts `low_confidence` soft-failures) are actually exercised. `classifyByText` +// maps a prompt text to the parsed model body `parseResult` will inspect (or +// `null`/undefined to simulate a call that produced nothing). +async function loadWithRealBatchRaw({ makeClassify }) { + return esmock('../../../src/support/serenity/intent-classification.js', { + '../../../src/support/intent-classifier.js': { + createIntentClassifier: (_ctx, spec) => makeClassify(spec), + classifyIntents: realClassifyIntents, + }, + ...metricsMock(), + }); +} + +async function loadWithRealBatch({ classifyByText }) { + return loadWithRealBatchRaw({ + makeClassify: (spec) => async (text) => { + const parsed = classifyByText(text); + return parsed === undefined || parsed === null ? null : spec.parseResult(parsed); + }, + }); +} + +describe('intent-classification.js — classifyPromptIntents (serenity-docs#32)', () => { + afterEach(() => { + sinon.reset(); + }); + + it('returns an empty map for an empty/falsy text list without touching the classifier', async () => { + const classifyIntentsStub = sinon.stub(); + const { classifyPromptIntents } = await loadWithClassifier({ + classify: () => {}, classifyIntentsStub, + }); + const result = await classifyPromptIntents([], { env: {}, log, deadline: Date.now() + 100000 }); + expect(result.size).to.equal(0); + expect(classifyIntentsStub).to.not.have.been.called; + }); + + it('treats a null/undefined texts argument as empty', async () => { + const classifyIntentsStub = sinon.stub(); + const { classifyPromptIntents } = await loadWithClassifier({ + classify: () => {}, classifyIntentsStub, + }); + const result = await classifyPromptIntents(null, { + env: {}, log, deadline: Date.now() + 100000, + }); + expect(result.size).to.equal(0); + expect(classifyIntentsStub).to.not.have.been.called; + }); + + it('hard skip-gate: defaults everything and logs budget_skipped when no room at entry, without constructing a classifier', async () => { + const classifyIntentsStub = sinon.stub(); + const { classifyPromptIntents, computeWriteDeadline } = await loadWithClassifier({ + classify: () => {}, classifyIntentsStub, + }); + // deadline already effectively exhausted (in the past). + const deadline = computeWriteDeadline(Date.now() - 100000); + const result = await classifyPromptIntents(['a', 'b'], { env: {}, log, deadline }); + expect(result.get('a')).to.equal('Informational'); + expect(result.get('b')).to.equal('Informational'); + expect(classifyIntentsStub).to.not.have.been.called; + expect(log.info).to.have.been.calledWithMatch(/budget_skipped/); + }); + + it('defaults everything with an info log (non-prod) when the classifier is unavailable (Azure not configured)', async () => { + const { classifyPromptIntents } = await loadWithClassifier({ classify: null }); + const result = await classifyPromptIntents(['a'], { + env: { AWS_ENV: 'dev' }, log, deadline: Date.now() + 100000, + }); + expect(result.get('a')).to.equal('Informational'); + expect(log.info).to.have.been.calledWithMatch(/Azure OpenAI is not configured/); + expect(log.warn).to.not.have.been.called; + }); + + it('defaults everything with a WARN + prod_llm_unavailable log in prod when the classifier is unavailable', async () => { + const { classifyPromptIntents } = await loadWithClassifier({ classify: null }); + const result = await classifyPromptIntents(['a'], { + env: { AWS_ENV: 'prod' }, log, deadline: Date.now() + 100000, + }); + expect(result.get('a')).to.equal('Informational'); + expect(log.warn).to.have.been.calledWithMatch(/prod_llm_unavailable/); + }); + + it('treats a null/undefined env or log as if omitted, without throwing', async () => { + const { classifyPromptIntents } = await loadWithClassifier({ classify: null }); + const result = await classifyPromptIntents(['a'], { + env: null, log: null, deadline: Date.now() + 100000, + }); + expect(result.get('a')).to.equal('Informational'); + }); + + it('uses the first-pass classification result and counts classified_ok, no retry when everything resolves', async () => { + const classifyIntentsStub = sinon.stub().resolves(new Map([ + ['a', 'Task'], ['b', 'Commercial'], + ])); + const { classifyPromptIntents } = await loadWithClassifier({ + classify: () => {}, classifyIntentsStub, + }); + const result = await classifyPromptIntents(['a', 'b'], { + env: {}, log, deadline: Date.now() + 100000, + }); + expect(result.get('a')).to.equal('Task'); + expect(result.get('b')).to.equal('Commercial'); + expect(classifyIntentsStub).to.have.been.calledOnce; + expect(log.info).to.have.been.calledWithMatch(/summary/, sinon.match({ classified_ok: 2, retry_attempted: 0 })); + }); + + it('retries once for unresolved texts when the budget allows, and uses the retry result on success', async () => { + const classifyIntentsStub = sinon.stub(); + classifyIntentsStub.onCall(0).resolves(new Map([['a', 'Task']])); // 'b' left unresolved + classifyIntentsStub.onCall(1).resolves(new Map([['b', 'Navigational']])); + const { classifyPromptIntents } = await loadWithClassifier({ + classify: () => {}, classifyIntentsStub, + }); + // Ample budget so the retry gate passes. + const result = await classifyPromptIntents(['a', 'b'], { + env: {}, log, deadline: Date.now() + 100000, + }); + expect(result.get('a')).to.equal('Task'); + expect(result.get('b')).to.equal('Navigational'); + expect(classifyIntentsStub).to.have.been.calledTwice; + expect(log.info).to.have.been.calledWithMatch(/summary/, sinon.match({ classified_ok: 1, retry_attempted: 1, retry_succeeded: 1 })); + }); + + it('defaults to Informational when the retry also fails to resolve a text', async () => { + const classifyIntentsStub = sinon.stub(); + classifyIntentsStub.onCall(0).resolves(new Map()); // nothing resolved + classifyIntentsStub.onCall(1).resolves(new Map()); // retry also resolves nothing + const { classifyPromptIntents } = await loadWithClassifier({ + classify: () => {}, classifyIntentsStub, + }); + const result = await classifyPromptIntents(['a'], { + env: {}, log, deadline: Date.now() + 100000, + }); + expect(result.get('a')).to.equal('Informational'); + expect(classifyIntentsStub).to.have.been.calledTwice; + expect(log.info).to.have.been.calledWithMatch(/summary/, sinon.match({ defaulted: 1, retry_attempted: 1 })); + }); + + it('skips the retry (defaults immediately) when the deadline has no room left for a second call', async () => { + const t0 = 1_000_000_000; + const clock = sinon.useFakeTimers(t0); + try { + // Entry hard-skip-gate passes (remaining ≈ PER_CALL_MS + 50ms of slack), but + // the mocked first-pass classify call "spends" 60ms of wall-clock via the + // fake timer, so by the time the retry-gate check runs, remaining budget has + // dropped below PER_CALL_MS — the retry is skipped, not attempted. + const deadline = t0 + CREATE_PUBLISH_RESERVE_MS + PER_CALL_MS + 50; + const classifyIntentsStub = sinon.stub().callsFake(async () => { + clock.tick(60); + return new Map(); // nothing resolved on the first pass + }); + const { classifyPromptIntents } = await loadWithClassifier({ + classify: () => {}, classifyIntentsStub, + }); + const result = await classifyPromptIntents(['a'], { env: {}, log, deadline }); + expect(result.get('a')).to.equal('Informational'); + expect(classifyIntentsStub).to.have.been.calledOnce; + expect(log.info).to.have.been.calledWithMatch(/summary/, sinon.match({ retry_attempted: 0, defaulted: 1 })); + } finally { + clock.restore(); + } + }); +}); + +describe('intent-classification.js — observability (serenity-docs#32)', () => { + afterEach(() => { + sinon.reset(); + }); + + it('emits IntentOutcome counters dimensioned by WritePath + Workspace, and IntentValueDistribution by WritePath', async () => { + const classifyIntentsStub = sinon.stub().resolves(new Map([ + ['a', 'Task'], ['b', 'Task'], ['c', 'Commercial'], + ])); + const { classifyPromptIntents } = await loadWithClassifier({ + classify: () => {}, classifyIntentsStub, + }); + await classifyPromptIntents(['a', 'b', 'c'], { + env: { AWS_ENV: 'stage' }, + log, + deadline: Date.now() + 100000, + writePath: 'create', + workspaceId: 'ws-42', + }); + + const outcome = emittedMetric('IntentOutcome').find((m) => m.dimensions.Outcome === 'classified_ok'); + expect(outcome).to.include({ value: 3, unit: 'Count' }); + expect(outcome.dimensions).to.include({ WritePath: 'create', Workspace: 'ws-42', Outcome: 'classified_ok' }); + + const values = emittedMetric('IntentValueDistribution'); + const task = values.find((m) => m.dimensions.Value === 'Task'); + const commercial = values.find((m) => m.dimensions.Value === 'Commercial'); + expect(task).to.include({ value: 2 }); + expect(task.dimensions).to.include({ WritePath: 'create', Value: 'Task' }); + expect(task.dimensions).to.not.have.property('Workspace'); + expect(commercial).to.include({ value: 1 }); + // Environment resolves via AWS_ENV for the emit sink options (not a dimension here). + expect(realResolveEnvironment({ AWS_ENV: 'stage' })).to.equal('stage'); + }); + + it('emits a budget_skipped IntentOutcome and a defaulted value bucket on the hard skip-gate', async () => { + const { classifyPromptIntents, computeWriteDeadline } = await loadWithClassifier({ + classify: () => {}, + }); + const deadline = computeWriteDeadline(Date.now() - 100000); + await classifyPromptIntents(['a', 'b'], { + env: {}, log, deadline, writePath: 'csv', workspaceId: 'ws-1', + }); + const skipped = emittedMetric('IntentOutcome').find((m) => m.dimensions.Outcome === 'budget_skipped'); + expect(skipped).to.include({ value: 2 }); + const defaulted = emittedMetric('IntentValueDistribution').find((m) => m.dimensions.Value === 'defaulted'); + expect(defaulted).to.include({ value: 2 }); + }); + + it('emits a ProdLlmUnavailable counter (and keeps the warn) when the classifier cannot be constructed in prod', async () => { + const { classifyPromptIntents } = await loadWithClassifier({ classify: null }); + await classifyPromptIntents(['a'], { + env: { AWS_ENV: 'prod' }, log, deadline: Date.now() + 100000, writePath: 'create', + }); + const prodUnavailable = emittedMetric('ProdLlmUnavailable'); + expect(prodUnavailable).to.have.length(1); + expect(prodUnavailable[0]).to.include({ value: 1 }); + expect(prodUnavailable[0].dimensions).to.include({ WritePath: 'create' }); + expect(log.warn).to.have.been.calledWithMatch(/prod_llm_unavailable/); + }); + + it('fires the prod path (warn + counter) when prod is signalled via ENV, not just AWS_ENV', async () => { + // Regression: prod detection resolves through resolveEnvironment (AWS_ENV || ENV), + // so ENV=prod alone must still trip the prod path — the old AWS_ENV-only check missed this. + const { classifyPromptIntents } = await loadWithClassifier({ classify: null }); + await classifyPromptIntents(['a'], { + env: { ENV: 'prod' }, log, deadline: Date.now() + 100000, writePath: 'create', + }); + expect(emittedMetric('ProdLlmUnavailable')).to.have.length(1); + expect(log.warn).to.have.been.calledWithMatch(/prod_llm_unavailable/); + }); + + it('does NOT emit ProdLlmUnavailable on the construction failure in non-prod', async () => { + const { classifyPromptIntents } = await loadWithClassifier({ classify: null }); + await classifyPromptIntents(['a'], { + env: { ENV: 'stage' }, log, deadline: Date.now() + 100000, writePath: 'create', + }); + expect(emittedMetric('ProdLlmUnavailable')).to.have.length(0); + }); + + it('counts low_confidence apart from defaulted, logs truncated reasoning, and folds it into the retry/default ladder', async () => { + // 'a' resolves; 'b' is a below-floor low-confidence soft failure on every pass. + const parsedByText = { + a: { intent: 'Task', confidence: 0.97, reasoning: 'clear delegation' }, + b: { + intent: 'Commercial', + confidence: 0.2, + reasoning: 'x'.repeat(500), + }, + }; + const { classifyPromptIntents } = await loadWithRealBatch({ + classifyByText: (t) => parsedByText[t], + }); + const result = await classifyPromptIntents(['a', 'b'], { + env: {}, log, deadline: Date.now() + 100000, writePath: 'create', workspaceId: 'ws-9', + }); + expect(result.get('a')).to.equal('Task'); + expect(result.get('b')).to.equal('Informational'); // defaulted after retry + + // low_confidence counted per soft-failure call (pass + retry = 2), distinct from defaulted (1). + expect(log.info).to.have.been.calledWithMatch( + /summary/, + sinon.match({ low_confidence: 2, defaulted: 1, classified_ok: 1 }), + ); + const lc = emittedMetric('IntentOutcome').find((m) => m.dimensions.Outcome === 'low_confidence'); + expect(lc).to.include({ value: 2 }); + + // Truncated reasoning is logged (sliced to 200 chars), not the full 500. + const softLog = log.info.getCalls().find((c) => /soft failures/.test(c.args[0])); + expect(softLog).to.not.equal(undefined); + expect(softLog.args[1].samples[0].reasoning).to.have.length(200); + expect(softLog.args[1].samples[0].reason).to.equal('low_confidence'); + }); + + it('logs only {reason, reasoning} per soft-failure sample — never the prompt text', async () => { + const secret = 'super-secret-prompt-text-that-must-not-be-logged'; + const { classifyPromptIntents } = await loadWithRealBatch({ + classifyByText: () => ({ intent: 'Commercial', confidence: 0.1, reasoning: 'low conf' }), + }); + await classifyPromptIntents([secret], { + env: {}, log, deadline: Date.now() + 100000, writePath: 'create', + }); + const softLog = log.info.getCalls().find((c) => /soft failures/.test(c.args[0])); + expect(softLog).to.not.equal(undefined); + expect(softLog.args[1].samples[0]).to.have.keys(['reason', 'reasoning']); + expect(JSON.stringify(softLog.args)).to.not.contain(secret); + }); + + it('caps the number of logged soft-failure reasoning samples at 10', async () => { + const texts = Array.from({ length: 15 }, (_, i) => `t${i}`); + const { classifyPromptIntents } = await loadWithRealBatch({ + classifyByText: () => ({ intent: 'bogus-value', confidence: 0.99 }), + }); + await classifyPromptIntents(texts, { + env: {}, log, deadline: Date.now() + 100000, writePath: 'create', + }); + const softLog = log.info.getCalls().find((c) => /soft failures/.test(c.args[0])); + expect(softLog.args[1].samples).to.have.length(10); + // invalid_value is logged but is NOT counted as low_confidence. + expect(softLog.args[1].samples[0].reason).to.equal('invalid_value'); + expect(softLog.args[1].low_confidence).to.equal(0); + }); + + it('emits per-call latency (p50/p95), a per-call timeout tally, and a prod repeated-invoke-failure signal', async () => { + const t0 = 1_000_000_000; + const clock = sinon.useFakeTimers(t0); + try { + // Every call "spends" the full per-call budget and resolves null → a + // heuristic timeout, and no text ever resolves → repeated-invoke-failure. + const { classifyPromptIntents } = await loadWithRealBatchRaw({ + makeClassify: () => async () => { + clock.tick(PER_CALL_MS); + return null; + }, + }); + const result = await classifyPromptIntents(['a'], { + env: { AWS_ENV: 'prod' }, + log, + deadline: t0 + 100000, + writePath: 'edit', + workspaceId: 'ws-7', + }); + expect(result.get('a')).to.equal('Informational'); + + const p50 = emittedMetric('PerCallLatencyP50Ms'); + const p95 = emittedMetric('PerCallLatencyP95Ms'); + expect(p50[0]).to.include({ value: PER_CALL_MS, unit: 'Milliseconds' }); + expect(p95[0]).to.include({ value: PER_CALL_MS, unit: 'Milliseconds' }); + + const timeouts = emittedMetric('PerCallTimeout'); + // first pass + retry, both timed out. + expect(timeouts[0]).to.include({ value: 2 }); + + const batch = emittedMetric('ClassifyBatchDurationMs'); + expect(batch[0].unit).to.equal('Milliseconds'); + + const prodUnavailable = emittedMetric('ProdLlmUnavailable'); + expect(prodUnavailable).to.have.length(1); + expect(prodUnavailable[0].dimensions).to.include({ WritePath: 'edit' }); + } finally { + clock.restore(); + } + }); + + it('does NOT emit the repeated-invoke-failure signal in non-prod even when nothing resolves', async () => { + const { classifyPromptIntents } = await loadWithRealBatch({ + classifyByText: () => null, + }); + await classifyPromptIntents(['a'], { + env: { AWS_ENV: 'stage' }, log, deadline: Date.now() + 100000, writePath: 'create', + }); + expect(emittedMetric('ProdLlmUnavailable')).to.have.length(0); + }); + + it('never throws into the classify path when emitMetric itself throws', async () => { + const classifyIntentsStub = sinon.stub().resolves(new Map([['a', 'Task']])); + const mod = await esmock('../../../src/support/serenity/intent-classification.js', { + '../../../src/support/intent-classifier.js': { + createIntentClassifier: sinon.stub().returns(() => {}), + classifyIntents: classifyIntentsStub, + }, + '../../../src/support/metrics-emf.js': { + emitMetric: () => { throw new Error('emf boom'); }, + resolveEnvironment: realResolveEnvironment, + }, + }); + const result = await mod.classifyPromptIntents(['a'], { + env: {}, log, deadline: Date.now() + 100000, writePath: 'create', + }); + expect(result.get('a')).to.equal('Task'); + }); + + it('never throws when emitMetric throws on the latency + prod_llm_unavailable paths', async () => { + // Real batch runner so the per-call timing wrapper runs (populating + // callDurations → latency emits), in prod with nothing resolving so the + // repeated-invoke-failure ProdLlmUnavailable site also fires — all while + // emitMetric throws on every call. + const mod = await esmock('../../../src/support/serenity/intent-classification.js', { + '../../../src/support/intent-classifier.js': { + createIntentClassifier: () => async () => null, + classifyIntents: realClassifyIntents, + }, + '../../../src/support/metrics-emf.js': { + emitMetric: () => { throw new Error('emf boom'); }, + resolveEnvironment: realResolveEnvironment, + }, + }); + const result = await mod.classifyPromptIntents(['a', 'b'], { + env: { AWS_ENV: 'prod' }, log, deadline: Date.now() + 100000, writePath: 'create', + }); + expect(result.get('a')).to.equal('Informational'); + expect(result.get('b')).to.equal('Informational'); + }); +}); diff --git a/test/support/serenity/intent-taxonomy.test.js b/test/support/serenity/intent-taxonomy.test.js new file mode 100644 index 000000000..bb90cf5c2 --- /dev/null +++ b/test/support/serenity/intent-taxonomy.test.js @@ -0,0 +1,101 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { expect } from 'chai'; + +import { + parseSerenityIntent, + inspectSerenityIntent, + SERENITY_INTENT_VALUES, + PROMPT_INTENT_MIN_CONFIDENCE, + SERENITY_INTENT_CATEGORY_SPEC, +} from '../../../src/support/serenity/intent-taxonomy.js'; + +describe('intent-taxonomy.js — parseSerenityIntent (serenity-docs#32)', () => { + it('returns the bare value for a valid value at/above the confidence floor', () => { + expect(parseSerenityIntent({ intent: 'Task', confidence: PROMPT_INTENT_MIN_CONFIDENCE })) + .to.equal('Task'); + expect(parseSerenityIntent({ intent: 'Informational', confidence: 0.99 })) + .to.equal('Informational'); + }); + + it('accepts every canonical value', () => { + SERENITY_INTENT_VALUES.forEach((value) => { + expect(parseSerenityIntent({ intent: value, confidence: 1 })).to.equal(value); + }); + }); + + it('returns null for a value outside the 5 canonical Capitalized literals (case-sensitive)', () => { + expect(parseSerenityIntent({ intent: 'informational', confidence: 1 })).to.equal(null); + expect(parseSerenityIntent({ intent: 'not-a-real-value', confidence: 1 })).to.equal(null); + expect(parseSerenityIntent({ intent: undefined, confidence: 1 })).to.equal(null); + }); + + it('returns null when confidence is below the validity floor', () => { + expect(parseSerenityIntent({ intent: 'Task', confidence: PROMPT_INTENT_MIN_CONFIDENCE - 0.01 })) + .to.equal(null); + }); + + it('returns null when confidence is missing, non-numeric, or non-finite', () => { + expect(parseSerenityIntent({ intent: 'Task' })).to.equal(null); + expect(parseSerenityIntent({ intent: 'Task', confidence: 'high' })).to.equal(null); + expect(parseSerenityIntent({ intent: 'Task', confidence: NaN })).to.equal(null); + expect(parseSerenityIntent({ intent: 'Task', confidence: Infinity })).to.equal(null); + }); + + it('returns null for a garbled/empty parsed body', () => { + expect(parseSerenityIntent({})).to.equal(null); + expect(parseSerenityIntent(null)).to.equal(null); + expect(parseSerenityIntent(undefined)).to.equal(null); + }); + + it('exposes the category spec with the fixed per-call timeout, not env-driven', () => { + expect(SERENITY_INTENT_CATEGORY_SPEC.parseResult).to.equal(parseSerenityIntent); + expect(SERENITY_INTENT_CATEGORY_SPEC.invokeTimeoutMs).to.be.a('number').greaterThan(0); + expect(SERENITY_INTENT_CATEGORY_SPEC.systemPrompt).to.be.a('string').with.length.greaterThan(0); + }); +}); + +describe('intent-taxonomy.js — inspectSerenityIntent (serenity-docs#32 observability)', () => { + it('returns ok with the bare value and surfaced fields for a valid, confident result', () => { + const r = inspectSerenityIntent({ intent: 'Task', confidence: 0.97, reasoning: 'delegated pick' }); + expect(r).to.deep.equal({ + value: 'Task', reason: 'ok', confidence: 0.97, reasoning: 'delegated pick', + }); + }); + + it('flags a recognized value below the confidence floor as low_confidence, still tag null', () => { + const r = inspectSerenityIntent({ + intent: 'Commercial', confidence: PROMPT_INTENT_MIN_CONFIDENCE - 0.01, reasoning: 'unsure', + }); + expect(r.value).to.equal(null); + expect(r.reason).to.equal('low_confidence'); + expect(r.reasoning).to.equal('unsure'); + }); + + it('flags an unrecognized/garbled value as invalid_value before the confidence check', () => { + expect(inspectSerenityIntent({ intent: 'nope', confidence: 0.99 }).reason).to.equal('invalid_value'); + expect(inspectSerenityIntent({}).reason).to.equal('invalid_value'); + expect(inspectSerenityIntent(null).reason).to.equal('invalid_value'); + }); + + it('defaults reasoning to an empty string when absent or non-string', () => { + expect(inspectSerenityIntent({ intent: 'Task', confidence: 1 }).reasoning).to.equal(''); + expect(inspectSerenityIntent({ intent: 'Task', confidence: 1, reasoning: 42 }).reasoning).to.equal(''); + }); + + it('parseSerenityIntent is the value projection of inspectSerenityIntent', () => { + const parsed = { intent: 'Navigational', confidence: 0.9, reasoning: 'go to site' }; + expect(parseSerenityIntent(parsed)).to.equal(inspectSerenityIntent(parsed).value); + expect(parseSerenityIntent({ intent: 'x' })).to.equal(inspectSerenityIntent({ intent: 'x' }).value); + }); +}); diff --git a/test/support/serenity/tag-tree.test.js b/test/support/serenity/tag-tree.test.js index 3ca43b296..0d12812a5 100644 --- a/test/support/serenity/tag-tree.test.js +++ b/test/support/serenity/tag-tree.test.js @@ -22,6 +22,7 @@ import { provisionDimensionTree, ensureClosedValue, resolveTypeValueInjection, + resolveIntentValueInjection, resolveClosedValueInjection, findTagsInTree, assertParentWithinDimension, @@ -573,6 +574,61 @@ describe('serenity tag-tree', () => { }); }); + // Twin of the `resolveTypeValueInjection` block above for the `intent` closed + // dimension (serenity-docs#32). Both are thin wrappers over + // `resolveClosedValueInjection`; this pins the intent-specific return key + // (`intentTagIds`) and the fail-closed contract independently of the type twin. + describe('resolveIntentValueInjection', () => { + it('returns the wanted value id plus EVERY id under the intent root', async () => { + const transport = { + listProjectTags: makeListProjectTagsStub(), + createProjectTags: sinon.stub(), + }; + const res = await resolveIntentValueInjection(transport, WS, PROJECT, 'Task', fakeLog()); + expect(res.computedId).to.equal(TAG_IDS.intentTask); + // The strip set is every intent id — the caller may not set the value itself. + expect(res.intentTagIds).to.have.members([ + TAG_IDS.intentInformational, + TAG_IDS.intentTask, + TAG_IDS.intentCommercial, + TAG_IDS.intentTransactional, + TAG_IDS.intentNavigational, + ]); + expect(transport.createProjectTags).to.not.have.been.called; + }); + + it('creates the value on demand when a project predating the taxonomy lacks it', async () => { + const { listProjectTags, createProjectTags } = makeProvisioningTransportStubs(); + const transport = { listProjectTags, createProjectTags }; + const res = await resolveIntentValueInjection(transport, WS, PROJECT, 'Task', fakeLog()); + expect(res.computedId).to.equal('created:created::intent:Task'); + expect(res.intentTagIds).to.deep.equal(['created:created::intent:Task']); + }); + + it('502s rather than skip injection when the intent root cannot be resolved', async () => { + // A prompt written without the server-computed `intent` tag stays + // unclassified forever: the client may not set that dimension itself. + const transport = { + listProjectTags: makeListProjectTagsStub({ '': [] }), + createProjectTags: sinon.stub().resolves([]), + }; + const err = await resolveIntentValueInjection(transport, WS, PROJECT, 'Task', fakeLog()) + .then(() => null, (e) => e); + expect(err).to.be.an('error'); + expect(err.status).to.equal(502); + }); + + it('propagates a transport failure while reading the tag tree', async () => { + const transport = { + listProjectTags: sinon.stub().rejects(new Error('listProjectTags 502')), + createProjectTags: sinon.stub(), + }; + await expect(resolveIntentValueInjection(transport, WS, PROJECT, 'Task', fakeLog())) + .to.be.rejectedWith(/listProjectTags 502/); + expect(transport.createProjectTags).to.not.have.been.called; + }); + }); + describe('findTagsInTree', () => { it('places several ids in one walk and reports each ancestry', async () => { const transport = { listProjectTags: makeListProjectTagsStub() };