From aa3ae8de67d23ff48f37b60a298c1b5ed888f971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hanu=C5=A1?= Date: Sun, 5 Jul 2026 00:51:52 +0200 Subject: [PATCH] docs(skills): add common Actor patterns to apify-actor-development SKILL.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a "Common patterns" section covering six gaps that repeatedly trip up agents (both human and LLM) authoring Actors: - Network robustness — wrap fetch() in try/catch with retry + exponential backoff, so a transient upstream 503 does not surface as an Actor-level RUNTIME_ERROR. Full reference at references/robust-fetch.ts. - API keys and secrets — mark input fields carrying tokens/keys with isSecret: true so they are encrypted at rest and censored in the UI. - Persisting state across scheduled / cron runs — two documented patterns (named KV store for full-scope tokens; previous-run-lookup via REST for LIMITED_PERMISSIONS scheduled runs where Actor.openKeyValueStore('name') fails with "Permission denied"). Full reference at references/cross-run-state.ts. - Key-value store key charset — explicit regex and one-line sanitizer for the ArgumentError thrown when a natural key contains colons, slashes, spaces, or unicode. - Progress reporting — Actor.setStatusMessage snippet with the isStatusTerminal option for terminal failure states. - Prefer MCP tools over raw REST — enumerate the common mcp__apify__* tools so agents reach for them before hand-rolling curl calls against api.apify.com. Each SKILL.md entry is a copy-pasteable snippet; the two larger reference implementations live under references/ following the existing layout. Surfaced during an evaluation of Apify surfaces for agent-driven Actor development. Co-Authored-By: Claude Opus 4.7 --- skills/apify-actor-development/SKILL.md | 163 ++++++++++++++++++ .../references/cross-run-state.ts | 97 +++++++++++ .../references/robust-fetch.ts | 76 ++++++++ 3 files changed, 336 insertions(+) create mode 100644 skills/apify-actor-development/references/cross-run-state.ts create mode 100644 skills/apify-actor-development/references/robust-fetch.ts diff --git a/skills/apify-actor-development/SKILL.md b/skills/apify-actor-development/SKILL.md index ce02412..c9ed7eb 100644 --- a/skills/apify-actor-development/SKILL.md +++ b/skills/apify-actor-development/SKILL.md @@ -122,6 +122,169 @@ Use the appropriate CLI command based on the user's language choice. Additional - Use `console.log()` or `print()` instead of the Apify logger — these bypass credential censoring - Disable standby mode without explicit permission +## Common patterns + +Six patterns come up in almost every non-trivial Actor. Each is small enough +to reproduce inline — copy the snippet and adapt. + +### Network robustness — wrap `fetch` in try/catch + retry + +Actors run in a shared cloud environment and hit upstream services that can +flake. Unwrapped `fetch()` calls turn a transient 503 into an Actor-level +`RUNTIME_ERROR`. Always wrap network calls in try/catch, retry transient +failures with exponential backoff, and route errors through `apify/log` so +they appear in the run log. + +```ts +import { log } from 'apify'; + +async function robustFetch(url: string, retries = 4): Promise { + for (let attempt = 0; attempt < retries; attempt++) { + try { + const res = await fetch(url); + if (res.ok) return (await res.json()) as T; + if (![408, 429, 500, 502, 503, 504].includes(res.status) || attempt === retries - 1) { + throw new Error(`HTTP ${res.status} for ${url}`); + } + } catch (err) { + if (attempt === retries - 1) { + log.exception(err as Error, `fetch ${url} failed`); + throw err; + } + log.warning(`fetch ${url} failed (${attempt + 1}/${retries}): ${(err as Error).message}`); + } + await new Promise((r) => setTimeout(r, 500 * 2 ** attempt)); + } + throw new Error('unreachable'); +} +``` + +A complete version with timeouts and configurable retry statuses lives in +[references/robust-fetch.ts](references/robust-fetch.ts). + +### API keys and secrets — always mark `isSecret: true` + +Any input field that carries an API key, token, or password MUST set +`isSecret: true` in the input schema. Without it the value is stored in +plaintext in the run object, shown in the Console UI, and included in +`Actor.getInput()` logs. + +```json +{ + "properties": { + "apiKey": { + "title": "API key", + "type": "string", + "description": "API key for the upstream service.", + "editor": "textfield", + "isSecret": true + } + }, + "required": ["apiKey"] +} +``` + +Read it with the usual `Actor.getInput()` call — the SDK decrypts secret +fields automatically: + +```ts +const { apiKey } = await Actor.getInput<{ apiKey: string }>() ?? {}; +``` + +Never log the value (`apify/log` censors known secret names, but only if you +use it — `console.log` bypasses censoring). See the "Do NOT" list under Best +practices above. + +### Persisting state across scheduled / cron runs + +Each run gets its own default key-value store, so `Actor.setValue(key, val)` +does not survive to the next run. Two patterns exist — pick based on the +token scope the Actor runs under. + +**Preferred: a named KV store.** Requires a full-scope token (a user token or +an Actor granted "Access to all key-value stores"). Declare it up front: + +```jsonc +// .actor/actor.json — environmentVariables MUST be an OBJECT, not an array. +{ + "environmentVariables": { + "STATE_STORE_NAME": "my-actor-state", + "UPSTREAM_API_KEY": "@upstreamApiKey" // resolves via `apify secrets add upstreamApiKey ` + } +} +``` + +```ts +const store = await Actor.openKeyValueStore(process.env.STATE_STORE_NAME!); +const prev = await store.getValue('cursor'); +await store.setValue('cursor', nextCursor); +``` + +**Fallback for LIMITED_PERMISSIONS tokens** (the default scope for scheduled +runs of a user's own Actor). `Actor.openKeyValueStore('some-name')` will fail +with "Permission denied" — instead, list previous SUCCEEDED runs of the same +Actor and read their default store: + +```ts +const client = Actor.newClient(); +const { items } = await client.actor(process.env.APIFY_ACTOR_ID!).runs().list({ + status: 'SUCCEEDED', desc: true, limit: 10, +}); +const prev = items.find((r) => r.id !== process.env.APIFY_ACTOR_RUN_ID); +const record = prev + ? await client.keyValueStore(prev.defaultKeyValueStoreId).getRecord('cursor') + : undefined; +``` + +Full reference implementation with helpers for both patterns: +[references/cross-run-state.ts](references/cross-run-state.ts). + +### Key-value store key charset + +KV keys must match `/^[a-zA-Z0-9!\-_.'()]{1,256}$/`. Natural keys with +colons, slashes, spaces, or unicode throw +`ArgumentError: (string \`key\`) must be at most 256 characters long and only contain: a-zA-Z0-9!-_.'()` +from `setValue()`. Sanitize before use: + +```ts +const kvKey = (raw: string) => raw.replace(/[^a-zA-Z0-9!\-_.'()]/g, '_').slice(0, 256); +await Actor.setValue(kvKey(`user:${userId}:${timestamp}`), payload); +``` + +### Reporting progress with `Actor.setStatusMessage` + +Long-running Actors should surface progress so the Console UI and any +watching agent can see what's happening. `Actor.setStatusMessage` sets a +single-line status that appears at the top of the run detail page. + +```ts +await Actor.setStatusMessage(`Scraped ${done}/${total} pages`); +// On failure, mark it terminal so retries do not overwrite it: +await Actor.setStatusMessage('Upstream API returned 403 — aborting', { isStatusTerminal: true }); +``` + +Call it at meaningful milestones (start of each phase, every N items, +before a slow network call), not on every iteration — status updates hit +the API. + +### Prefer MCP tools over raw REST for platform metadata + +When an Apify MCP server is available, use its tools rather than +constructing `curl` calls against `api.apify.com`. The MCP tools handle +auth, pagination, and error shapes, and stay in sync with API changes. +Common tools: + +- `mcp__apify__search-apify-docs` — search the docs. +- `mcp__apify__fetch-apify-docs` — fetch a full doc page. +- `mcp__apify__search-actors` — find Actors in the Store. +- `mcp__apify__get-actor` — inspect an Actor's README, input schema, and metadata. +- `mcp__apify__call-actor` — invoke an Actor and get its output. +- `mcp__apify__get-actor-run` / `mcp__apify__get-actor-run-log` — read run status and logs. +- `mcp__apify__get-dataset-items` / `mcp__apify__get-key-value-store-record` — pull storage contents. + +Fall back to `apify api ` or raw `fetch` against `api.apify.com` +only when no MCP tool covers the endpoint you need. + ## Logging See [references/logging.md](references/logging.md) for complete logging documentation including available log levels and best practices for JavaScript/TypeScript and Python. diff --git a/skills/apify-actor-development/references/cross-run-state.ts b/skills/apify-actor-development/references/cross-run-state.ts new file mode 100644 index 0000000..21480aa --- /dev/null +++ b/skills/apify-actor-development/references/cross-run-state.ts @@ -0,0 +1,97 @@ +// Persisting state across scheduled / cron Actor runs. +// +// Problem: each Actor run gets its own default key-value store, so state +// written with `Actor.setValue()` disappears when the run ends. Two options +// exist for making state visible to the next run — pick based on the token +// scope the Actor is granted. +// +// (A) Named KV store (preferred when you have full-scope credentials). +// Requires the Actor's token to be able to open a store by name across +// runs, which needs a user-scoped token or an "Access to all key-value +// stores" permission grant. +// +// (B) Fetch the previous run's default KV store via the REST API. Works +// under LIMITED_PERMISSIONS tokens (the default for scheduled runs of +// a user's own Actor), because listing your own runs and reading a run's +// default store only needs runs:read + key-value-stores:read on that +// specific store. Use this pattern when Actor.openKeyValueStore('name') +// fails with "Permission denied". +// +// Both patterns require sanitizing keys — see `sanitizeKvKey` below — because +// KV keys must match /^[a-zA-Z0-9!\-_.'()]{1,256}$/. + +import { Actor, log } from 'apify'; + +/** + * KV keys must match /^[a-zA-Z0-9!\-_.'()]{1,256}$/. Anything else — colons, + * slashes, spaces, unicode — throws `ArgumentError` from setValue(). + * Replace disallowed characters with `_` and truncate to 256 chars. + */ +export function sanitizeKvKey(raw: string): string { + const cleaned = raw.replace(/[^a-zA-Z0-9!\-_.'()]/g, '_'); + return cleaned.slice(0, 256); +} + +// --------------------------------------------------------------------------- +// (A) Named store — full-scope token +// --------------------------------------------------------------------------- + +export async function loadStateFromNamedStore( + storeName: string, + key: string, +): Promise { + const store = await Actor.openKeyValueStore(storeName); + return (await store.getValue(sanitizeKvKey(key))) ?? undefined; +} + +export async function saveStateToNamedStore( + storeName: string, + key: string, + value: T, +): Promise { + const store = await Actor.openKeyValueStore(storeName); + await store.setValue(sanitizeKvKey(key), value); +} + +// --------------------------------------------------------------------------- +// (B) Previous run's default KV store — LIMITED_PERMISSIONS token +// --------------------------------------------------------------------------- + +/** + * List this Actor's most recent SUCCEEDED runs (excluding the current one) + * and return the default KV store ID of the newest one, if any. + */ +export async function findPreviousRunKvStoreId(): Promise { + const client = Actor.newClient(); + const actorId = process.env.APIFY_ACTOR_ID; + const currentRunId = process.env.APIFY_ACTOR_RUN_ID; + + if (!actorId) { + log.warning('APIFY_ACTOR_ID not set — cross-run state lookup is only meaningful on the platform.'); + return undefined; + } + + const { items } = await client.actor(actorId).runs().list({ + status: 'SUCCEEDED', + desc: true, + limit: 10, + }); + + const previous = items.find((r) => r.id !== currentRunId); + return previous?.defaultKeyValueStoreId; +} + +export async function loadStateFromPreviousRun(key: string): Promise { + const storeId = await findPreviousRunKvStoreId(); + if (!storeId) return undefined; + + const client = Actor.newClient(); + const record = await client.keyValueStore(storeId).getRecord(sanitizeKvKey(key)); + return record?.value; +} + +// State written to the CURRENT run's default store is what the NEXT run will +// read via loadStateFromPreviousRun — no cross-store write needed. +export async function saveStateForNextRun(key: string, value: T): Promise { + await Actor.setValue(sanitizeKvKey(key), value); +} diff --git a/skills/apify-actor-development/references/robust-fetch.ts b/skills/apify-actor-development/references/robust-fetch.ts new file mode 100644 index 0000000..35209f4 --- /dev/null +++ b/skills/apify-actor-development/references/robust-fetch.ts @@ -0,0 +1,76 @@ +// Robust fetch helper for Actors — wraps every network call in try/catch, +// retries on transient failures with exponential backoff, and surfaces the +// final error through the Apify logger so it appears in the run log. +// +// Import from your Actor entry point: +// +// import { robustFetch } from './robust-fetch'; +// const data = await robustFetch('https://api.example.com/x'); + +import { Actor, log } from 'apify'; + +export interface RobustFetchOptions extends RequestInit { + /** Maximum number of attempts, including the first one. Default: 4. */ + retries?: number; + /** Base delay in milliseconds — the actual delay is `base * 2**attempt`. Default: 500. */ + backoffBaseMs?: number; + /** Fail after this many milliseconds per attempt. Default: 30_000. */ + timeoutMs?: number; + /** HTTP status codes that should be retried in addition to network errors. */ + retryStatuses?: number[]; +} + +const DEFAULT_RETRY_STATUSES = [408, 425, 429, 500, 502, 503, 504]; + +export async function robustFetch( + url: string, + options: RobustFetchOptions = {}, +): Promise { + const { + retries = 4, + backoffBaseMs = 500, + timeoutMs = 30_000, + retryStatuses = DEFAULT_RETRY_STATUSES, + ...init + } = options; + + let lastError: unknown; + + for (let attempt = 0; attempt < retries; attempt++) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + try { + const response = await fetch(url, { ...init, signal: controller.signal }); + clearTimeout(timer); + + if (!response.ok) { + if (retryStatuses.includes(response.status) && attempt < retries - 1) { + const wait = backoffBaseMs * 2 ** attempt; + log.warning(`fetch ${url} returned ${response.status}; retrying in ${wait}ms`); + await new Promise((r) => setTimeout(r, wait)); + continue; + } + throw new Error(`HTTP ${response.status} ${response.statusText} for ${url}`); + } + + // Assume JSON — callers that need a different content type should use fetch directly. + return (await response.json()) as T; + } catch (err) { + clearTimeout(timer); + lastError = err; + + const isLast = attempt >= retries - 1; + if (isLast) break; + + const wait = backoffBaseMs * 2 ** attempt; + log.warning(`fetch ${url} failed (attempt ${attempt + 1}/${retries}): ${(err as Error).message}; retrying in ${wait}ms`); + await new Promise((r) => setTimeout(r, wait)); + } + } + + // Surface the failure through the Apify logger so it appears in the run log, + // then rethrow so the caller can decide whether to Actor.fail() or continue. + log.exception(lastError as Error, `fetch ${url} failed after ${retries} attempts`); + throw lastError; +}