diff --git a/.agents/skills/wp-bench-execution-tests/SKILL.md b/.agents/skills/wp-bench-execution-tests/SKILL.md new file mode 100644 index 0000000..6c5beab --- /dev/null +++ b/.agents/skills/wp-bench-execution-tests/SKILL.md @@ -0,0 +1,125 @@ +--- +name: wp-bench-execution-tests +description: Add, revise, or review WP-Bench WordPress execution tests. Use when working on datasets/suites/*/execution JSON, runtime_checks, static_checks, reference_solution, expected_behavior, test ID filtering, WordPress API benchmark coverage, or PR review comments about execution test quality. +--- + +# WP-Bench Execution Tests + +Use this skill when adding or reviewing execution tests for WP-Bench. + +## Workflow + +1. Inspect nearby execution and knowledge tests before editing. Match the suite's organization, naming style, and category balance. +2. Treat the WordPress source/runtime as the authority. For modern APIs, verify behavior against WordPress 7.0 source or official field-guide docs before writing assertions. +3. Define the observable WordPress behavior first. Prompts should be specific enough to identify the intended API area and outcome, but should not give away exact implementation details that the test is meant to measure, such as a particular argument key, metadata field, or helper call. Ask for a behavior or artifact, not an arbitrary wrapper function, unless the function itself is the contract. +4. Keep `requirements` concise and model-facing. They are appended to the prompt. +5. Keep `expected_behavior` reviewer-facing. It documents the contract and review focus; it is not used for scoring. +6. Use `reference_solution` as the canonical passing implementation. It is for verification and maintenance, not model input. +7. Make static checks robust for the contract: require expected functions, methods, classes, hooks, slugs, schema keys, and other identifiers when their use is essential to the task. Do not require incidental helpers or checker calls that the runtime assertion can perform itself. +8. Make runtime checks test the behavior inside WordPress. Use built-in assertion types when they directly express the check, such as output containment or REST response checks. Use `custom_assertion` when the verifier needs PHP to inspect the result, such as checking a registered category, returned value, database state, capability result, dispatched hook, or computed WordPress output. +9. Verify `reference_solution` with `wp-bench run --check-reference-solution` for every new or modified execution test. + +## Field Semantics + +- `prompt`: The task sent to the model. +- `requirements`: Additional model-facing constraints. +- `expected_behavior`: Reviewer documentation. +- `reference_solution`: Canonical passing code used for author verification. +- `static_checks`: Coarse guardrails for required or forbidden code patterns. +- `runtime_checks.setup`: Optional PHP fixture setup evaluated before the submitted code. +- `runtime_checks.assertions`: WordPress-executed behavioral assertions evaluated after the submitted code. +- `runtime_checks.teardown`: Optional PHP cleanup evaluated after assertions, even when setup, submitted code, or assertions fail. Use it for cleanup, not correctness. +- `metadata.source_refs`: Required source pointers. + +## Prompt And Assertion Shape + +Write tests around the contract, not the harness mechanics. + +Good: + +```json +{ + "prompt": "Register an Abilities API category with the slug 'wpbp-tools' so it is discoverable by WordPress.", + "static_checks": { + "required_patterns": [ + { "pattern": "wp_register_ability_category", "description": "Uses the Abilities category API", "weight": 1 }, + { "pattern": "wpbp-tools", "description": "Registers the requested category slug", "weight": 1 }, + { "pattern": "wp_abilities_api_categories_init", "description": "Uses the category init hook", "weight": 1 } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "return wp_has_ability_category( 'wpbp-tools' );", + "description": "The wpbp-tools category is discoverable", + "weight": 1 + } + ] + } +} +``` + +Avoid: + +- Requiring a wrapper function name unless implementing that function is the real task. +- Requiring the model to call the same checker API that the runtime assertion can call. +- Putting fixture cleanup inside assertions instead of `runtime_checks.teardown`. +- Adding cleanup by habit when the state is process-local. +- Making `prompt` and `expected_behavior` duplicates. + +Static check patterns are regular expressions. Delimiterless patterns are wrapped by the runtime, so simple slugs like `wpbp/count-words` can be written without escaping. Use explicit regex delimiters only when flags are needed, such as `/pattern/i`. + +Use the pattern list to enforce important API surface, not just one token from the prompt. If a task requires retrieving an ability and executing it, check for the function, method, and ability name, such as `wp_get_ability`, `execute`, and `wpbp/add-one`. + +## Difficulty + +Treat `difficulty` as author-estimated implementation complexity, not scoring. + +- `basic`: One obvious API or behavior, minimal setup. +- `intermediate`: Combines multiple WordPress concepts or requires lifecycle timing, setup, teardown, or edge handling. +- `hard`: Requires newer/obscure APIs plus nontrivial interaction, permissions, schemas, REST exposure, block/editor internals, or runtime reasoning. + +Do not mark a test `hard` only because the API is new. + +## Setup, Teardown, And Isolation + +Runtime order is `setup`, submitted code, assertions, then `teardown`. + +Use `runtime_checks.setup` to create fixtures the submitted code or assertions need. Use `runtime_checks.teardown` to remove persistent fixtures and restore global state. Keep assertions focused on measuring behavior. + +Clean up state in `teardown` when it persists beyond the PHP process or can affect later assertions: + +- posts, users, terms, comments, options, metadata +- scheduled cron events and transients +- object cache values with reusable keys/groups +- files or uploads created during the test + +Avoid cleanup for in-process-only registries when each verifier run starts a fresh WP-CLI process. Extra cleanup can make failing cases noisy and less diagnostic. + +## Validation + +For each changed test, run: + +```bash +.venv/bin/python -m pytest python/tests/test_execution_dataset.py +.venv/bin/wp-bench run --config wp-bench.yaml --dry-run --test-type execution --test-id +.venv/bin/wp-bench run --config wp-bench.yaml --check-reference-solution --test-type execution --test-id +``` + +Require the dry run to select only the requested test ID or IDs. Require the reference-solution run to execute the selected tests through the real WordPress verifier, without model calls, and pass every selected test. + +For broad suite changes, also run: + +```bash +.venv/bin/wp-bench run --config wp-bench.yaml --dry-run --test-type execution +.venv/bin/wp-bench run --config wp-bench.yaml --check-reference-solution --test-type execution +.venv/bin/python datasets/export_dataset.py +git diff --check +``` + +## Determinism + +- AI Client tests must not make live provider calls or require credentials. +- Avoid network, uncontrolled time, random IDs without cleanup, and dependency on unrelated global state. +- Prefer deterministic WordPress fixtures created by setup code and removed by teardown when persistent. diff --git a/README.md b/README.md index 780c74c..8309286 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The official WordPress AI benchmark. Evaluate how well language models understan WP-Bench measures AI model capabilities across two dimensions: - **Knowledge** — Multiple-choice and short-answer questions testing WordPress concepts, APIs, and best practices -- **Execution** — Code generation tasks graded by a real WordPress runtime for correctness and quality +- **Execution** — Code generation tasks graded by static checks and runtime assertions in a real WordPress environment The benchmark uses WordPress itself as the grader, running generated code in a sandboxed environment with static analysis and runtime assertions. @@ -82,6 +82,8 @@ grader: run: suite: wp-core-v1 limit: 10 # limit tests (null = all) + test_ids: [] # optional explicit test IDs to run + dry_run: false # load/filter tests without calling models concurrency: 4 output: @@ -97,7 +99,9 @@ wp-bench run --config wp-bench.yaml # run with config file wp-bench run --model-name gpt-4o --limit 5 # quick single-model test wp-bench run --test-type knowledge # run only knowledge tests (no WordPress env needed) wp-bench run --test-type execution # run only execution tests -wp-bench dry-run --config wp-bench.yaml # validate config without calling models +wp-bench run --test-type execution --test-id e-abilities-api-001 +wp-bench run --test-id e-abilities-api-001 --test-id e-rest-api-001 +wp-bench run --config wp-bench.yaml --dry-run # validate config without calling models ``` ## Repository Structure @@ -146,16 +150,11 @@ The notebook generates: ## How Grading Works 1. The harness sends a prompt to the model requesting WordPress code -2. Generated code is sent to the WordPress runtime via WP-CLI +2. Generated code is sent to the WordPress runtime 3. The runtime performs static analysis (syntax, coding standards, security) 4. Code executes in a sandbox with test assertions 5. Results return as JSON with scores and detailed feedback -```bash -# Manual grading example (run from runtime/ directory) -npm run wp-bench -- verify --payload=$(echo '{"code":" list[dict]: "test_kind": "execution", "type": "execution", "prompt": t["prompt"], + "expected_behavior": t.get("expected_behavior", ""), "category": t.get("category", "general"), "difficulty": t.get("difficulty", "unknown"), "choices": orjson.dumps(t.get("choices", [])).decode(), @@ -44,7 +45,6 @@ def load_suite(suite_name: str) -> list[dict]: "requirements": orjson.dumps(t.get("requirements", [])).decode(), "static_checks": orjson.dumps(t.get("static_checks", {})).decode(), "runtime_checks": orjson.dumps(t.get("runtime_checks", {})).decode(), - "judge_config": orjson.dumps(t.get("judge_config", {})).decode(), "reference_solution": t.get("reference_solution", ""), }) @@ -60,6 +60,7 @@ def load_suite(suite_name: str) -> list[dict]: "test_kind": "knowledge", "type": t.get("type", "knowledge"), "prompt": t["prompt"], + "expected_behavior": "", "category": t.get("category", "general"), "difficulty": t.get("difficulty", "unknown"), "choices": orjson.dumps(t.get("choices", [])).decode(), @@ -68,7 +69,6 @@ def load_suite(suite_name: str) -> list[dict]: "requirements": "[]", "static_checks": "{}", "runtime_checks": "{}", - "judge_config": "{}", "reference_solution": "", }) diff --git a/datasets/suites/wp-core-v1/execution/abilities-api.json b/datasets/suites/wp-core-v1/execution/abilities-api.json index b6dfefa..3304730 100644 --- a/datasets/suites/wp-core-v1/execution/abilities-api.json +++ b/datasets/suites/wp-core-v1/execution/abilities-api.json @@ -1,53 +1,110 @@ { "id": "wp-core-execution-v1-abilities_api", - "version": "1.1.0", + "version": "2.0.0", "metadata": { "name": "WordPress Core Execution Tests - Abilities API", - "description": "Code generation tasks for WordPress Abilities API", - "wp_version": "6.9", - "created_at": "2025-12-16" + "description": "Code generation tasks for WordPress Abilities API APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" }, "tests": [ { - "id": "e-abilities-001", + "id": "e-abilities-api-001", "category": "abilities-api", - "difficulty": "intermediate", - "prompt": "Register an ability category 'site-tools' and an ability 'site-tools/clear-cache' that clears caches. The ability should require manage_options, expose itself over REST, and return an array with status 'cleared'.", - "expected_behavior": "After registration, wp_has_ability('site-tools/clear-cache') is true; executing the ability returns ['status' => 'cleared'].", + "difficulty": "basic", + "prompt": "Register an Abilities API category with the slug 'wpbp-tools' so it is discoverable by WordPress.", + "expected_behavior": "Registers the `wpbp-tools` ability category with the public Abilities API on `wp_abilities_api_categories_init`, so WordPress can discover it through `wp_has_ability_category()`. The test focuses on category discovery through WordPress, not a custom registry or helper.", "requirements": [ - "Register category on 'wp_abilities_api_categories_init'", - "Register ability on 'wp_abilities_api_init'", - "Ability has label and description", - "Permission callback checks current_user_can('manage_options')", - "Execute callback returns ['status' => 'cleared']", - "Enable REST exposure via meta.show_in_rest = true" + "Use WordPress Abilities API", + "Register the category slug as 'wpbp-tools'", + "Register the category on wp_abilities_api_categories_init" ], "static_checks": { "required_patterns": [ { "pattern": "wp_register_ability_category", - "description": "Registers ability category", - "weight": 1.0 + "description": "Uses wp_register_ability_category", + "weight": 1 + }, + { + "pattern": "wpbp-tools", + "description": "Registers the requested category slug", + "weight": 1 }, + { + "pattern": "wp_abilities_api_categories_init", + "description": "Uses wp_abilities_api_categories_init", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "return wp_has_ability_category( 'wpbp-tools' );", + "description": "The wpbp-tools category is discoverable", + "weight": 1 + } + ] + }, + "reference_solution": "add_action( 'wp_abilities_api_categories_init', function () { wp_register_ability_category( 'wpbp-tools', array( 'label' => 'WPBP Tools', 'description' => 'Abilities for WPBP tools.' ) ); } );", + "metadata": { + "source_refs": [ + "src/wp-includes/abilities-api.php", + "src/wp-includes/abilities-api/class-wp-ability-category.php", + "src/wp-includes/abilities-api/class-wp-ability-categories-registry.php" + ], + "release_focus": "6.9" + } + }, + { + "id": "e-abilities-api-002", + "category": "abilities-api", + "difficulty": "intermediate", + "prompt": "Register an Abilities API ability named 'wpbp/count-words' in the 'wpbp-content' category. It must accept an object input with a required 'text' string, return an integer, and count the words in the supplied text.", + "expected_behavior": "Registers the `wpbp-content` category and the `wpbp/count-words` ability through the Abilities API init hooks. The ability has an object input schema requiring a `text` string, an integer output schema, and an execute callback that returns the word count for the supplied text. Runtime validation inspects the registered `WP_Ability` and executes it with deterministic input.", + "requirements": [ + "Use WordPress Abilities API", + "Register the 'wpbp-content' category before the ability", + "Use input_schema and output_schema for the ability contract" + ], + "static_checks": { + "required_patterns": [ { "pattern": "wp_register_ability", - "description": "Registers ability", - "weight": 1.0 + "description": "Uses wp_register_ability", + "weight": 1 }, { - "pattern": "wp_abilities_api_init", - "description": "Uses correct init hook", - "weight": 0.6 + "pattern": "wp_register_ability_category", + "description": "Registers the custom ability category", + "weight": 1 }, { - "pattern": "permission_callback", - "description": "Declares permission callback", - "weight": 0.4 + "pattern": "wpbp/count-words", + "description": "Registers the requested ability name", + "weight": 1 }, { - "pattern": "show_in_rest", - "description": "Opt-in to REST exposure", - "weight": 0.3 + "pattern": "input_schema", + "description": "Uses input_schema", + "weight": 1 + }, + { + "pattern": "output_schema", + "description": "Uses output_schema", + "weight": 1 + }, + { + "pattern": "wp_abilities_api_init", + "description": "Uses wp_abilities_api_init", + "weight": 1 + }, + { + "pattern": "wp_abilities_api_categories_init", + "description": "Uses wp_abilities_api_categories_init", + "weight": 1 } ] }, @@ -55,62 +112,151 @@ "assertions": [ { "type": "custom_assertion", - "code": "return function_exists('wp_has_ability_category') && wp_has_ability_category('site-tools');", - "description": "Category exists", - "weight": 0.8 + "code": "WP_Abilities_Registry::get_instance(); $ability = wp_get_ability( 'wpbp/count-words' ); if ( ! $ability instanceof WP_Ability ) { return false; } $input_schema = $ability->get_input_schema(); $output_schema = $ability->get_output_schema(); return 'wpbp-content' === $ability->get_category() && 'object' === ( $input_schema['type'] ?? null ) && 'string' === ( $input_schema['properties']['text']['type'] ?? null ) && isset( $input_schema['required'] ) && in_array( 'text', $input_schema['required'], true ) && 'integer' === ( $output_schema['type'] ?? null ) && 3 === $ability->execute( array( 'text' => 'one two three' ) );", + "description": "The count-words ability is registered with schemas and executes correctly", + "weight": 1 + } + ] + }, + "reference_solution": "add_action( 'wp_abilities_api_categories_init', function () { wp_register_ability_category( 'wpbp-content', array( 'label' => 'Content', 'description' => 'Content analysis abilities.' ) ); } ); add_action( 'wp_abilities_api_init', function () { wp_register_ability( 'wpbp/count-words', array( 'label' => 'Count words', 'description' => 'Counts the words in supplied text.', 'category' => 'wpbp-content', 'input_schema' => array( 'type' => 'object', 'properties' => array( 'text' => array( 'type' => 'string' ) ), 'required' => array( 'text' ) ), 'output_schema' => array( 'type' => 'integer' ), 'execute_callback' => static fn( array $input ): int => str_word_count( $input['text'] ), 'permission_callback' => '__return_true' ) ); } );", + "metadata": { + "source_refs": [ + "src/wp-includes/abilities-api.php", + "src/wp-includes/abilities-api/class-wp-ability.php", + "src/wp-includes/abilities-api/class-wp-abilities-registry.php", + "src/wp-includes/abilities-api/class-wp-ability-categories-registry.php" + ], + "release_focus": "6.9" + } + }, + { + "id": "e-abilities-api-003", + "category": "abilities-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_abilities_api_003( int $number ) to retrieve the registered 'wpbp/add-one' ability and return the result of executing it with the supplied number.", + "expected_behavior": "`wpbp_abilities_api_003()` retrieves the pre-registered `wpbp/add-one` ability with `wp_get_ability()` and returns the result from `WP_Ability::execute()` for the supplied number. The setup registers the fixture ability, and runtime validation checks multiple inputs.", + "requirements": [ + "Use wp_get_ability to retrieve 'wpbp/add-one'", + "Call the ability object's execute method", + "Return the execution result" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_abilities_api_003", + "description": "Defines the requested function", + "weight": 0.5 }, { - "type": "custom_assertion", - "code": "return function_exists('wp_has_ability') && wp_has_ability('site-tools/clear-cache');", - "description": "Ability is registered", - "weight": 1.0 + "pattern": "wp_get_ability", + "description": "Uses wp_get_ability", + "weight": 1 + }, + { + "pattern": "wpbp/add-one", + "description": "Retrieves the requested ability name", + "weight": 1 }, + { + "pattern": "execute", + "description": "Uses execute", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "add_action( 'wp_abilities_api_categories_init', function () { wp_register_ability_category( 'wpbp-math', array( 'label' => 'Math', 'description' => 'Math abilities.' ) ); } ); add_action( 'wp_abilities_api_init', function () { wp_register_ability( 'wpbp/add-one', array( 'label' => 'Add one', 'description' => 'Adds one to an integer.', 'category' => 'wpbp-math', 'input_schema' => array( 'type' => 'integer' ), 'output_schema' => array( 'type' => 'integer' ), 'execute_callback' => static fn( int $input ): int => $input + 1, 'permission_callback' => '__return_true' ) ); } ); WP_Abilities_Registry::get_instance();", + "assertions": [ { "type": "custom_assertion", - "code": "$ability = function_exists('wp_get_ability') ? wp_get_ability('site-tools/clear-cache') : null; if ( ! $ability ) { return false; } $result = $ability->execute( array( 'force' => true ) ); return is_array( $result ) && isset( $result['status'] ) && 'cleared' === $result['status'];", - "description": "Ability executes and returns expected payload", - "weight": 1.0 + "code": "if ( ! function_exists( 'wpbp_abilities_api_003' ) ) { return false; } return 5 === wpbp_abilities_api_003( 4 ) && -1 === wpbp_abilities_api_003( -2 );", + "description": "The helper returns results from executing the registered ability", + "weight": 1 } ] }, - "judge_config": { - "rubric_id": "wp-judge-rubric-v1", - "context_for_judge": "Focus on correct hooks, permissions, REST exposure, and execution payload." + "reference_solution": "function wpbp_abilities_api_003( int $number ) { $ability = wp_get_ability( 'wpbp/add-one' ); if ( ! $ability instanceof WP_Ability ) { return new WP_Error( 'wpbp_missing_ability', 'The wpbp/add-one ability is not registered.' ); } return $ability->execute( $number ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/abilities-api.php", + "src/wp-includes/abilities-api/class-wp-ability.php", + "src/wp-includes/abilities-api/class-wp-abilities-registry.php" + ], + "release_focus": "6.9" } }, { - "id": "e-abilities-cache-001", + "id": "e-abilities-api-004", "category": "abilities-api", - "difficulty": "hard", - "prompt": "Register an Abilities API ability that returns a cached list of published 'movie' titles, uses salted cache keys, and stays discoverable via REST.", - "expected_behavior": "GET /wp-abilities/v1/abilities/demo/get-movies/run returns titles; second call is a cache hit; inserting a new movie causes the next call to reflect the new title without changing the cache key.", + "difficulty": "intermediate", + "prompt": "Register an Abilities API ability named 'wpbp/rest-visible' in the 'wpbp-rest-abilities' category and configure the ability to be available through the Abilities REST API.", + "expected_behavior": "Registers the `wpbp-rest-abilities` category and the `wpbp/rest-visible` ability through the Abilities API init hooks, with `meta.show_in_rest` set to true on the ability. Runtime validation inspects the registered ability metadata and executes its callback; REST visibility should come from Abilities API metadata, not a custom REST route.", "requirements": [ - "Register the ability on wp_abilities_api_init; category on wp_abilities_api_categories_init", - "Ability name demo/get-movies with input schema allowing optional 'genre' string; output is array of strings", - "Use wp_cache_get_salted/wp_cache_set_salted with post last_changed as salt; group 'post-queries'", - "Permission callback requires read for logged-in users; ability has meta.show_in_rest=true" + "Use WordPress Abilities API", + "Configure the ability for Abilities REST API availability", + "Do not create a custom REST route" ], "static_checks": { "required_patterns": [ - { "pattern": "wp_register_ability", "description": "Ability registration", "weight": 1.0 }, - { "pattern": "wp_cache_get_salted", "description": "Uses new cache helpers", "weight": 0.8 }, - { "pattern": "show_in_rest", "description": "REST exposure enabled", "weight": 0.4 } + { + "pattern": "show_in_rest", + "description": "Uses show_in_rest", + "weight": 1 + }, + { + "pattern": "wp_register_ability", + "description": "Uses wp_register_ability", + "weight": 1 + }, + { + "pattern": "wp_register_ability_category", + "description": "Registers the REST ability category", + "weight": 1 + }, + { + "pattern": "wpbp/rest-visible", + "description": "Registers the requested ability name", + "weight": 1 + }, + { + "pattern": "wp_abilities_api_init", + "description": "Uses wp_abilities_api_init", + "weight": 1 + }, + { + "pattern": "wp_abilities_api_categories_init", + "description": "Uses wp_abilities_api_categories_init", + "weight": 1 + } ], "forbidden_patterns": [ - { "pattern": "transient", "description": "Do not use transients for this cache", "severity": "error" } + { + "pattern": "register_rest_route", + "description": "Do not create a custom REST route for Abilities API exposure", + "severity": "error" + } ] }, "runtime_checks": { - "setup": "register_post_type('movie'); wp_insert_post(['post_type'=>'movie','post_title'=>'First']);", "assertions": [ - { "type": "http_body_contains", "target": "/wp-abilities/v1/abilities/demo/get-movies/run", "expected": "First", "description": "Ability returns data", "weight": 1.0 }, - { "type": "cache_miss_then_hit", "target": "/wp-abilities/v1/abilities/demo/get-movies/run", "description": "Second call reuses salted cache", "weight": 1.0 }, - { "type": "cache_invalidated_on_hook", "target": "save_post_movie", "description": "Adding a movie causes refreshed data but same cache key", "weight": 1.0 } - ], - "teardown": "unregister_post_type('movie');" + { + "type": "custom_assertion", + "code": "WP_Abilities_Registry::get_instance(); $ability = wp_get_ability( 'wpbp/rest-visible' ); if ( ! $ability instanceof WP_Ability ) { return false; } $meta = $ability->get_meta(); return 'wpbp-rest-abilities' === $ability->get_category() && true === ( $meta['show_in_rest'] ?? false ) && array( 'ok' => true ) === $ability->execute();", + "description": "The ability is registered with REST visibility metadata", + "weight": 1 + } + ] }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "add_action( 'wp_abilities_api_categories_init', function() {\n wp_register_ability_category( 'demo', array( 'label' => 'Demo' ) );\n} );\nadd_action( 'wp_abilities_api_init', function() {\n wp_register_ability( 'demo/get-movies', array(\n 'label' => 'Get Movies',\n 'category' => 'demo',\n 'input_schema' => array(\n 'type' => 'object',\n 'properties' => array( 'genre' => array( 'type' => 'string' ) ),\n ),\n 'output_schema' => array( 'type' => 'array', 'items' => array( 'type' => 'string' ) ),\n 'permission_callback' => fn() => is_user_logged_in() && current_user_can( 'read' ),\n 'callback' => function () {\n $salt = wp_cache_get_last_changed( 'posts' );\n $group = 'post-queries';\n $key = 'movies:list';\n $cached = wp_cache_get_salted( $key, $group, $salt, $found );\n if ( $found ) {\n return $cached;\n }\n $q = new WP_Query( array( 'post_type' => 'movie', 'post_status' => 'publish', 'fields' => 'titles', 'nopaging' => true ) );\n $titles = wp_list_pluck( $q->posts, 'post_title' );\n wp_cache_set_salted( $key, $titles, $group, $salt );\n return $titles;\n },\n 'meta' => array( 'show_in_rest' => true ),\n ) );\n} );\nadd_action( 'save_post_movie', fn() => wp_cache_delete( 'movies:list', 'post-queries' ) );\nadd_action( 'trashed_post', fn( $pid ) => wp_cache_delete( 'movies:list', 'post-queries' ) );" + "reference_solution": "add_action( 'wp_abilities_api_categories_init', function () { wp_register_ability_category( 'wpbp-rest-abilities', array( 'label' => 'REST Abilities', 'description' => 'Abilities exposed through REST.' ) ); } ); add_action( 'wp_abilities_api_init', function () { wp_register_ability( 'wpbp/rest-visible', array( 'label' => 'REST Visible', 'description' => 'Returns a small REST-visible payload.', 'category' => 'wpbp-rest-abilities', 'output_schema' => array( 'type' => 'object' ), 'execute_callback' => static fn() : array => array( 'ok' => true ), 'permission_callback' => '__return_true', 'meta' => array( 'show_in_rest' => true ) ) ); } );", + "metadata": { + "source_refs": [ + "src/wp-includes/abilities-api.php", + "src/wp-includes/abilities-api/class-wp-ability.php", + "src/wp-includes/abilities-api/class-wp-abilities-registry.php", + "src/wp-includes/abilities-api/class-wp-ability-categories-registry.php" + ], + "release_focus": "6.9" + } } ] } diff --git a/datasets/suites/wp-core-v1/execution/ai-client.json b/datasets/suites/wp-core-v1/execution/ai-client.json new file mode 100644 index 0000000..d8838d2 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/ai-client.json @@ -0,0 +1,310 @@ +{ + "id": "wp-core-execution-v1-ai_client", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - AI Client", + "description": "Code generation tasks for WordPress AI Client APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-ai-client-001", + "category": "ai-client", + "difficulty": "basic", + "prompt": "Implement wpbp_ai_client_001(): bool so it temporarily disables WordPress AI support for its own check, returns the disabled support result, and leaves AI support enabled afterward.", + "expected_behavior": "`wpbp_ai_client_001()` uses the `wp_supports_ai` filter only around a call to `wp_supports_ai()`, returns `false` for that temporary check, and removes the filter before returning. The test focuses on the AI support gate and cleanup, not a hard-coded return value.", + "requirements": [ + "Use wp_supports_ai() to read the support state", + "Use the wp_supports_ai filter to force support off for the temporary check", + "Remove the filter before returning" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_ai_client_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_supports_ai", + "description": "Uses wp_supports_ai", + "weight": 1 + }, + { + "pattern": "add_filter", + "description": "Uses add_filter", + "weight": 1 + }, + { + "pattern": "remove_filter", + "description": "Removes the temporary filter", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_ai_client_001' ) ) { return false; } $result = wpbp_ai_client_001(); return false === $result && true === wp_supports_ai();", + "description": "The support check is disabled only temporarily", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_ai_client_001(): bool { add_filter( 'wp_supports_ai', '__return_false' ); $supported = wp_supports_ai(); remove_filter( 'wp_supports_ai', '__return_false' ); return $supported; }", + "metadata": { + "source_refs": [ + "src/wp-includes/ai-client.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-ai-client-002", + "category": "ai-client", + "difficulty": "basic", + "prompt": "Implement wpbp_ai_client_002(): WP_AI_Client_Prompt_Builder to create a text prompt builder with a concise system instruction, temperature 0.2, and a model preference order of 'gpt-4o-mini' then 'gpt-4o'. Return the builder without generating content.", + "expected_behavior": "`wpbp_ai_client_002()` returns a `WP_AI_Client_Prompt_Builder` from `wp_ai_client_prompt()` after configuring text-generation options with snake_case AI Client methods. It must not call a generating method; runtime validation later prevents generation and confirms the returned builder is not already in an error state.", + "requirements": [ + "Use wp_ai_client_prompt()", + "Configure temperature, system instruction, and model preference", + "Return the builder without generating content" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_ai_client_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_ai_client_prompt", + "description": "Uses wp_ai_client_prompt", + "weight": 1 + }, + { + "pattern": "using_temperature", + "description": "Uses using_temperature", + "weight": 1 + }, + { + "pattern": "using_system_instruction", + "description": "Uses using_system_instruction", + "weight": 1 + }, + { + "pattern": "using_model_preference", + "description": "Uses using_model_preference", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "/\\b(?:generate_(?:result|text(?:s|_result)?|image(?:s|_result)?|speech(?:es|_result)?|video(?:s|_result)?)|convert_text_to_speech(?:es|_result)?)\\b/", + "description": "Does not generate content in the helper", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_ai_client_002' ) ) { return false; } $builder = wpbp_ai_client_002(); if ( ! $builder instanceof WP_AI_Client_Prompt_Builder ) { return false; } add_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); $result = $builder->generate_text(); remove_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); return is_wp_error( $result ) && 'prompt_prevented' === $result->get_error_code();", + "description": "The helper returns a usable builder without calling a provider", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_ai_client_002(): WP_AI_Client_Prompt_Builder { return wp_ai_client_prompt( 'Summarize WordPress.' )->using_system_instruction( 'Be concise.' )->using_temperature( 0.2 )->using_model_preference( 'gpt-4o-mini', 'gpt-4o' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/ai-client.php", + "src/wp-includes/ai-client/class-wp-ai-client-prompt-builder.php", + "src/wp-includes/php-ai-client/src/Builders/PromptBuilder.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-ai-client-003", + "category": "ai-client", + "difficulty": "intermediate", + "prompt": "Implement wpbp_ai_client_003(): string to prevent a single AI text generation attempt and return the resulting WP_Error code.", + "expected_behavior": "`wpbp_ai_client_003()` temporarily enables `wp_ai_client_prevent_prompt`, calls `generate_text()` on an AI Client prompt, returns the resulting `WP_Error` code, and removes the prevention filter. The expected code is `prompt_prevented`, with no live provider call.", + "requirements": [ + "Use wp_ai_client_prompt() and generate_text()", + "Use wp_ai_client_prevent_prompt to prevent the generation call", + "Remove the filter before returning" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_ai_client_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_ai_client_prevent_prompt", + "description": "Uses wp_ai_client_prevent_prompt", + "weight": 1 + }, + { + "pattern": "wp_ai_client_prompt", + "description": "Uses wp_ai_client_prompt", + "weight": 1 + }, + { + "pattern": "generate_text", + "description": "Uses generate_text", + "weight": 1 + }, + { + "pattern": "remove_filter", + "description": "Removes the temporary filter", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_ai_client_003' ) ) { return false; } $code = wpbp_ai_client_003(); $prevented_after = apply_filters( 'wp_ai_client_prevent_prompt', false, wp_ai_client_prompt( 'Probe' ) ); return 'prompt_prevented' === $code && false === $prevented_after;", + "description": "Prompt generation is prevented and the filter is restored", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_ai_client_003(): string { add_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); try { $result = wp_ai_client_prompt( 'Blocked' )->generate_text(); } finally { remove_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); } return is_wp_error( $result ) ? $result->get_error_code() : ''; }", + "metadata": { + "source_refs": [ + "src/wp-includes/ai-client.php", + "src/wp-includes/ai-client/class-wp-ai-client-prompt-builder.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-ai-client-004", + "category": "ai-client", + "difficulty": "basic", + "prompt": "Implement wpbp_ai_client_004( string $ability_name ): array to return the AI Client function-call name for the ability and the ability name obtained by converting it back.", + "expected_behavior": "`wpbp_ai_client_004()` uses `WP_AI_Client_Ability_Function_Resolver` conversion helpers to return `array( $function_name, $round_trip_ability_name )` for any supplied ability name. Runtime validation checks multiple ability names so hard-coded output does not pass.", + "requirements": [ + "Use WP_AI_Client_Ability_Function_Resolver", + "Use both conversion directions", + "Return the function-call name and round-tripped ability name" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_ai_client_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_AI_Client_Ability_Function_Resolver", + "description": "Uses WP_AI_Client_Ability_Function_Resolver", + "weight": 1 + }, + { + "pattern": "ability_name_to_function_name", + "description": "Uses ability_name_to_function_name", + "weight": 1 + }, + { + "pattern": "function_name_to_ability_name", + "description": "Uses function_name_to_ability_name", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_ai_client_004' ) ) { return false; } $first = wpbp_ai_client_004( 'wpbp/do-thing' ); $second = wpbp_ai_client_004( 'my-plugin/export_data' ); return is_array( $first ) && is_array( $second ) && 'wpab__wpbp__do-thing' === ( $first[0] ?? null ) && 'wpbp/do-thing' === ( $first[1] ?? null ) && 'wpab__my-plugin__export_data' === ( $second[0] ?? null ) && 'my-plugin/export_data' === ( $second[1] ?? null );", + "description": "Ability names round-trip through AI function-call names", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_ai_client_004( string $ability ): array { $function = WP_AI_Client_Ability_Function_Resolver::ability_name_to_function_name( $ability ); return array( $function, WP_AI_Client_Ability_Function_Resolver::function_name_to_ability_name( $function ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/ai-client/class-wp-ai-client-ability-function-resolver.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-ai-client-005", + "category": "ai-client", + "difficulty": "intermediate", + "prompt": "Implement wpbp_ai_client_005(): bool to return the text-generation support check result while AI prompt execution is temporarily prevented.", + "expected_behavior": "`wpbp_ai_client_005()` temporarily enables `wp_ai_client_prevent_prompt`, calls `is_supported_for_text_generation()` on an AI Client prompt, returns that support-check result, and removes the filter. The support check should return `false` without making a provider call.", + "requirements": [ + "Use wp_ai_client_prompt()", + "Use is_supported_for_text_generation()", + "Remove the prevention filter before returning" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_ai_client_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "is_supported_for_text_generation", + "description": "Uses is_supported_for_text_generation", + "weight": 1 + }, + { + "pattern": "wp_ai_client_prompt", + "description": "Uses wp_ai_client_prompt", + "weight": 1 + }, + { + "pattern": "wp_ai_client_prevent_prompt", + "description": "Uses wp_ai_client_prevent_prompt", + "weight": 1 + }, + { + "pattern": "remove_filter", + "description": "Removes the temporary filter", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "/\\b(?:generate_(?:result|text(?:s|_result)?|image(?:s|_result)?|speech(?:es|_result)?|video(?:s|_result)?)|convert_text_to_speech(?:es|_result)?)\\b/", + "description": "Does not generate content for a support check", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_ai_client_005' ) ) { return false; } $supported = wpbp_ai_client_005(); $prevented_after = apply_filters( 'wp_ai_client_prevent_prompt', false, wp_ai_client_prompt( 'Probe' ) ); return false === $supported && false === $prevented_after;", + "description": "The support check returns false while prevention is active and the filter is restored", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_ai_client_005(): bool { add_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); try { $supported = wp_ai_client_prompt( 'Check' )->is_supported_for_text_generation(); } finally { remove_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); } return $supported; }", + "metadata": { + "source_refs": [ + "src/wp-includes/ai-client.php", + "src/wp-includes/ai-client/class-wp-ai-client-prompt-builder.php" + ], + "release_focus": "7.0" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/block-api.json b/datasets/suites/wp-core-v1/execution/block-api.json deleted file mode 100644 index 68b0df5..0000000 --- a/datasets/suites/wp-core-v1/execution/block-api.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "id": "wp-core-execution-v1-block_api", - "version": "1.1.0", - "metadata": { - "name": "WordPress Core Execution Tests - Block API", - "description": "Code generation tasks for WordPress Block API", - "wp_version": "6.9", - "created_at": "2025-12-16" - }, - "tests": [ - { - "id": "e-block-processor-count-001", - "category": "block-api", - "difficulty": "hard", - "prompt": "Implement wpbp_count_block_types($html) that uses the WP_Block_Processor streaming parser (not parse_blocks) to count each block type in the provided content, including nested blocks.", - "expected_behavior": "Returns an associative array like ['core/group' => 1, 'core/paragraph' => 2, 'core/buttons' => 1, 'core/button' => 1] for nested block content; ignores freeform HTML chunks.", - "requirements": [ - "Use WP_Block_Processor to iterate blocks; do not call parse_blocks()", - "Count nested blocks and merge counts", - "Skip blocks with empty block_name or core/freeform fallback", - "Return associative array keyed by block name with integer counts" - ], - "static_checks": { - "required_patterns": [ - { "pattern": "WP_Block_Processor", "description": "Uses streaming block parser", "weight": 1.0 }, - { "pattern": "extract_full_block_and_advance", "description": "Extracts blocks while advancing", "weight": 0.7 } - ], - "forbidden_patterns": [ - { "pattern": "parse_blocks", "description": "Must avoid full parse for performance", "severity": "error" } - ] - }, - "runtime_checks": { - "assertions": [ - { - "type": "custom_assertion", - "code": "if ( ! function_exists('wpbp_count_block_types') ) { return false; } $html = '

One

Go

Two

Raw text outside blocks'; $result = wpbp_count_block_types( $html ); $expected = array( 'core/group' => 1, 'core/paragraph' => 2, 'core/buttons' => 1, 'core/button' => 1 ); foreach ( $expected as $key => $val ) { if ( ! isset( $result[ $key ] ) || (int) $result[ $key ] !== $val ) { return false; } } return empty( $result[''] ?? null );", - "description": "Counts nested blocks correctly and ignores freeform HTML", - "weight": 1.0 - } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_count_block_types( string $html ): array {\n $counts = array();\n $processor = new WP_Block_Processor( $html );\n while ( $block = $processor->extract_full_block_and_advance() ) {\n $name = $block->block_name;\n if ( ! $name || $name === 'core/freeform' ) {\n continue;\n }\n $counts[ $name ] = ($counts[ $name ] ?? 0) + 1;\n if ( ! empty( $block->inner_html ) ) {\n foreach ( wpbp_count_block_types( $block->inner_html ) as $k => $v ) {\n $counts[ $k ] = ($counts[ $k ] ?? 0) + $v;\n }\n }\n }\n return $counts;\n}" - } - ] -} diff --git a/datasets/suites/wp-core-v1/execution/block-bindings.json b/datasets/suites/wp-core-v1/execution/block-bindings.json deleted file mode 100644 index 9d380b9..0000000 --- a/datasets/suites/wp-core-v1/execution/block-bindings.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "id": "wp-core-execution-v1-block_bindings", - "version": "1.1.0", - "metadata": { - "name": "WordPress Core Execution Tests - Block Bindings", - "description": "Code generation tasks for WordPress Block Bindings API", - "wp_version": "6.9", - "created_at": "2025-12-16" - }, - "tests": [ - { - "id": "e-bindings-001", - "category": "block-bindings", - "difficulty": "intermediate", - "prompt": "Register a custom block bindings source named 'myplugin/meta-caption' that returns the post meta value 'featured_image_caption'. The source should require post context, be registered on 'init', and be usable by blocks like core/image captions.", - "expected_behavior": "A bindings source 'myplugin/meta-caption' exists; when invoked with a post context containing meta 'featured_image_caption', it returns that value.", - "requirements": [ - "Hook registration into the 'init' action", - "Call register_block_bindings_source('myplugin/meta-caption', ...)", - "Provide label and get_value_callback", - "Include uses_context with postId", - "Callback reads get_post_meta( $post_id, 'featured_image_caption', true )" - ], - "static_checks": { - "required_patterns": [ - { - "pattern": "register_block_bindings_source", - "description": "Must register a custom bindings source", - "weight": 1.0 - }, - { - "pattern": "myplugin/meta-caption", - "description": "Uses the correct source name", - "weight": 0.6 - }, - { - "pattern": "get_post_meta", - "description": "Retrieves meta for the caption", - "weight": 0.6 - }, - { - "pattern": "uses_context", - "description": "Declares context needed for callback", - "weight": 0.3 - }, - { - "pattern": "add_action\\s*\\(\\s*['\"]init['\"]", - "description": "Registers source during init", - "weight": 0.3 - } - ] - }, - "runtime_checks": { - "assertions": [ - { - "type": "custom_assertion", - "code": "$registry = WP_Block_Bindings_Registry::get_instance(); if ( ! method_exists( $registry, 'is_registered' ) ) { return false; } return $registry->is_registered('myplugin/meta-caption');", - "description": "Bindings source is registered", - "weight": 1.0 - }, - { - "type": "custom_assertion", - "code": "$post_id = wp_insert_post(['post_title' => 'Caption Test', 'post_status' => 'publish']);\\nupdate_post_meta($post_id, 'featured_image_caption', 'Bound Caption');\\n$registry = WP_Block_Bindings_Registry::get_instance();\\n$source = method_exists( $registry, 'get_registered' ) ? $registry->get_registered('myplugin/meta-caption') : null;\\n$callback = $source['get_value_callback'] ?? null;\\n$value = $callback ? call_user_func($callback, [], (object)['context' => ['postId' => $post_id]], 'caption') : null;\\nwp_delete_post($post_id, true);\\nreturn $value === 'Bound Caption';", - "description": "Callback returns stored caption meta", - "weight": 1.0 - } - ] - }, - "judge_config": { - "rubric_id": "wp-judge-rubric-v1", - "context_for_judge": "Ensure bindings source uses post context and returns the 'featured_image_caption' meta value." - } - }, - { - "id": "e-block-bindings-aria-001", - "category": "block-bindings", - "difficulty": "hard", - "prompt": "Extend Block Bindings so the core/button block can bind its ariaLabel attribute to post meta, then prove the bound value renders on the front end.", - "expected_behavior": "A page containing a button bound to meta key 'cta_label' renders with aria-label='Draft CTA' pulled from post meta.", - "requirements": [ - "Use block_bindings_supported_attributes_core/button filter to allow ariaLabel binding", - "Bind both text and ariaLabel to the same meta key via core/post-meta source", - "Sanitize the ariaLabel before output", - "Do not modify block.json; use PHP filters only" - ], - "static_checks": { - "required_patterns": [ - { "pattern": "block_bindings_supported_attributes_core/button", "description": "Attribute extension hook", "weight": 1.0 }, - { "pattern": "ariaLabel", "description": "Targets ariaLabel attribute", "weight": 0.8 } - ], - "forbidden_patterns": [ - { "pattern": "register_block_type_from_metadata", "description": "No block.json fork", "severity": "error" } - ] - }, - "runtime_checks": { - "setup": "create page with postmeta cta_label='Draft CTA' and content containing bound button", - "assertions": [ - { "type": "http_body_contains", "target": "/?page_id={created_page_id}", "expected": "aria-label=\"Draft CTA\"", "description": "Front-end renders bound aria-label", "weight": 1.0 }, - { "type": "http_body_contains", "target": "/?page_id={created_page_id}", "expected": ">Draft CTA<", "description": "Button text also bound", "weight": 0.8 } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "add_filter( 'block_bindings_supported_attributes_core/button', function( $attrs ) {\n if ( ! in_array( 'ariaLabel', $attrs, true ) ) {\n $attrs[] = 'ariaLabel';\n }\n return $attrs;\n} );\nadd_filter( 'render_block', function( $content, $block ) {\n if ( ( $block['blockName'] ?? '' ) !== 'core/button' ) {\n return $content;\n }\n if ( empty( $block['attrs']['ariaLabel'] ) ) {\n return $content;\n }\n $aria = esc_attr( $block['attrs']['ariaLabel'] );\n // Ensure sanitized aria-label is present even if theme filters markup.\n if ( str_contains( $content, 'aria-label=' ) ) {\n return preg_replace( '/aria-label=\"[^\"]*\"/', 'aria-label=\"' . $aria . '\"', $content );\n }\n return str_replace( ''Hook Target','post_status'=>'publish']); $page_id = wp_insert_post(['post_title'=>'Page Target','post_status'=>'publish','post_type'=>'page']); return true; ", - "assertions": [ - { "type": "http_body_contains", "target": "/?p={post_id}", "expected": "Hooked CTA", "description": "CTA appears on single post", "weight": 1.0 }, - { "type": "http_body_not_contains", "target": "/?page_id={page_id}", "expected": "Hooked CTA", "description": "CTA absent on page", "weight": 1.0 } - ], - "teardown": "wp_delete_post({post_id}, true); wp_delete_post({page_id}, true);" - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "add_action( 'wp', function () {\n if ( ! is_single() || get_post_type() !== 'post' ) {\n return;\n }\n add_filter( 'hooked_block_types', function( $hooks ) {\n $hooks['core/post-title'][] = array(\n 'blockName' => 'core/paragraph',\n 'attributes' => array( 'content' => 'Hooked CTA' ),\n 'position' => 'after',\n );\n return $hooks;\n } );\n} );" - } - ] -} diff --git a/datasets/suites/wp-core-v1/execution/caching.json b/datasets/suites/wp-core-v1/execution/caching.json index a608628..d8f8b13 100644 --- a/datasets/suites/wp-core-v1/execution/caching.json +++ b/datasets/suites/wp-core-v1/execution/caching.json @@ -1,80 +1,321 @@ { "id": "wp-core-execution-v1-caching", - "version": "1.1.0", + "version": "2.0.0", "metadata": { "name": "WordPress Core Execution Tests - Caching", - "description": "Code generation tasks for WordPress object caching", - "wp_version": "6.9", - "created_at": "2025-12-16" + "description": "Code generation tasks for WordPress Caching APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" }, "tests": [ { - "id": "e-cache-false-hit-001", + "id": "e-caching-001", "category": "caching", - "difficulty": "hard", - "prompt": "Implement wpbp_cached_flag() that caches a computed boolean false and returns the cached false on subsequent calls using the $found parameter to distinguish cache hits from misses.", - "expected_behavior": "First call computes false; second call returns false from cache without recomputing.", + "difficulty": "intermediate", + "prompt": "Implement wpbp_caching_001( string $key, string $group ): array to read an object-cache miss, store boolean false, then read the same key again while reporting both found flags.", + "expected_behavior": "Reviewer contract: `wpbp_caching_001()` returns array( $miss_value, $miss_found, $hit_value, $hit_found ) for the supplied object-cache key and group. Review focus: use the `$found` parameter on `wp_cache_get()` so a cached false is distinguishable from a miss, and store the false value through `wp_cache_set()`.", "requirements": [ - "Use wp_cache_get with $found to detect cached false values", - "Store results in cache group 'wpbp-cache-flag' with key 'flag'", - "Ensure recomputation does not occur on cache hit even when value is false" + "Use object cache APIs", + "Use the wp_cache_get() found flag to distinguish a cached false from a miss", + "Return array( $miss_value, $miss_found, $hit_value, $hit_found )" ], "static_checks": { "required_patterns": [ - { "pattern": "wp_cache_get\\s*\\(.*\\$found", "description": "Uses found flag on cache get", "weight": 1.0 }, - { "pattern": "wp_cache_set", "description": "Writes to cache", "weight": 0.8 } + { + "pattern": "wpbp_caching_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_cache_get\\s*\\([^)]*,\\s*[^)]*,\\s*[^)]*,\\s*\\$", + "description": "Reads cache values with the found flag", + "weight": 1 + }, + { + "pattern": "wp_cache_set", + "description": "Uses wp_cache_set", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "wp_cache_delete( 'wpbp_false_a', 'wpbp_found_a' ); wp_cache_delete( 'wpbp_false_b', 'wpbp_found_b' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_caching_001' ) ) { return false; } $cases = array( array( 'wpbp_false_a', 'wpbp_found_a' ), array( 'wpbp_false_b', 'wpbp_found_b' ) ); foreach ( $cases as $case ) { list( $key, $group ) = $case; wp_cache_delete( $key, $group ); $result = wpbp_caching_001( $key, $group ); $stored = wp_cache_get( $key, $group, false, $stored_found ); if ( array( false, false, false, true ) !== $result || false !== $stored || true !== $stored_found ) { return false; } } return true;", + "description": "The helper reports a miss before storing false and a found hit after storing it", + "weight": 1 + } ], - "forbidden_patterns": [ - { "pattern": "wp_cache_delete", "description": "No explicit deletes needed", "severity": "warning" } + "teardown": "wp_cache_delete( 'wpbp_false_a', 'wpbp_found_a' ); wp_cache_delete( 'wpbp_false_b', 'wpbp_found_b' );" + }, + "reference_solution": "function wpbp_caching_001( string $key, string $group ): array { $first = wp_cache_get( $key, $group, false, $found_first ); wp_cache_set( $key, false, $group ); $second = wp_cache_get( $key, $group, false, $found_second ); return array( $first, $found_first, $second, $found_second ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cache.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-caching-002", + "category": "caching", + "difficulty": "intermediate", + "prompt": "Implement wpbp_caching_002( array $values, array $keys, string $group ): array to store multiple object-cache values and fetch a requested key list without dropping falsey cached values.", + "expected_behavior": "Reviewer contract: `wpbp_caching_002()` stores the supplied key/value map in the supplied group with `wp_cache_set_multiple()` and returns `wp_cache_get_multiple()` for the requested keys. Review focus: false and zero values remain present in the returned associative array, while missing requested keys are returned as false.", + "requirements": [ + "Use the multiple-value object cache APIs", + "Return an associative array for every requested key", + "Preserve cached false and zero values" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_caching_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_cache_get_multiple", + "description": "Uses wp_cache_get_multiple", + "weight": 1 + }, + { + "pattern": "wp_cache_set_multiple", + "description": "Uses wp_cache_set_multiple", + "weight": 1 + } ] }, "runtime_checks": { + "setup": "wp_cache_delete_multiple( array( 'a', 'b', 'zero', 'missing' ), 'wpbp_multi' );", "assertions": [ { "type": "custom_assertion", - "code": "if ( ! function_exists('wpbp_cached_flag') ) { return false; } $a = wpbp_cached_flag(); $b = wpbp_cached_flag(); return $a === false && $b === false;", - "description": "Returns cached false without recompute", - "weight": 1.0 + "code": "if ( ! function_exists( 'wpbp_caching_002' ) ) { return false; } $out = wpbp_caching_002( array( 'a' => false, 'b' => 'bee', 'zero' => 0 ), array( 'a', 'b', 'zero', 'missing' ), 'wpbp_multi' ); return is_array( $out ) && array_key_exists( 'a', $out ) && false === $out['a'] && 'bee' === $out['b'] && array_key_exists( 'zero', $out ) && 0 === $out['zero'] && array_key_exists( 'missing', $out ) && false === $out['missing'];", + "description": "Multiple cache retrieval preserves falsey hits and includes false for a miss", + "weight": 1 + } + ], + "teardown": "wp_cache_delete_multiple( array( 'a', 'b', 'zero', 'missing' ), 'wpbp_multi' );" + }, + "reference_solution": "function wpbp_caching_002( array $values, array $keys, string $group ): array { wp_cache_set_multiple( $values, $group ); return wp_cache_get_multiple( $keys, $group ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cache.php", + "src/wp-includes/cache-compat.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-caching-003", + "category": "caching", + "difficulty": "intermediate", + "prompt": "Implement wpbp_caching_003( string $key, string $group, array $salt ): array to store an object-cache value with the supplied salt and return both a matching-salt read and a stale-salt read.", + "expected_behavior": "Reviewer contract: `wpbp_caching_003()` stores a value using `wp_cache_set_salted()` and returns array( $fresh, $stale ). Review focus: the matching salt returns the stored value, while the same key read with a different salt returns false.", + "requirements": [ + "Use WordPress salted object cache helpers", + "Accept the salt as an array", + "Return array( $fresh, $stale )" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_caching_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_cache_get_salted", + "description": "Uses wp_cache_get_salted", + "weight": 1 + }, + { + "pattern": "wp_cache_set_salted", + "description": "Uses wp_cache_set_salted", + "weight": 1 } ] }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_cached_flag(): bool {\n $cached = wp_cache_get( 'flag', 'wpbp-cache-flag', false, $found );\n if ( $found ) {\n return (bool) $cached;\n }\n // Simulate expensive computation that yields false first time.\n $value = false;\n wp_cache_set( 'flag', $value, 'wpbp-cache-flag' );\n return $value;\n}" + "runtime_checks": { + "setup": "wp_cache_delete( 'wpbp_salted_item', 'wpbp_salted' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_caching_003' ) ) { return false; } $out = wpbp_caching_003( 'wpbp_salted_item', 'wpbp_salted', array( 'posts-last', 'terms-last' ) ); return array( 'fresh-value', false ) === $out;", + "description": "A salted cache read succeeds only when the supplied salt matches", + "weight": 1 + } + ], + "teardown": "wp_cache_delete( 'wpbp_salted_item', 'wpbp_salted' );" + }, + "reference_solution": "function wpbp_caching_003( string $key, string $group, array $salt ): array { wp_cache_set_salted( $key, 'fresh-value', $group, $salt ); return array( wp_cache_get_salted( $key, $group, $salt ), wp_cache_get_salted( $key, $group, array_reverse( $salt ) ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cache.php", + "src/wp-includes/cache-compat.php" + ], + "release_focus": "6.9" + } }, { - "id": "e-cache-get-multiple-001", + "id": "e-caching-004", "category": "caching", - "difficulty": "hard", - "prompt": "Implement wpbp_get_options_cached(array $keys) that uses wp_cache_get_multiple to fetch options and fills misses from get_option without overwriting cached falsey values.", - "expected_behavior": "Returns associative array of option values; cached false is preserved; misses are saved back to cache.", + "difficulty": "intermediate", + "prompt": "Implement wpbp_caching_004( string $key, string $value, int $expiration ): bool to store a transient with the supplied expiration and confirm the stored value can be retrieved.", + "expected_behavior": "Reviewer contract: `wpbp_caching_004()` calls `set_transient()` with the caller-supplied expiration and returns true only when `get_transient()` retrieves the same value. Review focus: use the Transients API rather than writing transient options directly.", "requirements": [ - "Call wp_cache_get_multiple with a custom group (e.g., 'options') and capture missing keys via the miss list", - "For misses, call get_option and wp_cache_set_multiple", - "Do not treat falsey cached values as misses" + "Use set_transient() and get_transient()", + "Pass through the supplied expiration", + "Return true only when the retrieved value matches" ], "static_checks": { "required_patterns": [ - { "pattern": "wp_cache_get_multiple", "description": "Batch cache get", "weight": 1.0 }, - { "pattern": "wp_cache_set_multiple", "description": "Batch cache set for misses", "weight": 0.8 } + { + "pattern": "wpbp_caching_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "set_transient", + "description": "Uses set_transient", + "weight": 1 + }, + { + "pattern": "get_transient", + "description": "Uses get_transient", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "delete_transient( 'wpbp_transient_a' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_caching_004' ) ) { return false; } $seen_expiration = null; add_filter( 'expiration_of_transient_wpbp_transient_a', function ( $expiration ) use ( &$seen_expiration ) { $seen_expiration = $expiration; return $expiration; } ); $ok = wpbp_caching_004( 'wpbp_transient_a', 'cached-value', 90 ); return true === $ok && 90 === $seen_expiration && 'cached-value' === get_transient( 'wpbp_transient_a' );", + "description": "The transient uses the caller-supplied expiration and stores the expected value", + "weight": 1 + } + ], + "teardown": "delete_transient( 'wpbp_transient_a' ); remove_all_filters( 'expiration_of_transient_wpbp_transient_a' );" + }, + "reference_solution": "function wpbp_caching_004( string $key, string $value, int $expiration ): bool { set_transient( $key, $value, $expiration ); return $value === get_transient( $key ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/option.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-caching-005", + "category": "caching", + "difficulty": "intermediate", + "prompt": "Implement wpbp_caching_005( int $post_id ): bool to prime the post object cache for the supplied post, clean that post's cache, and report whether the post cache entry is gone afterward.", + "expected_behavior": "Reviewer contract: `wpbp_caching_005()` primes the supplied post ID with `_prime_post_caches()`, calls `clean_post_cache()`, and returns true only when a subsequent lookup in the `posts` cache group misses. Review focus: clean the specific post cache and rely on WordPress post-cache APIs rather than direct cache-key guessing alone.", + "requirements": [ + "Use _prime_post_caches() for the supplied post ID", + "Use clean_post_cache() for the supplied post ID", + "Return whether the posts cache no longer contains that post" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_caching_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "_prime_post_caches", + "description": "Uses _prime_post_caches", + "weight": 1 + }, + { + "pattern": "clean_post_cache", + "description": "Uses clean_post_cache", + "weight": 1 + }, + { + "pattern": "wp_cache_get", + "description": "Verifies the post cache lookup after cleaning", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_caching_005_post_id'] = wp_insert_post( array( 'post_title' => 'Cache Post', 'post_status' => 'publish' ) ); if ( $GLOBALS['wpbp_caching_005_post_id'] ) { wp_cache_delete( $GLOBALS['wpbp_caching_005_post_id'], 'posts' ); }", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_caching_005' ) ) { return false; } $post_id = (int) ( $GLOBALS['wpbp_caching_005_post_id'] ?? 0 ); if ( ! $post_id ) { return false; } $cleaned = array(); add_action( 'clean_post_cache', function ( $cleaned_post_id ) use ( &$cleaned ) { $cleaned[] = $cleaned_post_id; } ); wp_cache_delete( $post_id, 'posts' ); $ok = wpbp_caching_005( $post_id ); wp_cache_get( $post_id, 'posts', false, $found_after ); return true === $ok && in_array( $post_id, $cleaned, true ) && false === $found_after;", + "description": "The helper cleans the supplied post cache and leaves no posts-group cache hit", + "weight": 1 + } ], - "forbidden_patterns": [ - { "pattern": "wp_cache_delete", "description": "No deletes necessary", "severity": "warning" } + "teardown": "if ( ! empty( $GLOBALS['wpbp_caching_005_post_id'] ) ) { wp_delete_post( (int) $GLOBALS['wpbp_caching_005_post_id'], true ); }" + }, + "reference_solution": "function wpbp_caching_005( int $post_id ): bool { _prime_post_caches( array( $post_id ), false, false ); clean_post_cache( $post_id ); wp_cache_get( $post_id, 'posts', false, $found_after ); return false === $found_after; }", + "metadata": { + "source_refs": [ + "src/wp-includes/cache.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-caching-006", + "category": "caching", + "difficulty": "hard", + "prompt": "Implement wpbp_caching_006( array $values, array $keys, string $group, array $salt ): array to store a batch of salted object-cache values, then return both current-salt and stale-salt batch reads.", + "expected_behavior": "Reviewer contract: `wpbp_caching_006()` stores all supplied values with `wp_cache_set_multiple_salted()` and returns array( 'current' => ..., 'stale' => ... ) from `wp_cache_get_multiple_salted()`. Review focus: current reads preserve falsey cached values, requested missing keys are false, and stale-salt reads reject previously stored values.", + "requirements": [ + "Use the salted multiple-value object cache helpers", + "Preserve cached false values in the current-salt read", + "Return false for missing keys and stale-salt reads" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_caching_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_cache_set_multiple_salted", + "description": "Uses wp_cache_set_multiple_salted", + "weight": 1 + }, + { + "pattern": "wp_cache_get_multiple_salted", + "description": "Uses wp_cache_get_multiple_salted", + "weight": 1 + } ] }, "runtime_checks": { - "setup": "update_option('opt_false', false); update_option('opt_one', 'one');", + "setup": "wp_cache_delete_multiple( array( 'a', 'f', 'missing' ), 'wpbp_multi_salt' );", "assertions": [ { "type": "custom_assertion", - "code": "if ( ! function_exists('wpbp_get_options_cached') ) { return false; } $res = wpbp_get_options_cached( array( 'opt_false', 'opt_one', 'missing_opt' ) ); return array_key_exists('opt_false',$res) && $res['opt_false'] === false && $res['opt_one'] === 'one' && $res['missing_opt'] === null;", - "description": "Preserves cached false, fetches existing and missing options", - "weight": 1.0 + "code": "if ( ! function_exists( 'wpbp_caching_006' ) ) { return false; } $out = wpbp_caching_006( array( 'a' => 'A', 'f' => false ), array( 'a', 'f', 'missing' ), 'wpbp_multi_salt', array( 'posts-last', 'terms-last' ) ); return isset( $out['current'], $out['stale'] ) && 'A' === $out['current']['a'] && array_key_exists( 'f', $out['current'] ) && false === $out['current']['f'] && array_key_exists( 'missing', $out['current'] ) && false === $out['current']['missing'] && false === $out['stale']['a'] && false === $out['stale']['f'];", + "description": "Salted batch reads preserve falsey current values and reject stale-salt values", + "weight": 1 } ], - "teardown": "delete_option('opt_false'); delete_option('opt_one'); delete_option('missing_opt');" + "teardown": "wp_cache_delete_multiple( array( 'a', 'f', 'missing' ), 'wpbp_multi_salt' );" }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_get_options_cached( array $keys ): array {\n $group = 'wpbp-options';\n $fetched = wp_cache_get_multiple( $keys, $group );\n $misses = array();\n foreach ( $keys as $k ) {\n if ( ! array_key_exists( $k, $fetched ) ) {\n $misses[] = $k;\n }\n }\n if ( $misses ) {\n $fill = array();\n foreach ( $misses as $k ) {\n $val = get_option( $k, null );\n $fill[ $k ] = $val;\n $fetched[ $k ] = $val;\n }\n wp_cache_set_multiple( $fill, $group );\n }\n return $fetched;\n}" + "reference_solution": "function wpbp_caching_006( array $values, array $keys, string $group, array $salt ): array { wp_cache_set_multiple_salted( $values, $group, $salt ); return array( 'current' => wp_cache_get_multiple_salted( $keys, $group, $salt ), 'stale' => wp_cache_get_multiple_salted( $keys, $group, array_merge( $salt, array( 'stale' ) ) ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cache.php", + "src/wp-includes/cache-compat.php" + ], + "release_focus": "6.9" + } } ] } diff --git a/datasets/suites/wp-core-v1/execution/comments-users.json b/datasets/suites/wp-core-v1/execution/comments-users.json new file mode 100644 index 0000000..bc71e3c --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/comments-users.json @@ -0,0 +1,236 @@ +{ + "id": "wp-core-execution-v1-comments_users", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Comments and Users", + "description": "Code generation tasks for WordPress Comments and Users APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-comments-users-001", + "category": "comments-users", + "difficulty": "basic", + "prompt": "Implement wpbp_comments_users_001( string $login ) to create a Subscriber user with that login and return the resulting WP_User object.", + "expected_behavior": "Reviewer contract: The function should create a new Subscriber through WordPress user APIs and return the stored WP_User for the supplied login. The runtime verifies the returned object, persisted login, role, and basic read capability, then deletes the created user in teardown.", + "requirements": [ + "Use WordPress user APIs", + "Accept the login as input", + "Return the stored WP_User object" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_comments_users_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_insert_user", + "description": "Uses wp_insert_user", + "weight": 1 + }, + { + "pattern": "get_user_by", + "description": "Retrieves the stored user object", + "weight": 1 + }, + { + "pattern": "subscriber", + "description": "Assigns the Subscriber role", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_comments_users_001' ) ) { return false; } $login = 'wpbp_cu_001_' . wp_rand(); $GLOBALS['wpbp_comments_users_001_login'] = $login; $user = wpbp_comments_users_001( $login ); if ( ! $user instanceof WP_User ) { return false; } $GLOBALS['wpbp_comments_users_001_user_id'] = $user->ID; $stored = get_user_by( 'id', $user->ID ); return $stored instanceof WP_User && $stored->user_login === $login && $user->user_login === $login && in_array( 'subscriber', (array) $stored->roles, true ) && user_can( $stored, 'read' );", + "description": "A Subscriber is created for the supplied login and returned as a WP_User", + "weight": 1 + } + ], + "teardown": "$user_id = $GLOBALS['wpbp_comments_users_001_user_id'] ?? 0; $login = $GLOBALS['wpbp_comments_users_001_login'] ?? ''; if ( ( ! is_int( $user_id ) || $user_id <= 0 ) && is_string( $login ) && '' !== $login ) { $user_id = username_exists( $login ); } if ( is_int( $user_id ) && $user_id > 0 ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); } unset( $GLOBALS['wpbp_comments_users_001_user_id'], $GLOBALS['wpbp_comments_users_001_login'] );" + }, + "reference_solution": "function wpbp_comments_users_001( string $login ): WP_User|WP_Error { $id = wp_insert_user( array( 'user_login' => $login, 'user_pass' => wp_generate_password(), 'role' => 'subscriber' ) ); return is_wp_error( $id ) ? $id : get_user_by( 'id', $id ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/user.php", + "src/wp-includes/class-wp-user.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-comments-users-002", + "category": "comments-users", + "difficulty": "intermediate", + "prompt": "Implement wpbp_comments_users_002( string $meta_key, string $meta_value ) to return user IDs whose user meta exactly matches the supplied value.", + "expected_behavior": "Reviewer contract: The function should use WP_User_Query with the supplied meta key and value and return IDs only. The runtime creates matching and non-matching user fixtures, checks that only the matching user ID is present, and removes both users in teardown.", + "requirements": [ + "Use WordPress user-query APIs", + "Accept the meta key and value as inputs", + "Return matching user IDs" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_comments_users_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_User_Query", + "description": "Uses WP_User_Query", + "weight": 1 + }, + { + "pattern": "meta_key", + "description": "Uses meta_key", + "weight": 1 + }, + { + "pattern": "meta_value", + "description": "Uses meta_value", + "weight": 1 + }, + { + "pattern": "fields", + "description": "Requests IDs from the query", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_comments_users_002_target_id'] = wp_insert_user( array( 'user_login' => 'wpbp_user_meta_target_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'subscriber' ) ); $GLOBALS['wpbp_comments_users_002_other_id'] = wp_insert_user( array( 'user_login' => 'wpbp_user_meta_other_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'subscriber' ) ); if ( is_int( $GLOBALS['wpbp_comments_users_002_target_id'] ) ) { update_user_meta( $GLOBALS['wpbp_comments_users_002_target_id'], 'wpbp_team', 'blue' ); } if ( is_int( $GLOBALS['wpbp_comments_users_002_other_id'] ) ) { update_user_meta( $GLOBALS['wpbp_comments_users_002_other_id'], 'wpbp_team', 'red' ); }", + "assertions": [ + { + "type": "custom_assertion", + "code": "$target_id = $GLOBALS['wpbp_comments_users_002_target_id'] ?? 0; $other_id = $GLOBALS['wpbp_comments_users_002_other_id'] ?? 0; if ( ! function_exists( 'wpbp_comments_users_002' ) || ! is_int( $target_id ) || ! is_int( $other_id ) ) { return false; } $ids = wpbp_comments_users_002( 'wpbp_team', 'blue' ); if ( ! is_array( $ids ) ) { return false; } $ids = array_map( 'intval', $ids ); return in_array( $target_id, $ids, true ) && ! in_array( $other_id, $ids, true );", + "description": "Only users with the exact requested user-meta value are returned", + "weight": 1 + } + ], + "teardown": "foreach ( array( 'wpbp_comments_users_002_target_id', 'wpbp_comments_users_002_other_id' ) as $key ) { $user_id = $GLOBALS[ $key ] ?? 0; if ( is_int( $user_id ) ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); } unset( $GLOBALS[ $key ] ); }" + }, + "reference_solution": "function wpbp_comments_users_002( string $meta_key, string $meta_value ): array { $query = new WP_User_Query( array( 'meta_key' => $meta_key, 'meta_value' => $meta_value, 'fields' => 'ID' ) ); return $query->get_results(); }", + "metadata": { + "source_refs": [ + "src/wp-includes/user.php", + "src/wp-includes/class-wp-user-query.php", + "src/wp-includes/class-wp-meta-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-comments-users-003", + "category": "comments-users", + "difficulty": "basic", + "prompt": "Implement wpbp_comments_users_003( int $post_id, string $content ) to add an approved comment to the post and return the stored comment text.", + "expected_behavior": "Reviewer contract: The function should insert an approved comment for the supplied post and return the text retrieved from WordPress for that new comment. The runtime creates a post, verifies the stored approved comment content and returned text, then removes comments and the post in teardown.", + "requirements": [ + "Use WordPress comment APIs", + "Accept the post ID and comment content as inputs", + "Return the stored comment text" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_comments_users_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_insert_comment", + "description": "Uses wp_insert_comment", + "weight": 1 + }, + { + "pattern": "get_comment_text", + "description": "Uses get_comment_text", + "weight": 1 + }, + { + "pattern": "comment_approved", + "description": "Creates the comment as approved", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_comments_users_003_post_id'] = wp_insert_post( array( 'post_title' => 'WPBP Comment Text', 'post_status' => 'publish' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "$post_id = $GLOBALS['wpbp_comments_users_003_post_id'] ?? 0; if ( ! function_exists( 'wpbp_comments_users_003' ) || ! is_int( $post_id ) ) { return false; } $content = 'Approved text ' . wp_rand(); $returned = wpbp_comments_users_003( $post_id, $content ); $comments = get_comments( array( 'post_id' => $post_id, 'status' => 'approve' ) ); $matches = array(); foreach ( $comments as $comment ) { if ( $comment instanceof WP_Comment && $comment->comment_content === $content ) { $matches[] = $comment; } } return $returned === $content && 1 === count( $matches ) && '1' === (string) $matches[0]->comment_approved;", + "description": "An approved comment is stored on the post and its text is returned", + "weight": 1 + } + ], + "teardown": "$post_id = $GLOBALS['wpbp_comments_users_003_post_id'] ?? 0; if ( is_int( $post_id ) ) { foreach ( get_comments( array( 'post_id' => $post_id, 'status' => 'all' ) ) as $comment ) { if ( $comment instanceof WP_Comment ) { wp_delete_comment( (int) $comment->comment_ID, true ); } } wp_delete_post( $post_id, true ); } unset( $GLOBALS['wpbp_comments_users_003_post_id'] );" + }, + "reference_solution": "function wpbp_comments_users_003( int $post_id, string $content ): string { $id = wp_insert_comment( array( 'comment_post_ID' => $post_id, 'comment_content' => $content, 'comment_approved' => 1 ) ); return get_comment_text( $id ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/comment.php", + "src/wp-includes/comment-template.php", + "src/wp-includes/class-wp-comment.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-comments-users-004", + "category": "comments-users", + "difficulty": "basic", + "prompt": "Implement wpbp_comments_users_004( int $post_id ) to return the number of approved comments for that post.", + "expected_behavior": "Reviewer contract: The function should use WordPress comment-count APIs for the supplied post and return only the approved count. The runtime creates one approved and one pending comment, expects an exact count of one, and removes all fixtures in teardown.", + "requirements": [ + "Use WordPress comment APIs", + "Accept the post ID as input", + "Count only approved comments" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_comments_users_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_count_comments", + "description": "Uses wp_count_comments", + "weight": 1 + }, + { + "pattern": "approved", + "description": "Reads the approved count", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_comments_users_004_post_id'] = wp_insert_post( array( 'post_title' => 'WPBP Comment Count', 'post_status' => 'publish' ) ); if ( is_int( $GLOBALS['wpbp_comments_users_004_post_id'] ) ) { $GLOBALS['wpbp_comments_users_004_approved_id'] = wp_insert_comment( array( 'comment_post_ID' => $GLOBALS['wpbp_comments_users_004_post_id'], 'comment_content' => 'Approved', 'comment_approved' => 1 ) ); $GLOBALS['wpbp_comments_users_004_pending_id'] = wp_insert_comment( array( 'comment_post_ID' => $GLOBALS['wpbp_comments_users_004_post_id'], 'comment_content' => 'Pending', 'comment_approved' => 0 ) ); }", + "assertions": [ + { + "type": "custom_assertion", + "code": "$post_id = $GLOBALS['wpbp_comments_users_004_post_id'] ?? 0; if ( ! function_exists( 'wpbp_comments_users_004' ) || ! is_int( $post_id ) ) { return false; } return 1 === wpbp_comments_users_004( $post_id );", + "description": "Only the approved comment is counted for the supplied post", + "weight": 1 + } + ], + "teardown": "foreach ( array( 'wpbp_comments_users_004_approved_id', 'wpbp_comments_users_004_pending_id' ) as $key ) { $comment_id = $GLOBALS[ $key ] ?? 0; if ( is_int( $comment_id ) && $comment_id > 0 ) { wp_delete_comment( $comment_id, true ); } unset( $GLOBALS[ $key ] ); } $post_id = $GLOBALS['wpbp_comments_users_004_post_id'] ?? 0; if ( is_int( $post_id ) ) { wp_delete_post( $post_id, true ); } unset( $GLOBALS['wpbp_comments_users_004_post_id'] );" + }, + "reference_solution": "function wpbp_comments_users_004( int $post_id ): int { $counts = wp_count_comments( $post_id ); return (int) $counts->approved; }", + "metadata": { + "source_refs": [ + "src/wp-includes/comment.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/connectors-api.json b/datasets/suites/wp-core-v1/execution/connectors-api.json new file mode 100644 index 0000000..85d17b5 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/connectors-api.json @@ -0,0 +1,194 @@ +{ + "id": "wp-core-execution-v1-connectors_api", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Connectors API", + "description": "Code generation tasks for WordPress Connectors API APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-connectors-api-001", + "category": "connectors-api", + "difficulty": "basic", + "prompt": "Implement wpbp_connectors_api_001(): bool to report whether WordPress has registered the built-in Akismet connector through the public Connectors API.", + "expected_behavior": "`wpbp_connectors_api_001()` uses `wp_is_connector_registered()` to check the `akismet` connector after WordPress connector initialization. The test focuses on public connector discovery, not direct registry access or a hard-coded truthy value.", + "requirements": [ + "Use wp_is_connector_registered()", + "Check the connector ID 'akismet'", + "Do not access WP_Connector_Registry directly" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_connectors_api_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_is_connector_registered", + "description": "Uses wp_is_connector_registered", + "weight": 1 + }, + { + "pattern": "akismet", + "description": "Checks the built-in Akismet connector ID", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "WP_Connector_Registry", + "description": "Does not access the connector registry directly", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_connectors_api_001' ) ) { return false; } return true === wpbp_connectors_api_001() && wp_is_connector_registered( 'akismet' );", + "description": "The helper reports the built-in Akismet connector as registered", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_connectors_api_001(): bool { return wp_is_connector_registered( 'akismet' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/connectors.php", + "src/wp-includes/class-wp-connector-registry.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-connectors-api-002", + "category": "connectors-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_connectors_api_002( string $id ): ?array to safely return connector data for the supplied connector ID through the public Connectors API without triggering a not-found registry notice.", + "expected_behavior": "`wpbp_connectors_api_002()` checks whether the supplied ID is registered before retrieving connector data, returning `null` for unknown IDs without causing a `_doing_it_wrong()` notice. Runtime validation checks Akismet's type, authentication, and plugin metadata, plus a missing-ID case.", + "requirements": [ + "Use wp_is_connector_registered() before retrieving the connector", + "Use wp_get_connector() for registered connector IDs", + "Return null for unknown connector IDs", + "Do not access WP_Connector_Registry directly" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_connectors_api_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_get_connector", + "description": "Uses wp_get_connector", + "weight": 1 + }, + { + "pattern": "wp_is_connector_registered", + "description": "Checks registration before retrieval", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "WP_Connector_Registry", + "description": "Does not access the connector registry directly", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_connectors_api_002' ) ) { return false; } $connector = wpbp_connectors_api_002( 'akismet' ); $missing = wpbp_connectors_api_002( 'wpbp_missing_connector' ); return is_array( $connector ) && 'spam_filtering' === ( $connector['type'] ?? null ) && 'api_key' === ( $connector['authentication']['method'] ?? null ) && 'wordpress_api_key' === ( $connector['authentication']['setting_name'] ?? null ) && 'WPCOM_API_KEY' === ( $connector['authentication']['constant_name'] ?? null ) && 'akismet/akismet.php' === ( $connector['plugin']['file'] ?? null ) && is_callable( $connector['plugin']['is_active'] ?? null ) && null === $missing;", + "description": "The helper returns Akismet connector metadata and null for an unknown ID", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_connectors_api_002( string $id ): ?array { if ( ! wp_is_connector_registered( $id ) ) { return null; } return wp_get_connector( $id ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/connectors.php", + "src/wp-includes/class-wp-connector-registry.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-connectors-api-003", + "category": "connectors-api", + "difficulty": "intermediate", + "prompt": "Register a Connectors API connector with the ID 'wpbp_supportdesk' so it is discoverable by WordPress. It should represent a non-AI support desk provider with no authentication and the plugin file 'wpbp-supportdesk/wpbp-supportdesk.php'.", + "expected_behavior": "Adds a callback on `wp_connectors_init` that uses the provided `WP_Connector_Registry` instance to register `wpbp_supportdesk` with type `support_desk`, authentication method `none`, and the requested plugin file. Runtime validation dispatches the connector init hook and verifies discovery through the public query API.", + "requirements": [ + "Register on wp_connectors_init", + "Use the WP_Connector_Registry instance provided to the hook", + "Use connector type 'support_desk' and authentication method 'none'" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wp_connectors_init", + "description": "Uses the connector initialization hook", + "weight": 1 + }, + { + "pattern": "register", + "description": "Registers a connector through the registry", + "weight": 1 + }, + { + "pattern": "wpbp_supportdesk", + "description": "Registers the requested connector ID", + "weight": 1 + }, + { + "pattern": "support_desk", + "description": "Uses the requested connector type", + "weight": 1 + }, + { + "pattern": "authentication", + "description": "Configures connector authentication metadata", + "weight": 1 + }, + { + "pattern": "none", + "description": "Uses no-authentication connector metadata", + "weight": 1 + }, + { + "pattern": "wpbp-supportdesk/wpbp-supportdesk.php", + "description": "Sets the requested plugin file", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "$registry = WP_Connector_Registry::get_instance(); if ( ! $registry instanceof WP_Connector_Registry ) { return false; } if ( $registry->is_registered( 'wpbp_supportdesk' ) ) { $registry->unregister( 'wpbp_supportdesk' ); } do_action( 'wp_connectors_init', $registry ); $connector = wp_get_connector( 'wpbp_supportdesk' ); return wp_is_connector_registered( 'wpbp_supportdesk' ) && is_array( $connector ) && 'support_desk' === ( $connector['type'] ?? null ) && 'none' === ( $connector['authentication']['method'] ?? null ) && ! isset( $connector['authentication']['setting_name'] ) && 'wpbp-supportdesk/wpbp-supportdesk.php' === ( $connector['plugin']['file'] ?? null ) && is_callable( $connector['plugin']['is_active'] ?? null );", + "description": "The custom support desk connector is registered and publicly discoverable", + "weight": 1 + } + ] + }, + "reference_solution": "add_action( 'wp_connectors_init', function ( WP_Connector_Registry $registry ) { $registry->register( 'wpbp_supportdesk', array( 'name' => 'WPBP Support Desk', 'description' => 'Connects WPBP to a support desk provider.', 'type' => 'support_desk', 'authentication' => array( 'method' => 'none' ), 'plugin' => array( 'file' => 'wpbp-supportdesk/wpbp-supportdesk.php', 'is_active' => '__return_true' ) ) ); } );", + "metadata": { + "source_refs": [ + "src/wp-includes/connectors.php", + "src/wp-includes/class-wp-connector-registry.php" + ], + "release_focus": "7.0" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/cron.json b/datasets/suites/wp-core-v1/execution/cron.json new file mode 100644 index 0000000..d6a7e4a --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/cron.json @@ -0,0 +1,280 @@ +{ + "id": "wp-core-execution-v1-cron", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Cron", + "description": "Code generation tasks for WordPress Cron APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-cron-001", + "category": "cron", + "difficulty": "intermediate", + "prompt": "Implement wpbp_cron_001( int $timestamp ): bool to schedule a one-time event for the hook 'wpbp_single_event' only when that hook is not already scheduled.", + "expected_behavior": "`wpbp_cron_001()` uses `wp_next_scheduled()` to avoid enqueueing another `wpbp_single_event`, schedules exactly one one-time event with `wp_schedule_single_event()` when none exists, and returns false without changing the existing timestamp on later calls.", + "requirements": [ + "Use wp_next_scheduled() to detect an existing event", + "Return false when the hook already has a scheduled event" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_cron_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_schedule_single_event", + "description": "Uses wp_schedule_single_event", + "weight": 1 + }, + { + "pattern": "wp_next_scheduled", + "description": "Uses wp_next_scheduled", + "weight": 1 + }, + { + "pattern": "wpbp_single_event", + "description": "Targets the expected cron hook", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "wp_clear_scheduled_hook( 'wpbp_single_event' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_cron_001' ) ) { return false; } $first = time() + HOUR_IN_SECONDS; $second = $first + HOUR_IN_SECONDS; $scheduled = wpbp_cron_001( $first ); $first_timestamp = wp_next_scheduled( 'wpbp_single_event' ); $blocked = wpbp_cron_001( $second ); return true === $scheduled && $first === $first_timestamp && false === $blocked && $first === wp_next_scheduled( 'wpbp_single_event' );", + "description": "The first call schedules the single event, while a later call leaves the existing timestamp unchanged", + "weight": 1 + } + ], + "teardown": "wp_clear_scheduled_hook( 'wpbp_single_event' );" + }, + "reference_solution": "function wpbp_cron_001( int $timestamp ): bool { if ( false !== wp_next_scheduled( 'wpbp_single_event' ) ) { return false; } return wp_schedule_single_event( $timestamp, 'wpbp_single_event' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cron.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-cron-002", + "category": "cron", + "difficulty": "intermediate", + "prompt": "Implement wpbp_cron_002( int $timestamp, array $args ): bool to schedule the hook 'wpbp_hourly_event' as an hourly recurring event using the provided arguments.", + "expected_behavior": "`wpbp_cron_002()` schedules `wpbp_hourly_event` with `wp_schedule_event()` using the `hourly` recurrence and the provided argument array. Runtime validation inspects the scheduled event object for the timestamp, recurrence name, interval, and stored arguments.", + "requirements": [ + "Use wp_schedule_event() with the hourly recurrence", + "Pass the provided argument array to the scheduled event" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_cron_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_schedule_event", + "description": "Uses wp_schedule_event", + "weight": 1 + }, + { + "pattern": "hourly", + "description": "Uses hourly", + "weight": 1 + }, + { + "pattern": "wpbp_hourly_event", + "description": "Targets the expected cron hook", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "wp_clear_scheduled_hook( 'wpbp_hourly_event', array( 'site' => 'wp-bench' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_cron_002' ) ) { return false; } $ts = time() + HOUR_IN_SECONDS; $args = array( 'site' => 'wp-bench' ); $result = wpbp_cron_002( $ts, $args ); $event = wp_get_scheduled_event( 'wpbp_hourly_event', $args ); return true === $result && $event && $ts === $event->timestamp && 'hourly' === $event->schedule && HOUR_IN_SECONDS === $event->interval && $args === $event->args;", + "description": "The hourly cron event is scheduled with the requested timestamp, interval, and arguments", + "weight": 1 + } + ], + "teardown": "wp_clear_scheduled_hook( 'wpbp_hourly_event', array( 'site' => 'wp-bench' ) );" + }, + "reference_solution": "function wpbp_cron_002( int $timestamp, array $args ): bool { return wp_schedule_event( $timestamp, 'hourly', 'wpbp_hourly_event', $args ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cron.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-cron-003", + "category": "cron", + "difficulty": "intermediate", + "prompt": "Implement wpbp_cron_003( array $args ): bool to unschedule only the next event for the hook 'wpbp_unschedule' that matches the provided arguments.", + "expected_behavior": "`wpbp_cron_003()` finds the next scheduled `wpbp_unschedule` event for the provided arguments with `wp_next_scheduled()` and removes that timestamp with `wp_unschedule_event()`. Runtime validation schedules two matching events and verifies only the earlier event is removed.", + "requirements": [ + "Find the next matching timestamp with wp_next_scheduled()", + "Unschedule that timestamp with the same hook and arguments" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_cron_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_unschedule_event", + "description": "Uses wp_unschedule_event", + "weight": 1 + }, + { + "pattern": "wp_next_scheduled", + "description": "Uses wp_next_scheduled", + "weight": 1 + }, + { + "pattern": "wpbp_unschedule", + "description": "Targets the expected cron hook", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "wp_clear_scheduled_hook( 'wpbp_unschedule', array( 'id' => 31 ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_cron_003' ) ) { return false; } $args = array( 'id' => 31 ); $first = time() + 1000; $second = $first + HOUR_IN_SECONDS; wp_schedule_single_event( $first, 'wpbp_unschedule', $args ); wp_schedule_single_event( $second, 'wpbp_unschedule', $args ); $result = wpbp_cron_003( $args ); return true === $result && $second === wp_next_scheduled( 'wpbp_unschedule', $args );", + "description": "Only the next matching cron event is unscheduled", + "weight": 1 + } + ], + "teardown": "wp_clear_scheduled_hook( 'wpbp_unschedule', array( 'id' => 31 ) );" + }, + "reference_solution": "function wpbp_cron_003( array $args ): bool { $timestamp = wp_next_scheduled( 'wpbp_unschedule', $args ); if ( false === $timestamp ) { return false; } return wp_unschedule_event( $timestamp, 'wpbp_unschedule', $args ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cron.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-cron-004", + "category": "cron", + "difficulty": "intermediate", + "prompt": "Implement wpbp_cron_004(): void to add a custom recurrence named 'wpbp_five_minutes' to WordPress cron schedules with a 300-second interval and display text 'Every five minutes'.", + "expected_behavior": "`wpbp_cron_004()` attaches a `cron_schedules` filter that adds `wpbp_five_minutes` while preserving existing schedules. Runtime validation retrieves the schedules through WordPress and verifies the custom interval and display label.", + "requirements": [ + "Use the cron_schedules filter", + "Preserve existing schedules" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_cron_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "cron_schedules", + "description": "Uses cron_schedules", + "weight": 1 + }, + { + "pattern": "wpbp_five_minutes", + "description": "Registers the expected custom schedule", + "weight": 1 + }, + { + "pattern": "add_filter", + "description": "Registers the schedule filter", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_cron_004' ) ) { return false; } wpbp_cron_004(); $schedules = wp_get_schedules(); return isset( $schedules['wpbp_five_minutes'] ) && 300 === $schedules['wpbp_five_minutes']['interval'] && 'Every five minutes' === $schedules['wpbp_five_minutes']['display'];", + "description": "The custom five-minute cron recurrence is available through wp_get_schedules()", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_cron_004(): void { add_filter( 'cron_schedules', function ( $schedules ) { $schedules['wpbp_five_minutes'] = array( 'interval' => 300, 'display' => 'Every five minutes' ); return $schedules; } ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cron.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-cron-005", + "category": "cron", + "difficulty": "intermediate", + "prompt": "Implement wpbp_cron_005(): WP_Error to attempt scheduling the hook 'wpbp_invalid_event' with the invalid recurrence slug 'wpbp_missing_schedule' and return the WP_Error object produced by WordPress.", + "expected_behavior": "`wpbp_cron_005()` calls `wp_schedule_event()` with WordPress error returns enabled for a recurrence that is not registered. Runtime validation expects the returned object to be a `WP_Error` with the `invalid_schedule` code and no scheduled `wpbp_invalid_event` event.", + "requirements": [ + "Enable WP_Error returns when calling wp_schedule_event()", + "Return the error object instead of converting it to a boolean" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_cron_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_schedule_event", + "description": "Uses wp_schedule_event", + "weight": 1 + }, + { + "pattern": "wpbp_invalid_event", + "description": "Targets the expected cron hook", + "weight": 1 + }, + { + "pattern": "wpbp_missing_schedule", + "description": "Uses the invalid recurrence slug", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "wp_clear_scheduled_hook( 'wpbp_invalid_event' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_cron_005' ) ) { return false; } $result = wpbp_cron_005(); return is_wp_error( $result ) && 'invalid_schedule' === $result->get_error_code() && false === wp_next_scheduled( 'wpbp_invalid_event' );", + "description": "Invalid recurrence scheduling returns WordPress' invalid_schedule error and does not enqueue an event", + "weight": 1 + } + ], + "teardown": "wp_clear_scheduled_hook( 'wpbp_invalid_event' );" + }, + "reference_solution": "function wpbp_cron_005(): WP_Error { return wp_schedule_event( time() + HOUR_IN_SECONDS, 'wpbp_missing_schedule', 'wpbp_invalid_event', array(), true ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/cron.php" + ], + "release_focus": "6.9" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/database.json b/datasets/suites/wp-core-v1/execution/database.json index e9148cd..2cd6a0a 100644 --- a/datasets/suites/wp-core-v1/execution/database.json +++ b/datasets/suites/wp-core-v1/execution/database.json @@ -1,43 +1,368 @@ { "id": "wp-core-execution-v1-database", - "version": "1.1.0", + "version": "2.0.0", "metadata": { "name": "WordPress Core Execution Tests - Database", - "description": "Code generation tasks for WordPress database operations and meta queries", - "wp_version": "6.9", - "created_at": "2025-12-16" + "description": "Code generation tasks for WordPress Database APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" }, "tests": [ { - "id": "e-meta-query-zero-001", + "id": "e-database-001", "category": "database", - "difficulty": "hard", - "prompt": "Write wpbp_zero_price_products() that returns a WP_Query for 'product' posts whose numeric meta key 'price_cents' equals 0 without missing results due to type coercion.", - "expected_behavior": "Function returns one post when a product has meta price_cents stored as integer 0; does not return posts with null/missing meta.", + "difficulty": "intermediate", + "prompt": "Implement wpbp_database_001( string $item_key, string $item_value ) to add a row to the WordPress-prefixed custom table `wpbp_database_001` and return the new auto-increment ID.", + "expected_behavior": "`wpbp_database_001()` inserts into the fixture-owned custom table with `$wpdb->insert()`, writes `item_key`, `item_value`, and `status` as `active`, and returns `$wpdb->insert_id` rather than the rows-affected result. Runtime validation pre-seeds the table so returning `1` from `$wpdb->insert()` cannot pass.", "requirements": [ - "Query post_type=product and post_status=publish", - "Use a meta_query with key price_cents, compare '=' and type 'NUMERIC'; do not rely on meta_value=0 string", - "Meta results must include rows where meta_value is 0", - "Return the WP_Query object" + "Target the WordPress-prefixed `wpbp_database_001` custom table", + "Write `item_key`, `item_value`, and `status` columns, setting `status` to `active`", + "Return the new row's auto-increment ID, not a row count" ], "static_checks": { "required_patterns": [ - { "pattern": "meta_query", "description": "Uses meta_query", "weight": 1.0 }, - { "pattern": "type['\"]?\\s*=>\\s*['\"]NUMERIC['\"]", "description": "Numeric type coercion", "weight": 0.8 } + { + "pattern": "wpbp_database_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "->insert", + "description": "Uses $wpdb->insert", + "weight": 1 + }, + { + "pattern": "insert_id", + "description": "Returns $wpdb->insert_id", + "weight": 1 + }, + { + "pattern": "item_key", + "description": "Writes the item_key column", + "weight": 1 + }, + { + "pattern": "item_value", + "description": "Writes the item_value column", + "weight": 1 + }, + { + "pattern": "active", + "description": "Stores the active status", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_001'; $charset_collate = $wpdb->get_charset_collate(); $created = $wpdb->query( \"CREATE TABLE {$table} ( id bigint(20) unsigned NOT NULL AUTO_INCREMENT, item_key varchar(64) NOT NULL, item_value longtext NOT NULL, status varchar(20) NOT NULL DEFAULT 'new', PRIMARY KEY (id), KEY item_key (item_key) ) {$charset_collate}\" ); if ( false === $created ) { throw new \\RuntimeException( 'Could not create fixture table: ' . $wpdb->last_error ); } $wpdb->insert( $table, array( 'item_key' => 'seed', 'item_value' => 'seeded', 'status' => 'seed' ), array( '%s', '%s', '%s' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_database_001' ) ) { return false; } global $wpdb; $table = $wpdb->prefix . 'wpbp_database_001'; $id = wpbp_database_001( 'alpha', 'stored value' ); if ( $id <= 1 ) { return false; } $row = $wpdb->get_row( $wpdb->prepare( \"SELECT id, item_key, item_value, status FROM {$table} WHERE item_key = %s\", 'alpha' ) ); return $row && (int) $row->id === (int) $id && 'stored value' === $row->item_value && 'active' === $row->status;", + "description": "The custom table row has the returned insert ID and requested values", + "weight": 1 + } ], - "forbidden_patterns": [ - { "pattern": "meta_value\\s*=>\\s*0", "description": "Avoid direct meta_value=0 string shortcut", "severity": "warning" } + "teardown": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_001'; $wpdb->query( \"DROP TABLE IF EXISTS {$table}\" );" + }, + "reference_solution": "function wpbp_database_001( string $item_key, string $item_value ): int { global $wpdb; $table = $wpdb->prefix . 'wpbp_database_001'; $result = $wpdb->insert( $table, array( 'item_key' => $item_key, 'item_value' => $item_value, 'status' => 'active' ), array( '%s', '%s', '%s' ) ); return false === $result ? 0 : (int) $wpdb->insert_id; }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wpdb.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-database-002", + "category": "database", + "difficulty": "intermediate", + "prompt": "Implement wpbp_database_002( string $term ) to return a SQL query for the WordPress-prefixed custom table `wpbp_database_002`. Select that table's `id` column, require its `visibility` column to equal 'public', and require its `title` column to contain the supplied literal term.", + "expected_behavior": "`wpbp_database_002()` returns a SQL string prepared with `$wpdb->prepare()`, uses `%i` placeholders for the custom table plus its `id`, `visibility`, and `title` column identifiers, and escapes LIKE wildcards before adding surrounding `%` characters. Runtime validation executes the returned SQL against fixture rows, including a wildcard-looking distractor.", + "requirements": [ + "Return a safely parameterized SQL string for the custom table, not queried results", + "Treat the table name and `id`, `visibility`, and `title` column names as SQL identifiers", + "Treat percent and underscore characters in the term as literal text" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_database_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "prepare", + "description": "Uses prepare", + "weight": 1 + }, + { + "pattern": "%i", + "description": "Uses identifier placeholders", + "weight": 1 + }, + { + "pattern": "esc_like", + "description": "Uses esc_like", + "weight": 1 + }, + { + "pattern": "LIKE", + "description": "Builds a LIKE query", + "weight": 1 + } ] }, "runtime_checks": { - "setup": "$id = wp_insert_post(['post_title'=>'Zero Product','post_type'=>'product','post_status'=>'publish']); update_post_meta($id,'price_cents', 0); return true;", + "setup": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_002'; $charset_collate = $wpdb->get_charset_collate(); $created = $wpdb->query( \"CREATE TABLE {$table} ( id bigint(20) unsigned NOT NULL AUTO_INCREMENT, title varchar(191) NOT NULL, visibility varchar(20) NOT NULL, PRIMARY KEY (id), KEY visibility (visibility) ) {$charset_collate}\" ); if ( false === $created ) { throw new \\RuntimeException( 'Could not create fixture table: ' . $wpdb->last_error ); } $wpdb->insert( $table, array( 'title' => 'WPBP 100%_match target', 'visibility' => 'public' ), array( '%s', '%s' ) ); $target = (int) $wpdb->insert_id; $wpdb->insert( $table, array( 'title' => 'WPBP 100Xmatch target', 'visibility' => 'public' ), array( '%s', '%s' ) ); $distractor = (int) $wpdb->insert_id; $wpdb->insert( $table, array( 'title' => 'WPBP 100%_match hidden', 'visibility' => 'private' ), array( '%s', '%s' ) ); $private = (int) $wpdb->insert_id; $GLOBALS['wpbp_database_002_rows'] = array( 'target' => $target, 'distractor' => $distractor, 'private' => $private );", "assertions": [ - { "type": "custom_assertion", "code": "return function_exists('wpbp_zero_price_products') && wpbp_zero_price_products()->post_count === 1;", "description": "Finds zero-priced product", "weight": 1.0 } + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_database_002' ) ) { return false; } global $wpdb; $sql = wpbp_database_002( '100%_match' ); if ( ! is_string( $sql ) || ! str_contains( $sql, 'LIKE' ) ) { return false; } $ids = array_map( 'intval', $wpdb->get_col( $sql ) ); $rows = $GLOBALS['wpbp_database_002_rows'] ?? array(); return in_array( $rows['target'] ?? 0, $ids, true ) && ! in_array( $rows['distractor'] ?? 0, $ids, true ) && ! in_array( $rows['private'] ?? 0, $ids, true );", + "description": "The prepared SQL matches only public custom-table rows containing the literal LIKE term", + "weight": 1 + } ], - "teardown": "wp_delete_post({id}, true);" + "teardown": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_002'; $wpdb->query( \"DROP TABLE IF EXISTS {$table}\" ); unset( $GLOBALS['wpbp_database_002_rows'] );" }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_zero_price_products(): WP_Query {\n return new WP_Query( array(\n 'post_type' => 'product',\n 'post_status' => 'publish',\n 'fields' => 'ids',\n 'meta_query' => array(\n array(\n 'key' => 'price_cents',\n 'value' => 0,\n 'compare' => '=',\n 'type' => 'NUMERIC',\n ),\n ),\n ) );\n}" + "reference_solution": "function wpbp_database_002( string $term ): string { global $wpdb; $table = $wpdb->prefix . 'wpbp_database_002'; return $wpdb->prepare( 'SELECT %i FROM %i WHERE %i = %s AND %i LIKE %s', 'id', $table, 'visibility', 'public', 'title', '%' . $wpdb->esc_like( $term ) . '%' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wpdb.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-database-003", + "category": "database", + "difficulty": "basic", + "prompt": "Implement wpbp_database_003( string $lookup_key ) to read one `lookup_value` from the WordPress-prefixed custom table `wpbp_database_003` for the supplied `lookup_key`.", + "expected_behavior": "`wpbp_database_003()` uses `$wpdb->get_var()` with a prepared query against the fixture-owned custom table, returning the matching `lookup_value` for the supplied `lookup_key` and `null` when no row exists.", + "requirements": [ + "Target the WordPress-prefixed `wpbp_database_003` custom table", + "Handle the `lookup_key` comparison safely", + "Return null when the custom-table row is missing" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_database_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "get_var", + "description": "Uses get_var", + "weight": 1 + }, + { + "pattern": "prepare", + "description": "Uses prepare", + "weight": 1 + }, + { + "pattern": "lookup_value", + "description": "Reads the lookup_value column", + "weight": 1 + }, + { + "pattern": "lookup_key", + "description": "Matches rows by lookup_key", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_003'; $charset_collate = $wpdb->get_charset_collate(); $created = $wpdb->query( \"CREATE TABLE {$table} ( id bigint(20) unsigned NOT NULL AUTO_INCREMENT, lookup_key varchar(64) NOT NULL, lookup_value longtext NOT NULL, PRIMARY KEY (id), UNIQUE KEY lookup_key (lookup_key) ) {$charset_collate}\" ); if ( false === $created ) { throw new \\RuntimeException( 'Could not create fixture table: ' . $wpdb->last_error ); } $wpdb->insert( $table, array( 'lookup_key' => 'alpha', 'lookup_value' => 'stored' ), array( '%s', '%s' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_database_003' ) ) { return false; } return 'stored' === wpbp_database_003( 'alpha' ) && null === wpbp_database_003( 'missing' );", + "description": "The function reads the requested custom-table value and returns null for a missing row", + "weight": 1 + } + ], + "teardown": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_003'; $wpdb->query( \"DROP TABLE IF EXISTS {$table}\" );" + }, + "reference_solution": "function wpbp_database_003( string $lookup_key ): ?string { global $wpdb; $table = $wpdb->prefix . 'wpbp_database_003'; return $wpdb->get_var( $wpdb->prepare( \"SELECT lookup_value FROM {$table} WHERE lookup_key = %s\", $lookup_key ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wpdb.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-database-004", + "category": "database", + "difficulty": "basic", + "prompt": "Implement wpbp_database_004( string $lookup_key, string $value ) to update `lookup_value` in the WordPress-prefixed custom table `wpbp_database_004` for the supplied `lookup_key`.", + "expected_behavior": "`wpbp_database_004()` calls `$wpdb->update()` on the fixture-owned custom table, identifies rows by `lookup_key`, writes the supplied `lookup_value`, and returns the method's row-count result. Runtime validation checks the target row, an unrelated row, and the zero-row result for a missing key.", + "requirements": [ + "Target the WordPress-prefixed `wpbp_database_004` custom table", + "Accept the lookup key and new value as inputs", + "Return the affected row count from the database API" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_database_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "->update", + "description": "Uses $wpdb->update", + "weight": 1 + }, + { + "pattern": "lookup_value", + "description": "Writes the lookup_value column", + "weight": 1 + }, + { + "pattern": "lookup_key", + "description": "Matches rows by lookup_key", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_004'; $charset_collate = $wpdb->get_charset_collate(); $created = $wpdb->query( \"CREATE TABLE {$table} ( id bigint(20) unsigned NOT NULL AUTO_INCREMENT, lookup_key varchar(64) NOT NULL, lookup_value longtext NOT NULL, PRIMARY KEY (id), UNIQUE KEY lookup_key (lookup_key) ) {$charset_collate}\" ); if ( false === $created ) { throw new \\RuntimeException( 'Could not create fixture table: ' . $wpdb->last_error ); } $wpdb->insert( $table, array( 'lookup_key' => 'target', 'lookup_value' => 'old' ), array( '%s', '%s' ) ); $wpdb->insert( $table, array( 'lookup_key' => 'keep', 'lookup_value' => 'unchanged' ), array( '%s', '%s' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_database_004' ) ) { return false; } global $wpdb; $table = $wpdb->prefix . 'wpbp_database_004'; $updated = wpbp_database_004( 'target', 'new' ); $stored = $wpdb->get_var( $wpdb->prepare( \"SELECT lookup_value FROM {$table} WHERE lookup_key = %s\", 'target' ) ); $keep = $wpdb->get_var( $wpdb->prepare( \"SELECT lookup_value FROM {$table} WHERE lookup_key = %s\", 'keep' ) ); $missing = wpbp_database_004( 'missing', 'unused' ); return 1 === $updated && 'new' === $stored && 'unchanged' === $keep && 0 === $missing;", + "description": "The function updates only the matching custom-table row and returns row counts", + "weight": 1 + } + ], + "teardown": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_004'; $wpdb->query( \"DROP TABLE IF EXISTS {$table}\" );" + }, + "reference_solution": "function wpbp_database_004( string $lookup_key, string $value ): int|false { global $wpdb; $table = $wpdb->prefix . 'wpbp_database_004'; return $wpdb->update( $table, array( 'lookup_value' => $value ), array( 'lookup_key' => $lookup_key ), array( '%s' ), array( '%s' ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wpdb.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-database-005", + "category": "database", + "difficulty": "basic", + "prompt": "Implement wpbp_database_005( string $lookup_key ) to delete exactly the matching row from the WordPress-prefixed custom table `wpbp_database_005`.", + "expected_behavior": "`wpbp_database_005()` calls `$wpdb->delete()` on the fixture-owned custom table, matches only the supplied `lookup_key`, and returns the method's row-count result. Runtime validation checks that the target row is removed, an unrelated row remains, and a missing key returns zero affected rows.", + "requirements": [ + "Target the WordPress-prefixed `wpbp_database_005` custom table", + "Accept the lookup key as input", + "Return the affected row count from the database API" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_database_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "->delete", + "description": "Uses $wpdb->delete", + "weight": 1 + }, + { + "pattern": "lookup_key", + "description": "Matches rows by lookup_key", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_005'; $charset_collate = $wpdb->get_charset_collate(); $created = $wpdb->query( \"CREATE TABLE {$table} ( id bigint(20) unsigned NOT NULL AUTO_INCREMENT, lookup_key varchar(64) NOT NULL, lookup_value longtext NOT NULL, PRIMARY KEY (id), UNIQUE KEY lookup_key (lookup_key) ) {$charset_collate}\" ); if ( false === $created ) { throw new \\RuntimeException( 'Could not create fixture table: ' . $wpdb->last_error ); } $wpdb->insert( $table, array( 'lookup_key' => 'target', 'lookup_value' => 'remove' ), array( '%s', '%s' ) ); $wpdb->insert( $table, array( 'lookup_key' => 'keep', 'lookup_value' => 'keep' ), array( '%s', '%s' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_database_005' ) ) { return false; } global $wpdb; $table = $wpdb->prefix . 'wpbp_database_005'; $deleted = wpbp_database_005( 'target' ); $target = $wpdb->get_var( $wpdb->prepare( \"SELECT lookup_value FROM {$table} WHERE lookup_key = %s\", 'target' ) ); $keep = $wpdb->get_var( $wpdb->prepare( \"SELECT lookup_value FROM {$table} WHERE lookup_key = %s\", 'keep' ) ); $missing = wpbp_database_005( 'missing' ); return 1 === $deleted && null === $target && 'keep' === $keep && 0 === $missing;", + "description": "The function deletes only the matching custom-table row and returns row counts", + "weight": 1 + } + ], + "teardown": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_005'; $wpdb->query( \"DROP TABLE IF EXISTS {$table}\" );" + }, + "reference_solution": "function wpbp_database_005( string $lookup_key ): int|false { global $wpdb; $table = $wpdb->prefix . 'wpbp_database_005'; return $wpdb->delete( $table, array( 'lookup_key' => $lookup_key ), array( '%s' ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wpdb.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-database-006", + "category": "database", + "difficulty": "intermediate", + "prompt": "Implement wpbp_database_006() to create or update the WordPress-prefixed custom table `wpbp_database_006` using WordPress's schema-upgrade path, then return the upgrade result array.", + "expected_behavior": "`wpbp_database_006()` loads WordPress's upgrade functions, builds a schema for the fixture-owned custom table with `$wpdb->get_charset_collate()`, and passes it to `dbDelta()`. Runtime validation checks that the table exists with the expected columns, auto-increment primary key, default score value, and slug index.", + "requirements": [ + "Create `id`, `slug`, `score`, and `created_at` columns", + "Use the site's charset and collation for the table schema", + "Return the schema-upgrade result array" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_database_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "dbDelta", + "description": "Uses dbDelta", + "weight": 1 + }, + { + "pattern": "upgrade.php", + "description": "Loads the upgrade functions file", + "weight": 1 + }, + { + "pattern": "get_charset_collate", + "description": "Uses the site's charset/collation", + "weight": 1 + }, + { + "pattern": "PRIMARY KEY", + "description": "Defines the primary key for dbDelta", + "weight": 1 + }, + { + "pattern": "KEY slug", + "description": "Defines the slug index", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_006'; if ( $wpdb->get_var( $wpdb->prepare( 'SHOW TABLES LIKE %s', $table ) ) === $table ) { throw new \\RuntimeException( 'Fixture table already exists: ' . $table ); }", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_database_006' ) ) { return false; } global $wpdb; $table = $wpdb->prefix . 'wpbp_database_006'; $result = wpbp_database_006(); $exists = $wpdb->get_var( $wpdb->prepare( 'SHOW TABLES LIKE %s', $table ) ); if ( $exists !== $table || ! is_array( $result ) ) { return false; } $columns = $wpdb->get_results( \"SHOW COLUMNS FROM {$table}\", OBJECT_K ); if ( ! isset( $columns['id'], $columns['slug'], $columns['score'], $columns['created_at'] ) ) { return false; } $indexes = $wpdb->get_results( \"SHOW INDEX FROM {$table}\", ARRAY_A ); $has_slug_key = false; foreach ( $indexes as $index ) { if ( 'slug' === ( $index['Key_name'] ?? '' ) && 'slug' === ( $index['Column_name'] ?? '' ) ) { $has_slug_key = true; } } return str_contains( strtolower( (string) $columns['id']->Extra ), 'auto_increment' ) && '0' === (string) $columns['score']->Default && $has_slug_key;", + "description": "dbDelta creates the custom table with the expected columns and slug index", + "weight": 1 + } + ], + "teardown": "global $wpdb; $table = $wpdb->prefix . 'wpbp_database_006'; $wpdb->query( \"DROP TABLE IF EXISTS {$table}\" );" + }, + "reference_solution": "function wpbp_database_006(): array { global $wpdb; require_once ABSPATH . 'wp-admin/includes/upgrade.php'; $table = $wpdb->prefix . 'wpbp_database_006'; $charset_collate = $wpdb->get_charset_collate(); $sql = \"CREATE TABLE {$table} (\\nid bigint(20) unsigned NOT NULL AUTO_INCREMENT,\\nslug varchar(80) NOT NULL,\\nscore int(11) unsigned NOT NULL DEFAULT 0,\\ncreated_at datetime NOT NULL,\\nPRIMARY KEY (id),\\nKEY slug (slug)\\n) {$charset_collate};\"; return dbDelta( $sql ); }", + "metadata": { + "source_refs": [ + "src/wp-admin/includes/upgrade.php", + "src/wp-includes/class-wpdb.php" + ], + "release_focus": "classic" + } } ] } diff --git a/datasets/suites/wp-core-v1/execution/font-library.json b/datasets/suites/wp-core-v1/execution/font-library.json deleted file mode 100644 index 1fb3794..0000000 --- a/datasets/suites/wp-core-v1/execution/font-library.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "id": "wp-core-execution-v1-font_library", - "version": "1.1.0", - "metadata": { - "name": "WordPress Core Execution Tests - Font Library", - "description": "Code generation tasks for WordPress Font Library API", - "wp_version": "6.9", - "created_at": "2025-12-16" - }, - "tests": [ - { - "id": "e-font-library-disable-001", - "category": "font-library", - "difficulty": "hard", - "prompt": "Disable the Font Library UI everywhere via block_editor_settings_all so editors never see the Font Library panel.", - "expected_behavior": "Applying block_editor_settings_all yields fontLibraryEnabled=false.", - "requirements": [ - "Add a filter on block_editor_settings_all that sets fontLibraryEnabled to false", - "Avoid unregistering post types or removing menus", - "Filter must run in both site editor and post editor contexts" - ], - "static_checks": { - "required_patterns": [ - { "pattern": "block_editor_settings_all", "description": "Global editor settings filter", "weight": 1.0 }, - { "pattern": "fontLibraryEnabled", "description": "Sets flag to false", "weight": 0.8 } - ], - "forbidden_patterns": [ - { "pattern": "remove_menu_page", "description": "Should not hide via admin menu", "severity": "warning" } - ] - }, - "runtime_checks": { - "assertions": [ - { - "type": "custom_assertion", - "code": "$settings = apply_filters( 'block_editor_settings_all', array(), array() ); return isset( $settings['fontLibraryEnabled'] ) && $settings['fontLibraryEnabled'] === false;", - "description": "Font Library disabled in editor settings", - "weight": 1.0 - } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "add_filter( 'block_editor_settings_all', function( $settings, $context ) {\n $settings['fontLibraryEnabled'] = false;\n return $settings;\n}, 10, 2 );" - } - ] -} diff --git a/datasets/suites/wp-core-v1/execution/gb-block-api.json b/datasets/suites/wp-core-v1/execution/gb-block-api.json new file mode 100644 index 0000000..3b1fd1a --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/gb-block-api.json @@ -0,0 +1,351 @@ +{ + "id": "wp-core-execution-v1-gb_block_api", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Block API", + "description": "Code generation tasks for WordPress Block API APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-gb-block-api-001", + "category": "gb-block-api", + "difficulty": "basic", + "prompt": "Implement wpbp_gb_block_api_001(): bool to register a dynamic block type named 'wpbp/dynamic' that renders '

Dynamic

' on the server.", + "expected_behavior": "`wpbp_gb_block_api_001()` uses `register_block_type()` to register `wpbp/dynamic` with a server-side render callback. Runtime validation invokes the function, checks the block registry, verifies the block is dynamic, and renders the block through WordPress.", + "requirements": [ + "Use the server-side Block API", + "Register the block name 'wpbp/dynamic'", + "Return whether WordPress registered the block" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_api_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_block_type", + "description": "Uses register_block_type", + "weight": 1 + }, + { + "pattern": "wpbp/dynamic", + "description": "Registers the requested block name", + "weight": 1 + }, + { + "pattern": "render_callback", + "description": "Uses render_callback", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_api_001' ) ) { return false; } $ok = wpbp_gb_block_api_001(); $type = WP_Block_Type_Registry::get_instance()->get_registered( 'wpbp/dynamic' ); return true === $ok && $type instanceof WP_Block_Type && $type->is_dynamic() && '

Dynamic

' === trim( do_blocks( '' ) );", + "description": "The wpbp/dynamic block is registered as dynamic and renders through do_blocks()", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_api_001(): bool { register_block_type( 'wpbp/dynamic', array( 'render_callback' => fn() => '

Dynamic

' ) ); return WP_Block_Type_Registry::get_instance()->is_registered( 'wpbp/dynamic' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block-type.php", + "src/wp-includes/class-wp-block-type-registry.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-block-api-002", + "category": "gb-block-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_api_002(): bool to register a PHP-only block named 'wpbp/auto' that auto-registers for the editor, has a string 'message' attribute defaulting to 'Hi', and renders the message on the server.", + "expected_behavior": "`wpbp_gb_block_api_002()` registers `wpbp/auto` with the WordPress 7.0 PHP-only block auto-registration support, defines the requested message attribute, and provides a render callback. Runtime validation checks the registered block type, support flag, attribute default, dynamic status, and rendered output.", + "requirements": [ + "Use the server-side Block API", + "Register the block name 'wpbp/auto'", + "Enable PHP-only auto-registration", + "Return whether WordPress registered the block" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_api_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "autoRegister", + "description": "Uses autoRegister", + "weight": 1 + }, + { + "pattern": "register_block_type", + "description": "Uses register_block_type", + "weight": 1 + }, + { + "pattern": "wpbp/auto", + "description": "Registers the requested block name", + "weight": 1 + }, + { + "pattern": "message", + "description": "Defines the requested message attribute", + "weight": 1 + }, + { + "pattern": "render_callback", + "description": "Uses a server-side render callback", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_api_002' ) ) { return false; } $ok = wpbp_gb_block_api_002(); $type = WP_Block_Type_Registry::get_instance()->get_registered( 'wpbp/auto' ); if ( ! $type instanceof WP_Block_Type ) { return false; } $out = trim( do_blocks( '' ) ); return true === $ok && ! empty( $type->supports['autoRegister'] ) && $type->is_dynamic() && 'Hi' === ( $type->attributes['message']['default'] ?? null ) && '

Hello

' === $out;", + "description": "The wpbp/auto block exposes autoRegister support, a message default, and server rendering", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_api_002(): bool { register_block_type( 'wpbp/auto', array( 'title' => 'Auto', 'attributes' => array( 'message' => array( 'type' => 'string', 'default' => 'Hi' ) ), 'supports' => array( 'autoRegister' => true ), 'render_callback' => static fn( array $attrs ): string => '

' . esc_html( $attrs['message'] ?? '' ) . '

' ) ); return WP_Block_Type_Registry::get_instance()->is_registered( 'wpbp/auto' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block-type.php", + "src/wp-includes/class-wp-block-type-registry.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-gb-block-api-003", + "category": "gb-block-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_api_003( string $content ): array to count each explicit block type in the supplied block markup with WP_Block_Processor.", + "expected_behavior": "`wpbp_gb_block_api_003()` streams through the supplied block markup with `WP_Block_Processor`, counts every explicit block opening including nested blocks, and returns counts keyed by fully qualified block names. Runtime validation includes freeform HTML, nested blocks, and repeated block types.", + "requirements": [ + "Use WP_Block_Processor instead of parse_blocks()", + "Return an associative array keyed by fully qualified block names", + "Count nested blocks as they are encountered" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_api_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Block_Processor", + "description": "Uses WP_Block_Processor", + "weight": 1 + }, + { + "pattern": "next_block", + "description": "Uses next_block", + "weight": 1 + }, + { + "pattern": "get_block_type", + "description": "Reads fully qualified block names from the processor", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_api_003' ) ) { return false; } $content = '

Freeform

A

B

'; $counts = wpbp_gb_block_api_003( $content ); return is_array( $counts ) && 1 === ( $counts['core/group'] ?? null ) && 2 === ( $counts['core/paragraph'] ?? null ) && 1 === ( $counts['core/image'] ?? null ) && ! isset( $counts['core/freeform'] );", + "description": "The processor count includes nested and repeated explicit blocks and ignores freeform HTML", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_api_003( string $content ): array { $processor = new WP_Block_Processor( $content ); $counts = array(); while ( $processor->next_block() ) { $name = $processor->get_block_type(); if ( null !== $name ) { $counts[ $name ] = ( $counts[ $name ] ?? 0 ) + 1; } } return $counts; }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block.php", + "src/wp-includes/class-wp-block-processor.php" + ], + "release_focus": "6.9" + } + }, + { + "id": "e-gb-block-api-004", + "category": "gb-block-api", + "difficulty": "basic", + "prompt": "Implement wpbp_gb_block_api_004( array $block ): string to serialize a parsed block array back to WordPress block markup.", + "expected_behavior": "`wpbp_gb_block_api_004()` delegates parsed block serialization to WordPress. Runtime validation checks that core block names are serialized with the expected shorthand, inner HTML is preserved, and comment-sensitive attribute characters are escaped by WordPress.", + "requirements": [ + "Use the server-side block serialization helper", + "Return the serialized markup string" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_api_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "serialize_block", + "description": "Uses serialize_block", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_api_004' ) ) { return false; } $block = array( 'blockName' => 'core/paragraph', 'attrs' => array( 'metadata' => array( 'name' => 'A -- & \"quote\"' ) ), 'innerBlocks' => array(), 'innerHTML' => '

Hi

', 'innerContent' => array( '

Hi

' ) ); $out = wpbp_gb_block_api_004( $block ); return is_string( $out ) && str_starts_with( $out, '' );", + "description": "Serialized markup preserves inner HTML and escapes comment-sensitive attributes", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_api_004( array $block ): string { return serialize_block( $block ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block.php", + "src/wp-includes/class-wp-block-processor.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-block-api-005", + "category": "gb-block-api", + "difficulty": "basic", + "prompt": "Implement wpbp_gb_block_api_005(): bool to register a 'wpbp-muted' block style for 'core/paragraph' labeled 'Muted'.", + "expected_behavior": "`wpbp_gb_block_api_005()` uses the server-side block style API to register the `wpbp-muted` style for `core/paragraph`. Runtime validation inspects `WP_Block_Styles_Registry` to confirm the style name and label WordPress registered.", + "requirements": [ + "Use the server-side block style API", + "Register the style for 'core/paragraph'", + "Return whether WordPress registered the style" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_api_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_block_style", + "description": "Uses register_block_style", + "weight": 1 + }, + { + "pattern": "core/paragraph", + "description": "Targets the paragraph block", + "weight": 1 + }, + { + "pattern": "wpbp-muted", + "description": "Registers the requested style name", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_api_005' ) ) { return false; } $ok = wpbp_gb_block_api_005(); $style = WP_Block_Styles_Registry::get_instance()->get_registered( 'core/paragraph', 'wpbp-muted' ); return true === $ok && is_array( $style ) && 'wpbp-muted' === ( $style['name'] ?? null ) && 'Muted' === ( $style['label'] ?? null );", + "description": "The wpbp-muted style is registered for core/paragraph with the requested label", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_api_005(): bool { register_block_style( 'core/paragraph', array( 'name' => 'wpbp-muted', 'label' => 'Muted' ) ); return WP_Block_Styles_Registry::get_instance()->is_registered( 'core/paragraph', 'wpbp-muted' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block-styles-registry.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-block-api-006", + "category": "gb-block-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_api_006( string $content ): array to return the first image block found while scanning only the first ten block openings with WP_Block_Processor.", + "expected_behavior": "`wpbp_gb_block_api_006()` streams through block markup with `WP_Block_Processor`, matches image blocks through the processor, and returns the parsed block array from `extract_full_block_and_advance()`. Runtime validation checks both the extracted image attributes and that an image after the first ten block openings is ignored.", + "requirements": [ + "Use WP_Block_Processor", + "Return the parsed image block array from WordPress", + "Return an empty array when no image appears in the first ten block openings" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_api_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Block_Processor", + "description": "Uses WP_Block_Processor", + "weight": 1 + }, + { + "pattern": "next_block", + "description": "Scans block openings with the processor", + "weight": 1 + }, + { + "pattern": "is_block_type", + "description": "Matches the image block through the processor", + "weight": 1 + }, + { + "pattern": "extract_full_block_and_advance", + "description": "Uses extract_full_block_and_advance", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_api_006' ) ) { return false; } $out = wpbp_gb_block_api_006( '

A

' ); return is_array( $out ) && 'core/image' === ( $out['blockName'] ?? null ) && 7 === ( $out['attrs']['id'] ?? null ) && 'Hero' === ( $out['attrs']['alt'] ?? null );", + "description": "The first image block is extracted as a parsed block with attributes", + "weight": 1 + }, + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_api_006' ) ) { return false; } $content = str_repeat( '

A

', 10 ) . ''; return array() === wpbp_gb_block_api_006( $content );", + "description": "An image after the first ten block openings is ignored", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_api_006( string $content ): array { $processor = new WP_Block_Processor( $content ); $seen = 0; while ( $seen < 10 && $processor->next_block() ) { ++$seen; if ( $processor->is_block_type( 'image' ) ) { return $processor->extract_full_block_and_advance() ?? array(); } } return array(); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block-processor.php" + ], + "release_focus": "6.9" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/gb-block-bindings.json b/datasets/suites/wp-core-v1/execution/gb-block-bindings.json new file mode 100644 index 0000000..81e6d97 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/gb-block-bindings.json @@ -0,0 +1,268 @@ +{ + "id": "wp-core-execution-v1-gb_block_bindings", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Block Bindings", + "description": "Code generation tasks for WordPress Block Bindings APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-gb-block-bindings-001", + "category": "gb-block-bindings", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_bindings_001(): bool to register a server-side block binding source named 'wpbp/source' that can replace a bound paragraph's content with the binding's 'text' argument.", + "expected_behavior": "`wpbp_gb_block_bindings_001()` registers the `wpbp/source` binding source with WordPress and returns whether the source is registered. Runtime validation renders a paragraph whose `content` attribute is bound to the source and confirms WordPress replaces the fallback text with the binding argument.", + "requirements": [ + "Use register_block_bindings_source()", + "Register the source name 'wpbp/source'", + "Return whether WordPress registered the source" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_bindings_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_block_bindings_source", + "description": "Uses register_block_bindings_source", + "weight": 1 + }, + { + "pattern": "wpbp/source", + "description": "Registers the requested source name", + "weight": 1 + }, + { + "pattern": "get_value_callback", + "description": "Defines a source value callback", + "weight": 1 + }, + { + "pattern": "text", + "description": "Reads the requested binding argument", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_bindings_001' ) ) { return false; } $ok = wpbp_gb_block_bindings_001(); $source = get_block_bindings_source( 'wpbp/source' ); if ( true !== $ok || ! $source instanceof WP_Block_Bindings_Source ) { return false; } $content = '

Fallback

'; return '

Bound paragraph

' === trim( do_blocks( $content ) );", + "description": "The wpbp/source source is registered and supplies paragraph content through block rendering", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_bindings_001(): bool { register_block_bindings_source( 'wpbp/source', array( 'label' => 'WPBP Source', 'get_value_callback' => static function ( array $source_args, WP_Block $block_instance, string $attribute_name ): string { return isset( $source_args['text'] ) ? (string) $source_args['text'] : 'bound'; } ) ); return null !== get_block_bindings_source( 'wpbp/source' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/block-bindings.php", + "src/wp-includes/class-wp-block-bindings-source.php", + "src/wp-includes/class-wp-block-bindings-registry.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-block-bindings-002", + "category": "gb-block-bindings", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_bindings_002(): void so WordPress treats the 'title' attribute of the custom block type 'wpbp/card' as bindable without changing bindable attributes for unrelated block types.", + "expected_behavior": "`wpbp_gb_block_bindings_002()` hooks the Block Bindings supported-attributes API for `wpbp/card`, preserves existing attribute lists, and adds `title` only for that block type. Runtime validation registers a dynamic `wpbp/card` fixture, binds its `title` attribute to a source, and confirms WordPress passes the bound title into the block render callback.", + "requirements": [ + "Use the supported block attributes filter", + "Target only the 'wpbp/card' block type", + "Preserve existing supported attributes" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_bindings_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "block_bindings_supported_attributes", + "description": "Uses block_bindings_supported_attributes", + "weight": 1 + }, + { + "pattern": "add_filter", + "description": "Registers a filter callback", + "weight": 1 + }, + { + "pattern": "wpbp/card", + "description": "Targets the requested block type", + "weight": 1 + }, + { + "pattern": "title", + "description": "Adds the requested bindable attribute", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_bindings_002' ) ) { return false; } register_block_type( 'wpbp/card', array( 'attributes' => array( 'title' => array( 'type' => 'string', 'default' => 'Fallback Card' ) ), 'render_callback' => static fn( array $attrs ): string => '
' . esc_html( $attrs['title'] ?? '' ) . '
' ) ); register_block_bindings_source( 'wpbp/card-title-source', array( 'label' => 'Card Title Source', 'get_value_callback' => static fn(): string => 'Bound Card' ) ); wpbp_gb_block_bindings_002(); $card_attrs = get_block_bindings_supported_attributes( 'wpbp/card' ); $paragraph_attrs = get_block_bindings_supported_attributes( 'core/paragraph' ); $content = ''; $out = trim( do_blocks( $content ) ); return in_array( 'title', $card_attrs, true ) && ! in_array( 'title', $paragraph_attrs, true ) && '
Bound Card
' === $out;", + "description": "Only wpbp/card gains a bindable title attribute, and rendering receives the bound value", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_bindings_002(): void { add_filter( 'block_bindings_supported_attributes', static function ( array $supported_attributes, string $block_type ): array { if ( 'wpbp/card' !== $block_type ) { return $supported_attributes; } if ( ! in_array( 'title', $supported_attributes, true ) ) { $supported_attributes[] = 'title'; } return $supported_attributes; }, 10, 2 ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/block-bindings.php", + "src/wp-includes/class-wp-block-bindings-registry.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-gb-block-bindings-003", + "category": "gb-block-bindings", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_bindings_003(): bool to register a block binding source named 'wpbp/meta-title' that reads the current block's postId context and returns that post's '_wpbp_title' meta value for bound content.", + "expected_behavior": "`wpbp_gb_block_bindings_003()` registers `wpbp/meta-title`, declares the `postId` context needed by the source, and returns whether WordPress registered the source. Runtime validation creates a post fixture with `_wpbp_title`, renders a paragraph block with matching context, and confirms the bound content comes from post meta.", + "requirements": [ + "Use register_block_bindings_source()", + "Declare the postId context for the source", + "Return whether WordPress registered the source" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_bindings_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_block_bindings_source", + "description": "Uses register_block_bindings_source", + "weight": 1 + }, + { + "pattern": "uses_context", + "description": "Uses uses_context", + "weight": 1 + }, + { + "pattern": "wpbp/meta-title", + "description": "Registers the requested source name", + "weight": 1 + }, + { + "pattern": "postId", + "description": "Requests postId block context", + "weight": 1 + }, + { + "pattern": "get_post_meta", + "description": "Reads post meta through WordPress", + "weight": 1 + }, + { + "pattern": "_wpbp_title", + "description": "Reads the requested meta key", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$post_id = wp_insert_post( array( 'post_title' => 'WPBP Binding Fixture', 'post_status' => 'publish', 'post_type' => 'post' ) ); update_post_meta( $post_id, '_wpbp_title', 'Context Meta Title' ); update_option( 'wpbp_gb_block_bindings_003_post_id', $post_id, false );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_bindings_003' ) ) { return false; } $post_id = (int) get_option( 'wpbp_gb_block_bindings_003_post_id' ); if ( ! $post_id ) { return false; } $ok = wpbp_gb_block_bindings_003(); $source = get_block_bindings_source( 'wpbp/meta-title' ); if ( true !== $ok || ! $source instanceof WP_Block_Bindings_Source || ! in_array( 'postId', $source->uses_context ?? array(), true ) ) { return false; } $parsed = parse_blocks( '

Fallback

' ); $block = new WP_Block( $parsed[0], array( 'postId' => $post_id ) ); return '

Context Meta Title

' === trim( $block->render() );", + "description": "The wpbp/meta-title source uses postId context to render post meta as paragraph content", + "weight": 1 + } + ], + "teardown": "$post_id = (int) get_option( 'wpbp_gb_block_bindings_003_post_id' ); if ( $post_id ) { wp_delete_post( $post_id, true ); } delete_option( 'wpbp_gb_block_bindings_003_post_id' );" + }, + "reference_solution": "function wpbp_gb_block_bindings_003(): bool { register_block_bindings_source( 'wpbp/meta-title', array( 'label' => 'Meta Title', 'uses_context' => array( 'postId' ), 'get_value_callback' => static function ( array $source_args, WP_Block $block_instance, string $attribute_name ): ?string { if ( 'content' !== $attribute_name ) { return null; } $post_id = isset( $block_instance->context['postId'] ) ? (int) $block_instance->context['postId'] : 0; if ( ! $post_id ) { return null; } return (string) get_post_meta( $post_id, '_wpbp_title', true ); } ) ); return null !== get_block_bindings_source( 'wpbp/meta-title' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/block-bindings.php", + "src/wp-includes/class-wp-block-bindings-source.php", + "src/wp-includes/class-wp-block-bindings-registry.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-block-bindings-004", + "category": "gb-block-bindings", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_bindings_004(): void to filter block binding source values so only values from the 'wpbp/filter-source' source bound to the 'content' attribute are uppercased.", + "expected_behavior": "`wpbp_gb_block_bindings_004()` hooks `block_bindings_source_value` with access to the source name and attribute name, uppercases values only for `wpbp/filter-source` content bindings, and returns unrelated binding values unchanged. Runtime validation renders two bound paragraphs through WordPress to verify both the filtered and unfiltered paths.", + "requirements": [ + "Use the block_bindings_source_value filter", + "Check the source name and attribute name before changing the value", + "Leave unrelated sources unchanged" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_bindings_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "block_bindings_source_value", + "description": "Uses block_bindings_source_value", + "weight": 1 + }, + { + "pattern": "add_filter", + "description": "Registers a filter callback", + "weight": 1 + }, + { + "pattern": "wpbp/filter-source", + "description": "Targets the requested binding source", + "weight": 1 + }, + { + "pattern": "content", + "description": "Checks the requested bound attribute", + "weight": 1 + }, + { + "pattern": "strtoupper", + "description": "Uppercases the filtered value", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_bindings_004' ) ) { return false; } register_block_bindings_source( 'wpbp/filter-source', array( 'label' => 'Filter Source', 'get_value_callback' => static fn(): string => 'mixed case' ) ); register_block_bindings_source( 'wpbp/other-filter-source', array( 'label' => 'Other Filter Source', 'get_value_callback' => static fn(): string => 'mixed case' ) ); wpbp_gb_block_bindings_004(); $filtered = trim( do_blocks( '

Fallback

' ) ); $unfiltered = trim( do_blocks( '

Fallback

' ) ); return '

MIXED CASE

' === $filtered && '

mixed case

' === $unfiltered;", + "description": "Only wpbp/filter-source content bindings are uppercased during WordPress rendering", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_bindings_004(): void { add_filter( 'block_bindings_source_value', static function ( $value, string $source_name, array $source_args, WP_Block $block_instance, string $attribute_name ) { if ( 'wpbp/filter-source' !== $source_name || 'content' !== $attribute_name ) { return $value; } return strtoupper( (string) $value ); }, 10, 5 ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/block-bindings.php", + "src/wp-includes/class-wp-block-bindings-source.php", + "src/wp-includes/class-wp-block-bindings-registry.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/gb-block-editor.json b/datasets/suites/wp-core-v1/execution/gb-block-editor.json new file mode 100644 index 0000000..9303f29 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/gb-block-editor.json @@ -0,0 +1,261 @@ +{ + "id": "wp-core-execution-v1-gb_block_editor", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Block Editor", + "description": "Code generation tasks for WordPress Block Editor APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-gb-block-editor-001", + "category": "gb-block-editor", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_editor_001(): array to define dimension size presets in a WP_Theme_JSON object and return the presets WordPress exposes from theme settings.", + "expected_behavior": "`wpbp_gb_block_editor_001()` builds theme.json data with the latest schema, defines the requested dimension size presets, reads settings through `WP_Theme_JSON`, and returns the theme-origin presets exactly as WordPress normalizes them.", + "requirements": [ + "Use WP_Theme_JSON with the latest schema", + "Define Small and Large dimension size presets", + "Return the normalized theme-origin dimension size presets" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_editor_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Theme_JSON", + "description": "Uses WP_Theme_JSON", + "weight": 1 + }, + { + "pattern": "LATEST_SCHEMA", + "description": "Uses the latest theme.json schema", + "weight": 1 + }, + { + "pattern": "dimensionSizes", + "description": "Uses dimensionSizes", + "weight": 1 + }, + { + "pattern": "get_settings", + "description": "Reads normalized settings from WP_Theme_JSON", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_editor_001' ) ) { return false; } $presets = wpbp_gb_block_editor_001(); if ( ! is_array( $presets ) || 2 !== count( $presets ) ) { return false; } return 'Small' === ( $presets[0]['name'] ?? null ) && 'small' === ( $presets[0]['slug'] ?? null ) && '240px' === ( $presets[0]['size'] ?? null ) && 'Large' === ( $presets[1]['name'] ?? null ) && 'large' === ( $presets[1]['slug'] ?? null ) && '720px' === ( $presets[1]['size'] ?? null );", + "description": "The function returns WordPress-normalized theme-origin dimension size presets", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_editor_001(): array { $theme_json = new WP_Theme_JSON( array( 'version' => WP_Theme_JSON::LATEST_SCHEMA, 'settings' => array( 'dimensions' => array( 'dimensionSizes' => array( array( 'name' => 'Small', 'slug' => 'small', 'size' => '240px' ), array( 'name' => 'Large', 'slug' => 'large', 'size' => '720px' ) ) ) ) ) ); $settings = $theme_json->get_settings(); return $settings['dimensions']['dimensionSizes']['theme'] ?? array(); }", + "metadata": { + "source_refs": [ + "src/wp-includes/block-editor.php", + "src/wp-includes/class-wp-theme-json.php", + "src/wp-includes/block-supports/dimensions.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-gb-block-editor-002", + "category": "gb-block-editor", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_editor_002( string $content ): string to render supplied block markup through WordPress so blocks hidden with block visibility metadata are omitted.", + "expected_behavior": "`wpbp_gb_block_editor_002()` delegates rendering to the WordPress block parser/renderer. Runtime validation passes visible and hidden paragraph blocks together and expects only the visible block's rendered HTML to remain.", + "requirements": [ + "Use the server-side block rendering pipeline", + "Return rendered HTML for the supplied content", + "Do not strip hidden blocks with ad hoc string replacement" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_editor_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "do_blocks", + "description": "Uses do_blocks", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "preg_replace|str_replace", + "description": "Avoids manual string stripping instead of WordPress rendering", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_editor_002' ) ) { return false; } $content = '

Visible

Hidden

'; $out = trim( wpbp_gb_block_editor_002( $content ) ); return '' !== $out && str_contains( $out, 'Visible' ) && ! str_contains( $out, 'Hidden' );", + "description": "Visible block markup renders while a block with metadata.blockVisibility false is omitted", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_editor_002( string $content ): string { return do_blocks( $content ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block.php", + "src/wp-includes/block-supports/block-visibility.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-gb-block-editor-003", + "category": "gb-block-editor", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_editor_003(): bool to register a server-rendered block named 'wpbp/dimensions' that opts into width and height controls through dimensions support.", + "expected_behavior": "`wpbp_gb_block_editor_003()` registers `wpbp/dimensions` with the server-side Block API and declares `supports.dimensions.width` and `supports.dimensions.height`. Runtime validation inspects the registered `WP_Block_Type` rather than trusting the function's return value alone.", + "requirements": [ + "Use register_block_type()", + "Register the block name 'wpbp/dimensions'", + "Return whether WordPress registered the block" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_editor_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_block_type", + "description": "Uses register_block_type", + "weight": 1 + }, + { + "pattern": "wpbp/dimensions", + "description": "Registers the requested block name", + "weight": 1 + }, + { + "pattern": "dimensions", + "description": "Uses dimensions", + "weight": 1 + }, + { + "pattern": "width", + "description": "Enables width support", + "weight": 1 + }, + { + "pattern": "height", + "description": "Enables height support", + "weight": 1 + }, + { + "pattern": "render_callback", + "description": "Registers a server-rendered block", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_editor_003' ) ) { return false; } $ok = wpbp_gb_block_editor_003(); $type = WP_Block_Type_Registry::get_instance()->get_registered( 'wpbp/dimensions' ); return true === $ok && $type instanceof WP_Block_Type && ! empty( $type->supports['dimensions']['width'] ) && ! empty( $type->supports['dimensions']['height'] ) && $type->is_dynamic();", + "description": "The wpbp/dimensions block is registered with width and height dimension supports", + "weight": 1 + } + ], + "teardown": "if ( WP_Block_Type_Registry::get_instance()->is_registered( 'wpbp/dimensions' ) ) { WP_Block_Type_Registry::get_instance()->unregister( 'wpbp/dimensions' ); }" + }, + "reference_solution": "function wpbp_gb_block_editor_003(): bool { register_block_type( 'wpbp/dimensions', array( 'supports' => array( 'dimensions' => array( 'width' => true, 'height' => true ) ), 'render_callback' => static fn(): string => '
' ) ); return WP_Block_Type_Registry::get_instance()->is_registered( 'wpbp/dimensions' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block-type.php", + "src/wp-includes/block-supports/dimensions.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-gb-block-editor-004", + "category": "gb-block-editor", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_editor_004(): bool to register a server-rendered block named 'wpbp/text-indent' that opts into the typography textIndent support.", + "expected_behavior": "`wpbp_gb_block_editor_004()` registers `wpbp/text-indent` with the server-side Block API and declares `supports.typography.textIndent`. Runtime validation inspects the registered block type so a stubbed return value does not pass.", + "requirements": [ + "Use register_block_type()", + "Register the block name 'wpbp/text-indent'", + "Return whether WordPress registered the block" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_editor_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_block_type", + "description": "Uses register_block_type", + "weight": 1 + }, + { + "pattern": "wpbp/text-indent", + "description": "Registers the requested block name", + "weight": 1 + }, + { + "pattern": "textIndent", + "description": "Uses textIndent", + "weight": 1 + }, + { + "pattern": "typography", + "description": "Uses typography", + "weight": 1 + }, + { + "pattern": "render_callback", + "description": "Registers a server-rendered block", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_editor_004' ) ) { return false; } $ok = wpbp_gb_block_editor_004(); $type = WP_Block_Type_Registry::get_instance()->get_registered( 'wpbp/text-indent' ); return true === $ok && $type instanceof WP_Block_Type && ! empty( $type->supports['typography']['textIndent'] ) && $type->is_dynamic();", + "description": "The wpbp/text-indent block is registered with typography textIndent support", + "weight": 1 + } + ], + "teardown": "if ( WP_Block_Type_Registry::get_instance()->is_registered( 'wpbp/text-indent' ) ) { WP_Block_Type_Registry::get_instance()->unregister( 'wpbp/text-indent' ); }" + }, + "reference_solution": "function wpbp_gb_block_editor_004(): bool { register_block_type( 'wpbp/text-indent', array( 'supports' => array( 'typography' => array( 'textIndent' => true ) ), 'render_callback' => static fn(): string => '

' ) ); return WP_Block_Type_Registry::get_instance()->is_registered( 'wpbp/text-indent' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block-type.php", + "src/wp-includes/block-supports/typography.php" + ], + "release_focus": "7.0" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/gb-block-hooks.json b/datasets/suites/wp-core-v1/execution/gb-block-hooks.json new file mode 100644 index 0000000..fbebe5a --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/gb-block-hooks.json @@ -0,0 +1,292 @@ +{ + "id": "wp-core-execution-v1-gb_block_hooks", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Block Hooks", + "description": "Code generation tasks for WordPress Block Hooks APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-gb-block-hooks-001", + "category": "gb-block-hooks", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_hooks_001(): bool to register a dynamic block named 'wpbp/cta-hook' that hooks after every core/paragraph block and renders '

CTA

'.", + "expected_behavior": "`wpbp_gb_block_hooks_001()` registers `wpbp/cta-hook` with server-side Block Hooks metadata targeting `core/paragraph` in the `after` position and a render callback. Runtime validation checks the block registry, `get_hooked_blocks()`, serialized insertion from `apply_block_hooks_to_content()`, and rendered output through `do_blocks()`.", + "requirements": [ + "Use the server-side Block Hooks API", + "Register the block name 'wpbp/cta-hook'", + "Return whether WordPress registered the block" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_hooks_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_block_type", + "description": "Uses the server-side block registration API", + "weight": 1 + }, + { + "pattern": "wpbp/cta-hook", + "description": "Registers the requested hooked block name", + "weight": 1 + }, + { + "pattern": "block_hooks", + "description": "Declares Block Hooks metadata", + "weight": 1 + }, + { + "pattern": "core/paragraph", + "description": "Targets the paragraph block as the anchor", + "weight": 1 + }, + { + "pattern": "after", + "description": "Uses the requested after position", + "weight": 1 + }, + { + "pattern": "render_callback", + "description": "Provides dynamic server rendering", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_hooks_001' ) ) { return false; } $ok = wpbp_gb_block_hooks_001(); $registry = WP_Block_Type_Registry::get_instance(); $type = $registry->get_registered( 'wpbp/cta-hook' ); $hooked = get_hooked_blocks(); $content = '

A

'; $inserted = apply_block_hooks_to_content( $content, array( 'slug' => 'wpbp-pattern' ) ); $rendered = trim( do_blocks( $inserted ) ); $registry->unregister( 'wpbp/cta-hook' ); return true === $ok && $type instanceof WP_Block_Type && $type->is_dynamic() && 'after' === ( $type->block_hooks['core/paragraph'] ?? null ) && in_array( 'wpbp/cta-hook', $hooked['core/paragraph']['after'] ?? array(), true ) && str_contains( $inserted, '' ) && str_contains( $rendered, '

CTA

' );", + "description": "The wpbp/cta-hook block is registered, discovered by Block Hooks, inserted after a paragraph, and rendered dynamically", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_hooks_001(): bool { register_block_type( 'wpbp/cta-hook', array( 'block_hooks' => array( 'core/paragraph' => 'after' ), 'render_callback' => static fn(): string => '

CTA

' ) ); return WP_Block_Type_Registry::get_instance()->is_registered( 'wpbp/cta-hook' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block-type.php", + "src/wp-includes/class-wp-block-type-registry.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-block-hooks-002", + "category": "gb-block-hooks", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_hooks_002(): void to add 'wpbp/context-note' as a hooked block only as the first child of core/group blocks when the Block Hooks context array has slug 'wpbp-pattern'.", + "expected_behavior": "`wpbp_gb_block_hooks_002()` registers a `hooked_block_types` filter that preserves the incoming list, inspects the relative position, anchor block, and context, and appends `wpbp/context-note` only for `core/group` in the `first_child` position when the context slug is `wpbp-pattern`. Runtime validation checks direct filter output and serialized insertion.", + "requirements": [ + "Use the Block Hooks filter API", + "Preserve any existing hooked block types", + "Register the filter callback with access to the context argument" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_hooks_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_filter", + "description": "Registers a WordPress filter", + "weight": 1 + }, + { + "pattern": "hooked_block_types", + "description": "Uses the Block Hooks filter", + "weight": 1 + }, + { + "pattern": "wpbp/context-note", + "description": "Adds the requested hooked block", + "weight": 1 + }, + { + "pattern": "core/group", + "description": "Limits the hook to group anchors", + "weight": 1 + }, + { + "pattern": "first_child", + "description": "Uses the requested child insertion position", + "weight": 1 + }, + { + "pattern": "wpbp-pattern", + "description": "Checks the requested context slug", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_hooks_002' ) ) { return false; } wpbp_gb_block_hooks_002(); $matching = apply_filters( 'hooked_block_types', array( 'wpbp/existing' ), 'first_child', 'core/group', array( 'slug' => 'wpbp-pattern' ) ); $wrong_position = apply_filters( 'hooked_block_types', array(), 'after', 'core/group', array( 'slug' => 'wpbp-pattern' ) ); $wrong_context = apply_filters( 'hooked_block_types', array(), 'first_child', 'core/group', array( 'slug' => 'other-pattern' ) ); $content = '

A

'; $inserted = apply_block_hooks_to_content( $content, array( 'slug' => 'wpbp-pattern' ) ); $other = apply_block_hooks_to_content( $content, array( 'slug' => 'other-pattern' ) ); return in_array( 'wpbp/existing', $matching, true ) && in_array( 'wpbp/context-note', $matching, true ) && ! in_array( 'wpbp/context-note', $wrong_position, true ) && ! in_array( 'wpbp/context-note', $wrong_context, true ) && str_contains( $inserted, '' ) && strpos( $inserted, '' ) < strpos( $inserted, '' ) && ! str_contains( $other, '' );", + "description": "The filter appends wpbp/context-note only for the matching group first-child context and produces first-child serialized insertion", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_hooks_002(): void { add_filter( 'hooked_block_types', static function ( array $hooked_block_types, string $relative_position, string $anchor_block_type, $context ): array { if ( 'first_child' !== $relative_position || 'core/group' !== $anchor_block_type ) { return $hooked_block_types; } if ( ! is_array( $context ) || 'wpbp-pattern' !== ( $context['slug'] ?? '' ) ) { return $hooked_block_types; } $hooked_block_types[] = 'wpbp/context-note'; return $hooked_block_types; }, 10, 4 ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-block-hooks-003", + "category": "gb-block-hooks", + "difficulty": "hard", + "prompt": "Implement wpbp_gb_block_hooks_003( WP_Post $post ): array to apply Block Hooks to a post object and return both the updated raw content and the root-level ignored hooked block names WordPress reports.", + "expected_behavior": "`wpbp_gb_block_hooks_003()` runs the post-object Block Hooks algorithm with the WordPress callback that inserts hooked blocks and records ignored hooked block metadata. It returns an array with `content` containing the updated raw block markup and `ignored` containing the root-level ignored hooked block names populated by WordPress.", + "requirements": [ + "Use WordPress post-object Block Hooks processing", + "Use the callback that inserts hooked blocks and records ignored metadata", + "Return an array with keys 'content' and 'ignored'" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_hooks_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "apply_block_hooks_to_content_from_post_object", + "description": "Uses post-object Block Hooks processing", + "weight": 1 + }, + { + "pattern": "insert_hooked_blocks_and_set_ignored_hooked_blocks_metadata", + "description": "Uses the metadata-recording insertion callback", + "weight": 1 + }, + { + "pattern": "content", + "description": "Returns the updated content value", + "weight": 1 + }, + { + "pattern": "ignored", + "description": "Returns the ignored hooked blocks value", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_gb_block_hooks_003_post_id'] = wp_insert_post( array( 'post_title' => 'WPBP Block Hooks Fixture', 'post_status' => 'draft', 'post_type' => 'post', 'post_content' => '

A

' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_hooks_003' ) ) { return false; } register_block_type( 'wpbp/root-note', array( 'block_hooks' => array( 'core/post-content' => 'first_child' ), 'render_callback' => static fn(): string => '

Root

' ) ); $post = get_post( (int) ( $GLOBALS['wpbp_gb_block_hooks_003_post_id'] ?? 0 ) ); $result = $post instanceof WP_Post ? wpbp_gb_block_hooks_003( $post ) : null; WP_Block_Type_Registry::get_instance()->unregister( 'wpbp/root-note' ); if ( ! is_array( $result ) ) { return false; } $content = $result['content'] ?? ''; $ignored = $result['ignored'] ?? null; return is_string( $content ) && str_starts_with( $content, '' ) && str_contains( $content, '' ) && is_array( $ignored ) && in_array( 'wpbp/root-note', $ignored, true );", + "description": "Post-object processing inserts a root hooked block and reports the root ignored hooked block name by reference", + "weight": 1 + } + ], + "teardown": "if ( ! empty( $GLOBALS['wpbp_gb_block_hooks_003_post_id'] ) ) { wp_delete_post( (int) $GLOBALS['wpbp_gb_block_hooks_003_post_id'], true ); unset( $GLOBALS['wpbp_gb_block_hooks_003_post_id'] ); }" + }, + "reference_solution": "function wpbp_gb_block_hooks_003( WP_Post $post ): array { $ignored = array(); $content = apply_block_hooks_to_content_from_post_object( $post->post_content, $post, 'insert_hooked_blocks_and_set_ignored_hooked_blocks_metadata', $ignored ); return array( 'content' => $content, 'ignored' => $ignored ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-post.php", + "src/wp-includes/post.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-gb-block-hooks-004", + "category": "gb-block-hooks", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_block_hooks_004(): bool to register a dynamic block named 'wpbp/single-hook' that hooks after core/paragraph, renders '

Once

', and allows only one instance in a context.", + "expected_behavior": "`wpbp_gb_block_hooks_004()` registers `wpbp/single-hook` with Block Hooks metadata targeting `core/paragraph` after the anchor, a render callback, and block supports that disable multiple instances. Runtime validation checks the registry support flag and verifies that WordPress inserts the hooked block only once across repeated anchors and does not add another copy when one already exists.", + "requirements": [ + "Use the server-side Block Hooks API", + "Disable multiple instances with block supports", + "Return whether WordPress registered the block" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_block_hooks_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_block_type", + "description": "Uses the server-side block registration API", + "weight": 1 + }, + { + "pattern": "wpbp/single-hook", + "description": "Registers the requested hooked block name", + "weight": 1 + }, + { + "pattern": "block_hooks", + "description": "Declares Block Hooks metadata", + "weight": 1 + }, + { + "pattern": "core/paragraph", + "description": "Targets the paragraph block as the anchor", + "weight": 1 + }, + { + "pattern": "after", + "description": "Uses the requested after position", + "weight": 1 + }, + { + "pattern": "multiple", + "description": "Declares multiple-instance support", + "weight": 1 + }, + { + "pattern": "false", + "description": "Disables multiple instances", + "weight": 1 + }, + { + "pattern": "render_callback", + "description": "Provides dynamic server rendering", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_block_hooks_004' ) ) { return false; } $ok = wpbp_gb_block_hooks_004(); $registry = WP_Block_Type_Registry::get_instance(); $type = $registry->get_registered( 'wpbp/single-hook' ); $content = '

A

B

'; $inserted = apply_block_hooks_to_content( $content, array( 'slug' => 'wpbp-pattern' ) ); $existing = '' . $content; $existing_out = apply_block_hooks_to_content( $existing, array( 'slug' => 'wpbp-pattern' ) ); $rendered = do_blocks( $inserted ); $registry->unregister( 'wpbp/single-hook' ); return true === $ok && $type instanceof WP_Block_Type && false === ( $type->supports['multiple'] ?? true ) && 1 === substr_count( $inserted, '' ) && 1 === substr_count( $existing_out, '' ) && str_contains( $rendered, '

Once

' );", + "description": "The single-hook block is registered with multiple disabled and WordPress suppresses duplicate insertions in one context", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_block_hooks_004(): bool { register_block_type( 'wpbp/single-hook', array( 'block_hooks' => array( 'core/paragraph' => 'after' ), 'supports' => array( 'multiple' => false ), 'render_callback' => static fn(): string => '

Once

' ) ); return WP_Block_Type_Registry::get_instance()->is_registered( 'wpbp/single-hook' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/class-wp-block-type.php", + "src/wp-includes/class-wp-block-type-registry.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/gb-font-library.json b/datasets/suites/wp-core-v1/execution/gb-font-library.json new file mode 100644 index 0000000..16bf99c --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/gb-font-library.json @@ -0,0 +1,185 @@ +{ + "id": "wp-core-execution-v1-gb_font_library", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Font Library", + "description": "Code generation tasks for WordPress Font Library APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-gb-font-library-001", + "category": "gb-font-library", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_font_library_001(): void to disable the Font Library in every block editor by filtering the editor settings WordPress sends to the editor.", + "expected_behavior": "`wpbp_gb_font_library_001()` registers a `block_editor_settings_all` callback that sets `fontLibraryEnabled` to false while preserving unrelated editor settings. Runtime validation applies the filter with a real block editor context after the function is called.", + "requirements": [ + "Use the block editor settings filter", + "Disable only the Font Library setting", + "Do not remove Font Library post types or admin pages" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_font_library_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_filter", + "description": "Registers a filter callback", + "weight": 1 + }, + { + "pattern": "block_editor_settings_all", + "description": "Uses the editor settings filter", + "weight": 1 + }, + { + "pattern": "fontLibraryEnabled", + "description": "Updates the Font Library setting", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_font_library_001' ) ) { return false; } $before = apply_filters( 'block_editor_settings_all', array( 'fontLibraryEnabled' => true, 'codeEditingEnabled' => true ), new WP_Block_Editor_Context() ); wpbp_gb_font_library_001(); $settings = apply_filters( 'block_editor_settings_all', array( 'fontLibraryEnabled' => true, 'codeEditingEnabled' => true ), new WP_Block_Editor_Context( array( 'name' => 'core/edit-site' ) ) ); return true === ( $before['fontLibraryEnabled'] ?? null ) && false === ( $settings['fontLibraryEnabled'] ?? null ) && true === ( $settings['codeEditingEnabled'] ?? null );", + "description": "Calling the function disables the Font Library setting and preserves unrelated editor settings", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_font_library_001(): void { add_filter( 'block_editor_settings_all', function ( $settings ) { $settings['fontLibraryEnabled'] = false; return $settings; } ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/block-editor.php", + "src/wp-includes/class-wp-block-editor-context.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-font-library-002", + "category": "gb-font-library", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_font_library_002(): bool to register a Font Library collection with the slug 'wpbp-fonts' that contains an inline family named 'Bench Sans'.", + "expected_behavior": "`wpbp_gb_font_library_002()` registers `wpbp-fonts` through the Font Library collection API and returns true when WordPress accepts the collection. Runtime validation inspects `WP_Font_Library` and the collection data instead of trusting the return value alone.", + "requirements": [ + "Use wp_register_font_collection()", + "Set the collection name to WPBP Fonts", + "Define a Bench Sans family with slug bench-sans" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_font_library_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_register_font_collection", + "description": "Uses the Font Library collection API", + "weight": 1 + }, + { + "pattern": "wpbp-fonts", + "description": "Registers the requested collection slug", + "weight": 1 + }, + { + "pattern": "font_families", + "description": "Defines collection font families", + "weight": 1 + }, + { + "pattern": "font_family_settings", + "description": "Defines a font family settings object", + "weight": 1 + }, + { + "pattern": "Bench Sans", + "description": "Defines the requested family name", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_font_library_002' ) ) { return false; } $registered = wpbp_gb_font_library_002(); $collection = WP_Font_Library::get_instance()->get_font_collection( 'wpbp-fonts' ); if ( ! $collection instanceof WP_Font_Collection ) { return false; } $data = $collection->get_data(); if ( is_wp_error( $data ) ) { return false; } $family = $data['font_families'][0]['font_family_settings'] ?? array(); return true === $registered && 'WPBP Fonts' === ( $data['name'] ?? null ) && 'Bench Sans' === ( $family['name'] ?? null ) && 'bench-sans' === ( $family['slug'] ?? null ) && '\"Bench Sans\", system-ui, sans-serif' === ( $family['fontFamily'] ?? null );", + "description": "The wpbp-fonts collection is registered with the expected inline Bench Sans family", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_font_library_002(): bool { $collection = wp_register_font_collection( 'wpbp-fonts', array( 'name' => 'WPBP Fonts', 'font_families' => array( array( 'font_family_settings' => array( 'name' => 'Bench Sans', 'slug' => 'bench-sans', 'fontFamily' => 'Bench Sans, system-ui, sans-serif' ), 'categories' => array( 'sans-serif' ) ) ), 'categories' => array( array( 'name' => 'Sans Serif', 'slug' => 'sans-serif' ) ) ) ); return $collection instanceof WP_Font_Collection; }", + "metadata": { + "source_refs": [ + "src/wp-includes/fonts.php", + "src/wp-includes/fonts/class-wp-font-library.php", + "src/wp-includes/fonts/class-wp-font-collection.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-font-library-003", + "category": "gb-font-library", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_font_library_003( array $fonts ): string to generate WordPress Font Face style markup for the supplied font-face definitions and return it as a string.", + "expected_behavior": "`wpbp_gb_font_library_003()` captures the output from WordPress's Font Face printer and returns the generated style markup. Runtime validation supplies a local font definition and checks WordPress's normalized CSS, including the default `font-display: fallback` and source ordering.", + "requirements": [ + "Use wp_print_font_faces()", + "Return the captured style markup as a string", + "Let WordPress apply default font-face values" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_font_library_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_print_font_faces", + "description": "Uses the WordPress Font Face printer", + "weight": 1 + }, + { + "pattern": "ob_start", + "description": "Captures printed style markup", + "weight": 1 + }, + { + "pattern": "ob_get_clean", + "description": "Returns captured output", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_font_library_003' ) ) { return false; } $fonts = array( 'bench' => array( array( 'font-family' => 'Bench Sans', 'font-style' => 'italic', 'font-weight' => '700', 'src' => array( 'https://example.test/fonts/bench-sans.woff', 'https://example.test/fonts/bench-sans.woff2' ) ) ) ); ob_start(); $out = wpbp_gb_font_library_003( $fonts ); $leaked = ob_get_clean(); return '' === $leaked && is_string( $out ) && str_contains( $out, 'wp-fonts-local' ) && str_contains( $out, '@font-face{' ) && str_contains( $out, 'font-family:\"Bench Sans\";' ) && str_contains( $out, 'font-style:italic;' ) && str_contains( $out, 'font-weight:700;' ) && str_contains( $out, 'font-display:fallback;' ) && str_contains( $out, \"url('https://example.test/fonts/bench-sans.woff2') format('woff2')\" ) && str_contains( $out, \"url('https://example.test/fonts/bench-sans.woff') format('woff')\" );", + "description": "The function returns WordPress-generated font-face style markup with default fallback display and ordered sources", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_font_library_003( array $fonts ): string { ob_start(); wp_print_font_faces( $fonts ); return (string) ob_get_clean(); }", + "metadata": { + "source_refs": [ + "src/wp-includes/fonts.php", + "src/wp-includes/fonts/class-wp-font-face.php" + ], + "release_focus": "7.0" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/gb-interactivity-api.json b/datasets/suites/wp-core-v1/execution/gb-interactivity-api.json new file mode 100644 index 0000000..988f1cc --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/gb-interactivity-api.json @@ -0,0 +1,249 @@ +{ + "id": "wp-core-execution-v1-gb_interactivity_api", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Interactivity API", + "description": "Code generation tasks for WordPress Interactivity API APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-gb-interactivity-api-001", + "category": "gb-interactivity-api", + "difficulty": "basic", + "prompt": "Implement wpbp_gb_interactivity_api_001() to initialize the 'wpbp/store' Interactivity API state with a count of 2 and return the store state from WordPress.", + "expected_behavior": "`wpbp_gb_interactivity_api_001()` uses `wp_interactivity_state()` for the `wpbp/store` namespace and returns the state array reported by WordPress. Runtime validation confirms the requested `count` value is present without replacing unrelated state that was already registered for the same namespace.", + "requirements": [ + "Use the WordPress Interactivity API state helper", + "Use the store namespace 'wpbp/store'", + "Return the final state array" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_interactivity_api_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_interactivity_state", + "description": "Uses wp_interactivity_state", + "weight": 1 + }, + { + "pattern": "wpbp/store", + "description": "Uses the requested store namespace", + "weight": 1 + }, + { + "pattern": "count", + "description": "Sets the requested count state key", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "wp_interactivity_state( 'wpbp/store', array( 'label' => 'Preserved' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_interactivity_api_001' ) ) { return false; } $state = wpbp_gb_interactivity_api_001(); return is_array( $state ) && 2 === ( $state['count'] ?? null ) && 'Preserved' === ( $state['label'] ?? null );", + "description": "The wpbp/store state contains the new count and preserves existing keys", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_interactivity_api_001(): array { wp_interactivity_state( 'wpbp/store', array( 'count' => 2 ) ); return wp_interactivity_state( 'wpbp/store' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/interactivity-api/interactivity-api.php", + "src/wp-includes/interactivity-api/class-wp-interactivity-api.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-interactivity-api-002", + "category": "gb-interactivity-api", + "difficulty": "basic", + "prompt": "Implement wpbp_gb_interactivity_api_002( array $context ) to return a safe Interactivity API context attribute for the 'wpbp/store' namespace.", + "expected_behavior": "`wpbp_gb_interactivity_api_002()` delegates context attribute generation to `wp_interactivity_data_wp_context()` with the `wpbp/store` namespace. Runtime validation passes context containing quotes, angle brackets, and ampersands, then verifies WordPress returns a `data-wp-context` attribute with the namespace prefix and JSON_HEX-escaped data.", + "requirements": [ + "Use the WordPress Interactivity API context helper", + "Use the store namespace 'wpbp/store'", + "Return the complete attribute string" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_interactivity_api_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_interactivity_data_wp_context", + "description": "Uses wp_interactivity_data_wp_context", + "weight": 1 + }, + { + "pattern": "wpbp/store", + "description": "Uses the requested store namespace", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_interactivity_api_002' ) ) { return false; } $out = wpbp_gb_interactivity_api_002( array( 'open' => true, 'label' => 'A \"quote\" & more' ) ); return is_string( $out ) && str_starts_with( $out, \"data-wp-context='wpbp/store::\" ) && str_contains( $out, '\"open\":true' ) && str_contains( $out, '\\\\u0022quote\\\\u0022' ) && str_contains( $out, '\\\\u003Ctag\\\\u003E' ) && str_contains( $out, '\\\\u0026' );", + "description": "The context attribute is namespaced and safely JSON-encoded for HTML", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_interactivity_api_002( array $context ): string { return wp_interactivity_data_wp_context( $context, 'wpbp/store' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/interactivity-api/interactivity-api.php", + "src/wp-includes/interactivity-api/class-wp-interactivity-api.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-interactivity-api-003", + "category": "gb-interactivity-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_interactivity_api_003( string $html ) to register derived Interactivity API state for 'wpbp/iapi-derived' and return the server-processed markup.", + "expected_behavior": "`wpbp_gb_interactivity_api_003()` registers state for `wpbp/iapi-derived` with a base count and a PHP Closure-derived `summary` value, then runs the supplied HTML through `wp_interactivity_process_directives()`. Runtime validation checks that a `data-wp-text=\"state.summary\"` element is rendered with the derived text from WordPress instead of its fallback text.", + "requirements": [ + "Use wp_interactivity_state for the 'wpbp/iapi-derived' namespace", + "Use a PHP Closure for the derived summary value", + "Process the supplied HTML with the WordPress directive processor" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_interactivity_api_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_interactivity_process_directives", + "description": "Uses wp_interactivity_process_directives", + "weight": 1 + }, + { + "pattern": "wp_interactivity_state", + "description": "Uses wp_interactivity_state", + "weight": 1 + }, + { + "pattern": "wpbp/iapi-derived", + "description": "Uses the requested namespace", + "weight": 1 + }, + { + "pattern": "summary", + "description": "Defines the derived summary state key", + "weight": 1 + }, + { + "pattern": "static\\s+fn|function\\s*\\(", + "description": "Uses a Closure for derived state", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_interactivity_api_003' ) ) { return false; } $out = wpbp_gb_interactivity_api_003( '
Loading
' ); return is_string( $out ) && str_contains( $out, '>Items: 3' ) && ! str_contains( $out, '>Loading' );", + "description": "Server-side directive processing renders the derived summary text", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_interactivity_api_003( string $html ): string { wp_interactivity_state( 'wpbp/iapi-derived', array( 'count' => 3, 'summary' => static function (): string { $state = wp_interactivity_state(); return 'Items: ' . $state['count']; } ) ); return wp_interactivity_process_directives( $html ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/interactivity-api/interactivity-api.php", + "src/wp-includes/interactivity-api/class-wp-interactivity-api.php" + ], + "release_focus": "6.9" + } + }, + { + "id": "e-gb-interactivity-api-004", + "category": "gb-interactivity-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_gb_interactivity_api_004() to return server-processed markup for one interactive element where two independent plugins toggle the same class suffix using unique Interactivity API directive IDs.", + "expected_behavior": "`wpbp_gb_interactivity_api_004()` uses WordPress 6.9 unique directive ID syntax with `data-wp-class` directives on a single element, registers true state flags, and processes the markup through `wp_interactivity_process_directives()`. Runtime validation checks that WordPress preserves both independent directives by adding both unique class names to the rendered element.", + "requirements": [ + "Use the 'wpbp/iapi-unique' namespace", + "Use triple-dash unique IDs for both class directives", + "Return the markup after WordPress processes the directives" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_interactivity_api_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_interactivity_process_directives", + "description": "Processes directives on the server", + "weight": 1 + }, + { + "pattern": "wp_interactivity_state", + "description": "Registers directive state", + "weight": 1 + }, + { + "pattern": "data\\-wp\\-class", + "description": "Uses class directives", + "weight": 1 + }, + { + "pattern": "\\-\\-\\-plugin\\-a", + "description": "Uses a unique ID for plugin-a", + "weight": 1 + }, + { + "pattern": "\\-\\-\\-plugin\\-b", + "description": "Uses a unique ID for plugin-b", + "weight": 1 + }, + { + "pattern": "wpbp/iapi-unique", + "description": "Uses the requested namespace", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_interactivity_api_004' ) ) { return false; } $out = wpbp_gb_interactivity_api_004(); return is_string( $out ) && str_contains( $out, 'class=\"badge---plugin-a badge---plugin-b\"' ) && str_contains( $out, 'data-wp-class--badge---plugin-a=\"state.pluginA\"' ) && str_contains( $out, 'data-wp-class--badge---plugin-b=\"state.pluginB\"' );", + "description": "Both unique data-wp-class directives are applied to the same rendered element", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_interactivity_api_004(): string { wp_interactivity_state( 'wpbp/iapi-unique', array( 'pluginA' => true, 'pluginB' => true ) ); return wp_interactivity_process_directives( 'Badge' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/interactivity-api/interactivity-api.php", + "src/wp-includes/interactivity-api/class-wp-interactivity-api.php" + ], + "release_focus": "6.9" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/gb-templates-navigation.json b/datasets/suites/wp-core-v1/execution/gb-templates-navigation.json new file mode 100644 index 0000000..e3b87f1 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/gb-templates-navigation.json @@ -0,0 +1,178 @@ +{ + "id": "wp-core-execution-v1-gb_templates_navigation", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Templates and Navigation", + "description": "Code generation tasks for WordPress Templates and Navigation APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-gb-templates-navigation-001", + "category": "gb-templates-navigation", + "difficulty": "intermediate", + "prompt": "Register a plugin-provided block template named `wpbp//landing-special` for posts. The template should have the title `Landing Special`, the description `Landing page for benchmark products`, and paragraph block content containing `Landing Special Template`.", + "expected_behavior": "Reviewer contract: the submitted code registers the `wpbp//landing-special` block template with WordPress so it is available for the `post` post type and preserves the requested title, description, and block markup in the registered `WP_Block_Template` object.", + "requirements": [ + "Use the WordPress block template registration API.", + "Make the template available only to posts." + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "register_block_template|WP_Block_Templates_Registry", + "description": "Uses a WordPress block template registration API", + "weight": 1 + }, + { + "pattern": "wpbp//landing-special", + "description": "Registers the requested template name", + "weight": 1 + }, + { + "pattern": "post_types", + "description": "Scopes the template to post types", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "$template = WP_Block_Templates_Registry::get_instance()->get_registered( 'wpbp//landing-special' ); return $template instanceof WP_Block_Template && 'landing-special' === $template->slug && 'plugin' === $template->source && 'wp_template' === $template->type && 'Landing Special' === $template->title && 'Landing page for benchmark products' === $template->description && array( 'post' ) === $template->post_types && str_contains( $template->content, '' ) && str_contains( $template->content, 'Landing Special Template' );", + "description": "The registered template has the expected template object properties", + "weight": 1 + } + ] + }, + "reference_solution": "register_block_template( 'wpbp//landing-special', array( 'title' => 'Landing Special', 'description' => 'Landing page for benchmark products', 'content' => '

Landing Special Template

', 'post_types' => array( 'post' ) ) );", + "metadata": { + "source_refs": [ + "src/wp-includes/block-template.php", + "src/wp-includes/class-wp-block-templates-registry.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-gb-templates-navigation-002", + "category": "gb-templates-navigation", + "difficulty": "intermediate", + "prompt": "Define `wpbp_gb_templates_navigation_002( string $url, string $label ): string` to render a `core/navigation-link` block for the supplied URL and label using WordPress server-side block rendering.", + "expected_behavior": "Reviewer contract: `wpbp_gb_templates_navigation_002()` returns the markup produced by rendering a `core/navigation-link` block with the supplied `url` and `label` attributes. Review focus is that WordPress block rendering handles the navigation item markup and escaping.", + "requirements": [ + "Pass the URL and label as block attributes.", + "Return the rendered markup." + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_gb_templates_navigation_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "render_block", + "description": "Uses render_block", + "weight": 1 + }, + { + "pattern": "core\\/navigation\\-link", + "description": "Uses core/navigation-link", + "weight": 1 + }, + { + "pattern": "url", + "description": "Passes a URL attribute", + "weight": 0.5 + }, + { + "pattern": "label", + "description": "Passes a label attribute", + "weight": 0.5 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_gb_templates_navigation_002' ) ) { return false; } $out = wpbp_gb_templates_navigation_002( 'https://example.test/products?ref=bench&plan=pro', 'Products & Plans' ); $decoded = html_entity_decode( $out, ENT_QUOTES, 'UTF-8' ); return is_string( $out ) && str_contains( $out, 'wp-block-navigation-item' ) && str_contains( $decoded, 'https://example.test/products?ref=bench&plan=pro' ) && str_contains( $decoded, 'Products & Plans' );", + "description": "The rendered navigation link contains WordPress navigation item markup, the supplied URL, and the supplied label", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_gb_templates_navigation_002( string $url, string $label ): string { return render_block( array( 'blockName' => 'core/navigation-link', 'attrs' => array( 'url' => $url, 'label' => $label ), 'innerBlocks' => array(), 'innerHTML' => '', 'innerContent' => array() ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/blocks.php", + "src/wp-includes/blocks/navigation-link.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-gb-templates-navigation-003", + "category": "gb-templates-navigation", + "difficulty": "intermediate", + "prompt": "Register a block pattern named `wpbp/navigation-footer` for the Navigation block. Its content should be a `core/navigation` block with two custom links: `Docs` to `/docs` and `Support` to `/support`.", + "expected_behavior": "Reviewer contract: the submitted code registers the `wpbp/navigation-footer` block pattern with title `Navigation Footer`, marks it as relevant to `core/navigation`, and stores block markup containing a Navigation block with the requested custom links.", + "requirements": [ + "Use the WordPress block pattern registration API.", + "Set the pattern title to `Navigation Footer` and include `core/navigation` in `blockTypes`." + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "register_block_pattern|WP_Block_Patterns_Registry", + "description": "Uses a WordPress block pattern registration API", + "weight": 1 + }, + { + "pattern": "wpbp\\/navigation-footer", + "description": "Registers the requested pattern name", + "weight": 1 + }, + { + "pattern": "core\\/navigation", + "description": "Includes a Navigation block", + "weight": 1 + }, + { + "pattern": "wp:navigation\\-link|core\\/navigation\\-link", + "description": "Includes serialized Navigation Link blocks", + "weight": 1 + }, + { + "pattern": "blockTypes", + "description": "Scopes the pattern to relevant block types", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "$pattern = WP_Block_Patterns_Registry::get_instance()->get_registered( 'wpbp/navigation-footer' ); return is_array( $pattern ) && 'Navigation Footer' === $pattern['title'] && in_array( 'core/navigation', $pattern['blockTypes'] ?? array(), true ) && str_contains( $pattern['content'], '' ) );", + "metadata": { + "source_refs": [ + "src/wp-includes/block-patterns.php", + "src/wp-includes/class-wp-block-patterns-registry.php", + "src/wp-includes/blocks/navigation.php", + "src/wp-includes/blocks/navigation-link.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/hooks.json b/datasets/suites/wp-core-v1/execution/hooks.json index ee4905f..8c778ae 100644 --- a/datasets/suites/wp-core-v1/execution/hooks.json +++ b/datasets/suites/wp-core-v1/execution/hooks.json @@ -1,68 +1,328 @@ { "id": "wp-core-execution-v1-hooks", - "version": "1.1.0", + "version": "2.0.0", "metadata": { "name": "WordPress Core Execution Tests - Hooks", - "description": "Code generation tasks for WordPress hooks (actions and filters)", - "wp_version": "6.9", - "created_at": "2025-12-16" + "description": "Code generation tasks for WordPress Hooks APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" }, "tests": [ { "id": "e-hooks-001", "category": "hooks", + "difficulty": "basic", + "prompt": "Implement wpbp_hooks_001(): void to attach a filter to the_title that appends the literal suffix ' [WPBP]' to the incoming title string.", + "expected_behavior": "`wpbp_hooks_001()` registers a `the_title` filter whose callback returns the original title plus ` [WPBP]`. Runtime validation calls the function and applies the WordPress title filter to sample values to verify the appended result.", + "requirements": [ + "Use add_filter() on the the_title filter", + "Append the suffix without replacing the original title" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_hooks_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_filter", + "description": "Uses add_filter", + "weight": 1 + }, + { + "pattern": "the_title", + "description": "Hooks into the title filter", + "weight": 1 + }, + { + "pattern": "\\[WPBP\\]", + "description": "Uses the required suffix", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_hooks_001' ) ) { return false; } wpbp_hooks_001(); return 'Hello [WPBP]' === apply_filters( 'the_title', 'Hello', 123 ) && 'Draft [WPBP]' === apply_filters( 'the_title', 'Draft', 456 );", + "description": "The title filter appends the required suffix to filtered titles", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_hooks_001(): void { add_filter( 'the_title', function ( $title ) { return $title . ' [WPBP]'; }, 10, 1 ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/plugin.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-hooks-002", + "category": "hooks", "difficulty": "intermediate", - "prompt": "Create code that adds a custom column called 'Featured' to the Posts admin list table. The column should display 'Yes' if the post has a meta key '_is_featured' with value '1', otherwise display 'No'.", - "expected_behavior": "Admin posts list shows a 'Featured' column with Yes/No values based on post meta", + "prompt": "Implement wpbp_hooks_002(): void to register an action on wpbp_two_args. When the action fires with two arguments, store them as ':' in the option wpbp_two_args_result.", + "expected_behavior": "`wpbp_hooks_002()` registers an action callback for `wpbp_two_args` that accepts both action arguments and writes them to `wpbp_two_args_result` in order. Runtime validation fires the action with two values and reads the option written by the callback.", "requirements": [ - "Add column header using 'manage_posts_columns' filter", - "Populate column content using 'manage_posts_custom_column' action", - "Check post meta '_is_featured' for value '1'", - "Display 'Yes' or 'No' appropriately", - "Escape all output" + "Use add_action() with two accepted arguments", + "Preserve both action arguments in order" ], "static_checks": { "required_patterns": [ { - "pattern": "add_filter.*manage_posts_columns", - "description": "Must hook into manage_posts_columns", - "weight": 1.0 + "pattern": "wpbp_hooks_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_action", + "description": "Uses add_action", + "weight": 1 }, { - "pattern": "add_action.*manage_posts_custom_column", - "description": "Must hook into manage_posts_custom_column", - "weight": 1.0 + "pattern": "wpbp_two_args", + "description": "Registers the expected action hook", + "weight": 1 }, { - "pattern": "get_post_meta", - "description": "Must use get_post_meta to check meta value", + "pattern": "wpbp_two_args_result", + "description": "Writes the expected option", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "delete_option( 'wpbp_two_args_result' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_hooks_002' ) ) { return false; } wpbp_hooks_002(); do_action( 'wpbp_two_args', 'alpha', 'beta' ); return 'alpha:beta' === get_option( 'wpbp_two_args_result' );", + "description": "The action callback receives both arguments and stores them in order", + "weight": 1 + } + ], + "teardown": "delete_option( 'wpbp_two_args_result' );" + }, + "reference_solution": "function wpbp_hooks_002(): void { add_action( 'wpbp_two_args', function ( $a, $b ) { update_option( 'wpbp_two_args_result', $a . ':' . $b ); }, 10, 2 ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/plugin.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-hooks-003", + "category": "hooks", + "difficulty": "intermediate", + "prompt": "Implement wpbp_hooks_003( callable $callback ): bool to remove the supplied callback from the wpbp_remove_filter hook at priority 0 and return true only when WordPress reports it was present before removal and absent afterward.", + "expected_behavior": "`wpbp_hooks_003()` checks a supplied callback with `has_filter()`, removes it from `wpbp_remove_filter` at priority 0 with `remove_filter()`, and treats priority 0 as a valid registration rather than a falsey miss. Runtime validation pre-registers a callback at priority 0 and verifies the filter no longer changes a value.", + "requirements": [ + "Use has_filter() to inspect the callback registration", + "Use remove_filter() with the matching priority", + "Treat priority 0 as a valid registered priority" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_hooks_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "remove_filter", + "description": "Uses remove_filter", + "weight": 1 + }, + { + "pattern": "has_filter", + "description": "Uses has_filter", + "weight": 1 + }, + { + "pattern": "wpbp_remove_filter", + "description": "Targets the expected filter hook", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_hooks_003_callback'] = static function ( $value ) { return $value . ' changed'; }; add_filter( 'wpbp_remove_filter', $GLOBALS['wpbp_hooks_003_callback'], 0 );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_hooks_003' ) ) { return false; } $callback = $GLOBALS['wpbp_hooks_003_callback'] ?? null; if ( ! is_callable( $callback ) ) { return false; } $ok = wpbp_hooks_003( $callback ); return true === $ok && false === has_filter( 'wpbp_remove_filter', $callback ) && 'seed' === apply_filters( 'wpbp_remove_filter', 'seed' );", + "description": "The priority-0 callback is removed and no longer affects filtered values", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_hooks_003( callable $callback ): bool { $before = has_filter( 'wpbp_remove_filter', $callback ); $removed = remove_filter( 'wpbp_remove_filter', $callback, 0 ); $after = has_filter( 'wpbp_remove_filter', $callback ); return 0 === $before && true === $removed && false === $after; }", + "metadata": { + "source_refs": [ + "src/wp-includes/plugin.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-hooks-004", + "category": "hooks", + "difficulty": "intermediate", + "prompt": "Implement wpbp_hooks_004(): string to register two callbacks on wpbp_priority: a lower-priority callback appends 'A' and a later callback appends 'B', then return the filtered value starting from an empty string.", + "expected_behavior": "`wpbp_hooks_004()` uses `add_filter()` priorities on `wpbp_priority` and returns the result of `apply_filters()`. Runtime validation expects WordPress to run the lower numeric priority first so the final value is `AB`.", + "requirements": [ + "Use add_filter() with explicit priorities", + "Use apply_filters() to return the computed value" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_hooks_004", + "description": "Defines the requested function", "weight": 0.5 }, { - "pattern": "_is_featured", - "description": "Must reference the correct meta key", + "pattern": "add_filter", + "description": "Uses add_filter", + "weight": 1 + }, + { + "pattern": "apply_filters", + "description": "Uses apply_filters", + "weight": 1 + }, + { + "pattern": "wpbp_priority", + "description": "Uses the expected custom filter", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_hooks_004' ) ) { return false; } return 'AB' === wpbp_hooks_004();", + "description": "Filter callbacks run in WordPress priority order", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_hooks_004(): string { add_filter( 'wpbp_priority', fn( $v ) => $v . 'B', 20 ); add_filter( 'wpbp_priority', fn( $v ) => $v . 'A', 5 ); return apply_filters( 'wpbp_priority', '' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/plugin.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-hooks-005", + "category": "hooks", + "difficulty": "basic", + "prompt": "Implement wpbp_hooks_005(): int to fire the custom action wpbp_did_action twice and return WordPress's count for that action.", + "expected_behavior": "`wpbp_hooks_005()` dispatches `wpbp_did_action` twice with `do_action()` and returns `did_action( 'wpbp_did_action' )`. Runtime validation checks the returned count and WordPress's recorded action count.", + "requirements": [ + "Use do_action() to fire the action", + "Use did_action() to return the action count" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_hooks_005", + "description": "Defines the requested function", "weight": 0.5 + }, + { + "pattern": "do_action", + "description": "Uses do_action", + "weight": 1 + }, + { + "pattern": "did_action", + "description": "Uses did_action", + "weight": 1 + }, + { + "pattern": "wpbp_did_action", + "description": "Uses the expected action hook", + "weight": 1 } ] }, "runtime_checks": { "assertions": [ { - "type": "hook_registered", - "target": "manage_posts_columns", - "description": "Column filter must be registered", - "weight": 1.0 + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_hooks_005' ) ) { return false; } return 2 === wpbp_hooks_005() && 2 === did_action( 'wpbp_did_action' );", + "description": "did_action() reports the exact number of dispatched actions", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_hooks_005(): int { do_action( 'wpbp_did_action' ); do_action( 'wpbp_did_action' ); return did_action( 'wpbp_did_action' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/plugin.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-hooks-006", + "category": "hooks", + "difficulty": "basic", + "prompt": "Implement wpbp_hooks_006(): bool to remove every callback from the wpbp_remove_all filter and return true when applying that filter to 'abc' leaves it unchanged.", + "expected_behavior": "`wpbp_hooks_006()` calls `remove_all_filters()` for `wpbp_remove_all` and verifies through `apply_filters()` that callbacks on that hook no longer alter the value. Runtime validation pre-registers callbacks before the submitted code runs and confirms the hook is empty afterward.", + "requirements": [ + "Use remove_all_filters() on the custom filter", + "Use apply_filters() to verify the value is no longer changed" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_hooks_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "remove_all_filters", + "description": "Uses remove_all_filters", + "weight": 1 + }, + { + "pattern": "apply_filters", + "description": "Uses apply_filters", + "weight": 1 }, { - "type": "hook_registered", - "target": "manage_posts_custom_column", - "description": "Column action must be registered", - "weight": 1.0 + "pattern": "wpbp_remove_all", + "description": "Targets the expected filter hook", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "add_filter( 'wpbp_remove_all', 'strtoupper' ); add_filter( 'wpbp_remove_all', static function ( $value ) { return $value . '!'; }, 20 );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_hooks_006' ) ) { return false; } return true === wpbp_hooks_006() && false === has_filter( 'wpbp_remove_all' ) && 'abc' === apply_filters( 'wpbp_remove_all', 'abc' );", + "description": "All callbacks are removed from the filter hook", + "weight": 1 } ] }, - "judge_config": { - "rubric_id": "wp-judge-rubric-v1" + "reference_solution": "function wpbp_hooks_006(): bool { remove_all_filters( 'wpbp_remove_all' ); return 'abc' === apply_filters( 'wpbp_remove_all', 'abc' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/plugin.php" + ], + "release_focus": "classic" } } ] diff --git a/datasets/suites/wp-core-v1/execution/html-api.json b/datasets/suites/wp-core-v1/execution/html-api.json deleted file mode 100644 index 9faa05d..0000000 --- a/datasets/suites/wp-core-v1/execution/html-api.json +++ /dev/null @@ -1,242 +0,0 @@ -{ - "id": "wp-core-execution-v1-html_api", - "version": "1.1.0", - "metadata": { - "name": "WordPress Core Execution Tests - HTML API", - "description": "Code generation tasks for WordPress HTML API (WP_HTML_Tag_Processor, WP_HTML_Processor)", - "wp_version": "6.9", - "created_at": "2025-12-16" - }, - "tests": [ - { - "id": "e-html-serialize-001", - "category": "html-api", - "difficulty": "hard", - "prompt": "Write wpbp_first_paragraph_html($html) that returns the normalized outer HTML of the first

element using WP_HTML_Processor::serialize_token(), not regex or DOMDocument.", - "expected_behavior": "Given messy HTML like '

First two

Second', it returns '

First two

' with well-formed tags.", - "requirements": [ - "Instantiate WP_HTML_Processor and loop with next_token()", - "Detect the first p start tag and serialize it with serialize_token()", - "Stop after the first paragraph; return empty string if none", - "Do not use regex or DOMDocument for parsing" - ], - "static_checks": { - "required_patterns": [ - { "pattern": "WP_HTML_Processor", "description": "Uses HTML API", "weight": 1.0 }, - { "pattern": "serialize_token", "description": "Serializes current token", "weight": 0.8 }, - { "pattern": "next_token", "description": "Iterates tokens", "weight": 0.6 } - ], - "forbidden_patterns": [ - { "pattern": "preg_match", "description": "No regex parsing", "severity": "warning" }, - { "pattern": "DOMDocument", "description": "Must rely on HTML API", "severity": "warning" } - ] - }, - "runtime_checks": { - "assertions": [ - { - "type": "custom_assertion", - "code": "if ( ! function_exists('wpbp_first_paragraph_html') ) { return false; } $html = '

First two

Second'; $out = wpbp_first_paragraph_html( $html ); return '

First two

' === $out;", - "description": "Returns normalized first paragraph", - "weight": 1.0 - }, - { - "type": "custom_assertion", - "code": "if ( ! function_exists('wpbp_first_paragraph_html') ) { return false; } $html = '
No p here
'; $out = wpbp_first_paragraph_html( $html ); return $out === '';", - "description": "Gracefully handles missing paragraphs", - "weight": 0.5 - } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_first_paragraph_html( string $html ): string {\n $processor = WP_HTML_Processor::create_fragment( $html );\n $capturing = false;\n $depth = 0;\n $buffer = '';\n while ( $processor->next_token() ) {\n $type = $processor->get_token_type();\n if ( ! $capturing && 'start tag' === $type && 'p' === $processor->get_tag() ) {\n $capturing = true;\n $depth = 1;\n $buffer .= $processor->serialize_token();\n continue;\n }\n if ( ! $capturing ) {\n continue;\n }\n // While capturing, append every token until the matching

closes.\n $buffer .= $processor->serialize_token();\n if ( 'start tag' === $type && $processor->get_tag() === 'p' ) {\n $depth++;\n } elseif ( 'end tag' === $type && $processor->get_tag() === 'p' ) {\n $depth--;\n if ( $depth === 0 ) {\n return $buffer;\n }\n }\n }\n return '';\n}" - }, - { - "id": "e-html-strip-scripts-001", - "category": "html-api", - "difficulty": "hard", - "prompt": "Write wpbp_strip_scripts($html) that removes all

Keep

X
'; $out = wpbp_strip_scripts( $html ); return ! str_contains( $out, 'Keep

' ) && str_contains( $out, '' );", - "description": "Scripts removed, other tags kept", - "weight": 1.0 - } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_strip_scripts( string $html ): string {\n $processor = WP_HTML_Tag_Processor::create_fragment( $html );\n $output = '';\n while ( $processor->next_tag() ) {\n if ( 'script' === $processor->get_tag() ) {\n $processor->remove_tag();\n }\n }\n return $processor->get_updated_html();\n}" - }, - { - "id": "e-html-safe-script-text-001", - "category": "html-api", - "difficulty": "hard", - "prompt": "Write wpbp_replace_script_text($html, $replacement) that replaces the text content of the first

See A and B

'; return wpbp_first_url( $html ) === 'https://example.com/a';", - "description": "Gets first URL", - "weight": 1.0 - }, - { - "type": "custom_assertion", - "code": "if ( ! function_exists('wpbp_first_url') ) { return false; } return wpbp_first_url('

No links

') === '';", - "description": "Empty when none", - "weight": 0.5 - } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_first_url( string $html ): string {\n $url = get_url_in_content( $html );\n return $url ? $url : '';\n}" - }, - { - "id": "e-html-ie-conditional-001", - "category": "html-api", - "difficulty": "hard", - "prompt": "Using WP_HTML_Tag_Processor, strip legacy IE conditional comments () from HTML while leaving other comments intact.", - "expected_behavior": "Returns HTML with IE conditionals removed; other comments and markup remain.", - "requirements": [ - "Identify conditional comments by pattern

Keep

'; $out = wpbp_strip_ie_conditionals( $html ); return ! str_contains( $out, 'old' ) && str_contains( $out, '

Keep

' ) && str_contains( $out, '' );", - "description": "IE conditionals removed, other comments kept", - "weight": 1.0 - } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_strip_ie_conditionals( string $html ): string {\n $processor = WP_HTML_Processor::create_fragment( $html );\n $output = '';\n while ( $processor->next_token() ) {\n if ( 'comment' === $processor->get_token_type() && str_starts_with( $processor->get_comment_text(), '[if' ) ) {\n $processor->remove_token();\n }\n }\n return $processor->get_updated_html();\n}" - } - ] -} diff --git a/datasets/suites/wp-core-v1/execution/interactivity-api.json b/datasets/suites/wp-core-v1/execution/interactivity-api.json deleted file mode 100644 index fbd804c..0000000 --- a/datasets/suites/wp-core-v1/execution/interactivity-api.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "id": "wp-core-execution-v1-interactivity_api", - "version": "1.1.0", - "metadata": { - "name": "WordPress Core Execution Tests - Interactivity API", - "description": "Code generation tasks for WordPress Interactivity API", - "wp_version": "6.9", - "created_at": "2025-12-16" - }, - "tests": [ - { - "id": "e-interactivity-client-nav-001", - "category": "interactivity-api", - "difficulty": "hard", - "prompt": "Register a script module 'demo/nav' and mark it as client-navigation-compatible so it loads on SPA transitions. Prove the script tag contains data-wp-router-options with loadOnClientNavigation true.", - "expected_behavior": "Enqueuing the module prints a script tag with data-wp-router-options including \\\"loadOnClientNavigation\\\":true.", - "requirements": [ - "Call wp_register_script_module to register 'demo/nav' with a placeholder file", - "Use wp_interactivity()->add_client_navigation_support_to_script_module('demo/nav')", - "Enqueue the module so wp_print_script_modules outputs it", - "Script tag must include data-wp-router-options with loadOnClientNavigation true" - ], - "static_checks": { - "required_patterns": [ - { "pattern": "wp_register_script_module", "description": "Registers script module", "weight": 1.0 }, - { "pattern": "add_client_navigation_support_to_script_module", "description": "Marks module for client navigation", "weight": 0.8 }, - { "pattern": "wp_enqueue_script_module", "description": "Enqueues module", "weight": 0.6 } - ], - "forbidden_patterns": [ - { "pattern": "wp_enqueue_script\\s*\\(", "description": "Must use module API, not classic scripts", "severity": "warning" } - ] - }, - "runtime_checks": { - "assertions": [ - { - "type": "custom_assertion", - "code": "if ( ! function_exists('wp_register_script_module') ) { return false; } ob_start(); wp_enqueue_script_module( 'demo/nav' ); wp_print_script_modules(); $html = ob_get_clean(); return str_contains( $html, 'demo/nav' ) && str_contains( $html, 'data-wp-router-options' ) && str_contains( $html, 'loadOnClientNavigation' );", - "description": "Script tag contains client navigation directive", - "weight": 1.0 - } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "add_action( 'init', function() {\n wp_register_script_module( 'demo/nav', includes_url( 'js/wp-util.js' ) ); // placeholder URL available in core.\n wp_interactivity()->add_client_navigation_support_to_script_module( 'demo/nav' );\n wp_enqueue_script_module( 'demo/nav' );\n} );" - }, - { - "id": "e-interactivity-router-replace-001", - "category": "interactivity-api", - "difficulty": "hard", - "prompt": "Create wpbp_router_link($url, $text) that outputs an anchor using the Interactivity API router with history replacement and intent prefetch.", - "expected_behavior": "Returns with data-wp-router-replace=\"true\", data-wp-router-prefetch=\"intent\", href set to $url, and inner text.", - "requirements": [ - "Return the HTML string (do not echo)", - "Include data-wp-router-replace=\"true\" and data-wp-router-prefetch=\"intent\" attributes", - "Escape URL and text", - "Include rel=\"nofollow\" when URL is external (scheme not matching home_url scheme/host)" - ], - "static_checks": { - "required_patterns": [ - { "pattern": "data-wp-router-replace", "description": "Uses router replace attribute", "weight": 1.0 }, - { "pattern": "data-wp-router-prefetch", "description": "Prefetch intent attribute", "weight": 0.6 }, - { "pattern": "esc_url", "description": "Escapes URL", "weight": 0.6 } - ] - }, - "runtime_checks": { - "assertions": [ - { - "type": "custom_assertion", - "code": "if ( ! function_exists('wpbp_router_link') ) { return false; } $html = wpbp_router_link( 'https://example.com/path', 'Go' ); return str_contains($html,'data-wp-router-replace=\"true\"') && str_contains($html,'data-wp-router-prefetch=\"intent\"') && str_contains($html,'href=\"https://example.com/path\"') && str_contains($html,'>Go<') && str_contains($html,'rel=\"nofollow\"');", - "description": "Has router attrs and rel on external", - "weight": 1.0 - } - ] - }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "function wpbp_router_link( string $url, string $text ): string {\n $home = wp_parse_url( home_url() );\n $target = wp_parse_url( $url );\n $is_external = $target && ( ($target['host'] ?? '') !== ($home['host'] ?? '') || ($target['scheme'] ?? '') !== ($home['scheme'] ?? '') );\n $rel = $is_external ? ' rel=\"nofollow\"' : '';\n return sprintf(\n '%s',\n esc_url( $url ),\n $rel,\n esc_html( $text )\n );\n}" - } - ] -} diff --git a/datasets/suites/wp-core-v1/execution/internationalization.json b/datasets/suites/wp-core-v1/execution/internationalization.json new file mode 100644 index 0000000..a5604d5 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/internationalization.json @@ -0,0 +1,279 @@ +{ + "id": "wp-core-execution-v1-internationalization", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Internationalization", + "description": "Code generation tasks for WordPress Internationalization APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-internationalization-001", + "category": "internationalization", + "difficulty": "basic", + "prompt": "Implement `wpbp_internationalization_001(): string` to return the translatable label \"Checkout complete\" from the `wpbp-i18n` text domain.", + "expected_behavior": "`wpbp_internationalization_001()` uses the core translation helper with the exact source string `Checkout complete` and text domain `wpbp-i18n`. Runtime validation installs a gettext filter for that text/domain pair and expects the filtered translation to be returned.", + "requirements": [ + "Use a WordPress i18n return helper", + "Pass the source string and text domain directly to the translation call" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_internationalization_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "/\\b__\\s*\\(/", + "description": "Uses the return-oriented translation helper", + "weight": 1 + }, + { + "pattern": "Checkout complete", + "description": "Uses the requested source string", + "weight": 1 + }, + { + "pattern": "wpbp-i18n", + "description": "Uses the requested text domain", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_i18n_001_filter'] = static function ( $translation, $text, $domain ) { return ( 'Checkout complete' === $text && 'wpbp-i18n' === $domain ) ? 'Translated Checkout' : $translation; }; add_filter( 'gettext', $GLOBALS['wpbp_i18n_001_filter'], 10, 3 );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_internationalization_001' ) ) { return false; } return 'Translated Checkout' === wpbp_internationalization_001();", + "description": "The label is passed through WordPress translation with the requested text domain", + "weight": 1 + } + ], + "teardown": "if ( isset( $GLOBALS['wpbp_i18n_001_filter'] ) ) { remove_filter( 'gettext', $GLOBALS['wpbp_i18n_001_filter'], 10 ); unset( $GLOBALS['wpbp_i18n_001_filter'] ); }" + }, + "reference_solution": "function wpbp_internationalization_001(): string { return __( 'Checkout complete', 'wpbp-i18n' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/l10n.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-internationalization-002", + "category": "internationalization", + "difficulty": "basic", + "prompt": "Implement `wpbp_internationalization_002(): string` to return the label \"Open & tools\" translated from `wpbp-i18n` and escaped for safe HTML text output.", + "expected_behavior": "`wpbp_internationalization_002()` uses WordPress's combined translate-and-escape helper for HTML text. Runtime validation filters the translation to unsafe markup and expects the returned string to match WordPress HTML escaping.", + "requirements": [ + "Use the combined WordPress helper for translated HTML text", + "Pass the source string and text domain directly to the helper" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_internationalization_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "/\\besc_html__\\s*\\(/", + "description": "Uses esc_html__ for translated HTML text", + "weight": 1 + }, + { + "pattern": "Open & tools", + "description": "Uses the requested source string", + "weight": 1 + }, + { + "pattern": "wpbp-i18n", + "description": "Uses the requested text domain", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_i18n_002_filter'] = static function ( $translation, $text, $domain ) { return ( 'Open & tools' === $text && 'wpbp-i18n' === $domain ) ? 'Unsafe & label' : $translation; }; add_filter( 'gettext', $GLOBALS['wpbp_i18n_002_filter'], 10, 3 );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_internationalization_002' ) ) { return false; } return esc_html( 'Unsafe & label' ) === wpbp_internationalization_002();", + "description": "The filtered translation is escaped for safe HTML text output", + "weight": 1 + } + ], + "teardown": "if ( isset( $GLOBALS['wpbp_i18n_002_filter'] ) ) { remove_filter( 'gettext', $GLOBALS['wpbp_i18n_002_filter'], 10 ); unset( $GLOBALS['wpbp_i18n_002_filter'] ); }" + }, + "reference_solution": "function wpbp_internationalization_002(): string { return esc_html__( 'Open & tools', 'wpbp-i18n' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/l10n.php", + "src/wp-includes/formatting.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-internationalization-003", + "category": "internationalization", + "difficulty": "intermediate", + "prompt": "Implement `wpbp_internationalization_003( string $locale ): bool` to temporarily switch WordPress to the supplied locale and restore the previous locale before returning whether the switch succeeded.", + "expected_behavior": "`wpbp_internationalization_003()` calls `switch_to_locale()`, restores after a successful switch, returns the boolean switch result, and leaves no switched locale on the stack. Runtime validation forces a non-en_US effective locale so switching to `en_US` succeeds, then checks WordPress's switch and restore hooks.", + "requirements": [ + "Use WordPress locale switching APIs", + "Restore the previous locale after a successful switch" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_internationalization_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "switch_to_locale", + "description": "Uses switch_to_locale", + "weight": 1 + }, + { + "pattern": "restore_previous_locale", + "description": "Uses restore_previous_locale", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_i18n_003_counts'] = array( 'switch' => 0, 'restore' => 0 ); $GLOBALS['wpbp_i18n_003_determine'] = static function ( $locale ) { return is_locale_switched() ? $locale : 'wpbp_TEST'; }; add_filter( 'determine_locale', $GLOBALS['wpbp_i18n_003_determine'], 1 ); $GLOBALS['wpbp_i18n_003_switch'] = static function ( $locale ) { $GLOBALS['wpbp_i18n_003_counts']['switch']++; $GLOBALS['wpbp_i18n_003_counts']['switched_to'] = $locale; }; add_action( 'switch_locale', $GLOBALS['wpbp_i18n_003_switch'], 10, 1 ); $GLOBALS['wpbp_i18n_003_restore'] = static function ( $locale, $previous_locale ) { $GLOBALS['wpbp_i18n_003_counts']['restore']++; $GLOBALS['wpbp_i18n_003_counts']['restored_to'] = $locale; $GLOBALS['wpbp_i18n_003_counts']['previous_locale'] = $previous_locale; }; add_action( 'restore_previous_locale', $GLOBALS['wpbp_i18n_003_restore'], 10, 2 );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_internationalization_003' ) ) { return false; } $ok = wpbp_internationalization_003( 'en_US' ); $counts = $GLOBALS['wpbp_i18n_003_counts'] ?? array(); return true === $ok && false === is_locale_switched() && 1 === ( $counts['switch'] ?? 0 ) && 1 === ( $counts['restore'] ?? 0 ) && 'en_US' === ( $counts['switched_to'] ?? null );", + "description": "The locale switch succeeds, is restored, and fires the expected WordPress hooks", + "weight": 1 + } + ], + "teardown": "if ( isset( $GLOBALS['wpbp_i18n_003_determine'] ) ) { remove_filter( 'determine_locale', $GLOBALS['wpbp_i18n_003_determine'], 1 ); unset( $GLOBALS['wpbp_i18n_003_determine'] ); } if ( isset( $GLOBALS['wpbp_i18n_003_switch'] ) ) { remove_action( 'switch_locale', $GLOBALS['wpbp_i18n_003_switch'], 10 ); unset( $GLOBALS['wpbp_i18n_003_switch'] ); } if ( isset( $GLOBALS['wpbp_i18n_003_restore'] ) ) { remove_action( 'restore_previous_locale', $GLOBALS['wpbp_i18n_003_restore'], 10 ); unset( $GLOBALS['wpbp_i18n_003_restore'] ); } unset( $GLOBALS['wpbp_i18n_003_counts'] );" + }, + "reference_solution": "function wpbp_internationalization_003( string $locale ): bool { $switched = switch_to_locale( $locale ); if ( $switched ) { restore_previous_locale(); } return $switched; }", + "metadata": { + "source_refs": [ + "src/wp-includes/l10n.php", + "src/wp-includes/class-wp-locale-switcher.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-internationalization-004", + "category": "internationalization", + "difficulty": "intermediate", + "prompt": "Implement `wpbp_internationalization_004(): bool` to register the plugin text domain `wpbp-i18n-loader` with the relative languages directory `wp-bench-runtime/languages` when the function is called.", + "expected_behavior": "`wpbp_internationalization_004()` calls `load_plugin_textdomain()` with the requested domain and relative plugin languages path. Runtime validation checks WordPress's textdomain registry for the custom path instead of expecting translations to load immediately, matching the just-in-time loading behavior in modern WordPress.", + "requirements": [ + "Use load_plugin_textdomain()", + "Use the relative path parameter rather than an absolute MO file path" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_internationalization_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "load_plugin_textdomain", + "description": "Uses load_plugin_textdomain", + "weight": 1 + }, + { + "pattern": "wpbp-i18n-loader", + "description": "Uses the requested text domain", + "weight": 1 + }, + { + "pattern": "wp-bench-runtime/languages", + "description": "Uses the requested relative languages path", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_internationalization_004' ) ) { return false; } global $wp_textdomain_registry; if ( ! $wp_textdomain_registry instanceof WP_Textdomain_Registry ) { return false; } $ok = wpbp_internationalization_004(); $path = $wp_textdomain_registry->get( 'wpbp-i18n-loader', determine_locale() ); return true === $ok && is_string( $path ) && str_ends_with( rtrim( $path, '/' ), '/wp-bench-runtime/languages' );", + "description": "The text domain registry resolves the domain to the requested custom languages path", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_internationalization_004(): bool { return load_plugin_textdomain( 'wpbp-i18n-loader', false, 'wp-bench-runtime/languages' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/l10n.php", + "src/wp-includes/class-wp-textdomain-registry.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-internationalization-005", + "category": "internationalization", + "difficulty": "intermediate", + "prompt": "Implement `wpbp_internationalization_005( string $text, string $domain ): string` to return WordPress's translation for the supplied source text and text domain, allowing gettext filters to affect the result.", + "expected_behavior": "`wpbp_internationalization_005()` delegates to WordPress translation with both caller-supplied arguments. Runtime validation installs a gettext filter and verifies that only the matching source string and domain are changed.", + "requirements": [ + "Use the WordPress translation API that applies gettext filters", + "Pass through the caller-supplied text and domain" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_internationalization_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "/\\btranslate\\s*\\(/", + "description": "Uses translate to apply WordPress gettext behavior", + "weight": 1 + }, + { + "pattern": "\\$text", + "description": "Uses the supplied text argument", + "weight": 0.5 + }, + { + "pattern": "\\$domain", + "description": "Uses the supplied domain argument", + "weight": 0.5 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_i18n_005_filter'] = static function ( $translation, $text, $domain ) { return ( 'WPBP Source' === $text && 'wpbp-i18n' === $domain ) ? 'Filtered Translation' : $translation; }; add_filter( 'gettext', $GLOBALS['wpbp_i18n_005_filter'], 10, 3 );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_internationalization_005' ) ) { return false; } return 'Filtered Translation' === wpbp_internationalization_005( 'WPBP Source', 'wpbp-i18n' ) && 'WPBP Source' === wpbp_internationalization_005( 'WPBP Source', 'other-domain' ) && 'Other Source' === wpbp_internationalization_005( 'Other Source', 'wpbp-i18n' );", + "description": "The gettext filter can change only the matching text and domain", + "weight": 1 + } + ], + "teardown": "if ( isset( $GLOBALS['wpbp_i18n_005_filter'] ) ) { remove_filter( 'gettext', $GLOBALS['wpbp_i18n_005_filter'], 10 ); unset( $GLOBALS['wpbp_i18n_005_filter'] ); }" + }, + "reference_solution": "function wpbp_internationalization_005( string $text, string $domain ): string { return translate( $text, $domain ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/l10n.php", + "src/wp-includes/plugin.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/mail-url.json b/datasets/suites/wp-core-v1/execution/mail-url.json new file mode 100644 index 0000000..02518b3 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/mail-url.json @@ -0,0 +1,198 @@ +{ + "id": "wp-core-execution-v1-mail_url", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Mail and URL Handling", + "description": "Code generation tasks for WordPress Mail and URL Handling APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-mail-url-001", + "category": "mail-url", + "difficulty": "basic", + "prompt": "Implement `wpbp_mail_url_001( string $url ): string` to sanitize a URL for storage or redirects while preferring HTTPS when WordPress adds a missing scheme.", + "expected_behavior": "`wpbp_mail_url_001()` uses WordPress URL sanitization in a non-display context with HTTPS first in the allowed protocol list. A schemeless host should gain `https://`, an existing `http://` URL should remain HTTP, and unsafe protocols should be rejected.", + "requirements": [ + "Use WordPress URL sanitization", + "Allow both HTTPS and HTTP with HTTPS preferred" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_mail_url_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "sanitize_url", + "description": "Uses sanitize_url", + "weight": 1 + }, + { + "pattern": "/https/i", + "description": "Supplies HTTPS as the preferred protocol", + "weight": 0.5 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_mail_url_001' ) ) { return false; } return 'https://example.org/path?x=1&y=2' === wpbp_mail_url_001( 'example.org/path?x=1&y=2' ) && 'http://example.org/path' === wpbp_mail_url_001( 'http://example.org/path' ) && '' === wpbp_mail_url_001( 'javascript:alert(1)' );", + "description": "Schemeless URLs prefer HTTPS, existing HTTP is preserved, and disallowed protocols are removed", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_mail_url_001( string $url ): string { return sanitize_url( $url, array( 'https', 'http' ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/formatting.php" + ], + "release_focus": "6.9" + } + }, + { + "id": "e-mail-url-002", + "category": "mail-url", + "difficulty": "basic", + "prompt": "Implement `wpbp_mail_url_002( string $email ): string|false` to validate an email address and return WordPress's validation result.", + "expected_behavior": "`wpbp_mail_url_002()` delegates to `is_email()`, returning the original email string on success and `false` on failure. The test focuses on WordPress's return contract, not boolean-only validation or custom normalization.", + "requirements": [ + "Use WordPress email validation", + "Return the successful email string, not boolean true" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_mail_url_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "is_email", + "description": "Uses is_email", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_mail_url_002' ) ) { return false; } return 'Person+tag@example.co.uk' === wpbp_mail_url_002( 'Person+tag@example.co.uk' ) && false === wpbp_mail_url_002( 'bad@' ) && false === wpbp_mail_url_002( 'person@example' );", + "description": "Valid addresses are returned unchanged and invalid addresses return false", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_mail_url_002( string $email ): string|false { return is_email( $email ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/formatting.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-mail-url-003", + "category": "mail-url", + "difficulty": "basic", + "prompt": "Implement `wpbp_mail_url_003( string $location ): string` to sanitize a redirect target before it is passed to a redirect function.", + "expected_behavior": "`wpbp_mail_url_003()` uses `wp_sanitize_redirect()` and returns the sanitized location string without sending headers. It should encode spaces and strip CR/LF sequences; allowed-host validation belongs to `wp_safe_redirect()` or `wp_validate_redirect()`, not this helper.", + "requirements": [ + "Use WordPress redirect sanitization", + "Return the sanitized location without performing a redirect" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_mail_url_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_sanitize_redirect", + "description": "Uses wp_sanitize_redirect", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_mail_url_003' ) ) { return false; } $spaced = wpbp_mail_url_003( 'https://example.com/a b' ); $header = wpbp_mail_url_003( 'https://example.com/%0d%0Aadmin' ); return 'https://example.com/a%20b' === $spaced && 'https://example.com/admin' === $header;", + "description": "Redirect sanitization encodes spaces and removes CR/LF sequences", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_mail_url_003( string $url ): string { return wp_sanitize_redirect( $url ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/pluggable.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-mail-url-004", + "category": "mail-url", + "difficulty": "intermediate", + "prompt": "Implement `wpbp_mail_url_004( string $to, string $subject, string $html, array $embeds ): bool` to send an HTML email through WordPress with embedded image paths.", + "expected_behavior": "`wpbp_mail_url_004()` calls `wp_mail()` with the supplied recipient, subject, HTML body, a text/html Content-Type header, no attachments, and the supplied embeds array. The verifier short-circuits delivery with `pre_wp_mail` and checks the arguments passed to WordPress.", + "requirements": [ + "Use wp_mail() rather than PHPMailer directly", + "Pass a text/html Content-Type header", + "Pass the supplied embeds array to wp_mail() and return its boolean result" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_mail_url_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_mail", + "description": "Uses wp_mail", + "weight": 1 + }, + { + "pattern": "/Content-Type/i", + "description": "Passes an email content-type header", + "weight": 0.5 + }, + { + "pattern": "embeds", + "description": "Passes embedded image paths through wp_mail", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_mail_url_004_seen'] = array(); add_filter( 'pre_wp_mail', function ( $return, $atts ) { $GLOBALS['wpbp_mail_url_004_seen'][] = $atts; return true; }, 10, 2 );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_mail_url_004' ) ) { return false; } $embeds = array( 'logo' => '/tmp/wpbp-logo.png' ); $html = '

Logo

'; $sent = wpbp_mail_url_004( 'nobody@example.com', 'Embedded', $html, $embeds ); $seen = $GLOBALS['wpbp_mail_url_004_seen'] ?? array(); if ( true !== $sent || 1 !== count( $seen ) ) { return false; } $atts = $seen[0]; $headers = $atts['headers'] ?? array(); if ( is_string( $headers ) ) { $headers = array( $headers ); } $has_html_header = false; foreach ( $headers as $header ) { if ( is_string( $header ) && false !== stripos( $header, 'Content-Type' ) && false !== stripos( $header, 'text/html' ) ) { $has_html_header = true; } } return 'nobody@example.com' === ( $atts['to'] ?? null ) && 'Embedded' === ( $atts['subject'] ?? null ) && $html === ( $atts['message'] ?? null ) && $embeds === ( $atts['embeds'] ?? null ) && $has_html_header;", + "description": "wp_mail receives the supplied message fields, HTML header, and embeds array", + "weight": 1 + } + ], + "teardown": "remove_all_filters( 'pre_wp_mail' ); unset( $GLOBALS['wpbp_mail_url_004_seen'] );" + }, + "reference_solution": "function wpbp_mail_url_004( string $to, string $subject, string $html, array $embeds ): bool { return wp_mail( $to, $subject, $html, array( 'Content-Type: text/html; charset=UTF-8' ), array(), $embeds ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/pluggable.php" + ], + "release_focus": "6.9" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/media.json b/datasets/suites/wp-core-v1/execution/media.json new file mode 100644 index 0000000..dde87ed --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/media.json @@ -0,0 +1,238 @@ +{ + "id": "wp-core-execution-v1-media", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Media", + "description": "Code generation tasks for WordPress Media APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-media-001", + "category": "media", + "difficulty": "basic", + "prompt": "Implement wpbp_media_001( int $attachment_id, string|array $size = 'thumbnail' ) to return WordPress's image source data for an attachment.", + "expected_behavior": "`wpbp_media_001()` delegates to `wp_get_attachment_image_src()` with the supplied attachment ID and size, returning WordPress's four-item source array for full and intermediate image sizes or `false` when no image source is available. Runtime validation uses attachment metadata instead of a real upload file and verifies URL, dimensions, and intermediate-size state.", + "requirements": [ + "Use WordPress media APIs", + "Accept the attachment ID and requested image size as inputs", + "Return the WordPress image source array without parsing attachment metadata manually" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_media_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_get_attachment_image_src", + "description": "Uses wp_get_attachment_image_src", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$attachment_id = wp_insert_attachment( array( 'post_title' => 'WPBP Media 001', 'post_mime_type' => 'image/jpeg', 'guid' => home_url( '/wp-content/uploads/2026/06/wpbp-media-001.jpg' ) ) ); update_post_meta( $attachment_id, '_wp_attached_file', '2026/06/wpbp-media-001.jpg' ); update_post_meta( $attachment_id, '_wp_attachment_metadata', array( 'width' => 640, 'height' => 480, 'file' => '2026/06/wpbp-media-001.jpg', 'sizes' => array( 'medium' => array( 'file' => 'wpbp-media-001-300x225.jpg', 'width' => 300, 'height' => 225, 'mime-type' => 'image/jpeg' ) ) ) ); $GLOBALS['wpbp_media_001_attachment'] = (int) $attachment_id;", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_media_001' ) ) { return false; } $attachment_id = (int) $GLOBALS['wpbp_media_001_attachment']; $full = wpbp_media_001( $attachment_id, 'full' ); $medium = wpbp_media_001( $attachment_id, 'medium' ); return is_array( $full ) && str_ends_with( $full[0], '/2026/06/wpbp-media-001.jpg' ) && 640 === $full[1] && 480 === $full[2] && false === $full[3] && is_array( $medium ) && str_ends_with( $medium[0], '/2026/06/wpbp-media-001-300x225.jpg' ) && 300 === $medium[1] && 225 === $medium[2] && true === $medium[3] && false === wpbp_media_001( 0, 'full' );", + "description": "The function returns WordPress attachment image source arrays for full and intermediate sizes", + "weight": 1 + } + ], + "teardown": "if ( ! empty( $GLOBALS['wpbp_media_001_attachment'] ) ) { wp_delete_attachment( (int) $GLOBALS['wpbp_media_001_attachment'], true ); } unset( $GLOBALS['wpbp_media_001_attachment'] );" + }, + "reference_solution": "function wpbp_media_001( int $attachment_id, string|array $size = 'thumbnail' ): array|false { return wp_get_attachment_image_src( $attachment_id, $size ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/media.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-media-002", + "category": "media", + "difficulty": "basic", + "prompt": "Implement wpbp_media_002( string $html, ?string $context = null ) to return post content after WordPress applies media tag optimizations.", + "expected_behavior": "`wpbp_media_002()` passes the supplied HTML and optional context to `wp_filter_content_tags()`. Runtime validation creates image attachment metadata, filters an image tag with a matching `wp-image-*` class, and checks that WordPress adds dimensions and responsive image attributes rather than leaving the tag unchanged.", + "requirements": [ + "Use WordPress media APIs", + "Accept the HTML string and optional context as inputs", + "Return the filtered HTML string" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_media_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_filter_content_tags", + "description": "Uses wp_filter_content_tags", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$attachment_id = wp_insert_attachment( array( 'post_title' => 'WPBP Media 002', 'post_mime_type' => 'image/jpeg', 'guid' => home_url( '/wp-content/uploads/2026/06/wpbp-media-002.jpg' ) ) ); update_post_meta( $attachment_id, '_wp_attached_file', '2026/06/wpbp-media-002.jpg' ); update_post_meta( $attachment_id, '_wp_attachment_metadata', array( 'width' => 640, 'height' => 480, 'file' => '2026/06/wpbp-media-002.jpg', 'sizes' => array( 'medium' => array( 'file' => 'wpbp-media-002-300x225.jpg', 'width' => 300, 'height' => 225, 'mime-type' => 'image/jpeg' ) ) ) ); $GLOBALS['wpbp_media_002_attachment'] = (int) $attachment_id;", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_media_002' ) ) { return false; } $attachment_id = (int) $GLOBALS['wpbp_media_002_attachment']; $src = wp_get_attachment_url( $attachment_id ); $html = '

\"Fixture\"

'; $out = wpbp_media_002( $html, 'the_content' ); return str_contains( $out, 'width=\"640\"' ) && str_contains( $out, 'height=\"480\"' ) && str_contains( $out, 'srcset=' ) && str_contains( $out, 'sizes=' ) && $out !== $html;", + "description": "The filtered image tag receives WordPress dimensions and responsive image attributes", + "weight": 1 + } + ], + "teardown": "if ( ! empty( $GLOBALS['wpbp_media_002_attachment'] ) ) { wp_delete_attachment( (int) $GLOBALS['wpbp_media_002_attachment'], true ); } unset( $GLOBALS['wpbp_media_002_attachment'] );" + }, + "reference_solution": "function wpbp_media_002( string $html, ?string $context = null ): string { return wp_filter_content_tags( $html, $context ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/media.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-media-003", + "category": "media", + "difficulty": "intermediate", + "prompt": "Implement wpbp_media_003( int $orig_w, int $orig_h, int $dest_w, int $dest_h, bool|array $crop = false ) to return WordPress image resize dimensions.", + "expected_behavior": "`wpbp_media_003()` forwards the original size, requested size, and crop setting to `image_resize_dimensions()`. Runtime validation checks proportional downscaling, verifies WordPress refuses an upscale-only resize, and checks explicit crop positioning.", + "requirements": [ + "Use WordPress media APIs", + "Accept the original dimensions, requested dimensions, and crop setting as inputs", + "Return the WordPress resize dimensions array or false" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_media_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "image_resize_dimensions", + "description": "Uses image_resize_dimensions", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_media_003' ) ) { return false; } $scaled = wpbp_media_003( 600, 300, 300, 300, false ); $upscale = wpbp_media_003( 300, 150, 600, 300, false ); $cropped = wpbp_media_003( 600, 300, 200, 200, array( 'left', 'top' ) ); return array( 0, 0, 0, 0, 300, 150, 600, 300 ) === $scaled && false === $upscale && array( 0, 0, 0, 0, 200, 200, 300, 300 ) === $cropped;", + "description": "The function returns WordPress resize arrays for scale and crop cases and false for upscaling", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_media_003( int $orig_w, int $orig_h, int $dest_w, int $dest_h, bool|array $crop = false ): array|false { return image_resize_dimensions( $orig_w, $orig_h, $dest_w, $dest_h, $crop ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/media.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-media-004", + "category": "media", + "difficulty": "intermediate", + "prompt": "Implement wpbp_media_004( string $file ) to read embedded image alternative text from a file path.", + "expected_behavior": "`wpbp_media_004()` loads the admin image helper if needed and calls `wp_get_image_alttext()` for the supplied file path. Runtime validation uses a temporary image-like file containing IPTC/XMP `AltTextAccessibility` metadata and a second file without XMP data, expecting the embedded alternative text for the first and an empty string for the second.", + "requirements": [ + "Use WordPress media APIs", + "Load the admin image helper before calling wp_get_image_alttext", + "Return an empty string when no embedded alternative text is present" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_media_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_get_image_alttext", + "description": "Uses wp_get_image_alttext", + "weight": 1 + }, + { + "pattern": "wp\\-admin\\/includes\\/image\\.php", + "description": "Uses wp-admin/includes/image.php", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$base = sys_get_temp_dir() . '/wpbp-alttext-004-' . wp_rand(); $GLOBALS['wpbp_media_004_alt_file'] = $base . '.jpg'; $GLOBALS['wpbp_media_004_plain_file'] = $base . '-plain.jpg'; $xmp = 'Bench alt text'; file_put_contents( $GLOBALS['wpbp_media_004_alt_file'], 'jpeg-data' . $xmp ); file_put_contents( $GLOBALS['wpbp_media_004_plain_file'], 'jpeg-data-without-xmp' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_media_004' ) ) { return false; } return 'Bench alt text' === wpbp_media_004( $GLOBALS['wpbp_media_004_alt_file'] ) && '' === wpbp_media_004( $GLOBALS['wpbp_media_004_plain_file'] );", + "description": "The function reads embedded IPTC/XMP alternative text and returns an empty string when absent", + "weight": 1 + } + ], + "teardown": "foreach ( array( 'wpbp_media_004_alt_file', 'wpbp_media_004_plain_file' ) as $key ) { if ( ! empty( $GLOBALS[ $key ] ) && file_exists( $GLOBALS[ $key ] ) ) { unlink( $GLOBALS[ $key ] ); } unset( $GLOBALS[ $key ] ); }" + }, + "reference_solution": "function wpbp_media_004( string $file ): string { require_once ABSPATH . 'wp-admin/includes/image.php'; return (string) wp_get_image_alttext( $file ); }", + "metadata": { + "source_refs": [ + "src/wp-admin/includes/image.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-media-005", + "category": "media", + "difficulty": "basic", + "prompt": "Implement wpbp_media_005( string $filename, ?array $mimes = null ) to return WordPress's extension and MIME type result for an upload filename.", + "expected_behavior": "`wpbp_media_005()` calls `wp_check_filetype()` with the supplied filename and optional MIME map, returning the `ext` and `type` array from WordPress. Runtime validation checks a normal JPEG filename, an unknown extension, and a custom MIME map to ensure the implementation uses WordPress's extension-based file-type lookup.", + "requirements": [ + "Use WordPress file type APIs", + "Accept an optional MIME map", + "Do not inspect file contents" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_media_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_check_filetype", + "description": "Uses wp_check_filetype", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_media_005' ) ) { return false; } $jpeg = wpbp_media_005( 'photo.jpg' ); $unknown = wpbp_media_005( 'photo.notallowed' ); $custom = wpbp_media_005( 'payload.wpbp', array( 'wpbp' => 'application/x-wpbp' ) ); return 'jpg' === $jpeg['ext'] && 'image/jpeg' === $jpeg['type'] && false === $unknown['ext'] && false === $unknown['type'] && 'wpbp' === $custom['ext'] && 'application/x-wpbp' === $custom['type'];", + "description": "The function returns WordPress file-type results for allowed, unknown, and custom extensions", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_media_005( string $filename, ?array $mimes = null ): array { return wp_check_filetype( $filename, $mimes ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/functions.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/post-meta.json b/datasets/suites/wp-core-v1/execution/post-meta.json index a30378c..17090f7 100644 --- a/datasets/suites/wp-core-v1/execution/post-meta.json +++ b/datasets/suites/wp-core-v1/execution/post-meta.json @@ -1,69 +1,335 @@ { "id": "wp-core-execution-v1-post_meta", - "version": "1.1.0", + "version": "2.0.0", "metadata": { "name": "WordPress Core Execution Tests - Post Meta", - "description": "Code generation tasks for WordPress post meta operations", - "wp_version": "6.9", - "created_at": "2025-12-16" + "description": "Code generation tasks for WordPress Post Meta APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" }, "tests": [ { - "id": "e-meta-001", + "id": "e-post-meta-001", "category": "post-meta", - "difficulty": "basic", - "prompt": "Write a function called 'save_view_count' that takes a post ID and increments a post meta value '_view_count' by 1. If the meta doesn't exist, initialize it to 1. The function should return the new view count.", - "expected_behavior": "Calling save_view_count(123) increments _view_count meta for post 123 and returns the new count", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_meta_001( int $post_id, string $meta_key ) to increment a numeric post meta counter and return the new count.", + "expected_behavior": "`wpbp_post_meta_001()` reads the supplied post meta key with `get_post_meta()`, treats a missing value as zero, increments it, persists the new value with `update_post_meta()`, and returns the new integer count. Runtime validation checks consecutive calls and the stored meta value.", "requirements": [ - "Accept post ID as parameter", - "Get current _view_count meta value", - "If meta doesn't exist, treat as 0", - "Increment by 1 and save using update_post_meta()", - "Return the new count as an integer" + "Use WordPress metadata APIs", + "Accept the post ID and meta key as inputs", + "Persist and return the incremented integer value" ], "static_checks": { "required_patterns": [ { - "pattern": "function\\s+save_view_count", - "description": "Must define function save_view_count", - "weight": 1.0 + "pattern": "wpbp_post_meta_001", + "description": "Defines the requested function", + "weight": 0.5 }, { "pattern": "get_post_meta", - "description": "Must use get_post_meta to read current value", - "weight": 0.5 + "description": "Uses get_post_meta", + "weight": 1 }, { "pattern": "update_post_meta", - "description": "Must use update_post_meta to save value", + "description": "Uses update_post_meta", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_post_meta_001_post'] = wp_insert_post( array( 'post_title' => 'Meta Count', 'post_status' => 'publish' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_meta_001' ) ) { return false; } $post_id = (int) $GLOBALS['wpbp_post_meta_001_post']; $first = wpbp_post_meta_001( $post_id, '_wpbp_count' ); $second = wpbp_post_meta_001( $post_id, '_wpbp_count' ); return 1 === $first && 2 === $second && '2' === get_post_meta( $post_id, '_wpbp_count', true );", + "description": "The function increments and persists the supplied post meta counter", + "weight": 1 + } + ], + "teardown": "if ( ! empty( $GLOBALS['wpbp_post_meta_001_post'] ) ) { wp_delete_post( (int) $GLOBALS['wpbp_post_meta_001_post'], true ); }" + }, + "reference_solution": "function wpbp_post_meta_001( int $post_id, string $meta_key ): int { $count = (int) get_post_meta( $post_id, $meta_key, true ); $count++; update_post_meta( $post_id, $meta_key, $count ); return $count; }", + "metadata": { + "source_refs": [ + "src/wp-includes/meta.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-meta-002", + "category": "post-meta", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_meta_002() to register a protected string post meta field named '_wpbp_subtitle' for REST exposure on posts.", + "expected_behavior": "`wpbp_post_meta_002()` registers `_wpbp_subtitle` for the `post` post type with `register_post_meta()`, stores a single string value, exposes it through REST metadata, sanitizes incoming text, and permits access to the protected key through an auth callback. Runtime validation inspects WordPress's registered meta registry and the sanitize/auth filters.", + "requirements": [ + "Use WordPress metadata APIs", + "Expose the field through REST metadata", + "Sanitize the value as plain text" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_meta_002", + "description": "Defines the requested function", "weight": 0.5 }, { - "pattern": "_view_count", - "description": "Must use correct meta key", + "pattern": "register_post_meta", + "description": "Uses register_post_meta", + "weight": 1 + }, + { + "pattern": "_wpbp_subtitle", + "description": "Registers the requested meta key", + "weight": 1 + }, + { + "pattern": "show_in_rest", + "description": "Uses show_in_rest", + "weight": 1 + }, + { + "pattern": "sanitize_callback", + "description": "Defines a sanitize callback", + "weight": 1 + }, + { + "pattern": "auth_callback", + "description": "Defines an auth callback for the protected key", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_meta_002' ) ) { return false; } if ( true !== wpbp_post_meta_002() ) { return false; } $registered = get_registered_meta_keys( 'post', 'post' ); if ( empty( $registered['_wpbp_subtitle'] ) ) { return false; } $meta = $registered['_wpbp_subtitle']; $sanitized = sanitize_meta( '_wpbp_subtitle', 'Subtitle', 'post', 'post' ); $authorized = apply_filters( 'auth_post_meta__wpbp_subtitle_for_post', null, '_wpbp_subtitle', 0, 0, 'edit_post_meta', array() ); return 'string' === ( $meta['type'] ?? null ) && true === ( $meta['single'] ?? null ) && true === ( $meta['show_in_rest'] ?? null ) && 'Subtitle' === $sanitized && true === $authorized;", + "description": "The protected subtitle meta is registered for posts with REST exposure, sanitization, and access", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_post_meta_002(): bool { return register_post_meta( 'post', '_wpbp_subtitle', array( 'type' => 'string', 'single' => true, 'show_in_rest' => true, 'sanitize_callback' => 'sanitize_text_field', 'auth_callback' => '__return_true' ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/meta.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-meta-003", + "category": "post-meta", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_meta_003( int $post_id, string $meta_key, string $value ) to delete only post meta entries whose stored value matches the supplied value.", + "expected_behavior": "`wpbp_post_meta_003()` calls `delete_post_meta()` with the supplied previous value so non-matching rows are preserved. Runtime validation creates multiple values for the same key, verifies a wrong value deletes nothing, then verifies only the matching value is removed.", + "requirements": [ + "Use WordPress metadata APIs", + "Accept the post ID, meta key, and value as inputs", + "Do not delete non-matching values for the same key" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_meta_003", + "description": "Defines the requested function", "weight": 0.5 + }, + { + "pattern": "delete_post_meta", + "description": "Uses delete_post_meta", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$post_id = wp_insert_post( array( 'post_title' => 'Delete Meta', 'post_status' => 'publish' ) ); $GLOBALS['wpbp_post_meta_003_post'] = $post_id; add_post_meta( $post_id, '_wpbp_token', 'delete-me' ); add_post_meta( $post_id, '_wpbp_token', 'keep' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_meta_003' ) ) { return false; } $post_id = (int) $GLOBALS['wpbp_post_meta_003_post']; $wrong = wpbp_post_meta_003( $post_id, '_wpbp_token', 'wrong' ); $before = get_post_meta( $post_id, '_wpbp_token', false ); $deleted = wpbp_post_meta_003( $post_id, '_wpbp_token', 'delete-me' ); $after = get_post_meta( $post_id, '_wpbp_token', false ); return false === $wrong && array( 'delete-me', 'keep' ) === $before && true === $deleted && array( 'keep' ) === $after;", + "description": "Only post meta rows with the matching value are deleted", + "weight": 1 } ], - "forbidden_patterns": [ + "teardown": "if ( ! empty( $GLOBALS['wpbp_post_meta_003_post'] ) ) { wp_delete_post( (int) $GLOBALS['wpbp_post_meta_003_post'], true ); }" + }, + "reference_solution": "function wpbp_post_meta_003( int $post_id, string $meta_key, string $value ): bool { return delete_post_meta( $post_id, $meta_key, $value ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/meta.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-meta-004", + "category": "post-meta", + "difficulty": "basic", + "prompt": "Implement wpbp_post_meta_004( int $post_id, string $meta_key ) to return every stored value for a multi-value post meta key.", + "expected_behavior": "`wpbp_post_meta_004()` uses `get_post_meta()` in multi-value mode for the supplied key, returning the full array of stored values instead of collapsing to the first value. Runtime validation checks both populated and missing keys.", + "requirements": [ + "Use WordPress metadata APIs", + "Accept the post ID and meta key as inputs", + "Return an array of all stored values" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_meta_004", + "description": "Defines the requested function", + "weight": 0.5 + }, { - "pattern": "\\$wpdb->", - "description": "Should use WordPress meta functions, not direct queries", - "severity": "warning" + "pattern": "get_post_meta", + "description": "Uses get_post_meta", + "weight": 1 } ] }, "runtime_checks": { + "setup": "$post_id = wp_insert_post( array( 'post_title' => 'Multi Meta', 'post_status' => 'publish' ) ); $GLOBALS['wpbp_post_meta_004_post'] = $post_id; add_post_meta( $post_id, '_wpbp_multi', 'a' ); add_post_meta( $post_id, '_wpbp_multi', 'b' );", "assertions": [ { - "type": "function_exists", - "target": "save_view_count", - "description": "Function must exist", - "weight": 1.0 + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_meta_004' ) ) { return false; } $post_id = (int) $GLOBALS['wpbp_post_meta_004_post']; return array( 'a', 'b' ) === wpbp_post_meta_004( $post_id, '_wpbp_multi' ) && array() === wpbp_post_meta_004( $post_id, '_wpbp_missing' );", + "description": "All values for the supplied key are returned without collapsing to one value", + "weight": 1 + } + ], + "teardown": "if ( ! empty( $GLOBALS['wpbp_post_meta_004_post'] ) ) { wp_delete_post( (int) $GLOBALS['wpbp_post_meta_004_post'], true ); }" + }, + "reference_solution": "function wpbp_post_meta_004( int $post_id, string $meta_key ): array { return get_post_meta( $post_id, $meta_key, false ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/meta.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-meta-005", + "category": "post-meta", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_meta_005( int $post_id, string $meta_key, string $json ) to store JSON text containing escaped quotes without corrupting it.", + "expected_behavior": "`wpbp_post_meta_005()` compensates for WordPress's unslashing of post meta values by slashing JSON text before calling `update_post_meta()`, then returns the stored value. Runtime validation uses JSON containing escaped quotes and checks both the return value and stored meta.", + "requirements": [ + "Use WordPress metadata APIs", + "Accept the post ID, meta key, and JSON text as inputs", + "Preserve escaped quotes in the stored JSON text" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_meta_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_slash", + "description": "Uses wp_slash", + "weight": 1 + }, + { + "pattern": "update_post_meta", + "description": "Uses update_post_meta", + "weight": 1 } ] }, - "judge_config": { - "rubric_id": "wp-judge-rubric-v1" + "runtime_checks": { + "setup": "$GLOBALS['wpbp_post_meta_005_post'] = wp_insert_post( array( 'post_title' => 'Slash Meta', 'post_status' => 'publish' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_meta_005' ) ) { return false; } $post_id = (int) $GLOBALS['wpbp_post_meta_005_post']; $json = wp_json_encode( array( 'key' => 'value with \"escaped quotes\"' ) ); if ( ! is_string( $json ) ) { return false; } $stored = wpbp_post_meta_005( $post_id, '_wpbp_json', $json ); return $json === $stored && $json === get_post_meta( $post_id, '_wpbp_json', true );", + "description": "JSON text with escaped quotes is stored without being unslashed into invalid JSON", + "weight": 1 + } + ], + "teardown": "if ( ! empty( $GLOBALS['wpbp_post_meta_005_post'] ) ) { wp_delete_post( (int) $GLOBALS['wpbp_post_meta_005_post'], true ); }" + }, + "reference_solution": "function wpbp_post_meta_005( int $post_id, string $meta_key, string $json ): string { update_post_meta( $post_id, $meta_key, wp_slash( $json ) ); return get_post_meta( $post_id, $meta_key, true ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/meta.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-meta-006", + "category": "post-meta", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_meta_006() to register an integer post meta field named '_wpbp_rating' for the 'post' object subtype with a default value of 0.", + "expected_behavior": "`wpbp_post_meta_006()` registers `_wpbp_rating` with `register_meta()` for the `post` object type and `post` object subtype, using a single integer value, default `0`, and REST exposure. Runtime validation inspects WordPress's registered meta registry and verifies the default returned for a post without stored rating meta.", + "requirements": [ + "Use WordPress metadata APIs", + "Limit the registration to the post object subtype", + "Set integer type, single value, REST exposure, and default 0" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_meta_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_meta", + "description": "Uses register_meta", + "weight": 1 + }, + { + "pattern": "_wpbp_rating", + "description": "Registers the requested meta key", + "weight": 1 + }, + { + "pattern": "object_subtype", + "description": "Uses object_subtype", + "weight": 1 + }, + { + "pattern": "default", + "description": "Provides a default value", + "weight": 1 + }, + { + "pattern": "show_in_rest", + "description": "Exposes the registered meta through REST", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_post_meta_006_post'] = wp_insert_post( array( 'post_title' => 'Rating Meta', 'post_status' => 'publish' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_meta_006' ) ) { return false; } if ( true !== wpbp_post_meta_006() ) { return false; } $registered = get_registered_meta_keys( 'post', 'post' ); if ( empty( $registered['_wpbp_rating'] ) ) { return false; } $meta = $registered['_wpbp_rating']; $post_id = (int) $GLOBALS['wpbp_post_meta_006_post']; return 'integer' === ( $meta['type'] ?? null ) && true === ( $meta['single'] ?? null ) && 0 === ( $meta['default'] ?? null ) && true === ( $meta['show_in_rest'] ?? null ) && 0 === get_post_meta( $post_id, '_wpbp_rating', true );", + "description": "The subtype-specific rating meta is registered with REST exposure and default value", + "weight": 1 + } + ], + "teardown": "if ( ! empty( $GLOBALS['wpbp_post_meta_006_post'] ) ) { wp_delete_post( (int) $GLOBALS['wpbp_post_meta_006_post'], true ); }" + }, + "reference_solution": "function wpbp_post_meta_006(): bool { return register_meta( 'post', '_wpbp_rating', array( 'object_subtype' => 'post', 'type' => 'integer', 'single' => true, 'default' => 0, 'show_in_rest' => true ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/meta.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" } } ] diff --git a/datasets/suites/wp-core-v1/execution/post-types-taxonomy.json b/datasets/suites/wp-core-v1/execution/post-types-taxonomy.json new file mode 100644 index 0000000..3542b63 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/post-types-taxonomy.json @@ -0,0 +1,401 @@ +{ + "id": "wp-core-execution-v1-post_types_taxonomy", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Post Types and Taxonomy", + "description": "Code generation tasks for WordPress Post Types and Taxonomy APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-post-types-taxonomy-001", + "category": "post-types-taxonomy", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_types_taxonomy_001() to register a public book post type with REST support.", + "expected_behavior": "`wpbp_post_types_taxonomy_001()` registers a public `wpbp_book` post type with REST API exposure. Runtime validation checks the registered post type through WordPress and unregisters it afterward.", + "requirements": [ + "Use registration APIs", + "Clean up registered types when possible" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_types_taxonomy_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_post_type", + "description": "Uses register_post_type", + "weight": 1 + }, + { + "pattern": "show_in_rest", + "description": "Uses show_in_rest", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_types_taxonomy_001' ) ) { return false; } $ok = wpbp_post_types_taxonomy_001(); unregister_post_type( 'wpbp_book' ); return $ok;", + "description": "The book post type is registered and REST-enabled", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_post_types_taxonomy_001(): bool { register_post_type( 'wpbp_book', array( 'public' => true, 'show_in_rest' => true, 'label' => 'Books' ) ); return post_type_exists( 'wpbp_book' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/post.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-types-taxonomy-002", + "category": "post-types-taxonomy", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_types_taxonomy_002() to register a hierarchical genre taxonomy for posts.", + "expected_behavior": "`wpbp_post_types_taxonomy_002()` registers a hierarchical `wpbp_genre` taxonomy for posts. Runtime validation checks the taxonomy through WordPress and unregisters it afterward.", + "requirements": [ + "Use registration APIs", + "Clean up registered types when possible" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_types_taxonomy_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_taxonomy", + "description": "Uses register_taxonomy", + "weight": 1 + }, + { + "pattern": "hierarchical", + "description": "Uses hierarchical", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_types_taxonomy_002' ) ) { return false; } $ok = wpbp_post_types_taxonomy_002(); unregister_taxonomy( 'wpbp_genre' ); return $ok;", + "description": "The genre taxonomy is registered as hierarchical", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_post_types_taxonomy_002(): bool { register_taxonomy( 'wpbp_genre', 'post', array( 'hierarchical' => true, 'show_in_rest' => true, 'label' => 'Genres' ) ); return taxonomy_exists( 'wpbp_genre' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/post.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-types-taxonomy-003", + "category": "post-types-taxonomy", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_types_taxonomy_003() to attach an existing taxonomy to a custom post type.", + "expected_behavior": "`wpbp_post_types_taxonomy_003()` registers a fixture post type and taxonomy, then attaches the taxonomy to that post type with WordPress registration APIs. Runtime validation checks the attachment result.", + "requirements": [ + "Use registration APIs", + "Clean up registered types when possible" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_types_taxonomy_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_taxonomy_for_object_type", + "description": "Uses register_taxonomy_for_object_type", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_types_taxonomy_003' ) ) { return false; } return wpbp_post_types_taxonomy_003();", + "description": "The taxonomy is attached to the custom post type", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_post_types_taxonomy_003(): bool { register_post_type( 'wpbp_movie' ); register_taxonomy( 'wpbp_topic', 'post' ); $ok = register_taxonomy_for_object_type( 'wpbp_topic', 'wpbp_movie' ); unregister_post_type( 'wpbp_movie' ); unregister_taxonomy( 'wpbp_topic' ); return $ok; }", + "metadata": { + "source_refs": [ + "src/wp-includes/post.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-types-taxonomy-004", + "category": "post-types-taxonomy", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_types_taxonomy_004() to create a post type with custom capabilities mapped from meta caps.", + "expected_behavior": "`wpbp_post_types_taxonomy_004()` registers a custom post type using singular/plural capability types with meta-cap mapping enabled, then returns the generated edit capabilities from the registered post type object.", + "requirements": [ + "Use registration APIs", + "Clean up registered types when possible" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_types_taxonomy_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "capability_type", + "description": "Uses capability_type", + "weight": 1 + }, + { + "pattern": "map_meta_cap", + "description": "Uses map_meta_cap", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_types_taxonomy_004' ) ) { return false; } return array( 'edit_case', 'edit_cases' ) === wpbp_post_types_taxonomy_004();", + "description": "The custom post type generates mapped edit capabilities", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_post_types_taxonomy_004(): array { register_post_type( 'wpbp_case', array( 'capability_type' => array( 'case', 'cases' ), 'map_meta_cap' => true ) ); $caps = get_post_type_object( 'wpbp_case' )->cap; unregister_post_type( 'wpbp_case' ); return array( $caps->edit_post, $caps->edit_posts ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/post.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-types-taxonomy-005", + "category": "post-types-taxonomy", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_types_taxonomy_005() to insert a term and return its slug and taxonomy.", + "expected_behavior": "`wpbp_post_types_taxonomy_005()` inserts a category term with the supplied name, reads it back through WordPress term APIs, and returns its slug, taxonomy, and term ID.", + "requirements": [ + "Use registration APIs", + "Clean up registered types when possible" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_types_taxonomy_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_insert_term", + "description": "Uses wp_insert_term", + "weight": 1 + }, + { + "pattern": "get_term", + "description": "Uses get_term", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_post_types_taxonomy_005_name'] = 'WPBP Term Insert ' . wp_rand();", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_types_taxonomy_005' ) ) { return false; } $out = wpbp_post_types_taxonomy_005( $GLOBALS['wpbp_post_types_taxonomy_005_name'] ); return is_array( $out ) && 'category' === ( $out[1] ?? '' );", + "description": "The inserted category term can be read back from WordPress", + "weight": 1 + } + ], + "teardown": "$name = (string) ( $GLOBALS['wpbp_post_types_taxonomy_005_name'] ?? '' ); if ( '' !== $name ) { $term = get_term_by( 'name', $name, 'category' ); if ( $term ) { wp_delete_term( (int) $term->term_id, 'category' ); } } unset( $GLOBALS['wpbp_post_types_taxonomy_005_name'] );" + }, + "reference_solution": "function wpbp_post_types_taxonomy_005( string $name ): array { $term = wp_insert_term( $name, 'category' ); if ( is_wp_error( $term ) ) { return array( '', '', 0 ); } return array( get_term_field( 'slug', $term['term_id'], 'category' ), get_term( $term['term_id'] )->taxonomy, $term['term_id'] ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/post.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-types-taxonomy-006", + "category": "post-types-taxonomy", + "difficulty": "intermediate", + "prompt": "Provide a PHP function named wpbp_post_types_taxonomy_006( int $post_id, array $term_ids ): array that adds the given category term IDs to the post without removing category terms already assigned to it.", + "expected_behavior": "The helper appends the supplied category term IDs to the given post while preserving any category terms already assigned. Runtime validation creates a post with one existing category, invokes the helper with a second category, verifies both relationships remain, verifies the returned IDs include the appended term_taxonomy_id, and removes the fixtures in teardown.", + "requirements": [ + "Use WordPress taxonomy relationship APIs", + "Preserve existing category relationships", + "Return the term_taxonomy_ids reported by WordPress" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_types_taxonomy_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_set_object_terms", + "description": "Uses wp_set_object_terms", + "weight": 1 + }, + { + "pattern": "category", + "description": "Targets the category taxonomy", + "weight": 0.5 + }, + { + "pattern": "/wp_set_object_terms\\s*\\((?:(?!;).)*,\\s*true\\s*\\)/s", + "description": "Appends terms instead of replacing existing relationships", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "foreach ( array( 'wpbp-existing-006', 'wpbp-added-006' ) as $slug ) { $stale = get_term_by( 'slug', $slug, 'category' ); if ( $stale ) { wp_delete_term( $stale->term_id, 'category' ); } } $post_id = wp_insert_post( array( 'post_title' => 'WPBP Preserve Terms 006', 'post_status' => 'publish', 'post_type' => 'post' ) ); $existing = wp_insert_term( 'WPBP Existing 006', 'category', array( 'slug' => 'wpbp-existing-006' ) ); $added = wp_insert_term( 'WPBP Added 006', 'category', array( 'slug' => 'wpbp-added-006' ) ); if ( ! $post_id || is_wp_error( $existing ) || is_wp_error( $added ) ) { throw new RuntimeException( 'Failed to create post-types-taxonomy-006 fixtures.' ); } $assigned = wp_set_object_terms( $post_id, array( (int) $existing['term_id'] ), 'category', false ); if ( is_wp_error( $assigned ) ) { throw new RuntimeException( 'Failed to assign initial category fixture.' ); } $GLOBALS['wpbp_ptt_006'] = array( 'post_id' => (int) $post_id, 'existing_term_id' => (int) $existing['term_id'], 'added_term_id' => (int) $added['term_id'], 'added_tt_id' => (int) $added['term_taxonomy_id'] );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_types_taxonomy_006' ) || empty( $GLOBALS['wpbp_ptt_006'] ) ) { return false; } $fixture = $GLOBALS['wpbp_ptt_006']; $result = wpbp_post_types_taxonomy_006( $fixture['post_id'], array( $fixture['added_term_id'] ) ); if ( is_wp_error( $result ) ) { return false; } $assigned = wp_get_object_terms( $fixture['post_id'], 'category', array( 'fields' => 'ids', 'orderby' => 'none' ) ); if ( is_wp_error( $assigned ) ) { return false; } $assigned = array_map( 'intval', $assigned ); $result = array_map( 'intval', $result ); return in_array( $fixture['existing_term_id'], $assigned, true ) && in_array( $fixture['added_term_id'], $assigned, true ) && in_array( $fixture['added_tt_id'], $result, true );", + "description": "The helper appends the new category term and preserves the existing one", + "weight": 1 + } + ], + "teardown": "if ( ! empty( $GLOBALS['wpbp_ptt_006']['post_id'] ) ) { wp_delete_post( $GLOBALS['wpbp_ptt_006']['post_id'], true ); } foreach ( array( 'wpbp-existing-006', 'wpbp-added-006' ) as $slug ) { $term = get_term_by( 'slug', $slug, 'category' ); if ( $term ) { wp_delete_term( $term->term_id, 'category' ); } } unset( $GLOBALS['wpbp_ptt_006'] );" + }, + "reference_solution": "function wpbp_post_types_taxonomy_006( int $post_id, array $term_ids ): array { return wp_set_object_terms( $post_id, array_map( 'intval', $term_ids ), 'category', true ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/post.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-types-taxonomy-007", + "category": "post-types-taxonomy", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_types_taxonomy_007() to read a post type archive link for a public CPT.", + "expected_behavior": "`wpbp_post_types_taxonomy_007()` registers a public custom post type with archives enabled and returns the archive link generated by WordPress.", + "requirements": [ + "Use registration APIs", + "Clean up registered types when possible" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_types_taxonomy_007", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "get_post_type_archive_link", + "description": "Uses get_post_type_archive_link", + "weight": 1 + }, + { + "pattern": "has_archive", + "description": "Uses has_archive", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_types_taxonomy_007' ) ) { return false; } $link = wpbp_post_types_taxonomy_007(); return str_contains( $link, 'wpbp_archive' );", + "description": "The archive link is generated for the custom post type", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_post_types_taxonomy_007(): string { register_post_type( 'wpbp_archive', array( 'public' => true, 'has_archive' => true ) ); $link = get_post_type_archive_link( 'wpbp_archive' ); unregister_post_type( 'wpbp_archive' ); return (string) $link; }", + "metadata": { + "source_refs": [ + "src/wp-includes/post.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-post-types-taxonomy-008", + "category": "post-types-taxonomy", + "difficulty": "intermediate", + "prompt": "Implement wpbp_post_types_taxonomy_008() to register a taxonomy with a custom REST base.", + "expected_behavior": "`wpbp_post_types_taxonomy_008()` registers a taxonomy exposed through the REST API with the custom REST base `wpbp-tax`, then returns the REST base from the registered taxonomy object.", + "requirements": [ + "Use registration APIs", + "Clean up registered types when possible" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_post_types_taxonomy_008", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_taxonomy", + "description": "Uses register_taxonomy", + "weight": 1 + }, + { + "pattern": "rest_base", + "description": "Uses rest_base", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_post_types_taxonomy_008' ) ) { return false; } return 'wpbp-tax' === wpbp_post_types_taxonomy_008();", + "description": "The taxonomy stores the requested REST base", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_post_types_taxonomy_008(): string { register_taxonomy( 'wpbp_rest_tax', 'post', array( 'show_in_rest' => true, 'rest_base' => 'wpbp-tax' ) ); $base = get_taxonomy( 'wpbp_rest_tax' )->rest_base; unregister_taxonomy( 'wpbp_rest_tax' ); return $base; }", + "metadata": { + "source_refs": [ + "src/wp-includes/post.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/queries.json b/datasets/suites/wp-core-v1/execution/queries.json new file mode 100644 index 0000000..cf71d24 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/queries.json @@ -0,0 +1,657 @@ +{ + "id": "wp-core-execution-v1-queries", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Queries", + "description": "Code generation tasks for WordPress Queries APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-queries-001", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_001() to return recent published posts by category slug with no_found_rows.", + "expected_behavior": "`wpbp_queries_001()` returns a `WP_Query` for published posts matching the supplied category slug, limited by the supplied count and configured with `no_found_rows` for an efficient read-only query.", + "requirements": [ + "Use the relevant WordPress query class or helper", + "Return deterministic query data" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Query", + "description": "Uses WP_Query", + "weight": 1 + }, + { + "pattern": "no_found_rows", + "description": "Uses no_found_rows", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$suffix = (string) wp_rand(); $slug = 'wpbp-news-' . $suffix; $cat = wp_insert_term( 'WPBP News ' . $suffix, 'category', array( 'slug' => $slug ) ); $term_id = is_wp_error( $cat ) ? 0 : (int) $cat['term_id']; $post_id = $term_id ? (int) wp_insert_post( array( 'post_title' => 'Q1 ' . $suffix, 'post_status' => 'publish', 'post_category' => array( $term_id ) ) ) : 0; $GLOBALS['wpbp_queries_001'] = array( 'slug' => $slug, 'term' => $term_id, 'post' => $post_id );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_001' ) ) { return false; } $fixture = $GLOBALS['wpbp_queries_001'] ?? array(); if ( empty( $fixture['post'] ) ) { return false; } $q = wpbp_queries_001( $fixture['slug'], 3 ); return in_array( $fixture['post'], array_map( 'intval', wp_list_pluck( $q->posts, 'ID' ) ), true );", + "description": "The query returns the published post in the requested category", + "weight": 1 + } + ], + "teardown": "$fixture = $GLOBALS['wpbp_queries_001'] ?? array(); if ( ! empty( $fixture['post'] ) ) { wp_delete_post( (int) $fixture['post'], true ); } if ( ! empty( $fixture['term'] ) ) { wp_delete_term( (int) $fixture['term'], 'category' ); } unset( $GLOBALS['wpbp_queries_001'] );" + }, + "reference_solution": "function wpbp_queries_001( string $slug, int $limit = 5 ): WP_Query { return new WP_Query( array( 'category_name' => $slug, 'posts_per_page' => $limit, 'post_status' => 'publish', 'no_found_rows' => true ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-meta-query.php", + "src/wp-includes/class-wp-tax-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-002", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_002() to return a WP_Query for published posts whose price_cents post meta compares numerically equal to zero.", + "expected_behavior": "`wpbp_queries_002()` must use `WP_Query` with a numeric meta query for the `price_cents` key, returning published posts with a zero price while excluding non-zero prices, posts without the meta key, and drafts. Runtime validation creates all four cases and verifies the observable query results.", + "requirements": [ + "Use WP_Query with a meta query", + "Compare price_cents numerically", + "Limit results to published posts" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Query", + "description": "Uses WP_Query", + "weight": 1 + }, + { + "pattern": "meta_query", + "description": "Uses meta_query", + "weight": 1 + }, + { + "pattern": "NUMERIC", + "description": "Uses NUMERIC", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "foreach ( ( $GLOBALS['wpbp_queries_002_posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } $target = wp_insert_post( array( 'post_title' => 'WPBP Q002 Free', 'post_status' => 'publish' ) ); update_post_meta( $target, 'price_cents', '0' ); $nonzero = wp_insert_post( array( 'post_title' => 'WPBP Q002 Paid', 'post_status' => 'publish' ) ); update_post_meta( $nonzero, 'price_cents', '125' ); $missing = wp_insert_post( array( 'post_title' => 'WPBP Q002 Missing', 'post_status' => 'publish' ) ); $draft = wp_insert_post( array( 'post_title' => 'WPBP Q002 Draft', 'post_status' => 'draft' ) ); update_post_meta( $draft, 'price_cents', '0' ); $GLOBALS['wpbp_queries_002_posts'] = array( 'target' => (int) $target, 'nonzero' => (int) $nonzero, 'missing' => (int) $missing, 'draft' => (int) $draft );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_002' ) ) { return false; } $q = wpbp_queries_002(); if ( ! $q instanceof WP_Query ) { return false; } $ids = array_map( 'intval', wp_list_pluck( $q->posts, 'ID' ) ); $posts = $GLOBALS['wpbp_queries_002_posts'] ?? array(); return in_array( $posts['target'] ?? 0, $ids, true ) && ! in_array( $posts['nonzero'] ?? 0, $ids, true ) && ! in_array( $posts['missing'] ?? 0, $ids, true ) && ! in_array( $posts['draft'] ?? 0, $ids, true );", + "description": "The query returns only published posts whose price_cents meta is numerically zero", + "weight": 1 + } + ], + "teardown": "foreach ( ( $GLOBALS['wpbp_queries_002_posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } unset( $GLOBALS['wpbp_queries_002_posts'] );" + }, + "reference_solution": "function wpbp_queries_002(): WP_Query { return new WP_Query( array( 'post_type' => 'post', 'post_status' => 'publish', 'meta_query' => array( array( 'key' => 'price_cents', 'value' => 0, 'compare' => '=', 'type' => 'NUMERIC' ) ) ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-meta-query.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-003", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_003() to return only post IDs for a search query.", + "expected_behavior": "`wpbp_queries_003()` runs a published-post search with `WP_Query` and returns IDs only by using the `fields` query argument.", + "requirements": [ + "Use the relevant WordPress query class or helper", + "Return deterministic query data" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Query", + "description": "Uses WP_Query", + "weight": 1 + }, + { + "pattern": "fields", + "description": "Uses fields", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$needle = 'Needle' . wp_rand(); $id = (int) wp_insert_post( array( 'post_title' => $needle . ' Alpha', 'post_status' => 'publish' ) ); $GLOBALS['wpbp_queries_003'] = array( 'needle' => $needle, 'post' => $id );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_003' ) ) { return false; } $fixture = $GLOBALS['wpbp_queries_003'] ?? array(); if ( empty( $fixture['post'] ) ) { return false; } $ids = wpbp_queries_003( $fixture['needle'] ); return in_array( $fixture['post'], array_map( 'intval', (array) $ids ), true );", + "description": "The search query returns matching post IDs", + "weight": 1 + } + ], + "teardown": "$fixture = $GLOBALS['wpbp_queries_003'] ?? array(); if ( ! empty( $fixture['post'] ) ) { wp_delete_post( (int) $fixture['post'], true ); } unset( $GLOBALS['wpbp_queries_003'] );" + }, + "reference_solution": "function wpbp_queries_003( string $term ): array { $q = new WP_Query( array( 's' => $term, 'fields' => 'ids', 'post_status' => 'publish' ) ); return $q->posts; }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-meta-query.php", + "src/wp-includes/class-wp-tax-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-004", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_004( string $category_slug, array $tag_slugs ) to return a WP_Query for published posts that are in the supplied category slug and in either of the supplied tag slugs.", + "expected_behavior": "`wpbp_queries_004()` must query published posts with a taxonomy constraint requiring the category match and at least one matching tag. Runtime validation creates matching posts, category-only posts, tag-only posts, and draft posts to verify the AND/OR taxonomy behavior rather than a single-term tax query.", + "requirements": [ + "Use WP_Query with tax_query", + "Require the category constraint", + "Allow either supplied tag slug" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Query", + "description": "Uses WP_Query", + "weight": 1 + }, + { + "pattern": "tax_query", + "description": "Uses tax_query", + "weight": 1 + }, + { + "pattern": "relation", + "description": "Uses relation", + "weight": 1 + }, + { + "pattern": "/\\bAND\\b/i", + "description": "Uses an AND relation for the category constraint", + "weight": 0.5 + }, + { + "pattern": "/\\bOR\\b/i", + "description": "Uses an OR relation for the tag alternatives", + "weight": 0.5 + } + ] + }, + "runtime_checks": { + "setup": "foreach ( ( $GLOBALS['wpbp_queries_004']['posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } foreach ( array( 'wpbp-q004-cat' ) as $slug ) { $term = get_term_by( 'slug', $slug, 'category' ); if ( $term ) { wp_delete_term( $term->term_id, 'category' ); } } foreach ( array( 'wpbp-q004-tag-a', 'wpbp-q004-tag-b' ) as $slug ) { $term = get_term_by( 'slug', $slug, 'post_tag' ); if ( $term ) { wp_delete_term( $term->term_id, 'post_tag' ); } } $cat = wp_insert_term( 'WPBP Q004 Category', 'category', array( 'slug' => 'wpbp-q004-cat' ) ); $tag_a = wp_insert_term( 'WPBP Q004 Tag A', 'post_tag', array( 'slug' => 'wpbp-q004-tag-a' ) ); $tag_b = wp_insert_term( 'WPBP Q004 Tag B', 'post_tag', array( 'slug' => 'wpbp-q004-tag-b' ) ); if ( is_wp_error( $cat ) || is_wp_error( $tag_a ) || is_wp_error( $tag_b ) ) { return; } $target_a = wp_insert_post( array( 'post_title' => 'WPBP Q004 Target A', 'post_status' => 'publish' ) ); $target_b = wp_insert_post( array( 'post_title' => 'WPBP Q004 Target B', 'post_status' => 'publish' ) ); $category_only = wp_insert_post( array( 'post_title' => 'WPBP Q004 Category Only', 'post_status' => 'publish' ) ); $tag_only = wp_insert_post( array( 'post_title' => 'WPBP Q004 Tag Only', 'post_status' => 'publish' ) ); $draft = wp_insert_post( array( 'post_title' => 'WPBP Q004 Draft', 'post_status' => 'draft' ) ); foreach ( array( $target_a, $target_b, $category_only, $draft ) as $post_id ) { wp_set_post_terms( $post_id, array( (int) $cat['term_id'] ), 'category' ); } wp_set_post_terms( $target_a, array( (int) $tag_a['term_id'] ), 'post_tag' ); wp_set_post_terms( $target_b, array( (int) $tag_b['term_id'] ), 'post_tag' ); wp_set_post_terms( $tag_only, array( (int) $tag_a['term_id'] ), 'post_tag' ); wp_set_post_terms( $draft, array( (int) $tag_a['term_id'] ), 'post_tag' ); $GLOBALS['wpbp_queries_004'] = array( 'category_slug' => 'wpbp-q004-cat', 'tag_slugs' => array( 'wpbp-q004-tag-a', 'wpbp-q004-tag-b' ), 'posts' => array( 'target_a' => (int) $target_a, 'target_b' => (int) $target_b, 'category_only' => (int) $category_only, 'tag_only' => (int) $tag_only, 'draft' => (int) $draft ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_004' ) ) { return false; } $data = $GLOBALS['wpbp_queries_004'] ?? array(); $q = wpbp_queries_004( $data['category_slug'] ?? '', $data['tag_slugs'] ?? array() ); if ( ! $q instanceof WP_Query ) { return false; } $ids = array_map( 'intval', wp_list_pluck( $q->posts, 'ID' ) ); $posts = $data['posts'] ?? array(); return in_array( $posts['target_a'] ?? 0, $ids, true ) && in_array( $posts['target_b'] ?? 0, $ids, true ) && ! in_array( $posts['category_only'] ?? 0, $ids, true ) && ! in_array( $posts['tag_only'] ?? 0, $ids, true ) && ! in_array( $posts['draft'] ?? 0, $ids, true );", + "description": "The query returns only published posts matching the category and either supplied tag", + "weight": 1 + } + ], + "teardown": "foreach ( ( $GLOBALS['wpbp_queries_004']['posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } foreach ( array( 'wpbp-q004-cat' ) as $slug ) { $term = get_term_by( 'slug', $slug, 'category' ); if ( $term ) { wp_delete_term( $term->term_id, 'category' ); } } foreach ( array( 'wpbp-q004-tag-a', 'wpbp-q004-tag-b' ) as $slug ) { $term = get_term_by( 'slug', $slug, 'post_tag' ); if ( $term ) { wp_delete_term( $term->term_id, 'post_tag' ); } } unset( $GLOBALS['wpbp_queries_004'] );" + }, + "reference_solution": "function wpbp_queries_004( string $category_slug, array $tag_slugs ): WP_Query { $tag_slugs = array_values( $tag_slugs ); return new WP_Query( array( 'post_type' => 'post', 'post_status' => 'publish', 'posts_per_page' => -1, 'tax_query' => array( 'relation' => 'AND', array( 'taxonomy' => 'category', 'field' => 'slug', 'terms' => array( $category_slug ) ), array( 'relation' => 'OR', array( 'taxonomy' => 'post_tag', 'field' => 'slug', 'terms' => array( $tag_slugs[0] ?? '' ) ), array( 'taxonomy' => 'post_tag', 'field' => 'slug', 'terms' => array( $tag_slugs[1] ?? '' ) ) ) ) ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-tax-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-005", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_005() to query future posts after a supplied date.", + "expected_behavior": "`wpbp_queries_005()` builds a `WP_Query` for scheduled future posts after the supplied date, using `date_query` and including the `future` post status.", + "requirements": [ + "Use the relevant WordPress query class or helper", + "Return deterministic query data" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Query", + "description": "Uses WP_Query", + "weight": 1 + }, + { + "pattern": "date_query", + "description": "Uses date_query", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_queries_005_post'] = (int) wp_insert_post( array( 'post_title' => 'Date ' . wp_rand(), 'post_status' => 'future', 'post_date' => '2099-01-05 00:00:00' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_005' ) ) { return false; } $id = (int) ( $GLOBALS['wpbp_queries_005_post'] ?? 0 ); if ( ! $id ) { return false; } $q = wpbp_queries_005( '2099-01-01' ); return in_array( $id, array_map( 'intval', wp_list_pluck( $q->posts, 'ID' ) ), true );", + "description": "The query returns future posts after the supplied date", + "weight": 1 + } + ], + "teardown": "$id = (int) ( $GLOBALS['wpbp_queries_005_post'] ?? 0 ); if ( $id ) { wp_delete_post( $id, true ); } unset( $GLOBALS['wpbp_queries_005_post'] );" + }, + "reference_solution": "function wpbp_queries_005( string $after ): WP_Query { return new WP_Query( array( 'post_status' => 'future', 'posts_per_page' => -1, 'date_query' => array( array( 'after' => $after, 'inclusive' => true ) ) ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-meta-query.php", + "src/wp-includes/class-wp-tax-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-006", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_006() to build a WP_User_Query for users matching a role.", + "expected_behavior": "`wpbp_queries_006()` builds a `WP_User_Query` for users with the supplied role and returns user IDs.", + "requirements": [ + "Use the relevant WordPress query class or helper", + "Return deterministic query data" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_User_Query", + "description": "Uses WP_User_Query", + "weight": 1 + }, + { + "pattern": "role", + "description": "Uses role", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$u = wp_insert_user( array( 'user_login' => 'wpbp_query_user_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'subscriber' ) ); $GLOBALS['wpbp_queries_006_user'] = is_wp_error( $u ) ? 0 : (int) $u;", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_006' ) ) { return false; } $u = (int) ( $GLOBALS['wpbp_queries_006_user'] ?? 0 ); if ( ! $u ) { return false; } $q = wpbp_queries_006( 'subscriber' ); return in_array( $u, array_map( 'intval', $q->get_results() ), true );", + "description": "The user query returns users with the requested role", + "weight": 1 + } + ], + "teardown": "$u = (int) ( $GLOBALS['wpbp_queries_006_user'] ?? 0 ); if ( $u ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $u ); } unset( $GLOBALS['wpbp_queries_006_user'] );" + }, + "reference_solution": "function wpbp_queries_006( string $role ): WP_User_Query { return new WP_User_Query( array( 'role' => $role, 'fields' => 'ID' ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-meta-query.php", + "src/wp-includes/class-wp-tax-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-007", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_007() to return comments for a post using WP_Comment_Query.", + "expected_behavior": "`wpbp_queries_007()` uses `WP_Comment_Query` to return approved comments for the supplied post ID.", + "requirements": [ + "Use the relevant WordPress query class or helper", + "Return deterministic query data" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_007", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Comment_Query", + "description": "Uses WP_Comment_Query", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$p = (int) wp_insert_post( array( 'post_title' => 'Commented', 'post_status' => 'publish' ) ); $c = $p ? (int) wp_insert_comment( array( 'comment_post_ID' => $p, 'comment_content' => 'Hi', 'comment_approved' => 1 ) ) : 0; $GLOBALS['wpbp_queries_007'] = array( 'post' => $p, 'comment' => $c );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_007' ) ) { return false; } $fixture = $GLOBALS['wpbp_queries_007'] ?? array(); if ( empty( $fixture['comment'] ) ) { return false; } $comments = wpbp_queries_007( $fixture['post'] ); return in_array( $fixture['comment'], array_map( 'intval', wp_list_pluck( $comments, 'comment_ID' ) ), true );", + "description": "The comment query returns the approved comment for the post", + "weight": 1 + } + ], + "teardown": "$fixture = $GLOBALS['wpbp_queries_007'] ?? array(); if ( ! empty( $fixture['comment'] ) ) { wp_delete_comment( (int) $fixture['comment'], true ); } if ( ! empty( $fixture['post'] ) ) { wp_delete_post( (int) $fixture['post'], true ); } unset( $GLOBALS['wpbp_queries_007'] );" + }, + "reference_solution": "function wpbp_queries_007( int $post_id ): array { $q = new WP_Comment_Query( array( 'post_id' => $post_id, 'status' => 'approve' ) ); return $q->comments; }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-meta-query.php", + "src/wp-includes/class-wp-tax-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-008", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_008() to query terms with hide_empty disabled.", + "expected_behavior": "`wpbp_queries_008()` uses `WP_Term_Query` to return term IDs for the supplied taxonomy with `hide_empty` disabled, so empty terms are included.", + "requirements": [ + "Use the relevant WordPress query class or helper", + "Return deterministic query data" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_008", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Term_Query", + "description": "Uses WP_Term_Query", + "weight": 1 + }, + { + "pattern": "hide_empty", + "description": "Uses hide_empty", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$suffix = (string) wp_rand(); $term = wp_insert_term( 'WPBP Empty Term ' . $suffix, 'category', array( 'slug' => 'wpbp-empty-' . $suffix ) ); $GLOBALS['wpbp_queries_008_term'] = is_wp_error( $term ) ? 0 : (int) $term['term_id'];", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_008' ) ) { return false; } $term_id = (int) ( $GLOBALS['wpbp_queries_008_term'] ?? 0 ); if ( ! $term_id ) { return false; } $ids = wpbp_queries_008( 'category' ); return in_array( $term_id, array_map( 'intval', (array) $ids ), true );", + "description": "The term query includes empty terms", + "weight": 1 + } + ], + "teardown": "$term_id = (int) ( $GLOBALS['wpbp_queries_008_term'] ?? 0 ); if ( $term_id ) { wp_delete_term( $term_id, 'category' ); } unset( $GLOBALS['wpbp_queries_008_term'] );" + }, + "reference_solution": "function wpbp_queries_008( string $taxonomy ): array { $q = new WP_Term_Query( array( 'taxonomy' => $taxonomy, 'hide_empty' => false, 'fields' => 'ids' ) ); return $q->terms; }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-meta-query.php", + "src/wp-includes/class-wp-tax-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-009", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_009() to return the adjacent published post in the same category as the current global post.", + "expected_behavior": "`wpbp_queries_009()` must delegate to WordPress adjacent-post handling with same-term category constraints. Runtime validation sets a current global post, creates same-date previous and next category neighbors plus closer posts in another category, and verifies the returned `WP_Post` objects honor category matching and WordPress's ID fallback for identical post dates.", + "requirements": [ + "Use WordPress adjacent-post handling", + "Constrain results to the category taxonomy", + "Return the WP_Post object WordPress finds" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_009", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "get_adjacent_post", + "description": "Uses get_adjacent_post", + "weight": 1 + }, + { + "pattern": "category", + "description": "Constrains adjacent lookup to categories", + "weight": 0.5 + } + ] + }, + "runtime_checks": { + "setup": "foreach ( ( $GLOBALS['wpbp_queries_009']['posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } foreach ( array( 'wpbp-q009-same', 'wpbp-q009-other' ) as $slug ) { $term = get_term_by( 'slug', $slug, 'category' ); if ( $term ) { wp_delete_term( $term->term_id, 'category' ); } } $same = wp_insert_term( 'WPBP Q009 Same', 'category', array( 'slug' => 'wpbp-q009-same' ) ); $other = wp_insert_term( 'WPBP Q009 Other', 'category', array( 'slug' => 'wpbp-q009-other' ) ); if ( is_wp_error( $same ) || is_wp_error( $other ) ) { return; } $date = '2024-01-01 12:00:00'; $previous = wp_insert_post( array( 'post_title' => 'WPBP Q009 Previous', 'post_status' => 'publish', 'post_date' => $date, 'post_date_gmt' => $date ) ); $previous_other = wp_insert_post( array( 'post_title' => 'WPBP Q009 Previous Other', 'post_status' => 'publish', 'post_date' => $date, 'post_date_gmt' => $date ) ); $current = wp_insert_post( array( 'post_title' => 'WPBP Q009 Current', 'post_status' => 'publish', 'post_date' => $date, 'post_date_gmt' => $date ) ); $next_other = wp_insert_post( array( 'post_title' => 'WPBP Q009 Next Other', 'post_status' => 'publish', 'post_date' => $date, 'post_date_gmt' => $date ) ); $next = wp_insert_post( array( 'post_title' => 'WPBP Q009 Next', 'post_status' => 'publish', 'post_date' => $date, 'post_date_gmt' => $date ) ); foreach ( array( $previous, $current, $next ) as $post_id ) { wp_set_post_terms( $post_id, array( (int) $same['term_id'] ), 'category' ); } foreach ( array( $previous_other, $next_other ) as $post_id ) { wp_set_post_terms( $post_id, array( (int) $other['term_id'] ), 'category' ); } $GLOBALS['wpbp_queries_009'] = array( 'posts' => array( 'previous' => (int) $previous, 'previous_other' => (int) $previous_other, 'current' => (int) $current, 'next_other' => (int) $next_other, 'next' => (int) $next ) ); $GLOBALS['post'] = get_post( $current ); setup_postdata( $GLOBALS['post'] );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_009' ) ) { return false; } $posts = $GLOBALS['wpbp_queries_009']['posts'] ?? array(); $GLOBALS['post'] = get_post( $posts['current'] ?? 0 ); if ( ! $GLOBALS['post'] instanceof WP_Post ) { return false; } setup_postdata( $GLOBALS['post'] ); $previous = wpbp_queries_009( true ); $next = wpbp_queries_009( false ); return $previous instanceof WP_Post && (int) $previous->ID === (int) ( $posts['previous'] ?? 0 ) && $next instanceof WP_Post && (int) $next->ID === (int) ( $posts['next'] ?? 0 );", + "description": "The helper returns same-category adjacent posts even when neighboring posts share the same date", + "weight": 1 + } + ], + "teardown": "wp_reset_postdata(); unset( $GLOBALS['post'] ); foreach ( ( $GLOBALS['wpbp_queries_009']['posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } foreach ( array( 'wpbp-q009-same', 'wpbp-q009-other' ) as $slug ) { $term = get_term_by( 'slug', $slug, 'category' ); if ( $term ) { wp_delete_term( $term->term_id, 'category' ); } } unset( $GLOBALS['wpbp_queries_009'] );" + }, + "reference_solution": "function wpbp_queries_009( bool $previous = true ): mixed { return get_adjacent_post( true, '', $previous, 'category' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/link-template.php", + "src/wp-includes/class-wp-query.php", + "src/wp-includes/taxonomy.php" + ], + "release_focus": "6.9" + } + }, + { + "id": "e-queries-010", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_010() to preload post meta cache before reading many posts.", + "expected_behavior": "`wpbp_queries_010()` primes post meta for the supplied post IDs with `update_meta_cache()` before reading the requested meta key from each post.", + "requirements": [ + "Use the relevant WordPress query class or helper", + "Return deterministic query data" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_010", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "update_meta_cache", + "description": "Uses update_meta_cache", + "weight": 1 + }, + { + "pattern": "get_post_meta", + "description": "Uses get_post_meta", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$id = (int) wp_insert_post( array( 'post_title' => 'Meta Cache', 'post_status' => 'publish' ) ); if ( $id ) { update_post_meta( $id, 'wpbp_key', 'yes' ); } $GLOBALS['wpbp_queries_010_post'] = $id;", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_010' ) ) { return false; } $id = (int) ( $GLOBALS['wpbp_queries_010_post'] ?? 0 ); if ( ! $id ) { return false; } $out = wpbp_queries_010( array( $id ), 'wpbp_key' ); return isset( $out[ $id ] ) && 'yes' === $out[ $id ];", + "description": "The helper returns meta values after priming the post meta cache", + "weight": 1 + } + ], + "teardown": "$id = (int) ( $GLOBALS['wpbp_queries_010_post'] ?? 0 ); if ( $id ) { wp_delete_post( $id, true ); } unset( $GLOBALS['wpbp_queries_010_post'] );" + }, + "reference_solution": "function wpbp_queries_010( array $post_ids, string $key ): array { update_meta_cache( 'post', $post_ids ); $out = array(); foreach ( $post_ids as $post_id ) { $out[ $post_id ] = get_post_meta( $post_id, $key, true ); } return $out; }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/class-wp-meta-query.php", + "src/wp-includes/class-wp-tax-query.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-011", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_011() to return one published post with get_posts() while allowing WordPress query filters to modify the query.", + "expected_behavior": "`wpbp_queries_011()` must call `get_posts()` with filters enabled, overriding the helper's default suppressed-filter behavior. Runtime validation installs a `posts_where` filter that selects an older fixture post; without `suppress_filters` set to false, `get_posts()` returns the newer distractor instead.", + "requirements": [ + "Use get_posts() for the query", + "Allow WordPress query filters to run", + "Return the filtered posts" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_011", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "get_posts", + "description": "Uses get_posts", + "weight": 1 + }, + { + "pattern": "/suppress_filters['\\\"]?\\s*=>\\s*false/i", + "description": "Keeps get_posts filters enabled", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "foreach ( ( $GLOBALS['wpbp_queries_011_posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } $target = wp_insert_post( array( 'post_title' => 'WPBP Q011 Filtered Target', 'post_status' => 'publish', 'post_date' => '2024-01-01 10:00:00', 'post_date_gmt' => '2024-01-01 10:00:00' ) ); $distractor = wp_insert_post( array( 'post_title' => 'WPBP Q011 Newer Distractor', 'post_status' => 'publish', 'post_date' => '2024-01-02 10:00:00', 'post_date_gmt' => '2024-01-02 10:00:00' ) ); $GLOBALS['wpbp_queries_011_posts'] = array( 'target' => (int) $target, 'distractor' => (int) $distractor );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_011' ) ) { return false; } $target_id = (int) ( $GLOBALS['wpbp_queries_011_posts']['target'] ?? 0 ); $callback = static function ( $where, $query ) use ( $target_id ) { global $wpdb; return $where . $wpdb->prepare( ' AND ' . $wpdb->posts . '.ID = %d', $target_id ); }; $GLOBALS['wpbp_queries_011_filter'] = $callback; add_filter( 'posts_where', $callback, 10, 2 ); $posts = wpbp_queries_011(); remove_filter( 'posts_where', $callback, 10 ); unset( $GLOBALS['wpbp_queries_011_filter'] ); if ( ! is_array( $posts ) ) { return false; } $ids = array_map( 'intval', wp_list_pluck( $posts, 'ID' ) ); return array( $target_id ) === $ids;", + "description": "The get_posts query allows posts_where to select the older target post", + "weight": 1 + } + ], + "teardown": "if ( isset( $GLOBALS['wpbp_queries_011_filter'] ) ) { remove_filter( 'posts_where', $GLOBALS['wpbp_queries_011_filter'], 10 ); } foreach ( ( $GLOBALS['wpbp_queries_011_posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } unset( $GLOBALS['wpbp_queries_011_posts'], $GLOBALS['wpbp_queries_011_filter'] );" + }, + "reference_solution": "function wpbp_queries_011(): array { $posts = get_posts( array( 'numberposts' => 1, 'suppress_filters' => false, 'post_status' => 'publish' ) ); return $posts; }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/post.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-queries-012", + "category": "queries", + "difficulty": "intermediate", + "prompt": "Implement wpbp_queries_012() to return only published sticky posts as a WP_Query without relying on home-query sticky prepending.", + "expected_behavior": "`wpbp_queries_012()` must read WordPress's sticky post IDs and query those IDs explicitly with `WP_Query`, using `ignore_sticky_posts` so normal home-query sticky relocation is not part of the result. Runtime validation verifies the sticky post is returned, a regular published post is excluded, and an empty sticky list yields no posts.", + "requirements": [ + "Read the sticky post IDs from WordPress", + "Query only those posts and preserve sticky order", + "Return an empty result when no posts are sticky" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_queries_012", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_Query", + "description": "Uses WP_Query", + "weight": 1 + }, + { + "pattern": "get_option", + "description": "Reads the sticky_posts option", + "weight": 1 + }, + { + "pattern": "sticky_posts", + "description": "Uses WordPress sticky post IDs", + "weight": 1 + }, + { + "pattern": "post__in", + "description": "Restricts the query to sticky IDs", + "weight": 1 + }, + { + "pattern": "ignore_sticky_posts", + "description": "Uses ignore_sticky_posts", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_queries_012_original_sticky'] = get_option( 'sticky_posts', array() ); foreach ( ( $GLOBALS['wpbp_queries_012_posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } $sticky = wp_insert_post( array( 'post_title' => 'WPBP Q012 Sticky', 'post_status' => 'publish', 'post_date' => '2024-01-01 10:00:00', 'post_date_gmt' => '2024-01-01 10:00:00' ) ); $regular = wp_insert_post( array( 'post_title' => 'WPBP Q012 Regular', 'post_status' => 'publish', 'post_date' => '2024-01-02 10:00:00', 'post_date_gmt' => '2024-01-02 10:00:00' ) ); update_option( 'sticky_posts', array( (int) $sticky ) ); $GLOBALS['wpbp_queries_012_posts'] = array( 'sticky' => (int) $sticky, 'regular' => (int) $regular );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_queries_012' ) ) { return false; } $posts = $GLOBALS['wpbp_queries_012_posts'] ?? array(); $q = wpbp_queries_012(); if ( ! $q instanceof WP_Query ) { return false; } $ids = array_map( 'intval', wp_list_pluck( $q->posts, 'ID' ) ); $only_sticky = array( (int) ( $posts['sticky'] ?? 0 ) ) === $ids; update_option( 'sticky_posts', array() ); $empty = wpbp_queries_012(); return $only_sticky && $empty instanceof WP_Query && empty( $empty->posts );", + "description": "The query returns only configured sticky posts and handles an empty sticky list", + "weight": 1 + } + ], + "teardown": "if ( array_key_exists( 'wpbp_queries_012_original_sticky', $GLOBALS ) ) { update_option( 'sticky_posts', $GLOBALS['wpbp_queries_012_original_sticky'] ); } foreach ( ( $GLOBALS['wpbp_queries_012_posts'] ?? array() ) as $post_id ) { if ( $post_id ) { wp_delete_post( (int) $post_id, true ); } } unset( $GLOBALS['wpbp_queries_012_original_sticky'], $GLOBALS['wpbp_queries_012_posts'] );" + }, + "reference_solution": "function wpbp_queries_012(): WP_Query { $sticky_ids = array_map( 'intval', (array) get_option( 'sticky_posts', array() ) ); if ( empty( $sticky_ids ) ) { $sticky_ids = array( 0 ); } return new WP_Query( array( 'post_type' => 'post', 'post_status' => 'publish', 'post__in' => $sticky_ids, 'orderby' => 'post__in', 'posts_per_page' => count( $sticky_ids ), 'ignore_sticky_posts' => true ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp-query.php", + "src/wp-includes/post.php", + "src/wp-includes/option.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/rest-api.json b/datasets/suites/wp-core-v1/execution/rest-api.json index a03b434..dce9d14 100644 --- a/datasets/suites/wp-core-v1/execution/rest-api.json +++ b/datasets/suites/wp-core-v1/execution/rest-api.json @@ -1,99 +1,642 @@ { "id": "wp-core-execution-v1-rest_api", - "version": "1.1.0", + "version": "2.0.0", "metadata": { "name": "WordPress Core Execution Tests - REST API", - "description": "Code generation tasks for WordPress REST API development", - "wp_version": "6.9", - "created_at": "2025-12-16" + "description": "Code generation tasks for WordPress REST API APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" }, "tests": [ { - "id": "e-rest-001", + "id": "e-rest-api-001", "category": "rest-api", "difficulty": "intermediate", - "prompt": "Create a REST API endpoint at '/wp-json/myplugin/v1/status' that returns a JSON object with a 'status' key set to 'ok' and a 'timestamp' key with the current Unix timestamp. The endpoint should be publicly accessible without authentication.", - "expected_behavior": "GET request to /wp-json/myplugin/v1/status returns {\"status\": \"ok\", \"timestamp\": }", + "prompt": "Create a public REST endpoint /wp-json/wpbp/v1/status that returns status ok with a numeric timestamp.", + "expected_behavior": "Registers a public GET route in the wpbp/v1 namespace on rest_api_init. Runtime validation dispatches the route through WordPress REST internals and verifies a 200 response containing status ok and an integer timestamp.", "requirements": [ - "Use register_rest_route() to register the endpoint", - "Use namespace 'myplugin/v1'", - "Route should be 'status'", - "Accept GET requests", - "Return WP_REST_Response with status and timestamp", - "Endpoint should be publicly accessible (permission_callback returns true)" + "Register the route on rest_api_init", + "Use namespace wpbp/v1", + "Use a public permission callback", + "Return status ok and an integer timestamp" ], "static_checks": { "required_patterns": [ { "pattern": "register_rest_route", - "description": "Must use register_rest_route()", - "weight": 1.0 + "description": "Uses register_rest_route", + "weight": 1 }, { - "pattern": "myplugin/v1", - "description": "Must use correct namespace", - "weight": 0.5 + "pattern": "rest_api_init", + "description": "Registers the route on rest_api_init", + "weight": 1 + }, + { + "pattern": "wpbp/v1", + "description": "Uses the requested REST namespace", + "weight": 1 }, { "pattern": "permission_callback", - "description": "Must include permission_callback", + "description": "Uses permission_callback", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! did_action( 'rest_api_init' ) ) { do_action( 'rest_api_init' ); } $response = rest_do_request( new WP_REST_Request( 'GET', '/wpbp/v1/status' ) ); $data = $response->get_data(); return 200 === $response->get_status() && is_array( $data ) && 'ok' === ( $data['status'] ?? null ) && isset( $data['timestamp'] ) && is_int( $data['timestamp'] );", + "description": "The public status endpoint returns status ok with an integer timestamp", + "weight": 1 + } + ] + }, + "reference_solution": "add_action( 'rest_api_init', function () {\n register_rest_route( 'wpbp/v1', '/status', array(\n 'methods' => WP_REST_Server::READABLE,\n 'permission_callback' => '__return_true',\n 'callback' => function () {\n return rest_ensure_response( array( 'status' => 'ok', 'timestamp' => time() ) );\n },\n ) );\n} );", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-002", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Create a REST endpoint /wp-json/wpbp/v1/books/ that accepts numeric IDs, rejects nonnumeric IDs through REST argument validation, and returns valid IDs as integers.", + "expected_behavior": "Registers a wpbp/v1 books route with an id path parameter and a REST argument validator. Runtime validation confirms that /books/42 returns id 42 and /books/not-a-number is rejected by WordPress as an invalid parameter.", + "requirements": [ + "Register the route on rest_api_init with namespace wpbp/v1", + "Use a route parameter named id and a validate_callback", + "Reject nonnumeric IDs with a REST validation error", + "Return valid IDs as integers" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "register_rest_route", + "description": "Uses register_rest_route", + "weight": 1 + }, + { + "pattern": "validate_callback", + "description": "Uses validate_callback", + "weight": 1 + }, + { + "pattern": "\\?P", + "description": "Defines an id route parameter", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "rest_response", + "target": "/wpbp/v1/books/42", + "expected_status": 200, + "expected_data": { + "id": 42 + }, + "description": "ID is validated and returned" + }, + { + "type": "rest_response", + "target": "/wpbp/v1/books/not-a-number", + "expected_status": 400, + "body_contains": "rest_invalid_param", + "description": "Nonnumeric IDs fail REST argument validation" + } + ] + }, + "reference_solution": "add_action( 'rest_api_init', function () {\n register_rest_route( 'wpbp/v1', '/books/(?P[^/]+)', array(\n 'methods' => WP_REST_Server::READABLE,\n 'permission_callback' => '__return_true',\n 'callback' => fn( WP_REST_Request $request ) => array( 'id' => (int) $request['id'] ),\n 'args' => array( 'id' => array( 'validate_callback' => fn( $value ) => is_numeric( $value ) ) ),\n ) );\n} );", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-003", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Create a private REST endpoint /wp-json/wpbp/v1/private that requires manage_options and returns secret data.", + "expected_behavior": "Registers a private wpbp/v1 route whose permission callback checks manage_options. Runtime validation dispatches the route as a logged-out user and as an administrator fixture, confirming unauthorized requests are denied and authorized requests receive secret true.", + "requirements": [ + "Register the route on rest_api_init with namespace wpbp/v1", + "Use current_user_can( 'manage_options' ) in permission_callback", + "Return secret true when authorized" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "register_rest_route", + "description": "Uses register_rest_route", + "weight": 1 + }, + { + "pattern": "current_user_can", + "description": "Uses current_user_can", + "weight": 1 + }, + { + "pattern": "permission_callback", + "description": "Uses permission_callback", + "weight": 1 + }, + { + "pattern": "manage_options", + "description": "Checks the requested capability", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_rest_api_003_admin_id'] = wp_insert_user( array( 'user_login' => 'wpbp_rest_private_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'administrator' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! did_action( 'rest_api_init' ) ) { do_action( 'rest_api_init' ); } $admin_id = $GLOBALS['wpbp_rest_api_003_admin_id'] ?? 0; if ( ! is_int( $admin_id ) ) { return false; } wp_set_current_user( 0 ); $logged_out = rest_do_request( new WP_REST_Request( 'GET', '/wpbp/v1/private' ) ); wp_set_current_user( $admin_id ); $authorized = rest_do_request( new WP_REST_Request( 'GET', '/wpbp/v1/private' ) ); $data = $authorized->get_data(); return 401 === $logged_out->get_status() && 200 === $authorized->get_status() && is_array( $data ) && true === ( $data['secret'] ?? null );", + "description": "Private endpoint denies logged-out users and returns secret data to an administrator", + "weight": 1 + } + ], + "teardown": "$user_id = $GLOBALS['wpbp_rest_api_003_admin_id'] ?? 0; if ( is_int( $user_id ) ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); } wp_set_current_user( 0 ); unset( $GLOBALS['wpbp_rest_api_003_admin_id'] );" + }, + "reference_solution": "add_action( 'rest_api_init', function () {\n register_rest_route( 'wpbp/v1', '/private', array(\n 'methods' => 'GET',\n 'permission_callback' => fn() => current_user_can( 'manage_options' ),\n 'callback' => fn() => array( 'secret' => true ),\n ) );\n} );", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-004", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Create a REST endpoint /wp-json/wpbp/v1/options that sanitizes a title parameter and returns it.", + "expected_behavior": "The submission registers a POST REST endpoint at `/wp-json/wpbp/v1/options`, declares a `title` argument with `sanitize_text_field` as its sanitizer, and returns the sanitized title from WordPress REST request handling.", + "requirements": [ + "Accept POST requests", + "Declare a title argument with sanitize_callback", + "Return the sanitized title" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "register_rest_route", + "description": "Uses register_rest_route", + "weight": 1 + }, + { + "pattern": "sanitize_callback", + "description": "Uses sanitize_callback", + "weight": 1 + }, + { + "pattern": "sanitize_text_field", + "description": "Uses sanitize_text_field", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "rest_response", + "target": "/wpbp/v1/options", + "method": "POST", + "params": { + "title": "Hello" + }, + "expected_status": 200, + "expected_data": { + "title": "Hello" + }, + "description": "Parameter is sanitized" + } + ] + }, + "reference_solution": "add_action( 'rest_api_init', function () {\n register_rest_route( 'wpbp/v1', '/options', array(\n 'methods' => WP_REST_Server::CREATABLE,\n 'permission_callback' => '__return_true',\n 'args' => array( 'title' => array( 'sanitize_callback' => 'sanitize_text_field' ) ),\n 'callback' => fn( WP_REST_Request $request ) => array( 'title' => $request->get_param( 'title' ) ),\n ) );\n} );", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-005", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_rest_api_005() to normalize a WP_Error into a REST response array with code, message, and status.", + "expected_behavior": "`wpbp_rest_api_005()` converts a `WP_Error` to a REST response with `rest_convert_error_to_response()` and returns the response status plus response data, preserving the error code and HTTP status.", + "requirements": [ + "Use rest_convert_error_to_response", + "Preserve the status code" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rest_api_005", + "description": "Defines the requested function", "weight": 0.5 }, { - "pattern": "WP_REST_Response|rest_ensure_response", - "description": "Should return proper REST response", - "weight": 0.3 + "pattern": "rest_convert_error_to_response", + "description": "Uses rest_convert_error_to_response", + "weight": 1 + }, + { + "pattern": "WP_Error", + "description": "Uses WP_Error", + "weight": 1 } ] }, "runtime_checks": { "assertions": [ { - "type": "hook_registered", - "target": "rest_api_init", - "description": "REST route registration hook must be used", - "weight": 1.0 + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rest_api_005' ) ) { return false; } $out = wpbp_rest_api_005( new WP_Error( 'wpbp_forbidden', 'Nope', array( 'status' => 403 ) ) ); return 403 === $out['status'] && 'wpbp_forbidden' === $out['data']['code'];", + "description": "The WP_Error is converted to REST response data with status", + "weight": 1 } ] }, - "judge_config": { - "rubric_id": "wp-judge-rubric-v1", - "context_for_judge": "This is a simple status endpoint for a health check. Security is important - ensure permission_callback is properly defined." + "reference_solution": "function wpbp_rest_api_005( WP_Error $error ): array { $response = rest_convert_error_to_response( $error ); return array( 'status' => $response->get_status(), 'data' => $response->get_data() ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" } }, { - "id": "e-rest-cache-salted-001", + "id": "e-rest-api-006", "category": "rest-api", - "difficulty": "hard", - "prompt": "Create a REST endpoint /wp-json/demo/v1/books that returns published 'book' titles and uses wp_cache_get_salted/wp_cache_set_salted keyed by the posts last_changed value. Invalidate cache on save_post_book and deleted_post.", - "expected_behavior": "First call computes and caches list; second call is a cache hit; after inserting a new book, subsequent call reflects new title and cache resets.", + "difficulty": "intermediate", + "prompt": "Implement wpbp_rest_api_006() to ensure arbitrary callback data is converted to a WP_REST_Response.", + "expected_behavior": "`wpbp_rest_api_006()` passes arbitrary callback data through `rest_ensure_response()` and returns the resulting `WP_REST_Response` status and data.", "requirements": [ - "Register route on rest_api_init with namespace demo/v1 and route books", - "Use wp_cache_get_salted/wp_cache_set_salted with group 'books-list' and salt from wp_cache_get_last_changed('posts')", - "Capability: read for logged-in users, otherwise error", - "Add actions on save_post_book and deleted_post to delete the salted cache entry" + "Use rest_ensure_response", + "Return the response status and data" ], "static_checks": { "required_patterns": [ - { "pattern": "register_rest_route", "description": "Registers REST route", "weight": 1.0 }, - { "pattern": "wp_cache_get_salted", "description": "Uses salted cache get", "weight": 0.8 }, - { "pattern": "save_post_book", "description": "Cache invalidation hook", "weight": 0.6 } + { + "pattern": "wpbp_rest_api_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "rest_ensure_response", + "description": "Uses rest_ensure_response", + "weight": 1 + }, + { + "pattern": "WP_REST_Response", + "description": "Uses WP_REST_Response", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rest_api_006' ) ) { return false; } return array( 200, array( 'ok' => true ) ) === wpbp_rest_api_006( array( 'ok' => true ) );", + "description": "Callback data is normalized to a WP_REST_Response", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rest_api_006( array $data ): array { $response = rest_ensure_response( $data ); if ( ! $response instanceof WP_REST_Response ) { return array(); } return array( $response->get_status(), $response->get_data() ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" ], - "forbidden_patterns": [ - { "pattern": "transient", "description": "Do not use transients", "severity": "error" } + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-007", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_rest_api_007() to build a collection response with X-WP-Total and X-WP-TotalPages headers.", + "expected_behavior": "`wpbp_rest_api_007()` returns a WP_REST_Response for the provided collection items and sets WordPress pagination headers X-WP-Total and X-WP-TotalPages. Runtime validation checks the computed headers for a non-even page count.", + "requirements": [ + "Use WP_REST_Response", + "Set X-WP-Total and X-WP-TotalPages headers" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rest_api_007", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_REST_Response", + "description": "Uses WP_REST_Response", + "weight": 1 + }, + { + "pattern": "X-WP-Total", + "description": "Sets the total-items REST header", + "weight": 1 + }, + { + "pattern": "X-WP-TotalPages", + "description": "Sets the total-pages REST header", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rest_api_007' ) ) { return false; } $r = wpbp_rest_api_007( array( 'a' ), 21, 10 ); return '21' === $r->get_headers()['X-WP-Total'] && '3' === $r->get_headers()['X-WP-TotalPages'];", + "description": "The REST response includes total and total-pages headers", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rest_api_007( array $items, int $total, int $per_page ): WP_REST_Response { $response = new WP_REST_Response( $items ); $response->header( 'X-WP-Total', (string) $total ); $response->header( 'X-WP-TotalPages', (string) (int) ceil( $total / $per_page ) ); return $response; }", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-008", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_rest_api_008() to return a REST schema for settings data where mode is required and may only be light or dark.", + "expected_behavior": "`wpbp_rest_api_008()` returns an object schema with a required mode property constrained to the light and dark string enum values. Runtime validation uses WordPress REST schema validation to confirm valid data passes while missing or unsupported modes fail.", + "requirements": [ + "Return an object schema with a mode property", + "Require mode", + "Limit mode to light or dark" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rest_api_008", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "enum", + "description": "Uses enum", + "weight": 1 + }, + { + "pattern": "properties", + "description": "Uses properties", + "weight": 1 + }, + { + "pattern": "required", + "description": "Marks the mode property as required", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rest_api_008' ) ) { return false; } $schema = wpbp_rest_api_008(); $valid = rest_validate_value_from_schema( array( 'mode' => 'light' ), $schema, 'settings' ); $invalid = rest_validate_value_from_schema( array( 'mode' => 'blue' ), $schema, 'settings' ); $missing = rest_validate_value_from_schema( array(), $schema, 'settings' ); return true === $valid && is_wp_error( $invalid ) && is_wp_error( $missing );", + "description": "Schema accepts light/dark mode data and rejects missing or unsupported modes", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rest_api_008(): array { return array( 'type' => 'object', 'required' => array( 'mode' ), 'properties' => array( 'mode' => array( 'type' => 'string', 'enum' => array( 'light', 'dark' ) ) ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-009", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_rest_api_009() to check whether a value passes WordPress REST schema validation as a positive integer.", + "expected_behavior": "`wpbp_rest_api_009()` builds an integer schema with minimum 1 and returns true only when WordPress REST schema validation accepts the value. Runtime validation covers an integer, an integer-like request string, a below-minimum value, and a nonnumeric value.", + "requirements": [ + "Build a REST schema with integer type and minimum 1", + "Use WordPress REST schema validation", + "Return true only when validation passes" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rest_api_009", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "rest_validate_value_from_schema", + "description": "Uses rest_validate_value_from_schema", + "weight": 1 + }, + { + "pattern": "minimum", + "description": "Uses minimum", + "weight": 1 + }, + { + "pattern": "integer", + "description": "Declares integer type validation", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rest_api_009' ) ) { return false; } return wpbp_rest_api_009( 5 ) && wpbp_rest_api_009( '5' ) && ! wpbp_rest_api_009( 0 ) && ! wpbp_rest_api_009( 'abc' );", + "description": "Positive integer values pass REST schema validation while invalid values fail", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rest_api_009( mixed $value ): bool { $schema = array( 'type' => 'integer', 'minimum' => 1 ); return true === rest_validate_value_from_schema( $value, $schema, 'id' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-010", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_rest_api_010() to sanitize REST array input using rest_sanitize_value_from_schema.", + "expected_behavior": "`wpbp_rest_api_010()` sanitizes an array of values against a REST schema whose `items` are integers, returning the integer-normalized values from WordPress schema sanitization.", + "requirements": [ + "Declare an array item schema", + "Return sanitized integers" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rest_api_010", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "rest_sanitize_value_from_schema", + "description": "Uses rest_sanitize_value_from_schema", + "weight": 1 + }, + { + "pattern": "items", + "description": "Uses items", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rest_api_010' ) ) { return false; } return array( 1, 2, 3 ) === wpbp_rest_api_010( array( '1', '2', '3' ) );", + "description": "Array values are sanitized through the REST schema", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rest_api_010( array $values ): array { $schema = array( 'type' => 'array', 'items' => array( 'type' => 'integer' ) ); return rest_sanitize_value_from_schema( $values, $schema ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-011", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_rest_api_011() to return the correct authorization status for logged-out and logged-in users.", + "expected_behavior": "`wpbp_rest_api_011()` checks `rest_authorization_required_code()` as an anonymous user and again as a logged-in user, returning both status codes so WordPress determines the 401/403 distinction.", + "requirements": [ + "Use rest_authorization_required_code", + "Do not hard-code only one status" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rest_api_011", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "rest_authorization_required_code", + "description": "Uses rest_authorization_required_code", + "weight": 1 + }, + { + "pattern": "wp_set_current_user", + "description": "Uses wp_set_current_user", + "weight": 1 + } ] }, "runtime_checks": { - "setup": "register_post_type('book'); wp_insert_post(['post_type'=>'book','post_title'=>'Alpha','post_status'=>'publish']);", + "setup": "global $wpdb; $GLOBALS['wpbp_rest_api_011_max_user'] = (int) $wpdb->get_var( \"SELECT MAX(ID) FROM {$wpdb->users}\" );", "assertions": [ - { "type": "http_body_contains", "target": "/wp-json/demo/v1/books", "expected": "Alpha", "description": "Endpoint returns cached data", "weight": 1.0 }, - { "type": "cache_miss_then_hit", "target": "/wp-json/demo/v1/books", "description": "Second call hits cache", "weight": 1.0 }, - { "type": "cache_invalidated_on_hook", "target": "save_post_book", "description": "Cache clears when book updated", "weight": 1.0 } + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rest_api_011' ) ) { return false; } return array( 401, 403 ) === wpbp_rest_api_011();", + "description": "Authorization status differs for logged-out and logged-in users", + "weight": 1 + } + ], + "teardown": "global $wpdb; $max = (int) ( $GLOBALS['wpbp_rest_api_011_max_user'] ?? 0 ); if ( $max > 0 ) { $ids = $wpdb->get_col( $wpdb->prepare( \"SELECT ID FROM {$wpdb->users} WHERE ID > %d\", $max ) ); if ( $ids ) { require_once ABSPATH . 'wp-admin/includes/user.php'; foreach ( $ids as $user_id ) { wp_delete_user( (int) $user_id ); } } } unset( $GLOBALS['wpbp_rest_api_011_max_user'] );" + }, + "reference_solution": "function wpbp_rest_api_011(): array { wp_set_current_user( 0 ); $logged_out = rest_authorization_required_code(); $user_id = wp_insert_user( array( 'user_login' => 'wpbp_rest_user_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'subscriber' ) ); wp_set_current_user( $user_id ); $logged_in = rest_authorization_required_code(); require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); wp_set_current_user( 0 ); return array( $logged_out, $logged_in ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" ], - "teardown": "unregister_post_type('book');" + "release_focus": "classic" + } + }, + { + "id": "e-rest-api-012", + "category": "rest-api", + "difficulty": "intermediate", + "prompt": "Implement wpbp_rest_api_012() to add a custom link relation to a WP_REST_Response.", + "expected_behavior": "`wpbp_rest_api_012()` creates a `WP_REST_Response`, adds the supplied URL under the `https://api.w.org/related` link relation, and returns the response links.", + "requirements": [ + "Use add_link", + "Return links from response" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rest_api_012", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_REST_Response", + "description": "Uses WP_REST_Response", + "weight": 1 + }, + { + "pattern": "add_link", + "description": "Uses add_link", + "weight": 1 + } + ] }, - "judge_config": { "rubric_id": "wp-judge-rubric-v1" }, - "reference_solution": "add_action( 'rest_api_init', function () {\n register_rest_route( 'demo/v1', '/books', array(\n 'methods' => 'GET',\n 'permission_callback' => fn() => is_user_logged_in() && current_user_can( 'read' ),\n 'callback' => function () {\n $salt = wp_cache_get_last_changed( 'posts' );\n $group = 'books-list';\n $key = 'titles';\n $data = wp_cache_get_salted( $key, $group, $salt, $found );\n if ( $found ) {\n return rest_ensure_response( $data );\n }\n $q = new WP_Query( array( 'post_type' => 'book', 'post_status' => 'publish', 'fields' => 'titles', 'nopaging' => true ) );\n $titles = wp_list_pluck( $q->posts, 'post_title' );\n wp_cache_set_salted( $key, $titles, $group, $salt );\n return rest_ensure_response( $titles );\n },\n ) );\n} );\nadd_action( 'save_post_book', fn() => wp_cache_delete( 'titles', 'books-list' ) );\nadd_action( 'deleted_post', function ( $pid ) {\n if ( get_post_type( $pid ) === 'book' ) {\n wp_cache_delete( 'titles', 'books-list' );\n }\n} );" + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rest_api_012' ) ) { return false; } $links = wpbp_rest_api_012( 'https://example.test/item' ); return isset( $links['https://api.w.org/related'][0]['href'] ) && 'https://example.test/item' === $links['https://api.w.org/related'][0]['href'];", + "description": "The response exposes the custom related link", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rest_api_012( string $url ): array { $response = new WP_REST_Response( array( 'ok' => true ) ); $response->add_link( 'https://api.w.org/related', $url ); return $response->get_links(); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rest-api.php", + "src/wp-includes/rest-api/class-wp-rest-request.php" + ], + "release_focus": "classic" + } } ] } diff --git a/datasets/suites/wp-core-v1/execution/rewrite-permalinks.json b/datasets/suites/wp-core-v1/execution/rewrite-permalinks.json new file mode 100644 index 0000000..97e9eb8 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/rewrite-permalinks.json @@ -0,0 +1,226 @@ +{ + "id": "wp-core-execution-v1-rewrite_permalinks", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Rewrite and Permalinks", + "description": "Code generation tasks for WordPress Rewrite and Permalinks APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-rewrite-permalinks-001", + "category": "rewrite-permalinks", + "difficulty": "intermediate", + "prompt": "Provide a PHP function named wpbp_rewrite_permalinks_001(): void that registers item URLs under items/ as an extra permalink structure using a %wpbp_item% placeholder.", + "expected_behavior": "Reviewer contract: The helper must register the %wpbp_item% rewrite tag, expose the wpbp_item query variable, and add a wpbp_item permastruct whose structure contains items/%wpbp_item%. Review focus: Use WordPress rewrite APIs and do not flush rewrite rules during normal runtime. The runtime inspects WordPress rewrite state after calling the helper rather than trusting its return value.", + "requirements": [ + "Use rewrite APIs", + "Register the wpbp_item query variable", + "Avoid flushing rewrite rules during normal runtime" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rewrite_permalinks_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_rewrite_tag", + "description": "Uses add_rewrite_tag", + "weight": 1 + }, + { + "pattern": "add_permastruct", + "description": "Uses add_permastruct", + "weight": 1 + }, + { + "pattern": "%wpbp_item%", + "description": "Registers the requested rewrite tag", + "weight": 0.5 + }, + { + "pattern": "items/%wpbp_item%", + "description": "Registers the requested item permalink structure", + "weight": 0.5 + } + ], + "forbidden_patterns": [ + { + "pattern": "flush_rewrite_rules\\s*\\(", + "description": "Does not flush rewrite rules during normal runtime", + "severity": "error" + }, + { + "pattern": "->\\s*flush_rules\\s*\\(", + "description": "Does not flush rewrite rules during normal runtime", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rewrite_permalinks_001' ) ) { return false; } global $wp, $wp_rewrite; wpbp_rewrite_permalinks_001(); $tag_index = array_search( '%wpbp_item%', $wp_rewrite->rewritecode, true ); if ( false === $tag_index ) { return false; } $regex = $wp_rewrite->rewritereplace[ $tag_index ] ?? ''; $pattern = '~^' . str_replace( '~', '\\\\~', $regex ) . '$~'; $accepts_slug = 1 === @preg_match( $pattern, 'alpha-1' ); $rejects_nested_path = 1 !== @preg_match( $pattern, 'alpha/beta' ); $query_replace = $wp_rewrite->queryreplace[ $tag_index ] ?? ''; $struct = $wp_rewrite->extra_permastructs['wpbp_item']['struct'] ?? ''; return $accepts_slug && $rejects_nested_path && 'wpbp_item=' === $query_replace && in_array( 'wpbp_item', $wp->public_query_vars, true ) && str_contains( $struct, 'items/%wpbp_item%' );", + "description": "The helper registers the item rewrite tag, query var, and permastruct in WordPress", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rewrite_permalinks_001(): void { add_rewrite_tag( '%wpbp_item%', '([^/]+)' ); add_permastruct( 'wpbp_item', 'items/%wpbp_item%', array( 'with_front' => false ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rewrite.php", + "src/wp-includes/class-wp.php", + "src/wp-includes/class-wp-rewrite.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rewrite-permalinks-002", + "category": "rewrite-permalinks", + "difficulty": "basic", + "prompt": "Provide a PHP function named wpbp_rewrite_permalinks_002(): void that registers a top-priority rewrite rule for URLs like wpbp-rule/, mapping the captured code to the wpbp_rule query variable.", + "expected_behavior": "Reviewer contract: The helper must add a rewrite rule to WordPress' top rewrite-rule bucket for wpbp-rule/ URLs and map the first path segment to the wpbp_rule query variable. Review focus: Use the Rewrite API, use top priority, and do not flush rewrite rules during normal runtime. The runtime inspects $wp_rewrite->extra_rules_top after calling the helper.", + "requirements": [ + "Use rewrite APIs", + "Register the rule at top priority", + "Avoid flushing rewrite rules during normal runtime" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rewrite_permalinks_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_rewrite_rule", + "description": "Uses add_rewrite_rule", + "weight": 1 + }, + { + "pattern": "wpbp-rule", + "description": "Targets the requested URL prefix", + "weight": 0.5 + }, + { + "pattern": "wpbp_rule", + "description": "Maps to the requested query variable", + "weight": 0.5 + }, + { + "pattern": "top", + "description": "Adds the rule at top priority", + "weight": 0.5 + }, + { + "pattern": "\\$matches\\[1\\]", + "description": "Uses the first regex capture in the query mapping", + "weight": 0.5 + } + ], + "forbidden_patterns": [ + { + "pattern": "flush_rewrite_rules\\s*\\(", + "description": "Does not flush rewrite rules during normal runtime", + "severity": "error" + }, + { + "pattern": "->\\s*flush_rules\\s*\\(", + "description": "Does not flush rewrite rules during normal runtime", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rewrite_permalinks_002' ) ) { return false; } global $wp_rewrite; wpbp_rewrite_permalinks_002(); foreach ( $wp_rewrite->extra_rules_top as $regex => $query ) { $pattern = '~' . str_replace( '~', '\\\\~', $regex ) . '~'; if ( 1 !== @preg_match( $pattern, 'wpbp-rule/abc-123' ) ) { continue; } if ( is_array( $query ) ) { if ( isset( $query['wpbp_rule'] ) && '$matches[1]' === $query['wpbp_rule'] ) { return true; } continue; } if ( is_string( $query ) && str_contains( $query, 'wpbp_rule=$matches[1]' ) ) { return true; } } return false;", + "description": "The helper registers a top rewrite rule that captures wpbp-rule URLs into wpbp_rule", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rewrite_permalinks_002(): void { add_rewrite_rule( '^wpbp-rule/([^/]+)/?', 'index.php?wpbp_rule=$matches[1]', 'top' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/rewrite.php", + "src/wp-includes/class-wp-rewrite.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-rewrite-permalinks-003", + "category": "rewrite-permalinks", + "difficulty": "basic", + "prompt": "Provide a PHP function named wpbp_rewrite_permalinks_003(): void that makes wpbp_var available as a public query variable without removing existing public query vars.", + "expected_behavior": "Reviewer contract: The helper must hook query_vars, append wpbp_var, and preserve the incoming public query vars array. Review focus: Use WordPress hook/query-var APIs and do not flush rewrite rules during normal runtime. The runtime applies the filter to a seed array and verifies both the new and existing vars remain present.", + "requirements": [ + "Use the query_vars filter", + "Preserve existing public query vars", + "Avoid flushing rewrite rules during normal runtime" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_rewrite_permalinks_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "query_vars", + "description": "Uses query_vars", + "weight": 1 + }, + { + "pattern": "add_filter", + "description": "Uses add_filter", + "weight": 1 + }, + { + "pattern": "wpbp_var", + "description": "Adds the requested public query var", + "weight": 0.5 + } + ], + "forbidden_patterns": [ + { + "pattern": "flush_rewrite_rules\\s*\\(", + "description": "Does not flush rewrite rules during normal runtime", + "severity": "error" + }, + { + "pattern": "->\\s*flush_rules\\s*\\(", + "description": "Does not flush rewrite rules during normal runtime", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_rewrite_permalinks_003' ) ) { return false; } wpbp_rewrite_permalinks_003(); $vars = apply_filters( 'query_vars', array( 'p', 'page_id' ) ); return in_array( 'wpbp_var', $vars, true ) && in_array( 'p', $vars, true ) && in_array( 'page_id', $vars, true ) && 1 === count( array_keys( $vars, 'wpbp_var', true ) );", + "description": "The helper appends wpbp_var without dropping existing query vars", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_rewrite_permalinks_003(): void { add_filter( 'query_vars', fn( $vars ) => array_merge( $vars, array( 'wpbp_var' ) ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/class-wp.php", + "src/wp-includes/plugin.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/roles-caps.json b/datasets/suites/wp-core-v1/execution/roles-caps.json new file mode 100644 index 0000000..83573ae --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/roles-caps.json @@ -0,0 +1,443 @@ +{ + "id": "wp-core-execution-v1-roles_caps", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Roles and Capabilities", + "description": "Code generation tasks for WordPress Roles and Capabilities APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-roles-caps-001", + "category": "roles-caps", + "difficulty": "basic", + "prompt": "Implement wpbp_roles_caps_001() to register a role named wpbp_manager with the read and wpbp_manage capabilities, then report whether WordPress can retrieve that role with the custom capability.", + "expected_behavior": "Reviewer contract: The function should use the roles API to register wpbp_manager and verify the role through WordPress role helpers. The runtime checks call the function, inspect the registered role, and remove it during teardown.", + "requirements": [ + "Use WordPress roles/capabilities APIs", + "Verify the role through WordPress helpers" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_roles_caps_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_role", + "description": "Registers a role through the WordPress API", + "weight": 1 + }, + { + "pattern": "get_role", + "description": "Retrieves the role through the WordPress API", + "weight": 1 + }, + { + "pattern": "has_cap", + "description": "Checks the role capability through a helper", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_roles_caps_001' ) ) { return false; } $returned = wpbp_roles_caps_001(); $role = get_role( 'wpbp_manager' ); return (bool) $returned && $role instanceof WP_Role && $role->has_cap( 'read' ) && $role->has_cap( 'wpbp_manage' );", + "description": "wpbp_manager is registered with the requested capabilities", + "weight": 1 + } + ], + "teardown": "remove_role( 'wpbp_manager' );" + }, + "reference_solution": "function wpbp_roles_caps_001(): bool { add_role( 'wpbp_manager', 'WPBP Manager', array( 'read' => true, 'wpbp_manage' => true ) ); $role = get_role( 'wpbp_manager' ); return $role instanceof WP_Role && $role->has_cap( 'read' ) && $role->has_cap( 'wpbp_manage' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/capabilities.php", + "src/wp-includes/class-wp-roles.php", + "src/wp-includes/class-wp-role.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-roles-caps-002", + "category": "roles-caps", + "difficulty": "basic", + "prompt": "Implement wpbp_roles_caps_002() to add wpbp_temp_cap to the Subscriber role, verify the role has it, remove it again, and report whether both transitions succeeded.", + "expected_behavior": "Reviewer contract: The function should mutate the Subscriber role through WP_Role methods and leave the temporary capability removed. The runtime checks both the returned result and the final Subscriber role state.", + "requirements": [ + "Use WordPress roles/capabilities APIs", + "Leave the Subscriber role without the temporary capability" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_roles_caps_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "get_role", + "description": "Retrieves the Subscriber role through the WordPress API", + "weight": 1 + }, + { + "pattern": "add_cap", + "description": "Adds a capability through the role API", + "weight": 1 + }, + { + "pattern": "remove_cap", + "description": "Removes the temporary capability through the role API", + "weight": 1 + }, + { + "pattern": "has_cap", + "description": "Verifies capability state through a helper", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_roles_caps_002' ) ) { return false; } $ok = wpbp_roles_caps_002(); $role = get_role( 'subscriber' ); return (bool) $ok && $role instanceof WP_Role && ! $role->has_cap( 'wpbp_temp_cap' );", + "description": "The temporary Subscriber capability is added, observed, and removed", + "weight": 1 + } + ], + "teardown": "$role = get_role( 'subscriber' ); if ( $role instanceof WP_Role ) { $role->remove_cap( 'wpbp_temp_cap' ); }" + }, + "reference_solution": "function wpbp_roles_caps_002(): bool { $role = get_role( 'subscriber' ); if ( ! $role instanceof WP_Role ) { return false; } $role->add_cap( 'wpbp_temp_cap' ); $had_cap = $role->has_cap( 'wpbp_temp_cap' ); $role->remove_cap( 'wpbp_temp_cap' ); return $had_cap && ! $role->has_cap( 'wpbp_temp_cap' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/capabilities.php", + "src/wp-includes/class-wp-roles.php", + "src/wp-includes/class-wp-role.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-roles-caps-003", + "category": "roles-caps", + "difficulty": "intermediate", + "prompt": "Implement wpbp_roles_caps_003( int $user_id, string $cap ) to switch to the given user and report whether the current user can perform the requested capability.", + "expected_behavior": "Reviewer contract: The function should set the current user to the supplied user ID and delegate the capability check to WordPress. The runtime uses a Subscriber fixture to verify a granted capability and a denied capability, then deletes the user and resets the current user.", + "requirements": [ + "Use WordPress roles/capabilities APIs", + "Check the capability as the current user after switching" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_roles_caps_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_set_current_user", + "description": "Switches the current user", + "weight": 1 + }, + { + "pattern": "current_user_can", + "description": "Checks the current user's capability", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_roles_caps_003_user_id'] = wp_insert_user( array( 'user_login' => 'wpbp_cap_user_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'subscriber' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "$user_id = $GLOBALS['wpbp_roles_caps_003_user_id'] ?? 0; if ( ! function_exists( 'wpbp_roles_caps_003' ) || ! is_int( $user_id ) ) { return false; } return wpbp_roles_caps_003( $user_id, 'read' ) && ! wpbp_roles_caps_003( $user_id, 'manage_options' );", + "description": "A switched Subscriber can read but cannot manage options", + "weight": 1 + } + ], + "teardown": "$user_id = $GLOBALS['wpbp_roles_caps_003_user_id'] ?? 0; if ( is_int( $user_id ) ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); } wp_set_current_user( 0 ); unset( $GLOBALS['wpbp_roles_caps_003_user_id'] );" + }, + "reference_solution": "function wpbp_roles_caps_003( int $user_id, string $cap ): bool { wp_set_current_user( $user_id ); return current_user_can( $cap ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/capabilities.php", + "src/wp-includes/class-wp-user.php", + "src/wp-includes/pluggable.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-roles-caps-004", + "category": "roles-caps", + "difficulty": "intermediate", + "prompt": "Implement wpbp_roles_caps_004( int $user_id, int $post_id ) to return the primitive capabilities WordPress requires for that user to edit that post.", + "expected_behavior": "Reviewer contract: The function should use WordPress meta-cap mapping for edit_post. The runtime creates an Author-owned draft post and expects WordPress to map the edit request to edit_posts, then removes the post and user.", + "requirements": [ + "Use WordPress roles/capabilities APIs", + "Return the primitive capabilities from WordPress meta-cap mapping" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_roles_caps_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "map_meta_cap", + "description": "Uses WordPress meta-cap mapping", + "weight": 1 + }, + { + "pattern": "edit_post", + "description": "Maps the edit_post meta capability", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_roles_caps_004_user_id'] = wp_insert_user( array( 'user_login' => 'wpbp_meta_cap_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'author' ) ); if ( is_int( $GLOBALS['wpbp_roles_caps_004_user_id'] ) ) { $GLOBALS['wpbp_roles_caps_004_post_id'] = wp_insert_post( array( 'post_title' => 'WPBP owned draft', 'post_status' => 'draft', 'post_author' => $GLOBALS['wpbp_roles_caps_004_user_id'] ) ); }", + "assertions": [ + { + "type": "custom_assertion", + "code": "$user_id = $GLOBALS['wpbp_roles_caps_004_user_id'] ?? 0; $post_id = $GLOBALS['wpbp_roles_caps_004_post_id'] ?? 0; if ( ! function_exists( 'wpbp_roles_caps_004' ) || ! is_int( $user_id ) || ! is_int( $post_id ) ) { return false; } $caps = wpbp_roles_caps_004( $user_id, $post_id ); return is_array( $caps ) && array_values( $caps ) === array( 'edit_posts' );", + "description": "Editing an Author-owned draft maps to edit_posts", + "weight": 1 + } + ], + "teardown": "$post_id = $GLOBALS['wpbp_roles_caps_004_post_id'] ?? 0; if ( is_int( $post_id ) ) { wp_delete_post( $post_id, true ); } $user_id = $GLOBALS['wpbp_roles_caps_004_user_id'] ?? 0; if ( is_int( $user_id ) ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); } unset( $GLOBALS['wpbp_roles_caps_004_post_id'], $GLOBALS['wpbp_roles_caps_004_user_id'] );" + }, + "reference_solution": "function wpbp_roles_caps_004( int $user_id, int $post_id ): array { return map_meta_cap( 'edit_post', $user_id, $post_id ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/capabilities.php", + "src/wp-includes/post.php", + "src/wp-includes/class-wp-user.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-roles-caps-005", + "category": "roles-caps", + "difficulty": "basic", + "prompt": "Implement wpbp_roles_caps_005( int $user_id, string $cap ) to grant the capability directly to that user and return whether WordPress recognizes the grant.", + "expected_behavior": "Reviewer contract: The function should add a direct user capability through WP_User and verify it with WordPress capability checks. The runtime confirms both the return value and the user's observable capability before deleting the fixture user.", + "requirements": [ + "Use the WP_User capability API", + "Verify the grant with WordPress capability checks" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_roles_caps_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_User", + "description": "Uses the WP_User API", + "weight": 1 + }, + { + "pattern": "add_cap", + "description": "Adds a direct user capability", + "weight": 1 + }, + { + "pattern": "user_can", + "description": "Verifies the user capability through WordPress", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_roles_caps_005_user_id'] = wp_insert_user( array( 'user_login' => 'wpbp_direct_cap_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'subscriber' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "$user_id = $GLOBALS['wpbp_roles_caps_005_user_id'] ?? 0; if ( ! function_exists( 'wpbp_roles_caps_005' ) || ! is_int( $user_id ) ) { return false; } $ok = wpbp_roles_caps_005( $user_id, 'wpbp_direct' ); return (bool) $ok && user_can( $user_id, 'wpbp_direct' );", + "description": "The direct user capability is granted and recognized", + "weight": 1 + } + ], + "teardown": "$user_id = $GLOBALS['wpbp_roles_caps_005_user_id'] ?? 0; if ( is_int( $user_id ) ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); } unset( $GLOBALS['wpbp_roles_caps_005_user_id'] );" + }, + "reference_solution": "function wpbp_roles_caps_005( int $user_id, string $cap ): bool { $user = new WP_User( $user_id ); $user->add_cap( $cap ); return user_can( $user_id, $cap ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/capabilities.php", + "src/wp-includes/class-wp-user.php", + "src/wp-includes/pluggable.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-roles-caps-006", + "category": "roles-caps", + "difficulty": "intermediate", + "prompt": "Implement wpbp_roles_caps_006() to allow Editor in the default new-user role dropdown while Administrator remains excluded.", + "expected_behavior": "Reviewer contract: The function should register a callback on the WP 7.0 default_role_dropdown_excluded_roles filter that removes editor from the excluded-role list while preserving administrator. The runtime calls the function and applies the filter to the core default exclusions.", + "requirements": [ + "Use the default-role excluded roles filter", + "Register the filter inside the function" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_roles_caps_006", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_filter", + "description": "Registers a filter callback", + "weight": 1 + }, + { + "pattern": "default_role_dropdown_excluded_roles", + "description": "Uses the default-role exclusion filter", + "weight": 1 + }, + { + "pattern": "editor", + "description": "Adjusts the editor exclusion", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_roles_caps_006' ) ) { return false; } wpbp_roles_caps_006(); $excluded = apply_filters( 'default_role_dropdown_excluded_roles', array( 'administrator', 'editor' ) ); return in_array( 'administrator', $excluded, true ) && ! in_array( 'editor', $excluded, true );", + "description": "The filter preserves Administrator exclusion while allowing Editor", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_roles_caps_006(): void { add_filter( 'default_role_dropdown_excluded_roles', function ( array $excluded_roles ): array { return array_values( array_diff( $excluded_roles, array( 'editor' ) ) ); } ); }", + "metadata": { + "source_refs": [ + "src/wp-admin/options-general.php", + "src/wp-includes/plugin.php", + "src/wp-includes/capabilities.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-roles-caps-007", + "category": "roles-caps", + "difficulty": "basic", + "prompt": "Implement wpbp_roles_caps_007( int $user_id, string $cap ) to report whether that specific user has the requested capability without changing the current user.", + "expected_behavior": "Reviewer contract: The function should check the supplied user's capability with user_can rather than switching global current-user state. The runtime verifies both a granted and denied capability for a Subscriber and checks that the current user is unchanged.", + "requirements": [ + "Use WordPress roles/capabilities APIs", + "Do not switch the current user" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_roles_caps_007", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "user_can", + "description": "Checks a specific user's capability", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "wp_set_current_user", + "description": "Should not switch current-user state for a specific-user check", + "severity": "error" + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_roles_caps_007_user_id'] = wp_insert_user( array( 'user_login' => 'wpbp_user_can_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'subscriber' ) );", + "assertions": [ + { + "type": "custom_assertion", + "code": "$user_id = $GLOBALS['wpbp_roles_caps_007_user_id'] ?? 0; if ( ! function_exists( 'wpbp_roles_caps_007' ) || ! is_int( $user_id ) ) { return false; } $before = get_current_user_id(); $ok = wpbp_roles_caps_007( $user_id, 'read' ) && ! wpbp_roles_caps_007( $user_id, 'manage_options' ); return $ok && get_current_user_id() === $before;", + "description": "A specific Subscriber can read, cannot manage options, and current user is unchanged", + "weight": 1 + } + ], + "teardown": "$user_id = $GLOBALS['wpbp_roles_caps_007_user_id'] ?? 0; if ( is_int( $user_id ) ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); } unset( $GLOBALS['wpbp_roles_caps_007_user_id'] );" + }, + "reference_solution": "function wpbp_roles_caps_007( int $user_id, string $cap ): bool { return user_can( $user_id, $cap ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/capabilities.php", + "src/wp-includes/class-wp-user.php", + "src/wp-includes/pluggable.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-roles-caps-008", + "category": "roles-caps", + "difficulty": "intermediate", + "prompt": "Implement wpbp_roles_caps_008( int $user_id, int $post_id ) to return the primitive capabilities WordPress requires for that user to delete that post.", + "expected_behavior": "Reviewer contract: The function should use WordPress meta-cap mapping for delete_post. The runtime creates an Author-owned published post and expects WordPress to map the delete request to delete_published_posts, then removes the post and user.", + "requirements": [ + "Use WordPress roles/capabilities APIs", + "Return the primitive capabilities from WordPress meta-cap mapping" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_roles_caps_008", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "map_meta_cap", + "description": "Uses WordPress meta-cap mapping", + "weight": 1 + }, + { + "pattern": "delete_post", + "description": "Maps the delete_post meta capability", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "$GLOBALS['wpbp_roles_caps_008_user_id'] = wp_insert_user( array( 'user_login' => 'wpbp_delete_cap_' . wp_rand(), 'user_pass' => wp_generate_password(), 'role' => 'author' ) ); if ( is_int( $GLOBALS['wpbp_roles_caps_008_user_id'] ) ) { $GLOBALS['wpbp_roles_caps_008_post_id'] = wp_insert_post( array( 'post_title' => 'WPBP owned published post', 'post_status' => 'publish', 'post_author' => $GLOBALS['wpbp_roles_caps_008_user_id'] ) ); }", + "assertions": [ + { + "type": "custom_assertion", + "code": "$user_id = $GLOBALS['wpbp_roles_caps_008_user_id'] ?? 0; $post_id = $GLOBALS['wpbp_roles_caps_008_post_id'] ?? 0; if ( ! function_exists( 'wpbp_roles_caps_008' ) || ! is_int( $user_id ) || ! is_int( $post_id ) ) { return false; } $caps = wpbp_roles_caps_008( $user_id, $post_id ); return is_array( $caps ) && array_values( $caps ) === array( 'delete_published_posts' );", + "description": "Deleting an Author-owned published post maps to delete_published_posts", + "weight": 1 + } + ], + "teardown": "$post_id = $GLOBALS['wpbp_roles_caps_008_post_id'] ?? 0; if ( is_int( $post_id ) ) { wp_delete_post( $post_id, true ); } $user_id = $GLOBALS['wpbp_roles_caps_008_user_id'] ?? 0; if ( is_int( $user_id ) ) { require_once ABSPATH . 'wp-admin/includes/user.php'; wp_delete_user( $user_id ); } unset( $GLOBALS['wpbp_roles_caps_008_post_id'], $GLOBALS['wpbp_roles_caps_008_user_id'] );" + }, + "reference_solution": "function wpbp_roles_caps_008( int $user_id, int $post_id ): array { return map_meta_cap( 'delete_post', $user_id, $post_id ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/capabilities.php", + "src/wp-includes/post.php", + "src/wp-includes/class-wp-user.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/scripts-styles.json b/datasets/suites/wp-core-v1/execution/scripts-styles.json new file mode 100644 index 0000000..732bd0a --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/scripts-styles.json @@ -0,0 +1,345 @@ +{ + "id": "wp-core-execution-v1-scripts_styles", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Scripts and Styles", + "description": "Code generation tasks for WordPress Scripts and Styles APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-scripts-styles-001", + "category": "scripts-styles", + "difficulty": "intermediate", + "prompt": "Implement wpbp_scripts_styles_001(): void to register the script handle 'wpbp-script' at 'https://example.test/script.js' with 'jquery' as its dependency, version '1.0.0', and enqueue it for footer output.", + "expected_behavior": "`wpbp_scripts_styles_001()` registers `wpbp-script` through WordPress's script API, preserves the requested source, dependency, version, and footer placement, then queues the handle. Runtime validation calls the function and inspects the `WP_Scripts` registration and queue state.", + "requirements": [ + "Use wp_register_script() and wp_enqueue_script()", + "Register the requested handle, source, dependency, version, and footer placement", + "Do not print or return hard-coded script tags" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_scripts_styles_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_register_script", + "description": "Uses wp_register_script", + "weight": 1 + }, + { + "pattern": "wp_enqueue_script", + "description": "Uses wp_enqueue_script", + "weight": 1 + }, + { + "pattern": "wpbp-script", + "description": "Uses the requested script handle", + "weight": 1 + }, + { + "pattern": "jquery", + "description": "Declares the jQuery dependency", + "weight": 1 + }, + { + "pattern": "in_footer", + "description": "Requests footer output through script args", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "/registered['wpbp-script'] ?? null; return $registered instanceof \\_WP_Dependency && wp_script_is( 'wpbp-script', 'enqueued' ) && 'https://example.test/script.js' === $registered->src && array( 'jquery' ) === $registered->deps && '1.0.0' === $registered->ver && 1 === (int) $scripts->get_data( 'wpbp-script', 'group' );", + "description": "The script is registered with the requested metadata and queued for footer output", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_scripts_styles_001(): void { wp_register_script( 'wpbp-script', 'https://example.test/script.js', array( 'jquery' ), '1.0.0', array( 'in_footer' => true ) ); wp_enqueue_script( 'wpbp-script' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/functions.wp-scripts.php", + "src/wp-includes/class-wp-scripts.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-scripts-styles-002", + "category": "scripts-styles", + "difficulty": "intermediate", + "prompt": "Implement wpbp_scripts_styles_002(): void to register and enqueue the script handle 'wpbp-inline', then attach the inline JavaScript `window.wpbpInline = true;` after that handle.", + "expected_behavior": "`wpbp_scripts_styles_002()` prepares a queued WordPress script handle and attaches the requested inline code in the `after` position with `wp_add_inline_script()`. Runtime validation checks the queued handle and the inline data stored on `WP_Scripts`.", + "requirements": [ + "Use WordPress script APIs for the handle", + "Use wp_add_inline_script() with the after position", + "Do not print literal script tags" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_scripts_styles_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_add_inline_script", + "description": "Uses wp_add_inline_script", + "weight": 1 + }, + { + "pattern": "wp_register_script", + "description": "Registers the script handle", + "weight": 1 + }, + { + "pattern": "wp_enqueue_script", + "description": "Enqueues the script handle", + "weight": 1 + }, + { + "pattern": "wpbp-inline", + "description": "Uses the requested script handle", + "weight": 1 + }, + { + "pattern": "window\\.wpbpInline\\s*=\\s*true", + "description": "Adds the requested inline JavaScript", + "weight": 1 + }, + { + "pattern": "after", + "description": "Adds inline code after the script handle", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "/get_data( 'wpbp-inline', 'after' ); return wp_script_is( 'wpbp-inline', 'enqueued' ) && is_array( $after ) && in_array( 'window.wpbpInline = true;', $after, true );", + "description": "The handle is queued and has the requested inline code attached after it", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_scripts_styles_002(): void { wp_register_script( 'wpbp-inline', false, array(), null, array( 'in_footer' => true ) ); wp_enqueue_script( 'wpbp-inline' ); wp_add_inline_script( 'wpbp-inline', 'window.wpbpInline = true;', 'after' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/functions.wp-scripts.php", + "src/wp-includes/class-wp-scripts.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-scripts-styles-003", + "category": "scripts-styles", + "difficulty": "intermediate", + "prompt": "Implement wpbp_scripts_styles_003(): void to register the script module ID 'wpbp/module' at 'https://example.test/module.js' with version '1.0.0', then enqueue that module.", + "expected_behavior": "`wpbp_scripts_styles_003()` uses the Script Modules API to register `wpbp/module` with the requested source and version, then marks the module for enqueue. Runtime validation inspects the `WP_Script_Modules` registry and queue.", + "requirements": [ + "Use wp_register_script_module() and wp_enqueue_script_module()", + "Register and enqueue the requested module ID", + "Do not print module script tags manually" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_scripts_styles_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_register_script_module", + "description": "Uses wp_register_script_module", + "weight": 1 + }, + { + "pattern": "wp_enqueue_script_module", + "description": "Uses wp_enqueue_script_module", + "weight": 1 + }, + { + "pattern": "wpbp/module", + "description": "Uses the requested script module ID", + "weight": 1 + }, + { + "pattern": "https://example\\.test/module\\.js", + "description": "Registers the requested module source", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "/get_registered( 'wpbp/module' ); return is_array( $registered ) && in_array( 'wpbp/module', $modules->get_queue(), true ) && 'https://example.test/module.js' === $registered['src'] && '1.0.0' === $registered['version'];", + "description": "The script module is registered with the requested source and queued", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_scripts_styles_003(): void { wp_register_script_module( 'wpbp/module', 'https://example.test/module.js', array(), '1.0.0' ); wp_enqueue_script_module( 'wpbp/module' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/script-modules.php", + "src/wp-includes/class-wp-script-modules.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-scripts-styles-004", + "category": "scripts-styles", + "difficulty": "intermediate", + "prompt": "Implement wpbp_scripts_styles_004(): void to register the script module ID 'wpbp/dep-module' at 'https://example.test/dep.js', then register the classic script handle 'wpbp-script-with-module' at 'https://example.test/app.js' with that module listed in its module dependencies and footer output enabled.", + "expected_behavior": "`wpbp_scripts_styles_004()` registers a script module and a classic script whose `module_dependencies` data points at that module. Because classic scripts with module dependencies need the import map available first, runtime validation also checks that the classic script is registered for footer output.", + "requirements": [ + "Use wp_register_script_module() for the module", + "Use wp_register_script() with module_dependencies for the classic script", + "Enable footer output for the classic script" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_scripts_styles_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "module_dependencies", + "description": "Uses module_dependencies", + "weight": 1 + }, + { + "pattern": "wp_register_script_module", + "description": "Registers the module dependency", + "weight": 1 + }, + { + "pattern": "wp_register_script", + "description": "Uses wp_register_script", + "weight": 1 + }, + { + "pattern": "wpbp/dep-module", + "description": "Uses the requested module dependency ID", + "weight": 1 + }, + { + "pattern": "wpbp-script-with-module", + "description": "Uses the requested classic script handle", + "weight": 1 + }, + { + "pattern": "in_footer", + "description": "Requests footer output for the classic script", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "/registered['wpbp-script-with-module'] ?? null; $module = wp_script_modules()->get_registered( 'wpbp/dep-module' ); $module_dependencies = $scripts->get_data( 'wpbp-script-with-module', 'module_dependencies' ); $has_module_dependency = is_array( $module_dependencies ) && ( in_array( 'wpbp/dep-module', $module_dependencies, true ) || array_filter( $module_dependencies, static fn( $dependency ) => is_array( $dependency ) && 'wpbp/dep-module' === ( $dependency['id'] ?? null ) ) ); return $registered instanceof \\_WP_Dependency && is_array( $module ) && 'https://example.test/dep.js' === $module['src'] && 'https://example.test/app.js' === $registered->src && true === (bool) $has_module_dependency && 1 === (int) $scripts->get_data( 'wpbp-script-with-module', 'group' );", + "description": "The classic script is registered with the requested module dependency and footer placement", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_scripts_styles_004(): void { wp_register_script_module( 'wpbp/dep-module', 'https://example.test/dep.js' ); wp_register_script( 'wpbp-script-with-module', 'https://example.test/app.js', array(), '1.0.0', array( 'in_footer' => true, 'module_dependencies' => array( 'wpbp/dep-module' ) ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/functions.wp-scripts.php", + "src/wp-includes/class-wp-scripts.php", + "src/wp-includes/script-modules.php", + "src/wp-includes/class-wp-script-modules.php" + ], + "release_focus": "7.0" + } + }, + { + "id": "e-scripts-styles-005", + "category": "scripts-styles", + "difficulty": "basic", + "prompt": "Implement wpbp_scripts_styles_005( string $dataset_name ): ?string to return the HTML custom data attribute name WordPress would use for the given JavaScript dataset property name.", + "expected_behavior": "`wpbp_scripts_styles_005()` delegates dataset-property conversion to WordPress and returns `null` when the supplied property name cannot be represented as an HTML custom data attribute. Runtime validation checks camelCase, uppercase-leading, empty-string, and invalid-name cases from the core helper's contract.", + "requirements": [ + "Use wp_html_custom_data_attribute_name()", + "Return the helper result without hand-written conversion rules" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_scripts_styles_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_html_custom_data_attribute_name", + "description": "Uses wp_html_custom_data_attribute_name", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_scripts_styles_005' ) ) { return false; } return 'data-post-id' === wpbp_scripts_styles_005( 'postId' ) && 'data--before' === wpbp_scripts_styles_005( 'Before' ) && 'data---one---two---' === wpbp_scripts_styles_005( '-One--Two---' ) && 'data-' === wpbp_scripts_styles_005( '' ) && null === wpbp_scripts_styles_005( '/not-an-attribute/' ) && null === wpbp_scripts_styles_005( 'no spaces' );", + "description": "Dataset property names are converted or rejected according to WordPress core behavior", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_scripts_styles_005( string $prop ): ?string { return wp_html_custom_data_attribute_name( $prop ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/script-loader.php" + ], + "release_focus": "6.9" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/security.json b/datasets/suites/wp-core-v1/execution/security.json new file mode 100644 index 0000000..092bb5b --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/security.json @@ -0,0 +1,267 @@ +{ + "id": "wp-core-execution-v1-security", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Security", + "description": "Code generation tasks for WordPress Security APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-security-001", + "category": "security", + "difficulty": "intermediate", + "prompt": "Implement `wpbp_security_001( string $html, string $href ): string` to update the first anchor in an HTML fragment with the supplied href and return the updated HTML.", + "expected_behavior": "`wpbp_security_001()` uses `WP_HTML_Tag_Processor` to find the first anchor and write the href through `set_attribute()`, so URI values are escaped by WordPress's HTML API. It should return `get_updated_html()`, leave fragments without an anchor unchanged, and avoid string replacement or concatenated attributes.", + "requirements": [ + "Use WP_HTML_Tag_Processor", + "Set the first anchor's href through the processor", + "Return the updated HTML" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_security_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_HTML_Tag_Processor", + "description": "Uses WP_HTML_Tag_Processor", + "weight": 1 + }, + { + "pattern": "next_tag", + "description": "Finds the anchor through the HTML API", + "weight": 1 + }, + { + "pattern": "set_attribute", + "description": "Uses set_attribute", + "weight": 1 + }, + { + "pattern": "get_updated_html", + "description": "Returns the processor's updated HTML", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_security_001' ) ) { return false; } $updated = wpbp_security_001( '

Go

', 'https://example.test/?q=Tom&name=Jerry' ); $rejected = wpbp_security_001( 'Go', 'javascript:alert(1)' ); $plain = wpbp_security_001( 'No link', 'https://example.test/' ); return '

Go

' === $updated && 'Go' === $rejected && 'No link' === $plain;", + "description": "The first anchor href is updated with WordPress escaping, unsafe hrefs are rejected, and non-anchor fragments are unchanged", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_security_001( string $html, string $href ): string { $processor = new WP_HTML_Tag_Processor( $html ); if ( $processor->next_tag( 'a' ) ) { $processor->set_attribute( 'href', $href ); } return $processor->get_updated_html(); }", + "metadata": { + "source_refs": [ + "src/wp-includes/html-api/class-wp-html-tag-processor.php" + ], + "release_focus": "6.9" + } + }, + { + "id": "e-security-002", + "category": "security", + "difficulty": "intermediate", + "prompt": "Implement `wpbp_security_002( string $html ): string` to return the normalized serialization of the first token in a WordPress HTML fragment.", + "expected_behavior": "`wpbp_security_002()` creates a fragment `WP_HTML_Processor`, advances to the first token, and returns `serialize_token()`. The result should be only the current token's normalized serialization, not the full fragment or raw input.", + "requirements": [ + "Use WP_HTML_Processor::create_fragment", + "Advance to the first token", + "Return serialize_token output" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_security_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "WP_HTML_Processor", + "description": "Uses WP_HTML_Processor", + "weight": 1 + }, + { + "pattern": "create_fragment", + "description": "Creates a fragment processor", + "weight": 1 + }, + { + "pattern": "next_token", + "description": "Advances to the first token", + "weight": 1 + }, + { + "pattern": "serialize_token", + "description": "Uses serialize_token", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_security_002' ) ) { return false; } $tag = wpbp_security_002( '

Hi

' ); $text = wpbp_security_002( '5 < 8 & ok' ); $empty = wpbp_security_002( '' ); return '

' === $tag && '5 < 8 & ' === $text && '' === $empty;", + "description": "Only the first token is serialized, with WordPress normalization for tags, attributes, text, and empty fragments", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_security_002( string $html ): string { $processor = WP_HTML_Processor::create_fragment( $html ); $processor->next_token(); return $processor->serialize_token(); }", + "metadata": { + "source_refs": [ + "src/wp-includes/html-api/class-wp-html-processor.php" + ], + "release_focus": "6.9" + } + }, + { + "id": "e-security-003", + "category": "security", + "difficulty": "basic", + "prompt": "Implement `wpbp_security_003( string $html ): string` to filter a chunk of post content so only HTML allowed for WordPress post content remains.", + "expected_behavior": "`wpbp_security_003()` delegates to `wp_kses_post()`, preserving allowed post-content markup while removing disallowed tags and attributes. The test focuses on KSES post-context behavior, not generic escaping or hand-built allowlists.", + "requirements": [ + "Use KSES's post-content allowlist", + "Return the filtered HTML string" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_security_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "wp_kses_post", + "description": "Uses wp_kses_post", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_security_003' ) ) { return false; } $out = wpbp_security_003( '

Safe

' ); return str_contains( $out, '

Safe

' ) && str_contains( $out, 'alert(1)' ) && ! str_contains( $out, 'onclick' ) && ! str_contains( $out, 'ID; if ( ! $uid ) { $uid = apply_filters( 'nonce_user_logged_out', $uid, $action ); } $previous = substr( wp_hash( ( wp_nonce_tick( $action ) - 1 ) . '|' . $action . '|' . $uid . '|' . wp_get_session_token(), 'nonce' ), -12, 10 ); return wpbp_security_004( $nonce, $action ) && wpbp_security_004( $previous, $action ) && ! wpbp_security_004( $nonce, 'other' ) && ! wpbp_security_004( '', $action );", + "description": "Current and previous-tick nonces are valid for the matching action only", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_security_004( string $nonce, string $action ): bool { return false !== wp_verify_nonce( $nonce, $action ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/pluggable.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-security-005", + "category": "security", + "difficulty": "basic", + "prompt": "Implement `wpbp_security_005( string $url ): string` to escape a URL for HTML display while preferring HTTPS when WordPress has to add a missing scheme.", + "expected_behavior": "`wpbp_security_005()` calls `esc_url()` with HTTPS first and HTTP second in the allowed protocol list. Schemeless absolute URLs should gain `https://`, existing HTTP URLs should stay HTTP, display-context ampersands should be entity-escaped, and protocols outside the allowed list should be rejected.", + "requirements": [ + "Use esc_url()", + "Allow both HTTPS and HTTP with HTTPS preferred", + "Return the display-safe URL string" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_security_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "esc_url", + "description": "Uses esc_url", + "weight": 1 + }, + { + "pattern": "/['\"]https['\"]/i", + "description": "Supplies HTTPS as the preferred protocol", + "weight": 0.5 + }, + { + "pattern": "/['\"]http['\"]/i", + "description": "Allows HTTP as a valid protocol", + "weight": 0.5 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_security_005' ) ) { return false; } return 'https://example.com/path?x=1&y=2' === wpbp_security_005( 'example.com/path?x=1&y=2' ) && 'http://example.com/path?x=1&y=2' === wpbp_security_005( 'http://example.com/path?x=1&y=2' ) && '' === wpbp_security_005( 'mailto:person@example.com' ) && '' === wpbp_security_005( 'javascript:alert(1)' );", + "description": "Schemeless URLs prefer HTTPS, existing HTTP is preserved, display entities are escaped, and disallowed protocols are removed", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_security_005( string $url ): string { return esc_url( $url, array( 'https', 'http' ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/formatting.php", + "src/wp-includes/kses.php" + ], + "release_focus": "6.9" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/settings-options.json b/datasets/suites/wp-core-v1/execution/settings-options.json new file mode 100644 index 0000000..44edf84 --- /dev/null +++ b/datasets/suites/wp-core-v1/execution/settings-options.json @@ -0,0 +1,313 @@ +{ + "id": "wp-core-execution-v1-settings_options", + "version": "2.0.0", + "metadata": { + "name": "WordPress Core Execution Tests - Settings and Options", + "description": "Code generation tasks for WordPress Settings and Options APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" + }, + "tests": [ + { + "id": "e-settings-options-001", + "category": "settings-options", + "difficulty": "intermediate", + "prompt": "Implement wpbp_settings_options_001() to register the option-backed setting 'wpbp_setting_title' in the 'wpbp' group as a REST-exposed string setting whose saved value is sanitized as plain text.", + "expected_behavior": "`wpbp_settings_options_001()` registers `wpbp_setting_title` with `register_setting()` in the `wpbp` group, exposes it through REST settings metadata, declares string data, and registers a sanitizer that strips markup. Runtime validation inspects WordPress's registered settings registry and verifies that `sanitize_option()` applies the registered sanitizer.", + "requirements": [ + "Use the Settings API", + "Register the setting under group 'wpbp' with option name 'wpbp_setting_title'", + "Expose the setting to REST clients and sanitize stored values as plain text" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_settings_options_001", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "register_setting", + "description": "Uses register_setting", + "weight": 1 + }, + { + "pattern": "wpbp_setting_title", + "description": "Registers the requested setting name", + "weight": 1 + }, + { + "pattern": "show_in_rest", + "description": "Exposes the setting to REST", + "weight": 1 + }, + { + "pattern": "sanitize_callback", + "description": "Registers a sanitizer for saved values", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_settings_options_001' ) ) { return false; } wpbp_settings_options_001(); global $wp_registered_settings; if ( empty( $wp_registered_settings['wpbp_setting_title'] ) ) { return false; } $setting = $wp_registered_settings['wpbp_setting_title']; $rest = $setting['show_in_rest'] ?? false; $rest_ok = true === $rest || ( is_array( $rest ) && isset( $rest['schema'] ) ); return 'wpbp' === ( $setting['group'] ?? null ) && 'string' === ( $setting['type'] ?? null ) && $rest_ok && isset( $setting['sanitize_callback'] ) && is_callable( $setting['sanitize_callback'] ) && 'Title' === sanitize_option( 'wpbp_setting_title', 'Title' );", + "description": "The setting is registered as a REST-exposed sanitized string setting", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_settings_options_001(): void { register_setting( 'wpbp', 'wpbp_setting_title', array( 'type' => 'string', 'show_in_rest' => true, 'sanitize_callback' => 'sanitize_text_field', 'default' => '' ) ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/option.php", + "src/wp-includes/formatting.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-settings-options-002", + "category": "settings-options", + "difficulty": "intermediate", + "prompt": "Implement wpbp_settings_options_002( string $value ): bool to create the 'wpbp_noautoload' option with the provided value sanitized as plain text and excluded from WordPress autoloading.", + "expected_behavior": "`wpbp_settings_options_002()` adds `wpbp_noautoload` with `add_option()`, stores the provided value after plain-text sanitization, and disables autoloading using the modern Options API behavior. Runtime validation checks the stored value and confirms the options table autoload marker is not one of WordPress's autoloaded values.", + "requirements": [ + "Use the Options API", + "Store the sanitized value in 'wpbp_noautoload'", + "Disable autoloading for the option" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_settings_options_002", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_option", + "description": "Uses add_option", + "weight": 1 + }, + { + "pattern": "wpbp_noautoload", + "description": "Adds the requested option name", + "weight": 1 + }, + { + "pattern": "sanitize_text_field", + "description": "Sanitizes the stored text value", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "delete_option( 'wpbp_noautoload' );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_settings_options_002' ) ) { return false; } global $wpdb; $added = wpbp_settings_options_002( 'Value' ); $stored = get_option( 'wpbp_noautoload' ); $autoload = $wpdb->get_var( $wpdb->prepare( \"SELECT autoload FROM {$wpdb->options} WHERE option_name = %s\", 'wpbp_noautoload' ) ); $autoloaded_values = function_exists( 'wp_autoload_values_to_autoload' ) ? wp_autoload_values_to_autoload() : array( 'yes' ); return true === $added && 'Value' === $stored && is_string( $autoload ) && ! in_array( $autoload, $autoloaded_values, true );", + "description": "The option is stored sanitized and is not marked for autoloading", + "weight": 1 + } + ], + "teardown": "delete_option( 'wpbp_noautoload' );" + }, + "reference_solution": "function wpbp_settings_options_002( string $value ): bool { return add_option( 'wpbp_noautoload', sanitize_text_field( $value ), '', false ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/option.php", + "src/wp-includes/formatting.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-settings-options-003", + "category": "settings-options", + "difficulty": "intermediate", + "prompt": "Implement wpbp_settings_options_003( string $key, string $value ): array to update the 'wpbp_nested' option by storing a sanitized plain-text value under a sanitized key, preserving existing entries, and returning the full updated option array.", + "expected_behavior": "`wpbp_settings_options_003()` reads the existing `wpbp_nested` array with `get_option()`, normalizes the supplied key with `sanitize_key()`, sanitizes the supplied value as plain text, persists the updated array with `update_option()`, and returns the same persisted array. Runtime validation seeds an existing entry, checks preservation, and verifies the sanitized key and value in both the return value and stored option.", + "requirements": [ + "Use the Options API to read and update 'wpbp_nested'", + "Sanitize the supplied key and value before storing", + "Preserve existing option entries" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_settings_options_003", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "get_option", + "description": "Reads the existing option value", + "weight": 1 + }, + { + "pattern": "sanitize_key", + "description": "Uses sanitize_key", + "weight": 1 + }, + { + "pattern": "update_option", + "description": "Uses update_option", + "weight": 1 + }, + { + "pattern": "sanitize_text_field", + "description": "Sanitizes the stored text value", + "weight": 1 + }, + { + "pattern": "wpbp_nested", + "description": "Updates the requested option name", + "weight": 1 + } + ] + }, + "runtime_checks": { + "setup": "update_option( 'wpbp_nested', array( 'existing' => 'keep' ), false );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_settings_options_003' ) ) { return false; } $out = wpbp_settings_options_003( 'Some Key!', 'Value' ); $stored = get_option( 'wpbp_nested' ); return is_array( $out ) && is_array( $stored ) && array( 'existing' => 'keep', 'somekey' => 'Value' ) === $out && $out === $stored;", + "description": "The nested option is updated with sanitized data while preserving existing entries", + "weight": 1 + } + ], + "teardown": "delete_option( 'wpbp_nested' );" + }, + "reference_solution": "function wpbp_settings_options_003( string $key, string $value ): array { $options = get_option( 'wpbp_nested', array() ); if ( ! is_array( $options ) ) { $options = array(); } $options[ sanitize_key( $key ) ] = sanitize_text_field( $value ); update_option( 'wpbp_nested', $options ); return $options; }", + "metadata": { + "source_refs": [ + "src/wp-includes/option.php", + "src/wp-includes/formatting.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-settings-options-004", + "category": "settings-options", + "difficulty": "intermediate", + "prompt": "Implement wpbp_settings_options_004(): void to register a settings section 'wpbp_section' and a settings field 'wpbp_field' on the built-in 'reading' settings page, with the field assigned to that section.", + "expected_behavior": "`wpbp_settings_options_004()` uses the Settings API to register `wpbp_section` on the `reading` page and `wpbp_field` inside that section. Runtime validation inspects the Settings API globals populated by `add_settings_section()` and `add_settings_field()` and checks that both callbacks are callable.", + "requirements": [ + "Use the Settings API", + "Register the section and field on the 'reading' settings page", + "Assign the field to the registered section" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_settings_options_004", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_settings_section", + "description": "Uses add_settings_section", + "weight": 1 + }, + { + "pattern": "add_settings_field", + "description": "Uses add_settings_field", + "weight": 1 + }, + { + "pattern": "wpbp_section", + "description": "Registers the requested section", + "weight": 1 + }, + { + "pattern": "wpbp_field", + "description": "Registers the requested field", + "weight": 1 + }, + { + "pattern": "reading", + "description": "Targets the reading settings page", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_settings_options_004' ) ) { return false; } wpbp_settings_options_004(); global $wp_settings_sections, $wp_settings_fields; $section = $wp_settings_sections['reading']['wpbp_section'] ?? null; $field = $wp_settings_fields['reading']['wpbp_section']['wpbp_field'] ?? null; return is_array( $section ) && is_array( $field ) && 'wpbp_section' === ( $section['id'] ?? null ) && 'wpbp_field' === ( $field['id'] ?? null ) && is_callable( $section['callback'] ?? null ) && is_callable( $field['callback'] ?? null );", + "description": "The requested section and field are registered on the reading page", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_settings_options_004(): void { add_settings_section( 'wpbp_section', 'WPBP Settings', '__return_null', 'reading' ); add_settings_field( 'wpbp_field', 'WPBP Field', '__return_null', 'reading', 'wpbp_section' ); }", + "metadata": { + "source_refs": [ + "src/wp-admin/includes/template.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-settings-options-005", + "category": "settings-options", + "difficulty": "basic", + "prompt": "Implement wpbp_settings_options_005( string $name, mixed $fallback ): mixed to delete the named option and then return WordPress's fallback value for that option.", + "expected_behavior": "`wpbp_settings_options_005()` deletes the supplied option name with `delete_option()` and then returns `get_option()` using the supplied fallback, without creating or reseeding the option itself. Runtime validation creates the fixture option before execution, checks that the fallback is returned after deletion, and verifies the option no longer exists.", + "requirements": [ + "Use the Options API", + "Delete the provided option name", + "Return get_option() with the provided fallback after deletion" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_settings_options_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "delete_option", + "description": "Uses delete_option", + "weight": 1 + }, + { + "pattern": "get_option", + "description": "Uses get_option", + "weight": 1 + } + ], + "forbidden_patterns": [ + { + "pattern": "update_option", + "description": "Does not recreate the option before deleting it", + "severity": "error" + } + ] + }, + "runtime_checks": { + "setup": "update_option( 'wpbp_deleted_option', 'stored', false );", + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_settings_options_005' ) ) { return false; } $result = wpbp_settings_options_005( 'wpbp_deleted_option', 'fallback' ); return 'fallback' === $result && false === get_option( 'wpbp_deleted_option', false );", + "description": "The option is deleted and get_option returns the supplied fallback", + "weight": 1 + } + ], + "teardown": "delete_option( 'wpbp_deleted_option' );" + }, + "reference_solution": "function wpbp_settings_options_005( string $name, mixed $fallback ): mixed { delete_option( $name ); return get_option( $name, $fallback ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/option.php" + ], + "release_focus": "classic" + } + } + ] +} diff --git a/datasets/suites/wp-core-v1/execution/shortcodes.json b/datasets/suites/wp-core-v1/execution/shortcodes.json index 182b0c6..7f177f6 100644 --- a/datasets/suites/wp-core-v1/execution/shortcodes.json +++ b/datasets/suites/wp-core-v1/execution/shortcodes.json @@ -1,81 +1,266 @@ { "id": "wp-core-execution-v1-shortcodes", - "version": "1.1.0", + "version": "2.0.0", "metadata": { "name": "WordPress Core Execution Tests - Shortcodes", - "description": "Code generation tasks for WordPress shortcode development", - "wp_version": "6.9", - "created_at": "2025-12-16" + "description": "Code generation tasks for WordPress Shortcodes APIs", + "wp_version": "7.0", + "created_at": "2026-05-21" }, "tests": [ { - "id": "e-shortcode-001", + "id": "e-shortcodes-001", "category": "shortcodes", - "difficulty": "basic", - "prompt": "Create a WordPress shortcode called 'greeting' that accepts a 'name' attribute and outputs 'Hello, {name}!' where {name} is the attribute value. If no name is provided, default to 'World'.", - "expected_behavior": "A shortcode [greeting name=\"Alice\"] outputs 'Hello, Alice!' and [greeting] outputs 'Hello, World!'", + "difficulty": "intermediate", + "prompt": "Implement wpbp_shortcodes_001() to register a wpbp_greeting shortcode that returns a safe greeting for an optional name attribute.", + "expected_behavior": "`wpbp_shortcodes_001()` registers `wpbp_greeting`. The shortcode merges the optional `name` attribute with a `World` default via WordPress shortcode attribute handling and returns `Hello, !` with the name escaped for HTML. Runtime validation renders the shortcode with omitted, plain, and HTML-containing names.", "requirements": [ - "Register a shortcode with the tag 'greeting'", - "Accept a 'name' attribute with default value 'World'", - "Output must be properly escaped using esc_html()", - "Return the greeting string, do not echo" + "Register the shortcode tag wpbp_greeting", + "Default the name to World when omitted", + "Return escaped shortcode output; do not echo it" ], "static_checks": { "required_patterns": [ { - "pattern": "add_shortcode\\s*\\(\\s*['\"]greeting['\"]", - "description": "Must register shortcode with tag 'greeting'", - "weight": 1.0 + "pattern": "wpbp_shortcodes_001", + "description": "Defines the requested function", + "weight": 0.5 }, { - "pattern": "esc_html|esc_attr|wp_kses", - "description": "Must use WordPress escaping", - "weight": 0.5 + "pattern": "add_shortcode", + "description": "Uses add_shortcode", + "weight": 1 + }, + { + "pattern": "wpbp_greeting", + "description": "Registers the requested shortcode tag", + "weight": 1 }, { "pattern": "shortcode_atts", - "description": "Should use shortcode_atts for attribute defaults", + "description": "Uses shortcode_atts", + "weight": 1 + }, + { + "pattern": "esc_html", + "description": "Uses esc_html", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_shortcodes_001' ) ) { return false; } wpbp_shortcodes_001(); $default = do_shortcode( '[wpbp_greeting]' ); $named = do_shortcode( '[wpbp_greeting name=\"Ada\"]' ); $escaped = do_shortcode( '[wpbp_greeting name=\"Ada\"]' ); remove_shortcode( 'wpbp_greeting' ); return 'Hello, World!' === $default && 'Hello, Ada!' === $named && 'Hello, <b>Ada</b>!' === $escaped;", + "description": "The greeting shortcode applies defaults and escapes the rendered name", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_shortcodes_001(): void { add_shortcode( 'wpbp_greeting', function ( $atts ) { $atts = shortcode_atts( array( 'name' => 'World' ), $atts, 'wpbp_greeting' ); return 'Hello, ' . esc_html( $atts['name'] ) . '!'; } ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/shortcodes.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-shortcodes-002", + "category": "shortcodes", + "difficulty": "basic", + "prompt": "Implement wpbp_shortcodes_002( string $text ): array to parse a shortcode attribute string into WordPress's attribute array without running shortcode callbacks.", + "expected_behavior": "`wpbp_shortcodes_002()` delegates parsing to WordPress's shortcode attribute parser, returning lowercase named keys and string values from the supplied attribute text. It must not run shortcode callbacks while parsing. Runtime validation registers a probe shortcode, parses mixed-case attributes plus probe-looking text, and verifies the probe was not executed.", + "requirements": [ + "Use WordPress's shortcode attribute parser", + "Return the parsed attributes array", + "Do not execute shortcode callbacks" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_shortcodes_002", + "description": "Defines the requested function", "weight": 0.5 + }, + { + "pattern": "shortcode_parse_atts", + "description": "Uses shortcode_parse_atts", + "weight": 1 } ], "forbidden_patterns": [ { - "pattern": "\\becho\\b", - "description": "Should return, not echo", - "severity": "warning" + "pattern": "/\\b(?:do_shortcode|apply_shortcodes)\\b/", + "description": "Parsing attributes must not execute shortcode callbacks", + "severity": "error" } ] }, "runtime_checks": { - "setup": "", "assertions": [ { - "type": "shortcode_exists", - "target": "greeting", - "description": "Shortcode must be registered", - "weight": 1.0 + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_shortcodes_002' ) ) { return false; } $GLOBALS['wpbp_shortcodes_002_runs'] = 0; add_shortcode( 'wpbp_parse_probe', function () { $GLOBALS['wpbp_shortcodes_002_runs']++; return 'ran'; } ); $atts = wpbp_shortcodes_002( 'Name=\"Ada\" count=\"3\" [wpbp_parse_probe]' ); $runs = $GLOBALS['wpbp_shortcodes_002_runs']; remove_shortcode( 'wpbp_parse_probe' ); unset( $GLOBALS['wpbp_shortcodes_002_runs'] ); return is_array( $atts ) && 'Ada' === ( $atts['name'] ?? null ) && '3' === ( $atts['count'] ?? null ) && 0 === $runs;", + "description": "Attribute parsing lowercases named keys and does not execute shortcode callbacks", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_shortcodes_002( string $text ): array { return shortcode_parse_atts( $text ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/shortcodes.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-shortcodes-003", + "category": "shortcodes", + "difficulty": "basic", + "prompt": "Implement wpbp_shortcodes_003( string $content ): string to remove registered shortcode occurrences from content while leaving unknown shortcode-like text intact.", + "expected_behavior": "`wpbp_shortcodes_003()` uses WordPress shortcode stripping so only currently registered shortcode tags are removed. Enclosing shortcode matches are removed by WordPress, while unregistered bracketed content is preserved. Runtime validation registers one tag and compares the filtered content against an unknown tag.", + "requirements": [ + "Use WordPress Shortcode API", + "Accept and return content as a string", + "Preserve unregistered shortcode-like text" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_shortcodes_003", + "description": "Defines the requested function", + "weight": 0.5 }, { - "type": "output_equals", - "target": "echo do_shortcode('[greeting name=\"Alice\"]');", - "expected": "Hello, Alice!", - "description": "Shortcode with name attribute", - "weight": 1.0 + "pattern": "strip_shortcodes", + "description": "Uses strip_shortcodes", + "weight": 1 + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_shortcodes_003' ) ) { return false; } add_shortcode( 'wpbp_strip', fn() => 'ignored' ); $out = wpbp_shortcodes_003( 'A [wpbp_strip]Keep[/wpbp_strip] B [wpbp_unknown]Stay[/wpbp_unknown]' ); remove_shortcode( 'wpbp_strip' ); return 'A B [wpbp_unknown]Stay[/wpbp_unknown]' === $out;", + "description": "Only registered shortcode occurrences are stripped and unknown shortcode-like text remains", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_shortcodes_003( string $content ): string { return strip_shortcodes( $content ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/shortcodes.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-shortcodes-004", + "category": "shortcodes", + "difficulty": "basic", + "prompt": "Implement wpbp_shortcodes_004( string $content, string $tag ): bool to report whether content contains a registered shortcode tag, including nested shortcode content.", + "expected_behavior": "`wpbp_shortcodes_004()` delegates shortcode detection to WordPress, so it only reports true for shortcode tags that are registered and can find a tag nested inside another shortcode's enclosed content. Runtime validation registers outer/inner fixture shortcodes, checks nested detection, then verifies an unregistered tag is not reported.", + "requirements": [ + "Use WordPress Shortcode API", + "Accept content and tag inputs", + "Return a boolean result without rendering the shortcode" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_shortcodes_004", + "description": "Defines the requested function", + "weight": 0.5 }, { - "type": "output_equals", - "target": "echo do_shortcode('[greeting]');", - "expected": "Hello, World!", - "description": "Shortcode with default name", - "weight": 1.0 + "pattern": "has_shortcode", + "description": "Uses has_shortcode", + "weight": 1 } ], - "teardown": "remove_shortcode('greeting');" + "forbidden_patterns": [ + { + "pattern": "/\\b(?:do_shortcode|apply_shortcodes)\\b/", + "description": "Detection must not render shortcode callbacks", + "severity": "error" + } + ] + }, + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_shortcodes_004' ) ) { return false; } add_shortcode( 'wpbp_outer', fn( $atts, $content = '' ) => $content ); add_shortcode( 'wpbp_inner', fn() => 'inner' ); $content = '[wpbp_outer]Before [wpbp_inner] after[/wpbp_outer] [wpbp_missing]'; $nested = wpbp_shortcodes_004( $content, 'wpbp_inner' ); $missing = wpbp_shortcodes_004( $content, 'wpbp_missing' ); remove_shortcode( 'wpbp_outer' ); remove_shortcode( 'wpbp_inner' ); return true === $nested && false === $missing;", + "description": "Registered nested shortcodes are detected and unregistered tags are ignored", + "weight": 1 + } + ] + }, + "reference_solution": "function wpbp_shortcodes_004( string $content, string $tag ): bool { return has_shortcode( $content, $tag ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/shortcodes.php" + ], + "release_focus": "classic" + } + }, + { + "id": "e-shortcodes-005", + "category": "shortcodes", + "difficulty": "intermediate", + "prompt": "Implement wpbp_shortcodes_005() to register a wpbp_wrap enclosing shortcode that returns its enclosed content inside a span with the content escaped for HTML.", + "expected_behavior": "`wpbp_shortcodes_005()` registers `wpbp_wrap` as an enclosing shortcode. The callback receives enclosed content, returns `...escaped content...`, and does not echo output. Runtime validation renders plain and HTML-containing enclosed content through `do_shortcode()`.", + "requirements": [ + "Register the shortcode tag wpbp_wrap", + "Read the shortcode's enclosed content argument", + "Return escaped markup; do not echo it" + ], + "static_checks": { + "required_patterns": [ + { + "pattern": "wpbp_shortcodes_005", + "description": "Defines the requested function", + "weight": 0.5 + }, + { + "pattern": "add_shortcode", + "description": "Uses add_shortcode", + "weight": 1 + }, + { + "pattern": "wpbp_wrap", + "description": "Registers the requested shortcode tag", + "weight": 1 + }, + { + "pattern": "esc_html", + "description": "Uses esc_html", + "weight": 1 + } + ] }, - "judge_config": { - "rubric_id": "wp-judge-rubric-v1" + "runtime_checks": { + "assertions": [ + { + "type": "custom_assertion", + "code": "if ( ! function_exists( 'wpbp_shortcodes_005' ) ) { return false; } wpbp_shortcodes_005(); $plain = do_shortcode( '[wpbp_wrap]Safe[/wpbp_wrap]' ); $escaped = do_shortcode( '[wpbp_wrap]Ada & Co[/wpbp_wrap]' ); remove_shortcode( 'wpbp_wrap' ); return 'Safe' === $plain && '<strong>Ada</strong> & Co' === $escaped;", + "description": "The enclosing shortcode returns escaped content inside a span", + "weight": 1 + } + ] }, - "reference_solution": "function greeting_shortcode( $atts ) {\n $atts = shortcode_atts( array( 'name' => 'World' ), $atts, 'greeting' );\n return 'Hello, ' . esc_html( $atts['name'] ) . '!';\n}\nadd_shortcode( 'greeting', 'greeting_shortcode' );" + "reference_solution": "function wpbp_shortcodes_005(): void { add_shortcode( 'wpbp_wrap', fn( $atts, $content = '' ) => '' . esc_html( $content ) . '' ); }", + "metadata": { + "source_refs": [ + "src/wp-includes/shortcodes.php" + ], + "release_focus": "classic" + } } ] } diff --git a/datasets/suites/wp-core-v1/execution/wp-query.json b/datasets/suites/wp-core-v1/execution/wp-query.json deleted file mode 100644 index 25b4ef8..0000000 --- a/datasets/suites/wp-core-v1/execution/wp-query.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "id": "wp-core-execution-v1-wp_query", - "version": "1.1.0", - "metadata": { - "name": "WordPress Core Execution Tests - WP_Query", - "description": "Code generation tasks for WordPress WP_Query usage", - "wp_version": "6.9", - "created_at": "2025-12-16" - }, - "tests": [ - { - "id": "e-wpquery-001", - "category": "wp-query", - "difficulty": "intermediate", - "prompt": "Write a function called 'get_recent_news_posts' that returns the 5 most recently published posts from the 'news' category. The function should return an array of post objects.", - "expected_behavior": "Function returns up to 5 published posts from the 'news' category, ordered by date descending", - "requirements": [ - "Use WP_Query for the database query", - "Filter to 'post' post type", - "Filter to 'publish' status only", - "Filter to 'news' category by slug", - "Limit to 5 posts", - "Order by date descending", - "Return array of post objects", - "Reset post data after query using wp_reset_postdata()" - ], - "static_checks": { - "required_patterns": [ - { - "pattern": "new\\s+WP_Query", - "description": "Must use WP_Query class", - "weight": 1.0 - }, - { - "pattern": "posts_per_page", - "description": "Must specify posts_per_page", - "weight": 0.5 - }, - { - "pattern": "category_name|cat\\s*=>", - "description": "Must filter by category", - "weight": 0.5 - }, - { - "pattern": "wp_reset_postdata", - "description": "Should reset post data after query", - "weight": 0.3 - } - ], - "forbidden_patterns": [ - { - "pattern": "\\$wpdb->", - "description": "Should use WP_Query, not direct database queries", - "severity": "warning" - } - ] - }, - "runtime_checks": { - "assertions": [ - { - "type": "function_exists", - "target": "get_recent_news_posts", - "description": "Function must exist", - "weight": 1.0 - }, - { - "type": "custom_assertion", - "code": "$result = get_recent_news_posts(); return is_array($result);", - "description": "Must return an array", - "weight": 1.0 - } - ] - }, - "judge_config": { - "rubric_id": "wp-judge-rubric-v1", - "context_for_judge": "This function will be used in a theme to display recent news posts on the homepage. Consider query efficiency and proper WordPress patterns." - } - } - ] -} diff --git a/notebooks/results_report.ipynb b/notebooks/results_report.ipynb index eefaeb5..0ea2dcf 100644 --- a/notebooks/results_report.ipynb +++ b/notebooks/results_report.ipynb @@ -62,7 +62,6 @@ " 'Model': model_name,\n", " 'Knowledge': scores['knowledge'] * 100,\n", " 'Correctness': scores['correctness'] * 100,\n", - " 'Quality': scores['quality'] * 100 if scores['quality'] else None,\n", " 'Overall': scores['overall'] * 100,\n", " })\n", "\n", @@ -173,7 +172,6 @@ "styled = styled.format({\n", " 'Knowledge': '{:.1f}%',\n", " 'Correctness': '{:.1f}%',\n", - " 'Quality': lambda x: f'{x:.1f}%' if pd.notna(x) else 'N/A',\n", " 'Overall': '{:.1f}%'\n", "})\n", "styled" @@ -201,4 +199,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/python/tests/test_dataset_filtering.py b/python/tests/test_dataset_filtering.py new file mode 100644 index 0000000..27690d9 --- /dev/null +++ b/python/tests/test_dataset_filtering.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import pytest + +from wp_bench.cli import _normalize_test_ids +from wp_bench.datasets import KnowledgeTest, ensure_test_ids_match_type, filter_tests_by_ids + + +def _knowledge_test(test_id: str) -> KnowledgeTest: + return KnowledgeTest( + id=test_id, + suite="wp-core-v1", + prompt="Prompt", + test_type="short_answer", + category="general", + difficulty="basic", + ) + + +def test_normalize_test_ids_accepts_repeated_and_comma_separated_values() -> None: + assert _normalize_test_ids(["e-one,e-two", "e-two", " e-three "]) == [ + "e-one", + "e-two", + "e-three", + ] + + +def test_filter_tests_by_ids_returns_requested_tests() -> None: + tests = { + "knowledge": [_knowledge_test("k-one"), _knowledge_test("k-two")], + "execution": [], + } + + filtered = filter_tests_by_ids(tests, ["k-two"]) + + assert [test.id for test in filtered["knowledge"]] == ["k-two"] + assert filtered["execution"] == [] + + +def test_filter_tests_by_ids_rejects_unknown_ids() -> None: + tests = {"knowledge": [_knowledge_test("k-one")], "execution": []} + + with pytest.raises(ValueError, match="Unknown test id"): + filter_tests_by_ids(tests, ["missing"]) + + +def test_ensure_test_ids_match_type_rejects_mismatched_explicit_type() -> None: + tests = {"knowledge": [_knowledge_test("k-one")], "execution": []} + + with pytest.raises(ValueError, match="No execution tests matched"): + ensure_test_ids_match_type(tests, "execution", ["k-one"]) diff --git a/python/tests/test_environment.py b/python/tests/test_environment.py new file mode 100644 index 0000000..94058f7 --- /dev/null +++ b/python/tests/test_environment.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import base64 +import json +from pathlib import Path + +from wp_bench.config import GraderConfig +from wp_bench.environment import WordPressEnvironment + + +def test_execute_code_uses_internal_runtime_verifier_for_wp_env() -> None: + calls: list[list[str]] = [] + environment = WordPressEnvironment(GraderConfig(kind="docker", wp_env_dir=Path("runtime"))) + + def fake_exec(command: list[str]) -> tuple[str, str, int]: + calls.append(command) + payload = json.loads(base64.b64decode(command[3]).decode("utf-8")) + assert payload["code"] == "function demo() { return true; }" + assert payload["static_checks"] == {"required_patterns": []} + assert payload["runtime_checks"] == {"assertions": []} + return ('{"success": true}', "", 0) + + environment._exec = fake_exec # type: ignore[method-assign] + + result = environment.execute_code( + "function demo() { return true; }", + { + "static_checks": {"required_patterns": []}, + "runtime_checks": {"assertions": []}, + }, + ) + + assert result.success is True + assert calls == [ + [ + "wp", + "eval-file", + "/var/www/html/wp-content/plugins/runtime/verify-runtime.php", + calls[0][3], + ] + ] diff --git a/python/tests/test_execution_dataset.py b/python/tests/test_execution_dataset.py new file mode 100644 index 0000000..40e0988 --- /dev/null +++ b/python/tests/test_execution_dataset.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import orjson + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +EXECUTION_DIR = PROJECT_ROOT / "datasets" / "suites" / "wp-core-v1" / "execution" + +SUPPORTED_ASSERTIONS = { + "class_exists", + "custom_assertion", + "function_exists", + "hook_registered", + "option_value", + "output_contains", + "output_equals", + "output_matches", + "output_not_contains", + "post_meta_value", + "query_result", + "rest_response", + "returns_value", + "shortcode_exists", +} + + +def _execution_suites() -> list[dict[str, Any]]: + return [orjson.loads(path.read_bytes()) for path in sorted(EXECUTION_DIR.glob("*.json"))] + + +def _execution_tests() -> list[dict[str, Any]]: + tests: list[dict[str, Any]] = [] + for suite in _execution_suites(): + tests.extend(suite.get("tests", [])) + return tests + + +def test_execution_suite_has_exactly_150_tests() -> None: + assert len(_execution_tests()) == 150 + + +def test_execution_test_ids_are_unique() -> None: + ids = [test["id"] for test in _execution_tests()] + assert len(ids) == len(set(ids)) + + +def test_execution_tests_have_required_fields() -> None: + required = { + "category", + "difficulty", + "expected_behavior", + "id", + "metadata", + "prompt", + "reference_solution", + "requirements", + "runtime_checks", + "static_checks", + } + for test in _execution_tests(): + assert required <= test.keys(), test["id"] + assert "judge_config" not in test, test["id"] + assert test["reference_solution"].strip(), test["id"] + assert test["expected_behavior"].strip(), test["id"] + assert test["expected_behavior"] != test["prompt"], test["id"] + assert test["requirements"], test["id"] + assert test["metadata"].get("source_refs"), test["id"] + + +def test_execution_assertion_types_are_supported() -> None: + for test in _execution_tests(): + assertions = test.get("runtime_checks", {}).get("assertions", []) + assert assertions, test["id"] + for assertion in assertions: + assert assertion.get("type") in SUPPORTED_ASSERTIONS, test["id"] + + +def test_execution_suite_includes_modern_wordpress_coverage() -> None: + modern_tests = [ + test + for test in _execution_tests() + if test.get("metadata", {}).get("release_focus") in {"6.9", "7.0"} + ] + assert len(modern_tests) >= 35 diff --git a/python/tests/test_reference_solution_mode.py b/python/tests/test_reference_solution_mode.py new file mode 100644 index 0000000..9bfa62b --- /dev/null +++ b/python/tests/test_reference_solution_mode.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from wp_bench.config import ( + DatasetConfig, + GraderConfig, + HarnessConfig, + ModelConfig, + OutputConfig, + RunConfig, +) +from wp_bench.core import BenchmarkRunner +from wp_bench.datasets import ExecutionTest, KnowledgeTest +from wp_bench.environment import ExecutionResult + + +def _config(tmp_path: Path, test_ids: list[str] | None = None) -> HarnessConfig: + return HarnessConfig( + dataset=DatasetConfig(source="local", name="wp-core-v1"), + model=ModelConfig(name="test-model"), + grader=GraderConfig(kind="cli"), + run=RunConfig( + check_reference_solution=True, + concurrency=1, + test_ids=test_ids or [], + ), + output=OutputConfig(path=tmp_path / "results.json", jsonl_path=None), + ) + + +def _execution_test(test_id: str = "e-one") -> ExecutionTest: + return ExecutionTest( + id=test_id, + suite="wp-core-v1", + prompt="Prompt", + expected_behavior="Reviewer contract: expected", + test_type="execution", + category="general", + difficulty="basic", + requirements=["Requirement"], + static_checks={"required_patterns": []}, + runtime_checks={"assertions": []}, + reference_solution="function ref() { return true; }", + metadata={}, + ) + + +def _passing_result() -> ExecutionResult: + raw = { + "success": True, + "static": {"score": 1.0, "details": {"total_weight": 1}}, + "runtime": {"score": 1.0, "details": {"total_weight": 1}}, + } + return ExecutionResult(success=True, raw=raw, stdout="", stderr="") + + +def _failing_result() -> ExecutionResult: + raw = { + "success": False, + "static": {"score": 1.0, "details": {"total_weight": 1}}, + "runtime": {"score": 0.0, "details": {"total_weight": 1}}, + } + return ExecutionResult(success=False, raw=raw, stdout="", stderr="") + + +def test_reference_solution_mode_executes_reference_solution( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + test = _execution_test() + config = _config(tmp_path) + calls: list[tuple[str, dict]] = [] + monkeypatch.setattr( + "wp_bench.core.load_tests", + lambda dataset: {"execution": [test], "knowledge": []}, + ) + runner = BenchmarkRunner(config) + runner.environment.setup = lambda: None # type: ignore[method-assign] + + def fake_execute_code(code: str, verification_spec: dict) -> ExecutionResult: + calls.append((code, verification_spec)) + return _passing_result() + + runner.environment.execute_code = fake_execute_code # type: ignore[method-assign] + + result = runner.run() + + assert calls == [ + ( + test.reference_solution, + { + "static_checks": test.static_checks, + "runtime_checks": test.runtime_checks, + }, + ) + ] + assert result["metadata"]["mode"] == "reference_solution" + assert result["metadata"]["scores"]["correctness"] == 1.0 + assert result["results"][0]["passed"] is True + + +def test_reference_solution_mode_exits_nonzero_on_failed_reference( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + test = _execution_test() + config = _config(tmp_path) + monkeypatch.setattr( + "wp_bench.core.load_tests", + lambda dataset: {"execution": [test], "knowledge": []}, + ) + runner = BenchmarkRunner(config) + runner.environment.setup = lambda: None # type: ignore[method-assign] + + def fake_execute_code(code: str, verification_spec: dict[str, Any]) -> ExecutionResult: + return _failing_result() + + runner.environment.execute_code = fake_execute_code # type: ignore[method-assign] + + with pytest.raises(SystemExit) as exc: + runner.run() + + assert exc.value.code == 1 + assert runner.records[0]["passed"] is False + assert runner.records[0]["correctness"] == 0.5 + + +def test_reference_solution_mode_rejects_non_execution_test_ids( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + knowledge_test = KnowledgeTest( + id="k-one", + suite="wp-core-v1", + prompt="Prompt", + test_type="knowledge", + category="general", + difficulty="basic", + ) + monkeypatch.setattr( + "wp_bench.core.load_tests", + lambda dataset: {"execution": [], "knowledge": [knowledge_test]}, + ) + runner = BenchmarkRunner(_config(tmp_path, test_ids=["k-one"])) + + with pytest.raises(ValueError, match="only supports execution"): + runner.run() diff --git a/python/tests/test_static_analysis_runtime.py b/python/tests/test_static_analysis_runtime.py new file mode 100644 index 0000000..fa17172 --- /dev/null +++ b/python/tests/test_static_analysis_runtime.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] + + +def _run_static_analysis(pattern: str, subject: str) -> dict: + php_code = f""" +require_once {str(PROJECT_ROOT / "runtime/src/class-static-analysis.php")!r}; +$analysis = new WPBench\\Runtime\\Static_Analysis(); +$result = $analysis->check( + {subject!r}, + array( + 'required_patterns' => array( + array( + 'pattern' => {pattern!r}, + 'description' => 'test pattern', + 'weight' => 1, + ), + ), + ) +); +echo json_encode( $result ); +""" + proc = subprocess.run( + ["php", "-r", php_code], + capture_output=True, + text=True, + check=True, + ) + return json.loads(proc.stdout) + + +def test_static_analysis_matches_delimiterless_pattern_containing_slash() -> None: + result = _run_static_analysis( + "wpbp/count-words", + "wp_register_ability( 'wpbp/count-words', array() );", + ) + + assert result["score"] == 1 + assert result["details"]["required"][0]["found"] is True + + +def test_static_analysis_preserves_explicit_regex_delimiters_and_flags() -> None: + result = _run_static_analysis( + "/WPBP\\/COUNT-WORDS/i", + "wp_register_ability( 'wpbp/count-words', array() );", + ) + + assert result["score"] == 1 + assert result["details"]["required"][0]["found"] is True + + +def test_static_analysis_matches_delimiterless_pattern_containing_tilde() -> None: + result = _run_static_analysis( + "wpbp~tool", + "wp_register_ability_category( 'wpbp~tool', array() );", + ) + + assert result["score"] == 1 + assert result["details"]["required"][0]["found"] is True diff --git a/python/wp_bench/cli.py b/python/wp_bench/cli.py index 61596bd..853f438 100644 --- a/python/wp_bench/cli.py +++ b/python/wp_bench/cli.py @@ -2,7 +2,7 @@ from __future__ import annotations from pathlib import Path -from typing import Optional +from typing import List, Optional import typer from dotenv import load_dotenv @@ -10,7 +10,7 @@ from .config import HarnessConfig, ModelConfig from .core import BenchmarkRunner, MultiModelRunner -from .datasets import load_tests +from .datasets import ensure_test_ids_match_type, filter_tests_by_ids, load_tests # Load .env file for API keys load_dotenv() @@ -19,6 +19,65 @@ console = Console() +@app.callback() +def main() -> None: + """WP-Bench command line interface.""" + + +def _normalize_test_ids(values: Optional[List[str]]) -> List[str]: + """Normalize repeated and comma-separated --test-id values.""" + if not values: + return [] + + normalized: List[str] = [] + seen: set[str] = set() + for value in values: + for test_id in value.split(","): + test_id = test_id.strip() + if test_id and test_id not in seen: + normalized.append(test_id) + seen.add(test_id) + return normalized + + +def _load_filtered_tests(harness_config: HarnessConfig) -> dict[str, list[object]]: + """Load tests and apply explicit test ID filtering.""" + tests = filter_tests_by_ids(load_tests(harness_config.dataset), harness_config.run.test_ids) + ensure_test_ids_match_type( + tests, + harness_config.run.test_type, + harness_config.run.test_ids, + ) + return tests + + +def _count_selected_tests(tests: list[object], harness_config: HarnessConfig) -> int: + """Count tests selected by current run settings.""" + if harness_config.run.test_ids: + return len(tests) + if harness_config.run.limit is None: + return len(tests) + return min(harness_config.run.limit, len(tests)) + + +def _print_dry_run_counts( + tests: dict[str, list[object]], + harness_config: HarnessConfig, +) -> None: + """Print test counts for a dry run.""" + test_type = harness_config.run.test_type + if test_type == "knowledge": + console.print(f"Knowledge tests: {_count_selected_tests(tests['knowledge'], harness_config)}") + elif test_type == "execution": + console.print(f"Execution tests: {_count_selected_tests(tests['execution'], harness_config)}") + else: + console.print( + "Execution tests: " + f"{_count_selected_tests(tests['execution'], harness_config)}, " + f"Knowledge tests: {_count_selected_tests(tests['knowledge'], harness_config)}" + ) + + @app.command() def run( config: Optional[Path] = typer.Option(None, help="Path to wp-bench YAML config"), @@ -26,6 +85,16 @@ def run( model_name: Optional[str] = typer.Option(None, help="Override model name (single model mode)"), limit: Optional[int] = typer.Option(None, help="Limit number of tests"), test_type: Optional[str] = typer.Option(None, help="Run only 'knowledge' or 'execution' tests"), + dry_run: bool = typer.Option(False, help="Load and filter tests without calling models"), + check_reference_solution: bool = typer.Option( + False, + help="Run execution tests with their reference_solution instead of calling models", + ), + test_id: Optional[List[str]] = typer.Option( + None, + "--test-id", + help="Run only the given dataset test ID. May be repeated or comma-separated.", + ), ) -> None: """Run the benchmark end-to-end.""" harness_config = HarnessConfig.from_file(config) if config else HarnessConfig() @@ -34,52 +103,76 @@ def run( harness_config.dataset.name = suite if "/" not in suite else suite if limit is not None: harness_config.run.limit = limit + if dry_run: + harness_config.run.dry_run = True + if check_reference_solution: + harness_config.run.check_reference_solution = True + normalized_test_ids = _normalize_test_ids(test_id) + if normalized_test_ids: + harness_config.run.test_ids = normalized_test_ids if test_type is not None: if test_type not in ("knowledge", "execution"): console.print(f"[red]Invalid --test-type: {test_type}. Must be 'knowledge' or 'execution'.[/red]") raise typer.Exit(1) harness_config.run.test_type = test_type # type: ignore[assignment] + if harness_config.run.dry_run and harness_config.run.check_reference_solution: + console.print("[red]--dry-run and --check-reference-solution cannot be used together.[/red]") + raise typer.Exit(1) + + if harness_config.run.dry_run: + try: + tests = _load_filtered_tests(harness_config) + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + _print_dry_run_counts(tests, harness_config) + return + + if harness_config.run.check_reference_solution: + reference_runner = BenchmarkRunner(harness_config) + try: + result = reference_runner.run() + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + console.print("[bold green]Reference solution check completed[/bold green]", result["metadata"]["scores"]) + return + # Check if multi-model mode models = harness_config.get_models() if model_name: # Override to single model harness_config.model = ModelConfig(name=model_name) harness_config.models = None - runner = BenchmarkRunner(harness_config) - result = runner.run() + single_runner = BenchmarkRunner(harness_config) + try: + result = single_runner.run() + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc console.print("[bold green]WP-Bench completed[/bold green]", result["metadata"]["scores"]) elif len(models) > 1: # Multi-model mode - runner = MultiModelRunner(harness_config) - runner.run() + multi_runner = MultiModelRunner(harness_config) + try: + multi_runner.run() + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc console.print("\n[bold green]WP-Bench completed[/bold green]") else: # Single model mode (legacy) if not harness_config.model: harness_config.model = models[0] - runner = BenchmarkRunner(harness_config) - result = runner.run() + single_runner = BenchmarkRunner(harness_config) + try: + result = single_runner.run() + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc console.print("[bold green]WP-Bench completed[/bold green]", result["metadata"]["scores"]) -@app.command() -def dry_run( - config: Optional[Path] = typer.Option(None, help="Config file"), - test_type: Optional[str] = typer.Option(None, help="Filter to 'knowledge' or 'execution' tests"), -) -> None: - """Load dataset and render prompt statistics without hitting models.""" - harness_config = HarnessConfig.from_file(config) if config else HarnessConfig() - tests = load_tests(harness_config.dataset) - if test_type == "knowledge": - console.print(f"Knowledge tests: {len(tests['knowledge'])}") - elif test_type == "execution": - console.print(f"Execution tests: {len(tests['execution'])}") - else: - console.print( - f"Execution tests: {len(tests['execution'])}, Knowledge tests: {len(tests['knowledge'])}" - ) - - if __name__ == "__main__": # pragma: no cover app() diff --git a/python/wp_bench/config.py b/python/wp_bench/config.py index bd2ecdc..7650894 100644 --- a/python/wp_bench/config.py +++ b/python/wp_bench/config.py @@ -53,10 +53,11 @@ class RunConfig(BaseModel): suite: str = "wp-core-v1" test_type: Optional[Literal["knowledge", "execution"]] = None limit: Optional[int] = None + test_ids: List[str] = Field(default_factory=list) seed: int = 1337 concurrency: int = 5 dry_run: bool = False - skip_judge: bool = False + check_reference_solution: bool = False skip_runtime: bool = False skip_static: bool = False diff --git a/python/wp_bench/core.py b/python/wp_bench/core.py index ace63cc..15187c4 100644 --- a/python/wp_bench/core.py +++ b/python/wp_bench/core.py @@ -11,7 +11,13 @@ import orjson from .config import HarnessConfig, ModelConfig -from .datasets import ExecutionTest, KnowledgeTest, load_tests +from .datasets import ( + ExecutionTest, + KnowledgeTest, + ensure_test_ids_match_type, + filter_tests_by_ids, + load_tests, +) from .environment import WordPressEnvironment from .knowledge import render_knowledge_prompt, score_knowledge_answer from .models import ModelInterface @@ -20,6 +26,7 @@ print_abort_message, print_comparison_table, print_model_header, + print_reference_solution_failures, print_results_path, print_test_error, ) @@ -44,6 +51,14 @@ def _timestamped_path(path: Path) -> Path: return path.parent / f"{path.stem}_{timestamp}{path.suffix}" +def _limit_tests(tests: List[Any], config: HarnessConfig) -> List[Any]: + """Apply run limit unless explicit test IDs are selected.""" + if config.run.test_ids: + return tests + limit = config.run.limit or len(tests) + return tests[:limit] + + class BenchmarkRunner: """Primary benchmark orchestrator for single-model evaluation. @@ -58,7 +73,7 @@ def __init__(self, config: HarnessConfig): config: Full harness configuration including model, grader, and output settings. """ self.config = config - self.model = ModelInterface(config.model) + self.model = ModelInterface(config.model or config.get_models()[0]) self.environment = WordPressEnvironment(config.grader) self.aggregator = ScoreAggregator() self.records: List[Dict[str, Any]] = [] @@ -76,16 +91,27 @@ def run(self) -> Dict[str, Any]: Raises: SystemExit: If a test fails, prints error details and exits with code 1. """ - tests = load_tests(self.config.dataset) + tests = filter_tests_by_ids(load_tests(self.config.dataset), self.config.run.test_ids) test_type = self.config.run.test_type - run_knowledge = test_type in (None, "knowledge") - run_execution = test_type in (None, "execution") + reference_mode = self.config.run.check_reference_solution + if reference_mode: + if test_type == "knowledge": + raise ValueError("--check-reference-solution only supports execution tests") + self._ensure_reference_solution_ids_are_execution_tests(tests) + run_knowledge = False + run_execution = True + else: + ensure_test_ids_match_type(tests, test_type, self.config.run.test_ids) + run_knowledge = test_type in (None, "knowledge") + run_execution = test_type in (None, "execution") if run_execution: self.environment.setup() try: if run_knowledge: self._run_knowledge_tests(tests["knowledge"]) - if run_execution: + if reference_mode: + self._run_reference_solution_tests(tests["execution"]) + elif run_execution: self._run_execution_tests(tests["execution"]) except TestError as e: print_test_error(e) @@ -94,24 +120,47 @@ def run(self) -> Dict[str, Any]: print_abort_message() raise SystemExit(130) from None summary = self.aggregator.finalize() + model_config = self.config.model.model_dump(mode="json") if self.config.model else None payload = { "metadata": { "suite": self.config.run.suite, - "model": self.config.model.model_dump(mode="json"), + "mode": "reference_solution" if reference_mode else "model", + "model": model_config, "grader": self.config.grader.model_dump(mode="json"), "dataset": self.config.dataset.model_dump(mode="json"), "scores": { "knowledge": summary.knowledge, "correctness": summary.correctness, - "quality": summary.quality, "overall": summary.overall(), }, }, "results": self.records, } self._write_outputs(payload) + if reference_mode: + failures = [record for record in self.records if not record.get("passed", False)] + if failures: + print_reference_solution_failures(failures) + raise SystemExit(1) return payload + def _ensure_reference_solution_ids_are_execution_tests( + self, + tests: Dict[str, List[Any]], + ) -> None: + """Ensure reference-solution mode is scoped to execution tests.""" + selected_ids = self.config.run.test_ids + if not selected_ids: + return + + execution_ids = {test.id for test in tests["execution"]} + non_execution_ids = [test_id for test_id in selected_ids if test_id not in execution_ids] + if non_execution_ids: + raise ValueError( + "--check-reference-solution only supports execution test id(s): " + + ", ".join(non_execution_ids) + ) + def _run_knowledge_tests(self, tests: List[KnowledgeTest]) -> None: """Run knowledge tests in parallel. @@ -124,8 +173,7 @@ def _run_knowledge_tests(self, tests: List[KnowledgeTest]) -> None: Raises: TestError: If any test fails, stops execution and raises with details. """ - limit = self.config.run.limit or len(tests) - tests_to_run = tests[:limit] + tests_to_run = _limit_tests(tests, self.config) concurrency = self.config.run.concurrency def process_test(test: KnowledgeTest) -> Dict[str, Any]: @@ -172,8 +220,7 @@ def _run_execution_tests(self, tests: List[ExecutionTest]) -> None: Raises: TestError: If any test fails, stops execution and raises with details. """ - limit = self.config.run.limit or len(tests) - tests_to_run = tests[:limit] + tests_to_run = _limit_tests(tests, self.config) concurrency = self.config.run.concurrency def process_test(test: ExecutionTest) -> Dict[str, Any]: @@ -185,11 +232,9 @@ def process_test(test: ExecutionTest) -> Dict[str, Any]: verification_spec = { "static_checks": test.static_checks, "runtime_checks": test.runtime_checks, - "judge_config": test.judge_config, } env_result = self.environment.execute_code(code, verification_spec) correctness = self._score_assertions(env_result.raw) - quality = env_result.raw.get("quality", {}).get("score") if env_result.raw else None return { "test_id": test.id, "type": "execution", @@ -199,7 +244,6 @@ def process_test(test: ExecutionTest) -> Dict[str, Any]: "stdout": env_result.stdout, "stderr": env_result.stderr, "correctness": correctness, - "quality": quality, } except Exception as e: raise TestError(test.id, "execution", e) from e @@ -212,7 +256,52 @@ def process_test(test: ExecutionTest) -> Dict[str, Any]: try: result = future.result() with self._lock: - self.aggregator.add_execution(result["correctness"], result["quality"]) + self.aggregator.add_execution(result["correctness"]) + self.records.append(result) + progress.update(task, advance=1) + except TestError: + for f in futures: + f.cancel() + raise + + def _run_reference_solution_tests(self, tests: List[ExecutionTest]) -> None: + """Run execution tests using their reference_solution as candidate code.""" + tests_to_run = _limit_tests(tests, self.config) + concurrency = self.config.run.concurrency + + def process_test(test: ExecutionTest) -> Dict[str, Any]: + try: + if not test.reference_solution: + raise ValueError("Missing reference_solution") + verification_spec = { + "static_checks": test.static_checks, + "runtime_checks": test.runtime_checks, + } + env_result = self.environment.execute_code(test.reference_solution, verification_spec) + correctness = self._score_assertions(env_result.raw) + return { + "test_id": test.id, + "type": "execution", + "mode": "reference_solution", + "code": test.reference_solution, + "result": env_result.raw, + "stdout": env_result.stdout, + "stderr": env_result.stderr, + "correctness": correctness, + "passed": env_result.success and correctness >= 0.999, + } + except Exception as e: + raise TestError(test.id, "execution", e) from e + + with create_progress() as progress: + task = progress.add_task("Reference solutions", total=len(tests_to_run)) + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = {executor.submit(process_test, test): test for test in tests_to_run} + for future in as_completed(futures): + try: + result = future.result() + with self._lock: + self.aggregator.add_execution(result["correctness"]) self.records.append(result) progress.update(task, advance=1) except TestError: @@ -252,14 +341,29 @@ def _render_execution_prompt(test: ExecutionTest) -> str: @staticmethod def _score_assertions(raw: Dict[str, Any]) -> float: - """Calculate correctness score from assertion results. + """Calculate correctness from static and runtime verifier scores. + + Only sections that actually defined checks contribute to the score. + An empty static (or runtime) section reports a perfect score of 1.0 + (or 0.0), so averaging it in unconditionally would inflate or deflate + correctness for tests that exercise only one section. Args: - raw: Raw result dict from WordPress environment containing assertions. + raw: Raw result dict from WordPress environment. Returns: - Float between 0.0 and 1.0 representing fraction of passed assertions. + Float between 0.0 and 1.0 representing combined verifier score. """ + scores: List[float] = [] + for section in ("static", "runtime"): + result = raw.get(section) or {} + score = result.get("score") + total_weight = result.get("details", {}).get("total_weight", 0) + if isinstance(score, (int, float)) and total_weight: + scores.append(float(score)) + if scores: + return round(sum(scores) / len(scores), 4) + assertions = raw.get("assertions") or [] if not assertions: return 0.0 @@ -315,7 +419,12 @@ def run(self) -> Dict[str, Any]: SystemExit: If a test fails, prints error details and exits with code 1. """ models = self.config.get_models() - tests = load_tests(self.config.dataset) + tests = filter_tests_by_ids(load_tests(self.config.dataset), self.config.run.test_ids) + ensure_test_ids_match_type( + tests, + self.config.run.test_type, + self.config.run.test_ids, + ) if self.config.run.test_type != "knowledge": self.environment.setup() @@ -414,7 +523,6 @@ def run(self) -> Dict[str, Any]: "scores": { "knowledge": summary.knowledge, "correctness": summary.correctness, - "quality": summary.quality, "overall": summary.overall(), }, "results": self.records, @@ -422,8 +530,7 @@ def run(self) -> Dict[str, Any]: def _run_knowledge_tests(self, tests: List[KnowledgeTest]) -> None: """Run knowledge tests in parallel. See BenchmarkRunner._run_knowledge_tests.""" - limit = self.config.run.limit or len(tests) - tests_to_run = tests[:limit] + tests_to_run = _limit_tests(tests, self.config) concurrency = self.config.run.concurrency def process_test(test: KnowledgeTest) -> Dict[str, Any]: @@ -459,8 +566,7 @@ def process_test(test: KnowledgeTest) -> Dict[str, Any]: def _run_execution_tests(self, tests: List[ExecutionTest]) -> None: """Run execution tests in parallel. See BenchmarkRunner._run_execution_tests.""" - limit = self.config.run.limit or len(tests) - tests_to_run = tests[:limit] + tests_to_run = _limit_tests(tests, self.config) concurrency = self.config.run.concurrency def process_test(test: ExecutionTest) -> Dict[str, Any]: @@ -471,17 +577,14 @@ def process_test(test: ExecutionTest) -> Dict[str, Any]: verification_spec = { "static_checks": test.static_checks, "runtime_checks": test.runtime_checks, - "judge_config": test.judge_config, } env_result = self.environment.execute_code(code, verification_spec) correctness = BenchmarkRunner._score_assertions(env_result.raw) - quality = env_result.raw.get("quality", {}).get("score") if env_result.raw else None return { "test_id": test.id, "type": "execution", "code": code, "correctness": correctness, - "quality": quality, } except Exception as e: raise TestError(test.id, "execution", e) from e @@ -494,7 +597,7 @@ def process_test(test: ExecutionTest) -> Dict[str, Any]: try: result = future.result() with self._lock: - self.aggregator.add_execution(result["correctness"], result["quality"]) + self.aggregator.add_execution(result["correctness"]) self.records.append(result) progress.update(task, advance=1) except TestError: diff --git a/python/wp_bench/datasets.py b/python/wp_bench/datasets.py index 97ae55e..b9a52f9 100644 --- a/python/wp_bench/datasets.py +++ b/python/wp_bench/datasets.py @@ -19,13 +19,13 @@ class ExecutionTest: id: str suite: str prompt: str + expected_behavior: str test_type: str category: str difficulty: str requirements: List[str] static_checks: Dict[str, Any] runtime_checks: Dict[str, Any] - judge_config: Optional[Dict[str, Any]] reference_solution: Optional[str] metadata: Dict[str, Any] @@ -52,6 +52,40 @@ def load_tests(config: DatasetConfig) -> Dict[str, List[Any]]: return _load_from_local_files(config) +def filter_tests_by_ids( + tests: Dict[str, List[Any]], + test_ids: List[str], +) -> Dict[str, List[Any]]: + """Filter loaded tests to the requested dataset test IDs.""" + if not test_ids: + return tests + + available_ids = {test.id for group in tests.values() for test in group} + missing_ids = [test_id for test_id in test_ids if test_id not in available_ids] + if missing_ids: + raise ValueError(f"Unknown test id(s): {', '.join(missing_ids)}") + + wanted = set(test_ids) + return { + kind: [test for test in group if test.id in wanted] + for kind, group in tests.items() + } + + +def ensure_test_ids_match_type( + tests: Dict[str, List[Any]], + test_type: str | None, + test_ids: List[str], +) -> None: + """Fail clearly when requested IDs do not match an explicit test type.""" + if not test_ids or test_type is None: + return + if not tests[test_type]: + raise ValueError( + f"No {test_type} tests matched requested test id(s): {', '.join(test_ids)}" + ) + + def _load_from_huggingface(config: DatasetConfig) -> Dict[str, List[Any]]: """Load dataset from Hugging Face Hub (Parquet format).""" dataset = hf_load_dataset( @@ -68,7 +102,6 @@ def _load_from_huggingface(config: DatasetConfig) -> Dict[str, List[Any]]: requirements = _parse_json_field(row.get("requirements", "[]")) static_checks = _parse_json_field(row.get("static_checks", "{}")) runtime_checks = _parse_json_field(row.get("runtime_checks", "{}")) - judge_config = _parse_json_field(row.get("judge_config", "{}")) choices = _parse_json_field(row.get("choices", "[]")) if row.get("test_kind") == "execution": @@ -77,13 +110,13 @@ def _load_from_huggingface(config: DatasetConfig) -> Dict[str, List[Any]]: id=row["id"], suite=row.get("suite", config.name), prompt=row["prompt"], + expected_behavior=row.get("expected_behavior", ""), test_type="execution", category=row.get("category", "general"), difficulty=row.get("difficulty", "unknown"), requirements=requirements if isinstance(requirements, list) else [], static_checks=static_checks if isinstance(static_checks, dict) else {}, runtime_checks=runtime_checks if isinstance(runtime_checks, dict) else {}, - judge_config=judge_config if isinstance(judge_config, dict) else None, reference_solution=row.get("reference_solution"), metadata={}, ) @@ -152,13 +185,13 @@ def _parse_execution_suite(path: Path) -> List[ExecutionTest]: id=test["id"], suite=suite_id, prompt=test["prompt"], + expected_behavior=test.get("expected_behavior", ""), test_type="execution", category=test.get("category", "general"), difficulty=test.get("difficulty", "unknown"), requirements=test.get("requirements", []), static_checks=test.get("static_checks", {}), runtime_checks=test.get("runtime_checks", {}), - judge_config=test.get("judge_config"), reference_solution=test.get("reference_solution"), metadata={"suite_metadata": metadata}, ) diff --git a/python/wp_bench/environment.py b/python/wp_bench/environment.py index 148a497..b0408ab 100644 --- a/python/wp_bench/environment.py +++ b/python/wp_bench/environment.py @@ -45,12 +45,12 @@ def execute_code(self, code: str, verification_spec: Dict[str, Any]) -> Executio **verification_spec, } encoded = base64.b64encode(json.dumps(payload).encode("utf-8")).decode("utf-8") + verifier_path = self._runtime_verifier_path() cmd = [ "wp", - "bench", - "verify", - f"--payload={encoded}", - "--format=json", + "eval-file", + verifier_path, + encoded, ] stdout, stderr, rc = self._exec(cmd) data: Dict[str, Any] = {} @@ -63,6 +63,11 @@ def execute_code(self, code: str, verification_spec: Dict[str, Any]) -> Executio return ExecutionResult(success=success, raw=data, stdout=stdout, stderr=stderr) # Internal helpers -------------------------------------------------- + def _runtime_verifier_path(self) -> str: + if self.config.wp_env_dir: + return "/var/www/html/wp-content/plugins/runtime/verify-runtime.php" + return "/var/www/html/wp-content/plugins/wp-bench-runtime/verify-runtime.php" + def _container_exists(self) -> bool: result = subprocess.run( [ diff --git a/python/wp_bench/output.py b/python/wp_bench/output.py index 63ef9b0..c96eac6 100644 --- a/python/wp_bench/output.py +++ b/python/wp_bench/output.py @@ -82,7 +82,6 @@ def print_comparison_table(results: Dict[str, Dict[str, Any]]) -> None: table.add_column("Model", style="cyan") table.add_column("Knowledge", justify="right") table.add_column("Correctness", justify="right") - table.add_column("Quality", justify="right") table.add_column("Overall", justify="right", style="bold") def _fmt_score(value: float | None) -> str: @@ -94,13 +93,35 @@ def _fmt_score(value: float | None) -> str: model_name, _fmt_score(scores["knowledge"]), _fmt_score(scores["correctness"]), - _fmt_score(scores["quality"]), f"{scores['overall']*100:.1f}%", ) console.print(table) +def print_reference_solution_failures(records: list[Dict[str, Any]]) -> None: + """Print reference solution failures in a compact table.""" + table = Table(title="Reference Solution Failures") + table.add_column("Test ID", style="cyan") + table.add_column("Correctness", justify="right") + table.add_column("Static", justify="right") + table.add_column("Runtime", justify="right") + + def _fmt_score(value: Any) -> str: + return f"{value*100:.1f}%" if isinstance(value, (int, float)) else "N/A" + + for record in records: + result = record.get("result") or {} + table.add_row( + str(record.get("test_id", "")), + _fmt_score(record.get("correctness")), + _fmt_score((result.get("static") or {}).get("score")), + _fmt_score((result.get("runtime") or {}).get("score")), + ) + + console.print(table) + + def create_progress() -> Progress: """Create a Progress instance for tracking test execution.""" return Progress() diff --git a/python/wp_bench/scoring.py b/python/wp_bench/scoring.py index 6ef2759..6122db5 100644 --- a/python/wp_bench/scoring.py +++ b/python/wp_bench/scoring.py @@ -10,9 +10,8 @@ class ScoreBreakdown: knowledge: Optional[float] = None correctness: Optional[float] = None - quality: Optional[float] = None weights: Dict[str, float] = field( - default_factory=lambda: {"knowledge": 0.3, "correctness": 0.4, "quality": 0.3} + default_factory=lambda: {"knowledge": 0.3, "correctness": 0.7} ) def overall(self) -> float: @@ -28,12 +27,9 @@ class ScoreAggregator: def __init__(self) -> None: self.knowledge_scores: List[float] = [] self.correctness_scores: List[float] = [] - self.quality_scores: List[float] = [] - def add_execution(self, correctness: float, quality: float | None = None) -> None: + def add_execution(self, correctness: float) -> None: self.correctness_scores.append(correctness) - if quality is not None: - self.quality_scores.append(quality) def add_knowledge(self, score: float) -> None: self.knowledge_scores.append(score) @@ -44,6 +40,4 @@ def finalize(self) -> ScoreBreakdown: breakdown.knowledge = mean(self.knowledge_scores) if self.correctness_scores: breakdown.correctness = mean(self.correctness_scores) - if self.quality_scores: - breakdown.quality = mean(self.quality_scores) return breakdown diff --git a/runtime/.wp-env.json b/runtime/.wp-env.json index 096fc01..b98da51 100644 --- a/runtime/.wp-env.json +++ b/runtime/.wp-env.json @@ -1,5 +1,5 @@ { - "core": "WordPress/WordPress#6.9", + "core": "WordPress/WordPress#7.0", "phpVersion": "8.2", "plugins": [ "./" diff --git a/runtime/Dockerfile b/runtime/Dockerfile index d06bff4..5b48c04 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -1,6 +1,6 @@ FROM wordpress:cli-php8.2 -ENV WORDPRESS_VERSION=6.9 \ +ENV WORDPRESS_VERSION=7.0 \ WP_CLI_ALLOW_ROOT=1 RUN apt-get update && apt-get install -y default-mysql-client && rm -rf /var/lib/apt/lists/* @@ -11,6 +11,7 @@ RUN rm -rf ./* && \ wp core download --version=${WORDPRESS_VERSION} --force COPY wp-bench-runtime.php /var/www/html/wp-content/plugins/wp-bench-runtime/wp-bench-runtime.php +COPY verify-runtime.php /var/www/html/wp-content/plugins/wp-bench-runtime/verify-runtime.php COPY src /var/www/html/wp-content/plugins/wp-bench-runtime/src COPY docker-entrypoint.sh /usr/local/bin/wp-bench-entrypoint RUN chmod +x /usr/local/bin/wp-bench-entrypoint diff --git a/runtime/src/class-cli-verifier.php b/runtime/src/class-cli-verifier.php deleted file mode 100644 index 76018e3..0000000 --- a/runtime/src/class-cli-verifier.php +++ /dev/null @@ -1,76 +0,0 @@ -static_analysis = new Static_Analysis(); - $this->sandbox = new Sandbox(); - } - - /** - * Execute verification from a Base64 payload. - * - * ## OPTIONS - * - * [--payload=] - * : Base64 encoded JSON payload containing code and verification spec. - * - * [--format=] - * : Output format (json or table). - */ - public function __invoke( array $args, array $assoc_args ): void { - $payload_b64 = $assoc_args['payload'] ?? null; - if ( ! $payload_b64 ) { - WP_CLI::error( 'Missing --payload argument.' ); - } - - $payload_json = base64_decode( (string) $payload_b64, true ); - if ( false === $payload_json ) { - WP_CLI::error( 'Invalid payload encoding.' ); - } - - $payload = json_decode( $payload_json, true ); - if ( ! is_array( $payload ) ) { - WP_CLI::error( 'Payload is not valid JSON.' ); - } - - $code_value = $payload['code'] ?? ''; - $code = is_string( $code_value ) ? $code_value : ''; - - $static_checks = $payload['static_checks'] ?? []; - $runtime_checks = $payload['runtime_checks'] ?? []; - - $static_result = $this->static_analysis->check( $code, is_array( $static_checks ) ? $static_checks : [] ); - $runtime_result = $this->sandbox->execute_and_verify( $code, is_array( $runtime_checks ) ? $runtime_checks : [] ); - - $success = $runtime_result['score'] >= 0.999 && $static_result['score'] >= 0.999; - - $response = [ - 'success' => $success, - 'static' => $static_result, - 'runtime' => $runtime_result, - 'assertions' => $runtime_result['details']['assertions'] ?? [], - 'version' => '1.0.0', - ]; - - $format = $assoc_args['format'] ?? 'json'; - if ( 'json' === $format ) { - WP_CLI::line( \wp_json_encode( $response, JSON_PRETTY_PRINT ) ); - return; - } - - WP_CLI::success( sprintf( 'Static: %.2f, Runtime: %.2f', $static_result['score'], $runtime_result['score'] ) ); - } -} diff --git a/runtime/src/class-sandbox.php b/runtime/src/class-sandbox.php index 8030b9f..eb94786 100644 --- a/runtime/src/class-sandbox.php +++ b/runtime/src/class-sandbox.php @@ -149,21 +149,23 @@ private function run_assertion( array $assertion ): array { ]; try { - $result = match ( $type ) { - 'function_exists' => $this->assert_function_exists( $target, $result ), - 'class_exists' => $this->assert_class_exists( $target, $result ), - 'shortcode_exists' => $this->assert_shortcode_exists( $target, $result ), - 'hook_registered' => $this->assert_hook_registered( $target, $assertion, $result ), - 'output_contains' => $this->assert_output_contains( $target, $expected_str, $result ), - 'output_equals' => $this->assert_output_equals( $target, $expected_str, $result ), - 'output_matches' => $this->assert_output_matches( $target, $expected_str, $result ), - 'returns_value' => $this->assert_returns_value( $target, $expected, $result ), - 'query_result' => $this->assert_query_result( $target, $expected, $result ), - 'option_value' => $this->assert_option_value( $target, $expected, $result ), - 'post_meta_value' => $this->assert_post_meta_value( $assertion, $result ), - 'custom_assertion' => $this->assert_custom( $assertion, $result ), - default => array_merge( $result, [ 'error' => "Unknown assertion type: {$type}" ] ), - }; + $result = match ( $type ) { + 'function_exists' => $this->assert_function_exists( $target, $result ), + 'class_exists' => $this->assert_class_exists( $target, $result ), + 'shortcode_exists' => $this->assert_shortcode_exists( $target, $result ), + 'hook_registered' => $this->assert_hook_registered( $target, $assertion, $result ), + 'output_contains' => $this->assert_output_contains( $target, $expected_str, $result ), + 'output_not_contains' => $this->assert_output_not_contains( $target, $expected_str, $result ), + 'output_equals' => $this->assert_output_equals( $target, $expected_str, $result ), + 'output_matches' => $this->assert_output_matches( $target, $expected_str, $result ), + 'rest_response' => $this->assert_rest_response( $target, $assertion, $result ), + 'returns_value' => $this->assert_returns_value( $target, $expected, $result ), + 'query_result' => $this->assert_query_result( $target, $expected, $result ), + 'option_value' => $this->assert_option_value( $target, $expected, $result ), + 'post_meta_value' => $this->assert_post_meta_value( $assertion, $result ), + 'custom_assertion' => $this->assert_custom( $assertion, $result ), + default => array_merge( $result, [ 'error' => "Unknown assertion type: {$type}" ] ), + }; } catch ( \Throwable $e ) { $result['error'] = $e->getMessage(); } @@ -274,6 +276,26 @@ private function assert_output_contains( string $code_or_callable, string $expec return $result; } + /** + * Assert that code output does not contain a string. + * + * @param string $code_or_callable Code to execute. + * @param string $expected Forbidden substring. + * @param array $result Base result array. + * + * @return array + */ + private function assert_output_not_contains( string $code_or_callable, string $expected, array $result ): array { + ob_start(); + $this->safe_eval( $code_or_callable ); + $output = ob_get_clean(); + $output = false !== $output ? $output : ''; + + $result['actual'] = $output; + $result['passed'] = ! str_contains( $output, $expected ); + return $result; + } + /** * Assert that code output equals a string. * @@ -320,6 +342,106 @@ private function assert_output_matches( string $code_or_callable, string $patter return $result; } + /** + * Assert a REST API response by dispatching through WordPress internals. + * + * @param string $route REST route, e.g. /demo/v1/item. + * @param array $assertion Full assertion config. + * @param array $result Base result array. + * + * @return array + */ + private function assert_rest_response( string $route, array $assertion, array $result ): array { + if ( ! did_action( 'rest_api_init' ) ) { + do_action( 'rest_api_init' ); + } + + $method_value = $assertion['method'] ?? 'GET'; + $method = is_string( $method_value ) ? strtoupper( $method_value ) : 'GET'; + $request = new \WP_REST_Request( $method, $route ); + + $params = $assertion['params'] ?? []; + if ( is_array( $params ) ) { + foreach ( $params as $key => $value ) { + if ( is_string( $key ) ) { + $request->set_param( $key, $value ); + } + } + } + + $headers = $assertion['headers'] ?? []; + if ( is_array( $headers ) ) { + foreach ( $headers as $key => $value ) { + if ( is_string( $key ) && is_string( $value ) ) { + $request->set_header( $key, $value ); + } + } + } + + if ( array_key_exists( 'body', $assertion ) ) { + $request->set_body( wp_json_encode( $assertion['body'] ) ); + $request->set_header( 'content-type', 'application/json' ); + } + + $response = rest_do_request( $request ); + $server = rest_get_server(); + $data = $server->response_to_data( $response, false ); + $status = $response->get_status(); + + $result['actual'] = [ + 'status' => $status, + 'data' => $data, + ]; + + $expected_status = $assertion['expected_status'] ?? null; + if ( null !== $expected_status ) { + if ( ! is_numeric( $expected_status ) ) { + $result['error'] = 'rest_response expected_status must be numeric.'; + return $result; + } + if ( (int) $expected_status !== $status ) { + $result['expected'] = [ 'status' => (int) $expected_status ]; + return $result; + } + } + + if ( array_key_exists( 'expected_data', $assertion ) ) { + $result['expected'] = $assertion['expected_data']; + // phpcs:ignore Universal.Operators.StrictComparisons.LooseEqual -- REST data can contain arrays/objects. + $result['passed'] = $assertion['expected_data'] == $data; + return $result; + } + + $encoded = wp_json_encode( $data ); + $encoded = is_string( $encoded ) ? $encoded : ''; + + $contains = $assertion['body_contains'] ?? null; + if ( is_string( $contains ) && ! str_contains( $encoded, $contains ) ) { + $result['expected'] = [ 'body_contains' => $contains ]; + return $result; + } + + $not_contains = $assertion['body_not_contains'] ?? null; + if ( is_string( $not_contains ) && str_contains( $encoded, $not_contains ) ) { + $result['expected'] = [ 'body_not_contains' => $not_contains ]; + return $result; + } + + // Require at least one expectation; an assertion that checks nothing + // must not silently pass for any response. + $has_expectation = null !== $expected_status + || array_key_exists( 'expected_data', $assertion ) + || is_string( $contains ) + || is_string( $not_contains ); + if ( ! $has_expectation ) { + $result['error'] = 'rest_response assertion requires expected_status, expected_data, body_contains, or body_not_contains.'; + return $result; + } + + $result['passed'] = true; + return $result; + } + /** * Assert that code returns a specific value. * diff --git a/runtime/src/class-static-analysis.php b/runtime/src/class-static-analysis.php index 21cacb2..72d7299 100644 --- a/runtime/src/class-static-analysis.php +++ b/runtime/src/class-static-analysis.php @@ -206,9 +206,10 @@ public function check_security( string $code ): array { * @return bool True if pattern matches. */ private function safe_preg_match( string $pattern, string $subject ): bool { - // Ensure pattern has delimiters. + // Ensure pattern has delimiters. Delimiterless patterns are still regexes, + // but use ~ so WordPress slugs like wpbp/example do not need escaping. if ( ! preg_match( '/^[\/\#\~\@\!]/', $pattern ) ) { - $pattern = '/' . $pattern . '/'; + $pattern = '~' . str_replace( '~', '\~', $pattern ) . '~'; } // Suppress regex warnings. diff --git a/runtime/src/class-verifier.php b/runtime/src/class-verifier.php new file mode 100644 index 0000000..8fc491a --- /dev/null +++ b/runtime/src/class-verifier.php @@ -0,0 +1,43 @@ +static_analysis = new Static_Analysis(); + $this->sandbox = new Sandbox(); + } + + /** + * Verify candidate code against static and runtime checks. + * + * @param array $payload Verification payload. + * @return array + */ + public function verify_payload( array $payload ): array { + $code_value = $payload['code'] ?? ''; + $code = is_string( $code_value ) ? $code_value : ''; + + $static_checks = $payload['static_checks'] ?? []; + $runtime_checks = $payload['runtime_checks'] ?? []; + + $static_result = $this->static_analysis->check( $code, is_array( $static_checks ) ? $static_checks : [] ); + $runtime_result = $this->sandbox->execute_and_verify( $code, is_array( $runtime_checks ) ? $runtime_checks : [] ); + + return [ + 'success' => $runtime_result['score'] >= 0.999 && $static_result['score'] >= 0.999, + 'static' => $static_result, + 'runtime' => $runtime_result, + 'assertions' => $runtime_result['details']['assertions'] ?? [], + 'version' => '1.0.0', + ]; + } +} diff --git a/runtime/verify-runtime.php b/runtime/verify-runtime.php new file mode 100644 index 0000000..f01f1d0 --- /dev/null +++ b/runtime/verify-runtime.php @@ -0,0 +1,28 @@ +verify_payload( $payload ), JSON_PRETTY_PRINT ) ); diff --git a/runtime/wp-bench-runtime.php b/runtime/wp-bench-runtime.php index 7b8de8a..687f801 100644 --- a/runtime/wp-bench-runtime.php +++ b/runtime/wp-bench-runtime.php @@ -20,9 +20,5 @@ } else { require_once __DIR__ . '/src/class-sandbox.php'; require_once __DIR__ . '/src/class-static-analysis.php'; - require_once __DIR__ . '/src/class-cli-verifier.php'; -} - -if ( defined( 'WP_CLI' ) && WP_CLI ) { - \WP_CLI::add_command( 'bench verify', \WPBench\Runtime\CLI_Verifier::class ); + require_once __DIR__ . '/src/class-verifier.php'; } diff --git a/wp-bench.example.yaml b/wp-bench.example.yaml index 4ba3351..e7fd34d 100644 --- a/wp-bench.example.yaml +++ b/wp-bench.example.yaml @@ -30,6 +30,8 @@ grader: run: suite: wp-core-v1 # limit: 10 # limit tests per model (null = all) + # test_ids: [] # optional explicit test IDs to run + # dry_run: false # load/filter tests without calling models concurrency: 5 # Output paths