From 6c00d581283c1c4a3cbf717fe917e0168c8af30e Mon Sep 17 00:00:00 2001 From: Finbarr Timbers Date: Mon, 4 May 2026 14:51:02 -0600 Subject: [PATCH 1/8] added ifbench --- pyproject.toml | 5 + src/olmo_eval/common/metrics/__init__.py | 10 ++ src/olmo_eval/common/scorers/__init__.py | 2 + uv.lock | 168 +++++++++++++++++------ 4 files changed, 146 insertions(+), 39 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7051cc1b5..8ddc11809 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,9 @@ dependencies = [ "textual-plot>=0.10.1", "tree-sitter>=0.24.0", "tree-sitter-python>=0.23.0,<0.26.0", + "ifbench @ git+https://github.com/allenai/IFBench.git", + # syllapy (transitive via ifbench) imports pkg_resources, which setuptools>=81 removes. + "setuptools<81", ] [project.urls] @@ -222,6 +225,8 @@ allowed-unresolved-imports = [ "google.cloud.**", "matplotlib.**", "scipy.**", + "instructions_registry.**", + "instructions.**", ] [tool.gantry] diff --git a/src/olmo_eval/common/metrics/__init__.py b/src/olmo_eval/common/metrics/__init__.py index f4b06406a..c20521f9e 100644 --- a/src/olmo_eval/common/metrics/__init__.py +++ b/src/olmo_eval/common/metrics/__init__.py @@ -19,6 +19,12 @@ SQuADF1Metric, ToolAccuracyMetric, ) +from .ifeval import ( + IFEvalInstLooseAccuracy, + IFEvalInstStrictAccuracy, + IFEvalPromptLooseAccuracy, + IFEvalPromptStrictAccuracy, +) __all__ = [ "AccuracyMetric", @@ -27,6 +33,10 @@ "CorpusPerplexityMetric", "F1Metric", "GreedyAccuracyMetric", + "IFEvalInstLooseAccuracy", + "IFEvalInstStrictAccuracy", + "IFEvalPromptLooseAccuracy", + "IFEvalPromptStrictAccuracy", "LogprobMCAccuracyMetric", "LogprobPerCharMCAccuracyMetric", "LogprobPerTokenMCAccuracyMetric", diff --git a/src/olmo_eval/common/scorers/__init__.py b/src/olmo_eval/common/scorers/__init__.py index 38b078863..83e131316 100644 --- a/src/olmo_eval/common/scorers/__init__.py +++ b/src/olmo_eval/common/scorers/__init__.py @@ -15,6 +15,7 @@ ) from .code_execution import CodeExecutionScorer, MultiplEScorer from .execution import ContextScorer, ExecutionScorer, SandboxRequiredError +from .ifeval import IFEvalScorer from .llm_judge import ( JudgeFn, LLMJudgeScorer, @@ -45,6 +46,7 @@ "ExactMatchScorer", "ExecutionScorer", "F1Scorer", + "IFEvalScorer", "JudgeFn", "LLMJudgeScorer", "LogprobScorer", diff --git a/uv.lock b/uv.lock index d673a4c76..c8f30cfc1 100644 --- a/uv.lock +++ b/uv.lock @@ -50,6 +50,15 @@ conflicts = [[ [manifest] overrides = [{ name = "transformers", specifier = ">=5.4.0" }] +[[package]] +name = "absl-py" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/64/c7/8de93764ad66968d19329a7e0c147a2bb3c7054c554d4a119111b8f9440f/absl_py-2.4.0.tar.gz", hash = "sha256:8c6af82722b35cf71e0f4d1d47dcaebfff286e27110a99fc359349b247dfb5d4", size = 116543, upload-time = "2026-01-28T10:17:05.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750, upload-time = "2026-01-28T10:17:04.19Z" }, +] + [[package]] name = "aiofile" version = "3.9.0" @@ -1906,6 +1915,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "emoji" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/78/0d2db9382c92a163d7095fc08efff7800880f830a152cfced40161e7638d/emoji-2.15.0.tar.gz", hash = "sha256:eae4ab7d86456a70a00a985125a03263a5eac54cd55e51d7e184b1ed3b6757e4", size = 615483, upload-time = "2025-09-21T12:13:02.755Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/5e/4b5aaaabddfacfe36ba7768817bd1f71a7a810a43705e531f3ae4c690767/emoji-2.15.0-py3-none-any.whl", hash = "sha256:205296793d66a89d88af4688fa57fd6496732eb48917a87175a023c8138995eb", size = 608433, upload-time = "2025-09-21T12:13:01.197Z" }, +] + [[package]] name = "et-xmlfile" version = "2.0.0" @@ -3092,6 +3110,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629, upload-time = "2026-04-22T16:42:40.909Z" }, ] +[[package]] +name = "ifbench" +version = "0.1.0" +source = { git = "https://github.com/allenai/IFBench.git#1091c4c3de6c1f6ed12c012ed68f11ea450b0117" } +dependencies = [ + { name = "absl-py" }, + { name = "emoji" }, + { name = "httpx" }, + { name = "immutabledict" }, + { name = "langdetect" }, + { name = "nltk" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "setuptools" }, + { name = "syllapy" }, + { name = "tqdm" }, + { name = "unicodedata2" }, +] + [[package]] name = "ijson" version = "3.5.0" @@ -3155,6 +3192,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/28/96711503245339084c8086b892c47415895eba49782d6cc52d9f4ee50301/ijson-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4f24b78d4ef028d17eb57ad1b16c0aed4a17bdd9badbf232dc5d9305b7e13854", size = 58965, upload-time = "2026-02-24T03:58:11.278Z" }, ] +[[package]] +name = "immutabledict" +version = "4.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/e6/718471048fea0366c3e3d1df3acfd914ca66d571cdffcf6d37bbcd725708/immutabledict-4.3.1.tar.gz", hash = "sha256:f844a669106cfdc73f47b1a9da003782fb17dc955a54c80972e0d93d1c63c514", size = 7806, upload-time = "2026-02-15T10:32:34.668Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/ce/f9018bf69ae91b273b6391a095e7c93fa5e1617f25b6ba81ad4b20c9df10/immutabledict-4.3.1-py3-none-any.whl", hash = "sha256:c9facdc0ff30fdb8e35bd16532026cac472a549e182c94fa201b51b25e4bf7bf", size = 5000, upload-time = "2026-02-15T10:32:33.672Z" }, +] + [[package]] name = "importlib-metadata" version = "8.7.1" @@ -3799,6 +3845,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/70/05b685ea2dffcb2adbf3cdcea5d8865b7bc66f67249084cf845012a0ff13/kubernetes-35.0.0-py2.py3-none-any.whl", hash = "sha256:39e2b33b46e5834ef6c3985ebfe2047ab39135d41de51ce7641a7ca5b372a13d", size = 2017602, upload-time = "2026-01-16T01:05:25.991Z" }, ] +[[package]] +name = "langdetect" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0", size = 981474, upload-time = "2021-05-07T07:54:13.562Z" } + [[package]] name = "lark" version = "1.2.2" @@ -4395,7 +4450,7 @@ dependencies = [ { name = "httpx", marker = "sys_platform != 'darwin'" }, { name = "jmespath", marker = "sys_platform != 'darwin'" }, { name = "pydantic", marker = "sys_platform != 'darwin'" }, - { name = "setuptools", version = "80.10.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "setuptools", marker = "sys_platform != 'darwin'" }, { name = "starlette", version = "0.52.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, { name = "supervisor", marker = "sys_platform != 'darwin'" }, ] @@ -4747,6 +4802,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, ] +[[package]] +name = "nltk" +version = "3.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -5150,11 +5220,13 @@ source = { editable = "." } dependencies = [ { name = "click" }, { name = "datasets" }, + { name = "ifbench" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'darwin' and extra == 'extra-18-olmo-eval-internal-agents') or (sys_platform != 'darwin' and extra == 'extra-18-olmo-eval-internal-clients') or (sys_platform != 'darwin' and extra != 'extra-18-olmo-eval-internal-openhands') or (extra == 'extra-18-olmo-eval-internal-agents' and extra == 'extra-18-olmo-eval-internal-openhands') or (extra == 'extra-18-olmo-eval-internal-clients' and extra == 'extra-18-olmo-eval-internal-openhands') or (extra == 'extra-18-olmo-eval-internal-openhands' and extra == 'extra-18-olmo-eval-internal-vllm')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin' or extra == 'extra-18-olmo-eval-internal-openhands'" }, { name = "omegaconf" }, { name = "prometheus-client" }, { name = "rich" }, + { name = "setuptools" }, { name = "sympy" }, { name = "textual-plot" }, { name = "tree-sitter" }, @@ -5241,6 +5313,7 @@ requires-dist = [ { name = "click", specifier = "~=8.3.2" }, { name = "datasets", specifier = ">=3.2.0" }, { name = "httpx", marker = "extra == 'agents'", specifier = "~=0.28.1" }, + { name = "ifbench", git = "https://github.com/allenai/IFBench.git" }, { name = "litellm", marker = "extra == 'litellm'", specifier = "~=1.80.11" }, { name = "matplotlib", marker = "extra == 'analysis'", specifier = "~=3.10.8" }, { name = "numpy", specifier = ">=1.20.0" }, @@ -5256,6 +5329,7 @@ requires-dist = [ { name = "rich", specifier = "~=14.3.4" }, { name = "scipy", marker = "extra == 'analysis'", specifier = "~=1.17.1" }, { name = "seaborn", marker = "extra == 'analysis'", specifier = "~=0.13.2" }, + { name = "setuptools", specifier = "<81" }, { name = "smart-open", extras = ["s3"], marker = "extra == 's3'", specifier = ">=7.0.0" }, { name = "sqlalchemy", marker = "extra == 'postgres'", specifier = ">=2.0.49" }, { name = "swe-rex", extras = ["modal"], marker = "extra == 'sandbox'", git = "https://github.com/jdahm/SWE-ReX.git?rev=johannd%2Fai2" }, @@ -5562,7 +5636,7 @@ dependencies = [ { name = "rapidfuzz" }, { name = "redis" }, { name = "requests" }, - { name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "setuptools" }, { name = "shellingham" }, { name = "sqlalchemy", extra = ["asyncio"], marker = "extra == 'extra-18-olmo-eval-internal-openhands'" }, { name = "sse-starlette" }, @@ -6960,9 +7034,9 @@ name = "pydantic-settings" version = "2.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic", marker = "sys_platform != 'darwin' or extra == 'extra-18-olmo-eval-internal-agents' or extra == 'extra-18-olmo-eval-internal-openhands'" }, - { name = "python-dotenv", marker = "sys_platform != 'darwin' or extra == 'extra-18-olmo-eval-internal-agents' or extra == 'extra-18-olmo-eval-internal-openhands'" }, - { name = "typing-inspection", marker = "sys_platform != 'darwin' or extra == 'extra-18-olmo-eval-internal-agents' or extra == 'extra-18-olmo-eval-internal-openhands'" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/98/c8345dccdc31de4228c039a98f6467a941e39558da41c1744fbe29fa5666/pydantic_settings-2.14.0.tar.gz", hash = "sha256:24285fd4b0e0c06507dd9fdfd331ee23794305352aaec8fc4eb92d4047aeb67d", size = 235709, upload-time = "2026-04-20T13:37:40.293Z" } wheels = [ @@ -11035,42 +11109,11 @@ wheels = [ name = "setuptools" version = "80.10.2" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] sdist = { url = "https://files.pythonhosted.org/packages/76/95/faf61eb8363f26aa7e1d762267a8d602a1b26d4f3a1e758e92cb3cb8b054/setuptools-80.10.2.tar.gz", hash = "sha256:8b0e9d10c784bf7d262c4e5ec5d4ec94127ce206e8738f29a437945fbc219b70", size = 1200343, upload-time = "2026-01-25T22:38:17.252Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" }, ] -[[package]] -name = "setuptools" -version = "82.0.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" @@ -11344,6 +11387,15 @@ modal = [ { name = "modal" }, ] +[[package]] +name = "syllapy" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/a1/7bc1ce4852e14ab5f3153262639742ae63fbfa626507dac4ab919a1e5232/syllapy-0.7.2.tar.gz", hash = "sha256:e55a7ad97d8b232e174b83f91b8f9be0c355d2a8e1208c7f6229055189605564", size = 25561, upload-time = "2022-08-29T01:55:03.366Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/cc/ffc9bddc146f14e8792a9b05b2bd1bc5f23f3b752a06e96b244780ce55b9/syllapy-0.7.2-py3-none-any.whl", hash = "sha256:198a7413033c32d7b31e21962efb3f284bcea80d3346e954b938ca1ebe6bee20", size = 24882, upload-time = "2022-08-29T01:55:01.1Z" }, +] + [[package]] name = "sympy" version = "1.14.0" @@ -11646,7 +11698,7 @@ dependencies = [ { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools", version = "80.10.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "setuptools", marker = "sys_platform != 'darwin'" }, { name = "sympy", marker = "sys_platform != 'darwin'" }, { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform != 'darwin'" }, @@ -12072,6 +12124,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/e1/7ec67882ad8fc9f86384bef6421fa252c9cbe5744f8df6ce77afc9eca1f5/uncalled_for-0.3.1-py3-none-any.whl", hash = "sha256:074cdc92da8356278f93d0ded6f2a66dd883dbecaf9bc89437646ee2289cc200", size = 11361, upload-time = "2026-04-07T13:05:05.341Z" }, ] +[[package]] +name = "unicodedata2" +version = "17.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/cb/520721a715da85530e21c71953b9b9a85a44e0d80d3b34bf9303c422d208/unicodedata2-17.0.1.tar.gz", hash = "sha256:d79943d153f5f6bfbe3f55a5ec611985184bda37fcedb3ecc75322d82ae6ad3b", size = 679856, upload-time = "2026-02-12T10:05:41.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/7b/52359e301fe53fc742e1f9ef8b2aec4b8b0d5509c365eb1819072a46506e/unicodedata2-17.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4bd5990d17c8415ed44d4cb2642a06c0377551540d151c458dcdff5426719661", size = 973395, upload-time = "2026-02-12T10:04:29.234Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b8/d2140512697cd1085b2a7004cd7813c1e6b26033587f19aa5f02ab25a54a/unicodedata2-17.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cbc1d0a458294ae5af4a7ca560dae976a6a88534fc2c61289f0b8ab25684aeb1", size = 495277, upload-time = "2026-02-12T10:04:30.644Z" }, + { url = "https://files.pythonhosted.org/packages/56/f5/17f8ad3a780043d2aff2b8bbb60280ecebc54cf45596942942fae28cd33d/unicodedata2-17.0.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88a32dc5372787f9cc0bfaf0a39b38a72e3f703111efa84a550cc228b85f2a30", size = 531996, upload-time = "2026-02-12T10:04:32.274Z" }, + { url = "https://files.pythonhosted.org/packages/32/b5/f1ef0d3944f30ebc0fc436a5be98e53969d76f975ce1005aeb5d99acc7be/unicodedata2-17.0.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b9cceda1f25498670245c79a6a1b3e2bc310b629693934075c388cc02585a686", size = 531920, upload-time = "2026-02-12T10:04:33.617Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d0/837bec637a3076496627d24d9921f7b5855122e2558d25417f1146f9e63f/unicodedata2-17.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c9eee5d5bad5ced178f14d946f106dff4419cb76000b77b49bc78b50fabe7b36", size = 530573, upload-time = "2026-02-12T10:04:35.17Z" }, + { url = "https://files.pythonhosted.org/packages/1d/89/c73a4e430d59d80f9e07ffb85294cfcb47c5050249728b1049e7b51ed351/unicodedata2-17.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e9e01d1525f57695fbb4aed3efad8814ae6fdbb9a0b710ec9dec18d1b07dd2f0", size = 531410, upload-time = "2026-02-12T10:04:36.631Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f7/731641438894f9c11e5281b6ebb7cc01467565c4a63e2c9be72a0d0dd741/unicodedata2-17.0.1-cp312-cp312-win32.whl", hash = "sha256:fd6d360f2063547e9ed8e01e9335bbc0fe0b71dd19649a614a76acb37e72eba4", size = 483974, upload-time = "2026-02-12T10:04:38.145Z" }, + { url = "https://files.pythonhosted.org/packages/dd/4d/24523557cfd632fc70fc031a83b280ae60d618e55afd54a6298c5d8b80f7/unicodedata2-17.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:d1439ad3ee0daace878196de4466a86aa5015cb244b9b1d5d00db74344649722", size = 484194, upload-time = "2026-02-12T10:04:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/4e/93/b30e22fc5c06827a112bf399fa9d583ccd3209ae5b6bb9fa101823d85cc3/unicodedata2-17.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e700dbedf1ec4466e1f36d2172fa2c856141e8d8ee256f5d1ef32507f7afa2b8", size = 973390, upload-time = "2026-02-12T10:04:41.854Z" }, + { url = "https://files.pythonhosted.org/packages/6b/71/61d7694139f62e9bbfc9d51c36e1fb7ce578d530074a75573ca59fb76c97/unicodedata2-17.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc83524357ea99933ff0b33eb781f35381b6da0e89fb466bbfd9e52d3b701602", size = 495284, upload-time = "2026-02-12T10:04:43.184Z" }, + { url = "https://files.pythonhosted.org/packages/67/c8/4a227db7c366c10e6135dc53c7389060f295ae234e696ae41406ace2c92c/unicodedata2-17.0.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3aefd1fd71d95e87ba8df580f54ae4d52076e4394eddc97d0f4609849f01ae4", size = 531970, upload-time = "2026-02-12T10:04:44.532Z" }, + { url = "https://files.pythonhosted.org/packages/ae/62/4dfe56801216acfff25c07a68d2a0b5330e802fbd1f89cf8ba4ae3374b2e/unicodedata2-17.0.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8a4c70a9b1438b2303ab577c6c3a80c650952c76f496418ef7ab6755ff1ffbea", size = 531750, upload-time = "2026-02-12T10:04:45.952Z" }, + { url = "https://files.pythonhosted.org/packages/4a/eb/c078cc851bf3212e56711bbd56e63579645f9a2d14cc1aa04397b90ba552/unicodedata2-17.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:761c53084ca0f3a1beda52f1c3980a0d561221f860a56a1342087c67879fd547", size = 530549, upload-time = "2026-02-12T10:04:47.83Z" }, + { url = "https://files.pythonhosted.org/packages/a5/c9/f95deb4a197bdd9af29bcbd8091a9847431ca16fcc5f9981efcdbe0e057e/unicodedata2-17.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ba5f094d6ff67c130fe36fbdad64371c7b2e7672f4e35166f98ae6c88646ccda", size = 531457, upload-time = "2026-02-12T10:04:49.254Z" }, + { url = "https://files.pythonhosted.org/packages/30/f6/de84a37d8fbbb2cd57ce6b6ba1af8bba9ffbecd05c046630e47629e74147/unicodedata2-17.0.1-cp313-cp313-win32.whl", hash = "sha256:e314cd8a8f64aa0d2bcf940b9d8e742cb726566574bf241a02b23d0ea9c779c9", size = 483971, upload-time = "2026-02-12T10:04:51.11Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7e/94116c542f4348b2def3e34654b4e1af66862e93e884c7405b361c6e92d6/unicodedata2-17.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:7ec69751488663ad5644b2bce67be9bbf9a178ba9ad3089ca652b05cf5a2aa49", size = 484193, upload-time = "2026-02-12T10:04:52.585Z" }, + { url = "https://files.pythonhosted.org/packages/1c/37/abd5d5babb5d7a898c06b3f4635a81741beae0ff96986ef2656614bb4cb9/unicodedata2-17.0.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6ac68ff0d4d8a1c1ccece9a29f5c2dd34c456a6ef5631eea7d926cbf288be3c7", size = 973471, upload-time = "2026-02-12T10:04:53.984Z" }, + { url = "https://files.pythonhosted.org/packages/44/41/749dfd9ebef778425a0d1b3bdd47d70413743d0df8e618663db930071122/unicodedata2-17.0.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9a8e82d999809157006c8f5942ef86d980d770798f77bffa6ec1c99150bfe1f0", size = 495320, upload-time = "2026-02-12T10:04:55.498Z" }, + { url = "https://files.pythonhosted.org/packages/08/90/7c929314a0fe1a867f795d220148bef7943db9a462e68ec9332cdf6402b4/unicodedata2-17.0.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21c3307f72d85b4e886037f100dc1ea10fddaea4086d81136712a1b7c278ec1d", size = 531904, upload-time = "2026-02-12T10:04:57.067Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b1/fcc2231f554f6c2d2b2132b6e4d3526e81961b0ad1593dd869872de0f75e/unicodedata2-17.0.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:730c3e33320dcfbafd6b1e91d5233a07d5c44f3ff713c0c16d76f72debeb0ac4", size = 531691, upload-time = "2026-02-12T10:04:58.4Z" }, + { url = "https://files.pythonhosted.org/packages/df/00/b06450a610c53a6e2aa3ca9bcf6863028b6c95700a3b0b99ab31c77b00ca/unicodedata2-17.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0d1c9efe0380568b68efc6620b3b5780845b17a1bd916b27c890a4d4a80f8078", size = 530495, upload-time = "2026-02-12T10:04:59.831Z" }, + { url = "https://files.pythonhosted.org/packages/14/04/35df12e1febace8696311bf5f8800172831885340e9d58b2ab6e59a70364/unicodedata2-17.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:410d6461ff1a98bea61c960a993e19ac60cff878e0d7774cc0c9ad6a60409f92", size = 531388, upload-time = "2026-02-12T10:05:01.376Z" }, + { url = "https://files.pythonhosted.org/packages/86/89/e42e4a6243c0c28bc9598ad3694d32be1f171fa95fb3a652aa64e45e119f/unicodedata2-17.0.1-cp314-cp314-win32.whl", hash = "sha256:4dd637b5ea44fd215e6fccf4111b6b8b8895da8e869fbb90e20626f54afee9bb", size = 577494, upload-time = "2026-02-12T10:05:03.339Z" }, + { url = "https://files.pythonhosted.org/packages/d2/08/7442582477e909de2312e76ebe9ce61c84594ce068429da7bdc6ba865e35/unicodedata2-17.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:6c085978d6e655716ba606f736bf1e7f957fa3a3f56df5e5d6aa87b4d1cb9ca0", size = 577747, upload-time = "2026-02-12T10:05:04.817Z" }, + { url = "https://files.pythonhosted.org/packages/ec/95/0b3e8da4a09b8985e20a5ececf6c58e94c9a9ebf34a9d18a85a0c703d50e/unicodedata2-17.0.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d4e40a701508e705b52f4df0d41f9cccf6eaa1160bb92b52348c98341b508461", size = 973976, upload-time = "2026-02-12T10:05:06.973Z" }, + { url = "https://files.pythonhosted.org/packages/67/55/4576286b7a89378d237fb140aa7474f1f2dcbf3cf6e4f14766b5aadefe11/unicodedata2-17.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b05a1469280f94e8fbc022563b6751a8a30640361ec2c6de4f73e7f5f6c9db6b", size = 495694, upload-time = "2026-02-12T10:05:08.915Z" }, + { url = "https://files.pythonhosted.org/packages/ab/aa/bf2da5223c5a016bc2975add15524ec08d860142ccecb4951143ba04c453/unicodedata2-17.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e5800972e1aefd6da8f4661b389f095249f0ff63530bc2a3fbabf00d2b3b52b6", size = 533692, upload-time = "2026-02-12T10:05:10.286Z" }, + { url = "https://files.pythonhosted.org/packages/2e/aa/fb8d97881b5289e551425a8e36391d7633f83dea29b4c3374fa574927b86/unicodedata2-17.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d7cd7d058be81fce6e62dbcb8b1f8eddf77407f82fb108fb5ca4b4623bdb5beb", size = 533415, upload-time = "2026-02-12T10:05:11.941Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0f/7bb66d630a050c78350a2e689ecf76b3c9fd4108df4fd30545465361d317/unicodedata2-17.0.1-cp314-cp314t-win32.whl", hash = "sha256:164867ee8d64b2eb62e1b9e3fbc21eb6d3d99d7c491888eebb45a5131d49fc6c", size = 577672, upload-time = "2026-02-12T10:05:13.819Z" }, + { url = "https://files.pythonhosted.org/packages/15/ff/6c5a0f561eeb47cd09f20caebc0416ec1d253b68cc7264a85f420b6350eb/unicodedata2-17.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:31647c7ff10197487d60a1bbb92d13a095c87e5ca3e1c218510e5cce39b2de21", size = 578003, upload-time = "2026-02-12T10:05:15.249Z" }, +] + [[package]] name = "uri-template" version = "1.3.0" @@ -12237,7 +12327,7 @@ dependencies = [ { name = "requests", marker = "sys_platform != 'darwin'" }, { name = "sentencepiece", marker = "sys_platform != 'darwin'" }, { name = "setproctitle", marker = "sys_platform != 'darwin'" }, - { name = "setuptools", version = "80.10.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "setuptools", marker = "sys_platform != 'darwin'" }, { name = "six", marker = "sys_platform != 'darwin'" }, { name = "tiktoken", marker = "sys_platform != 'darwin'" }, { name = "tokenizers", marker = "sys_platform != 'darwin'" }, @@ -12822,7 +12912,7 @@ name = "zope-interface" version = "7.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "setuptools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/30/93/9210e7606be57a2dfc6277ac97dcc864fd8d39f142ca194fdc186d596fda/zope.interface-7.2.tar.gz", hash = "sha256:8b49f1a3d1ee4cdaf5b32d2e738362c7f5e40ac8b46dd7d1a65e82a4872728fe", size = 252960, upload-time = "2024-11-28T08:45:39.224Z" } wheels = [ From bb07320475c32ede79c758175b1297927521786b Mon Sep 17 00:00:00 2001 From: Finbarr Timbers Date: Mon, 4 May 2026 14:56:17 -0600 Subject: [PATCH 2/8] Add missing IFBench scorer, metrics, task and tests Co-Authored-By: Claude Opus 4.7 --- src/olmo_eval/common/metrics/ifeval.py | 87 ++++++++++++++ src/olmo_eval/common/scorers/ifeval.py | 108 +++++++++++++++++ src/olmo_eval/evals/tasks/ifbench.py | 78 ++++++++++++ tests/evals/tasks/test_ifbench.py | 157 +++++++++++++++++++++++++ 4 files changed, 430 insertions(+) create mode 100644 src/olmo_eval/common/metrics/ifeval.py create mode 100644 src/olmo_eval/common/scorers/ifeval.py create mode 100644 src/olmo_eval/evals/tasks/ifbench.py create mode 100644 tests/evals/tasks/test_ifbench.py diff --git a/src/olmo_eval/common/metrics/ifeval.py b/src/olmo_eval/common/metrics/ifeval.py new file mode 100644 index 000000000..01b85a040 --- /dev/null +++ b/src/olmo_eval/common/metrics/ifeval.py @@ -0,0 +1,87 @@ +"""IFBench / IFEval metrics. + +All four metrics share :class:`IFEvalScorer`, which writes per-instruction +strict and loose pass lists to ``output.metadata["ifeval"]``. Each metric +aggregates that side-band data along the prompt or instruction axis. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import ClassVar + +from olmo_eval.common.metrics.base import Metric +from olmo_eval.common.scorers import IFEvalScorer, Scorer +from olmo_eval.common.types import Response + + +def _iter_results(responses: Sequence[Response], key: str) -> list[list[bool]]: + out: list[list[bool]] = [] + for response in responses: + if not response.outputs: + out.append([]) + continue + meta = response.outputs[0].metadata or {} + ifeval = meta.get("ifeval") or {} + out.append(list(ifeval.get(key, []))) + return out + + +def _prompt_level(results: list[list[bool]]) -> float: + if not results: + return 0.0 + correct = sum(1 for r in results if r and all(r)) + return correct / len(results) + + +def _instruction_level(results: list[list[bool]]) -> float: + total = sum(len(r) for r in results) + if total == 0: + return 0.0 + correct = sum(1 for r in results for v in r if v) + return correct / total + + +@dataclass(frozen=True, slots=True) +class IFEvalPromptStrictAccuracy(Metric): + """Fraction of prompts where every instruction passes under strict scoring.""" + + name: ClassVar[str] = "prompt_level_strict_acc" + scorer: ClassVar[type[Scorer]] = IFEvalScorer + + def compute(self, responses: Sequence[Response]) -> float: + return _prompt_level(_iter_results(responses, "strict")) + + +@dataclass(frozen=True, slots=True) +class IFEvalPromptLooseAccuracy(Metric): + """Fraction of prompts where every instruction passes under loose scoring.""" + + name: ClassVar[str] = "prompt_level_loose_acc" + scorer: ClassVar[type[Scorer]] = IFEvalScorer + + def compute(self, responses: Sequence[Response]) -> float: + return _prompt_level(_iter_results(responses, "loose")) + + +@dataclass(frozen=True, slots=True) +class IFEvalInstStrictAccuracy(Metric): + """Fraction of individual instructions passing under strict scoring.""" + + name: ClassVar[str] = "inst_level_strict_acc" + scorer: ClassVar[type[Scorer]] = IFEvalScorer + + def compute(self, responses: Sequence[Response]) -> float: + return _instruction_level(_iter_results(responses, "strict")) + + +@dataclass(frozen=True, slots=True) +class IFEvalInstLooseAccuracy(Metric): + """Fraction of individual instructions passing under loose scoring.""" + + name: ClassVar[str] = "inst_level_loose_acc" + scorer: ClassVar[type[Scorer]] = IFEvalScorer + + def compute(self, responses: Sequence[Response]) -> float: + return _instruction_level(_iter_results(responses, "loose")) diff --git a/src/olmo_eval/common/scorers/ifeval.py b/src/olmo_eval/common/scorers/ifeval.py new file mode 100644 index 000000000..3cf46e68b --- /dev/null +++ b/src/olmo_eval/common/scorers/ifeval.py @@ -0,0 +1,108 @@ +"""Scorer for IFBench / IFEval instruction-following evaluation. + +Wraps the upstream `allenai/IFBench` verifier registry. The scorer evaluates a +response against per-instance instructions (looked up in +``instance.metadata["instruction_id_list"]`` / ``"kwargs"``) and writes both +strict and loose pass/fail lists for each instruction into +``output.metadata["ifeval"]``. The four IFEval metrics consume that field. +""" + +from __future__ import annotations + +import importlib +from dataclasses import dataclass +from typing import Any, ClassVar + +from olmo_eval.common.scorers.base import Scorer +from olmo_eval.common.types import Instance, LMOutput + +_INSTRUCTION_DICT: dict[str, Any] | None = None + + +def _load_instruction_dict() -> dict[str, Any]: + global _INSTRUCTION_DICT + if _INSTRUCTION_DICT is None: + registry = importlib.import_module("instructions_registry") + _INSTRUCTION_DICT = registry.INSTRUCTION_DICT + return _INSTRUCTION_DICT + + +def _loose_response_variants(response: str) -> list[str]: + """Generate the eight response variants used by upstream loose scoring.""" + lines = response.split("\n") + remove_first = "\n".join(lines[1:]).strip() + remove_last = "\n".join(lines[:-1]).strip() + remove_both = "\n".join(lines[1:-1]).strip() + return [ + response, + response.replace("*", ""), + remove_first, + remove_last, + remove_both, + remove_first.replace("*", ""), + remove_last.replace("*", ""), + remove_both.replace("*", ""), + ] + + +def _check_one( + instruction_cls: Any, + instruction_id: str, + kwargs: dict[str, Any], + prompt: str, + response: str, +) -> bool: + instruction = instruction_cls(instruction_id) + cleaned_kwargs = {k: v for k, v in kwargs.items() if v is not None} + instruction.build_description(**cleaned_kwargs) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=prompt) + return bool(response.strip()) and bool(instruction.check_following(response)) + + +@dataclass(frozen=True) +class IFEvalScorer(Scorer): + """Run IFBench/IFEval instruction verifiers against a response. + + The numeric ``score()`` return is the prompt-level loose accuracy (1.0 if + every instruction passes under at least one loose variant, else 0.0). The + full strict + loose pass lists are written to ``output.metadata["ifeval"]`` + so the four metric classes can derive prompt/inst × strict/loose figures. + """ + + name: ClassVar[str] = "ifeval" + + def score(self, instance: Instance, output: LMOutput) -> float: + instruction_ids: list[str] = instance.metadata.get("instruction_id_list", []) + kwargs_list: list[dict[str, Any]] = instance.metadata.get("kwargs", []) + prompt: str = instance.metadata.get("prompt", instance.question) + response: str = output.text or "" + + strict_results: list[bool] = [] + loose_results: list[bool] = [] + + if instruction_ids: + registry = _load_instruction_dict() + loose_variants = _loose_response_variants(response) + for inst_id, inst_kwargs in zip(instruction_ids, kwargs_list, strict=True): + instruction_cls = registry[inst_id] + strict_results.append( + _check_one(instruction_cls, inst_id, inst_kwargs, prompt, response) + ) + loose_pass = any( + _check_one(instruction_cls, inst_id, inst_kwargs, prompt, variant) + for variant in loose_variants + ) + loose_results.append(loose_pass) + + if output.metadata is None: + output.metadata = {} + output.metadata["ifeval"] = { + "strict": strict_results, + "loose": loose_results, + } + + if not loose_results: + return 0.0 + return 1.0 if all(loose_results) else 0.0 diff --git a/src/olmo_eval/evals/tasks/ifbench.py b/src/olmo_eval/evals/tasks/ifbench.py new file mode 100644 index 000000000..d76cbc349 --- /dev/null +++ b/src/olmo_eval/evals/tasks/ifbench.py @@ -0,0 +1,78 @@ +"""IFBench: out-of-distribution instruction-following benchmark. + +Dataset: ``allenai/IFBench_test`` (300 prompts). Each row carries a list of +instruction IDs and per-instruction kwargs; verifiers come from the upstream +``allenai/IFBench`` package, accessed lazily via :class:`IFEvalScorer`. + +Reports prompt-level and instruction-level accuracy in both strict and loose +modes, matching upstream IFEval reporting. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from olmo_eval.common.metrics import ( + IFEvalInstLooseAccuracy, + IFEvalInstStrictAccuracy, + IFEvalPromptLooseAccuracy, + IFEvalPromptStrictAccuracy, +) +from olmo_eval.common.types import ( + Instance, + LMOutput, + LMRequest, + RequestType, + SamplingParams, + Split, +) +from olmo_eval.data import DataSource +from olmo_eval.evals.tasks.common import Task, register + +_PRIMARY_METRIC = IFEvalInstLooseAccuracy() + + +@register("ifbench") +class IFBench(Task): + data_source = DataSource(path="allenai/IFBench_test", split="train") + split = Split.TRAIN + metrics = ( + IFEvalPromptStrictAccuracy(), + IFEvalPromptLooseAccuracy(), + IFEvalInstStrictAccuracy(), + IFEvalInstLooseAccuracy(), + ) + primary_metric = _PRIMARY_METRIC + sampling_params = SamplingParams( + max_tokens=1280, + temperature=0.0, + do_sample=False, + ) + + @property + def instances(self) -> Iterator[Instance]: + yield from self._load_instances_cached() + + def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None: + prompt = doc["prompt"] + instruction_id_list = list(doc.get("instruction_id_list") or []) + raw_kwargs = doc.get("kwargs") or [] + kwargs_list = [{k: v for k, v in (kw or {}).items() if v is not None} for kw in raw_kwargs] + return Instance( + question=prompt, + gold_answer=None, + metadata={ + "id": doc.get("key", index), + "key": doc.get("key", index), + "prompt": prompt, + "instruction_id_list": instruction_id_list, + "kwargs": kwargs_list, + }, + ) + + def format_request(self, instance: Instance) -> LMRequest: + return LMRequest(request_type=RequestType.COMPLETION, prompt=instance.question) + + def extract_answer(self, output: LMOutput) -> str: + return output.text diff --git a/tests/evals/tasks/test_ifbench.py b/tests/evals/tasks/test_ifbench.py new file mode 100644 index 000000000..12e25b66d --- /dev/null +++ b/tests/evals/tasks/test_ifbench.py @@ -0,0 +1,157 @@ +"""Tests for the IFBench task and IFEval scoring stack.""" + +from __future__ import annotations + +import unittest +from typing import Any + +import pytest + +from olmo_eval.common.metrics import ( + IFEvalInstLooseAccuracy, + IFEvalInstStrictAccuracy, + IFEvalPromptLooseAccuracy, + IFEvalPromptStrictAccuracy, +) +from olmo_eval.common.scorers import IFEvalScorer +from olmo_eval.common.types import Instance, LMOutput, LMRequest, RequestType, Response +from olmo_eval.evals.tasks.common import get_task + +# Skip the whole module if upstream IFBench isn't installed. +pytest.importorskip("instructions_registry") + + +def _make_instance( + prompt: str, + instruction_id_list: list[str], + kwargs_list: list[dict[str, Any]], +) -> Instance: + return Instance( + question=prompt, + gold_answer=None, + metadata={ + "id": "test", + "key": "test", + "prompt": prompt, + "instruction_id_list": instruction_id_list, + "kwargs": kwargs_list, + }, + ) + + +def _make_response(instance: Instance, response_text: str) -> Response: + return Response( + instance=instance, + request=LMRequest(request_type=RequestType.COMPLETION, prompt=instance.question), + outputs=[LMOutput(text=response_text)], + ) + + +class TestIFBenchTask(unittest.TestCase): + def test_registered(self) -> None: + task = get_task("ifbench") + self.assertIsNotNone(task) + metric_names = {m.name for m in task.config.metrics} + self.assertEqual( + metric_names, + { + "prompt_level_strict_acc", + "prompt_level_loose_acc", + "inst_level_strict_acc", + "inst_level_loose_acc", + }, + ) + + def test_process_doc_strips_none_kwargs(self) -> None: + task = get_task("ifbench") + doc = { + "key": 0, + "prompt": "hi", + "instruction_id_list": ["count:numbers"], + "kwargs": [{"N": 2, "keyword": None, "frequency": None}], + } + instance = task.process_doc(doc, index=0) + self.assertIsNotNone(instance) + assert instance is not None + self.assertEqual(instance.metadata["instruction_id_list"], ["count:numbers"]) + self.assertEqual(instance.metadata["kwargs"], [{"N": 2}]) + self.assertEqual(instance.metadata["key"], 0) + + +class TestIFEvalScorer(unittest.TestCase): + """Use ``count:numbers`` (must include exactly N digits) — a deterministic verifier.""" + + INSTRUCTION_ID = "count:numbers" + KWARGS: dict[str, Any] = {"N": 2} + + def _score(self, response_text: str) -> dict[str, list[bool]]: + instance = _make_instance( + "Include exactly 2 numbers.", [self.INSTRUCTION_ID], [self.KWARGS] + ) + output = LMOutput(text=response_text) + IFEvalScorer().score(instance, output) + return output.metadata["ifeval"] + + def test_satisfied(self) -> None: + result = self._score("I have 3 apples and 4 oranges.") + self.assertEqual(result["strict"], [True]) + self.assertEqual(result["loose"], [True]) + + def test_violated_too_few(self) -> None: + result = self._score("I have apples and oranges.") + self.assertEqual(result["strict"], [False]) + self.assertEqual(result["loose"], [False]) + + def test_violated_too_many(self) -> None: + result = self._score("I have 1 apple, 2 oranges, and 3 grapes.") + self.assertEqual(result["strict"], [False]) + self.assertEqual(result["loose"], [False]) + + def test_loose_strips_leading_line(self) -> None: + # Strict fails: leading "Sure!" line adds no number, body has 2. + # But loose strips first line, leaving exactly 2 — passes. + result = self._score( + "Sure! Here's a response with 5 trailing extras at the end.\n" + "I have 3 apples and 4 oranges." + ) + self.assertEqual(result["strict"], [False]) + self.assertEqual(result["loose"], [True]) + + +class TestIFEvalMetrics(unittest.TestCase): + INSTRUCTION_ID = "count:numbers" + KWARGS: dict[str, Any] = {"N": 2} + + def _scored_response(self, response_text: str) -> Response: + instance = _make_instance("count.", [self.INSTRUCTION_ID], [self.KWARGS]) + response = _make_response(instance, response_text) + IFEvalScorer().score(instance, response.outputs[0]) + return response + + def test_aggregation(self) -> None: + responses = [ + self._scored_response("I have 3 apples and 4 oranges."), # passes + self._scored_response("Just words, no digits."), # fails + ] + self.assertAlmostEqual(IFEvalPromptStrictAccuracy().compute(responses), 0.5) + self.assertAlmostEqual(IFEvalPromptLooseAccuracy().compute(responses), 0.5) + self.assertAlmostEqual(IFEvalInstStrictAccuracy().compute(responses), 0.5) + self.assertAlmostEqual(IFEvalInstLooseAccuracy().compute(responses), 0.5) + + def test_multi_instruction_prompt_requires_all(self) -> None: + instance = _make_instance( + "Two numbers and an emoji-terminated sentence.", + ["count:numbers", "format:emoji"], + [{"N": 2}, {}], + ) + # Has 2 numbers (passes count) but no emoji at end of sentence (fails emoji). + response = _make_response(instance, "I have 3 apples and 4 oranges.") + IFEvalScorer().score(instance, response.outputs[0]) + prompt_acc = IFEvalPromptStrictAccuracy().compute([response]) + inst_acc = IFEvalInstStrictAccuracy().compute([response]) + self.assertEqual(prompt_acc, 0.0) # not all instructions satisfied + self.assertEqual(inst_acc, 0.5) # 1 of 2 satisfied + + +if __name__ == "__main__": + unittest.main() From 1e850b75658e57f0573ffacbe737dd6ba060ee95 Mon Sep 17 00:00:00 2001 From: Finbarr Timbers Date: Mon, 4 May 2026 17:18:18 -0600 Subject: [PATCH 3/8] Align ifbench with oe-eval-internal: ifeval_ood + ifeval_mt suite, vendored IFEval registry Co-Authored-By: Claude Opus 4.7 --- pyproject.toml | 6 + src/olmo_eval/common/scorers/ifeval.py | 15 +- .../common/scorers/ifeval_deps/__init__.py | 7 + .../scorers/ifeval_deps/instructions.py | 3923 +++++++++++++++++ .../ifeval_deps/instructions_registry.py | 251 ++ .../scorers/ifeval_deps/instructions_util.py | 1777 ++++++++ src/olmo_eval/evals/suites/ifbench.py | 13 + src/olmo_eval/evals/tasks/ifeval_mt.py | 104 + .../evals/tasks/{ifbench.py => ifeval_ood.py} | 31 +- tests/evals/tasks/test_ifbench.py | 62 +- 10 files changed, 6160 insertions(+), 29 deletions(-) create mode 100644 src/olmo_eval/common/scorers/ifeval_deps/__init__.py create mode 100644 src/olmo_eval/common/scorers/ifeval_deps/instructions.py create mode 100644 src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py create mode 100644 src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py create mode 100644 src/olmo_eval/evals/suites/ifbench.py create mode 100644 src/olmo_eval/evals/tasks/ifeval_mt.py rename src/olmo_eval/evals/tasks/{ifbench.py => ifeval_ood.py} (67%) diff --git a/pyproject.toml b/pyproject.toml index 8ddc11809..db5c79177 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,6 +145,12 @@ vllm = [ [tool.ruff] line-length = 100 target-version = "py312" +extend-exclude = [ + # Vendored from oe-eval-internal (oe_eval/dependencies/ifeval/), kept as-is. + "src/olmo_eval/common/scorers/ifeval_deps/instructions.py", + "src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py", + "src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py", +] [tool.ruff.lint] select = [ diff --git a/src/olmo_eval/common/scorers/ifeval.py b/src/olmo_eval/common/scorers/ifeval.py index 3cf46e68b..498b18c9e 100644 --- a/src/olmo_eval/common/scorers/ifeval.py +++ b/src/olmo_eval/common/scorers/ifeval.py @@ -1,7 +1,10 @@ """Scorer for IFBench / IFEval instruction-following evaluation. -Wraps the upstream `allenai/IFBench` verifier registry. The scorer evaluates a -response against per-instance instructions (looked up in +Uses the vendored ``olmo_eval.common.scorers.ifeval_deps`` registry, which +covers the original IFEval (DEFAULT) verifiers, the OOD verifiers used by +``allenai/IFBench_test2``, and the verifiers used by the multi-turn +``VGraf/ifeval_mt`` slices. The scorer evaluates a response against +per-instance instructions (looked up in ``instance.metadata["instruction_id_list"]`` / ``"kwargs"``) and writes both strict and loose pass/fail lists for each instruction into ``output.metadata["ifeval"]``. The four IFEval metrics consume that field. @@ -9,21 +12,17 @@ from __future__ import annotations -import importlib from dataclasses import dataclass from typing import Any, ClassVar from olmo_eval.common.scorers.base import Scorer +from olmo_eval.common.scorers.ifeval_deps import instructions_registry from olmo_eval.common.types import Instance, LMOutput -_INSTRUCTION_DICT: dict[str, Any] | None = None +_INSTRUCTION_DICT: dict[str, Any] = instructions_registry.INSTRUCTION_DICT def _load_instruction_dict() -> dict[str, Any]: - global _INSTRUCTION_DICT - if _INSTRUCTION_DICT is None: - registry = importlib.import_module("instructions_registry") - _INSTRUCTION_DICT = registry.INSTRUCTION_DICT return _INSTRUCTION_DICT diff --git a/src/olmo_eval/common/scorers/ifeval_deps/__init__.py b/src/olmo_eval/common/scorers/ifeval_deps/__init__.py new file mode 100644 index 000000000..39dcea997 --- /dev/null +++ b/src/olmo_eval/common/scorers/ifeval_deps/__init__.py @@ -0,0 +1,7 @@ +"""Vendored IFEval/IFBench instruction registry. + +Sourced from allenai/oe-eval-internal (oe_eval/dependencies/ifeval/) so that +both the original IFEval verifiers (DEFAULT) and the OOD verifiers used by +``allenai/IFBench_test2`` plus the multi-turn ``VGraf/ifeval_mt`` slices are +available from a single registry. +""" diff --git a/src/olmo_eval/common/scorers/ifeval_deps/instructions.py b/src/olmo_eval/common/scorers/ifeval_deps/instructions.py new file mode 100644 index 000000000..7268d8387 --- /dev/null +++ b/src/olmo_eval/common/scorers/ifeval_deps/instructions.py @@ -0,0 +1,3923 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library of instructions.""" + +import collections +import csv +import io +import json +import logging +import random +import re +import string +import unicodedata +from collections import Counter +from collections.abc import Sequence +from typing import Optional + +import emoji +import langdetect +import nltk +import syllapy + +from olmo_eval.common.scorers.ifeval_deps import instructions_util + +logger = logging.getLogger(__name__) + +_InstructionArgsDtype = Optional[dict[str, int | str | Sequence[str]]] + +_LANGUAGES = instructions_util.LANGUAGE_CODES + +# The relational operation for comparison. +_COMPARISON_RELATION = ("less than", "at least") + +# The maximum number of sentences. +_MAX_NUM_SENTENCES = 20 + +# The number of placeholders. +_NUM_PLACEHOLDERS = 4 + +# The number of bullet lists. +_NUM_BULLETS = 5 + +# The options of constrained response. +_CONSTRAINED_RESPONSE_OPTIONS = ( + "My answer is yes.", + "My answer is no.", + "My answer is maybe.", +) + +# The options of starter keywords. +_STARTER_OPTIONS = ( + "I would say", + "My answer is", + "I believe", + "In my opinion", + "I think", + "I reckon", + "I feel", + "From my perspective", + "As I see it", + "According to me", + "As far as I'm concerned", + "To my understanding", + "In my view", + "My take on it is", + "As per my perception", +) + +# The options of ending keywords. +# TODO(jeffreyzhou) add more ending options +_ENDING_OPTIONS = ("Any other questions?", "Is there anything else I can help with?") + +# The number of highlighted sections. +_NUM_HIGHLIGHTED_SECTIONS = 4 + +# The section spliter. +_SECTION_SPLITER = ("Section", "SECTION") + +# The number of sections. +_NUM_SECTIONS = 5 + +# The number of paragraphs. +_NUM_PARAGRAPHS = 5 + +# The postscript marker. +_POSTSCRIPT_MARKER = ("P.S.", "P.P.S") + +# The number of keywords. +_NUM_KEYWORDS = 2 + +# The occurrences of a single keyword. +_KEYWORD_FREQUENCY = 3 + +# The occurrences of a single letter. +_LETTER_FREQUENCY = 10 + +# The occurrences of words with all capital letters. +_ALL_CAPITAL_WORD_FREQUENCY = 20 + +# The number of words in the response. +_NUM_WORDS_LOWER_LIMIT = 100 +_NUM_WORDS_UPPER_LIMIT = 500 + +# The number of numbers. +_NUM_NUMBERS = 6 + +# Period length for periodic words. +_NUM_WORD_CYCLE = 30 + +# Maximum number of times a word can be repeated. +_MAX_REPEATS = 5 + +# Which sentence must contain a keyword. +_NUM_KEYWORD_SENTENCE = 20 + +# Minimum number of pronouns. +_NUM_PRONOUNS = 25 + +# The size of increment for lengths. +_NUM_INCREMENT = 5 + +# The number of coordinating conjunctions. +_NUM_CONJUNCTIONS = 6 + +# The Levenshtein distance of a response. +_LEV_DISTANCE_LOWER_LIMIT = 5 +_LEV_DISTANCE_UPPER_LIMIT = 40 + + +class Instruction: + """An instruction template.""" + + def __init__(self, instruction_id): + self.id = instruction_id + + def build_description(self, **kwargs): + raise NotImplementedError("`build_description` not implemented.") + + def get_instruction_args(self): + raise NotImplementedError("`get_instruction_args` not implemented.") + + def get_instruction_args_keys(self): + raise NotImplementedError("`get_instruction_args_keys` not implemented.") + + def check_following(self, value): + raise NotImplementedError("`check_following` not implemented.") + + +class ResponseLanguageChecker(Instruction): + """Check the language of the entire response.""" + + def build_description(self, *, language=None): + """Build the instruction description. + + Args: + language: A string representing the expected language of the response. The + language has to comply to the 97 types defined in + `langid.py` (https://pypi.org/project/langid/1.1.5/), which follows + ISO 639-1 codes (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes); + for example, `en` for English, `zh` for Chinese, `fr` for French. + + Returns: + A string representing the instruction description. + """ + self._language = language + if self._language is None: + self._language = random.choice(list(_LANGUAGES.keys())) + # TODO(tianjianlu): opens the description generation to more choices. + self._description_pattern = ( + "Your ENTIRE response should be in {language} language, no other " + + "language is allowed." + ) + return self._description_pattern.format(language=_LANGUAGES[self._language]) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"language": self._language} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["language"] + + def check_following(self, value): + """Check if the language of the entire response follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the language of `value` follows instruction; otherwise False. + """ + assert isinstance(value, str) + + try: + return langdetect.detect(value) == self._language + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class NumberOfSentences(Instruction): + """Check the number of sentences.""" + + def build_description(self, *, num_sentences=None, relation=None): + """Build the instruction description. + + Args: + num_sentences: An integer specifying the number of sentences as a + threshold. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of sentences < the threshold; + if 'at least', the actual number of sentences >= the threshold. + + Returns: + A string representing the instruction description. + """ + # The number of sentences as a threshold for comparison. + self._num_sentences_threshold = num_sentences + if self._num_sentences_threshold is None or self._num_sentences_threshold < 0: + self._num_sentences_threshold = random.randint(1, _MAX_NUM_SENTENCES) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "Your response should contain {relation} {num_sentences} sentences." + ) + return self._description_pattern.format( + relation=self._comparison_relation, + num_sentences=self._num_sentences_threshold, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_sentences": self._num_sentences_threshold, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "relation"] + + def check_following(self, value): + """Check if the number of sentences follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the response follows the instruction. + + Raise: + ValueError if the string in `instruction_args` is not in + [`less_than`, `at_least`]. + """ + num_sentences = instructions_util.count_sentences(value) + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_sentences < self._num_sentences_threshold + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_sentences >= self._num_sentences_threshold + + +class PlaceholderChecker(Instruction): + """Check the placeholders in template writing.""" + + def build_description(self, *, num_placeholders=None): + """Build the instruction description. + + Args: + num_placeholders: An integer denoting the minimum number of + placeholders required in the response. + + Returns: + A string representing the instruction description. + """ + self._num_placeholders = num_placeholders + if self._num_placeholders is None or self._num_placeholders < 0: + self._num_placeholders = random.randint(1, _NUM_PLACEHOLDERS) + self._description_pattern = ( + "The response must contain at least {num_placeholders} placeholders " + + "represented by square brackets, such as [address]." + ) + return self._description_pattern.format(num_placeholders=self._num_placeholders) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_placeholders": self._num_placeholders} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_placeholders"] + + def check_following(self, value): + """Check if the number of placeholders follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the actual number of placeholders in the response is greater than + or equal to `num_placeholders`; otherwise, False. + """ + placeholders = re.findall(r"\[.*?\]", value) + num_placeholders = len(placeholders) + return num_placeholders >= self._num_placeholders + + +class BulletListChecker(Instruction): + """Checks the bullet list in the prompt.""" + + def build_description(self, *, num_bullets=None): + """Build the instruction description. + + Args: + num_bullets: An integer specifying the exact number of bullet lists + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._num_bullets = num_bullets + if self._num_bullets is None or self._num_bullets < 0: + self._num_bullets = random.randint(1, _NUM_BULLETS) + self._description_pattern = ( + "Your answer must contain exactly {num_bullets} bullet points. " + + "Use the markdown bullet points such as:\n" + + "* This is point 1. \n" + + "* This is point 2" + ) + return self._description_pattern.format(num_bullets=self._num_bullets) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_bullets": self._num_bullets} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_bullets"] + + def check_following(self, value): + r"""Check if the number of bullet lists meets the requirement. + + Args: + value: A string representing the response. The response is expected to + contain some bullet lists that start with `\*`. + + Returns: + True if the actual number of bullet lists in the response meets the + requirement. + """ + bullet_lists = re.findall(r"^\s*\*[^\*].*$", value, flags=re.MULTILINE) + bullet_lists_2 = re.findall(r"^\s*-.*$", value, flags=re.MULTILINE) + num_bullet_lists = len(bullet_lists) + len(bullet_lists_2) + return num_bullet_lists == self._num_bullets + + +class ConstrainedResponseChecker(Instruction): + """Checks the constrained response.""" + + def build_description(self): + """Build the instruction description.""" + # A sequence of string(s) representing the options of the expected response. + self._constrained_responses = _CONSTRAINED_RESPONSE_OPTIONS + self._description_pattern = "Answer with one of the following options: {response_options}" + return self._description_pattern.format(response_options=self._constrained_responses) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response matches the constrained options. + + Args: + value: A string representing the response. + + Returns: + True if the actual response contains one of the options in the constrained + responses; otherwise False. + """ + value = value.strip() + for constrained_response in self._constrained_responses: + if constrained_response in value: + return True + return False + + +class ConstrainedStartChecker(Instruction): + """Checks the response start.""" + + def build_description(self, *, starter=None): + """Build the instruction description. + + Args: + starter: A string representing the keyword that the response should start + with. + + Returns: + A string representing the instruction description. + """ + self._starter = starter.strip() if isinstance(starter, str) else starter + if self._starter is None: + self._starter = random.choice(_STARTER_OPTIONS) + self._description_pattern = ( + "During the conversation, when it is your turn, " + "please always start with {starter}" + ) + return self._description_pattern.format(starter=self._starter) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"starter": self._starter} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["starter"] + + def check_following(self, value): + """Checks if the response starts with the constrained keyword or phrase. + + Args: + value: A string representing the response. + + Returns: + True if the response starts with the given phrase or keyword that is + contained in `instruction_args`; otherwise, False. + """ + response_pattern = r"^\s*" + self._starter + r".*$" + response_with_constrained_start = re.search(response_pattern, value, flags=re.MULTILINE) + return True if response_with_constrained_start else False + + +class HighlightSectionChecker(Instruction): + """Checks the highlighted section.""" + + def build_description(self, *, num_highlights=None): + """Build the instruction description. + + Args: + num_highlights: An integer specifying the minimum number of highlighted + sections. + + Returns: + A string representing the instruction description. + """ + self._num_highlights = num_highlights + if self._num_highlights is None or self._num_highlights < 0: + self._num_highlights = random.randint(1, _NUM_HIGHLIGHTED_SECTIONS) + + self._description_pattern = ( + "Highlight at least {num_highlights} sections in your answer with " + + "markdown, i.e. *highlighted section*." + ) + + return self._description_pattern.format(num_highlights=self._num_highlights) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_highlights": self._num_highlights} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_highlights"] + + def check_following(self, value): + """Checks if the number of highlighted sections meets the requirement. + + Args: + value: a string repesenting the response. The response is expected to + contain highlighted sections in the format of *highlighted*. + + Returns: + True if the actual number of highlighted sections in the format of + *highlighed sections* meets the minimum requirement; otherwise False. + """ + num_highlights = 0 + highlights = re.findall(r"\*[^\n\*]*\*", value) + double_highlights = re.findall(r"\*\*[^\n\*]*\*\*", value) + for highlight in highlights: + if highlight.strip("*").strip(): + num_highlights += 1 + for highlight in double_highlights: + if highlight.removeprefix("**").removesuffix("**").strip(): + num_highlights += 1 + + return num_highlights >= self._num_highlights + + +class SectionChecker(Instruction): + """Checks the sections.""" + + def build_description(self, *, section_spliter=None, num_sections=None): + """Build the instruction description. + + Args: + section_spliter: A string represents the section spliter keyword that + marks a new section, i.e., `Section` or `SECTION`. + num_sections: An integer specifying the number of sections. + + Returns: + A string representing the instruction description. + """ + self._section_spliter = ( + section_spliter.strip() if isinstance(section_spliter, str) else section_spliter + ) + if self._section_spliter is None: + self._section_spliter = random.choice(_SECTION_SPLITER) + + self._num_sections = num_sections + if self._num_sections is None or self._num_sections < 0: + self._num_sections = random.randint(1, _NUM_SECTIONS) + + self._description_pattern = ( + "Your response must have {num_sections} sections. Mark the beginning " + + "of each section with {section_spliter} X, such as:\n" + + "{section_spliter} 1\n" + + "[content of section 1]\n" + + "{section_spliter} 2\n" + + "[content of section 2]" + ) + + return self._description_pattern.format( + num_sections=self._num_sections, section_spliter=self._section_spliter + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "section_spliter": self._section_spliter, + "num_sections": self._num_sections, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["section_spliter", "num_sections"] + + def check_following(self, value): + """Checks the response contains multiple sections. + + Args: + value: A string representing the response. The response is expected + to contain multiple sections (number of sections is greater than 1). + A new section starts with `Section 1`, where the number denotes the + section index. + + Returns: + True if the number of sections in the response is greater than or equal to + the minimum number of sections; otherwise, False. + """ + section_splitter_patten = r"\s?" + self._section_spliter + r"\s?\d+\s?" + sections = re.split(section_splitter_patten, value) + num_sections = len(sections) - 1 + return num_sections >= self._num_sections + + +class ParagraphChecker(Instruction): + """Checks the paragraphs.""" + + def build_description(self, *, num_paragraphs=None): + """Build the instruction description. + + Args: + num_paragraphs: An integer specifying the number of paragraphs. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs are separated with the markdown divider: ***" + ) + + return self._description_pattern.format(num_paragraphs=self._num_paragraphs) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_paragraphs": self._num_paragraphs} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs"] + + def check_following(self, value): + """Checks the response contains required number of paragraphs. + + Args: + value: A string representing the response. The response may contain + paragraphs that are separated by the markdown divider: `***`. + + Returns: + True if the actual number of paragraphs is the same as required; + otherwise, False. + """ + paragraphs = re.split(r"\s?\*\*\*\s?", value) + num_paragraphs = len(paragraphs) + + for index, paragraph in enumerate(paragraphs): + if not paragraph.strip(): + if index == 0 or index == len(paragraphs) - 1: + num_paragraphs -= 1 + else: + return False + + return num_paragraphs == self._num_paragraphs + + +class PostscriptChecker(Instruction): + """Checks the postscript.""" + + def build_description(self, *, postscript_marker=None): + """Build the instruction description. + + Args: + postscript_marker: A string containing the keyword that marks the start + of the postscript section. + + Returns: + A string representing the instruction description. + """ + self._postscript_marker = ( + postscript_marker.strip() if isinstance(postscript_marker, str) else postscript_marker + ) + if self._postscript_marker is None: + self._postscript_marker = random.choice(_POSTSCRIPT_MARKER) + + self._description_pattern = ( + "At the end of your response, please explicitly add a postscript " + + "starting with {postscript}" + ) + + return self._description_pattern.format(postscript=self._postscript_marker) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"postscript_marker": self._postscript_marker} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["postscript_marker"] + + def check_following(self, value): + """Checks if the response follows the postscript format. + + Args: + value: a string representing the response. The response is expected to + contain a postscript section. + + Returns: + True if the response contains a postscript section starting with + the keyword containing in the `instruction_args`; otherwise False. + """ + value = value.lower() + if self._postscript_marker == "P.P.S": + postscript_pattern = r"\s*p\.\s?p\.\s?s.*$" + elif self._postscript_marker == "P.S.": + postscript_pattern = r"\s*p\.\s?s\..*$" + else: + postscript_pattern = r"\s*" + self._postscript_marker.lower() + r".*$" + postscript = re.findall(postscript_pattern, value, flags=re.MULTILINE) + return True if postscript else False + + +class RephraseChecker(Instruction): + """Checks the repharse.""" + + def build_description(self, *, original_message): + """Build the instruction description. + + Args: + original_message: A string representing the original message. The + rephrased response should only change its words/sentences in between + its two asterisks, for example, *change me*. Both original and rephrased + messages should contain the changes in the form of *change me*. + + Returns: + A string representing the instruction description. + """ + if not self.is_change(original_message): + raise ValueError( + f"Message {original_message} does not contain changes in the form of *change me*." + ) + + self._reference_without_change = original_message + self._description = ( + "Rephrasing: Your rephrased response should only" + + "change the words/sentences in between two asterisks" + + "such as *change me*." + ) + return self._description + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"original_message": self._reference_without_change} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_message"] + + def check_following(self, value): + r"""Checks if the rephrasing follows the instruction. + + Args: + value: A string representing the response, which is expected to rephras + the string of `instruction_args`. + + Returns: + True if `value` and `instruction_args` only differ by the words/sentences + in between two asterisks such as *change me*; otherwise, False. + """ + + if not self.is_change(value): + raise ValueError(f"value {value} does not contain changes in the form of *change me*.") + + response_without_changes = self.strip_changes(value) + reference_without_changes = self.strip_changes(self._reference_without_change) + + return response_without_changes == reference_without_changes + + def is_change(self, response): + """Check if there is change in the response in the form of *change me*.""" + return re.search(r"\*.*\*", response) + + def strip_changes(self, response): + """Strips off the changes.""" + return re.sub(r"\*.*\*", "", response) + + +class KeywordChecker(Instruction): + """Check the exisitence of certain keywords.""" + + def build_description(self, *, keywords=None): + """Build the instruction description. + + Args: + keywords: A sequence of strings representing the keywords that are + expected in the response. + + Returns: + A string representing the instruction description. + """ + + if not keywords: + self._keywords = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS) + else: + self._keywords = keywords + self._keywords = sorted(self._keywords) + + self._description_pattern = "Include keywords {keywords} in the response." + + return self._description_pattern.format(keywords=self._keywords) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"keywords": self._keywords} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keywords"] + + def check_following(self, value): + """Check if the response contain the expected keywords.""" + for keyword in self._keywords: + if not re.search(keyword, value, flags=re.IGNORECASE): + return False + return True + + +class KeywordFrequencyChecker(Instruction): + """Check the keyword frequency.""" + + def build_description(self, *, keyword=None, frequency=None, relation=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + frequency: An integer specifying the number of times `keyword` is expected + to appear in the response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of occurrences < frequency; + if 'at least', the actual number of occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if not keyword: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + + self._frequency = frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _KEYWORD_FREQUENCY) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "In your response, the word {keyword} should appear {relation} " + "{frequency} times." + ) + + return self._description_pattern.format( + keyword=self._keyword, + relation=self._comparison_relation, + frequency=self._frequency, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "keyword": self._keyword, + "frequency": self._frequency, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword", "frequency", "relation"] + + def check_following(self, value): + """Checks if the response contain the keyword with required frequency.""" + actual_occurrences = len(re.findall(self._keyword, value, flags=re.IGNORECASE)) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return actual_occurrences < self._frequency + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return actual_occurrences >= self._frequency + + +class NumberOfWords(Instruction): + """Checks the number of words.""" + + def build_description(self, *, num_words=None, relation=None): + """Build the instruction description. + + Args: + num_words: An integer specifying the number of words contained in the + response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of words < num_words; + if 'at least', the actual number of words >= num_words. + + Returns: + A string representing the instruction description. + """ + + self._num_words = num_words + if self._num_words is None or self._num_words < 0: + self._num_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = "Answer with {relation} {num_words} words." + + return self._description_pattern.format( + relation=self._comparison_relation, num_words=self._num_words + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_words": self._num_words, "relation": self._comparison_relation} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_words", "relation"] + + def check_following(self, value): + """Checks if the response contains the expected number of words.""" + num_words = instructions_util.count_words(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_words < self._num_words + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_words >= self._num_words + + +class JsonFormat(Instruction): + """Check the Json format.""" + + def build_description(self): + self._description_pattern = ( + "Entire output should be wrapped in JSON format. You can use markdown" + " ticks such as ```." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + value = ( + value.strip() + .removeprefix("```json") + .removeprefix("```Json") + .removeprefix("```JSON") + .removeprefix("```") + .removesuffix("```") + .strip() + ) + try: + json.loads(value) + except ValueError: + return False + return True + + +class ParagraphFirstWordCheck(Instruction): + """Check the paragraph and the first word of the nth paragraph.""" + + def build_description(self, num_paragraphs=None, nth_paragraph=None, first_word=None): + r"""Build the instruction description. + + Args: + num_paragraphs: An integer indicating the number of paragraphs expected + in the response. A paragraph is a subset of the string that is + expected to be separated by '\n\n'. + nth_paragraph: An integer indicating the paragraph number that we look at. + Note that n starts from 1. + first_word: A string that represent the first word of the bth paragraph. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._nth_paragraph = nth_paragraph + if ( + self._nth_paragraph is None + or self._nth_paragraph <= 0 + or self._nth_paragraph > self._num_paragraphs + ): + self._nth_paragraph = random.randint(1, self._num_paragraphs + 1) + + self._first_word = first_word + if self._first_word is None: + self._first_word = instructions_util.generate_keywords(num_keywords=1)[0] + self._first_word = self._first_word.lower() + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs and only paragraphs are separated with each other by two " + + "new lines as if it was '\\n\\n' in python. " + + "Paragraph {nth_paragraph} must start with word {first_word}." + ) + + return self._description_pattern.format( + num_paragraphs=self._num_paragraphs, + nth_paragraph=self._nth_paragraph, + first_word=self._first_word, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_paragraphs": self._num_paragraphs, + "nth_paragraph": self._nth_paragraph, + "first_word": self._first_word, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs", "nth_paragraph", "first_word"] + + def check_following(self, value): + """Checks for required number of paragraphs and correct first word. + + Args: + value: a string representing the response. The response may contain + paragraphs that are separated by two new lines and the first word of + the nth paragraph will have to match a specified word. + + Returns: + True if the number of paragraphs is the same as required and the first + word of the specified paragraph is the same as required. Otherwise, false. + """ + + paragraphs = re.split(r"\n\n", value) + num_paragraphs = len(paragraphs) + + for paragraph in paragraphs: + if not paragraph.strip(): + num_paragraphs -= 1 + + # check that index doesn't go out of bounds + if self._nth_paragraph <= num_paragraphs: + paragraph = paragraphs[self._nth_paragraph - 1].strip() + if not paragraph: + return False + else: + return False + + first_word = "" + punctuation = {".", ",", "?", "!", "'", '"'} + + # get first word and remove punctuation + word = paragraph.split()[0].strip() + # TODO(jeffrey): make more complex? + word = word.lstrip("'") + word = word.lstrip('"') + + for letter in word: + if letter in punctuation: + break + first_word += letter.lower() + + return num_paragraphs == self._num_paragraphs and first_word == self._first_word + + +# TODO(jeffrey) add relation - at least/at most? +class KeySentenceChecker(Instruction): + """Check the existence of certain key sentences.""" + + def build_description(self, key_sentences=None, num_sentences=None): + """Build the instruction description. + + Args: + key_sentences: A sequences of strings representing the key sentences that + are expected in the response. + num_sentences: The number of key sentences that are expected to be seen in + the response. + + Returns: + A string representing the instruction description. + """ + + if not key_sentences: + # TODO(jeffrey) make a generate sentences function? wonderwords package + self._key_sentences = set(["For now, this is fine."]) + else: + self._key_sentences = key_sentences + + if not num_sentences: + self._num_sentences = random.randint(1, len(self._key_sentences)) + else: + self._num_sentences = num_sentences + + self._description_pattern = ( + "Include {num_sentences} of the following sentences {key_sentences}" + ) + + return self._description_pattern.format( + num_sentences=self._num_sentences, key_sentences=self._key_sentences + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_sentences": self._num_sentences, + "key_sentences": list(self._key_sentences), + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "key_sentences"] + + def check_following(self, value): + """Checks if the response contains the expected key sentences.""" + count = 0 + sentences = instructions_util.split_into_sentences(value) + for sentence in self._key_sentences: + if sentence in sentences: + count += 1 + + return count == self._num_sentences + + +class ForbiddenWords(Instruction): + """Checks that specified words are not used in response.""" + + def build_description(self, forbidden_words=None): + """Build the instruction description. + + Args: + forbidden_words: A sequences of strings respresenting words that are not + allowed in the response. + + Returns: + A string representing the instruction description. + """ + + if not forbidden_words: + self._forbidden_words = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS) + else: + self._forbidden_words = list(set(forbidden_words)) + self._forbidden_words = sorted(self._forbidden_words) + self._description_pattern = "Do not include keywords {forbidden_words} in the response." + + return self._description_pattern.format(forbidden_words=self._forbidden_words) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"forbidden_words": self._forbidden_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["forbidden_words"] + + def check_following(self, value): + """Check if the response does not contain the expected keywords.""" + for word in self._forbidden_words: + if re.search(r"\b" + word + r"\b", value, flags=re.IGNORECASE): + return False + return True + + +class RephraseParagraph(Instruction): + """Checks that the paragraph is rephrased.""" + + def build_description(self, *, original_paragraph, low, high): + """Builds the instruction description. + + Args: + original_paragraph: A string presenting the original paragraph. The + rephrases response should have betweeb low-high words in common. + low: An integer presenting the lower bound of similar words. + high: An integer representing the upper bound of similar words. + + Returns: + A string representing the instruction description. + """ + # TODO(jeffrey) make more encompassing + self._original_paragraph = original_paragraph + self._low = low + self._high = high + + self._description = ( + "Rephrase the following paragraph: " + + "{original_paragraph}\nYour response should have " + + "between {low} and {high} of the same words. " + + "Words are the same if and only if all of the " + + "letters, ignoring cases, are the same. For " + + "example, 'run' is the same as 'Run' but different " + + "to 'ran'." + ) + + return self._description.format( + original_paragraph=original_paragraph, low=self._low, high=self._high + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "original_paragraph": self._original_paragraph, + "low": self._low, + "high": self._high, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_paragraph", "low", "high"] + + def check_following(self, value): + val_words = re.findall(r"\w+", value.lower()) + original_words = re.findall(r"\w+", self._original_paragraph.lower()) + similar_words = 0 + + dict_val = collections.Counter(val_words) + dict_original = collections.Counter(original_words) + + for word in dict_original: + similar_words += min(dict_original[word], dict_val[word]) + + return similar_words >= self._low and similar_words <= self._high + + +class TwoResponsesChecker(Instruction): + """Check that two responses were given.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Give two different responses. Responses and only responses should" + " be separated by 6 asterisk symbols: ******." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has two different answers. + + Args: + value: A string representing the response. + + Returns: + True if two responses are detected and false otherwise. + """ + valid_responses = list() + responses = value.split("******") + for index, response in enumerate(responses): + if not response.strip(): + if index != 0 and index != len(responses) - 1: + return False + else: + valid_responses.append(response) + return ( + len(valid_responses) == 2 and valid_responses[0].strip() != valid_responses[1].strip() + ) + + +class RepeatPromptThenAnswer(Instruction): + """Checks that Prompt is first repeated then answered.""" + + def build_description(self, *, prompt_to_repeat=None): + """Build the instruction description. + + Args: + prompt_to_repeat: The prompt that is meant to be repeated. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + self._description_pattern = ( + "First repeat the request word for word without change," + " then give your answer (1. do not say any words or characters" + " before repeating the request; 2. the request you need to repeat" + " does not include this sentence)" + ) + return self._description_pattern + + def get_instruction_args(self): + return {"prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["prompt_to_repeat"] + + def check_following(self, value): + if value.strip().lower().startswith(self._prompt_to_repeat.strip().lower()): + return True + return False + + +class EndChecker(Instruction): + """Checks that the prompt ends with a given phrase.""" + + def build_description(self, *, end_phrase=None): + """Build the instruction description. + + Args: + end_phrase: A string representing the phrase the response should end with. + + Returns: + A string representing the instruction description. + """ + self._end_phrase = end_phrase.strip() if isinstance(end_phrase, str) else end_phrase + if self._end_phrase is None: + self._end_phrase = random.choice(_ENDING_OPTIONS) + self._description_pattern = ( + "Finish your response with this exact phrase {ender}. " + "No other words should follow this phrase." + ) + return self._description_pattern.format(ender=self._end_phrase) + + def get_instruction_args(self): + return {"end_phrase": self._end_phrase} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["end_phrase"] + + def check_following(self, value): + """Checks if the response ends with the expected phrase.""" + value = value.strip().strip('"').lower() + self._end_phrase = self._end_phrase.strip().lower() + return value.endswith(self._end_phrase) + + +class TitleChecker(Instruction): + """Checks the response for a title.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your answer must contain a title, wrapped in double angular brackets," + " such as <>." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains a title.""" + pattern = r"<<[^\n]+>>" + re_pattern = re.compile(pattern) + titles = re.findall(re_pattern, value) + + for title in titles: + if title.lstrip("<").rstrip(">").strip(): + return True + return False + + +class LetterFrequencyChecker(Instruction): + """Checks letter frequency.""" + + def build_description(self, *, letter=None, let_frequency=None, let_relation=None): + """Build the instruction description. + + Args: + letter: A string representing a letter that is expected in the response. + let_frequency: An integer specifying the number of times `keyword` is + expected to appear in the response. + let_relation: A string in (`less than`, `at least`), defining the + relational operator for comparison. Two relational comparisons are + supported for now; if 'less than', the actual number of + occurrences < frequency; if 'at least', the actual number of + occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if not letter or len(letter) > 1 or ord(letter.lower()) < 97 or ord(letter.lower()) > 122: + self._letter = random.choice(list(string.ascii_letters)) + else: + self._letter = letter.strip() + self._letter = self._letter.lower() + + self._frequency = let_frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _LETTER_FREQUENCY) + + if let_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif let_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {let_relation} is given." + ) + else: + self._comparison_relation = let_relation + + self._description_pattern = ( + "In your response, the letter {letter} should appear {let_relation}" + " {let_frequency} times." + ) + + return self._description_pattern.format( + letter=self._letter, + let_frequency=self._frequency, + let_relation=self._comparison_relation, + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "letter": self._letter, + "let_frequency": self._frequency, + "let_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["letter", "let_frequency", "let_relation"] + + def check_following(self, value): + """Checks that the response contains the letter at the right frequency.""" + value = value.lower() + letters = collections.Counter(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return letters[self._letter] < self._frequency + else: + return letters[self._letter] >= self._frequency + + +class CapitalLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all capital letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all capital letters." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all capital letters.""" + assert isinstance(value, str) + + try: + return value.isupper() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class LowercaseLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all lowercase letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all lowercase" + " letters. No capital letters are allowed." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all lowercase letters.""" + assert isinstance(value, str) + + try: + return value.islower() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class CommaChecker(Instruction): + """Checks the response for no commas.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "In your entire response, refrain from the use of any commas." + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response does not contain commas.""" + return not re.search(r"\,", value) + + +class CapitalWordFrequencyChecker(Instruction): + """Checks frequency of words with all capital letters.""" + + def build_description( + self, + capital_frequency=None, + capital_relation=None, + ): + """Build the instruction description. + + Args: + capital_frequency: An integer that represents the number of words that + should be in all capital letters. + capital_relation: A string that is 'at least' or 'at most' that refers to + the frequency. + + Returns: + A string representing the instruction description. + """ + self._frequency = capital_frequency + if self._frequency is None: + self._frequency = random.randint(1, _ALL_CAPITAL_WORD_FREQUENCY) + + self._comparison_relation = capital_relation + if capital_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif capital_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {capital_relation} is given." + ) + + self._description_pattern = ( + "In your response, words with all capital letters should appear" + " {relation} {frequency} times." + ) + + return self._description_pattern.format( + frequency=self._frequency, relation=self._comparison_relation + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "capital_frequency": self._frequency, + "capital_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["capital_frequency", "capital_relation"] + + def check_following(self, value): + """Checks the frequency of words with all capital letters.""" + # Hyphenated words will count as one word + words = instructions_util.nltk.word_tokenize(value) + capital_words = [word for word in words if word.isupper()] + + capital_words = len(capital_words) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return capital_words < self._frequency + else: + return capital_words >= self._frequency + + +class QuotationChecker(Instruction): + """Checks response is wrapped with double quotation marks.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Wrap your entire response with double quotation marks." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is wrapped with double quotation marks.""" + value = value.strip() + return len(value) > 1 and value[0] == '"' and value[-1] == '"' + + +# Everything as follows is part of OOD IFEval + + +class WordCountRangeChecker(Instruction): + """Word Count Range: The response must contain between X and Y words.""" + + def build_description(self, *, min_words=None, max_words=None): + """Build the instruction description. + + Args: + min_words: An integer specifying the minimum number of words contained in the response. + max_words: An integer specifying the maximum number of words contained in the response. + + Returns: + A string representing the instruction description. + """ + self._min_words = min_words + self._max_words = max_words + + if self._min_words is None or self._min_words < 0: + self._min_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) + + # Make the range small + if self._max_words is None or self._max_words < 0: + self._max_words = self._min_words + random.randint( + int(self._min_words * 0.05), int(self._min_words * 0.1) + ) + + self._description_pattern = ( + "The response must contain between {min_words} and {max_words} words." + ) + + return self._description_pattern.format( + min_words=self._min_words, max_words=self._max_words + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"min_words": self._min_words, "max_words": self._max_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["min_words", "max_words"] + + def check_following(self, value): + """Checks if the response contains the expected number of words.""" + num_words = instructions_util.count_words(value) + return self._min_words <= num_words <= self._max_words + + +class UniqueWordCountChecker(Instruction): + """Unique Word Count: The response must contain X unique words.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + n: An integer specifying the number of unique words contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_unique_words = N + + if self._num_unique_words is None or self._num_unique_words < 0: + self._num_unique_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) + + self._description_pattern = "Use at least {N} unique words in the response." + + return self._description_pattern.format(N=self._num_unique_words) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_unique_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response contains the expected number of unique words.""" + words = value.lower().split() + unique_words = set() + for word in words: + unique_words.add(word.strip("".join(string.punctuation) + " ")) + # Convert to set to get unique words + return len(unique_words) >= self._num_unique_words + + +class StopWordPercentageChecker(Instruction): + """Ensure that stop words constitute no more than {percentage}% of the total words in your response.""" + + def build_description(self, *, percentage=None): + """Build the instruction description. + + Args: + percentage: An integer specifying the percentage of stop words that are allowed in the response. + + Returns: + A string representing the instruction description. + """ + self._percentage = percentage + + if self._percentage is None or self._percentage < 0: + self._percentage = random.randint(1, 100) + + self._description_pattern = "Ensure that stop words constitute no more than {percentage}% of the total words in your response." + + return self._description_pattern.format(percentage=self._percentage) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"percentage": self._percentage} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["percentage"] + + def check_following(self, value): + """Checks if the response contains the expected percentage of stop words.""" + num_words = instructions_util.count_words(value) + num_stopwords = instructions_util.count_stopwords(value) + stopword_percentage = (num_stopwords / num_words) * 100 + return stopword_percentage <= self._percentage + + +class SentTypeRatioChecker(Instruction): + """Maintain a 2:1 ratio of declarative to interrogative sentences.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Maintain a 2:1 ratio of declarative to interrogative sentences." + ) + nltk.download("punkt_tab") + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains the expected ratio of declarative to interrogative sentences.""" + # Split the text into sentences + sentences = instructions_util.split_into_sentences(value) + # Count the number of declarative and interrogative sentences + declarative_count = sum(1 for sentence in sentences if sentence.endswith(".")) + interrogative_count = sum(1 for sentence in sentences if sentence.endswith("?")) + # Check if the ratio is 2:1 + return declarative_count == 2 * interrogative_count + + +class SentBalanceChecker(Instruction): + """Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced.""" + + def build_description(self): + """Build the instruction description.""" + nltk.download("punkt_tab") + self._description_pattern = "Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced." + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains a balanced ratio of sentence types.""" + # Split the text into sentences + sentences = instructions_util.split_into_sentences(value) + # Count the number of each sentence type + declarative_count = sum(1 for sentence in sentences if sentence.endswith(".")) + interrogative_count = sum(1 for sentence in sentences if sentence.endswith("?")) + exclamatory_count = sum(1 for sentence in sentences if sentence.endswith("!")) + # Check if the ratio of sentence types is balanced + return declarative_count == interrogative_count == exclamatory_count + + +class ConjunctionCountChecker(Instruction): + """Use at least {small_n} different coordinating conjunctions in the response.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the number of different coordinating conjunctions contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_conjunctions = small_n + + if self._num_conjunctions is None or self._num_conjunctions < 0: + self._num_conjunctions = random.randint(2, _NUM_CONJUNCTIONS) + + self._description_pattern = ( + "Use at least {small_n} different coordinating conjunctions in the response." + ) + + return self._description_pattern.format(small_n=self._num_conjunctions) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._num_conjunctions} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if the response contains the expected number of different coordinating conjunctions.""" + # Split the text into words + words = value.split() + # Count the number of coordinating conjunctions + conjunctions = [ + word + for word in words + if word.strip("".join(string.punctuation) + " ").lower() + in ["and", "but", "for", "nor", "or", "so", "yet"] + ] + unique_conjunctions = set(conjunctions) + return len(unique_conjunctions) >= self._num_conjunctions + + +class PersonNameCountChecker(Instruction): + """Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia...""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the minimum number of unique person names contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_person_names = N + + if self._num_person_names is None or self._num_person_names < 0: + self._num_person_names = random.randint(1, 50) + + self._description_pattern = "Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia, Jackson, Olivia, Noah, Ava, Lucas, Isabella, Mason, Mia, Ethan, Charlotte, Alexander, Amelia, Benjamin, Harper, Leo, Zoe, Daniel, Chloe, Samuel, Lily, Matthew, Grace, Owen, Abigail, Gabriel, Ella, Jacob, Scarlett, Nathan, Victoria, Elijah, Layla, Nicholas, Audrey, David, Hannah, Christopher, Penelope, Thomas, Nora, Andrew, Aria, Joseph, Claire, Ryan, Stella, Jonathan ." + return self._description_pattern.format(N=self._num_person_names) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_person_names} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response contains at least the expected number of unique person names.""" + person_name_list = [ + "Emma", + "Liam", + "Sophia", + "Jackson", + "Olivia", + "Noah", + "Ava", + "Lucas", + "Isabella", + "Mason", + "Mia", + "Ethan", + "Charlotte", + "Alexander", + "Amelia", + "Benjamin", + "Harper", + "Leo", + "Zoe", + "Daniel", + "Chloe", + "Samuel", + "Lily", + "Matthew", + "Grace", + "Owen", + "Abigail", + "Gabriel", + "Ella", + "Jacob", + "Scarlett", + "Nathan", + "Victoria", + "Elijah", + "Layla", + "Nicholas", + "Audrey", + "David", + "Hannah", + "Christopher", + "Penelope", + "Thomas", + "Nora", + "Andrew", + "Aria", + "Joseph", + "Claire", + "Ryan", + "Stella", + "Jonathan", + ] + # Extract the named entities + person_names = [] + for name in person_name_list: + if name in value: + person_names.append(name) + unique_person_names = set(person_names) + + return len(unique_person_names) >= self._num_person_names + + +class NGramOverlapChecker(Instruction): + """Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text.""" + + def build_description(self, *, reference_text=None, percentage=None): + """Build the instruction description. + + Args: + reference_text: A string representing the reference text. + percentage: An integer specifying the percent trigram overlap + to maintain in the response. + + Returns: + A string representing the instruction description. + """ + self._reference_text = reference_text + self._percentage = percentage + if self._percentage is None or self._percentage < 0: + self._percentage = random.randint(1, 100) + + self._description_pattern = ( + "Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text." + ) + return self._description_pattern.format(percentage=self._percentage) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"reference_text": self._reference_text, "percentage": self._percentage} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["reference_text", "percentage"] + + def check_following(self, value): + """Checks if the response maintains a trigram overlap with the reference text within 2% of {percent}.""" + n = 3 + ngrams = set(nltk.ngrams(value, n)) + ref_ngrams = set(nltk.ngrams(self._reference_text, n)) + overlap = len(ngrams.intersection(ref_ngrams)) / len(ngrams) + return self._percentage - 2 <= overlap * 100 <= self._percentage + 2 + + +class NumbersCountChecker(Instruction): + """Include exactly {N} numbers in the response.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the exact number of numbers + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._count_numbers = N + if self._count_numbers is None or self._count_numbers < 0: + self._count_numbers = random.randint(1, _NUM_NUMBERS) + + self._description_pattern = "Include exactly {N} numbers in the response." + return self._description_pattern.format(N=self._count_numbers) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._count_numbers} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response includes exactly {N} numbers.""" + # Strip punctuation to handle decimals and commas in numbers correctly + value = value.translate(str.maketrans("", "", string.punctuation)) + numbers = re.findall(r"\d+", value) + return len(numbers) == self._count_numbers + + +class AlphabetLoopChecker(Instruction): + """Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each word of the response starts with the next letter of the alphabet.""" + value = value.translate(str.maketrans("", "", string.punctuation)) + words = value.strip("".join(string.punctuation) + " ").split() + alphabet = string.ascii_lowercase + correct_letter = words[0][0].lower() + if correct_letter not in alphabet: # numbers are fails + return False + for word in words[1:]: + word = word.strip("".join(string.punctuation) + " ").lower() + if not word: + continue + correct_letter = alphabet[(alphabet.index(correct_letter) + 1) % 26] + if word[0] != correct_letter: + return False + return True + + +class SingleVowelParagraphChecker(Instruction): + """Write a paragraph using words that contain only three type of vowels.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Write a paragraph using words that contain only three types of vowels." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if no more than three types of vowels are used in the response and the response is only 1 paragraph.""" + paragraphs = value.strip().split("\n") + if len(paragraphs) != 1: + return False + paragraph = paragraphs[0].lower() + + vowels = set("aeiou") + paragraph_vowels = set([char for char in paragraph if char in vowels]) + return len(paragraph_vowels) <= 3 + + +class ConsonantClusterChecker(Instruction): + """Ensure each word in your response has at least one consonant cluster (two or more consonants together).""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Ensure each word in your response has at least one consonant cluster (two or more consonants together)." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each word in the response includes at least one consonant cluster.""" + words = value.lower().strip().split() + consonants = set("bcdfghjklmnpqrstvwxyz") + for word in words: + cluster = False + for i in range(len(word) - 1): + if word[i] in consonants and word[i + 1] in consonants: + cluster = True + break + if not cluster: + return False + return True + + +class IncrementingAlliterationChecker(Instruction): + """Each sentence must have a longer sequence of consecutive alliterative words than the previous one.""" + + def build_description(self): + """Build the instruction description.""" + nltk.download("punkt_tab") + self._description_pattern = "Each sentence must have a longer sequence of consecutive alliterative words than the previous one." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each sentence in the response has more alliterative words (determined by common first letter) than the previous sentence.""" + sentences = instructions_util.split_into_sentences(value) + prev_alliteration = -1 + for sentence in sentences: + words = sentence.lower().split() + alliteration = 0 + prev_alliterative = False + new_words = [] + for word in words: + clean = word.lstrip("".join(string.punctuation) + " ") + if clean: + new_words.append(clean) + for i in range(len(new_words) - 1): + if new_words[i][0] == new_words[i + 1][0]: + if prev_alliterative: + alliteration += 1 + else: + alliteration += 2 + prev_alliterative = True + else: + prev_alliterative = False + if alliteration <= prev_alliteration: + return False + prev_alliteration = alliteration + return True + + +class PalindromeChecker(Instruction): + """Include at least 10 single-word palindromes, each at least 5 characters long.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Include at least 10 single-word palindromes, each at least 5 characters long." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes at least 10 single-word palindromes of length at least 5.""" + value = value.translate(str.maketrans("", "", string.punctuation)) + words = value.lower().split() + palindromes = [word for word in words if word == word[::-1] and len(word) >= 5] + return len(palindromes) >= 10 + + +class PunctuationCoverChecker(Instruction): + """Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!).""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!)." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes every standard punctuation mark at least once, including the interrobang (?!).""" + punctuation = {".", ",", "!", "?", ";", ":"} + if not ("!?" in value or "?!" in value or "‽" in value): + return False + new_value = value.replace("?!", "", 1) + if len(new_value) == len(value): + new_value = value.replace("!?", "", 1) + for char in new_value: + if char in punctuation: + punctuation.remove(char) + return not punctuation + + +class NestedParenthesesChecker(Instruction): + """Nest parentheses (and [brackets {and braces}]) at least 5 levels deep.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Nest parentheses (and [brackets {and braces}]) at least 5 levels deep." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes a correctly closed set of at least 5 nested brackets.""" + levels = [] + min_levels = 5 + max_depth = 0 + depth_stack = [] # Track depth per matched group + + for char in value: + if char in "([{": + levels.append(char) + if len(levels) > max_depth: + max_depth = len(levels) + elif char in ")]}": + if levels and ( + (levels[-1] == "(" and char == ")") + or (levels[-1] == "[" and char == "]") + or (levels[-1] == "{" and char == "}") + ): + levels.pop() + # Check if we just closed a group that reached 5+ depth + if max_depth >= min_levels and len(levels) < max_depth: + return True + else: + # Mismatch — reset + levels = [] + max_depth = 0 + + return False + + +class NestedQuotesChecker(Instruction): + """Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes nested quotes to at least 3 levels + alternating between " and ' starting with either character.""" + levels = [] + min_levels = 3 + reached_depth = 0 + current_depth = 0 + for char in value: + if len(levels) != 0 and char == levels[-1]: + levels.pop() + current_depth -= 1 + if reached_depth - current_depth >= min_levels: + return True + elif char == '"' or char == "'": + levels.append(char) + current_depth += 1 + if current_depth > reached_depth: + reached_depth = current_depth + return False + + +class PrimeLengthsChecker(Instruction): + """Use only words with lengths that are prime numbers.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Use only words with lengths that are prime numbers." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response only includes words with prime length.""" + value = value.translate(str.maketrans("", "", string.punctuation)) + words = value.split() + primes = set( + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + ] + ) + for word in words: + if len(word) not in primes: + return False + return True + + +class OptionsResponseChecker(Instruction): + """Answer with one of the following options: {options}. Do not give any explanation.""" + + def build_description(self, *, options=None): + """Build the instruction description. + + Args: + options: A string specifying the permitted options for + the response. + + Returns: + A string representing the instruction description. + """ + # Options string may be: yes/no/maybe, I know or I don't know, a), b), c), d) + # Can be separated by "/", "or", "," + options_bank = ["yes/no/maybe", "I know or I don't know", "a), b), c), d)"] + if options is None: + options = random.choice(options_bank) + + # Be more strict about format for multiple choice letters than for text options + self._strict = False + if re.match(r"\W*[aA]\W*[bB]\W*[cC]\W*", options) is not None: + self._strict = True + if "/" in options: + separator = "/" + elif "or" in options: + separator = "or" + else: + separator = "," + self._options = [option.strip() for option in options.split(separator)] + self._options_text = options # in text, shouldn't be formatted as a list + self._description_pattern = ( + "Answer with one of the following options: {options}. Do not give any explanation." + ) + return self._description_pattern.format(options=self._options_text) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"options": self._options_text} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["options"] + + def check_following(self, value): + """Checks if the response is exactly one of {options}.""" + if self._strict: + return value in self._options + value = value.strip("".join(string.punctuation) + " ").lower() + for option in self._options: + if option.strip("".join(string.punctuation) + " ").lower() == value: + return True + return False + + +class NewLineWordsChecker(Instruction): + """Write each word on a new line.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Write each word on a new line." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has each word on a new line.""" + value = value.translate(str.maketrans("", "", string.punctuation)) + lines = value.strip().split("\n") + while "" in lines: + lines.remove("") + return len(lines) == len(value.strip().split()) + + +class EmojiSentenceChecker(Instruction): + """Please use an emoji at the end of every sentence.""" + + def build_description(self): + """Build the instruction description.""" + nltk.download("punkt_tab") + self._description_pattern = "Please use an emoji at the end of every sentence." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes an emoji at the end of every sentence.""" + + sentences = instructions_util.split_into_sentences(value) + for i, sentence in enumerate(sentences): + stripped = sentence.translate(str.maketrans("", "", string.punctuation)).strip() + # check for empty string + if not stripped: + return False + last_char = stripped[-1] + # because blank spaces are treated oddly + second_last_char = stripped[-2] if len(stripped) > 1 else stripped[-1] + if not emoji.is_emoji(last_char) and not emoji.is_emoji(second_last_char): + if i < len(sentences) - 1: + stripped = ( + sentences[i + 1] + .translate(str.maketrans("", "", string.punctuation)) + .strip() + ) + # fixed empty string + if not stripped: + return False + first_char = stripped[0] + if not emoji.is_emoji(first_char): + return False + else: + return False + return True + + +class CharacterCountUniqueWordsChecker(Instruction): + """Respond with three sentences, all containing the same number of characters but using all different words.""" + + def build_description(self): + """Build the instruction description.""" + nltk.download("punkt_tab") + self._description_pattern = "Respond with three sentences, all containing the same number of characters but using all different words." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has exactly 3 sentences containing the same number of characters but different words.""" + sentences = instructions_util.split_into_sentences(value) + if len(sentences) != 3: + return False + char_count = len(sentences[0].strip()) + for sentence in sentences: + if len(sentence.strip()) != char_count: + return False + return True + + +class NthWordJapaneseChecker(Instruction): + """Every {N}th word of your response must be in Japanese.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the cycle length for + Japanese words to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._japanese_position = N + if self._japanese_position is None or self._japanese_position < 0: + self._japanese_position = random.randint(1, _NUM_WORD_CYCLE) + + self._description_pattern = "Every {N}th word of your response must be in Japanese." + if N % 10 == 1: + self._description_pattern = "Every {N}st of your response must be in Japanese." + if N % 10 == 2: + self._description_pattern = "Every {N}nd of your response must be in Japanese." + elif N % 10 == 3: + self._description_pattern = "Every {N}rd of your response must be in Japanese." + return self._description_pattern.format(N=self._japanese_position) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._japanese_position} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if every {N}th word of the response is in Japanese.""" + + def is_japanese(text): + """ + Checks if a string contains Japanese characters (Hiragana, Katakana, or Kanji). + + Args: + text: The string to check. + + Returns: + True if the string contains Japanese characters, False otherwise. + """ + japanese_pattern = re.compile(r"[\u3040-\u30ff\u4e00-\u9fff]") + return bool(japanese_pattern.search(text)) + + words = value.split() + for i, word in enumerate(words): + word = word.strip("".join(string.punctuation) + " ") + if (i + 1) % self._japanese_position == 0 and word and not word.isdigit(): + if not is_japanese(word): + return False + return True + + +class StartWithVerbChecker(Instruction): + """The response must start with a verb.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "The response must start with a verb." + nltk.download("averaged_perceptron_tagger_eng") + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response starts with a verb.""" + text = nltk.word_tokenize(value) + return len(text) > 0 and len(nltk.pos_tag(text)) > 0 and "VB" in nltk.pos_tag(text)[0][1] + + +class LimitedWordRepeatChecker(Instruction): + """The response should not repeat any word more than {small_n} times.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the maximum number of times + that a word can be repeated in the response. + + Returns: + A string representing the instruction description. + """ + self._max_repeats = small_n + if self._max_repeats is None or self._max_repeats < 0: + self._max_repeats = random.randint(1, _MAX_REPEATS) + + self._description_pattern = ( + "The response should not repeat any word more than {small_n} times." + ) + return self._description_pattern.format(small_n=self._max_repeats) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._max_repeats} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if the response repeats any word more than {small_n} times.""" + words = value.lower().translate(str.maketrans("", "", string.punctuation)).split() + word_count = Counter(words) + for word, count in word_count.items(): + if count > self._max_repeats: + return False + return True + + +class IncludeKeywordChecker(Instruction): + """The response must include keyword {word} in the {N}-th sentence.""" + + def build_description(self, *, word=None, N=None): + """Build the instruction description. + + Args: + word: A string specifying the keyword that is + required to appear in the response. + N: An integer specifying which sentence of the + response is required to have the keyword. + + Returns: + A string representing the instruction description. + """ + nltk.download("punkt_tab") + + if not word: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = word + self._keyword_position = N + if self._keyword_position is None or self._keyword_position < 0: + self._keyword_position = random.randint(1, _NUM_KEYWORD_SENTENCE) + + self._description_pattern = ( + 'The response must include keyword "{word}" in the {N}-th sentence.' + ) + return self._description_pattern.format(word=self._keyword, N=self._keyword_position) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"word": self._keyword, "N": self._keyword_position} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["word", "N"] + + def check_following(self, value): + """Checks if the {N}th sentence of the response includes keyword {word}.""" + sentences = instructions_util.split_into_sentences(value) + if len(sentences) < self._keyword_position: + return False + return self._keyword.lower() in sentences[int(self._keyword_position - 1)].lower() + + +class PronounCountChecker(Instruction): + """The response should include at least {N} pronouns.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the minimum number of pronouns + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._num_pronouns = N + if self._num_pronouns is None or self._num_pronouns < 0: + self._num_pronouns = random.randint(1, _NUM_PRONOUNS) + + self._description_pattern = "The response should include at least {N} pronouns." + return self._description_pattern.format(N=self._num_pronouns) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_pronouns} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response includes at least {N} pronouns.""" + pronouns = set( + [ + "i", + "me", + "my", + "mine", + "myself", + "we", + "us", + "our", + "ours", + "ourselves", + "you", + "your", + "yours", + "yourself", + "yourselves", + "he", + "him", + "his", + "himself", + "she", + "her", + "hers", + "herself", + "it", + "its", + "itself", + "they", + "them", + "their", + "theirs", + "themselves", + ] + ) + value = value.replace( + "/", " " + ) # to correctly count pronoun sets like she/her/hers, a common use case of pronouns + value = value.lower().translate(str.maketrans("", "", string.punctuation)) + words = value.split() + pronoun_count = sum(1 for word in words if word in pronouns) + return pronoun_count >= self._num_pronouns + + +class AlternateParitySyllablesChecker(Instruction): + """Alternate between words with odd and even numbers of syllables.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Alternate between words with odd and even numbers of syllables." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response alternates between words with odd and even numbers of syllables.""" + words = value.translate(str.maketrans("", "", string.punctuation)).lower().split() + syllables = [syllapy.count(word) % 2 for word in words if word.strip()] + return all(syllables[i] != syllables[i + 1] for i in range(len(syllables) - 1)) + + +class LastWordFirstNextChecker(Instruction): + """The last word of each sentence must become the first word of the next sentence.""" + + def build_description(self): + """Build the instruction description.""" + nltk.download("punkt_tab") + self._description_pattern = ( + "The last word of each sentence must become the first word of the next sentence." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the last word of each sentence in the response is the first word of the next sentence.""" + sentences = instructions_util.split_into_sentences(value) + for i in range(len(sentences) - 1): + last_word = sentences[i].rstrip("".join(string.punctuation) + " ").split()[-1] + first_word = sentences[i + 1].lstrip("".join(string.punctuation) + " ").split()[0] + if last_word.lower() != first_word.lower(): + return False + return True + + +class ParagraphLastFirstWordMatchChecker(Instruction): + """Each paragraph must end with the same word it started with, separate paragraphs with a newline.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Each paragraph must end with the same word it started with, separate paragraphs with a newline." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each paragraph of the response ends with the same word it started with.""" + paragraphs = value.split("\n") + for paragraph in paragraphs: + paragraph = paragraph.strip().lower() + if not paragraph: + continue + words = paragraph.strip("".join(string.punctuation) + " ").split() + if not words: + continue + if words[0] != words[-1]: + return False + return True + + +class IncrementingWordCountChecker(Instruction): + """Each sentence must contain exactly {small_n} more words than the previous one.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the exact increment for + the number of words in each sentence of the response. + + Returns: + A string representing the instruction description. + """ + self._num_increment = small_n + if self._num_increment is None or self._num_increment < 0: + self._num_increment = random.randint(1, _NUM_INCREMENT) + + nltk.download("punkt_tab") + + self._description_pattern = ( + "Each sentence must contain exactly {small_n} more words than the previous one." + ) + return self._description_pattern.format(small_n=self._num_increment) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._num_increment} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if each sentence of the response uses exactly {small_n} more words than the previous sentence.""" + sentences = instructions_util.split_into_sentences(value) + words = sentences[0].translate(str.maketrans("", "", string.punctuation)).strip().split() + while "" in words: + words.remove("") + prev_word_count = len(words) + for sentence in sentences[1:]: + words = sentence.translate(str.maketrans("", "", string.punctuation)).strip().split() + while "" in words: + words.remove("") + if len(words) != prev_word_count + self._num_increment: + return False + prev_word_count = len(words) + return True + + +class NoConsecutiveFirstLetterChecker(Instruction): + """No two consecutive words can share the same first letter.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "No two consecutive words can share the same first letter." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if no two consecutive words in the response share the same first letter.""" + words = value.lower().translate(str.maketrans("", "", string.punctuation)).split() + while "" in words: + words.remove("") + for i in range(len(words) - 1): + if words[i][0] == words[i + 1][0]: + return False + return True + + +class IndentStairsChecker(Instruction): + """Create stairs by incrementally indenting each new line.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Create stairs by incrementally indenting each new line." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response incrementally indents each new line.""" + lines = value.split("\n") + for line in lines: + if not line.strip(): + lines.remove(line) + for i in range(len(lines) - 1): + if len(lines[i + 1]) - len(lines[i + 1].lstrip(" ")) <= len(lines[i]) - len( + lines[i].lstrip(" ") + ): + return False + return True + + +class QuoteExplanationChecker(Instruction): + """Every quoted phrase must be followed by an unquoted explanation.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Every quoted phrase must be followed by an unquoted explanation." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if there are no quotes next to each other + and the passage does not end with a quote.""" + value = value.replace("“", '"').replace("”", '"') + value = value.replace("'\"'", "") # remove references to the character '"' + value = "".join(value.split()) # remove all whitespace + if '""' in value: + return False + if value.strip(string.digits + string.punctuation.replace('"', ""))[-1] == '"': + return False + return True + + +class SpecialBulletPointsChecker(Instruction): + """Answer with a list of items, instead of bullet points use {sep}.""" + + def build_description(self, *, sep=None): + """Build the instruction description. + + Args: + sep: A string specifying the bullet point marker for + the list in the response. + + Returns: + A string representing the instruction description. + """ + self._bullet_marker = sep + if sep is None: + self._bullet_marker = random.choice(["...", "SEPARATOR", "!?!?", "-"]) + self._description_pattern = ( + "Answer with a list of items, instead of bullet points use {sep}." + ) + return self._description_pattern.format(sep=self._bullet_marker) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"sep": self._bullet_marker} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["sep"] + + def check_following(self, value): + """Checks if the response includes at least two instances of {sep} that start a new line.""" + return len(re.findall(re.escape(self._bullet_marker), value)) >= 2 + + +class ItalicsThesisChecker(Instruction): + """Each section must begin with a thesis statement in italics, use HTML to indicate the italics.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Each section must begin with a thesis statement in italics, use HTML to indicate the italics." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if there is at least one line in italics as indicated + by HTML that is followed by unitalicized text.""" + index = value.find("") + if index == -1: + index = value.find("") + if index == -1: + return False + value = value[index:] + end_thesis = value.find("") + if end_thesis == -1: + end_thesis = value.find("") + if end_thesis == -1: + return False + thesis = value[3:end_thesis] + if thesis.strip() == "": + return False + text = value[end_thesis + 4 :] + return text.strip() != "" + + +class SubBulletPointsChecker(Instruction): + """Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that there is at least one * that starts a line and each * that starts a line + is followed by at least one line starting with -.""" + bullets = value.split("*") + for bullet in bullets[1:]: + if "-" not in bullet: + return False + return True + + +class SomeBulletPointsChecker(Instruction): + """Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes at least two sentences + followed by at least two lines that start with *.""" + lines = value.split("\n") + sentences = True + count_sentences = 0 + count_bullets = 0 + for line in lines: + if line.strip().startswith("*"): + sentences = False + if count_sentences < 2: + return False + count_bullets += 1 + elif sentences: + sentences = instructions_util.split_into_sentences(line.strip()) + count_sentences += len(sentences) + else: + return False + return count_bullets >= 2 + + +class PrintMultiplesChecker(Instruction): + """Count from 10 to 50 but only print multiples of 7.""" + + def build_description(self, **kwargs): + self._description_pattern = "Count from 10 to 50 but only print multiples of 7." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response prints multiples of 7 from 10 to 50.""" + value = value.replace(",", ", ") + numbers = re.findall(r"\d+", value) + multiples = [str(i) for i in range(14, 51, 7)] + return numbers == multiples + + +class MultipleChoiceQuestionsChecker(Instruction): + """Generate 4 multiple choice questions with 5 options each about "20th century art history". Each question should start with the label "Question". The questions should get progressively longer. Do not provide an explanation.""" + + def build_description(self, **kwargs): + self._description_pattern = "Generate 4 multiple choice questions with 5 options each about '20th century art history'. Each question should start with the label \"Question\". The questions should get progressively longer. Do not provide an explanation." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response generates 4 multiple choice questions with 5 options.""" + # Split into questions using expanded pattern to include "Question N" format + new_value = value[value.find("Question") :] + if new_value != value: + return False # failed no explanation + value = new_value + questions = re.split(r"\n*(?:Question \d+[\.|\):;]?\s*)", value) + if questions[0] == "": + questions = questions[1:] + questions = [q.strip() for q in questions if q.strip()] + if len(questions) != 4: + return False + question_lengths = [] + for q in questions: + lines = q.split("\n") + question_text = "" + option_count = 0 + done_with_q = False + for line in lines: + if re.match(r"^[A-Ea-e][\.|\)]\s*\w+", line.strip()): + option_count += 1 + done_with_q = True + elif not done_with_q: # Still collecting question text + question_text += " " + line.strip() + if option_count != 5: + return False + question_lengths.append(len(question_text.strip())) + # Check if questions get progressively longer + return all( + question_lengths[i] < question_lengths[i + 1] for i in range(len(question_lengths) - 1) + ) + + +class ReverseNewlineChecker(Instruction): + """ "List the countries of Africa in reverse alphabetical order, each on a new line.""" + + def build_description(self, **kwargs): + self._description_pattern = ( + "List the countries of Africa in reverse alphabetical order, each on a new line." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ + Checks if text satisfies the following constraints: + 1. Contains at least 53 newlines with text + 2. Lines are in reverse alphabetical order + 3. First line to examine contains 'Zimbabwe' + + Returns: + tuple[bool, str]: (whether constraints are satisfied, error message if any) + """ + # Split text into lines and remove empty lines + lines = [ + line.strip("".join(string.punctuation) + " ") + for line in value.split("\n") + if line.strip("".join(string.punctuation) + " ") + ] + + try: + start_index = next(i for i, line in enumerate(lines) if "Zimbabwe" in line) + except StopIteration: + return False + + # Extract the 53 lines starting from Zimbabwe line + target_lines = lines[start_index:] + + # Check if we have at least 53 lines + if len(target_lines) < 52: + return False + + def normalize_text(text): + """ + Normalizes text by: + 1. Converting to NFKD form (separates combined characters) + 2. Removes diacritical marks + 3. Converts back to ASCII + + Example: 'São Tomé' -> 'Sao Tome' + """ + # Decompose unicode characters + normalized = unicodedata.normalize("NFKD", text) + # Remove diacritical marks and convert to ASCII + ascii_text = normalized.encode("ASCII", "ignore").decode("ASCII") + return ascii_text + + # Create normalized versions for comparison while keeping originals for error messages + normalized_lines = [normalize_text(line) for line in target_lines] + sorted_normalized = sorted(normalized_lines, reverse=True) + return normalized_lines == sorted_normalized + + +class WordReverseOrderChecker(Instruction): + """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word.""" + + def build_description(self, **kwargs): + nltk.download("punkt_tab") + self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the reverse of the sentence is a valid English sentence.""" + value = value.lower().strip().translate(str.maketrans("", "", string.punctuation)) + value = " ".join(value.split()[::-1]) + if "bald eagle" not in value: + return False + return value in instructions_util.split_into_sentences(value) + + +class CharacterReverseOrderChecker(Instruction): + """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter.""" + + def build_description(self, **kwargs): + self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + value = value.lower() + return "elgae dlab" in value + + +class SentenceAlphabetChecker(Instruction): + """Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order.""" + + def build_description(self, **kwargs): + nltk.download("punkt_tab") + self._description_pattern = "Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + sentences = instructions_util.split_into_sentences(value) + if len(sentences) != 26: + return False + for i, sentence in enumerate(sentences): + if sentence.lstrip().split()[0].lower()[0] != chr(97 + i): + return False + return True + + +class EuropeanCapitalsSortChecker(Instruction): + """Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = "Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response lists the relevant capitals of Europe in correct order.""" + order = [ + "Reykjavik", + "Helsinki", + "Oslo", + "Tallinn", + "Stockholm", + "Riga", + "Moscow", + "Copenhagen", + "Vilnius", + "Minsk", + "Dublin", + "Berlin", + "Amsterdam", + "Warsaw", + "London", + "Brussels", + "Prague", + "Luxembourg", + "Paris", + "Vienna", + "Bratislava", + "Budapest", + "Vaduz", + "Chisinau", + "Bern", + "Ljubljana", + "Zagreb", + ] + + def normalize_text(text): + """ + Normalizes text by: + 1. Converting to NFKD form (separates combined characters) + 2. Removes diacritical marks + 3. Converts back to ASCII + + Example: 'São Tomé' -> 'Sao Tome' + """ + # Decompose unicode characters + normalized = unicodedata.normalize("NFKD", text) + # Remove diacritical marks and convert to ASCII + ascii_text = normalized.encode("ASCII", "ignore").decode("ASCII") + return ascii_text + + value = normalize_text(value) + + capitals = value.split(",") + capitals = [cap for cap in capitals if cap.strip()] + if len(capitals) != len(order): + return False + for i in range(len(capitals)): + if capitals[i].strip() != order[i]: + return False + return True + + +class CityCSVChecker(Instruction): + """Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is valid csv data with column names + ["ID", "Country", "City", "Year", "Count"] and 7 rows.""" + string_io = io.StringIO(value) + reader = csv.reader(string_io) + data = list(reader) + if len(data) != 8: + return False + header = data[0] + if header != ["ID", "Country", "City", "Year", "Count"]: + return False + for row in data[1:]: + if len(row) != 5: + return False + return True + + +class SpecialCharacterCSVChecker(Instruction): + """Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ "Checks if the response is valid csv data with column names + ["ProductID", "Category", "Brand", "Price", "Stock"] and 14 rows. + Also checks if one field contains a special character enclosed in double quotes.""" + header = value.split("\n")[0].strip() + if not re.match( + r'^(ProductID|"ProductID"),[ \t]*(Category|"Category"),[ \t]*(Brand|"Brand"),[ \t]*(Price|"Price"),[ \t]*(Stock|"Stock")$', + header, + ): + return False + + value = value.replace('"', '"""') + string_io = io.StringIO(value) + reader = csv.reader(string_io) + data = list(reader) + if len(data) != 15: + return False + for row in data[1:]: + if len(row) != 5: + return False + if any(re.match(r'".*[^\d\w\s].*"', field) for field in row): + return True + return False + + +class QuotesCSVChecker(Instruction): + """Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ "Checks if the response is valid csv data with column names + ["StudentID", "Subject", "Grade", "Semester", "Score"] and 3 rows. + Also checks if each field is enclosed in double quotes.""" + header = value.split("\n")[0].strip() + if not re.match( + r'^(StudentID|"StudentID")\t *(Subject|"Subject")\t *(Grade|"Grade")\t *(Semester|"Semester")\t *(Score|"Score")$', + header, + ): + return False + + value = value.replace('"', '"""') + string_io = io.StringIO(value) + reader = csv.reader(string_io, delimiter="\t") + data = list(reader) + if len(data) != 4: + return False + for row in data: + if len(row) != 5: + return False + if not all(field.strip()[0] == '"' and field.strip()[-1] == '"' for field in row): + return False + return True + + +class DateFormatListChecker(Instruction): + """List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = "List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ "Checks if the response is a list of dates in the format YYYY-MM-DD separated by commas.""" + value = value.strip() + dates = value.split(",") + for date in dates: + date = date.strip() + if not re.match(r"^\d{4}-\d{2}-\d{2}$", date): + return False + date = date.split("-") + if int(date[0]) < 1769 or int(date[0]) > 1821: + return False + if int(date[1]) > 12: + return False + if int(date[1]) in [1, 3, 5, 7, 8, 10, 12] and int(date[2]) > 31: + return False + if int(date[1]) in [4, 6, 9, 11] and int(date[2]) > 30: + return False + if int(date[1]) == 2 and int(date[2]) > 29: + return False + return True + + +class KeywordsMultipleChecker(Instruction): + """Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response.""" + + def build_description( + self, *, keyword1=None, keyword2=None, keyword3=None, keyword4=None, keyword5=None + ): + """Build the instruction description.""" + if keyword1 is None: + self._keyword1 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword1 = keyword1.strip() + if keyword2 is None: + self._keyword2 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword2 = keyword2.strip() + if keyword3 is None: + self._keyword3 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword3 = keyword3.strip() + if keyword4 is None: + self._keyword4 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword4 = keyword4.strip() + if keyword5 is None: + self._keyword5 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword5 = keyword5.strip() + self._description_pattern = "Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response." + return self._description_pattern.format( + keyword1=self._keyword1, + keyword2=self._keyword2, + keyword3=self._keyword3, + keyword4=self._keyword4, + keyword5=self._keyword5, + ) + + def get_instruction_args(self): + return { + "keyword1": self._keyword1, + "keyword2": self._keyword2, + "keyword3": self._keyword3, + "keyword4": self._keyword4, + "keyword5": self._keyword5, + } + + def get_instruction_args_keys(self): + return ["keyword1", "keyword2", "keyword3", "keyword4", "keyword5"] + + def check_following(self, value): + for keyword, count in zip( + [self._keyword1, self._keyword2, self._keyword3, self._keyword4, self._keyword5], + [1, 2, 3, 5, 7], strict=False, + ): + if value.lower().count(keyword.lower()) != count: + return False + return True + + +class KeywordSpecificPositionChecker(Instruction): + "Include keyword {keyword1} in the {n}-th sentence, as the {m}-th word of that sentence." + + def build_description(self, keyword=None, n=None, m=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + n: An integer representing the sentence number. + m: An integer representing the word number. + + Returns: + A string representing the instruction description. + """ + if not keyword: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + if not n: + self._n = random.randint(20, 30) + else: + self._n = n + if not m: + self._m = random.randint(30, 40) + else: + self._m = m + + self._description_pattern = ( + "Include keyword {keyword} in the {n}-th sentence, as the {m}-th word of that sentence." + ) + + return self._description_pattern.format(keyword=self._keyword, n=self._n, m=self._m) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"keyword": self._keyword, "n": self._n, "m": self._m} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword", "n", "m"] + + def check_following(self, value): + """Checks if the response contains the expected number of keywords. + + Args: + value: A string representing the response. + + Returns: + True if the response contains the expected number of keywords; + otherwise, False. + """ + sentences = instructions_util.split_into_sentences(value) + if len(sentences) < self._n: + return False + words = instructions_util.nltk.word_tokenize(sentences[self._n - 1]) + if len(words) < self._m: + return False + if words[self._m - 1] == self._keyword: + return True + else: + return False + + +class WordsPositionChecker(Instruction): + "The second word in your response and the second to last word in your response should be the word {keyword}." + + def build_description(self, *, keyword=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + + Returns: + A string representing the instruction description. + """ + if keyword is None: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + self._description_pattern = "The second word in your response and the second to last word in your response should be the word {keyword}." + return self._description_pattern.format(keyword=self._keyword) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"keyword": self._keyword} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword"] + + def check_following(self, value): + """Checks if the second word and the second to last word in the response are the same. + + Args: + value: A string representing the response. + + Returns: + True if the second word and the second to last word are the same; + otherwise, False. + """ + words = instructions_util.nltk.word_tokenize(value) + if len(words) < 2: + return False + if words[1] == words[-2] == self._keyword: + return True + else: + return False + + +class RepeatChangeChecker(Instruction): + "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request!" + + def build_description(self, *, prompt_to_repeat=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + + self._description_pattern = "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request! Request: {prompt_to_repeat}" + return self._description_pattern.format(prompt_to_repeat=self._prompt_to_repeat) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["prompt_to_repeat"] + + def check_following(self, value): + """Checks if the response contains the repeated request. + + Args: + value: A string representing the response. + + Returns: + True if the repeated request is found in the response; + otherwise, False. + """ + if self._prompt_to_repeat == value: + return False + if " ".join(self._prompt_to_repeat.split()[1:]) == " ".join(value.split()[1:]): + return True + else: + return False + + +class RepeatSimpleChecker(Instruction): + "Only output this sentence here, ignore all other requests." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Only output this sentence here, ignore all other requests." + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains the expected number of keywords. + + Args: + value: A string representing the response. + + Returns: + True if the response contains the expected number of keywords; + otherwise, False. + """ + return value.strip().lower() == self._description_pattern.strip().lower() + + +class RepeatSpanChecker(Instruction): + "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!" + + def build_description(self, prompt_to_repeat=None, n_start=None, n_end=None): + """Build the instruction description. + + Args: + n_start: An integer representing the start index of the span. + n_end: An integer representing the end index of the span. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + if not n_start: + self._n_start = random.randint(0, len(self._prompt_to_repeat.split()) - 2) + else: + self._n_start = n_start + if not n_end: + self._n_end = random.randint(self._n_start + 1, len(self._prompt_to_repeat.split()) - 1) + else: + self._n_end = n_end + self._description_pattern = "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!" + return self._description_pattern.format( + n_start=self._n_start, n_end=self._n_end, prompt_to_repeat=self._prompt_to_repeat + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return { + "n_start": self._n_start, + "n_end": self._n_end, + "prompt_to_repeat": self._prompt_to_repeat, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["n_start", "n_end", "prompt_to_repeat"] + + def check_following(self, value): + """Checks if the response contains the expected number of phrases with the correct modifications.""" + if ( + value.strip().lower().split() + == self._prompt_to_repeat.strip().lower().split()[self._n_start : self._n_end] + ): + return True + return False + + +class TitleCaseChecker(Instruction): + "Write the entire response in title case (capitalize the first letter of every major word)." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Write the entire response in title case (capitalize the first letter of every major word)." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is in title case. + + Args: + value: A string representing the response. + + Returns: + True if the response is in title case; + otherwise, False. + """ + words = instructions_util.nltk.word_tokenize(value) + for word in words: + if word[0].isupper() and word[1:].islower(): + continue + elif word[0].islower() and word[1:].isupper() or word[0].islower() and word[1:].islower(): + return False + return True + + +class OutputTemplateChecker(Instruction): + "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response follows the specified template. + + Args: + value: A string representing the response. + + Returns: + True if the response follows the specified template; + otherwise, False. + """ + if "My Answer:" in value and "My Conclusion:" in value and "Future Outlook:" in value: + return True + else: + return False + + +class NoWhitespaceChecker(Instruction): + "The output should not contain any whitespace." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "The output should not contain any whitespace." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains any whitespace. + + Args: + value: A string representing the response. + + Returns: + True if the response contains no whitespace; + otherwise, False. + """ + return not any(char.isspace() for char in value) diff --git a/src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py b/src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py new file mode 100644 index 000000000..fd7ce2d22 --- /dev/null +++ b/src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py @@ -0,0 +1,251 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registry of all instructions.""" + +from olmo_eval.common.scorers.ifeval_deps import instructions + +_KEYWORD = "keywords:" + +_LANGUAGE = "language:" + +_LENGTH = "length_constraints:" + +_CONTENT = "detectable_content:" + +_FORMAT = "detectable_format:" + +_MULTITURN = "multi-turn:" + +_COMBINATION = "combination:" + +_STARTEND = "startend:" + +_CHANGE_CASES = "change_case:" + +_PUNCTUATION = "punctuation:" + +INSTRUCTION_DICT = { + _KEYWORD + "existence": instructions.KeywordChecker, + _KEYWORD + "frequency": instructions.KeywordFrequencyChecker, + # TODO(jeffreyzhou): make a proper set of sentences to choose from + # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, + _KEYWORD + "forbidden_words": instructions.ForbiddenWords, + _KEYWORD + "letter_frequency": instructions.LetterFrequencyChecker, + _LANGUAGE + "response_language": instructions.ResponseLanguageChecker, + _LENGTH + "number_sentences": instructions.NumberOfSentences, + _LENGTH + "number_paragraphs": instructions.ParagraphChecker, + _LENGTH + "number_words": instructions.NumberOfWords, + _LENGTH + "nth_paragraph_first_word": instructions.ParagraphFirstWordCheck, + _CONTENT + "number_placeholders": instructions.PlaceholderChecker, + _CONTENT + "postscript": instructions.PostscriptChecker, + _FORMAT + "number_bullet_lists": instructions.BulletListChecker, + # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace + # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, + _FORMAT + "constrained_response": instructions.ConstrainedResponseChecker, + _FORMAT + "number_highlighted_sections": (instructions.HighlightSectionChecker), + _FORMAT + "multiple_sections": instructions.SectionChecker, + # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. + # _FORMAT + "rephrase": instructions.RephraseChecker, + _FORMAT + "json_format": instructions.JsonFormat, + _FORMAT + "title": instructions.TitleChecker, + # TODO(tianjianlu): Re-enable with specific prompts. + # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, + _COMBINATION + "two_responses": instructions.TwoResponsesChecker, + _COMBINATION + "repeat_prompt": instructions.RepeatPromptThenAnswer, + _STARTEND + "end_checker": instructions.EndChecker, + _CHANGE_CASES + "capital_word_frequency": instructions.CapitalWordFrequencyChecker, + _CHANGE_CASES + "english_capital": instructions.CapitalLettersEnglishChecker, + _CHANGE_CASES + "english_lowercase": instructions.LowercaseLettersEnglishChecker, + _PUNCTUATION + "no_comma": instructions.CommaChecker, + _STARTEND + "quotation": instructions.QuotationChecker, + # the following are added for ifeval_mt variants + # "type:translate": instructions.LanguageChecker, + # "type:repeat": instructions.RepeatChecker, + # "type:repeat_N": instructions.RewriteSentChecker, + # "type:num_words": instructions.NumWordsChecker, + # "type:num_words_N": instructions.NumWordsSentChecker, + # "constraint:increment": instructions.IncrementChecker, + # "constraint:digits": instructions.DigitsChecker, + # "constraint:nonalpha": instructions.NonAlphaNumChecker, + # "constraint:palindrome": instructions.PalindromeCountChecker, + # "constraint:chars": instructions.DiffNumCharsChecker, + # "constraint:punctuation": instructions.NumPunctuationChecker, + # "constraint:nochar": instructions.NoLetterChecker, + # "constraint:conjunctions": instructions.NoConjunctionsChecker, + # "constraint:vowels": instructions.VowelStartWordsChecker, + # "constraint:top_bottom": instructions.BeginEndLetterChecker, + # back to ood + "count:word_count_range": instructions.WordCountRangeChecker, + "count:unique_word_count": instructions.UniqueWordCountChecker, + "ratio:stop_words": instructions.StopWordPercentageChecker, + "ratio:sentence_type": instructions.SentTypeRatioChecker, + "ratio:sentence_balance": instructions.SentBalanceChecker, + "count:conjunctions": instructions.ConjunctionCountChecker, + "count:person_names": instructions.PersonNameCountChecker, + "ratio:overlap": instructions.NGramOverlapChecker, + "count:numbers": instructions.NumbersCountChecker, + "words:alphabet": instructions.AlphabetLoopChecker, + "words:vowel": instructions.SingleVowelParagraphChecker, + "words:consonants": instructions.ConsonantClusterChecker, + "sentence:alliteration_increment": instructions.IncrementingAlliterationChecker, + "words:palindrome": instructions.PalindromeChecker, + "count:punctuation": instructions.PunctuationCoverChecker, + "format:parentheses": instructions.NestedParenthesesChecker, + "format:quotes": instructions.NestedQuotesChecker, + "words:prime_lengths": instructions.PrimeLengthsChecker, + "format:options": instructions.OptionsResponseChecker, + "format:newline": instructions.NewLineWordsChecker, + "format:emoji": instructions.EmojiSentenceChecker, + "ratio:sentence_words": instructions.CharacterCountUniqueWordsChecker, + "count:words_japanese": instructions.NthWordJapaneseChecker, + "words:start_verb": instructions.StartWithVerbChecker, + "words:repeats": instructions.LimitedWordRepeatChecker, + "sentence:keyword": instructions.IncludeKeywordChecker, + "count:pronouns": instructions.PronounCountChecker, + "words:odd_even_syllables": instructions.AlternateParitySyllablesChecker, + "words:last_first": instructions.LastWordFirstNextChecker, + "words:paragraph_last_first": instructions.ParagraphLastFirstWordMatchChecker, + "sentence:increment": instructions.IncrementingWordCountChecker, + "words:no_consecutive": instructions.NoConsecutiveFirstLetterChecker, + "format:line_indent": instructions.IndentStairsChecker, + "format:quote_unquote": instructions.QuoteExplanationChecker, + "format:list": instructions.SpecialBulletPointsChecker, + "format:thesis": instructions.ItalicsThesisChecker, + "format:sub-bullets": instructions.SubBulletPointsChecker, + "format:no_bullets_bullets": instructions.SomeBulletPointsChecker, + "custom:multiples": instructions.PrintMultiplesChecker, + "custom:mcq_count_length": instructions.MultipleChoiceQuestionsChecker, + "custom:reverse_newline": instructions.ReverseNewlineChecker, + "custom:word_reverse": instructions.WordReverseOrderChecker, + "custom:character_reverse": instructions.CharacterReverseOrderChecker, + "custom:sentence_alphabet": instructions.SentenceAlphabetChecker, + "custom:european_capitals_sort": instructions.EuropeanCapitalsSortChecker, + "custom:csv_city": instructions.CityCSVChecker, + "custom:csv_special_character": instructions.SpecialCharacterCSVChecker, + "custom:csv_quotes": instructions.QuotesCSVChecker, + "custom:date_format_list": instructions.DateFormatListChecker, + "count:keywords_multiple": instructions.KeywordsMultipleChecker, + "words:keywords_specific_position": instructions.KeywordSpecificPositionChecker, + "words:words_position": instructions.WordsPositionChecker, + "repeat:repeat_change": instructions.RepeatChangeChecker, + "repeat:repeat_simple": instructions.RepeatSimpleChecker, + "repeat:repeat_span": instructions.RepeatSpanChecker, + "format:title_case": instructions.TitleCaseChecker, + "format:output_template": instructions.OutputTemplateChecker, + "format:no_whitespace": instructions.NoWhitespaceChecker, +} + +INSTRUCTION_CONFLICTS = { + _KEYWORD + "existence": {_KEYWORD + "existence"}, + _KEYWORD + "frequency": {_KEYWORD + "frequency"}, + # TODO(jeffreyzhou): make a proper set of sentences to choose from + # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, + _KEYWORD + "forbidden_words": {_KEYWORD + "forbidden_words"}, + _KEYWORD + "letter_frequency": {_KEYWORD + "letter_frequency"}, + _LANGUAGE + "response_language": { + _LANGUAGE + "response_language", + _FORMAT + "multiple_sections", + _KEYWORD + "existence", + _KEYWORD + "frequency", + _KEYWORD + "forbidden_words", + _STARTEND + "end_checker", + _CHANGE_CASES + "english_capital", + _CHANGE_CASES + "english_lowercase", + }, + _LENGTH + "number_sentences": {_LENGTH + "number_sentences"}, + _LENGTH + "number_paragraphs": { + _LENGTH + "number_paragraphs", + _LENGTH + "nth_paragraph_first_word", + _LENGTH + "number_sentences", + _LENGTH + "nth_paragraph_first_word", + }, + _LENGTH + "number_words": {_LENGTH + "number_words"}, + _LENGTH + "nth_paragraph_first_word": { + _LENGTH + "nth_paragraph_first_word", + _LENGTH + "number_paragraphs", + }, + _CONTENT + "number_placeholders": {_CONTENT + "number_placeholders"}, + _CONTENT + "postscript": {_CONTENT + "postscript"}, + _FORMAT + "number_bullet_lists": {_FORMAT + "number_bullet_lists"}, + # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace + # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, + _FORMAT + "constrained_response": set(INSTRUCTION_DICT.keys()), + _FORMAT + "number_highlighted_sections": {_FORMAT + "number_highlighted_sections"}, + _FORMAT + "multiple_sections": { + _FORMAT + "multiple_sections", + _LANGUAGE + "response_language", + _FORMAT + "number_highlighted_sections", + }, + # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. + # _FORMAT + "rephrase": instructions.RephraseChecker, + _FORMAT + "json_format": set(INSTRUCTION_DICT.keys()).difference( + {_KEYWORD + "forbidden_words", _KEYWORD + "existence"} + ), + _FORMAT + "title": {_FORMAT + "title"}, + # TODO(tianjianlu): Re-enable with specific prompts. + # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, + _COMBINATION + "two_responses": set(INSTRUCTION_DICT.keys()).difference( + { + _KEYWORD + "forbidden_words", + _KEYWORD + "existence", + _LANGUAGE + "response_language", + _FORMAT + "title", + _PUNCTUATION + "no_comma", + } + ), + _COMBINATION + "repeat_prompt": set(INSTRUCTION_DICT.keys()).difference( + {_KEYWORD + "existence", _FORMAT + "title", _PUNCTUATION + "no_comma"} + ), + _STARTEND + "end_checker": {_STARTEND + "end_checker"}, + _CHANGE_CASES + "capital_word_frequency": { + _CHANGE_CASES + "capital_word_frequency", + _CHANGE_CASES + "english_lowercase", + _CHANGE_CASES + "english_capital", + }, + _CHANGE_CASES + "english_capital": {_CHANGE_CASES + "english_capital"}, + _CHANGE_CASES + "english_lowercase": { + _CHANGE_CASES + "english_lowercase", + _CHANGE_CASES + "english_capital", + }, + _PUNCTUATION + "no_comma": {_PUNCTUATION + "no_comma"}, + _STARTEND + "quotation": {_STARTEND + "quotation", _FORMAT + "title"}, + "count:keywords_multiple": {"count:keywords_multiple"}, + "words:words_position": {"words:words_position"}, + "repeat:repeat_change": {"repeat:repeat_change"}, + "repeat:repeat_simple": {"repeat:repeat_simple"}, + "repeat:repeat_span": {"repeat:repeat_span"}, + "format:title_case": {"format:title_case"}, + "format:output_template": {"format:output_template"}, + "format:no_whitespace": {"format:no_whitespace"}, + "words:keywords_specific_position": {"words:keywords_specific_position"}, +} + + +def conflict_make(conflicts): + """Makes sure if A conflicts with B, B will conflict with A. + + Args: + conflicts: Dictionary of potential conflicts where key is instruction id + and value is set of instruction ids that it conflicts with. + + Returns: + Revised version of the dictionary. All instructions conflict with + themselves. If A conflicts with B, B will conflict with A. + """ + for key in conflicts: + for k in conflicts[key]: + conflicts[k].add(key) + conflicts[key].add(key) + return conflicts diff --git a/src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py b/src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py new file mode 100644 index 000000000..5743e7972 --- /dev/null +++ b/src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py @@ -0,0 +1,1777 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility library of instructions.""" + +import ast +import functools +import random +import re + +import immutabledict +import nltk + + +def download_nltk_resources(): + """Download 'punkt' if not already installed""" + try: + nltk.data.find("tokenizers/punkt") + except LookupError: + nltk.download("punkt") + + +download_nltk_resources() + +WORD_LIST = [ + "western", + "sentence", + "signal", + "dump", + "spot", + "opposite", + "bottom", + "potato", + "administration", + "working", + "welcome", + "morning", + "good", + "agency", + "primary", + "wish", + "responsibility", + "press", + "problem", + "president", + "steal", + "brush", + "read", + "type", + "beat", + "trainer", + "growth", + "lock", + "bone", + "case", + "equal", + "comfortable", + "region", + "replacement", + "performance", + "mate", + "walk", + "medicine", + "film", + "thing", + "rock", + "tap", + "total", + "competition", + "ease", + "south", + "establishment", + "gather", + "parking", + "world", + "plenty", + "breath", + "claim", + "alcohol", + "trade", + "dear", + "highlight", + "street", + "matter", + "decision", + "mess", + "agreement", + "studio", + "coach", + "assist", + "brain", + "wing", + "style", + "private", + "top", + "brown", + "leg", + "buy", + "procedure", + "method", + "speed", + "high", + "company", + "valuable", + "pie", + "analyst", + "session", + "pattern", + "district", + "pleasure", + "dinner", + "swimming", + "joke", + "order", + "plate", + "department", + "motor", + "cell", + "spend", + "cabinet", + "difference", + "power", + "examination", + "engine", + "horse", + "dimension", + "pay", + "toe", + "curve", + "literature", + "bother", + "fire", + "possibility", + "debate", + "activity", + "passage", + "hello", + "cycle", + "background", + "quiet", + "author", + "effect", + "actor", + "page", + "bicycle", + "error", + "throat", + "attack", + "character", + "phone", + "tea", + "increase", + "outcome", + "file", + "specific", + "inspector", + "internal", + "potential", + "staff", + "building", + "employer", + "shoe", + "hand", + "direction", + "garden", + "purchase", + "interview", + "study", + "recognition", + "member", + "spiritual", + "oven", + "sandwich", + "weird", + "passenger", + "particular", + "response", + "reaction", + "size", + "variation", + "a", + "cancel", + "candy", + "exit", + "guest", + "condition", + "fly", + "price", + "weakness", + "convert", + "hotel", + "great", + "mouth", + "mind", + "song", + "sugar", + "suspect", + "telephone", + "ear", + "roof", + "paint", + "refrigerator", + "organization", + "jury", + "reward", + "engineering", + "day", + "possession", + "crew", + "bar", + "road", + "description", + "celebration", + "score", + "mark", + "letter", + "shower", + "suggestion", + "sir", + "luck", + "national", + "progress", + "hall", + "stroke", + "theory", + "offer", + "story", + "tax", + "definition", + "history", + "ride", + "medium", + "opening", + "glass", + "elevator", + "stomach", + "question", + "ability", + "leading", + "village", + "computer", + "city", + "grand", + "confidence", + "candle", + "priest", + "recommendation", + "point", + "necessary", + "body", + "desk", + "secret", + "horror", + "noise", + "culture", + "warning", + "water", + "round", + "diet", + "flower", + "bus", + "tough", + "permission", + "week", + "prompt", + "connection", + "abuse", + "height", + "save", + "corner", + "border", + "stress", + "drive", + "stop", + "rip", + "meal", + "listen", + "confusion", + "girlfriend", + "living", + "relation", + "significance", + "plan", + "creative", + "atmosphere", + "blame", + "invite", + "housing", + "paper", + "drink", + "roll", + "silver", + "drunk", + "age", + "damage", + "smoke", + "environment", + "pack", + "savings", + "influence", + "tourist", + "rain", + "post", + "sign", + "grandmother", + "run", + "profit", + "push", + "clerk", + "final", + "wine", + "swim", + "pause", + "stuff", + "singer", + "funeral", + "average", + "source", + "scene", + "tradition", + "personal", + "snow", + "nobody", + "distance", + "sort", + "sensitive", + "animal", + "major", + "negotiation", + "click", + "mood", + "period", + "arrival", + "expression", + "holiday", + "repeat", + "dust", + "closet", + "gold", + "bad", + "sail", + "combination", + "clothes", + "emphasis", + "duty", + "black", + "step", + "school", + "jump", + "document", + "professional", + "lip", + "chemical", + "front", + "wake", + "while", + "inside", + "watch", + "row", + "subject", + "penalty", + "balance", + "possible", + "adult", + "aside", + "sample", + "appeal", + "wedding", + "depth", + "king", + "award", + "wife", + "blow", + "site", + "camp", + "music", + "safe", + "gift", + "fault", + "guess", + "act", + "shame", + "drama", + "capital", + "exam", + "stupid", + "record", + "sound", + "swing", + "novel", + "minimum", + "ratio", + "machine", + "shape", + "lead", + "operation", + "salary", + "cloud", + "affair", + "hit", + "chapter", + "stage", + "quantity", + "access", + "army", + "chain", + "traffic", + "kick", + "analysis", + "airport", + "time", + "vacation", + "philosophy", + "ball", + "chest", + "thanks", + "place", + "mountain", + "advertising", + "red", + "past", + "rent", + "return", + "tour", + "house", + "construction", + "net", + "native", + "war", + "figure", + "fee", + "spray", + "user", + "dirt", + "shot", + "task", + "stick", + "friend", + "software", + "promotion", + "interaction", + "surround", + "block", + "purpose", + "practice", + "conflict", + "routine", + "requirement", + "bonus", + "hole", + "state", + "junior", + "sweet", + "catch", + "tear", + "fold", + "wall", + "editor", + "life", + "position", + "pound", + "respect", + "bathroom", + "coat", + "script", + "job", + "teach", + "birth", + "view", + "resolve", + "theme", + "employee", + "doubt", + "market", + "education", + "serve", + "recover", + "tone", + "harm", + "miss", + "union", + "understanding", + "cow", + "river", + "association", + "concept", + "training", + "recipe", + "relationship", + "reserve", + "depression", + "proof", + "hair", + "revenue", + "independent", + "lift", + "assignment", + "temporary", + "amount", + "loss", + "edge", + "track", + "check", + "rope", + "estimate", + "pollution", + "stable", + "message", + "delivery", + "perspective", + "mirror", + "assistant", + "representative", + "witness", + "nature", + "judge", + "fruit", + "tip", + "devil", + "town", + "emergency", + "upper", + "drop", + "stay", + "human", + "neck", + "speaker", + "network", + "sing", + "resist", + "league", + "trip", + "signature", + "lawyer", + "importance", + "gas", + "choice", + "engineer", + "success", + "part", + "external", + "worker", + "simple", + "quarter", + "student", + "heart", + "pass", + "spite", + "shift", + "rough", + "lady", + "grass", + "community", + "garage", + "youth", + "standard", + "skirt", + "promise", + "blind", + "television", + "disease", + "commission", + "positive", + "energy", + "calm", + "presence", + "tune", + "basis", + "preference", + "head", + "common", + "cut", + "somewhere", + "presentation", + "current", + "thought", + "revolution", + "effort", + "master", + "implement", + "republic", + "floor", + "principle", + "stranger", + "shoulder", + "grade", + "button", + "tennis", + "police", + "collection", + "account", + "register", + "glove", + "divide", + "professor", + "chair", + "priority", + "combine", + "peace", + "extension", + "maybe", + "evening", + "frame", + "sister", + "wave", + "code", + "application", + "mouse", + "match", + "counter", + "bottle", + "half", + "cheek", + "resolution", + "back", + "knowledge", + "make", + "discussion", + "screw", + "length", + "accident", + "battle", + "dress", + "knee", + "log", + "package", + "it", + "turn", + "hearing", + "newspaper", + "layer", + "wealth", + "profile", + "imagination", + "answer", + "weekend", + "teacher", + "appearance", + "meet", + "bike", + "rise", + "belt", + "crash", + "bowl", + "equivalent", + "support", + "image", + "poem", + "risk", + "excitement", + "remote", + "secretary", + "public", + "produce", + "plane", + "display", + "money", + "sand", + "situation", + "punch", + "customer", + "title", + "shake", + "mortgage", + "option", + "number", + "pop", + "window", + "extent", + "nothing", + "experience", + "opinion", + "departure", + "dance", + "indication", + "boy", + "material", + "band", + "leader", + "sun", + "beautiful", + "muscle", + "farmer", + "variety", + "fat", + "handle", + "director", + "opportunity", + "calendar", + "outside", + "pace", + "bath", + "fish", + "consequence", + "put", + "owner", + "go", + "doctor", + "information", + "share", + "hurt", + "protection", + "career", + "finance", + "force", + "golf", + "garbage", + "aspect", + "kid", + "food", + "boot", + "milk", + "respond", + "objective", + "reality", + "raw", + "ring", + "mall", + "one", + "impact", + "area", + "news", + "international", + "series", + "impress", + "mother", + "shelter", + "strike", + "loan", + "month", + "seat", + "anything", + "entertainment", + "familiar", + "clue", + "year", + "glad", + "supermarket", + "natural", + "god", + "cost", + "conversation", + "tie", + "ruin", + "comfort", + "earth", + "storm", + "percentage", + "assistance", + "budget", + "strength", + "beginning", + "sleep", + "other", + "young", + "unit", + "fill", + "store", + "desire", + "hide", + "value", + "cup", + "maintenance", + "nurse", + "function", + "tower", + "role", + "class", + "camera", + "database", + "panic", + "nation", + "basket", + "ice", + "art", + "spirit", + "chart", + "exchange", + "feedback", + "statement", + "reputation", + "search", + "hunt", + "exercise", + "nasty", + "notice", + "male", + "yard", + "annual", + "collar", + "date", + "platform", + "plant", + "fortune", + "passion", + "friendship", + "spread", + "cancer", + "ticket", + "attitude", + "island", + "active", + "object", + "service", + "buyer", + "bite", + "card", + "face", + "steak", + "proposal", + "patient", + "heat", + "rule", + "resident", + "broad", + "politics", + "west", + "knife", + "expert", + "girl", + "design", + "salt", + "baseball", + "grab", + "inspection", + "cousin", + "couple", + "magazine", + "cook", + "dependent", + "security", + "chicken", + "version", + "currency", + "ladder", + "scheme", + "kitchen", + "employment", + "local", + "attention", + "manager", + "fact", + "cover", + "sad", + "guard", + "relative", + "county", + "rate", + "lunch", + "program", + "initiative", + "gear", + "bridge", + "breast", + "talk", + "dish", + "guarantee", + "beer", + "vehicle", + "reception", + "woman", + "substance", + "copy", + "lecture", + "advantage", + "park", + "cold", + "death", + "mix", + "hold", + "scale", + "tomorrow", + "blood", + "request", + "green", + "cookie", + "church", + "strip", + "forever", + "beyond", + "debt", + "tackle", + "wash", + "following", + "feel", + "maximum", + "sector", + "sea", + "property", + "economics", + "menu", + "bench", + "try", + "language", + "start", + "call", + "solid", + "address", + "income", + "foot", + "senior", + "honey", + "few", + "mixture", + "cash", + "grocery", + "link", + "map", + "form", + "factor", + "pot", + "model", + "writer", + "farm", + "winter", + "skill", + "anywhere", + "birthday", + "policy", + "release", + "husband", + "lab", + "hurry", + "mail", + "equipment", + "sink", + "pair", + "driver", + "consideration", + "leather", + "skin", + "blue", + "boat", + "sale", + "brick", + "two", + "feed", + "square", + "dot", + "rush", + "dream", + "location", + "afternoon", + "manufacturer", + "control", + "occasion", + "trouble", + "introduction", + "advice", + "bet", + "eat", + "kill", + "category", + "manner", + "office", + "estate", + "pride", + "awareness", + "slip", + "crack", + "client", + "nail", + "shoot", + "membership", + "soft", + "anybody", + "web", + "official", + "individual", + "pizza", + "interest", + "bag", + "spell", + "profession", + "queen", + "deal", + "resource", + "ship", + "guy", + "chocolate", + "joint", + "formal", + "upstairs", + "car", + "resort", + "abroad", + "dealer", + "associate", + "finger", + "surgery", + "comment", + "team", + "detail", + "crazy", + "path", + "tale", + "initial", + "arm", + "radio", + "demand", + "single", + "draw", + "yellow", + "contest", + "piece", + "quote", + "pull", + "commercial", + "shirt", + "contribution", + "cream", + "channel", + "suit", + "discipline", + "instruction", + "concert", + "speech", + "low", + "effective", + "hang", + "scratch", + "industry", + "breakfast", + "lay", + "join", + "metal", + "bedroom", + "minute", + "product", + "rest", + "temperature", + "many", + "give", + "argument", + "print", + "purple", + "laugh", + "health", + "credit", + "investment", + "sell", + "setting", + "lesson", + "egg", + "middle", + "marriage", + "level", + "evidence", + "phrase", + "love", + "self", + "benefit", + "guidance", + "affect", + "you", + "dad", + "anxiety", + "special", + "boyfriend", + "test", + "blank", + "payment", + "soup", + "obligation", + "reply", + "smile", + "deep", + "complaint", + "addition", + "review", + "box", + "towel", + "minor", + "fun", + "soil", + "issue", + "cigarette", + "internet", + "gain", + "tell", + "entry", + "spare", + "incident", + "family", + "refuse", + "branch", + "can", + "pen", + "grandfather", + "constant", + "tank", + "uncle", + "climate", + "ground", + "volume", + "communication", + "kind", + "poet", + "child", + "screen", + "mine", + "quit", + "gene", + "lack", + "charity", + "memory", + "tooth", + "fear", + "mention", + "marketing", + "reveal", + "reason", + "court", + "season", + "freedom", + "land", + "sport", + "audience", + "classroom", + "law", + "hook", + "win", + "carry", + "eye", + "smell", + "distribution", + "research", + "country", + "dare", + "hope", + "whereas", + "stretch", + "library", + "if", + "delay", + "college", + "plastic", + "book", + "present", + "use", + "worry", + "champion", + "goal", + "economy", + "march", + "election", + "reflection", + "midnight", + "slide", + "inflation", + "action", + "challenge", + "guitar", + "coast", + "apple", + "campaign", + "field", + "jacket", + "sense", + "way", + "visual", + "remove", + "weather", + "trash", + "cable", + "regret", + "buddy", + "beach", + "historian", + "courage", + "sympathy", + "truck", + "tension", + "permit", + "nose", + "bed", + "son", + "person", + "base", + "meat", + "usual", + "air", + "meeting", + "worth", + "game", + "independence", + "physical", + "brief", + "play", + "raise", + "board", + "she", + "key", + "writing", + "pick", + "command", + "party", + "yesterday", + "spring", + "candidate", + "physics", + "university", + "concern", + "development", + "change", + "string", + "target", + "instance", + "room", + "bitter", + "bird", + "football", + "normal", + "split", + "impression", + "wood", + "long", + "meaning", + "stock", + "cap", + "leadership", + "media", + "ambition", + "fishing", + "essay", + "salad", + "repair", + "today", + "designer", + "night", + "bank", + "drawing", + "inevitable", + "phase", + "vast", + "chip", + "anger", + "switch", + "cry", + "twist", + "personality", + "attempt", + "storage", + "being", + "preparation", + "bat", + "selection", + "white", + "technology", + "contract", + "side", + "section", + "station", + "till", + "structure", + "tongue", + "taste", + "truth", + "difficulty", + "group", + "limit", + "main", + "move", + "feeling", + "light", + "example", + "mission", + "might", + "wait", + "wheel", + "shop", + "host", + "classic", + "alternative", + "cause", + "agent", + "consist", + "table", + "airline", + "text", + "pool", + "craft", + "range", + "fuel", + "tool", + "partner", + "load", + "entrance", + "deposit", + "hate", + "article", + "video", + "summer", + "feature", + "extreme", + "mobile", + "hospital", + "flight", + "fall", + "pension", + "piano", + "fail", + "result", + "rub", + "gap", + "system", + "report", + "suck", + "ordinary", + "wind", + "nerve", + "ask", + "shine", + "note", + "line", + "mom", + "perception", + "brother", + "reference", + "bend", + "charge", + "treat", + "trick", + "term", + "homework", + "bake", + "bid", + "status", + "project", + "strategy", + "orange", + "let", + "enthusiasm", + "parent", + "concentrate", + "device", + "travel", + "poetry", + "business", + "society", + "kiss", + "end", + "vegetable", + "employ", + "schedule", + "hour", + "brave", + "focus", + "process", + "movie", + "illegal", + "general", + "coffee", + "ad", + "highway", + "chemistry", + "psychology", + "hire", + "bell", + "conference", + "relief", + "show", + "neat", + "funny", + "weight", + "quality", + "club", + "daughter", + "zone", + "touch", + "tonight", + "shock", + "burn", + "excuse", + "name", + "survey", + "landscape", + "advance", + "satisfaction", + "bread", + "disaster", + "item", + "hat", + "prior", + "shopping", + "visit", + "east", + "photo", + "home", + "idea", + "father", + "comparison", + "cat", + "pipe", + "winner", + "count", + "lake", + "fight", + "prize", + "foundation", + "dog", + "keep", + "ideal", + "fan", + "struggle", + "peak", + "safety", + "solution", + "hell", + "conclusion", + "population", + "strain", + "alarm", + "measurement", + "second", + "train", + "race", + "due", + "insurance", + "boss", + "tree", + "monitor", + "sick", + "course", + "drag", + "appointment", + "slice", + "still", + "care", + "patience", + "rich", + "escape", + "emotion", + "royal", + "female", + "childhood", + "government", + "picture", + "will", + "sock", + "big", + "gate", + "oil", + "cross", + "pin", + "improvement", + "championship", + "silly", + "help", + "sky", + "pitch", + "man", + "diamond", + "most", + "transition", + "work", + "science", + "committee", + "moment", + "fix", + "teaching", + "dig", + "specialist", + "complex", + "guide", + "people", + "dead", + "voice", + "original", + "break", + "topic", + "data", + "degree", + "reading", + "recording", + "bunch", + "reach", + "judgment", + "lie", + "regular", + "set", + "painting", + "mode", + "list", + "player", + "bear", + "north", + "wonder", + "carpet", + "heavy", + "officer", + "negative", + "clock", + "unique", + "baby", + "pain", + "assumption", + "disk", + "iron", + "bill", + "drawer", + "look", + "double", + "mistake", + "finish", + "future", + "brilliant", + "contact", + "math", + "rice", + "leave", + "restaurant", + "discount", + "sex", + "virus", + "bit", + "trust", + "event", + "wear", + "juice", + "failure", + "bug", + "context", + "mud", + "whole", + "wrap", + "intention", + "draft", + "pressure", + "cake", + "dark", + "explanation", + "space", + "angle", + "word", + "efficiency", + "management", + "habit", + "star", + "chance", + "finding", + "transportation", + "stand", + "criticism", + "flow", + "door", + "injury", + "insect", + "surprise", + "apartment", +] # pylint: disable=line-too-long + +# ISO 639-1 codes to language names. +LANGUAGE_CODES = immutabledict.immutabledict( + { + "en": "English", + "es": "Spanish", + "pt": "Portuguese", + "ar": "Arabic", + "hi": "Hindi", + "fr": "French", + "ru": "Russian", + "de": "German", + "ja": "Japanese", + "it": "Italian", + "bn": "Bengali", + "uk": "Ukrainian", + "th": "Thai", + "ur": "Urdu", + "ta": "Tamil", + "te": "Telugu", + "bg": "Bulgarian", + "ko": "Korean", + "pl": "Polish", + "he": "Hebrew", + "fa": "Persian", + "vi": "Vietnamese", + "ne": "Nepali", + "sw": "Swahili", + "kn": "Kannada", + "mr": "Marathi", + "gu": "Gujarati", + "pa": "Punjabi", + "ml": "Malayalam", + "fi": "Finnish", + } +) + +_ALPHABETS = "([A-Za-z])" +_PREFIXES = "(Mr|St|Mrs|Ms|Dr)[.]" +_SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)" +_STARTERS = r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" +_ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" +_WEBSITES = "[.](com|net|org|io|gov|edu|me)" +_DIGITS = "([0-9])" +_MULTIPLE_DOTS = r"\.{2,}" + + +def split_into_sentences(text): + """Split the text into sentences. + + Args: + text: A string that consists of more than or equal to one sentences. + + Returns: + A list of strings where each string is a sentence. + """ + text = " " + text + " " + text = text.replace("\n", " ") + text = re.sub(_PREFIXES, "\\1", text) + text = re.sub(_WEBSITES, "\\1", text) + text = re.sub(_DIGITS + "[.]" + _DIGITS, "\\1\\2", text) + text = re.sub( + _MULTIPLE_DOTS, + lambda match: "" * len(match.group(0)) + "", + text, + ) + if "Ph.D" in text: + text = text.replace("Ph.D.", "PhD") + text = re.sub(r"\s" + _ALPHABETS + "[.] ", " \\1 ", text) + text = re.sub(_ACRONYMS + " " + _STARTERS, "\\1 \\2", text) + text = re.sub( + _ALPHABETS + "[.]" + _ALPHABETS + "[.]" + _ALPHABETS + "[.]", + "\\1\\2\\3", + text, + ) + text = re.sub(_ALPHABETS + "[.]" + _ALPHABETS + "[.]", "\\1\\2", text) + text = re.sub(" " + _SUFFIXES + "[.] " + _STARTERS, " \\1 \\2", text) + text = re.sub(" " + _SUFFIXES + "[.]", " \\1", text) + text = re.sub(" " + _ALPHABETS + "[.]", " \\1", text) + if "”" in text: + text = text.replace(".”", "”.") + if '"' in text: + text = text.replace('."', '".') + if "!" in text: + text = text.replace('!"', '"!') + if "?" in text: + text = text.replace('?"', '"?') + text = text.replace(".", ".") + text = text.replace("?", "?") + text = text.replace("!", "!") + text = text.replace("", ".") + sentences = text.split("") + sentences = [s.strip() for s in sentences] + if sentences and not sentences[-1]: + sentences = sentences[:-1] + return sentences + + +def count_words(text): + """Counts the number of words.""" + tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_words = len(tokens) + return num_words + + +@functools.cache +def _get_sentence_tokenizer(): + return nltk.data.load("nltk:tokenizers/punkt/english.pickle") + + +def count_sentences(text): + """Count the number of sentences.""" + tokenizer = _get_sentence_tokenizer() + tokenized_sentences = tokenizer.tokenize(text) + return len(tokenized_sentences) + + +def count_stopwords(text): + """Counts the number of stopwords.""" + nltk.download("stopwords") + stopwords = nltk.corpus.stopwords.words("english") + tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_stopwords = len([t for t in tokens if t.lower() in stopwords]) + return num_stopwords + + +def generate_keywords(num_keywords): + """Randomly generates a few keywords.""" + return random.sample(WORD_LIST, k=num_keywords) + + +# below, helper class and function for CamelCaseVariablesChecker +class VariableVisitor(ast.NodeVisitor): + """AST Visitor that collects variable names from different contexts.""" + + def __init__(self): + self.variables: Set[str] = set() + self.assignments: Set[str] = set() + self.function_params: Set[str] = set() + + def visit_Name(self, node: ast.Name): + """Visit a name node in the AST.""" + if isinstance(node.ctx, ast.Store): + self.assignments.add(node.id) + self.variables.add(node.id) + + def visit_arg(self, node: ast.arg): + """Visit a function argument node.""" + self.function_params.add(node.arg) + + +def extract_variables(code: str) -> dict: + """ + Extract variable names from a Python code string. + + Args: + code (str): Python code as a string + + Returns: + dict: Dictionary containing different types of variables found + """ + try: + # Parse the code into an AST + tree = ast.parse(code) + + # Create and run our visitor + visitor = VariableVisitor() + visitor.visit(tree) + + return { + "all_variables": sorted(visitor.variables), + "assignments": sorted(visitor.assignments), + "function_params": sorted(visitor.function_params), + } + + except SyntaxError as e: + return {"error": f"Invalid Python code: {str(e)}"} + + +def is_camelcase(variable_name: str) -> bool: + """ + Verify if a variable name follows camelCase convention. + + Rules for camelCase: + 1. Starts with lowercase letter + 2. No underscores or hyphens + 3. Contains only letters and numbers + 4. Each new word starts with uppercase letter + + Args: + variable_name: String to check for camelCase compliance + + Returns: + bool: True if variable name is in camelCase, False otherwise + """ + # Assume empty strings are errors, do not penalize + if not variable_name: + return True + + # Must start with lowercase letter + if not variable_name[0].strip().islower(): + return False + + # Check for invalid characters (only letters and numbers allowed) + if not variable_name.isalnum(): + return False + + # No consecutive uppercase letters (would indicate CONSTANT or PascalCase) + for i in range(len(variable_name) - 1): + if variable_name[i].isupper() and variable_name[i + 1].isupper(): + return False + + return True diff --git a/src/olmo_eval/evals/suites/ifbench.py b/src/olmo_eval/evals/suites/ifbench.py new file mode 100644 index 000000000..df42f037c --- /dev/null +++ b/src/olmo_eval/evals/suites/ifbench.py @@ -0,0 +1,13 @@ +"""IFBench (Tulu) instruction-following suite.""" + +from olmo_eval.evals.suites.registry import make_suite + +IFBENCH = make_suite( + "ifbench", + ( + "ifeval_mt_wildchat_unused_withRewrite", + "ifeval_mt_ood_wildchat_unused_withRewrite", + "ifeval_ood", + ), + description="IFBench (Tulu): OOD + multi-turn instruction following", +) diff --git a/src/olmo_eval/evals/tasks/ifeval_mt.py b/src/olmo_eval/evals/tasks/ifeval_mt.py new file mode 100644 index 000000000..4a21e5107 --- /dev/null +++ b/src/olmo_eval/evals/tasks/ifeval_mt.py @@ -0,0 +1,104 @@ +"""IFEval-MT: multi-turn instruction following from ``VGraf/ifeval_mt``. + +Each row of the dataset is a multi-turn conversation that ends in a user turn +asking the assistant to rewrite/repeat its prior reply under a list of +instructions. ``instruction_id_list`` and ``kwargs`` are scored against the +final assistant reply using the same IFEval verifiers as ``ifeval_ood``. + +Two registered variants follow ``IFBENCH_MT_TASKS`` in oe-eval-internal: + +- ``ifeval_mt_wildchat_unused_withRewrite`` (HF subset + ``wildchat_unused_withRewrite``) +- ``ifeval_mt_ood_wildchat_unused_withRewrite`` (HF subset + ``ood_wildchat_unused_withRewrite``) +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from olmo_eval.common.metrics import ( + IFEvalInstLooseAccuracy, + IFEvalInstStrictAccuracy, + IFEvalPromptLooseAccuracy, + IFEvalPromptStrictAccuracy, +) +from olmo_eval.common.types import ( + Instance, + LMOutput, + LMRequest, + RequestType, + SamplingParams, + Split, +) +from olmo_eval.data import DataSource +from olmo_eval.evals.tasks.common import Task, register + +_PRIMARY_METRIC = IFEvalPromptLooseAccuracy() +_DATASET_PATH = "VGraf/ifeval_mt" +_SAMPLING_PARAMS = SamplingParams( + max_tokens=2048, + temperature=0.0, + do_sample=False, +) + + +class _IFEvalMTBase(Task): + split = Split.TEST + metrics = ( + IFEvalPromptStrictAccuracy(), + IFEvalPromptLooseAccuracy(), + IFEvalInstStrictAccuracy(), + IFEvalInstLooseAccuracy(), + ) + primary_metric = _PRIMARY_METRIC + sampling_params = _SAMPLING_PARAMS + + @property + def instances(self) -> Iterator[Instance]: + yield from self._load_instances_cached() + + @property + def request_type(self) -> RequestType: + return RequestType.CHAT + + def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None: + prompt = doc["prompt"] + instruction_id_list = list(doc.get("instruction_id_list") or []) + raw_kwargs = doc.get("kwargs") or [] + kwargs_list = [{k: v for k, v in (kw or {}).items() if v is not None} for kw in raw_kwargs] + messages = tuple({"role": m["role"], "content": m["content"]} for m in doc["messages"]) + return Instance( + question=prompt, + gold_answer=None, + metadata={ + "id": doc.get("id", doc.get("key", index)), + "key": doc.get("key", doc.get("id", index)), + "prompt": prompt, + "instruction_id_list": instruction_id_list, + "kwargs": kwargs_list, + "messages": messages, + }, + ) + + def format_request(self, instance: Instance) -> LMRequest: + return LMRequest( + request_type=RequestType.CHAT, + messages=tuple(instance.metadata["messages"]), + ) + + def extract_answer(self, output: LMOutput) -> str: + return output.text + + +@register("ifeval_mt_wildchat_unused_withRewrite") +class IFEvalMTWildchatUnusedWithRewrite(_IFEvalMTBase): + data_source = DataSource(path=_DATASET_PATH, subset="wildchat_unused_withRewrite", split="test") + + +@register("ifeval_mt_ood_wildchat_unused_withRewrite") +class IFEvalMTOODWildchatUnusedWithRewrite(_IFEvalMTBase): + data_source = DataSource( + path=_DATASET_PATH, subset="ood_wildchat_unused_withRewrite", split="test" + ) diff --git a/src/olmo_eval/evals/tasks/ifbench.py b/src/olmo_eval/evals/tasks/ifeval_ood.py similarity index 67% rename from src/olmo_eval/evals/tasks/ifbench.py rename to src/olmo_eval/evals/tasks/ifeval_ood.py index d76cbc349..d3c65991c 100644 --- a/src/olmo_eval/evals/tasks/ifbench.py +++ b/src/olmo_eval/evals/tasks/ifeval_ood.py @@ -1,11 +1,11 @@ -"""IFBench: out-of-distribution instruction-following benchmark. +"""IFEval OOD: out-of-distribution instruction-following slice of IFBench. -Dataset: ``allenai/IFBench_test`` (300 prompts). Each row carries a list of -instruction IDs and per-instruction kwargs; verifiers come from the upstream -``allenai/IFBench`` package, accessed lazily via :class:`IFEvalScorer`. +Dataset: ``allenai/IFBench_test2`` (300 prompts), each carrying a list of +instruction IDs and per-instruction kwargs. Verifiers come from the vendored +registry in :mod:`olmo_eval.common.scorers.ifeval_deps`. -Reports prompt-level and instruction-level accuracy in both strict and loose -modes, matching upstream IFEval reporting. +Mirrors the ``ifeval_ood::tulu`` configuration in oe-eval-internal: chat +format, ``max_gen_toks=2048``, primary metric ``prompt_level_loose_acc``. """ from __future__ import annotations @@ -30,12 +30,12 @@ from olmo_eval.data import DataSource from olmo_eval.evals.tasks.common import Task, register -_PRIMARY_METRIC = IFEvalInstLooseAccuracy() +_PRIMARY_METRIC = IFEvalPromptLooseAccuracy() -@register("ifbench") -class IFBench(Task): - data_source = DataSource(path="allenai/IFBench_test", split="train") +@register("ifeval_ood") +class IFEvalOOD(Task): + data_source = DataSource(path="allenai/IFBench_test2", split="train") split = Split.TRAIN metrics = ( IFEvalPromptStrictAccuracy(), @@ -45,7 +45,7 @@ class IFBench(Task): ) primary_metric = _PRIMARY_METRIC sampling_params = SamplingParams( - max_tokens=1280, + max_tokens=2048, temperature=0.0, do_sample=False, ) @@ -54,6 +54,10 @@ class IFBench(Task): def instances(self) -> Iterator[Instance]: yield from self._load_instances_cached() + @property + def request_type(self) -> RequestType: + return RequestType.CHAT + def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None: prompt = doc["prompt"] instruction_id_list = list(doc.get("instruction_id_list") or []) @@ -72,7 +76,10 @@ def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None: ) def format_request(self, instance: Instance) -> LMRequest: - return LMRequest(request_type=RequestType.COMPLETION, prompt=instance.question) + return LMRequest( + request_type=RequestType.CHAT, + messages=({"role": "user", "content": instance.question},), + ) def extract_answer(self, output: LMOutput) -> str: return output.text diff --git a/tests/evals/tasks/test_ifbench.py b/tests/evals/tasks/test_ifbench.py index 12e25b66d..ae61409b4 100644 --- a/tests/evals/tasks/test_ifbench.py +++ b/tests/evals/tasks/test_ifbench.py @@ -1,12 +1,10 @@ -"""Tests for the IFBench task and IFEval scoring stack.""" +"""Tests for IFBench tasks (ifeval_ood + ifeval_mt) and the IFEval scoring stack.""" from __future__ import annotations import unittest from typing import Any -import pytest - from olmo_eval.common.metrics import ( IFEvalInstLooseAccuracy, IFEvalInstStrictAccuracy, @@ -15,11 +13,9 @@ ) from olmo_eval.common.scorers import IFEvalScorer from olmo_eval.common.types import Instance, LMOutput, LMRequest, RequestType, Response +from olmo_eval.evals.suites.registry import get_suite from olmo_eval.evals.tasks.common import get_task -# Skip the whole module if upstream IFBench isn't installed. -pytest.importorskip("instructions_registry") - def _make_instance( prompt: str, @@ -47,9 +43,9 @@ def _make_response(instance: Instance, response_text: str) -> Response: ) -class TestIFBenchTask(unittest.TestCase): +class TestIFEvalOODTask(unittest.TestCase): def test_registered(self) -> None: - task = get_task("ifbench") + task = get_task("ifeval_ood") self.assertIsNotNone(task) metric_names = {m.name for m in task.config.metrics} self.assertEqual( @@ -61,9 +57,10 @@ def test_registered(self) -> None: "inst_level_loose_acc", }, ) + self.assertEqual(task.request_type, RequestType.CHAT) def test_process_doc_strips_none_kwargs(self) -> None: - task = get_task("ifbench") + task = get_task("ifeval_ood") doc = { "key": 0, "prompt": "hi", @@ -78,6 +75,53 @@ def test_process_doc_strips_none_kwargs(self) -> None: self.assertEqual(instance.metadata["key"], 0) +class TestIFEvalMTTask(unittest.TestCase): + def test_registered_variants(self) -> None: + for name in ( + "ifeval_mt_wildchat_unused_withRewrite", + "ifeval_mt_ood_wildchat_unused_withRewrite", + ): + task = get_task(name) + self.assertIsNotNone(task) + self.assertEqual(task.request_type, RequestType.CHAT) + + def test_process_doc_preserves_messages(self) -> None: + task = get_task("ifeval_mt_wildchat_unused_withRewrite") + messages = [ + {"role": "user", "content": "say hi"}, + {"role": "assistant", "content": "hi"}, + {"role": "user", "content": "Rewrite with exactly 2 numbers."}, + ] + doc = { + "id": "x", + "key": 0, + "prompt": "Rewrite with exactly 2 numbers.", + "instruction_id_list": ["count:numbers"], + "kwargs": [{"N": 2}], + "messages": messages, + } + instance = task.process_doc(doc, index=0) + assert instance is not None + self.assertEqual(len(instance.metadata["messages"]), 3) + self.assertEqual(instance.metadata["messages"][-1]["content"], messages[-1]["content"]) + request = task.format_request(instance) + self.assertEqual(request.request_type, RequestType.CHAT) + self.assertEqual(len(request.messages), 3) + + +class TestIFBenchSuite(unittest.TestCase): + def test_suite_registered(self) -> None: + suite = get_suite("ifbench") + self.assertEqual( + tuple(suite.tasks), + ( + "ifeval_mt_wildchat_unused_withRewrite", + "ifeval_mt_ood_wildchat_unused_withRewrite", + "ifeval_ood", + ), + ) + + class TestIFEvalScorer(unittest.TestCase): """Use ``count:numbers`` (must include exactly N digits) — a deterministic verifier.""" From 2f79d28d1a3cc721aaf5edbcc46cd8d5bb16bdce Mon Sep 17 00:00:00 2001 From: Finbarr Timbers Date: Wed, 13 May 2026 08:40:49 -0600 Subject: [PATCH 4/8] Replace vendored IFBench with ifbench package dependency Co-Authored-By: Claude Opus 4.7 --- pyproject.toml | 8 +- src/olmo_eval/common/scorers/ifeval.py | 16 +- .../common/scorers/ifeval_deps/__init__.py | 7 - .../scorers/ifeval_deps/instructions.py | 3923 ----------------- .../ifeval_deps/instructions_registry.py | 251 -- .../scorers/ifeval_deps/instructions_util.py | 1777 -------- uv.lock | 6 +- 7 files changed, 12 insertions(+), 5976 deletions(-) delete mode 100644 src/olmo_eval/common/scorers/ifeval_deps/__init__.py delete mode 100644 src/olmo_eval/common/scorers/ifeval_deps/instructions.py delete mode 100644 src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py delete mode 100644 src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py diff --git a/pyproject.toml b/pyproject.toml index db5c79177..e6a047b6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "textual-plot>=0.10.1", "tree-sitter>=0.24.0", "tree-sitter-python>=0.23.0,<0.26.0", - "ifbench @ git+https://github.com/allenai/IFBench.git", + "ifbench @ git+https://github.com/allenai/IFBench.git@finbarr/clean-install", # syllapy (transitive via ifbench) imports pkg_resources, which setuptools>=81 removes. "setuptools<81", ] @@ -145,12 +145,6 @@ vllm = [ [tool.ruff] line-length = 100 target-version = "py312" -extend-exclude = [ - # Vendored from oe-eval-internal (oe_eval/dependencies/ifeval/), kept as-is. - "src/olmo_eval/common/scorers/ifeval_deps/instructions.py", - "src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py", - "src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py", -] [tool.ruff.lint] select = [ diff --git a/src/olmo_eval/common/scorers/ifeval.py b/src/olmo_eval/common/scorers/ifeval.py index 498b18c9e..aef57c131 100644 --- a/src/olmo_eval/common/scorers/ifeval.py +++ b/src/olmo_eval/common/scorers/ifeval.py @@ -1,12 +1,11 @@ """Scorer for IFBench / IFEval instruction-following evaluation. -Uses the vendored ``olmo_eval.common.scorers.ifeval_deps`` registry, which -covers the original IFEval (DEFAULT) verifiers, the OOD verifiers used by -``allenai/IFBench_test2``, and the verifiers used by the multi-turn -``VGraf/ifeval_mt`` slices. The scorer evaluates a response against -per-instance instructions (looked up in -``instance.metadata["instruction_id_list"]`` / ``"kwargs"``) and writes both -strict and loose pass/fail lists for each instruction into +Uses the ``ifbench`` package registry, which covers the original IFEval +(DEFAULT) verifiers, the OOD verifiers used by ``allenai/IFBench_test2``, +and the verifiers used by the multi-turn ``VGraf/ifeval_mt`` slices. The +scorer evaluates a response against per-instance instructions (looked up +in ``instance.metadata["instruction_id_list"]`` / ``"kwargs"``) and writes +both strict and loose pass/fail lists for each instruction into ``output.metadata["ifeval"]``. The four IFEval metrics consume that field. """ @@ -15,8 +14,9 @@ from dataclasses import dataclass from typing import Any, ClassVar +from ifbench import instructions_registry + from olmo_eval.common.scorers.base import Scorer -from olmo_eval.common.scorers.ifeval_deps import instructions_registry from olmo_eval.common.types import Instance, LMOutput _INSTRUCTION_DICT: dict[str, Any] = instructions_registry.INSTRUCTION_DICT diff --git a/src/olmo_eval/common/scorers/ifeval_deps/__init__.py b/src/olmo_eval/common/scorers/ifeval_deps/__init__.py deleted file mode 100644 index 39dcea997..000000000 --- a/src/olmo_eval/common/scorers/ifeval_deps/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Vendored IFEval/IFBench instruction registry. - -Sourced from allenai/oe-eval-internal (oe_eval/dependencies/ifeval/) so that -both the original IFEval verifiers (DEFAULT) and the OOD verifiers used by -``allenai/IFBench_test2`` plus the multi-turn ``VGraf/ifeval_mt`` slices are -available from a single registry. -""" diff --git a/src/olmo_eval/common/scorers/ifeval_deps/instructions.py b/src/olmo_eval/common/scorers/ifeval_deps/instructions.py deleted file mode 100644 index 7268d8387..000000000 --- a/src/olmo_eval/common/scorers/ifeval_deps/instructions.py +++ /dev/null @@ -1,3923 +0,0 @@ -# Copyright 2023 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Library of instructions.""" - -import collections -import csv -import io -import json -import logging -import random -import re -import string -import unicodedata -from collections import Counter -from collections.abc import Sequence -from typing import Optional - -import emoji -import langdetect -import nltk -import syllapy - -from olmo_eval.common.scorers.ifeval_deps import instructions_util - -logger = logging.getLogger(__name__) - -_InstructionArgsDtype = Optional[dict[str, int | str | Sequence[str]]] - -_LANGUAGES = instructions_util.LANGUAGE_CODES - -# The relational operation for comparison. -_COMPARISON_RELATION = ("less than", "at least") - -# The maximum number of sentences. -_MAX_NUM_SENTENCES = 20 - -# The number of placeholders. -_NUM_PLACEHOLDERS = 4 - -# The number of bullet lists. -_NUM_BULLETS = 5 - -# The options of constrained response. -_CONSTRAINED_RESPONSE_OPTIONS = ( - "My answer is yes.", - "My answer is no.", - "My answer is maybe.", -) - -# The options of starter keywords. -_STARTER_OPTIONS = ( - "I would say", - "My answer is", - "I believe", - "In my opinion", - "I think", - "I reckon", - "I feel", - "From my perspective", - "As I see it", - "According to me", - "As far as I'm concerned", - "To my understanding", - "In my view", - "My take on it is", - "As per my perception", -) - -# The options of ending keywords. -# TODO(jeffreyzhou) add more ending options -_ENDING_OPTIONS = ("Any other questions?", "Is there anything else I can help with?") - -# The number of highlighted sections. -_NUM_HIGHLIGHTED_SECTIONS = 4 - -# The section spliter. -_SECTION_SPLITER = ("Section", "SECTION") - -# The number of sections. -_NUM_SECTIONS = 5 - -# The number of paragraphs. -_NUM_PARAGRAPHS = 5 - -# The postscript marker. -_POSTSCRIPT_MARKER = ("P.S.", "P.P.S") - -# The number of keywords. -_NUM_KEYWORDS = 2 - -# The occurrences of a single keyword. -_KEYWORD_FREQUENCY = 3 - -# The occurrences of a single letter. -_LETTER_FREQUENCY = 10 - -# The occurrences of words with all capital letters. -_ALL_CAPITAL_WORD_FREQUENCY = 20 - -# The number of words in the response. -_NUM_WORDS_LOWER_LIMIT = 100 -_NUM_WORDS_UPPER_LIMIT = 500 - -# The number of numbers. -_NUM_NUMBERS = 6 - -# Period length for periodic words. -_NUM_WORD_CYCLE = 30 - -# Maximum number of times a word can be repeated. -_MAX_REPEATS = 5 - -# Which sentence must contain a keyword. -_NUM_KEYWORD_SENTENCE = 20 - -# Minimum number of pronouns. -_NUM_PRONOUNS = 25 - -# The size of increment for lengths. -_NUM_INCREMENT = 5 - -# The number of coordinating conjunctions. -_NUM_CONJUNCTIONS = 6 - -# The Levenshtein distance of a response. -_LEV_DISTANCE_LOWER_LIMIT = 5 -_LEV_DISTANCE_UPPER_LIMIT = 40 - - -class Instruction: - """An instruction template.""" - - def __init__(self, instruction_id): - self.id = instruction_id - - def build_description(self, **kwargs): - raise NotImplementedError("`build_description` not implemented.") - - def get_instruction_args(self): - raise NotImplementedError("`get_instruction_args` not implemented.") - - def get_instruction_args_keys(self): - raise NotImplementedError("`get_instruction_args_keys` not implemented.") - - def check_following(self, value): - raise NotImplementedError("`check_following` not implemented.") - - -class ResponseLanguageChecker(Instruction): - """Check the language of the entire response.""" - - def build_description(self, *, language=None): - """Build the instruction description. - - Args: - language: A string representing the expected language of the response. The - language has to comply to the 97 types defined in - `langid.py` (https://pypi.org/project/langid/1.1.5/), which follows - ISO 639-1 codes (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes); - for example, `en` for English, `zh` for Chinese, `fr` for French. - - Returns: - A string representing the instruction description. - """ - self._language = language - if self._language is None: - self._language = random.choice(list(_LANGUAGES.keys())) - # TODO(tianjianlu): opens the description generation to more choices. - self._description_pattern = ( - "Your ENTIRE response should be in {language} language, no other " - + "language is allowed." - ) - return self._description_pattern.format(language=_LANGUAGES[self._language]) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"language": self._language} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["language"] - - def check_following(self, value): - """Check if the language of the entire response follows the instruction. - - Args: - value: A string representing the response. - - Returns: - True if the language of `value` follows instruction; otherwise False. - """ - assert isinstance(value, str) - - try: - return langdetect.detect(value) == self._language - except langdetect.LangDetectException as e: - # Count as instruction is followed. - logging.error( - "Unable to detect language for text %s due to %s", value, e - ) # refex: disable=pytotw.037 - return True - - -class NumberOfSentences(Instruction): - """Check the number of sentences.""" - - def build_description(self, *, num_sentences=None, relation=None): - """Build the instruction description. - - Args: - num_sentences: An integer specifying the number of sentences as a - threshold. - relation: A string in (`less than`, `at least`), defining the relational - operator for comparison. - Two relational comparisons are supported for now: - if 'less than', the actual number of sentences < the threshold; - if 'at least', the actual number of sentences >= the threshold. - - Returns: - A string representing the instruction description. - """ - # The number of sentences as a threshold for comparison. - self._num_sentences_threshold = num_sentences - if self._num_sentences_threshold is None or self._num_sentences_threshold < 0: - self._num_sentences_threshold = random.randint(1, _MAX_NUM_SENTENCES) - - if relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif relation not in _COMPARISON_RELATION: - raise ValueError( - "The supported relation for comparison must be in " - f"{_COMPARISON_RELATION}, but {relation} is given." - ) - else: - self._comparison_relation = relation - - self._description_pattern = ( - "Your response should contain {relation} {num_sentences} sentences." - ) - return self._description_pattern.format( - relation=self._comparison_relation, - num_sentences=self._num_sentences_threshold, - ) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return { - "num_sentences": self._num_sentences_threshold, - "relation": self._comparison_relation, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_sentences", "relation"] - - def check_following(self, value): - """Check if the number of sentences follows the instruction. - - Args: - value: A string representing the response. - - Returns: - True if the response follows the instruction. - - Raise: - ValueError if the string in `instruction_args` is not in - [`less_than`, `at_least`]. - """ - num_sentences = instructions_util.count_sentences(value) - if self._comparison_relation == _COMPARISON_RELATION[0]: - return num_sentences < self._num_sentences_threshold - elif self._comparison_relation == _COMPARISON_RELATION[1]: - return num_sentences >= self._num_sentences_threshold - - -class PlaceholderChecker(Instruction): - """Check the placeholders in template writing.""" - - def build_description(self, *, num_placeholders=None): - """Build the instruction description. - - Args: - num_placeholders: An integer denoting the minimum number of - placeholders required in the response. - - Returns: - A string representing the instruction description. - """ - self._num_placeholders = num_placeholders - if self._num_placeholders is None or self._num_placeholders < 0: - self._num_placeholders = random.randint(1, _NUM_PLACEHOLDERS) - self._description_pattern = ( - "The response must contain at least {num_placeholders} placeholders " - + "represented by square brackets, such as [address]." - ) - return self._description_pattern.format(num_placeholders=self._num_placeholders) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"num_placeholders": self._num_placeholders} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_placeholders"] - - def check_following(self, value): - """Check if the number of placeholders follows the instruction. - - Args: - value: A string representing the response. - - Returns: - True if the actual number of placeholders in the response is greater than - or equal to `num_placeholders`; otherwise, False. - """ - placeholders = re.findall(r"\[.*?\]", value) - num_placeholders = len(placeholders) - return num_placeholders >= self._num_placeholders - - -class BulletListChecker(Instruction): - """Checks the bullet list in the prompt.""" - - def build_description(self, *, num_bullets=None): - """Build the instruction description. - - Args: - num_bullets: An integer specifying the exact number of bullet lists - that is required to appear in the response. - - Returns: - A string representing the instruction description. - """ - self._num_bullets = num_bullets - if self._num_bullets is None or self._num_bullets < 0: - self._num_bullets = random.randint(1, _NUM_BULLETS) - self._description_pattern = ( - "Your answer must contain exactly {num_bullets} bullet points. " - + "Use the markdown bullet points such as:\n" - + "* This is point 1. \n" - + "* This is point 2" - ) - return self._description_pattern.format(num_bullets=self._num_bullets) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"num_bullets": self._num_bullets} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_bullets"] - - def check_following(self, value): - r"""Check if the number of bullet lists meets the requirement. - - Args: - value: A string representing the response. The response is expected to - contain some bullet lists that start with `\*`. - - Returns: - True if the actual number of bullet lists in the response meets the - requirement. - """ - bullet_lists = re.findall(r"^\s*\*[^\*].*$", value, flags=re.MULTILINE) - bullet_lists_2 = re.findall(r"^\s*-.*$", value, flags=re.MULTILINE) - num_bullet_lists = len(bullet_lists) + len(bullet_lists_2) - return num_bullet_lists == self._num_bullets - - -class ConstrainedResponseChecker(Instruction): - """Checks the constrained response.""" - - def build_description(self): - """Build the instruction description.""" - # A sequence of string(s) representing the options of the expected response. - self._constrained_responses = _CONSTRAINED_RESPONSE_OPTIONS - self._description_pattern = "Answer with one of the following options: {response_options}" - return self._description_pattern.format(response_options=self._constrained_responses) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response matches the constrained options. - - Args: - value: A string representing the response. - - Returns: - True if the actual response contains one of the options in the constrained - responses; otherwise False. - """ - value = value.strip() - for constrained_response in self._constrained_responses: - if constrained_response in value: - return True - return False - - -class ConstrainedStartChecker(Instruction): - """Checks the response start.""" - - def build_description(self, *, starter=None): - """Build the instruction description. - - Args: - starter: A string representing the keyword that the response should start - with. - - Returns: - A string representing the instruction description. - """ - self._starter = starter.strip() if isinstance(starter, str) else starter - if self._starter is None: - self._starter = random.choice(_STARTER_OPTIONS) - self._description_pattern = ( - "During the conversation, when it is your turn, " + "please always start with {starter}" - ) - return self._description_pattern.format(starter=self._starter) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"starter": self._starter} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["starter"] - - def check_following(self, value): - """Checks if the response starts with the constrained keyword or phrase. - - Args: - value: A string representing the response. - - Returns: - True if the response starts with the given phrase or keyword that is - contained in `instruction_args`; otherwise, False. - """ - response_pattern = r"^\s*" + self._starter + r".*$" - response_with_constrained_start = re.search(response_pattern, value, flags=re.MULTILINE) - return True if response_with_constrained_start else False - - -class HighlightSectionChecker(Instruction): - """Checks the highlighted section.""" - - def build_description(self, *, num_highlights=None): - """Build the instruction description. - - Args: - num_highlights: An integer specifying the minimum number of highlighted - sections. - - Returns: - A string representing the instruction description. - """ - self._num_highlights = num_highlights - if self._num_highlights is None or self._num_highlights < 0: - self._num_highlights = random.randint(1, _NUM_HIGHLIGHTED_SECTIONS) - - self._description_pattern = ( - "Highlight at least {num_highlights} sections in your answer with " - + "markdown, i.e. *highlighted section*." - ) - - return self._description_pattern.format(num_highlights=self._num_highlights) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"num_highlights": self._num_highlights} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_highlights"] - - def check_following(self, value): - """Checks if the number of highlighted sections meets the requirement. - - Args: - value: a string repesenting the response. The response is expected to - contain highlighted sections in the format of *highlighted*. - - Returns: - True if the actual number of highlighted sections in the format of - *highlighed sections* meets the minimum requirement; otherwise False. - """ - num_highlights = 0 - highlights = re.findall(r"\*[^\n\*]*\*", value) - double_highlights = re.findall(r"\*\*[^\n\*]*\*\*", value) - for highlight in highlights: - if highlight.strip("*").strip(): - num_highlights += 1 - for highlight in double_highlights: - if highlight.removeprefix("**").removesuffix("**").strip(): - num_highlights += 1 - - return num_highlights >= self._num_highlights - - -class SectionChecker(Instruction): - """Checks the sections.""" - - def build_description(self, *, section_spliter=None, num_sections=None): - """Build the instruction description. - - Args: - section_spliter: A string represents the section spliter keyword that - marks a new section, i.e., `Section` or `SECTION`. - num_sections: An integer specifying the number of sections. - - Returns: - A string representing the instruction description. - """ - self._section_spliter = ( - section_spliter.strip() if isinstance(section_spliter, str) else section_spliter - ) - if self._section_spliter is None: - self._section_spliter = random.choice(_SECTION_SPLITER) - - self._num_sections = num_sections - if self._num_sections is None or self._num_sections < 0: - self._num_sections = random.randint(1, _NUM_SECTIONS) - - self._description_pattern = ( - "Your response must have {num_sections} sections. Mark the beginning " - + "of each section with {section_spliter} X, such as:\n" - + "{section_spliter} 1\n" - + "[content of section 1]\n" - + "{section_spliter} 2\n" - + "[content of section 2]" - ) - - return self._description_pattern.format( - num_sections=self._num_sections, section_spliter=self._section_spliter - ) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return { - "section_spliter": self._section_spliter, - "num_sections": self._num_sections, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["section_spliter", "num_sections"] - - def check_following(self, value): - """Checks the response contains multiple sections. - - Args: - value: A string representing the response. The response is expected - to contain multiple sections (number of sections is greater than 1). - A new section starts with `Section 1`, where the number denotes the - section index. - - Returns: - True if the number of sections in the response is greater than or equal to - the minimum number of sections; otherwise, False. - """ - section_splitter_patten = r"\s?" + self._section_spliter + r"\s?\d+\s?" - sections = re.split(section_splitter_patten, value) - num_sections = len(sections) - 1 - return num_sections >= self._num_sections - - -class ParagraphChecker(Instruction): - """Checks the paragraphs.""" - - def build_description(self, *, num_paragraphs=None): - """Build the instruction description. - - Args: - num_paragraphs: An integer specifying the number of paragraphs. - - Returns: - A string representing the instruction description. - """ - self._num_paragraphs = num_paragraphs - if self._num_paragraphs is None or self._num_paragraphs < 0: - self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) - - self._description_pattern = ( - "There should be {num_paragraphs} paragraphs. " - + "Paragraphs are separated with the markdown divider: ***" - ) - - return self._description_pattern.format(num_paragraphs=self._num_paragraphs) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"num_paragraphs": self._num_paragraphs} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_paragraphs"] - - def check_following(self, value): - """Checks the response contains required number of paragraphs. - - Args: - value: A string representing the response. The response may contain - paragraphs that are separated by the markdown divider: `***`. - - Returns: - True if the actual number of paragraphs is the same as required; - otherwise, False. - """ - paragraphs = re.split(r"\s?\*\*\*\s?", value) - num_paragraphs = len(paragraphs) - - for index, paragraph in enumerate(paragraphs): - if not paragraph.strip(): - if index == 0 or index == len(paragraphs) - 1: - num_paragraphs -= 1 - else: - return False - - return num_paragraphs == self._num_paragraphs - - -class PostscriptChecker(Instruction): - """Checks the postscript.""" - - def build_description(self, *, postscript_marker=None): - """Build the instruction description. - - Args: - postscript_marker: A string containing the keyword that marks the start - of the postscript section. - - Returns: - A string representing the instruction description. - """ - self._postscript_marker = ( - postscript_marker.strip() if isinstance(postscript_marker, str) else postscript_marker - ) - if self._postscript_marker is None: - self._postscript_marker = random.choice(_POSTSCRIPT_MARKER) - - self._description_pattern = ( - "At the end of your response, please explicitly add a postscript " - + "starting with {postscript}" - ) - - return self._description_pattern.format(postscript=self._postscript_marker) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"postscript_marker": self._postscript_marker} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["postscript_marker"] - - def check_following(self, value): - """Checks if the response follows the postscript format. - - Args: - value: a string representing the response. The response is expected to - contain a postscript section. - - Returns: - True if the response contains a postscript section starting with - the keyword containing in the `instruction_args`; otherwise False. - """ - value = value.lower() - if self._postscript_marker == "P.P.S": - postscript_pattern = r"\s*p\.\s?p\.\s?s.*$" - elif self._postscript_marker == "P.S.": - postscript_pattern = r"\s*p\.\s?s\..*$" - else: - postscript_pattern = r"\s*" + self._postscript_marker.lower() + r".*$" - postscript = re.findall(postscript_pattern, value, flags=re.MULTILINE) - return True if postscript else False - - -class RephraseChecker(Instruction): - """Checks the repharse.""" - - def build_description(self, *, original_message): - """Build the instruction description. - - Args: - original_message: A string representing the original message. The - rephrased response should only change its words/sentences in between - its two asterisks, for example, *change me*. Both original and rephrased - messages should contain the changes in the form of *change me*. - - Returns: - A string representing the instruction description. - """ - if not self.is_change(original_message): - raise ValueError( - f"Message {original_message} does not contain changes in the form of *change me*." - ) - - self._reference_without_change = original_message - self._description = ( - "Rephrasing: Your rephrased response should only" - + "change the words/sentences in between two asterisks" - + "such as *change me*." - ) - return self._description - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"original_message": self._reference_without_change} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["original_message"] - - def check_following(self, value): - r"""Checks if the rephrasing follows the instruction. - - Args: - value: A string representing the response, which is expected to rephras - the string of `instruction_args`. - - Returns: - True if `value` and `instruction_args` only differ by the words/sentences - in between two asterisks such as *change me*; otherwise, False. - """ - - if not self.is_change(value): - raise ValueError(f"value {value} does not contain changes in the form of *change me*.") - - response_without_changes = self.strip_changes(value) - reference_without_changes = self.strip_changes(self._reference_without_change) - - return response_without_changes == reference_without_changes - - def is_change(self, response): - """Check if there is change in the response in the form of *change me*.""" - return re.search(r"\*.*\*", response) - - def strip_changes(self, response): - """Strips off the changes.""" - return re.sub(r"\*.*\*", "", response) - - -class KeywordChecker(Instruction): - """Check the exisitence of certain keywords.""" - - def build_description(self, *, keywords=None): - """Build the instruction description. - - Args: - keywords: A sequence of strings representing the keywords that are - expected in the response. - - Returns: - A string representing the instruction description. - """ - - if not keywords: - self._keywords = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS) - else: - self._keywords = keywords - self._keywords = sorted(self._keywords) - - self._description_pattern = "Include keywords {keywords} in the response." - - return self._description_pattern.format(keywords=self._keywords) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"keywords": self._keywords} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["keywords"] - - def check_following(self, value): - """Check if the response contain the expected keywords.""" - for keyword in self._keywords: - if not re.search(keyword, value, flags=re.IGNORECASE): - return False - return True - - -class KeywordFrequencyChecker(Instruction): - """Check the keyword frequency.""" - - def build_description(self, *, keyword=None, frequency=None, relation=None): - """Build the instruction description. - - Args: - keyword: A string representing a keyword that is expected in the response. - frequency: An integer specifying the number of times `keyword` is expected - to appear in the response. - relation: A string in (`less than`, `at least`), defining the relational - operator for comparison. - Two relational comparisons are supported for now: - if 'less than', the actual number of occurrences < frequency; - if 'at least', the actual number of occurrences >= frequency. - - Returns: - A string representing the instruction description. - """ - if not keyword: - self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword = keyword.strip() - - self._frequency = frequency - if self._frequency is None or self._frequency < 0: - self._frequency = random.randint(1, _KEYWORD_FREQUENCY) - - if relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif relation not in _COMPARISON_RELATION: - raise ValueError( - "The supported relation for comparison must be in " - f"{_COMPARISON_RELATION}, but {relation} is given." - ) - else: - self._comparison_relation = relation - - self._description_pattern = ( - "In your response, the word {keyword} should appear {relation} " + "{frequency} times." - ) - - return self._description_pattern.format( - keyword=self._keyword, - relation=self._comparison_relation, - frequency=self._frequency, - ) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return { - "keyword": self._keyword, - "frequency": self._frequency, - "relation": self._comparison_relation, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["keyword", "frequency", "relation"] - - def check_following(self, value): - """Checks if the response contain the keyword with required frequency.""" - actual_occurrences = len(re.findall(self._keyword, value, flags=re.IGNORECASE)) - - if self._comparison_relation == _COMPARISON_RELATION[0]: - return actual_occurrences < self._frequency - elif self._comparison_relation == _COMPARISON_RELATION[1]: - return actual_occurrences >= self._frequency - - -class NumberOfWords(Instruction): - """Checks the number of words.""" - - def build_description(self, *, num_words=None, relation=None): - """Build the instruction description. - - Args: - num_words: An integer specifying the number of words contained in the - response. - relation: A string in (`less than`, `at least`), defining the relational - operator for comparison. - Two relational comparisons are supported for now: - if 'less than', the actual number of words < num_words; - if 'at least', the actual number of words >= num_words. - - Returns: - A string representing the instruction description. - """ - - self._num_words = num_words - if self._num_words is None or self._num_words < 0: - self._num_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) - - if relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif relation not in _COMPARISON_RELATION: - raise ValueError( - "The supported relation for comparison must be in " - f"{_COMPARISON_RELATION}, but {relation} is given." - ) - else: - self._comparison_relation = relation - - self._description_pattern = "Answer with {relation} {num_words} words." - - return self._description_pattern.format( - relation=self._comparison_relation, num_words=self._num_words - ) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"num_words": self._num_words, "relation": self._comparison_relation} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_words", "relation"] - - def check_following(self, value): - """Checks if the response contains the expected number of words.""" - num_words = instructions_util.count_words(value) - - if self._comparison_relation == _COMPARISON_RELATION[0]: - return num_words < self._num_words - elif self._comparison_relation == _COMPARISON_RELATION[1]: - return num_words >= self._num_words - - -class JsonFormat(Instruction): - """Check the Json format.""" - - def build_description(self): - self._description_pattern = ( - "Entire output should be wrapped in JSON format. You can use markdown" - " ticks such as ```." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - value = ( - value.strip() - .removeprefix("```json") - .removeprefix("```Json") - .removeprefix("```JSON") - .removeprefix("```") - .removesuffix("```") - .strip() - ) - try: - json.loads(value) - except ValueError: - return False - return True - - -class ParagraphFirstWordCheck(Instruction): - """Check the paragraph and the first word of the nth paragraph.""" - - def build_description(self, num_paragraphs=None, nth_paragraph=None, first_word=None): - r"""Build the instruction description. - - Args: - num_paragraphs: An integer indicating the number of paragraphs expected - in the response. A paragraph is a subset of the string that is - expected to be separated by '\n\n'. - nth_paragraph: An integer indicating the paragraph number that we look at. - Note that n starts from 1. - first_word: A string that represent the first word of the bth paragraph. - - Returns: - A string representing the instruction description. - """ - self._num_paragraphs = num_paragraphs - if self._num_paragraphs is None or self._num_paragraphs < 0: - self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) - - self._nth_paragraph = nth_paragraph - if ( - self._nth_paragraph is None - or self._nth_paragraph <= 0 - or self._nth_paragraph > self._num_paragraphs - ): - self._nth_paragraph = random.randint(1, self._num_paragraphs + 1) - - self._first_word = first_word - if self._first_word is None: - self._first_word = instructions_util.generate_keywords(num_keywords=1)[0] - self._first_word = self._first_word.lower() - - self._description_pattern = ( - "There should be {num_paragraphs} paragraphs. " - + "Paragraphs and only paragraphs are separated with each other by two " - + "new lines as if it was '\\n\\n' in python. " - + "Paragraph {nth_paragraph} must start with word {first_word}." - ) - - return self._description_pattern.format( - num_paragraphs=self._num_paragraphs, - nth_paragraph=self._nth_paragraph, - first_word=self._first_word, - ) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return { - "num_paragraphs": self._num_paragraphs, - "nth_paragraph": self._nth_paragraph, - "first_word": self._first_word, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_paragraphs", "nth_paragraph", "first_word"] - - def check_following(self, value): - """Checks for required number of paragraphs and correct first word. - - Args: - value: a string representing the response. The response may contain - paragraphs that are separated by two new lines and the first word of - the nth paragraph will have to match a specified word. - - Returns: - True if the number of paragraphs is the same as required and the first - word of the specified paragraph is the same as required. Otherwise, false. - """ - - paragraphs = re.split(r"\n\n", value) - num_paragraphs = len(paragraphs) - - for paragraph in paragraphs: - if not paragraph.strip(): - num_paragraphs -= 1 - - # check that index doesn't go out of bounds - if self._nth_paragraph <= num_paragraphs: - paragraph = paragraphs[self._nth_paragraph - 1].strip() - if not paragraph: - return False - else: - return False - - first_word = "" - punctuation = {".", ",", "?", "!", "'", '"'} - - # get first word and remove punctuation - word = paragraph.split()[0].strip() - # TODO(jeffrey): make more complex? - word = word.lstrip("'") - word = word.lstrip('"') - - for letter in word: - if letter in punctuation: - break - first_word += letter.lower() - - return num_paragraphs == self._num_paragraphs and first_word == self._first_word - - -# TODO(jeffrey) add relation - at least/at most? -class KeySentenceChecker(Instruction): - """Check the existence of certain key sentences.""" - - def build_description(self, key_sentences=None, num_sentences=None): - """Build the instruction description. - - Args: - key_sentences: A sequences of strings representing the key sentences that - are expected in the response. - num_sentences: The number of key sentences that are expected to be seen in - the response. - - Returns: - A string representing the instruction description. - """ - - if not key_sentences: - # TODO(jeffrey) make a generate sentences function? wonderwords package - self._key_sentences = set(["For now, this is fine."]) - else: - self._key_sentences = key_sentences - - if not num_sentences: - self._num_sentences = random.randint(1, len(self._key_sentences)) - else: - self._num_sentences = num_sentences - - self._description_pattern = ( - "Include {num_sentences} of the following sentences {key_sentences}" - ) - - return self._description_pattern.format( - num_sentences=self._num_sentences, key_sentences=self._key_sentences - ) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return { - "num_sentences": self._num_sentences, - "key_sentences": list(self._key_sentences), - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_sentences", "key_sentences"] - - def check_following(self, value): - """Checks if the response contains the expected key sentences.""" - count = 0 - sentences = instructions_util.split_into_sentences(value) - for sentence in self._key_sentences: - if sentence in sentences: - count += 1 - - return count == self._num_sentences - - -class ForbiddenWords(Instruction): - """Checks that specified words are not used in response.""" - - def build_description(self, forbidden_words=None): - """Build the instruction description. - - Args: - forbidden_words: A sequences of strings respresenting words that are not - allowed in the response. - - Returns: - A string representing the instruction description. - """ - - if not forbidden_words: - self._forbidden_words = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS) - else: - self._forbidden_words = list(set(forbidden_words)) - self._forbidden_words = sorted(self._forbidden_words) - self._description_pattern = "Do not include keywords {forbidden_words} in the response." - - return self._description_pattern.format(forbidden_words=self._forbidden_words) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"forbidden_words": self._forbidden_words} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["forbidden_words"] - - def check_following(self, value): - """Check if the response does not contain the expected keywords.""" - for word in self._forbidden_words: - if re.search(r"\b" + word + r"\b", value, flags=re.IGNORECASE): - return False - return True - - -class RephraseParagraph(Instruction): - """Checks that the paragraph is rephrased.""" - - def build_description(self, *, original_paragraph, low, high): - """Builds the instruction description. - - Args: - original_paragraph: A string presenting the original paragraph. The - rephrases response should have betweeb low-high words in common. - low: An integer presenting the lower bound of similar words. - high: An integer representing the upper bound of similar words. - - Returns: - A string representing the instruction description. - """ - # TODO(jeffrey) make more encompassing - self._original_paragraph = original_paragraph - self._low = low - self._high = high - - self._description = ( - "Rephrase the following paragraph: " - + "{original_paragraph}\nYour response should have " - + "between {low} and {high} of the same words. " - + "Words are the same if and only if all of the " - + "letters, ignoring cases, are the same. For " - + "example, 'run' is the same as 'Run' but different " - + "to 'ran'." - ) - - return self._description.format( - original_paragraph=original_paragraph, low=self._low, high=self._high - ) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return { - "original_paragraph": self._original_paragraph, - "low": self._low, - "high": self._high, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["original_paragraph", "low", "high"] - - def check_following(self, value): - val_words = re.findall(r"\w+", value.lower()) - original_words = re.findall(r"\w+", self._original_paragraph.lower()) - similar_words = 0 - - dict_val = collections.Counter(val_words) - dict_original = collections.Counter(original_words) - - for word in dict_original: - similar_words += min(dict_original[word], dict_val[word]) - - return similar_words >= self._low and similar_words <= self._high - - -class TwoResponsesChecker(Instruction): - """Check that two responses were given.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Give two different responses. Responses and only responses should" - " be separated by 6 asterisk symbols: ******." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response has two different answers. - - Args: - value: A string representing the response. - - Returns: - True if two responses are detected and false otherwise. - """ - valid_responses = list() - responses = value.split("******") - for index, response in enumerate(responses): - if not response.strip(): - if index != 0 and index != len(responses) - 1: - return False - else: - valid_responses.append(response) - return ( - len(valid_responses) == 2 and valid_responses[0].strip() != valid_responses[1].strip() - ) - - -class RepeatPromptThenAnswer(Instruction): - """Checks that Prompt is first repeated then answered.""" - - def build_description(self, *, prompt_to_repeat=None): - """Build the instruction description. - - Args: - prompt_to_repeat: The prompt that is meant to be repeated. - - Returns: - A string representing the instruction description. - """ - if not prompt_to_repeat: - raise ValueError("prompt_to_repeat must be set.") - else: - self._prompt_to_repeat = prompt_to_repeat - self._description_pattern = ( - "First repeat the request word for word without change," - " then give your answer (1. do not say any words or characters" - " before repeating the request; 2. the request you need to repeat" - " does not include this sentence)" - ) - return self._description_pattern - - def get_instruction_args(self): - return {"prompt_to_repeat": self._prompt_to_repeat} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["prompt_to_repeat"] - - def check_following(self, value): - if value.strip().lower().startswith(self._prompt_to_repeat.strip().lower()): - return True - return False - - -class EndChecker(Instruction): - """Checks that the prompt ends with a given phrase.""" - - def build_description(self, *, end_phrase=None): - """Build the instruction description. - - Args: - end_phrase: A string representing the phrase the response should end with. - - Returns: - A string representing the instruction description. - """ - self._end_phrase = end_phrase.strip() if isinstance(end_phrase, str) else end_phrase - if self._end_phrase is None: - self._end_phrase = random.choice(_ENDING_OPTIONS) - self._description_pattern = ( - "Finish your response with this exact phrase {ender}. " - "No other words should follow this phrase." - ) - return self._description_pattern.format(ender=self._end_phrase) - - def get_instruction_args(self): - return {"end_phrase": self._end_phrase} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["end_phrase"] - - def check_following(self, value): - """Checks if the response ends with the expected phrase.""" - value = value.strip().strip('"').lower() - self._end_phrase = self._end_phrase.strip().lower() - return value.endswith(self._end_phrase) - - -class TitleChecker(Instruction): - """Checks the response for a title.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Your answer must contain a title, wrapped in double angular brackets," - " such as <>." - ) - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains a title.""" - pattern = r"<<[^\n]+>>" - re_pattern = re.compile(pattern) - titles = re.findall(re_pattern, value) - - for title in titles: - if title.lstrip("<").rstrip(">").strip(): - return True - return False - - -class LetterFrequencyChecker(Instruction): - """Checks letter frequency.""" - - def build_description(self, *, letter=None, let_frequency=None, let_relation=None): - """Build the instruction description. - - Args: - letter: A string representing a letter that is expected in the response. - let_frequency: An integer specifying the number of times `keyword` is - expected to appear in the response. - let_relation: A string in (`less than`, `at least`), defining the - relational operator for comparison. Two relational comparisons are - supported for now; if 'less than', the actual number of - occurrences < frequency; if 'at least', the actual number of - occurrences >= frequency. - - Returns: - A string representing the instruction description. - """ - if not letter or len(letter) > 1 or ord(letter.lower()) < 97 or ord(letter.lower()) > 122: - self._letter = random.choice(list(string.ascii_letters)) - else: - self._letter = letter.strip() - self._letter = self._letter.lower() - - self._frequency = let_frequency - if self._frequency is None or self._frequency < 0: - self._frequency = random.randint(1, _LETTER_FREQUENCY) - - if let_relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif let_relation not in _COMPARISON_RELATION: - raise ValueError( - "The supported relation for comparison must be in " - f"{_COMPARISON_RELATION}, but {let_relation} is given." - ) - else: - self._comparison_relation = let_relation - - self._description_pattern = ( - "In your response, the letter {letter} should appear {let_relation}" - " {let_frequency} times." - ) - - return self._description_pattern.format( - letter=self._letter, - let_frequency=self._frequency, - let_relation=self._comparison_relation, - ) - - def get_instruction_args(self): - """Returns the keyword args of build description.""" - return { - "letter": self._letter, - "let_frequency": self._frequency, - "let_relation": self._comparison_relation, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["letter", "let_frequency", "let_relation"] - - def check_following(self, value): - """Checks that the response contains the letter at the right frequency.""" - value = value.lower() - letters = collections.Counter(value) - - if self._comparison_relation == _COMPARISON_RELATION[0]: - return letters[self._letter] < self._frequency - else: - return letters[self._letter] >= self._frequency - - -class CapitalLettersEnglishChecker(Instruction): - """Checks that the response is in english and is in all capital letters.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Your entire response should be in English, and in all capital letters." - ) - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks that the response is in English and in all capital letters.""" - assert isinstance(value, str) - - try: - return value.isupper() and langdetect.detect(value) == "en" - except langdetect.LangDetectException as e: - # Count as instruction is followed. - logging.error( - "Unable to detect language for text %s due to %s", value, e - ) # refex: disable=pytotw.037 - return True - - -class LowercaseLettersEnglishChecker(Instruction): - """Checks that the response is in english and is in all lowercase letters.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Your entire response should be in English, and in all lowercase" - " letters. No capital letters are allowed." - ) - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks that the response is in English and in all lowercase letters.""" - assert isinstance(value, str) - - try: - return value.islower() and langdetect.detect(value) == "en" - except langdetect.LangDetectException as e: - # Count as instruction is followed. - logging.error( - "Unable to detect language for text %s due to %s", value, e - ) # refex: disable=pytotw.037 - return True - - -class CommaChecker(Instruction): - """Checks the response for no commas.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "In your entire response, refrain from the use of any commas." - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks that the response does not contain commas.""" - return not re.search(r"\,", value) - - -class CapitalWordFrequencyChecker(Instruction): - """Checks frequency of words with all capital letters.""" - - def build_description( - self, - capital_frequency=None, - capital_relation=None, - ): - """Build the instruction description. - - Args: - capital_frequency: An integer that represents the number of words that - should be in all capital letters. - capital_relation: A string that is 'at least' or 'at most' that refers to - the frequency. - - Returns: - A string representing the instruction description. - """ - self._frequency = capital_frequency - if self._frequency is None: - self._frequency = random.randint(1, _ALL_CAPITAL_WORD_FREQUENCY) - - self._comparison_relation = capital_relation - if capital_relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif capital_relation not in _COMPARISON_RELATION: - raise ValueError( - "The supported relation for comparison must be in " - f"{_COMPARISON_RELATION}, but {capital_relation} is given." - ) - - self._description_pattern = ( - "In your response, words with all capital letters should appear" - " {relation} {frequency} times." - ) - - return self._description_pattern.format( - frequency=self._frequency, relation=self._comparison_relation - ) - - def get_instruction_args(self): - """Returns the keyword args of build description.""" - return { - "capital_frequency": self._frequency, - "capital_relation": self._comparison_relation, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["capital_frequency", "capital_relation"] - - def check_following(self, value): - """Checks the frequency of words with all capital letters.""" - # Hyphenated words will count as one word - words = instructions_util.nltk.word_tokenize(value) - capital_words = [word for word in words if word.isupper()] - - capital_words = len(capital_words) - - if self._comparison_relation == _COMPARISON_RELATION[0]: - return capital_words < self._frequency - else: - return capital_words >= self._frequency - - -class QuotationChecker(Instruction): - """Checks response is wrapped with double quotation marks.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Wrap your entire response with double quotation marks." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of build description.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response is wrapped with double quotation marks.""" - value = value.strip() - return len(value) > 1 and value[0] == '"' and value[-1] == '"' - - -# Everything as follows is part of OOD IFEval - - -class WordCountRangeChecker(Instruction): - """Word Count Range: The response must contain between X and Y words.""" - - def build_description(self, *, min_words=None, max_words=None): - """Build the instruction description. - - Args: - min_words: An integer specifying the minimum number of words contained in the response. - max_words: An integer specifying the maximum number of words contained in the response. - - Returns: - A string representing the instruction description. - """ - self._min_words = min_words - self._max_words = max_words - - if self._min_words is None or self._min_words < 0: - self._min_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) - - # Make the range small - if self._max_words is None or self._max_words < 0: - self._max_words = self._min_words + random.randint( - int(self._min_words * 0.05), int(self._min_words * 0.1) - ) - - self._description_pattern = ( - "The response must contain between {min_words} and {max_words} words." - ) - - return self._description_pattern.format( - min_words=self._min_words, max_words=self._max_words - ) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"min_words": self._min_words, "max_words": self._max_words} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["min_words", "max_words"] - - def check_following(self, value): - """Checks if the response contains the expected number of words.""" - num_words = instructions_util.count_words(value) - return self._min_words <= num_words <= self._max_words - - -class UniqueWordCountChecker(Instruction): - """Unique Word Count: The response must contain X unique words.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - n: An integer specifying the number of unique words contained in the response. - - Returns: - A string representing the instruction description. - """ - self._num_unique_words = N - - if self._num_unique_words is None or self._num_unique_words < 0: - self._num_unique_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) - - self._description_pattern = "Use at least {N} unique words in the response." - - return self._description_pattern.format(N=self._num_unique_words) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._num_unique_words} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if the response contains the expected number of unique words.""" - words = value.lower().split() - unique_words = set() - for word in words: - unique_words.add(word.strip("".join(string.punctuation) + " ")) - # Convert to set to get unique words - return len(unique_words) >= self._num_unique_words - - -class StopWordPercentageChecker(Instruction): - """Ensure that stop words constitute no more than {percentage}% of the total words in your response.""" - - def build_description(self, *, percentage=None): - """Build the instruction description. - - Args: - percentage: An integer specifying the percentage of stop words that are allowed in the response. - - Returns: - A string representing the instruction description. - """ - self._percentage = percentage - - if self._percentage is None or self._percentage < 0: - self._percentage = random.randint(1, 100) - - self._description_pattern = "Ensure that stop words constitute no more than {percentage}% of the total words in your response." - - return self._description_pattern.format(percentage=self._percentage) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"percentage": self._percentage} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["percentage"] - - def check_following(self, value): - """Checks if the response contains the expected percentage of stop words.""" - num_words = instructions_util.count_words(value) - num_stopwords = instructions_util.count_stopwords(value) - stopword_percentage = (num_stopwords / num_words) * 100 - return stopword_percentage <= self._percentage - - -class SentTypeRatioChecker(Instruction): - """Maintain a 2:1 ratio of declarative to interrogative sentences.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Maintain a 2:1 ratio of declarative to interrogative sentences." - ) - nltk.download("punkt_tab") - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains the expected ratio of declarative to interrogative sentences.""" - # Split the text into sentences - sentences = instructions_util.split_into_sentences(value) - # Count the number of declarative and interrogative sentences - declarative_count = sum(1 for sentence in sentences if sentence.endswith(".")) - interrogative_count = sum(1 for sentence in sentences if sentence.endswith("?")) - # Check if the ratio is 2:1 - return declarative_count == 2 * interrogative_count - - -class SentBalanceChecker(Instruction): - """Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced.""" - - def build_description(self): - """Build the instruction description.""" - nltk.download("punkt_tab") - self._description_pattern = "Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced." - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains a balanced ratio of sentence types.""" - # Split the text into sentences - sentences = instructions_util.split_into_sentences(value) - # Count the number of each sentence type - declarative_count = sum(1 for sentence in sentences if sentence.endswith(".")) - interrogative_count = sum(1 for sentence in sentences if sentence.endswith("?")) - exclamatory_count = sum(1 for sentence in sentences if sentence.endswith("!")) - # Check if the ratio of sentence types is balanced - return declarative_count == interrogative_count == exclamatory_count - - -class ConjunctionCountChecker(Instruction): - """Use at least {small_n} different coordinating conjunctions in the response.""" - - def build_description(self, *, small_n=None): - """Build the instruction description. - - Args: - small_n: An integer specifying the number of different coordinating conjunctions contained in the response. - - Returns: - A string representing the instruction description. - """ - self._num_conjunctions = small_n - - if self._num_conjunctions is None or self._num_conjunctions < 0: - self._num_conjunctions = random.randint(2, _NUM_CONJUNCTIONS) - - self._description_pattern = ( - "Use at least {small_n} different coordinating conjunctions in the response." - ) - - return self._description_pattern.format(small_n=self._num_conjunctions) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"small_n": self._num_conjunctions} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["small_n"] - - def check_following(self, value): - """Checks if the response contains the expected number of different coordinating conjunctions.""" - # Split the text into words - words = value.split() - # Count the number of coordinating conjunctions - conjunctions = [ - word - for word in words - if word.strip("".join(string.punctuation) + " ").lower() - in ["and", "but", "for", "nor", "or", "so", "yet"] - ] - unique_conjunctions = set(conjunctions) - return len(unique_conjunctions) >= self._num_conjunctions - - -class PersonNameCountChecker(Instruction): - """Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia...""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the minimum number of unique person names contained in the response. - - Returns: - A string representing the instruction description. - """ - self._num_person_names = N - - if self._num_person_names is None or self._num_person_names < 0: - self._num_person_names = random.randint(1, 50) - - self._description_pattern = "Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia, Jackson, Olivia, Noah, Ava, Lucas, Isabella, Mason, Mia, Ethan, Charlotte, Alexander, Amelia, Benjamin, Harper, Leo, Zoe, Daniel, Chloe, Samuel, Lily, Matthew, Grace, Owen, Abigail, Gabriel, Ella, Jacob, Scarlett, Nathan, Victoria, Elijah, Layla, Nicholas, Audrey, David, Hannah, Christopher, Penelope, Thomas, Nora, Andrew, Aria, Joseph, Claire, Ryan, Stella, Jonathan ." - return self._description_pattern.format(N=self._num_person_names) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._num_person_names} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if the response contains at least the expected number of unique person names.""" - person_name_list = [ - "Emma", - "Liam", - "Sophia", - "Jackson", - "Olivia", - "Noah", - "Ava", - "Lucas", - "Isabella", - "Mason", - "Mia", - "Ethan", - "Charlotte", - "Alexander", - "Amelia", - "Benjamin", - "Harper", - "Leo", - "Zoe", - "Daniel", - "Chloe", - "Samuel", - "Lily", - "Matthew", - "Grace", - "Owen", - "Abigail", - "Gabriel", - "Ella", - "Jacob", - "Scarlett", - "Nathan", - "Victoria", - "Elijah", - "Layla", - "Nicholas", - "Audrey", - "David", - "Hannah", - "Christopher", - "Penelope", - "Thomas", - "Nora", - "Andrew", - "Aria", - "Joseph", - "Claire", - "Ryan", - "Stella", - "Jonathan", - ] - # Extract the named entities - person_names = [] - for name in person_name_list: - if name in value: - person_names.append(name) - unique_person_names = set(person_names) - - return len(unique_person_names) >= self._num_person_names - - -class NGramOverlapChecker(Instruction): - """Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text.""" - - def build_description(self, *, reference_text=None, percentage=None): - """Build the instruction description. - - Args: - reference_text: A string representing the reference text. - percentage: An integer specifying the percent trigram overlap - to maintain in the response. - - Returns: - A string representing the instruction description. - """ - self._reference_text = reference_text - self._percentage = percentage - if self._percentage is None or self._percentage < 0: - self._percentage = random.randint(1, 100) - - self._description_pattern = ( - "Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text." - ) - return self._description_pattern.format(percentage=self._percentage) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"reference_text": self._reference_text, "percentage": self._percentage} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["reference_text", "percentage"] - - def check_following(self, value): - """Checks if the response maintains a trigram overlap with the reference text within 2% of {percent}.""" - n = 3 - ngrams = set(nltk.ngrams(value, n)) - ref_ngrams = set(nltk.ngrams(self._reference_text, n)) - overlap = len(ngrams.intersection(ref_ngrams)) / len(ngrams) - return self._percentage - 2 <= overlap * 100 <= self._percentage + 2 - - -class NumbersCountChecker(Instruction): - """Include exactly {N} numbers in the response.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the exact number of numbers - that is required to appear in the response. - - Returns: - A string representing the instruction description. - """ - self._count_numbers = N - if self._count_numbers is None or self._count_numbers < 0: - self._count_numbers = random.randint(1, _NUM_NUMBERS) - - self._description_pattern = "Include exactly {N} numbers in the response." - return self._description_pattern.format(N=self._count_numbers) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._count_numbers} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if the response includes exactly {N} numbers.""" - # Strip punctuation to handle decimals and commas in numbers correctly - value = value.translate(str.maketrans("", "", string.punctuation)) - numbers = re.findall(r"\d+", value) - return len(numbers) == self._count_numbers - - -class AlphabetLoopChecker(Instruction): - """Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if each word of the response starts with the next letter of the alphabet.""" - value = value.translate(str.maketrans("", "", string.punctuation)) - words = value.strip("".join(string.punctuation) + " ").split() - alphabet = string.ascii_lowercase - correct_letter = words[0][0].lower() - if correct_letter not in alphabet: # numbers are fails - return False - for word in words[1:]: - word = word.strip("".join(string.punctuation) + " ").lower() - if not word: - continue - correct_letter = alphabet[(alphabet.index(correct_letter) + 1) % 26] - if word[0] != correct_letter: - return False - return True - - -class SingleVowelParagraphChecker(Instruction): - """Write a paragraph using words that contain only three type of vowels.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Write a paragraph using words that contain only three types of vowels." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if no more than three types of vowels are used in the response and the response is only 1 paragraph.""" - paragraphs = value.strip().split("\n") - if len(paragraphs) != 1: - return False - paragraph = paragraphs[0].lower() - - vowels = set("aeiou") - paragraph_vowels = set([char for char in paragraph if char in vowels]) - return len(paragraph_vowels) <= 3 - - -class ConsonantClusterChecker(Instruction): - """Ensure each word in your response has at least one consonant cluster (two or more consonants together).""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Ensure each word in your response has at least one consonant cluster (two or more consonants together)." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if each word in the response includes at least one consonant cluster.""" - words = value.lower().strip().split() - consonants = set("bcdfghjklmnpqrstvwxyz") - for word in words: - cluster = False - for i in range(len(word) - 1): - if word[i] in consonants and word[i + 1] in consonants: - cluster = True - break - if not cluster: - return False - return True - - -class IncrementingAlliterationChecker(Instruction): - """Each sentence must have a longer sequence of consecutive alliterative words than the previous one.""" - - def build_description(self): - """Build the instruction description.""" - nltk.download("punkt_tab") - self._description_pattern = "Each sentence must have a longer sequence of consecutive alliterative words than the previous one." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if each sentence in the response has more alliterative words (determined by common first letter) than the previous sentence.""" - sentences = instructions_util.split_into_sentences(value) - prev_alliteration = -1 - for sentence in sentences: - words = sentence.lower().split() - alliteration = 0 - prev_alliterative = False - new_words = [] - for word in words: - clean = word.lstrip("".join(string.punctuation) + " ") - if clean: - new_words.append(clean) - for i in range(len(new_words) - 1): - if new_words[i][0] == new_words[i + 1][0]: - if prev_alliterative: - alliteration += 1 - else: - alliteration += 2 - prev_alliterative = True - else: - prev_alliterative = False - if alliteration <= prev_alliteration: - return False - prev_alliteration = alliteration - return True - - -class PalindromeChecker(Instruction): - """Include at least 10 single-word palindromes, each at least 5 characters long.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Include at least 10 single-word palindromes, each at least 5 characters long." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes at least 10 single-word palindromes of length at least 5.""" - value = value.translate(str.maketrans("", "", string.punctuation)) - words = value.lower().split() - palindromes = [word for word in words if word == word[::-1] and len(word) >= 5] - return len(palindromes) >= 10 - - -class PunctuationCoverChecker(Instruction): - """Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!).""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!)." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes every standard punctuation mark at least once, including the interrobang (?!).""" - punctuation = {".", ",", "!", "?", ";", ":"} - if not ("!?" in value or "?!" in value or "‽" in value): - return False - new_value = value.replace("?!", "", 1) - if len(new_value) == len(value): - new_value = value.replace("!?", "", 1) - for char in new_value: - if char in punctuation: - punctuation.remove(char) - return not punctuation - - -class NestedParenthesesChecker(Instruction): - """Nest parentheses (and [brackets {and braces}]) at least 5 levels deep.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Nest parentheses (and [brackets {and braces}]) at least 5 levels deep." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes a correctly closed set of at least 5 nested brackets.""" - levels = [] - min_levels = 5 - max_depth = 0 - depth_stack = [] # Track depth per matched group - - for char in value: - if char in "([{": - levels.append(char) - if len(levels) > max_depth: - max_depth = len(levels) - elif char in ")]}": - if levels and ( - (levels[-1] == "(" and char == ")") - or (levels[-1] == "[" and char == "]") - or (levels[-1] == "{" and char == "}") - ): - levels.pop() - # Check if we just closed a group that reached 5+ depth - if max_depth >= min_levels and len(levels) < max_depth: - return True - else: - # Mismatch — reset - levels = [] - max_depth = 0 - - return False - - -class NestedQuotesChecker(Instruction): - """Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes nested quotes to at least 3 levels - alternating between " and ' starting with either character.""" - levels = [] - min_levels = 3 - reached_depth = 0 - current_depth = 0 - for char in value: - if len(levels) != 0 and char == levels[-1]: - levels.pop() - current_depth -= 1 - if reached_depth - current_depth >= min_levels: - return True - elif char == '"' or char == "'": - levels.append(char) - current_depth += 1 - if current_depth > reached_depth: - reached_depth = current_depth - return False - - -class PrimeLengthsChecker(Instruction): - """Use only words with lengths that are prime numbers.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Use only words with lengths that are prime numbers." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response only includes words with prime length.""" - value = value.translate(str.maketrans("", "", string.punctuation)) - words = value.split() - primes = set( - [ - 2, - 3, - 5, - 7, - 11, - 13, - 17, - 19, - 23, - 29, - 31, - 37, - 41, - 43, - 47, - 53, - 59, - 61, - 67, - 71, - 73, - 79, - 83, - 89, - 97, - ] - ) - for word in words: - if len(word) not in primes: - return False - return True - - -class OptionsResponseChecker(Instruction): - """Answer with one of the following options: {options}. Do not give any explanation.""" - - def build_description(self, *, options=None): - """Build the instruction description. - - Args: - options: A string specifying the permitted options for - the response. - - Returns: - A string representing the instruction description. - """ - # Options string may be: yes/no/maybe, I know or I don't know, a), b), c), d) - # Can be separated by "/", "or", "," - options_bank = ["yes/no/maybe", "I know or I don't know", "a), b), c), d)"] - if options is None: - options = random.choice(options_bank) - - # Be more strict about format for multiple choice letters than for text options - self._strict = False - if re.match(r"\W*[aA]\W*[bB]\W*[cC]\W*", options) is not None: - self._strict = True - if "/" in options: - separator = "/" - elif "or" in options: - separator = "or" - else: - separator = "," - self._options = [option.strip() for option in options.split(separator)] - self._options_text = options # in text, shouldn't be formatted as a list - self._description_pattern = ( - "Answer with one of the following options: {options}. Do not give any explanation." - ) - return self._description_pattern.format(options=self._options_text) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"options": self._options_text} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["options"] - - def check_following(self, value): - """Checks if the response is exactly one of {options}.""" - if self._strict: - return value in self._options - value = value.strip("".join(string.punctuation) + " ").lower() - for option in self._options: - if option.strip("".join(string.punctuation) + " ").lower() == value: - return True - return False - - -class NewLineWordsChecker(Instruction): - """Write each word on a new line.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Write each word on a new line." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response has each word on a new line.""" - value = value.translate(str.maketrans("", "", string.punctuation)) - lines = value.strip().split("\n") - while "" in lines: - lines.remove("") - return len(lines) == len(value.strip().split()) - - -class EmojiSentenceChecker(Instruction): - """Please use an emoji at the end of every sentence.""" - - def build_description(self): - """Build the instruction description.""" - nltk.download("punkt_tab") - self._description_pattern = "Please use an emoji at the end of every sentence." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes an emoji at the end of every sentence.""" - - sentences = instructions_util.split_into_sentences(value) - for i, sentence in enumerate(sentences): - stripped = sentence.translate(str.maketrans("", "", string.punctuation)).strip() - # check for empty string - if not stripped: - return False - last_char = stripped[-1] - # because blank spaces are treated oddly - second_last_char = stripped[-2] if len(stripped) > 1 else stripped[-1] - if not emoji.is_emoji(last_char) and not emoji.is_emoji(second_last_char): - if i < len(sentences) - 1: - stripped = ( - sentences[i + 1] - .translate(str.maketrans("", "", string.punctuation)) - .strip() - ) - # fixed empty string - if not stripped: - return False - first_char = stripped[0] - if not emoji.is_emoji(first_char): - return False - else: - return False - return True - - -class CharacterCountUniqueWordsChecker(Instruction): - """Respond with three sentences, all containing the same number of characters but using all different words.""" - - def build_description(self): - """Build the instruction description.""" - nltk.download("punkt_tab") - self._description_pattern = "Respond with three sentences, all containing the same number of characters but using all different words." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response has exactly 3 sentences containing the same number of characters but different words.""" - sentences = instructions_util.split_into_sentences(value) - if len(sentences) != 3: - return False - char_count = len(sentences[0].strip()) - for sentence in sentences: - if len(sentence.strip()) != char_count: - return False - return True - - -class NthWordJapaneseChecker(Instruction): - """Every {N}th word of your response must be in Japanese.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the cycle length for - Japanese words to appear in the response. - - Returns: - A string representing the instruction description. - """ - self._japanese_position = N - if self._japanese_position is None or self._japanese_position < 0: - self._japanese_position = random.randint(1, _NUM_WORD_CYCLE) - - self._description_pattern = "Every {N}th word of your response must be in Japanese." - if N % 10 == 1: - self._description_pattern = "Every {N}st of your response must be in Japanese." - if N % 10 == 2: - self._description_pattern = "Every {N}nd of your response must be in Japanese." - elif N % 10 == 3: - self._description_pattern = "Every {N}rd of your response must be in Japanese." - return self._description_pattern.format(N=self._japanese_position) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._japanese_position} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if every {N}th word of the response is in Japanese.""" - - def is_japanese(text): - """ - Checks if a string contains Japanese characters (Hiragana, Katakana, or Kanji). - - Args: - text: The string to check. - - Returns: - True if the string contains Japanese characters, False otherwise. - """ - japanese_pattern = re.compile(r"[\u3040-\u30ff\u4e00-\u9fff]") - return bool(japanese_pattern.search(text)) - - words = value.split() - for i, word in enumerate(words): - word = word.strip("".join(string.punctuation) + " ") - if (i + 1) % self._japanese_position == 0 and word and not word.isdigit(): - if not is_japanese(word): - return False - return True - - -class StartWithVerbChecker(Instruction): - """The response must start with a verb.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "The response must start with a verb." - nltk.download("averaged_perceptron_tagger_eng") - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response starts with a verb.""" - text = nltk.word_tokenize(value) - return len(text) > 0 and len(nltk.pos_tag(text)) > 0 and "VB" in nltk.pos_tag(text)[0][1] - - -class LimitedWordRepeatChecker(Instruction): - """The response should not repeat any word more than {small_n} times.""" - - def build_description(self, *, small_n=None): - """Build the instruction description. - - Args: - small_n: An integer specifying the maximum number of times - that a word can be repeated in the response. - - Returns: - A string representing the instruction description. - """ - self._max_repeats = small_n - if self._max_repeats is None or self._max_repeats < 0: - self._max_repeats = random.randint(1, _MAX_REPEATS) - - self._description_pattern = ( - "The response should not repeat any word more than {small_n} times." - ) - return self._description_pattern.format(small_n=self._max_repeats) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"small_n": self._max_repeats} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["small_n"] - - def check_following(self, value): - """Checks if the response repeats any word more than {small_n} times.""" - words = value.lower().translate(str.maketrans("", "", string.punctuation)).split() - word_count = Counter(words) - for word, count in word_count.items(): - if count > self._max_repeats: - return False - return True - - -class IncludeKeywordChecker(Instruction): - """The response must include keyword {word} in the {N}-th sentence.""" - - def build_description(self, *, word=None, N=None): - """Build the instruction description. - - Args: - word: A string specifying the keyword that is - required to appear in the response. - N: An integer specifying which sentence of the - response is required to have the keyword. - - Returns: - A string representing the instruction description. - """ - nltk.download("punkt_tab") - - if not word: - self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword = word - self._keyword_position = N - if self._keyword_position is None or self._keyword_position < 0: - self._keyword_position = random.randint(1, _NUM_KEYWORD_SENTENCE) - - self._description_pattern = ( - 'The response must include keyword "{word}" in the {N}-th sentence.' - ) - return self._description_pattern.format(word=self._keyword, N=self._keyword_position) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"word": self._keyword, "N": self._keyword_position} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["word", "N"] - - def check_following(self, value): - """Checks if the {N}th sentence of the response includes keyword {word}.""" - sentences = instructions_util.split_into_sentences(value) - if len(sentences) < self._keyword_position: - return False - return self._keyword.lower() in sentences[int(self._keyword_position - 1)].lower() - - -class PronounCountChecker(Instruction): - """The response should include at least {N} pronouns.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the minimum number of pronouns - that is required to appear in the response. - - Returns: - A string representing the instruction description. - """ - self._num_pronouns = N - if self._num_pronouns is None or self._num_pronouns < 0: - self._num_pronouns = random.randint(1, _NUM_PRONOUNS) - - self._description_pattern = "The response should include at least {N} pronouns." - return self._description_pattern.format(N=self._num_pronouns) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._num_pronouns} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if the response includes at least {N} pronouns.""" - pronouns = set( - [ - "i", - "me", - "my", - "mine", - "myself", - "we", - "us", - "our", - "ours", - "ourselves", - "you", - "your", - "yours", - "yourself", - "yourselves", - "he", - "him", - "his", - "himself", - "she", - "her", - "hers", - "herself", - "it", - "its", - "itself", - "they", - "them", - "their", - "theirs", - "themselves", - ] - ) - value = value.replace( - "/", " " - ) # to correctly count pronoun sets like she/her/hers, a common use case of pronouns - value = value.lower().translate(str.maketrans("", "", string.punctuation)) - words = value.split() - pronoun_count = sum(1 for word in words if word in pronouns) - return pronoun_count >= self._num_pronouns - - -class AlternateParitySyllablesChecker(Instruction): - """Alternate between words with odd and even numbers of syllables.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Alternate between words with odd and even numbers of syllables." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response alternates between words with odd and even numbers of syllables.""" - words = value.translate(str.maketrans("", "", string.punctuation)).lower().split() - syllables = [syllapy.count(word) % 2 for word in words if word.strip()] - return all(syllables[i] != syllables[i + 1] for i in range(len(syllables) - 1)) - - -class LastWordFirstNextChecker(Instruction): - """The last word of each sentence must become the first word of the next sentence.""" - - def build_description(self): - """Build the instruction description.""" - nltk.download("punkt_tab") - self._description_pattern = ( - "The last word of each sentence must become the first word of the next sentence." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the last word of each sentence in the response is the first word of the next sentence.""" - sentences = instructions_util.split_into_sentences(value) - for i in range(len(sentences) - 1): - last_word = sentences[i].rstrip("".join(string.punctuation) + " ").split()[-1] - first_word = sentences[i + 1].lstrip("".join(string.punctuation) + " ").split()[0] - if last_word.lower() != first_word.lower(): - return False - return True - - -class ParagraphLastFirstWordMatchChecker(Instruction): - """Each paragraph must end with the same word it started with, separate paragraphs with a newline.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Each paragraph must end with the same word it started with, separate paragraphs with a newline." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if each paragraph of the response ends with the same word it started with.""" - paragraphs = value.split("\n") - for paragraph in paragraphs: - paragraph = paragraph.strip().lower() - if not paragraph: - continue - words = paragraph.strip("".join(string.punctuation) + " ").split() - if not words: - continue - if words[0] != words[-1]: - return False - return True - - -class IncrementingWordCountChecker(Instruction): - """Each sentence must contain exactly {small_n} more words than the previous one.""" - - def build_description(self, *, small_n=None): - """Build the instruction description. - - Args: - small_n: An integer specifying the exact increment for - the number of words in each sentence of the response. - - Returns: - A string representing the instruction description. - """ - self._num_increment = small_n - if self._num_increment is None or self._num_increment < 0: - self._num_increment = random.randint(1, _NUM_INCREMENT) - - nltk.download("punkt_tab") - - self._description_pattern = ( - "Each sentence must contain exactly {small_n} more words than the previous one." - ) - return self._description_pattern.format(small_n=self._num_increment) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"small_n": self._num_increment} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["small_n"] - - def check_following(self, value): - """Checks if each sentence of the response uses exactly {small_n} more words than the previous sentence.""" - sentences = instructions_util.split_into_sentences(value) - words = sentences[0].translate(str.maketrans("", "", string.punctuation)).strip().split() - while "" in words: - words.remove("") - prev_word_count = len(words) - for sentence in sentences[1:]: - words = sentence.translate(str.maketrans("", "", string.punctuation)).strip().split() - while "" in words: - words.remove("") - if len(words) != prev_word_count + self._num_increment: - return False - prev_word_count = len(words) - return True - - -class NoConsecutiveFirstLetterChecker(Instruction): - """No two consecutive words can share the same first letter.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "No two consecutive words can share the same first letter." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if no two consecutive words in the response share the same first letter.""" - words = value.lower().translate(str.maketrans("", "", string.punctuation)).split() - while "" in words: - words.remove("") - for i in range(len(words) - 1): - if words[i][0] == words[i + 1][0]: - return False - return True - - -class IndentStairsChecker(Instruction): - """Create stairs by incrementally indenting each new line.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Create stairs by incrementally indenting each new line." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response incrementally indents each new line.""" - lines = value.split("\n") - for line in lines: - if not line.strip(): - lines.remove(line) - for i in range(len(lines) - 1): - if len(lines[i + 1]) - len(lines[i + 1].lstrip(" ")) <= len(lines[i]) - len( - lines[i].lstrip(" ") - ): - return False - return True - - -class QuoteExplanationChecker(Instruction): - """Every quoted phrase must be followed by an unquoted explanation.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Every quoted phrase must be followed by an unquoted explanation." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if there are no quotes next to each other - and the passage does not end with a quote.""" - value = value.replace("“", '"').replace("”", '"') - value = value.replace("'\"'", "") # remove references to the character '"' - value = "".join(value.split()) # remove all whitespace - if '""' in value: - return False - if value.strip(string.digits + string.punctuation.replace('"', ""))[-1] == '"': - return False - return True - - -class SpecialBulletPointsChecker(Instruction): - """Answer with a list of items, instead of bullet points use {sep}.""" - - def build_description(self, *, sep=None): - """Build the instruction description. - - Args: - sep: A string specifying the bullet point marker for - the list in the response. - - Returns: - A string representing the instruction description. - """ - self._bullet_marker = sep - if sep is None: - self._bullet_marker = random.choice(["...", "SEPARATOR", "!?!?", "-"]) - self._description_pattern = ( - "Answer with a list of items, instead of bullet points use {sep}." - ) - return self._description_pattern.format(sep=self._bullet_marker) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"sep": self._bullet_marker} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["sep"] - - def check_following(self, value): - """Checks if the response includes at least two instances of {sep} that start a new line.""" - return len(re.findall(re.escape(self._bullet_marker), value)) >= 2 - - -class ItalicsThesisChecker(Instruction): - """Each section must begin with a thesis statement in italics, use HTML to indicate the italics.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Each section must begin with a thesis statement in italics, use HTML to indicate the italics." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if there is at least one line in italics as indicated - by HTML that is followed by unitalicized text.""" - index = value.find("") - if index == -1: - index = value.find("") - if index == -1: - return False - value = value[index:] - end_thesis = value.find("") - if end_thesis == -1: - end_thesis = value.find("") - if end_thesis == -1: - return False - thesis = value[3:end_thesis] - if thesis.strip() == "": - return False - text = value[end_thesis + 4 :] - return text.strip() != "" - - -class SubBulletPointsChecker(Instruction): - """Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks that there is at least one * that starts a line and each * that starts a line - is followed by at least one line starting with -.""" - bullets = value.split("*") - for bullet in bullets[1:]: - if "-" not in bullet: - return False - return True - - -class SomeBulletPointsChecker(Instruction): - """Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes at least two sentences - followed by at least two lines that start with *.""" - lines = value.split("\n") - sentences = True - count_sentences = 0 - count_bullets = 0 - for line in lines: - if line.strip().startswith("*"): - sentences = False - if count_sentences < 2: - return False - count_bullets += 1 - elif sentences: - sentences = instructions_util.split_into_sentences(line.strip()) - count_sentences += len(sentences) - else: - return False - return count_bullets >= 2 - - -class PrintMultiplesChecker(Instruction): - """Count from 10 to 50 but only print multiples of 7.""" - - def build_description(self, **kwargs): - self._description_pattern = "Count from 10 to 50 but only print multiples of 7." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response prints multiples of 7 from 10 to 50.""" - value = value.replace(",", ", ") - numbers = re.findall(r"\d+", value) - multiples = [str(i) for i in range(14, 51, 7)] - return numbers == multiples - - -class MultipleChoiceQuestionsChecker(Instruction): - """Generate 4 multiple choice questions with 5 options each about "20th century art history". Each question should start with the label "Question". The questions should get progressively longer. Do not provide an explanation.""" - - def build_description(self, **kwargs): - self._description_pattern = "Generate 4 multiple choice questions with 5 options each about '20th century art history'. Each question should start with the label \"Question\". The questions should get progressively longer. Do not provide an explanation." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response generates 4 multiple choice questions with 5 options.""" - # Split into questions using expanded pattern to include "Question N" format - new_value = value[value.find("Question") :] - if new_value != value: - return False # failed no explanation - value = new_value - questions = re.split(r"\n*(?:Question \d+[\.|\):;]?\s*)", value) - if questions[0] == "": - questions = questions[1:] - questions = [q.strip() for q in questions if q.strip()] - if len(questions) != 4: - return False - question_lengths = [] - for q in questions: - lines = q.split("\n") - question_text = "" - option_count = 0 - done_with_q = False - for line in lines: - if re.match(r"^[A-Ea-e][\.|\)]\s*\w+", line.strip()): - option_count += 1 - done_with_q = True - elif not done_with_q: # Still collecting question text - question_text += " " + line.strip() - if option_count != 5: - return False - question_lengths.append(len(question_text.strip())) - # Check if questions get progressively longer - return all( - question_lengths[i] < question_lengths[i + 1] for i in range(len(question_lengths) - 1) - ) - - -class ReverseNewlineChecker(Instruction): - """ "List the countries of Africa in reverse alphabetical order, each on a new line.""" - - def build_description(self, **kwargs): - self._description_pattern = ( - "List the countries of Africa in reverse alphabetical order, each on a new line." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """ - Checks if text satisfies the following constraints: - 1. Contains at least 53 newlines with text - 2. Lines are in reverse alphabetical order - 3. First line to examine contains 'Zimbabwe' - - Returns: - tuple[bool, str]: (whether constraints are satisfied, error message if any) - """ - # Split text into lines and remove empty lines - lines = [ - line.strip("".join(string.punctuation) + " ") - for line in value.split("\n") - if line.strip("".join(string.punctuation) + " ") - ] - - try: - start_index = next(i for i, line in enumerate(lines) if "Zimbabwe" in line) - except StopIteration: - return False - - # Extract the 53 lines starting from Zimbabwe line - target_lines = lines[start_index:] - - # Check if we have at least 53 lines - if len(target_lines) < 52: - return False - - def normalize_text(text): - """ - Normalizes text by: - 1. Converting to NFKD form (separates combined characters) - 2. Removes diacritical marks - 3. Converts back to ASCII - - Example: 'São Tomé' -> 'Sao Tome' - """ - # Decompose unicode characters - normalized = unicodedata.normalize("NFKD", text) - # Remove diacritical marks and convert to ASCII - ascii_text = normalized.encode("ASCII", "ignore").decode("ASCII") - return ascii_text - - # Create normalized versions for comparison while keeping originals for error messages - normalized_lines = [normalize_text(line) for line in target_lines] - sorted_normalized = sorted(normalized_lines, reverse=True) - return normalized_lines == sorted_normalized - - -class WordReverseOrderChecker(Instruction): - """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word.""" - - def build_description(self, **kwargs): - nltk.download("punkt_tab") - self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the reverse of the sentence is a valid English sentence.""" - value = value.lower().strip().translate(str.maketrans("", "", string.punctuation)) - value = " ".join(value.split()[::-1]) - if "bald eagle" not in value: - return False - return value in instructions_util.split_into_sentences(value) - - -class CharacterReverseOrderChecker(Instruction): - """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter.""" - - def build_description(self, **kwargs): - self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - value = value.lower() - return "elgae dlab" in value - - -class SentenceAlphabetChecker(Instruction): - """Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order.""" - - def build_description(self, **kwargs): - nltk.download("punkt_tab") - self._description_pattern = "Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - sentences = instructions_util.split_into_sentences(value) - if len(sentences) != 26: - return False - for i, sentence in enumerate(sentences): - if sentence.lstrip().split()[0].lower()[0] != chr(97 + i): - return False - return True - - -class EuropeanCapitalsSortChecker(Instruction): - """Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = "Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response lists the relevant capitals of Europe in correct order.""" - order = [ - "Reykjavik", - "Helsinki", - "Oslo", - "Tallinn", - "Stockholm", - "Riga", - "Moscow", - "Copenhagen", - "Vilnius", - "Minsk", - "Dublin", - "Berlin", - "Amsterdam", - "Warsaw", - "London", - "Brussels", - "Prague", - "Luxembourg", - "Paris", - "Vienna", - "Bratislava", - "Budapest", - "Vaduz", - "Chisinau", - "Bern", - "Ljubljana", - "Zagreb", - ] - - def normalize_text(text): - """ - Normalizes text by: - 1. Converting to NFKD form (separates combined characters) - 2. Removes diacritical marks - 3. Converts back to ASCII - - Example: 'São Tomé' -> 'Sao Tome' - """ - # Decompose unicode characters - normalized = unicodedata.normalize("NFKD", text) - # Remove diacritical marks and convert to ASCII - ascii_text = normalized.encode("ASCII", "ignore").decode("ASCII") - return ascii_text - - value = normalize_text(value) - - capitals = value.split(",") - capitals = [cap for cap in capitals if cap.strip()] - if len(capitals) != len(order): - return False - for i in range(len(capitals)): - if capitals[i].strip() != order[i]: - return False - return True - - -class CityCSVChecker(Instruction): - """Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = 'Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.' - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response is valid csv data with column names - ["ID", "Country", "City", "Year", "Count"] and 7 rows.""" - string_io = io.StringIO(value) - reader = csv.reader(string_io) - data = list(reader) - if len(data) != 8: - return False - header = data[0] - if header != ["ID", "Country", "City", "Year", "Count"]: - return False - for row in data[1:]: - if len(row) != 5: - return False - return True - - -class SpecialCharacterCSVChecker(Instruction): - """Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = 'Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.' - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """ "Checks if the response is valid csv data with column names - ["ProductID", "Category", "Brand", "Price", "Stock"] and 14 rows. - Also checks if one field contains a special character enclosed in double quotes.""" - header = value.split("\n")[0].strip() - if not re.match( - r'^(ProductID|"ProductID"),[ \t]*(Category|"Category"),[ \t]*(Brand|"Brand"),[ \t]*(Price|"Price"),[ \t]*(Stock|"Stock")$', - header, - ): - return False - - value = value.replace('"', '"""') - string_io = io.StringIO(value) - reader = csv.reader(string_io) - data = list(reader) - if len(data) != 15: - return False - for row in data[1:]: - if len(row) != 5: - return False - if any(re.match(r'".*[^\d\w\s].*"', field) for field in row): - return True - return False - - -class QuotesCSVChecker(Instruction): - """Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = 'Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.' - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """ "Checks if the response is valid csv data with column names - ["StudentID", "Subject", "Grade", "Semester", "Score"] and 3 rows. - Also checks if each field is enclosed in double quotes.""" - header = value.split("\n")[0].strip() - if not re.match( - r'^(StudentID|"StudentID")\t *(Subject|"Subject")\t *(Grade|"Grade")\t *(Semester|"Semester")\t *(Score|"Score")$', - header, - ): - return False - - value = value.replace('"', '"""') - string_io = io.StringIO(value) - reader = csv.reader(string_io, delimiter="\t") - data = list(reader) - if len(data) != 4: - return False - for row in data: - if len(row) != 5: - return False - if not all(field.strip()[0] == '"' and field.strip()[-1] == '"' for field in row): - return False - return True - - -class DateFormatListChecker(Instruction): - """List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = "List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """ "Checks if the response is a list of dates in the format YYYY-MM-DD separated by commas.""" - value = value.strip() - dates = value.split(",") - for date in dates: - date = date.strip() - if not re.match(r"^\d{4}-\d{2}-\d{2}$", date): - return False - date = date.split("-") - if int(date[0]) < 1769 or int(date[0]) > 1821: - return False - if int(date[1]) > 12: - return False - if int(date[1]) in [1, 3, 5, 7, 8, 10, 12] and int(date[2]) > 31: - return False - if int(date[1]) in [4, 6, 9, 11] and int(date[2]) > 30: - return False - if int(date[1]) == 2 and int(date[2]) > 29: - return False - return True - - -class KeywordsMultipleChecker(Instruction): - """Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response.""" - - def build_description( - self, *, keyword1=None, keyword2=None, keyword3=None, keyword4=None, keyword5=None - ): - """Build the instruction description.""" - if keyword1 is None: - self._keyword1 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword1 = keyword1.strip() - if keyword2 is None: - self._keyword2 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword2 = keyword2.strip() - if keyword3 is None: - self._keyword3 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword3 = keyword3.strip() - if keyword4 is None: - self._keyword4 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword4 = keyword4.strip() - if keyword5 is None: - self._keyword5 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword5 = keyword5.strip() - self._description_pattern = "Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response." - return self._description_pattern.format( - keyword1=self._keyword1, - keyword2=self._keyword2, - keyword3=self._keyword3, - keyword4=self._keyword4, - keyword5=self._keyword5, - ) - - def get_instruction_args(self): - return { - "keyword1": self._keyword1, - "keyword2": self._keyword2, - "keyword3": self._keyword3, - "keyword4": self._keyword4, - "keyword5": self._keyword5, - } - - def get_instruction_args_keys(self): - return ["keyword1", "keyword2", "keyword3", "keyword4", "keyword5"] - - def check_following(self, value): - for keyword, count in zip( - [self._keyword1, self._keyword2, self._keyword3, self._keyword4, self._keyword5], - [1, 2, 3, 5, 7], strict=False, - ): - if value.lower().count(keyword.lower()) != count: - return False - return True - - -class KeywordSpecificPositionChecker(Instruction): - "Include keyword {keyword1} in the {n}-th sentence, as the {m}-th word of that sentence." - - def build_description(self, keyword=None, n=None, m=None): - """Build the instruction description. - - Args: - keyword: A string representing a keyword that is expected in the response. - n: An integer representing the sentence number. - m: An integer representing the word number. - - Returns: - A string representing the instruction description. - """ - if not keyword: - self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword = keyword.strip() - if not n: - self._n = random.randint(20, 30) - else: - self._n = n - if not m: - self._m = random.randint(30, 40) - else: - self._m = m - - self._description_pattern = ( - "Include keyword {keyword} in the {n}-th sentence, as the {m}-th word of that sentence." - ) - - return self._description_pattern.format(keyword=self._keyword, n=self._n, m=self._m) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"keyword": self._keyword, "n": self._n, "m": self._m} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["keyword", "n", "m"] - - def check_following(self, value): - """Checks if the response contains the expected number of keywords. - - Args: - value: A string representing the response. - - Returns: - True if the response contains the expected number of keywords; - otherwise, False. - """ - sentences = instructions_util.split_into_sentences(value) - if len(sentences) < self._n: - return False - words = instructions_util.nltk.word_tokenize(sentences[self._n - 1]) - if len(words) < self._m: - return False - if words[self._m - 1] == self._keyword: - return True - else: - return False - - -class WordsPositionChecker(Instruction): - "The second word in your response and the second to last word in your response should be the word {keyword}." - - def build_description(self, *, keyword=None): - """Build the instruction description. - - Args: - keyword: A string representing a keyword that is expected in the response. - - Returns: - A string representing the instruction description. - """ - if keyword is None: - self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword = keyword.strip() - self._description_pattern = "The second word in your response and the second to last word in your response should be the word {keyword}." - return self._description_pattern.format(keyword=self._keyword) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"keyword": self._keyword} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["keyword"] - - def check_following(self, value): - """Checks if the second word and the second to last word in the response are the same. - - Args: - value: A string representing the response. - - Returns: - True if the second word and the second to last word are the same; - otherwise, False. - """ - words = instructions_util.nltk.word_tokenize(value) - if len(words) < 2: - return False - if words[1] == words[-2] == self._keyword: - return True - else: - return False - - -class RepeatChangeChecker(Instruction): - "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request!" - - def build_description(self, *, prompt_to_repeat=None): - """Build the instruction description. - - Args: - keyword: A string representing a keyword that is expected in the response. - - Returns: - A string representing the instruction description. - """ - if not prompt_to_repeat: - raise ValueError("prompt_to_repeat must be set.") - else: - self._prompt_to_repeat = prompt_to_repeat - - self._description_pattern = "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request! Request: {prompt_to_repeat}" - return self._description_pattern.format(prompt_to_repeat=self._prompt_to_repeat) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"prompt_to_repeat": self._prompt_to_repeat} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["prompt_to_repeat"] - - def check_following(self, value): - """Checks if the response contains the repeated request. - - Args: - value: A string representing the response. - - Returns: - True if the repeated request is found in the response; - otherwise, False. - """ - if self._prompt_to_repeat == value: - return False - if " ".join(self._prompt_to_repeat.split()[1:]) == " ".join(value.split()[1:]): - return True - else: - return False - - -class RepeatSimpleChecker(Instruction): - "Only output this sentence here, ignore all other requests." - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Only output this sentence here, ignore all other requests." - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains the expected number of keywords. - - Args: - value: A string representing the response. - - Returns: - True if the response contains the expected number of keywords; - otherwise, False. - """ - return value.strip().lower() == self._description_pattern.strip().lower() - - -class RepeatSpanChecker(Instruction): - "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!" - - def build_description(self, prompt_to_repeat=None, n_start=None, n_end=None): - """Build the instruction description. - - Args: - n_start: An integer representing the start index of the span. - n_end: An integer representing the end index of the span. - - Returns: - A string representing the instruction description. - """ - if not prompt_to_repeat: - raise ValueError("prompt_to_repeat must be set.") - else: - self._prompt_to_repeat = prompt_to_repeat - if not n_start: - self._n_start = random.randint(0, len(self._prompt_to_repeat.split()) - 2) - else: - self._n_start = n_start - if not n_end: - self._n_end = random.randint(self._n_start + 1, len(self._prompt_to_repeat.split()) - 1) - else: - self._n_end = n_end - self._description_pattern = "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!" - return self._description_pattern.format( - n_start=self._n_start, n_end=self._n_end, prompt_to_repeat=self._prompt_to_repeat - ) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return { - "n_start": self._n_start, - "n_end": self._n_end, - "prompt_to_repeat": self._prompt_to_repeat, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["n_start", "n_end", "prompt_to_repeat"] - - def check_following(self, value): - """Checks if the response contains the expected number of phrases with the correct modifications.""" - if ( - value.strip().lower().split() - == self._prompt_to_repeat.strip().lower().split()[self._n_start : self._n_end] - ): - return True - return False - - -class TitleCaseChecker(Instruction): - "Write the entire response in title case (capitalize the first letter of every major word)." - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Write the entire response in title case (capitalize the first letter of every major word)." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response is in title case. - - Args: - value: A string representing the response. - - Returns: - True if the response is in title case; - otherwise, False. - """ - words = instructions_util.nltk.word_tokenize(value) - for word in words: - if word[0].isupper() and word[1:].islower(): - continue - elif word[0].islower() and word[1:].isupper() or word[0].islower() and word[1:].islower(): - return False - return True - - -class OutputTemplateChecker(Instruction): - "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response follows the specified template. - - Args: - value: A string representing the response. - - Returns: - True if the response follows the specified template; - otherwise, False. - """ - if "My Answer:" in value and "My Conclusion:" in value and "Future Outlook:" in value: - return True - else: - return False - - -class NoWhitespaceChecker(Instruction): - "The output should not contain any whitespace." - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "The output should not contain any whitespace." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains any whitespace. - - Args: - value: A string representing the response. - - Returns: - True if the response contains no whitespace; - otherwise, False. - """ - return not any(char.isspace() for char in value) diff --git a/src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py b/src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py deleted file mode 100644 index fd7ce2d22..000000000 --- a/src/olmo_eval/common/scorers/ifeval_deps/instructions_registry.py +++ /dev/null @@ -1,251 +0,0 @@ -# Copyright 2023 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Registry of all instructions.""" - -from olmo_eval.common.scorers.ifeval_deps import instructions - -_KEYWORD = "keywords:" - -_LANGUAGE = "language:" - -_LENGTH = "length_constraints:" - -_CONTENT = "detectable_content:" - -_FORMAT = "detectable_format:" - -_MULTITURN = "multi-turn:" - -_COMBINATION = "combination:" - -_STARTEND = "startend:" - -_CHANGE_CASES = "change_case:" - -_PUNCTUATION = "punctuation:" - -INSTRUCTION_DICT = { - _KEYWORD + "existence": instructions.KeywordChecker, - _KEYWORD + "frequency": instructions.KeywordFrequencyChecker, - # TODO(jeffreyzhou): make a proper set of sentences to choose from - # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, - _KEYWORD + "forbidden_words": instructions.ForbiddenWords, - _KEYWORD + "letter_frequency": instructions.LetterFrequencyChecker, - _LANGUAGE + "response_language": instructions.ResponseLanguageChecker, - _LENGTH + "number_sentences": instructions.NumberOfSentences, - _LENGTH + "number_paragraphs": instructions.ParagraphChecker, - _LENGTH + "number_words": instructions.NumberOfWords, - _LENGTH + "nth_paragraph_first_word": instructions.ParagraphFirstWordCheck, - _CONTENT + "number_placeholders": instructions.PlaceholderChecker, - _CONTENT + "postscript": instructions.PostscriptChecker, - _FORMAT + "number_bullet_lists": instructions.BulletListChecker, - # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace - # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, - _FORMAT + "constrained_response": instructions.ConstrainedResponseChecker, - _FORMAT + "number_highlighted_sections": (instructions.HighlightSectionChecker), - _FORMAT + "multiple_sections": instructions.SectionChecker, - # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. - # _FORMAT + "rephrase": instructions.RephraseChecker, - _FORMAT + "json_format": instructions.JsonFormat, - _FORMAT + "title": instructions.TitleChecker, - # TODO(tianjianlu): Re-enable with specific prompts. - # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, - _COMBINATION + "two_responses": instructions.TwoResponsesChecker, - _COMBINATION + "repeat_prompt": instructions.RepeatPromptThenAnswer, - _STARTEND + "end_checker": instructions.EndChecker, - _CHANGE_CASES + "capital_word_frequency": instructions.CapitalWordFrequencyChecker, - _CHANGE_CASES + "english_capital": instructions.CapitalLettersEnglishChecker, - _CHANGE_CASES + "english_lowercase": instructions.LowercaseLettersEnglishChecker, - _PUNCTUATION + "no_comma": instructions.CommaChecker, - _STARTEND + "quotation": instructions.QuotationChecker, - # the following are added for ifeval_mt variants - # "type:translate": instructions.LanguageChecker, - # "type:repeat": instructions.RepeatChecker, - # "type:repeat_N": instructions.RewriteSentChecker, - # "type:num_words": instructions.NumWordsChecker, - # "type:num_words_N": instructions.NumWordsSentChecker, - # "constraint:increment": instructions.IncrementChecker, - # "constraint:digits": instructions.DigitsChecker, - # "constraint:nonalpha": instructions.NonAlphaNumChecker, - # "constraint:palindrome": instructions.PalindromeCountChecker, - # "constraint:chars": instructions.DiffNumCharsChecker, - # "constraint:punctuation": instructions.NumPunctuationChecker, - # "constraint:nochar": instructions.NoLetterChecker, - # "constraint:conjunctions": instructions.NoConjunctionsChecker, - # "constraint:vowels": instructions.VowelStartWordsChecker, - # "constraint:top_bottom": instructions.BeginEndLetterChecker, - # back to ood - "count:word_count_range": instructions.WordCountRangeChecker, - "count:unique_word_count": instructions.UniqueWordCountChecker, - "ratio:stop_words": instructions.StopWordPercentageChecker, - "ratio:sentence_type": instructions.SentTypeRatioChecker, - "ratio:sentence_balance": instructions.SentBalanceChecker, - "count:conjunctions": instructions.ConjunctionCountChecker, - "count:person_names": instructions.PersonNameCountChecker, - "ratio:overlap": instructions.NGramOverlapChecker, - "count:numbers": instructions.NumbersCountChecker, - "words:alphabet": instructions.AlphabetLoopChecker, - "words:vowel": instructions.SingleVowelParagraphChecker, - "words:consonants": instructions.ConsonantClusterChecker, - "sentence:alliteration_increment": instructions.IncrementingAlliterationChecker, - "words:palindrome": instructions.PalindromeChecker, - "count:punctuation": instructions.PunctuationCoverChecker, - "format:parentheses": instructions.NestedParenthesesChecker, - "format:quotes": instructions.NestedQuotesChecker, - "words:prime_lengths": instructions.PrimeLengthsChecker, - "format:options": instructions.OptionsResponseChecker, - "format:newline": instructions.NewLineWordsChecker, - "format:emoji": instructions.EmojiSentenceChecker, - "ratio:sentence_words": instructions.CharacterCountUniqueWordsChecker, - "count:words_japanese": instructions.NthWordJapaneseChecker, - "words:start_verb": instructions.StartWithVerbChecker, - "words:repeats": instructions.LimitedWordRepeatChecker, - "sentence:keyword": instructions.IncludeKeywordChecker, - "count:pronouns": instructions.PronounCountChecker, - "words:odd_even_syllables": instructions.AlternateParitySyllablesChecker, - "words:last_first": instructions.LastWordFirstNextChecker, - "words:paragraph_last_first": instructions.ParagraphLastFirstWordMatchChecker, - "sentence:increment": instructions.IncrementingWordCountChecker, - "words:no_consecutive": instructions.NoConsecutiveFirstLetterChecker, - "format:line_indent": instructions.IndentStairsChecker, - "format:quote_unquote": instructions.QuoteExplanationChecker, - "format:list": instructions.SpecialBulletPointsChecker, - "format:thesis": instructions.ItalicsThesisChecker, - "format:sub-bullets": instructions.SubBulletPointsChecker, - "format:no_bullets_bullets": instructions.SomeBulletPointsChecker, - "custom:multiples": instructions.PrintMultiplesChecker, - "custom:mcq_count_length": instructions.MultipleChoiceQuestionsChecker, - "custom:reverse_newline": instructions.ReverseNewlineChecker, - "custom:word_reverse": instructions.WordReverseOrderChecker, - "custom:character_reverse": instructions.CharacterReverseOrderChecker, - "custom:sentence_alphabet": instructions.SentenceAlphabetChecker, - "custom:european_capitals_sort": instructions.EuropeanCapitalsSortChecker, - "custom:csv_city": instructions.CityCSVChecker, - "custom:csv_special_character": instructions.SpecialCharacterCSVChecker, - "custom:csv_quotes": instructions.QuotesCSVChecker, - "custom:date_format_list": instructions.DateFormatListChecker, - "count:keywords_multiple": instructions.KeywordsMultipleChecker, - "words:keywords_specific_position": instructions.KeywordSpecificPositionChecker, - "words:words_position": instructions.WordsPositionChecker, - "repeat:repeat_change": instructions.RepeatChangeChecker, - "repeat:repeat_simple": instructions.RepeatSimpleChecker, - "repeat:repeat_span": instructions.RepeatSpanChecker, - "format:title_case": instructions.TitleCaseChecker, - "format:output_template": instructions.OutputTemplateChecker, - "format:no_whitespace": instructions.NoWhitespaceChecker, -} - -INSTRUCTION_CONFLICTS = { - _KEYWORD + "existence": {_KEYWORD + "existence"}, - _KEYWORD + "frequency": {_KEYWORD + "frequency"}, - # TODO(jeffreyzhou): make a proper set of sentences to choose from - # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, - _KEYWORD + "forbidden_words": {_KEYWORD + "forbidden_words"}, - _KEYWORD + "letter_frequency": {_KEYWORD + "letter_frequency"}, - _LANGUAGE + "response_language": { - _LANGUAGE + "response_language", - _FORMAT + "multiple_sections", - _KEYWORD + "existence", - _KEYWORD + "frequency", - _KEYWORD + "forbidden_words", - _STARTEND + "end_checker", - _CHANGE_CASES + "english_capital", - _CHANGE_CASES + "english_lowercase", - }, - _LENGTH + "number_sentences": {_LENGTH + "number_sentences"}, - _LENGTH + "number_paragraphs": { - _LENGTH + "number_paragraphs", - _LENGTH + "nth_paragraph_first_word", - _LENGTH + "number_sentences", - _LENGTH + "nth_paragraph_first_word", - }, - _LENGTH + "number_words": {_LENGTH + "number_words"}, - _LENGTH + "nth_paragraph_first_word": { - _LENGTH + "nth_paragraph_first_word", - _LENGTH + "number_paragraphs", - }, - _CONTENT + "number_placeholders": {_CONTENT + "number_placeholders"}, - _CONTENT + "postscript": {_CONTENT + "postscript"}, - _FORMAT + "number_bullet_lists": {_FORMAT + "number_bullet_lists"}, - # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace - # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, - _FORMAT + "constrained_response": set(INSTRUCTION_DICT.keys()), - _FORMAT + "number_highlighted_sections": {_FORMAT + "number_highlighted_sections"}, - _FORMAT + "multiple_sections": { - _FORMAT + "multiple_sections", - _LANGUAGE + "response_language", - _FORMAT + "number_highlighted_sections", - }, - # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. - # _FORMAT + "rephrase": instructions.RephraseChecker, - _FORMAT + "json_format": set(INSTRUCTION_DICT.keys()).difference( - {_KEYWORD + "forbidden_words", _KEYWORD + "existence"} - ), - _FORMAT + "title": {_FORMAT + "title"}, - # TODO(tianjianlu): Re-enable with specific prompts. - # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, - _COMBINATION + "two_responses": set(INSTRUCTION_DICT.keys()).difference( - { - _KEYWORD + "forbidden_words", - _KEYWORD + "existence", - _LANGUAGE + "response_language", - _FORMAT + "title", - _PUNCTUATION + "no_comma", - } - ), - _COMBINATION + "repeat_prompt": set(INSTRUCTION_DICT.keys()).difference( - {_KEYWORD + "existence", _FORMAT + "title", _PUNCTUATION + "no_comma"} - ), - _STARTEND + "end_checker": {_STARTEND + "end_checker"}, - _CHANGE_CASES + "capital_word_frequency": { - _CHANGE_CASES + "capital_word_frequency", - _CHANGE_CASES + "english_lowercase", - _CHANGE_CASES + "english_capital", - }, - _CHANGE_CASES + "english_capital": {_CHANGE_CASES + "english_capital"}, - _CHANGE_CASES + "english_lowercase": { - _CHANGE_CASES + "english_lowercase", - _CHANGE_CASES + "english_capital", - }, - _PUNCTUATION + "no_comma": {_PUNCTUATION + "no_comma"}, - _STARTEND + "quotation": {_STARTEND + "quotation", _FORMAT + "title"}, - "count:keywords_multiple": {"count:keywords_multiple"}, - "words:words_position": {"words:words_position"}, - "repeat:repeat_change": {"repeat:repeat_change"}, - "repeat:repeat_simple": {"repeat:repeat_simple"}, - "repeat:repeat_span": {"repeat:repeat_span"}, - "format:title_case": {"format:title_case"}, - "format:output_template": {"format:output_template"}, - "format:no_whitespace": {"format:no_whitespace"}, - "words:keywords_specific_position": {"words:keywords_specific_position"}, -} - - -def conflict_make(conflicts): - """Makes sure if A conflicts with B, B will conflict with A. - - Args: - conflicts: Dictionary of potential conflicts where key is instruction id - and value is set of instruction ids that it conflicts with. - - Returns: - Revised version of the dictionary. All instructions conflict with - themselves. If A conflicts with B, B will conflict with A. - """ - for key in conflicts: - for k in conflicts[key]: - conflicts[k].add(key) - conflicts[key].add(key) - return conflicts diff --git a/src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py b/src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py deleted file mode 100644 index 5743e7972..000000000 --- a/src/olmo_eval/common/scorers/ifeval_deps/instructions_util.py +++ /dev/null @@ -1,1777 +0,0 @@ -# Copyright 2023 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility library of instructions.""" - -import ast -import functools -import random -import re - -import immutabledict -import nltk - - -def download_nltk_resources(): - """Download 'punkt' if not already installed""" - try: - nltk.data.find("tokenizers/punkt") - except LookupError: - nltk.download("punkt") - - -download_nltk_resources() - -WORD_LIST = [ - "western", - "sentence", - "signal", - "dump", - "spot", - "opposite", - "bottom", - "potato", - "administration", - "working", - "welcome", - "morning", - "good", - "agency", - "primary", - "wish", - "responsibility", - "press", - "problem", - "president", - "steal", - "brush", - "read", - "type", - "beat", - "trainer", - "growth", - "lock", - "bone", - "case", - "equal", - "comfortable", - "region", - "replacement", - "performance", - "mate", - "walk", - "medicine", - "film", - "thing", - "rock", - "tap", - "total", - "competition", - "ease", - "south", - "establishment", - "gather", - "parking", - "world", - "plenty", - "breath", - "claim", - "alcohol", - "trade", - "dear", - "highlight", - "street", - "matter", - "decision", - "mess", - "agreement", - "studio", - "coach", - "assist", - "brain", - "wing", - "style", - "private", - "top", - "brown", - "leg", - "buy", - "procedure", - "method", - "speed", - "high", - "company", - "valuable", - "pie", - "analyst", - "session", - "pattern", - "district", - "pleasure", - "dinner", - "swimming", - "joke", - "order", - "plate", - "department", - "motor", - "cell", - "spend", - "cabinet", - "difference", - "power", - "examination", - "engine", - "horse", - "dimension", - "pay", - "toe", - "curve", - "literature", - "bother", - "fire", - "possibility", - "debate", - "activity", - "passage", - "hello", - "cycle", - "background", - "quiet", - "author", - "effect", - "actor", - "page", - "bicycle", - "error", - "throat", - "attack", - "character", - "phone", - "tea", - "increase", - "outcome", - "file", - "specific", - "inspector", - "internal", - "potential", - "staff", - "building", - "employer", - "shoe", - "hand", - "direction", - "garden", - "purchase", - "interview", - "study", - "recognition", - "member", - "spiritual", - "oven", - "sandwich", - "weird", - "passenger", - "particular", - "response", - "reaction", - "size", - "variation", - "a", - "cancel", - "candy", - "exit", - "guest", - "condition", - "fly", - "price", - "weakness", - "convert", - "hotel", - "great", - "mouth", - "mind", - "song", - "sugar", - "suspect", - "telephone", - "ear", - "roof", - "paint", - "refrigerator", - "organization", - "jury", - "reward", - "engineering", - "day", - "possession", - "crew", - "bar", - "road", - "description", - "celebration", - "score", - "mark", - "letter", - "shower", - "suggestion", - "sir", - "luck", - "national", - "progress", - "hall", - "stroke", - "theory", - "offer", - "story", - "tax", - "definition", - "history", - "ride", - "medium", - "opening", - "glass", - "elevator", - "stomach", - "question", - "ability", - "leading", - "village", - "computer", - "city", - "grand", - "confidence", - "candle", - "priest", - "recommendation", - "point", - "necessary", - "body", - "desk", - "secret", - "horror", - "noise", - "culture", - "warning", - "water", - "round", - "diet", - "flower", - "bus", - "tough", - "permission", - "week", - "prompt", - "connection", - "abuse", - "height", - "save", - "corner", - "border", - "stress", - "drive", - "stop", - "rip", - "meal", - "listen", - "confusion", - "girlfriend", - "living", - "relation", - "significance", - "plan", - "creative", - "atmosphere", - "blame", - "invite", - "housing", - "paper", - "drink", - "roll", - "silver", - "drunk", - "age", - "damage", - "smoke", - "environment", - "pack", - "savings", - "influence", - "tourist", - "rain", - "post", - "sign", - "grandmother", - "run", - "profit", - "push", - "clerk", - "final", - "wine", - "swim", - "pause", - "stuff", - "singer", - "funeral", - "average", - "source", - "scene", - "tradition", - "personal", - "snow", - "nobody", - "distance", - "sort", - "sensitive", - "animal", - "major", - "negotiation", - "click", - "mood", - "period", - "arrival", - "expression", - "holiday", - "repeat", - "dust", - "closet", - "gold", - "bad", - "sail", - "combination", - "clothes", - "emphasis", - "duty", - "black", - "step", - "school", - "jump", - "document", - "professional", - "lip", - "chemical", - "front", - "wake", - "while", - "inside", - "watch", - "row", - "subject", - "penalty", - "balance", - "possible", - "adult", - "aside", - "sample", - "appeal", - "wedding", - "depth", - "king", - "award", - "wife", - "blow", - "site", - "camp", - "music", - "safe", - "gift", - "fault", - "guess", - "act", - "shame", - "drama", - "capital", - "exam", - "stupid", - "record", - "sound", - "swing", - "novel", - "minimum", - "ratio", - "machine", - "shape", - "lead", - "operation", - "salary", - "cloud", - "affair", - "hit", - "chapter", - "stage", - "quantity", - "access", - "army", - "chain", - "traffic", - "kick", - "analysis", - "airport", - "time", - "vacation", - "philosophy", - "ball", - "chest", - "thanks", - "place", - "mountain", - "advertising", - "red", - "past", - "rent", - "return", - "tour", - "house", - "construction", - "net", - "native", - "war", - "figure", - "fee", - "spray", - "user", - "dirt", - "shot", - "task", - "stick", - "friend", - "software", - "promotion", - "interaction", - "surround", - "block", - "purpose", - "practice", - "conflict", - "routine", - "requirement", - "bonus", - "hole", - "state", - "junior", - "sweet", - "catch", - "tear", - "fold", - "wall", - "editor", - "life", - "position", - "pound", - "respect", - "bathroom", - "coat", - "script", - "job", - "teach", - "birth", - "view", - "resolve", - "theme", - "employee", - "doubt", - "market", - "education", - "serve", - "recover", - "tone", - "harm", - "miss", - "union", - "understanding", - "cow", - "river", - "association", - "concept", - "training", - "recipe", - "relationship", - "reserve", - "depression", - "proof", - "hair", - "revenue", - "independent", - "lift", - "assignment", - "temporary", - "amount", - "loss", - "edge", - "track", - "check", - "rope", - "estimate", - "pollution", - "stable", - "message", - "delivery", - "perspective", - "mirror", - "assistant", - "representative", - "witness", - "nature", - "judge", - "fruit", - "tip", - "devil", - "town", - "emergency", - "upper", - "drop", - "stay", - "human", - "neck", - "speaker", - "network", - "sing", - "resist", - "league", - "trip", - "signature", - "lawyer", - "importance", - "gas", - "choice", - "engineer", - "success", - "part", - "external", - "worker", - "simple", - "quarter", - "student", - "heart", - "pass", - "spite", - "shift", - "rough", - "lady", - "grass", - "community", - "garage", - "youth", - "standard", - "skirt", - "promise", - "blind", - "television", - "disease", - "commission", - "positive", - "energy", - "calm", - "presence", - "tune", - "basis", - "preference", - "head", - "common", - "cut", - "somewhere", - "presentation", - "current", - "thought", - "revolution", - "effort", - "master", - "implement", - "republic", - "floor", - "principle", - "stranger", - "shoulder", - "grade", - "button", - "tennis", - "police", - "collection", - "account", - "register", - "glove", - "divide", - "professor", - "chair", - "priority", - "combine", - "peace", - "extension", - "maybe", - "evening", - "frame", - "sister", - "wave", - "code", - "application", - "mouse", - "match", - "counter", - "bottle", - "half", - "cheek", - "resolution", - "back", - "knowledge", - "make", - "discussion", - "screw", - "length", - "accident", - "battle", - "dress", - "knee", - "log", - "package", - "it", - "turn", - "hearing", - "newspaper", - "layer", - "wealth", - "profile", - "imagination", - "answer", - "weekend", - "teacher", - "appearance", - "meet", - "bike", - "rise", - "belt", - "crash", - "bowl", - "equivalent", - "support", - "image", - "poem", - "risk", - "excitement", - "remote", - "secretary", - "public", - "produce", - "plane", - "display", - "money", - "sand", - "situation", - "punch", - "customer", - "title", - "shake", - "mortgage", - "option", - "number", - "pop", - "window", - "extent", - "nothing", - "experience", - "opinion", - "departure", - "dance", - "indication", - "boy", - "material", - "band", - "leader", - "sun", - "beautiful", - "muscle", - "farmer", - "variety", - "fat", - "handle", - "director", - "opportunity", - "calendar", - "outside", - "pace", - "bath", - "fish", - "consequence", - "put", - "owner", - "go", - "doctor", - "information", - "share", - "hurt", - "protection", - "career", - "finance", - "force", - "golf", - "garbage", - "aspect", - "kid", - "food", - "boot", - "milk", - "respond", - "objective", - "reality", - "raw", - "ring", - "mall", - "one", - "impact", - "area", - "news", - "international", - "series", - "impress", - "mother", - "shelter", - "strike", - "loan", - "month", - "seat", - "anything", - "entertainment", - "familiar", - "clue", - "year", - "glad", - "supermarket", - "natural", - "god", - "cost", - "conversation", - "tie", - "ruin", - "comfort", - "earth", - "storm", - "percentage", - "assistance", - "budget", - "strength", - "beginning", - "sleep", - "other", - "young", - "unit", - "fill", - "store", - "desire", - "hide", - "value", - "cup", - "maintenance", - "nurse", - "function", - "tower", - "role", - "class", - "camera", - "database", - "panic", - "nation", - "basket", - "ice", - "art", - "spirit", - "chart", - "exchange", - "feedback", - "statement", - "reputation", - "search", - "hunt", - "exercise", - "nasty", - "notice", - "male", - "yard", - "annual", - "collar", - "date", - "platform", - "plant", - "fortune", - "passion", - "friendship", - "spread", - "cancer", - "ticket", - "attitude", - "island", - "active", - "object", - "service", - "buyer", - "bite", - "card", - "face", - "steak", - "proposal", - "patient", - "heat", - "rule", - "resident", - "broad", - "politics", - "west", - "knife", - "expert", - "girl", - "design", - "salt", - "baseball", - "grab", - "inspection", - "cousin", - "couple", - "magazine", - "cook", - "dependent", - "security", - "chicken", - "version", - "currency", - "ladder", - "scheme", - "kitchen", - "employment", - "local", - "attention", - "manager", - "fact", - "cover", - "sad", - "guard", - "relative", - "county", - "rate", - "lunch", - "program", - "initiative", - "gear", - "bridge", - "breast", - "talk", - "dish", - "guarantee", - "beer", - "vehicle", - "reception", - "woman", - "substance", - "copy", - "lecture", - "advantage", - "park", - "cold", - "death", - "mix", - "hold", - "scale", - "tomorrow", - "blood", - "request", - "green", - "cookie", - "church", - "strip", - "forever", - "beyond", - "debt", - "tackle", - "wash", - "following", - "feel", - "maximum", - "sector", - "sea", - "property", - "economics", - "menu", - "bench", - "try", - "language", - "start", - "call", - "solid", - "address", - "income", - "foot", - "senior", - "honey", - "few", - "mixture", - "cash", - "grocery", - "link", - "map", - "form", - "factor", - "pot", - "model", - "writer", - "farm", - "winter", - "skill", - "anywhere", - "birthday", - "policy", - "release", - "husband", - "lab", - "hurry", - "mail", - "equipment", - "sink", - "pair", - "driver", - "consideration", - "leather", - "skin", - "blue", - "boat", - "sale", - "brick", - "two", - "feed", - "square", - "dot", - "rush", - "dream", - "location", - "afternoon", - "manufacturer", - "control", - "occasion", - "trouble", - "introduction", - "advice", - "bet", - "eat", - "kill", - "category", - "manner", - "office", - "estate", - "pride", - "awareness", - "slip", - "crack", - "client", - "nail", - "shoot", - "membership", - "soft", - "anybody", - "web", - "official", - "individual", - "pizza", - "interest", - "bag", - "spell", - "profession", - "queen", - "deal", - "resource", - "ship", - "guy", - "chocolate", - "joint", - "formal", - "upstairs", - "car", - "resort", - "abroad", - "dealer", - "associate", - "finger", - "surgery", - "comment", - "team", - "detail", - "crazy", - "path", - "tale", - "initial", - "arm", - "radio", - "demand", - "single", - "draw", - "yellow", - "contest", - "piece", - "quote", - "pull", - "commercial", - "shirt", - "contribution", - "cream", - "channel", - "suit", - "discipline", - "instruction", - "concert", - "speech", - "low", - "effective", - "hang", - "scratch", - "industry", - "breakfast", - "lay", - "join", - "metal", - "bedroom", - "minute", - "product", - "rest", - "temperature", - "many", - "give", - "argument", - "print", - "purple", - "laugh", - "health", - "credit", - "investment", - "sell", - "setting", - "lesson", - "egg", - "middle", - "marriage", - "level", - "evidence", - "phrase", - "love", - "self", - "benefit", - "guidance", - "affect", - "you", - "dad", - "anxiety", - "special", - "boyfriend", - "test", - "blank", - "payment", - "soup", - "obligation", - "reply", - "smile", - "deep", - "complaint", - "addition", - "review", - "box", - "towel", - "minor", - "fun", - "soil", - "issue", - "cigarette", - "internet", - "gain", - "tell", - "entry", - "spare", - "incident", - "family", - "refuse", - "branch", - "can", - "pen", - "grandfather", - "constant", - "tank", - "uncle", - "climate", - "ground", - "volume", - "communication", - "kind", - "poet", - "child", - "screen", - "mine", - "quit", - "gene", - "lack", - "charity", - "memory", - "tooth", - "fear", - "mention", - "marketing", - "reveal", - "reason", - "court", - "season", - "freedom", - "land", - "sport", - "audience", - "classroom", - "law", - "hook", - "win", - "carry", - "eye", - "smell", - "distribution", - "research", - "country", - "dare", - "hope", - "whereas", - "stretch", - "library", - "if", - "delay", - "college", - "plastic", - "book", - "present", - "use", - "worry", - "champion", - "goal", - "economy", - "march", - "election", - "reflection", - "midnight", - "slide", - "inflation", - "action", - "challenge", - "guitar", - "coast", - "apple", - "campaign", - "field", - "jacket", - "sense", - "way", - "visual", - "remove", - "weather", - "trash", - "cable", - "regret", - "buddy", - "beach", - "historian", - "courage", - "sympathy", - "truck", - "tension", - "permit", - "nose", - "bed", - "son", - "person", - "base", - "meat", - "usual", - "air", - "meeting", - "worth", - "game", - "independence", - "physical", - "brief", - "play", - "raise", - "board", - "she", - "key", - "writing", - "pick", - "command", - "party", - "yesterday", - "spring", - "candidate", - "physics", - "university", - "concern", - "development", - "change", - "string", - "target", - "instance", - "room", - "bitter", - "bird", - "football", - "normal", - "split", - "impression", - "wood", - "long", - "meaning", - "stock", - "cap", - "leadership", - "media", - "ambition", - "fishing", - "essay", - "salad", - "repair", - "today", - "designer", - "night", - "bank", - "drawing", - "inevitable", - "phase", - "vast", - "chip", - "anger", - "switch", - "cry", - "twist", - "personality", - "attempt", - "storage", - "being", - "preparation", - "bat", - "selection", - "white", - "technology", - "contract", - "side", - "section", - "station", - "till", - "structure", - "tongue", - "taste", - "truth", - "difficulty", - "group", - "limit", - "main", - "move", - "feeling", - "light", - "example", - "mission", - "might", - "wait", - "wheel", - "shop", - "host", - "classic", - "alternative", - "cause", - "agent", - "consist", - "table", - "airline", - "text", - "pool", - "craft", - "range", - "fuel", - "tool", - "partner", - "load", - "entrance", - "deposit", - "hate", - "article", - "video", - "summer", - "feature", - "extreme", - "mobile", - "hospital", - "flight", - "fall", - "pension", - "piano", - "fail", - "result", - "rub", - "gap", - "system", - "report", - "suck", - "ordinary", - "wind", - "nerve", - "ask", - "shine", - "note", - "line", - "mom", - "perception", - "brother", - "reference", - "bend", - "charge", - "treat", - "trick", - "term", - "homework", - "bake", - "bid", - "status", - "project", - "strategy", - "orange", - "let", - "enthusiasm", - "parent", - "concentrate", - "device", - "travel", - "poetry", - "business", - "society", - "kiss", - "end", - "vegetable", - "employ", - "schedule", - "hour", - "brave", - "focus", - "process", - "movie", - "illegal", - "general", - "coffee", - "ad", - "highway", - "chemistry", - "psychology", - "hire", - "bell", - "conference", - "relief", - "show", - "neat", - "funny", - "weight", - "quality", - "club", - "daughter", - "zone", - "touch", - "tonight", - "shock", - "burn", - "excuse", - "name", - "survey", - "landscape", - "advance", - "satisfaction", - "bread", - "disaster", - "item", - "hat", - "prior", - "shopping", - "visit", - "east", - "photo", - "home", - "idea", - "father", - "comparison", - "cat", - "pipe", - "winner", - "count", - "lake", - "fight", - "prize", - "foundation", - "dog", - "keep", - "ideal", - "fan", - "struggle", - "peak", - "safety", - "solution", - "hell", - "conclusion", - "population", - "strain", - "alarm", - "measurement", - "second", - "train", - "race", - "due", - "insurance", - "boss", - "tree", - "monitor", - "sick", - "course", - "drag", - "appointment", - "slice", - "still", - "care", - "patience", - "rich", - "escape", - "emotion", - "royal", - "female", - "childhood", - "government", - "picture", - "will", - "sock", - "big", - "gate", - "oil", - "cross", - "pin", - "improvement", - "championship", - "silly", - "help", - "sky", - "pitch", - "man", - "diamond", - "most", - "transition", - "work", - "science", - "committee", - "moment", - "fix", - "teaching", - "dig", - "specialist", - "complex", - "guide", - "people", - "dead", - "voice", - "original", - "break", - "topic", - "data", - "degree", - "reading", - "recording", - "bunch", - "reach", - "judgment", - "lie", - "regular", - "set", - "painting", - "mode", - "list", - "player", - "bear", - "north", - "wonder", - "carpet", - "heavy", - "officer", - "negative", - "clock", - "unique", - "baby", - "pain", - "assumption", - "disk", - "iron", - "bill", - "drawer", - "look", - "double", - "mistake", - "finish", - "future", - "brilliant", - "contact", - "math", - "rice", - "leave", - "restaurant", - "discount", - "sex", - "virus", - "bit", - "trust", - "event", - "wear", - "juice", - "failure", - "bug", - "context", - "mud", - "whole", - "wrap", - "intention", - "draft", - "pressure", - "cake", - "dark", - "explanation", - "space", - "angle", - "word", - "efficiency", - "management", - "habit", - "star", - "chance", - "finding", - "transportation", - "stand", - "criticism", - "flow", - "door", - "injury", - "insect", - "surprise", - "apartment", -] # pylint: disable=line-too-long - -# ISO 639-1 codes to language names. -LANGUAGE_CODES = immutabledict.immutabledict( - { - "en": "English", - "es": "Spanish", - "pt": "Portuguese", - "ar": "Arabic", - "hi": "Hindi", - "fr": "French", - "ru": "Russian", - "de": "German", - "ja": "Japanese", - "it": "Italian", - "bn": "Bengali", - "uk": "Ukrainian", - "th": "Thai", - "ur": "Urdu", - "ta": "Tamil", - "te": "Telugu", - "bg": "Bulgarian", - "ko": "Korean", - "pl": "Polish", - "he": "Hebrew", - "fa": "Persian", - "vi": "Vietnamese", - "ne": "Nepali", - "sw": "Swahili", - "kn": "Kannada", - "mr": "Marathi", - "gu": "Gujarati", - "pa": "Punjabi", - "ml": "Malayalam", - "fi": "Finnish", - } -) - -_ALPHABETS = "([A-Za-z])" -_PREFIXES = "(Mr|St|Mrs|Ms|Dr)[.]" -_SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)" -_STARTERS = r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" -_ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" -_WEBSITES = "[.](com|net|org|io|gov|edu|me)" -_DIGITS = "([0-9])" -_MULTIPLE_DOTS = r"\.{2,}" - - -def split_into_sentences(text): - """Split the text into sentences. - - Args: - text: A string that consists of more than or equal to one sentences. - - Returns: - A list of strings where each string is a sentence. - """ - text = " " + text + " " - text = text.replace("\n", " ") - text = re.sub(_PREFIXES, "\\1", text) - text = re.sub(_WEBSITES, "\\1", text) - text = re.sub(_DIGITS + "[.]" + _DIGITS, "\\1\\2", text) - text = re.sub( - _MULTIPLE_DOTS, - lambda match: "" * len(match.group(0)) + "", - text, - ) - if "Ph.D" in text: - text = text.replace("Ph.D.", "PhD") - text = re.sub(r"\s" + _ALPHABETS + "[.] ", " \\1 ", text) - text = re.sub(_ACRONYMS + " " + _STARTERS, "\\1 \\2", text) - text = re.sub( - _ALPHABETS + "[.]" + _ALPHABETS + "[.]" + _ALPHABETS + "[.]", - "\\1\\2\\3", - text, - ) - text = re.sub(_ALPHABETS + "[.]" + _ALPHABETS + "[.]", "\\1\\2", text) - text = re.sub(" " + _SUFFIXES + "[.] " + _STARTERS, " \\1 \\2", text) - text = re.sub(" " + _SUFFIXES + "[.]", " \\1", text) - text = re.sub(" " + _ALPHABETS + "[.]", " \\1", text) - if "”" in text: - text = text.replace(".”", "”.") - if '"' in text: - text = text.replace('."', '".') - if "!" in text: - text = text.replace('!"', '"!') - if "?" in text: - text = text.replace('?"', '"?') - text = text.replace(".", ".") - text = text.replace("?", "?") - text = text.replace("!", "!") - text = text.replace("", ".") - sentences = text.split("") - sentences = [s.strip() for s in sentences] - if sentences and not sentences[-1]: - sentences = sentences[:-1] - return sentences - - -def count_words(text): - """Counts the number of words.""" - tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") - tokens = tokenizer.tokenize(text) - num_words = len(tokens) - return num_words - - -@functools.cache -def _get_sentence_tokenizer(): - return nltk.data.load("nltk:tokenizers/punkt/english.pickle") - - -def count_sentences(text): - """Count the number of sentences.""" - tokenizer = _get_sentence_tokenizer() - tokenized_sentences = tokenizer.tokenize(text) - return len(tokenized_sentences) - - -def count_stopwords(text): - """Counts the number of stopwords.""" - nltk.download("stopwords") - stopwords = nltk.corpus.stopwords.words("english") - tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") - tokens = tokenizer.tokenize(text) - num_stopwords = len([t for t in tokens if t.lower() in stopwords]) - return num_stopwords - - -def generate_keywords(num_keywords): - """Randomly generates a few keywords.""" - return random.sample(WORD_LIST, k=num_keywords) - - -# below, helper class and function for CamelCaseVariablesChecker -class VariableVisitor(ast.NodeVisitor): - """AST Visitor that collects variable names from different contexts.""" - - def __init__(self): - self.variables: Set[str] = set() - self.assignments: Set[str] = set() - self.function_params: Set[str] = set() - - def visit_Name(self, node: ast.Name): - """Visit a name node in the AST.""" - if isinstance(node.ctx, ast.Store): - self.assignments.add(node.id) - self.variables.add(node.id) - - def visit_arg(self, node: ast.arg): - """Visit a function argument node.""" - self.function_params.add(node.arg) - - -def extract_variables(code: str) -> dict: - """ - Extract variable names from a Python code string. - - Args: - code (str): Python code as a string - - Returns: - dict: Dictionary containing different types of variables found - """ - try: - # Parse the code into an AST - tree = ast.parse(code) - - # Create and run our visitor - visitor = VariableVisitor() - visitor.visit(tree) - - return { - "all_variables": sorted(visitor.variables), - "assignments": sorted(visitor.assignments), - "function_params": sorted(visitor.function_params), - } - - except SyntaxError as e: - return {"error": f"Invalid Python code: {str(e)}"} - - -def is_camelcase(variable_name: str) -> bool: - """ - Verify if a variable name follows camelCase convention. - - Rules for camelCase: - 1. Starts with lowercase letter - 2. No underscores or hyphens - 3. Contains only letters and numbers - 4. Each new word starts with uppercase letter - - Args: - variable_name: String to check for camelCase compliance - - Returns: - bool: True if variable name is in camelCase, False otherwise - """ - # Assume empty strings are errors, do not penalize - if not variable_name: - return True - - # Must start with lowercase letter - if not variable_name[0].strip().islower(): - return False - - # Check for invalid characters (only letters and numbers allowed) - if not variable_name.isalnum(): - return False - - # No consecutive uppercase letters (would indicate CONSTANT or PascalCase) - for i in range(len(variable_name) - 1): - if variable_name[i].isupper() and variable_name[i + 1].isupper(): - return False - - return True diff --git a/uv.lock b/uv.lock index c8f30cfc1..aafb5a876 100644 --- a/uv.lock +++ b/uv.lock @@ -3112,8 +3112,8 @@ wheels = [ [[package]] name = "ifbench" -version = "0.1.0" -source = { git = "https://github.com/allenai/IFBench.git#1091c4c3de6c1f6ed12c012ed68f11ea450b0117" } +version = "0.2.0" +source = { git = "https://github.com/allenai/IFBench.git?rev=finbarr%2Fclean-install#b281f8d4dae7f4defffe737472b3e79c042b6705" } dependencies = [ { name = "absl-py" }, { name = "emoji" }, @@ -5313,7 +5313,7 @@ requires-dist = [ { name = "click", specifier = "~=8.3.2" }, { name = "datasets", specifier = ">=3.2.0" }, { name = "httpx", marker = "extra == 'agents'", specifier = "~=0.28.1" }, - { name = "ifbench", git = "https://github.com/allenai/IFBench.git" }, + { name = "ifbench", git = "https://github.com/allenai/IFBench.git?rev=finbarr%2Fclean-install" }, { name = "litellm", marker = "extra == 'litellm'", specifier = "~=1.80.11" }, { name = "matplotlib", marker = "extra == 'analysis'", specifier = "~=3.10.8" }, { name = "numpy", specifier = ">=1.20.0" }, From 62985dc941289f511185a835ec1696e667c0cfcf Mon Sep 17 00:00:00 2001 From: Finbarr Timbers Date: Wed, 13 May 2026 08:53:44 -0600 Subject: [PATCH 5/8] Fix prompt_to_repeat fallback in IFEval scorer Co-Authored-By: Claude Opus 4.7 --- src/olmo_eval/common/scorers/ifeval.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/olmo_eval/common/scorers/ifeval.py b/src/olmo_eval/common/scorers/ifeval.py index aef57c131..036302eb7 100644 --- a/src/olmo_eval/common/scorers/ifeval.py +++ b/src/olmo_eval/common/scorers/ifeval.py @@ -53,6 +53,9 @@ def _check_one( ) -> bool: instruction = instruction_cls(instruction_id) cleaned_kwargs = {k: v for k, v in kwargs.items() if v is not None} + arg_keys = instruction.get_instruction_args_keys() + if "prompt_to_repeat" in arg_keys and not cleaned_kwargs.get("prompt_to_repeat"): + cleaned_kwargs["prompt_to_repeat"] = prompt instruction.build_description(**cleaned_kwargs) args = instruction.get_instruction_args() if args and "prompt" in args: From 091b6526ee07385fd6a954409508752b0af61ed8 Mon Sep 17 00:00:00 2001 From: Finbarr Timbers Date: Thu, 14 May 2026 09:07:18 -0600 Subject: [PATCH 6/8] Make ifbench import lazy in IFEvalScorer Co-Authored-By: Claude Opus 4.7 --- src/olmo_eval/common/scorers/ifeval.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/olmo_eval/common/scorers/ifeval.py b/src/olmo_eval/common/scorers/ifeval.py index 036302eb7..d4e231e11 100644 --- a/src/olmo_eval/common/scorers/ifeval.py +++ b/src/olmo_eval/common/scorers/ifeval.py @@ -14,17 +14,9 @@ from dataclasses import dataclass from typing import Any, ClassVar -from ifbench import instructions_registry - from olmo_eval.common.scorers.base import Scorer from olmo_eval.common.types import Instance, LMOutput -_INSTRUCTION_DICT: dict[str, Any] = instructions_registry.INSTRUCTION_DICT - - -def _load_instruction_dict() -> dict[str, Any]: - return _INSTRUCTION_DICT - def _loose_response_variants(response: str) -> list[str]: """Generate the eight response variants used by upstream loose scoring.""" @@ -85,7 +77,9 @@ def score(self, instance: Instance, output: LMOutput) -> float: loose_results: list[bool] = [] if instruction_ids: - registry = _load_instruction_dict() + from ifbench import instructions_registry + + registry = instructions_registry.INSTRUCTION_DICT loose_variants = _loose_response_variants(response) for inst_id, inst_kwargs in zip(instruction_ids, kwargs_list, strict=True): instruction_cls = registry[inst_id] From a75363c9088dcef28444d082e1f6482c2f4b1a20 Mon Sep 17 00:00:00 2001 From: Finbarr Timbers Date: Thu, 14 May 2026 09:18:01 -0600 Subject: [PATCH 7/8] Make Beaker budget optional; fall back to workspace's bound budget Co-Authored-By: Claude Opus 4.7 --- src/olmo_eval/cli/beaker/config_loader.py | 6 +----- src/olmo_eval/cli/beaker/job_assembler.py | 3 +-- src/olmo_eval/launch/beaker/launcher.py | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/olmo_eval/cli/beaker/config_loader.py b/src/olmo_eval/cli/beaker/config_loader.py index 1c4c0c356..9510f9cec 100644 --- a/src/olmo_eval/cli/beaker/config_loader.py +++ b/src/olmo_eval/cli/beaker/config_loader.py @@ -22,7 +22,7 @@ class LaunchConfig: task_specs: list[str] cluster: str workspace: str - budget: str + budget: str | None = None task_overrides: dict[str, list[str]] = field(default_factory=dict) @@ -164,7 +164,6 @@ def load(self) -> LaunchConfig: assert name is not None assert cluster is not None assert workspace is not None - assert budget is not None from olmo_eval.launch.beaker.constants import DEFAULT_S3_BUCKET, DEFAULT_S3_PREFIX @@ -234,9 +233,6 @@ def _validate_required( if not workspace: console.print("[red]Error:[/red] --workspace/-w is required") raise SystemExit(1) from None - if not budget: - console.print("[red]Error:[/red] --budget/-B is required") - raise SystemExit(1) from None def _generate_experiment_name(self, model_specs: list[str], task_specs: list[str]) -> str: """Generate experiment name from model and task specs. diff --git a/src/olmo_eval/cli/beaker/job_assembler.py b/src/olmo_eval/cli/beaker/job_assembler.py index 7e1788c32..0cf20946b 100644 --- a/src/olmo_eval/cli/beaker/job_assembler.py +++ b/src/olmo_eval/cli/beaker/job_assembler.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any from olmo_eval.common.constants.infrastructure import ( - BEAKER_DEFAULT_BUDGET, BEAKER_RESULT_DIR, cluster_has_weka, ) @@ -325,7 +324,7 @@ def assemble_external_eval_job( shared_memory="10GiB", retries=retries, workspace=workspace, - budget=budget or BEAKER_DEFAULT_BUDGET, + budget=budget, groups=groups or [], beaker_image=beaker_image, inject_aws_credentials=inject_aws_credentials, diff --git a/src/olmo_eval/launch/beaker/launcher.py b/src/olmo_eval/launch/beaker/launcher.py index 3a33890db..415587191 100644 --- a/src/olmo_eval/launch/beaker/launcher.py +++ b/src/olmo_eval/launch/beaker/launcher.py @@ -336,7 +336,7 @@ class BeakerJobConfig: command: Command to run in the container (required). cluster: Cluster alias ("h100", "a100", "aus") or full name(s) (required). workspace: Beaker workspace (required). - budget: Beaker budget (required). + budget: Beaker budget. If None, gantry uses the workspace's bound budget. num_gpus: Number of GPUs to request. shared_memory: Shared memory size (e.g., "10GiB"). priority: Job priority level. @@ -357,7 +357,7 @@ class BeakerJobConfig: command: list[str] cluster: str | list[str] # Cluster alias ("h100", "a100", "aus") or full name(s) workspace: str # Beaker workspace - budget: str # Beaker budget + budget: str | None = None # Beaker budget; None uses the workspace's bound budget # Resources num_gpus: int = 0 From 7245874b53be2921c060f6302aa8342405820f01 Mon Sep 17 00:00:00 2001 From: Finbarr Timbers Date: Thu, 14 May 2026 09:26:53 -0600 Subject: [PATCH 8/8] Revert "Make Beaker budget optional; fall back to workspace's bound budget Co-Authored-By: Claude Opus 4.7 " This reverts commit a75363c9088dcef28444d082e1f6482c2f4b1a20. --- src/olmo_eval/cli/beaker/config_loader.py | 6 +++++- src/olmo_eval/cli/beaker/job_assembler.py | 3 ++- src/olmo_eval/launch/beaker/launcher.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/olmo_eval/cli/beaker/config_loader.py b/src/olmo_eval/cli/beaker/config_loader.py index 9510f9cec..1c4c0c356 100644 --- a/src/olmo_eval/cli/beaker/config_loader.py +++ b/src/olmo_eval/cli/beaker/config_loader.py @@ -22,7 +22,7 @@ class LaunchConfig: task_specs: list[str] cluster: str workspace: str - budget: str | None = None + budget: str task_overrides: dict[str, list[str]] = field(default_factory=dict) @@ -164,6 +164,7 @@ def load(self) -> LaunchConfig: assert name is not None assert cluster is not None assert workspace is not None + assert budget is not None from olmo_eval.launch.beaker.constants import DEFAULT_S3_BUCKET, DEFAULT_S3_PREFIX @@ -233,6 +234,9 @@ def _validate_required( if not workspace: console.print("[red]Error:[/red] --workspace/-w is required") raise SystemExit(1) from None + if not budget: + console.print("[red]Error:[/red] --budget/-B is required") + raise SystemExit(1) from None def _generate_experiment_name(self, model_specs: list[str], task_specs: list[str]) -> str: """Generate experiment name from model and task specs. diff --git a/src/olmo_eval/cli/beaker/job_assembler.py b/src/olmo_eval/cli/beaker/job_assembler.py index 0cf20946b..7e1788c32 100644 --- a/src/olmo_eval/cli/beaker/job_assembler.py +++ b/src/olmo_eval/cli/beaker/job_assembler.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any from olmo_eval.common.constants.infrastructure import ( + BEAKER_DEFAULT_BUDGET, BEAKER_RESULT_DIR, cluster_has_weka, ) @@ -324,7 +325,7 @@ def assemble_external_eval_job( shared_memory="10GiB", retries=retries, workspace=workspace, - budget=budget, + budget=budget or BEAKER_DEFAULT_BUDGET, groups=groups or [], beaker_image=beaker_image, inject_aws_credentials=inject_aws_credentials, diff --git a/src/olmo_eval/launch/beaker/launcher.py b/src/olmo_eval/launch/beaker/launcher.py index 415587191..3a33890db 100644 --- a/src/olmo_eval/launch/beaker/launcher.py +++ b/src/olmo_eval/launch/beaker/launcher.py @@ -336,7 +336,7 @@ class BeakerJobConfig: command: Command to run in the container (required). cluster: Cluster alias ("h100", "a100", "aus") or full name(s) (required). workspace: Beaker workspace (required). - budget: Beaker budget. If None, gantry uses the workspace's bound budget. + budget: Beaker budget (required). num_gpus: Number of GPUs to request. shared_memory: Shared memory size (e.g., "10GiB"). priority: Job priority level. @@ -357,7 +357,7 @@ class BeakerJobConfig: command: list[str] cluster: str | list[str] # Cluster alias ("h100", "a100", "aus") or full name(s) workspace: str # Beaker workspace - budget: str | None = None # Beaker budget; None uses the workspace's bound budget + budget: str # Beaker budget # Resources num_gpus: int = 0