From b11dfaf4db3355d53c41964ee51652d71450117f Mon Sep 17 00:00:00 2001 From: HenryByte Date: Fri, 31 Jul 2026 16:28:01 +0100 Subject: [PATCH] Adding in performance tests to see what improvement iterative will give. --- ...a7_20260731_152455_uncommited-changes.json | 307 ++++++++++++++++++ Makefile | 10 +- pyproject.toml | 6 + tests/perf/__init__.py | 0 tests/perf/test_fountain_perf.py | 70 ++++ uv.lock | 24 ++ 6 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 .benchmarks/Linux-CPython-3.12-64bit/0002_05c33b050b92b213cbd02a71ba63bcf6469fdfa7_20260731_152455_uncommited-changes.json create mode 100644 tests/perf/__init__.py create mode 100644 tests/perf/test_fountain_perf.py diff --git a/.benchmarks/Linux-CPython-3.12-64bit/0002_05c33b050b92b213cbd02a71ba63bcf6469fdfa7_20260731_152455_uncommited-changes.json b/.benchmarks/Linux-CPython-3.12-64bit/0002_05c33b050b92b213cbd02a71ba63bcf6469fdfa7_20260731_152455_uncommited-changes.json new file mode 100644 index 0000000..120bbd0 --- /dev/null +++ b/.benchmarks/Linux-CPython-3.12-64bit/0002_05c33b050b92b213cbd02a71ba63bcf6469fdfa7_20260731_152455_uncommited-changes.json @@ -0,0 +1,307 @@ +{ + "machine_info": { + "node": "vapula", + "processor": "x86_64", + "machine": "x86_64", + "python_compiler": "GCC 13.3.0", + "python_implementation": "CPython", + "python_implementation_version": "3.12.3", + "python_version": "3.12.3", + "python_build": [ + "main", + "Jun 19 2026 12:46:00" + ], + "release": "7.0.0-28-generic", + "system": "Linux", + "cpu": { + "python_version": "3.12.3.final.0 (64 bit)", + "cpuinfo_version": [ + 9, + 0, + 0 + ], + "cpuinfo_version_string": "9.0.0", + "arch": "X86_64", + "bits": 64, + "count": 12, + "arch_string_raw": "x86_64", + "vendor_id_raw": "AuthenticAMD", + "brand_raw": "AMD Ryzen 5 3600X 6-Core Processor", + "hz_advertised_friendly": "4.2751 GHz", + "hz_actual_friendly": "4.2751 GHz", + "hz_advertised": [ + 4275120000, + 0 + ], + "hz_actual": [ + 4275120000, + 0 + ], + "model": 113, + "family": 23, + "flags": [ + "3dnowprefetch", + "abm", + "adx", + "aes", + "aperfmperf", + "apic", + "arat", + "avic", + "avx", + "avx2", + "bmi1", + "bmi2", + "bpext", + "cat_l3", + "cdp_l3", + "clflush", + "clflushopt", + "clwb", + "clzero", + "cmov", + "cmp_legacy", + "constant_tsc", + "cpb", + "cpuid", + "cqm", + "cqm_llc", + "cqm_mbm_local", + "cqm_mbm_total", + "cqm_occup_llc", + "cr8_legacy", + "cx16", + "cx8", + "dbx", + "de", + "decodeassists", + "extapic", + "extd_apicid", + "f16c", + "flushbyasid", + "fma", + "fpu", + "fsgsbase", + "fxsr", + "fxsr_opt", + "ht", + "hw_pstate", + "ibpb", + "ibs", + "irperf", + "lahf_lm", + "lbrv", + "lm", + "mba", + "mca", + "mce", + "misalignsse", + "mmx", + "mmxext", + "monitor", + "movbe", + "msr", + "mtrr", + "mwaitx", + "nonstop_tsc", + "nopl", + "npt", + "nrip_save", + "nx", + "osvw", + "osxsave", + "overflow_recov", + "pae", + "pat", + "pausefilter", + "pci_l2i", + "pclmulqdq", + "pdpe1gb", + "perfctr_core", + "perfctr_llc", + "perfctr_nb", + "pfthreshold", + "pge", + "pni", + "popcnt", + "pqe", + "pqm", + "pse", + "pse36", + "rapl", + "rdpid", + "rdpru", + "rdrand", + "rdrnd", + "rdseed", + "rdt_a", + "rdtscp", + "rep_good", + "sep", + "sev", + "sev_es", + "sha", + "sha_ni", + "skinit", + "smap", + "smca", + "smep", + "ssbd", + "sse", + "sse2", + "sse4_1", + "sse4_2", + "sse4a", + "ssse3", + "stibp", + "succor", + "svm", + "svm_lock", + "syscall", + "tce", + "topoext", + "tsc", + "tsc_scale", + "umip", + "v_spec_ctrl", + "v_vmsave_vmload", + "vgif", + "vmcb_clean", + "vme", + "vmmcall", + "wbnoinvd", + "wdt", + "x2apic", + "xgetbv1", + "xsave", + "xsavec", + "xsaveerptr", + "xsaveopt", + "xtopology" + ], + "l3_cache_size": 524288, + "l2_cache_size": 3145728, + "l1_data_cache_size": 196608, + "l1_instruction_cache_size": 196608, + "l2_cache_line_size": 512, + "l2_cache_associativity": 6 + } + }, + "commit_info": { + "id": "05c33b050b92b213cbd02a71ba63bcf6469fdfa7", + "time": "2026-07-30T13:03:55+01:00", + "author_time": "2026-07-30T13:03:55+01:00", + "dirty": true, + "project": "stageplayplain", + "branch": "master" + }, + "benchmarks": [ + { + "group": null, + "name": "test_parse_big_fish", + "fullname": "tests/perf/test_fountain_perf.py::test_parse_big_fish", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.02635018900036812, + "max": 0.034267336002812954, + "mean": 0.027683898942424484, + "stddev": 0.002088606934198818, + "rounds": 35, + "median": 0.026874704999499954, + "iqr": 0.0005243965024419595, + "q1": 0.026677764747546462, + "q3": 0.02720216124998842, + "iqr_outliers": 5, + "stddev_outliers": 5, + "outliers": "5;5", + "ld15iqr": 0.02635018900036812, + "hd15iqr": 0.03158795700073824, + "ops": 36.12207955533097, + "total": 0.9689364629848569, + "iterations": 1 + } + }, + { + "group": null, + "name": "test_parse_lines_big_fish", + "fullname": "tests/perf/test_fountain_perf.py::test_parse_lines_big_fish", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.025079914998059394, + "max": 0.0376777330020559, + "mean": 0.027860122417021356, + "stddev": 0.0031526301670151643, + "rounds": 36, + "median": 0.026500305501031107, + "iqr": 0.002220674499767483, + "q1": 0.02589295950019732, + "q3": 0.028113633999964804, + "iqr_outliers": 6, + "stddev_outliers": 6, + "outliers": "6;6", + "ld15iqr": 0.025079914998059394, + "hd15iqr": 0.032402565997472266, + "ops": 35.89359677002145, + "total": 1.0029644070127688, + "iterations": 1 + } + }, + { + "group": null, + "name": "test_parse_lines_large_document", + "fullname": "tests/perf/test_fountain_perf.py::test_parse_lines_large_document", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.27378064400181756, + "max": 0.30207301399786957, + "mean": 0.2912719439998909, + "stddev": 0.012344043391590424, + "rounds": 5, + "median": 0.29817993200049386, + "iqr": 0.01958478900269256, + "q1": 0.28056161149834224, + "q3": 0.3001464005010348, + "iqr_outliers": 0, + "stddev_outliers": 1, + "outliers": "1;0", + "ld15iqr": 0.27378064400181756, + "hd15iqr": 0.30207301399786957, + "ops": 3.433217721787768, + "total": 1.4563597199994547, + "iterations": 1 + } + } + ], + "datetime": "2026-07-31T15:25:01.929901+00:00", + "version": "5.2.3" +} \ No newline at end of file diff --git a/Makefile b/Makefile index abe7a29..eeaea2d 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,16 @@ .PHONY: test test: - uv run pytest --doctest-modules -W error + uv run pytest --doctest-modules -W error -m "not perf" uv run screenplain tests/files/simple.fountain /tmp/simple.pdf +# Benchmark the Fountain parser against examples/Big-Fish.fountain. +# Pass ARGS to save/compare runs, e.g. +# make perf ARGS=--benchmark-autosave +# make perf ARGS='--benchmark-compare --benchmark-compare-fail=mean:10%' +.PHONY: perf +perf: + uv run pytest tests/perf -m perf --benchmark-columns=min,mean,stddev,rounds $(ARGS) + .PHONY: lint lint: uvx prek run -a \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 16ef79b..e6bfef4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,11 +35,17 @@ screenplain = "screenplain.main:cli" [dependency-groups] dev = [ "pytest>=9.1.1", + "pytest-benchmark>=5.2.3", "ruff>=0.16.0", "setuptools>=82.0.1", "ty>=0.0.65", ] +[tool.pytest.ini_options] +markers = [ + "perf: performance benchmarks, excluded from the normal test run", +] + [tool.ruff] src = ["."] include = ["*.py"] diff --git a/tests/perf/__init__.py b/tests/perf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/perf/test_fountain_perf.py b/tests/perf/test_fountain_perf.py new file mode 100644 index 0000000..f0ee983 --- /dev/null +++ b/tests/perf/test_fountain_perf.py @@ -0,0 +1,70 @@ +# Copyright (c) 2011 Martin Vilcans +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license.php + +"""Performance benchmarks for the Fountain parser. + +Run with: + + make perf + +These are excluded from the normal test run (see the ``perf`` marker). +""" + +from io import StringIO +from pathlib import Path + +import pytest +from pytest_benchmark.fixture import BenchmarkFixture + +from screenplain.parsers import fountain + +EXAMPLE = Path(__file__).parent.parent.parent / "examples" / "Big-Fish.fountain" + +# Number of copies of the example used for the "large document" benchmark. +LARGE_MULTIPLIER = 10 + + +@pytest.fixture(scope="module") +def source() -> str: + return EXAMPLE.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def lines(source: str) -> list[str]: + """The example split into lines, as parse() would hand them to parse_lines.""" + return fountain.linebreak_re.split(fountain.boneyard_re.sub("", source)) + + +@pytest.fixture(scope="module") +def large_lines(lines: list[str]) -> list[str]: + """A body-only document repeated to get a workload larger than one screenplay. + + The title page is dropped from the copies so that the result is still a + single valid document instead of title page keys appearing mid-body. + """ + body = lines[lines.index("") :] + return lines + body * (LARGE_MULTIPLIER - 1) + + +@pytest.mark.perf +def test_parse_big_fish(benchmark: BenchmarkFixture, source: str) -> None: + """Benchmark the whole entry point: read, strip boneyard, split, parse.""" + screenplay = benchmark(lambda: fountain.parse(StringIO(source))) + assert len(screenplay.paragraphs) > 0 + + +@pytest.mark.perf +def test_parse_lines_big_fish(benchmark: BenchmarkFixture, lines: list[str]) -> None: + """Benchmark parse_lines alone, without the read/boneyard/split overhead.""" + screenplay = benchmark(lambda: fountain.parse_lines(list(lines))) + assert len(screenplay.paragraphs) > 0 + + +@pytest.mark.perf +def test_parse_lines_large_document( + benchmark: BenchmarkFixture, large_lines: list[str] +) -> None: + """Benchmark parse_lines on a document ~10x the size of one screenplay.""" + screenplay = benchmark(lambda: fountain.parse_lines(list(large_lines))) + assert len(screenplay.paragraphs) > 0 diff --git a/uv.lock b/uv.lock index bcaf788..610991f 100644 --- a/uv.lock +++ b/uv.lock @@ -164,6 +164,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -189,6 +198,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-benchmark" +version = "5.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" }, +] + [[package]] name = "reportlab" version = "4.4.7" @@ -238,6 +260,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "pytest" }, + { name = "pytest-benchmark" }, { name = "ruff" }, { name = "setuptools" }, { name = "ty" }, @@ -249,6 +272,7 @@ requires-dist = [{ name = "reportlab", specifier = ">=4.4.7" }] [package.metadata.requires-dev] dev = [ { name = "pytest", specifier = ">=9.1.1" }, + { name = "pytest-benchmark", specifier = ">=5.2.3" }, { name = "ruff", specifier = ">=0.16.0" }, { name = "setuptools", specifier = ">=82.0.1" }, { name = "ty", specifier = ">=0.0.65" },