diff --git a/README.md b/README.md index 12d5074..f413f5b 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,27 @@ new Benchmark() ->toConsole(); ``` +### Warm-up + +Use the `warmup` method to run each callback a few times before the measured iterations. Warm-up runs behave +identically to regular ones (`beforeEach` / `afterEach` are invoked, the progress bar advances), but their +time and memory are not included in the final statistics. + +This stabilizes results by absorbing one-time costs such as cold caches, JIT/opcache priming and lazy autoloading. + +Disabled by default. + +```php +use DragonCode\Benchmark\Benchmark; + +new Benchmark() + ->warmup(3) // run 3 warm-up iterations per callback before measuring + ->compare( + foo: fn () => /* some code */, + bar: fn () => /* some code */, + ); +``` + ### Round Precision Use the `round` method to set the number of decimal places in console output: diff --git a/src/Benchmark.php b/src/Benchmark.php index e2446ee..bb76e21 100644 --- a/src/Benchmark.php +++ b/src/Benchmark.php @@ -27,6 +27,8 @@ class Benchmark protected int $deviations = 1; + protected int $warmup = 0; + public function __construct( protected RunnerService $runner = new RunnerService, protected ViewService $view = new ViewService, @@ -57,6 +59,19 @@ public function snapshots(string $directory): static return $this; } + /** + * Specifies the number of warm-up iterations for each callback. + * + * @param int<1, max> $count + * @return $this + */ + public function warmup(int $count = 1): static + { + $this->warmup = max(1, abs($count)); + + return $this; + } + /** * Sets a callback to be executed before all iterations for each comparison. * @@ -292,7 +307,10 @@ protected function withProgress(Closure $callback, int $total): void */ protected function steps(array $callbacks, int $multiplier = 1): int { - return count($callbacks) * $this->iterations * $multiplier; + $count = count($callbacks); + $warmup = $count * $this->warmup; + + return $count * $this->iterations * $multiplier + $warmup; } /** @@ -321,14 +339,18 @@ protected function chunks(array $callbacks, ProgressBar $progressBar): void */ protected function run(mixed $name, Closure $callback, ProgressBar $progressBar): void { - for ($i = 1; $i <= $this->iterations; $i++) { + $warmedUp = $this->warmup === 0; + + for ($i = 1; $i <= $this->iterations + $this->warmup; $i++) { $result = $this->callbacks->performBeforeEach($name, $i); [$time, $memory] = $this->call($callback, [$i, $result]); $this->callbacks->performAfterEach($name, $i, $time, $memory); - $this->push($name, $time, $memory); + if ($warmedUp) { + $this->push($name, $time, $memory); + } $progressBar->advance(); } diff --git a/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___10__.snap b/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___10__.snap new file mode 100644 index 0000000..34b8915 --- /dev/null +++ b/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___10__.snap @@ -0,0 +1,16 @@ + + + 0/26 [░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 0% 1/26 [▓░░░░░░░░░░░░░░░░░░░░░░░░░░░] 3% 2/26 [▓▓░░░░░░░░░░░░░░░░░░░░░░░░░░] 7% 3/26 [▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░░] 11% 4/26 [▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░] 15% 5/26 [▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░░] 19% 6/26 [▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░] 23% 7/26 [▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░] 26% 8/26 [▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░] 30% 9/26 [▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░] 34% 10/26 [▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░] 38% 11/26 [▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░] 42% 12/26 [▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░] 46% 13/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░] 50% 14/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░] 53% 15/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░] 57% 16/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░] 61% 17/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░] 65% 18/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░] 69% 19/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░] 73% 20/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░] 76% 21/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░] 80% 22/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░] 84% 23/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░] 88% 24/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░] 92% 25/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░] 96% 26/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% 26/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% + + + ++-------+------------------------+------------------------+ +| # | 0 | 1 | ++-------+------------------------+------------------------+ +| min | 15.6789 ms - 202 bytes | 2.3457 ms - 102 bytes | +| max | 112.789 ms - 209 bytes | 9.7568 ms - 109 bytes | +| avg | 53.0252 ms - 205 bytes | 5.9429 ms - 105 bytes | +| total | 424.202 ms - 1.61 KB | 47.5432 ms - 844 bytes | ++-------+------------------------+------------------------+ +| order | 2 | 1 | ++-------+------------------------+------------------------+ diff --git a/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___1__.snap b/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___1__.snap new file mode 100644 index 0000000..bd2ccfb --- /dev/null +++ b/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___1__.snap @@ -0,0 +1,16 @@ + + + 0/8 [░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 0% 1/8 [▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░░] 12% 2/8 [▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░] 25% 3/8 [▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░] 37% 4/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░] 50% 5/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░] 62% 6/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░] 75% 7/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░] 87% 8/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% 8/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% + + + ++-------+------------------------+------------------------+ +| # | 0 | 1 | ++-------+------------------------+------------------------+ +| min | 15.6789 ms - 202 bytes | 2.3457 ms - 102 bytes | +| max | 112.789 ms - 209 bytes | 9.7568 ms - 109 bytes | +| avg | 53.0252 ms - 205 bytes | 5.9429 ms - 105 bytes | +| total | 424.202 ms - 1.61 KB | 47.5432 ms - 844 bytes | ++-------+------------------------+------------------------+ +| order | 2 | 1 | ++-------+------------------------+------------------------+ diff --git a/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___5__.snap b/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___5__.snap new file mode 100644 index 0000000..f070740 --- /dev/null +++ b/tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___5__.snap @@ -0,0 +1,16 @@ + + + 0/16 [░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 0% 1/16 [▓░░░░░░░░░░░░░░░░░░░░░░░░░░░] 6% 2/16 [▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░░] 12% 3/16 [▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░░] 18% 4/16 [▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░] 25% 5/16 [▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░] 31% 6/16 [▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░] 37% 7/16 [▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░] 43% 8/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░] 50% 9/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░] 56% 10/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░] 62% 11/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░] 68% 12/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░] 75% 13/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░] 81% 14/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░] 87% 15/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░] 93% 16/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% 16/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% + + + ++-------+------------------------+------------------------+ +| # | 0 | 1 | ++-------+------------------------+------------------------+ +| min | 15.6789 ms - 202 bytes | 2.3457 ms - 102 bytes | +| max | 112.789 ms - 209 bytes | 9.7568 ms - 109 bytes | +| avg | 53.0252 ms - 205 bytes | 5.9429 ms - 105 bytes | +| total | 424.202 ms - 1.61 KB | 47.5432 ms - 844 bytes | ++-------+------------------------+------------------------+ +| order | 2 | 1 | ++-------+------------------------+------------------------+ diff --git a/tests/Unit/WarmUp/WarmUpTest.php b/tests/Unit/WarmUp/WarmUpTest.php new file mode 100644 index 0000000..11ea0ce --- /dev/null +++ b/tests/Unit/WarmUp/WarmUpTest.php @@ -0,0 +1,28 @@ +iterations(3) + ->round(4) + ->warmup($warmup) + ->compare( + foo: function () use (&$count) { + $count++; + }, + bar: function () use (&$count) { + $count++; + }, + ) + ->toConsole(); + + expect($count)->toBe(6 + $warmup * 2); + + expectOutputToMatchSnapshot(); +})->with([1, 5, 10]);