From 6d9d94dc80474185116a896c1cd29421c25ec351 Mon Sep 17 00:00:00 2001 From: Dennis Snell Date: Tue, 24 Jan 2023 22:43:59 -0700 Subject: [PATCH 1/2] Perf Tests: Wild attempt at improving test determinism In this patch we're tossing in Chrome CLI args in an attempt to run things in a more deterministic manner, as well as trying to improve the timer precision, which is normally reduced as a mitigation against speculative execution attacks. The goal is to be able to run the performance tests CI workflow on a branch against itself or against another branch with a no-code change (such as a Markdown doc update) and end up with performance test results that are close enough to each other to be effectively equal. Currently the tests run with variation in the results that exceeds any actual variation between the branches, whereas we find statistical confidence that random noise is unlikely to account for the differences in the readings that we measure. Hopefully we can adjust some command-line arguments and figure out that some of them will help with the test reliability and we can add those to the repository. https://peter.sh/experiments/chromium-command-line-switches/ https://chromium.googlesource.com/v8/v8/+/master/src/flags/flag-definitions.h#188 --- .../jest-environment-puppeteer/config.js | 26 +++++++++++++++++++ packages/scripts/config/puppeteer.config.js | 25 ++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/packages/scripts/config/jest-environment-puppeteer/config.js b/packages/scripts/config/jest-environment-puppeteer/config.js index 12e9886800481c..07d322b20d67e4 100644 --- a/packages/scripts/config/jest-environment-puppeteer/config.js +++ b/packages/scripts/config/jest-environment-puppeteer/config.js @@ -38,6 +38,32 @@ const DEFAULT_CONFIG_CI = merge( DEFAULT_CONFIG, { '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-renderer-backgrounding', + '--disable-web-security', + '--bwsi', + '--deterministic-mode', + '--disable-back-forward-cache', + '--disable-background-networking', + '--disable-domain-reliability', + '--disable-extensions', + '--disable-hang-monitor', + '--disable-machine-cert-request', + '--disable-oobe-chromevox-hint-timer-for-testing', + '--disable-oobe-network-screen-skipping-for-testing', + '--disable-popup-blocking', + '--disable-pushstate-throttle', + '--disable-skia-runtime-opts', + '--disable-timeouts-for-profiling', + '--disable-v8-idle-tasks', + '--disable-vsync-for-tests', + // '--enable-low-end-device-mode', + '--enable-precise-memory-info', + '--enable-features=SharedArrayBuffer', + '--js-flags="--gc_experiment_less_compaction --expose-gc --sampling_heap_profiler_suppress_randomness --predictable --predictable_gc_schedule --single_threaded"', + '--no-pings', + '--single-process', + '--isolation-by-default', // reducedPrecisionTimers? + '--site-per-process', // reducedPrecisionTimers? + '--shared-array-buffer-allowed-origins="localhost,localhost:8888,localhost:8889"', ], }, } ); diff --git a/packages/scripts/config/puppeteer.config.js b/packages/scripts/config/puppeteer.config.js index dc222b0e8d1caf..0065e230b2932c 100644 --- a/packages/scripts/config/puppeteer.config.js +++ b/packages/scripts/config/puppeteer.config.js @@ -6,6 +6,31 @@ module.exports = { args: [ '--enable-blink-features=ComputedAccessibilityInfo', '--disable-web-security', + '--bwsi', + '--deterministic-mode', + '--disable-back-forward-cache', + '--disable-background-networking', + '--disable-domain-reliability', + '--disable-extensions', + '--disable-hang-monitor', + '--disable-machine-cert-request', + '--disable-oobe-chromevox-hint-timer-for-testing', + '--disable-oobe-network-screen-skipping-for-testing', + '--disable-popup-blocking', + '--disable-pushstate-throttle', + '--disable-skia-runtime-opts', + '--disable-timeouts-for-profiling', + '--disable-v8-idle-tasks', + '--disable-vsync-for-tests', + // '--enable-low-end-device-mode', + '--enable-precise-memory-info', + '--enable-features=SharedArrayBuffer', + '--js-flags="--gc_experiment_less_compaction --expose-gc --sampling_heap_profiler_suppress_randomness --predictable --predictable_gc_schedule --single_threaded"', + '--no-pings', + '--single-process', + '--isolation-by-default', // reducedPrecisionTimers? + '--site-per-process', // reducedPrecisionTimers? + '--shared-array-buffer-allowed-origins="localhost,localhost:8888,localhost:8889"', ], }, }; From 9c5b722fa047f83c5b2ad6dd622006247d79bca3 Mon Sep 17 00:00:00 2001 From: Dennis Snell Date: Tue, 7 Feb 2023 14:23:09 -0700 Subject: [PATCH 2/2] Try these switches --- .../config/jest-environment-puppeteer/config.js | 16 ---------------- packages/scripts/config/puppeteer.config.js | 15 --------------- 2 files changed, 31 deletions(-) diff --git a/packages/scripts/config/jest-environment-puppeteer/config.js b/packages/scripts/config/jest-environment-puppeteer/config.js index 07d322b20d67e4..12363d93fe7346 100644 --- a/packages/scripts/config/jest-environment-puppeteer/config.js +++ b/packages/scripts/config/jest-environment-puppeteer/config.js @@ -38,32 +38,16 @@ const DEFAULT_CONFIG_CI = merge( DEFAULT_CONFIG, { '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-renderer-backgrounding', - '--disable-web-security', '--bwsi', - '--deterministic-mode', '--disable-back-forward-cache', - '--disable-background-networking', '--disable-domain-reliability', '--disable-extensions', - '--disable-hang-monitor', - '--disable-machine-cert-request', - '--disable-oobe-chromevox-hint-timer-for-testing', - '--disable-oobe-network-screen-skipping-for-testing', '--disable-popup-blocking', - '--disable-pushstate-throttle', - '--disable-skia-runtime-opts', '--disable-timeouts-for-profiling', - '--disable-v8-idle-tasks', '--disable-vsync-for-tests', // '--enable-low-end-device-mode', - '--enable-precise-memory-info', - '--enable-features=SharedArrayBuffer', '--js-flags="--gc_experiment_less_compaction --expose-gc --sampling_heap_profiler_suppress_randomness --predictable --predictable_gc_schedule --single_threaded"', - '--no-pings', '--single-process', - '--isolation-by-default', // reducedPrecisionTimers? - '--site-per-process', // reducedPrecisionTimers? - '--shared-array-buffer-allowed-origins="localhost,localhost:8888,localhost:8889"', ], }, } ); diff --git a/packages/scripts/config/puppeteer.config.js b/packages/scripts/config/puppeteer.config.js index 0065e230b2932c..388f49204bd352 100644 --- a/packages/scripts/config/puppeteer.config.js +++ b/packages/scripts/config/puppeteer.config.js @@ -7,30 +7,15 @@ module.exports = { '--enable-blink-features=ComputedAccessibilityInfo', '--disable-web-security', '--bwsi', - '--deterministic-mode', '--disable-back-forward-cache', - '--disable-background-networking', '--disable-domain-reliability', '--disable-extensions', - '--disable-hang-monitor', - '--disable-machine-cert-request', - '--disable-oobe-chromevox-hint-timer-for-testing', - '--disable-oobe-network-screen-skipping-for-testing', '--disable-popup-blocking', - '--disable-pushstate-throttle', - '--disable-skia-runtime-opts', '--disable-timeouts-for-profiling', - '--disable-v8-idle-tasks', '--disable-vsync-for-tests', // '--enable-low-end-device-mode', - '--enable-precise-memory-info', - '--enable-features=SharedArrayBuffer', '--js-flags="--gc_experiment_less_compaction --expose-gc --sampling_heap_profiler_suppress_randomness --predictable --predictable_gc_schedule --single_threaded"', - '--no-pings', '--single-process', - '--isolation-by-default', // reducedPrecisionTimers? - '--site-per-process', // reducedPrecisionTimers? - '--shared-array-buffer-allowed-origins="localhost,localhost:8888,localhost:8889"', ], }, };