From 9e1e642abbe5648d3fe43259ff372176f0628066 Mon Sep 17 00:00:00 2001 From: Paul Kilmurray Date: Fri, 19 Jun 2026 10:24:55 +0200 Subject: [PATCH 1/3] fix(analytics): bootstrap landing-variant server-side, stop early identify The landing-variant A/B test was stuck on the free-plus fallback for every visitor. Two root causes: 1. The self-hosted PostHog /flags endpoint rejects the public project token (401) while /capture works, so posthog-js never resolves the flag over the network (zero render_source:flag events). The infra needs a separate fix. 2. The plugin's early posthog.init + identify() on the welcome page wrote the bundle's shared `ph__posthog` localStorage identity first, flipping the bundle's hasPersistedIdentity() gate and breaking the deliberate flag-before-identify exposure ordering. Fix (plugin side): - Add Services\Feature_Flags: resolve landing-variant via PostHog's standard consistent-hash locally (no network, no secret), so assignment matches the server SDK's get_all_feature_flags() and works regardless of the broken /flags endpoint. - Landing_Profile: inject `bootstrap_flags` into the always-present functional data (anon, consent-independent) so the bundle can seed posthog.init bootstrap.featureFlags at first paint. - Menu: stop initialising PostHog / identifying on the welcome page and remove the redundant DOM CTA tracker (it double-fired upgrade_cta_clicked against the bundle's own React tracking). The bundle now solely owns PostHog init + identify ordering. Companion bundle change required: wcpos/wp-admin-landing#39 (read bootstrap_flags into bootstrap.featureFlags). This PR is necessary but the experiment only goes green once both ship. Refs #1188 --- includes/Admin/Menu.php | 41 +---- includes/Services/Feature_Flags.php | 159 ++++++++++++++++++ includes/Services/Landing_Profile.php | 18 +- .../Admin/Test_Menu_Landing_Bootstrap.php | 114 +++++++++++++ .../includes/Services/Test_Feature_Flags.php | 141 ++++++++++++++++ .../Services/Test_Landing_Profile.php | 29 ++++ 6 files changed, 461 insertions(+), 41 deletions(-) create mode 100644 includes/Services/Feature_Flags.php create mode 100644 tests/includes/Admin/Test_Menu_Landing_Bootstrap.php create mode 100644 tests/includes/Services/Test_Feature_Flags.php diff --git a/includes/Admin/Menu.php b/includes/Admin/Menu.php index fb0f53b2..693e92bf 100644 --- a/includes/Admin/Menu.php +++ b/includes/Admin/Menu.php @@ -277,9 +277,13 @@ public function enqueue_landing_scripts_and_styles( $hook_suffix ): void { true ); + // Inject functional data (locale, version, anon_id, experiment + // bootstrap flags). The landing bundle owns PostHog initialisation, + // flag resolution, and identify ordering (flag-before-identify, spec + // §5.1); the plugin must NOT init PostHog or identify here — doing so + // shares the bundle's localStorage identity, breaks the anon-bucket + // exposure, and double-fires CTA tracking. wp_add_inline_script( 'wcpos-welcome', $this->landing_inline_script(), 'before' ); - wp_add_inline_script( 'wcpos-welcome', self::get_posthog_inline_script(), 'before' ); - wp_add_inline_script( 'wcpos-welcome', $this->landing_tracking_inline_script(), 'after' ); } } @@ -430,39 +434,6 @@ private function landing_inline_script(): string { ); } - /** - * Track clicks from the remote upgrade landing content. - * - * @return string - */ - private function landing_tracking_inline_script(): string { - return << + */ + const LANDING_VARIANTS = array( + array( + 'key' => 'indie', + 'rollout' => 50, + ), + array( + 'key' => 'free-plus', + 'rollout' => 50, + ), + ); + + /** + * Build the feature-flag bootstrap map for the landing page. + * + * Returns a map of resolved flags keyed for PostHog's `bootstrap.featureFlags` + * (e.g. `array( 'landing-variant' => 'indie' )`). Empty when no distinct id is + * available, in which case the bundle resolves the variant itself. + * + * @param string $distinct_id Stable anonymous identifier for the visitor/site. + * + * @return array + */ + public function get_landing_bootstrap_flags( string $distinct_id ): array { + $flags = array(); + $variant = $this->get_landing_variant( $distinct_id ); + + if ( null !== $variant ) { + $flags[ self::LANDING_FLAG_KEY ] = $variant; + } + + /** + * Filters the feature flags bootstrapped into the landing page. + * + * Allows forcing a variant (for QA) or extending the bootstrap with + * additional flags. Values must be strings or booleans, matching + * PostHog's `bootstrap.featureFlags` contract. + * + * @since 1.9.7 + * + * @param array $flags Resolved flag map. + * @param string $distinct_id The visitor distinct id. + */ + return apply_filters( 'woocommerce_pos_landing_bootstrap_flags', $flags, $distinct_id ); + } + + /** + * Resolve the `landing-variant` value for a distinct id. + * + * @param string $distinct_id Stable anonymous identifier. + * + * @return null|string The variant key, or null when no distinct id is given. + */ + public function get_landing_variant( string $distinct_id ): ?string { + if ( '' === $distinct_id ) { + return null; + } + + return $this->match_variant( self::LANDING_FLAG_KEY, $distinct_id, self::LANDING_VARIANTS ); + } + + /** + * Map a distinct id to a variant using PostHog's consistent-hash lookup. + * + * The variant hash is salted with `variant` and compared against the + * cumulative rollout ranges built from the variant order. + * + * @param string $key Flag key. + * @param string $distinct_id Distinct id. + * @param array $variants Ordered variants. + * + * @return null|string + */ + private function match_variant( string $key, string $distinct_id, array $variants ): ?string { + $hash_value = $this->hash( $key, $distinct_id, 'variant' ); + + $value_min = 0.0; + foreach ( $variants as $variant ) { + $value_max = $value_min + ( $variant['rollout'] / 100 ); + if ( $hash_value >= $value_min && $hash_value < $value_max ) { + return $variant['key']; + } + $value_min = $value_max; + } + + return null; + } + + /** + * PostHog consistent hash: SHA-1 of `key.distinct_id+salt`, first 15 hex + * digits scaled to a deterministic [0, 1) float. + * + * @param string $key Flag key. + * @param string $distinct_id Distinct id. + * @param string $salt Hash salt (`variant` for variant selection). + * + * @return float + */ + private function hash( string $key, string $distinct_id, string $salt = '' ): float { + $hex = substr( sha1( $key . '.' . $distinct_id . $salt ), 0, 15 ); + + return (float) ( hexdec( $hex ) / self::LONG_SCALE ); + } +} diff --git a/includes/Services/Landing_Profile.php b/includes/Services/Landing_Profile.php index 633ee9e9..e992f458 100644 --- a/includes/Services/Landing_Profile.php +++ b/includes/Services/Landing_Profile.php @@ -46,17 +46,23 @@ class Landing_Profile { /** * Get functional data that is always available (no consent required). * - * Used for translations, feature gating, and schema versioning. + * Used for translations, feature gating, and schema versioning. Includes the + * anonymous identity and the server-resolved experiment flags so the landing + * bundle can bootstrap its A/B variant at first paint without a network flag + * fetch (landing-experiments spec §5.1). * * @return array */ public function get_functional_data(): array { + $anon_id = ( new Anon_ID() )->get(); + return array( - 'schema_version' => 2, // bumped: anon_id added (landing-experiments spec §5.1). - 'locale' => get_locale(), - 'plugin_version' => PLUGIN_VERSION, - 'pro_active' => class_exists( '\WCPOS\WooCommercePOSPro\WooCommercePOSPro' ), - 'anon_id' => ( new Anon_ID() )->get(), + 'schema_version' => 2, // bumped: anon_id added (landing-experiments spec §5.1). + 'locale' => get_locale(), + 'plugin_version' => PLUGIN_VERSION, + 'pro_active' => class_exists( '\WCPOS\WooCommercePOSPro\WooCommercePOSPro' ), + 'anon_id' => $anon_id, + 'bootstrap_flags' => ( new Feature_Flags() )->get_landing_bootstrap_flags( $anon_id ), ); } diff --git a/tests/includes/Admin/Test_Menu_Landing_Bootstrap.php b/tests/includes/Admin/Test_Menu_Landing_Bootstrap.php new file mode 100644 index 00000000..913a2635 --- /dev/null +++ b/tests/includes/Admin/Test_Menu_Landing_Bootstrap.php @@ -0,0 +1,114 @@ +user->create( array( 'role' => 'administrator' ) ); + wp_set_current_user( $user_id ); + wp_get_current_user()->add_cap( 'manage_woocommerce_pos' ); + + // Consent ON: under the old code this is exactly when the plugin emitted + // a real posthog.init + identify, so the regression assertions below are + // meaningful rather than vacuous. + $settings = (array) woocommerce_pos_get_settings( 'general' ); + $settings['tracking_consent'] = 'allowed'; + SettingsService::instance()->save_settings( 'general', $settings ); + + $this->menu = new Menu(); + } + + /** + * Tear down test fixtures. + */ + public function tearDown(): void { + remove_filter( 'pre_http_request', '__return_empty_array' ); + Analytics::reset_instance(); + wp_set_current_user( 0 ); + wp_deregister_script( 'wcpos-welcome' ); + + parent::tearDown(); + } + + /** + * Reads the inline scripts attached to the landing bundle at a position. + * + * @param string $position Either `before` or `after`. + * + * @return string Concatenated inline script source. + */ + private function inline_scripts( string $position ): string { + $data = wp_scripts()->get_data( 'wcpos-welcome', $position ); + + return \is_array( $data ) ? implode( "\n", $data ) : (string) $data; + } + + /** + * The landing page must still inject the functional data block so the bundle + * can read locale/version/anon_id/bootstrap_flags. + */ + public function test_landing_page_injects_functional_data(): void { + $this->menu->enqueue_landing_scripts_and_styles( $this->menu->toplevel_screen_id ); + + $this->assertStringContainsString( 'wcpos.landing', $this->inline_scripts( 'before' ) ); + } + + /** + * The plugin must NOT initialise PostHog on the landing page — the bundle + * owns init/identify (flag-before-identify). An early init here shares the + * bundle's localStorage identity and breaks anon-bucket exposure. + */ + public function test_landing_page_does_not_initialise_posthog(): void { + $this->menu->enqueue_landing_scripts_and_styles( $this->menu->toplevel_screen_id ); + + $before = $this->inline_scripts( 'before' ); + $this->assertStringNotContainsString( 'posthog.init', $before ); + $this->assertStringNotContainsString( 'posthog.identify', $before ); + } + + /** + * The redundant DOM-level CTA tracker (which depended on window.wcpos.posthog + * and double-fired against the bundle's own React tracking) must be gone. + */ + public function test_landing_page_has_no_duplicate_cta_tracking(): void { + $this->menu->enqueue_landing_scripts_and_styles( $this->menu->toplevel_screen_id ); + + $this->assertStringNotContainsString( 'upgrade_cta_clicked', $this->inline_scripts( 'after' ) ); + } +} diff --git a/tests/includes/Services/Test_Feature_Flags.php b/tests/includes/Services/Test_Feature_Flags.php new file mode 100644 index 00000000..aa40e100 --- /dev/null +++ b/tests/includes/Services/Test_Feature_Flags.php @@ -0,0 +1,141 @@ +flags = new Feature_Flags(); + } + + /** + * The same distinct id must always resolve to the same variant. + */ + public function test_get_landing_variant_is_deterministic_for_a_given_id(): void { + $id = '8c2f7e10-1234-4abc-89de-0123456789ab'; + + $this->assertSame( + $this->flags->get_landing_variant( $id ), + $this->flags->get_landing_variant( $id ) + ); + } + + /** + * Local assignment must match PostHog's consistent-hash algorithm, so that + * `get_all_feature_flags()` (server-side local eval) agrees with the + * bootstrapped value. Vectors generated from the reference implementation. + */ + public function test_get_landing_variant_matches_posthog_reference_vectors(): void { + $vectors = array( + 'a1b2c3d4-0000-4000-8000-000000000001' => 'indie', + '11111111-1111-4111-8111-111111111111' => 'free-plus', + 'test-distinct-id' => 'indie', + 'wcpos' => 'free-plus', + '00000000-0000-4000-8000-000000000000' => 'free-plus', + 'f47ac10b-58cc-4372-a567-0e02b2c3d479' => 'free-plus', + ); + + foreach ( $vectors as $distinct_id => $expected ) { + $this->assertSame( + $expected, + $this->flags->get_landing_variant( $distinct_id ), + "Variant mismatch for distinct id {$distinct_id}" + ); + } + } + + /** + * The flag is only ever assigned one of the two configured variants. + */ + public function test_get_landing_variant_only_returns_configured_variants(): void { + for ( $i = 0; $i < 100; $i++ ) { + $variant = $this->flags->get_landing_variant( wp_generate_uuid4() ); + $this->assertContains( $variant, array( 'indie', 'free-plus' ) ); + } + } + + /** + * Traffic must split roughly 50/50 across many anonymous visitors. + */ + public function test_get_landing_variant_splits_traffic_roughly_evenly(): void { + $counts = array( + 'indie' => 0, + 'free-plus' => 0, + ); + + $total = 2000; + for ( $i = 0; $i < $total; $i++ ) { + $variant = $this->flags->get_landing_variant( wp_generate_uuid4() ); + ++$counts[ $variant ]; + } + + // Generous bounds (40%–60%) so the assertion never flakes while still + // catching a broken split (e.g. everyone landing on one arm). + $this->assertGreaterThan( $total * 0.4, $counts['indie'], 'indie share too low' ); + $this->assertLessThan( $total * 0.6, $counts['indie'], 'indie share too high' ); + $this->assertSame( $total, $counts['indie'] + $counts['free-plus'] ); + } + + /** + * An empty distinct id yields no assignment (the bundle resolves it instead). + */ + public function test_get_landing_variant_returns_null_for_empty_id(): void { + $this->assertNull( $this->flags->get_landing_variant( '' ) ); + } + + /** + * The bootstrap map carries the landing-variant flag keyed for PostHog. + */ + public function test_get_landing_bootstrap_flags_contains_landing_variant(): void { + $flags = $this->flags->get_landing_bootstrap_flags( 'test-distinct-id' ); + + $this->assertArrayHasKey( 'landing-variant', $flags ); + $this->assertSame( 'indie', $flags['landing-variant'] ); + } + + /** + * No distinct id means an empty bootstrap map (no bootstrap injected). + */ + public function test_get_landing_bootstrap_flags_is_empty_for_empty_id(): void { + $this->assertSame( array(), $this->flags->get_landing_bootstrap_flags( '' ) ); + } + + /** + * The bootstrap map is filterable for QA / kill-switch overrides. + */ + public function test_bootstrap_flags_filter_can_override_variant(): void { + $override = static function (): array { + return array( 'landing-variant' => 'free-plus' ); + }; + add_filter( 'woocommerce_pos_landing_bootstrap_flags', $override ); + + $flags = $this->flags->get_landing_bootstrap_flags( 'test-distinct-id' ); + + remove_filter( 'woocommerce_pos_landing_bootstrap_flags', $override ); + + $this->assertSame( 'free-plus', $flags['landing-variant'] ); + } +} diff --git a/tests/includes/Services/Test_Landing_Profile.php b/tests/includes/Services/Test_Landing_Profile.php index e48ece74..bd3774d4 100644 --- a/tests/includes/Services/Test_Landing_Profile.php +++ b/tests/includes/Services/Test_Landing_Profile.php @@ -10,6 +10,7 @@ use Automattic\WooCommerce\RestApi\UnitTests\Helpers\OrderHelper; use Automattic\WooCommerce\Utilities\OrderUtil; use ReflectionMethod; +use WCPOS\WooCommercePOS\Services\Feature_Flags; use WCPOS\WooCommercePOS\Services\Landing_Profile; use WP_UnitTestCase; @@ -156,4 +157,32 @@ public function test_functional_data_anon_id_is_stable_across_pageloads(): void $profile->get_functional_data()['anon_id'] ); } + + /** + * Functional data must carry the server-resolved experiment bootstrap flags + * so the bundle can seed PostHog at first paint (no client flag fetch). + */ + public function test_functional_data_carries_landing_variant_bootstrap_flag(): void { + $profile = new Landing_Profile(); + $data = $profile->get_functional_data(); + + $this->assertArrayHasKey( 'bootstrap_flags', $data ); + $this->assertArrayHasKey( 'landing-variant', $data['bootstrap_flags'] ); + $this->assertContains( $data['bootstrap_flags']['landing-variant'], array( 'indie', 'free-plus' ) ); + } + + /** + * The bootstrapped variant must equal what the resolver assigns for the + * payload's own anon_id — the client buckets on that id, so a mismatch would + * corrupt assignment. + */ + public function test_functional_data_bootstrap_variant_matches_anon_id_assignment(): void { + $profile = new Landing_Profile(); + $data = $profile->get_functional_data(); + + $this->assertSame( + ( new Feature_Flags() )->get_landing_variant( $data['anon_id'] ), + $data['bootstrap_flags']['landing-variant'] + ); + } } From 36e7306601531f7d2fe25aca966c712faf111221 Mon Sep 17 00:00:00 2001 From: "wcpos-agents[bot]" <2860316+wcpos-agents[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:51:04 +0000 Subject: [PATCH 2/3] fix: restore landing bootstrap test settings --- tests/includes/Admin/Test_Menu_Landing_Bootstrap.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/includes/Admin/Test_Menu_Landing_Bootstrap.php b/tests/includes/Admin/Test_Menu_Landing_Bootstrap.php index 913a2635..a0d78fa6 100644 --- a/tests/includes/Admin/Test_Menu_Landing_Bootstrap.php +++ b/tests/includes/Admin/Test_Menu_Landing_Bootstrap.php @@ -25,6 +25,13 @@ class Test_Menu_Landing_Bootstrap extends WP_UnitTestCase { */ private $menu; + /** + * Original general settings snapshot for restoration. + * + * @var array + */ + private $original_general_settings = array(); + /** * Set up test fixtures. */ @@ -47,7 +54,8 @@ public function setUp(): void { // Consent ON: under the old code this is exactly when the plugin emitted // a real posthog.init + identify, so the regression assertions below are // meaningful rather than vacuous. - $settings = (array) woocommerce_pos_get_settings( 'general' ); + $this->original_general_settings = (array) woocommerce_pos_get_settings( 'general' ); + $settings = $this->original_general_settings; $settings['tracking_consent'] = 'allowed'; SettingsService::instance()->save_settings( 'general', $settings ); @@ -58,6 +66,7 @@ public function setUp(): void { * Tear down test fixtures. */ public function tearDown(): void { + SettingsService::instance()->save_settings( 'general', $this->original_general_settings ); remove_filter( 'pre_http_request', '__return_empty_array' ); Analytics::reset_instance(); wp_set_current_user( 0 ); From fb8a325276fcb548333e1bf19889e29f9f8c74a3 Mon Sep 17 00:00:00 2001 From: "wcpos-agents[bot]" <2860316+wcpos-agents[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:51:40 +0000 Subject: [PATCH 3/3] fix: clean up feature flag test filter --- tests/includes/Services/Test_Feature_Flags.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/includes/Services/Test_Feature_Flags.php b/tests/includes/Services/Test_Feature_Flags.php index aa40e100..9b20375b 100644 --- a/tests/includes/Services/Test_Feature_Flags.php +++ b/tests/includes/Services/Test_Feature_Flags.php @@ -132,9 +132,11 @@ public function test_bootstrap_flags_filter_can_override_variant(): void { }; add_filter( 'woocommerce_pos_landing_bootstrap_flags', $override ); - $flags = $this->flags->get_landing_bootstrap_flags( 'test-distinct-id' ); - - remove_filter( 'woocommerce_pos_landing_bootstrap_flags', $override ); + try { + $flags = $this->flags->get_landing_bootstrap_flags( 'test-distinct-id' ); + } finally { + remove_filter( 'woocommerce_pos_landing_bootstrap_flags', $override ); + } $this->assertSame( 'free-plus', $flags['landing-variant'] ); }