test(ffe): use canonical fixtures

.github/CODEOWNERS

@@ -12,6 +12,7 @@
 .dockerignore                        @DataDog/libdatadog-core
 .github/                             @DataDog/apm-common-components-core
 .gitignore                           @DataDog/libdatadog
+.gitmodules                          @DataDog/libdatadog
 .gitlab-ci.yml                       @DataDog/apm-common-components-core
 .gitlab/benchmarks.yml               @DataDog/apm-common-components-core
 .gitlab/fuzz.yml                     @DataDog/chaos-platform
@@ -25,6 +26,7 @@ cmake/                               @DataDog/apm-common-components-core
 CONTRIBUTING.md                      @DataDog/libdatadog-core
 Cross.toml                           @DataDog/apm-common-components-core
 datadog-ffe                          @DataDog/feature-flagging-and-experimentation-sdk
+datadog-ffe-test-suite               @DataDog/feature-flagging-and-experimentation-sdk
 datadog-ffe-ffi                      @DataDog/feature-flagging-and-experimentation-sdk
 datadog-ipc*/                        @DataDog/libdatadog-php
 datadog-live-debugger*/              @DataDog/libdatadog-php @DataDog/libdatadog-apm

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "gitsubmodule"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/test-ffi.yml

@@ -78,6 +78,8 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        with:
+          submodules: recursive
       - name: Setup output dir
         shell: bash
         run: |
@@ -224,6 +226,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        with:
+          submodules: recursive
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # 3.10.0
         with:

.gitlab/benchmarks.yml

@@ -17,6 +17,29 @@ benchmarks:
     - export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :)
     - git clone --branch libdatadog/benchmarks https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform /platform && cd /platform
     - ./steps/capture-hardware-software-info.sh
+    - |
+      SCRIPT_DIR="$(pwd)/steps"
+      source ./steps/config-benchmark-env.sh
+
+      checkout_source_with_submodules() {
+        local repo_url=$1
+        local dest=$2
+        local target=$3
+
+        if [[ -d "${dest}" ]]; then
+          return
+        fi
+
+        git clone "${repo_url}" "${dest}"
+        git -C "${dest}" checkout "${target}"
+        git -C "${dest}" submodule update --init --recursive
+      }
+
+      checkout_source_with_submodules "${UPSTREAM_REPO_URL}" "${CANDIDATE_PATH}" "${CANDIDATE_IDENTIFIER}"
+
+      # Refresh benchmark-platform's derived baseline after candidate checkout.
+      VERBOSE="true" source ./steps/config-benchmark-env.sh
+      checkout_source_with_submodules "${UPSTREAM_REPO_URL}" "${BASELINE_PATH}" "${BASELINE_IDENTIFIER}"
     - ./steps/run-benchmarks.sh
     - ./steps/analyze-results.sh
     - "./steps/upload-results-to-s3.sh || :"

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "datadog-ffe-test-suite/ffe-system-test-data"]
+	path = datadog-ffe-test-suite/ffe-system-test-data
+	url = https://github.com/DataDog/ffe-system-test-data.git

Cargo.toml

@@ -8,6 +8,7 @@ members = [
   "libdd-crashtracker",
   "libdd-crashtracker-ffi",
   "datadog-ffe",
+  "datadog-ffe-test-suite",
   "datadog-ffe-ffi",
   "datadog-ipc",
   "datadog-ipc-macros",

README.md

@@ -28,6 +28,14 @@ See [`CONTRIBUTING.md`](CONTRIBUTING.md).
 
 Build `libdatadog` as usual with `cargo build`.
 
+This repository uses git submodules for shared test data. If tests that depend
+on fixture data fail because files are missing, initialize submodules from the
+repository root:
+
+```bash
+git submodule update --init --recursive
+```
+
 #### Builder crate
 
 You can generate a release using the builder crate. This will trigger all the necessary steps to create the libraries, binaries, headers and package config files needed to use a pre-built libdatadog binary in a (non-rust) project.

datadog-ffe-test-suite/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "datadog-ffe-test-suite"
+version.workspace = true
+rust-version.workspace = true
+edition.workspace = true
+license.workspace = true
+autobenches = false
+publish = false
+
+[lib]
+bench = false
+
+[dev-dependencies]
+datadog-ffe = { path = "../datadog-ffe" }
+chrono = { version = "0.4.38", default-features = false, features = ["now", "serde"] }
+criterion = { version = "0.5", features = ["html_reports"] }
+env_logger = "0.10"
+serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
+serde_json = { version = "1.0", default-features = false, features = ["std", "raw_value"] }
+
+[[bench]]
+name = "ffe-eval"
+harness = false
+path = "benches/eval.rs"

datadog-ffe/benches/eval.rs → datadog-ffe-test-suite/benches/eval.rs

@@ -10,8 +10,17 @@ use datadog_ffe::rules_based::{
     UniversalFlagConfig,
 };
 
+const UFC_CONFIG_PATH: &str = concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/ffe-system-test-data/ufc-config.json"
+);
+const EVALUATION_CASES_DIR: &str = concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/ffe-system-test-data/evaluation-cases"
+);
+
 fn load_configuration_bytes() -> Vec<u8> {
-    fs::read("tests/data/flags-v1.json").expect("Failed to read flags-v1.json")
+    fs::read(UFC_CONFIG_PATH).expect("Failed to read ufc-config.json")
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -33,7 +42,7 @@ struct TestResult {
 fn load_test_cases() -> Vec<TestCase> {
     let mut test_cases = Vec::new();
 
-    if let Ok(entries) = fs::read_dir("tests/data/tests") {
+    if let Ok(entries) = fs::read_dir(EVALUATION_CASES_DIR) {
         for entry in entries.flatten() {
             if let Some(path_str) = entry.path().to_str() {
                 if path_str.ends_with(".json") {

datadog-ffe-test-suite/src/lib.rs

@@ -0,0 +1,4 @@
+// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+//! Internal test-suite crate for datadog-ffe.

datadog-ffe-test-suite/tests/canonical_fixtures.rs

@@ -0,0 +1,81 @@
+// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{collections::HashMap, fs, path::PathBuf, sync::Arc};
+
+use chrono::Utc;
+use datadog_ffe::rules_based::{
+    get_assignment, Attribute, Configuration, EvaluationContext, FlagType, Str, UniversalFlagConfig,
+};
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct TestCase {
+    flag: String,
+    variation_type: FlagType,
+    default_value: serde_json::Value,
+    targeting_key: Option<Str>,
+    attributes: Arc<HashMap<Str, Attribute>>,
+    result: TestResult,
+}
+
+#[derive(Debug, Deserialize)]
+struct TestResult {
+    value: serde_json::Value,
+}
+
+fn fixture_root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("ffe-system-test-data")
+}
+
+#[test]
+#[cfg_attr(miri, ignore)] // this test is too slow on miri
+fn evaluates_canonical_json_fixtures() {
+    let _ = env_logger::builder().is_test(true).try_init();
+
+    let root = fixture_root();
+    let config_path = root.join("ufc-config.json");
+    let cases_dir = root.join("evaluation-cases");
+
+    let config = UniversalFlagConfig::from_json(fs::read(&config_path).unwrap()).unwrap();
+    let config = Configuration::from_server_response(config);
+    let now = Utc::now();
+
+    let mut fixture_count = 0;
+    for entry in fs::read_dir(&cases_dir).unwrap() {
+        let path = entry.unwrap().path();
+        if path.extension().and_then(|ext| ext.to_str()) != Some("json") {
+            continue;
+        }
+
+        fixture_count += 1;
+        let test_cases: Vec<TestCase> = serde_json::from_reader(fs::File::open(&path).unwrap())
+            .unwrap_or_else(|err| panic!("failed to parse fixture {}: {err}", path.display()));
+
+        for test_case in test_cases {
+            let subject = EvaluationContext::new(test_case.targeting_key, test_case.attributes);
+            let result = get_assignment(
+                Some(&config),
+                &test_case.flag,
+                &subject,
+                test_case.variation_type.into(),
+                now,
+            );
+
+            let actual = result
+                .map(|assignment| assignment.value.variation_value())
+                .unwrap_or(test_case.default_value);
+
+            assert_eq!(
+                actual,
+                test_case.result.value,
+                "unexpected value for flag {} in {}",
+                test_case.flag,
+                path.display()
+            );
+        }
+    }
+
+    assert!(fixture_count > 0, "no canonical FFE fixtures loaded");
+}

datadog-ffe/Cargo.toml

@@ -26,14 +26,5 @@ thiserror = { version = "2.0.3", default-features = false }
 url = { version = "2.5.0", default-features = false, features = ["std"] }
 pyo3 = { version = "0.28", optional = true, default-features = false, features = ["macros"] }
 
-[dev-dependencies]
-env_logger = "0.10"
-criterion = { version = "0.5", features = ["html_reports"] }
-
-[[bench]]
-name = "ffe-eval"
-harness = false
-path = "benches/eval.rs"
-
 [features]
 pyo3 = ["dep:pyo3"]

datadog-ffe/src/rules_based/eval/eval_assignment.rs

@@ -198,88 +198,3 @@ impl Shard {
         self.ranges.iter().any(|range| range.contains(h))
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use std::{
-        collections::HashMap,
-        fs::{self, File},
-        sync::Arc,
-    };
-
-    use chrono::Utc;
-    use serde::{Deserialize, Serialize};
-
-    use crate::rules_based::{
-        eval::get_assignment,
-        ufc::{AssignmentValue, UniversalFlagConfig},
-        Attribute, Configuration, EvaluationContext, FlagType, Str,
-    };
-
-    #[derive(Debug, Serialize, Deserialize)]
-    #[serde(rename_all = "camelCase")]
-    struct TestCase {
-        flag: String,
-        variation_type: FlagType,
-        default_value: Arc<serde_json::value::RawValue>,
-        targeting_key: Option<Str>,
-        attributes: Arc<HashMap<Str, Attribute>>,
-        result: TestResult,
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    struct TestResult {
-        value: Arc<serde_json::value::RawValue>,
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // this test is way too slow on miri
-    fn evaluation_sdk_test_data() {
-        let _ = env_logger::builder().is_test(true).try_init();
-
-        let config =
-            UniversalFlagConfig::from_json(std::fs::read("tests/data/flags-v1.json").unwrap())
-                .unwrap();
-        let config = Configuration::from_server_response(config);
-        let now = Utc::now();
-
-        for entry in fs::read_dir("tests/data/tests/").unwrap() {
-            let entry = entry.unwrap();
-            println!("Processing test file: {:?}", entry.path());
-
-            let f = File::open(entry.path()).unwrap();
-            let test_cases: Vec<TestCase> = serde_json::from_reader(f).unwrap();
-
-            for test_case in test_cases {
-                let default_assignment = AssignmentValue::from_wire(
-                    test_case.variation_type.into(),
-                    test_case.default_value,
-                )
-                .unwrap();
-
-                print!("test subject {:?} ... ", test_case.targeting_key);
-                let subject = EvaluationContext::new(test_case.targeting_key, test_case.attributes);
-                let result = get_assignment(
-                    Some(&config),
-                    &test_case.flag,
-                    &subject,
-                    test_case.variation_type.into(),
-                    now,
-                );
-
-                let result_assingment = result
-                    .as_ref()
-                    .map(|assignment| &assignment.value)
-                    .unwrap_or(&default_assignment);
-                let expected_assignment = AssignmentValue::from_wire(
-                    test_case.variation_type.into(),
-                    test_case.result.value,
-                )
-                .unwrap();
-
-                assert_eq!(result_assingment, &expected_assignment);
-                println!("ok");
-            }
-        }
-    }
-}

datadog-ffe/src/rules_based/ufc/models.rs

@@ -558,24 +558,6 @@ impl ShardRange {
 mod tests {
     use super::{TryParse, UniversalFlagConfigWire};
 
-    #[test]
-    #[cfg_attr(miri, ignore)] // this test is way too slow on miri
-    fn parse_flags_v1() {
-        let json_content = std::fs::read_to_string("tests/data/flags-v1.json").unwrap();
-        let ufc: UniversalFlagConfigWire = serde_json::from_str(&json_content).unwrap();
-
-        let failures = ufc
-            .flags
-            .values()
-            .filter(|it| matches!(it, TryParse::ParseFailed(_)))
-            .count();
-        assert!(
-            failures == 0,
-            "failed to parse {failures}/{} flags",
-            ufc.flags.len()
-        );
-    }
-
     #[test]
     fn parse_partially_if_unexpected() {
         let ufc: UniversalFlagConfigWire = serde_json::from_str(