diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3e6cc8..14c6094 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -794,404 +794,6 @@ jobs: name: test-results-windows-msvc-debug path: build\build\Debug\test_results_msvc_debug.xml - # ============================================================================ - # Performance Baseline Tracking - # ============================================================================ - - performance-tracking: - name: "Performance Baseline Tracking" - runs-on: ubuntu-24.04 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - permissions: - contents: write - pages: write - id-token: write - actions: read - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - fetch-depth: 0 - - - name: Install system dependencies - run: | - sudo apt update - sudo apt install -y cmake python3-pip git - # Install Clang 18 for high-performance release builds - sudo apt install -y clang-18 libc++-18-dev libc++abi-18-dev - # Install profiling tools for flame graph generation - # Install kernel-specific perf tools to avoid version mismatch - KERNEL_VERSION=$(uname -r) - echo "Kernel version: $KERNEL_VERSION" - # Try to install kernel-specific perf first, fallback to generic - sudo apt install -y linux-tools-$KERNEL_VERSION || sudo apt install -y linux-tools-generic linux-tools-common - sudo apt install -y perf-tools-unstable - # Install FlameGraph tools - git clone https://github.com/brendangregg/FlameGraph.git /tmp/FlameGraph - sudo cp /tmp/FlameGraph/*.pl /usr/local/bin/ - sudo chmod +x /usr/local/bin/*.pl - # Set up alternatives to use Clang 18 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 - - - name: Set perf permissions - run: | - # Allow perf to run without root for non-paranoid mode - sudo sysctl -w kernel.perf_event_paranoid=-1 - sudo sysctl -w kernel.kptr_restrict=0 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install Conan - run: | - pip3 install --user conan - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Setup Conan profile - run: | - conan profile detect --force - # Use Clang 18 with libstdc++ for performance builds (avoids GoogleTest linking issues) - sed -i 's/compiler=.*/compiler=clang/' ~/.conan2/profiles/default - sed -i 's/compiler.version=.*/compiler.version=18/' ~/.conan2/profiles/default - sed -i 's/compiler.libcxx=.*/compiler.libcxx=libstdc++11/' ~/.conan2/profiles/default - sed -i 's/compiler.cppstd=.*/compiler.cppstd=23/' ~/.conan2/profiles/default - sed -i 's/os=.*/os=Linux/' ~/.conan2/profiles/default - sed -i 's/arch=.*/arch=x86_64/' ~/.conan2/profiles/default - echo "=== Conan Profile ===" - conan profile show --profile:host=default - - - name: Cache Conan packages - uses: actions/cache@v4 - with: - path: ~/.conan2 - key: conan-performance-baseline-${{ hashFiles('conanfile.py') }} - restore-keys: | - conan-performance-baseline- - - - name: Install dependencies - run: | - export CC=clang-18 - export CXX=clang++-18 - conan install . -s os=Linux -s build_type=Release --output-folder=build --build=missing -o with_gperftools=True - - - name: Configure and Build (Release) - run: | - export CC=clang-18 - export CXX=clang++-18 - cmake --preset conan-release \ - -DCOMPILER_TYPE=CLANG \ - -DUSE_LTO=OFF \ - -DUSE_NATIVE_ARCH=ON \ - -DUSE_LIBC_PLUS_PLUS=OFF \ - -DENABLE_SHARED_LIBRARY=ON - cmake --build --preset conan-release --parallel - - - name: Run Performance Benchmarks and Generate Flame Graphs - run: | - cd build/build - - # Create docs/performance directory for all performance reports - mkdir -p ../../docs/performance - - # 1. Standard benchmark run for historical tracking - echo "Running standard benchmarks..." - ./utf_strings-bench --benchmark_min_time=1.0s --benchmark_format=json --benchmark_out=performance_baseline.json - - # 2. Extended benchmark run for detailed analysis - echo "Running extended benchmarks..." - ./utf_strings-bench --benchmark_min_time=3.0s --benchmark_format=json --benchmark_out=../../docs/performance/detailed_benchmark.json - - # 3. Generate flame graphs with perf (if available) - echo "Generating flame graphs..." - if command -v perf >/dev/null 2>&1; then - # Try to run perf - may fail in containerized environment - echo "Perf version info:" - perf --version || echo "Could not get perf version" - uname -r | echo "Kernel version: $(cat)" - echo "Attempting perf record with diagnostic output..." - if perf record -F 99 -g --call-graph dwarf -- ./utf_strings-bench --benchmark_min_time=2.0s --benchmark_repetitions=5; then - echo "Perf recording successful, checking file format..." - ls -la perf.data - - # Check perf file format and try to process - echo "Testing perf script compatibility..." - if perf script -v > /dev/null 2>&1; then - echo "Perf file format compatible, generating flame graph..." - perf script > ../../docs/performance/perf.script - if stackcollapse-perf.pl ../../docs/performance/perf.script > ../../docs/performance/perf.folded; then - flamegraph.pl ../../docs/performance/perf.folded > ../../docs/performance/flamegraph.svg - echo "Flame graph generated successfully" - else - echo "Stack collapse failed, creating placeholder" - echo "Perf data processing failed during stack collapse step. File format may be incompatible." > ../../docs/performance/flamegraph_unavailable.txt - fi - else - echo "Perf file format incompatible, trying alternative approach..." - perf script -v 2>&1 | head -20 > ../../docs/performance/perf_debug.txt || true - - # Try an alternative approach with simpler call graph format - echo "Attempting alternative perf record with fp call-graph..." - rm -f perf.data - if perf record -F 99 -g --call-graph fp -- ./utf_strings-bench --benchmark_min_time=1.0s --benchmark_repetitions=3; then - echo "Alternative perf recording successful, trying script..." - if perf script > ../../docs/performance/perf.script 2>/dev/null; then - echo "Alternative perf script successful, generating flame graph..." - if stackcollapse-perf.pl ../../docs/performance/perf.script > ../../docs/performance/perf.folded; then - flamegraph.pl ../../docs/performance/perf.folded > ../../docs/performance/flamegraph.svg - echo "Flame graph generated successfully with alternative method" - else - echo "Stack collapse failed with alternative method" - echo "Perf data processing failed during stack collapse step. File format may be incompatible." > ../../docs/performance/flamegraph_unavailable.txt - fi - else - echo "Alternative perf script also failed" - echo "Perf file format incompatible with current perf version. This can happen in containerized environments. Debug info saved to perf_debug.txt" > ../../docs/performance/flamegraph_unavailable.txt - fi - else - echo "Alternative perf record also failed" - echo "Perf file format incompatible with current perf version. This can happen in containerized environments. Debug info saved to perf_debug.txt" > ../../docs/performance/flamegraph_unavailable.txt - fi - fi - else - echo "Perf recording failed (likely due to container restrictions), creating placeholder" - echo "Flame graph generation requires privileged container access. Run locally with: perf record -g ./utf_strings-bench && perf script | stackcollapse-perf.pl | flamegraph.pl > flamegraph.svg" > ../../docs/performance/flamegraph_unavailable.txt - fi - else - echo "Perf not available, skipping flame graph generation" - echo "Perf tools not found. Install with: sudo apt install linux-tools-generic" > ../../docs/performance/flamegraph_unavailable.txt - fi - - # 4. Generate comprehensive benchmark report with multiple formats - echo "Generating comprehensive reports..." - ./utf_strings-bench --benchmark_format=console --benchmark_min_time=1.0s > ../../docs/performance/benchmark_console.txt - ./utf_strings-bench --benchmark_format=csv --benchmark_min_time=1.0s --benchmark_out=../../docs/performance/benchmark_results.csv - - # 5. System information for performance context - echo "Collecting system information..." - cat > ../../docs/performance/system_info.txt << EOF - Performance Benchmark System Information - Generated: $(date) - Build: Clang 18 Release with libc++ - - CPU Information: - $(cat /proc/cpuinfo | grep "model name" | head -1) - $(cat /proc/cpuinfo | grep "cpu MHz" | head -1) - $(cat /proc/cpuinfo | grep "cache size" | head -1) - CPU Cores: $(nproc) - - Memory Information: - $(free -h) - - Compiler Version: - $(clang++-18 --version) - - Build Flags Used: - Release build with -O3 -march=native -mtune=native -flto - EOF - - echo "Performance analysis complete!" - ls -la ../../docs/performance/ - - - name: Upload docs for Pages deployment - uses: actions/upload-artifact@v4 - with: - name: docs-for-pages - path: docs/ - retention-days: 1 - - - name: Setup docs directory and create performance index - run: | - # Configure git identity for GitHub Actions - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - - # Set up authentication for git push - git remote set-url origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" - - # Ensure docs directories exist - mkdir -p docs/dev/bench - mkdir -p docs/performance - - # Ensure .nojekyll exists to disable Jekyll processing - touch docs/.nojekyll - - # Create performance index page - cat > docs/performance/index.html << 'EOF' - - - - - - UTF Strings - Performance Analysis - - - -
-

UTF Strings Performance Analysis

-

Comprehensive performance benchmarks and profiling results

-
- Build Configuration: Clang 18 Release with libc++, -O3 -march=native -flto -
Last Updated: -
-
- -
-
-

Flame Graph Analysis

-

Interactive flame graph showing CPU time distribution and hot paths in the UTF string processing functions.

- View Flame Graph - Raw Perf Data -
- -
-

Historical Benchmarks

-

Interactive charts showing performance trends over time with automatic regression detection.

- View Historical Charts -
- -
-

Detailed Results

-

Comprehensive benchmark results in multiple formats for analysis and comparison.

- JSON Results - CSV Data - Console Output -
- -
-

System Information

-

Hardware and build environment details for performance context.

- System Info -
-
- -
-

Interactive Flame Graph

-

Embedded flame graph showing function call patterns and CPU time distribution:

- -
- - - - - EOF - - # Create a basic README for docs if it doesn't exist - if [ ! -f docs/README.md ]; then - echo "# UTF Strings Documentation" > docs/README.md - echo "This directory contains automated performance benchmark results and documentation." >> docs/README.md - fi - - - name: Commit Performance Analysis Reports - run: | - # Add all generated performance files to git - git add docs/performance/ - - # Check if there are changes to commit - if ! git diff --cached --quiet; then - git commit -m "Update performance analysis reports - - - Detailed benchmark results (JSON, CSV, console) - - Flame graph analysis (if available) - - System information and build context - - Generated on: $(date) - - Build: Clang 18 Release with libc++" - - # Push the performance reports - git push origin main - - echo "Performance reports committed and pushed successfully" - else - echo "No changes in performance reports to commit" - fi - - - name: Store Performance Results - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: "googlecpp" - output-file-path: build/build/performance_baseline.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - alert-threshold: "150%" - comment-on-alert: true - fail-on-alert: false - # Use gh-pages branch for benchmark data storage - gh-pages-branch: "gh-pages" - benchmark-data-dir-path: "dev/bench" - # ============================================================================ # Publish Test Results # ============================================================================ diff --git a/.github/workflows/performance-baseline.yml b/.github/workflows/performance-baseline.yml new file mode 100644 index 0000000..ddb94a5 --- /dev/null +++ b/.github/workflows/performance-baseline.yml @@ -0,0 +1,464 @@ +name: Performance Baseline Tracking + +on: + # Trigger on pushes to main branch + push: + branches: [main] + + # Allow manual triggering + workflow_dispatch: + + # Trigger on workflow runs completion to get artifacts + workflow_run: + workflows: ["CI/CD Pipeline"] + branches: [main] + types: + - completed + +permissions: + contents: write + pages: write + id-token: write + actions: read + +jobs: + performance-tracking: + name: "Performance Baseline Tracking" + runs-on: ubuntu-24.04 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update + sudo apt install -y cmake python3-pip git + # Install Clang 18 for high-performance release builds + sudo apt install -y clang-18 libc++-18-dev libc++abi-18-dev + # Install profiling tools for flame graph generation + # Install kernel-specific perf tools to avoid version mismatch + KERNEL_VERSION=$(uname -r) + echo "Kernel version: $KERNEL_VERSION" + # Try to install kernel-specific perf first, fallback to generic + sudo apt install -y linux-tools-$KERNEL_VERSION || sudo apt install -y linux-tools-generic linux-tools-common + sudo apt install -y perf-tools-unstable + # Install FlameGraph tools + git clone https://github.com/brendangregg/FlameGraph.git /tmp/FlameGraph + sudo cp /tmp/FlameGraph/*.pl /usr/local/bin/ + sudo chmod +x /usr/local/bin/*.pl + # Set up alternatives to use Clang 18 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 + + - name: Set perf permissions + run: | + # Allow perf to run without root for non-paranoid mode + sudo sysctl -w kernel.perf_event_paranoid=-1 + sudo sysctl -w kernel.kptr_restrict=0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install Conan + run: | + pip3 install --user conan + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Setup Conan profile + run: | + conan profile detect --force + # Use Clang 18 with libstdc++ for performance builds (avoids GoogleTest linking issues) + sed -i 's/compiler=.*/compiler=clang/' ~/.conan2/profiles/default + sed -i 's/compiler.version=.*/compiler.version=18/' ~/.conan2/profiles/default + sed -i 's/compiler.libcxx=.*/compiler.libcxx=libstdc++11/' ~/.conan2/profiles/default + sed -i 's/compiler.cppstd=.*/compiler.cppstd=23/' ~/.conan2/profiles/default + sed -i 's/os=.*/os=Linux/' ~/.conan2/profiles/default + sed -i 's/arch=.*/arch=x86_64/' ~/.conan2/profiles/default + echo "=== Conan Profile ===" + conan profile show --profile:host=default + + - name: Cache Conan packages + uses: actions/cache@v4 + with: + path: ~/.conan2 + key: conan-performance-baseline-${{ hashFiles('conanfile.py') }} + restore-keys: | + conan-performance-baseline- + + - name: Install dependencies + run: | + export CC=clang-18 + export CXX=clang++-18 + conan install . -s os=Linux -s build_type=Release --output-folder=build --build=missing -o with_gperftools=True + + - name: Configure and Build (Release) + run: | + export CC=clang-18 + export CXX=clang++-18 + cmake --preset conan-release \ + -DCOMPILER_TYPE=CLANG \ + -DUSE_LTO=OFF \ + -DUSE_NATIVE_ARCH=ON \ + -DUSE_LIBC_PLUS_PLUS=OFF \ + -DENABLE_SHARED_LIBRARY=ON + cmake --build --preset conan-release --parallel + + - name: Run Performance Benchmarks and Generate Flame Graphs + run: | + cd build/build + + # Create docs/performance directory for all performance reports + mkdir -p ../../docs/performance + + # 1. Standard benchmark run for historical tracking + echo "Running standard benchmarks..." + ./utf_strings-bench --benchmark_min_time=1.0s --benchmark_format=json --benchmark_out=performance_baseline.json + + # 2. Extended benchmark run for detailed analysis + echo "Running extended benchmarks..." + ./utf_strings-bench --benchmark_min_time=3.0s --benchmark_format=json --benchmark_out=../../docs/performance/detailed_benchmark.json + + # 3. Generate flame graphs with perf (if available) + echo "Generating flame graphs..." + if command -v perf >/dev/null 2>&1; then + # Try to run perf - may fail in containerized environment + echo "Perf version info:" + perf --version || echo "Could not get perf version" + uname -r | echo "Kernel version: $(cat)" + echo "Attempting perf record with diagnostic output..." + if perf record -F 99 -g --call-graph dwarf -- ./utf_strings-bench --benchmark_min_time=2.0s --benchmark_repetitions=5; then + echo "Perf recording successful, checking file format..." + ls -la perf.data + + # Check perf file format and try to process + echo "Testing perf script compatibility..." + if perf script -v > /dev/null 2>&1; then + echo "Perf file format compatible, generating flame graph..." + perf script > ../../docs/performance/perf.script + if stackcollapse-perf.pl ../../docs/performance/perf.script > ../../docs/performance/perf.folded; then + flamegraph.pl ../../docs/performance/perf.folded > ../../docs/performance/flamegraph.svg + echo "Flame graph generated successfully" + else + echo "Stack collapse failed, creating placeholder" + echo "Perf data processing failed during stack collapse step. File format may be incompatible." > ../../docs/performance/flamegraph_unavailable.txt + fi + else + echo "Perf file format incompatible, trying alternative approach..." + perf script -v 2>&1 | head -20 > ../../docs/performance/perf_debug.txt || true + + # Try an alternative approach with simpler call graph format + echo "Attempting alternative perf record with fp call-graph..." + rm -f perf.data + if perf record -F 99 -g --call-graph fp -- ./utf_strings-bench --benchmark_min_time=1.0s --benchmark_repetitions=3; then + echo "Alternative perf recording successful, trying script..." + if perf script > ../../docs/performance/perf.script 2>/dev/null; then + echo "Alternative perf script successful, generating flame graph..." + if stackcollapse-perf.pl ../../docs/performance/perf.script > ../../docs/performance/perf.folded; then + flamegraph.pl ../../docs/performance/perf.folded > ../../docs/performance/flamegraph.svg + echo "Flame graph generated successfully with alternative method" + else + echo "Stack collapse failed with alternative method" + echo "Perf data processing failed during stack collapse step. File format may be incompatible." > ../../docs/performance/flamegraph_unavailable.txt + fi + else + echo "Alternative perf script also failed" + echo "Perf file format incompatible with current perf version. This can happen in containerized environments. Debug info saved to perf_debug.txt" > ../../docs/performance/flamegraph_unavailable.txt + fi + else + echo "Alternative perf record also failed" + echo "Perf file format incompatible with current perf version. This can happen in containerized environments. Debug info saved to perf_debug.txt" > ../../docs/performance/flamegraph_unavailable.txt + fi + fi + else + echo "Perf recording failed (likely due to container restrictions), creating placeholder" + echo "Flame graph generation requires privileged container access. Run locally with: perf record -g ./utf_strings-bench && perf script | stackcollapse-perf.pl | flamegraph.pl > flamegraph.svg" > ../../docs/performance/flamegraph_unavailable.txt + fi + else + echo "Perf not available, skipping flame graph generation" + echo "Perf tools not found. Install with: sudo apt install linux-tools-generic" > ../../docs/performance/flamegraph_unavailable.txt + fi + + # 4. Generate comprehensive benchmark report with multiple formats + echo "Generating comprehensive reports..." + ./utf_strings-bench --benchmark_format=console --benchmark_min_time=1.0s > ../../docs/performance/benchmark_console.txt + ./utf_strings-bench --benchmark_format=csv --benchmark_min_time=1.0s --benchmark_out=../../docs/performance/benchmark_results.csv + + # 5. System information for performance context + echo "Collecting system information..." + cat > ../../docs/performance/system_info.txt << EOF + Performance Benchmark System Information + Generated: $(date) + Build: Clang 18 Release with libc++ + + CPU Information: + $(cat /proc/cpuinfo | grep "model name" | head -1) + $(cat /proc/cpuinfo | grep "cpu MHz" | head -1) + $(cat /proc/cpuinfo | grep "cache size" | head -1) + CPU Cores: $(nproc) + + Memory Information: + $(free -h) + + Compiler Version: + $(clang++-18 --version) + + Build Flags Used: + Release build with -O3 -march=native -mtune=native -flto + EOF + + echo "Performance analysis complete!" + ls -la ../../docs/performance/ + + - name: Upload docs for Pages deployment + uses: actions/upload-artifact@v4 + with: + name: docs-for-pages + path: docs/ + retention-days: 1 + + - name: Setup docs directory and create performance index + run: | + # Configure git identity for GitHub Actions + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + # Set up authentication for git push + git remote set-url origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" + + # Ensure docs directories exist + mkdir -p docs/dev/bench + mkdir -p docs/performance + + # Create performance index page with dynamic file checking + cat > docs/performance/index.html << 'EOF' + + + + + + UTF Strings - Performance Analysis + + + +
+

UTF Strings Performance Analysis

+

Comprehensive performance benchmarks and profiling results

+
+ Build Configuration: Clang 18 Release with libc++, -O3 -march=native -flto +
Last Updated: +
+
+ +
+
+

Flame Graph Analysis

+

Interactive flame graph showing CPU time distribution and hot paths in the UTF string processing functions.

+ +
+
+ +
+

Historical Benchmarks

+

Interactive charts showing performance trends over time with automatic regression detection.

+ View Historical Charts +
+ +
+

Detailed Results

+

Comprehensive benchmark results in multiple formats for analysis and comparison.

+ JSON Results + CSV Data + Console Output +
+ +
+

System Information

+

Hardware and build environment details for performance context.

+ System Info +
+
+ +
+

Interactive Flame Graph

+

Embedded flame graph showing function call patterns and CPU time distribution:

+
+ +
+
+ + + + + EOF + + # Create a basic README for docs if it doesn't exist + if [ ! -f docs/README.md ]; then + echo "# UTF Strings Documentation" > docs/README.md + echo "This directory contains automated performance benchmark results and documentation." >> docs/README.md + fi + + - name: Commit Performance Analysis Reports + run: | + # Add all generated performance files to git + git add docs/performance/ + + # Check if there are changes to commit + if ! git diff --cached --quiet; then + git commit -m "Update performance analysis reports + + - Detailed benchmark results (JSON, CSV, console) + - Flame graph analysis (if available) + - System information and build context + - Generated on: $(date) + - Build: Clang 18 Release with libc++" + + # Push the performance reports + git push origin main + + echo "Performance reports committed and pushed successfully" + else + echo "No changes in performance reports to commit" + fi + + - name: Store Performance Results + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "googlecpp" + output-file-path: build/build/performance_baseline.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + alert-threshold: "150%" + comment-on-alert: true + fail-on-alert: false + # Use gh-pages branch for benchmark data storage + gh-pages-branch: "gh-pages" + benchmark-data-dir-path: "dev/bench" diff --git a/.github/workflows/sast-scanning.yml b/.github/workflows/sast-scanning.yml index f1c4cab..69d65c6 100644 --- a/.github/workflows/sast-scanning.yml +++ b/.github/workflows/sast-scanning.yml @@ -663,6 +663,8 @@ jobs: contents: read security-events: write actions: read + issues: write + pull-requests: write steps: - name: Download all SAST artifacts