Add generating benchmarks for Zephelin execution

.github/workflows/docs.yml

@@ -14,6 +14,40 @@ env:
   DEBIAN_FRONTEND: noninteractive
   CMAKE_PREFIX_PATH: /opt/toolchains
 jobs:
+  Run-Benchmark:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Free space on runner
+        run: |
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ggc
+          sudo rm -rf /usr/local/.ghcup
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+      - name: Install dependencies
+        run: |
+          ./scripts/ci/install_deps.sh
+      - name: Setup Zephyr
+        run: |
+          ./scripts/ci/setup_zephyr.sh
+      - name: Run Twister
+        run: |
+          export ZPL_BASE="$(pwd)"
+          west twister -v -p max32690fthr/max32690/m4 -p stm32f746g_disco -T benchmark -j 1 --timeout-multiplier 3
+      - name: Run Benchmark
+        run: |
+          python3 benchmark/scripts/extract_measurements.py twister-out results.json
+          python3 benchmark/scripts/generate_report.py results.json report.md
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: "results"
+          path: |
+              results.json
+              report.md
+
   Run-GatherTrace:
     runs-on: ubuntu-22.04
     steps:
@@ -40,7 +74,9 @@ jobs:
 
   Run-BuildDocs:
     runs-on: ubuntu-22.04
-    needs: [Run-GatherTrace]
+    needs:
+      - Run-GatherTrace
+      - Run-Benchmark
     steps:
       - name: Free space on runner
         run: |
@@ -66,6 +102,11 @@ jobs:
         with:
           name: tef-traces
           path: docs/source/_static/trace_viewer
+      - name: Download benchmark results
+        uses: actions/download-artifact@v4
+        with:
+          name: results
+          path: benchmark-results
       - uses: actions/setup-python@v5
         with:
           python-version: 3.11
@@ -76,6 +117,8 @@ jobs:
           python -m pip install -r docs/requirements.txt
       - name: Generate documentation
         run: |
+          mkdir -p docs/source/generated
+          cp benchmark-results/report.md docs/source/generated/benchmark-report.md
           cd docs
           make html latexpdf
           cp build/latex/*.pdf build/html

docs/source/benchmark_report.md

@@ -1,4 +1,6 @@
-# Benchmark
+# Zephelin benchmarking
+
+This chapter demonstrates overhead introduced by Zephelin library based in different variants.
 
 ```{include} generated/benchmark-report.md
 ```

docs/source/index.md

@@ -14,7 +14,7 @@ instrumentation
 ctf_to_tef
 new_runtime
 visual_interface
-server_endpoints
+live_tracing
 examples
 development
 benchmark_report

docs/source/introduction.md

@@ -13,6 +13,7 @@ This documentation describes the following aspects of Zephelin and the associate
 * {doc}`ctf_to_tef` - describes how traces are converted and processed
 * {doc}`new_runtime` - depicts how to track a new runtime
 * {doc}`visual_interface` - describes the tool for trace visualization and provides sample applications for various tracing features
-* {doc}`server_endpoints` - lists the API endpoints supported by the real-time tracing server.
+* {doc}`live_tracing` - describes how to trace the execution in real-time
 * {doc}`examples` - aggregates and describes various example applications present in the project
 * {doc}`development` - summarizes and links resources regarding development of Zephelin
+* {doc}`benchmark_report` - provides details on the overhead introduced by the library based on different options

docs/source/library.md

@@ -236,6 +236,7 @@ against using both methods at the same time. Improper use will result in a compi
 The library can be configured both during building and during a run on a device.
 To find out how to configure the library and how to add new configurations, check {doc}`configuration`.
 
+(trace-collection)=
 ### Trace collection
 
 To enable Zephelin tracing support, the user should enable the symbol `CONFIG_ZPL_TRACE` in Kconfig.
@@ -271,6 +272,8 @@ Depending on the tracing backend used, the following commands can be used for tr
     -h, --help       show this help message and exit
     --send-enable    Send 'enable' to device before collecting data to enable tracing, requires
                      CONFIG_TRACING_HANDLE_HOST_CMD to be enabled in the app
+    --send-to-remote
+                     Forward collected data to specified address
   ```
 
 #### USB
@@ -293,6 +296,7 @@ Depending on the tracing backend used, the following commands can be used for tr
                           Timeout of the USB capture in seconds
     -w, --wait-for-device
                           When this flag is set, the command will wait for the device to connect
+    --send-to-remote      Forward collected data to specified address
   ```
 
 #### Debugger
@@ -319,6 +323,7 @@ Depending on the tracing backend used, the following commands can be used for tr
     --openocd OPENOCD    Path to custom OpenOCD
     --buffer-full        Run application until trace buffer is full
     --n-bytes N_BYTES    Run application until there is at least n in trace buffer
+    --send-to-remote     Forward collected data to specified address
   ```
 
 #### Trivial UART in Renode

docs/source/live_tracing.md

@@ -0,0 +1,83 @@
+# Real-time tracing
+
+This chapter covers the usage of [Zephelin Trace Viewer](visual_interface) for live visualization of the traces being gathered during application execution.
+
+:::{note}
+Real-time visualization requires running a local Python server responsible for ingesting CTF traces, parsing them into TEF, and forwarding them to the visualizer.
+:::
+
+## Prerequisites
+
+Before running the server, you need to install required backend dependencies and compile the frontend visualizer.
+
+**1. Install server dependencies**
+
+From the root of the project, run the following command:
+
+```bash
+pip install -r server/requirements.txt
+```
+
+**2. Build the Visualizer**
+
+The server requires a compiled version of the [Zephelin Trace Viewer](visual_interface).
+After cloning the repository, build the frontend by running:
+```bash
+yarn build
+```
+
+## Running the server
+
+To start the Zephelin Server and host the visualizer, execute the `run_backend.py` script.
+
+### Basic usage
+To run the server with default settings and serve the frontend, point it to the `dist` directory with the compiled frontend:
+
+```bash
+python server/run_backend.py --frontend-directory <path_to_dist>
+```
+
+### Configuration Options
+
+You can customize the server's networking interfaces, and specify trace parsing options that will be used in [CTF to TEF](ctf_to_tef) conversion.
+
+* `--tcp-server-host` - Address of the Zephelin TCP socket for CTF trace ingestion (Default: `127.0.0.1`).
+* `--tcp-server-port`- Port of the Zephelin TCP socket for CTF trace ingestion (Default: `5000`).
+* `--backend-host` - Address where the backend API and visualizer are hosted (Default: `127.0.0.1`).
+* `--backend-port` - Port where the backend API and visualizer are hosted (Default: `8000`).
+* `--frontend-directory` - Path to compiled frontend directory.
+* `--build-dir` - Path to the traced application build directory.
+* `--tflm-model-paths` - Paths to the TFLM models.
+* `--tvm-model-paths` - Paths to the TVM models.
+* `--tvm-model-metadata-paths` - Paths to the TVM model metadata files.
+* `--tvm-model-op-remove-prefix` - Regex pattern used for removing TVM operator prefixes.
+* `--tvm-model-op-remove-suffix` - Regex pattern used for removing TVM operator suffixes.
+* `--verbosity` - Set the logging verbosity level (`DEBUG, INFO, WARNING, ERROR, CRITICAL`).
+
+## Providing traces to the server
+
+Once the server is running, it listens for CTF traces on the TCP socket.
+To route the traces read from one of the available backends (described in [Trace Collection](trace-collection) section), `--send-to-remote` argument can be provided with TCP socket address specified.
+
+**Example**:
+
+```bash
+west zpl-uart-capture /dev/pts/12 115200 ./trace-hw.ctf --send-to-remote 127.0.0.1:5000
+```
+
+## Live-tracing controls
+
+If the Visualizer is connected to the server, the live-tracing controls are available on the top-bar.
+
+* `Start streaming` - enables continuous rendering of received traces.
+* `Stop streaming` - disables continuous rendering.
+* `Collect` - visualizes all the traces gathered so far.
+* `Stop Tailing` - Stops the default behavior in which viewport follows live edge of visualized trace.
+* `Resume Tailing` - Snaps the viewport back to the live edge.
+
+
+## Server API reference
+
+The Zephelin real-time tracing server supports the following JSON-RPC requests sent as rpc_request WebSocket event:
+
+{{endpoints}}

scripts/prepare_trace.py

@@ -51,7 +51,25 @@
 # Prefix for scheduling events
 INSTR_SCHED_PREFIX = "instr_sched_switch"
 
-CPPFILT_CMD = [os.environ.get("ZPL_DEMANGLE_CMD", "c++filt")]
+CPPFILT_CMD = os.environ.get("ZPL_DEMANGLE_CMD", None)
+
+
+def get_sdk_bin_path(build_dir: str | Path):
+    """
+    Extracts the SDK path from CMakeCache.
+    """
+    build_dir = Path(build_dir)
+    cmake_cache = build_dir / "CMakeCache.txt"
+    if not cmake_cache.exists():
+        return None
+
+    with cmake_cache.open("r") as f:
+        for line in f:
+            if line.startswith("CMAKE_CXX_COMPILER:STRING="):
+                path = line.removeprefix("CMAKE_CXX_COMPILER:STRING=")
+                return Path(path).parent
+
+    return None
 
 
 # The list of custom events definitions
@@ -183,7 +201,10 @@ def inference_additional_args(msg: bt2._EventMessageConst) -> dict:
 
     @functools.lru_cache(maxsize=1024)
     def demangle(func: str):
-        cmd = CPPFILT_CMD + [func]
+        if CPPFILT_CMD is None:
+            return func
+
+        cmd = [CPPFILT_CMD, func]
         try:
             func_demangled = subprocess.check_output(cmd, text=True).strip()
         except subprocess.CalledProcessError as e:
@@ -788,6 +809,20 @@ def process_ram_report(ram: dict) -> float:
     return s
 
 
+def ensure_cppfilt(sdk_bin_path: str | Path):
+    """Ensures that CPPFILT_CMD is set."""
+    global CPPFILT_CMD
+    if CPPFILT_CMD is not None:
+        return
+
+    sdk_bin_path = Path(sdk_bin_path)
+    CPPFILT_CMD = next(sdk_bin_path.glob("*-c++filt"), None)
+    if CPPFILT_CMD:
+        print(f"Found c++filt at '{CPPFILT_CMD}'")
+    else:
+        print("c++filt is missing")
+
+
 def prepare(args: argparse.Namespace):
     """
     Prepares CTF trace to be visualized.
@@ -802,6 +837,10 @@ def prepare(args: argparse.Namespace):
     if args.zephyr_elf_path is None:
         args.zephyr_elf_path = Path(".") / "build" / "zephyr" / "zephyr.elf"
 
+    if Path(args.build_dir).exists():
+        if sdk_bin_path := get_sdk_bin_path(args.build_dir):
+            ensure_cppfilt(sdk_bin_path)
+
     multiple_models = False
 
     if args.ctf_trace is None and args.instrumentation is None: