feat(logs): add modules support

.github/workflows/ci.yml

@@ -0,0 +1,96 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-test:
+    name: build & test (${{ matrix.name }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux
+            os: ubuntu-24.04
+            build_dir: _Build_lnx
+          - name: macos
+            os: macos-14
+            build_dir: _Build_mac
+          - name: windows
+            os: windows-2022
+            build_dir: _Build_wnd
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      # Runners ship CMake ~3.31; the kit dependency requires CMake >= 4.0.
+      - name: Install CMake 4.x
+        uses: lukka/get-cmake@latest
+        with:
+          cmakeVersion: "~4.0"
+
+      # The kit dependency is pulled by CMake FetchContent over SSH
+      # (git@github.com:RiantZ/kit.git). SSH has no keys in CI, so rewrite that
+      # URL to plain HTTPS. kit is public, so no token is needed.
+      - name: Rewrite SSH GitHub URLs to HTTPS
+        shell: bash
+        run: |
+          git config --global \
+            url."https://github.com/".insteadOf \
+            "git@github.com:"
+
+      - name: Configure & build
+        shell: bash
+        run: python scripts/cmake_build.py --build
+
+      - name: Test
+        shell: bash
+        run: ctest --test-dir ${{ matrix.build_dir }} -C Release --output-on-failure
+
+  format:
+    name: clang-format check
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      # clang-format output drifts between major versions. Pin to 22 to match
+      # the version developers use locally (Homebrew clang-format); the runner's
+      # default is v18, which reports spurious violations.
+      - name: Install clang-format 22
+        run: |
+          wget -q https://apt.llvm.org/llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 22
+          sudo apt-get install -y clang-format-22
+          # /usr/local/bin precedes /usr/bin in PATH, so this wins over the
+          # runner's default clang-format.
+          sudo ln -sf /usr/bin/clang-format-22 /usr/local/bin/clang-format
+          clang-format --version
+
+      # code_format.py lives in the kit repo. Clone it outside the p8 checkout
+      # (into RUNNER_TEMP) so its own sources are not picked up by the
+      # formatter's recursive scan of the project tree.
+      - name: Fetch formatter from kit
+        shell: bash
+        run: |
+          git clone --depth 1 \
+            "https://github.com/RiantZ/kit.git" \
+            "$RUNNER_TEMP/kit"
+
+      - name: Check formatting
+        shell: bash
+        run: python "$RUNNER_TEMP/kit/scripts/code_format.py" --check --all

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.20...4.0 FATAL_ERROR)
+cmake_minimum_required(VERSION 4.0 FATAL_ERROR)
 
 project(Protocol8)
 
@@ -54,6 +54,10 @@ set(ROOT_P8_PATH ${PROJECT_SOURCE_DIR} CACHE INTERNAL "")
 if(MSVC)
     add_compile_definitions(_UNICODE UNICODE)
 
+    # <windows.h> (pulled in via kit) defines min()/max() macros that clobber
+    # std::min/std::max; NOMINMAX suppresses them.
+    add_compile_definitions(NOMINMAX)
+
     if(NOT COMMAND set_ide_folder)
         macro(set_ide_folder _project_name)
             string(LENGTH ${ROOT_P8_PATH} ROOT_P8_PATH_LEN)
@@ -71,6 +75,10 @@ if(MSVC)
     #enable_cpp_c_compiler_flag_if_supported("/Wall")
     enable_cpp_c_compiler_flag_if_supported("/W4")
     enable_cpp_c_compiler_flag_if_supported("/WX")
+    # C4324: structure padded due to alignas() specifier. The padding is
+    # intentional (cache-line alignment via alignas(64)), so silence it rather
+    # than let /WX turn it into an error.
+    enable_cpp_c_compiler_flag_if_supported("/wd4324")
 else()
     enable_c_compiler_flag_if_supported("-std=gnu99")
 

CMakePresets.json

@@ -1,6 +1,6 @@
 {
   "version": 6,
-  "cmakeMinimumRequired": { "major": 3, "minor": 21 },
+  "cmakeMinimumRequired": { "major": 4, "minor": 0 },
   "configurePresets": [
     {
       "name": "windows",

README.md

@@ -1,9 +1,12 @@
 # Protocol8
+
+[![CI](https://github.com/RiantZ/p8/actions/workflows/ci.yml/badge.svg)](https://github.com/RiantZ/p8/actions/workflows/ci.yml)
+
 ## Building
 
 ### Prerequisites
 
-- CMake 3.20+ (tested up to 4.0)
+- CMake 4.0+
 - Python 3
 - C++20 compatible compiler (MSVC, GCC, Clang)
 
@@ -78,14 +81,20 @@ Single-thread emit latency (`c_log_perf_test`, 32 batches x 1 000 000 iterations
 | `send_hello_d_no_attrs` | 26.8 ns | 1 000 000 |
 | `send_hello_d_3_attrs` (str + i64 + f64) | 39.3 ns | 1 000 000 |
 
-`full_cycle` throughput (init + emit + release) through the in-memory null sink, 1 000 000 iterations per thread:
+`full_cycle` throughput (init + emit + release) through the in-memory null sink,
+1 000 000 iterations per thread. Numbers are aggregated over 25 runs of the test
+(`ns/call` = full-cycle wall time / total calls):
+
+| Threads | Total calls | Median ns/call | p95 ns/call | Stdev | Throughput (median) |
+|---------|-------------|----------------|-------------|-------|---------------------|
+| 1 | 1 000 000 | 26.6 ns | 32.8 ns | 3.16 ns | 37.6 M calls/s |
+| 2 | 2 000 000 | 13.7 ns | 16.1 ns | 1.85 ns | 72.8 M calls/s |
+| 4 | 4 000 000 | 7.9 ns  | 11.0 ns | 1.29 ns | 125.9 M calls/s |
+| 8 | 8 000 000 | 4.5 ns  | 6.5 ns  | 0.94 ns | 221.1 M calls/s |
 
-| Threads | Total calls | ns/call | Throughput |
-|---------|-------------|---------|------------|
-| 1 | 1 000 000 | 22.6 ns | 44.2 M calls/s |
-| 2 | 2 000 000 | 12.7 ns | 78.6 M calls/s |
-| 4 | 4 000 000 | 9.8 ns  | 102.4 M calls/s |
-| 8 | 8 000 000 | 6.4 ns  | 157.0 M calls/s |
+Relative spread (stdev/mean) grows with thread count — from ~12 % at 1 thread to
+~19 % at 8 threads — so multi-threaded runs are less stable and p95 sits noticeably
+above the median.
 
 ### Profiling with Tracy
 

api/p8_protocol.h

@@ -127,6 +127,18 @@ extern "C"
         //* padding to align total size on 8 bytes boundary
     };
 
+    // Serialized representation of s_p8_log_mod
+    // Fixed-size header; variable-length data follows in the buffer.
+    // Name (string utf8) include a NUL terminator.
+    struct s_p8_log_mod_svc
+    {
+        s_p8_svc_hdr ms_hdr;  // service item header, 4
+        uint16_t     mu_id;   // ID of the module
+        uint8_t      mu_verb; // verbosity == e_p8_level
+        //* Serialized name
+        //* padding to align total size on 8 bytes boundary
+    };
+
     // Serialized representation of s_p8_log_desc (the immutable log descriptor).
     // Fixed-size header; variable-length data follows in the buffer.
     // String lengths are byte counts and do NOT include a NUL terminator.
@@ -187,9 +199,12 @@ extern "C"
         uint16_t mu_args_size; // serialized variable arguments size in bytes
 
         // 32 bits
-        uint16_t mu_size;        // total item size in bytes (header + args + attrs + padding)
-        uint8_t  mu_attrs_count; // number of serialized attributes
-        uint8_t  mu_flags;       // todo
+        uint16_t mu_size;   // total item size in bytes (header + args + attrs + padding)
+        uint16_t mu_mod_id; // module ID
+
+        // 16 bits
+        uint8_t mu_attrs_count; // number of serialized attributes
+        uint8_t mu_flags;       // todo
         //* Serialized Variable agruments [int32][int64][double][length in bytes (16b) + string data] ...
         //* Serialized attributes [p8_attr_id + data][...]
         //* padding to tail of the buffer to 8 bytes boundary, to be sure that following buffer will start from 8b

dep/CMakeLists.txt

@@ -1,8 +1,23 @@
 include(FetchContent)
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.30)
+    cmake_policy(SET CMP0205 NEW)
+endif()
+
+# SYSTEM: consume kit as a third-party dependency so its headers are included
+# with -isystem. That suppresses warnings originating inside kit headers (e.g.
+# GCC's -Wclass-memaccess from list.hpp's memset on non-trivial types when a
+# kit template is instantiated in a p8 translation unit) without weakening the
+# project's -Werror on p8's own code.
+# p8 only ever links the static kit archive (kit::kit). Skip building kit's
+# shared library so we don't waste build time on an unused DLL/.so (and, on
+# Windows, avoid its import lib colliding with the static kit.lib).
+set(KIT_BUILD_SHARED OFF CACHE BOOL "" FORCE)
 FetchContent_Declare(
     kit
     GIT_REPOSITORY git@github.com:RiantZ/kit.git
     GIT_TAG        main
+    SYSTEM
 )
 FetchContent_MakeAvailable(kit)
 
@@ -15,6 +30,7 @@ FetchContent_Declare(
     nlohmann_json
     GIT_REPOSITORY https://github.com/nlohmann/json.git
     GIT_TAG        v3.12.0
+    SYSTEM
 )
 FetchContent_MakeAvailable(nlohmann_json)
 
@@ -29,7 +45,7 @@ FetchContent_MakeAvailable(xxhash)
 
 add_library(xxhash_hdr INTERFACE)
 add_library(xxhash::xxhash ALIAS xxhash_hdr)
-target_include_directories(xxhash_hdr INTERFACE ${xxhash_SOURCE_DIR})
+target_include_directories(xxhash_hdr SYSTEM INTERFACE ${xxhash_SOURCE_DIR})
 
 # Tracy profiler -- optional, controlled by P8_ENABLE_TRACY (OFF by default).
 #

doc/todo.md

@@ -1,17 +1,12 @@
 # TODO
 
 ## Next Sprint
-- Добавление для логов модулей и их сериализация
-- мониторинг потерь данных - мониторинг в логах, трейсах, метриках и регулярный пулл из core - создание статистики которую можно использовать в тестах
-- тест производительности 
-  - логов против принтф в файл из нескольких тредов, замер времени и места на диске
-  - СДК от Датадога С++
-  - СДК от Датадога для профайлинга?
 - Создание презентации с цифрами (оттолкнуться от )
   - Анализ рынка и конкурентов
   - Потребности клиентов
   - Архитектура
   - Что мы предлагаем (p8, microP8, server, datadog integration)  
+- Добавление режима обратного давления - когда буфера нет - мы ждем
 - Найдена нестабильность (flaky crash). На втором запросе «100 раз» один из прогонов упал: бинарник P8_RegressionTests был убит (subprocess killed, rc=8) на ~60-й итерации, вывод оборвался на тесте c_writer_registry_test.no_core_writer_is_noop.
 Воспроизвести не удалось:
 Изолированный прогон c_writer_registry_test.* — прошёл.
@@ -26,6 +21,12 @@
 
 ## Done
 - add pre-hook for format & auto-install by cmake
+- мониторинг потерь данных - мониторинг в логах, трейсах, метриках и регулярный пулл из core - создание статистики которую можно использовать в тестах
+- Добавление для логов модулей и их сериализация
+- тест производительности 
+  - логов против принтф в файл из нескольких тредов, замер времени и места на диске
+  - СДК от Датадога С++
+  - СДК от Датадога для профайлинга?
 
 Создание синков
 - ответить на вопрос поддержки мульти-синков? **Нет, такую поддержку мы делать не будем**

engine/p8_buffer_pool.cpp

@@ -130,6 +130,29 @@ void cp8_buffer_pool::recycle(uint8_t *ip_buf)
     mo_free.push_last(ip_buf);
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+void cp8_buffer_pool::recycle(kit::c_lst<uint8_t *> &io_bufs)
+{
+    if(0 == io_bufs.size())
+    {
+        return;
+    }
+
+    std::lock_guard<std::mutex> lo_guard(mo_mutex);
+
+    io_bufs.clear(
+        [this](uint8_t *ip_buf)
+        {
+            if(!ip_buf)
+            {
+                return;
+            }
+            mz_acquired -= mz_buffer_size;
+            mo_free.push_last(ip_buf);
+        },
+        kit::e_c_lst_pool_policy::e_keep);
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 size_t cp8_buffer_pool::get_buffer_size() const
 {

engine/p8_buffer_pool.hpp

@@ -56,6 +56,12 @@ class cp8_buffer_pool
     // Released memory is kept in the pool — it is not given back to the OS.
     void recycle(uint8_t *ip_buf);
 
+    // Batch variant: returns every buffer in io_bufs to the free list under a
+    // single lock acquisition, then empties io_bufs (keeping its node pool).
+    // Equivalent to calling recycle() per element but pays the mutex cost once,
+    // which matters on the worker's flush path where whole batches are recycled.
+    void recycle(kit::c_lst<uint8_t *> &io_bufs);
+
     size_t get_buffer_size() const;
     size_t get_free_count();
 

engine/p8_config.cpp

@@ -31,7 +31,12 @@ extern "C"
         }
 
 #ifdef G_OS_WINDOWS
-        lp_file = _wfopen(ip_path, L"rb");
+        // _wfopen is flagged unsafe (C4996) by MSVC; use the _s variant, which
+        // sets lp_file to NULL on failure so the shared check below still works.
+        if(_wfopen_s(&lp_file, ip_path, L"rb") != 0)
+        {
+            lp_file = nullptr;
+        }
 #else
         lp_file = std::fopen(ip_path, "rb");
 #endif