From bee842e565067688c26abba8115e618382010806 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Jun 2026 17:38:28 +0000 Subject: [PATCH 1/7] docs: Initialize documentation pipeline [skip ci] --- .doc-pipeline-status.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .doc-pipeline-status.md diff --git a/.doc-pipeline-status.md b/.doc-pipeline-status.md new file mode 100644 index 00000000000..321a7aedce1 --- /dev/null +++ b/.doc-pipeline-status.md @@ -0,0 +1,5 @@ +# Documentation Pipeline Started + +Run ID: doc-orchestrator-2026-06-04T17-38-04.379Z +Status: In Progress +Started: 2026-06-04 17:38:28 UTC From ebabe1c315bffd9f9127fdfa5ee497eb1b8421df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Jun 2026 17:38:35 +0000 Subject: [PATCH 2/7] chore(docs): Clean slate - remove all documentation (101 files) [skip ci] --- docs/README.md | 119 ----- docs/development/.gitignore | 7 - docs/development/README.md | 96 ---- docs/development/architecture/README.md | 239 ---------- docs/development/contributing/guidelines.md | 272 ----------- docs/development/security/README.md | 239 ---------- docs/development/setup/environment.md | 200 -------- docs/development/setup/local-development.md | 279 ------------ docs/development/testing/README.md | 310 ------------- docs/diagrams/architecture/.gitignore | 8 - docs/diagrams/architecture/README-10.mmd | 4 - docs/diagrams/architecture/README-11.mmd | 4 - docs/diagrams/architecture/README-2.mmd | 11 - docs/diagrams/architecture/README-3.mmd | 7 - docs/diagrams/architecture/README-4.mmd | 6 - docs/diagrams/architecture/README-5.mmd | 6 - docs/diagrams/architecture/README-6.mmd | 6 - docs/diagrams/architecture/README-7.mmd | 5 - docs/diagrams/architecture/README-8.mmd | 7 - docs/diagrams/architecture/README-9.mmd | 5 - docs/diagrams/architecture/README.mmd | 22 - .../configuration-and-packs-2.mmd | 6 - .../configuration-and-packs-3.mmd | 7 - .../configuration-and-packs-4.mmd | 8 - .../configuration-and-packs-5.mmd | 11 - .../architecture/configuration-and-packs.mmd | 15 - .../architecture/core-init-and-runtime-2.mmd | 15 - .../architecture/core-init-and-runtime-3.mmd | 8 - .../architecture/core-init-and-runtime-4.mmd | 7 - .../architecture/core-init-and-runtime-5.mmd | 9 - .../architecture/core-init-and-runtime-6.mmd | 7 - .../architecture/core-init-and-runtime-7.mmd | 6 - .../architecture/core-init-and-runtime-8.mmd | 9 - .../architecture/core-init-and-runtime.mmd | 16 - .../database-and-storage-plugins-2.mmd | 4 - .../database-and-storage-plugins-3.mmd | 6 - .../database-and-storage-plugins-4.mmd | 5 - .../database-and-storage-plugins-5.mmd | 6 - .../database-and-storage-plugins.mmd | 19 - .../architecture/distributed-querying-2.mmd | 11 - .../architecture/distributed-querying-3.mmd | 6 - .../architecture/distributed-querying-4.mmd | 11 - .../architecture/distributed-querying.mmd | 10 - ...eventing-framework-and-subscriptions-2.mmd | 7 - ...eventing-framework-and-subscriptions-3.mmd | 7 - ...eventing-framework-and-subscriptions-4.mmd | 7 - ...eventing-framework-and-subscriptions-5.mmd | 5 - ...eventing-framework-and-subscriptions-6.mmd | 7 - ...eventing-framework-and-subscriptions-7.mmd | 16 - .../eventing-framework-and-subscriptions.mmd | 9 - .../architecture/extensions-and-ipc-2.mmd | 7 - .../architecture/extensions-and-ipc-3.mmd | 10 - .../architecture/extensions-and-ipc-4.mmd | 4 - .../architecture/extensions-and-ipc-5.mmd | 5 - .../architecture/extensions-and-ipc-6.mmd | 5 - .../architecture/extensions-and-ipc-7.mmd | 6 - .../architecture/extensions-and-ipc.mmd | 12 - .../filesystem-and-path-utilities-2.mmd | 5 - .../filesystem-and-path-utilities-3.mmd | 4 - .../filesystem-and-path-utilities-4.mmd | 5 - .../filesystem-and-path-utilities-5.mmd | 5 - .../filesystem-and-path-utilities-6.mmd | 3 - .../filesystem-and-path-utilities-7.mmd | 4 - .../filesystem-and-path-utilities-8.mmd | 9 - .../filesystem-and-path-utilities.mmd | 21 - .../logging-and-query-observability-2.mmd | 7 - .../logging-and-query-observability-3.mmd | 5 - .../logging-and-query-observability-4.mmd | 4 - .../logging-and-query-observability-5.mmd | 5 - .../logging-and-query-observability-6.mmd | 9 - .../logging-and-query-observability-7.mmd | 5 - .../logging-and-query-observability.mmd | 15 - .../architecture/remote-http-client-2.mmd | 5 - .../architecture/remote-http-client-3.mmd | 4 - .../architecture/remote-http-client-4.mmd | 14 - .../architecture/remote-http-client-5.mmd | 6 - .../architecture/remote-http-client-6.mmd | 9 - .../architecture/remote-http-client.mmd | 13 - .../sql-engine-and-virtual-tables-2.mmd | 5 - .../sql-engine-and-virtual-tables-3.mmd | 4 - .../sql-engine-and-virtual-tables-4.mmd | 5 - .../sql-engine-and-virtual-tables-5.mmd | 12 - .../sql-engine-and-virtual-tables-6.mmd | 10 - .../sql-engine-and-virtual-tables.mmd | 11 - docs/getting-started/.gitignore | 7 - docs/getting-started/first-steps.md | 178 -------- docs/getting-started/introduction.md | 113 ----- docs/getting-started/prerequisites.md | 167 ------- docs/getting-started/quick-start.md | 177 -------- docs/reference/architecture/.gitignore | 8 - docs/reference/architecture/README.md | 366 --------------- .../configuration-and-packs.md | 383 ---------------- .../core-init-and-runtime.md | 428 ------------------ .../database-and-storage-plugins.md | 357 --------------- .../distributed-querying.md | 376 --------------- .../eventing-framework-and-subscriptions.md | 363 --------------- .../extensions-and-ipc/extensions-and-ipc.md | 375 --------------- .../filesystem-and-path-utilities.md | 338 -------------- .../logging-and-query-observability.md | 409 ----------------- .../remote-http-client/remote-http-client.md | 345 -------------- .../sql-engine-and-virtual-tables.md | 351 -------------- 101 files changed, 7106 deletions(-) delete mode 100644 docs/README.md delete mode 100644 docs/development/.gitignore delete mode 100644 docs/development/README.md delete mode 100644 docs/development/architecture/README.md delete mode 100644 docs/development/contributing/guidelines.md delete mode 100644 docs/development/security/README.md delete mode 100644 docs/development/setup/environment.md delete mode 100644 docs/development/setup/local-development.md delete mode 100644 docs/development/testing/README.md delete mode 100644 docs/diagrams/architecture/.gitignore delete mode 100644 docs/diagrams/architecture/README-10.mmd delete mode 100644 docs/diagrams/architecture/README-11.mmd delete mode 100644 docs/diagrams/architecture/README-2.mmd delete mode 100644 docs/diagrams/architecture/README-3.mmd delete mode 100644 docs/diagrams/architecture/README-4.mmd delete mode 100644 docs/diagrams/architecture/README-5.mmd delete mode 100644 docs/diagrams/architecture/README-6.mmd delete mode 100644 docs/diagrams/architecture/README-7.mmd delete mode 100644 docs/diagrams/architecture/README-8.mmd delete mode 100644 docs/diagrams/architecture/README-9.mmd delete mode 100644 docs/diagrams/architecture/README.mmd delete mode 100644 docs/diagrams/architecture/configuration-and-packs-2.mmd delete mode 100644 docs/diagrams/architecture/configuration-and-packs-3.mmd delete mode 100644 docs/diagrams/architecture/configuration-and-packs-4.mmd delete mode 100644 docs/diagrams/architecture/configuration-and-packs-5.mmd delete mode 100644 docs/diagrams/architecture/configuration-and-packs.mmd delete mode 100644 docs/diagrams/architecture/core-init-and-runtime-2.mmd delete mode 100644 docs/diagrams/architecture/core-init-and-runtime-3.mmd delete mode 100644 docs/diagrams/architecture/core-init-and-runtime-4.mmd delete mode 100644 docs/diagrams/architecture/core-init-and-runtime-5.mmd delete mode 100644 docs/diagrams/architecture/core-init-and-runtime-6.mmd delete mode 100644 docs/diagrams/architecture/core-init-and-runtime-7.mmd delete mode 100644 docs/diagrams/architecture/core-init-and-runtime-8.mmd delete mode 100644 docs/diagrams/architecture/core-init-and-runtime.mmd delete mode 100644 docs/diagrams/architecture/database-and-storage-plugins-2.mmd delete mode 100644 docs/diagrams/architecture/database-and-storage-plugins-3.mmd delete mode 100644 docs/diagrams/architecture/database-and-storage-plugins-4.mmd delete mode 100644 docs/diagrams/architecture/database-and-storage-plugins-5.mmd delete mode 100644 docs/diagrams/architecture/database-and-storage-plugins.mmd delete mode 100644 docs/diagrams/architecture/distributed-querying-2.mmd delete mode 100644 docs/diagrams/architecture/distributed-querying-3.mmd delete mode 100644 docs/diagrams/architecture/distributed-querying-4.mmd delete mode 100644 docs/diagrams/architecture/distributed-querying.mmd delete mode 100644 docs/diagrams/architecture/eventing-framework-and-subscriptions-2.mmd delete mode 100644 docs/diagrams/architecture/eventing-framework-and-subscriptions-3.mmd delete mode 100644 docs/diagrams/architecture/eventing-framework-and-subscriptions-4.mmd delete mode 100644 docs/diagrams/architecture/eventing-framework-and-subscriptions-5.mmd delete mode 100644 docs/diagrams/architecture/eventing-framework-and-subscriptions-6.mmd delete mode 100644 docs/diagrams/architecture/eventing-framework-and-subscriptions-7.mmd delete mode 100644 docs/diagrams/architecture/eventing-framework-and-subscriptions.mmd delete mode 100644 docs/diagrams/architecture/extensions-and-ipc-2.mmd delete mode 100644 docs/diagrams/architecture/extensions-and-ipc-3.mmd delete mode 100644 docs/diagrams/architecture/extensions-and-ipc-4.mmd delete mode 100644 docs/diagrams/architecture/extensions-and-ipc-5.mmd delete mode 100644 docs/diagrams/architecture/extensions-and-ipc-6.mmd delete mode 100644 docs/diagrams/architecture/extensions-and-ipc-7.mmd delete mode 100644 docs/diagrams/architecture/extensions-and-ipc.mmd delete mode 100644 docs/diagrams/architecture/filesystem-and-path-utilities-2.mmd delete mode 100644 docs/diagrams/architecture/filesystem-and-path-utilities-3.mmd delete mode 100644 docs/diagrams/architecture/filesystem-and-path-utilities-4.mmd delete mode 100644 docs/diagrams/architecture/filesystem-and-path-utilities-5.mmd delete mode 100644 docs/diagrams/architecture/filesystem-and-path-utilities-6.mmd delete mode 100644 docs/diagrams/architecture/filesystem-and-path-utilities-7.mmd delete mode 100644 docs/diagrams/architecture/filesystem-and-path-utilities-8.mmd delete mode 100644 docs/diagrams/architecture/filesystem-and-path-utilities.mmd delete mode 100644 docs/diagrams/architecture/logging-and-query-observability-2.mmd delete mode 100644 docs/diagrams/architecture/logging-and-query-observability-3.mmd delete mode 100644 docs/diagrams/architecture/logging-and-query-observability-4.mmd delete mode 100644 docs/diagrams/architecture/logging-and-query-observability-5.mmd delete mode 100644 docs/diagrams/architecture/logging-and-query-observability-6.mmd delete mode 100644 docs/diagrams/architecture/logging-and-query-observability-7.mmd delete mode 100644 docs/diagrams/architecture/logging-and-query-observability.mmd delete mode 100644 docs/diagrams/architecture/remote-http-client-2.mmd delete mode 100644 docs/diagrams/architecture/remote-http-client-3.mmd delete mode 100644 docs/diagrams/architecture/remote-http-client-4.mmd delete mode 100644 docs/diagrams/architecture/remote-http-client-5.mmd delete mode 100644 docs/diagrams/architecture/remote-http-client-6.mmd delete mode 100644 docs/diagrams/architecture/remote-http-client.mmd delete mode 100644 docs/diagrams/architecture/sql-engine-and-virtual-tables-2.mmd delete mode 100644 docs/diagrams/architecture/sql-engine-and-virtual-tables-3.mmd delete mode 100644 docs/diagrams/architecture/sql-engine-and-virtual-tables-4.mmd delete mode 100644 docs/diagrams/architecture/sql-engine-and-virtual-tables-5.mmd delete mode 100644 docs/diagrams/architecture/sql-engine-and-virtual-tables-6.mmd delete mode 100644 docs/diagrams/architecture/sql-engine-and-virtual-tables.mmd delete mode 100644 docs/getting-started/.gitignore delete mode 100644 docs/getting-started/first-steps.md delete mode 100644 docs/getting-started/introduction.md delete mode 100644 docs/getting-started/prerequisites.md delete mode 100644 docs/getting-started/quick-start.md delete mode 100644 docs/reference/architecture/.gitignore delete mode 100644 docs/reference/architecture/README.md delete mode 100644 docs/reference/architecture/configuration-and-packs/configuration-and-packs.md delete mode 100644 docs/reference/architecture/core-init-and-runtime/core-init-and-runtime.md delete mode 100644 docs/reference/architecture/database-and-storage-plugins/database-and-storage-plugins.md delete mode 100644 docs/reference/architecture/distributed-querying/distributed-querying.md delete mode 100644 docs/reference/architecture/eventing-framework-and-subscriptions/eventing-framework-and-subscriptions.md delete mode 100644 docs/reference/architecture/extensions-and-ipc/extensions-and-ipc.md delete mode 100644 docs/reference/architecture/filesystem-and-path-utilities/filesystem-and-path-utilities.md delete mode 100644 docs/reference/architecture/logging-and-query-observability/logging-and-query-observability.md delete mode 100644 docs/reference/architecture/remote-http-client/remote-http-client.md delete mode 100644 docs/reference/architecture/sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 19d2db24188..00000000000 --- a/docs/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# osquery β€” OpenFrame Edition Documentation - -Welcome to the documentation for **osquery with OpenFrame** β€” the cross-platform OS instrumentation framework extended with AI-driven MSP automation by [Flamingo](https://flamingo.run) and [OpenFrame](https://openframe.ai). - ---- - -## πŸ“š Table of Contents - -- [Getting Started](#-getting-started) -- [Development](#-development) -- [Reference Architecture](#-reference-architecture) -- [Architecture Diagrams](#-architecture-diagrams) -- [Quick Links](#-quick-links) -- [Community](#-community) - ---- - -## πŸš€ Getting Started - -New to osquery? Start here. - -| Guide | Description | -|---|---| -| [Introduction](./getting-started/introduction.md) | What is osquery with OpenFrame? Features and target audience | -| [Prerequisites](./getting-started/prerequisites.md) | System requirements, supported platforms, required software | -| [Quick Start](./getting-started/quick-start.md) | Clone, build, and run osquery in under 10 minutes | -| [First Steps](./getting-started/first-steps.md) | Explore virtual tables, scheduled packs, FIM, extensions, and OpenFrame | - -**Recommended reading order:** Introduction β†’ Prerequisites β†’ Quick Start β†’ First Steps - ---- - -## πŸ›  Development - -Guides for contributors and developers building on or extending osquery. - -### Setup - -| Guide | Description | -|---|---| -| [Environment Setup](./development/setup/environment.md) | IDE recommendations, clangd, ccache, editor extensions | -| [Local Development](./development/setup/local-development.md) | Build configurations, debug flags, GDB/LLDB, VS Code debugging | - -### Architecture - -| Guide | Description | -|---|---| -| [Architecture Overview](./development/architecture/README.md) | High-level module breakdown, runtime lifecycle, design decisions | - -### Quality - -| Guide | Description | -|---|---| -| [Testing Guide](./development/testing/README.md) | GTest/GMock structure, running tests, writing unit and integration tests | -| [Security Best Practices](./development/security/README.md) | Auth patterns, AES-256-GCM, SQL authorizer, TLS, secrets management | - -### Contributing - -| Guide | Description | -|---|---| -| [Contributing Guidelines](./development/contributing/guidelines.md) | Code style, branch naming, commit format, PR process, virtual table creation | - ---- - -## πŸ“– Reference Architecture - -Deep technical documentation for each core subsystem β€” generated directly from source code analysis. - -| Module | Description | -|---|---| -| [Core Init And Runtime](./reference/architecture/core-init-and-runtime/core-init-and-runtime.md) | Process bootstrap, flags, watcher/worker model, watchdog | -| [SQL Engine And Virtual Tables](./reference/architecture/sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md) | SQLite engine, authorizer, virtual table binding, constraint pushdown | -| [Configuration And Packs](./reference/architecture/configuration-and-packs/configuration-and-packs.md) | Config loading, packs, schedulers, decorators, change detection | -| [Eventing Framework And Subscriptions](./reference/architecture/eventing-framework-and-subscriptions/eventing-framework-and-subscriptions.md) | Publisher/subscriber system, inotify, BPF, FSEvents, ETW | -| [Extensions And IPC](./reference/architecture/extensions-and-ipc/extensions-and-ipc.md) | Apache Thrift IPC, runtime plugin model, UUID routing | -| [Distributed Querying](./reference/architecture/distributed-querying/distributed-querying.md) | Remote SQL orchestration, TLS transport, fleet denylisting | -| [Remote HTTP Client](./reference/architecture/remote-http-client/remote-http-client.md) | Boost.Asio/Beast HTTPS client, TLS handshake, peer verification | -| [Logging And Query Observability](./reference/architecture/logging-and-query-observability/logging-and-query-observability.md) | Differential result tracking, JSON serialization, pluggable backends | -| [Database And Storage Plugins](./reference/architecture/database-and-storage-plugins/database-and-storage-plugins.md) | RocksDB persistent backend, ephemeral in-memory store, key-value interface | -| [Filesystem And Path Utilities](./reference/architecture/filesystem-and-path-utilities/filesystem-and-path-utilities.md) | Cross-platform file abstraction, glob resolution, permission enforcement | - ---- - -## πŸ—Ί Architecture Diagrams - -Visual Mermaid diagrams for each subsystem are available in: - -```text -docs/diagrams/architecture/ -``` - -Diagrams cover all major subsystems including the SQL engine, eventing framework, configuration pipeline, distributed querying, OpenFrame auth layer, and more. Open any `.mmd` file in a Mermaid-compatible viewer. - ---- - -## πŸ”— Quick Links - -| Resource | Link | -|---|---| -| [Project README](../README.md) | Main project overview, quick start, and features | -| [Contributing Guide](../CONTRIBUTING.md) | How to contribute code, docs, and virtual tables | -| [License](../LICENSE.md) | License information | - ---- - -## πŸ’¬ Community - -> We do **not** use GitHub Issues or GitHub Discussions. All support and collaboration happens on **OpenMSP Slack**. - -| Resource | Link | -|---|---| -| OpenMSP Community Slack | [Join here](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) | -| OpenMSP Website | [https://www.openmsp.ai/](https://www.openmsp.ai/) | -| OpenFrame Platform | [https://openframe.ai](https://openframe.ai) | -| Flamingo | [https://flamingo.run](https://flamingo.run) | - ---- - -*Documentation generated by [OpenFrame Doc Orchestrator](https://github.com/flamingo-stack/openframe-oss-tenant)* diff --git a/docs/development/.gitignore b/docs/development/.gitignore deleted file mode 100644 index a5d01be5236..00000000000 --- a/docs/development/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# VoltAgent temp files -temp/ - -# JSON intermediate files (except schema/config) -*.json -!*-schema.json -!*-config.json diff --git a/docs/development/README.md b/docs/development/README.md deleted file mode 100644 index 83ce9e027c8..00000000000 --- a/docs/development/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Development Documentation - -Welcome to the osquery with OpenFrame development documentation. This section covers everything you need to build, run, test, and contribute to the project. - ---- - -## Overview - -osquery is a C++ cross-platform OS instrumentation framework. The codebase is organized around a modular, plugin-driven architecture where every major subsystem is independently testable and extensible. - -The **OpenFrame** layer adds authentication and encryption components that integrate with the [Flamingo/OpenFrame MSP platform](https://openframe.ai). - ---- - -## Documentation Index - -| Guide | Description | -|---|---| -| [Environment Setup](setup/environment.md) | IDE recommendations, development tools, editor extensions | -| [Local Development](setup/local-development.md) | Clone, build, run locally, debug configuration | -| [Architecture Overview](architecture/README.md) | High-level architecture diagrams and core component descriptions | -| [Security Best Practices](security/README.md) | Auth patterns, encryption, secrets management | -| [Testing Guide](testing/README.md) | Test structure, running tests, writing new tests | -| [Contributing Guidelines](contributing/guidelines.md) | Code style, branch naming, PR process, commit format | - ---- - -## Technology Stack - -| Layer | Technology | -|---|---| -| **Language** | C++17 | -| **Build System** | CMake 3.21+ with Ninja | -| **SQL Engine** | SQLite (embedded, in-memory) | -| **IPC** | Apache Thrift (UNIX sockets / named pipes) | -| **Encryption** | OpenSSL (AES-256-GCM via OpenFrame layer) | -| **Networking** | Boost.Asio + Boost.Beast | -| **Event Systems** | inotify (Linux), FSEvents (macOS), ETW (Windows), BPF (Linux) | -| **Database** | RocksDB (persistent), ephemeral in-memory store | -| **Code Generation** | Python scripts for table schema, API, and amalgamation | -| **Testing** | Google Test + Google Mock | - ---- - -## Repository Structure - -```text -osquery/ -β”œβ”€β”€ osquery/ # Core C++ source β€” SQL, eventing, config, logging -β”‚ β”œβ”€β”€ core/ # Initialization, flags, watcher/worker model -β”‚ β”œβ”€β”€ sql/ # SQLite engine, virtual tables, authorizer -β”‚ β”œβ”€β”€ config/ # Configuration loading, packs, parsers -β”‚ β”œβ”€β”€ events/ # Eventing framework (publishers/subscribers) -β”‚ β”œβ”€β”€ database/ # Storage backend abstraction (RocksDB, ephemeral) -β”‚ β”œβ”€β”€ distributed/ # Distributed query orchestration -β”‚ β”œβ”€β”€ extensions/ # Thrift-based extension/IPC framework -β”‚ β”œβ”€β”€ remote/ # HTTP client, TLS transport -β”‚ β”œβ”€β”€ logger/ # Logging plugins and observability -β”‚ └── tables/ # Virtual table implementations -β”œβ”€β”€ openframe/ # OpenFrame authentication and encryption layer -β”œβ”€β”€ plugins/ # Logger, config, database, and distributed plugins -β”œβ”€β”€ libraries/ # CMake-managed third-party dependencies -β”œβ”€β”€ tools/ # Codegen scripts, CI tools, formatting -β”œβ”€β”€ tests/ # Integration test suite -└── external/ # Extension examples -``` - ---- - -## Quick Commands - -```bash -# Configure build -cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo - -# Build everything -cmake --build build --parallel $(nproc) - -# Run interactive shell -./build/osquery/osqueryi - -# Run tests -cmake --build build --target osquery_tests -cd build && ctest --output-on-failure -``` - ---- - -## Getting Help - -All development discussions happen in the **OpenMSP Slack community**: - -- [Join OpenMSP Slack](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) -- [https://www.openmsp.ai/](https://www.openmsp.ai/) - -> We do not use GitHub Issues or GitHub Discussions. All support and collaboration is on Slack. diff --git a/docs/development/architecture/README.md b/docs/development/architecture/README.md deleted file mode 100644 index a7b03cb1fce..00000000000 --- a/docs/development/architecture/README.md +++ /dev/null @@ -1,239 +0,0 @@ -# Architecture Overview - -osquery is built as a modular, plugin-driven system. Each subsystem is independently composable, testable, and extensible. The codebase is written in C++17 and targets Linux, macOS, and Windows. - ---- - -## High-Level Architecture - -```mermaid -graph TD - CLI["osqueryi / osqueryd"] --> Core["Core Init And Runtime"] - Core --> Config["Configuration And Packs"] - Core --> SQL["SQL Engine And Virtual Tables"] - Core --> Events["Eventing Framework"] - Core --> Logging["Logging And Observability"] - Core --> DB["Database And Storage Plugins"] - Core --> Dist["Distributed Querying"] - Core --> Ext["Extensions And IPC"] - Core --> HTTP["Remote HTTP Client"] - Core --> OF["OpenFrame Auth Layer"] - Config --> SQL - Config --> Events - SQL --> Logging - Events --> DB - Dist --> SQL - Dist --> HTTP - Ext --> SQL - Ext --> DB - OF --> HTTP -``` - ---- - -## Core Components - -| Module | Location | Description | -|---|---|---| -| **Core Init And Runtime** | `osquery/core/` | Process bootstrap, flags, watchdog, shutdown | -| **SQL Engine And Virtual Tables** | `osquery/sql/` | SQLite engine, authorizer, virtual table binding | -| **Configuration And Packs** | `osquery/config/` | Config loading, packs, parsers, scheduled queries | -| **Eventing Framework** | `osquery/events/` | Pub/sub system for OS events (inotify, BPF, ETW) | -| **Logging And Observability** | `osquery/logger/` | Differential logging, JSON serialization, logger plugins | -| **Database And Storage Plugins** | `osquery/database/` | Key-value storage (RocksDB, ephemeral) | -| **Distributed Querying** | `osquery/distributed/` | Remote SQL orchestration across fleets | -| **Extensions And IPC** | `osquery/extensions/` | Apache Thrift-based extension framework | -| **Remote HTTP Client** | `osquery/remote/` | Boost.Asio/Beast HTTPS client with TLS | -| **OpenFrame Auth Layer** | `openframe/` | JWT token management, AES-256-GCM encryption | -| **Virtual Tables** | `osquery/tables/` | 300+ OS-specific table implementations | - ---- - -## Runtime Lifecycle - -```mermaid -sequenceDiagram - participant Main - participant Initializer - participant Registry - participant Config - participant Extensions - participant Events - - Main->>Initializer: Construct(argc, argv) - Initializer->>Initializer: Parse flags - Initializer->>Registry: registryAndPluginInit() - Initializer->>Extensions: startExtensionManager() - Initializer->>Config: load() - Initializer->>Events: attachEvents() - Initializer->>Initializer: start() -``` - -osquery can operate in four runtime modes: - -| Mode | Binary | Description | -|---|---|---| -| **Interactive Shell** | `osqueryi` | REPL for ad-hoc SQL queries | -| **Daemon** | `osqueryd` | Background daemon with scheduled queries | -| **Watcher** | Internal | Supervisor process that monitors the worker | -| **Extension** | External | External plugin process communicating via Thrift | - ---- - -## SQL Engine and Virtual Tables - -The heart of osquery is its SQL engine β€” an in-memory SQLite instance hardened with a strict authorizer. - -```mermaid -graph TD - Query["SQL Query"] --> SQLPlugin["SQLiteSQLPlugin"] - SQLPlugin --> DBManager["SQLiteDBManager"] - DBManager --> DBInst["SQLiteDBInstance"] - DBInst --> SQLiteCore["In-Memory SQLite Engine"] - SQLiteCore --> VTabModule["sqlite3_module"] - VTabModule --> VirtualTable["VirtualTable Wrapper"] - VirtualTable --> TablePlugin["TablePlugin"] - TablePlugin --> OS["Operating System Data"] -``` - -**Key properties:** -- All queries run against an in-memory SQLite database -- A strict authorizer allowlists only safe SQL opcodes -- Virtual tables are attached dynamically from the registry -- Constraint pushdown and projection optimization reduce OS calls - ---- - -## Eventing Framework - -Event-driven monitoring is handled by a publish/subscribe system: - -```mermaid -graph TD - Publisher["EventPublisher"] --> EventFactory["EventFactory"] - Subscriber["EventSubscriber"] --> EventFactory - EventFactory --> Callback["EventCallback"] - Callback --> Storage["Database Storage"] - SQL["SELECT from events tables"] --> Subscriber -``` - -| Platform | Event Technology | -|---|---| -| Linux | inotify, BPF, Audit netlink | -| macOS | FSEvents, EndpointSecurity, OpenBSM | -| Windows | ETW, USN Journal, Windows Event Log | - ---- - -## Configuration and Scheduling - -```mermaid -graph TD - Source["Config Source"] --> ConfigCore["Config Singleton"] - ConfigCore --> Parsers["ConfigParserPlugins"] - Parsers --> Schedule["Scheduled Queries"] - Schedule --> Scheduler["Query Scheduler"] - Scheduler --> SQL["SQL Engine"] - SQL --> Logging["Logging And Observability"] -``` - -Configuration sources: -- **FilesystemConfigPlugin** β€” local JSON file with `.d/` fragment support -- **TLS Config Plugin** β€” remote configuration over HTTPS -- **Extension Config Plugin** β€” configuration provided by an extension process - ---- - -## OpenFrame Authentication Layer - -The `openframe/` directory contains the Flamingo/OpenFrame integration: - -```mermaid -graph LR - Provider["AuthorizationManagerProvider"] --> Manager["AuthorizationManager"] - Extractor["TokenExtractor"] --> Manager - Refresher["TokenRefresher"] --> Extractor - Manager --> Token["JWT Bearer Token"] - Token --> HTTP["Remote HTTP Client"] - EncSvc["EncryptionService"] --> OpenSSL["OpenSSL AES-256-GCM"] -``` - -| Component | Responsibility | -|---|---| -| `OpenframeAuthorizationManager` | Singleton token store with provider-controlled lifecycle | -| `OpenframeAuthorizationManagerProvider` | Sole factory for the token manager | -| `OpenframeEncryptionService` | AES-256-GCM decryption via OpenSSL | -| `OpenframeTokenExtractor` | Fetches tokens from OpenFrame services | -| `OpenframeTokenRefresher` | Background thread for token renewal | - ---- - -## Extension System - -Extensions communicate with osquery core via Apache Thrift over UNIX domain sockets (Linux/macOS) or named pipes (Windows): - -```mermaid -graph LR - Core["osquery Core"] --> Manager["Extension Manager"] - Manager --> Registry["RegistryFactory"] - ExtProc["Extension Process"] --> Manager - Manager --> ExtProc -``` - -Extensions can provide: -- Custom virtual tables -- Logger plugins -- Config plugins -- Distributed plugins - ---- - -## Data Flow: Query Execution to Log - -```mermaid -sequenceDiagram - participant Scheduler - participant SQL as SQL Engine - participant DB as Database - participant Logger - - Scheduler->>SQL: Execute scheduled query - SQL->>SQL: Run against virtual tables - SQL-->>Scheduler: QueryData (current results) - Scheduler->>DB: Load previous results - DB-->>Scheduler: Previous QueryData - Scheduler->>Scheduler: Compute DiffResults - Scheduler->>Logger: logQueryLogItem(diff) - Logger->>Logger: Serialize to JSON - Logger-->>Logger: Forward to backend sink -``` - ---- - -## Key Design Decisions - -| Decision | Rationale | -|---|---| -| **In-memory SQLite** | Zero persistent SQL state; each query is a fresh execution | -| **Plugin/Registry pattern** | All major subsystems (loggers, config, tables) are hot-swappable | -| **Watcher/Worker isolation** | Worker crashes don't bring down the watchdog supervisor | -| **Thrift for extensions** | Language-agnostic, versioned, cross-platform RPC | -| **Differential logging** | Only changed rows are logged, dramatically reducing volume | -| **AES-256-GCM encryption** | OpenFrame credentials protected with authenticated encryption | - ---- - -## Reference Documentation - -For deep dives into each module, see the reference architecture docs: - -- [Core Init And Runtime](./reference/architecture/core-init-and-runtime/core-init-and-runtime.md) -- [SQL Engine And Virtual Tables](./reference/architecture/sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md) -- [Configuration And Packs](./reference/architecture/configuration-and-packs/configuration-and-packs.md) -- [Eventing Framework And Subscriptions](./reference/architecture/eventing-framework-and-subscriptions/eventing-framework-and-subscriptions.md) -- [Extensions And IPC](./reference/architecture/extensions-and-ipc/extensions-and-ipc.md) -- [Distributed Querying](./reference/architecture/distributed-querying/distributed-querying.md) -- [Remote HTTP Client](./reference/architecture/remote-http-client/remote-http-client.md) -- [Logging And Query Observability](./reference/architecture/logging-and-query-observability/logging-and-query-observability.md) -- [Database And Storage Plugins](./reference/architecture/database-and-storage-plugins/database-and-storage-plugins.md) -- [Filesystem And Path Utilities](./reference/architecture/filesystem-and-path-utilities/filesystem-and-path-utilities.md) diff --git a/docs/development/contributing/guidelines.md b/docs/development/contributing/guidelines.md deleted file mode 100644 index de082c732cb..00000000000 --- a/docs/development/contributing/guidelines.md +++ /dev/null @@ -1,272 +0,0 @@ -# Contributing Guidelines - -Thank you for contributing to osquery with OpenFrame! This guide covers code style, branching, commit messages, and the pull request process. - ---- - -## Community First - -All collaboration happens on the **OpenMSP Slack community** β€” not GitHub Issues or GitHub Discussions. - -- πŸ’¬ **Join Slack**: [OpenMSP Community](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) -- 🌐 **OpenMSP**: [https://www.openmsp.ai/](https://www.openmsp.ai/) - -Before starting significant work, please discuss your changes in the Slack community to align with the team's roadmap. - ---- - -## Code Style and Conventions - -### C++ Standards - -- Use **C++17** features where appropriate -- Follow the existing code style in each file you modify -- All new code must pass `clang-format` with the repository's `.clang-format` config - -### Formatting - -osquery enforces `clang-format`. Run it before every commit: - -```bash -# Format a single file -clang-format -i path/to/your/file.cpp - -# Format all changed files (compared to main branch) -git diff --name-only main | grep -E '\.(cpp|h)$' | xargs clang-format -i - -# Check without modifying -clang-format --dry-run --Werror path/to/your/file.cpp -``` - -### Naming Conventions - -| Item | Convention | Example | -|---|---|---| -| Classes | `PascalCase` | `EventSubscriberPlugin` | -| Methods | `camelCase` | `generateRows()` | -| Member variables | `snake_case_` (trailing underscore) | `running_` | -| Constants | `kPascalCase` | `kSQLOpcodes` | -| Macros | `UPPER_SNAKE_CASE` | `DECLARE_FLAG` | -| Namespaces | `snake_case` | `osquery` | -| Files | `snake_case.cpp` / `snake_case.h` | `event_subscriber.cpp` | - -### Code Organization - -- Keep headers (`*.h`) as minimal as possible β€” forward declare where possible -- Use the `osquery` namespace for all production code -- Place tests in `tests/` subdirectories alongside the source -- New virtual tables go in `osquery/tables//` - -### Include Order - -Follow this include order, with blank lines between groups: - -```cpp -// 1. Standard library -#include -#include -#include - -// 2. Third-party libraries -#include -#include - -// 3. osquery headers -#include "osquery/core/core.h" -#include "osquery/sql/sql.h" - -// 4. Local headers (same directory) -#include "my_local_header.h" -``` - ---- - -## Branch Naming - -| Type | Pattern | Example | -|---|---|---| -| Feature | `feature/` | `feature/bpf-socket-events` | -| Bug fix | `fix/` | `fix/config-refresh-race` | -| OpenFrame integration | `openframe/` | `openframe/token-refresh-retry` | -| Documentation | `docs/` | `docs/virtual-table-guide` | -| Refactor | `refactor/` | `refactor/sql-authorizer` | -| Test | `test/` | `test/events-integration` | -| Release | `release/v` | `release/v5.13.0` | - -Always branch from the latest `main`: - -```bash -git checkout main -git pull origin main -git checkout -b feature/my-new-feature -``` - ---- - -## Commit Message Format - -Use the following format for all commits: - -```text -(): - - - - -``` - -### Types - -| Type | When to Use | -|---|---| -| `feat` | New feature or capability | -| `fix` | Bug fix | -| `docs` | Documentation changes only | -| `style` | Code formatting, no logic change | -| `refactor` | Code refactoring without behavior change | -| `test` | Adding or fixing tests | -| `perf` | Performance improvement | -| `chore` | Build, CI, tooling changes | -| `openframe` | OpenFrame platform-specific changes | - -### Scopes - -| Scope | Area | -|---|---| -| `core` | Core init and runtime | -| `sql` | SQL engine and virtual tables | -| `config` | Configuration and packs | -| `events` | Eventing framework | -| `logger` | Logging and observability | -| `db` | Database and storage | -| `distributed` | Distributed querying | -| `extensions` | Extension IPC | -| `http` | Remote HTTP client | -| `openframe` | OpenFrame auth layer | -| `tables` | Virtual table implementations | - -### Examples - -```text -feat(events): add BPF socket event publisher for Linux - -Implements a new BPF-based publisher that captures socket connect/accept -events and exposes them via the bpf_socket_events virtual table. - -Closes #1234 -``` - -```text -fix(openframe): handle token refresh failure with exponential backoff - -When OpenframeTokenRefresher encounters a network error, it now retries -with exponential backoff instead of immediately stopping the refresh loop. -``` - -```text -test(config): add pack discovery query unit tests - -Adds unit tests for the discovery query evaluation logic in Pack::shouldPackExecute() -covering platform, version, and shard constraints. -``` - ---- - -## Pull Request Process - -### Before Submitting - -- [ ] Branch is up-to-date with `main` -- [ ] All tests pass: `cd build && ctest --output-on-failure` -- [ ] Code is formatted: `clang-format --dry-run --Werror` -- [ ] Copyright headers are present on new files -- [ ] New virtual tables have integration tests in `tests/integration/tables/` -- [ ] OpenFrame-specific changes include updated documentation - -### PR Title - -Use the same format as commit messages: - -```text -feat(sql): add query result caching for repeated virtual table scans -``` - -### PR Description Template - -```markdown -## Summary - - -## Changes - - -## Testing - - -## Platform Support - - -## Checklist -- [ ] Tests pass (ctest) -- [ ] clang-format applied -- [ ] Documentation updated (if applicable) -- [ ] Discussed in OpenMSP Slack (if significant change) -``` - ---- - -## Copyright Headers - -All new source files must include a copyright header. Use the format already present in existing source files: - -```cpp -/** - * Copyright (c) 2014-present, The osquery authors - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ -``` - -The CI script `tools/ci/scripts/check_copyright_headers.py` enforces this on all pull requests. - ---- - -## Review Checklist for Reviewers - -When reviewing a PR: - -- [ ] Logic is correct and handles error paths -- [ ] No secrets or credentials in source -- [ ] SQL inputs validated (if applicable) -- [ ] Thread safety considered for shared state -- [ ] Platform-specific code properly guarded with `#ifdef` -- [ ] Tests added for new functionality -- [ ] No debug/temporary code left in -- [ ] Commit messages follow format convention -- [ ] Performance impact considered for hot paths (scheduler, SQL engine) - ---- - -## Adding a New Virtual Table - -1. Define the table schema in `osquery/tables//.table` -2. Run the code generator: - ```bash - python3 tools/codegen/gentable.py osquery/tables//.table - ``` -3. Implement the `generate()` method in `.cpp` -4. Register in the CMakefile for your category -5. Add an integration test in `tests/integration/tables/.cpp` -6. Test locally: - ```bash - cmake --build build --target osqueryi - ./build/osquery/osqueryi - osquery> SELECT * FROM ; - ``` - ---- - -## Code of Conduct - -Be respectful, collaborative, and constructive. All contributors are expected to maintain a professional and welcoming environment in both Slack and code reviews. diff --git a/docs/development/security/README.md b/docs/development/security/README.md deleted file mode 100644 index 87ae6702cae..00000000000 --- a/docs/development/security/README.md +++ /dev/null @@ -1,239 +0,0 @@ -# Security Best Practices - -This guide covers security patterns used in osquery with OpenFrame, including authentication, encryption, secrets management, input validation, and common vulnerability mitigations. - ---- - -## Authentication and Authorization - -### OpenFrame JWT Token Management - -The OpenFrame layer uses a provider-controlled singleton pattern to manage JWT authentication tokens: - -```mermaid -graph LR - Provider["AuthorizationManagerProvider"] --> Manager["AuthorizationManager"] - Extractor["TokenExtractor"] --> Manager - Refresher["TokenRefresher"] --> Extractor - Manager --> Token["Bearer JWT Token"] - Token --> HTTP["Outbound HTTPS Requests"] -``` - -**Key principles:** - -- `OpenframeAuthorizationManager` is **non-copyable** (`boost::noncopyable`) β€” tokens cannot be accidentally duplicated -- Only `OpenframeAuthorizationManagerProvider` can construct or destroy the manager (private constructor/destructor) -- The `OpenframeTokenRefresher` runs in a dedicated background thread using `std::atomic` for lock-free lifecycle control -- Token updates and reads are centralized through `updateToken()` and `getToken()` - -**Never:** -- Store JWT tokens in plain text files or environment variables without appropriate OS-level access controls -- Log token values, even at debug level -- Pass tokens as command-line arguments - ---- - -## Encryption - -### AES-256-GCM via OpenSSL - -The `OpenframeEncryptionService` provides symmetric encryption using AES-256-GCM β€” an authenticated encryption scheme: - -```cpp -// Initialize with a 256-bit secret key -OpenframeEncryptionService enc("my-32-byte-secret-key-goes-here!"); - -// Decrypt a Base64-encoded AES-256-GCM ciphertext -std::string plaintext = enc.decrypt(ciphertext_base64); -``` - -**Encryption properties:** - -| Property | Value | -|---|---| -| Algorithm | AES-256-GCM | -| Key size | 32 bytes (256-bit) | -| IV (nonce) size | 12 bytes (96-bit) | -| Authentication tag | 16 bytes (128-bit) | -| Encoding | Base64 | - -**Security rules:** -- The symmetric key must be exactly 32 bytes -- GCM authentication tag is verified on every decryption β€” tampered ciphertext throws `std::runtime_error` -- Never reuse nonces for the same key -- Store keys using OS keystore mechanisms or secret management systems, not in source code - ---- - -## SQL Security β€” The Authorizer - -osquery's embedded SQLite engine includes a strict **authorizer** (`sqliteAuthorizer`) that allowlists only safe SQL operations: - -```mermaid -graph TD - Prepare["sqlite3_prepare_v2"] --> Authorizer["sqliteAuthorizer"] - Authorizer -->|"Allowed"| Continue["Execute Statement"] - Authorizer -->|"Denied"| Reject["SQLITE_DENY"] -``` - -**Explicitly allowlisted:** -- `SELECT`, `READ` operations -- Controlled `INSERT`, `UPDATE`, `DELETE` (virtual tables only) -- Virtual table creation and drop -- Limited `PRAGMA` commands - -**Explicitly denied:** -- `SQLITE_ATTACH` β€” cannot attach external database files -- Any non-allowlisted opcode - -This prevents osquery from being abused to read arbitrary files via SQLite's `ATTACH` mechanism or execute unsafe operations. - ---- - -## Input Validation - -### SQL Query Validation - -All SQL submitted through the distributed querying or config channels is: -1. Parsed by the SQLite authorizer before execution -2. Validated against the virtual table schema -3. Size-limited by configuration constraints - -### Configuration Validation - -The `Config` singleton enforces: -- JSON max depth limits -- JSON max document size limits -- Comment stripping (non-standard JSON is rejected) -- Hash-based change detection to prevent replay attacks - -### Extension Registration - -Extension processes are validated during registration: -- SDK version compatibility is enforced -- Duplicate extension names are rejected -- Route UUIDs are generated server-side (not client-controlled) - ---- - -## Secrets Management - -### Environment-Level Secrets - -Never embed secrets in: -- Source code -- CMake configuration -- Build scripts -- Log output - -Use OS-level secrets management: - -| Platform | Recommended Tool | -|---|---| -| Linux | systemd credentials, HashiCorp Vault, AWS Secrets Manager | -| macOS | Keychain, AWS Secrets Manager | -| Windows | Windows Credential Manager, Azure Key Vault | - -### osquery Flag Files - -Sensitive flags (TLS certificates, enrollment secrets) should be stored in a flagfile with restricted permissions: - -```bash -# Create flagfile with restricted permissions -sudo touch /etc/osquery/osquery.secret.flags -sudo chmod 600 /etc/osquery/osquery.secret.flags -sudo chown root:root /etc/osquery/osquery.secret.flags - -# Write sensitive flags -echo "--tls_server_certs=/etc/osquery/server.pem" | sudo tee -a /etc/osquery/osquery.secret.flags -echo "--enroll_secret_path=/etc/osquery/enroll.secret" | sudo tee -a /etc/osquery/osquery.secret.flags -``` - -Reference the flagfile: - -```bash -sudo ./osqueryd --flagfile=/etc/osquery/osquery.secret.flags -``` - ---- - -## TLS / HTTPS Communication - -### Remote HTTP Client Security - -The `Remote HTTP Client` module (`osquery/remote/`) enforces: -- TLS by default for all remote communication -- Peer certificate verification -- Configurable cipher suites -- Custom CA certificate paths -- Timeout enforcement (no hanging connections) - -```text -Client::Options options; -options.ssl_connection(true) - .verify_peer(true) - .ca_verify_path("/etc/ssl/certs/ca-certificates.crt") - .timeout(30); -``` - -**Never disable peer verification in production.** The `verify_peer` flag should only be `false` in isolated development environments. - ---- - -## Common Vulnerabilities and Mitigations - -| Vulnerability | Mitigation in osquery | -|---|---| -| **SQL Injection** | Authorizer blocks unsafe opcodes; only allowlisted operations execute | -| **Credential Theft** | JWT tokens non-copyable, stored in singleton with restricted access | -| **Ciphertext Tampering** | AES-256-GCM authentication tag verified on every decrypt | -| **Privilege Escalation** | Watcher/Worker model isolates query execution from the supervisor | -| **External DB Injection** | `SQLITE_ATTACH` explicitly denied by authorizer | -| **Token Replay** | `OpenframeTokenRefresher` rotates tokens on a schedule | -| **Extension Impersonation** | SDK version check and UUID server-assignment prevent spoofing | -| **Resource Exhaustion** | Watchdog enforces CPU/memory limits and respawns workers | -| **Config Tampering** | Hash-based change detection and JSON size/depth limits | - ---- - -## Security Testing - -### Running Security-Relevant Tests - -```bash -# Build with tests enabled -cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DOSQUERY_BUILD_TESTS=ON -cmake --build build --parallel $(nproc) - -# Run all tests -cd build && ctest --output-on-failure - -# Run specific security-related tests -cd build && ctest -R "tls" --output-on-failure -cd build && ctest -R "config" --output-on-failure -cd build && ctest -R "sql" --output-on-failure -``` - -### Code Review Checklist for Security - -Before submitting any PR that touches security-sensitive code: - -- [ ] No secrets or credentials hardcoded -- [ ] All SQL inputs pass through the authorizer -- [ ] AES-GCM nonces are generated freshly (not reused) -- [ ] TLS peer verification is not disabled -- [ ] Error paths do not leak sensitive information in logs -- [ ] New extension APIs validate input size and type -- [ ] Thread-shared state uses proper synchronization primitives -- [ ] New config keys have size/depth validation - ---- - -## Reporting Security Issues - -Security vulnerabilities should be reported directly to the Flamingo team through the **OpenMSP Slack community**: - -- πŸ’¬ [Join OpenMSP Slack](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) -- 🌐 [https://www.openmsp.ai/](https://www.openmsp.ai/) - -> Do not open public GitHub Issues for security vulnerabilities. diff --git a/docs/development/setup/environment.md b/docs/development/setup/environment.md deleted file mode 100644 index 882e48c34fb..00000000000 --- a/docs/development/setup/environment.md +++ /dev/null @@ -1,200 +0,0 @@ -# Development Environment Setup - -This guide covers recommended IDE configurations, development tools, editor extensions, and environment variables for contributing to osquery with OpenFrame. - ---- - -## Recommended IDEs - -### Visual Studio Code (All Platforms) - -VS Code with the C++ extension pack is the recommended cross-platform IDE for osquery development. - -**Required Extensions:** - -| Extension | ID | Purpose | -|---|---|---| -| C/C++ | `ms-vscode.cpptools` | IntelliSense, debugging, syntax highlighting | -| CMake Tools | `ms-vscode.cmake-tools` | CMake integration, build configuration | -| CMake | `twxs.cmake` | CMake syntax highlighting | -| clangd | `llvm-vs-code-extensions.vscode-clangd` | Fast code navigation, diagnostics | - -**Optional but Recommended:** - -| Extension | ID | Purpose | -|---|---|---| -| GitLens | `eamodio.gitlens` | Enhanced Git history and annotations | -| Clang-Format | `xaver.clang-format` | Auto-formatting on save | -| Error Lens | `usernamehakki.error-lens` | Inline error display | - -**Install all at once:** - -```bash -code --install-extension ms-vscode.cpptools -code --install-extension ms-vscode.cmake-tools -code --install-extension twxs.cmake -code --install-extension llvm-vs-code-extensions.vscode-clangd -code --install-extension eamodio.gitlens -``` - -**Recommended `.vscode/settings.json`:** - -```json -{ - "cmake.buildDirectory": "${workspaceFolder}/build", - "cmake.generator": "Ninja", - "cmake.buildType": "Debug", - "editor.formatOnSave": true, - "clangd.arguments": [ - "--compile-commands-dir=${workspaceFolder}/build", - "--header-insertion=iwyu", - "--clang-tidy" - ], - "C_Cpp.intelliSenseEngine": "disabled" -} -``` - -> When using `clangd`, disable the built-in IntelliSense engine to avoid conflicts. - ---- - -### CLion (JetBrains) - -CLion provides first-class CMake support. Open the project root and CLion will auto-detect the `CMakeLists.txt`. - -**Recommended Settings:** - -- Set the CMake build directory to `build/` -- Enable `clang-format` in **Settings β†’ Editor β†’ Code Style β†’ C/C++** -- Use the built-in CMake tab for build configuration - ---- - -### Xcode (macOS Only) - -Generate an Xcode project from CMake: - -```bash -cmake -S . -B build-xcode -G Xcode -open build-xcode/osquery.xcodeproj -``` - ---- - -## Development Tools - -Install these tools before starting development: - -### All Platforms - -```bash -# Python tools for code generation -pip3 install jinja2 pexpect six -``` - -### Linux - -```bash -sudo apt-get install -y \ - clang-format \ - clang-tidy \ - ccache \ - valgrind \ - gdb -``` - -### macOS - -```bash -brew install \ - clang-format \ - ccache \ - llvm -``` - -### Windows - -- Install [LLVM](https://releases.llvm.org/) for `clang-format` and `clang-tidy` -- Configure `PATH` to include the LLVM `bin` directory - ---- - -## Setting Up ccache (Highly Recommended) - -`ccache` dramatically speeds up incremental rebuilds: - -```bash -# Install ccache -sudo apt-get install ccache # Linux -brew install ccache # macOS - -# Enable in CMake -cmake -S . -B build \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -``` - ---- - -## Generate compile_commands.json for clangd - -For full IntelliSense support with `clangd`, generate the compilation database: - -```bash -cmake -S . -B build \ - -G Ninja \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - -# Link to project root (required by clangd) -ln -sf build/compile_commands.json compile_commands.json -``` - ---- - -## Environment Variables for Development - -| Variable | Purpose | Example | -|---|---|---| -| `OSQUERY_EXTENSIONS_SOCKET` | Extension manager socket path | `/tmp/osquery.em` | -| `OSQUERY_CONFIG_PATH` | Override default config location | `./test.conf` | -| `OSQUERY_DB_PATH` | Custom database path for development | `/tmp/osquery-dev.db` | -| `OPENSSL_ROOT_DIR` | OpenSSL installation path (Windows/macOS) | `/usr/local/opt/openssl` | - -Set in your shell profile or export before building: - -```bash -export OPENSSL_ROOT_DIR="/usr/local/opt/openssl@3" -``` - -On Windows: - -```bash -set OPENSSL_ROOT_DIR=C:\OpenSSL-Win64 -``` - ---- - -## Code Formatting - -osquery enforces formatting with `clang-format`. The configuration is in `.clang-format` at the repository root. - -```bash -# Format a single file -clang-format -i osquery/core/init.cpp - -# Check formatting without modifying -clang-format --dry-run --Werror osquery/core/init.cpp - -# Format all C++ files in a directory -find osquery/core -name "*.cpp" -o -name "*.h" | xargs clang-format -i -``` - -The CI pipeline enforces formatting. Run format checks before submitting PRs. - ---- - -## Next Steps - -Once your environment is set up, proceed to the [Local Development Guide](local-development.md) for instructions on building, running, and debugging osquery locally. diff --git a/docs/development/setup/local-development.md b/docs/development/setup/local-development.md deleted file mode 100644 index da84e7144e8..00000000000 --- a/docs/development/setup/local-development.md +++ /dev/null @@ -1,279 +0,0 @@ -# Local Development Guide - -This guide walks you through cloning, building, running, and debugging osquery with OpenFrame on your local machine. - ---- - -## Clone the Repository - -```bash -git clone https://github.com/flamingo-stack/osquery.git -cd osquery -``` - ---- - -## Build Configurations - -### Debug Build (Recommended for Development) - -```bash -cmake -S . -B build \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - -cmake --build build --parallel $(nproc) -``` - -> Debug builds include full debug symbols and disable optimizations, making debugging straightforward. - -### RelWithDebInfo (Recommended for Testing) - -```bash -cmake -S . -B build \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo - -cmake --build build --parallel $(nproc) -``` - -### Release Build - -```bash -cmake -S . -B build \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Release - -cmake --build build --parallel $(nproc) -``` - -### Build with Tests Enabled - -```bash -cmake -S . -B build \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DOSQUERY_BUILD_TESTS=ON - -cmake --build build --parallel $(nproc) -``` - ---- - -## Running osquery Locally - -### Interactive Shell (`osqueryi`) - -```bash -./build/osquery/osqueryi -``` - -```sql -osquery> SELECT hostname, cpu_brand FROM system_info; -osquery> .tables -osquery> .schema processes -osquery> .quit -``` - -### Daemon (`osqueryd`) - -```bash -# Create a minimal development config -cat > /tmp/osquery-dev.conf <<'EOF' -{ - "options": { - "logger_plugin": "filesystem", - "logger_path": "/tmp/osquery-dev-logs", - "database_path": "/tmp/osquery-dev.db", - "schedule_splay_percent": 0 - }, - "schedule": { - "system_info": { - "query": "SELECT hostname, cpu_brand FROM system_info;", - "interval": 60 - } - } -} -EOF - -mkdir -p /tmp/osquery-dev-logs - -./build/osquery/osqueryd \ - --config_path=/tmp/osquery-dev.conf \ - --verbose \ - --ephemeral -``` - ---- - -## Development Flags - -These flags are useful during local development: - -| Flag | Purpose | -|---|---| -| `--verbose` | Enable verbose logging output | -| `--ephemeral` | Use in-memory database (no disk writes) | -| `--disable_watchdog` | Disable resource watchdog (easier debugging) | -| `--disable_events` | Disable event publishers (faster startup) | -| `--disable_logging` | Suppress all logger output | -| `--allow_unsafe` | Allow loading unsigned extensions | -| `--extensions_timeout=10` | Extension connection timeout in seconds | - -Example for a minimal debug session: - -```bash -./build/osquery/osqueryi \ - --verbose \ - --disable_events \ - --ephemeral -``` - ---- - -## Incremental Builds - -After making source changes, only rebuild changed targets: - -```bash -# Rebuild only the osqueryi binary -cmake --build build --target osqueryi - -# Rebuild only the osqueryd binary -cmake --build build --target osqueryd - -# Rebuild a specific test target -cmake --build build --target osquery_core_tests -``` - ---- - -## Debugging - -### Using GDB (Linux) - -```bash -# Build with debug symbols -cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -cmake --build build --parallel $(nproc) - -# Launch with GDB -gdb ./build/osquery/osqueryi - -# In GDB: -# (gdb) set args --verbose --ephemeral -# (gdb) break osquery::Initializer::start -# (gdb) run -``` - -### Using LLDB (macOS) - -```bash -lldb ./build/osquery/osqueryi - -# In LLDB: -# (lldb) settings set -- target.run-args "--verbose" "--ephemeral" -# (lldb) b osquery::Initializer::start -# (lldb) run -``` - -### Using VS Code Debugger - -Add to `.vscode/launch.json`: - -```json -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Debug osqueryi", - "type": "cppdbg", - "request": "launch", - "program": "${workspaceFolder}/build/osquery/osqueryi", - "args": ["--verbose", "--ephemeral", "--disable_events"], - "stopAtEntry": false, - "cwd": "${workspaceFolder}", - "environment": [], - "externalConsole": false, - "MIMode": "gdb", - "miDebuggerPath": "/usr/bin/gdb", - "setupCommands": [ - { - "description": "Enable pretty-printing", - "text": "-enable-pretty-printing", - "ignoreFailures": true - } - ] - } - ] -} -``` - ---- - -## Developing the OpenFrame Layer - -The OpenFrame components live in the `openframe/` directory: - -```text -openframe/ -β”œβ”€β”€ openframe_authorization_manager.h/.cpp # JWT token lifecycle -β”œβ”€β”€ openframe_authorization_manager_provider.h -β”œβ”€β”€ openframe_encryption_service.h/.cpp # AES-256-GCM via OpenSSL -β”œβ”€β”€ openframe_token_extractor.h/.cpp # Token acquisition -└── openframe_token_refresher.h/.cpp # Background token renewal -``` - -When modifying OpenFrame components, rebuild only the affected target: - -```bash -cmake --build build --target openframe_auth -``` - -Test the encryption service independently by linking against it in a test binary. - ---- - -## Working with Virtual Tables - -To add a new virtual table: - -1. Create schema and implementation in `osquery/tables//` -2. Register the table in the appropriate CMake target -3. Run the code generator to update schema definitions: - -```bash -python3 tools/codegen/gentable.py osquery/tables/my_category/my_table.table -``` - -4. Rebuild and test: - -```bash -cmake --build build --target osqueryi -./build/osquery/osqueryi -osquery> .tables my_table -``` - ---- - -## Log Output Locations - -| Mode | Default Log Path | -|---|---| -| Daemon (Linux) | `/var/log/osquery/` | -| Daemon (macOS) | `/var/log/osquery/` | -| Daemon (Windows) | `C:\ProgramData\osquery\log\` | -| Development override | Set `--logger_path=/tmp/my-logs` | - ---- - -## Clean Rebuild - -If you encounter stale build artifacts: - -```bash -rm -rf build/ -cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -cmake --build build --parallel $(nproc) -``` diff --git a/docs/development/testing/README.md b/docs/development/testing/README.md deleted file mode 100644 index 550d7e68a8e..00000000000 --- a/docs/development/testing/README.md +++ /dev/null @@ -1,310 +0,0 @@ -# Testing Guide - -osquery uses **Google Test** (GTest) and **Google Mock** (GMock) for unit and integration testing. Tests are organized alongside the source code in `tests/` subdirectories within each module. - ---- - -## Test Structure and Organization - -```text -osquery/ -β”œβ”€β”€ core/ -β”‚ β”œβ”€β”€ tests/ -β”‚ β”‚ β”œβ”€β”€ flags_tests.cpp -β”‚ β”‚ β”œβ”€β”€ query_tests.cpp -β”‚ β”‚ β”œβ”€β”€ system_test.cpp -β”‚ β”‚ └── tables_tests.cpp -β”œβ”€β”€ sql/ -β”‚ └── tests/ -β”‚ β”œβ”€β”€ sql.cpp -β”‚ └── virtual_table.cpp -β”œβ”€β”€ events/ -β”‚ └── tests/ -β”‚ β”œβ”€β”€ events_tests.cpp -β”‚ └── linux/ -β”‚ β”œβ”€β”€ inotify_tests.cpp -β”‚ └── bpf/ -β”œβ”€β”€ config/ -β”‚ └── tests/ -β”‚ β”œβ”€β”€ config_tests.cpp -β”‚ └── packs.cpp -β”œβ”€β”€ database/ -β”‚ └── tests/ -β”œβ”€β”€ distributed/ -β”‚ └── tests/ -└── extensions/ - └── tests/ - -tests/ -└── integration/ - └── tables/ # Integration tests for all virtual tables - β”œβ”€β”€ processes.cpp - β”œβ”€β”€ users.cpp - β”œβ”€β”€ listening_ports.cpp - └── ... - -plugins/ -└── */tests/ # Plugin-level tests -``` - -### Test Categories - -| Category | Location | Description | -|---|---|---| -| **Unit Tests** | `osquery/*/tests/` | Fast, isolated, no OS dependencies | -| **Integration Tests** | `tests/integration/tables/` | Live table queries against the real OS | -| **Benchmark Tests** | `osquery/*/benchmarks/` | Performance measurements | -| **Extension Tests** | `osquery/extensions/tests/` | IPC and Thrift extension round-trips | -| **Plugin Tests** | `plugins/*/tests/` | Logger, config, and database plugins | - ---- - -## Building Tests - -Tests must be explicitly enabled at CMake configure time: - -```bash -cmake -S . -B build \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DOSQUERY_BUILD_TESTS=ON - -cmake --build build --parallel $(nproc) -``` - ---- - -## Running Tests - -### Run All Tests - -```bash -cd build -ctest --output-on-failure -``` - -### Run Tests in Parallel - -```bash -cd build -ctest --output-on-failure --parallel $(nproc) -``` - -### Run a Specific Test Suite - -```bash -cd build - -# Run core tests -ctest -R "osquery_core_tests" --output-on-failure - -# Run SQL tests -ctest -R "osquery_sql_tests" --output-on-failure - -# Run config tests -ctest -R "osquery_config_tests" --output-on-failure - -# Run events tests -ctest -R "osquery_events_tests" --output-on-failure - -# Run integration table tests -ctest -R "integration" --output-on-failure -``` - -### Run Tests by Keyword - -```bash -cd build -ctest -R "tls" --output-on-failure -ctest -R "database" --output-on-failure -ctest -R "extension" --output-on-failure -``` - -### Verbose Test Output - -```bash -cd build -ctest -V -R "osquery_core_tests" -``` - ---- - -## Writing New Tests - -### Unit Test Template - -```cpp -#include -#include - -#include "osquery/my_module/my_class.h" - -namespace osquery { - -class MyClassTest : public ::testing::Test { - protected: - void SetUp() override { - // Per-test setup - } - - void TearDown() override { - // Per-test cleanup - } -}; - -TEST_F(MyClassTest, BasicOperation) { - MyClass obj; - EXPECT_EQ(obj.doSomething(), expected_value); -} - -TEST_F(MyClassTest, HandlesEdgeCase) { - MyClass obj; - EXPECT_THROW(obj.doSomethingInvalid(), std::runtime_error); -} - -} // namespace osquery -``` - -### Virtual Table Integration Test Template - -```cpp -#include "tests/integration/tables/helper.h" - -namespace osquery { -namespace table_tests { - -class MyTableTest : public testing::Test {}; - -TEST_F(MyTableTest, test_sanity) { - // Validate table schema and basic query - auto const data = execute_query("SELECT * FROM my_table;"); - ASSERT_GT(data.size(), 0ul); - - // Validate specific column presence and types - ValidateSchema( - data, - { - {"column_one", BIGINT_TYPE, CHECK_GT, 0, CHECK_LT, INT_MAX}, - {"column_two", TEXT_TYPE, CHECK_NON_EMPTY}, - } - ); -} - -} // namespace table_tests -} // namespace osquery -``` - -### Using GMock for Dependencies - -```cpp -#include -#include "osquery/database/database.h" - -class MockDatabase : public IDatabaseInterface { - public: - MOCK_METHOD(Status, putBatch, - (const std::string& domain, - const DatabaseStringValueList& data), - (override)); - - MOCK_METHOD(Status, get, - (const std::string& domain, - const std::string& key, - std::string& value), - (const, override)); -}; - -TEST(MyServiceTest, PersistsData) { - MockDatabase db; - EXPECT_CALL(db, putBatch(testing::_, testing::_)) - .Times(1) - .WillOnce(testing::Return(Status::success())); - - MyService service(db); - auto result = service.persist("key", "value"); - EXPECT_TRUE(result.ok()); -} -``` - ---- - -## Test Helpers - -The integration test suite includes shared helpers in `tests/integration/tables/helper.h`: - -| Helper | Purpose | -|---|---| -| `execute_query(sql)` | Execute SQL and return rows | -| `ValidateSchema(data, validators)` | Assert column types and value ranges | -| `CHECK_GT`, `CHECK_LT` | Numeric range validators | -| `CHECK_NON_EMPTY` | String non-emptiness check | - ---- - -## Benchmark Tests - -Benchmarks measure performance of hot paths: - -```bash -# Build benchmarks -cmake -S . -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DOSQUERY_BUILD_BENCHMARKS=ON -cmake --build build --parallel $(nproc) - -# Run SQL benchmarks -./build/osquery/sql/benchmarks/sql_benchmarks - -# Run events benchmarks -./build/osquery/events/benchmarks/events_benchmarks -``` - ---- - -## Coverage Requirements - -While no mandatory coverage threshold is enforced in CI currently, the project encourages: - -- **Unit tests** for all new core module functionality -- **Integration tests** for every new virtual table -- **Error path coverage** for all error handling branches -- **Mock-based tests** for components with external dependencies (network, database) - -When contributing new virtual tables, a corresponding integration test in `tests/integration/tables/` is expected. - ---- - -## Platform-Specific Tests - -Some tests only run on specific platforms: - -```cpp -// Linux-only test -#ifdef __linux__ -TEST_F(AuditTest, LinuxAuditPublisher) { - // Linux-specific audit test -} -#endif - -// macOS-only test -#ifdef __APPLE__ -TEST_F(FSEventsTest, FseventsPublisher) { - // macOS FSEvents test -} -#endif -``` - -The CMake build system automatically includes platform-appropriate test targets. - ---- - -## Continuous Integration - -Tests run automatically in CI on every pull request. The CI pipeline: - -1. Builds with `-DOSQUERY_BUILD_TESTS=ON` -2. Runs `ctest --output-on-failure` -3. Enforces `clang-format` compliance -4. Checks copyright headers (via `tools/ci/scripts/check_copyright_headers.py`) - -All tests must pass before a PR can be merged. diff --git a/docs/diagrams/architecture/.gitignore b/docs/diagrams/architecture/.gitignore deleted file mode 100644 index b4f4c7a6c9c..00000000000 --- a/docs/diagrams/architecture/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# CodeWiki temp files (dependency graphs can be 7GB+) -temp/ -dependency_graphs/ - -# JSON intermediate files (except schema/config) -*.json -!*-schema.json -!*-config.json diff --git a/docs/diagrams/architecture/README-10.mmd b/docs/diagrams/architecture/README-10.mmd deleted file mode 100644 index 9de352b6a66..00000000000 --- a/docs/diagrams/architecture/README-10.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart TD - Caller["Module"] --> Client["HTTP Client"] - Client --> TLS["TLS Handshake"] - TLS --> Remote["Remote Server"] diff --git a/docs/diagrams/architecture/README-11.mmd b/docs/diagrams/architecture/README-11.mmd deleted file mode 100644 index dbc77c3eaf7..00000000000 --- a/docs/diagrams/architecture/README-11.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart TD - Caller["Subsystem"] --> FS["Filesystem API"] - FS --> POSIX["POSIX Implementation"] - FS --> Windows["Windows Implementation"] diff --git a/docs/diagrams/architecture/README-2.mmd b/docs/diagrams/architecture/README-2.mmd deleted file mode 100644 index 15cf680aafe..00000000000 --- a/docs/diagrams/architecture/README-2.mmd +++ /dev/null @@ -1,11 +0,0 @@ -flowchart TD - Start["Process Start"] --> Init["Initializer"] - Init --> Flags["Parse Flags"] - Init --> Registry["Initialize Registry"] - Init --> DBInit["Initialize Database"] - Init --> ExtMgr["Start Extension Manager"] - Init --> ConfigLoad["Load Configuration"] - ConfigLoad --> Schedule["Build Schedule"] - Schedule --> QueryExec["Execute Queries"] - QueryExec --> Log["Log Results"] - Log --> End["Runtime Loop"] diff --git a/docs/diagrams/architecture/README-3.mmd b/docs/diagrams/architecture/README-3.mmd deleted file mode 100644 index 2bec5262176..00000000000 --- a/docs/diagrams/architecture/README-3.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - Query["SQL Query"] --> SQLite["SQLite Engine"] - SQLite --> Module["sqlite3_module"] - Module --> VirtualTable["VirtualTable Wrapper"] - VirtualTable --> TablePlugin["TablePlugin"] - TablePlugin --> OS["Operating System Data"] - OS --> Rows["Rows Returned"] diff --git a/docs/diagrams/architecture/README-4.mmd b/docs/diagrams/architecture/README-4.mmd deleted file mode 100644 index 7cb68c8186b..00000000000 --- a/docs/diagrams/architecture/README-4.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - Source["Config Source (File / TLS / Extension)"] --> ConfigCore["Config Singleton"] - ConfigCore --> Parsers["ConfigParserPlugins"] - Parsers --> Schedule["Scheduled Queries"] - Schedule --> Scheduler["Query Scheduler"] - Scheduler --> SQL["SQL Engine"] diff --git a/docs/diagrams/architecture/README-5.mmd b/docs/diagrams/architecture/README-5.mmd deleted file mode 100644 index 15b3d86b3cc..00000000000 --- a/docs/diagrams/architecture/README-5.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - Publisher["EventPublisher"] --> EventFactory["EventFactory"] - Subscriber["EventSubscriber"] --> EventFactory - EventFactory --> Callback["EventCallback"] - Callback --> Storage["Database Storage"] - SQL["SELECT *_events"] --> Subscriber diff --git a/docs/diagrams/architecture/README-6.mmd b/docs/diagrams/architecture/README-6.mmd deleted file mode 100644 index f6966640dec..00000000000 --- a/docs/diagrams/architecture/README-6.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - Exec["Query Execution"] --> Diff["DiffResults"] - Diff --> LogItem["QueryLogItem"] - LogItem --> Serialize["JSON Serialization"] - Serialize --> Logger["LoggerPlugin"] - Logger --> Sink["External Backend"] diff --git a/docs/diagrams/architecture/README-7.mmd b/docs/diagrams/architecture/README-7.mmd deleted file mode 100644 index b995aa1bd07..00000000000 --- a/docs/diagrams/architecture/README-7.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - Modules["Core Modules"] --> Interface["IDatabaseInterface"] - Interface --> Plugin["Database Plugin"] - Plugin --> RocksDB["Persistent Backend"] - Plugin --> Ephemeral["In-Memory Backend"] diff --git a/docs/diagrams/architecture/README-8.mmd b/docs/diagrams/architecture/README-8.mmd deleted file mode 100644 index f90914fba8e..00000000000 --- a/docs/diagrams/architecture/README-8.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - Server["Central Server"] --> TLS["TLS Distributed Plugin"] - TLS --> Core["Distributed Core"] - Core --> SQL["SQL Engine"] - SQL --> Results["Results"] - Results --> TLS - TLS --> Server diff --git a/docs/diagrams/architecture/README-9.mmd b/docs/diagrams/architecture/README-9.mmd deleted file mode 100644 index 05f3210aec0..00000000000 --- a/docs/diagrams/architecture/README-9.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart LR - Core["osquery Core"] --> Manager["Extension Manager"] - Extension["External Extension"] --> Manager - Manager --> Registry["RegistryFactory"] - Core --> Extension diff --git a/docs/diagrams/architecture/README.mmd b/docs/diagrams/architecture/README.mmd deleted file mode 100644 index abd03ad683d..00000000000 --- a/docs/diagrams/architecture/README.mmd +++ /dev/null @@ -1,22 +0,0 @@ -flowchart TD - CLI["osqueryi / osqueryd"] --> Core["Core Init And Runtime"] - - Core --> Config["Configuration And Packs"] - Core --> SQL["SQL Engine And Virtual Tables"] - Core --> Events["Eventing Framework"] - Core --> Logging["Logging And Observability"] - Core --> DB["Database And Storage Plugins"] - Core --> Distributed["Distributed Querying"] - Core --> Extensions["Extensions And IPC"] - Core --> HTTP["Remote HTTP Client"] - Core --> FS["Filesystem And Path Utilities"] - - Config --> SQL - Config --> Events - SQL --> Logging - Events --> DB - Distributed --> SQL - Distributed --> HTTP - Logging --> DB - Extensions --> SQL - Extensions --> DB diff --git a/docs/diagrams/architecture/configuration-and-packs-2.mmd b/docs/diagrams/architecture/configuration-and-packs-2.mmd deleted file mode 100644 index 058d1366de2..00000000000 --- a/docs/diagrams/architecture/configuration-and-packs-2.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - Start["ConfigRefreshRunner Start"] --> Wait["Sleep refresh_sec"] - Wait --> Check["Interrupted?"] - Check -->|No| Refresh["Config.refresh()"] - Refresh --> Wait - Check -->|Yes| End["End"] diff --git a/docs/diagrams/architecture/configuration-and-packs-3.mmd b/docs/diagrams/architecture/configuration-and-packs-3.mmd deleted file mode 100644 index 14e23b18fcf..00000000000 --- a/docs/diagrams/architecture/configuration-and-packs-3.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - ConfigUpdate["Config Update"] --> BuildPacks["Create Pack Objects"] - BuildPacks --> DiscoveryCheck["Discovery Queries"] - DiscoveryCheck --> ActivePack["Active Pack"] - ActivePack --> Scheduler["ScheduledQuery Execution"] - Scheduler --> Performance["Query Performance Recording"] - Scheduler --> Denylist["Denylist On Failure"] diff --git a/docs/diagrams/architecture/configuration-and-packs-4.mmd b/docs/diagrams/architecture/configuration-and-packs-4.mmd deleted file mode 100644 index 31827beb892..00000000000 --- a/docs/diagrams/architecture/configuration-and-packs-4.mmd +++ /dev/null @@ -1,8 +0,0 @@ -flowchart TD - ConfigLoad["Config Load"] --> RunLoad["Run Load Decorators"] - QueryExec["Query Execution"] --> RunAlways["Run Always Decorators"] - IntervalTick["Interval Timer"] --> RunInterval["Run Interval Decorators"] - RunLoad --> Decorations[("Decoration Store")] - RunAlways --> Decorations - RunInterval --> Decorations - Decorations --> Logger["Status And Query Logs"] diff --git a/docs/diagrams/architecture/configuration-and-packs-5.mmd b/docs/diagrams/architecture/configuration-and-packs-5.mmd deleted file mode 100644 index 91bbcadc931..00000000000 --- a/docs/diagrams/architecture/configuration-and-packs-5.mmd +++ /dev/null @@ -1,11 +0,0 @@ -flowchart TD - Source["Config Source"] --> GenConfig["genConfig()"] - GenConfig --> Update["Config.update()"] - Update --> Purge["Purge Stale State"] - Purge --> UpdateSource["updateSource()"] - UpdateSource --> ParseJSON["Validate And Parse JSON"] - ParseJSON --> BuildSchedule["Create Packs And Schedule"] - BuildSchedule --> ApplyParsers["Apply ConfigParserPlugins"] - ApplyParsers --> Reconfigure["Registry.configure()"] - Reconfigure --> EventNotify["EventFactory.configUpdate()"] - EventNotify --> Ready["Runtime Updated"] diff --git a/docs/diagrams/architecture/configuration-and-packs.mmd b/docs/diagrams/architecture/configuration-and-packs.mmd deleted file mode 100644 index 3a720d6d6c3..00000000000 --- a/docs/diagrams/architecture/configuration-and-packs.mmd +++ /dev/null @@ -1,15 +0,0 @@ -flowchart TD - ConfigPlugin["Config Plugin Registry"] --> ConfigCore["Config Singleton"] - ConfigCore --> RefreshRunner["ConfigRefreshRunner"] - ConfigCore --> Schedule["Schedule"] - Schedule --> Pack["Pack"] - ConfigCore --> Parsers["ConfigParserPlugins"] - Parsers --> OptionsParser["Options Parser"] - Parsers --> FilePathsParser["File Paths Parser"] - Parsers --> DecoratorsParser["Decorators Parser"] - Parsers --> EventsParser["Events Parser"] - Parsers --> ViewsParser["Views Parser"] - ConfigCore --> Database[("Persistent Database")] - Schedule --> Logging["Logging And Query Observability"] - Schedule --> SQLEngine["SQL Engine And Virtual Tables"] - Parsers --> EventFramework["Eventing Framework"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-2.mmd b/docs/diagrams/architecture/core-init-and-runtime-2.mmd deleted file mode 100644 index 12303b48e3e..00000000000 --- a/docs/diagrams/architecture/core-init-and-runtime-2.mmd +++ /dev/null @@ -1,15 +0,0 @@ -sequenceDiagram - participant Main - participant Initializer - participant Registry - participant Config - participant Extensions - participant Events - - Main->>Initializer: Construct(argc, argv) - Initializer->>Initializer: Parse flags - Initializer->>Registry: registryAndPluginInit() - Initializer->>Extensions: startExtensionManager() - Initializer->>Config: load() - Initializer->>Events: attachEvents() - Initializer->>Initializer: start() diff --git a/docs/diagrams/architecture/core-init-and-runtime-3.mmd b/docs/diagrams/architecture/core-init-and-runtime-3.mmd deleted file mode 100644 index 6f1c2a8d3fd..00000000000 --- a/docs/diagrams/architecture/core-init-and-runtime-3.mmd +++ /dev/null @@ -1,8 +0,0 @@ -flowchart LR - Macro["FLAG / CLI_FLAG Macros"] --> FlagCreate["Flag::create()"] - FlagCreate --> FlagRegistry["Internal Flag Map"] - - CLI["Command Line"] --> Parse["ParseCommandLineFlags"] - Config["Config Options"] --> Update["Flag::updateValue()"] - - FlagRegistry --> Runtime["Runtime Access"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-4.mmd b/docs/diagrams/architecture/core-init-and-runtime-4.mmd deleted file mode 100644 index 3464c298d66..00000000000 --- a/docs/diagrams/architecture/core-init-and-runtime-4.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - Watcher["Watcher Process"] --> Spawn["Spawn Worker"] - Spawn --> Monitor["Monitor CPU And Memory"] - Monitor --> Check{{"Limit Exceeded?"}} - Check -->|"No"| Continue["Continue Monitoring"] - Check -->|"Yes"| Kill["Stop Worker"] - Kill --> Respawn["Respawn Worker"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-5.mmd b/docs/diagrams/architecture/core-init-and-runtime-5.mmd deleted file mode 100644 index 50b08395e59..00000000000 --- a/docs/diagrams/architecture/core-init-and-runtime-5.mmd +++ /dev/null @@ -1,9 +0,0 @@ -flowchart TD - AnyThread["Any Thread"] --> Request["requestShutdown()"] - Request --> Signal["Condition Variable"] - Signal --> MainThread["Main Thread"] - MainThread --> StopServices["Dispatcher::stopServices()"] - StopServices --> Join["Join Services"] - Join --> StopEvents["EventFactory::end()"] - StopEvents --> CloseDB["shutdownDatabase()"] - CloseDB --> Exit["Return Exit Code"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-6.mmd b/docs/diagrams/architecture/core-init-and-runtime-6.mmd deleted file mode 100644 index 83e50573800..00000000000 --- a/docs/diagrams/architecture/core-init-and-runtime-6.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart LR - Flag["host_identifier Flag"] --> Mode{{"Mode"}} - Mode -->|"uuid"| Hardware["Hardware UUID"] - Mode -->|"instance"| Instance["Instance UUID"] - Mode -->|"ephemeral"| Ephemeral["Random UUID"] - Mode -->|"specified"| Specified["Configured Identifier"] - Mode -->|"default"| Hostname["System Hostname"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-7.mmd b/docs/diagrams/architecture/core-init-and-runtime-7.mmd deleted file mode 100644 index 0533bd82ae3..00000000000 --- a/docs/diagrams/architecture/core-init-and-runtime-7.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - SQL["SQL Query"] --> SQLite["SQLite Virtual Table API"] - SQLite --> QueryContext["QueryContext"] - QueryContext --> ConstraintMap["ConstraintMap"] - ConstraintMap --> ConstraintList["ConstraintList"] - ConstraintList --> TablePlugin["TablePlugin::generate()"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-8.mmd b/docs/diagrams/architecture/core-init-and-runtime-8.mmd deleted file mode 100644 index dd5a802fe24..00000000000 --- a/docs/diagrams/architecture/core-init-and-runtime-8.mmd +++ /dev/null @@ -1,9 +0,0 @@ -flowchart TD - Core["Core Init And Runtime"] - - Core --> Config["Configuration And Packs"] - Core --> Logging["Logging And Query Observability"] - Core --> Database["Database And Storage Plugins"] - Core --> SQL["SQL Engine And Virtual Tables"] - Core --> Events["Eventing Framework And Subscriptions"] - Core --> Extensions["Extensions And IPC"] diff --git a/docs/diagrams/architecture/core-init-and-runtime.mmd b/docs/diagrams/architecture/core-init-and-runtime.mmd deleted file mode 100644 index 75bf4fac1e6..00000000000 --- a/docs/diagrams/architecture/core-init-and-runtime.mmd +++ /dev/null @@ -1,16 +0,0 @@ -flowchart TD - Main["Main Entry Point"] --> Initializer["Initializer"] - - Initializer --> Flags["Flag System"] - Initializer --> Registry["Registry And Plugins"] - Initializer --> Database["Database Initialization"] - Initializer --> ExtensionManager["Extension Manager"] - Initializer --> Events["Event Factory"] - Initializer --> Watchdog["Watcher / Worker Model"] - - Watchdog --> Worker["Worker Process"] - Watchdog --> Extensions["Extension Processes"] - - Worker --> Tables["Virtual Tables"] - Worker --> Scheduler["Query Scheduler"] - Worker --> Logger["Logger Plugins"] diff --git a/docs/diagrams/architecture/database-and-storage-plugins-2.mmd b/docs/diagrams/architecture/database-and-storage-plugins-2.mmd deleted file mode 100644 index 605504b8b71..00000000000 --- a/docs/diagrams/architecture/database-and-storage-plugins-2.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart TD - DB["EphemeralDatabasePlugin"] --> Map["std::map>"] - Map --> DomainMap["Domain Map"] - DomainMap --> KeyValue["boost::variant"] diff --git a/docs/diagrams/architecture/database-and-storage-plugins-3.mmd b/docs/diagrams/architecture/database-and-storage-plugins-3.mmd deleted file mode 100644 index cea2ef35f02..00000000000 --- a/docs/diagrams/architecture/database-and-storage-plugins-3.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - Start["Startup"] --> CheckFlag{{"disable_database?"}} - CheckFlag -->|"No"| UseRocksDB["Activate RocksDB Plugin"] - CheckFlag -->|"Yes"| UseEphemeral["Activate Ephemeral Plugin"] - UseRocksDB --> InitDone["Database Initialized"] - UseEphemeral --> InitDone diff --git a/docs/diagrams/architecture/database-and-storage-plugins-4.mmd b/docs/diagrams/architecture/database-and-storage-plugins-4.mmd deleted file mode 100644 index 721f5928c67..00000000000 --- a/docs/diagrams/architecture/database-and-storage-plugins-4.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - ResetCall["resetDatabase()"] --> LockWrite["Acquire Write Lock"] - LockWrite --> TearDown["Plugin tearDown()"] - TearDown --> SetUp["Plugin setUp()"] - SetUp --> Unlock["Release Lock"] diff --git a/docs/diagrams/architecture/database-and-storage-plugins-5.mmd b/docs/diagrams/architecture/database-and-storage-plugins-5.mmd deleted file mode 100644 index eed79ca554a..00000000000 --- a/docs/diagrams/architecture/database-and-storage-plugins-5.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - ReadVersion["Read results_version"] --> Compare["Compare with target version"] - Compare -->|"Older"| RunMigration["Execute migrateVxVy()"] - RunMigration --> UpdateVersion["Persist new version"] - UpdateVersion --> Compare - Compare -->|"Match"| Done["Migration Complete"] diff --git a/docs/diagrams/architecture/database-and-storage-plugins.mmd b/docs/diagrams/architecture/database-and-storage-plugins.mmd deleted file mode 100644 index a4efc77fa93..00000000000 --- a/docs/diagrams/architecture/database-and-storage-plugins.mmd +++ /dev/null @@ -1,19 +0,0 @@ -flowchart TD - CoreModules["Core And Feature Modules"] --> DBInterface["IDatabaseInterface"] - DBInterface --> OsqueryDB["OsqueryDatabase"] - OsqueryDB --> RegistryLayer["Database Plugin Registry"] - RegistryLayer --> RocksDBPlugin["RocksDB Plugin (Persistent)"] - RegistryLayer --> EphemeralPlugin["Ephemeral Database Plugin (In-Memory)"] - - subgraph storage_domains["Logical Storage Domains"] - PersistentSettings["configurations"] - Queries["queries"] - Events["events"] - Logs["logs"] - Carves["carves"] - Distributed["distributed"] - QueryPerformance["query_performance"] - end - - RocksDBPlugin --> storage_domains - EphemeralPlugin --> storage_domains diff --git a/docs/diagrams/architecture/distributed-querying-2.mmd b/docs/diagrams/architecture/distributed-querying-2.mmd deleted file mode 100644 index af1e3628736..00000000000 --- a/docs/diagrams/architecture/distributed-querying-2.mmd +++ /dev/null @@ -1,11 +0,0 @@ -flowchart TD - Start["Pull Updates"] --> Accept["acceptWork()"] - Accept --> Pending{"Pending Queries?"} - Pending -->|"Yes"| Run["runQueries()"] - Run --> Deny{"Denylisted?"} - Deny -->|"No"| Execute["Execute via SQL Engine"] - Execute --> Record["recordQueryPerformance()"] - Record --> Queue["addResult()"] - Queue --> Flush["flushCompleted()"] - Deny -->|"Yes"| Skip["Skip Execution"] - Flush --> End["Cycle Complete"] diff --git a/docs/diagrams/architecture/distributed-querying-3.mmd b/docs/diagrams/architecture/distributed-querying-3.mmd deleted file mode 100644 index 2850535d935..00000000000 --- a/docs/diagrams/architecture/distributed-querying-3.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - Incoming["Incoming Query"] --> Check["checkAndSetAsRunning()"] - Check --> Running{"Already Running?"} - Running -->|"Within Duration"| Block["Denylisted"] - Running -->|"Expired"| Allow["Allow Execution"] - Check -->|"First Time"| Allow diff --git a/docs/diagrams/architecture/distributed-querying-4.mmd b/docs/diagrams/architecture/distributed-querying-4.mmd deleted file mode 100644 index da9275ef2f4..00000000000 --- a/docs/diagrams/architecture/distributed-querying-4.mmd +++ /dev/null @@ -1,11 +0,0 @@ -sequenceDiagram - participant Node - participant TLS as "TLS Distributed Plugin" - participant Server - - Node->>TLS: pullUpdates() - TLS->>Server: POST read endpoint - Server->>TLS: JSON queries - TLS->>Node: return JSON - Node->>TLS: writeResults(JSON) - TLS->>Server: POST write endpoint diff --git a/docs/diagrams/architecture/distributed-querying.mmd b/docs/diagrams/architecture/distributed-querying.mmd deleted file mode 100644 index c57619110a1..00000000000 --- a/docs/diagrams/architecture/distributed-querying.mmd +++ /dev/null @@ -1,10 +0,0 @@ -flowchart TD - Server["Central Server"] -->|"HTTPS"| TLSPlugin["TLS Distributed Plugin"] - TLSPlugin -->|"getQueries()"| DistributedCore["Distributed Core"] - DistributedCore -->|"runQueries()"| SQLEngine["SQL Engine"] - SQLEngine -->|"QueryData"| DistributedCore - DistributedCore -->|"writeResults()"| TLSPlugin - TLSPlugin -->|"HTTPS"| Server - - DistributedCore -->|"recordQueryPerformance"| Observability["Query Observability"] - DistributedCore -->|"running state"| Storage["Database Plugins"] diff --git a/docs/diagrams/architecture/eventing-framework-and-subscriptions-2.mmd b/docs/diagrams/architecture/eventing-framework-and-subscriptions-2.mmd deleted file mode 100644 index 0f4dd5e4690..00000000000 --- a/docs/diagrams/architecture/eventing-framework-and-subscriptions-2.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - RegisterSub["registerEventSubscriber()"] --> SetupSub["setUp()"] - SetupSub --> InitSub["init()"] - InitSub --> RunningSub["EVENT_RUNNING"] - - RegisterPub["registerEventPublisher()"] --> SetupPub["setUp()"] - SetupPub --> ReadyPub["EVENT_SETUP"] diff --git a/docs/diagrams/architecture/eventing-framework-and-subscriptions-3.mmd b/docs/diagrams/architecture/eventing-framework-and-subscriptions-3.mmd deleted file mode 100644 index 22c93bf1ac5..00000000000 --- a/docs/diagrams/architecture/eventing-framework-and-subscriptions-3.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - Delay["delay()"] --> ThreadSpawn["spawn thread per publisher"] - ThreadSpawn --> RunLoop["run(type_id)"] - RunLoop --> PublisherRun["publisher->run()"] - PublisherRun --> Pause["pause(200ms)"] - Pause --> RunLoop - RunLoop -->|"isEnding()"| TearDown["tearDown()"] diff --git a/docs/diagrams/architecture/eventing-framework-and-subscriptions-4.mmd b/docs/diagrams/architecture/eventing-framework-and-subscriptions-4.mmd deleted file mode 100644 index 13f5eab16f8..00000000000 --- a/docs/diagrams/architecture/eventing-framework-and-subscriptions-4.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - Callback["EventCallback"] --> AddBatch["addBatch()"] - AddBatch --> AssignID["generateEventIdentifier()"] - AssignID --> Persist["Database write"] - Persist --> IndexUpdate["update time index"] - - ExpireCheck["expireEventBatches()"] --> RemoveOld["delete expired batches"] diff --git a/docs/diagrams/architecture/eventing-framework-and-subscriptions-5.mmd b/docs/diagrams/architecture/eventing-framework-and-subscriptions-5.mmd deleted file mode 100644 index e165a93a604..00000000000 --- a/docs/diagrams/architecture/eventing-framework-and-subscriptions-5.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - SQLQuery["SELECT * FROM example_events"] --> GenTable["genTable()"] - GenTable --> GenerateRows["generateRows()"] - GenerateRows --> FilterTime["apply start/stop window"] - FilterTime --> YieldRows["yield Row to SQL engine"] diff --git a/docs/diagrams/architecture/eventing-framework-and-subscriptions-6.mmd b/docs/diagrams/architecture/eventing-framework-and-subscriptions-6.mmd deleted file mode 100644 index 2de3bd7f42b..00000000000 --- a/docs/diagrams/architecture/eventing-framework-and-subscriptions-6.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart LR - FactoryLock["factory_lock_"] --> ProtectPub["event_pubs_"] - FactoryLock --> ProtectSub["event_subs_"] - - SubscriberLock1["event_id_lock_"] --> IDGen["EventID generation"] - SubscriberLock2["event_record_lock_"] --> RecordWrite["record time bins"] - SubscriberLock3["event_query_record_"] --> QueryTracking["query usage tracking"] diff --git a/docs/diagrams/architecture/eventing-framework-and-subscriptions-7.mmd b/docs/diagrams/architecture/eventing-framework-and-subscriptions-7.mmd deleted file mode 100644 index 58804b2f0a6..00000000000 --- a/docs/diagrams/architecture/eventing-framework-and-subscriptions-7.mmd +++ /dev/null @@ -1,16 +0,0 @@ -sequenceDiagram - participant Config - participant Factory as EventFactory - participant Pub as EventPublisher - participant Sub as EventSubscriber - participant DB as Database - participant SQL - - Config->>Factory: configUpdate() - Factory->>Sub: setMinExpiry() - Factory->>Pub: spawn run loop - Pub->>Sub: fire event (callback) - Sub->>DB: addBatch() - SQL->>Sub: SELECT from event table - Sub->>DB: generateRows() - Sub->>SQL: yield rows diff --git a/docs/diagrams/architecture/eventing-framework-and-subscriptions.mmd b/docs/diagrams/architecture/eventing-framework-and-subscriptions.mmd deleted file mode 100644 index 169b0f41f29..00000000000 --- a/docs/diagrams/architecture/eventing-framework-and-subscriptions.mmd +++ /dev/null @@ -1,9 +0,0 @@ -flowchart TD - Config["Configuration And Packs"] -->|"scheduled queries"| EventFactory["EventFactory"] - EventFactory -->|"registers"| Publisher["EventPublisherPlugin"] - EventFactory -->|"registers"| Subscriber["EventSubscriberPlugin"] - Subscriber -->|"creates"| SubscriptionObj["Subscription"] - Publisher -->|"dispatches"| Callback["EventCallback"] - Callback -->|"addBatch()"| Storage["Database And Storage Plugins"] - SQL["SQL Engine And Virtual Tables"] -->|"SELECT from _events tables"| Subscriber - Subscriber -->|"generateRows()"| SQL diff --git a/docs/diagrams/architecture/extensions-and-ipc-2.mmd b/docs/diagrams/architecture/extensions-and-ipc-2.mmd deleted file mode 100644 index 566b1b1c48c..00000000000 --- a/docs/diagrams/architecture/extensions-and-ipc-2.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - Client["ExtensionClient"] --> Transport["Thrift Transport"] - Transport --> Socket["UNIX Socket or Named Pipe"] - Socket --> Server["ExtensionRunner or ManagerRunner"] - Server --> Handler["ExtensionHandler or ExtensionManagerHandler"] - Handler --> Interface["ExtensionInterface / ExtensionManagerInterface"] - Interface --> Registry["RegistryFactory"] diff --git a/docs/diagrams/architecture/extensions-and-ipc-3.mmd b/docs/diagrams/architecture/extensions-and-ipc-3.mmd deleted file mode 100644 index 9aabee40835..00000000000 --- a/docs/diagrams/architecture/extensions-and-ipc-3.mmd +++ /dev/null @@ -1,10 +0,0 @@ -sequenceDiagram - participant Ext as Extension Process - participant EM as Extension Manager - participant Reg as RegistryFactory - - Ext->>EM: registerExtension(info, registry) - EM->>EM: Validate name uniqueness - EM->>EM: Check SDK compatibility - EM->>Reg: addBroadcast(uuid, registry) - EM-->>Ext: Return UUID diff --git a/docs/diagrams/architecture/extensions-and-ipc-4.mmd b/docs/diagrams/architecture/extensions-and-ipc-4.mmd deleted file mode 100644 index 642a20aeb10..00000000000 --- a/docs/diagrams/architecture/extensions-and-ipc-4.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart LR - SQL["SQL Engine"] --> ExternalSQL["ExternalSQLPlugin"] - ExternalSQL --> ManagerClient["ExtensionManagerClient"] - ManagerClient --> Extension["Extension Process"] diff --git a/docs/diagrams/architecture/extensions-and-ipc-5.mmd b/docs/diagrams/architecture/extensions-and-ipc-5.mmd deleted file mode 100644 index a4a24d258ed..00000000000 --- a/docs/diagrams/architecture/extensions-and-ipc-5.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - Watcher["ExtensionManagerWatcher"] --> UUIDs["Registered UUIDs"] - UUIDs --> Ping["Ping Extension"] - Ping -->|"Success"| Keep["Keep Registered"] - Ping -->|"Failure x2"| Remove["Remove Broadcast Route"] diff --git a/docs/diagrams/architecture/extensions-and-ipc-6.mmd b/docs/diagrams/architecture/extensions-and-ipc-6.mmd deleted file mode 100644 index 1844fc82272..00000000000 --- a/docs/diagrams/architecture/extensions-and-ipc-6.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - Start["Core Start"] --> StartManager["startExtensionManager()"] - StartManager --> Watcher["ExtensionManagerWatcher"] - Watcher --> Runner["ExtensionManagerRunner"] - Runner --> Listen["Thrift Listen"] diff --git a/docs/diagrams/architecture/extensions-and-ipc-7.mmd b/docs/diagrams/architecture/extensions-and-ipc-7.mmd deleted file mode 100644 index 82e492bb100..00000000000 --- a/docs/diagrams/architecture/extensions-and-ipc-7.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - ExtStart["Extension Binary Start"] --> Connect["Connect to Manager Socket"] - Connect --> Register["registerExtension()"] - Register --> UUID["Receive UUID"] - UUID --> StartServer["Start ExtensionRunner"] - StartServer --> Serve["Serve Thrift Requests"] diff --git a/docs/diagrams/architecture/extensions-and-ipc.mmd b/docs/diagrams/architecture/extensions-and-ipc.mmd deleted file mode 100644 index 343a33e6b65..00000000000 --- a/docs/diagrams/architecture/extensions-and-ipc.mmd +++ /dev/null @@ -1,12 +0,0 @@ -flowchart LR - Core["osquery Core"] -->|"Starts"| Manager["Extension Manager"] - Manager -->|"Registers routes"| Registry["RegistryFactory"] - - ExtensionProc["Extension Process"] -->|"registerExtension()"| Manager - Manager -->|"Assign UUID"| ExtensionProc - - Core -->|"callExtension()"| Manager - Manager -->|"Route to UUID"| ExtensionProc - - ExtensionProc -->|"Response"| Manager - Manager -->|"PluginResponse"| Core diff --git a/docs/diagrams/architecture/filesystem-and-path-utilities-2.mmd b/docs/diagrams/architecture/filesystem-and-path-utilities-2.mmd deleted file mode 100644 index 58aae8b080a..00000000000 --- a/docs/diagrams/architecture/filesystem-and-path-utilities-2.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - Open["Open PlatformFile"] --> Validate["Handle Valid?"] - Validate -->|No| Fail["Return Error"] - Validate -->|Yes| IO["Read / Write / Seek"] - IO --> Close["Destructor Closes Handle"] diff --git a/docs/diagrams/architecture/filesystem-and-path-utilities-3.mmd b/docs/diagrams/architecture/filesystem-and-path-utilities-3.mmd deleted file mode 100644 index 26f4fd78970..00000000000 --- a/docs/diagrams/architecture/filesystem-and-path-utilities-3.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart LR - Path["File Path"] --> StatCall["platformStat()"] - StatCall -->|POSIX| POSIXStat["struct stat"] - StatCall -->|Windows| WinStat["WINDOWS_STAT"] diff --git a/docs/diagrams/architecture/filesystem-and-path-utilities-4.mmd b/docs/diagrams/architecture/filesystem-and-path-utilities-4.mmd deleted file mode 100644 index 8f8a50b070d..00000000000 --- a/docs/diagrams/architecture/filesystem-and-path-utilities-4.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - Pattern["Input Pattern"] --> Normalize["replaceGlobWildcards()"] - Normalize --> Expand["platformGlob()"] - Expand --> Filter["Apply GLOB_FILES / GLOB_FOLDERS"] - Filter --> Results["Resolved Paths"] diff --git a/docs/diagrams/architecture/filesystem-and-path-utilities-5.mmd b/docs/diagrams/architecture/filesystem-and-path-utilities-5.mmd deleted file mode 100644 index 9efad4a8202..00000000000 --- a/docs/diagrams/architecture/filesystem-and-path-utilities-5.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - ReadCall["readFile()"] --> Open["PlatformFile"] - Open --> SizeCheck["checkFileReadLimit()"] - SizeCheck --> Loop["Block Read Loop"] - Loop --> Predicate["Callback / Buffer"] diff --git a/docs/diagrams/architecture/filesystem-and-path-utilities-6.mmd b/docs/diagrams/architecture/filesystem-and-path-utilities-6.mmd deleted file mode 100644 index 8b63dd13550..00000000000 --- a/docs/diagrams/architecture/filesystem-and-path-utilities-6.mmd +++ /dev/null @@ -1,3 +0,0 @@ -flowchart LR - getMounted["getMountedFilesystems()"] --> MountInfo["MountInformation"] - MountInfo --> StatFS["StatFsInfo"] diff --git a/docs/diagrams/architecture/filesystem-and-path-utilities-7.mmd b/docs/diagrams/architecture/filesystem-and-path-utilities-7.mmd deleted file mode 100644 index 60fca930f80..00000000000 --- a/docs/diagrams/architecture/filesystem-and-path-utilities-7.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart TD - Proc["/proc"] --> Enumerate["procEnumerateProcesses()"] - Enumerate --> FDEnum["procEnumerateProcessDescriptors()"] - FDEnum --> SocketMap["SocketInodeToProcessInfoMap"] diff --git a/docs/diagrams/architecture/filesystem-and-path-utilities-8.mmd b/docs/diagrams/architecture/filesystem-and-path-utilities-8.mmd deleted file mode 100644 index 1359e2b787f..00000000000 --- a/docs/diagrams/architecture/filesystem-and-path-utilities-8.mmd +++ /dev/null @@ -1,9 +0,0 @@ -flowchart TD - File["Candidate File"] --> Exists["Exists?"] - Exists -->|No| Reject["Reject"] - Exists --> Owner["Owner Root or Current User?"] - Owner -->|No| Reject - Owner --> Writable["World Writable?"] - Writable -->|Yes| Reject - Writable --> Exec["Executable Required?"] - Exec --> Accept["Safe"] diff --git a/docs/diagrams/architecture/filesystem-and-path-utilities.mmd b/docs/diagrams/architecture/filesystem-and-path-utilities.mmd deleted file mode 100644 index 44c2abfbebc..00000000000 --- a/docs/diagrams/architecture/filesystem-and-path-utilities.mmd +++ /dev/null @@ -1,21 +0,0 @@ -flowchart TD - Caller["Core / SQL / Extensions"] --> FSAPI["Filesystem API"] - - subgraph abstraction_layer["Cross-Platform Abstraction"] - FSAPI --> PlatformFile["PlatformFile"] - FSAPI --> Glob["platformGlob()"] - FSAPI --> Perms["platformChmod()"] - FSAPI --> Access["platformAccess()"] - FSAPI --> Stat["platformStat() / lstat()"] - end - - subgraph posix_layer["POSIX Implementation"] - PlatformFile --> POSIXFile["open/read/write/lstat"] - Glob --> POSIXGlob["glob()"] - end - - subgraph windows_layer["Windows Implementation"] - PlatformFile --> WinFile["CreateFile / ReadFile"] - Glob --> WinGlob["FindFirstFileW"] - Perms --> WinACL["ACL Manipulation"] - end diff --git a/docs/diagrams/architecture/logging-and-query-observability-2.mmd b/docs/diagrams/architecture/logging-and-query-observability-2.mmd deleted file mode 100644 index 2feabab2fc7..00000000000 --- a/docs/diagrams/architecture/logging-and-query-observability-2.mmd +++ /dev/null @@ -1,7 +0,0 @@ -flowchart TD - Init["System Initialization"] --> Buffer["Buffer Early Status Logs"] - Buffer --> LoadConfig["Load Configuration"] - LoadConfig --> Discover["Discover Logger Plugin"] - Discover --> InitLogger["LoggerPlugin.init()"] - InitLogger --> Flush["Flush Buffered StatusLogLine Entries"] - Flush --> Runtime["Runtime Logging"] diff --git a/docs/diagrams/architecture/logging-and-query-observability-3.mmd b/docs/diagrams/architecture/logging-and-query-observability-3.mmd deleted file mode 100644 index 7c5a848377e..00000000000 --- a/docs/diagrams/architecture/logging-and-query-observability-3.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart LR - Old["Previous Results"] --> DiffEngine["diff()"] - New["Current Results"] --> DiffEngine - DiffEngine --> Added["Added Rows"] - DiffEngine --> Removed["Removed Rows"] diff --git a/docs/diagrams/architecture/logging-and-query-observability-4.mmd b/docs/diagrams/architecture/logging-and-query-observability-4.mmd deleted file mode 100644 index 93386e9e467..00000000000 --- a/docs/diagrams/architecture/logging-and-query-observability-4.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart TD - Diff["DiffResults"] --> Serialize["serializeDiffResults()"] - Serialize --> JSONDoc["JSON Document"] - JSONDoc --> Logger["LoggerPlugin"] diff --git a/docs/diagrams/architecture/logging-and-query-observability-5.mmd b/docs/diagrams/architecture/logging-and-query-observability-5.mmd deleted file mode 100644 index 30d12735a02..00000000000 --- a/docs/diagrams/architecture/logging-and-query-observability-5.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - Item["QueryLogItem"] --> JSON1["serializeQueryLogItem()"] - Item --> JSON2["serializeQueryLogItemJSON()"] - Item --> Events1["serializeQueryLogItemAsEvents()"] - Item --> Events2["serializeQueryLogItemAsEventsJSON()"] diff --git a/docs/diagrams/architecture/logging-and-query-observability-6.mmd b/docs/diagrams/architecture/logging-and-query-observability-6.mmd deleted file mode 100644 index 99c77b8ea17..00000000000 --- a/docs/diagrams/architecture/logging-and-query-observability-6.mmd +++ /dev/null @@ -1,9 +0,0 @@ -flowchart TD - Config["ScheduledQuery"] --> Exec["SQL Execution"] - Exec --> Results["QueryDataTyped"] - Results --> QueryState["Query.addNewResults()"] - QueryState --> Diff["DiffResults"] - Diff --> LogItem["QueryLogItem"] - LogItem --> Serialize["JSON Serialization"] - Serialize --> Logger["LoggerPlugin"] - Logger --> Sink["External Log System"] diff --git a/docs/diagrams/architecture/logging-and-query-observability-7.mmd b/docs/diagrams/architecture/logging-and-query-observability-7.mmd deleted file mode 100644 index 88b889b3b8e..00000000000 --- a/docs/diagrams/architecture/logging-and-query-observability-7.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - SQ["ScheduledQuery snapshot=true"] --> Exec["Execute Query"] - Exec --> Full["Full Result Set"] - Full --> LogItem["QueryLogItem isSnapshot=true"] - LogItem --> Logger["logSnapshot()"] diff --git a/docs/diagrams/architecture/logging-and-query-observability.mmd b/docs/diagrams/architecture/logging-and-query-observability.mmd deleted file mode 100644 index 82b06f91f54..00000000000 --- a/docs/diagrams/architecture/logging-and-query-observability.mmd +++ /dev/null @@ -1,15 +0,0 @@ -flowchart TD - Config["Configuration And Packs"] --> Scheduler["Scheduled Query"] - Scheduler --> SQ["ScheduledQuery Struct"] - SQ --> QueryObj["Query State Manager"] - QueryObj --> DB["Database And Storage Plugins"] - - QueryObj --> Diff["DiffResults Engine"] - Diff --> LogItem["QueryLogItem"] - LogItem --> Serializer["JSON Serialization"] - Serializer --> Logger["LoggerPlugin"] - - StatusLogs["Internal Status Events"] --> StatusLine["StatusLogLine"] - StatusLine --> Logger - - Logger --> External["External Logging Backend"] diff --git a/docs/diagrams/architecture/remote-http-client-2.mmd b/docs/diagrams/architecture/remote-http-client-2.mmd deleted file mode 100644 index 810bffb2210..00000000000 --- a/docs/diagrams/architecture/remote-http-client-2.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart LR - OldOpts["Current Options"] --> Compare["operator=="] - NewOpts["New Options"] --> Compare - Compare -->|"Different"| Reconfigure["Reinitialize Connection"] - Compare -->|"Same"| Reuse["Reuse Existing Settings"] diff --git a/docs/diagrams/architecture/remote-http-client-3.mmd b/docs/diagrams/architecture/remote-http-client-3.mmd deleted file mode 100644 index 6445d351559..00000000000 --- a/docs/diagrams/architecture/remote-http-client-3.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart TD - Response["HTTP_Response"] --> Headers["Headers"] - Headers --> Iterator["Iterator"] - Iterator --> Pair["(name, value)"] diff --git a/docs/diagrams/architecture/remote-http-client-4.mmd b/docs/diagrams/architecture/remote-http-client-4.mmd deleted file mode 100644 index 52c275c5be9..00000000000 --- a/docs/diagrams/architecture/remote-http-client-4.mmd +++ /dev/null @@ -1,14 +0,0 @@ -sequenceDiagram - participant Caller - participant Client - participant Resolver - participant Server - - Caller->>Client: get(Request) - Client->>Resolver: async_resolve() - Resolver-->>Client: endpoints - Client->>Server: async_connect() - Client->>Server: async_write(request) - Server-->>Client: HTTP response - Client->>Server: async_read() - Client-->>Caller: Response diff --git a/docs/diagrams/architecture/remote-http-client-5.mmd b/docs/diagrams/architecture/remote-http-client-5.mmd deleted file mode 100644 index 1f2bd2fd1e2..00000000000 --- a/docs/diagrams/architecture/remote-http-client-5.mmd +++ /dev/null @@ -1,6 +0,0 @@ -flowchart TD - Connect["TCP Connect"] --> TLSCheck{"SSL Enabled?"} - TLSCheck -->|"Yes"| Handshake["TLS Handshake"] - TLSCheck -->|"No"| Plain["Plain HTTP"] - Handshake --> Send["Send Request"] - Plain --> Send diff --git a/docs/diagrams/architecture/remote-http-client-6.mmd b/docs/diagrams/architecture/remote-http-client-6.mmd deleted file mode 100644 index 01586684843..00000000000 --- a/docs/diagrams/architecture/remote-http-client-6.mmd +++ /dev/null @@ -1,9 +0,0 @@ -flowchart TD - Start["Start Network Call"] --> Timer["Start Deadline Timer"] - Timer --> Async["Async Operation"] - Async --> Complete{"Completed?"} - Complete -->|"Yes"| CancelTimer["Cancel Timer"] - Complete -->|"No"| Timeout["Timer Fires"] - Timeout --> Abort["Abort Socket"] - CancelTimer --> Return["Return Response"] - Abort --> Return diff --git a/docs/diagrams/architecture/remote-http-client.mmd b/docs/diagrams/architecture/remote-http-client.mmd deleted file mode 100644 index f6bb4f64dff..00000000000 --- a/docs/diagrams/architecture/remote-http-client.mmd +++ /dev/null @@ -1,13 +0,0 @@ -flowchart TD - Caller["Caller Module"] --> Client["Client"] - Client --> Options["Client::Options"] - Client --> Request["HTTP_Request"] - Client --> Response["HTTP_Response"] - - Client --> Resolver["TCP Resolver"] - Client --> Socket["TCP Socket"] - Client --> TLSSocket["SSL Stream"] - Client --> Timer["Deadline Timer"] - - Request --> URI["URI Parser"] - Response --> Headers["Header Iterator"] diff --git a/docs/diagrams/architecture/sql-engine-and-virtual-tables-2.mmd b/docs/diagrams/architecture/sql-engine-and-virtual-tables-2.mmd deleted file mode 100644 index 09946020084..00000000000 --- a/docs/diagrams/architecture/sql-engine-and-virtual-tables-2.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart LR - Caller["Caller"] --> GetConn["SQLiteDBManager.get()"] - GetConn --> PrimaryCheck{"Primary Available?"} - PrimaryCheck -->|Yes| Primary["Primary SQLiteDBInstance"] - PrimaryCheck -->|No| Transient["Transient SQLiteDBInstance"] diff --git a/docs/diagrams/architecture/sql-engine-and-virtual-tables-3.mmd b/docs/diagrams/architecture/sql-engine-and-virtual-tables-3.mmd deleted file mode 100644 index 6bded7bfd12..00000000000 --- a/docs/diagrams/architecture/sql-engine-and-virtual-tables-3.mmd +++ /dev/null @@ -1,4 +0,0 @@ -flowchart TD - Prepare["sqlite3_prepare_v2"] --> Authorizer["sqliteAuthorizer"] - Authorizer -->|Allowed| Continue["Execute Statement"] - Authorizer -->|Denied| Reject["SQLITE_DENY"] diff --git a/docs/diagrams/architecture/sql-engine-and-virtual-tables-4.mmd b/docs/diagrams/architecture/sql-engine-and-virtual-tables-4.mmd deleted file mode 100644 index d79fe38ce6a..00000000000 --- a/docs/diagrams/architecture/sql-engine-and-virtual-tables-4.mmd +++ /dev/null @@ -1,5 +0,0 @@ -flowchart TD - Query["SQL Query"] --> ExplainPlan["EXPLAIN QUERY PLAN"] - Query --> Explain["EXPLAIN"] - Explain --> OpcodeMap["kSQLOpcodes Mapping"] - OpcodeMap --> TypeInference["Apply Column Types"] diff --git a/docs/diagrams/architecture/sql-engine-and-virtual-tables-5.mmd b/docs/diagrams/architecture/sql-engine-and-virtual-tables-5.mmd deleted file mode 100644 index 5a569905d36..00000000000 --- a/docs/diagrams/architecture/sql-engine-and-virtual-tables-5.mmd +++ /dev/null @@ -1,12 +0,0 @@ -sequenceDiagram - participant SQLite - participant Module as "sqlite3_module" - participant Table as "TablePlugin" - - SQLite->>Module: xBestIndex - Module->>SQLite: Constraint plan - - SQLite->>Module: xFilter - Module->>Table: generate(context) - Table-->>Module: Row set - Module-->>SQLite: Rows diff --git a/docs/diagrams/architecture/sql-engine-and-virtual-tables-6.mmd b/docs/diagrams/architecture/sql-engine-and-virtual-tables-6.mmd deleted file mode 100644 index ebfb533e38c..00000000000 --- a/docs/diagrams/architecture/sql-engine-and-virtual-tables-6.mmd +++ /dev/null @@ -1,10 +0,0 @@ -flowchart TD - Start["Query Submitted"] --> Prepare["sqlite3_prepare_v2"] - Prepare --> Execute["sqlite3_step Loop"] - Execute -->|Virtual Table Access| VTab - VTab --> xBestIndex - xBestIndex --> xFilter - xFilter --> Generate["TablePlugin.generate()"] - Generate --> Rows["Return Rows"] - Rows --> Finalize["sqlite3_finalize"] - Finalize --> End["Results Returned"] diff --git a/docs/diagrams/architecture/sql-engine-and-virtual-tables.mmd b/docs/diagrams/architecture/sql-engine-and-virtual-tables.mmd deleted file mode 100644 index f435ddde934..00000000000 --- a/docs/diagrams/architecture/sql-engine-and-virtual-tables.mmd +++ /dev/null @@ -1,11 +0,0 @@ -flowchart TD - Client["SQL Caller (Scheduler / CLI / Extension)"] --> SQLPlugin["SQLiteSQLPlugin"] - SQLPlugin --> DBManager["SQLiteDBManager"] - DBManager --> DBInstance["SQLiteDBInstance"] - DBInstance --> SQLiteCore[("In-Memory SQLite Engine")] - - SQLiteCore --> VTabModule["sqlite3_module (Virtual Table Module)"] - VTabModule --> VirtualTable["VirtualTable Wrapper"] - VirtualTable --> TablePlugin["TablePlugin (Registry)"] - - SQLiteCore --> Planner["QueryPlanner"] diff --git a/docs/getting-started/.gitignore b/docs/getting-started/.gitignore deleted file mode 100644 index a5d01be5236..00000000000 --- a/docs/getting-started/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# VoltAgent temp files -temp/ - -# JSON intermediate files (except schema/config) -*.json -!*-schema.json -!*-config.json diff --git a/docs/getting-started/first-steps.md b/docs/getting-started/first-steps.md deleted file mode 100644 index 3b002d66b32..00000000000 --- a/docs/getting-started/first-steps.md +++ /dev/null @@ -1,178 +0,0 @@ -# First Steps - -Now that osquery is running, here are the first things to explore and configure. - ---- - -## 1. Explore Available Tables - -osquery ships with hundreds of virtual tables. Discover them with the `.tables` command in the interactive shell: - -```bash -./build/osquery/osqueryi -``` - -```sql --- List all available tables -osquery> .tables - --- Search for tables by name -osquery> .tables process -osquery> .tables network -osquery> .tables user - --- Inspect the schema of any table -osquery> .schema processes -osquery> .schema listening_ports -osquery> .schema users -``` - -Tables are organized by platform: - -| Category | Example Tables | -|---|---| -| **System** | `system_info`, `uptime`, `cpu_info`, `memory_info` | -| **Processes** | `processes`, `process_open_files`, `process_open_sockets` | -| **Networking** | `listening_ports`, `interface_addresses`, `arp_cache`, `routes` | -| **Users & Groups** | `users`, `groups`, `logged_in_users`, `sudoers` | -| **Packages** | `deb_packages`, `rpm_packages`, `homebrew_packages`, `chocolatey_packages` | -| **Events** | `file_events`, `process_events`, `socket_events` | -| **Security** | `authorized_keys`, `certificates`, `secureboot`, `disk_encryption` | - ---- - -## 2. Write Your First Scheduled Query Pack - -Create a configuration pack to run queries on a schedule: - -```bash -sudo tee /etc/osquery/packs/first-steps.json <<'EOF' -{ - "queries": { - "users_snapshot": { - "query": "SELECT uid, username, shell, directory FROM users;", - "interval": 3600, - "description": "Snapshot of all local users every hour" - }, - "listening_ports": { - "query": "SELECT pid, port, protocol, address FROM listening_ports;", - "interval": 300, - "description": "Check open ports every 5 minutes" - }, - "new_processes": { - "query": "SELECT pid, name, path, cmdline, start_time FROM processes;", - "interval": 60, - "description": "Poll running processes every minute" - } - } -} -EOF -``` - -Reference it in your `osquery.conf`: - -```json -{ - "options": { - "logger_plugin": "filesystem", - "schedule_splay_percent": 10 - }, - "packs": { - "first-steps": "/etc/osquery/packs/first-steps.json" - } -} -``` - ---- - -## 3. Enable File Integrity Monitoring - -osquery can monitor filesystem changes using platform-native event APIs (inotify on Linux, FSEvents on macOS, ETW on Windows): - -```json -{ - "file_paths": { - "etc": ["/etc/%%"], - "homes": ["/root/.ssh/%%", "/home/%/.ssh/%%"], - "sensitive": ["/usr/bin/%%", "/usr/sbin/%%"] - }, - "schedule": { - "file_events": { - "query": "SELECT * FROM file_events;", - "interval": 30 - } - } -} -``` - -Then query the event table: - -```sql -osquery> SELECT time, target_path, action, md5 FROM file_events; -``` - ---- - -## 4. Connect to OpenFrame (Flamingo Platform) - -If you are part of the [OpenFrame](https://openframe.ai) / [Flamingo](https://flamingo.run) platform: - -1. Obtain your OpenFrame credentials from your platform administrator -2. Configure the TLS enrollment endpoint in your daemon flags -3. The `OpenframeAuthorizationManager` handles token lifecycle automatically -4. The `OpenframeTokenRefresher` keeps sessions active in the background - -The OpenFrame layer uses AES-256-GCM encryption via the `OpenframeEncryptionService` β€” all credential handling is fully automated once configured. - -Refer to your environment configuration for the specific connection details. - ---- - -## 5. Load an Extension - -osquery's extension system lets you add custom tables, loggers, and configuration plugins as separate processes: - -```bash -# Start osqueryd with extension support enabled -sudo ./build/osquery/osqueryd \ - --config_path=/etc/osquery/osquery.conf \ - --extensions_socket=/var/osquery/osquery.em \ - --extensions_autoload=/etc/osquery/extensions.load - -# In a separate terminal, run your extension -./my_extension --socket=/var/osquery/osquery.em -``` - -Extensions communicate via Apache Thrift over UNIX domain sockets (Linux/macOS) or named pipes (Windows). - ---- - -## Key Configuration Flags - -| Flag | Description | Default | -|---|---|---| -| `--config_path` | Path to the JSON configuration file | Platform-specific | -| `--logger_plugin` | Logging backend (`filesystem`, `tls`, `syslog`) | `filesystem` | -| `--schedule_splay_percent` | Random splay to distribute query load | `10` | -| `--extensions_socket` | Path to the Thrift extension socket | Platform-specific | -| `--config_refresh` | How often (seconds) to refresh remote config | `0` (disabled) | -| `--watchdog_level` | Resource limit profile (0=normal, 1=restrictive, -1=disabled) | `0` | - ---- - -## Where to Get Help - -| Resource | Link | -|---|---| -| **OpenMSP Community Slack** | [Join here](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) | -| **OpenMSP Website** | [https://www.openmsp.ai/](https://www.openmsp.ai/) | -| **Flamingo Platform** | [https://flamingo.run](https://flamingo.run) | -| **OpenFrame** | [https://openframe.ai](https://openframe.ai) | - -> All support and discussions are managed on the **OpenMSP Slack community** β€” not GitHub Issues or GitHub Discussions. - ---- - -## Watch: osquery in Action - -[![osquery Overview](https://img.youtube.com/vi/1UcWGiHbLVo/hqdefault.jpg)](https://www.youtube.com/watch?v=1UcWGiHbLVo) diff --git a/docs/getting-started/introduction.md b/docs/getting-started/introduction.md deleted file mode 100644 index a15e0308dbb..00000000000 --- a/docs/getting-started/introduction.md +++ /dev/null @@ -1,113 +0,0 @@ -# Introduction to osquery with OpenFrame - -**osquery** is a cross-platform operating system instrumentation framework that exposes live system state and activity as relational data β€” enabling you to query your operating system using SQL. - -Built and extended by the [Flamingo / OpenFrame](https://openframe.ai) platform, this distribution integrates osquery's powerful SQL telemetry engine with OpenFrame's AI-driven MSP automation infrastructure. - ---- - -## What is osquery? - -osquery transforms every host into a **SQL-queryable telemetry node**. Instead of parsing flat log files or calling proprietary APIs, you write standard SQL queries to explore: - -- Running processes and open sockets -- Installed packages and browser extensions -- Kernel modules and hardware inventory -- Filesystem events and user activity -- Network interfaces and firewall rules -- Windows Registry, launchd, systemd units, and much more - -```sql --- Which processes are listening on ports? -SELECT pid, name, port, protocol FROM listening_ports JOIN processes USING (pid); - --- What Chrome extensions are installed? -SELECT u.username, e.name, e.version, e.permissions -FROM users u, chrome_extensions e -WHERE e.uid = u.uid; -``` - ---- - -## OpenFrame Integration - -This repository extends osquery with the **OpenFrame** authentication and encryption layer: - -- **`OpenframeAuthorizationManager`** β€” Secure, lifecycle-controlled JWT token management -- **`OpenframeEncryptionService`** β€” AES-256-GCM symmetric encryption via OpenSSL -- **`OpenframeTokenExtractor`** β€” Token acquisition from OpenFrame services -- **`OpenframeTokenRefresher`** β€” Background thread for seamless token renewal - -These components are part of the [OpenFrame platform](https://www.flamingo.run/openframe) β€” the unified MSP interface that integrates IT tools under a single AI-driven experience. - ---- - -## Key Features - -| Feature | Description | -|---|---| -| **SQL Querying** | Query live OS data with standard SQL via embedded SQLite | -| **Virtual Tables** | 300+ platform-specific tables across Linux, macOS, and Windows | -| **Event Monitoring** | inotify, BPF, FSEvents, ETW, audit β€” all exposed as queryable tables | -| **Scheduled Queries** | Pack-based query scheduling with differential result tracking | -| **Distributed Queries** | Remote SQL orchestration across fleets via TLS | -| **Extension System** | Runtime plugin model with Apache Thrift IPC | -| **OpenFrame Auth** | AES-256-GCM encrypted JWT token management for OpenFrame services | -| **Cross-Platform** | Linux (x86_64, aarch64), macOS, and Windows support | - ---- - -## Target Audience - -osquery with OpenFrame is designed for: - -- **IT Operations Teams** using Flamingo/OpenFrame for managed service delivery -- **Security Engineers** building detection and response workflows -- **Fleet Administrators** who need SQL-level visibility across endpoints -- **Developers** building extensions, plugins, or custom table providers -- **MSP Technicians** leveraging Mingo AI and Fae automation through the OpenFrame platform - ---- - -## Architecture Overview - -```mermaid -graph TD - CLI["osqueryi / osqueryd"] --> Core["Core Init And Runtime"] - Core --> Config["Configuration And Packs"] - Core --> SQL["SQL Engine And Virtual Tables"] - Core --> Events["Eventing Framework"] - Core --> Logging["Logging And Observability"] - Core --> DB["Database And Storage Plugins"] - Core --> Dist["Distributed Querying"] - Core --> Ext["Extensions And IPC"] - Core --> HTTP["Remote HTTP Client"] - Core --> OF["OpenFrame Auth Layer"] - Config --> SQL - Config --> Events - SQL --> Logging - Events --> DB - Dist --> SQL - Dist --> HTTP - Ext --> SQL - OF --> HTTP -``` - ---- - -## Getting Started - -- Review system and software prerequisites in the [Prerequisites Guide](prerequisites.md) -- Follow the [Quick Start Guide](quick-start.md) to build and run osquery in minutes -- Explore common workflows in [First Steps](first-steps.md) - ---- - -## Community and Support - -osquery with OpenFrame is part of the [Flamingo](https://flamingo.run) open-source ecosystem. - -- πŸ’¬ **Community Slack**: [OpenMSP Community](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) -- 🌐 **OpenMSP**: [https://www.openmsp.ai/](https://www.openmsp.ai/) -- πŸš€ **OpenFrame Platform**: [https://openframe.ai](https://openframe.ai) -- 🦩 **Flamingo**: [https://flamingo.run](https://flamingo.run) diff --git a/docs/getting-started/prerequisites.md b/docs/getting-started/prerequisites.md deleted file mode 100644 index 8e6027a2725..00000000000 --- a/docs/getting-started/prerequisites.md +++ /dev/null @@ -1,167 +0,0 @@ -# Prerequisites - -Before building or deploying osquery with OpenFrame, ensure your environment meets the requirements below. - ---- - -## System Requirements - -| Tier | RAM | CPU Cores | Disk Space | -|---|---|---|---| -| **Minimum** | 24 GB | 6 cores | 50 GB | -| **Recommended** | 32 GB | 12 cores | 100 GB | - -> Building osquery from source is resource-intensive. The recommended configuration significantly reduces build times and prevents out-of-memory failures during compilation. - ---- - -## Supported Operating Systems - -| Platform | Architecture | Notes | -|---|---|---| -| Linux | x86_64, aarch64 | Ubuntu 20.04+, RHEL 8+, Debian 11+ | -| macOS | x86_64, aarch64 (Apple Silicon) | macOS 12+ recommended | -| Windows | x86_64, aarch64 | Windows 10/11, Server 2019+ | - ---- - -## Required Software - -| Tool | Minimum Version | Purpose | -|---|---|---| -| **CMake** | 3.21+ | Build system generator | -| **Python** | 3.8+ | Code generation, tooling scripts | -| **Git** | 2.x | Source control | -| **C++ Compiler** | GCC 9+ / Clang 10+ / MSVC 2019+ | Compiling C++17 sources | -| **Ninja** | 1.10+ | Fast parallel builds (recommended) | -| **OpenSSL** | 1.1.1+ | TLS support and AES-256-GCM encryption (OpenFrame) | - ---- - -## Optional but Recommended - -| Tool | Purpose | -|---|---| -| **ccache** | Speeds up incremental C++ builds significantly | -| **clang-format** | Code formatting (enforced by CI) | -| **Docker** | Isolated build environments for Linux targets | - ---- - -## Platform-Specific Notes - -### Linux - -On Debian/Ubuntu, install build essentials: - -```bash -sudo apt-get update -sudo apt-get install -y \ - build-essential \ - cmake \ - ninja-build \ - python3 \ - python3-pip \ - git \ - openssl \ - libssl-dev \ - clang-format -``` - -On RHEL/CentOS/Fedora: - -```bash -sudo dnf install -y \ - gcc-c++ \ - cmake \ - ninja-build \ - python3 \ - git \ - openssl-devel \ - clang -``` - -### macOS - -Install Xcode Command Line Tools and Homebrew tools: - -```bash -xcode-select --install -brew install cmake ninja python3 openssl git -``` - -### Windows - -1. Install [Visual Studio 2019 or 2022](https://visualstudio.microsoft.com/) with the **Desktop development with C++** workload -2. Install [CMake](https://cmake.org/download/) 3.21+ -3. Install [Git for Windows](https://git-scm.com/download/win) -4. Install [Python 3.8+](https://www.python.org/downloads/windows/) -5. Optionally install [Ninja](https://github.com/ninja-build/ninja/releases) - ---- - -## OpenFrame Platform Requirements - -If you are connecting to the [OpenFrame platform](https://openframe.ai), you also need: - -| Requirement | Description | -|---|---| -| **OpenFrame Account** | Active account on the Flamingo/OpenFrame platform | -| **Network Access** | Outbound HTTPS (port 443) to OpenFrame services | -| **AES-256 Key** | Symmetric secret key for the `OpenframeEncryptionService` | -| **JWT Token** | Authorization token managed by `OpenframeAuthorizationManager` | - -Refer to your environment configuration and OpenFrame administrator for credential details. - ---- - -## Environment Variables - -The following environment variables may be required depending on your deployment: - -| Variable | Purpose | -|---|---| -| `OSQUERY_FLAGS_FILE` | Path to a flagfile for osquery runtime configuration | -| `OSQUERY_EXTENSIONS_SOCKET` | Path to the Thrift extension manager socket | -| `OSQUERY_CONFIG_PATH` | Custom configuration file path (overrides default) | - -Set variables using your shell profile or deployment tooling. For example: - -```bash -export OSQUERY_FLAGS_FILE="/etc/osquery/osquery.flags" -export OSQUERY_EXTENSIONS_SOCKET="/var/osquery/osquery.em" -``` - ---- - -## Verifying Your Environment - -Run the following checks to confirm your build environment is ready: - -```bash -# Check CMake version -cmake --version - -# Check C++ compiler -c++ --version - -# Check Python -python3 --version - -# Check Git -git --version - -# Check OpenSSL -openssl version - -# Check Ninja (if installed) -ninja --version -``` - -All tools should report versions meeting the minimums in the table above. - ---- - -## Next Steps - -Once prerequisites are confirmed, proceed to the [Quick Start Guide](quick-start.md) to clone, build, and run osquery. diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md deleted file mode 100644 index 0ab657e8ce7..00000000000 --- a/docs/getting-started/quick-start.md +++ /dev/null @@ -1,177 +0,0 @@ -# Quick Start - -Get osquery with OpenFrame running in under 10 minutes. - ---- - -## TL;DR β€” Fastest Path - -```bash -# 1. Clone the repository -git clone https://github.com/flamingo-stack/osquery.git -cd osquery - -# 2. Configure the build -cmake -S . -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo -G Ninja - -# 3. Build osquery (this takes 5–30 minutes depending on hardware) -cmake --build build --parallel $(nproc) - -# 4. Run the interactive shell -./build/osquery/osqueryi -``` - -> **Hardware note**: Building from source requires at minimum 24 GB RAM and 6 CPU cores. See the [Prerequisites Guide](prerequisites.md) for full system requirements. - ---- - -## Step 1: Clone the Repository - -```bash -git clone https://github.com/flamingo-stack/osquery.git -cd osquery -``` - ---- - -## Step 2: Configure the Build - -### Linux / macOS - -```bash -cmake -S . -B build \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -G Ninja -``` - -### Windows (Visual Studio) - -```bash -cmake -S . -B build ^ - -DCMAKE_BUILD_TYPE=RelWithDebInfo ^ - -G "Visual Studio 17 2022" -``` - -### Common CMake Options - -| Option | Description | -|---|---| -| `-DCMAKE_BUILD_TYPE=RelWithDebInfo` | Optimized build with debug symbols (recommended) | -| `-DCMAKE_BUILD_TYPE=Debug` | Full debug build (slower, larger binaries) | -| `-DCMAKE_BUILD_TYPE=Release` | Fully optimized production build | -| `-G Ninja` | Use Ninja for faster parallel builds | -| `-DOSQUERY_BUILD_TESTS=ON` | Include test suite in the build | - ---- - -## Step 3: Build - -```bash -# Linux/macOS β€” use all available CPU cores -cmake --build build --parallel $(nproc) - -# Windows β€” use all available CPU cores -cmake --build build --parallel %NUMBER_OF_PROCESSORS% -``` - -The first build downloads and compiles all third-party dependencies. Subsequent builds with `ccache` are significantly faster. - ---- - -## Step 4: Run Your First Query - -Launch the osquery interactive shell: - -```bash -./build/osquery/osqueryi -``` - -Expected output: - -```text -Using a virtual database. Need help, type '.help' -osquery> -``` - -Now try your first query: - -```sql -osquery> SELECT hostname, cpu_brand, physical_memory FROM system_info; -``` - -Example output: - -```text -+------------------+-------------------------------+------------------+ -| hostname | cpu_brand | physical_memory | -+------------------+-------------------------------+------------------+ -| my-linux-host | Intel(R) Core(TM) i9-12900K | 34207285248 | -+------------------+-------------------------------+------------------+ -``` - ---- - -## Step 5: Explore More Tables - -```sql --- List all running processes -osquery> SELECT pid, name, cmdline FROM processes LIMIT 10; - --- Check listening network ports -osquery> SELECT pid, port, protocol, address FROM listening_ports LIMIT 10; - --- List installed packages (Linux) -osquery> SELECT name, version, arch FROM deb_packages LIMIT 10; - --- Show users on the system -osquery> SELECT uid, gid, username, shell FROM users; -``` - ---- - -## Step 6: Run as a Daemon - -To run osquery as a background daemon with scheduled queries: - -```bash -# Create a basic configuration -sudo mkdir -p /etc/osquery -sudo tee /etc/osquery/osquery.conf <<'EOF' -{ - "options": { - "logger_plugin": "filesystem", - "schedule_splay_percent": 10 - }, - "schedule": { - "system_info": { - "query": "SELECT hostname, cpu_brand, physical_memory FROM system_info;", - "interval": 3600 - } - } -} -EOF - -# Run the daemon -sudo ./build/osquery/osqueryd --config_path=/etc/osquery/osquery.conf -``` - ---- - -## Expected Results Summary - -| Command | What You Should See | -|---|---| -| `osqueryi` | Interactive SQL prompt | -| `SELECT * FROM system_info;` | Host identity, CPU, RAM details | -| `SELECT * FROM processes LIMIT 5;` | Running process list | -| `osqueryd` | Daemon starts, logs to `/var/log/osquery/` | - ---- - -## Next Steps - -After completing this quick start: - -- Follow the [First Steps Guide](first-steps.md) to explore key features and common workflows -- Review the [Prerequisites Guide](prerequisites.md) for full system and software requirements -- Explore the [Development section](../development/README.md) to understand the architecture and contribute diff --git a/docs/reference/architecture/.gitignore b/docs/reference/architecture/.gitignore deleted file mode 100644 index b4f4c7a6c9c..00000000000 --- a/docs/reference/architecture/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# CodeWiki temp files (dependency graphs can be 7GB+) -temp/ -dependency_graphs/ - -# JSON intermediate files (except schema/config) -*.json -!*-schema.json -!*-config.json diff --git a/docs/reference/architecture/README.md b/docs/reference/architecture/README.md deleted file mode 100644 index d8bef8be03c..00000000000 --- a/docs/reference/architecture/README.md +++ /dev/null @@ -1,366 +0,0 @@ -# osquery Repository Overview - -## 1. Purpose of the Repository - -**osquery** is a cross-platform operating system instrumentation framework that exposes system state and activity as relational data. It embeds a hardened SQLite engine and represents operating system concepts (processes, files, sockets, users, registry keys, etc.) as **virtual tables**, enabling users to query live system data using SQL. - -The repository implements: - -- A secure in-memory SQL engine -- A virtual table plugin framework -- Configuration-driven scheduled querying -- Differential result logging and observability -- Distributed query orchestration -- Event-driven monitoring infrastructure -- Persistent and ephemeral storage backends -- Remote HTTP/TLS communication -- Extension-based runtime plugin model - -At a high level, osquery transforms a host into a **SQL-queryable telemetry node** with local and distributed control capabilities. - ---- - -# 2. End-to-End Architecture - -The system is modular and plugin-driven. Below is the full end-to-end architecture across core subsystems. - -## 2.1 System-Level Architecture - -```mermaid -flowchart TD - CLI["osqueryi / osqueryd"] --> Core["Core Init And Runtime"] - - Core --> Config["Configuration And Packs"] - Core --> SQL["SQL Engine And Virtual Tables"] - Core --> Events["Eventing Framework"] - Core --> Logging["Logging And Observability"] - Core --> DB["Database And Storage Plugins"] - Core --> Distributed["Distributed Querying"] - Core --> Extensions["Extensions And IPC"] - Core --> HTTP["Remote HTTP Client"] - Core --> FS["Filesystem And Path Utilities"] - - Config --> SQL - Config --> Events - SQL --> Logging - Events --> DB - Distributed --> SQL - Distributed --> HTTP - Logging --> DB - Extensions --> SQL - Extensions --> DB -``` - ---- - -# 3. Runtime Execution Model - -osquery operates in several runtime modes: - -- **Daemon (`osqueryd`)** -- **Interactive shell (`osqueryi`)** -- **Watcher / Worker model** -- **Extension process** - -## Runtime Lifecycle - -```mermaid -flowchart TD - Start["Process Start"] --> Init["Initializer"] - Init --> Flags["Parse Flags"] - Init --> Registry["Initialize Registry"] - Init --> DBInit["Initialize Database"] - Init --> ExtMgr["Start Extension Manager"] - Init --> ConfigLoad["Load Configuration"] - ConfigLoad --> Schedule["Build Schedule"] - Schedule --> QueryExec["Execute Queries"] - QueryExec --> Log["Log Results"] - Log --> End["Runtime Loop"] -``` - -The **Core Init And Runtime** module governs this lifecycle and ensures safe startup, resource enforcement (watchdog), and graceful shutdown. - -πŸ“˜ Reference: -`osquery/core` β†’ *Core Init And Runtime* - ---- - -# 4. SQL Engine and Virtual Table Layer - -The SQL engine is built around an in-memory SQLite instance hardened with a strict authorizer. All system data is exposed via dynamically attached **virtual tables** backed by registry plugins. - -```mermaid -flowchart TD - Query["SQL Query"] --> SQLite["SQLite Engine"] - SQLite --> Module["sqlite3_module"] - Module --> VirtualTable["VirtualTable Wrapper"] - VirtualTable --> TablePlugin["TablePlugin"] - TablePlugin --> OS["Operating System Data"] - OS --> Rows["Rows Returned"] -``` - -Key characteristics: - -- In-memory execution -- Strict opcode allowlisting -- Constraint pushdown -- Projection optimization -- Extension-backed writable tables - -πŸ“˜ Reference: -`osquery/sql` β†’ *Sql Engine And Virtual Tables* - ---- - -# 5. Configuration and Packs - -Configuration defines: - -- Scheduled queries -- Packs (query groups) -- Decorators -- File monitoring rules -- Event subscriptions -- Logger configuration -- Runtime options - -```mermaid -flowchart TD - Source["Config Source (File / TLS / Extension)"] --> ConfigCore["Config Singleton"] - ConfigCore --> Parsers["ConfigParserPlugins"] - Parsers --> Schedule["Scheduled Queries"] - Schedule --> Scheduler["Query Scheduler"] - Scheduler --> SQL["SQL Engine"] -``` - -Supports: - -- Hash-based change detection -- Background refresh runner -- Query denylisting -- Dynamic registry reconfiguration - -πŸ“˜ Reference: -`osquery/config` β†’ *Configuration And Packs* - ---- - -# 6. Eventing Framework - -The eventing subsystem provides a publish/subscribe model for event-driven monitoring. - -```mermaid -flowchart TD - Publisher["EventPublisher"] --> EventFactory["EventFactory"] - Subscriber["EventSubscriber"] --> EventFactory - EventFactory --> Callback["EventCallback"] - Callback --> Storage["Database Storage"] - SQL["SELECT *_events"] --> Subscriber -``` - -Capabilities: - -- Thread-isolated publishers -- Time-window indexing -- Config-driven retention -- Persistent event buffering -- Optimized incremental queries - -πŸ“˜ Reference: -`osquery/events` β†’ *Eventing Framework And Subscriptions* - ---- - -# 7. Logging and Query Observability - -osquery converts query executions into structured log artifacts. - -```mermaid -flowchart TD - Exec["Query Execution"] --> Diff["DiffResults"] - Diff --> LogItem["QueryLogItem"] - LogItem --> Serialize["JSON Serialization"] - Serialize --> Logger["LoggerPlugin"] - Logger --> Sink["External Backend"] -``` - -Features: - -- Differential result tracking -- Snapshot support -- Structured JSON output -- Performance telemetry -- Pluggable logging backends - -πŸ“˜ Reference: -`osquery/core` β†’ *Logging And Query Observability* - ---- - -# 8. Database and Storage Plugins - -All persistent state is stored through a pluggable key-value interface. - -```mermaid -flowchart TD - Modules["Core Modules"] --> Interface["IDatabaseInterface"] - Interface --> Plugin["Database Plugin"] - Plugin --> RocksDB["Persistent Backend"] - Plugin --> Ephemeral["In-Memory Backend"] -``` - -Used for: - -- Query state -- Event buffers -- Distributed query tracking -- Configuration backup -- Performance metrics - -πŸ“˜ Reference: -`osquery/database` β†’ *Database And Storage Plugins* - ---- - -# 9. Distributed Querying - -Enables centralized orchestration of SQL across fleets. - -```mermaid -flowchart TD - Server["Central Server"] --> TLS["TLS Distributed Plugin"] - TLS --> Core["Distributed Core"] - Core --> SQL["SQL Engine"] - SQL --> Results["Results"] - Results --> TLS - TLS --> Server -``` - -Includes: - -- Discovery query gating -- SHA256-based denylisting -- Execution performance tracking -- Pluggable transport interface - -πŸ“˜ Reference: -`osquery/distributed` β†’ *Distributed Querying* - ---- - -# 10. Extensions and IPC - -osquery supports runtime plugin extensions via Apache Thrift IPC. - -```mermaid -flowchart LR - Core["osquery Core"] --> Manager["Extension Manager"] - Extension["External Extension"] --> Manager - Manager --> Registry["RegistryFactory"] - Core --> Extension -``` - -Capabilities: - -- Dynamic table plugins -- Remote SQL execution -- UUID-based routing -- Health monitoring -- Secure socket validation - -πŸ“˜ Reference: -`osquery/extensions` β†’ *Extensions And IPC* - ---- - -# 11. Remote HTTP Client - -Provides secure HTTP/TLS transport used by: - -- Distributed querying -- Remote logging -- Remote configuration plugins - -```mermaid -flowchart TD - Caller["Module"] --> Client["HTTP Client"] - Client --> TLS["TLS Handshake"] - TLS --> Remote["Remote Server"] -``` - -Built on Boost.Asio and Boost.Beast with strict TLS configuration. - -πŸ“˜ Reference: -`osquery/remote` β†’ *Remote Http Client* - ---- - -# 12. Filesystem and Path Utilities - -Cross-platform abstraction for: - -- Safe file access -- Glob resolution -- Permission enforcement -- Linux `/proc` inspection -- Windows ACL and metadata handling - -```mermaid -flowchart TD - Caller["Subsystem"] --> FS["Filesystem API"] - FS --> POSIX["POSIX Implementation"] - FS --> Windows["Windows Implementation"] -``` - -Security-first design ensures safe extension loading and configuration handling. - -πŸ“˜ Reference: -`osquery/filesystem` β†’ *Filesystem And Path Utilities* - ---- - -# 13. Repository Structure Summary - -| Module | Path | Responsibility | -|--------|------|----------------| -| Core Init And Runtime | `osquery/core` | Process lifecycle, flags, watchdog | -| Configuration And Packs | `osquery/config` | Scheduled queries, packs, decorators | -| SQL Engine And Virtual Tables | `osquery/sql` | SQLite engine, virtual table layer | -| Eventing Framework | `osquery/events` | Publisher/subscriber event system | -| Logging And Query Observability | `osquery/core` | Result diffing, structured logs | -| Database And Storage Plugins | `osquery/database` | Persistent state abstraction | -| Distributed Querying | `osquery/distributed` | Remote fleet orchestration | -| Extensions And IPC | `osquery/extensions` | Thrift-based runtime extensions | -| Remote HTTP Client | `osquery/remote` | Secure HTTP/TLS transport | -| Filesystem And Path Utilities | `osquery/filesystem` | Cross-platform file abstraction | - ---- - -# 14. Architectural Principles - -1. **SQL as a Universal Interface** -2. **Plugin-Driven Extensibility** -3. **Security-First Execution (SQLite authorizer + permission checks)** -4. **In-Memory Analytical Core** -5. **Config-Driven Behavior** -6. **Differential Logging Efficiency** -7. **Distributed Fleet Control** -8. **Cross-Platform Abstraction Layer** - ---- - -# 15. Conclusion - -The `osquery` repository implements a secure, extensible, SQL-based operating system telemetry platform. - -It combines: - -- A hardened SQLite execution engine -- A virtual table plugin architecture -- Event-driven monitoring -- Differential logging -- Distributed query orchestration -- Runtime extension via IPC -- Secure remote communication - -Together, these modules form a scalable, production-grade system observability framework capable of operating both locally and as part of a centrally managed fleet. \ No newline at end of file diff --git a/docs/reference/architecture/configuration-and-packs/configuration-and-packs.md b/docs/reference/architecture/configuration-and-packs/configuration-and-packs.md deleted file mode 100644 index f9ca515ae14..00000000000 --- a/docs/reference/architecture/configuration-and-packs/configuration-and-packs.md +++ /dev/null @@ -1,383 +0,0 @@ -# Configuration And Packs - -The **Configuration And Packs** module is responsible for loading, validating, parsing, distributing, and refreshing osquery configuration data. It transforms raw configuration sources (files, extensions, or remote plugins) into: - -- Scheduled queries and packs -- File monitoring rules -- Event subscriptions -- Logger and runtime options -- SQL views -- Decorations and metadata - -This module acts as the *control plane* for osquery runtime behavior. It connects configuration sources with the scheduler, SQL engine, eventing framework, logging pipeline, and database layer. - ---- - -## Architecture Overview - -At a high level, the module consists of: - -- A **Config registry** for loading configuration sources -- A **ConfigRefreshRunner** background service -- A **Schedule** abstraction built from Packs -- Multiple **ConfigParserPlugin** implementations -- Backup, hashing, and validation logic - -```mermaid -flowchart TD - ConfigPlugin["Config Plugin Registry"] --> ConfigCore["Config Singleton"] - ConfigCore --> RefreshRunner["ConfigRefreshRunner"] - ConfigCore --> Schedule["Schedule"] - Schedule --> Pack["Pack"] - ConfigCore --> Parsers["ConfigParserPlugins"] - Parsers --> OptionsParser["Options Parser"] - Parsers --> FilePathsParser["File Paths Parser"] - Parsers --> DecoratorsParser["Decorators Parser"] - Parsers --> EventsParser["Events Parser"] - Parsers --> ViewsParser["Views Parser"] - ConfigCore --> Database[("Persistent Database")] - Schedule --> Logging["Logging And Query Observability"] - Schedule --> SQLEngine["SQL Engine And Virtual Tables"] - Parsers --> EventFramework["Eventing Framework"] -``` - ---- - -## Core Responsibilities - -### 1. Configuration Loading - -The module defines a `config` registry that supports pluggable configuration sources: - -- `FilesystemConfigPlugin` (default) -- `UpdateConfigPlugin` (internal chaining for extensions) - -Each plugin implements: - -- `genConfig()` β†’ produce configuration map -- `genPack()` β†’ resolve pack references -- `update()` β†’ apply configuration updates - -The active plugin is selected via the `config_plugin` flag. - ---- - -### 2. Config Singleton - -The `Config` class is a process-wide singleton that: - -- Loads configuration -- Maintains the active Schedule -- Applies parser plugins -- Tracks source hashes -- Backs up and restores configuration -- Coordinates plugin reconfiguration - -Key behaviors: - -- JSON validation (max depth and size constraints) -- Comment stripping -- Hash-based change detection -- Safe concurrent access using mutexes -- Performance tracking per scheduled query - ---- - -### 3. Refresh Lifecycle - -The `ConfigRefreshRunner` runs in the background when `config_refresh` is enabled. - -```mermaid -flowchart TD - Start["ConfigRefreshRunner Start"] --> Wait["Sleep refresh_sec"] - Wait --> Check["Interrupted?"] - Check -->|No| Refresh["Config.refresh()"] - Refresh --> Wait - Check -->|Yes| End["End"] -``` - -Features: - -- Normal refresh interval -- Accelerated retry interval on failure -- Optional backup restoration on first failure -- Dynamic reconfiguration of registries and loggers - ---- - -## Packs and Scheduling - -The Schedule is composed of `Pack` objects. - -### Pack - -A **Pack** represents a group of scheduled queries plus optional discovery logic. - -Capabilities: - -- Platform constraints -- Version constraints -- Shard percentage -- Discovery queries -- ScheduledQuery collection -- Execution eligibility (`shouldPackExecute()`) - -### Schedule - -The Schedule: - -- Maintains active packs -- Filters packs via discovery checks -- Tracks denylisted queries -- Restores failed queries on restart -- Provides query iteration to the scheduler - -```mermaid -flowchart TD - ConfigUpdate["Config Update"] --> BuildPacks["Create Pack Objects"] - BuildPacks --> DiscoveryCheck["Discovery Queries"] - DiscoveryCheck --> ActivePack["Active Pack"] - ActivePack --> Scheduler["ScheduledQuery Execution"] - Scheduler --> Performance["Query Performance Recording"] - Scheduler --> Denylist["Denylist On Failure"] -``` - -### Denylist Logic - -If a scheduled query: - -- Causes watchdog termination -- Was executing during shutdown - -It is denylisted temporarily and persisted in the database. - -Expiration logic considers: - -- Time-based expiry -- Event-based exemptions -- Explicit `denylist=false` option - ---- - -## Config Parser Plugins - -The module defines a `config_parser` registry. Each parser consumes specific top-level configuration keys. - -Parsers are applied in this order: - -1. Options parser (always first) -2. Remaining parsers in registry order - -### FilesystemConfigPlugin - -Loads configuration from: - -- Primary JSON file -- Optional `.d` directory fragments -- Multi-pack glob patterns - -Produces a map of `source β†’ JSON content`. - ---- - -### OptionsConfigParserPlugin - -Handles the `options` key. - -- Updates runtime flags -- Prevents modification of CLI-only flags -- Reconfigures logger verbosity dynamically -- Supports `custom_` prefixed flags - ---- - -### FilePathsConfigParserPlugin - -Handles: - -- `file_paths` -- `file_paths_query` -- `file_accesses` -- `exclude_paths` - -Responsibilities: - -- Adds file monitoring rules to Config -- Executes SQL-based path discovery -- Normalizes wildcard globs -- Maintains access category mappings - -This integrates directly with the **Filesystem And Path Utilities** and **Eventing Framework And Subscriptions** modules. - ---- - -### DecoratorsConfigParserPlugin - -Handles the `decorators` key. - -Decorators: - -- Execute SQL queries -- Extract first-row column values -- Attach results to logs - -Supported execution points: - -- `load` -- `always` -- `interval` - -```mermaid -flowchart TD - ConfigLoad["Config Load"] --> RunLoad["Run Load Decorators"] - QueryExec["Query Execution"] --> RunAlways["Run Always Decorators"] - IntervalTick["Interval Timer"] --> RunInterval["Run Interval Decorators"] - RunLoad --> Decorations[("Decoration Store")] - RunAlways --> Decorations - RunInterval --> Decorations - Decorations --> Logger["Status And Query Logs"] -``` - -Thread safety is enforced with dedicated mutexes. - ---- - -### EventsConfigParserPlugin - -Consumes the `events` key and exposes configuration to event publishers and subscribers. - -Acts as a bridge to the **Eventing Framework And Subscriptions** module. - ---- - -### ViewsConfigParserPlugin - -Handles the `views` key. - -- Creates SQL views -- Drops outdated views -- Persists view definitions -- Ensures idempotent updates - -This integrates with the **SQL Engine And Virtual Tables** module. - ---- - -## Backup and Persistence - -If `config_enable_backup` is enabled: - -- Config sources are stored in the database -- Failed refresh attempts restore backup -- Backup keys are namespaced with `config_persistence.` - -The module also persists: - -- Query performance statistics -- Executing query markers -- Denylist entries -- Query result timestamps - -Database interactions use the **Database And Storage Plugins** module. - ---- - -## Hashing and Change Detection - -Each configuration source is hashed (SHA1): - -- Prevents unnecessary reconfiguration -- Enables selective updates -- Supports global config hash generation - -If a source hash does not change: - -- Packs are not rebuilt -- Parsers are not re-applied - ---- - -## Interaction With Other Modules - -### Logging And Query Observability - -- Query performance stats recorded -- Decorations attached to logs -- Denylist logging - -### SQL Engine And Virtual Tables - -- Discovery queries -- Scheduled queries -- View creation - -### Eventing Framework And Subscriptions - -- Event configuration via `events` -- File path monitoring integration - -### Database And Storage Plugins - -- Config backup -- Query stats -- Denylist persistence -- Result expiration - -### Extensions And IPC - -- External extensions can call `Config::update()` -- UpdateConfigPlugin enables update chaining -- External registries propagate updates back to core - ---- - -## Configuration Processing Flow - -```mermaid -flowchart TD - Source["Config Source"] --> GenConfig["genConfig()"] - GenConfig --> Update["Config.update()"] - Update --> Purge["Purge Stale State"] - Purge --> UpdateSource["updateSource()"] - UpdateSource --> ParseJSON["Validate And Parse JSON"] - ParseJSON --> BuildSchedule["Create Packs And Schedule"] - BuildSchedule --> ApplyParsers["Apply ConfigParserPlugins"] - ApplyParsers --> Reconfigure["Registry.configure()"] - Reconfigure --> EventNotify["EventFactory.configUpdate()"] - EventNotify --> Ready["Runtime Updated"] -``` - ---- - -## Concurrency and Safety - -The module uses: - -- Recursive mutexes for schedule and files -- Dedicated mutexes for: - - Hash map - - Backup store - - Performance stats - - Decorations - -Design goals: - -- Safe asynchronous refresh -- Deterministic schedule rebuilds -- Minimal runtime disruption -- Atomic config transitions - ---- - -## Summary - -The **Configuration And Packs** module is the orchestration layer of osquery runtime behavior. It: - -- Loads configuration from pluggable sources -- Builds and manages query packs -- Applies runtime options -- Coordinates parser plugins -- Manages refresh and backup logic -- Integrates deeply with scheduler, SQL engine, event system, logging, and database layers - -Without this module, osquery would have no dynamic runtime control plane. It is the central authority that transforms static configuration data into active system behavior. diff --git a/docs/reference/architecture/core-init-and-runtime/core-init-and-runtime.md b/docs/reference/architecture/core-init-and-runtime/core-init-and-runtime.md deleted file mode 100644 index 03460755d71..00000000000 --- a/docs/reference/architecture/core-init-and-runtime/core-init-and-runtime.md +++ /dev/null @@ -1,428 +0,0 @@ -# Core Init And Runtime - -The **Core Init And Runtime** module is the foundational execution layer of osquery. It is responsible for: - -- Process initialization and bootstrap -- Command-line flag management -- Worker / watcher lifecycle orchestration -- Graceful and forced shutdown handling -- Platform and identity utilities (UUID, host identity, privilege control) -- Virtual table execution context and constraint handling -- Watchdog-based resource governance - -This module defines the runtime contract that all higher-level subsystems rely on, including: - -- [Configuration And Packs](../configuration-and-packs/configuration-and-packs.md) -- [Logging And Query Observability](../logging-and-query-observability/logging-and-query-observability.md) -- [Database And Storage Plugins](../database-and-storage-plugins/database-and-storage-plugins.md) -- [SQL Engine And Virtual Tables](../sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md) -- [Eventing Framework And Subscriptions](../eventing-framework-and-subscriptions/eventing-framework-and-subscriptions.md) -- [Extensions And IPC](../extensions-and-ipc/extensions-and-ipc.md) - ---- - -## 1. Architectural Overview - -At runtime, osquery can operate in multiple modes: - -- **Daemon** (`osqueryd`) -- **Shell** (`osqueryi`) -- **Watcher (watchdog)** -- **Worker** (child process of watcher) -- **Extension process** - -The Core Init And Runtime module orchestrates how these modes are selected, started, monitored, and shut down. - -### High-Level Runtime Architecture - -```mermaid -flowchart TD - Main["Main Entry Point"] --> Initializer["Initializer"] - - Initializer --> Flags["Flag System"] - Initializer --> Registry["Registry And Plugins"] - Initializer --> Database["Database Initialization"] - Initializer --> ExtensionManager["Extension Manager"] - Initializer --> Events["Event Factory"] - Initializer --> Watchdog["Watcher / Worker Model"] - - Watchdog --> Worker["Worker Process"] - Watchdog --> Extensions["Extension Processes"] - - Worker --> Tables["Virtual Tables"] - Worker --> Scheduler["Query Scheduler"] - Worker --> Logger["Logger Plugins"] -``` - -The **Initializer** class is the central orchestrator of this lifecycle. - ---- - -## 2. Initialization Lifecycle - -The `Initializer` class (defined in `system.h` and implemented in `init.cpp`) governs startup behavior. - -### Core Responsibilities - -- Parse CLI flags and flagfiles -- Determine tool type (daemon, shell, extension) -- Configure runtime defaults -- Initialize registries and plugins -- Start extension manager -- Load configuration -- Attach event publishers -- Activate logging and distributed plugins -- Launch watcher/worker model if enabled - -### Startup Sequence - -```mermaid -sequenceDiagram - participant Main - participant Initializer - participant Registry - participant Config - participant Extensions - participant Events - - Main->>Initializer: Construct(argc, argv) - Initializer->>Initializer: Parse flags - Initializer->>Registry: registryAndPluginInit() - Initializer->>Extensions: startExtensionManager() - Initializer->>Config: load() - Initializer->>Events: attachEvents() - Initializer->>Initializer: start() -``` - -The `start()` method finalizes runtime activation after construction. - ---- - -## 3. Flag System - -Core Components: - -- `FlagDetail` -- `FlagInfo` -- `Flag` - -The flag system wraps **Google GFlags** to provide: - -- Structured metadata (shell-only, CLI-only, hidden, extension-only) -- Default value tracking -- Runtime updates -- Custom flags from configuration - -### Flag Architecture - -```mermaid -flowchart LR - Macro["FLAG / CLI_FLAG Macros"] --> FlagCreate["Flag::create()"] - FlagCreate --> FlagRegistry["Internal Flag Map"] - - CLI["Command Line"] --> Parse["ParseCommandLineFlags"] - Config["Config Options"] --> Update["Flag::updateValue()"] - - FlagRegistry --> Runtime["Runtime Access"] -``` - -Important behaviors: - -- Flags can be set via CLI, flagfile, or config (unless CLI-only) -- `Flag::isDefault()` detects overridden values -- Aliases support backwards compatibility - -Flags influence: - -- Database activation -- Watchdog enablement -- Logging and distributed plugins -- OpenFrame mode - ---- - -## 4. Worker and Watcher Model - -Core Components: - -- `LimitDefinition` -- `PerformanceChange` - -The watcher/worker architecture isolates execution for reliability and resource control. - -### Roles - -| Role | Responsibility | -|------|----------------| -| Watcher | Monitors worker and extensions | -| Worker | Executes queries and plugins | -| Extension | External plugin process | - -### Watchdog Control Flow - -```mermaid -flowchart TD - Watcher["Watcher Process"] --> Spawn["Spawn Worker"] - Spawn --> Monitor["Monitor CPU And Memory"] - Monitor --> Check{{"Limit Exceeded?"}} - Check -->|"No"| Continue["Continue Monitoring"] - Check -->|"Yes"| Kill["Stop Worker"] - Kill --> Respawn["Respawn Worker"] -``` - -### Resource Limits - -`LimitDefinition` defines profiles: - -- Normal -- Restrictive -- Disabled - -Enforced metrics: - -- Memory footprint (MB) -- CPU utilization percentage -- Sustained latency duration -- Respawn frequency - -`PerformanceChange` calculates CPU window utilization based on interval and CPU count. - -If limits are exceeded: - -- Worker is gracefully terminated -- Forced kill occurs if needed -- Restart logic applies exponential backoff - ---- - -## 5. Shutdown Coordination - -Core Component: - -- `ShutdownData` - -Shutdown is centralized and thread-safe. - -### Shutdown Model - -```mermaid -flowchart TD - AnyThread["Any Thread"] --> Request["requestShutdown()"] - Request --> Signal["Condition Variable"] - Signal --> MainThread["Main Thread"] - MainThread --> StopServices["Dispatcher::stopServices()"] - StopServices --> Join["Join Services"] - Join --> StopEvents["EventFactory::end()"] - StopEvents --> CloseDB["shutdownDatabase()"] - CloseDB --> Exit["Return Exit Code"] -``` - -Key properties: - -- Shutdown can only be requested once -- Exit code is stored atomically -- `AlarmRunnable` enforces maximum shutdown time -- `shutdownNow()` performs immediate `_Exit()` - -This guarantees deterministic teardown even during deadlocks. - ---- - -## 6. Platform and System Utilities - -Core Components: - -- `PrivateData` -- `stat` -- `tm` - -### Host Identity - -The module manages multiple UUID strategies: - -- Hardware UUID -- Instance UUID (persistent) -- Ephemeral UUID -- Specified UUID -- Hostname fallback - -Flow: - -```mermaid -flowchart LR - Flag["host_identifier Flag"] --> Mode{{"Mode"}} - Mode -->|"uuid"| Hardware["Hardware UUID"] - Mode -->|"instance"| Instance["Instance UUID"] - Mode -->|"ephemeral"| Ephemeral["Random UUID"] - Mode -->|"specified"| Specified["Configured Identifier"] - Mode -->|"default"| Hostname["System Hostname"] -``` - -Persistent identifiers are stored via the database layer. - -### Privilege Dropping (POSIX) - -- Temporarily reduce effective UID/GID -- Restore original groups on destruction -- Used when interacting with filesystem-backed tables - -### Thread Naming - -Cross-platform thread naming for observability and debugging. - ---- - -## 7. Virtual Table Execution Context - -Core Components: - -- `Constraint` -- `ConstraintList` -- `QueryContext` -- `VirtualTableContent` - -This subsystem forms the bridge between: - -- SQLite virtual table engine -- osquery table plugins - -### Constraint Model - -```mermaid -flowchart TD - SQL["SQL Query"] --> SQLite["SQLite Virtual Table API"] - SQLite --> QueryContext["QueryContext"] - QueryContext --> ConstraintMap["ConstraintMap"] - ConstraintMap --> ConstraintList["ConstraintList"] - ConstraintList --> TablePlugin["TablePlugin::generate()"] -``` - -`ConstraintList` supports: - -- Operator-aware filtering -- Affinity-aware matching (TEXT, INTEGER, BIGINT) -- Efficient pre-filter evaluation - -`QueryContext` provides: - -- Column usage tracking -- Constraint expansion -- Cache control flags - -### VirtualTableContent - -Stores: - -- Column definitions -- Attributes (CACHEABLE, EVENT_BASED, etc.) -- Aliases -- Query-scoped caches - -This metadata allows high-performance virtual table execution without repeated registry lookups. - ---- - -## 8. Table Plugin Integration - -Although fully described in [SQL Engine And Virtual Tables](../sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md), Core Init And Runtime provides: - -- Context serialization -- Cache freshness checks -- Generator-based row streaming -- Extension table attachment - -Caching model: - -- Interval-based freshness -- Disabled when constraints are complex -- Serialized into backing database - ---- - -## 9. Signal Handling - -Signals handled: - -- `SIGTERM` -- `SIGINT` -- `SIGUSR1` - -Behavior: - -- `SIGUSR1` β†’ mark resource limit hit -- `SIGTERM` / `SIGINT` β†’ request graceful shutdown -- Alarm timeout enforces upper bound on shutdown time - ---- - -## 10. OpenFrame Mode Integration - -Additional flags: - -- `openframe_mode` -- `openframe_secret` -- `openframe_token_path` - -When enabled: - -- Initializes encryption service -- Extracts token -- Starts token refresher -- Updates authorization manager - -This integrates osquery runtime with OpenFrame-managed authentication. - ---- - -## 11. How This Module Fits the System - -The Core Init And Runtime module: - -- Bootstraps **all subsystems** -- Owns lifecycle and teardown guarantees -- Provides runtime isolation (watchdog model) -- Supplies identity and environment context -- Enables safe virtual table execution -- Enforces resource governance - -Every other module depends on it either directly or indirectly. - -### Dependency Positioning - -```mermaid -flowchart TD - Core["Core Init And Runtime"] - - Core --> Config["Configuration And Packs"] - Core --> Logging["Logging And Query Observability"] - Core --> Database["Database And Storage Plugins"] - Core --> SQL["SQL Engine And Virtual Tables"] - Core --> Events["Eventing Framework And Subscriptions"] - Core --> Extensions["Extensions And IPC"] -``` - -Without this module: - -- No flags would be parsed -- No plugins would activate -- No scheduler would run -- No graceful shutdown would occur - -It is the execution spine of the osquery system. - ---- - -## Summary - -The **Core Init And Runtime** module is responsible for transforming a process invocation into a fully operational, resource-governed, plugin-enabled osquery runtime. - -It provides: - -- Deterministic initialization -- Mode-aware execution (daemon, shell, extension) -- Resource watchdog enforcement -- Safe shutdown semantics -- Host identity management -- Virtual table execution scaffolding - -All higher-level capabilities build on top of this foundational runtime layer. \ No newline at end of file diff --git a/docs/reference/architecture/database-and-storage-plugins/database-and-storage-plugins.md b/docs/reference/architecture/database-and-storage-plugins/database-and-storage-plugins.md deleted file mode 100644 index c3210d09ee9..00000000000 --- a/docs/reference/architecture/database-and-storage-plugins/database-and-storage-plugins.md +++ /dev/null @@ -1,357 +0,0 @@ -# Database And Storage Plugins - -## Overview - -The **Database And Storage Plugins** module provides the persistent and ephemeral key-value storage abstraction used by osquery core. It exposes a unified database interface that: - -- Persists configuration state, scheduled query metadata, event buffers, logs, and distributed query state. -- Abstracts the underlying storage engine (e.g., RocksDB or in-memory ephemeral storage). -- Provides safe concurrent access with reset and migration support. -- Enables plugin-based database backends via the internal registry system. - -At runtime, this module acts as the storage backbone for higher-level modules such as: - -- [Configuration And Packs](../configuration-and-packs/configuration-and-packs.md) -- [Logging And Query Observability](../logging-and-query-observability/logging-and-query-observability.md) -- [Distributed Querying](../distributed-querying/distributed-querying.md) -- [Eventing Framework And Subscriptions](../eventing-framework-and-subscriptions/eventing-framework-and-subscriptions.md) -- [SQL Engine And Virtual Tables](../sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md) - -It is implemented using a registry-driven plugin architecture, allowing the storage backend to be replaced or disabled. - ---- - -## Architectural Overview - -### High-Level Architecture - -```mermaid -flowchart TD - CoreModules["Core And Feature Modules"] --> DBInterface["IDatabaseInterface"] - DBInterface --> OsqueryDB["OsqueryDatabase"] - OsqueryDB --> RegistryLayer["Database Plugin Registry"] - RegistryLayer --> RocksDBPlugin["RocksDB Plugin (Persistent)"] - RegistryLayer --> EphemeralPlugin["Ephemeral Database Plugin (In-Memory)"] - - subgraph storage_domains["Logical Storage Domains"] - PersistentSettings["configurations"] - Queries["queries"] - Events["events"] - Logs["logs"] - Carves["carves"] - Distributed["distributed"] - QueryPerformance["query_performance"] - end - - RocksDBPlugin --> storage_domains - EphemeralPlugin --> storage_domains -``` - -The module separates concerns into three layers: - -1. **Public Database Interface** – Used by the rest of the system. -2. **Plugin Registry Layer** – Selects and manages the active database backend. -3. **Concrete Database Plugins** – Implement actual storage logic. - ---- - -## Core Components - -### OsqueryDatabase - -**Component:** `osquery.osquery.database.database.OsqueryDatabase` - -`OsqueryDatabase` implements the `IDatabaseInterface` and serves as the primary entry point for all database operations inside osquery. - -It provides: - -- `getDatabaseValue` (string and integer variants) -- `setDatabaseValue` -- `setDatabaseBatch` -- `deleteDatabaseValue` -- `deleteDatabaseRange` -- `scanDatabaseKeys` - -Internally, it delegates all operations to free functions such as: - -- `getDatabaseValue(...)` -- `setDatabaseValue(...)` -- `scanDatabaseKeys(...)` - -These functions: - -- Acquire read/write locks to protect against concurrent resets. -- Route calls through the database plugin registry. -- Support both internal and external (extension-based) registry execution. - -This design ensures that the rest of the system depends only on the interface, not the concrete backend. - ---- - -### EphemeralDatabasePlugin - -**Component:** `osquery.osquery.database.ephemeral.EphemeralDatabasePlugin` - -The `EphemeralDatabasePlugin` is an in-memory implementation of the `DatabasePlugin` interface. - -It is: - -- Registered internally as the `ephemeral` database plugin. -- Used when the `disable_database` flag is enabled. -- Used for testing and fallback scenarios. - -#### Internal Data Structure - -```mermaid -flowchart TD - DB["EphemeralDatabasePlugin"] --> Map["std::map>"] - Map --> DomainMap["Domain Map"] - DomainMap --> KeyValue["boost::variant"] -``` - -Storage model: - -- Outer map: `domain β†’ domain_map` -- Inner map: `key β†’ boost::variant` - -Supported operations: - -- `get` -- `put` -- `putBatch` -- `remove` -- `removeRange` -- `scan` - -Since it is purely in-memory: - -- No persistence across restarts. -- No disk I/O. -- Ideal for unit testing and ephemeral deployments. - ---- - -## Database Domains - -The module defines several logical domains used throughout osquery: - -| Domain | Purpose | -|--------|----------| -| `configurations` | Persistent settings and metadata | -| `queries` | Scheduled query state and results metadata | -| `events` | Event framework buffers | -| `logs` | Buffered logging data | -| `carves` | File carving state | -| `distributed` | Distributed query tasks | -| `distributed_running` | Active distributed query tracking | -| `query_performance` | Performance metrics for queries | - -Domains act as logical namespaces within a single database backend. - ---- - -## Plugin Lifecycle and Initialization - -### Initialization Flow - -```mermaid -flowchart TD - Start["Startup"] --> CheckFlag{{"disable_database?"}} - CheckFlag -->|"No"| UseRocksDB["Activate RocksDB Plugin"] - CheckFlag -->|"Yes"| UseEphemeral["Activate Ephemeral Plugin"] - UseRocksDB --> InitDone["Database Initialized"] - UseEphemeral --> InitDone -``` - -Key behaviors: - -- If `disable_database` is false, the internal persistent plugin (RocksDB) is selected. -- If `disable_database` is true, the `ephemeral` plugin is activated. -- Initialization retries up to 25 times to handle file locks or shutdown races. -- `kDBInitialized` ensures safe access. - ---- - -## Concurrency and Reset Safety - -To protect database operations: - -- A global mutex `kDatabaseReset` is used. -- Read locks guard standard operations. -- Write locks guard reset flows. - -### Reset Flow - -```mermaid -flowchart TD - ResetCall["resetDatabase()"] --> LockWrite["Acquire Write Lock"] - LockWrite --> TearDown["Plugin tearDown()"] - TearDown --> SetUp["Plugin setUp()"] - SetUp --> Unlock["Release Lock"] -``` - -If reset fails: - -- The registry falls back to the `ephemeral` backend. -- A warning is logged. - -This prevents total system failure when persistent storage is corrupted. - ---- - -## Database Migrations - -The module supports schema/data migrations via version tracking. - -- Version key: `results_version` -- Stored under domain: `configurations` - -### Migration Flow - -```mermaid -flowchart TD - ReadVersion["Read results_version"] --> Compare["Compare with target version"] - Compare -->|"Older"| RunMigration["Execute migrateVxVy()"] - RunMigration --> UpdateVersion["Persist new version"] - UpdateVersion --> Compare - Compare -->|"Match"| Done["Migration Complete"] -``` - -Examples of migrations: - -- Converting legacy property-tree JSON to RapidJSON format. -- Renaming event publisher keys. - -If migration fails: - -- Error is logged. -- Database upgrade aborts. - ---- - -## Interaction with Other Modules - -### Configuration And Packs - -Configuration refreshers persist: - -- Pack execution counters. -- Schedule metadata. -- Decorator state. - -All stored under structured domains using `setDatabaseValue` and `scanDatabaseKeys`. - ---- - -### Logging And Query Observability - -This module persists: - -- Scheduled query state. -- Query performance metrics. -- Result differentials. - -The `query_performance` domain ensures metrics survive restarts when persistent storage is enabled. - ---- - -### Distributed Querying - -The distributed module uses domains: - -- `distributed` -- `distributed_running` - -These track: - -- Pending tasks. -- Active query execution state. - ---- - -### Eventing Framework And Subscriptions - -Event publishers and subscribers use the `events` domain to: - -- Persist event cursors. -- Maintain audit or publisher state. - ---- - -## Extension and External Registry Behavior - -If osquery is running as an **extension process**: - -- It does not directly access the database backend. -- Database requests are routed via `Registry::call("database", ...)`. -- The main daemon handles storage. - -This enforces: - -- Centralized persistence. -- No extension-level database corruption risk. - ---- - -## Key Design Characteristics - -### 1. Plugin-Based Storage - -- Storage backend selected via registry. -- Easily replaceable or disabled. -- Uniform request interface (`get`, `put`, `scan`, etc.). - -### 2. Domain Isolation - -Logical partitioning prevents key collisions across features. - -### 3. Migration Safety - -- Explicit version tracking. -- Incremental migration steps. -- Failure-safe version commit. - -### 4. Reset and Recovery Support - -- Safe reset workflow. -- Automatic fallback to ephemeral storage. -- Multi-attempt initialization. - -### 5. Thread Safety - -- Reader/writer locking around reset flows. -- Atomic flags guarding initialization state. - ---- - -## When to Use Each Backend - -| Backend | Use Case | -|----------|----------| -| RocksDB (Persistent) | Production deployments requiring durable state | -| Ephemeral | Testing, debugging, or stateless environments | - -Testing environments typically use: - -```text -initDatabasePluginForTesting() -``` - -Which: - -- Forces `disable_database = true` -- Activates ephemeral storage -- Resets the database - ---- - -## Summary - -The **Database And Storage Plugins** module provides the foundational storage abstraction for osquery. It: - -- Decouples storage implementation from core logic. -- Enables plugin-based extensibility. -- Protects against corruption via reset and migration controls. -- Supports both persistent and in-memory storage. - -Every major subsystem in osquery relies on this module for durable state, making it a critical backbone component of the overall architecture. \ No newline at end of file diff --git a/docs/reference/architecture/distributed-querying/distributed-querying.md b/docs/reference/architecture/distributed-querying/distributed-querying.md deleted file mode 100644 index 37c3bf57f6c..00000000000 --- a/docs/reference/architecture/distributed-querying/distributed-querying.md +++ /dev/null @@ -1,376 +0,0 @@ -# Distributed Querying - -The **Distributed Querying** module enables remote orchestration and execution of SQL queries across a fleet of osquery nodes. It allows a central server to: - -- Push SQL queries to enrolled agents -- Collect query results asynchronously -- Monitor execution performance -- Denylist problematic or long-running queries - -This module acts as the execution bridge between: - -- The **SQL Engine and Virtual Tables** subsystem (local query execution) -- The **Database and Storage Plugins** subsystem (state management) -- The **Logging and Query Observability** subsystem (performance + result logging) -- Remote transport implementations such as the TLS-based plugin - -At its core, Distributed Querying defines the request/response model, execution lifecycle, denylist logic, and plugin interface for retrieving and submitting distributed work. - ---- - -## Architectural Overview - -```mermaid -flowchart TD - Server["Central Server"] -->|"HTTPS"| TLSPlugin["TLS Distributed Plugin"] - TLSPlugin -->|"getQueries()"| DistributedCore["Distributed Core"] - DistributedCore -->|"runQueries()"| SQLEngine["SQL Engine"] - SQLEngine -->|"QueryData"| DistributedCore - DistributedCore -->|"writeResults()"| TLSPlugin - TLSPlugin -->|"HTTPS"| Server - - DistributedCore -->|"recordQueryPerformance"| Observability["Query Observability"] - DistributedCore -->|"running state"| Storage["Database Plugins"] -``` - -### Key Responsibilities - -| Component | Responsibility | -|------------|----------------| -| Distributed Core | Lifecycle, queuing, denylisting, performance tracking | -| Distributed Plugin | Abstract interface for remote work retrieval and result submission | -| TLS Distributed Plugin | HTTPS implementation of Distributed Plugin | -| SQL Engine | Executes queries against virtual tables | -| Database Plugins | Persist running state and deduplication | -| Observability | Tracks performance and execution metrics | - ---- - -# Core Data Structures - -## DistributedQueryRequest - -Represents a single unit of distributed work. - -```text -Fields: -- id: Unique server-assigned identifier -- query: SQL string to execute -``` - -This structure is serialized/deserialized using: - -- `serializeDistributedQueryRequest` -- `deserializeDistributedQueryRequest` -- JSON string helpers for transport safety - -### Example Server Payload - -```json -{ - "queries": { - "id1": "select * from osquery_info", - "id2": "select * from processes" - } -} -``` - -Each key becomes a `DistributedQueryRequest` instance. - ---- - -## DistributedQueryResult - -Encapsulates execution output and metadata. - -```text -Fields: -- request: Original DistributedQueryRequest -- results: QueryData (rows) -- columns: ColumnNames -- status: Execution Status -- message: Optional error or informational message -``` - -Serialized using: - -- `serializeDistributedQueryResult` -- `deserializeDistributedQueryResult` - -This object ensures transport-neutral packaging of results. - ---- - -# Distributed Execution Lifecycle - -The `Distributed` class orchestrates the runtime behavior. - -```mermaid -flowchart TD - Start["Pull Updates"] --> Accept["acceptWork()"] - Accept --> Pending{"Pending Queries?"} - Pending -->|"Yes"| Run["runQueries()"] - Run --> Deny{"Denylisted?"} - Deny -->|"No"| Execute["Execute via SQL Engine"] - Execute --> Record["recordQueryPerformance()"] - Record --> Queue["addResult()"] - Queue --> Flush["flushCompleted()"] - Deny -->|"Yes"| Skip["Skip Execution"] - Flush --> End["Cycle Complete"] -``` - ---- - -## 1. Pull Phase - -```cpp -Status pullUpdates(); -``` - -- Calls the active `DistributedPlugin` -- Retrieves raw JSON work payload -- Delegates parsing to `acceptWork()` - ---- - -## 2. Work Acceptance - -```cpp -Status acceptWork(const std::string& work); -``` - -Behavior: - -- Parses `queries` section -- Evaluates optional `discovery` queries -- Enqueues only valid work - -### Discovery Query Logic - -- If no discovery query exists β†’ enqueue -- If discovery returns rows β†’ enqueue -- If discovery returns zero rows β†’ skip - -This prevents unnecessary execution on nodes where the query is irrelevant. - ---- - -## 3. Denylisting Protection - -To prevent repeated execution of problematic queries, the module includes: - -- `checkAndSetAsRunning()` -- `setAsNotRunning()` -- `denylistedQueryTimestampExpired()` -- `denylistDuration()` -- `hashQuery()` - -### Denylist Mechanism - -```mermaid -flowchart TD - Incoming["Incoming Query"] --> Check["checkAndSetAsRunning()"] - Check --> Running{"Already Running?"} - Running -->|"Within Duration"| Block["Denylisted"] - Running -->|"Expired"| Allow["Allow Execution"] - Check -->|"First Time"| Allow -``` - -The denylist key is derived from: - -```text -SHA256(query) -``` - -This ensures consistent identification across restarts and transports. - ---- - -## 4. Query Execution - -```cpp -Status runQueries(); -``` - -For each pending request: - -1. Pop request from internal storage -2. Execute via SQL engine -3. Measure execution time -4. Record performance -5. Store result -6. Clear running state - -Execution uses: - -```cpp -SQL monitorNonnumeric(const std::string& name, const std::string& query); -``` - ---- - -## 5. Performance Recording - -Performance metrics are stored in: - -```text -std::map performance_ -``` - -Tracked metrics include: - -- Execution duration -- Row count -- Sample process table deltas - -These integrate with the **Logging and Query Observability** module. - ---- - -## 6. Result Flushing - -```cpp -Status flushCompleted(); -``` - -- Serializes accumulated `DistributedQueryResult` objects -- Sends them to the plugin via `writeResults()` -- Clears local result queue - ---- - -# Plugin Interface - -## DistributedPlugin (Abstract) - -Defines the extension point for remote orchestration. - -```cpp -virtual Status getQueries(std::string& json) = 0; -virtual Status writeResults(const std::string& json) = 0; -``` - -The `call()` method acts as the unified entry point for plugin requests. - -Any transport (TLS, filesystem, custom RPC) can implement this interface. - ---- - -# TLS Distributed Plugin - -The **TLS Distributed Plugin** provides an HTTPS implementation of the plugin interface. - -Registered as: - -```text -REGISTER(TLSDistributedPlugin, "distributed", "tls") -``` - -## Configuration Flags - -```text ---distributed_tls_read_endpoint ---distributed_tls_write_endpoint ---distributed_tls_max_attempts -``` - -## Setup Phase - -```cpp -Status setUp(); -``` - -- Builds read/write URIs -- Uses TLS request helper utilities - -## Query Retrieval - -```cpp -Status getQueries(std::string& json); -``` - -- Sends POST to read endpoint -- Returns JSON payload containing work - -## Result Submission - -```cpp -Status writeResults(const std::string& json); -``` - -- Parses JSON -- POSTs to write endpoint -- Ignores server response body - ---- - -## TLS Data Flow - -```mermaid -sequenceDiagram - participant Node - participant TLS as "TLS Distributed Plugin" - participant Server - - Node->>TLS: pullUpdates() - TLS->>Server: POST read endpoint - Server->>TLS: JSON queries - TLS->>Node: return JSON - Node->>TLS: writeResults(JSON) - TLS->>Server: POST write endpoint -``` - ---- - -# State Management - -Distributed Querying relies on persistent state to: - -- Track running queries -- Store denylist timestamps -- Maintain queued work - -This state is handled through the Database and Storage Plugins subsystem. - ---- - -# Security Model - -Key security properties: - -1. Query integrity validated via SHA-256 hashing -2. HTTPS transport in TLS plugin -3. Configurable retry attempts -4. Denylist prevents execution storms - -The denylist duration is configurable via runtime flags. - ---- - -# Integration Points - -Distributed Querying integrates with: - -- SQL Engine and Virtual Tables (local execution) -- Database and Storage Plugins (persistent state) -- Logging and Query Observability (performance tracking) -- Extensions and IPC (optional distributed plugin implementations) -- Remote HTTP Client (transport layer headers and networking) - -Sibling module reference: - -- [Remote HTTP Client](../remote-http-client/remote-http-client.md) - ---- - -# Summary - -The **Distributed Querying** module provides: - -- A structured request/response model -- Pluggable transport abstraction -- Controlled query lifecycle management -- Built-in denylisting and performance tracking -- Secure TLS-based remote orchestration - -It transforms osquery from a local query engine into a centrally orchestrated distributed telemetry system while preserving execution safety and observability guarantees. diff --git a/docs/reference/architecture/eventing-framework-and-subscriptions/eventing-framework-and-subscriptions.md b/docs/reference/architecture/eventing-framework-and-subscriptions/eventing-framework-and-subscriptions.md deleted file mode 100644 index a18e9b822dc..00000000000 --- a/docs/reference/architecture/eventing-framework-and-subscriptions/eventing-framework-and-subscriptions.md +++ /dev/null @@ -1,363 +0,0 @@ -# Eventing Framework And Subscriptions - -## Overview - -The **Eventing Framework And Subscriptions** module provides the core publish/subscribe infrastructure for osquery’s event-driven data model. It enables: - -- Event publishers to monitor operating system activity (e.g., file changes, process activity). -- Event subscribers to register interest in specific event types. -- Persistent storage of event data for later retrieval via virtual tables. -- Automatic lifecycle, expiration, and optimization of event data based on query schedules. - -This module acts as the bridge between: - -- The SQL Engine And Virtual Tables module (which exposes event tables to queries). -- The Database And Storage Plugins module (which persists event data). -- The Configuration And Packs module (which defines scheduled queries and event usage). - ---- - -## High-Level Architecture - -```mermaid -flowchart TD - Config["Configuration And Packs"] -->|"scheduled queries"| EventFactory["EventFactory"] - EventFactory -->|"registers"| Publisher["EventPublisherPlugin"] - EventFactory -->|"registers"| Subscriber["EventSubscriberPlugin"] - Subscriber -->|"creates"| SubscriptionObj["Subscription"] - Publisher -->|"dispatches"| Callback["EventCallback"] - Callback -->|"addBatch()"| Storage["Database And Storage Plugins"] - SQL["SQL Engine And Virtual Tables"] -->|"SELECT from _events tables"| Subscriber - Subscriber -->|"generateRows()"| SQL -``` - -### Core Roles - -- **EventFactory** – Central coordinator for publishers and subscribers. -- **EventPublisherPlugin** – Produces system events. -- **EventSubscriberPlugin** – Consumes events and exposes them as queryable tables. -- **Subscription** – Binds subscriber logic to publisher events. -- **PathSet utilities** – Optimized path pattern matching for filesystem-related subscriptions. - ---- - -## EventFactory - -The `EventFactory` is a singleton responsible for managing the lifecycle and coordination of all event publishers and subscribers. - -### Responsibilities - -- Registering and deregistering event publishers and subscribers. -- Creating and forwarding subscriptions. -- Spawning publisher run-loop threads. -- Coordinating shutdown and cleanup. -- Applying configuration updates (e.g., scheduled query analysis). - -### Publisher and Subscriber Registration - -```mermaid -flowchart TD - RegisterSub["registerEventSubscriber()"] --> SetupSub["setUp()"] - SetupSub --> InitSub["init()"] - InitSub --> RunningSub["EVENT_RUNNING"] - - RegisterPub["registerEventPublisher()"] --> SetupPub["setUp()"] - SetupPub --> ReadyPub["EVENT_SETUP"] -``` - -During registration: - -- Subscribers are optionally enabled/disabled via configuration (`events.enable_subscribers`, `events.disable_subscribers`). -- Publishers are validated by type and initialized if events are enabled. -- State transitions are enforced (`EVENT_NONE`, `EVENT_SETUP`, `EVENT_RUNNING`, `EVENT_PAUSED`, `EVENT_FAILED`). - -### Publisher Execution Model - -Each publisher runs in its own thread: - -```mermaid -flowchart TD - Delay["delay()"] --> ThreadSpawn["spawn thread per publisher"] - ThreadSpawn --> RunLoop["run(type_id)"] - RunLoop --> PublisherRun["publisher->run()"] - PublisherRun --> Pause["pause(200ms)"] - Pause --> RunLoop - RunLoop -->|"isEnding()"| TearDown["tearDown()"] -``` - -Key behaviors: - -- The factory prevents publisher restarts once started. -- A default cool-off pause avoids CPU thrashing. -- Shutdown via `end()` gracefully deregisters publishers and joins threads. - ---- - -## Subscription Model - -The `Subscription` structure binds a subscriber to a publisher. - -### Subscription Structure - -```text -Subscription - β”œβ”€β”€ subscriber_name - β”œβ”€β”€ context (SubscriptionContextRef) - └── callback (EventCallback) -``` - -Defined in: - -- `osquery.osquery.events.subscription.Subscription` - -### EventCallback - -```text -Status(const EventContextRef&, const SubscriptionContextRef&) -``` - -When an event fires: - -1. The publisher evaluates matching subscriptions. -2. The associated callback executes. -3. The subscriber stores structured rows using `addBatch()`. - -Subscriptions are usually created via: - -```cpp -EventFactory::addSubscription("PublisherType", subscription); -``` - ---- - -## EventSubscriberPlugin - -Defined in: - -- `osquery.osquery.events.eventsubscriberplugin.Context` -- `osquery.osquery.events.eventsubscriberplugin.GenerateRowsResult` - -The `EventSubscriberPlugin` class encapsulates event storage, indexing, expiration, and query-time retrieval. - -### Core Responsibilities - -- Registering subscriptions in `init()`. -- Storing events using `addBatch()`. -- Managing time-based indexing and expiration. -- Exposing events as virtual tables via `genTable()`. - -### Internal Context - -```text -Context - β”œβ”€β”€ database_namespace - β”œβ”€β”€ event_index - β”œβ”€β”€ last_query_time - └── last_event_id -``` - -The context ensures: - -- Namespace isolation between subscribers. -- Efficient indexing by time and event identifier. -- Thread-safe updates using mutexes. - -### Event Storage and Expiration - -```mermaid -flowchart TD - Callback["EventCallback"] --> AddBatch["addBatch()"] - AddBatch --> AssignID["generateEventIdentifier()"] - AssignID --> Persist["Database write"] - Persist --> IndexUpdate["update time index"] - - ExpireCheck["expireEventBatches()"] --> RemoveOld["delete expired batches"] -``` - -Expiration behavior is governed by: - -- `getEventsExpiry()` – Default from flags. -- `getEventBatchesMax()` – Max retained batches. -- `setMinExpiry()` – Adjusted dynamically based on scheduled query intervals. - -### Query-Time Retrieval - -When a query selects from an event table: - -```mermaid -flowchart TD - SQLQuery["SELECT * FROM example_events"] --> GenTable["genTable()"] - GenTable --> GenerateRows["generateRows()"] - GenerateRows --> FilterTime["apply start/stop window"] - FilterTime --> YieldRows["yield Row to SQL engine"] -``` - -The `GenerateRowsResult` structure tracks: - -- Whether the scan reached the end. -- The last processed timestamp. -- The last processed event identifier. - -This enables optimized incremental queries. - ---- - -## Configuration-Driven Expiration Logic - -One of the most advanced responsibilities of this module is aligning event retention with scheduled queries. - -### SubscriberExpirationDetails - -Defined in: - -- `osquery.osquery.events.eventfactory.SubscriberExpirationDetails` - -Structure: - -```text -SubscriberExpirationDetails - β”œβ”€β”€ max_interval - └── query_count -``` - -During `configUpdate()`: - -1. The schedule is scanned for queries referencing `_events` tables. -2. For each subscriber table: - - The maximum interval is recorded. - - The number of queries referencing it is counted. -3. The minimum expiration window is set to: - -```text -min_expiry = (max_interval * 3), rounded to next minute -``` - -This ensures: - -- Events are retained long enough for scheduled queries to observe them. -- Storage is not retained unnecessarily. - ---- - -## PathSet and Pattern Matching - -Defined in: - -- `osquery.osquery.events.pathset.Compare` - -The `PathSet` template provides a thread-safe multiset implementation for path-based matching. - -### Supported Patterns - -- `*` – Match a single path component. -- `**` – Recursive wildcard. - -Example equivalence: - -```text -/This/Path/is -/This/Path/* -/This/Path/** -``` - -### Comparison Logic - -The `Compare` functor: - -- Compares path components lexicographically. -- Treats `*` as a wildcard. -- Treats `**` as recursive match. -- Maintains ordering semantics required by `std::multiset`. - -This is especially important for filesystem event publishers that need efficient path matching across many subscriptions. - ---- - -## Concurrency Model - -```mermaid -flowchart LR - FactoryLock["factory_lock_"] --> ProtectPub["event_pubs_"] - FactoryLock --> ProtectSub["event_subs_"] - - SubscriberLock1["event_id_lock_"] --> IDGen["EventID generation"] - SubscriberLock2["event_record_lock_"] --> RecordWrite["record time bins"] - SubscriberLock3["event_query_record_"] --> QueryTracking["query usage tracking"] -``` - -Key thread-safety mechanisms: - -- Recursive locks in `EventFactory`. -- Mutex-protected event indexing. -- Atomic counters for event identifiers and expiration windows. -- Dedicated threads per publisher. - ---- - -## Integration With Other Modules - -### SQL Engine And Virtual Tables - -- Event subscribers expose data through `genTable()`. -- `_events` tables are queried like normal virtual tables. - -### Database And Storage Plugins - -- Event rows are persisted using `IDatabaseInterface`. -- Time-based indexing enables efficient windowed queries. - -### Configuration And Packs - -- Scheduled queries determine retention windows. -- Config parsers can enable/disable subscribers. - -### Logging And Query Observability - -- Events may be forwarded using `addForwarder()` and `forwardEvent()`. -- Logger plugins can receive structured event payloads. - ---- - -## End-to-End Flow - -```mermaid -sequenceDiagram - participant Config - participant Factory as EventFactory - participant Pub as EventPublisher - participant Sub as EventSubscriber - participant DB as Database - participant SQL - - Config->>Factory: configUpdate() - Factory->>Sub: setMinExpiry() - Factory->>Pub: spawn run loop - Pub->>Sub: fire event (callback) - Sub->>DB: addBatch() - SQL->>Sub: SELECT from event table - Sub->>DB: generateRows() - Sub->>SQL: yield rows -``` - ---- - -## Design Principles - -1. **Separation of Concerns** – Publishers generate, subscribers transform and persist, SQL retrieves. -2. **Config-Aware Retention** – Event expiration adapts to scheduled queries. -3. **Thread Isolation** – Each publisher runs independently. -4. **Optimized Query Windows** – Incremental scanning via time and ID indexes. -5. **Extensibility** – New publishers and subscribers can be registered dynamically. - ---- - -## Summary - -The **Eventing Framework And Subscriptions** module is the backbone of osquery’s event-driven capabilities. It provides: - -- A robust publish/subscribe infrastructure. -- Persistent and optimized event storage. -- Config-driven lifecycle management. -- Tight integration with the SQL engine and scheduler. - -Through careful coordination between `EventFactory`, `EventSubscriberPlugin`, `Subscription`, and `PathSet`, this module enables scalable, queryable system event monitoring across platforms. \ No newline at end of file diff --git a/docs/reference/architecture/extensions-and-ipc/extensions-and-ipc.md b/docs/reference/architecture/extensions-and-ipc/extensions-and-ipc.md deleted file mode 100644 index b6bf3ce6127..00000000000 --- a/docs/reference/architecture/extensions-and-ipc/extensions-and-ipc.md +++ /dev/null @@ -1,375 +0,0 @@ -# Extensions And Ipc - -The **Extensions And Ipc** module provides the inter-process communication (IPC) layer and extension framework that allows osquery to be extended at runtime. It enables external processes (extensions) to: - -- Register new registry plugins (tables, config plugins, logger plugins, etc.) -- Execute SQL queries via the core engine -- Expose custom functionality to the osquery core -- Communicate securely over platform-specific IPC channels - -This module is the foundation for osquery’s pluggable architecture, separating the core daemon from externally developed functionality while maintaining a controlled and versioned API boundary. - ---- - -## 1. Purpose and Design Goals - -The Extensions And Ipc module is designed to: - -- βœ… Allow third-party or internal extensions to run as separate processes -- βœ… Provide a stable RPC interface using Apache Thrift -- βœ… Enforce SDK compatibility and version checks -- βœ… Isolate crashes or faults from the core daemon -- βœ… Support cross-platform IPC (UNIX domain sockets on POSIX, named pipes on Windows) - -At a high level, it implements: - -- An **Extension Manager** (running inside osquery core) -- One or more **Extension Processes** (external binaries) -- A **Thrift-based RPC layer** -- Health monitoring and watchdog services - ---- - -## 2. High-Level Architecture - -The Extensions And Ipc module sits between: - -- The **Registry subsystem** (plugin routing) -- The **SQL engine** (query delegation) -- The **Dispatcher and runtime threads** -- External extension processes - -### 2.1 Core–Extension Interaction Model - -```mermaid -flowchart LR - Core["osquery Core"] -->|"Starts"| Manager["Extension Manager"] - Manager -->|"Registers routes"| Registry["RegistryFactory"] - - ExtensionProc["Extension Process"] -->|"registerExtension()"| Manager - Manager -->|"Assign UUID"| ExtensionProc - - Core -->|"callExtension()"| Manager - Manager -->|"Route to UUID"| ExtensionProc - - ExtensionProc -->|"Response"| Manager - Manager -->|"PluginResponse"| Core -``` - -### 2.2 IPC and Thrift Layer - -```mermaid -flowchart TD - Client["ExtensionClient"] --> Transport["Thrift Transport"] - Transport --> Socket["UNIX Socket or Named Pipe"] - Socket --> Server["ExtensionRunner or ManagerRunner"] - Server --> Handler["ExtensionHandler or ExtensionManagerHandler"] - Handler --> Interface["ExtensionInterface / ExtensionManagerInterface"] - Interface --> Registry["RegistryFactory"] -``` - ---- - -## 3. Core Components - -### 3.1 Extension Metadata - -**Component:** `ExtensionInfo` - -Defined in `extensions.h`, this struct mirrors the Thrift `InternalExtensionInfo` type and contains: - -- `name` -- `version` -- `sdk_version` -- `min_sdk_version` - -This metadata is validated during registration and stored in an `ExtensionList` keyed by a transient `RouteUUID`. - ---- - -### 3.2 Extension Manager (Core Side) - -**Key Classes:** - -- `ExtensionManagerInterface` -- `ExtensionManagerHandler` -- `ExtensionManagerRunner` -- `ExtensionManagerWatcher` - -#### Responsibilities - -1. Accept extension registrations -2. Assign unique `RouteUUID` values (via `UuidGenerator`) -3. Validate SDK compatibility -4. Maintain active extension metadata -5. Route plugin calls to registered extensions -6. Monitor extension health - -#### Registration Flow - -```mermaid -sequenceDiagram - participant Ext as Extension Process - participant EM as Extension Manager - participant Reg as RegistryFactory - - Ext->>EM: registerExtension(info, registry) - EM->>EM: Validate name uniqueness - EM->>EM: Check SDK compatibility - EM->>Reg: addBroadcast(uuid, registry) - EM-->>Ext: Return UUID -``` - -If duplicate names or incompatible SDK versions are detected, registration fails. - ---- - -### 3.3 Extension Process (External Binary) - -**Key Classes:** - -- `ExtensionRunner` -- `ExtensionInterface` -- `ExtensionHandler` - -An extension process: - -1. Connects to the Extension Manager socket -2. Registers its broadcasted registry routes -3. Receives a `RouteUUID` -4. Starts a Thrift server loop -5. Waits for calls from the core - -Each extension serves the `Extension` Thrift service defined in the generated files. - ---- - -### 3.4 Thrift RPC Layer - -**Key Files:** - -- `impl_thrift.cpp` -- Generated files under `thrift/gen/` - -The module uses Apache Thrift to define two services: - -1. `Extension` – implemented by extension processes -2. `ExtensionManager` – implemented by osquery core - -#### Server-Side Wrappers - -- `ExtensionHandler` implements `ExtensionIf` -- `ExtensionManagerHandler` implements `ExtensionManagerIf` - -These handlers: - -- Translate Thrift structures into internal `PluginRequest`/`PluginResponse` -- Delegate logic to `ExtensionInterface` or `ExtensionManagerInterface` - -#### Client-Side Wrappers - -- `ExtensionClient` -- `ExtensionManagerClient` - -They abstract: - -- Transport creation -- Timeout configuration -- Method invocation -- Status translation - ---- - -### 3.5 Extension API Contracts - -The module defines abstract APIs: - -- `ExtensionAPI` -- `ExtensionManagerAPI` - -These interfaces enforce a separation between: - -- Thrift transport concerns -- Business logic - -This design allows alternative RPC backends in the future while preserving core logic. - ---- - -### 3.6 UUID Management - -**Component:** `UuidGenerator` - -- Generates 16-bit UUID values -- Tracks active UUIDs in a thread-safe set -- Removes UUIDs on deregistration - -UUIDs uniquely identify extension routes inside `RegistryFactory`. - ---- - -### 3.7 External SQL Plugin - -**Component:** `ExternalSQLPlugin` - -This special plugin allows the core SQL registry to delegate SQL queries to extensions. - -```mermaid -flowchart LR - SQL["SQL Engine"] --> ExternalSQL["ExternalSQLPlugin"] - ExternalSQL --> ManagerClient["ExtensionManagerClient"] - ManagerClient --> Extension["Extension Process"] -``` - -Used when tables or query logic are implemented outside the core. - ---- - -### 3.8 Watchers and Health Monitoring - -**Key Classes:** - -- `ExtensionWatcher` -- `ExtensionManagerWatcher` - -#### ExtensionWatcher - -- Runs inside extension processes -- Periodically pings the manager -- Exits fatally if the manager disappears - -#### ExtensionManagerWatcher - -- Runs inside core -- Pings all registered extensions -- Removes routes if extensions become unreachable - -```mermaid -flowchart TD - Watcher["ExtensionManagerWatcher"] --> UUIDs["Registered UUIDs"] - UUIDs --> Ping["Ping Extension"] - Ping -->|"Success"| Keep["Keep Registered"] - Ping -->|"Failure x2"| Remove["Remove Broadcast Route"] -``` - ---- - -### 3.9 Autoloading and Security - -Extensions can be autoloaded using: - -- `--extensions_autoload` -- `--extension` (shell-only) - -Security checks include: - -- File extension validation (`.ext`, `.exe` on Windows) -- Directory permission safety checks -- Ownership validation - -Unsafe or improperly permissioned binaries are rejected. - ---- - -### 3.10 IPC Channel Integration - -On POSIX systems, IPC is primarily UNIX domain sockets. Additionally, worker IPC may use pipe-based channels. - -**Component:** `GetChannelType` - -This specialization maps a `PipeChannelFactory` to its underlying `PipeChannel` type, integrating extension communication with the worker IPC framework. - -This ensures consistent abstraction across: - -- Extension IPC -- Worker-table IPC -- Internal task communication - ---- - -## 4. Lifecycle Overview - -### 4.1 Core Startup - -```mermaid -flowchart TD - Start["Core Start"] --> StartManager["startExtensionManager()"] - StartManager --> Watcher["ExtensionManagerWatcher"] - Watcher --> Runner["ExtensionManagerRunner"] - Runner --> Listen["Thrift Listen"] -``` - -### 4.2 Extension Startup - -```mermaid -flowchart TD - ExtStart["Extension Binary Start"] --> Connect["Connect to Manager Socket"] - Connect --> Register["registerExtension()"] - Register --> UUID["Receive UUID"] - UUID --> StartServer["Start ExtensionRunner"] - StartServer --> Serve["Serve Thrift Requests"] -``` - -### 4.3 Shutdown Flow - -- Extension calls `shutdown()` or receives manager failure -- Manager deregisters UUID -- `RegistryFactory::removeBroadcast()` is invoked -- Stale socket paths are cleaned via `removeStalePaths()` - ---- - -## 5. Interaction with Other Subsystems - -The Extensions And Ipc module interacts with: - -- **RegistryFactory** – route registration and broadcast -- **SQL Engine** – delegated queries -- **Dispatcher** – background service threads -- **Flags subsystem** – option propagation -- **Filesystem utilities** – socket and permission checks - -It acts as a boundary layer rather than a business-logic module. - ---- - -## 6. Error Handling and Status Model - -All RPC responses use: - -- `ExtensionStatus` -- `ExtensionResponse` -- `ExtensionCode` (SUCCESS, FAILED, FATAL) - -This ensures: - -- Consistent status propagation across process boundaries -- Clear failure semantics for SDK mismatches and duplicate registrations - ---- - -## 7. Security Considerations - -The module enforces: - -- SDK version compatibility checks -- Duplicate extension prevention -- File permission validation for autoload -- Socket path isolation per UUID -- Controlled shutdown behavior - -On Windows, named pipes are secured using explicit security descriptors. - ---- - -## 8. Summary - -The **Extensions And Ipc** module is the runtime extension backbone of osquery. It: - -- Implements a bidirectional Thrift RPC system -- Manages extension lifecycle and UUID assignment -- Routes registry plugin calls across process boundaries -- Monitors extension health -- Enforces compatibility and security constraints - -Without this module, osquery would be a static binary. With it, osquery becomes a dynamic, pluggable platform capable of safely integrating externally developed functionality at runtime. diff --git a/docs/reference/architecture/filesystem-and-path-utilities/filesystem-and-path-utilities.md b/docs/reference/architecture/filesystem-and-path-utilities/filesystem-and-path-utilities.md deleted file mode 100644 index 579007ad3ba..00000000000 --- a/docs/reference/architecture/filesystem-and-path-utilities/filesystem-and-path-utilities.md +++ /dev/null @@ -1,338 +0,0 @@ -# Filesystem And Path Utilities - -## Overview - -The **Filesystem And Path Utilities** module provides a cross-platform abstraction layer for file operations, path handling, globbing, permissions, and filesystem metadata inspection across Linux, macOS (POSIX), and Windows systems. - -It is a foundational module used by higher-level components to: - -- Read and write files safely -- Enforce secure permission models -- Resolve glob patterns -- Inspect file metadata -- Enumerate directories and mounts -- Interact with `/proc` on Linux -- Provide consistent APIs across POSIX and Windows - -This module hides operating system differences while preserving security guarantees and performance characteristics. - ---- - -## Architectural Overview - -```mermaid -flowchart TD - Caller["Core / SQL / Extensions"] --> FSAPI["Filesystem API"] - - subgraph abstraction_layer["Cross-Platform Abstraction"] - FSAPI --> PlatformFile["PlatformFile"] - FSAPI --> Glob["platformGlob()"] - FSAPI --> Perms["platformChmod()"] - FSAPI --> Access["platformAccess()"] - FSAPI --> Stat["platformStat() / lstat()"] - end - - subgraph posix_layer["POSIX Implementation"] - PlatformFile --> POSIXFile["open/read/write/lstat"] - Glob --> POSIXGlob["glob()"] - end - - subgraph windows_layer["Windows Implementation"] - PlatformFile --> WinFile["CreateFile / ReadFile"] - Glob --> WinGlob["FindFirstFileW"] - Perms --> WinACL["ACL Manipulation"] - end -``` - -The abstraction layer ensures that callers use a unified API regardless of the underlying OS. - ---- - -## Core Components - -### 1. PlatformFile - -**Component:** `osquery.osquery.filesystem.fileops.PlatformFile` - -`PlatformFile` is the central abstraction for file I/O. - -It provides: - -- Cross-platform file opening modes (`PF_READ`, `PF_WRITE`, etc.) -- Non-blocking I/O support -- Ownership validation -- Executable checks -- Safe permission verification -- Seek and size inspection -- File time retrieval - -### Lifecycle Model - -```mermaid -flowchart TD - Open["Open PlatformFile"] --> Validate["Handle Valid?"] - Validate -->|No| Fail["Return Error"] - Validate -->|Yes| IO["Read / Write / Seek"] - IO --> Close["Destructor Closes Handle"] -``` - -Platform-specific implementations exist for: - -- POSIX (`open`, `read`, `write`, `fstat`) -- Windows (`CreateFileW`, `ReadFile`, overlapped I/O) - ---- - -### 2. File Metadata and Stat Structures - -#### POSIX -- `osquery.osquery.filesystem.posix.fileops.stat` - -Uses: -- `stat` -- `lstat` -- `fstat` - -#### Windows -- `osquery.osquery.filesystem.fileops.WINDOWS_STAT` -- `osquery.osquery.filesystem.fileops.win_stat` -- `osquery.osquery.filesystem.windows.fileops.WindowsFindFiles` - -Windows provides: -- File ID -- Volume serial -- ACL-derived ownership -- File version metadata -- Attributes (hidden, system, archive) - -```mermaid -flowchart LR - Path["File Path"] --> StatCall["platformStat()"] - StatCall -->|POSIX| POSIXStat["struct stat"] - StatCall -->|Windows| WinStat["WINDOWS_STAT"] -``` - ---- - -### 3. Globbing and Pattern Resolution - -**Core APIs:** -- `platformGlob()` -- `resolveFilePattern()` -- `replaceGlobWildcards()` - -Supports: -- `*` and `?` -- `**` recursive globs -- `%` SQL-style wildcard translation -- Brace expansion (Windows regex translation) - -Recursive globbing includes loop detection using inode tracking. - -```mermaid -flowchart TD - Pattern["Input Pattern"] --> Normalize["replaceGlobWildcards()"] - Normalize --> Expand["platformGlob()"] - Expand --> Filter["Apply GLOB_FILES / GLOB_FOLDERS"] - Filter --> Results["Resolved Paths"] -``` - ---- - -### 4. Permissions and Security Enforcement - -Security is a major responsibility of this module. - -#### Ownership Checks -- `isOwnerRoot()` -- `isOwnerCurrentUser()` - -#### Safe Permission Enforcement -- `hasSafePermissions()` -- `platformSetSafeDbPerms()` - -On Windows, safe permissions require: -- Only Administrators and SYSTEM have write privileges -- Explicit ACL validation - -On POSIX, safety approximates: -- Mode `0700` for sensitive files -- No world-writable flags - -#### Execution Safety - -The `safePermissions()` helper ensures: - -- File exists -- Not located in temporary directory -- Proper ownership -- Proper executable bits - -This protects extension loading and module execution paths. - ---- - -### 5. File Reading and Writing Utilities - -High-level helpers include: - -- `readFile()` (streaming and full-buffer versions) -- `writeTextFile()` -- `isReadable()` -- `isWritable()` -- `pathExists()` -- `removePath()` -- `movePath()` - -A configurable limit (`read_max`) prevents excessive file reads. - -```mermaid -flowchart TD - ReadCall["readFile()"] --> Open["PlatformFile"] - Open --> SizeCheck["checkFileReadLimit()"] - SizeCheck --> Loop["Block Read Loop"] - Loop --> Predicate["Callback / Buffer"] -``` - ---- - -### 6. Home Directory and System Paths - -Cross-platform resolution of: - -- Current user home directory (`getHomeDirectory()`) -- Osquery working directory (`osqueryHomeDirectory()`) -- System root (`getSystemRoot()`) - -Fallback behavior: - -- Uses environment variables first -- Falls back to system APIs -- Falls back to temp directory if needed - ---- - -## Linux-Specific Extensions - -### Mounted Filesystems - -**Component:** `osquery.osquery.filesystem.linux.mounts.MountInformation` - -Provides structured information for mounted filesystems: - -- Device path -- Filesystem type -- Mount flags -- `statfs` block and inode counts - -```mermaid -flowchart LR - getMounted["getMountedFilesystems()"] --> MountInfo["MountInformation"] - MountInfo --> StatFS["StatFsInfo"] -``` - ---- - -### /proc Parsing and Socket Inspection - -**Components:** -- `osquery.osquery.filesystem.linux.proc.SocketInfo` -- `osquery.osquery.filesystem.linux.proc.SocketProcessInfo` - -Capabilities: - -- Enumerate processes via `/proc` -- Map socket inode to process -- Decode hex-encoded network addresses -- Parse `/proc/net/*` files - -```mermaid -flowchart TD - Proc["/proc"] --> Enumerate["procEnumerateProcesses()"] - Enumerate --> FDEnum["procEnumerateProcessDescriptors()"] - FDEnum --> SocketMap["SocketInodeToProcessInfoMap"] -``` - -This enables socket tables and network observability features. - ---- - -## Windows-Specific Enhancements - -Windows includes additional capabilities: - -- NTFS ACL manipulation -- File version extraction -- Original filename extraction from PE metadata -- Overlapped (async) I/O (`AsyncEvent`) -- Named pipe detection (`socketExists()`) - -Windows globbing is implemented using: -- `FindFirstFileW` -- Regex translation for brace patterns - ---- - -## Security Model Summary - -```mermaid -flowchart TD - File["Candidate File"] --> Exists["Exists?"] - Exists -->|No| Reject["Reject"] - Exists --> Owner["Owner Root or Current User?"] - Owner -->|No| Reject - Owner --> Writable["World Writable?"] - Writable -->|Yes| Reject - Writable --> Exec["Executable Required?"] - Exec --> Accept["Safe"] -``` - -This ensures: - -- No unsafe extension loading -- No world-writable execution -- No loading from temp directories - ---- - -## Integration Within the System - -The Filesystem And Path Utilities module is used by: - -- SQL virtual tables for file metadata -- Configuration loading -- Extension management -- Logging subsystems -- Distributed query execution -- Event subscribers - -It acts as a **trusted boundary layer** between: - -- User-supplied file paths -- The operating system -- Internal execution logic - ---- - -## Key Design Principles - -1. **Cross-platform consistency** -2. **Security-first permission enforcement** -3. **Non-blocking support where possible** -4. **Recursive glob loop detection** -5. **Minimal exposure of OS-specific details** - ---- - -## Conclusion - -The **Filesystem And Path Utilities** module is a foundational cross-platform subsystem responsible for: - -- File I/O abstraction -- Secure permission validation -- Glob resolution -- Filesystem metadata access -- Linux `/proc` inspection -- Windows ACL and metadata handling - -It enables higher-level components to operate safely and consistently across heterogeneous operating systems while enforcing strict security guarantees. \ No newline at end of file diff --git a/docs/reference/architecture/logging-and-query-observability/logging-and-query-observability.md b/docs/reference/architecture/logging-and-query-observability/logging-and-query-observability.md deleted file mode 100644 index cf98d585c16..00000000000 --- a/docs/reference/architecture/logging-and-query-observability/logging-and-query-observability.md +++ /dev/null @@ -1,409 +0,0 @@ -# Logging And Query Observability - -## Overview - -The **Logging And Query Observability** module is responsible for transforming raw query executions and internal status events into structured, serialized, and transport-ready log artifacts. - -It sits at the intersection of: - -- The **SQL execution layer** (see [SQL Engine And Virtual Tables](../sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md)) -- The **persistent state layer** (see [Database And Storage Plugins](../database-and-storage-plugins/database-and-storage-plugins.md)) -- The **configuration and scheduling layer** (see [Configuration And Packs](../configuration-and-packs/configuration-and-packs.md)) -- The **pluggable logging infrastructure** - -This module provides: - -- Query result diffing and snapshot handling -- Query execution metadata and performance tracking -- Structured log serialization (JSON, event-based) -- Pluggable logger interface for external sinks (TLS, filesystem, syslog, etc.) -- Status log forwarding and event streaming support - ---- - -## Architectural Position - -```mermaid -flowchart TD - Config["Configuration And Packs"] --> Scheduler["Scheduled Query"] - Scheduler --> SQ["ScheduledQuery Struct"] - SQ --> QueryObj["Query State Manager"] - QueryObj --> DB["Database And Storage Plugins"] - - QueryObj --> Diff["DiffResults Engine"] - Diff --> LogItem["QueryLogItem"] - LogItem --> Serializer["JSON Serialization"] - Serializer --> Logger["LoggerPlugin"] - - StatusLogs["Internal Status Events"] --> StatusLine["StatusLogLine"] - StatusLine --> Logger - - Logger --> External["External Logging Backend"] -``` - -The Logging And Query Observability module acts as a transformation and routing layer: - -1. Queries execute. -2. Results are compared with historical state. -3. Changes are wrapped in structured log items. -4. Logs are serialized. -5. A `LoggerPlugin` implementation forwards them upstream. - ---- - -## Core Components - -This module consists of five primary building blocks: - -- **LoggerPlugin** and **StatusLogLine** – Pluggable logging interface -- **QueryLogItem** – Structured representation of query output -- **DiffResults** – Result set comparison engine -- **QueryPerformance** – Execution metrics and telemetry -- **ScheduledQuery** – Query scheduling metadata model - -Each component addresses a specific concern in the observability pipeline. - ---- - -## LoggerPlugin And StatusLogLine - -### Purpose - -The logging subsystem provides a pluggable abstraction for forwarding: - -- Query results -- Snapshot outputs -- Event batches -- Internal status logs - -### StatusLogLine - -`StatusLogLine` represents a single internal status entry emitted by the runtime. It includes: - -- Severity (`O_INFO`, `O_WARNING`, `O_ERROR`, `O_FATAL`) -- Source file and line number -- Human-readable message -- UNIX timestamp and calendar time -- Host identifier - -This structure decouples internal logging (e.g., Glog) from external log sinks. - -### LoggerPlugin - -`LoggerPlugin` is an abstract plugin interface that allows custom logging backends. - -Key extensibility points: - -- `logString` (required) – Primary logging entrypoint -- `logStatus` – Handles structured status logs -- `logSnapshot` – Special handling for large snapshot queries -- `logEvent` / `logStringBatch` – Direct event forwarding -- `usesLogStatus` – Opt-in takeover of internal status handling -- `usesLogEvent` – Opt-in direct event streaming - -### Logger Feature Flags - -Logger plugins can advertise support for: - -- `LOGGER_FEATURE_LOGSTATUS` -- `LOGGER_FEATURE_LOGEVENT` - -This enables selective routing decisions at runtime. - -### Logger Lifecycle - -```mermaid -flowchart TD - Init["System Initialization"] --> Buffer["Buffer Early Status Logs"] - Buffer --> LoadConfig["Load Configuration"] - LoadConfig --> Discover["Discover Logger Plugin"] - Discover --> InitLogger["LoggerPlugin.init()"] - InitLogger --> Flush["Flush Buffered StatusLogLine Entries"] - Flush --> Runtime["Runtime Logging"] -``` - -Before the logger initializes, early status logs are buffered and later flushed during `init()`. - -This ensures no observability gaps during startup. - ---- - -## ScheduledQuery - -`ScheduledQuery` models the runtime attributes of a scheduled query. - -Key attributes: - -- `pack_name` – Configuration pack source -- `name` – Unique identifier -- `query` – SQL statement -- `interval` – Execution frequency -- `startup_priority` – Startup execution control -- `denylisted` – Configuration-level suppression -- `options` – Behavioral flags (e.g., snapshot, removed) - -### Behavioral Flags - -- `snapshot` β†’ Always log full results -- `removed` β†’ Control reporting of removed rows - -This structure originates from configuration parsing (see [Configuration And Packs](../configuration-and-packs/configuration-and-packs.md)) and feeds directly into execution and logging logic. - ---- - -## Query State Management - -### Query - -The `Query` class maintains persistent state for a scheduled query. - -Responsibilities: - -- Retrieve previous results from storage -- Compute diffs between executions -- Track query epoch -- Maintain execution counter -- Persist updated result sets - -It interacts with the database abstraction (see [Database And Storage Plugins](../database-and-storage-plugins/database-and-storage-plugins.md)). - -### Differential Execution Model - -```mermaid -flowchart LR - Old["Previous Results"] --> DiffEngine["diff()"] - New["Current Results"] --> DiffEngine - DiffEngine --> Added["Added Rows"] - DiffEngine --> Removed["Removed Rows"] -``` - -The `addNewResults` method: - -1. Loads historical state. -2. Computes a `DiffResults` structure. -3. Updates persistent storage. -4. Returns differential output for logging. - -This avoids repeatedly logging identical datasets and reduces log volume. - ---- - -## DiffResults - -`DiffResults` represents the delta between two query executions. - -Structure: - -- `added` – Rows present in new results but not old -- `removed` – Rows present in old results but not new - -### Properties - -- Move-only semantics (efficient handling of large result sets) -- Equality comparison support -- JSON serialization helpers - -### Serialization Flow - -```mermaid -flowchart TD - Diff["DiffResults"] --> Serialize["serializeDiffResults()"] - Serialize --> JSONDoc["JSON Document"] - JSONDoc --> Logger["LoggerPlugin"] -``` - -Diff serialization ensures: - -- Stable output format -- Optional numeric preservation -- Compatibility with downstream analytics pipelines - ---- - -## QueryLogItem - -`QueryLogItem` is the canonical structured representation of a query execution. - -It includes: - -- `isSnapshot` – Snapshot vs differential indicator -- `results` – `DiffResults` -- `snapshot_results` – Full results (if snapshot) -- `name` – Query name -- `identifier` – Host ID -- `time` / `calendar_time` -- `epoch` -- `counter` -- `decorations` – Arbitrary metadata - -### Equality Semantics - -Two `QueryLogItem` instances are equal if: - -- Their `DiffResults` match -- Their query names match - -This enables deduplication logic in higher-level processing. - -### Serialization Variants - -The module supports multiple output formats: - -1. Structured JSON object -2. JSON string -3. Event-style records (split per action) - -```mermaid -flowchart TD - Item["QueryLogItem"] --> JSON1["serializeQueryLogItem()"] - Item --> JSON2["serializeQueryLogItemJSON()"] - Item --> Events1["serializeQueryLogItemAsEvents()"] - Item --> Events2["serializeQueryLogItemAsEventsJSON()"] -``` - -Event-based serialization allows each row addition/removal to be emitted as an individual log record. - ---- - -## QueryPerformance - -`QueryPerformance` provides execution telemetry. - -Metrics tracked: - -- Total executions -- Last execution time -- Wall time (total and last) -- User/system CPU time -- Memory usage -- Output size - -### CSV Serialization - -Performance data is stored and transferred using a CSV representation: - -- Constructor accepts CSV -- `toCSV()` exports current state - -This lightweight format reduces overhead in persistent storage. - -### Observability Use Cases - -- Detect slow queries -- Identify resource-heavy packs -- Enforce denylisting via configuration -- Feed monitoring dashboards - ---- - -## End-To-End Query Logging Flow - -```mermaid -flowchart TD - Config["ScheduledQuery"] --> Exec["SQL Execution"] - Exec --> Results["QueryDataTyped"] - Results --> QueryState["Query.addNewResults()"] - QueryState --> Diff["DiffResults"] - Diff --> LogItem["QueryLogItem"] - LogItem --> Serialize["JSON Serialization"] - Serialize --> Logger["LoggerPlugin"] - Logger --> Sink["External Log System"] -``` - -### Snapshot Query Flow - -```mermaid -flowchart TD - SQ["ScheduledQuery snapshot=true"] --> Exec["Execute Query"] - Exec --> Full["Full Result Set"] - Full --> LogItem["QueryLogItem isSnapshot=true"] - LogItem --> Logger["logSnapshot()"] -``` - -Snapshot queries bypass diff computation and emit complete result sets. - ---- - -## Relationship With Other Modules - -### SQL Engine And Virtual Tables - -Query execution originates from the SQL engine layer: - -- Virtual table resolution -- SQLite execution -- Result materialization - -See: [SQL Engine And Virtual Tables](../sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md) - -### Database And Storage Plugins - -Persistent state for query history and counters is managed by the storage abstraction. - -See: [Database And Storage Plugins](../database-and-storage-plugins/database-and-storage-plugins.md) - -### Configuration And Packs - -Scheduled queries and behavioral flags originate from configuration packs. - -See: [Configuration And Packs](../configuration-and-packs/configuration-and-packs.md) - -### Distributed Querying - -When queries are executed via distributed frameworks, results still flow through `QueryLogItem` and the logger interface. - -See: [Distributed Querying](../distributed-querying/distributed-querying.md) - ---- - -## Design Principles - -### 1. Pluggability - -Logging is abstracted via `LoggerPlugin` to allow: - -- TLS streaming -- Filesystem logging -- Syslog integration -- Custom enterprise pipelines - -### 2. Efficiency - -- Differential logging reduces noise -- Move semantics for result sets -- CSV encoding for performance stats - -### 3. Deterministic Serialization - -All log structures have explicit JSON serializers to guarantee: - -- Backward compatibility -- Stable schema -- Predictable downstream parsing - -### 4. Observability-First Model - -Every query execution produces: - -- State delta -- Execution metadata -- Performance metrics -- Host attribution - -This ensures complete auditability of query behavior. - ---- - -## Summary - -The **Logging And Query Observability** module transforms raw SQL execution results and runtime status events into structured, serialized, and pluggable log outputs. - -It provides: - -- Differential result tracking (`DiffResults`) -- Query metadata encapsulation (`QueryLogItem`) -- Performance telemetry (`QueryPerformance`) -- Scheduling metadata (`ScheduledQuery`) -- Extensible logging backends (`LoggerPlugin`) - -By cleanly separating execution, state management, serialization, and transport, this module forms the backbone of osquery’s observability and audit pipeline. \ No newline at end of file diff --git a/docs/reference/architecture/remote-http-client/remote-http-client.md b/docs/reference/architecture/remote-http-client/remote-http-client.md deleted file mode 100644 index 83b5f95de29..00000000000 --- a/docs/reference/architecture/remote-http-client/remote-http-client.md +++ /dev/null @@ -1,345 +0,0 @@ -# Remote Http Client - -The **Remote Http Client** module provides a general-purpose HTTP and HTTPS client implementation used across the system for secure remote communication. Built on top of Boost.Asio and Boost.Beast, it enables synchronous-style HTTP operations backed by asynchronous networking primitives, with full TLS support, timeout handling, redirect control, and proxy configuration. - -This module acts as the foundational networking layer for components that need to communicate with remote services, such as: - -- [Distributed Querying](distributed-querying/distributed-querying.md) -- [Logging and Query Observability](logging-and-query-observability/logging-and-query-observability.md) -- [Configuration and Packs](configuration-and-packs/configuration-and-packs.md) - -It abstracts low-level socket management, TLS setup, request serialization, and response parsing into a reusable and secure HTTP client interface. - ---- - -## 1. Purpose and Responsibilities - -The Remote Http Client module is responsible for: - -- Creating and managing TCP and TLS connections -- Executing HTTP methods: `GET`, `POST`, `PUT`, `HEAD`, `DELETE` -- Managing request headers and body serialization -- Parsing HTTP responses -- Enforcing timeouts and connection lifecycle rules -- Handling TLS verification, certificates, and cipher configuration -- Supporting optional proxy routing -- Supporting redirect and keep-alive behavior - -It provides a high-level API while internally orchestrating asynchronous Boost networking operations. - ---- - -## 2. High-Level Architecture - -The module is centered around three main abstractions: - -- `Client` – Connection lifecycle and HTTP execution engine -- `HTTP_Request` – URI-aware request wrapper -- `HTTP_Response` – Response wrapper with header iteration support - -### 2.1 Architectural Overview - -```mermaid -flowchart TD - Caller["Caller Module"] --> Client["Client"] - Client --> Options["Client::Options"] - Client --> Request["HTTP_Request"] - Client --> Response["HTTP_Response"] - - Client --> Resolver["TCP Resolver"] - Client --> Socket["TCP Socket"] - Client --> TLSSocket["SSL Stream"] - Client --> Timer["Deadline Timer"] - - Request --> URI["URI Parser"] - Response --> Headers["Header Iterator"] -``` - -The `Client` orchestrates the network stack and delegates URI parsing to `HTTP_Request`, while wrapping Boost.Beast responses inside `HTTP_Response`. - ---- - -## 3. Core Components - -### 3.1 Client - -The `Client` class implements a general-purpose HTTP/HTTPS client built on: - -- `boost::asio::io_context` -- `boost::asio::ip::tcp::resolver` -- `boost::asio::ip::tcp::socket` -- `boost::asio::ssl::stream` -- `boost::beast::http` - -It exposes the following public methods: - -- `get(Request&)` -- `post(Request&, body, content_type)` -- `put(Request&, body, content_type)` -- `head(Request&)` -- `delete_(Request&)` - -Each method: -1. Initializes the request -2. Establishes or reuses a connection -3. Sends the request asynchronously -4. Waits for response or timeout -5. Returns a `Response` object - -The destructor ensures that open sockets are closed safely. - ---- - -### 3.2 Client Options - -The `Client::Options` class drives client behavior. - -It supports configuration for: - -- TLS enablement (`ssl_connection`) -- Keep-alive behavior -- Redirect following -- Peer verification enforcement -- Timeout configuration -- Cipher selection -- Certificate and private key files -- CA verify path -- Proxy hostname -- Remote hostname and port overrides - -#### Options Comparison - -The equality operator allows the `Client` to detect configuration changes: - -```mermaid -flowchart LR - OldOpts["Current Options"] --> Compare["operator=="] - NewOpts["New Options"] --> Compare - Compare -->|"Different"| Reconfigure["Reinitialize Connection"] - Compare -->|"Same"| Reuse["Reuse Existing Settings"] -``` - -This prevents unnecessary TLS reconfiguration and socket churn. - ---- - -### 3.3 HTTP_Request - -`HTTP_Request` extends Boost.Beast request objects with: - -- URI parsing support -- Host, port, path extraction -- Protocol detection -- Header insertion via operator overloading - -#### URI Handling - -The request wraps an internal `Uri` object and exposes: - -- `remoteHost()` -- `remotePort()` -- `remotePath()` -- `protocol()` - -This allows the `Client` to resolve and connect without requiring manual URL parsing. - -#### Header Injection - -Headers are added using a helper struct: - -```text -Request req("https://example.com/api"); -req << HTTP_Request::Header("Authorization", "Bearer token"); -``` - -This keeps header logic encapsulated within the request abstraction. - ---- - -### 3.4 HTTP_Response - -`HTTP_Response` extends Boost.Beast response objects and provides: - -- `status()` – numeric HTTP status -- `body()` – response body string -- `headers()` – iterable header collection - -#### Header Iteration Model - -```mermaid -flowchart TD - Response["HTTP_Response"] --> Headers["Headers"] - Headers --> Iterator["Iterator"] - Iterator --> Pair["(name, value)"] -``` - -Example usage pattern: - -```text -for (const auto& header : resp.headers()) { - header.first; - header.second; -} -``` - -This abstraction avoids exposing raw Boost iterators directly. - ---- - -## 4. Request Lifecycle - -The `Client` wraps asynchronous Boost operations into a controlled flow. - -### 4.1 End-to-End Flow - -```mermaid -sequenceDiagram - participant Caller - participant Client - participant Resolver - participant Server - - Caller->>Client: get(Request) - Client->>Resolver: async_resolve() - Resolver-->>Client: endpoints - Client->>Server: async_connect() - Client->>Server: async_write(request) - Server-->>Client: HTTP response - Client->>Server: async_read() - Client-->>Caller: Response -``` - -### 4.2 TLS Flow (HTTPS) - -If `ssl_connection` is enabled: - -```mermaid -flowchart TD - Connect["TCP Connect"] --> TLSCheck{"SSL Enabled?"} - TLSCheck -->|"Yes"| Handshake["TLS Handshake"] - TLSCheck -->|"No"| Plain["Plain HTTP"] - Handshake --> Send["Send Request"] - Plain --> Send -``` - -TLS configuration respects: - -- Cipher restrictions -- Verify path -- Server certificate pinning -- Client certificate and private key - -Legacy SSL versions and MD5 are explicitly disabled at compile time. - ---- - -## 5. Timeout and Error Handling - -The client uses a `boost::asio::deadline_timer` to enforce network timeouts. - -### 5.1 callNetworkOperation Wrapper - -The `callNetworkOperation` method: - -1. Starts the timeout timer -2. Executes the async operation -3. Waits for completion or timeout -4. Cancels the opposing operation -5. Sets the final error code - -```mermaid -flowchart TD - Start["Start Network Call"] --> Timer["Start Deadline Timer"] - Timer --> Async["Async Operation"] - Async --> Complete{"Completed?"} - Complete -->|"Yes"| CancelTimer["Cancel Timer"] - Complete -->|"No"| Timeout["Timer Fires"] - Timeout --> Abort["Abort Socket"] - CancelTimer --> Return["Return Response"] - Abort --> Return -``` - -### 5.2 TLS Short Read Handling - -During TLS shutdown, some servers may not perform proper `shutdown()` calls. The client treats certain short-read conditions as success in post-response handling to avoid false-negative errors. - ---- - -## 6. Proxy and Metadata Support - -### 6.1 Proxy Routing - -If a proxy hostname is configured: - -- Connection is established to the proxy -- Request routing logic is adapted accordingly - -This enables deployment behind corporate proxies. - -### 6.2 Cloud Metadata Authority - -The constant `kInstanceMetadataAuthority` (`169.254.169.254`) identifies the authority used by cloud metadata services (e.g., EC2, Azure). - -This allows safe and consistent access to instance metadata endpoints when required by higher-level modules. - ---- - -## 7. Interaction with Other Modules - -The Remote Http Client module acts as an infrastructure dependency for modules that require outbound HTTP communication. - -### 7.1 Distributed Querying - -The [Distributed Querying](distributed-querying/distributed-querying.md) module typically relies on HTTP transport (e.g., TLS-based distributed plugins) to: - -- Fetch remote queries -- Submit query results - -The Remote Http Client provides the secure communication backbone for these operations. - -### 7.2 Logging and Query Observability - -The [Logging and Query Observability](logging-and-query-observability/logging-and-query-observability.md) module may use HTTP/TLS transports for remote log submission. - -The client ensures: - -- TLS enforcement -- Certificate verification -- Controlled timeout behavior - -### 7.3 Configuration and Packs - -The [Configuration and Packs](configuration-and-packs/configuration-and-packs.md) module can retrieve configuration from remote endpoints via HTTP plugins. The Remote Http Client supplies: - -- Secure GET/POST support -- Redirect handling -- Proxy awareness - ---- - -## 8. Security Model - -The module enforces multiple security measures: - -- SSLv2 and SSLv3 disabled -- MD5 disabled -- Deprecated OpenSSL features disabled -- Optional strict peer verification -- Optional custom cipher suites -- Support for certificate pinning - -Security behavior is fully driven by `Client::Options`, allowing different modules to enforce different policies. - ---- - -## 9. Summary - -The **Remote Http Client** module provides a robust, TLS-capable HTTP client abstraction built on Boost.Asio and Boost.Beast. It encapsulates: - -- Connection lifecycle management -- TLS negotiation and verification -- Asynchronous networking orchestration -- Timeout enforcement -- Request and response abstraction - -By centralizing HTTP communication logic, it ensures consistent, secure, and configurable remote connectivity across the system, serving as the networking foundation for distributed querying, logging, and remote configuration workflows. diff --git a/docs/reference/architecture/sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md b/docs/reference/architecture/sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md deleted file mode 100644 index e450a4f1d9d..00000000000 --- a/docs/reference/architecture/sql-engine-and-virtual-tables/sql-engine-and-virtual-tables.md +++ /dev/null @@ -1,351 +0,0 @@ -# Sql Engine And Virtual Tables - -## Overview - -The **Sql Engine And Virtual Tables** module is the execution core of osquery. It embeds and configures SQLite as an in-memory analytical engine and exposes operating system data through dynamically attached virtual tables. - -This module is responsible for: - -- Managing and optimizing SQLite database instances -- Enforcing SQL security through authorizers and allowlists -- Attaching and detaching virtual tables backed by TablePlugin implementations -- Planning and introspecting queries -- Executing SQL statements and returning typed results -- Bridging SQLite’s virtual table API with osquery’s plugin system - -It acts as the execution layer between: - -- Table plugins (from the Registry) -- The scheduler and query interfaces -- Extensions providing additional tables - ---- - -## High-Level Architecture - -```mermaid -flowchart TD - Client["SQL Caller (Scheduler / CLI / Extension)"] --> SQLPlugin["SQLiteSQLPlugin"] - SQLPlugin --> DBManager["SQLiteDBManager"] - DBManager --> DBInstance["SQLiteDBInstance"] - DBInstance --> SQLiteCore[("In-Memory SQLite Engine")] - - SQLiteCore --> VTabModule["sqlite3_module (Virtual Table Module)"] - VTabModule --> VirtualTable["VirtualTable Wrapper"] - VirtualTable --> TablePlugin["TablePlugin (Registry)"] - - SQLiteCore --> Planner["QueryPlanner"] -``` - -### Flow Summary - -1. A SQL query is submitted via the SQL registry. -2. `SQLiteSQLPlugin` obtains a database connection from `SQLiteDBManager`. -3. The query is prepared and executed using `queryInternal`. -4. When a virtual table is accessed, SQLite invokes the `sqlite3_module` callbacks. -5. The virtual table implementation forwards execution to a `TablePlugin`. -6. Results are returned to SQLite and then to the caller as typed rows. - ---- - -## Core Components - -### SQLiteSQLPlugin - -Implements the internal `sql` registry plugin. - -**Responsibilities:** - -- Execute SQL queries (`query`) -- Introspect query columns (`getQueryColumns`) -- Determine scanned tables (`getQueryTables`) -- Attach/detach virtual tables dynamically - -It acts as the primary entry point for SQL execution inside osquery. - ---- - -### SQLiteDBManager - -Singleton responsible for lifecycle and concurrency management of SQLite instances. - -**Key Features:** - -- Maintains a primary in-memory SQLite database -- Creates transient database instances under contention -- Applies memory optimizations (PRAGMA tuning) -- Attaches all registered virtual tables -- Enforces table enable/disable flags - -```mermaid -flowchart LR - Caller["Caller"] --> GetConn["SQLiteDBManager.get()"] - GetConn --> PrimaryCheck{"Primary Available?"} - PrimaryCheck -->|Yes| Primary["Primary SQLiteDBInstance"] - PrimaryCheck -->|No| Transient["Transient SQLiteDBInstance"] -``` - -This design minimizes resource usage while ensuring thread safety. - ---- - -### SQLiteDBInstance - -RAII wrapper around a `sqlite3*` pointer. - -**Responsibilities:** - -- Provide safe access to SQLite handle -- Manage locking and attach mutexes -- Track affected virtual tables per query -- Clear per-query table state -- Support query result caching - -Each query receives a scoped database instance. On destruction: - -- Transient connections are closed -- Primary connections remain managed - ---- - -### Security Enforcement (Authorizer) - -The module enforces strict SQL safety using `sqliteAuthorizer`. - -**Allowlisted Actions:** - -- `SELECT`, `READ` -- Controlled `INSERT`, `UPDATE`, `DELETE` -- Virtual table creation and drop -- Limited `PRAGMA` commands - -**Explicitly Denied:** - -- `SQLITE_ATTACH` -- Any non-allowlisted opcode - -```mermaid -flowchart TD - Prepare["sqlite3_prepare_v2"] --> Authorizer["sqliteAuthorizer"] - Authorizer -->|Allowed| Continue["Execute Statement"] - Authorizer -->|Denied| Reject["SQLITE_DENY"] -``` - -This ensures osquery cannot be abused to modify arbitrary files or attach external databases. - ---- - -### QueryPlanner - -Performs query introspection using: - -- `EXPLAIN QUERY PLAN` -- `EXPLAIN` - -Used for: - -- Inferring column types for expressions -- Determining scanned tables -- Inspecting SQLite opcodes - -The planner maps SQLite opcodes to osquery `ColumnType` using `kSQLOpcodes`. - -```mermaid -flowchart TD - Query["SQL Query"] --> ExplainPlan["EXPLAIN QUERY PLAN"] - Query --> Explain["EXPLAIN"] - Explain --> OpcodeMap["kSQLOpcodes Mapping"] - OpcodeMap --> TypeInference["Apply Column Types"] -``` - -This enables accurate schema introspection even when SQLite cannot directly determine expression types. - ---- - -### SQLInternal - -A lower-level SQL execution wrapper used internally. - -**Capabilities:** - -- Executes queries bypassing registry lookup -- Returns typed results (`QueryDataTyped`) -- Detects event-based tables -- Escapes non-printable bytes -- Calculates result size - -It is used when deeper inspection of query attributes is required. - ---- - -## Virtual Table Subsystem - -The Virtual Table subsystem connects SQLite’s virtual table API with osquery’s `TablePlugin` abstraction. - -### VirtualTable - -Wraps: - -- `sqlite3_vtab` -- `VirtualTableContent` (metadata, schema, constraints) -- Associated `SQLiteDBInstance` - -It maintains: - -- Column definitions -- Aliases -- Attributes (EVENT_BASED, USER_BASED, etc.) -- Constraint tracking - ---- - -### BaseCursor - -Represents an active scan of a virtual table. - -Tracks: - -- Row set or generator -- Cursor position -- Current row -- Planner ID - -Supports both: - -- Pre-generated row sets -- Streaming generator-based tables - ---- - -### sqlite3_module Implementation - -Implements the SQLite virtual table callbacks: - -- `xCreate` -- `xBestIndex` -- `xFilter` -- `xNext` -- `xColumn` -- `xRowid` -- `xUpdate` - -```mermaid -sequenceDiagram - participant SQLite - participant Module as "sqlite3_module" - participant Table as "TablePlugin" - - SQLite->>Module: xBestIndex - Module->>SQLite: Constraint plan - - SQLite->>Module: xFilter - Module->>Table: generate(context) - Table-->>Module: Row set - Module-->>SQLite: Rows -``` - ---- - -## Constraint Optimization - -`xBestIndex` evaluates constraints provided by SQLite and: - -- Identifies indexed and required columns -- Calculates estimated cost -- Rewrites `IN` constraints for optimized handling -- Tracks used columns for projection pushdown - -If required constraints are missing: - -- Cost is set to a maximum -- Query may fail with `SQLITE_CONSTRAINT` - -This enables efficient filtering at the table implementation level. - ---- - -## Query Execution Flow - -```mermaid -flowchart TD - Start["Query Submitted"] --> Prepare["sqlite3_prepare_v2"] - Prepare --> Execute["sqlite3_step Loop"] - Execute -->|Virtual Table Access| VTab - VTab --> xBestIndex - xBestIndex --> xFilter - xFilter --> Generate["TablePlugin.generate()"] - Generate --> Rows["Return Rows"] - Rows --> Finalize["sqlite3_finalize"] - Finalize --> End["Results Returned"] -``` - ---- - -## Extension and Writable Tables - -If a table originates from an extension: - -- `xUpdate` is enabled -- INSERT / UPDATE / DELETE are forwarded -- Parameters are serialized into JSON -- Responses are validated - -This allows controlled writable virtual tables. - ---- - -## Memory and Performance Optimizations - -When opening the in-memory database: - -- `journal_mode=OFF` -- `synchronous=OFF` -- `auto_vacuum=FULL` -- Custom function extensions registered -- Authorizer installed - -Additionally: - -- SQLite soft heap limit configured -- Memory released after each query -- Transient DB instances created only under contention - ---- - -## Interaction with Other Modules - -The Sql Engine And Virtual Tables module integrates closely with: - -- Table plugins (Registry) -- Extensions (for remote virtual tables) -- Logging and query observability (for execution results) -- Eventing framework (for event-based tables) -- Core initialization (for flag configuration) - -It does not duplicate table logic; it provides the execution infrastructure. - ---- - -## Key Design Principles - -1. **Security First** – Strict SQLite authorizer and allowlists -2. **In-Memory Execution** – No persistent database files -3. **Plugin-Based Tables** – Data sources abstracted behind registry -4. **Query-Aware Optimization** – Constraint and projection pushdown -5. **Thread-Safe Resource Management** – Managed primary DB with transient fallback -6. **Extension-Friendly** – Writable virtual tables supported - ---- - -## Summary - -The **Sql Engine And Virtual Tables** module is the execution backbone of osquery. - -It transforms SQLite into a secure, extensible, in-memory analytics engine that: - -- Executes SQL safely -- Dynamically attaches system-backed virtual tables -- Optimizes query plans -- Bridges plugins and extensions with SQLite’s execution engine - -Without this module, osquery would not be able to expose system data through a unified SQL interface. From b60a56740b31d29d5199ca205989e7b88b39ec3d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Jun 2026 19:03:53 +0000 Subject: [PATCH 3/7] docs: Stage 1 - Inline documentation (235 files) [skip ci] --- .../generated/linux/aarch64/code/.parser.md | 84 ++++++++++------- .../generated/linux/x86_64/code/.parser.md | 73 +++++++-------- .../generated/macos/aarch64/code/.parser.md | 82 +++++++++------- .../generated/macos/x86_64/code/.parser.md | 72 +++++++------- openframe/.openframe_authorization_manager.md | 37 ++++---- openframe/.openframe_encryption_service.md | 30 +++--- openframe/.openframe_token_extractor.md | 34 +++---- openframe/.openframe_token_refresher.md | 37 ++++---- osquery/carver/.carver.md | 39 ++++---- osquery/carver/.carver_utils.md | 39 ++++---- osquery/config/.config.md | 70 ++++++-------- osquery/config/.packs.md | 45 +++++---- osquery/config/tests/.test_utils.md | 37 ++++---- osquery/core/.flags.md | 66 +++++++------ osquery/core/.query.md | 93 +++++++++++-------- osquery/core/.shutdown.md | 56 +++++------ osquery/core/.system.md | 65 +++++++------ osquery/core/.tables.md | 55 +++++------ osquery/core/.watcher.md | 23 +++-- osquery/core/plugins/.logger.md | 59 +++++------- osquery/core/plugins/.plugin.md | 70 +++++++------- osquery/core/plugins/.sql.md | 30 +++--- osquery/core/sql/.column.md | 45 ++++----- osquery/core/sql/.diff_results.md | 44 ++++----- osquery/core/sql/.query_data.md | 49 +++++----- osquery/core/sql/.query_performance.md | 39 ++++---- osquery/core/sql/.row.md | 63 +++++++------ osquery/core/sql/.scheduled_query.md | 42 ++++----- osquery/core/sql/.table_rows.md | 42 +++------ .../windows/.global_users_groups_cache.md | 37 +++++--- osquery/core/windows/.handle.md | 37 ++++---- osquery/core/windows/.wmi.md | 46 +++++---- osquery/database/.database.md | 50 +++++----- osquery/database/tests/.test_utils.md | 42 ++++----- osquery/dispatcher/.dispatcher.md | 75 ++++++--------- osquery/dispatcher/.distributed_runner.md | 23 ++--- osquery/dispatcher/.scheduler.md | 54 +++++------ osquery/distributed/.distributed.md | 49 ++++------ osquery/events/.eventer.md | 66 ++++++------- osquery/events/.eventfactory.md | 69 ++++++++------ osquery/events/.eventpublisherplugin.md | 47 +++++----- osquery/events/.events.md | 18 ++-- osquery/events/.eventsubscriberplugin.md | 52 +++++------ osquery/events/.subscription.md | 44 +++++---- osquery/events/darwin/.diskarbitration.md | 60 ++++++------ osquery/events/darwin/.endpointsecurity.md | 45 ++++----- osquery/events/darwin/.es_utils.md | 30 +++--- osquery/events/darwin/.event_taps.md | 44 +++++---- osquery/events/darwin/.fsevents.md | 43 ++++----- osquery/events/darwin/.iokit.md | 61 ++++++------ osquery/events/darwin/.openbsm.md | 60 ++++++------ osquery/events/darwin/.scnetwork.md | 43 ++++----- osquery/events/linux/.auditdnetlink.md | 46 ++++----- osquery/events/linux/.auditeventpublisher.md | 79 +++++++--------- osquery/events/linux/.inotify.md | 66 ++++++++----- osquery/events/linux/.socket_events.md | 20 ++-- osquery/events/linux/.syslog.md | 35 ++++--- osquery/events/linux/.udev.md | 46 +++++---- osquery/events/linux/bpf/.bpferrorstate.md | 39 ++++---- .../events/linux/bpf/.bpfeventpublisher.md | 66 ++++++------- osquery/events/linux/bpf/.filesystem.md | 32 +++---- .../linux/bpf/.processcontextfactory.md | 51 +++++----- osquery/events/linux/bpf/.serializers.md | 29 +++--- osquery/events/linux/bpf/.setrlimit.md | 19 ++-- .../events/linux/bpf/.systemstatetracker.md | 73 ++++++++------- .../events/tests/.mockedosquerydatabase.md | 58 ++++++------ .../tests/linux/bpf/.mockedfilesystem.md | 40 ++++---- .../linux/bpf/.mockedprocesscontextfactory.md | 44 +++++---- osquery/events/tests/linux/bpf/.utils.md | 54 +++++------ osquery/events/windows/.evtsubscription.md | 44 +++++---- .../events/windows/.ntfs_event_publisher.md | 59 ++++++------ osquery/events/windows/.usn_journal_reader.md | 66 ++++++------- .../events/windows/.windowseventlogparser.md | 65 ++++++------- .../windows/.windowseventlogparserservice.md | 50 +++++----- .../windows/.windowseventlogpublisher.md | 51 +++++----- osquery/events/windows/etw/.etw_controller.md | 58 ++++++------ .../events/windows/etw/.etw_kernel_session.md | 53 ++++++----- .../etw/.etw_post_processing_pipeline.md | 55 ++++++----- .../windows/etw/.etw_provider_config.md | 48 ++++++---- osquery/events/windows/etw/.etw_publisher.md | 45 +++++---- .../events/windows/etw/.etw_publisher_dns.md | 55 +++++------ .../windows/etw/.etw_publisher_processes.md | 80 ++++++++-------- .../events/windows/etw/.etw_user_session.md | 51 ++++------ .../events_stream/.events_stream.md | 10 +- .../events_stream/.events_stream_registry.md | 23 ++--- .../linuxevents/src/.bpfprocesseventstable.md | 49 +++++----- .../linuxevents/src/.linuxeventsservice.md | 39 ++++---- osquery/extensions/.extensions.md | 51 +++++----- osquery/extensions/.interface.md | 59 +++++++----- osquery/extensions/thrift/gen/.Extension.md | 39 ++++---- .../thrift/gen/.ExtensionManager.md | 65 ++++++------- .../thrift/gen/.osquery_constants.md | 23 +++-- .../extensions/thrift/gen/.osquery_types.md | 38 +++++--- osquery/filesystem/.filesystem.md | 86 ++++++++--------- osquery/filesystem/.mock_file_structure.md | 36 +++---- osquery/filesystem/linux/.mounts.md | 63 ++++++------- osquery/filesystem/linux/.proc.md | 74 ++++++++------- osquery/filesystem/posix/.xattrs.md | 52 ++++++----- osquery/hashing/.hashing.md | 64 ++++++------- osquery/logger/.logger.md | 35 ++----- osquery/main/.main.md | 28 +++--- osquery/main/harnesses/.fuzz_utils.md | 6 +- .../numeric_monitoring/.numeric_monitoring.md | 49 +++++----- .../numeric_monitoring/.plugin_interface.md | 30 +++--- .../.pre_aggregation_cache.md | 41 ++++---- osquery/registry/.registry_factory.md | 71 ++++++++------ osquery/registry/.registry_interface.md | 62 +++++++------ osquery/remote/.http_client.md | 43 ++++----- osquery/remote/.requests.md | 68 ++++++-------- osquery/remote/.uri.md | 46 +++++---- osquery/remote/enroll/.enroll.md | 49 +++++----- osquery/remote/serializers/.json.md | 30 +++--- osquery/remote/tests/.test_utils.md | 39 ++++---- osquery/remote/transports/.tls.md | 61 +++++------- osquery/sql/.dynamic_table_row.md | 49 +++++----- osquery/sql/.sql.md | 55 ++++++----- osquery/sql/.sqlite_util.md | 72 ++++++-------- osquery/sql/.virtual_table.md | 56 +++++------ osquery/sql/tests/.sql_test_utils.md | 44 ++++----- osquery/system/network/.hostname.md | 48 +++++----- .../usersgroups/windows/.groups_service.md | 37 ++++---- .../windows/.users_groups_cache.md | 67 ++++++------- .../usersgroups/windows/.users_service.md | 46 +++++---- .../tables/applications/.jetbrains_plugins.md | 44 +++++---- osquery/tables/applications/chrome/.utils.md | 60 ++++++------ .../applications/posix/.prometheus_metrics.md | 50 +++++----- osquery/tables/events/.event_utils.md | 26 +++--- .../tables/events/linux/.apparmor_events.md | 55 +++++------ .../events/linux/.bpf_process_events.md | 53 +++++------ .../tables/events/linux/.bpf_socket_events.md | 40 ++++---- .../tables/events/linux/.process_events.md | 40 ++++---- .../events/linux/.process_file_events.md | 57 ++++++------ .../tables/events/linux/.seccomp_events.md | 56 +++++------ .../tables/events/linux/.selinux_events.md | 39 ++++---- osquery/tables/events/linux/.socket_events.md | 28 ++---- .../events/windows/.dns_lookup_events.md | 55 +++++------ .../events/windows/.etw_process_events.md | 26 +++--- .../events/windows/.ntfs_journal_events.md | 45 ++++----- .../events/windows/.powershell_events.md | 62 ++++++------- .../tables/events/windows/.windows_events.md | 42 +++------ .../tables/networking/linux/.iptc_proxy.md | 55 ++++++----- .../tables/networking/posix/.interfaces.md | 10 +- osquery/tables/networking/posix/.utils.md | 41 ++++---- .../tables/networking/windows/.interfaces.md | 18 ++-- .../windows/.windows_firewall_rules.md | 52 +++++------ osquery/tables/system/.system_utils.md | 38 +++----- osquery/tables/system/darwin/.asl_utils.md | 50 ++++++---- osquery/tables/system/linux/.apt_sources.md | 39 ++++---- osquery/tables/system/linux/.md_tables.md | 41 ++++---- osquery/tables/system/linux/.pci_devices.md | 46 +++++---- osquery/tables/system/linux/.processes.md | 14 +-- .../linux/dbus/.uniquedbusconnection.md | 38 +++----- .../system/linux/dbus/.uniquedbusmessage.md | 52 ++++++----- .../linux/dbus/methods/.getstringproperty.md | 23 ++--- .../dbus/methods/.listunitsmethodhandler.md | 50 +++++----- .../tables/system/posix/.authorized_keys.md | 32 +++---- osquery/tables/system/posix/.known_hosts.md | 37 +++----- osquery/tables/system/posix/.last.md | 38 ++++---- osquery/tables/system/posix/.openssl_utils.md | 35 ++++--- osquery/tables/system/posix/.shell_history.md | 34 +++---- osquery/tables/system/posix/.sudoers.md | 27 +++--- .../tables/system/windows/.certificates.md | 52 +++++++---- osquery/tables/system/windows/.programs.md | 19 ++-- osquery/tables/system/windows/.registry.md | 44 ++++----- .../windows/.security_profile_info_utils.md | 64 +++++-------- .../tables/system/windows/.startup_items.md | 36 ++++--- .../system/windows/.windows_eventlog.md | 53 ++++++----- .../system/windows/.windows_update_history.md | 61 ++++++------ osquery/tables/yara/.yara_utils.md | 60 ++++++------ osquery/utils/.base64.md | 18 ++-- osquery/utils/.chars.md | 52 +++++------ osquery/utils/.only_movable.md | 40 ++++---- osquery/utils/.rot13.md | 6 +- osquery/utils/aws/.aws_util.md | 63 +++++++------ osquery/utils/azure/.azure_util.md | 38 +++----- osquery/utils/config/.default_paths.md | 48 +++++----- osquery/utils/conversions/.split.md | 36 +++---- osquery/utils/conversions/.trim.md | 8 +- osquery/utils/conversions/.tryto.md | 33 ++++--- osquery/utils/conversions/darwin/.cfdata.md | 17 ++-- .../utils/conversions/darwin/.cfdictionary.md | 33 +++---- osquery/utils/conversions/darwin/.cfnumber.md | 39 ++++---- osquery/utils/conversions/darwin/.cfstring.md | 17 ++-- osquery/utils/conversions/darwin/.cftime.md | 16 ++-- osquery/utils/conversions/darwin/.iokit.md | 63 ++++++------- osquery/utils/conversions/windows/.strings.md | 37 ++++---- .../conversions/windows/.windows_time.md | 42 ++++----- osquery/utils/info/.platform_type.md | 36 ++++--- osquery/utils/info/.tool_type.md | 35 ++++--- osquery/utils/info/.version.md | 47 +++++----- osquery/utils/json/.json.md | 89 ++++++++++-------- osquery/utils/linux/dpkg/.dpkgquery.md | 46 +++++---- osquery/utils/linux/dpkg/.idpkgquery.md | 57 ++++-------- osquery/utils/pidfile/.pidfile.md | 68 +++++++------- osquery/utils/status/.status.md | 44 +++++---- osquery/utils/system/.time.md | 27 +++--- osquery/utils/system/.uptime.md | 5 +- osquery/utils/system/linux/.cpu.md | 46 +++++---- osquery/utils/system/linux/proc/.proc.md | 22 +++-- osquery/utils/system/posix/.errno.md | 34 +++---- osquery/utils/system/posix/.system.md | 57 ++++++------ osquery/utils/system/windows/.etw_helpers.md | 42 ++++----- osquery/utils/system/windows/.system.md | 30 +++--- .../system/windows/.users_groups_helpers.md | 65 +++++++------ osquery/utils/windows/.lzxpress.md | 26 +++--- osquery/utils/windows/.shellitem.md | 62 +++++++------ osquery/utils/ycloud/.ycloud_util.md | 38 ++++---- .../worker/ipc/.table_ipc_json_converter.md | 58 ++++++------ .../ipc/linux/.linux_table_container_ipc.md | 43 ++++----- osquery/worker/ipc/linux/.linux_table_ipc.md | 56 ++++++----- osquery/worker/ipc/posix/.pipe_channel.md | 44 +++++---- .../worker/ipc/posix/.pipe_channel_factory.md | 48 +++++----- osquery/worker/logging/glog/.glog_logger.md | 30 +++--- plugins/config/.tls_config.md | 37 ++++---- .../parsers/.auto_constructed_tables.md | 60 ++++++------ plugins/config/parsers/.decorators.md | 45 +++++---- plugins/config/parsers/.feature_vectors.md | 32 ++++--- plugins/config/parsers/.kafka_topics.md | 36 +++---- plugins/config/parsers/.logger.md | 43 +++++---- plugins/config/parsers/.prometheus_targets.md | 40 ++++---- plugins/database/.rocksdb.md | 72 +++++++------- plugins/database/.sqlite.md | 66 ++++++------- plugins/logger/.aws_firehose.md | 69 +++++++------- plugins/logger/.aws_kinesis.md | 69 +++++++------- plugins/logger/.buffered.md | 72 +++++++------- plugins/logger/.filesystem_logger.md | 54 +++++------ plugins/logger/.kafka_producer.md | 59 ++++++------ plugins/logger/.stdout.md | 46 ++++----- plugins/logger/.syslog_logger.md | 47 ++++------ plugins/logger/.tls_logger.md | 68 +++++++------- plugins/logger/.windows_event_log.md | 57 +++++------- plugins/numeric_monitoring/.filesystem.md | 55 ++++++----- plugins/remote/enroll/.tls_enroll.md | 43 +++++---- tests/.test_util.md | 55 +++++------ tests/integration/tables/.helper.md | 82 +++++++--------- 235 files changed, 5376 insertions(+), 5624 deletions(-) diff --git a/libraries/cmake/source/augeas/generated/linux/aarch64/code/.parser.md b/libraries/cmake/source/augeas/generated/linux/aarch64/code/.parser.md index a9ebb2bd29b..059bdc11868 100644 --- a/libraries/cmake/source/augeas/generated/linux/aarch64/code/.parser.md +++ b/libraries/cmake/source/augeas/generated/linux/aarch64/code/.parser.md @@ -1,65 +1,85 @@ -Auto-generated Bison parser interface header for the Augeas lens language (`augl`) parser, defining token types, semantic value types, and location tracking structures used by the generated LALR(1) parser. +Auto-generated Bison parser interface header for the Augeas lens language (`augl`) parser, defining tokens, semantic value types, and location tracking structures used by the generated LALR(1) parser. ## Key Components -### Token Enum (`yytokentype`) -Defines all terminal tokens recognized by the lexer: +### Token Definitions (`yytokentype` enum + macros) +Lexical tokens recognized by the scanner: | Token | Value | Description | -|---|---|---| +|-------|-------|-------------| | `DQUOTED` | 258 | Double-quoted string literal | | `REGEXP` | 259 | Regular expression literal | | `LIDENT` / `UIDENT` / `QIDENT` | 260–262 | Lower/Upper/Qualified identifiers | -| `ARROW` | 263 | `->` arrow operator | +| `ARROW` | 263 | `->` operator | | `KW_MODULE` … `KW_AFTER` | 264–275 | Language keywords | -### Semantic Value Union (`YYSTYPE`) -Holds the value associated with each parsed token or grammar rule: +### `YYSTYPE` β€” Semantic Value Union +Holds the parsed value for any token or grammar rule: -- `struct term *term` β€” AST expression node -- `struct type *type` β€” Type annotation node -- `struct ident *ident` β€” Identifier reference -- `struct tree *tree` β€” Parse tree node -- `char *string` β€” String value -- `regexp` β€” Struct with `pattern` (string) and `nocase` (int) flag -- `int intval` β€” Integer literal -- `enum quant_tag quant` β€” Quantifier (`*`, `+`, `?`) +```c +union YYSTYPE { + struct term *term; // AST expression node + struct type *type; // Type annotation + struct ident *ident; // Identifier reference + struct tree *tree; // Parse tree node + char *string; // String value + struct { + int nocase; // Case-insensitive flag + char *pattern; // Regex pattern string + } regexp; + int intval; // Integer literal + enum quant_tag quant; // Quantifier (*, +, ?) +}; +``` -### Location Type (`YYLTYPE`) -Tracks source positions with `first_line`, `first_column`, `last_line`, `last_column` for error reporting. +### `YYLTYPE` β€” Source Location Tracking +Tracks line/column ranges for error reporting: -### Parser Entry Point ```c -int augl_parse(struct term **term, yyscan_t scanner); +struct YYLTYPE { + int first_line, first_column; + int last_line, last_column; +}; ``` -### Scanner State (`struct state`) +### `struct state` β€” Scanner State +Custom reentrant scanner context attached to `yyscan_t`: + ```c struct state { - struct info *info; // Source file info - unsigned int comment_depth; // Nested comment tracking + struct info *info; // File/source metadata + unsigned int comment_depth; // Nested comment nesting level }; ``` +### Entry Point +```c +int augl_parse(struct term **term, yyscan_t scanner); +``` +Parses a full Augeas lens module; populates `*term` with the root AST node on success. + ## Usage Example ```c #include "parser.h" #include "info.h" -struct term *result = NULL; -yyscan_t scanner; +int parse_lens_file(const char *filename) { + struct term *root = NULL; + yyscan_t scanner; -augl_lex_init(&scanner); -// configure scanner input ... + augl_lex_init(&scanner); + /* attach struct state with file info to scanner */ -int rc = augl_parse(&result, scanner); -if (rc == 0 && result != NULL) { - // process the parsed AST term -} + int rc = augl_parse(&root, scanner); + if (rc == 0) { + /* root now points to the parsed AST */ + } -augl_lex_destroy(scanner); + augl_lex_destroy(scanner); + return rc; +} ``` -> **Note:** This file is auto-generated by GNU Bison 3.0.4 from `parser.y`. Do not edit directly β€” modify the grammar source and regenerate. \ No newline at end of file +> **Note:** This file is auto-generated by GNU Bison 3.0.4 from `parser.y`. Edit `parser.y` directly rather than modifying this header. \ No newline at end of file diff --git a/libraries/cmake/source/augeas/generated/linux/x86_64/code/.parser.md b/libraries/cmake/source/augeas/generated/linux/x86_64/code/.parser.md index b54e83a94b8..4dc7c98f47d 100644 --- a/libraries/cmake/source/augeas/generated/linux/x86_64/code/.parser.md +++ b/libraries/cmake/source/augeas/generated/linux/x86_64/code/.parser.md @@ -1,55 +1,51 @@ -Auto-generated Bison parser interface header defining token types, semantic value union (`YYSTYPE`), location tracking (`YYLTYPE`), and scanner state for parsing Augeas lens definition files. +Auto-generated Bison parser interface header defining token types, semantic value union (`YYSTYPE`), and location tracking (`YYLTYPE`) for a lens/module language parser. ## Key Components ### Token Definitions (`yytokentype` enum) | Token | Value | Description | -|---|---|---| +|-------|-------|-------------| | `DQUOTED` | 258 | Double-quoted string literal | | `REGEXP` | 259 | Regular expression literal | -| `LIDENT` / `UIDENT` / `QIDENT` | 260–262 | Lower/Upper/Qualified identifiers | -| `ARROW` | 263 | `->` arrow operator | +| `LIDENT` / `UIDENT` / `QIDENT` | 260–262 | Lower/upper/qualified identifiers | +| `ARROW` | 263 | `->` operator | | `KW_MODULE` … `KW_AFTER` | 264–275 | Language keywords | ### `YYSTYPE` β€” Semantic Value Union -Holds the value associated with each parsed token or grammar rule: +Holds the parsed value for any grammar symbol: ```c -union YYSTYPE { - struct term *term; // Expression/term node - struct type *type; // Type annotation - struct ident *ident; // Identifier reference - struct tree *tree; // Parse tree node - char *string; // String value +typedef union YYSTYPE { + struct term *term; // AST term node + struct type *type; // Type annotation + struct ident *ident; // Identifier + struct tree *tree; // Tree node + char *string; // String value struct { - int nocase; // Case-insensitive flag - char *pattern; // Regex pattern string - } regexp; - int intval; // Integer literal - enum quant_tag quant; // Quantifier (*, +, ?) -}; + int nocase; + char *pattern; + } regexp; // Regex with case flag + int intval; // Integer literal + enum quant_tag quant; // Quantifier (*, +, ?) +} YYSTYPE; ``` -### `YYLTYPE` β€” Location Tracking -Tracks source positions for error reporting: +### `YYLTYPE` β€” Source Location Tracking +Tracks token positions for error reporting: ```c typedef struct YYLTYPE { - int first_line; - int first_column; - int last_line; - int last_column; + int first_line, first_column; + int last_line, last_column; } YYLTYPE; ``` -### `state` β€” Custom Scanner State -Carries context between the lexer and parser: - +### `struct state` β€” Custom Scanner State ```c struct state { - struct info *info; // Source file metadata - unsigned int comment_depth; // Nested comment tracking + struct info *info; // Source file info + unsigned int comment_depth; // Nested comment depth }; ``` @@ -58,17 +54,16 @@ struct state { ```c #include "parser.h" -// Access token location after parsing -void report_error(YYLTYPE *loc, const char *msg) { - fprintf(stderr, "Error at line %d, col %d: %s\n", - loc->first_line, loc->first_column, msg); -} +/* Access token type in a lexer rule */ +YYSTYPE yylval; +YYLTYPE yylloc; -// Initialize scanner state before invoking yyparse() -struct state scanner_state = { - .info = load_info("myfile.aug"), - .comment_depth = 0 -}; +/* Assign a parsed string token */ +yylval.string = strdup(yytext); + +/* Check source location after a parse error */ +fprintf(stderr, "Error at line %d, col %d\n", + yylloc.first_line, yylloc.first_column); ``` -> **Note:** This file is auto-generated by GNU Bison 2.4.1 from `parser.y`. Do not edit directly β€” modify the grammar source instead. \ No newline at end of file +> **Note:** This file is auto-generated by GNU Bison 2.4.1 from `parser.y`. Edit the source grammar file rather than this header directly. \ No newline at end of file diff --git a/libraries/cmake/source/augeas/generated/macos/aarch64/code/.parser.md b/libraries/cmake/source/augeas/generated/macos/aarch64/code/.parser.md index e4d32a0c30c..c657e74e105 100644 --- a/libraries/cmake/source/augeas/generated/macos/aarch64/code/.parser.md +++ b/libraries/cmake/source/augeas/generated/macos/aarch64/code/.parser.md @@ -1,38 +1,60 @@ -Auto-generated Bison parser interface header for the `augl` grammar, defining token types, semantic value unions, and location tracking structures used by the Augeas lens language parser. +Auto-generated Bison parser interface header for the `augl` grammar, defining token types, semantic value types, and location tracking structures used by the Augeas lens language parser. ## Key Components ### Token Enum (`yytokentype`) -Defines all terminal tokens recognized by the lexer: +Defines all lexical tokens recognized by the parser: | Token | Value | Description | -|---|---|---| +|-------|-------|-------------| | `DQUOTED` | 258 | Double-quoted string literal | -| `REGEXP` | 259 | Regular expression literal | +| `REGEXP` | 259 | Regular expression | | `LIDENT` / `UIDENT` / `QIDENT` | 260–262 | Lower/upper/qualified identifiers | -| `ARROW` | 263 | `->` arrow operator | +| `ARROW` | 263 | `->` operator | | `KW_MODULE` … `KW_AFTER` | 264–275 | Language keywords | ### Semantic Value Union (`YYSTYPE`) -Holds the value of each parsed token/rule: +Holds the parsed value for each token/rule: -| Field | Type | Purpose | -|---|---|---| -| `term` | `struct term *` | AST term node | -| `type` | `struct type *` | Type annotation | -| `ident` | `struct ident *` | Identifier reference | -| `tree` | `struct tree *` | Parse tree node | -| `regexp` | `struct { int nocase; char *pattern; }` | Regex with case flag | -| `quant` | `enum quant_tag` | Quantifier (`*`, `+`, `?`) | +```c +union YYSTYPE { + struct term *term; // AST expression node + struct type *type; // Type annotation + struct ident *ident; // Identifier reference + struct tree *tree; // Parse tree node + char *string; // String value + struct { + int nocase; + char *pattern; + } regexp; // Regex with case flag + int intval; // Integer literal + enum quant_tag quant; // Quantifier (*, +, ?) +}; +``` ### Location Type (`YYLTYPE`) -Tracks source positions with `first_line`, `first_column`, `last_line`, `last_column` for error reporting. +Tracks source positions for error reporting: + +```c +struct YYLTYPE { + int first_line, first_column; + int last_line, last_column; +}; +``` ### Scanner State (`struct state`) -Custom lexer state carrying an `info` context pointer and `comment_depth` for nested comment handling. +Custom lexer state tracking source info and nested comment depth: + +```c +struct state { + struct info *info; + unsigned int comment_depth; +}; +``` + +### Main Parse Function -### Entry Point ```c int augl_parse(struct term **term, yyscan_t scanner); ``` @@ -43,25 +65,17 @@ int augl_parse(struct term **term, yyscan_t scanner); #include "parser.h" #include "info.h" -int parse_lens_file(const char *filename) { - struct term *root = NULL; - yyscan_t scanner; - - /* Initialize scanner state with source info */ - struct state st = { - .info = make_info(filename), - .comment_depth = 0 - }; - - augl_lex_init(&scanner); - augl_set_extra(&st, scanner); +struct term *result = NULL; +yyscan_t scanner; - /* Run the parser; result stored in root */ - int rc = augl_parse(&root, scanner); +augl_lex_init(&scanner); +// attach input source to scanner ... +int rc = augl_parse(&result, scanner); +augl_lex_destroy(scanner); - augl_lex_destroy(scanner); - return rc; /* 0 on success */ +if (rc == 0) { + // result points to the root AST term } ``` -> **Note:** Do not use any symbols prefixed with `yy_` or `YY_` β€” these are private Bison internals subject to change without notice. \ No newline at end of file +> **Note:** This file is auto-generated by GNU Bison 3.8.2 from `parser.y`. Do not edit manually β€” modify the grammar source instead. \ No newline at end of file diff --git a/libraries/cmake/source/augeas/generated/macos/x86_64/code/.parser.md b/libraries/cmake/source/augeas/generated/macos/x86_64/code/.parser.md index aebbb9ace67..fa52a800177 100644 --- a/libraries/cmake/source/augeas/generated/macos/x86_64/code/.parser.md +++ b/libraries/cmake/source/augeas/generated/macos/x86_64/code/.parser.md @@ -1,38 +1,33 @@ -Auto-generated Bison parser interface header for the Augeas lens language (`augl`) grammar, defining token types, semantic value union, and location tracking structures used by the LALR(1) parser. +Auto-generated Bison parser interface header for the `augl` language parser, defining token types, semantic value types, and location tracking structures used by the Yacc-compatible parser. ## Key Components -### Token Enum (`yytokentype`) -Defines all terminal symbols recognized by the lexer: - -| Token | Value | Description | -|---|---|---| -| `DQUOTED` | 258 | Double-quoted string literal | -| `REGEXP` | 259 | Regular expression literal | -| `LIDENT` / `UIDENT` / `QIDENT` | 260–262 | Lower/upper/qualified identifiers | -| `ARROW` | 263 | `->` operator | -| `KW_MODULE` … `KW_AFTER` | 264–275 | Language keywords | - -### Semantic Value Union (`YYSTYPE`) -Holds the parsed value for each grammar symbol: - -- `struct term *term` β€” AST expression node -- `struct type *type` β€” type annotation node -- `struct ident *ident` β€” identifier node -- `struct tree *tree` β€” parse tree node -- `char *string` β€” raw string value -- `regexp` β€” struct with `nocase` flag and `pattern` string -- `int intval` β€” integer literal -- `enum quant_tag quant` β€” quantifier (`*`, `+`, `?`) - -### Location Type (`YYLTYPE`) -Tracks source positions with `first_line`, `first_column`, `last_line`, `last_column` for error reporting. - -### Scanner State (`struct state`) -Custom lexer state attached to the reentrant scanner, carrying an `info` context and `comment_depth` counter for nested comment handling. - -### Parser Entry Point +### Token Types (`yytokentype` enum) +Defines all lexical tokens recognized by the parser: +- **Literals:** `DQUOTED` (258), `REGEXP` (259) +- **Identifiers:** `LIDENT` (260), `UIDENT` (261), `QIDENT` (262) +- **Operators:** `ARROW` (263) +- **Keywords:** `KW_MODULE`, `KW_AUTOLOAD`, `KW_LET`, `KW_LET_REC`, `KW_IN`, `KW_STRING`, `KW_REGEXP`, `KW_LENS`, `KW_TEST`, `KW_GET`, `KW_PUT`, `KW_AFTER` + +### `YYSTYPE` Union +Semantic value type holding the possible values a token or grammar symbol can carry: +- `struct term *` β€” AST term node +- `struct type *` β€” type annotation +- `struct ident *` β€” identifier +- `struct tree *` β€” parse tree node +- `char *string` β€” string literal +- `regexp` struct β€” pattern with `nocase` flag and `char *pattern` +- `int intval` β€” integer value +- `enum quant_tag quant` β€” quantifier tag + +### `YYLTYPE` Struct +Source location tracking with `first_line`, `first_column`, `last_line`, `last_column` fields for error reporting. + +### `struct state` +Custom scanner state tracking a `struct info *` pointer and `comment_depth` counter for nested comment handling. + +### Entry Point ```c int augl_parse(struct term **term, yyscan_t scanner); ``` @@ -43,22 +38,21 @@ int augl_parse(struct term **term, yyscan_t scanner); #include "parser.h" #include "info.h" -int parse_lens_file(const char *filename) { +int parse_module(struct info *info) { struct term *result = NULL; yyscan_t scanner; - /* Initialize reentrant scanner, then invoke parser */ + /* Initialize scanner with custom state */ + struct state st = { .info = info, .comment_depth = 0 }; + augl_lex_init(&scanner); + augl_set_extra(&st, scanner); int rc = augl_parse(&result, scanner); - if (rc == 0 && result != NULL) { - /* Walk the resulting AST term */ - process_term(result); - } augl_lex_destroy(scanner); - return rc; + return rc; /* 0 on success */ } ``` -> **Note:** This file is auto-generated by GNU Bison 3.8.2 from `parser.y`. Do not edit directly β€” modify the grammar source instead and regenerate. \ No newline at end of file +> **Note:** This file is auto-generated by GNU Bison 3.8.2 from `parser.y`. Do not edit directly β€” modify the grammar source instead. \ No newline at end of file diff --git a/openframe/.openframe_authorization_manager.md b/openframe/.openframe_authorization_manager.md index 5203e846d2e..ee26f75c1a3 100644 --- a/openframe/.openframe_authorization_manager.md +++ b/openframe/.openframe_authorization_manager.md @@ -1,34 +1,37 @@ -Manages the OpenFrame authorization token used for authentication with OpenFrame services. This singleton-style class provides controlled access to storing and retrieving the auth token via a provider-based pattern. +Manages the OpenFrame authorization token lifecycle within the `osquery` namespace, providing thread-safe storage and retrieval of authentication credentials for OpenFrame services. ## Key Components | Symbol | Type | Description | -|---|---|---| -| `OpenframeAuthorizationManager` | Class | Non-copyable token manager; instantiation restricted to `OpenframeAuthorizationManagerProvider` | -| `updateToken(token)` | Method | Replaces the stored authorization token | +|--------|------|-------------| +| `OpenframeAuthorizationManager` | Class | Singleton-style, non-copyable manager for the OpenFrame auth token | +| `updateToken(token)` | Method | Stores a new authorization token | | `getToken()` | Method | Returns the current authorization token | -| `token_` | Private Field | Internal string storage for the token | -| `OpenframeAuthorizationManagerProvider` | Friend Class | Sole class permitted to construct/destroy instances | +| `OpenframeAuthorizationManagerProvider` | Friend class | Sole class permitted to construct/destroy the manager | + +## Design Notes + +- Inherits `boost::noncopyable` β€” copy construction and assignment are disabled +- Constructor and destructor are `private`, enforcing creation exclusively through `OpenframeAuthorizationManagerProvider` +- Follows a provider/consumer pattern separating token lifecycle management from token access ## Usage Example ```cpp -// Access is granted only through the provider β€” never constructed directly. -// Typical usage via the provider pattern: +// Access is granted via the provider (not directly constructed) +// Typical usage pattern within the osquery namespace: -OpenframeAuthorizationManager& manager = provider.getManager(); +OpenframeAuthorizationManager& manager = getManagerFromProvider(); -// Store a new token received from OpenFrame auth flow +// Store a new token received from OpenFrame authentication manager.updateToken("Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9..."); -// Retrieve the token for an outbound API request +// Retrieve the token when making authenticated API calls std::string token = manager.getToken(); -httpClient.setHeader("Authorization", token); -``` -## Design Notes +// Use token in an outbound OpenFrame service request +sendRequest(endpoint, token); +``` -- **Non-copyable** (`boost::noncopyable`): Prevents accidental token duplication across object copies. -- **Private constructor/destructor**: Enforces that only `OpenframeAuthorizationManagerProvider` can create or destroy instances, ensuring centralized lifecycle management. -- **`osquery` namespace**: Integrates with the broader osquery agent infrastructure used by the OpenFrame platform. \ No newline at end of file +> **Note:** Direct instantiation of `OpenframeAuthorizationManager` is not permitted. All instances must be obtained through `OpenframeAuthorizationManagerProvider`, which acts as the sole factory for this class. \ No newline at end of file diff --git a/openframe/.openframe_encryption_service.md b/openframe/.openframe_encryption_service.md index 7319497e8f5..ca40e8f820a 100644 --- a/openframe/.openframe_encryption_service.md +++ b/openframe/.openframe_encryption_service.md @@ -1,18 +1,17 @@ -AES-256-GCM decryption service header for the OpenFrame platform, providing symmetric key encryption utilities via OpenSSL. +AES-GCM decryption service header for the OpenFrame platform, providing symmetric encryption utilities backed by OpenSSL. ## Key Components | Symbol | Type | Description | -|---|---|---| -| `OpenframeEncryptionService` | Class | Main encryption service wrapping OpenSSL EVP AES-GCM operations | -| `OpenframeEncryptionService(secret)` | Constructor | Initializes the service with a symmetric secret key | -| `decrypt(data)` | Method | Decrypts a Base64-encoded AES-256-GCM ciphertext, returns plaintext string | -| `base64Decode(encoded)` | Method | Decodes a Base64 string into raw bytes | -| `handleOpenSSLError()` | Private Method | Throws `std::runtime_error` with OpenSSL error details on failure | +|--------|------|-------------| +| `OpenframeEncryptionService` | Class | Main encryption service instantiated with a shared secret | +| `decrypt()` | Method | Decrypts a Base64-encoded AES-256-GCM ciphertext string | +| `base64Decode()` | Method | Decodes a Base64 string into raw bytes | | `KEY_SIZE` | Constant | 32 bytes (256-bit AES key) | | `IV_SIZE` | Constant | 12 bytes (96-bit GCM nonce) | | `TAG_SIZE` | Constant | 16 bytes (128-bit GCM authentication tag) | +| `handleOpenSSLError()` | Private | Translates OpenSSL error stack into `std::runtime_error` | ## Usage Example @@ -21,23 +20,20 @@ AES-256-GCM decryption service header for the OpenFrame platform, providing symm #include int main() { - // Initialize with a 256-bit secret key - OpenframeEncryptionService enc("my-32-byte-secret-key-goes-here!"); + // Instantiate with a 256-bit shared secret + OpenframeEncryptionService svc("my-32-byte-secret-key-goes-here!"); - // Decrypt Base64-encoded AES-256-GCM payload + // Decrypt a Base64-encoded AES-256-GCM payload try { - std::string ciphertext = "BASE64_ENCODED_ENCRYPTED_PAYLOAD=="; - std::string plaintext = enc.decrypt(ciphertext); + std::string plaintext = svc.decrypt(""); std::cout << "Decrypted: " << plaintext << std::endl; } catch (const std::runtime_error& e) { std::cerr << "Decryption failed: " << e.what() << std::endl; } - // Standalone Base64 decode - std::vector raw = enc.base64Decode("SGVsbG8gV29ybGQ="); - - return 0; + // Standalone Base64 decoding + std::vector raw = svc.base64Decode(""); } ``` -> **Note:** The secret passed to the constructor must produce a 256-bit (32-byte) derived key. Any OpenSSL failure during decryption throws `std::runtime_error` via `handleOpenSSLError()`. \ No newline at end of file +> **Note:** The constructor accepts the raw secret string; callers are responsible for ensuring it meets the 32-byte (256-bit) key length requirement. All decryption errors surface as `std::runtime_error` via OpenSSL's error stack. \ No newline at end of file diff --git a/openframe/.openframe_token_extractor.md b/openframe/.openframe_token_extractor.md index 2d3e20928e6..ef65d78870a 100644 --- a/openframe/.openframe_token_extractor.md +++ b/openframe/.openframe_token_extractor.md @@ -1,37 +1,37 @@ -Provides a class interface for reading and decrypting authentication tokens stored in encrypted files, bridging file I/O with the OpenFrame encryption layer. +Declares the `OpenframeTokenExtractor` class, responsible for reading and decrypting authentication tokens from a file using the `OpenframeEncryptionService`. ## Key Components -### `OpenframeTokenExtractor` (class) - -| Member | Type | Description | -|--------|------|-------------| -| `OpenframeTokenExtractor(encryption_service, token_file_path)` | Constructor | Initializes the extractor with a shared encryption service and the path to the token file | -| `extractToken()` | `std::string` | Reads the token file, decrypts its contents via the injected `OpenframeEncryptionService`, and returns the plaintext token | -| `token_file_path_` | `std::string` (private) | Stored path to the encrypted token file on disk | -| `encryption_service_` | `std::shared_ptr` (private) | Shared ownership of the encryption service used for decryption | +- **`OpenframeTokenExtractor`** β€” Main class that wraps token extraction logic +- **Constructor** β€” Accepts a shared pointer to an `OpenframeEncryptionService` instance and the path to the token file +- **`extractToken()`** β€” Public method that reads the token file and returns the decrypted token string +- **`token_file_path_`** β€” Private member storing the filesystem path to the encrypted token file +- **`encryption_service_`** β€” Private shared pointer to the encryption service used for decryption ## Usage Example -```cpp +```c #include "openframe_token_extractor.h" #include "openframe_encryption_service.h" -// Create a shared encryption service +// Create shared encryption service auto enc_service = std::make_shared(/* config */); -// Instantiate the extractor with the service and token file path -OpenframeTokenExtractor extractor(enc_service, "/etc/openframe/auth.token"); +// Initialize extractor with token file path +OpenframeTokenExtractor extractor(enc_service, "/etc/openframe/token.enc"); // Extract and decrypt the token std::string token = extractor.extractToken(); -// Use the plaintext token for authenticated API calls +// Use token for authenticated API calls +if (!token.empty()) { + // proceed with authenticated request +} ``` ## Notes -- Uses shared ownership (`std::shared_ptr`) for the encryption service, making it safe to share one service instance across multiple extractors. -- The destructor is defaulted, so no manual resource cleanup is required. -- Depends on `openframe_encryption_service.h` β€” ensure the encryption service is properly initialized before calling `extractToken()`. \ No newline at end of file +- Ownership of `OpenframeEncryptionService` is shared via `std::shared_ptr`, allowing safe reuse across multiple extractor instances +- The destructor is defaulted, relying on RAII for safe cleanup of owned resources +- Token file path is stored at construction time and cannot be changed after initialization \ No newline at end of file diff --git a/openframe/.openframe_token_refresher.md b/openframe/.openframe_token_refresher.md index 3ec04ff3e39..9d71f38fa59 100644 --- a/openframe/.openframe_token_refresher.md +++ b/openframe/.openframe_token_refresher.md @@ -1,17 +1,21 @@ -Manages background token refresh for OpenFrame authentication by running a periodic refresh loop in a dedicated thread. +Manages background token refresh for the OpenFrame authorization system, periodically re-extracting credentials at a fixed interval using a dedicated thread. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `OpenframeTokenRefresher` | Class | Background service that periodically refreshes authentication tokens | -| `OpenframeTokenRefresher(extractor)` | Constructor | Accepts a shared `OpenframeTokenExtractor` instance | -| `start()` | Method | Spawns the background refresh thread | -| `stop()` | Method | Signals the refresh loop to terminate and joins the thread | -| `process()` | Private Method | Core refresh loop executed on the background thread | -| `running_` | `std::atomic` | Thread-safe flag controlling the refresh loop lifecycle | -| `extractor_` | `shared_ptr` | Token extraction dependency injected at construction | +- **`OpenframeTokenRefresher`** β€” Main class that runs a background refresh loop + - `OpenframeTokenRefresher(extractor)` β€” Constructor accepting a shared `OpenframeTokenExtractor` instance + - `start()` β€” Launches the background refresh thread + - `stop()` β€” Signals the thread to terminate and joins it + - `process()` β€” Private loop body executed on the refresh thread + +## Dependencies + +| Dependency | Role | +|---|---| +| `OpenframeTokenExtractor` | Performs the actual token extraction/renewal | +| `OpenframeAuthorizationManager` | Manages downstream authorization state | +| `std::thread` + `std::atomic` | Thread lifecycle and safe stop signaling | ## Usage Example @@ -19,20 +23,15 @@ Manages background token refresh for OpenFrame authentication by running a perio #include "openframe_token_refresher.h" #include "openframe_token_extractor.h" -// Inject extractor and start background refresh +// Create shared extractor and start background refresh auto extractor = std::make_shared(); osquery::OpenframeTokenRefresher refresher(extractor); -refresher.start(); // begins periodic token refresh loop +refresher.start(); // Begins periodic token renewal in background // ... application runs ... -refresher.stop(); // gracefully stops the background thread +refresher.stop(); // Gracefully stops the refresh thread on shutdown ``` -## Notes - -- Lives in the `osquery` namespace alongside other OpenFrame authentication components -- Depends on `OpenframeTokenExtractor` for token retrieval and `OpenframeAuthorizationManager` for token storage/validation -- Uses `std::atomic` for lock-free lifecycle control and `std::mutex` for any shared state access within `process()` -- `stop()` should always be called before destruction to avoid detached thread behavior \ No newline at end of file +> `running_` is an `std::atomic`, ensuring the stop signal is visible to the refresh thread without requiring an explicit lock on the control path. \ No newline at end of file diff --git a/osquery/carver/.carver.md b/osquery/carver/.carver.md index 50e1e949180..14331c7a012 100644 --- a/osquery/carver/.carver.md +++ b/osquery/carver/.carver.md @@ -1,39 +1,44 @@ -Defines the file carving subsystem for osquery, providing classes and utilities to copy files from disk, archive them, compress them, and POST the results to a remote endpoint. +Defines the file carving subsystem for osquery, enabling collection and exfiltration of files from disk to a remote endpoint via compressed archives. ## Key Components ### `CarverRunnable` -Abstract base class extending `InternalRunnable`. Manages the carver dispatch lifecycle via a static `running_` atomic flag. The `start()` method serially processes all pending carve requests; subclasses must implement `doCarve()`. +Abstract base class extending `InternalRunnable`. Serially processes pending carve requests from the dispatcher queue. Tracks global running state via `static std::atomic running_`. + +- `start()` β€” Scans and executes all pending carve requests +- `doCarve()` β€” Pure virtual; performs the actual carve for a given set of paths, GUID, and request ID +- `running()` β€” Static check for whether a carver is currently dispatched ### `CarverRunner` -Templated concrete runner that instantiates a carver of type `T` for each request. Designed to allow test-fake injection by swapping the carver type. Tracks the number of carves attempted in the current run cycle via `carves()`. +Concrete template subclass of `CarverRunnable`. Instantiates a carver of type `T` per request, enabling test fakes via template substitution. + +- `doCarve()` β€” Constructs `T(paths, guid, requestId)` and calls `carve()` +- `carves()` β€” Returns the count of carves attempted in the current runner lifetime ### `Carver` -Core carving class. Constructed with a set of file paths, a GUID, and a request ID. Orchestrates the full pipeline: +Core carving implementation. Manages the full lifecycle: file copy β†’ tar archive β†’ zstd compression β†’ HTTP POST. -| Method | Description | -|---|---| -| `carve()` | End-to-end entry point: carve β†’ compress β†’ POST | -| `createPaths()` | Sets up temp directory, archive, and compressed output paths | -| `carveAll()` | Blockwise-copies source files into a temp directory | -| `blockwiseCopy()` | Low-level copy using `FLAGS_carver_block_size` (default 8 KB) | -| `postCarve()` | POSTs the compressed archive to the configured TLS endpoint | +- `carve()` β€” End-to-end orchestration method +- `createPaths()` β€” Initializes temp directory, archive, and compress paths +- `carveAll()` β€” Copies source files into the temp carve directory +- `blockwiseCopy()` β€” Block-level file copy (default 8KB blocks via `FLAGS_carver_block_size`) +- `postCarve()` β€” POSTs the compressed archive to configured TLS endpoints ### `scheduleCarves()` -Free function called by the scheduler to dispatch a `CarverRunner` if one is not already running. +Free function that dispatches a `CarverRunner` if one is not already active. Intended to be called periodically by the osquery scheduler. ## Usage Example ```cpp -// Dispatch carves from the scheduler +// Dispatch carving from the scheduler osquery::scheduleCarves(); -// Direct use of the Carver pipeline (e.g., in tests via CarverRunner) -std::set paths = {"/etc/passwd", "/var/log/syslog"}; -osquery::Carver carver(paths, "guid-1234", "request-5678"); +// Direct carver instantiation (e.g., in tests) +std::set paths = {"/etc/hosts", "/etc/passwd"}; +osquery::Carver carver(paths, "guid-1234", "req-5678"); +osquery::Status status = carver.carve(); -Status status = carver.carve(); // carve β†’ archive β†’ compress β†’ POST if (!status.ok()) { LOG(ERROR) << "Carve failed: " << status.getMessage(); } diff --git a/osquery/carver/.carver_utils.md b/osquery/carver/.carver_utils.md index da5baa2fa91..ef1d2bc391a 100644 --- a/osquery/carver/.carver_utils.md +++ b/osquery/carver/.carver_utils.md @@ -1,40 +1,41 @@ -Utility header providing core primitives for osquery's file carving subsystem β€” constants, database helpers, and scheduling functions used across carver components. +Utility header for osquery's file carving subsystem, providing constants, a database update helper, and functions to schedule and identify file carve operations. ## Key Components ### Constants -| Constant | Value | Purpose | -|---|---|---| -| `kCarvePathPrefix` | `"osquery_carve_"` | Temp filesystem directory prefix | -| `kCarveNamePrefix` | `"carve_"` | Tar archive filename prefix | -| `kCarverDBPrefix` | `"carves."` | Database key namespace prefix | -| `kCarverStatusSuccess` | `"SUCCESS"` | Terminal success state | -| `kCarverStatusScheduled` | `"SCHEDULED"` | Pending/queued state | +| Name | Value | Purpose | +|------|-------|---------| +| `kCarvePathPrefix` | `"osquery_carve_"` | Prefix for temp FS directory holding carved files | +| `kCarveNamePrefix` | `"carve_"` | Prefix for the output tar archive | +| `kCarverDBPrefix` | `"carves."` | Database key namespace for carve entries | +| `kCarverStatusSuccess` | `"SUCCESS"` | Terminal success status string | +| `kCarverStatusScheduled` | `"SCHEDULED"` | Status set when a carve is queued | ### Globals -- **`kCarverPendingCarves`** (`std::atomic`) β€” Optimization flag for `CarverRunner`; avoids spinning up scheduler threads when no carves are queued. +- **`kCarverPendingCarves`** (`std::atomic`) β€” Optimization flag; set `true` when a carve is requested, `false` after the `CarverRunner` drains the queue, preventing unnecessary thread wake-ups. ### Functions -- **`updateCarveValue(guid, key, value)`** β€” Template function that reads a carve entry from the database by GUID, parses its JSON, updates a single key, and writes it back. -- **`createCarveGuid()`** β€” Generates and returns a new UUID string for identifying a carve request. -- **`carvePaths(paths, request_id, carve_guid)`** β€” Schedules a deferred file carve for the given paths; populates `carve_guid` on success and returns a `Status`. +- **`updateCarveValue(guid, key, value)`** β€” Template helper that reads a carve's JSON blob from the database by GUID, merges a new key/value pair, and writes it back. +- **`createCarveGuid()`** β€” Returns a new UUID string to uniquely identify a carve request. +- **`carvePaths(paths, request_id, carve_guid)`** β€” Schedules a deferred file carve for the given set of paths. Carving is intentionally asynchronous to avoid blocking query execution or waiting on remote services. ## Usage Example ```c +#include + // Schedule a carve for specific paths std::set targets = {"/etc/passwd", "/var/log/auth.log"}; -std::string guid; +std::string request_id = "incident-42"; +std::string carve_guid; -Status s = carvePaths(targets, "incident-42", guid); +Status s = osquery::carvePaths(targets, request_id, carve_guid); if (s.ok()) { - // Update a metadata field on the scheduled carve - updateCarveValue(guid, "priority", std::string("high")); + // carve_guid now identifies this request in the database + osquery::updateCarveValue(carve_guid, "priority", 1); } -``` - -> **Note:** `carvePaths` is non-blocking by design β€” actual carving is deferred to the scheduler to avoid blocking query execution or parallelism issues. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/config/.config.md b/osquery/config/.config.md index c963af8f4d6..73853b3e1dc 100644 --- a/osquery/config/.config.md +++ b/osquery/config/.config.md @@ -1,5 +1,5 @@ -Programmatic representation of osquery's runtime configuration, providing a singleton interface for loading, updating, validating, and querying configuration state including scheduled queries, file watches, and performance metrics. +Programmatic representation of osquery's configuration system, providing a singleton interface for loading, updating, validating, and querying configuration data including scheduled queries, packs, file categories, and performance metrics. ## Key Components @@ -7,56 +7,46 @@ Programmatic representation of osquery's runtime configuration, providing a sing The core configuration manager. Access via `Config::get()`. **Public Methods:** - -| Method | Description | -|---|---| -| `update(config)` | Apply new configuration data from a source map | -| `load()` | Check for a registered config plugin and trigger initial load | -| `refresh()` | Re-invoke the config plugin's `genConfig` (supports periodic refresh) | -| `recordQueryPerformance(...)` | Store pre/post process metrics for a scheduled query | -| `recordQueryStart(name)` | Mark a query as "dirty" (running) for crash detection | -| `genHash(hash)` | Compute SHA1 over all config sources | -| `hashSource(source, content)` | Hash a single source; returns `true` if content changed | -| `scheduledQueries(predicate)` | Iterate scheduled queries via callback | -| `packs(predicate)` | Iterate all loaded packs via callback | -| `files(predicate)` | Iterate watched file categories via callback | -| `getPerformanceStats(name, predicate)` | Retrieve `QueryPerformance` stats for a named query | -| `getParser(name)` | Fetch a registered `ConfigParserPlugin` by name | -| `validateConfig(document)` | Validate JSON depth and root type | -| `restoreConfigBackup()` | Retrieve config persisted in the backing database | +- `update()` β€” Apply new config data from a source map +- `restoreConfigBackup()` β€” Retrieve persisted config from the database +- `recordQueryPerformance()` β€” Store pre/post query process metrics +- `recordQueryStart()` β€” Mark a query as dirty/initialized before execution +- `genHash()` / `getHash()` / `hashSource()` β€” SHA1 hashing per source +- `scheduledQueries()` β€” Iterate scheduled queries via predicate +- `packs()` β€” Iterate registered packs via predicate +- `files()` β€” Iterate file categories via predicate +- `getPerformanceStats()` β€” Access `QueryPerformance` data by query name +- `getParser()` β€” Retrieve a `ConfigParserPlugin` by name +- `validateConfig()` β€” Verify JSON document structure and depth +- `addPack()` / `removePack()` β€” Manage query packs +- `addFile()` / `removeFiles()` β€” Manage monitored file paths + +**Protected Methods:** +- `refresh()` / `load()` β€” Trigger config reload from plugin +- `updateSource()` β€” Process a single source update +- `applyParsers()` β€” Dispatch JSON keys to registered `ConfigParserPlugin` instances +- `genPack()` β€” Resolve a pack resource via `ConfigPlugin` +- `purge()` / `reset()` β€” Cleanup and test utilities ### `kExecutingQuery` -Global string constant identifying the currently executing scheduled query. - ---- +Global string tracking the currently executing scheduled query name. ## Usage Example ```cpp // Access the singleton -auto& cfg = Config::get(); - -// Iterate all scheduled queries -cfg.scheduledQueries([](std::string name, const ScheduledQuery& query) { - LOG(INFO) << "Query: " << name << " interval: " << query.interval; -}); +auto& config = Config::get(); -// Iterate watched file categories -cfg.files([](const std::string& category, - const std::vector& paths) { - for (const auto& p : paths) { - LOG(INFO) << category << ": " << p; - } +// Iterate scheduled queries +config.scheduledQueries([](std::string name, const ScheduledQuery& query) { + std::cout << "Query: " << name << std::endl; }); -// Retrieve performance stats for a specific query +// Retrieve performance stats Config::getPerformanceStats("my_query", [](const QueryPerformance& perf) { - LOG(INFO) << "Executions: " << perf.executions; + std::cout << "Executions: " << perf.executions << std::endl; }); -// Restore backed-up config from database -auto result = Config::restoreConfigBackup(); -if (result) { - cfg.update(result.take()); -} +// Update config from a source map +Status s = config.update({{"filesystem", "{\"schedule\": {}}"}}); ``` \ No newline at end of file diff --git a/osquery/config/.packs.md b/osquery/config/.packs.md index 153833cf1fe..83815acb439 100644 --- a/osquery/config/.packs.md +++ b/osquery/config/.packs.md @@ -1,51 +1,50 @@ -Defines the `Pack` class and related utilities for managing osquery query packs β€” collections of scheduled queries with platform/version constraints and discovery logic. +Defines the `Pack` class and related utilities for managing osquery query packs β€” named collections of scheduled queries with platform/version constraints and discovery logic. ## Key Components -### `PackStats` struct -Tracks discovery query execution statistics: `total`, `hits`, and `misses` counts. +### `PackStats` +Struct tracking discovery query execution statistics: `total`, `hits`, and `misses` counts. -### `Pack` class -Non-copyable class representing a query pack loaded from configuration. +### `Pack` (class) +Non-copyable representation of a query pack loaded from configuration. | Method | Description | |---|---| | `initialize()` | Parses and loads pack content from a `rapidjson::Value` | | `shouldPackExecute()` | Determines if the pack should run on this host | -| `checkPlatform()` | Validates platform compatibility | +| `checkPlatform()` | Validates OS compatibility | | `checkVersion()` | Validates minimum osquery version requirement | -| `checkDiscovery()` | Runs discovery queries to determine pack applicability | -| `isActive()` | Returns active state without triggering discovery queries | -| `getSchedule()` | Returns the vector of `ScheduledQuery` entries | -| `getStats()` | Returns accumulated `PackStats` | +| `checkDiscovery()` | Runs discovery queries to determine pack eligibility | +| `isActive()` | Returns active state without re-running discovery queries | +| `getSchedule()` | Returns the list of `ScheduledQuery` entries | +| `getStats()` | Returns discovery execution statistics | ### Free Functions | Function | Description | |---|---| -| `splayValue(original, splay_percent)` | Generates a splayed interval from a base interval and percent | -| `restoreSplayedValue(name, interval)` | Retrieves or generates a cached splay for a name/interval pair | +| `splayValue(original, splay_percent)` | Computes a randomized interval offset based on a splay percentage | +| `restoreSplayedValue(name, interval)` | Retrieves or generates a cached deterministic splay for a query name/interval pair | ## Usage Example ```cpp #include -// Construct a pack from parsed JSON -rapidjson::Value packObj; // populated from config -osquery::Pack pack("my_pack", "config_source", packObj); +// Load a pack from parsed JSON +rapidjson::Document doc; +doc.Parse(pack_json_string.c_str()); + +osquery::Pack pack("my_pack", "config_source", doc); // Check if the pack should run on this host if (pack.shouldPackExecute()) { - for (const auto& query : pack.getSchedule()) { - // queue query for execution - } + for (const auto& query : pack.getSchedule()) { + // queue query for execution + } } -// Generate a splayed interval (e.g., 10% splay on 60s interval) -uint64_t splayed = osquery::splayValue(60, 10); - -// Retrieve or create a stable cached splay -uint64_t stable = osquery::restoreSplayedValue("my_pack_query", 60); +// Compute a splayed interval for a 60s query with 10% splay +uint64_t interval = osquery::restoreSplayedValue("my_query", 60); ``` \ No newline at end of file diff --git a/osquery/config/tests/.test_utils.md b/osquery/config/tests/.test_utils.md index 0771b51ebad..f5f41f3b792 100644 --- a/osquery/config/tests/.test_utils.md +++ b/osquery/config/tests/.test_utils.md @@ -1,37 +1,38 @@ -Utility header providing test infrastructure helpers for osquery configuration and pack loading in unit tests. +Utility header providing test infrastructure helpers for osquery configuration and pack testing. ## Key Components | Symbol | Type | Description | -|---|---|---| +|--------|------|-------------| | `getTestConfigDirectory()` | Function | Returns path to the test configuration directory | | `getTestHelperScriptsDirectory()` | Function | Returns path to the test helper scripts directory | | `getTestConfigMap()` | Function | Builds a config map from a file, mapping source name to JSON content | | `getExamplePacksConfig()` | Function | Returns a sample packs configuration as a `JSON` object | | `getUnrestrictedPack()` | Function | Returns a pack with no platform/version restrictions | | `getRestrictedPack()` | Function | Returns a pack with platform/version restrictions applied | -| `getPackWithDiscovery()` | Function | Returns a pack containing discovery queries | -| `getPackWithValidDiscovery()` | Function | Returns a pack with a discovery query expected to pass | -| `getPackWithFakeVersion()` | Function | Returns a pack targeting a non-existent/fake osquery version | +| `getPackWithDiscovery()` | Function | Returns a pack that includes discovery queries | +| `getPackWithValidDiscovery()` | Function | Returns a pack with a valid, passing discovery query | +| `getPackWithFakeVersion()` | Function | Returns a pack targeting a non-existent osquery version | ## Usage Example -```c +```cpp #include "test_utils.h" -// Resolve test config files at runtime -auto config_dir = osquery::getTestConfigDirectory(); -auto config_map = osquery::getTestConfigMap("example_config.conf"); +// Resolve the test config directory at runtime +auto configDir = osquery::getTestConfigDirectory(); -// Load fixture packs for scheduler/config unit tests -osquery::JSON unrestricted = osquery::getUnrestrictedPack(); -osquery::JSON restricted = osquery::getRestrictedPack(); -osquery::JSON discovered = osquery::getPackWithValidDiscovery(); -``` +// Load a config file into a source-to-content map +auto configMap = osquery::getTestConfigMap("test.conf"); + +// Retrieve pre-built pack fixtures for unit tests +osquery::JSON unrestrictedPack = osquery::getUnrestrictedPack(); +osquery::JSON restrictedPack = osquery::getRestrictedPack(); +osquery::JSON discoveryPack = osquery::getPackWithDiscovery(); -## Notes +// Use a fake version pack to test version-gating logic +osquery::JSON fakePack = osquery::getPackWithFakeVersion(); +``` -- All functions are declared inside the `osquery` namespace. -- Path helpers return `const` references β€” do not store the result beyond test scope. -- Pack fixture functions return `JSON` objects (from `osquery/utils/json/json.h`) and are intended exclusively for use in unit/integration tests. \ No newline at end of file +> All symbols are declared within the `osquery` namespace. These helpers are intended exclusively for use in unit and integration tests β€” they should not be linked into production binaries. \ No newline at end of file diff --git a/osquery/core/.flags.md b/osquery/core/.flags.md index 3168ef7b20f..bd4f4f59970 100644 --- a/osquery/core/.flags.md +++ b/osquery/core/.flags.md @@ -1,55 +1,51 @@ -Declares osquery's flag management system, wrapping gflags to support categorized, queryable, and runtime-updatable configuration options across shell, CLI, extension, and hidden flag contexts. +Defines osquery's flag management system, wrapping Google's gflags library with additional metadata tracking for shell, extension, CLI, and hidden flag categories. ## Key Components ### Structs - **`FlagDetail`** β€” Metadata for a flag: description, and boolean markers (`shell`, `external`, `cli`, `hidden`) -- **`FlagInfo`** β€” Full flag snapshot including type, description, default value, current value, and its `FlagDetail` +- **`FlagInfo`** β€” Full flag snapshot including type, description, default value, current value, and a `FlagDetail` ### Class: `Flag` (singleton, non-copyable) | Method | Description | -|---|---| -| `create()` | Registers a new named flag with its detail metadata | -| `createAlias()` | Registers a gflags alias pointing to an existing flag | -| `flags()` | Returns a map of all registered flags and their info | -| `getValue()` / `getInt32Value()` | Retrieves flag value by name | -| `getDefaultValue()` | Retrieves the default value for a flag | -| `updateValue()` | Updates a flag's value at runtime | -| `isDefault()` | Checks if a flag still holds its default value | -| `getType()` / `getDescription()` | Introspection helpers | -| `isCLIOnlyFlag()` | Returns true if the flag is CLI/flagfile-only | -| `printFlags()` | Prints help output, filterable by shell/external/cli | -| `resetCustomFlags()` | Clears custom flags (used by fuzzers) | +|--------|-------------| +| `create()` / `createAlias()` | Register a new flag or alias | +| `instance()` | Access the singleton | +| `flags()` | Return all registered flags as a map | +| `getValue()` / `getInt32Value()` | Read flag value by name | +| `getDefaultValue()` | Retrieve the original default | +| `updateValue()` | Override a flag's value at runtime | +| `isDefault()` | Check if a flag is still at its default | +| `getType()` / `getDescription()` | Introspect flag metadata | +| `isCLIOnlyFlag()` | Check if a flag is restricted to CLI/flagfile | +| `printFlags()` | Print help output filtered by flag category | +| `resetCustomFlags()` | Clear custom flags (used by fuzz targets) | ### Macros -| Macro | Behavior | -|---|---| -| `FLAG(t,n,v,d)` | Standard flag β€” settable via flagfile, CLI, or config options | -| `SHELL_FLAG` | Only available in `osqueryi` | -| `EXTENSION_FLAG` | Only available to extensions | -| `CLI_FLAG` | Cannot be set via config `options` | -| `HIDDEN_FLAG` | Excluded from `--help` output | +| Macro | Description | +|-------|-------------| +| `FLAG(t,n,v,d)` | Standard flag, settable via CLI, flagfile, or config options | +| `SHELL_FLAG` | Only available in `osqueryi` (interactive shell) | +| `EXTENSION_FLAG` | Only available to osquery extensions | +| `CLI_FLAG` | CLI/flagfile only; cannot be set in config `options` | +| `HIDDEN_FLAG` | Registered but not shown in `--help` output | ## Usage Example ```cpp -// Declare flags in a .cpp file -FLAG(bool, enable_feature, false, "Enable the experimental feature"); -CLI_FLAG(string, config_path, "/etc/osquery/osquery.conf", "Path to config file"); -SHELL_FLAG(int32, query_timeout, 300, "Query timeout in seconds (osqueryi only)"); +// Declare a standard flag +FLAG(string, config_path, "/etc/osquery/osquery.conf", "Path to config file"); -// Read a flag value at runtime +// Declare a CLI-only flag +CLI_FLAG(bool, force, false, "Force operation regardless of state"); + +// Read flag values at runtime std::string path = Flag::getValue("config_path"); -// Update a flag value programmatically -Status s = Flag::updateValue("enable_feature", "true"); -if (!s.ok()) { - LOG(WARNING) << "Failed to update flag: " << s.getMessage(); -} +// Override a flag value programmatically +Status s = Flag::updateValue("config_path", "/custom/path.conf"); -// Check if a flag was explicitly set or still at default -if (Flag::isDefault("query_timeout")) { - LOG(INFO) << "Using default query timeout"; -} +// Check if still at default +if (Flag::isDefault("config_path")) { /* use default behavior */ } ``` \ No newline at end of file diff --git a/osquery/core/.query.md b/osquery/core/.query.md index 5cfec8773ce..65ef250ea91 100644 --- a/osquery/core/.query.md +++ b/osquery/core/.query.md @@ -3,58 +3,71 @@ Defines the `Query` class and `QueryLogItem` struct for managing scheduled query ## Key Components -### `QueryLogItem` struct +### `QueryLogItem` (struct) +Holds metadata and results for a single query execution log entry. + | Field | Type | Description | -|---|---|---| -| `isSnapshot` | `bool` | Flags snapshot vs. differential results | -| `results` | `DiffResults` | Differential query results | -| `snapshot_results` | `QueryDataTyped` | Full snapshot results | -| `name` / `identifier` | `std::string` | Query name and host identifier | -| `epoch` / `counter` | `uint64_t` | Execution epoch and invocation counter | -| `decorations` | `map` | Additional log line metadata | +|-------|------|-------------| +| `isSnapshot` | `bool` | Snapshot vs. differential mode | +| `results` | `DiffResults` | Differential row changes | +| `snapshot_results` | `QueryDataTyped` | Full snapshot rows | +| `name` | `string` | Scheduled query name | +| `identifier` | `string` | Host identifier (hostname/UUID) | +| `time` / `epoch` / `counter` | `uint64_t` | Execution timing and counters | +| `decorations` | `map` | Extra metadata fields | ### Serialization Functions -- `serializeQueryLogItem()` β€” Serializes a `QueryLogItem` into a JSON document -- `serializeQueryLogItemJSON()` β€” Serializes a `QueryLogItem` into a JSON string -- `serializeQueryLogItemAsEvents()` β€” Serializes as an event-list JSON document -- `serializeQueryLogItemAsEventsJSON()` β€” Serializes as a vector of event JSON strings +| Function | Output | +|----------|--------| +| `serializeQueryLogItem()` | JSON document object | +| `serializeQueryLogItemJSON()` | JSON string | +| `serializeQueryLogItemAsEvents()` | JSON document (event list) | +| `serializeQueryLogItemAsEventsJSON()` | Vector of JSON strings | + +### `Query` (class) +Manages persistent storage and diffing of scheduled query results against a RocksDB backend. -### `Query` class | Method | Description | -|---|---| -| `getPreviousQueryResults()` | Loads prior results from RocksDB into a `QueryDataSet` | -| `saveQueryResults()` | Persists updated results JSON and epoch to the database | -| `addNewResults()` | Stores new results and computes differential vs. last run | -| `addNewEvents()` | Variant of `addNewResults` for event-based queries | -| `getQueryCounter()` | Returns/increments the invocation counter, handling resets | -| `isNewQuerySql()` | Detects if the query SQL has changed since last run | -| `getStoredQueryNames()` | Static; returns all query names persisted in RocksDB | +|--------|-------------| +| `getPreviousQueryResults()` | Loads prior results as a `QueryDataSet` | +| `saveQueryResults()` | Persists new results JSON with epoch | +| `getPreviousEpoch()` | Returns stored epoch value | +| `getQueryCounter()` | Returns/increments invocation counter | +| `isQueryNameInDatabase()` | Checks if query has prior history | +| `isNewQuerySql()` | Detects SQL changes since last run | +| `addNewResults()` | Stores results and computes diff | +| `addNewEvents()` | Variant for event-based queries | +| `getStoredQueryNames()` | *(static)* Lists all stored query names | ## Usage Example -```c -// Initialize and run a scheduled query, capturing differential results -ScheduledQuery sq; -sq.query = "SELECT * FROM processes;"; +```cpp +#include -Query q("process_monitor", sq); +// Construct a Query object for a named scheduled query +osquery::ScheduledQuery sq; +sq.query = "SELECT * FROM processes"; +osquery::Query q("my_query", sq); -uint64_t counter = 0; -DiffResults diff; -QueryDataTyped results = executeQuery(sq.query); +// Check if this is a known query, then compute differential results +if (q.isQueryNameInDatabase() && !q.isNewQuerySql()) { + osquery::DiffResults diff; + uint64_t counter = 0; + osquery::QueryDataTyped results = fetchCurrentResults(); -Status status = q.addNewResults(std::move(results), current_epoch, counter, diff); + auto status = q.addNewResults( + std::move(results), current_epoch, counter, diff + ); -if (status.ok()) { - QueryLogItem item; - item.name = "process_monitor"; - item.results = diff; - item.epoch = current_epoch; - item.counter = counter; - item.isSnapshot = false; + if (status.ok()) { + osquery::QueryLogItem item; + item.name = "my_query"; + item.results = diff; + item.epoch = current_epoch; + item.counter = counter; - std::string json; - serializeQueryLogItemJSON(item, json); - logger.log(json); + std::string json_out; + osquery::serializeQueryLogItemJSON(item, json_out); + } } ``` \ No newline at end of file diff --git a/osquery/core/.shutdown.md b/osquery/core/.shutdown.md index 6480b232439..f1b2011f5ef 100644 --- a/osquery/core/.shutdown.md +++ b/osquery/core/.shutdown.md @@ -1,47 +1,49 @@ -Declares the osquery shutdown coordination API, providing functions to request, wait for, and inspect application shutdown state across daemon services and threads. +Declares the `osquery` shutdown coordination API, providing functions to request, monitor, and wait for application shutdown across multi-threaded osquery components. ## Key Components | Function | Description | |---|---| -| `requestShutdown(int, string)` | Primary shutdown trigger; signals all dispatched services to stop gracefully | -| `waitForShutdown()` | Blocks the calling thread until shutdown is requested | -| `waitTimeoutOrShutdown(milliseconds)` | Interruptible sleep β€” returns early if shutdown is requested | -| `shutdownRequested()` | Non-blocking check for shutdown state (shell/tool use only) | -| `getShutdownExitCode()` | Retrieves the stored exit code set during shutdown request | -| `setShutdownExitCode(int)` | Directly sets the exit code without triggering shutdown | -| `resetShutdown()` | Resets shutdown state to `false`; intended for internal/testing use only | +| `requestShutdown(retcode, system_log)` | Primary shutdown trigger; signals all dispatched services to stop gracefully | +| `waitForShutdown()` | Blocks the calling thread until a shutdown is requested | +| `waitTimeoutOrShutdown(timeout)` | Interruptible sleep β€” returns early if shutdown is requested | +| `shutdownRequested()` | Non-blocking check for shutdown state (intended for shell tools) | +| `getShutdownExitCode()` / `setShutdownExitCode()` | Get/set the process exit code | +| `resetShutdown()` | Resets shutdown state β€” for internal use and testing only | ## Usage Example ```cpp #include -// In main thread β€” block until shutdown is signaled -void runDaemon() { - waitForShutdown(); - // Proceed with cleanup after shutdown is requested - Initializer::shutdown(getShutdownExitCode()); +// In a signal handler or failure path: +void onFatalError() { + osquery::requestShutdown(EXIT_FAILURE, "Unrecoverable config error"); } -// In a retry loop β€” sleep interruptibly -void enrollWithRetry() { - while (!shutdownRequested()) { - bool done = attemptEnrollment(); - if (done) break; +// In main(), block until shutdown is signaled: +int main() { + // ... initialize services ... - // Wait 30s or abort early if shutdown is requested - if (waitTimeoutOrShutdown(std::chrono::milliseconds(30000))) { - break; // Shutdown was requested - } - } + osquery::waitForShutdown(); + + // ... teardown ... + return osquery::getShutdownExitCode(); } -// Request shutdown from an error handler -void onFatalError(const std::string& reason) { - requestShutdown(EXIT_FAILURE, reason); +// In a retry loop, sleep interruptibly: +bool enrollWithRetry() { + while (true) { + if (tryEnroll()) return true; + + // Wait 30s, but exit early if shutdown is requested + bool shutdown = osquery::waitTimeoutOrShutdown( + std::chrono::milliseconds(30000) + ); + if (shutdown) return false; + } } ``` -> **Note:** Prefer `requestShutdown()` over direct `exit()` calls to ensure all event threads and Thrift service pools are properly notified before process termination. \ No newline at end of file +> **Note:** Prefer `requestShutdown()` over `stdlib exit()` in all osquery components to ensure graceful teardown of event threads and Thrift service pools. \ No newline at end of file diff --git a/osquery/core/.system.md b/osquery/core/.system.md index 50183a002ee..130850fe1f2 100644 --- a/osquery/core/.system.md +++ b/osquery/core/.system.md @@ -1,52 +1,63 @@ -Defines the `Initializer` class and supporting utilities for bootstrapping osquery process lifecycle, identity, and platform setup within the `osquery` namespace. +Provides the core initialization, lifecycle management, and system identity utilities for osquery processes, including daemon/worker/watcher setup and UUID generation. ## Key Components ### `Initializer` Class -The primary entry point for osquery process initialization. Non-copyable, intended for use in `main()`. +Non-copyable RAII class that bootstraps osquery process state. Must be instantiated in `main()`. | Method | Description | -|---|---| -| `Initializer(argc, argv, tool, init_glog)` | Constructor; configures flags, logging, and tool type | -| `initDaemon()` | Sets up process as an osquery daemon | -| `initShell()` | Sets up process as an interactive shell | -| `initWorkerWatcher(name)` | Spawns and monitors child worker processes | -| `shutdown(retcode)` | Cleanly stops all services and returns exit code | -| `requestShutdown(retcode)` | Static; requests graceful shutdown | -| `shutdownNow(retcode)` | Static; immediate exit without dispatcher teardown | -| `isWorker()` / `isWatcher()` | Static; detects process role via environment | +|--------|-------------| +| `Initializer(argc, argv, tool, init_glog)` | Parses args, sets up logging and platform state | +| `initDaemon()` | Configures process as a full osquery daemon | +| `initShell()` | Configures process as a lightweight interactive shell | +| `initWorkerWatcher(name)` | Spawns and monitors child worker processes with CPU/memory bounds | +| `start()` | Begins active work after initialization | +| `shutdown(retcode)` | Stops all services and returns appropriate exit code | +| `requestShutdown(retcode)` | Static signal to trigger graceful shutdown | +| `shutdownNow(retcode)` | Immediate exit bypassing dispatcher teardown | +| `isWorker()` / `isWatcher()` | Detect process role via `OSQUERY_WORKER` env var | ### UUID & Identity Utilities + +| Function | Description | +|----------|-------------| +| `generateNewUUID()` | Returns a new random UUID string | +| `getInstanceUUID(ident)` | Persistent per-instance UUID | +| `getEphemeralUUID(ident)` | Session-scoped UUID | +| `getHostUUID(ident)` | Hardware-derived host UUID | +| `isPlaceholderHardwareUUID(uuid)` | Detects known invalid/placeholder UUIDs | +| `getHostIdentifier()` | Returns the configured unique machine identifier | + +### System Helpers + | Function | Description | -|---|---| -| `generateNewUUID()` | Generates a random UUID string | -| `getInstanceUUID(ident)` | Retrieves persistent instance UUID | -| `getEphemeralUUID(ident)` | Retrieves session-scoped UUID | -| `getHostUUID(ident)` | Retrieves hardware-based host UUID | -| `getHostIdentifier()` | Returns configured unique machine identifier | -| `isPlaceholderHardwareUUID(uuid)` | Detects invalid/placeholder UUIDs | - -### Platform & Process Utilities -- `platformSetup()` / `platformTeardown()` β€” COM/platform init on Windows -- `isUserAdmin()` β€” Checks admin/root privileges -- `setThreadName(name)` β€” Names the current thread -- `getStartTime()` / `setStartTime(st)` β€” Tracks tool start time +|----------|-------------| +| `isUserAdmin()` | Checks if the process runs with admin privileges | +| `setThreadName(name)` | Sets the current thread's name | +| `getStartTime()` / `setStartTime(st)` | Manages tool start timestamp | +| `platformSetup()` / `platformTeardown()` | Windows COM and platform-specific init/teardown | +| `checkPlatform(platform)` | Validates a platform name string | ## Usage Example -```cpp +```c #include int main(int argc, char* argv[]) { + // Bootstrap osquery as a daemon process osquery::Initializer runner(argc, argv, osquery::ToolType::DAEMON); - runner.initDaemon(); + + // Spawn and monitor worker processes runner.initWorkerWatcher("osqueryd"); - // Only worker processes return here + // Workers return here; begin processing runner.start(); + // Retrieve host identity + std::string hostId = osquery::getHostIdentifier(); + return runner.shutdown(EXIT_SUCCESS); } ``` \ No newline at end of file diff --git a/osquery/core/.tables.md b/osquery/core/.tables.md index 7dabd713bd7..ba2f06d21d7 100644 --- a/osquery/core/.tables.md +++ b/osquery/core/.tables.md @@ -1,43 +1,38 @@ -Defines the core data structures and type system for osquery's virtual table infrastructure, including SQLite type affinity macros, constraint handling, and table metadata used when implementing and querying osquery tables. +Defines the core data structures and type system for osquery's virtual table infrastructure, including SQLite type affinity macros, constraint operators, and the `ConstraintList` used to filter table query results. ## Key Components -### Type Affinity Macros -- `SQL_TEXT(x)`, `INTEGER(x)`, `BIGINT(x)`, `UNSIGNED_BIGINT(x)`, `DOUBLE(x)` β€” convert native C++ values to SQLite string representations via `__sqliteField()` -- `TEXT_LITERAL`, `INTEGER_LITERAL`, `BIGINT_LITERAL`, `UNSIGNED_BIGINT_LITERAL`, `DOUBLE_LITERAL` β€” C++ type aliases for table column literals - -### Enumerations -- `ConstraintOperator` β€” SQL predicate operators (`EQUALS`, `LESS_THAN`, `GLOB`, `REGEXP`, `IN_OP`, etc.) -- `TableAttributes` β€” bitflag enum describing table behavior (`CACHEABLE`, `EVENT_BASED`, `USER_BASED`, `PENDING`, etc.) - -### Structs & Classes -- `Constraint` β€” pairs a `ConstraintOperator` with a string expression -- `ConstraintList` β€” ordered set of `Constraint` objects for a single column; supports `matches()`, `exists()`, `existsAndMatches()`, `notExistsOrMatches()`, and `getAll()` - -### Type Aliases -- `TableColumns` β€” ordered vector of `(name, ColumnType, ColumnOptions)` tuples -- `ConstraintMap` β€” maps column names to their `ConstraintList` -- `ConstraintSet` β€” parsed predicate pairs used to populate a `ConstraintMap` -- `UsedColumns` / `UsedColumnsBitset` β€” tracks which columns are referenced in a query +- **`__sqliteField` / Type Macros** β€” Overloaded template functions and macros (`SQL_TEXT`, `INTEGER`, `BIGINT`, `DOUBLE`, etc.) that convert native C++ types to SQLite-compatible string representations +- **`ConstraintOperator`** β€” Enum defining SQL predicate operators (`EQUALS`, `GREATER_THAN`, `LIKE`, `GLOB`, `REGEXP`, etc.) +- **`Constraint`** β€” Struct pairing an operator with a string expression, representing a single WHERE clause condition +- **`TableAttributes`** β€” Flags enum describing table behavior (`CACHEABLE`, `EVENT_BASED`, `USER_BASED`, `PENDING`, etc.) +- **`ConstraintList`** β€” Core struct holding all constraints for a single column, with methods for matching, existence checks, and serialization +- **Type Aliases** β€” `TableColumns`, `ConstraintMap`, `ConstraintSet`, `UsedColumns`, `UsedColumnsBitset` for query context plumbing ## Usage Example -```cpp -// Check if a constraint exists and matches a value +```c +// Check if a query constrains a column to a specific value ConstraintList cl; -cl.add(Constraint(EQUALS, "root")); - -if (cl.existsAndMatches(std::string("root"))) { - // Filter results to user "root" +cl.affinity = TEXT_TYPE; +cl.add(Constraint(EQUALS, "chrome")); + +// In table generation logic: +if (cl.exists(EQUALS)) { + for (const auto& val : cl.getAll(EQUALS)) { + // val == "chrome" β€” use to filter rows + } } -// Get all EQUALS expressions for iteration -for (const auto& expr : cl.getAll(EQUALS)) { - generateRowsForUser(expr); +// Match a candidate value against all constraints +std::string candidate = "chrome"; +if (cl.matches(candidate)) { + // Row passes the predicate β€” include in results } -// Convert a native value to SQLite text representation -std::string val = INTEGER(42); // "42" -std::string txt = SQL_TEXT("hi"); // "hi" +// Convert native types to SQLite text fields +std::string row_pid = INTEGER(1234); +std::string row_name = SQL_TEXT("osqueryd"); +std::string row_bytes = BIGINT(9876543210LL); ``` \ No newline at end of file diff --git a/osquery/core/.watcher.md b/osquery/core/.watcher.md index 93cd774f62e..1e69fab1d2c 100644 --- a/osquery/core/.watcher.md +++ b/osquery/core/.watcher.md @@ -1,41 +1,40 @@ -Defines the watchdog infrastructure for osquery, managing the lifecycle and performance monitoring of worker processes and autoloaded extension processes. +Defines the watchdog subsystem for osquery, providing thread-safe process supervision of worker daemons and autoloaded extensions with configurable performance enforcement. ## Key Components ### Enums & Structs - **`WatchdogLimitType`** β€” Categorizes performance limits: `MEMORY_LIMIT`, `UTILIZATION_LIMIT`, `RESPAWN_LIMIT`, `RESPAWN_DELAY`, `LATENCY_LIMIT`, `INTERVAL` -- **`PerformanceState`** β€” Snapshot struct tracking CPU time, memory footprint, respawn timestamps, and sustained latency counters for a monitored process +- **`PerformanceState`** β€” Snapshot struct tracking CPU times, respawn timestamps, and sustained latency counters for a monitored process ### Classes -- **`Watcher`** β€” Thread-safe, noncopyable singleton managing the worker and extension process map. Tracks performance states, restart counts, and process handles. Only accessible by `WatcherRunner` -- **`WatcherRunner`** β€” `InternalRunnable` thread that spawns, monitors, and respawns worker/extension processes. Enforces resource limits and handles clean/forced shutdowns -- **`WatcherWatcherRunner`** β€” A reverse-watchdog spawned inside the worker process; monitors the parent watcher process and exits if it disappears +- **`Watcher`** β€” Thread-safe (noncopyable) singleton-style manager holding references to the worker `PlatformProcess` and all extension processes, along with their `PerformanceState` histories. Exposes restart counters and fate-binding logic. Only `WatcherRunner` has private access. +- **`WatcherRunner`** β€” `InternalRunnable` thread that spawns, polls, and respawns worker/extension processes. Kills children that exceed resource limits, signals clean shutdown via `stopChild()`, and delegates health checks to `isChildSane()` / `isWatcherHealthy()`. +- **`WatcherWatcherRunner`** β€” Lightweight `InternalRunnable` spawned *inside* the worker process to monitor the parent watchdog; exits the worker if the watcher disappears. ### Free Functions -- **`getWorkerLimit(WatchdogLimitType)`** β€” Returns a numeric performance threshold for a given limit category and configured watchdog level +- **`getWorkerLimit(WatchdogLimitType)`** β€” Returns a numeric threshold for a given limit category, respecting the `--watchdog_level` flag ### Type Aliases -- **`ExtensionMap`** β€” `std::map>` mapping extension paths to their process handles +- **`ExtensionMap`** β€” `std::map>` mapping extension paths to their processes ## Usage Example ```cpp -// Construct and bind a Watcher, then launch the WatcherRunner thread +// Instantiate the watcher and launch the runner thread auto watcher = std::make_shared(); watcher->loadExtensions(); auto runner = std::make_shared( argc, argv, /*use_worker=*/true, watcher); -// Query a specific performance limit +osquery::Dispatcher::addService(runner); + +// Query a resource threshold at the configured watchdog level uint64_t memLimit = osquery::getWorkerLimit( osquery::WatchdogLimitType::MEMORY_LIMIT); - -// Bind watcher/worker fates (watcher exit kills worker) -watcher->bindFates(); ``` \ No newline at end of file diff --git a/osquery/core/plugins/.logger.md b/osquery/core/plugins/.logger.md index 201e0d11239..4f900546584 100644 --- a/osquery/core/plugins/.logger.md +++ b/osquery/core/plugins/.logger.md @@ -1,28 +1,26 @@ -Defines the pluggable logging interface for osquery, providing the base class and supporting types for implementing custom log destinations (e.g., Flume, Splunk, syslog). +Pluggable logger plugin interface for osquery, defining the base class and supporting types for integrating custom logging backends (e.g., Splunk, syslog, Flume). ## Key Components -### Enumerations -- **`LoggerFeatures`** β€” Opt-in feature flags: `LOGGER_FEATURE_BLANK`, `LOGGER_FEATURE_LOGSTATUS`, `LOGGER_FEATURE_LOGEVENT` -- **`StatusLogSeverity`** β€” Severity levels mirroring Glog: `O_INFO`, `O_WARNING`, `O_ERROR`, `O_FATAL` - -### Structs -- **`StatusLogLine`** β€” Intermediate log entry holding severity, filename, line number, message, timestamp, UNIX time, and host identifier - -### Classes -- **`LoggerPlugin`** β€” Abstract base class (extends `Plugin`) for all logger implementations - -### `LoggerPlugin` Virtual Methods - -| Method | Required | Description | -|---|---|---| -| `logString(s)` | βœ… Pure virtual | Core log output handler | -| `init(binary_name, log)` | βœ… Pure virtual | Called once at startup with buffered pre-init logs | -| `logStatus(log)` | Optional | Handles Glog status lines | -| `logSnapshot(s)` | Optional | Handles snapshot query results; defaults to `logString` | -| `logEvent(s)` | Optional | Handles individual forwarded events | -| `logStringBatch(batch)` | Optional | Handles a JSON array batch of events | +**Enumerations** +- `LoggerFeatures` β€” Opt-in capability flags (`LOGGER_FEATURE_LOGSTATUS`, `LOGGER_FEATURE_LOGEVENT`) for advanced logger behaviors +- `StatusLogSeverity` β€” Severity levels (`O_INFO`, `O_WARNING`, `O_ERROR`, `O_FATAL`) mirroring Glog's `LogSeverity` + +**Structs** +- `StatusLogLine` β€” Represents a single buffered status log entry, including severity, filename, line number, message, timestamp, and host identifier + +**Class: `LoggerPlugin`** (extends `Plugin`) +| Method | Description | +|---|---| +| `logString(s)` | **Pure virtual.** Core method subclasses must implement to write a log string | +| `init(binary_name, log)` | **Pure virtual.** Called once at startup with buffered pre-init status logs | +| `logStatus(log)` | Optional. Handles Glog status lines directly | +| `logSnapshot(s)` | Optional. Handles snapshot query results; defaults to `logString` | +| `logEvent(s)` | Optional. Handles individual forwarded events | +| `logStringBatch(event_batch)` | Optional. Parses a JSON array and delegates each entry to `logString` | +| `usesLogStatus()` | Returns `true` to take exclusive ownership of Glog status output | +| `usesLogEvent()` | Returns `true` to receive events directly, bypassing the database | ## Usage Example @@ -32,26 +30,19 @@ Defines the pluggable logging interface for osquery, providing the base class an class MyLoggerPlugin : public osquery::LoggerPlugin { public: osquery::Status logString(const std::string& s) override { - writeToMyBackend(s); - return osquery::Status::success(); - } - - bool usesLogStatus() override { return true; } - - osquery::Status logStatus( - const std::vector& log) override { - for (const auto& line : log) { - writeStatusToMyBackend(line.severity, line.message); - } + sendToMyBackend(s); return osquery::Status::success(); } protected: void init(const std::string& binary_name, const std::vector& log) override { - setProcessName(binary_name); - logStatus(log); // flush buffered pre-init logs + for (const auto& entry : log) { + sendToMyBackend(entry.message); + } } + + bool usesLogStatus() override { return true; } }; REGISTER(MyLoggerPlugin, "logger", "my_logger"); diff --git a/osquery/core/plugins/.plugin.md b/osquery/core/plugins/.plugin.md index 57a9afb5479..3fa7d11643e 100644 --- a/osquery/core/plugins/.plugin.md +++ b/osquery/core/plugins/.plugin.md @@ -1,57 +1,57 @@ -Defines the core `Plugin` base class and associated types for the osquery plugin/registry system, providing the interface that all registry items must implement. +Defines the core `Plugin` base class and supporting types for osquery's registry/plugin system, providing the interface contract all plugins must implement. ## Key Components -| Symbol | Type | Description | -|--------|------|-------------| -| `PluginRequest` | `std::map` | Key-value input map passed to a plugin call, typically includes an `"action"` key | -| `PluginResponse` | `std::vector` | Vector of key-value maps returned from a successful plugin call | -| `Plugin` | Abstract class | Non-copyable base class all plugins must inherit; requires `call()` implementation | -| `PluginRef` | `std::shared_ptr` | Shared pointer alias for registry-managed plugin instances | -| `tableRowsToPluginResponse()` | Free function | Converts a `TableRows` result into a `PluginResponse` | - -### `Plugin` Virtual Methods - -| Method | Required | Description | -|--------|----------|-------------| -| `call()` | βœ… Pure virtual | Handles plugin invocation with request/response | -| `setUp()` | ❌ Optional | One-time initialization logic | -| `tearDown()` | ❌ Optional | Cleanup/release logic | -| `configure()` | ❌ Optional | React to configuration changes | -| `routeInfo()` | ❌ Optional | Publish additional routing metadata | -| `addExternal()` | ❌ Static | Hook for when an external plugin is registered | -| `removeExternal()` | ❌ Static | Hook for when an external plugin is removed | +**Type Aliases** +- `PluginRequest` β€” `std::map` representing input to a plugin call (typically includes an `"action"` key) +- `PluginResponse` β€” `std::vector` representing output from a plugin call +- `PluginRef` β€” `std::shared_ptr` convenience alias -## Usage Example +**`Plugin` Class (abstract, non-copyable)** -```cpp -#include +| Method | Description | +|--------|-------------| +| `call()` | **Pure virtual** β€” must be implemented; handles request/response dispatch | +| `setUp()` | Optional initialization hook | +| `tearDown()` | Optional cleanup hook | +| `configure()` | Called on configuration updates | +| `routeInfo()` | Returns plugin routing metadata | +| `setName()` / `getName()` | Manages the plugin's registry name (final) | +| `addExternal()` | Static hook when an external plugin is registered | +| `removeExternal()` | Static hook when an external plugin is removed | -namespace osquery { +**Free Function** +- `tableRowsToPluginResponse()` β€” Converts a `TableRows` object into a `PluginResponse` -class MyPlugin : public Plugin { +## Usage Example + +```cpp +class MyPlugin : public osquery::Plugin { public: - Status call(const PluginRequest& request, - PluginResponse& response) override { + osquery::Status call(const osquery::PluginRequest& request, + osquery::PluginResponse& response) override { auto action = request.find("action"); if (action == request.end()) { - return Status::failure("Missing action key"); + return osquery::Status::failure("No action specified"); } if (action->second == "query") { - response.push_back({{"result", "hello from MyPlugin"}}); - return Status::success(); + response.push_back({{"result", "value"}}); + return osquery::Status::success(); } - return Status::failure("Unknown action: " + action->second); + return osquery::Status::failure("Unknown action: " + action->second); } - Status setUp() override { - // Optional: initialize resources - return Status::success(); + osquery::Status setUp() override { + // Optional one-time initialization + return osquery::Status::success(); } }; -} // namespace osquery +// Using via registry +osquery::PluginRequest req = {{"action", "query"}}; +osquery::PluginResponse resp; +auto status = myPlugin->call(req, resp); ``` \ No newline at end of file diff --git a/osquery/core/plugins/.sql.md b/osquery/core/plugins/.sql.md index c0e5554f802..87e7465b163 100644 --- a/osquery/core/plugins/.sql.md +++ b/osquery/core/plugins/.sql.md @@ -1,49 +1,43 @@ -Defines the abstract `SQLPlugin` interface that enables osquery's pluggable SQL execution layer, decoupling the SQL API from its underlying implementation (e.g., SQLite). +Defines the abstract `SQLPlugin` interface for osquery's pluggable SQL backend, allowing the SQL implementation (e.g., SQLite) to be decoupled from the core SDK via the osquery registry system. ## Key Components -- **`SQLPlugin`** β€” Abstract base class inheriting from `Plugin` that defines the contract for SQL backend implementations registered under the `"sql"` registry key. - -| Method | Description | -|---|---| -| `query()` | Executes a SQL query string and populates a `QueryData` result set | -| `getQueryColumns()` | Parses a query and returns column metadata (`name`, `type`) | -| `getQueryTables()` | Returns the list of virtual tables scanned by a given query | -| `attach()` | Optionally attaches a virtual table at runtime (no-op default) | -| `detach()` | Optionally detaches a virtual table by name (no-op default) | -| `call()` | Dispatches plugin registry requests to the appropriate method | +- **`SQLPlugin`** β€” Abstract plugin class inheriting from `Plugin` that defines the SQL implementation contract. Registered under the `"sql"` registry name. + - `query()` β€” Executes a SQL query string, returning results into a `QueryData` object with optional cache support. + - `getQueryColumns()` β€” Parses a query and returns column metadata (`TableColumns`) without executing it. + - `getQueryTables()` β€” Extracts the list of virtual tables referenced in a query. + - `attach()` β€” Optional hook to attach a virtual table at runtime (no-op default). + - `detach()` β€” Optional hook to detach a virtual table by name (no-op default). + - `call()` β€” Dispatches `PluginRequest` messages to the appropriate method (overrides `Plugin::call`). ## Usage Example ```cpp -#include - // Implementing a custom SQL backend plugin class MySQLPlugin : public osquery::SQLPlugin { public: osquery::Status query(const std::string& q, osquery::QueryData& results, bool use_cache) const override { - // Execute query against your SQL backend + // Execute query against your SQL engine return osquery::Status::success(); } osquery::Status getQueryColumns(const std::string& q, osquery::TableColumns& columns) const override { - // Parse and return column descriptors + // Parse and return column metadata return osquery::Status::success(); } osquery::Status getQueryTables(const std::string& q, std::vector& tables) const override { - // Return scanned table names + // Extract referenced table names return osquery::Status::success(); } }; -// Register the plugin under the "sql" registry REGISTER(MySQLPlugin, "sql", "sql"); ``` -> **Note:** The default `attach()` and `detach()` implementations return `Status::success()` β€” override them only if your backend requires runtime virtual table management (as SQLite does during initialization). \ No newline at end of file +> The default `attach()`/`detach()` implementations return `Status::success()` and can be overridden when the backend requires explicit virtual table lifecycle management (as SQLite does). \ No newline at end of file diff --git a/osquery/core/sql/.column.md b/osquery/core/sql/.column.md index 997711a199b..1bebaa37f41 100644 --- a/osquery/core/sql/.column.md +++ b/osquery/core/sql/.column.md @@ -1,45 +1,48 @@ -Defines column metadata types and enumerations used to describe osquery table schema β€” column options, column types, and related type aliases for table definitions. +Defines column metadata types used throughout osquery's table plugin system, providing the building blocks for describing table schema, query optimization hints, and column behavior. ## Key Components ### `ColumnOptions` (enum class) -Bitmask flags controlling column behavior in SQLite table implementations: +Bitmask flags controlling column behavior: -| Flag | Value | Description | -|------|-------|-------------| +| Flag | Value | Purpose | +|------|-------|---------| | `DEFAULT` | 0 | No special behavior | | `INDEX` | 1 | Treat as primary key | | `REQUIRED` | 2 | Must appear in query predicate | -| `ADDITIONAL` | 4 | Generates extra data when in predicate | -| `OPTIMIZED` | 8 | Enables query-time optimization | +| `ADDITIONAL` | 4 | Unlocks additional result context when in predicate | +| `OPTIMIZED` | 8 | Enables query path optimization when in predicate | | `HIDDEN` | 16 | Excluded from `SELECT *` results | -| `COLLATEBINARY` / `COLLATE*` | 32–2048 | SQLite collation sequences | +| `COLLATEBINARY/NOCASE/RTRIM/VERSION*` | 32–2048 | Custom SQLite collation sequences | -Bitwise `|` and `&` operators are overloaded for flag composition. +**Operators:** `|` and `&` are overloaded to support bitmask composition. ### `ColumnType` (enum) -Supported SQLite column data types: `TEXT_TYPE`, `INTEGER_TYPE`, `BIGINT_TYPE`, `UNSIGNED_BIGINT_TYPE`, `DOUBLE_TYPE`, `BLOB_TYPE`, `UNKNOWN_TYPE`. +SQLite-compatible column data types: `TEXT_TYPE`, `INTEGER_TYPE`, `BIGINT_TYPE`, `UNSIGNED_BIGINT_TYPE`, `DOUBLE_TYPE`, `BLOB_TYPE`, `UNKNOWN_TYPE`. ### Type Aliases -- `TableName` β€” `std::string` alias for plugin/table names -- `TableColumns` β€” ordered vector of `(name, ColumnType, ColumnOptions)` tuples describing a table's schema -- `ColumnAliasSet` β€” map of column name to a set of alias strings +- **`TableName`** β€” `std::string` alias for table plugin names +- **`TableColumns`** β€” Ordered vector of `(name, ColumnType, ColumnOptions)` tuples describing a table's schema +- **`ColumnAliasSet`** β€” Map of column names to their alias sets ### `kColumnTypeNames` -External map from `ColumnType` to its SQLite string representation (e.g., `TEXT`, `INTEGER`). +External map from `ColumnType` to its SQLite string representation (e.g., `TEXT_TYPE` β†’ `"TEXT"`). ## Usage Example -```cpp +```c #include #include "column.h" -osquery::TableColumns myTableColumns = { - // Column name, type, options - {"uid", osquery::INTEGER_TYPE, osquery::ColumnOptions::INDEX}, - {"name", osquery::TEXT_TYPE, osquery::ColumnOptions::DEFAULT}, - {"token", osquery::TEXT_TYPE, osquery::ColumnOptions::HIDDEN | - osquery::ColumnOptions::REQUIRED}, -}; +using namespace osquery; + +// Define schema for a table plugin +TableColumns MyTable::columns() const { + return { + {"uid", BIGINT_TYPE, ColumnOptions::INDEX}, + {"username", TEXT_TYPE, ColumnOptions::DEFAULT}, + {"domain", TEXT_TYPE, ColumnOptions::OPTIMIZED | ColumnOptions::HIDDEN}, + }; +} ``` \ No newline at end of file diff --git a/osquery/core/sql/.diff_results.md b/osquery/core/sql/.diff_results.md index fdf548bc014..a965d235f62 100644 --- a/osquery/core/sql/.diff_results.md +++ b/osquery/core/sql/.diff_results.md @@ -1,44 +1,46 @@ -Defines the `DiffResults` structure and related utilities for computing and serializing the difference between two query result sets in osquery. +Defines the `DiffResults` data structure and related utilities for computing and serializing the difference between two osquery query result sets. ## Key Components ### `DiffResults` (struct) -A move-only struct representing the delta between two `QueryDataTyped` result sets: +A move-only struct representing the delta between two `QueryDataTyped` result sets. | Member | Type | Description | -|---|---|---| +|--------|------|-------------| | `added` | `QueryDataTyped` | Rows present in the new result but not the old | | `removed` | `QueryDataTyped` | Rows present in the old result but not the new | - -**Methods:** -- `hasNoResults()` β€” Returns `true` if both `added` and `removed` are empty -- `operator==` / `operator!=` β€” Equality comparison between two `DiffResults` +| `hasNoResults()` | `bool` | Returns `true` if both `added` and `removed` are empty | +| `operator==` / `operator!=` | `bool` | Equality comparison between two `DiffResults` | ### Free Functions | Function | Description | -|---|---| -| `diff(old_, new_)` | Computes delta between a `QueryDataSet` and `QueryDataTyped`, returns a `DiffResults` | +|----------|-------------| +| `diff(old_, new_)` | Computes a `DiffResults` from a `QueryDataSet` (old) and `QueryDataTyped` (new) | | `serializeDiffResults(...)` | Serializes a `DiffResults` into a `rapidjson::Document` object | -| `serializeDiffResultsJSON(...)` | Serializes a `DiffResults` into a JSON string | +| `serializeDiffResultsJSON(...)` | Serializes a `DiffResults` into a JSON `std::string` | ## Usage Example ```cpp -// Compute diff between old and new query results -QueryDataSet oldResults = getStoredResults(); -QueryDataTyped newResults = runQuery(); +#include + +// Compute diff between two query result sets +QueryDataSet previousResults = getPreviousResults(); +QueryDataTyped currentResults = getCurrentResults(); -DiffResults delta = osquery::diff(oldResults, newResults); +DiffResults delta = osquery::diff(previousResults, currentResults); if (!delta.hasNoResults()) { - std::string jsonOutput; - auto status = osquery::serializeDiffResultsJSON(delta, jsonOutput, false); + // Process added and removed rows + for (const auto& row : delta.added) { /* handle new rows */ } + for (const auto& row : delta.removed) { /* handle removed rows */ } - if (status.ok()) { - // jsonOutput contains {"added": [...], "removed": [...]} - processChanges(jsonOutput); - } + // Serialize to JSON string + std::string jsonOutput; + osquery::serializeDiffResultsJSON(delta, jsonOutput, /*asNumeric=*/false); } -``` \ No newline at end of file +``` + +> **Note:** `DiffResults` is move-only (`only_movable`). Use `std::move` when passing instances between scopes. \ No newline at end of file diff --git a/osquery/core/sql/.query_data.md b/osquery/core/sql/.query_data.md index 554cf1f10c3..f401525b2df 100644 --- a/osquery/core/sql/.query_data.md +++ b/osquery/core/sql/.query_data.md @@ -1,47 +1,48 @@ -Defines the core result set types and serialization utilities for osquery SQL query results, providing `QueryData`, `QueryDataTyped`, and `QueryDataSet` container types along with JSON serialization/deserialization functions. +Defines the core result set types and serialization interfaces for osquery SQL query results, providing `QueryData`, `QueryDataTyped`, and `QueryDataSet` container types along with JSON conversion utilities. ## Key Components ### Type Aliases -| Type | Definition | Purpose | -|---|---|---| -| `QueryData` | `std::vector` | Standard untyped SQL result set | -| `QueryDataTyped` | `std::vector` | Typed SQL result set | +| Type | Underlying Type | Purpose | +|------|----------------|---------| +| `QueryData` | `std::vector` | Standard untyped query result set | +| `QueryDataTyped` | `std::vector` | Typed query result set | | `QueryDataSet` | `std::multiset` | Multiset for fast row lookup | ### Serialization Functions -- **`serializeQueryData`** β€” Converts `QueryData` or `QueryDataTyped` into a RapidJSON array, with optional numeric value preservation -- **`serializeQueryDataJSON`** β€” Serializes query results directly to a `JSON` document or `std::string` +- **`serializeQueryData`** β€” Converts `QueryData` or `QueryDataTyped` into a RapidJSON array; typed variant supports numeric value preservation via `asNumeric` +- **`serializeQueryDataJSON`** β€” Overloaded convenience wrappers serializing directly to a `JSON` document or `std::string` ### Deserialization Functions -- **`deserializeQueryData`** β€” Overloaded for `QueryData`, `QueryDataTyped`, and `QueryDataSet` targets from a `rapidjson::Value` -- **`deserializeQueryDataJSON`** β€” Overloaded for `QueryData` and `QueryDataSet` targets from a JSON string or `JSON` document +- **`deserializeQueryData`** β€” Overloaded for `QueryData`, `QueryDataTyped`, and `QueryDataSet` from a `rapidjson::Value` +- **`deserializeQueryDataJSON`** β€” Overloaded for `QueryData` and `QueryDataSet` from a `JSON` doc or raw JSON string ### Utility -- **`addUniqueRowToQueryData`** β€” Appends a `RowTyped` to a `QueryDataTyped` only if it does not already exist (linear scan) +- **`addUniqueRowToQueryData`** β€” Appends a `RowTyped` to a `QueryDataTyped` only if not already present (linear scan) ## Usage Example ```cpp #include -// Serialize query results to JSON string -osquery::QueryData results = runMyQuery(); -std::string jsonOutput; -auto status = osquery::serializeQueryDataJSON(results, jsonOutput); -if (!status.ok()) { - LOG(ERROR) << "Serialization failed: " << status.getMessage(); +// Serialize query results to a JSON string +QueryData results = runMyQuery(); +std::string json; +Status s = serializeQueryDataJSON(results, json); +if (s.ok()) { + LOG(INFO) << "Results: " << json; } -// Deserialize back from JSON -osquery::QueryData restored; -status = osquery::deserializeQueryDataJSON(jsonOutput, restored); +// Round-trip: deserialize back to QueryData +QueryData restored; +Status ds = deserializeQueryDataJSON(json, restored); -// Add unique rows only -osquery::QueryDataTyped typedResults; -osquery::RowTyped row = buildRow(); -bool added = osquery::addUniqueRowToQueryData(typedResults, row); +// Append only unique typed rows +QueryDataTyped typed; +RowTyped row = {{"column", SQLiteValue{42LL}}}; +addUniqueRowToQueryData(typed, row); // added +addUniqueRowToQueryData(typed, row); // skipped (duplicate) ``` -> **Note:** `addUniqueRowToQueryData` performs a linear scan β€” avoid using it in performance-critical loops with large result sets. \ No newline at end of file +> **Note:** `addUniqueRowToQueryData` performs a linear scan β€” avoid calling it in tight loops over large result sets. \ No newline at end of file diff --git a/osquery/core/sql/.query_performance.md b/osquery/core/sql/.query_performance.md index 2441d260de2..6bff80f6201 100644 --- a/osquery/core/sql/.query_performance.md +++ b/osquery/core/sql/.query_performance.md @@ -1,28 +1,27 @@ -Defines the `QueryPerformance` struct within the `osquery` namespace for tracking runtime performance statistics of scheduled queries. +Defines the `QueryPerformance` struct within the `osquery` namespace for tracking execution statistics of scheduled queries, including timing, memory, and output metrics. ## Key Components ### `QueryPerformance` (struct) -Aggregates cumulative and per-execution metrics for a single query: +Aggregates per-query performance data across all executions: | Field | Type | Description | |---|---|---| | `executions` | `size_t` | Total number of times the query has run | -| `last_executed` | `uint64_t` | UNIX timestamp (seconds) of last successful execution | -| `wall_time` / `wall_time_ms` | `uint64_t` | Cumulative wall clock time (seconds / milliseconds) | +| `last_executed` | `uint64_t` | UNIX timestamp (seconds) of last successful run | +| `wall_time` / `wall_time_ms` | `uint64_t` | Cumulative wall clock time (seconds / ms) | | `last_wall_time_ms` | `uint64_t` | Wall time of the most recent execution (ms) | | `user_time` / `last_user_time` | `uint64_t` | Cumulative and latest user-space CPU time (ms) | -| `system_time` / `last_system_time` | `uint64_t` | Cumulative and latest kernel CPU time (ms) | -| `average_memory` | `uint64_t` | Average resident memory (bytes) after result collection | -| `last_memory` | `uint64_t` | Resident memory of the most recent execution (bytes) | -| `output_size` | `uint64_t` | Total bytes produced by the query | +| `system_time` / `last_system_time` | `uint64_t` | Cumulative and latest kernel-space CPU time (ms) | +| `average_memory` | `uint64_t` | Mean resident memory (bytes) post-result collection | +| `last_memory` | `uint64_t` | Resident memory (bytes) after the latest execution | +| `output_size` | `uint64_t` | Total bytes of query output | **Methods:** - - `QueryPerformance(const std::string& csv)` β€” Deserializes a `QueryPerformance` from a CSV string -- `toCSV() const` β€” Serializes the struct to a CSV string for persistence +- `toCSV()` β€” Serializes the struct to a CSV string (nodiscard) - `operator==` β€” Equality comparison between two instances ## Usage Example @@ -30,16 +29,20 @@ Aggregates cumulative and per-execution metrics for a single query: ```cpp #include "query_performance.h" -// Default-initialize and inspect after a query run +// Default-constructed (all zeros) osquery::QueryPerformance perf; -perf.executions++; -perf.last_wall_time_ms = 42; +perf.executions = 1; +perf.wall_time_ms = 42; +perf.last_memory = 204800; -// Persist to CSV -std::string serialized = perf.toCSV(); +// Serialize for persistent storage +std::string csv = perf.toCSV(); -// Restore from CSV -osquery::QueryPerformance restored(serialized); +// Deserialize from stored CSV +osquery::QueryPerformance restored(csv); -assert(perf == restored); +// Compare +if (perf == restored) { + // metrics match +} ``` \ No newline at end of file diff --git a/osquery/core/sql/.row.md b/osquery/core/sql/.row.md index e0e3d010bcc..b1a664982ba 100644 --- a/osquery/core/sql/.row.md +++ b/osquery/core/sql/.row.md @@ -1,57 +1,56 @@ -Defines the core data types and serialization utilities for representing single database query rows in osquery, supporting both string-based and typed (SQLite affinity) row formats. +Defines the core data structures and serialization utilities for representing individual database query rows in osquery, supporting both untyped string-based and SQLite type-affinity-aware typed variants. ## Key Components ### Type Aliases - -| Type | Definition | -|------|-----------| -| `RowData` | `std::string` β€” raw column value | -| `RowDataTyped` | `boost::variant` β€” typed column value | -| `Row` | `std::map` β€” untyped string row | -| `RowTyped` | `std::map` β€” typed variant row | -| `ColumnNames` | `std::vector` β€” ordered column name list | +| Alias | Underlying Type | Description | +|-------|----------------|-------------| +| `RowData` | `std::string` | Base column value type | +| `RowDataTyped` | `boost::variant` | Typed column value (maps to SQLite affinities) | +| `Row` | `std::map` | Untyped row as column-name β†’ string | +| `RowTyped` | `std::map` | Typed row as column-name β†’ variant | +| `ColumnNames` | `std::vector` | Ordered list of column names from a query | ### Serialization Functions - -- **`serializeRow`** β€” Serializes a `Row` or `RowTyped` into a `rapidjson::Value` object, with optional column ordering via `ColumnNames` -- **`serializeRowJSON`** β€” Serializes a `Row` or `RowTyped` directly to a JSON string; typed variant accepts an `asNumeric` flag to preserve numeric types +- **`serializeRow`** β€” Serializes a `Row` or `RowTyped` into a `rapidjson::Value` object within a managed `JSON` document +- **`serializeRowJSON`** β€” Serializes a `Row` or `RowTyped` directly to a JSON string; typed variant accepts `asNumeric` flag to preserve numeric types ### Deserialization Functions +- **`deserializeRow`** β€” Populates a `Row` or `RowTyped` from a `rapidjson::Value` object +- **`deserializeRowJSON`** β€” Populates a `Row` or `RowTyped` from a raw JSON string -- **`deserializeRow`** β€” Deserializes a `rapidjson::Value` object into a `Row` or `RowTyped` -- **`deserializeRowJSON`** β€” Deserializes a JSON string into a `Row` or `RowTyped` - -All functions return an osquery `Status` object indicating success or failure. +All functions return an osquery `Status` indicating success or failure. ## Usage Example ```c #include -#include +#include -// Serialize a Row to JSON string +// Build a simple untyped row osquery::Row r; -r["pid"] = "1234"; +r["pid"] = "1234"; r["name"] = "my_process"; -std::string json_output; -auto status = osquery::serializeRowJSON(r, json_output); +// Serialize to JSON string +std::string json; +auto status = osquery::serializeRowJSON(r, json); if (status.ok()) { - // json_output -> {"pid":"1234","name":"my_process"} + // json => {"pid":"1234","name":"my_process"} } -// Deserialize back from JSON string -osquery::Row restored; -osquery::deserializeRowJSON(json_output, restored); +// Deserialize back +osquery::Row r2; +osquery::deserializeRowJSON(json, r2); -// Using typed rows with numeric preservation -osquery::RowTyped typed_row; -typed_row["pid"] = (long long)1234; -typed_row["load"] = 0.75; +// Typed row β€” numeric values preserved when asNumeric=true +osquery::RowTyped rt; +rt["pid"] = (long long)1234; +rt["load"] = 0.75; +rt["name"] = std::string("my_process"); -std::string typed_json; -osquery::serializeRowJSON(typed_row, typed_json, /*asNumeric=*/true); -// typed_json -> {"pid":1234,"load":0.75} +std::string typedJson; +osquery::serializeRowJSON(rt, typedJson, /*asNumeric=*/true); +// typedJson => {"pid":1234,"load":0.75,"name":"my_process"} ``` \ No newline at end of file diff --git a/osquery/core/sql/.scheduled_query.md b/osquery/core/sql/.scheduled_query.md index 42d5c135e19..4439f3b1101 100644 --- a/osquery/core/sql/.scheduled_query.md +++ b/osquery/core/sql/.scheduled_query.md @@ -1,48 +1,46 @@ -Defines the `ScheduledQuery` struct used within osqueryd to represent all relevant parameters and metadata for a scheduled SQL query. +Defines the `ScheduledQuery` struct used within osqueryd to represent all relevant attributes of a scheduled SQL query, including execution interval, ownership, and runtime options. ## Key Components -### `ScheduledQuery` (struct) -A move-only data structure (inherits `only_movable`) containing: +### `ScheduledQuery` (struct, `osquery` namespace) +A move-only data structure (inherits `only_movable`) holding the configuration and state of a scheduled query. + +**Fields:** | Field | Type | Description | |---|---|---| -| `pack_name` | `std::string` | Name of the containing pack | -| `name` | `std::string` | Query identifier | -| `query` | `std::string` | Raw SQL query string | -| `oncall` | `std::string` | Query owner/team | +| `pack_name` | `string` | Name of the containing pack | +| `name` | `string` | Query identifier | +| `query` | `string` | The SQL query string | +| `oncall` | `string` | Owner/responsible team | | `interval` | `uint64_t` | Execution frequency in seconds | | `splayed_interval` | `uint64_t` | Temporary splayed interval value | | `startup_priority` | `uint64_t` | Startup execution priority (`UINT64_MAX` = disabled) | -| `denylisted` | `bool` | Whether query is currently denylisted | -| `options` | `std::map` | Key/value query option flags | +| `denylisted` | `bool` | Whether the query is denylisted | +| `options` | `map` | Runtime option flags (e.g. `snapshot`, `removed`) | **Methods:** -- `isSnapshotQuery()` β€” Returns `true` if the `"snapshot"` option is set -- `reportRemovedRows()` β€” Returns `true` if the `"removed"` option is set or absent (default behavior) -- `operator==` / `operator!=` β€” Equality based on `query` and `interval` fields + +- `isSnapshotQuery()` β€” Returns `true` if the `snapshot` option is set +- `reportRemovedRows()` β€” Returns `true` if the `removed` option is set or absent (defaults to reporting) +- `operator==` / `operator!=` β€” Equality based on `query` and `interval` only ## Usage Example ```cpp #include -// Construct with required fields -osquery::ScheduledQuery sq("my_pack", "active_users", "SELECT * FROM users;"); +// Construct with pack, name, and SQL +osquery::ScheduledQuery sq("my_pack", "running_procs", "SELECT * FROM processes;"); sq.interval = 60; sq.options["snapshot"] = false; sq.options["removed"] = true; -// Check query behavior -if (sq.isSnapshotQuery()) { - // handle snapshot mode -} - -if (sq.reportRemovedRows()) { - // include removed row diffs in results +if (!sq.isSnapshotQuery() && sq.reportRemovedRows()) { + // Differential query β€” schedule for execution } -// Move semantics only β€” copies are disabled +// Move semantics only; copy construction is deleted osquery::ScheduledQuery sq2 = std::move(sq); ``` \ No newline at end of file diff --git a/osquery/core/sql/.table_rows.md b/osquery/core/sql/.table_rows.md index 91963c102ad..96c827d7419 100644 --- a/osquery/core/sql/.table_rows.md +++ b/osquery/core/sql/.table_rows.md @@ -3,15 +3,13 @@ Defines the `TableRows` type alias and declares serialization/deserialization ut ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `TableRows` | Type alias | `std::vector` β€” an ordered collection of table row objects | -| `serializeTableRows` | Function | Serializes a `TableRows` into a `rapidjson::Document` JSON array | -| `serializeTableRowsJSON` | Function | Serializes a `TableRows` directly into a JSON string | -| `deserializeTableRows` | Function | Converts a `rapidjson::Value` JSON array back into a `TableRows` | -| `deserializeTableRowsJSON` | Function | Converts a raw JSON string back into a `TableRows` | +- **`TableRows`** β€” Type alias for `std::vector`, representing a collection of query result rows. +- **`serializeTableRows`** β€” Serializes a `TableRows` object into a `rapidjson::Document` JSON array. +- **`serializeTableRowsJSON`** β€” Serializes a `TableRows` object directly into a JSON-formatted `std::string`. +- **`deserializeTableRows`** β€” Deserializes a `rapidjson::Value` JSON array back into a `TableRows` object. +- **`deserializeTableRowsJSON`** β€” Deserializes a JSON string back into a `TableRows` object. -All functions return `osquery::Status` to indicate success or failure. +All functions return a `Status` indicating success or failure. ## Usage Example @@ -19,29 +17,19 @@ All functions return `osquery::Status` to indicate success or failure. #include "table_rows.h" // Serialize to JSON string -osquery::TableRows rows = getMyTableRows(); -std::string jsonOutput; - -osquery::Status s = osquery::serializeTableRowsJSON(rows, jsonOutput); -if (!s.ok()) { - LOG(ERROR) << "Serialization failed: " << s.getMessage(); +osquery::TableRows rows = getQueryResults(); +std::string json; +osquery::Status s = osquery::serializeTableRowsJSON(rows, json); +if (s.ok()) { + // use json string } // Deserialize from JSON string osquery::TableRows restored; -osquery::Status ds = osquery::deserializeTableRowsJSON(jsonOutput, restored); -if (!ds.ok()) { - LOG(ERROR) << "Deserialization failed: " << ds.getMessage(); -} +osquery::Status s2 = osquery::deserializeTableRowsJSON(json, restored); -// Serialize into an existing JSON document/array +// Serialize to JSON document array osquery::JSON doc; rapidjson::Document arr; -osquery::serializeTableRows(rows, doc, arr); -``` - -## Dependencies - -- `osquery/utils/json/json.h` β€” JSON document wrapper -- `query_data.h` β€” underlying query result types -- `table_row.h` β€” `TableRowHolder` definition \ No newline at end of file +osquery::Status s3 = osquery::serializeTableRows(rows, doc, arr); +``` \ No newline at end of file diff --git a/osquery/core/windows/.global_users_groups_cache.md b/osquery/core/windows/.global_users_groups_cache.md index 42bc31f30e6..b59ca910038 100644 --- a/osquery/core/windows/.global_users_groups_cache.md +++ b/osquery/core/windows/.global_users_groups_cache.md @@ -1,34 +1,45 @@ -Declares the `GlobalUsersGroupsCache` class, which provides synchronized global access to Windows user and group cache data initialized by background services. +Provides a thread-safe singleton interface for accessing globally shared Windows user and group caches, blocking until the caches are fully initialized before returning references. ## Key Components ### `GlobalUsersGroupsCache` (class) +A static-only class managing lazy-initialized, shared caches for Windows users and groups. -A singleton-style accessor class for Windows users and groups cache data, designed to block until cache initialization completes before returning references. +**Public Static Methods** + +| Method | Returns | Description | +|--------|---------|-------------| +| `getUsersCache()` | `const UsersCache&` | Blocks until the users cache is ready, then returns a const reference | +| `getGroupsCache()` | `const GroupsCache&` | Blocks until the groups cache is ready, then returns a const reference | + +**Private Members** | Member | Type | Description | |--------|------|-------------| -| `getUsersCache()` | `static const UsersCache&` | Blocks until the users cache is ready, then returns a const reference | -| `getGroupsCache()` | `static const GroupsCache&` | Blocks until the groups cache is ready, then returns a const reference | -| `global_users_cache_future_` | `std::shared_future` | Synchronization primitive tracking users cache initialization | -| `global_groups_cache_future_` | `std::shared_future` | Synchronization primitive tracking groups cache initialization | -| `global_users_cache_` | `std::shared_ptr` | Shared ownership pointer to the users cache instance | -| `global_groups_cache_` | `std::shared_ptr` | Shared ownership pointer to the groups cache instance | +| `global_users_cache_future_` | `std::shared_future` | Synchronization primitive for users cache initialization | +| `global_users_cache_` | `std::shared_ptr` | Shared pointer to the users cache instance | +| `global_groups_cache_future_` | `std::shared_future` | Synchronization primitive for groups cache initialization | +| `global_groups_cache_` | `std::shared_ptr` | Shared pointer to the groups cache instance | -**Friend declarations:** `Initializer`, `initUsersAndGroupsServices`, `deinitUsersAndGroupsServices` β€” only these can populate the private cache state. +**Friend Declarations** +- `Initializer` β€” manages startup lifecycle +- `initUsersAndGroupsServices(bool, bool)` β€” populates the caches +- `deinitUsersAndGroupsServices(bool, bool)` β€” cleans up the caches ## Usage Example ```cpp #include -// Blocks until the background service has finished populating the cache, -// then returns a safe const reference for querying. +// Blocks until initialization is complete, then accesses the cache const UsersCache& users = GlobalUsersGroupsCache::getUsersCache(); const GroupsCache& groups = GlobalUsersGroupsCache::getGroupsCache(); -// Use the caches for lookup operations (Windows only) +// Safe to use from multiple threads; futures handle synchronization +for (const auto& user : users) { + // process user entries +} ``` -> **Note:** Both accessors are blocking calls. They rely on `std::shared_future` to ensure the background initialization service has completed before returning. This header is Windows-specific and wraps `users_groups_cache.h`. \ No newline at end of file +> **Note:** This header is Windows-specific. Access is safe from multiple threads β€” calls to `getUsersCache()` and `getGroupsCache()` will block until `initUsersAndGroupsServices` signals completion via the underlying `shared_future`. \ No newline at end of file diff --git a/osquery/core/windows/.handle.md b/osquery/core/windows/.handle.md index d38911d1f47..7529dd06a39 100644 --- a/osquery/core/windows/.handle.md +++ b/osquery/core/windows/.handle.md @@ -1,41 +1,36 @@ -RAII-style wrapper class for Windows kernel object handles (`HANDLE`), providing safe open/close lifecycle management for directory and symbolic link objects. +RAII wrapper for Windows kernel object handles within the osquery framework, providing safe lifecycle management and typed access to Windows directory and symbolic link objects. ## Key Components -### `Handle` class - -| Member | Description | -|---|---| -| `openDirObj(directory)` | Opens a Windows object directory by name using `OPEN_DIRECTORY` access mask | -| `openSymLinkObj(symlink)` | Opens a Windows symbolic link by name using `SYMBOLIC_LINK_QUERY` access mask | -| `getAsHandle()` | Returns the underlying `HANDLE` value | -| `close()` | Explicitly releases the handle | -| `valid()` | Returns `true` if the handle is currently open and valid | -| `~Handle()` | Destructor automatically closes the handle on scope exit | +- **`Handle` class** β€” Encapsulates a Windows `HANDLE`, ensuring proper acquisition and release via constructor/destructor +- **`openDirObj()`** β€” Opens a Windows object directory by name using the `OPEN_DIRECTORY` access mask; returns a `Status` +- **`openSymLinkObj()`** β€” Opens a Windows symbolic link by name using the `SYMBOLIC_LINK_QUERY` access mask; returns a `Status` +- **`getAsHandle()`** β€” Returns the underlying raw `HANDLE` for use with Win32 APIs +- **`close()`** β€” Explicitly releases the handle before destruction +- **`valid()`** β€” Checks whether the handle is currently open and valid ## Usage Example -```c +```cpp #include "handle.h" osquery::Handle h; // Open a Windows object directory -osquery::Status s = h.openDirObj(L"\\BaseNamedObjects"); -if (!s.ok()) { - // handle error +auto status = h.openDirObj(L"\\BaseNamedObjects"); +if (!status.ok()) { + LOG(ERROR) << "Failed to open dir: " << status.getMessage(); + return; } +// Use the raw handle with Win32 APIs if needed HANDLE raw = h.getAsHandle(); -// Check validity before use +// Explicit close (also called automatically on destruction) if (h.valid()) { - // perform operations on raw handle + h.close(); } - -// Explicitly close, or destructor will close on scope exit -h.close(); ``` -> **Note:** This class is Windows-only and relies on `osquery/utils/system/system.h`. The RAII destructor ensures handles are not leaked even if exceptions occur, making it safer than manual `CloseHandle` calls. \ No newline at end of file +> **Note:** This class is Windows-only and depends on `osquery/utils/system/system.h`. The RAII pattern ensures handles are not leaked even when exceptions occur β€” prefer letting the destructor manage cleanup over calling `close()` manually. \ No newline at end of file diff --git a/osquery/core/windows/.wmi.md b/osquery/core/windows/.wmi.md index c5bf928f37f..ab38dfe76d8 100644 --- a/osquery/core/windows/.wmi.md +++ b/osquery/core/windows/.wmi.md @@ -1,28 +1,23 @@ -Windows Management Instrumentation (WMI) query abstraction layer for osquery, providing C++ wrapper classes to execute WMI queries and retrieve typed results on Windows systems. +Windows Management Instrumentation (WMI) abstraction layer for osquery, providing a C++ interface to execute WMI queries and retrieve typed results on Windows systems. ## Key Components ### `WmiMethodArgs` -Constructs and holds arguments for WMI method calls. Used alongside `WmiResultItem::ExecMethod`. -- `Put(name, value)` β€” Adds a typed argument to the call -- `GetArguments()` β€” Returns the underlying `WmiMethodArgsMap` +Helper class for constructing WMI method call arguments. Supports a templated `Put()` method to add named arguments and `GetArguments()` to retrieve the underlying `WmiMethodArgsMap`. ### `WmiResultItem` -Wraps a single `IWbemClassObject` result, providing typed getters for WMI properties: -- `GetBool`, `GetString`, `GetLong`, `GetUnsignedInt32`, `GetDateTime`, etc. -- `GetVectorOfStrings`, `GetVectorOfLongs` β€” For multi-value properties -- `PrintType(name)` β€” Debug helper to inspect property VARIANT types +Wraps a single `IWbemClassObject` result. Provides typed getters for extracting WMI property values: +- Scalar types: `GetBool`, `GetUChar`, `GetUnsignedShort`, `GetUnsignedInt32`, `GetLong`, `GetUnsignedLong`, `GetLongLong`, `GetUnsignedLongLong` +- String types: `GetString` (narrow and wide), `GetVectorOfStrings` +- Collections: `GetVectorOfLongs` +- Time: `GetDateTime` ### `WmiRequest` -Main entry point for executing WMI queries against a namespace. -- `CreateWmiRequest(query, nspace)` β€” Static factory; returns `Expected` -- `results()` β€” Returns `std::vector` from the query -- `getStatus()` β€” Retrieves execution status -- `ExecMethod(object, method, args, out)` β€” Executes a WMI method on a result object +Main query class. Created via the factory `CreateWmiRequest(query, namespace)` which returns an `Expected`. Executes WQL queries against a WMI namespace (default `ROOT\CIMV2`) and exposes results as `std::vector`. Also supports `ExecMethod()` for invoking WMI methods on result objects. ### `impl::WmiObjectDeleter` -Internal RAII deleter for COM `IUnknown` pointers, ensuring `Release()` is called on cleanup. +Internal RAII deleter for COM `IUnknown` pointers, used with `std::unique_ptr` to ensure proper `Release()` calls. ## Usage Example @@ -31,20 +26,21 @@ Internal RAII deleter for COM `IUnknown` pointers, ensuring `Release()` is calle // Execute a WMI query auto request = osquery::WmiRequest::CreateWmiRequest( - "SELECT * FROM Win32_OperatingSystem" -); + "SELECT Name, ProcessId FROM Win32_Process"); if (request) { - for (const auto& item : request->results()) { - std::string caption; - if (item.GetString("Caption", caption).ok()) { - std::cout << "OS: " << caption << std::endl; - } + for (const auto& item : request->results()) { + std::string name; + unsigned int pid = 0; + + item.GetString("Name", name); + item.GetUnsignedInt32("ProcessId", pid); - unsigned long long totalMemory = 0; - if (item.GetUnsignedLongLong("TotalVisibleMemorySize", totalMemory).ok()) { - std::cout << "RAM (KB): " << totalMemory << std::endl; + std::cout << name << " [" << pid << "]\n"; } - } } + +// Execute a WMI method with arguments +osquery::WmiMethodArgs args; +args.Put("CommandLine", std::string("notepad.exe")); ``` \ No newline at end of file diff --git a/osquery/database/.database.md b/osquery/database/.database.md index e9e76edce13..e3e95777a43 100644 --- a/osquery/database/.database.md +++ b/osquery/database/.database.md @@ -1,58 +1,50 @@ -Defines the `DatabasePlugin` abstract interface and supporting free functions for osquery's key-value backing storage system, abstracting RocksDB (and SQLite) behind a domain/key API. +Header defining osquery's persistent backing storage abstraction layer, including the `DatabasePlugin` base class, storage domain constants, and free-function wrappers for database operations. ## Key Components -### Domain Constants +**Storage Domain Constants** | Constant | Purpose | |---|---| | `kPersistentSettings` | Node UUID, enrollment keys, persisted config | | `kQueries` | Scheduled query results | | `kEvents` | Queued event results | -| `kCarves` | File carve query results | | `kLogs` | Buffered logger plugin output | +| `kCarves` | File carve query results | | `kDistributedQueries` | Distributed query storage | | `kQueryPerformance` | Query performance statistics | -### `DatabasePlugin` (abstract class) -Extends `Plugin`; all backing store implementations must override: -- `get()` β€” retrieve a string or int value by domain + key -- `put()` β€” store a string or int value -- `putBatch()` β€” write multiple key/value pairs atomically -- `remove()` / `removeRange()` β€” delete single or ranged keys -- `scan()` β€” list keys with optional prefix filter -- `call()` β€” registry/extension routing entrypoint - -### Free Functions -- `getDatabaseValue` / `setDatabaseValue` / `setDatabaseBatch` β€” primary read/write API for all osquery components -- `deleteDatabaseValue` / `deleteDatabaseRange` β€” key removal -- `scanDatabaseKeys` β€” enumerate keys in a domain -- `initDatabasePlugin` / `initDatabasePluginForTesting` β€” lifecycle setup -- `shutdownDatabase` / `resetDatabase` β€” teardown and reset -- `upgradeDatabase` β€” migrates legacy ptree JSON to RapidJSON format -- `getOsqueryDatabase` β€” returns the active `IDatabaseInterface` +**`DatabasePlugin`** β€” Abstract base class (`Plugin`) all backing store implementations must extend. Pure virtual methods: `get`, `put`, `putBatch`, `remove`, `removeRange`. Concrete helpers: `reset()`, `checkDB()`. + +**Free-Function API** β€” Wrappers that route through the active plugin registry: +- `getDatabaseValue` / `setDatabaseValue` / `setDatabaseBatch` / `deleteDatabaseValue` +- `scanDatabaseKeys` β€” list keys in a domain with optional prefix/limit +- `initDatabasePlugin` / `shutdownDatabase` / `resetDatabase` +- `upgradeDatabase` β€” migrates legacy Boost ptree JSON to RapidJSON format ## Usage Example -```c +```cpp #include +// Initialize before use +osquery::initDatabasePlugin(); + // Write a persistent value -osquery::Status s = osquery::setDatabaseValue( - osquery::kPersistentSettings, "node_key", "abc-123"); +osquery::setDatabaseValue(osquery::kPersistentSettings, "node_key", "abc123"); // Read it back std::string node_key; -s = osquery::getDatabaseValue( +auto status = osquery::getDatabaseValue( osquery::kPersistentSettings, "node_key", node_key); +if (status.ok()) { + // use node_key +} + // Scan all keys in a domain std::vector keys; osquery::scanDatabaseKeys(osquery::kQueries, keys); -// Batch write -osquery::DatabaseStringValueList batch = { - {"key1", "value1"}, - {"key2", "value2"}}; -osquery::setDatabaseBatch(osquery::kLogs, batch); +osquery::shutdownDatabase(); ``` \ No newline at end of file diff --git a/osquery/database/tests/.test_utils.md b/osquery/database/tests/.test_utils.md index f27cb6fbe72..4986a92564e 100644 --- a/osquery/database/tests/.test_utils.md +++ b/osquery/database/tests/.test_utils.md @@ -1,41 +1,41 @@ -Provides a reusable Google Test base class and macro for testing osquery database plugin implementations. +Shared test infrastructure for osquery database plugin unit tests, providing a reusable base fixture class and macro-driven test registration for validating `DatabasePlugin` implementations. ## Key Components -### Macro +### `CREATE_DATABASE_TESTS(n)` Macro +Registers a standard suite of 9 Google Test cases against a given fixture class `n`, covering: +- `test_plugin_check` β€” validates plugin initialization +- `test_reset` β€” verifies reset behavior +- `test_put` / `test_putBatch` β€” single and batch write operations +- `test_get` β€” key lookup +- `test_delete` / `test_delete_range` β€” single and range deletion +- `test_scan` / `test_scan_limit` β€” full and limited key scanning -- **`CREATE_DATABASE_TESTS(n)`** β€” Expands into a full suite of `TEST_F` cases for a given fixture class `n`, covering all standard database operations. Use this in a `.cpp` file to instantiate tests for a specific plugin. +### `DatabasePluginTests` (class) +Abstract Google Test fixture (`testing::Test`) that manages plugin lifecycle across tests. -### Class: `DatabasePluginTests` - -A `testing::Test` subclass within the `osquery` namespace that serves as the base fixture for all database plugin test suites. - -| Member | Type | Description | +| Member | Type | Purpose | |---|---|---| | `path_` | `std::string` | Path to the temporary test database | -| `previous_path_` | `std::string` | Database path saved before `SetUp` runs | -| `name()` | pure virtual | Must return the plugin name under test | -| `SetUp()` / `TearDown()` | lifecycle | Initializes and cleans up the plugin under test | - -**Protected test methods** (called by the macro-generated test cases): - -`testPluginCheck` Β· `testReset` Β· `testPut` Β· `testPutBatch` Β· `testGet` Β· `testDelete` Β· `testDeleteRange` Β· `testScan` Β· `testScanLimit` +| `previous_path_` | `std::string` | Saved database path before `SetUp` | +| `name()` | pure virtual | Subclass must return the plugin name | +| `SetUp()` / `TearDown()` | override | Initializes and cleans up the plugin under test | +| `plugin_` | `shared_ptr` | Plugin instance ready for test execution | ## Usage Example ```cpp -#include "test_utils.h" - -class RocksDBDatabaseTests : public osquery::DatabasePluginTests { +// Declare a concrete fixture for a specific database plugin +class RocksDBPluginTests : public DatabasePluginTests { protected: std::string name() override { return "rocksdb"; } }; -// Expands all standard TEST_F cases for RocksDBDatabaseTests -CREATE_DATABASE_TESTS(RocksDBDatabaseTests); +// Expand all standard database tests for this fixture +CREATE_DATABASE_TESTS(RocksDBPluginTests); ``` -Adding a new database plugin test suite requires only subclassing `DatabasePluginTests`, implementing `name()`, and invoking `CREATE_DATABASE_TESTS` β€” no boilerplate test bodies needed. \ No newline at end of file +The macro expands into individual `TEST_F` definitions, so each test appears separately in GTest output with a descriptive name (e.g., `RocksDBPluginTests.test_put`). \ No newline at end of file diff --git a/osquery/dispatcher/.dispatcher.md b/osquery/dispatcher/.dispatcher.md index 6df777c8bb4..c0f1b957fbf 100644 --- a/osquery/dispatcher/.dispatcher.md +++ b/osquery/dispatcher/.dispatcher.md @@ -1,74 +1,59 @@ -Manages thread lifecycle and parallel task execution for osquery services, providing a singleton dispatcher that coordinates interruptible background threads. +Manages thread lifecycle and parallel task execution for osquery services, providing a singleton dispatcher that coordinates asynchronous runnable services across the application. ## Key Components ### `InterruptibleRunnable` -Base class for any runnable that supports clean interruption. - -| Member | Description | -|---|---| -| `interrupt()` | Signals the runnable to stop (final, non-overridable) | -| `interrupted()` | Checks if an interrupt has been requested | -| `pause(milliseconds)` | Interruptible sleep using a condition variable | -| `stop()` | Pure virtual β€” subclasses define their shutdown logic | -| `runnable_name_` | Thread display name | +Base class for any task that can be interrupted mid-execution. +- `interrupt()` β€” signals the runnable to stop +- `interrupted()` β€” checks current interruption state +- `pause(milliseconds)` β€” interruptible sleep using a condition variable +- `stop()` β€” pure virtual; subclasses must define their shutdown logic ### `InternalRunnable` -Extends `InterruptibleRunnable` with a thread entrypoint. Non-copyable. - -| Member | Description | -|---|---| -| `run()` | Thread entrypoint called by `Dispatcher` (final) | -| `start()` | Pure virtual β€” subclasses define their work loop | -| `hasRun()` | Returns whether the thread context has started | - -### `Dispatcher` (Singleton) -Manages all `InternalRunnable` service threads. - -| Method | Description | -|---|---| -| `instance()` | Returns the singleton instance | -| `addService(ref)` | Spawns a new service thread (unlimited pool) | -| `joinServices()` | Blocks until all services complete | -| `stopServices()` | Interrupts and destroys all running services | -| `serviceCount()` | Returns the number of active services | +Extends `InterruptibleRunnable` with a concrete thread entrypoint. Non-copyable. +- `run()` β€” thread entrypoint, called exclusively by `Dispatcher` +- `hasRun()` β€” returns whether the thread has started +- `start()` β€” pure virtual; subclasses define their main loop here + +### `Dispatcher` +Singleton managing all service threads. Non-copyable. +- `instance()` β€” returns the singleton instance +- `addService(InternalRunnableRef)` β€” spawns a service on a new thread +- `joinServices()` β€” blocks until all services complete +- `stopServices()` β€” signals and tears down all running services +- `serviceCount()` β€” returns active service count ### Type Aliases - -```c -using InternalRunnableRef = std::shared_ptr; -using InternalThreadRef = std::unique_ptr; -``` +- `InternalRunnableRef` β€” `std::shared_ptr` +- `InternalThreadRef` β€” `std::unique_ptr` ## Usage Example -```c -// Define a custom background service -class MyService : public InternalRunnable { +```cpp +// Define a custom service +class MyService : public osquery::InternalRunnable { public: MyService() : InternalRunnable("MyService") {} protected: void start() override { while (!interrupted()) { - // do work - pause(std::chrono::milliseconds(500)); + // do periodic work + pause(std::chrono::milliseconds(1000)); } } void stop() override { - // cleanup + // optional cleanup } }; // Register and launch the service -auto svc = std::make_shared(); -osquery::Dispatcher::addService(svc); +auto service = std::make_shared(); +osquery::Dispatcher::addService(service); -// Later, cleanly shut everything down +// On shutdown osquery::Dispatcher::stopServices(); osquery::Dispatcher::joinServices(); -``` - -> **Note:** `Dispatcher` is a singleton β€” never construct it directly. Use `Dispatcher::instance()`. Services are not subject to a fixed thread pool size, unlike the underlying Thrift pool. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/dispatcher/.distributed_runner.md b/osquery/dispatcher/.distributed_runner.md index c52d1a98aff..c764a382e03 100644 --- a/osquery/dispatcher/.distributed_runner.md +++ b/osquery/dispatcher/.distributed_runner.md @@ -1,32 +1,21 @@ -Declares the `DistributedRunner` service thread responsible for executing distributed queries within the osquery dispatcher framework. +Defines the `DistributedRunner` service thread responsible for executing distributed queries within the osquery dispatcher framework. ## Key Components -- **`DistributedRunner`** β€” A `InternalRunnable` subclass that runs as a long-lived Dispatcher service thread, implementing the distributed query polling/execution loop. - - `start()` β€” Entry point invoked by the Dispatcher when the thread is scheduled. -- **`startDistributed()`** β€” Factory/helper function to initialize and register the `DistributedRunner` with the osquery Dispatcher. +- **`DistributedRunner`** β€” A `InternalRunnable` subclass that runs as a managed Dispatcher service thread. Implements the distributed query polling/execution loop via `start()`. +- **`startDistributed()`** β€” Factory/launch function that initializes and registers the `DistributedRunner` with the Dispatcher, returning a `Status` indicating success or failure. ## Usage Example -```cpp +```c #include -// Start the distributed query service (typically called during osquery startup) +// Start the distributed query service (typically called during daemon init) Status s = osquery::startDistributed(); if (!s.ok()) { LOG(ERROR) << "Failed to start distributed runner: " << s.getMessage(); } ``` -Internally, the runner is registered via the Dispatcher: - -```cpp -// Inside startDistributed() β€” conceptual implementation -Status startDistributed() { - Dispatcher::addService(std::make_shared()); - return Status::success(); -} -``` - -> **Note:** `DistributedRunner` is intended to be managed exclusively by the osquery `Dispatcher`. Do not instantiate or call `start()` directly β€” use `startDistributed()` instead to ensure proper lifecycle management. \ No newline at end of file +The `DistributedRunner` thread is managed by the `Dispatcher` β€” it should not be instantiated directly in most cases. Use `startDistributed()` to register it properly within the service lifecycle. \ No newline at end of file diff --git a/osquery/dispatcher/.scheduler.md b/osquery/dispatcher/.scheduler.md index 01692c14232..10e003dd145 100644 --- a/osquery/dispatcher/.scheduler.md +++ b/osquery/dispatcher/.scheduler.md @@ -1,54 +1,44 @@ -Defines the `SchedulerRunner` class and supporting utilities for executing scheduled osquery SQL queries at configurable intervals within the osquery dispatcher framework. +Defines the `SchedulerRunner` class and related utilities for executing scheduled osquery SQL queries at configurable intervals within the osquery dispatcher framework. ## Key Components ### `SchedulerRunner` (class) -A `InternalRunnable` dispatcher service thread that drives osquery's schedule execution loop. +A `Dispatcher` service thread (`InternalRunnable`) that drives the osquery schedule loop. | Member | Description | -|---|---| -| `SchedulerRunner(timeout, interval, max_time_drift)` | Constructor; configures loop timeout, step interval, and optional drift cap | -| `start()` | Thread entry point; runs the scheduling loop | -| `stop()` | Interrupt/stop hook (no-op implementation) | -| `getCurrentTimeDrift()` | Returns accumulated time drift in milliseconds | - -### Private Helpers -| Method | Description | -|---|---| -| `calculateTimeDriftAndMaybePause()` | Tracks loop step duration and compensates for drift | -| `maybeRunDecorators()` | Fires interval-based query decorators | -| `maybeReloadSchedule()` | Checks and applies config schedule changes | -| `maybeFlushLogs()` | Conditionally flushes buffered status logs | -| `maybeScheduleCarves()` | Triggers file carve requests on schedule | +|--------|-------------| +| `SchedulerRunner(timeout, interval, max_time_drift)` | Constructor; sets loop interval, max steps, and optional drift cap | +| `start()` | Thread entry point; runs the scheduler loop | +| `stop()` | Interrupt/shutdown hook (no-op) | +| `getCurrentTimeDrift()` | Returns accumulated scheduling time drift in milliseconds | +| `calculateTimeDriftAndMaybePause()` | Compensates for loop execution drift by adjusting sleep | +| `maybeRunDecorators(time_step)` | Fires interval-based query decorators | +| `maybeReloadSchedule(time_step)` | Checks and reloads config schedule flags | +| `maybeFlushLogs(time_step)` | Flushes buffered status logs on schedule | +| `maybeScheduleCarves(time_step)` | Enqueues file carve requests | ### Free Functions + | Function | Description | -|---|---| -| `monitor(name, query)` | Executes a named `ScheduledQuery` and returns an `SQLInternal` result | +|----------|-------------| +| `monitor(name, query)` | Executes a `ScheduledQuery` by name and returns an `SQLInternal` result | | `startScheduler()` | Starts the scheduler using config-defined settings | | `startScheduler(timeout, interval)` | Starts the scheduler with explicit timeout and interval (used in tests) | ## Usage Example ```cpp -#include "osquery/scheduler/scheduler.h" - -// Start the scheduler using the loaded osquery config schedule +// Production startup β€” reads schedule from osquery config osquery::startScheduler(); -// Or start with explicit parameters (e.g., in tests) -osquery::startScheduler( - /*timeout=*/3600UL, // run for 3600 steps - /*interval=*/60 // execute every 60 seconds -); +// Test startup β€” explicit timeout (steps) and interval (seconds) +osquery::startScheduler(/*timeout=*/100UL, /*interval=*/1); -// Manually monitor a specific scheduled query +// Manually execute a single scheduled query osquery::ScheduledQuery sq; -sq.query = "SELECT * FROM processes;"; -sq.interval = 60; - -osquery::SQLInternal result = osquery::monitor("process_monitor", sq); +sq.query = "SELECT * FROM processes;"; +auto result = osquery::monitor("process_monitor", sq); ``` -> **Note:** `request_shutdown_on_expiration` defaults to `true` in production but is suppressed during unit tests via `FRIEND_TEST(TLSConfigTests, test_runner_and_scheduler)` to prevent premature process exit. \ No newline at end of file +> **Note:** `request_shutdown_on_expiration` defaults to `true` in production. Tests override this via `FRIEND_TEST(TLSConfigTests, test_runner_and_scheduler)` to prevent forced shutdown when the scheduler timeout expires. \ No newline at end of file diff --git a/osquery/distributed/.distributed.md b/osquery/distributed/.distributed.md index e9b7444e2a4..c474e36c380 100644 --- a/osquery/distributed/.distributed.md +++ b/osquery/distributed/.distributed.md @@ -1,27 +1,26 @@ -Defines the interfaces and data structures for osquery's distributed query system, enabling remote query execution, result serialization, and denylist management across endpoints. +Defines the interfaces and data structures for osquery's distributed query system, enabling remote query dispatch, execution, result collection, and serialization within the `osquery` namespace. ## Key Components ### Structs - **`DistributedQueryRequest`** β€” Holds a query string and its unique ID for a pending distributed query -- **`DistributedQueryResult`** β€” Holds the request, result rows, column names, status, and message from a completed query execution +- **`DistributedQueryResult`** β€” Contains the originating request, `QueryData` results, column names, execution `Status`, and an optional message ### Free Functions - -| Function | Description | +| Function | Purpose | |---|---| -| `serializeDistributedQueryRequest[JSON]` | Serialize a request to JSON tree or string | -| `deserializeDistributedQueryRequest[JSON]` | Deserialize a request from JSON tree or string | -| `serializeDistributedQueryResult[JSON]` | Serialize a result to JSON tree or string | -| `deserializeDistributedQueryResult[JSON]` | Deserialize a result from JSON tree or string | -| `hashQuery` | Returns SHA-256 hex digest of a query string | -| `denylistDuration` | Returns configured denylist duration in seconds | -| `denylistedQueryTimestampExpired` | Checks if a denylisted query's timeout has elapsed | +| `serializeDistributedQueryRequest[JSON]` | Serialize a request to a property tree or JSON string | +| `deserializeDistributedQueryRequest[JSON]` | Deserialize a request from a property tree or JSON string | +| `serializeDistributedQueryResult[JSON]` | Serialize a result to a property tree or JSON string | +| `deserializeDistributedQueryResult[JSON]` | Deserialize a result from a property tree or JSON string | +| `hashQuery` | Returns the SHA-256 hex digest of a query string | +| `denylistDuration` | Returns the configured denylist duration for distributed queries | +| `denylistedQueryTimestampExpired` | Checks whether a denylisted query's timestamp has expired | ### Classes -- **`DistributedPlugin`** β€” Abstract plugin base; implementors must provide `getQueries()` and `writeResults()` to integrate with a remote query backend -- **`Distributed`** β€” Orchestrates the full distributed query lifecycle: pulling, executing, tracking running state, recording performance, and flushing results +- **`DistributedPlugin`** β€” Abstract plugin base defining `getQueries()` and `writeResults()` for remote query I/O +- **`Distributed`** β€” Orchestrates the full query lifecycle: pulling, executing, denylist tracking, performance recording, and flushing results ## Usage Example @@ -30,22 +29,14 @@ Defines the interfaces and data structures for osquery's distributed query syste auto dist = osquery::Distributed(); while (true) { - // Pull pending queries from remote server - dist.pullUpdates(); - - if (dist.getPendingQueryCount() > 0) { - // Execute all pending queries and queue results - dist.runQueries(); - } + dist.pullUpdates(); - // Serialize and inspect buffered results - std::string json; - dist.serializeResults(json); + if (dist.getPendingQueryCount() > 0) { + dist.runQueries(); + } } -``` - -## Notes -- Query deduplication is enforced via SHA-256 hashing (`hashQuery`) combined with denylist tracking (`checkAndSetAsRunning`) to prevent re-execution within the configured denylist window -- Discovery queries gate execution: if a discovery query returns no rows, its associated distributed query is skipped -- Performance statistics per query are recorded in `performance_` and reported alongside results \ No newline at end of file +// Serialize and inspect completed results +std::string json; +dist.serializeResults(json); +``` \ No newline at end of file diff --git a/osquery/events/.eventer.md b/osquery/events/.eventer.md index 739769904da..e0643fb8aa7 100644 --- a/osquery/events/.eventer.md +++ b/osquery/events/.eventer.md @@ -1,51 +1,55 @@ -Defines the base `Eventer` class and `EventState` enum used to track the lifecycle state of event publishers and subscribers within osquery's eventing system. +Defines the base `Eventer` class and `EventState` enum used to track the lifecycle state of event subscribers and publishers within the osquery eventing framework. ## Key Components -### `EventState` (enum class) -Represents the possible lifecycle states of an event publisher or subscriber: +**`EventState` (enum class)** +Represents the possible lifecycle states of an event subscriber or publisher: | Value | Description | |---|---| | `EVENT_NONE` | Default uninitialized state | -| `EVENT_SETUP` | Subscriber attached and setup complete | -| `EVENT_RUNNING` | Ready to receive events | +| `EVENT_SETUP` | Attached and setup complete | +| `EVENT_RUNNING` | Ready to accept events | | `EVENT_PAUSED` | Initialized but not accepting events | -| `EVENT_FAILED` | Initialization failed or offline | +| `EVENT_FAILED` | Failed to initialize or offline | -### `Eventer` (class) -Base class providing state management for event publishers and subscribers. +**`Eventer` (class)** +Base class providing state management for event subscribers and publishers. -| Member | Access | Description | -|---|---|---| -| `state() const` | `public` | Returns current `EventState` | -| `state(EventState)` | `protected` | Sets the current state | -| `state_` | `private` | Internal state storage, defaults to `EVENT_NONE` | - -`EventFactory` is declared a `friend` class, granting it direct access to internal state. +- `state() const` β€” Public getter returning the current `EventState` +- `state(EventState)` β€” Protected setter for transitioning state (subclasses only) +- `EventFactory` is declared a `friend`, granting it direct access to internal state ## Usage Example ```cpp -#include +#include "eventer.h" + +namespace osquery { class MySubscriber : public Eventer { public: - void initialize() { - // Transition to running after setup - state(EventState::EVENT_SETUP); - if (setupSucceeded()) { - state(EventState::EVENT_RUNNING); - } else { - state(EventState::EVENT_FAILED); - } - } + void initialize() { + // Transition to setup before registering subscriptions + state(EventState::EVENT_SETUP); - void checkStatus() { - if (state() == EventState::EVENT_RUNNING) { - // Accept and process events - } - } + if (!registerSubscriptions()) { + state(EventState::EVENT_FAILED); + return; + } + + state(EventState::EVENT_RUNNING); + } }; -``` \ No newline at end of file + +void checkSubscriber(const MySubscriber& sub) { + if (sub.state() == EventState::EVENT_RUNNING) { + // Subscriber is active and accepting events + } +} + +} // namespace osquery +``` + +> `Eventer` is intended as a base class; direct state mutation is restricted to subclasses and `EventFactory` to enforce controlled lifecycle transitions. \ No newline at end of file diff --git a/osquery/events/.eventfactory.md b/osquery/events/.eventfactory.md index c83bdcca4f3..60356f0e76c 100644 --- a/osquery/events/.eventfactory.md +++ b/osquery/events/.eventfactory.md @@ -1,47 +1,62 @@ -Singleton factory class for managing osquery event publishers, subscribers, and subscriptions throughout the application lifecycle. +Singleton factory class for managing the lifecycle of event publishers, subscribers, and subscriptions within the osquery eventing framework. ## Key Components -| Member | Type | Description | -|--------|------|-------------| -| `getInstance()` | Static method | Returns the singleton `EventFactory` instance | -| `registerEventPublisher()` | Static method | Registers an `EventPublisher` plugin with the factory | -| `registerEventSubscriber()` | Template method | Registers an `EventSubscriber` by type or instance | -| `addSubscription()` | Static method | Binds a `SubscriptionContext` and `EventCallback` to a publisher | -| `deregisterEventPublisher()` | Static method | Halts a publisher's run loop and removes it from the factory | -| `getEventPublisher()` | Static method | Retrieves a registered publisher by name | -| `getEventSubscriber()` | Static method | Retrieves a registered subscriber by name | -| `fire()` | Template method | Fires an `EventContext` into a publisher's subscriber callbacks | -| `getType()` | Template method | Resolves the registry plugin name for a publisher type | -| `delay()` | Static method | Spawns all registered publisher run loop threads | -| `end()` | Static method | Terminates all publisher run loops and optionally joins threads | -| `configUpdate()` | Static method | Notifies publishers/subscribers of configuration changes | +### `EventFactory` (Singleton Class) +The central coordinator for osquery's event-driven architecture. Manages registration, dispatch, and teardown of all event-related components. + +**Static Management Methods** +- `getInstance()` β€” Returns the singleton `EventFactory` instance +- `registerEventPublisher(pub)` β€” Registers an `EventPublisher` plugin with the factory +- `registerEventSubscriber()` / `registerEventSubscriber(sub)` β€” Registers an `EventSubscriber` by type or instance +- `deregisterEventPublisher(pub/type_id)` β€” Halts and removes an `EventPublisher` by reference or name +- `deregisterEventSubscriber(sub)` β€” Removes an `EventSubscriber` by name + +**Subscription Methods** +- `addSubscription(type_id, name_id, sc, cb)` β€” Creates and attaches a `Subscription` to a publisher using a context and callback +- `addSubscription(type_id, subscription)` β€” Attaches a pre-built `Subscription` instance +- `numSubscriptions(type_id)` β€” Returns subscription count for a given publisher + +**Lookup & Introspection** +- `getEventPublisher(pub)` β€” Returns a reference to a registered publisher +- `getEventSubscriber(sub)` β€” Returns a reference to a registered subscriber +- `exists(sub)` β€” Checks if a subscriber is registered +- `publisherTypes()` / `subscriberNames()` β€” Lists all registered publisher or subscriber names + +**Runtime Control** +- `run(type_id)` β€” Entry point for a publisher's event thread +- `delay()` β€” Spawns all publisher run loops +- `end(join)` β€” Stops all publishers and optionally joins their threads +- `fire(ec)` β€” Fires an event context into a publisher's subscriber callbacks +- `getType()` β€” Resolves the registry plugin name for a publisher type + +**Event Forwarding** +- `addForwarder(logger)` β€” Registers a logger plugin to receive forwarded events +- `forwardEvent(event)` β€” Sends an event string to all registered loggers +- `configUpdate()` β€” Notifies the factory of configuration changes ## Usage Example ```cpp -// Register a publisher plugin -EventFactory::registerEventPublisher(pub_plugin_ref); - -// Register a subscriber by type +// Register a custom publisher and subscriber +EventFactory::registerEventPublisher(std::make_shared()); EventFactory::registerEventSubscriber(); -// Add a subscription with context and callback +// Add a subscription with a context and callback auto sc = std::make_shared(); EventFactory::addSubscription("my_publisher", "my_subscriber", sc, [](const EventContextRef& ec, const SubscriptionContextRef& sc) { - // Handle event + // handle event return Status::success(); }); -// Fire an event from a static publisher callback -EventContextRef ec = std::make_shared(); -EventFactory::fire(ec); - // Start all publisher run loops EventFactory::delay(); -// Shutdown all publishers (with thread join) -EventFactory::end(true); +// Fire an event from a static publisher callback +EventFactory::fire(event_context); + +// Shut down all publishers +EventFactory::end(true /* join threads */); ``` \ No newline at end of file diff --git a/osquery/events/.eventpublisherplugin.md b/osquery/events/.eventpublisherplugin.md index 9903fac46e1..eb3967f320e 100644 --- a/osquery/events/.eventpublisherplugin.md +++ b/osquery/events/.eventpublisherplugin.md @@ -1,5 +1,5 @@ -Defines the `EventPublisherPlugin` base class for osquery's event-driven architecture, providing the lifecycle interface that all event publishers must implement to register OS-level callbacks, manage subscriptions, and fire events to subscribers. +Defines the `EventPublisherPlugin` base class for osquery's event-driven plugin system, providing the lifecycle interface for publishing events to registered subscribers. ## Key Components @@ -9,29 +9,29 @@ Inherits from `Plugin`, `InterruptibleRunnable`, and `Eventer`. Serves as the ab **Lifecycle Methods** | Method | Description | |--------|-------------| -| `setUp()` | Initialize handles and register OS API callbacks (called before run loop) | -| `configure()` | Re-optimize subscriptions (e.g., dedup inotify paths) when subscriptions change | -| `tearDown()` | Release handles and clean up resources before exit | -| `run()` | Single iteration of the publisher's run loop; return `FAILED` to exit | -| `stop()` | Signal the run loop to terminate (called by `EventFactory`) | +| `setUp()` | Handle/resource initialization before the run loop starts | +| `configure()` | Optimize state based on current subscriptions (e.g., dedup paths) | +| `run()` | Single step of the publisher's run loop; returning `FAILED` exits the loop | +| `tearDown()` | Cleanup handles and resources on shutdown | +| `stop()` | Signal the run loop to halt (called by `EventFactory`) | **Subscription Management** | Method | Description | |--------|-------------| -| `addSubscription(sub)` | Register a new `SubscriptionRef` with this publisher | +| `addSubscription(subscription)` | Register a new `SubscriptionRef` with this publisher | | `removeSubscriptions(subscriber)` | Remove all subscriptions for a named subscriber | -| `numSubscriptions()` | Returns active subscription count | +| `numSubscriptions()` | Returns current subscriber count | **State Inspection** -- `isEnding()` / `hasStarted()` β€” get/set publisher lifecycle state -- `numEvents()` β€” total fired event count -- `restartCount()` β€” run loop restart iterations +- `isEnding()` / `hasStarted()` β€” get/set publisher run loop state +- `numEvents()` β€” total `EventContext` IDs issued +- `restartCount()` β€” number of run loop restarts -### Protected Members -- `fire(ec, time)` β€” dispatches an `EventContext` to all matching subscribers -- `fireCallback(sub, ec)` β€” pure virtual; typed publishers implement actual dispatch -- `subscriptions_` β€” the internal `SubscriptionVector` -- `next_ec_id_` β€” atomic event context ID counter +**Protected** +- `fire(ec, time)` β€” dispatches an `EventContext` to all matching subscriptions +- `fireCallback(sub, ec)` β€” **pure virtual**; implemented by typed publishers to invoke subscriber callbacks +- `subscriptions_` β€” `SubscriptionVector` guarded by `subscription_lock_` +- `next_ec_id_` β€” atomic `EventContextID` counter ## Usage Example @@ -39,24 +39,23 @@ Inherits from `Plugin`, `InterruptibleRunnable`, and `Eventer`. Serves as the ab class MyPublisher : public EventPublisherPlugin { public: Status setUp() override { - // Open OS handle or register callback + // Register OS API callbacks return Status::success(); } Status run() override { + // Poll or block for events, then fire auto ec = createEventContext(); - fire(ec); // dispatches to all subscribers + fire(ec); return Status::success(); } - void tearDown() override { - // Close handles - } - protected: void fireCallback(const SubscriptionRef& sub, const EventContextRef& ec) const override { - // Invoke subscriber's typed callback + // Invoke the subscriber's typed callback } }; -``` \ No newline at end of file +``` + +> **Note:** Copy construction and assignment are explicitly deleted. Do not override `fire`; implement `fireCallback` and use `shouldFire` logic in typed subclasses instead. \ No newline at end of file diff --git a/osquery/events/.events.md b/osquery/events/.events.md index 185e2df9624..574d5fdebde 100644 --- a/osquery/events/.events.md +++ b/osquery/events/.events.md @@ -1,26 +1,26 @@ -Utility header declaring event system lifecycle functions within the `osquery` namespace for managing event publisher run loops and query denylist enforcement. +Declares two utility functions for managing osquery's event publisher system, providing event loop initialization and query denylist enforcement for event-based tables. ## Key Components -| Function | Description | -|---|---| -| `attachEvents()` | Iterates the event publisher registry and creates run loops for each publisher via the event factory | -| `enforceEventsDenylist(query)` | Returns `true` if the given query operates exclusively on event-based tables and should be denylisted | +| Symbol | Type | Description | +|--------|------|-------------| +| `attachEvents()` | `void` | Iterates the event publisher registry and spawns run loops for each publisher via the event factory | +| `enforceEventsDenylist()` | `bool` | Checks whether a query operates exclusively on event-based tables and should be denied execution | ## Usage Example ```c #include "events.h" -// Initialize all registered event publishers +// Initialize all registered event publishers at startup osquery::attachEvents(); -// Check if a query should be denied based on event-table usage +// Before executing a query, check if it should be blocked std::string query = "SELECT * FROM process_events"; if (osquery::enforceEventsDenylist(query)) { - // Query is denylist-eligible; skip or block execution + // Query targets only event-based tables β€” skip or reject } ``` -> **Note:** `attachEvents()` is typically called once during daemon startup to wire up all registered publishers. `enforceEventsDenylist` is used during query scheduling to prevent high-frequency polling on purely event-driven tables. \ No newline at end of file +> **Note:** Both functions operate within the `osquery` namespace. `attachEvents()` is typically called once during daemon initialization. `enforceEventsDenylist()` is used at query evaluation time to prevent abuse of high-frequency event-driven tables. \ No newline at end of file diff --git a/osquery/events/.eventsubscriberplugin.md b/osquery/events/.eventsubscriberplugin.md index 9338289cef4..1215cfc0cc5 100644 --- a/osquery/events/.eventsubscriberplugin.md +++ b/osquery/events/.eventsubscriberplugin.md @@ -1,29 +1,29 @@ -Defines the `EventSubscriberPlugin` class, the base interface for osquery event subscribers that receive, store, and expose event data as queryable table rows via a backing database store. +Defines the `EventSubscriberPlugin` class, the base plugin interface for osquery event subscribers that receive, store, and expose event data as queryable table rows via a backing store. ## Key Components ### Class: `EventSubscriberPlugin` -Inherits from `Plugin` and `Eventer`. Manages event ingestion, indexed storage, expiration, and query-time row generation. +Inherits from `Plugin` and `Eventer`. Provides the full lifecycle for event subscription, storage, indexing, expiration, and table generation. -### Core Methods +### Core Virtual Methods +- **`init()`** β€” Override to register `Subscription`s with an `EventPublisher` +- **`genTable(yield, ctx)`** β€” Entrypoint for table generation; retrieves stored events within a query time window +- **`getType()`** β€” Pure virtual; returns the associated publisher type name +- **`getEventsExpiry()`** / **`getEventBatchesMax()`** β€” Override to customize retention per subscriber -| Method | Description | -|---|---| -| `init()` | Override to register `Subscription`s with an `EventPublisher` | -| `call(PluginRequest, PluginResponse)` | Plugin dispatch entrypoint | -| `addBatch(row_list)` | Store a batch of parsed event rows to the backing store (preferred API) | -| `add(row)` | *(Deprecated)* Store a single event row β€” use `addBatch()` instead | -| `genTable(yield, ctx)` | Table generation entrypoint; retrieves stored events filtered by query time window | -| `generateRows(...)` | Static: enumerate stored events within a `[start_time, end_time]` range | -| `expireEventBatches(...)` | Static: prune old event batches from backing store | -| `removeOverflowingEventBatches(...)` | Static: enforce maximum batch count | -| `getOptimizeData(...)` / `setOptimizeData(...)` | Static: manage query optimization checkpoints | +### Storage Methods +- **`addBatch(row_list)`** β€” Stores a batch of event rows into the backing database (preferred over deprecated `add()`) +- **`add(row)`** β€” *(Deprecated)* Single-row storage; migrate to `addBatch()` -### Supporting Structures +### Static Utilities +- **`generateRows(context, db, callback, start, end, last_eid)`** β€” Retrieves events within a time range from the backing store +- **`expireEventBatches()`** / **`removeOverflowingEventBatches()`** β€” Manage event retention and overflow +- **`generateEventDataIndex()`** β€” Rebuilds the time-based event index from the database +- **`setOptimizeData()`** / **`getOptimizeData()`** β€” Read/write optimization cursors for incremental queries -- **`Context`** β€” Holds the database namespace, event index, last query/event timestamps -- **`GenerateRowsResult`** β€” Returns pagination state (`isEnd`, `last_time`, `last_id`) from `generateRows` +### `Context` struct +Holds per-subscriber state: `database_namespace`, `event_index`, `last_query_time`, and `last_event_id`. ## Usage Example @@ -31,19 +31,19 @@ Inherits from `Plugin` and `Eventer`. Manages event ingestion, indexed storage, class MySubscriber : public EventSubscriber { public: Status init() override { - auto sub = createSubscription(); - return EventFactory::addSubscription(getType(), sub); + auto sub = createSubscription(); + return subscribe(&MySubscriber::eventCallback, sub); } - Status Callback(const EventContextRef& ec, const SubscriptionContextRef& sc) { + Status eventCallback(const ECRef& ec, const SCRef& sc) { + std::vector rows; Row r; r["action"] = ec->action; - r["time"] = INTEGER(ec->time); - - std::vector batch = {r}; - return addBatch(batch); // preferred over deprecated add() + r["time"] = std::to_string(ec->time); + rows.push_back(r); + return addBatch(rows); // preferred over deprecated add() } }; -``` -> **Note:** `add()` is deprecated. Always use `addBatch()` to group events together for efficient indexed storage and reduced write overhead. \ No newline at end of file +REGISTER(MySubscriber, "event_subscriber", "my_subscriber"); +``` \ No newline at end of file diff --git a/osquery/events/.subscription.md b/osquery/events/.subscription.md index b7de73f51af..390b675dae3 100644 --- a/osquery/events/.subscription.md +++ b/osquery/events/.subscription.md @@ -1,40 +1,44 @@ -Defines the `Subscription` struct used to configure an `EventPublisher` and bind event callbacks to a `SubscriptionContext` within osquery's event framework. +Defines the `Subscription` struct used to configure an `EventPublisher` and bind a callback to a `SubscriptionContext` within the osquery event framework. ## Key Components -| Symbol | Type | Description | +**Type Aliases** +- `EventCallback` β€” `std::function` callback signature fired when a matching event occurs +- `SubscriptionRef` β€” `std::shared_ptr` smart pointer alias +- `SubscriptionVector` β€” `std::vector` for tracking multiple subscriptions per publisher + +**`Subscription` struct** *(non-copyable)* +| Member | Type | Description | |---|---|---| -| `EventCallback` | `using` alias | Function signature `Status(const EventContextRef&, const SubscriptionContextRef&)` invoked when a matching event fires | -| `SubscriptionRef` | `using` alias | `shared_ptr` for shared ownership of a subscription | -| `SubscriptionVector` | `using` alias | `vector` used by publishers to track all active subscriptions | -| `Subscription` | `struct` | Non-copyable aggregate holding a subscriber name, context, and callback | -| `Subscription::create()` | static factory | Two overloads β€” name-only, or name + context + optional callback | +| `subscriber_name` | `std::string` | Owning `EventSubscriber` name | +| `context` | `SubscriptionContextRef` | Publisher-specific filter/scope context | +| `callback` | `EventCallback` | Fired when a matching event is dispatched | + +**Static Factory Methods** +- `create(name)` β€” Creates a bare subscription with no context or callback +- `create(name, context, callback)` β€” Creates a fully configured subscription ready for registration ## Usage Example ```cpp #include -// Define a callback matching the EventCallback signature -Status onFileEvent(const EventContextRef& ec, - const SubscriptionContextRef& sc) { - // Handle the event +// Define a callback +EventCallback my_callback = [](const EventContextRef& ec, + const SubscriptionContextRef& sc) -> Status { + // Handle event return Status::success(); -} +}; -// Create a subscription with context and callback +// Create a subscription context (publisher-specific) auto ctx = std::make_shared(); ctx->path = "/etc/passwd"; -SubscriptionRef sub = Subscription::create( - "my_subscriber", // EventSubscriber name - ctx, // Publisher-specific context - onFileEvent // Callback fired on matching events -); +// Create and register the subscription +auto sub = Subscription::create("my_subscriber", ctx, my_callback); -// Register via EventFactory EventFactory::addSubscription("MyEventPublisher", sub); ``` -> **Note:** `Subscription` inherits `boost::noncopyable` β€” always pass instances via `SubscriptionRef` (`shared_ptr`). The default constructor is explicitly deleted; use the static `create()` factory methods. \ No newline at end of file +> **Note:** The default constructor is deleted β€” subscriptions must always be created via the named `create()` factory methods to ensure a valid `subscriber_name` is assigned. \ No newline at end of file diff --git a/osquery/events/darwin/.diskarbitration.md b/osquery/events/darwin/.diskarbitration.md index fe2b809a169..e0fb11c680c 100644 --- a/osquery/events/darwin/.diskarbitration.md +++ b/osquery/events/darwin/.diskarbitration.md @@ -1,50 +1,50 @@ -Provides the osquery event publisher interface for monitoring macOS disk mount/unmount events via the DiskArbitration framework. +Defines the `DiskArbitrationEventPublisher` for monitoring macOS disk mount/unmount events using the DiskArbitration framework within the osquery event system. ## Key Components ### Macros - -| Macro | Purpose | -|---|---| -| `kIOPropertyProtocolCharacteristicsKey_` | IOKit key for protocol characteristics lookup | -| `kVirtualInterfaceLocation_` | Key identifying virtual disk interface paths | -| `kDAAppearanceTime_` | Key for disk appearance timestamp | +- `kIOPropertyProtocolCharacteristicsKey_` β€” IOKit key for protocol characteristics lookup +- `kVirtualInterfaceLocation_` β€” Key identifying virtual disk interface location paths +- `kDAAppearanceTime_` β€” Key for disk appearance timestamp ### Structs - -- **`DiskArbitrationSubscriptionContext`** β€” Subscription filter; `physical_disks` flag limits events to physical vs. virtual (DMG) disks -- **`DiskArbitrationEventContext`** β€” Event payload carrying disk metadata: `action`, `path`, `device_path`, `uuid`, `size`, `filesystem`, `vendor`, `checksum`, and mount capability flags +- **`DiskArbitrationSubscriptionContext`** β€” Subscription config; `physical_disks` flag filters events to physical disks only (default: `false`, virtual/DMG only) +- **`DiskArbitrationEventContext`** β€” Event payload carrying disk metadata: `action`, `path`, `device_path`, `name`, `uuid`, `size`, `filesystem`, `vendor`, `checksum`, and mount flags (`ejectable`, `mountable`, `writable`) ### Type Aliases - - `DiskArbitrationEventContextRef` β€” `shared_ptr` - `DiskArbitrationSubscriptionContextRef` β€” `shared_ptr` ### Class: `DiskArbitrationEventPublisher` - -Extends `EventPublisher` to wrap macOS `DASession` callbacks into osquery events. +Extends `EventPublisher`. | Member | Description | -|---|---| -| `run()` | Starts the `CFRunLoop` and registers DA session callbacks | -| `tearDown()` | Releases DA session and run loop resources | -| `shouldFire()` | Filters events against subscription context | -| `DiskAppearedCallback()` | Static DA callback for disk mount events | -| `DiskDisappearedCallback()` | Static DA callback for disk unmount events | -| `extractUdifChecksum()` | Reads checksum from UDIF (DMG) disk images | -| `fire()` | Builds and dispatches `DiskArbitrationEventContext` from a `CFDictionaryRef` | +|--------|-------------| +| `run()` | Starts the DA session and CFRunLoop | +| `tearDown()` / `stop()` | Cleans up session and run loop | +| `shouldFire()` | Filters events per subscription context | +| `DiskAppearedCallback()` | Static DA callback on disk mount | +| `DiskDisappearedCallback()` | Static DA callback on disk unmount | +| `getProperty()` | Extracts a string value from a `CFDictionaryRef` | +| `extractUdifChecksum()` | Computes checksum for UDIF (DMG) disk images | +| `fire()` | Builds and dispatches the event context | ## Usage Example ```cpp -// Subscribe to physical disk events only -auto sc = std::make_shared(); -sc->physical_disks = true; - -auto sub = EventFactory::addSubscription( - "diskarbitration", - EventSubscriberPlugin::SubscriptionAction::ADD, - sc, - callback); +auto sub_ctx = std::make_shared(); +sub_ctx->physical_disks = true; // Subscribe to physical disk events only + +// In an EventSubscriber: +subscribe(&MySubscriber::handleDiskEvent, sub_ctx); + +Status MySubscriber::handleDiskEvent( + const DiskArbitrationEventContextRef& ec, + const DiskArbitrationSubscriptionContextRef& sc) { + LOG(INFO) << "Disk action: " << ec->action + << " path: " << ec->path + << " filesystem: " << ec->filesystem; + return Status::success(); +} ``` \ No newline at end of file diff --git a/osquery/events/darwin/.endpointsecurity.md b/osquery/events/darwin/.endpointsecurity.md index 4e48c562096..9a4fc889436 100644 --- a/osquery/events/darwin/.endpointsecurity.md +++ b/osquery/events/darwin/.endpointsecurity.md @@ -1,49 +1,40 @@ -macOS Endpoint Security event publisher and subscriber definitions for osquery, providing process and file system monitoring via Apple's EndpointSecurity framework (requires macOS 10.15+). +macOS Endpoint Security event publisher and subscriber definitions for osquery, providing process and file system event monitoring via Apple's EndpointSecurity framework (requires macOS 10.15+). ## Key Components -### Context Structs +### Subscription & Event Contexts -| Struct | Purpose | -|---|---| +| Type | Purpose | +|------|---------| | `EndpointSecuritySubscriptionContext` | Holds ES event type subscriptions and row buffer for process events | -| `EndpointSecurityEventContext` | Full process event payload β€” PID, credentials, code signing, exec args, fork/exit data | -| `EndpointSecurityFileSubscriptionContext` | Holds ES event type subscriptions and row buffer for file events | -| `EndpointSecurityFileEventContext` | Extends process context with `filename` and `dest_filename` for file operations | +| `EndpointSecurityEventContext` | Carries full process event data: PID, credentials, code signing, exec args, fork/exit info | +| `EndpointSecurityFileSubscriptionContext` | Holds ES event type subscriptions for file integrity monitoring (FIM) | +| `EndpointSecurityFileEventContext` | Extends process context with source/destination filename fields | ### Publishers -- **`EndpointSecurityPublisher`** β€” Registers as `"endpointsecurity"`, manages an `es_client_s*` handle, dispatches process-level ES events (`exec`, `fork`, `exit`) to subscribers -- **`EndpointSecurityFileEventPublisher`** β€” Registers as `"endpointsecurity_fim"`, manages a separate `es_file_client_s*` handle with configurable path muting/filtering for file integrity monitoring (FIM); ships with a default muted path list covering high-noise system daemons (e.g., `WindowServer`, `tccd`, `securityd`) +- **`EndpointSecurityPublisher`** β€” Registers as `"endpointsecurity"`, manages an `es_client_s*`, dispatches process lifecycle events (`exec`, `fork`, `exit`) to subscribers +- **`EndpointSecurityFileEventPublisher`** β€” Registers as `"endpointsecurity_fim"`, manages a separate `es_client_s*` with configurable path muting/filtering and a default mute list of noisy system binaries (e.g. `WindowServer`, `tccd`, `securityd`) ### Subscribers -- **`ESProcessEventSubscriber`** β€” Subscribes to `EndpointSecurityPublisher`, populates the `es_process_events` osquery table -- **`ESProcessFileEventSubscriber`** β€” Subscribes to `EndpointSecurityFileEventPublisher`, populates the `es_process_file_events` osquery table +- **`ESProcessEventSubscriber`** β€” Subscribes to `EndpointSecurityPublisher`, emits to the `es_process_events` table +- **`ESProcessFileEventSubscriber`** β€” Subscribes to `EndpointSecurityFileEventPublisher`, emits to the `es_process_file_events` table ## Usage Example ```cpp -// Subscribing to process events in a custom subscriber -class MyProcessSubscriber - : public EventSubscriber { - public: - MyProcessSubscriber() { setName("my_process_events"); } - - Status init() override { +// Subscribing to process events in an osquery plugin +Status ESProcessEventSubscriber::init() { auto sc = createSubscriptionContext(); sc->es_event_subscriptions_.push_back(ES_EVENT_TYPE_NOTIFY_EXEC); - subscribe(&MyProcessSubscriber::Callback, sc); - return Status::success(); - } + sc->es_event_subscriptions_.push_back(ES_EVENT_TYPE_NOTIFY_FORK); + sc->es_event_subscriptions_.push_back(ES_EVENT_TYPE_NOTIFY_EXIT); - Status Callback(const EndpointSecurityEventContextRef& ec, - const EndpointSecuritySubscriptionContextRef& sc) { - // Access ec->pid, ec->path, ec->args, ec->signing_id, etc. + subscribe(&ESProcessEventSubscriber::Callback, sc); return Status::success(); - } -}; +} ``` -> **Platform note:** All publisher and subscriber methods are gated with `API_AVAILABLE(macos(10.15))`. This header has no effect on non-macOS builds. \ No newline at end of file +> **Note:** All publisher and subscriber methods are annotated `API_AVAILABLE(macos(10.15))`. This entire subsystem is macOS-only and will not compile or run on other platforms. \ No newline at end of file diff --git a/osquery/events/darwin/.es_utils.md b/osquery/events/darwin/.es_utils.md index ee9049cde42..15bacba45fc 100644 --- a/osquery/events/darwin/.es_utils.md +++ b/osquery/events/darwin/.es_utils.md @@ -1,40 +1,38 @@ -Utility header for extracting and formatting process and string properties from macOS Endpoint Security (`es_*`) data structures within the osquery framework. +Utility header providing helper functions for extracting and formatting data from Apple's Endpoint Security framework (`es_*`) types within the osquery Darwin event pipeline. ## Key Components | Function | Description | -|---|---| +|----------|-------------| | `getEsNewClientErrorMessage` | Converts an `es_new_client_result_t` error code into a human-readable string | -| `getPath` | Extracts the executable path from an `es_process_t` process struct | +| `getPath` | Extracts the executable path from an `es_process_t` | | `getSigningId` | Retrieves the code-signing identifier from an `es_process_t` | | `getTeamId` | Retrieves the Apple Developer Team ID from an `es_process_t` | -| `getStringFromToken` | Converts a mutable or const `es_string_token_t` to a `std::string` | +| `getStringFromToken` | Converts an `es_string_token_t` (mutable or const) to a `std::string` | | `getCwdPathFromPid` | Resolves the current working directory path for a given PID | -| `getCDHash` | Extracts the code-directory hash from an `es_process_t` | -| `getProcessProperties` | Populates an `EndpointSecurityEventContextRef` with properties from a process | -| `appendQuotedString` | Appends a delimiter-quoted string to an output stream | +| `getCDHash` | Returns the CDHash (code directory hash) from an `es_process_t` | +| `getProcessProperties` | Populates an `EndpointSecurityEventContextRef` with properties from an `es_process_t` | +| `appendQuotedString` | Writes a delimiter-quoted string to an output stream | ## Usage Example ```c #include -// Inside an Endpoint Security event handler: +// Inside an ES message handler: void handleEvent(const es_message_t* msg) { const es_process_t* proc = msg->process; - std::string path = osquery::getPath(proc); - std::string signingId = osquery::getSigningId(proc); - std::string teamId = osquery::getTeamId(proc); - std::string cdHash = osquery::getCDHash(proc); - std::string cwd = osquery::getCwdPathFromPid(audit_token_to_pid(proc->audit_token)); + std::string path = osquery::getPath(proc); + std::string sigId = osquery::getSigningId(proc); + std::string teamId = osquery::getTeamId(proc); + std::string cdHash = osquery::getCDHash(proc); + std::string cwd = osquery::getCwdPathFromPid(audit_token_to_pid(proc->audit_token)); osquery::getProcessProperties(proc, eventContext); std::ostringstream out; osquery::appendQuotedString(out, path, '"'); } -``` - -> **Note:** This header is macOS-only and depends on the Apple Endpoint Security framework (`es_process_t`, `es_string_token_t`). It is used internally by osquery's EndpointSecurity event publisher to normalize raw ES API types into standard C++ types before query processing. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/events/darwin/.event_taps.md b/osquery/events/darwin/.event_taps.md index f8b09845338..82bfe968060 100644 --- a/osquery/events/darwin/.event_taps.md +++ b/osquery/events/darwin/.event_taps.md @@ -1,46 +1,44 @@ -macOS event tap publisher that monitors and intercepts system-level input events (keyboard, mouse, etc.) using the CoreGraphics `CGEventTap` API, exposing them to osquery subscribers. +Declares the macOS Event Tapping publisher for osquery, enabling system-wide input event monitoring via the CoreGraphics `CGEventTap` API. ## Key Components | Symbol | Type | Description | -|---|---|---| +|--------|------|-------------| | `EventTappingSubscriptionContext` | Struct | Empty subscription context for event tap subscriptions | -| `EventTappingEventContext` | Struct | Empty event context carrying tapped event data | -| `EventTappingEventPublisher` | Class | Core publisher managing the CGEventTap lifecycle and run loop | -| `EventTappingConsumerRunner` | Class (forward) | Dispatched service that processes published event tap events | - -### `EventTappingEventPublisher` Methods - -| Method | Description | -|---|---| -| `setUp()` | Initializes the CGEventTap and run loop source | -| `tearDown()` | Cleans up tap resources | -| `stop()` | Stops the run loop | -| `run()` | Starts the publisher run loop | -| `restart()` | Reinitializes the event tap (e.g., after system re-enables it) | -| `eventCallback()` | Static CGEventTap callback receiving raw `CGEventRef` events | -| `shouldFire()` | Filters events against subscription context | +| `EventTappingEventContext` | Struct | Empty event context carrying dispatched tap events | +| `EventTappingEventPublisher` | Class | Core publisher that installs a `CGEventTap`, runs a `CFRunLoop`, and dispatches input events to subscribers | +| `EventTappingConsumerRunner` | Class (fwd) | Dispatched service that processes published event tapping events | +| `eventCallback` | Static method | CoreGraphics callback invoked on each intercepted system event | ## Usage Example ```cpp -// Subscribing to event tap events in an osquery EventSubscriber +// Subscribing to event tapping events in an osquery EventSubscriber +#include "event_taps.h" + class MyEventTapSubscriber : public EventSubscriber { + DECLARE_SUBSCRIBER("my_event_tap_subscriber"); + public: Status init() override { auto sc = createSubscriptionContext(); - subscribe(&MyEventTapSubscriber::onEvent, sc); + subscribe(&MyEventTapSubscriber::eventHandler, sc); return Status::success(); } - Status onEvent(const EventTappingEventContextRef& ec, - const EventTappingSubscriptionContextRef& sc) { - // Process intercepted macOS input event + Status eventHandler(const EventTappingEventContextRef& ec, + const EventTappingSubscriptionContextRef& sc) { + // Process the intercepted macOS input event return Status::success(); } }; ``` -> **Note:** This publisher is macOS-only and requires the `ApplicationServices` framework. The process may need Accessibility permissions (`TCC`) for the event tap to function correctly in macOS 10.15+. \ No newline at end of file +## Notes + +- **macOS only** β€” depends on `ApplicationServices.framework` and the `CGEventTap` API +- The publisher manages a `CFRunLoopSourceRef` + `CFRunLoopRef` pair; `setUp()` installs the tap, `tearDown()`/`stop()` removes it +- `restart()` allows re-initializing the tap if it becomes disabled (e.g., due to a timeout imposed by macOS) +- Thread safety for the run loop is handled via an internal `Mutex` (`run_loop_mutex_`) \ No newline at end of file diff --git a/osquery/events/darwin/.fsevents.md b/osquery/events/darwin/.fsevents.md index 68064e595fb..8cd2c06a949 100644 --- a/osquery/events/darwin/.fsevents.md +++ b/osquery/events/darwin/.fsevents.md @@ -1,47 +1,48 @@ -Wraps Apple's FSEvents API as an osquery `EventPublisher`, enabling filesystem change monitoring with path globbing, recursive watching, and event filtering on macOS. +Defines the macOS FSEvents-based filesystem event publisher for osquery, wrapping Apple's `CoreServices` FSEvents API to deliver real-time filesystem change notifications to subscribers. ## Key Components ### `FSEventsSubscriptionContext` -Defines what to watch: -- `path` β€” filesystem path to monitor (supports glob wildcards) +Holds per-subscription configuration: +- `path` β€” filesystem path to watch - `mask` β€” optional `FSEventStreamEventFlags` filter -- `recursive` β€” enables recursive directory watching -- `category` β€” config-originated category label +- `recursive` β€” whether the watch is recursive (double-wildcard) +- `category` β€” originating config category - `requireAction()` β€” appends a required action filter -- Private fields `discovered_` and `recursive_match` used internally by the publisher for path resolution and event matching +- Private `discovered_` / `recursive_match` β€” resolved at configure-time for efficient re-matching ### `FSEventsEventContext` -Carries data for a fired event: +Carries per-event data fired to subscribers: - `fsevent_stream`, `fsevent_flags`, `transaction_id` β€” raw FSEvents metadata -- `path`, `action` β€” resolved path and human-readable action string +- `path`, `action` β€” normalized path and action string ### `FSEventsEventPublisher` Core publisher class inheriting `EventPublisher`: -| Method | Purpose | +| Member | Purpose | |---|---| -| `configure()` | Rebuilds watch paths and exclude sets on config change | -| `run()` | Starts the CFRunLoop-based event loop | -| `tearDown()` / `stop()` | Stops the stream and cleans up | -| `Callback()` | Static FSEvents client callback; fires event contexts | -| `shouldFire()` | Filters events against subscription criteria | +| `configure()` | Rebuilds watched path set on config load/update | +| `run()` | Enters the `CFRunLoop` event loop | +| `tearDown()` / `stop()` | Gracefully halts stream and run loop | +| `Callback()` | Static FSEvents client callback; dispatches events | +| `shouldFire()` | Filters events against subscription context | | `transformSubscription()` | Resolves globs/wildcards at configure time | | `buildExcludePathsSet()` | Populates `ExcludePathSet` for suppressed paths | +| `flush()` | Forces kernel-buffered events to drain | ## Usage Example ```cpp // Subscribe to all changes under /etc recursively auto sc = std::make_shared(); -sc->path = "/etc"; +sc->path = "/etc/"; sc->recursive = true; -sc->category = "config_files"; sc->requireAction("UPDATED"); -// The publisher matches fired events against this context via shouldFire() -// and delivers FSEventsEventContext to the subscriber callback -``` - -> **Note:** This publisher is macOS-only and requires `CoreServices.framework`. Path inputs should use filesystem glob syntax (e.g., `**`), not SQL wildcards. \ No newline at end of file +// Publisher is registered automatically via DECLARE_PUBLISHER("fsevents") +// Subscribers receive FSEventsEventContext on match: +void eventCallback(const FSEventsEventContextRef& ec) { + std::cout << ec->action << ": " << ec->path << "\n"; +} +``` \ No newline at end of file diff --git a/osquery/events/darwin/.iokit.md b/osquery/events/darwin/.iokit.md index d99f0934fde..f84d596499c 100644 --- a/osquery/events/darwin/.iokit.md +++ b/osquery/events/darwin/.iokit.md @@ -1,57 +1,50 @@ -Provides the `IOKitEventPublisher` class and associated context structures for subscribing to and handling macOS IOKit hardware device attach/detach events within the osquery event framework. +Provides the IOKit event publisher interface for monitoring macOS hardware device attach/detach events using Apple's IOKit framework. ## Key Components ### Structs -| Struct | Description | -|---|---| -| `IOKitSubscriptionContext` | Subscription filter with `model_id`, `vendor_id`, and bus `type` (e.g., `"USB"`) | -| `IOKitEventContext` | Event payload carrying device metadata: `action`, `vendor`, `model`, `path`, `driver`, `version`, `serial` | +- **`IOKitSubscriptionContext`** β€” Subscription filter context; supports filtering by `model_id`, `vendor_id`, and bus `type` (e.g., `"USB"`) +- **`IOKitEventContext`** β€” Event payload containing device metadata: `action`, `type`, `vendor`, `model`, `vendor_id`, `model_id`, `path`, `driver`, `version`, and `serial` +- **`DeviceTracker`** β€” Internal struct (forward-declared) used to track active device detach notifications ### Enums - **`IOKitEventContext::Action`** β€” `DEVICE_ATTACH (0)` or `DEVICE_DETACH` -### Type Aliases - -- `IOKitEventContextRef` β€” `std::shared_ptr` -- `IOKitSubscriptionContextRef` β€” `std::shared_ptr` - ### Class: `IOKitEventPublisher` -Extends `EventPublisher`. Manages a `CFRunLoop` and `IONotificationPort` to listen for IOKit device notifications. +Core publisher registered as `"iokit"`, extending `EventPublisher`. | Method | Description | -|---|---| -| `run()` | Starts the CFRunLoop and notification monitoring | -| `tearDown()` | Cleans up run loop and IOKit resources | -| `shouldFire()` | Filters events against subscription context | -| `deviceAttach()` *(static)* | IOKit callback for device connect events | -| `deviceDetach()` *(static)* | IOKit callback for device disconnect events | -| `newEvent()` | Constructs and fires an `IOKitEventContext` | -| `restart()` | Resets the run loop and re-registers notifications | +|--------|-------------| +| `run()` | Starts the IOKit CFRunLoop and registers device notifications | +| `tearDown()` | Cleans up run loop and IOKit port resources | +| `stop()` | Signals the run loop to halt | +| `restart()` | Rebuilds the notification port and iterators | +| `shouldFire()` | Filters events against subscription context filters | +| `deviceAttach()` *(static)* | IOKit callback for device connection events | +| `deviceDetach()` *(static)* | IOKit callback for device removal events | +| `newEvent()` | Constructs and fires an `IOKitEventContext` for a given device action | ## Usage Example -```cpp -// Subscribe to USB device events +```c +// Define a subscriber filtering for USB devices from a specific vendor auto sc = std::make_shared(); -sc->type = "USB"; +sc->type = "USB"; sc->vendor_id = "05ac"; // Apple vendor ID -EventFactory::addSubscription( - "iokit", - Subscription::create("usb_monitor", sc, - [](const EventContextRef& ec, const void*) { - auto event = std::static_pointer_cast(ec); - if (event->action == IOKitEventContext::DEVICE_ATTACH) { - LOG(INFO) << "Device attached: " << event->model - << " path=" << event->path; - } - }) -); +// Subscribe to iokit events +EventFactory::addSubscription("iokit", sc, callback); + +// Event context fields available in callback +void callback(const IOKitEventContextRef& ec) { + if (ec->action == IOKitEventContext::DEVICE_ATTACH) { + // ec->vendor, ec->model, ec->serial, ec->path, etc. + } +} ``` -> **Note:** `publisher_started_` is intentionally set `false` during initial iterator seeding to suppress synthetic attach events at startup β€” only real plug/unplug events are emitted after `restart()` completes. \ No newline at end of file +> **Note:** `publisher_started_` is set to `false` during initial iterator seeding to suppress spurious attach events at startup β€” only post-seed events are emitted to subscribers. \ No newline at end of file diff --git a/osquery/events/darwin/.openbsm.md b/osquery/events/darwin/.openbsm.md index 4e4167c36be..70d70c9e1bb 100644 --- a/osquery/events/darwin/.openbsm.md +++ b/osquery/events/darwin/.openbsm.md @@ -1,37 +1,43 @@ -Defines the OpenBSM event publisher and associated context structures for subscribing to and processing macOS/BSD audit log events within the osquery eventing framework. +Header file defining the OpenBSM audit event publisher for osquery, enabling subscription to and processing of BSM (Basic Security Module) audit log events on macOS/BSD systems. ## Key Components -| Name | Type | Description | -|---|---|---| -| `OpenBSMSubscriptionContext` | Struct | Holds the `event_id` a subscriber wants to monitor (e.g., `23` for `execve`) | -| `OpenBSMEventContext` | Struct | Carries a fired event's `event_id`, parsed `tokenstr_t` tokens, and a shared memory buffer | -| `OpenBSMEventContextRef` | Type alias | `std::shared_ptr` | -| `OpenBSMSubscriptionContextRef` | Type alias | `std::shared_ptr` | -| `OpenBSMEventPublisher` | Class | Core publisher that reads from the audit pipe, filters events by subscribed IDs, and fires matching event contexts | - -### `OpenBSMEventPublisher` Methods - -| Method | Description | -|---|---| -| `setUp()` | Initializes the publisher | -| `configure()` | Rebuilds the set of subscribed event IDs | -| `tearDown()` | Closes the audit pipe and cleans up resources | -| `run()` | Polling loop β€” reads audit records until interrupted | -| `acquireMessages()` | Dequeues and parses raw records from the audit pipe | -| `configureAuditPipe()` | Opens and configures `/dev/auditpipe` | -| `shouldFire()` | Matches incoming events against subscription contexts | +### Structs +- **`OpenBSMSubscriptionContext`** β€” Subscription filter context; holds `event_id` (e.g., `23` for `execve`) to match specific BSM audit event types +- **`OpenBSMEventContext`** β€” Event payload carrying `event_id`, a vector of parsed `tokenstr_t` tokens, and a shared buffer pointing to raw OpenBSM memory + +### Type Aliases +- **`OpenBSMEventContextRef`** β€” `std::shared_ptr` +- **`OpenBSMSubscriptionContextRef`** β€” `std::shared_ptr` + +### Classes +- **`OpenBSMEventPublisher`** β€” Core publisher inheriting from `EventPublisher`; manages the audit pipe lifecycle and dispatches matching events to subscribers + + | Method | Description | + |---|---| + | `setUp()` | Initializes the audit pipe connection | + | `configure()` | Rebuilds the set of subscribed event IDs | + | `tearDown()` | Closes the audit pipe and cleans up | + | `run()` | Polling loop; reads from the audit descriptor | + | `acquireMessages()` | Dequeues and parses raw BSM records | + | `shouldFire()` | Filters events against subscription context | ## Usage Example -```c -// Subscribe to execve events (event_id = 23) +```cpp +// Define a subscriber for execve events (BSM event ID 23) auto sub_ctx = std::make_shared(); -sub_ctx->event_id = 23; - -// Register the subscription β€” the publisher handles routing -EventFactory::addSubscription("openbsm", sub_ctx, callback); +sub_ctx->event_id = 23; // AUE_EXECVE + +// Subscriber callback receives OpenBSMEventContext +Status MySubscriber::Callback(const OpenBSMEventContextRef& ec, + const OpenBSMSubscriptionContextRef& sc) { + for (const auto& token : ec->tokens) { + // Process individual BSM tokens + } + return Status::success(); +} ``` -> **Note:** `OpenBSMEventPublisher` is a dispatched service β€” `run()` is called by the osquery eventing thread, not directly by subscribers. Access to `audit_pipe_` and `event_ids_` is guarded by dedicated `Mutex` members. \ No newline at end of file +> **Note:** The publisher uses two separate mutexes β€” `audit_pipe_mutex_` and `event_ids_mutex_` β€” to guard concurrent access to the audit pipe and the event ID set respectively. `OpenBSMConsumerRunner` is a forward-declared dispatched service that handles published replies asynchronously. \ No newline at end of file diff --git a/osquery/events/darwin/.scnetwork.md b/osquery/events/darwin/.scnetwork.md index 4f5dc1b2f1c..0e9c925b335 100644 --- a/osquery/events/darwin/.scnetwork.md +++ b/osquery/events/darwin/.scnetwork.md @@ -3,44 +3,41 @@ Provides the osquery event publisher interface for monitoring macOS network reac ## Key Components -### Enumerations - -- **`SCNetworkSubscriptionType`** β€” Defines target types: `ADDRESS_TARGET` (IP address) or `NAME_TARGET` (hostname) +### Enums +- **`SCNetworkSubscriptionType`** β€” Specifies the target type: `ADDRESS_TARGET` (IP address) or `NAME_TARGET` (hostname) ### Structs - -- **`SCNetworkSubscriptionContext`** β€” Subscription configuration containing target type, hostname/address string, address family, and a reachability flags bitmask filter -- **`SCNetworkEventContext`** β€” Event payload carrying the originating subscription context and observed `SCNetworkReachabilityFlags` +- **`SCNetworkSubscriptionContext`** β€” Subscription configuration holding target type, hostname/address string, address family, and a reachability flags bitmask filter +- **`SCNetworkEventContext`** β€” Event payload carrying the originating subscription context and the current `SCNetworkReachabilityFlags` ### Class: `SCNetworkEventPublisher` - -An `EventPublisher` subclass registered under the `"scnetwork"` identifier. Manages a CoreFoundation run loop with registered reachability targets. +Extends `EventPublisher`. Key members: | Method | Description | |---|---| -| `configure()` | Rebuilds and registers reachability targets | -| `run()` | Starts the CFRunLoop for async callbacks | -| `tearDown()` | Cleans up all registered targets | +| `configure()` | Registers hostname/address targets with the SCNetwork API | +| `run()` | Starts the `CFRunLoop` for receiving reachability callbacks | +| `tearDown()` | Cleans up registered targets and run loop | +| `Callback()` (static) | SCNetwork client callback invoked on reachability changes | | `shouldFire()` | Filters events against subscription flag masks | -| `Callback()` (static) | SCNetwork callback invoked on reachability change | -| `addHostname()` / `addAddress()` | Register name or address targets | -| `clearAll()` | Removes all tracked targets and contexts | +| `addHostname()` / `addAddress()` | Register individual targets | +| `clearAll()` | Removes all registered reachability targets | + +**Private state:** vectors of target names, addresses, `SCNetworkReachabilityRef` handles, associated contexts, a `CFRunLoopRef`, and a mutex for thread safety. ## Usage Example ```cpp -// Define a subscription context to monitor a hostname +// Define a subscription to monitor a hostname auto sc = std::make_shared(); -sc->type = NAME_TARGET; +sc->type = SCNetworkSubscriptionType::NAME_TARGET; sc->target = "example.com"; sc->mask = kSCNetworkReachabilityFlagsReachable; -// Subscribe an event subscriber using this context -auto sub = EventFactory::createSubscription("scnetwork", sc); +// Subscribe and receive SCNetworkEventContext events +// when reachability flags change for the given target +auto sub = SCNetworkEventPublisher::createSubscriptionContext(); +EventFactory::addSubscription("scnetwork", sub); ``` -## Notes - -- macOS-only; requires `SystemConfiguration.framework` -- `setUp()` always returns `Status(1, "Publisher not used")` β€” publisher lifecycle is driven by `configure()` and `run()` -- Internal state is protected by a `Mutex` for thread-safe target management \ No newline at end of file +> **Note:** `setUp()` always returns a failure status β€” this publisher is configured exclusively via `configure()` and the static `Callback()` delegate, not a polling loop. \ No newline at end of file diff --git a/osquery/events/linux/.auditdnetlink.md b/osquery/events/linux/.auditdnetlink.md index d1fb75c544f..b51e9ea0adb 100644 --- a/osquery/events/linux/.auditdnetlink.md +++ b/osquery/events/linux/.auditdnetlink.md @@ -1,43 +1,47 @@ -Header file defining the audit netlink interface for osquery's Linux audit subsystem integration, providing classes and data structures for reading, parsing, and consuming kernel audit events via the Linux netlink socket. +Header defining the audit netlink interface for reading and parsing Linux audit events in osquery. ## Key Components ### Enums & Type Aliases -- **`NetlinkStatus`** β€” Enum indicating netlink handle state: `ActiveMutable`, `ActiveImmutable`, `Disabled`, or `Error` +- **`NetlinkStatus`** β€” Enum describing the audit handle state: `ActiveMutable`, `ActiveImmutable`, `Disabled`, `Error` - **`AuditRuleDataObject`** β€” `std::vector` wrapping raw audit rule data -- **`AuditdContextRef`** β€” `shared_ptr` for shared ownership across services +- **`AuditdContextRef`** β€” `shared_ptr` shared between reader and parser threads ### Structs -- **`AuditEventRecord`** β€” A parsed audit event containing type, timestamp, audit ID, key-value fields, and raw message data -- **`AuditdContext`** β€” Thread-safe shared state between reader and parser services; holds unprocessed/processed queues, mutexes, condition variables, and throttling counters +- **`AuditEventRecord`** β€” Parsed audit event with `type`, `time`, `audit_id`, `fields` map, and `raw_data` for SELinux/AppArmor records +- **`AuditdContext`** β€” Thread-safe shared state holding unprocessed `audit_reply` records, parsed `AuditEventRecord` events, synchronization primitives (`mutex`, `condition_variable`), and throttling counters ### Classes -- **`AuditdNetlinkReader`** β€” `InternalRunnable` service that acquires the netlink handle, configures audit rules, reads raw `audit_reply` records, and manages audit service lifecycle -- **`AuditdNetlinkParser`** β€” `InternalRunnable` service that parses raw `audit_reply` structures into `AuditEventRecord` objects; exposes static `ParseAuditReply()` and `AdjustAuditReply()` utilities -- **`AuditdNetlink`** β€” Top-level consumer interface; calls `getEvents()` to retrieve fully processed `AuditEventRecord` vectors +- **`AuditdNetlinkReader`** (`InternalRunnable`) β€” Background service that acquires the netlink handle, configures audit rules, reads raw `audit_reply` messages, and manages rule lifecycle (`configureAuditService`, `restoreAuditServiceConfiguration`) +- **`AuditdNetlinkParser`** (`InternalRunnable`) β€” Background service that parses raw `audit_reply` structs into `AuditEventRecord` objects; exposes two static utilities: `ParseAuditReply` and `AdjustAuditReply` +- **`AuditdNetlink`** β€” Consumer-facing facade; calls `getEvents()` to retrieve fully parsed `AuditEventRecord` vectors from the shared context -### Utility -- **`DecodeAuditPathValues()`** β€” Inline function that strips surrounding quotes or hex-decodes audit field values using `boost::algorithm::unhex` +### Inline Utility +- **`DecodeAuditPathValues`** β€” Strips surrounding quotes from quoted strings or hex-decodes raw audit field values using `boost::algorithm::unhex` ## Usage Example ```cpp #include "auditdnetlink.h" -// Initialize shared context and the top-level netlink interface +// Instantiate the netlink interface (starts reader/parser threads internally) osquery::AuditdNetlink netlink; -// Poll for new audit events in a publisher loop -while (running) { - std::vector events = netlink.getEvents(); - for (const auto& record : events) { - // Access parsed fields - auto it = record.fields.find("exe"); - if (it != record.fields.end()) { - std::string exe = osquery::DecodeAuditPathValues(it->second); - // process exe path... - } +// Poll for parsed audit events +std::vector events = netlink.getEvents(); + +for (const auto& record : events) { + // Access structured fields + auto it = record.fields.find("syscall"); + if (it != record.fields.end()) { + std::cout << "Syscall: " << it->second << "\n"; + } + + // Decode a path field + if (record.fields.count("name")) { + std::string path = osquery::DecodeAuditPathValues(record.fields.at("name")); + std::cout << "Path: " << path << "\n"; } } ``` \ No newline at end of file diff --git a/osquery/events/linux/.auditeventpublisher.md b/osquery/events/linux/.auditeventpublisher.md index b3b36217a6a..110b807bbad 100644 --- a/osquery/events/linux/.auditeventpublisher.md +++ b/osquery/events/linux/.auditeventpublisher.md @@ -1,60 +1,51 @@ -Header file defining the Linux audit event publisher for osquery, providing data structures and interfaces for processing kernel audit events including syscalls, AppArmor, SELinux, and seccomp notifications via the Linux Audit netlink subsystem. +Header defining the Linux audit event publisher for osquery, providing data structures and interfaces for processing kernel audit events (syscalls, AppArmor, seccomp, user events) via the Linux audit netlink subsystem. ## Key Components ### Event Data Structures - -| Struct | Description | -|---|---| -| `UserAuditEventData` | Holds a user-space audit event ID | -| `SyscallAuditEventData` | Captures syscall metadata: PID, UID/GID sets, executable path, success status | -| `AppArmorAuditEventData` | Key-value fields for AppArmor audit records (profile, operation, masks, etc.) | -| `SeccompAuditEventData` | Key-value fields for seccomp audit records (arch, syscall, signal, instruction pointer, etc.) | -| `AuditEvent` | Top-level event descriptor wrapping all event types via `boost::variant` with a raw `record_list` | - -### Publisher Infrastructure - -| Type | Description | -|---|---| -| `AuditEventPublisher` | Core `EventPublisher` subclass; manages netlink I/O, assembles audit records into events | -| `AuditEventContext` | Carries a `vector` dispatched to subscribers | -| `AuditSubscriptionContext` | Subscription handle (opaque; controlled by the publisher) | -| `AuditTraceContext` | `map` tracking in-flight multi-record audit sequences | - -### Free Functions - -| Function | Description | +- **`UserAuditEventData`** β€” Holds a user-space audit event ID +- **`SyscallAuditEventData`** β€” Captures syscall metadata: number, success, PID, UID/GID sets, and executable path +- **`AppArmorAuditEventData`** β€” Key/value fields for AppArmor audit records (profile, operation, masks, capabilities) +- **`SeccompAuditEventData`** β€” Key/value fields for seccomp audit records (PID, syscall, arch, signal, instruction pointer) +- **`AuditEvent`** β€” Top-level descriptor combining a `Type` enum (`UserEvent`, `Syscall`, `SELinux`, `AppArmor`, `Seccomp`), a `boost::variant` data payload, and a raw record list + +### Publisher Class +- **`AuditEventPublisher`** β€” Core publisher implementing `setUp()`, `configure()`, `tearDown()`, and `run()` lifecycle methods; owns the `AuditdNetlink` reader and the `AuditTraceContext` assembly map +- **`ProcessEvents()`** β€” Static method aggregating raw `AuditEventRecord` lists into structured `AuditEvent` objects + +### Helper Functions +| Function | Purpose | |---|---| -| `ProcessEvents()` | Aggregates raw `AuditEventRecord` lists into typed `AuditEvent` objects | -| `GetEventRecord()` | Extracts a record of a specific type from an `AuditEvent` | -| `GetStringFieldFromMap()` | Safe string field lookup with default value | -| `GetIntegerFieldFromMap()` | Safe integer field lookup with configurable base and default | +| `GetEventRecord()` | Extracts a record by type from an `AuditEvent` | +| `GetStringFieldFromMap()` | Safe string field extraction with default | +| `GetIntegerFieldFromMap()` | Safe integer field extraction with base/default | | `CopyFieldFromMap()` | Copies a named field into an osquery `Row` | -| `StripQuotes()` | Removes surrounding quotes from audit field values | +| `StripQuotes()` | Removes surrounding quotes from a string value | | `parseSeccompEvent()` | Parses a raw seccomp record into `SeccompAuditEventData` | ## Usage Example ```cpp -// Subscribing to audit events -auto sub_ctx = AuditEventPublisher::createSubscriptionContext(); -EventFactory::addSubscription("auditeventpublisher", sub_ctx, - [](const AuditEventContextRef& ctx, const AuditSubscriptionContextRef&) { - for (const auto& event : ctx->audit_events) { - if (event.type == AuditEvent::Type::Syscall) { +#include + +// Process incoming audit records into structured events +AuditEventContextRef ctx = std::make_shared(); +AuditTraceContext traceCtx; +std::set allowedFailures = {59}; // e.g., execve + +AuditEventPublisher::ProcessEvents( + ctx, rawRecordList, traceCtx, allowedFailures); + +for (const auto& event : ctx->audit_events) { + if (event.type == AuditEvent::Type::Syscall) { const auto& data = boost::get(event.data); - LOG(INFO) << "Syscall " << data.syscall_number - << " by PID " << data.process_id - << " exe: " << data.executable_path; - } + std::cout << "Syscall " << data.syscall_number + << " by PID " << data.process_id << "\n"; } - return Status::success(); - }); +} -// Assembling raw records into typed events (used internally / in tests) -AuditEventContextRef ctx = std::make_shared(); -AuditTraceContext trace; -std::set allowed_failures = {257}; // e.g. openat allowed to fail -AuditEventPublisher::ProcessEvents(ctx, raw_records, trace, allowed_failures); +// Extract a specific field from a parsed record +std::string path; +GetStringFieldFromMap(path, record.fields, "exe", "/unknown"); ``` \ No newline at end of file diff --git a/osquery/events/linux/.inotify.md b/osquery/events/linux/.inotify.md index 920966d4189..9955076b240 100644 --- a/osquery/events/linux/.inotify.md +++ b/osquery/events/linux/.inotify.md @@ -1,44 +1,62 @@ -Linux `inotify`-based filesystem event publisher header for osquery, defining subscription contexts, event contexts, and the `INotifyEventPublisher` class used to monitor file and directory changes on Linux. +Linux `inotify`-based filesystem event publisher for osquery, providing watch descriptor management, subscription context handling, and event routing for file/directory monitoring. ## Key Components ### Constants & Type Aliases -- `kMaskActions` β€” maps `inotify` mask bits to human-readable action strings -- `kFileDefaultMasks` / `kFileAccessMasks` β€” predefined `inotify` event mask sets -- `PathDescriptorMap` / `DescriptorPathMap` β€” bidirectional path ↔ watch descriptor lookup maps -- `ExcludePathSet` β€” pattern-aware set of paths excluded from event propagation +- `kMaskActions` β€” maps `inotify` bitmask values to human-readable action strings +- `kFileDefaultMasks` / `kFileAccessMasks` β€” predefined event mask groups +- `PathDescriptorMap` / `DescriptorPathMap` β€” bidirectional path ↔ watch descriptor lookups +- `DescriptorINotifySubCtxMap` β€” maps watch descriptors to subscription contexts +- `ExcludePathSet` β€” patterned path set for suppressing unwanted events ### `INotifySubscriptionContext` -Subscription configuration struct holding the watched path, optional action mask, recursive flag, category label, and lazy-deletion marker. Provides `requireAction()` to map string action names to `inotify` bitmask values. +Subscription configuration struct. Fields include: +- `path` / `opath` β€” resolved and original config paths +- `mask` β€” `inotify` event filter bitmask (0 = all events) +- `recursive` β€” enables recursive directory watching +- `category` β€” config-originating category label +- `mark_for_deletion` β€” lazy subscription removal flag +- `requireAction(action)` β€” maps a string action name to the appropriate bitmask bit ### `INotifyEventContext` -Event payload struct carrying the raw `inotify_event`, resolved path string, action string, transaction ID, and a reference back to the originating subscription context. +Event payload struct carrying the raw `inotify_event`, resolved `path`, string `action`, and a reference back to the originating `INotifySubscriptionContextRef`. ### `INotifyEventPublisher` -Core publisher class (inherits `EventPublisher`) managing the `inotify` file descriptor lifecycle, watch descriptor bookkeeping, and event dispatch loop. Key methods: +Core publisher class inheriting `EventPublisher`. Key methods: | Method | Purpose | |---|---| -| `setUp()` | Opens the `inotify` handle | -| `configure()` | Applies updated subscriptions from config | -| `run()` | Main event polling loop | -| `addMonitor()` | Registers a path watch, optionally recursive | -| `needMonitoring()` | Guards `addMonitor` using status-change time | -| `removeMonitor()` | Unregisters a watch descriptor | -| `shouldFire()` | Matches an event against a subscription's path/mask | +| `setUp()` | Opens the `inotify` file descriptor | +| `configure()` | Applies updated subscription config | +| `tearDown()` | Releases the `inotify` handle and scratch buffer | +| `run()` | Main event loop thread entry point | +| `addMonitor()` | Registers an `inotify` watch on a path, optionally recursive | +| `removeMonitor()` | Deregisters a watch descriptor | +| `shouldFire()` | Matches an event context against a subscription context | | `handleOverflow()` | Recovers from `IN_Q_OVERFLOW` by increasing read batch size | ## Usage Example ```cpp -// Subscribe to file creation events under a directory -auto sc = std::make_shared(); -sc->path = "/etc/myapp/"; -sc->recursive = true; -sc->requireAction("CREATED"); - -// The publisher dispatches INotifyEventContext to matching subscribers -// ec->path β†’ absolute path of the affected file -// ec->action β†’ "CREATED", "UPDATED", "DELETED", etc. +// Define a subscriber using inotify contexts +class MySubscriber + : public EventSubscriber { + public: + Status init() override { + auto sc = createSubscriptionContext(); + sc->path = "/etc/hosts"; + sc->recursive = false; + sc->requireAction("UPDATED"); // Filter to write/modify events only + subscribe(&MySubscriber::handleEvent, sc); + return Status::success(); + } + + Status handleEvent(const INotifyEventContextRef& ec, + const INotifySubscriptionContextRef& sc) { + LOG(INFO) << "File changed: " << ec->path + << " Action: " << ec->action; + return Status::success(); + } +}; ``` \ No newline at end of file diff --git a/osquery/events/linux/.socket_events.md b/osquery/events/linux/.socket_events.md index e6e1e3b6741..5ab67534840 100644 --- a/osquery/events/linux/.socket_events.md +++ b/osquery/events/linux/.socket_events.md @@ -1,26 +1,20 @@ -Declares the interface for retrieving the set of system call numbers monitored by the socket events subsystem in osquery. +Declares the `getSocketEventsSyscalls` utility function for retrieving the set of system call identifiers related to socket events within the osquery framework. ## Key Components -- **`getSocketEventsSyscalls()`** β€” Returns a `std::set` containing the syscall numbers relevant to socket event monitoring (e.g., `connect`, `bind`, `accept`). +- **`getSocketEventsSyscalls()`** β€” Returns a `std::set` containing the syscall numbers that correspond to socket-related kernel events (e.g., `connect`, `bind`, `accept`). Used to configure event subscriptions or filters for network socket monitoring. ## Usage Example ```c #include "socket_events.h" -#include -// Retrieve all syscalls tracked for socket events +// Retrieve all socket-related syscall IDs std::set syscalls = osquery::getSocketEventsSyscalls(); -for (int syscall_nr : syscalls) { - std::cout << "Monitoring syscall: " << syscall_nr << std::endl; +for (int syscall_id : syscalls) { + // Register or filter events for each syscall + registerSyscallFilter(syscall_id); } -``` - -## Notes - -- Lives within the `osquery` namespace. -- Typically consumed by the audit-based event publisher to register which syscalls to subscribe to for network socket activity tracking. -- The returned set drives kernel-level audit rule registration β€” only syscalls in this set will generate socket events in the osquery event pipeline. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/events/linux/.syslog.md b/osquery/events/linux/.syslog.md index cd8b52ae6e8..9efc232ff9f 100644 --- a/osquery/events/linux/.syslog.md +++ b/osquery/events/linux/.syslog.md @@ -1,40 +1,47 @@ -Declares the event publisher and supporting types for ingesting syslog entries forwarded from rsyslog via a named pipe, publishing parsed log fields to osquery subscribers. +Defines the event publisher infrastructure for ingesting syslog entries forwarded via rsyslog through a named pipe, publishing parsed JSON log fields to osquery subscribers. ## Key Components | Symbol | Type | Description | -|---|---|---| -| `SyslogSubscriptionContext` | struct | Empty subscription context; no filtering criteria needed | -| `SyslogEventContext` | struct | Holds a syslog message as a `map` of parsed fields | -| `NonBlockingFStream` | class | Non-blocking pipe reader with internal ring buffer; exposes `openReadOnly`, `getline`, and `close` | -| `SyslogEventPublisher` | class | osquery event publisher; reads JSON syslog lines from a named pipe, parses them, and fires events | -| `RsyslogCsvSeparator` | class | Boost `TokenizerFunction` functor that handles rsyslog's CSV dialect (`""` escaping, no backslash escaping) | +|--------|------|-------------| +| `SyslogSubscriptionContext` | struct | Empty subscription context for syslog event filtering | +| `SyslogEventContext` | struct | Holds tokenized syslog fields as a `stringβ†’string` map | +| `NonBlockingFStream` | class | Non-blocking pipe reader with internal line-buffering; avoids blocking reads on named pipes | +| `SyslogEventPublisher` | class | EventPublisher that reads JSON syslog lines from a named pipe, parses them, and fires events to subscribers | +| `RsyslogCsvSeparator` | class | Boost `TokenizerFunction` functor handling rsyslog's CSV quirks (`""` escaping, no backslash escaping) | ## Usage Example ```cpp -// Subscribing to syslog events in an EventSubscriber +// Subscriber example β€” receive syslog events class MySyslogSubscriber : public EventSubscriber { public: Status init() override { - // No filter criteria needed; SyslogSubscriptionContext is empty auto sc = createSubscriptionContext(); - subscribe(&MySyslogSubscriber::onEvent, sc); + subscribe(&MySyslogSubscriber::handleEvent, sc); return Status::success(); } - Status onEvent(const SyslogEventContextRef& ec, - const SyslogSubscriptionContextRef&) { + Status handleEvent(const SyslogEventContextRef& ec, + const SyslogSubscriptionContextRef&) { // Access parsed syslog fields auto it = ec->fields.find("message"); if (it != ec->fields.end()) { - LOG(INFO) << "syslog message: " << it->second; + LOG(INFO) << "Syslog message: " << it->second; } return Status::success(); } }; + +// NonBlockingFStream standalone usage +osquery::NonBlockingFStream stream; +stream.openReadOnly("/var/osquery/syslog.pipe"); +std::string line; +while (stream.getline(line).ok()) { + // process line +} ``` -> **Note:** `SyslogEventPublisher` requires rsyslog to be configured to write JSON-formatted entries to the named pipe path defined in osquery's configuration. Only the first osquery process to acquire the pipe lock (`lockPipe`) will consume events; subsequent processes (e.g. `osqueryi`) are silently skipped to avoid corrupting `osqueryd` reads. \ No newline at end of file +> **Note:** `SyslogEventPublisher` requires rsyslog to be configured to forward JSON-formatted entries to the named pipe path configured in osquery. Only the first osquery process to acquire the pipe lock will read from it β€” concurrent `osqueryd`/`osqueryi` instances are prevented from splitting reads. \ No newline at end of file diff --git a/osquery/events/linux/.udev.md b/osquery/events/linux/.udev.md index 2dcfc678f43..70fe15027a1 100644 --- a/osquery/events/linux/.udev.md +++ b/osquery/events/linux/.udev.md @@ -1,49 +1,47 @@ -Defines the Linux `udev` event publisher interface for osquery, enabling subscribers to monitor hardware device events (add, remove, change) via the `libudev` API. +Defines the Linux `udev` event publisher interface for osquery, enabling hardware device event monitoring (add/remove/change) via the `libudev` kernel device management subsystem. ## Key Components -### Enums -- **`udev_event_action`** β€” Device event types: `ADD`, `REMOVE`, `CHANGE`, `UNKNOWN`, and `ALL` (subscriber catch-all) +### Enumerations +- **`udev_event_action`** β€” Device event action types: `ADD`, `REMOVE`, `CHANGE`, `UNKNOWN`, and `ALL` (subscriber catch-all) ### Structs -- **`UdevSubscriptionContext`** β€” Subscription filter criteria; restrict events by `action`, `subsystem`, `devnode`, `devtype`, or `driver` -- **`UdevEventContext`** β€” Fired event payload carrying a `udev_device*` pointer, resolved `action`, and device metadata strings +- **`UdevSubscriptionContext`** β€” Subscription filter criteria: action type, subsystem, devnode, devtype, and driver name +- **`UdevEventContext`** β€” Event payload: raw `udev_device*` pointer, action enum, action string, and device metadata fields ### Type Aliases - **`UdevEventContextRef`** β€” `std::shared_ptr` - **`UdevSubscriptionContextRef`** β€” `std::shared_ptr` ### Class: `UdevEventPublisher` -Extends `EventPublisher` and manages the udev socket monitor lifecycle. +Extends `EventPublisher`. | Method | Description | -|---|---| +|--------|-------------| | `setUp()` | Initializes the udev handle and monitor | | `tearDown()` | Releases udev resources | -| `run()` | Polls the monitor and dispatches events | -| `getValue(device, property)` | Returns a udev property string | -| `getAttr(device, attr)` | Returns a udev sysattr string | -| `shouldFire(...)` | Filters events against subscription context | -| `createEventContextFrom(device)` | Builds an `UdevEventContextRef` from a raw device pointer | +| `run()` | Main event loop; polls the udev monitor socket | +| `getValue(device, property)` | Returns a udev property string for a device | +| `getAttr(device, attr)` | Returns a udev sysattr string for a device | +| `shouldFire(...)` | Matches subscription filters against incoming events | +| `createEventContextFrom(device)` | Builds an `UdevEventContext` from a raw `udev_device*` | ## Usage Example ```cpp -// Create a subscription context filtering USB add events +// Define a subscription context to watch USB block device additions auto sc = createSubscriptionContext(); -sc->action = UDEV_EVENT_ACTION_ADD; -sc->subsystem = "usb"; +sc->action = UDEV_EVENT_ACTION_ADD; +sc->subsystem = "block"; +sc->devtype = "disk"; -// Subscribe with a callback -subscribe(&MySubscriber::onUsbAdd, sc); +// Subscribe and handle the event +subscribe(&MySubscriber::Callback, sc); -// In the callback, inspect event context -void onUsbAdd(const UdevEventContextRef& ec) { - auto node = ec->devnode; // e.g. "/dev/bus/usb/001/002" - auto driver = ec->driver; - auto device = ec->device; // raw udev_device* for deeper inspection - - auto vendor = UdevEventPublisher::getValue(device, "ID_VENDOR"); +// Inside callback β€” read device metadata from context +void Callback(const UdevEventContextRef& ec) { + auto vendor = UdevEventPublisher::getValue(ec->device, "ID_VENDOR"); + auto sysfs_path = UdevEventPublisher::getAttr(ec->device, "size"); } ``` \ No newline at end of file diff --git a/osquery/events/linux/bpf/.bpferrorstate.md b/osquery/events/linux/bpf/.bpferrorstate.md index 219a15712c6..3051178d76a 100644 --- a/osquery/events/linux/bpf/.bpferrorstate.md +++ b/osquery/events/linux/bpf/.bpferrorstate.md @@ -1,38 +1,43 @@ -Defines the `BPFErrorState` structure and related utilities for tracking and reporting errors encountered during BPF-based event collection in the osquery BPF publisher. +Defines the `BPFErrorState` structure and utility functions for tracking and reporting errors in osquery's BPF (Berkeley Packet Filter) event publisher. ## Key Components ### `BPFErrorState` (struct) -A container aggregating all BPF publisher error counters: +Aggregates all error counters for the BPF publisher: | Field | Type | Description | -|---|---|---| +|-------|------|-------------| | `perf_error_counters` | `IPerfEventReader::ErrorCounters` | Errors from `perf_output` ring buffer reads | -| `probe_error_counter` | `std::size_t` | Failures from BPF probes (e.g. string/buffer capture failures) | +| `probe_error_counter` | `std::size_t` | Failures from loaded BPF probes (e.g., string/buffer capture failures) | | `errored_tracer_list` | `std::unordered_set` | IDs of tracers that produced unprocessable events | -### `updateBpfErrorState()` -Merges incoming `perf_output` error counters into an existing `BPFErrorState` instance. +### Functions -### `reportAndClearBpfErrorState()` -Logs the current error state (typically to the osquery logger) and resets all counters and sets to their zero/empty defaults. +- **`updateBpfErrorState`** β€” Merges new `perf_error_counters` into an existing `BPFErrorState` instance. +- **`reportAndClearBpfErrorState`** β€” Logs the current error state to the osquery logger and resets all counters/sets to zero. ## Usage Example ```c -BPFErrorState error_state{}; +#include "bpferrorstate.h" -// After each perf reader poll cycle, merge new counters in -IPerfEventReader::ErrorCounters new_counters = reader->getErrorCounters(); -updateBpfErrorState(error_state, new_counters); +osquery::BPFErrorState error_state; -// Periodically flush errors to logs and reset state -reportAndClearBpfErrorState(error_state); +// Called periodically as perf events are read +tob::ebpfpub::IPerfEventReader::ErrorCounters new_counters = reader->getErrorCounters(); +osquery::updateBpfErrorState(error_state, new_counters); + +// Mark a tracer as having produced bad events +error_state.probe_error_counter++; +error_state.errored_tracer_list.insert(tracer_id); + +// Flush errors to logs and reset state +osquery::reportAndClearBpfErrorState(error_state); ``` ## Notes -- `probe_error_counter` covers soft failures such as truncated strings in `open` syscall captures or malformed `sockaddr_in` buffers β€” not fatal crashes. -- `errored_tracer_list` deduplicates affected tracer IDs, making it useful for identifying persistently failing probes. -- Both functions operate on `BPFErrorState` by reference, making them safe to use with a long-lived state object across polling intervals. \ No newline at end of file +- Declared in the `osquery` namespace. +- Depends on `ebpfpub/iperfeventreader.h` from the `tob::ebpfpub` library. +- Probe errors typically indicate lost events or failed syscall argument capture (e.g., `open` path strings, `sockaddr_in` buffers). \ No newline at end of file diff --git a/osquery/events/linux/bpf/.bpfeventpublisher.md b/osquery/events/linux/bpf/.bpfeventpublisher.md index 1e344a84b2d..f96a686af21 100644 --- a/osquery/events/linux/bpf/.bpfeventpublisher.md +++ b/osquery/events/linux/bpf/.bpfeventpublisher.md @@ -1,54 +1,56 @@ -Header defining the `BPFEventPublisher` class for osquery's Linux BPF-based event publishing system, which captures and processes kernel syscall events via eBPF tracing. +BPF-based event publisher for osquery on Linux that captures and processes kernel-level system call events using eBPF tracing infrastructure. ## Key Components ### Structs -| Name | Base | Description | -|------|------|-------------| -| `BPFEventSC` | `SubscriptionContext` | Subscription context for BPF event subscriptions (opaque to subscribers) | -| `BPFEventEC` | `EventContext` | Event context carrying a list of system state events (`ISystemStateTracker::EventList`) | +| Name | Description | +|------|-------------| +| `BPFEventSC` | Subscription context for BPF events (opaque; managed by `BPFEventPublisher`) | +| `BPFEventEC` | Event context carrying a list of `ISystemStateTracker` events | ### Class: `BPFEventPublisher` -Extends `EventPublisher`. Manages the eBPF perf event reader lifecycle and dispatches processed syscall events to subscribers. - -**Lifecycle methods:** +Extends `EventPublisher` and implements the full publisher lifecycle. | Method | Description | |--------|-------------| | `setUp()` | Initializes eBPF tracers and perf event reader | -| `configure()` | Applies runtime configuration | -| `run()` | Poll loop consuming BPF perf events | +| `configure()` | Applies subscription configuration | +| `run()` | Polls eBPF perf events and dispatches to subscribers | | `tearDown()` | Cleans up eBPF resources | -**Static helper β€” `getEventMapValue`** - -Extracts a typed value from a `IFunctionTracer::Event::FieldMap` by key using `std::variant`. Returns `false` if the key is missing or the type doesn't match. - -**Static syscall processors** - -One handler per traced syscall, each updating an `ISystemStateTracker` with the parsed event: +### Static Helpers -- **Process lifecycle:** `processForkEvent`, `processVforkEvent`, `processCloneEvent`, `processExecveEvent`, `processExecveatEvent` -- **File descriptors:** `processCloseEvent`, `processDupEvent`, `processDup2Event`, `processDup3Event`, `processFcntlEvent` -- **File system:** `processCreatEvent`, `processMknodatEvent`, `processOpenEvent`, `processOpenatEvent`, `processOpenat2Event`, `processChdirEvent`, `processFchdirEvent`, `processNameToHandleAtEvent`, `processOpenByHandleAtEvent` -- **Networking:** `processSocketEvent`, `processConnectEvent`, `processAcceptEvent`, `processAccept4Event`, `processBindEvent`, `processListenEvent` +| Method | Description | +|--------|-------------| +| `getEventMapValue()` | Type-safe extraction of a named field from an eBPF event field map | +| `processForkEvent()` / `processVforkEvent()` / `processCloneEvent()` | Track process creation syscalls | +| `processExecveEvent()` / `processExecveatEvent()` | Track program execution | +| `processOpenEvent()` / `processOpenatEvent()` / `processOpenat2Event()` / `processCreatEvent()` | Track file open/create operations | +| `processDupEvent()` / `processDup2Event()` / `processDup3Event()` | Track file descriptor duplication | +| `processCloseEvent()` | Track file descriptor closure | +| `processSocketEvent()` / `processConnectEvent()` / `processAcceptEvent()` / `processBindEvent()` / `processListenEvent()` | Track socket lifecycle | +| `processChdirEvent()` / `processFchdirEvent()` | Track working directory changes | +| `processFcntlEvent()` | Track file control operations | ## Usage Example ```cpp -// Retrieve a typed field from a BPF event map -uint64_t fd = 0; -bool ok = BPFEventPublisher::getEventMapValue( - fd, event.field_map, "fd"); - -if (ok) { - // Use fd value +// Subscribing to BPF events +auto sc = std::make_shared(); +EventFactory::addSubscription("BPFEventPublisher", sc, + [](const BPFEventEC& ec, const BPFEventSC&) { + for (const auto& event : ec.event_list) { + // handle system state event + } + return Status::success(); + }); + +// Extracting a field from an eBPF event inside a process handler +uint64_t pid = 0; +if (BPFEventPublisher::getEventMapValue(pid, event.field_map, "pid")) { + // use pid } - -// Dispatch a clone syscall event to the state tracker -bool handled = BPFEventPublisher::processCloneEvent( - stateTracker, bpfEvent); ``` \ No newline at end of file diff --git a/osquery/events/linux/bpf/.filesystem.md b/osquery/events/linux/bpf/.filesystem.md index b295a6a391e..5ba269a48c8 100644 --- a/osquery/events/linux/bpf/.filesystem.md +++ b/osquery/events/linux/bpf/.filesystem.md @@ -1,48 +1,46 @@ -Concrete implementation of the `IFilesystem` interface for Linux BPF event processing, providing real filesystem operations backed by system calls. +Concrete implementation of the `IFilesystem` interface providing real filesystem operations for Linux BPF event processing within osquery. ## Key Components **`Filesystem`** β€” Final class implementing `IFilesystem` with the following methods: | Method | Description | -|---|---| -| `open()` | Opens a file at the given path with specified flags, returning a unique fd | +|--------|-------------| +| `open()` | Opens a file by path with specified flags, returning a unique fd | | `openAt()` | Opens a file relative to a directory fd (`dirfd`) | -| `readLinkAt()` | Reads a symbolic link target relative to a directory fd | +| `readLinkAt()` | Resolves a symbolic link path relative to a directory fd | | `read()` | Reads up to `max_size` bytes from an open fd into a buffer | | `enumFiles()` | Enumerates files in a directory, invoking a callback per entry | | `fileExists()` | Checks whether a named file exists within a directory fd | -> Construction is private (`Filesystem() = default`), enforcing instantiation exclusively through the `IFilesystem` factory (friend class pattern). +The constructor is private β€” instantiation is controlled exclusively by `IFilesystem` (factory pattern via `friend class IFilesystem`). ## Usage Example ```cpp #include -// Instantiate via the IFilesystem factory (not directly) -auto fs = IFilesystem::create(); +// Obtain instance through the IFilesystem factory +auto fs = osquery::IFilesystem::create(); -// Open a file +// Open a file and read its contents tob::utils::UniqueFd fd; -if (fs->open(fd, "/proc/self/maps", O_RDONLY)) { +if (fs->open(fd, "/proc/self/status", O_RDONLY)) { std::vector buffer; - fs->read(buffer, fd.get(), 4096); + if (fs->read(buffer, fd.get(), 4096)) { + // process buffer + } } -// Check file existence relative to a directory fd +// Check if a file exists relative to a directory fd bool exists = false; fs->fileExists(exists, AT_FDCWD, "config.json"); // Enumerate directory entries fs->enumFiles(AT_FDCWD, [](const std::string& name) { - // process each entry + // handle each entry }); ``` -## Notes - -- Scoped to the `osquery` namespace -- Designed for Linux only (BPF event subsystem) -- All methods return `bool` indicating success/failure; output is via reference parameters \ No newline at end of file +> **Note:** This class is Linux-only and intended for use within osquery's BPF event subsystem. Direct instantiation is not permitted β€” always obtain instances via the `IFilesystem` interface factory. \ No newline at end of file diff --git a/osquery/events/linux/bpf/.processcontextfactory.md b/osquery/events/linux/bpf/.processcontextfactory.md index dad39b819ef..3b1ace636c2 100644 --- a/osquery/events/linux/bpf/.processcontextfactory.md +++ b/osquery/events/linux/bpf/.processcontextfactory.md @@ -1,51 +1,44 @@ -Concrete implementation of `IProcessContextFactory` for capturing Linux process context data by reading `/proc` filesystem entries via BPF event handling. +Concrete implementation of `IProcessContextFactory` for capturing Linux process context data via BPF event handling in osquery. ## Key Components -### Class: `ProcessContextFactory` - -Final implementation of `IProcessContextFactory` with both instance and static utility methods. +### `ProcessContextFactory` (class) +Final class implementing `IProcessContextFactory`, providing both instance and static methods for process context capture. **Instance Methods** - -| Method | Description | -|---|---| -| `captureSingleProcess(ProcessContext&, pid_t)` | Captures context for a single process by PID | -| `captureAllProcesses(ProcessContextMap&)` | Captures context for all running processes | +- `captureSingleProcess(ProcessContext&, pid_t)` β€” Captures context for a single process by PID +- `captureAllProcesses(ProcessContextMap&)` β€” Enumerates and captures context for all running processes **Static Utility Methods** - -| Method | Description | -|---|---| -| `captureSingleProcess(IFilesystem&, ProcessContext&, pid_t)` | Filesystem-injectable single process capture | -| `captureAllProcesses(IFilesystem&, ProcessContextMap&)` | Filesystem-injectable bulk process capture | -| `getArgvFromCmdlineFile(IFilesystem&, vector&, int fd)` | Parses argv from `/proc//cmdline` | -| `getParentPidFromStatFile(IFilesystem&, pid_t&, int fd)` | Extracts parent PID from `/proc//stat` | +- `captureSingleProcess(IFilesystem&, ProcessContext&, pid_t)` β€” Filesystem-injected variant for single process capture +- `captureAllProcesses(IFilesystem&, ProcessContextMap&)` β€” Filesystem-injected variant for bulk capture +- `getArgvFromCmdlineFile(IFilesystem&, std::vector&, int fd)` β€” Parses `/proc//cmdline` into an argv vector +- `getParentPidFromStatFile(IFilesystem&, pid_t&, int fd)` β€” Extracts parent PID from `/proc//stat` **Private Members** -- `IFilesystem::Ref fs` β€” injected filesystem abstraction for testability -- Private constructor (instantiated via `IProcessContextFactory` friend) +- `fs` (`IFilesystem::Ref`) β€” Injected filesystem abstraction used by instance methods +- Private constructor β€” instantiation is controlled via `IProcessContextFactory` (friend class) ## Usage Example -```c -// Typically instantiated through IProcessContextFactory -auto factory = IProcessContextFactory::create(fs_ref); +```cpp +// Factory instantiation is managed by IProcessContextFactory +auto factory = IProcessContextFactory::create(fs); // Capture a single process by PID -ProcessContext ctx; +osquery::ProcessContext ctx; if (factory->captureSingleProcess(ctx, 1234)) { - // ctx now contains argv, parent PID, etc. + // ctx is populated with argv, parent PID, etc. } // Capture all running processes -ProcessContextMap all_procs; -factory->captureAllProcesses(all_procs); +osquery::ProcessContextMap process_map; +factory->captureAllProcesses(process_map); -// Static form β€” useful in tests with mock filesystem -ProcessContext ctx2; -ProcessContextFactory::captureSingleProcess(mock_fs, ctx2, 1234); +// Static helper: parse cmdline directly +std::vector argv; +osquery::ProcessContextFactory::getArgvFromCmdlineFile(fs, argv, fd); ``` -> **Note:** The private constructor and `friend class IProcessContextFactory` pattern enforces creation through the factory interface, enabling dependency injection of `IFilesystem` for unit testing. \ No newline at end of file +> **Note:** Construction is restricted via a private constructor and `friend class IProcessContextFactory`, enforcing factory-pattern instantiation. The `IFilesystem` abstraction enables testability by allowing mock filesystem injection. \ No newline at end of file diff --git a/osquery/events/linux/bpf/.serializers.md b/osquery/events/linux/bpf/.serializers.md index e55d7396631..825c552aa11 100644 --- a/osquery/events/linux/bpf/.serializers.md +++ b/osquery/events/linux/bpf/.serializers.md @@ -1,28 +1,31 @@ -Declares the `ParameterListMap` type and the `kParameterListMap` lookup table used to map eBPF function tracer names to their corresponding parameter lists. +Defines the `ParameterListMap` type alias and declares the `kParameterListMap` constant, which maps syscall/function names to their eBPF parameter list definitions for use in osquery's Linux event serialization pipeline. ## Key Components | Symbol | Type | Description | -|---|---|---| -| `ParameterListMap` | Type alias | `unordered_map` keyed by `std::string`, mapping to `IFunctionTracer::ParameterList` | -| `kParameterListMap` | `extern const` | Global registry of eBPF-traced function names to their parameter definitions | +|--------|------|-------------| +| `ParameterListMap` | Type alias | `unordered_map` keyed by `std::string`, mapping to `tob::ebpfpub::IFunctionTracer::ParameterList` | +| `kParameterListMap` | `extern const` | Global map of function/syscall names to their expected eBPF parameter lists | ## Usage Example -```cpp +```c #include "serializers.h" -// Look up parameters for a known syscall tracer -auto it = osquery::kParameterListMap.find("open"); +// Look up parameter list for a specific syscall +auto it = osquery::kParameterListMap.find("openat"); if (it != osquery::kParameterListMap.end()) { - const auto& params = it->second; - // iterate or pass params to a function tracer + const auto& paramList = it->second; + // Use paramList to deserialize eBPF-captured syscall arguments } + +// Check if a function is tracked +bool isTracked = osquery::kParameterListMap.count("execve") > 0; ``` -## Notes +## Dependencies + +- **`ebpfpub/ifunctiontracer.h`** β€” Provides `tob::ebpfpub::IFunctionTracer::ParameterList`, the eBPF parameter descriptor type from the [`ebpfpub`](https://github.com/trailofbits/ebpfpub) library. -- Depends on `ebpfpub/ifunctiontracer.h` from the `tob::ebpfpub` namespace. -- `kParameterListMap` is defined externally (in the corresponding `.cpp`); this header only declares it. -- Intended for use within osquery's eBPF event tracing subsystem to drive syscall parameter serialization. \ No newline at end of file +> The actual contents of `kParameterListMap` are defined in the corresponding `.cpp` implementation file, keeping syscall-to-parameter mappings centralized and reusable across serializers. \ No newline at end of file diff --git a/osquery/events/linux/bpf/.setrlimit.md b/osquery/events/linux/bpf/.setrlimit.md index 7af86cfb756..3cacb6a53ba 100644 --- a/osquery/events/linux/bpf/.setrlimit.md +++ b/osquery/events/linux/bpf/.setrlimit.md @@ -3,25 +3,30 @@ Declares a utility function for configuring BPF (Berkeley Packet Filter) memory ## Key Components -- **`configureBPFMemoryLimits()`** β€” Sets system resource limits (`rlimit`) required for BPF operations. Returns a `Status` object indicating success or failure. +- **`configureBPFMemoryLimits()`** β€” Sets system resource limits (`rlimit`) required for BPF operations. Returns an osquery `Status` object indicating success or failure. ## Usage Example ```c #include "setrlimit.h" -void initBPF() { - osquery::Status status = osquery::configureBPFMemoryLimits(); +namespace osquery { + +void initializeBPF() { + Status status = configureBPFMemoryLimits(); if (!status.ok()) { - // Handle failure: status.getMessage() provides details + LOG(ERROR) << "Failed to configure BPF memory limits: " + << status.getMessage(); return; } // Proceed with BPF initialization } + +} // namespace osquery ``` ## Notes -- Part of the `osquery` namespace. -- Relies on `osquery::Status` from `osquery/utils/status/status.h` for error propagation. -- Typically called during BPF subsystem initialization to ensure adequate locked memory (`RLIMIT_MEMLOCK`) is available for eBPF map and program loading. \ No newline at end of file +- Must be called before initializing BPF-based event publishers or subscribers. +- Internally uses `setrlimit(2)` to raise memory-locking limits (`RLIMIT_MEMLOCK`) needed by BPF maps and programs. +- Returns a non-OK `Status` if the syscall fails (e.g., insufficient privileges). \ No newline at end of file diff --git a/osquery/events/linux/bpf/.systemstatetracker.md b/osquery/events/linux/bpf/.systemstatetracker.md index 5c345dc8594..1c924bdc603 100644 --- a/osquery/events/linux/bpf/.systemstatetracker.md +++ b/osquery/events/linux/bpf/.systemstatetracker.md @@ -1,52 +1,53 @@ -Concrete implementation of `ISystemStateTracker` that tracks Linux process and system state via BPF event hooks, maintaining per-process context maps and generating an event list from observed kernel activity. +Concrete implementation of `ISystemStateTracker` that tracks Linux process and file descriptor state via BPF event hooks within the osquery eventing subsystem. ## Key Components -### Factory Methods -- `create()` β€” Instantiates tracker with a default `IProcessContextFactory` -- `create(IProcessContextFactory::Ref)` β€” Instantiates tracker with an injected factory (for testing) - -### Instance Methods (virtual overrides) -| Method | Description | -|---|---| -| `restart()` | Reinitializes tracker state | -| `createProcess()` | Records a fork/clone event | -| `executeBinary()` | Records an exec event with argv | -| `setWorkingDirectory()` | Updates CWD by `dirfd` or path string | -| `openFile()` / `closeHandle()` / `duplicateHandle()` | Tracks file descriptor lifecycle | -| `createSocket()` / `bind()` / `listen()` / `connect()` / `accept()` | Tracks socket state transitions | -| `nameToHandleAt()` / `openByHandleAt()` | Tracks opaque file handle resolution | -| `eventList()` | Returns accumulated BPF events | - -### Internal Static Helpers -- `getProcessContext()` β€” Looks up or creates a `ProcessContext` for a given PID -- `expireProcessContexts()` β€” Prunes stale process entries -- `parseUnixSockaddr()` / `parseInetSockaddr()` / `parseInet6Sockaddr()` / `parseNetlinkSockaddr()` β€” Deserializes raw sockaddr bytes into address/port strings -- `parseSocketAddress()` β€” Dispatches to the appropriate sockaddr parser -- `saveFileHandle()` / `expireFileHandleEntries()` β€” Manages the file handle index/map - -### Key Data Structures -- `Context` β€” Holds the `ProcessContextMap`, accumulated `EventList`, and file handle index/map -- `FileHandleStruct` β€” Associates a handle with its originating `dfd`, name, and flags -- `PrivateData` β€” Opaque pImpl struct holding internal state +### Class: `SystemStateTracker` + +| Member | Description | +|--------|-------------| +| `create()` | Factory methods returning a `Ref`; overload accepts a custom `IProcessContextFactory` | +| `restart()` | Resets tracker state and re-enumerates existing processes | +| `Context` | Inner struct holding the `ProcessContextMap`, `EventList`, and file handle index/map | +| `PrivateData` | Pimpl struct owning internal state behind a `unique_ptr` | +| `FileHandleStruct` / `FileHandleStructMap` | Stores `name_to_handle_at` results for later `open_by_handle_at` resolution | + +### Virtual Event Handlers (override `ISystemStateTracker`) + +| Method | Syscall Tracked | +|--------|----------------| +| `createProcess` | `fork` / `clone` | +| `executeBinary` | `execveat` / `execve` | +| `setWorkingDirectory` | `chdir` / `fchdir` | +| `openFile` / `duplicateHandle` / `closeHandle` | `open`, `dup2`, `close` | +| `createSocket` / `bind` / `listen` / `connect` / `accept` | Socket lifecycle | +| `nameToHandleAt` / `openByHandleAt` | File handle open | + +### Static Helpers + +- `getProcessContext` β€” lazily creates a `ProcessContext` for a PID +- `expireProcessContexts` β€” prunes stale entries against the filesystem +- `parseUnixSockaddr` / `parseInetSockaddr` / `parseInet6Sockaddr` / `parseNetlinkSockaddr` β€” deserialize raw `sockaddr` bytes into address/port +- `parseSocketAddress` β€” dispatches to the correct parser and populates `SocketData` +- `saveFileHandle` / `expireFileHandleEntries` β€” manage the bounded file handle cache ## Usage Example ```cpp -// Default construction +// Create tracker with default process context factory auto tracker = osquery::SystemStateTracker::create(); -// Process a fork event +// Handle a fork event received from BPF tracker->createProcess(event_header, parent_pid, child_pid); -// Track a binary execution -tracker->executeBinary(event_header, pid, AT_FDCWD, 0, "/usr/bin/ls", argv); - -// Retrieve accumulated events +// Retrieve and process accumulated events auto events = tracker->eventList(); +for (const auto& ev : events) { + // dispatch ev to osquery table publishers +} -// Inject a custom factory (e.g., in unit tests) +// Inject a custom factory for testing auto mock_factory = std::make_shared(); -auto tracker = osquery::SystemStateTracker::create(mock_factory); +auto test_tracker = osquery::SystemStateTracker::create(mock_factory); ``` \ No newline at end of file diff --git a/osquery/events/tests/.mockedosquerydatabase.md b/osquery/events/tests/.mockedosquerydatabase.md index adf5afcc7e9..c407946c5f0 100644 --- a/osquery/events/tests/.mockedosquerydatabase.md +++ b/osquery/events/tests/.mockedosquerydatabase.md @@ -1,41 +1,45 @@ -In-memory mock implementation of `IDatabaseInterface` for use in osquery unit tests, providing a simple `std::map`-backed database without requiring a real storage backend. +A test utility header providing an in-memory mock implementation of the osquery database interface for unit testing purposes. ## Key Components -- **`MockedOsqueryDatabase`** β€” Final class implementing `IDatabaseInterface` with an in-memory `std::map key_map` as the backing store. -- **`generateEvents()`** β€” Helper to simulate event generation for a given publisher and event name during tests. -- **`getDatabaseValue()`** β€” Overloaded for `std::string` and `int` return types; reads from `key_map`. -- **`setDatabaseValue()`** β€” Overloaded for `std::string` and `int` values; writes to `key_map`. -- **`setDatabaseBatch()`** β€” Bulk insert of string key-value pairs into the mock store. -- **`deleteDatabaseValue()`** / **`deleteDatabaseRange()`** β€” Remove single or range-bounded entries from `key_map`. -- **`scanDatabaseKeys()`** β€” Overloaded to list keys within a domain, with optional prefix filtering and a result size limit. +### `MockedOsqueryDatabase` +A `final` class implementing `IDatabaseInterface` backed by an in-memory `std::map`. Allows tests to exercise database-dependent code without a real database backend. + +**Public Members:** +- `key_map` β€” mutable in-memory store holding all key/value pairs (domain+key concatenated as map key) +- `generateEvents()` β€” seeds the mock with synthetic event data for a given publisher and event name + +**Overridden Interface Methods:** + +| Method | Description | +|---|---| +| `getDatabaseValue` | Retrieves `string` or `int` value by domain/key | +| `setDatabaseValue` | Stores `string` or `int` value by domain/key | +| `setDatabaseBatch` | Stores multiple string values in one call | +| `deleteDatabaseValue` | Removes a single key from the store | +| `deleteDatabaseRange` | Removes all keys between `low` and `high` within a domain | +| `scanDatabaseKeys` | Lists keys in a domain, with optional prefix filter and max count | ## Usage Example ```cpp #include "mockedosquerydatabase.h" -using namespace osquery; - -// Instantiate mock database for a unit test -MockedOsqueryDatabase db; - -// Write a value -db.setDatabaseValue("config", "refresh_interval", std::string("60")); +TEST(MyFeatureTest, StoresAndRetrieves) { + osquery::MockedOsqueryDatabase db; -// Read it back -std::string value; -Status s = db.getDatabaseValue("config", "refresh_interval", value); -assert(s.ok()); -assert(value == "60"); + // Write a value + db.setDatabaseValue("config", "refresh_interval", std::string("60")); -// Scan keys in a domain -std::vector keys; -db.scanDatabaseKeys("config", keys, 100); + // Read it back + std::string val; + auto status = db.getDatabaseValue("config", "refresh_interval", val); -// Simulate event generation -db.generateEvents("file_events", "file_created"); -``` + EXPECT_TRUE(status.ok()); + EXPECT_EQ(val, "60"); -> **Note:** `key_map` is `mutable` to allow `const`-qualified read/write methods to modify the in-memory store β€” a deliberate design choice for test convenience. Not intended for production use. \ No newline at end of file + // Seed synthetic events for testing event-driven logic + db.generateEvents("file_events", "file_created"); +} +``` \ No newline at end of file diff --git a/osquery/events/tests/linux/bpf/.mockedfilesystem.md b/osquery/events/tests/linux/bpf/.mockedfilesystem.md index 8f8daa9374c..f31d357641e 100644 --- a/osquery/events/tests/linux/bpf/.mockedfilesystem.md +++ b/osquery/events/tests/linux/bpf/.mockedfilesystem.md @@ -1,37 +1,37 @@ -A test double implementation of the `IFilesystem` interface that provides a mockable filesystem abstraction for unit testing BPF-related Linux event components without requiring real filesystem access. +A test double implementation of the `IFilesystem` interface for use in unit tests, providing a mockable filesystem abstraction for BPF event testing on Linux. ## Key Components -### `MockedFilesystem` class - -Concrete final class inheriting from `IFilesystem`, overriding all virtual methods: +**`MockedFilesystem`** β€” Concrete class implementing `IFilesystem` that serves as a test stub, overriding all virtual filesystem operations: | Method | Description | -|---|---| -| `open()` | Mocked file open by path with flags | -| `openAt()` | Mocked file open relative to a directory fd | -| `readLinkAt()` | Mocked symlink resolution relative to a directory fd | -| `read()` | Mocked buffered file read up to a max size | -| `enumFiles()` | Mocked directory file enumeration via callback | -| `fileExists()` | Mocked existence check relative to a directory fd | +|--------|-------------| +| `open()` | Opens a file by path with specified flags | +| `openAt()` | Opens a file relative to a directory file descriptor | +| `readLinkAt()` | Reads a symlink target relative to a directory fd | +| `read()` | Reads file contents up to a maximum size | +| `enumFiles()` | Enumerates files in a directory via callback | +| `fileExists()` | Checks whether a named file exists within a directory | ## Usage Example ```cpp #include "mockedfilesystem.h" -// Inject MockedFilesystem into a component under test -osquery::MockedFilesystem mockFs; +// Inject MockedFilesystem instead of a real filesystem +// implementation into components under test +osquery::MockedFilesystem fs; -// Use with a BPF event processor that depends on IFilesystem tob::utils::UniqueFd fd; -bool success = mockFs.open(fd, "/proc/self/maps", O_RDONLY); +bool opened = fs.open(fd, "/proc/1/maps", O_RDONLY); + +bool exists = false; +fs.fileExists(exists, AT_FDCWD, "somefile"); -// Typical usage: inject via IFilesystem interface pointer -std::unique_ptr fs = - std::make_unique(); -MyBpfComponent component(std::move(fs)); +// Pass to BPF event processor as IFilesystem dependency +MyBpfProcessor processor(fs); +processor.run(); ``` -> **Note:** Method bodies are defined in the accompanying `.cpp` file where mock behavior (e.g., via GoogleMock `MOCK_METHOD` macros or manual stubs) is implemented. This header only declares the override surface. \ No newline at end of file +> **Note:** `MockedFilesystem` is intended exclusively for testing. In production code, a real `IFilesystem` implementation is injected. Override individual methods in a derived mock (e.g., via GoogleMock) to simulate specific filesystem behaviors or failure conditions. \ No newline at end of file diff --git a/osquery/events/tests/linux/bpf/.mockedprocesscontextfactory.md b/osquery/events/tests/linux/bpf/.mockedprocesscontextfactory.md index 24fb05c7b91..c00d9daa23c 100644 --- a/osquery/events/tests/linux/bpf/.mockedprocesscontextfactory.md +++ b/osquery/events/tests/linux/bpf/.mockedprocesscontextfactory.md @@ -1,42 +1,40 @@ -Provides a test double implementation of `IProcessContextFactory` for unit testing BPF process context capture logic in osquery's Linux event system. +Provides a mock implementation of `IProcessContextFactory` for unit testing BPF-based Linux process context capture without requiring real process data. ## Key Components -| Member | Description | -|---|---| -| `MockedProcessContextFactory` | Concrete mock class implementing `IProcessContextFactory` | -| `invocationCount()` | Returns the number of times capture methods have been called | -| `failNextRequest()` | Configures the mock to return failure on the next capture call | -| `captureSingleProcess()` | Mock override; captures context for a single PID | -| `captureAllProcesses()` | Mock override; captures context for all processes | +- **`MockedProcessContextFactory`** β€” Concrete mock class implementing `IProcessContextFactory`, designed for test environments +- **`invocationCount()`** β€” Returns the number of times capture methods have been called, useful for verifying call behavior in tests +- **`failNextRequest()`** β€” Configures the mock to simulate a failure on the next capture call, enabling error-path testing +- **`captureSingleProcess()`** β€” Mock override that captures context for a single process by PID +- **`captureAllProcesses()`** β€” Mock override that populates a `ProcessContextMap` with all process contexts + +### Private State + +| Member | Type | Purpose | +|--------|------|---------| +| `fail_next_request` | `mutable bool` | Triggers simulated failure on next call | +| `invocation_count` | `mutable size_t` | Tracks total capture invocations | ## Usage Example -```c +```cpp #include "mockedprocesscontextfactory.h" -// Basic happy-path test +// Basic invocation tracking MockedProcessContextFactory factory; ProcessContext ctx; - -bool ok = factory.captureSingleProcess(ctx, 1234); -assert(ok); +factory.captureSingleProcess(ctx, 1234); assert(factory.invocationCount() == 1); -// Simulate capture failure +// Simulate a capture failure factory.failNextRequest(); -bool failed = factory.captureSingleProcess(ctx, 5678); -assert(!failed); +bool success = factory.captureSingleProcess(ctx, 5678); +assert(!success); // failure path exercised -// Bulk capture +// Capture all processes ProcessContextMap process_map; factory.captureAllProcesses(process_map); -assert(factory.invocationCount() == 3); ``` -## Notes - -- Both `fail_next_request` and `invocation_count` are `mutable`, allowing state tracking inside `const` method overrides. -- Marked `final` β€” not intended to be subclassed further. -- Depends on `IProcessContextFactory` from `osquery/events/linux/bpf/`. \ No newline at end of file +> **Note:** Both `fail_next_request` and `invocation_count` are declared `mutable`, allowing state tracking through `const` method calls without breaking the `const` interface contract inherited from `IProcessContextFactory`. \ No newline at end of file diff --git a/osquery/events/tests/linux/bpf/.utils.md b/osquery/events/tests/linux/bpf/.utils.md index e03ac74cf11..030d1ef6aad 100644 --- a/osquery/events/tests/linux/bpf/.utils.md +++ b/osquery/events/tests/linux/bpf/.utils.md @@ -1,27 +1,19 @@ -Utility header for managing and validating file and socket descriptors within BPF process context tracking on Linux. +Utility header for managing and validating file and socket descriptors within BPF-tracked process contexts in the osquery Linux event system. ## Key Components All functions reside in the `osquery` namespace and operate on either a single `ProcessContext` or a `ProcessContextMap` (keyed by `pid_t`). ### Set Functions - -| Function | Description | -|---|---| -| `setFileDescriptor` | Registers a file descriptor with its path and `close_on_exec` flag into a process context | -| `setSocketDescriptor` | Registers a socket descriptor with full network metadata (domain, type, protocol, local/remote address and port) | - -Both setters are overloaded to accept either a direct `ProcessContext&` or a `ProcessContextMap&` + `pid_t` pair. +- **`setFileDescriptor`** β€” Registers a file descriptor with its path and `close_on_exec` flag into a process context. +- **`setSocketDescriptor`** β€” Registers a socket descriptor with full network metadata (domain, type, protocol, local/remote address and port) into a process context. ### Validate Functions +- **`validateFileDescriptor`** β€” Asserts that a file descriptor entry in a process context matches the expected path and flags; returns `bool`. +- **`validateSocketDescriptor`** β€” Asserts that a socket descriptor entry matches all provided network parameters; returns `bool`. -| Function | Description | -|---|---| -| `validateFileDescriptor` | Returns `true` if the given fd matches the expected path and `close_on_exec` state in the context | -| `validateSocketDescriptor` | Returns `true` if the given fd matches all expected socket properties in the context | - -Same overload pattern as the setters. +Each function is overloaded to accept either a direct `ProcessContext&` or a `ProcessContextMap&` + `pid_t` pair for map-based lookup. ## Usage Example @@ -29,25 +21,25 @@ Same overload pattern as the setters. #include "utils.h" osquery::ProcessContext ctx; -osquery::ProcessContextMap ctx_map; -// Register a file descriptor on a single context +// Register a file descriptor osquery::setFileDescriptor(ctx, 3, false, "/etc/passwd"); -// Register a socket on a context map entry for pid 1234 -osquery::setSocketDescriptor(ctx_map, 1234, 5, false, - AF_INET, SOCK_STREAM, IPPROTO_TCP, - "127.0.0.1", 8080, - "10.0.0.1", 443); - -// Validate the file descriptor +// Validate it was stored correctly bool ok = osquery::validateFileDescriptor(ctx, 3, false, "/etc/passwd"); -// Validate the socket via map -bool valid = osquery::validateSocketDescriptor(ctx_map, 1234, 5, false, - AF_INET, SOCK_STREAM, IPPROTO_TCP, - "127.0.0.1", 8080, - "10.0.0.1", 443); -``` - -> Primarily used in BPF event processing tests to populate and assert process context state without requiring live kernel events. \ No newline at end of file +// Register a socket descriptor via process map +osquery::ProcessContextMap ctx_map; +pid_t pid = 1234; +osquery::setSocketDescriptor( + ctx_map, pid, 5, false, + AF_INET, SOCK_STREAM, IPPROTO_TCP, + "127.0.0.1", 8080, + "10.0.0.1", 443); + +bool valid = osquery::validateSocketDescriptor( + ctx_map, pid, 5, false, + AF_INET, SOCK_STREAM, IPPROTO_TCP, + "127.0.0.1", 8080, + "10.0.0.1", 443); +``` \ No newline at end of file diff --git a/osquery/events/windows/.evtsubscription.md b/osquery/events/windows/.evtsubscription.md index 524810ccc31..a3dc2e958f0 100644 --- a/osquery/events/windows/.evtsubscription.md +++ b/osquery/events/windows/.evtsubscription.md @@ -1,17 +1,26 @@ -Defines the `EvtSubscription` class and its Windows Event Log callback dispatcher for subscribing to and collecting Windows Event Log entries within osquery. +Windows Event Log subscription wrapper that provides a C++ RAII interface for subscribing to and collecting Windows Event Log entries via the WinEvt API. ## Key Components -- **`EvtSubscriptionCallbackDispatcher`** β€” Windows callback function (`DWORD WINAPI`) invoked by the WinEvt API on new events; dispatches to the appropriate `EvtSubscription` instance via the `context` pointer. -- **`EvtSubscription`** β€” RAII-style, non-copyable class managing a Windows Event Log channel subscription. - - `Ref` β€” `std::unique_ptr` alias for ownership semantics. - - `Event` / `EventList` β€” Type aliases for `std::wstring` and `std::vector` representing collected events. - - `create(Ref&, channel)` β€” Static factory method; constructs a subscription for the given channel name, returning an osquery `Status`. - - `channel()` β€” Returns the subscribed channel name. - - `getEvents()` β€” Drains and returns all buffered events collected since the last call. - - `processEvent(EVT_HANDLE)` β€” Private handler called by the dispatcher to render and buffer an incoming event. - - `PrivateData` β€” Pimpl struct hiding WinEvt handles and internal state. +### `EvtSubscription` (class) +A non-copyable, final class managing a Windows Event Log channel subscription. + +| Member | Description | +|--------|-------------| +| `Ref` | `std::unique_ptr` β€” ownership handle | +| `Event` | `std::wstring` β€” single event payload | +| `EventList` | `std::vector` β€” collection of events | +| `create(Ref&, channel)` | Factory method; allocates and subscribes to the named channel, returns `Status` | +| `channel()` | Returns the subscribed channel name | +| `getEvents()` | Drains and returns all buffered events since last call | +| `~EvtSubscription()` | Cleans up WinEvt handles | + +### `EvtSubscriptionCallbackDispatcher` (free function) +A `WINAPI` callback registered with the WinEvt API. Dispatches incoming `EVT_HANDLE` events to the owning `EvtSubscription` instance via the `context` pointer. + +### `PrivateData` (private struct) +Pimpl pattern β€” hides WinEvt handle storage and internal state from the public header. ## Usage Example @@ -19,18 +28,19 @@ Defines the `EvtSubscription` class and its Windows Event Log callback dispatche #include "evtsubscription.h" osquery::EvtSubscription::Ref sub; -auto status = osquery::EvtSubscription::create(sub, "System"); +auto status = osquery::EvtSubscription::create(sub, "System"); if (!status.ok()) { - LOG(ERROR) << "Subscription failed: " << status.getMessage(); - return; + LOG(ERROR) << "Subscription failed: " << status.getMessage(); + return; } -// Later, poll for collected events +// Poll for new events auto events = sub->getEvents(); -for (const auto& event : events) { - // process wide-string XML event data +for (const auto& evt : events) { + // evt is a std::wstring containing the rendered XML event + processEvent(evt); } ``` -> **Platform note:** This header is Windows-only. It depends on `` and `` and must only be compiled in Windows build targets. \ No newline at end of file +> **Note:** This header is Windows-only (``). Guard any cross-platform code with `#ifdef OSQUERY_WINDOWS` or equivalent CMake platform checks. \ No newline at end of file diff --git a/osquery/events/windows/.ntfs_event_publisher.md b/osquery/events/windows/.ntfs_event_publisher.md index bc37d986214..31bc72b87f9 100644 --- a/osquery/events/windows/.ntfs_event_publisher.md +++ b/osquery/events/windows/.ntfs_event_publisher.md @@ -1,54 +1,49 @@ -Declares the NTFS event publisher for osquery on Windows, which consumes raw USN journal records and emits structured file change events to subscribers. +Declares the `NTFSEventPublisher` class and associated data structures for monitoring NTFS file system changes on Windows via the USN (Update Sequence Number) journal. ## Key Components ### Structs -| Name | Purpose | -|---|---| -| `NTFSEventSubscriptionContext` | Subscriber filter: paths/FRNs for write-only or always-include (read+write) monitoring | -| `NTFSEventRecord` | Single normalized NTFS event with path, type, timestamps, FRNs, and drive letter | -| `NTFSEventContext` | Batch event context holding a `vector` emitted to subscribers | -| `VolumeData` | Holds volume/root folder HANDLE and root FRN for a monitored drive | -| `USNJournalReaderInstance` | Tracks a running reader service, its shared context, parent FRN cache, and rename merge map | +| Name | Description | +|------|-------------| +| `NTFSEventSubscriptionContext` | Subscription filter containing watched paths and file reference numbers (FRNs) for both write/delete and read-access events | +| `NTFSEventRecord` | Single normalized NTFS event record with path, timestamp, attributes, USN, and drive letter | +| `NTFSEventContext` | Batch event context holding a vector of `NTFSEventRecord` entries emitted to subscribers | +| `VolumeData` | Holds NTFS volume/root folder handles and root FRN for path resolution | +| `USNJournalReaderInstance` | Tracks a live `USNJournalReader` service, its shared context, parent FRN cache, and rename merge map | ### Type Aliases -- `NTFSEventPublisherConfiguration` β€” `unordered_set` of drive letters to monitor -- `ParentFRNCache` β€” `unordered_map` for resolving parent directory paths +- `NTFSEventSubscriptionContextRef` β€” `shared_ptr` +- `NTFSEventContextRef` β€” `shared_ptr` +- `ParentFRNCache` β€” `unordered_map` for FRN-to-directory resolution +- `NTFSEventPublisherConfiguration` β€” `unordered_set` of drive letters being monitored ### Class: `NTFSEventPublisher` Extends `EventPublisher`. Core lifecycle methods: -| Method | Role | -|---|---| -| `setUp()` | One-time initialization | -| `configure()` | Reads drive list from config; called on reload | -| `run()` | Main loop: acquires journal records, resolves paths, fires events | -| `tearDown()` | Releases handles and cleans up resources | +- `setUp()` β€” Initializes publisher resources +- `configure()` β€” Reads config and updates monitored drives +- `run()` β€” Main loop: acquires USN journal records and fires events +- `tearDown()` β€” Releases handles and cleans up + +Key private helpers: `restartJournalReaderServices()`, `acquireJournalRecords()`, `getPathFromReferenceNumber()`, `getPathFromParentFRN()`, `getVolumeData()` ### Free Function -- `GetNativeFileIdFromUSNReference()` β€” Converts a `USNFileReferenceNumber` to a `FILE_ID_DESCRIPTOR` for use with the `OpenFileById` Win32 API +- `GetNativeFileIdFromUSNReference()` β€” Converts a `USNFileReferenceNumber` to a `FILE_ID_DESCRIPTOR` for use with the Windows `OpenFileById` API ## Usage Example ```cpp // Subscribing to NTFS write events on C:\Windows -auto sc = std::make_shared(); -sc->category = "system_files"; -sc->write_paths.insert("C:\\Windows\\System32"); - -// The publisher resolves FRNs to full paths internally -// and emits NTFSEventContext batches containing NTFSEventRecord entries -// Subscribers inspect each record's type, path, and old_path (renames) -for (const auto& record : context->event_list) { - if (record.type == USNJournalEventRecord::Type::FileWrite) { - LOG(INFO) << "Modified: " << record.path; - } -} -``` - -> **Note:** This publisher is Windows-only and depends on the USN change journal. Partial events (`record.partial == true`) indicate only a filename was resolved, not a full path. \ No newline at end of file +auto ctx = NTFSEventPublisher::createSubscriptionContext(); +ctx->category = "file_changes"; +ctx->write_paths.insert("C:\\Windows\\System32"); + +// Framework wires the publisher automatically via DECLARE_PUBLISHER +// The publisher calls run() in a loop, emitting NTFSEventContext batches +// to all matching subscribers +``` \ No newline at end of file diff --git a/osquery/events/windows/.usn_journal_reader.md b/osquery/events/windows/.usn_journal_reader.md index 144bfdc2d1e..8ad96f700eb 100644 --- a/osquery/events/windows/.usn_journal_reader.md +++ b/osquery/events/windows/.usn_journal_reader.md @@ -1,52 +1,54 @@ -Windows NTFS USN (Update Sequence Number) Journal reader interface for osquery's file change event monitoring on Windows. +Windows NTFS USN (Update Sequence Number) Journal reader interface for osquery's file change event monitoring system. ## Key Components -### Structs +### `USNFileReferenceNumber` +A portable wrapper around Windows file reference numbers supporting 64-bit (`DWORDLONG`), 128-bit (`FILE_ID_128`), and GUID formats. Implements full comparison operators and `std::hash` specialization for use in unordered containers. -| Name | Description | -|------|-------------| -| `USNFileReferenceNumber` | Uniquely identifies a file within a volume; supports 64-bit, 128-bit (`FILE_ID_128`), and GUID formats | -| `USNJournalEventRecord` | Represents a single file system change event, including type, path, timestamps, and attributes | -| `USNJournalReaderContext` | Shared state between `USNJournalReader` service and `FileChangePublisher`; thread-safe via mutex and condition variable | +### `USNJournalEventRecord` +Event record struct representing a single NTFS journal change, containing: +- `Type` enum β€” 32 distinct event types covering file/directory creation, deletion, rename, write, truncation, and attribute changes +- Metadata fields: `drive_letter`, `update_sequence_number`, `node_ref_number`, `parent_ref_number`, `record_timestamp`, `attributes`, `name` -### Classes +### `USNJournalReaderContext` +Shared state between the reader thread and `FileChangePublisher`, holding the processed record list, synchronization primitives (`Mutex`, `ConditionVariable`), and a termination flag. -- **`USNJournalReader`** β€” Background service thread (`InternalRunnable`) that asynchronously reads USN journal records from a volume, decompresses compound events, and dispatches them to the publisher. +### `USNJournalReader` (`InternalRunnable`) +Background service thread that drives the journal read loop via `initialize()`, `acquireRecords()`, `processAcquiredRecords()`, and `dispatchEventRecords()`. Key static methods: +- `DecompressRecord()` β€” expands a compound reason bitmask into discrete event records +- `ProcessAndAppendUSNRecord()` β€” parses a raw `USN_RECORD*` and appends typed events -### Namespace `USNParsers` - -Utility functions for extracting individual fields from raw `USN_RECORD` pointers: - -- `GetUpdateSequenceNumber`, `GetFileReferenceNumber`, `GetParentFileReferenceNumber` -- `GetTimeStamp`, `GetAttributes`, `GetReason`, `GetEventType`, `GetEventString` - -### Key Type Aliases - -- `USNPerFileLastRecordType` β€” `std::map` used to deduplicate event types per file reference -- `USNJournalReaderRef` / `USNJournalReaderContextRef` β€” `shared_ptr` wrappers for shared ownership +### `USNParsers` Namespace +Low-level parser helpers that extract individual fields from raw `USN_RECORD*` pointers: `GetUpdateSequenceNumber`, `GetFileReferenceNumber`, `GetParentFileReferenceNumber`, `GetTimeStamp`, `GetAttributes`, `GetReason`, `GetEventType`, `GetEventString`. ## Usage Example ```cpp -// Create a shared context for the reader and publisher -auto context = std::make_shared(); -context->drive_letter = 'C'; +// Create shared context for a drive +auto ctx = std::make_shared(); +ctx->drive_letter = 'C'; -// Instantiate and start the reader service -auto reader = std::make_shared(context); -reader->start(); +// Start the reader service +auto reader = std::make_shared(ctx); +reader->run(); -// Consume processed records (publisher side) +// Publisher side: consume processed records { - std::lock_guard lock(context->processed_records_mutex); - for (const auto& record : context->processed_record_list) { + std::unique_lock lock(ctx->processed_records_mutex); + ctx->processed_records_cv.wait(lock, [&] { + return !ctx->processed_record_list.empty() || ctx->terminate; + }); + + for (const auto& record : ctx->processed_record_list) { std::cout << record << "\n"; // uses operator<< overload } - context->processed_record_list.clear(); + ctx->processed_record_list.clear(); } -// Signal shutdown -context->terminate.store(true); +// Parse a raw record manually +std::vector out; +osquery::USNPerFileLastRecordType lastTypeMap; +osquery::USNJournalReader::ProcessAndAppendUSNRecord( + out, rawUsnRecord, lastTypeMap, 'C'); ``` \ No newline at end of file diff --git a/osquery/events/windows/.windowseventlogparser.md b/osquery/events/windows/.windowseventlogparser.md index 9c48abcf6f1..1f1da347a01 100644 --- a/osquery/events/windows/.windowseventlogparser.md +++ b/osquery/events/windows/.windowseventlogparser.md @@ -1,57 +1,48 @@ -Windows Event Log parser interface for osquery, providing data structures and parsing utilities to convert raw Windows Event Log XML into structured C++ objects. +Header defining types and functions for parsing Windows Event Log (WEL) entries from XML into structured C++ objects within the osquery framework. ## Key Components -### `WELEvent` Struct -A plain data container representing a parsed Windows Event Log entry with the following fields: +### `WELEvent` struct +A plain data structure holding all normalized fields extracted from a Windows Event Log entry: | Field | Type | Description | -|---|---|---| +|-------|------|-------------| | `osquery_time` | `std::time_t` | Timestamp when osquery captured the event | -| `datetime` | `std::string` | Human-readable event datetime | +| `datetime` | `std::string` | Event datetime string | | `source` | `std::string` | Event log source channel | -| `provider_name` | `std::string` | Publisher/provider name | -| `provider_guid` | `std::string` | Provider GUID | -| `computer_name` | `std::string` | Originating machine name | -| `event_id` | `int64_t` | Windows Event ID | -| `task_id` | `int64_t` | Task category ID | -| `level` | `int64_t` | Severity level | -| `pid` / `tid` | `int64_t` | Process and thread IDs | -| `keywords` | `std::string` | Event keywords bitmask string | -| `data` | `std::string` | Raw event data payload | - -### `parseWindowsEventLogXML()` -Parses a raw Windows Event Log XML string (`std::wstring`) into a Boost `property_tree` object for further processing. - -### `parseWindowsEventLogPTree()` -Converts a populated Boost `property_tree` into a `WELEvent` struct, extracting all structured fields. +| `provider_name` / `provider_guid` | `std::string` | Event provider identity | +| `computer_name` | `std::string` | Originating host | +| `event_id`, `task_id`, `level` | `std::int64_t` | Event classification fields | +| `pid`, `tid` | `std::int64_t` | Process/thread IDs | +| `keywords`, `data` | `std::string` | Additional event metadata and payload | + +### Functions + +- **`parseWindowsEventLogXML`** β€” Parses a raw wide-string XML event into a `boost::property_tree::ptree` intermediate representation. Returns an osquery `Status` indicating success or failure. + +- **`parseWindowsEventLogPTree`** β€” Converts a populated `ptree` object into a `WELEvent` struct. Intended to be called after `parseWindowsEventLogXML`. ## Usage Example ```cpp #include "windowseventlogparser.h" -// Step 1: Parse raw XML into a property tree -boost::property_tree::ptree event_object; -std::wstring raw_xml = /* ... raw WEL XML ... */; +std::wstring raw_xml_event = /* event XML from Windows API */; -osquery::Status parse_status = - osquery::parseWindowsEventLogXML(event_object, raw_xml); - -if (!parse_status.ok()) { - // handle error -} +boost::property_tree::ptree event_tree; +osquery::Status s = osquery::parseWindowsEventLogXML(event_tree, raw_xml_event); -// Step 2: Populate a WELEvent struct from the property tree -osquery::WELEvent wel_event; -osquery::Status map_status = - osquery::parseWindowsEventLogPTree(wel_event, event_object); +if (s.ok()) { + osquery::WELEvent wel_event; + s = osquery::parseWindowsEventLogPTree(wel_event, event_tree); -if (map_status.ok()) { - // Access structured fields - std::cout << wel_event.provider_name << " - ID: " << wel_event.event_id; + if (s.ok()) { + // Access parsed fields + std::cout << wel_event.provider_name << "\n"; + std::cout << wel_event.event_id << "\n"; + } } ``` -The two-step design separates XML deserialization from field extraction, allowing either stage to be used independently or tested in isolation. \ No newline at end of file +The two-stage parsing design (XML β†’ ptree β†’ struct) allows intermediate inspection of the property tree before final deserialization into a `WELEvent`. \ No newline at end of file diff --git a/osquery/events/windows/.windowseventlogparserservice.md b/osquery/events/windows/.windowseventlogparserservice.md index 03bf850b8cd..02054c6cf21 100644 --- a/osquery/events/windows/.windowseventlogparserservice.md +++ b/osquery/events/windows/.windowseventlogparserservice.md @@ -1,50 +1,46 @@ -Parses raw Windows Event Log subscription data into structured property trees for consumption by osquery event publishers. +Background service class for parsing raw Windows Event Log subscription data into structured property trees, organized by channel. ## Key Components -### Class: `WindowsEventLogParserService` - -Extends `InternalRunnable` (osquery dispatcher thread). Runs as a background service that deserializes Windows Event Log entries from active `EvtSubscription` callbacks. +### `WindowsEventLogParserService` +Extends `InternalRunnable` (osquery dispatcher thread) to run as a background service. **Type Aliases** - -| Type | Definition | -|---|---| -| `PropertyTreeList` | `std::vector` | -| `ChannelEventObjects` | `std::unordered_map` | +- `PropertyTreeList` β€” `std::vector` β€” ordered list of parsed event objects +- `ChannelEventObjects` β€” `std::unordered_map` β€” maps channel names to their parsed event lists **Public Methods** | Method | Description | -|---|---| -| `start()` | Begins the parser service loop (called by dispatcher) | -| `stop()` | Signals the service loop to terminate | -| `addEventList(channel, event_list)` | Queues raw `EvtSubscription::EventList` entries for a named channel | -| `getChannelEventObjects()` | Returns parsed property trees grouped by channel name, draining the internal buffer | - -**Implementation Detail** +|--------|-------------| +| `start()` | Begins the background parsing loop | +| `stop()` | Signals the service to halt | +| `addEventList(channel, event_list)` | Enqueues raw `EvtSubscription::EventList` data for a given channel | +| `getChannelEventObjects()` | Returns all parsed events grouped by channel name, consuming the internal buffer | -Uses the PIMPL idiom (`struct PrivateData` + `std::unique_ptr d_`) to hide internal state (likely mutex, event queue, and parse buffers) from the header. +**Private Implementation** +Uses the [Pimpl idiom](https://en.cppreference.com/w/cpp/language/pimpl) (`PrivateData` via `std::unique_ptr`) to hide threading primitives and internal state. ## Usage Example ```cpp -// Instantiated and managed by the osquery dispatcher -auto parser = std::make_shared(); +#include -// From a subscription callback, queue raw events for a channel -parser->addEventList("Security", std::move(event_list)); +// Instantiate and register with the osquery dispatcher +auto parser = std::make_shared(); +Dispatcher::addService(parser); -// From the publisher, drain parsed results -WindowsEventLogParserService::ChannelEventObjects parsed = - parser->getChannelEventObjects(); +// Feed raw subscription events from a Windows Event Log channel +parser->addEventList("Security", subscriptionHandle.drainEvents()); -for (const auto& [channel, trees] : parsed) { +// Retrieve all parsed property trees grouped by channel +auto parsedEvents = parser->getChannelEventObjects(); +for (const auto& [channel, trees] : parsedEvents) { for (const auto& tree : trees) { - // Process each boost::property_tree representing one event record + // Process each boost::property_tree::ptree event object } } ``` -> **Note:** `addEventList` and `getChannelEventObjects` are intended to be called from different threads β€” the subscription callback thread writes, while the publisher thread reads. Internal synchronization is handled inside `PrivateData`. \ No newline at end of file +> **Note:** `getChannelEventObjects()` is intended to be called from the consumer side (e.g., an event publisher) after the service has processed incoming raw events. Thread safety is managed internally via the hidden `PrivateData` struct. \ No newline at end of file diff --git a/osquery/events/windows/.windowseventlogpublisher.md b/osquery/events/windows/.windowseventlogpublisher.md index e9c5eeae94a..b2ad9b33f4c 100644 --- a/osquery/events/windows/.windowseventlogpublisher.md +++ b/osquery/events/windows/.windowseventlogpublisher.md @@ -1,49 +1,48 @@ -Windows Event Log event publisher interface for osquery, providing subscription-based access to Windows Event Log channels with channel filtering and character frequency analysis for event matching. +Windows Event Log event publisher that subscribes to and distributes Windows Event Log entries to osquery subscribers, with support for channel filtering and anomaly detection via cosine similarity. ## Key Components ### Structs -| Name | Description | -|------|-------------| -| `WindowsEventLogSubscriptionContext` | Holds subscriber configuration β€” a set of channel names to monitor and a character frequency map used for cosine similarity filtering | -| `WindowsEventLogEC` | Event context payload carrying the source channel name and a list of parsed property tree event objects | +- **`WindowsEventLogSubscriptionContext`** β€” Subscription configuration holding a set of Windows Event Log channel names and a character frequency map for similarity analysis +- **`WindowsEventLogEC`** (Event Context) β€” Carries the source channel name and a list of parsed event property trees delivered to subscribers ### Type Aliases -| Alias | Underlying Type | -|-------|----------------| -| `WindowsEventLogECRef` | `std::shared_ptr` | -| `WindowsEventLogSCRef` | `std::shared_ptr` | +- **`WindowsEventLogECRef`** β€” `shared_ptr` +- **`WindowsEventLogSCRef`** β€” `shared_ptr` ### Class: `WindowsEventLogPublisher` -Extends `EventPublisher`. +Inherits `EventPublisher`. | Method | Description | |--------|-------------| -| `shouldFire()` | Determines if an event should be dispatched to a given subscriber based on channel and frequency matching | -| `configure()` | Sets up event log channel subscriptions | -| `tearDown()` | Cleans up active subscriptions and resources | -| `run()` | Main loop β€” polls/receives Windows Event Log entries and fires events | -| `cosineSimilarity()` | Static utility computing cosine similarity between a buffer's character frequency and a reference frequency map | +| `configure()` | Sets up channel subscriptions from registered contexts | +| `run()` | Main loop that reads and dispatches Windows Event Log entries | +| `tearDown()` | Cleans up handles and parser service resources | +| `shouldFire()` | Filters events against a subscription's channel list | +| `cosineSimilarity()` | Static utility β€” computes cosine similarity between a string's character frequency and a reference frequency vector (used for anomaly scoring) | + +Private state is encapsulated via the `PrivateData` pImpl pattern (`d_`). ## Usage Example ```cpp -// Define a subscriber with specific channels to monitor +// Register a subscription to specific Windows Event Log channels auto sc = std::make_shared(); -sc->channel_list.insert("Security"); -sc->channel_list.insert("System"); +sc->channel_list = {"Security", "System", "Application"}; -// Compute similarity between an event buffer and known frequency baseline -std::vector baseline = loadBaselineFrequencies(); -double score = WindowsEventLogPublisher::cosineSimilarity(rawEventBuffer, baseline); +// Optionally set a character frequency map for similarity filtering +sc->character_frequency_map = getBaselineFrequencies(); -if (score > threshold) { - // Process suspicious event -} -``` +// Use cosine similarity to score an event string against a baseline +double score = WindowsEventLogPublisher::cosineSimilarity( + rawEventBuffer, sc->character_frequency_map +); -> **Note:** The `PrivateData` pimpl pattern hides platform-specific WinAPI subscription handles, keeping the public interface clean and ABI-stable. \ No newline at end of file +if (score < 0.85) { + // Flag as anomalous +} +``` \ No newline at end of file diff --git a/osquery/events/windows/etw/.etw_controller.md b/osquery/events/windows/etw/.etw_controller.md index a386ff138e5..d605275273e 100644 --- a/osquery/events/windows/etw/.etw_controller.md +++ b/osquery/events/windows/etw/.etw_controller.md @@ -1,49 +1,45 @@ -Singleton controller class for managing Windows Event Tracing (ETW) sessions and event processing pipelines within the osquery framework. +Manages ETW (Event Tracing for Windows) sessions and event processing pipeline within the osquery framework, providing a singleton interface for subscribing to Windows kernel and userspace ETW providers. ## Key Components -| Member | Type | Description | -|---|---|---| -| `instance()` | Static method | Returns the singleton `EtwController` instance | -| `addProvider()` | Public method | Registers an ETW provider with pre/post-processor callbacks | -| `dispatchETWEvents()` | Public method | Routes captured ETW events into the processing pipeline | -| `startProcessing()` | Private method | Starts ETW session listening and event processing | -| `initialize()` | Private method | Bootstraps ETW sessions and the post-processing engine | +| Symbol | Type | Description | +|--------|------|-------------| +| `EtwController` | Class | Singleton controller managing ETW sessions and event dispatch | +| `instance()` | Static method | Returns the single global `EtwController` instance | +| `addProvider()` | Public method | Registers an ETW provider with pre/post-processor callbacks via `EtwProviderConfig` | +| `dispatchETWEvents()` | Public method | Routes captured ETW events from preprocessor callbacks into the processing pipeline | +| `startProcessing()` | Private method | Starts userspace/kernel ETW listening sessions | +| `initialize()` | Private method | Bootstraps ETW sessions and post-processing engine | -**Internal Sessions:** +**Internal session runnables:** -| Field | Purpose | -|---|---| -| `etwUserSession_` | Manages userspace ETW provider session | -| `etwKernelSession_` | Manages kernel ETW provider session | -| `etwPostProcessingEngine_` | Runs post-processor callbacks on captured events | -| `concurrentQueue_` | Thread-safe event queue bridging capture and processing | +- `UserEtwSessionRunnable` β€” handles userspace ETW providers +- `KernelEtwSessionRunnable` β€” handles kernel ETW providers +- `EtwPostProcessorsRunnable` β€” runs registered post-processor callbacks +- `ConcurrentEventQueue` β€” thread-safe queue bridging capture and processing ## Usage Example -```cpp +```c #include +#include -// Retrieve the singleton instance -EtwController& controller = EtwController::instance(); +// Obtain the singleton instance +auto& controller = osquery::EtwController::instance(); -// Build a provider configuration -EtwProviderConfig config; -// ... populate config with provider GUID, pre/post-processor callbacks ... +// Build provider configuration with desired callbacks +osquery::EtwProviderConfig config; +// ... populate config with provider GUID, pre/post-processor lambdas ... -// Register the provider β€” starts sessions if not already running -Status status = controller.addProvider(config); +// Register the provider β€” starts listening if not already initialized +osquery::Status status = controller.addProvider(config); if (!status.ok()) { - LOG(ERROR) << "Failed to add ETW provider: " << status.getMessage(); + // handle error } -// Events are automatically dispatched internally via: -// controller.dispatchETWEvents(eventDataRef); +// Events are automatically dispatched via dispatchETWEvents() +// through the internal concurrent queue to post-processors ``` -## Notes - -- Inherits `boost::noncopyable` β€” copy and assignment are explicitly disabled. -- Thread-safety is enforced via `std::mutex` and `std::atomic initialized_`. -- Session names are fixed: `OsqueryUserETWSession`, `OsqueryKernelETWSession`, and `EtwPostProcessingEngine`. \ No newline at end of file +> **Note:** `EtwController` inherits from `boost::noncopyable`, enforcing singleton semantics. Direct construction is private β€” always access via `EtwController::instance()`. Thread safety is managed internally via `std::mutex`. \ No newline at end of file diff --git a/osquery/events/windows/etw/.etw_kernel_session.md b/osquery/events/windows/etw/.etw_kernel_session.md index 26982a77877..f6b9cdb33b5 100644 --- a/osquery/events/windows/etw/.etw_kernel_session.md +++ b/osquery/events/windows/etw/.etw_kernel_session.md @@ -1,42 +1,47 @@ -Manages the ETW (Event Tracing for Windows) kernel-space trace session as a dedicated runnable thread within osquery, wrapping the KrabsETW library to listen for and process kernel-level ETW events. +Manages the ETW (Event Tracing for Windows) kernel-space trace session as a dedicated thread, using the KrabsETW library to listen for and process kernel-mode ETW events within osquery. ## Key Components -### Class: `KernelEtwSessionRunnable` -Extends `InternalRunnable` to run a kernel ETW trace session on a dedicated thread. +### `KernelEtwSessionRunnable` (class) +Inherits from `InternalRunnable` to manage a kernel ETW trace session on a dedicated thread. | Member | Type | Description | |--------|------|-------------| -| `addProvider()` | `Status` | Registers a kernel ETW provider with pre/post-processor callbacks | -| `start()` | `void` | Starts the InternalRunnable thread | -| `stop()` | `void` | Stops the InternalRunnable thread | -| `pause()` / `resume()` | `void` | Suspends or resumes event listening | +| `addProvider()` | `Status` | Registers a kernel ETW provider with preprocessor/postprocessor callbacks | +| `start()` | `void` | Starts the underlying `InternalRunnable` thread | +| `stop()` | `void` | Stops the underlying `InternalRunnable` thread | +| `pause()` | `void` | Suspends active event listening | +| `resume()` | `void` | Resumes suspended event listening | | `initKernelTraceSession()` | `void` | Best-effort initialization of the kernel trace session | | `stopKernelTraceSession()` | `void` | Best-effort teardown of the kernel trace session | -| `kernelTraceSession_` | `shared_ptr` | Underlying KrabsETW kernel trace object | -| `runningProviders_` | `KernelProvidersCollection` | Cache of active kernel-space providers | + +**Private state:** + +- `kernelTraceSession_` β€” KrabsETW `kernel_trace` session handle +- `runningProviders_` β€” Cache of active `krabs::kernel_provider` instances +- `mutex_` β€” Guards thread-safe access +- `endTraceSession_` / `traceSessionStopped_` β€” Atomic lifecycle flags +- `condition_` β€” Signals session readiness for resume ## Usage Example -```c -// Instantiate and start the kernel ETW session runnable -auto kernelSession = std::make_shared("KernelEtwSession"); +```cpp +#include -// Register a kernel provider with its event config -EtwProviderConfig providerConfig = /* ... */; -Status status = kernelSession->addProvider(providerConfig); +// Create and start the kernel ETW session thread +auto kernelSession = std::make_shared( + "KernelEtwSession" +); +// Register a kernel provider before starting +EtwProviderConfig providerConfig; // configure provider details +Status status = kernelSession->addProvider(providerConfig); if (!status.ok()) { - LOG(ERROR) << "Failed to add ETW kernel provider: " << status.getMessage(); + LOG(ERROR) << "Failed to add kernel ETW provider: " + << status.getMessage(); } -// Thread lifecycle is managed via InternalRunnable (Dispatcher) +// Start listening for kernel ETW events on a dedicated thread Dispatcher::addService(kernelSession); -``` - -## Thread Safety - -- All shared state is protected by `std::mutex mutex_` -- Session lifecycle flags use `std::atomic` (`endTraceSession_`, `traceSessionStopped_`) -- A `std::condition_variable` coordinates pause/resume signaling \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/events/windows/etw/.etw_post_processing_pipeline.md b/osquery/events/windows/etw/.etw_post_processing_pipeline.md index 2b4f5be28aa..46d84b9b396 100644 --- a/osquery/events/windows/etw/.etw_post_processing_pipeline.md +++ b/osquery/events/windows/etw/.etw_post_processing_pipeline.md @@ -3,42 +3,41 @@ Manages the ETW (Event Tracing for Windows) post-processing pipeline by collecti ## Key Components -### `EtwPostProcessorsRunnable` -A thread-safe runnable class (extending `InternalRunnable`) that processes ETW events from a shared concurrent queue and dispatches them to registered event subscribers. +**`EtwPostProcessorsRunnable`** β€” A thread-safe runnable class (extends `InternalRunnable`) that dequeues ETW events from a shared concurrent queue and routes them to registered post-processor callbacks. | Member | Type | Description | -|---|---|---| -| `addProvider()` | `Status` | Registers a post-processor callback for specific ETW event types | -| `start()` | `void` | Starts the processing thread | -| `stop()` | `void` | Signals the thread to stop | -| `CommonPostProcessing()` | `bool` (private) | Applies common enrichment logic to every ETW event | -| `etwPostProcessors_` | `ProviderProcessors` | Map of ETW event types to their post-processor callbacks | -| `shouldRun_` | `atomic` | Atomic flag controlling thread lifecycle | -| `concurrentQueue_` | `ConcurrentEventQueueRef&` | Reference to the shared concurrent event queue | +|--------|------|-------------| +| `addProvider()` | `Status` | Registers a post-processor callback for one or more ETW event types | +| `start()` | `void` | Starts the processing thread loop | +| `stop()` | `void` | Signals the thread to halt via `shouldRun_` flag | +| `CommonPostProcessing()` | `bool` | Applies common enrichment to every ETW event before dispatch | +| `etwPostProcessors_` | `ProviderProcessors` | Map of `EtwEventType` β†’ callback handler | +| `concurrentQueue_` | `ConcurrentEventQueueRef&` | Shared queue feeding raw ETW events into this pipeline | ## Usage Example ```cpp -// Create a shared concurrent queue -ConcurrentEventQueueRef sharedQueue = std::make_shared(); - -// Instantiate the post-processing pipeline runnable -auto postProcessor = std::make_shared( - "etw_post_processor", sharedQueue); - -// Register an ETW provider with its post-processing callback -EtwProviderConfig providerConfig; -// ... configure providerConfig with event types and callback ... -Status status = postProcessor->addProvider(providerConfig); - -if (status.ok()) { - // Dispatcher manages the thread lifecycle - Dispatcher::addService(postProcessor); +// Create the shared concurrent queue +auto sharedQueue = std::make_shared(); + +// Instantiate the post-processing runnable +auto pipeline = std::make_shared( + "ETWPostProcessor", sharedQueue); + +// Register a provider with its post-processor callback +EtwProviderConfig config; +// ... configure provider ... +auto status = pipeline->addProvider(config); +if (!status.ok()) { + LOG(ERROR) << "Failed to register ETW provider: " << status.getMessage(); } + +// Dispatch via osquery's dispatcher +Dispatcher::addService(pipeline); ``` ## Notes -- Uses `ProviderProcessors` (`std::map`) to route events to their handlers. -- `CommonPostProcessing()` runs on **every** event before provider-specific callbacks, enabling consistent event enrichment across all ETW sources. -- Thread safety is enforced via `std::atomic` for the run flag and the shared `ConcurrentEventQueueRef`. \ No newline at end of file +- `shouldRun_` is an `std::atomic` ensuring safe cross-thread shutdown signaling. +- `ProviderProcessors` is a `std::map` β€” each event type maps to exactly one callback. +- `CommonPostProcessing()` runs on **every** event prior to type-specific dispatch, making it suitable for timestamp normalization or process context enrichment. \ No newline at end of file diff --git a/osquery/events/windows/etw/.etw_provider_config.md b/osquery/events/windows/etw/.etw_provider_config.md index 0374d01d841..1a064f04278 100644 --- a/osquery/events/windows/etw/.etw_provider_config.md +++ b/osquery/events/windows/etw/.etw_provider_config.md @@ -1,13 +1,14 @@ -Defines the `EtwProviderConfig` class, which encapsulates the configurable parameters and callback hooks for an ETW (Event Tracing for Windows) provider within osquery's Windows event pipeline. +Configuration class for ETW (Event Tracing for Windows) providers in osquery, encapsulating provider settings, optional filtering flags, and pre/post-processing callbacks for event handling. ## Key Components -### `EtwKernelProviderType` (enum) -Enumerates supported kernel provider categories: -- `File`, `ImageLoad`, `Network`, `Process`, `Registry`, `ObjectManager`, `Invalid` +### `EtwProviderConfig` Class -### Type Aliases +**Enum** +- `EtwKernelProviderType` β€” Identifies supported kernel providers: `File`, `ImageLoad`, `Network`, `Process`, `Registry`, `ObjectManager`, `Invalid` + +**Type Aliases** | Alias | Underlying Type | |---|---| | `EtwProviderName` | `std::string` | @@ -16,36 +17,43 @@ Enumerates supported kernel provider categories: | `EventProviderPreProcessor` | `krabs::c_provider_callback` | | `EventProviderPostProcessor` | `std::function` | -### Core Methods -- **`setPreProcessor` / `getPreProcessor`** β€” Attach a raw `krabs` callback for low-level event pre-processing -- **`setPostProcessor` / `getPostProcessor`** β€” Attach a `std::function` for structured post-processing of `EtwEventDataRef` -- **`setEventTypes` / `addEventTypeToHandle`** β€” Define which ETW event types this provider handles -- **`isValid()`** β€” Validates the configuration before use -- **Optional flag setters/getters** β€” `AnyBitmask`, `AllBitmask`, `Level`, `TraceFlags`, `Tag` +**Core Methods** +- `setPreProcessor` / `getPreProcessor` β€” Attach a raw `krabs` callback for low-level event pre-processing +- `setPostProcessor` / `getPostProcessor` β€” Attach a `std::function` callback for structured post-processing +- `setEventTypes` / `addEventTypeToHandle` β€” Define which `EtwEventType` values this provider handles +- `isValid()` β€” Validates the configuration before use + +**Optional Flag Helpers** +- `isAnyBitmaskSet()`, `isAllBitmaskSet()`, `isLevelSet()`, `isTraceFlagsSet()`, `isTagSet()` β€” Check if optional filtering parameters were configured +- Corresponding `get*` / `set*` accessors for `AnyBitmask`, `AllBitmask`, `Level`, `TraceFlags`, `Tag` -### `EtwProviderConfigRef` -Convenience alias: `std::shared_ptr` +**Shared Pointer Alias** +- `EtwProviderConfigRef` β€” `std::shared_ptr` ## Usage Example ```cpp auto config = std::make_shared(); -// Configure a user-mode provider -config->setName("Microsoft-Windows-Kernel-Process"); -config->setLevel(osquery::EtwProviderConfig::EtwLevel(4)); // Informational -config->setAnyBitmask(osquery::EtwProviderConfig::EtwBitmask(0x10)); +// Configure a user-mode provider by name +config->setName("Microsoft-Windows-Security-Auditing"); + +// Set keyword bitmasks for event filtering +config->setAnyBitmask(0x8000000000000000ULL); +config->setLevel(4); // Information level -// Attach processing callbacks -config->setPreProcessor(myKrabsCallback); +// Register pre- and post-processing callbacks +config->setPreProcessor(&myKrabsCallback); config->setPostProcessor([](const osquery::EtwEventDataRef& event) { // Handle structured event data }); +// Add specific event types to handle config->addEventTypeToHandle(osquery::EtwEventType::ProcessStart); +// Validate before registering with the ETW controller auto status = config->isValid(); if (!status.ok()) { - // Handle invalid configuration + LOG(ERROR) << "Invalid ETW provider config: " << status.getMessage(); } ``` \ No newline at end of file diff --git a/osquery/events/windows/etw/.etw_publisher.md b/osquery/events/windows/etw/.etw_publisher.md index c9744ec7b2c..f20d877b5e8 100644 --- a/osquery/events/windows/etw/.etw_publisher.md +++ b/osquery/events/windows/etw/.etw_publisher.md @@ -1,46 +1,45 @@ -Windows ETW (Event Tracing for Windows) publisher base class that abstracts event collection and dispatching infrastructure for osquery's ETW event system. +Brief 1-2 sentence description: Defines the `EtwPublisherBase` abstract class and supporting macros for building Windows ETW (Event Tracing for Windows) event publishers within the osquery event framework, providing lifecycle management, pre/post-processing callbacks, and utility helpers for ETW event collection. ## Key Components ### Macros | Macro | Purpose | -|---|---| -| `REGISTER_ETW_PUBLISHER` | Registers a class as an ETW event publisher plugin | -| `REGISTER_ETW_SUBSCRIBER` | Registers a class as an ETW event subscriber plugin | -| `getPreProcessorCallback()` | Generates a lambda wrapping `providerPreProcessor` with structured error logging for raw ETW events | +|-------|---------| +| `REGISTER_ETW_PUBLISHER` | Registers a class as an `event_publisher` plugin | +| `REGISTER_ETW_SUBSCRIBER` | Registers a class as an `event_subscriber` plugin | +| `getPreProcessorCallback()` | Returns a lambda wrapping `providerPreProcessor` with exception-safe ETW event handling | ### Class: `EtwPublisherBase` -Abstract base class for all ETW publishers. Concrete publishers must inherit from this and implement `providerPostProcessor`. +Abstract base class for ETW event publishers. -| Member | Type | Description | -|---|---|---| -| `EtwEngine()` | Method | Returns reference to the global `EtwController` instance | -| `run()` | Method | Signals dispatcher that no dedicated running thread is required | -| `getPostProcessorCallback()` | Method | Returns a `std::function` wrapping the publisher's post-processor | -| `updateUserInfo()` | Protected | Resolves and caches username from a SID string | -| `updateHardVolumeWithLogicalDrive()` | Protected | Converts device paths (e.g., `\Device\HarddiskVolume3\`) to logical drive paths (e.g., `C:\`) | -| `providerPostProcessor()` | Private/Pure virtual | Must be implemented by concrete ETW publishers to handle parsed event data | +| Member | Description | +|--------|-------------| +| `EtwPublisherBase(name)` | Constructor accepting a provider name | +| `EtwEngine()` | Returns a reference to the global `EtwController` instance | +| `run()` | Signals the dispatcher that no dedicated thread is needed | +| `getPostProcessorCallback()` | Returns a lambda wrapping the virtual `providerPostProcessor` | +| `updateUserInfo(userSid, username)` | Resolves a SID to a username using an internal cache | +| `updateHardVolumeWithLogicalDrive(path)` | Converts a device path (e.g. `\Device\HarddiskVolume3`) to a logical drive path (e.g. `C:`) | +| `providerPostProcessor(data)` | **Pure virtual** β€” must be implemented by each concrete publisher | ## Usage Example ```cpp -class MyEtwPublisher : public EtwPublisherBase { +class MyEtwPublisher : public EtwPublisherBase, + public EventPublisher { public: - MyEtwPublisher() : EtwPublisherBase("my_etw_publisher") {} + MyEtwPublisher() : EtwPublisherBase("MyProvider") {} private: void providerPostProcessor(const EtwEventDataRef& data) override { - // Process and dispatch parsed ETW event data - std::string username; - updateUserInfo(data->userSid, username); - - std::string path = data->imagePath; - updateHardVolumeWithLogicalDrive(path); + // Parse and dispatch ETW event data to subscribers } }; REGISTER_ETW_PUBLISHER(MyEtwPublisher, "my_etw_publisher"); -``` \ No newline at end of file +``` + +> **Note:** Concrete publishers must implement `providerPostProcessor`. The `getPreProcessorCallback()` macro should be used when registering the provider with the ETW engine to ensure exception-safe raw event parsing. \ No newline at end of file diff --git a/osquery/events/windows/etw/.etw_publisher_dns.md b/osquery/events/windows/etw/.etw_publisher_dns.md index 99ba0945ddf..1fb5f149487 100644 --- a/osquery/events/windows/etw/.etw_publisher_dns.md +++ b/osquery/events/windows/etw/.etw_publisher_dns.md @@ -1,51 +1,52 @@ -ETW publisher header defining the DNS event collection pipeline for Windows ETW-based DNS monitoring in osquery. +ETW (Event Tracing for Windows) publisher header for capturing and dispatching DNS resolution events from the Windows kernel and user-mode ETW providers. ## Key Components ### Structs -- **`EtwDNSEventSubContext`** β€” Subscription context for DNS ETW events; friend-scoped to `EtwPublisherDNS` -- **`EtwDNSEventContext`** β€” Event context carrying an `EtwEventDataRef` payload dispatched to subscribers +- **`EtwDNSEventSubContext`** β€” Subscription context for DNS event subscribers; access restricted to `EtwPublisherDNS` +- **`EtwDNSEventContext`** β€” Event payload container holding an `EtwEventDataRef` data reference dispatched to subscribers ### Type Aliases -- **`EtwDNSEventContextRef`** β€” `shared_ptr` -- **`EtwDNSEventSubContextRef`** β€” `shared_ptr` +- **`EtwDNSEventContextRef`** β€” `std::shared_ptr` +- **`EtwDNSEventSubContextRef`** β€” `std::shared_ptr` ### Constants -- **`kEtwDNSPublisherName`** β€” Publisher identifier string `"etw_dns_publisher"` +- **`kEtwDNSPublisherName`** β€” Publisher identifier string: `"etw_dns_publisher"` ### Class: `EtwPublisherDNS` -Inherits from `EtwPublisherBase` and `EventPublisher`. +Inherits `EtwPublisherBase` and `EventPublisher`. | Member | Description | |---|---| -| `setUp()` | Configures ETW providers, parameters, and processing callbacks | -| `providerPreProcessor()` | Static C-function callback; lightweight entry point for raw ETW events from the OS | -| `providerPostProcessor()` | Post-processing hook for enriching/aggregating event data before dispatch | -| `hardVolumeDrives_` | Maps volume paths for drive resolution | -| `usernamesBySIDs_` | Caches SID-to-username lookups | +| `setUp()` | Registers ETW providers, configuration, and processing callbacks | +| `providerPreProcessor()` | Static C-function callback; lightweight entry point for raw `EVENT_RECORD` data from the OS | +| `providerPostProcessor()` | Post-processing callback; enriches and aggregates event data before subscriber dispatch | +| `hardVolumeDrives_` | Maps hard volume paths to drive letters | +| `usernamesBySIDs_` | Caches SID-to-username resolutions | -**Supported event versions:** -- DNS Start events: `Version0–3` -- DNS Stop events: `Version0–2` -- Kernel DNS events: `Version3–4` +**Supported ETW Event IDs/Versions:** + +| Event | IDs / Versions | +|---|---| +| Kernel DNS | Versions 3–4 | +| User DNS Start | Versions 0–3 (ID: 1) | +| User DNS Stop | Versions 0–2 (ID: 2) | ## Usage Example ```cpp #include -// Register and start the DNS ETW publisher -auto publisher = std::make_shared(); -publisher->setUp(); - // Subscribe to DNS events -auto sub = std::make_shared(); -auto callback = [](const EtwDNSEventContextRef& ctx, - const EtwDNSEventSubContextRef&) { - // Access enriched DNS event data +auto sub_ctx = std::make_shared(); + +EventFactory::registerEventPublisher(); + +// Access event data in a subscriber callback +void onDNSEvent(const EtwDNSEventContextRef& ctx, + const EtwDNSEventSubContextRef& sub) { auto& data = ctx->data; - // process data... - return Status::success(); -}; + // Process DNS resolution event data +} ``` \ No newline at end of file diff --git a/osquery/events/windows/etw/.etw_publisher_processes.md b/osquery/events/windows/etw/.etw_publisher_processes.md index 8578b68d793..ececcb7488f 100644 --- a/osquery/events/windows/etw/.etw_publisher_processes.md +++ b/osquery/events/windows/etw/.etw_publisher_processes.md @@ -1,59 +1,53 @@ -ETW publisher header for collecting and dispatching Windows process lifecycle events (start/stop) using the Event Tracing for Windows (ETW) infrastructure within osquery. +ETW publisher header that defines the `EtwPublisherProcesses` class for capturing and dispatching Windows ETW (Event Tracing for Windows) process start and stop events within the osquery framework. ## Key Components -### Structs / Type Aliases +**Structs / Type Aliases** +- `EtwProcEventSubContext` β€” Subscription context for process ETW events (friend-restricted to `EtwPublisherProcesses`) +- `EtwProcessEventContext` β€” Event context carrying an `EtwEventDataRef` payload +- `EtwProcessEventContextRef` / `EtwProcEventSubContextRef` β€” `shared_ptr` aliases for the above -| Type | Description | -|---|---| -| `EtwProcEventSubContext` | Subscription context for process event subscribers | -| `EtwProcessEventContext` | Event context carrying `EtwEventDataRef` payload data | -| `EtwProcessEventContextRef` | `shared_ptr` alias for `EtwProcessEventContext` | -| `EtwProcEventSubContextRef` | `shared_ptr` alias for `EtwProcEventSubContext` | - -### Constants - -- `kEtwProcessPublisherName` β€” Publisher identifier string: `"etw_process_publisher"` - -### Class: `EtwPublisherProcesses` +**Constants** +- `kEtwProcessPublisherName` β€” Publisher identifier string (`"etw_process_publisher"`) -Inherits from `EtwPublisherBase` and `EventPublisher`. Listens to kernel and user-mode ETW providers for process start/stop events. +**Class: `EtwPublisherProcesses`** +Inherits `EtwPublisherBase` and `EventPublisher`. | Member | Description | |---|---| -| `setUp()` | Registers ETW providers, configuration, and callbacks | -| `providerPreProcessor()` | Static C-function callback; lightweight ETW event entry point called per OS event | -| `providerPostProcessor()` | Enriches and aggregates event data before dispatching to subscribers | +| `setUp()` | Configures ETW providers, parameters, and processing callbacks | +| `providerPreProcessor()` | Static C-function callback; lightweight entry point called by the OS per ETW event | +| `providerPostProcessor()` | Post-processing: enriches, aggregates, and dispatches event data to subscribers | | `cleanOldAggregationCacheEntries()` | Purges stale cache entries | | `updateTokenInfo()` | Resolves token type metadata | -| `updateImagePath()` | Resolves process image paths via composite key lookup | -| `processStartAggregationCache_` | Cache keyed by composed `uint64` key for process start correlation | -| `processImageCache_` | Cache mapping composite keys to image path strings | +| `updateImagePath()` | Resolves process image path by composite key | +| `isSupportedEvent()` / `isKernelEvent()` / etc. | Inline header-level event classification helpers | +| `processStartAggregationCache_` | Maps composite key β†’ `EtwProcStartDataRef` | +| `processImageCache_` | Maps composite key β†’ image path string | -**Supported event versions tracked internally:** -- User process start: `Version0–3` -- User process stop: `Version0–2` -- Kernel process events: `Version3–4` +**Supported ETW Event Versions** + +| Enum | Versions | +|---|---| +| `etwUserProcStartVersion` | 0–3 | +| `etwUserProcStopVersion` | 0–2 | +| `etwKernelProcVersion` | 3–4 | ## Usage Example -```cpp -// Register and start the ETW process publisher -auto publisher = std::make_shared(); - -Status status = publisher->setUp(); -if (!status.ok()) { - LOG(ERROR) << "ETW process publisher setup failed: " << status.getMessage(); -} - -// Subscribe to process events -auto subscription = EtwProcEventSubContext::create(); -EventFactory::addSubscription(kEtwProcessPublisherName, subscription, - [](const EtwProcessEventContextRef& ctx, - const EtwProcEventSubContextRef& /*sub*/) -> Status { - auto& data = ctx->data; - // Handle process start/stop event data - return Status::success(); - }); +```c +// Register a subscriber to process events +auto sub_ctx = std::make_shared(); + +EventSubscriber::subscribe( + [](const EtwProcessEventContextRef& event_ctx, + const EtwProcEventSubContextRef& sub_ctx) -> Status { + + auto& data = event_ctx->data; + // Inspect process start/stop data + return Status::success(); + }, + sub_ctx +); ``` \ No newline at end of file diff --git a/osquery/events/windows/etw/.etw_user_session.md b/osquery/events/windows/etw/.etw_user_session.md index 96784ea8cb2..24667dc3d53 100644 --- a/osquery/events/windows/etw/.etw_user_session.md +++ b/osquery/events/windows/etw/.etw_user_session.md @@ -1,49 +1,36 @@ -Manages the ETW (Event Tracing for Windows) user-space trace session for osquery, running on a dedicated thread via the `InternalRunnable` abstraction to listen for and process user-space ETW events. +Manages the ETW (Event Tracing for Windows) user-space trace session within osquery, running the session on a dedicated thread via the `InternalRunnable` abstraction. ## Key Components -### `UserEtwSessionRunnable` (class) -Inherits from `InternalRunnable`. Owns and manages a `krabs::user_trace` session and a collection of registered ETW providers. +**`UserEtwSessionRunnable`** β€” Final class inheriting from `InternalRunnable` that owns and controls a KrabsETW `user_trace` session. | Member | Type | Description | |---|---|---| -| `addProvider()` | `Status` | Registers an ETW user-space provider with pre/post-processor callbacks | -| `start()` | `void` | Starts the dedicated trace session thread | -| `stop()` | `void` | Stops the dedicated trace session thread | -| `pause()` / `resume()` | `void` | Pauses and resumes event listening | -| `initUserTraceSession()` | `void` | Best-effort initialization of the KrabsETW user trace | -| `stopUserTraceSession()` | `void` | Best-effort teardown of the KrabsETW user trace | - -**Internal state:** -- `userTraceSession_` β€” KrabsETW `krabs::user_trace` instance -- `runningProviders_` β€” vector of active `krabs::provider<>` references -- `mutex_` β€” thread-safety guard -- `endTraceSession_` / `traceSessionStopped_` β€” atomic lifecycle flags -- `condition_` β€” condition variable for pause/resume signaling +| `addProvider()` | Public | Registers an ETW user-space provider with pre/post-processor callbacks | +| `start()` / `stop()` | Protected | Thread lifecycle hooks from `InternalRunnable` | +| `pause()` / `resume()` | Private | Suspends and resumes event listening | +| `initUserTraceSession()` | Private | Best-effort session initialization by name | +| `stopUserTraceSession()` | Private | Best-effort session teardown by name | +| `userTraceSession_` | `shared_ptr` | Underlying KrabsETW session handle | +| `runningProviders_` | `vector>>` | Cache of active providers | + +Thread safety is maintained via `std::mutex`, with `std::atomic` flags for session state and a `std::condition_variable` for pause/resume signaling. ## Usage Example ```cpp -#include - // Create and start a user ETW session runnable -auto session = std::make_shared("MyUserSession"); +auto session = std::make_shared("my_etw_session"); -// Register a provider using its configuration -osquery::EtwProviderConfig config; -// ... populate config with provider GUID, callbacks, etc. +// Register a provider before starting +EtwProviderConfig config; // populated with GUID, keywords, callbacks +Status s = session->addProvider(config); -auto status = session->addProvider(config); -if (!status.ok()) { - LOG(ERROR) << "Failed to add ETW provider: " << status.getMessage(); +if (s.ok()) { + // Start is called by the dispatcher thread manager + Dispatcher::addService(session); } - -// Start listening (runs on dedicated internal thread) -session->start(); - -// Stop when done -session->stop(); ``` -> **Note:** This class is Windows-only and depends on the [KrabsETW](https://github.com/microsoft/krabsetw) library (`krabs::user_trace`). It is `final` and not intended for subclassing. \ No newline at end of file +> **Note:** `addProvider()` must be called before the session starts processing. The class is `final` and not intended for further subclassing. \ No newline at end of file diff --git a/osquery/experimental/events_stream/.events_stream.md b/osquery/experimental/events_stream/.events_stream.md index 8e0dc05217d..6d7cbde5a06 100644 --- a/osquery/experimental/events_stream/.events_stream.md +++ b/osquery/experimental/events_stream/.events_stream.md @@ -1,9 +1,9 @@ -Declares a single function for dispatching serialized events within the osquery events subsystem. +Declares a single function for dispatching pre-serialized events within the osquery events subsystem. ## Key Components -- **`dispatchSerializedEvent(const std::string& event)`** β€” Accepts a serialized event string and dispatches it through the osquery event pipeline. +- **`dispatchSerializedEvent(const std::string& event)`** β€” Accepts a serialized event string and dispatches it through the osquery event stream pipeline. ## Usage Example @@ -11,8 +11,8 @@ Declares a single function for dispatching serialized events within the osquery #include "events_stream.h" // Dispatch a pre-serialized event payload -std::string serialized = serializeMyEvent(data); -osquery::events::dispatchSerializedEvent(serialized); +std::string serializedPayload = R"({"type":"process","pid":1234})"; +osquery::events::dispatchSerializedEvent(serializedPayload); ``` -> **Note:** The function expects the event to already be serialized before being passed in. Serialization format is determined by the calling subsystem. \ No newline at end of file +> **Note:** The function operates within the `osquery::events` namespace. Callers are responsible for serializing the event data before passing it to this function. \ No newline at end of file diff --git a/osquery/experimental/events_stream/.events_stream_registry.md b/osquery/experimental/events_stream/.events_stream_registry.md index 2f172cfff4c..f5d6fd837c9 100644 --- a/osquery/experimental/events_stream/.events_stream_registry.md +++ b/osquery/experimental/events_stream/.events_stream_registry.md @@ -1,13 +1,10 @@ -Declares the `EventsStreamPlugin` class and supporting utilities for the osquery events stream plugin registry, enabling event data to be dispatched through the plugin framework. +Defines the `EventsStreamPlugin` and registry name for osquery's events stream plugin system, enabling plugins to process and forward event data through the plugin registry. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `EventsStreamPlugin` | Class | Plugin subclass that handles event stream dispatch via the osquery plugin registry | -| `EventsStreamPlugin::call` | Method | Processes incoming `PluginRequest` and populates a `PluginResponse`; overrides `Plugin::call` | -| `events::streamRegistryName` | Function | Returns the canonical string name used to register the events stream in the plugin registry | +- **`EventsStreamPlugin`** β€” Extends `Plugin` base class; implements the `call()` method to handle incoming plugin requests and populate responses for event stream processing +- **`events::streamRegistryName()`** β€” Returns the string identifier used to register and look up the events stream plugin in osquery's plugin registry ## Usage Example @@ -17,22 +14,16 @@ class MyEventsStreamPlugin : public EventsStreamPlugin { public: Status call(const PluginRequest& request, PluginResponse& response) override { - // Handle the incoming event stream request - auto event_data = request.at("data"); + // Process incoming event stream data response.push_back({{"status", "ok"}}); return Status::success(); } }; -// Register using the canonical registry name +// Register with the events stream registry REGISTER(MyEventsStreamPlugin, events::streamRegistryName(), - "my_events_stream"); + "my_stream_plugin"); ``` -## Dependencies - -- `osquery/core/plugins/plugin.h` β€” base `Plugin` class and `PluginRequest`/`PluginResponse` types -- `osquery/core/query.h` β€” core query types shared across the osquery plugin framework -- `osquery/utils/expected/expected.h` β€” `Expected<>` utility for error-propagating return types -- `osquery/numeric_monitoring/numeric_monitoring.h` β€” numeric metrics instrumentation support \ No newline at end of file +> **Note:** Plugins extending `EventsStreamPlugin` are registered under the name returned by `events::streamRegistryName()`, ensuring correct routing within osquery's plugin dispatch system. \ No newline at end of file diff --git a/osquery/experimental/experiments/linuxevents/src/.bpfprocesseventstable.md b/osquery/experimental/experiments/linuxevents/src/.bpfprocesseventstable.md index 136755aa829..ba935c0d998 100644 --- a/osquery/experimental/experiments/linuxevents/src/.bpfprocesseventstable.md +++ b/osquery/experimental/experiments/linuxevents/src/.bpfprocesseventstable.md @@ -1,43 +1,40 @@ -Header defining the `BPFProcessEventsTable` plugin, an osquery table that collects Linux process events captured via BPF (Berkeley Packet Filter) tracing. +Declares the `BPFProcessEventsTable` plugin, an osquery table that ingests Linux process events captured via BPF (Berkeley Packet Filter) and exposes them as queryable SQL rows. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `BPFProcessEventsTable` | Class | osquery `TablePlugin` that exposes BPF process events as a queryable table | -| `ErrorCode` | Enum | Factory error states β€” currently `MemoryAllocationFailure` | -| `Ptr` | Type alias | `std::shared_ptr` for safe ownership | -| `create()` | Static factory | Returns `Expected` β€” preferred construction path | -| `addEvents()` | Method | Ingests a batch of `ILinuxEvents::EventList` events into the table buffer | -| `name()` | Method | Returns the registered osquery table name | -| `columns()` | Override | Declares the table schema to the osquery runtime | -| `generate()` | Override | Produces `TableRows` in response to SQL queries | +| Symbol | Kind | Description | +|--------|------|-------------| +| `BPFProcessEventsTable` | Class | `TablePlugin` subclass that buffers and serves BPF process events | +| `ErrorCode` | Enum | Factory error variants (`MemoryAllocationFailure`) | +| `Ptr` | Type alias | `std::shared_ptr` | +| `create()` | Static factory | Returns `Expected`; only safe construction path | +| `addEvents()` | Public method | Ingests a batch of `ILinuxEvents::EventList` into the internal buffer | +| `name()` | Public method | Returns the registered SQL table name | +| `columns()` | Private override | Declares the table schema to the osquery core | +| `generate()` | Private override | Converts buffered events into `TableRows` on query execution | +| `PrivateData` | Private struct | Pimpl idiom; hides internal state from consumers | ## Usage Example ```cpp #include "bpfprocesseventstable.h" -// Create the table plugin via factory -auto result = osquery::BPFProcessEventsTable::create(); -if (result.isError()) { - // Handle MemoryAllocationFailure - return; +// Create the table (returns Expected, check for errors) +auto tableOrError = osquery::BPFProcessEventsTable::create(); +if (tableOrError.isError()) { + LOG(ERROR) << "Failed to create BPFProcessEventsTable"; + return; } -osquery::BPFProcessEventsTable::Ptr table = result.take(); +osquery::BPFProcessEventsTable::Ptr table = tableOrError.take(); -// Push captured BPF events into the table -tob::linuxevents::ILinuxEvents::EventList events = captureEvents(); +// Feed incoming BPF events into the table +tob::linuxevents::ILinuxEvents::EventList events = linuxEvents->poll(); table->addEvents(std::move(events)); -// osquery runtime calls generate() internally when -// a query like `SELECT * FROM bpf_process_events` is executed +// osquery core calls generate() automatically on SQL queries: +// SELECT * FROM bpf_process_events; ``` -## Notes - -- Non-copyable by design β€” copy constructor and assignment operator are explicitly deleted -- Uses the **PIMPL idiom** (`PrivateData`) to hide implementation details and reduce compile-time dependencies -- Depends on `tob::linuxevents::ILinuxEvents` from the `linuxevents` library for raw event ingestion \ No newline at end of file +> **Note:** Copy construction and copy assignment are explicitly deleted. Always manage instances through the `Ptr` (`shared_ptr`) returned by `create()`. \ No newline at end of file diff --git a/osquery/experimental/experiments/linuxevents/src/.linuxeventsservice.md b/osquery/experimental/experiments/linuxevents/src/.linuxeventsservice.md index b8afb1daee8..d7976701341 100644 --- a/osquery/experimental/experiments/linuxevents/src/.linuxeventsservice.md +++ b/osquery/experimental/experiments/linuxevents/src/.linuxeventsservice.md @@ -1,15 +1,18 @@ -A background service class that runs as an osquery internal thread, consuming BPF-sourced Linux process events and feeding them into the `BPFProcessEventsTable`. +Background service that runs as an osquery internal thread, consuming BPF-based Linux process events and forwarding them to a `BPFProcessEventsTable` for query access. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `LinuxEventsService` | Class | Final class inheriting from `InternalRunnable`; manages the lifecycle of the BPF event processing loop | -| `LinuxEventsService(BPFProcessEventsTable&)` | Constructor | Binds the service to a specific `BPFProcessEventsTable` instance | -| `start()` | Protected method | Entry point called by the dispatcher to begin event consumption | -| `stop()` | Protected method | Signals the service to halt gracefully | -| `PrivateData` | Private struct (PIMPL) | Opaque implementation detail; hides internal state behind a `unique_ptr` | +### `LinuxEventsService` +Extends `InternalRunnable` to provide a long-running background dispatcher thread. + +| Member | Description | +|--------|-------------| +| `LinuxEventsService(BPFProcessEventsTable&)` | Constructor accepting a reference to the target events table | +| `~LinuxEventsService()` | Virtual destructor for safe polymorphic cleanup | +| `start()` | Overridden lifecycle method; begins event consumption loop | +| `stop()` | Overridden lifecycle method; signals the service to halt | +| `PrivateData` | Pimpl struct hiding internal implementation details | ## Usage Example @@ -18,18 +21,22 @@ A background service class that runs as an osquery internal thread, consuming BP #include "bpfprocesseventstable.h" #include -// Instantiate the table that will receive BPF process events -osquery::BPFProcessEventsTable bpfTable; +// Instantiate the BPF events table +BPFProcessEventsTable eventsTable; + +// Create and register the background service +auto service = std::make_shared(eventsTable); -// Create and register the service with the osquery dispatcher -auto service = std::make_shared(bpfTable); +// Dispatch via osquery's internal runner osquery::Dispatcher::addService(service); -// The dispatcher calls start() internally; stop() is invoked on shutdown +// Service runs start() in background thread until stop() is called +// or the dispatcher shuts down ``` ## Notes -- Uses the **PIMPL idiom** (`PrivateData` + `unique_ptr`) to keep implementation details out of the header and reduce recompilation overhead. -- Marked `final` β€” not intended to be subclassed. -- Depends on `osquery::InternalRunnable` (from `dispatcher/dispatcher.h`) for thread lifecycle management. \ No newline at end of file +- Uses the **Pimpl idiom** (`PrivateData`) to hide BPF polling internals and keep the header lean +- Marked `final` β€” not intended to be subclassed further +- Lifecycle (`start`/`stop`) is managed by the osquery `Dispatcher`, not called directly by consumers +- Depends on `BPFProcessEventsTable` defined in `bpfprocesseventstable.h` \ No newline at end of file diff --git a/osquery/extensions/.extensions.md b/osquery/extensions/.extensions.md index d9e0be1f5b4..7ed62bd561d 100644 --- a/osquery/extensions/.extensions.md +++ b/osquery/extensions/.extensions.md @@ -7,46 +7,47 @@ Defines the public API for osquery's extension system, providing interfaces for | Flag | Type | Description | |------|------|-------------| | `extensions_socket` | string | Path to the extension manager socket | -| `extensions_autoload` | string | Search path for autoloading extensions | -| `extensions_timeout` | string | Global timeout for extension operations | -| `disable_extensions` | bool | Toggle to disable the extension subsystem | +| `extensions_autoload` | string | Search path for auto-loaded extensions | +| `extensions_timeout` | string | Timeout duration for extension operations | +| `disable_extensions` | bool | Disables the extension subsystem | ### Structs & Types -- **`ExtensionInfo`** β€” Metadata container (name, version, sdk_version, min_sdk_version) matching Thrift's `InternalExtensionInfo` +- **`ExtensionInfo`** β€” Metadata container for an extension (`name`, `version`, `sdk_version`, `min_sdk_version`) - **`ExtensionList`** β€” `std::map` mapping route IDs to extension metadata ### Classes -- **`ExternalSQLPlugin`** β€” `SQLPlugin` implementation that routes SQL queries through an extension registry, exposing `query()` and `getQueryColumns()` +- **`ExternalSQLPlugin`** β€” `SQLPlugin` implementation that routes `query()` and `getQueryColumns()` calls through the extension registry ### Key Functions | Function | Description | |----------|-------------| -| `getExtensionSocket()` | Derives a socket path for a given `RouteUUID` | +| `getExtensionSocket()` | Resolves the socket path for a given `RouteUUID` | | `loadExtensions()` | Reads autoload flags and returns valid extension paths | | `startExtension()` | Launches an `ExtensionRunner` thread for an extension process | | `startExtensionManager()` | Launches the `ExtensionManagerRunner` thread | -| `startExtensionWatcher()` | Starts a watcher thread monitoring a manager connection | -| `callExtension()` | Invokes a plugin exposed by an extension over a socket | -| `getExtensions()` | Retrieves the list of currently active extensions | -| `pingExtension()` | Health-checks an extension or manager socket | -| `applyExtensionDelay()` | Runs a predicate in a retry loop bounded by the extension timeout | -| `initShellSocket()` | Resolves a unique socket path for `osqueryi` shell sessions | +| `callExtension()` | Invokes a plugin exposed by an extension over its socket | +| `getExtensions()` | Returns the list of currently active extensions | +| `pingExtension()` | Health-checks an extension or manager via its socket path | +| `applyExtensionDelay()` | Polls a predicate until it succeeds or the configured timeout expires | +| `initShellSocket()` | Generates a unique socket path for `osqueryi` shell instances | ## Usage Example ```c -// Autoload extensions at startup -std::set paths = osquery::loadExtensions(); - -// Start the extension manager -osquery::startExtensionManager(); - -// Call a plugin exposed by a registered extension -osquery::PluginRequest req = {{"action", "query"}}; -osquery::PluginResponse resp; -osquery::callExtension(uuid, "table", "my_table", req, resp); - -// Start an extension process and connect to the manager -osquery::startExtension("my_extension", "1.0.0", "2.0.0"); +// Start an extension with version and SDK constraint +Status s = startExtension("my_extension", "1.0.0", "2.0.0"); + +// Call a plugin exposed by an extension +PluginRequest req = {{"action", "query"}}; +PluginResponse resp; +Status call_status = callExtension(uuid, "table", "my_table", req, resp); + +// Poll until an extension is ready or timeout expires +Status result = applyExtensionDelay([](bool& stop) -> Status { + ExtensionList exts; + auto s = getExtensions(exts); + if (!exts.empty()) stop = true; + return s; +}); ``` \ No newline at end of file diff --git a/osquery/extensions/.interface.md b/osquery/extensions/.interface.md index 5b471e23d4f..7c284b354a8 100644 --- a/osquery/extensions/.interface.md +++ b/osquery/extensions/.interface.md @@ -1,40 +1,51 @@ -Defines the abstract interfaces, data structures, and concrete classes for osquery's Thrift-based extension IPC system, providing both server-side handlers and client-side accessors for extension/extension-manager communication. +Defines the abstract interfaces, data structures, and server/client classes for osquery's Thrift-based extension communication layer, enabling extensions to register, communicate with, and be managed by the osquery core process. ## Key Components -| Class / Type | Role | -|---|---| -| `ExtensionAPI` | Pure abstract base defining `ping()`, `call()`, and `shutdown()` | -| `ExtensionManagerAPI` | Pure abstract base for manager operations: register, deregister, query, options | -| `ExtensionInterface` | Concrete `ExtensionAPI` server implementation with a transient `RouteUUID` | -| `ExtensionManagerInterface` | Full server implementation combining both APIs; manages route table and extension list | -| `ExtensionRunnerInterface` | PIMPL wrapper around the Thrift server setup (`serve`, `connect`, `init`) | -| `ExtensionRunnerCore` | `InternalRunnable` base for dispatcher threads with start/stop lifecycle | -| `ExtensionRunner` | Dispatcher thread serving a single extension handler | -| `ExtensionManagerRunner` | Dispatcher thread serving the extension manager handler | -| `ExtensionClientCore` | Non-copyable PIMPL base for UNIX socket clients | -| `ExtensionClient` | Client to an individual extension (from the manager side) | -| `ExtensionManagerClient` | Client to the extension manager (from an extension side) | -| `Option` | Holds a flag's current value, default, and type string | -| `ExtensionCode` | Enum mirroring Thrift IDL status codes (`EXT_SUCCESS`, `EXT_FAILED`, `EXT_FATAL`) | +### Data Structures +- **`Option`** β€” Holds a flag's current value, default value, and type string +- **`ExtensionCode`** β€” Enum for extension response codes (`EXT_SUCCESS`, `EXT_FAILED`, `EXT_FATAL`) +- **Type aliases** β€” `OptionList`, `ExtensionRouteTable`, `ExtensionRegistry` + +### Abstract API Interfaces +- **`ExtensionAPI`** β€” Pure virtual base requiring `ping()`, `call()`, and `shutdown()` +- **`ExtensionManagerAPI`** β€” Pure virtual base requiring `extensions()`, `options()`, `registerExtension()`, `deregisterExtension()`, `query()`, and `getQueryColumns()` + +### Server-Side Classes +- **`ExtensionInterface`** β€” Implements `ExtensionAPI`; serves the extension's Thrift endpoint with an assigned `RouteUUID` +- **`ExtensionManagerInterface`** β€” Combines `ExtensionInterface` + `ExtensionManagerAPI`; manages extension registration, route tracking, and SQL passthrough +- **`ExtensionRunnerInterface`** β€” PIMPL wrapper around the Thrift server lifecycle (`serve`, `connect`, `init`, `stopServer`) +- **`ExtensionRunnerCore`** β€” Dispatcher-integrated runnable base with start/stop thread safety +- **`ExtensionRunner`** β€” Runs a single extension's Thrift server +- **`ExtensionManagerRunner`** β€” Runs the extension manager's Thrift server + +### Client-Side Classes +- **`ExtensionClientCore`** β€” PIMPL wrapper managing the UNIX socket client connection and timeouts +- **`ExtensionClient`** β€” Client for calling into an extension (`ping`, `call`, `shutdown`) +- **`ExtensionManagerClient`** β€” Client for calling into the extension manager ## Usage Example ```cpp // Start an extension manager server -auto manager_runner = std::make_unique( - "/var/osquery/osquery.em"); -Dispatcher::addService(std::move(manager_runner)); +auto manager_runner = std::make_shared("/var/osquery/osquery.em"); +Dispatcher::addService(manager_runner); -// Connect a client from an extension back to the manager +// Connect a client from an extension process ExtensionManagerClient client("/var/osquery/osquery.em"); RouteUUID uuid; ExtensionRegistry registry; +ExtensionInfo info; + Status s = client.registerExtension(info, registry, uuid); +if (s.ok()) { + // Extension registered; begin serving ExtensionRunner + auto runner = std::make_shared("/var/osquery/osquery.em", uuid); + Dispatcher::addService(runner); +} -// Call a plugin on a connected extension from the manager -ExtensionClient ext_client("/var/osquery/ext.sock"); -PluginResponse response; -ext_client.call("table", "users", {{"action", "generate"}}, response); +// Query osquery core SQL from an extension +QueryData results; +client.query("SELECT * FROM users", results); ``` \ No newline at end of file diff --git a/osquery/extensions/thrift/gen/.Extension.md b/osquery/extensions/thrift/gen/.Extension.md index 9af9061d1fe..9bba553829a 100644 --- a/osquery/extensions/thrift/gen/.Extension.md +++ b/osquery/extensions/thrift/gen/.Extension.md @@ -1,22 +1,24 @@ -Autogenerated Apache Thrift RPC interface header for the osquery Extension service, defining the client, processor, and service interface for inter-process communication between osquery core and its extensions. +Autogenerated Thrift (0.13.0) header defining the `Extension` service interface for osquery's IPC extension protocol within the `osquery::extensions` namespace. ## Key Components -| Class | Description | -|---|---| -| `ExtensionIf` | Pure virtual service interface defining the three RPC methods all extensions must implement | -| `ExtensionNull` | No-op implementation of `ExtensionIf`; useful as a stub or placeholder | -| `ExtensionIfFactory` / `ExtensionIfSingletonFactory` | Factory abstractions for creating/releasing handler instances per connection | -| `ExtensionClient` | Thrift-generated RPC client for calling remote extension methods over a protocol transport | -| `ExtensionProcessor` | Server-side dispatcher that routes incoming RPC calls to the appropriate handler method | -| Arg/Result structs | Serialization containers (`Extension_ping_args`, `Extension_call_args`, etc.) for each RPC method's parameters and return values | +- **`ExtensionIf`** β€” Pure virtual interface declaring the three service methods all extension handlers must implement: + - `ping()` β€” Health check returning `ExtensionStatus` + - `call()` β€” Invoke a plugin by registry/item name with a request payload, returning `ExtensionResponse` + - `shutdown()` β€” Signal the extension to terminate -## RPC Methods +- **`ExtensionIfFactory` / `ExtensionIfSingletonFactory`** β€” Factory abstractions for creating/releasing handler instances per connection; singleton variant wraps a shared handler -- **`ping`** β€” Health check; returns an `ExtensionStatus` -- **`call`** β€” Invokes a plugin by registry name and item key, passing an `ExtensionPluginRequest`; returns an `ExtensionResponse` -- **`shutdown`** β€” Signals the extension to terminate +- **`ExtensionNull`** β€” No-op implementation of `ExtensionIf`; useful as a stub or base class + +- **`Extension_*_args` / `Extension_*_pargs`** β€” Thrift-generated argument containers for each RPC method, with `read()`/`write()` for protocol serialization + +- **`Extension_*_result` / `Extension_*_presult`** β€” Result containers holding return values and `__isset` bitmask structs for optional field tracking + +- **`ExtensionClient`** β€” Concrete Thrift client implementing `ExtensionIf`; wraps input/output `TProtocol` instances and exposes split `send_*/recv_*` async-style calls + +- **`ExtensionProcessor`** β€” `TDispatchProcessor` subclass that routes incoming RPC calls by method name to the appropriate `process_*` handler via a `ProcessMap` ## Usage Example @@ -25,14 +27,13 @@ Autogenerated Apache Thrift RPC interface header for the osquery Extension servi #include #include -// Connect to a running extension over a socket +// Connect to a running osquery extension auto socket = std::make_shared("localhost", 9090); auto transport = std::make_shared(socket); auto protocol = std::make_shared(transport); -transport->open(); - osquery::extensions::ExtensionClient client(protocol); +transport->open(); // Health check osquery::extensions::ExtensionStatus status; @@ -43,9 +44,5 @@ osquery::extensions::ExtensionResponse response; osquery::extensions::ExtensionPluginRequest req; client.call(response, "table", "processes", req); -// Shut down the extension -client.shutdown(); transport->close(); -``` - -> **Note:** This file is autogenerated by Thrift Compiler 0.13.0. Do not edit manually β€” regenerate from the `.thrift` IDL definition instead. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/extensions/thrift/gen/.ExtensionManager.md b/osquery/extensions/thrift/gen/.ExtensionManager.md index 39e9729e8f1..730206b965a 100644 --- a/osquery/extensions/thrift/gen/.ExtensionManager.md +++ b/osquery/extensions/thrift/gen/.ExtensionManager.md @@ -1,52 +1,53 @@ -Autogenerated Thrift RPC header (Thrift Compiler 0.13.0) defining the `ExtensionManager` service interface for osquery's inter-process extension communication within the `osquery::extensions` namespace. +Autogenerated Thrift RPC header (Thrift Compiler 0.13.0) defining the `ExtensionManager` service interface for osquery's extension IPC layer. Extends the base `Extension` service with manager-specific operations for registering, listing, and querying extensions. ## Key Components -| Class | Description | +| Class | Purpose | |---|---| -| `ExtensionManagerIf` | Pure virtual interface extending `ExtensionIf`; defines all RPC method contracts | -| `ExtensionManagerIfFactory` | Abstract factory for creating/releasing `ExtensionManagerIf` handler instances | -| `ExtensionManagerIfSingletonFactory` | Concrete factory wrapping a shared singleton handler instance | -| `ExtensionManagerNull` | No-op implementation of all interface methods; useful for testing/stubbing | -| `ExtensionManager_*_args / _pargs` | Thrift-generated argument containers for each RPC call | -| `ExtensionManager_*_result / _presult` | Thrift-generated result containers with `__isset` tracking for optional fields | - -### RPC Methods (via `ExtensionManagerIf`) - -| Method | Description | -|---|---| -| `extensions()` | Returns list of registered extensions (`InternalExtensionList`) | -| `options()` | Returns available options (`InternalOptionList`) | -| `registerExtension()` | Registers an extension with its info and route registry | -| `deregisterExtension()` | Removes an extension by UUID | -| `query()` | Executes a SQL query via an extension | -| `getQueryColumns()` | Returns column metadata for a SQL query | +| `ExtensionManagerIf` | Pure virtual interface; defines the 6 RPC methods all manager implementations must provide | +| `ExtensionManagerIfFactory` | Abstract factory for creating per-connection handler instances | +| `ExtensionManagerIfSingletonFactory` | Concrete factory wrapping a single shared `ExtensionManagerIf` instance | +| `ExtensionManagerNull` | No-op stub implementing all methods as empty returns (useful for testing/mocking) | +| `ExtensionManager_*_args / _pargs` | Thrift-generated request serialization structs per RPC method | +| `ExtensionManager_*_result / _presult` | Thrift-generated response deserialization structs per RPC method | + +### RPC Methods (defined on `ExtensionManagerIf`) + +```text +extensions() β†’ InternalExtensionList List all registered extensions +options() β†’ InternalOptionList Retrieve osquery config options +registerExtension() β†’ ExtensionStatus Register a new extension + its routes +deregisterExtension() β†’ ExtensionStatus Remove an extension by UUID +query() β†’ ExtensionResponse Execute a SQL query +getQueryColumns() β†’ ExtensionResponse Introspect SQL query columns +``` ## Usage Example ```cpp -#include "ExtensionManager.h" - -// Implement the interface +// Implement the interface to create a custom extension manager class MyExtensionManager : virtual public osquery::extensions::ExtensionManagerIf { public: - void extensions(osquery::extensions::InternalExtensionList& _return) override { - // populate _return with registered extensions + void extensions(InternalExtensionList& _return) override { + // populate _return with active extensions } - void query(osquery::extensions::ExtensionResponse& _return, - const std::string& sql) override { - // execute SQL and populate _return + void registerExtension( + ExtensionStatus& _return, + const InternalExtensionInfo& info, + const ExtensionRegistry& registry) override { + // validate and register the extension + _return.code = 0; // success } - // ... implement remaining pure virtual methods + // implement remaining pure virtual methods... }; -// Wrap in singleton factory for Thrift server -auto handler = std::make_shared(); +// Use singleton factory to serve one shared instance over Thrift transport +auto manager = std::make_shared(); auto factory = std::make_shared< - osquery::extensions::ExtensionManagerIfSingletonFactory>(handler); + osquery::extensions::ExtensionManagerIfSingletonFactory>(manager); ``` -> **Note:** This file is autogenerated β€” do not edit manually. Regenerate using Thrift Compiler 0.13.0 from the corresponding `.thrift` service definition. \ No newline at end of file +> **Note:** Do not edit this file directly β€” it is regenerated by the Thrift compiler from the osquery `.thrift` service definition. Modify the upstream `.thrift` source instead. \ No newline at end of file diff --git a/osquery/extensions/thrift/gen/.osquery_constants.md b/osquery/extensions/thrift/gen/.osquery_constants.md index 2d1f433767c..8de9e09fceb 100644 --- a/osquery/extensions/thrift/gen/.osquery_constants.md +++ b/osquery/extensions/thrift/gen/.osquery_constants.md @@ -1,12 +1,10 @@ -Auto-generated Thrift constants header for the `osquery` extensions namespace. Declares the `osqueryConstants` class and its global instance used across the osquery Thrift service interface. +Autogenerated Thrift constants header for the `osquery` extensions namespace. Declares the `osqueryConstants` class and its global instance, produced by the Thrift IDL compiler (v0.13.0). ## Key Components -| Symbol | Type | Description | -|--------|------|-------------| -| `osqueryConstants` | Class | Thrift-generated constants container for the osquery extensions namespace | -| `g_osquery_constants` | `extern const osqueryConstants` | Global singleton instance of the constants class | +- **`osqueryConstants`** β€” Auto-generated class intended to hold Thrift-defined constants for the osquery extensions interface. Currently contains only a constructor with no declared constant fields. +- **`g_osquery_constants`** β€” Extern global instance of `osqueryConstants`, accessible across translation units that include this header. ## Usage Example @@ -14,8 +12,17 @@ Auto-generated Thrift constants header for the `osquery` extensions namespace. D #include "osquery_constants.h" // Access the global constants instance -const osquery::extensions::osqueryConstants& constants = - osquery::extensions::g_osquery_constants; +const osquery::extensions::osqueryConstants& consts = osquery::extensions::g_osquery_constants; + +// Use within the namespace +namespace osquery { +namespace extensions { + void example() { + // Reference the global constants object directly + (void)g_osquery_constants; + } +} +} ``` -> **⚠️ Do not manually edit this file.** It is auto-generated by the Thrift Compiler (v0.13.0). Any changes will be overwritten on regeneration. Modify the upstream `.thrift` schema definition instead. \ No newline at end of file +> **Note:** This file is auto-generated by the [Apache Thrift](https://thrift.apache.org/) compiler and should not be edited manually. Any changes will be overwritten on the next Thrift compilation. Modify the upstream `.thrift` IDL file instead. \ No newline at end of file diff --git a/osquery/extensions/thrift/gen/.osquery_types.md b/osquery/extensions/thrift/gen/.osquery_types.md index 117febb4c4b..1e4e2892509 100644 --- a/osquery/extensions/thrift/gen/.osquery_types.md +++ b/osquery/extensions/thrift/gen/.osquery_types.md @@ -1,10 +1,10 @@ -Autogenerated Thrift header defining the core data types and structures used for osquery extension IPC communication within the `osquery::extensions` namespace. +Autogenerated Thrift header defining the core C++ data types and structures used for osquery's extension IPC (inter-process communication) protocol within the `osquery::extensions` namespace. ## Key Components ### Enum -- **`ExtensionCode`** β€” Status codes: `EXT_SUCCESS` (0), `EXT_FAILED` (1), `EXT_FATAL` (2) +- **`ExtensionCode`** β€” Result codes: `EXT_SUCCESS` (0), `EXT_FAILED` (1), `EXT_FATAL` (2) ### Type Aliases | Typedef | Underlying Type | @@ -18,33 +18,41 @@ Autogenerated Thrift header defining the core data types and structures used for | `InternalExtensionList` | `map` | ### Classes -- **`InternalOptionInfo`** β€” Holds a config option's `value`, `default_value`, and `type` strings +- **`InternalOptionInfo`** β€” Holds an option's `value`, `default_value`, and `type` as strings - **`InternalExtensionInfo`** β€” Describes a registered extension: `name`, `version`, `sdk_version`, `min_sdk_version` - **`ExtensionStatus`** β€” Response status with integer `code`, `message`, and `uuid` -- **`ExtensionResponse`** β€” Combines an `ExtensionStatus` with an `ExtensionPluginResponse` payload -- **`ExtensionException`** β€” Throwable Thrift exception carrying `code`, `message`, and `uuid` +- **`ExtensionResponse`** β€” Wraps an `ExtensionStatus` alongside an `ExtensionPluginResponse` payload +- **`ExtensionException`** β€” Thrift exception type mirroring `ExtensionStatus` fields; implements `what()` -All classes inherit from `apache::thrift::TBase` and support Thrift protocol `read()`/`write()`, equality operators, and `std::ostream` output. +All classes inherit from `apache::thrift::TBase` (or `TException`) and support Thrift serialization via `read()`/`write()` methods. ## Usage Example ```cpp #include "osquery_types.h" -osquery::extensions::ExtensionStatus status; -status.__set_code(osquery::extensions::ExtensionCode::EXT_SUCCESS); +using namespace osquery::extensions; + +// Build a successful status response +ExtensionStatus status; +status.__set_code(ExtensionCode::EXT_SUCCESS); status.__set_message("OK"); status.__set_uuid(42); -osquery::extensions::ExtensionResponse resp; -resp.__set_status(status); +// Attach a payload +ExtensionPluginResponse rows = { + {{"name", "osqueryd"}, {"pid", "1234"}} +}; -// Serialize over a Thrift protocol -resp.write(protocol); +ExtensionResponse resp; +resp.__set_status(status); +resp.__set_response(rows); -// Check result -if (resp.status.code == osquery::extensions::ExtensionCode::EXT_SUCCESS) { - // handle success +// Handle an exception +try { + // ... Thrift call ... +} catch (const ExtensionException& ex) { + std::cerr << "Extension error: " << ex.what() << "\n"; } ``` diff --git a/osquery/filesystem/.filesystem.md b/osquery/filesystem/.filesystem.md index 0867794ccee..867540a2f92 100644 --- a/osquery/filesystem/.filesystem.md +++ b/osquery/filesystem/.filesystem.md @@ -1,39 +1,40 @@ -Filesystem utility header for osquery providing cross-platform file I/O, directory traversal, glob pattern resolution, and Linux-specific `/proc` filesystem helpers. +Filesystem utility header for osquery providing cross-platform file I/O, directory traversal, glob pattern resolution, and Linux `/proc` introspection primitives. ## Key Components ### Enums & Constants -- **`GlobLimits`** β€” Bitmask enum controlling glob traversal: `GLOB_FILES`, `GLOB_FOLDERS`, `GLOB_ALL`, `GLOB_NO_CANON` -- **`kSQLGlobWildcard`** / **`kSQLGlobRecursive`** β€” SQL `%` wildcard constants used in path pattern matching +- **`GlobLimits`** β€” Bitmask flags controlling glob traversal: `GLOB_FILES`, `GLOB_FOLDERS`, `GLOB_ALL`, `GLOB_NO_CANON` +- **`kSQLGlobWildcard`** / **`kSQLGlobRecursive`** β€” SQL-style `%` and `%%` wildcard constants ### File I/O -- **`readFile()`** β€” Reads file contents into a string or streams chunks via callback; respects `read_max` flag and handles special files (FIFOs, named pipes) safely -- **`writeTextFile()`** β€” Writes text to disk with configurable permissions and open mode flags -- **`isWritable()` / `isReadable()`** β€” Permission checks with optional effective UID support - -### Path Utilities -- **`pathExists()`** β€” Checks disk presence; returns `-1` (no input), `0` (not found), or `1` (found) -- **`isDirectory()`** β€” Tests if a path is a directory -- **`removePath()` / `movePath()`** β€” Delete or relocate files and directories -- **`createDirectory()`** β€” Creates directories with optional recursive and ignore-existing flags -- **`safePermissions()`** β€” Validates file ownership is root or current process UID (not world-writable) - -### Glob & Listing -- **`resolveFilePattern()`** β€” Expands SQL-style glob patterns into matching path lists -- **`replaceGlobWildcards()`** β€” Converts SQL `%` wildcards to filesystem `*` with optional path canonicalization -- **`listFilesInDirectory()` / `listDirectoriesInDirectory()`** β€” Enumerate directory contents, optionally recursive - -### System Helpers -- **`getHomeDirectories()`** β€” Returns all user home directories on the system -- **`osqueryHomeDirectory()`** β€” Returns osquery's protected local storage path -- **`parseJSON()`** β€” Parses a JSON file from disk into a Boost property tree -- **`lsperms()`** β€” Converts numeric permission mode to `ls`-style string - -### Linux-Only (`#ifdef __linux__`) -- **`procProcesses()`** β€” Lists running PIDs from `/proc` -- **`procDescriptors()`** β€” Enumerates open file descriptors for a given PID -- **`procReadDescriptor()`** β€” Resolves a descriptor's virtual symlink path +- **`readFile()`** β€” Read file contents into a string or stream via chunk callback; non-blocking, respects `read_max` flag +- **`writeTextFile()`** β€” Write string content to disk with configurable permissions and open mode +- **`isWritable()` / `isReadable()`** β€” Permission checks with optional effective-UID semantics +- **`pathExists()`** β€” Disk existence check returning coded `Status` + +### Directory Operations +- **`listFilesInDirectory()` / `listDirectoriesInDirectory()`** β€” Enumerate entries, optionally recursive +- **`resolveFilePattern()`** β€” Expand SQL/glob wildcard patterns into matching path lists +- **`replaceGlobWildcards()`** β€” Translate SQL `%` wildcards to filesystem `*`, with path canonicalization +- **`createDirectory()` / `removePath()` / `movePath()`** β€” Directory lifecycle helpers +- **`isDirectory()`** β€” Type check for a given path +- **`getHomeDirectories()`** β€” Returns all system home directory paths + +### Security & Permissions +- **`safePermissions()`** β€” Validates file is not in a world-writable `/tmp`-like directory and is root/process-owned +- **`lsperms()`** β€” Converts a numeric mode to an `ls`-style permission string + +### Utilities +- **`osqueryHomeDirectory()`** β€” Returns the osquery local storage home path +- **`parseJSON()`** β€” Deserializes a JSON file into a Boost property tree +- **`initializeFilesystemAPILocale()`** β€” Windows-only locale initialization + +### Linux `/proc` (ifdef `__linux__`) +- **`procProcesses()`** β€” Lists all PIDs from `/proc` +- **`procDescriptors()`** β€” Lists open file descriptors for a process +- **`procReadDescriptor()`** β€” Resolves a descriptor's symlink target +- **`readRawMemory()`** *(partial)* β€” Physical memory reads via `/dev/mem` ## Usage Example @@ -43,29 +44,20 @@ Filesystem utility header for osquery providing cross-platform file I/O, directo // Read a file std::string content; auto status = osquery::readFile("/etc/os-release", content); -if (!status.ok()) { - LOG(ERROR) << status.getMessage(); +if (status.ok()) { + LOG(INFO) << content; } -// Stream large file in chunks -osquery::readFile("/var/log/syslog", [](std::string_view chunk) { - processChunk(chunk); -}); - -// Resolve glob pattern (SQL-style wildcards) -std::vector matches; -osquery::resolveFilePattern("/home/%/.ssh/%.pub", matches, osquery::GLOB_FILES); - -// Write with explicit permissions -osquery::writeTextFile("/tmp/report.txt", "data", 0644); +// Glob-resolve all .conf files under /etc +std::vector results; +osquery::resolveFilePattern("/etc/%.conf", results, osquery::GLOB_FILES); +for (const auto& path : results) { + LOG(INFO) << path; +} -// Linux: iterate /proc +// Linux: list running PIDs #ifdef __linux__ std::set pids; osquery::procProcesses(pids); -for (const auto& pid : pids) { - std::map fds; - osquery::procDescriptors(pid, fds); -} #endif ``` \ No newline at end of file diff --git a/osquery/filesystem/.mock_file_structure.md b/osquery/filesystem/.mock_file_structure.md index 0ecfcc8c2c9..d8b786bdf4e 100644 --- a/osquery/filesystem/.mock_file_structure.md +++ b/osquery/filesystem/.mock_file_structure.md @@ -1,34 +1,34 @@ -Provides a test utility for generating a temporary mock directory structure used in osquery filesystem-related unit tests. +Utility header for creating a temporary mock directory structure used in unit tests. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `kTopLevelMockFolderName` | `extern const std::string` | Name constant for the top-level mock folder | -| `createMockFileStructure()` | `boost::filesystem::path` | Creates a small temporary directory tree for testing | +- **`kTopLevelMockFolderName`** β€” Extern constant string holding the name of the top-level mock folder. +- **`createMockFileStructure()`** β€” Generates a small temporary directory tree on disk and returns its root path as a `boost::filesystem::path`. ## Usage Example ```c #include "mock_file_structure.h" -#include -namespace osquery { +using namespace osquery; -void myFilesystemTest() { - // Create a temporary mock directory tree - boost::filesystem::path mockRoot = createMockFileStructure(); +void myTest() { + // Create a temporary directory tree for testing + boost::filesystem::path mockRoot = createMockFileStructure(); - // Use mockRoot in test assertions - // e.g., verify files exist under kTopLevelMockFolderName - boost::filesystem::path topLevel = mockRoot / kTopLevelMockFolderName; + // Use mockRoot as a base for file system operations under test + boost::filesystem::path target = mockRoot / kTopLevelMockFolderName; - // Clean up after test - boost::filesystem::remove_all(mockRoot); -} + // ... run assertions against the mock structure ... -} // namespace osquery + // Clean up after test (caller is responsible for removal) + boost::filesystem::remove_all(mockRoot); +} ``` -> **Note:** This header is intended exclusively for test environments. Do not use `createMockFileStructure()` in production code paths, as it generates temporary filesystem artifacts on the host. \ No newline at end of file +## Notes + +- Intended for **test environments only** β€” not for production use. +- The caller is responsible for cleaning up the generated directory structure after use. +- Requires Boost.Filesystem linkage. \ No newline at end of file diff --git a/osquery/filesystem/linux/.mounts.md b/osquery/filesystem/linux/.mounts.md index d3dfe9984dd..796192acc33 100644 --- a/osquery/filesystem/linux/.mounts.md +++ b/osquery/filesystem/linux/.mounts.md @@ -1,33 +1,30 @@ -Declares data structures and a utility function for querying mounted filesystem information within the osquery framework. +Defines data structures and the primary interface for querying mounted filesystem information within the osquery framework. ## Key Components -### Structs +### `MountInformation::StatFsInfo` +Nested struct holding raw `statfs` metrics for a mounted filesystem: +- `block_size` β€” optimal transfer block size +- `block_count` / `free_block_count` / `unprivileged_free_block_count` β€” block usage stats +- `inode_count` / `free_inode_count` β€” inode usage stats -- **`MountInformation::StatFsInfo`** β€” Nested struct holding low-level `statfs` metrics: - - `block_size` β€” Optimal transfer block size (`f_bsize`) - - `block_count` β€” Total data blocks (`f_blocks`) - - `free_block_count` β€” Free blocks (`f_bfree`) - - `unprivileged_free_block_count` β€” Free blocks available to unprivileged users (`f_bavail`) - - `inode_count` β€” Total inodes (`f_files`) - - `free_inode_count` β€” Free inodes (`f_ffree`) +### `MountInformation` +Top-level struct describing a single mounted filesystem: +- `type` β€” filesystem type (e.g., `ext4`, `tmpfs`) +- `device` / `device_alias` β€” raw and canonicalized device paths +- `path` β€” mount point +- `flags` β€” mount options string +- `optional_statfs_info` β€” `boost::optional`, absent if `statfs` failed -- **`MountInformation`** β€” Describes a single mounted filesystem: - - `type` β€” Filesystem type (e.g., `ext4`, `tmpfs`) - - `device` β€” Raw device path - - `device_alias` β€” Canonicalized device path - - `path` β€” Mount point - - `flags` β€” Mount options - - `optional_statfs_info` β€” Optional `StatFsInfo`; absent if `statfs` failed +### `MountedFilesystems` +Type alias for `std::vector` β€” the full collection of active mounts. -### Type Aliases - -- **`MountedFilesystems`** β€” `std::vector` representing all active mounts - -### Functions - -- **`getMountedFilesystems(MountedFilesystems&)`** β€” Populates the provided vector with information about all currently mounted filesystems; returns an osquery `Status` +### `getMountedFilesystems()` +```c +Status getMountedFilesystems(MountedFilesystems& mounted_fs_info); +``` +Populates the provided vector with information about all currently mounted filesystems. Returns an osquery `Status` indicating success or failure. ## Usage Example @@ -38,15 +35,15 @@ osquery::MountedFilesystems mounts; auto status = osquery::getMountedFilesystems(mounts); if (status.ok()) { - for (const auto& mount : mounts) { - std::cout << mount.path << " -> " << mount.device - << " [" << mount.type << "]\n"; - - if (mount.optional_statfs_info) { - auto& fs = *mount.optional_statfs_info; - std::cout << " Blocks: " << fs.block_count - << ", Free: " << fs.free_block_count << "\n"; + for (const auto& mount : mounts) { + std::cout << mount.path << " (" << mount.type << ")\n"; + + if (mount.optional_statfs_info.has_value()) { + const auto& fs = mount.optional_statfs_info.value(); + std::cout << " Blocks free: " << fs.free_block_count << "\n"; + } } - } } -``` \ No newline at end of file +``` + +The `optional_statfs_info` field should always be checked before access, as `statfs` may fail for certain virtual or remote filesystems. \ No newline at end of file diff --git a/osquery/filesystem/linux/.proc.md b/osquery/filesystem/linux/.proc.md index b43d18e1a1f..3ce17a360b1 100644 --- a/osquery/filesystem/linux/.proc.md +++ b/osquery/filesystem/linux/.proc.md @@ -1,57 +1,59 @@ -Linux `/proc` filesystem parsing utilities for osquery, providing socket enumeration, process namespace inspection, and file descriptor traversal on Linux systems. +Linux `/proc` filesystem utilities for osquery, providing process enumeration, socket inspection, and namespace resolution on Linux systems. ## Key Components ### Data Structures - -| Type | Description | -|------|-------------| -| `SocketInfo` | Holds socket metadata: family, protocol, local/remote address+port, state, and network namespace | -| `SocketProcessInfo` | Maps a socket to its owning process (`pid` + `fd`) | -| `SocketInodeToProcessInfoMap` | `map` keyed by socket inode | -| `ProcessNamespaceList` | `map` of namespace name to inode | +- **`SocketInfo`** β€” Holds socket metadata: family, protocol, local/remote address+port, Unix socket path, state, and network namespace inode +- **`SocketProcessInfo`** β€” Maps a socket inode to its owning process ID and file descriptor +- **`ProcessNamespaceList`** β€” Maps namespace type names to their inode numbers ### Constants +- **`kLinuxProcPath`** β€” Base path `/proc` +- **`kLinuxProtocolNames`** β€” Maps `IPPROTO_*` values to `/proc/net` filename suffixes (`tcp`, `udp`, etc.) +- **`tcp_states`** β€” Ordered TCP state name strings indexed by kernel state number -- `kLinuxProcPath` β€” `/proc` root path -- `kLinuxProtocolNames` β€” maps `IPPROTO_*` values to `/proc/net` filenames -- `tcp_states` β€” ordered TCP state name lookup by index - -### Functions - -| Function | Description | -|----------|-------------| -| `procGetSocketList` | Builds `SocketInfoList` from `/proc//net` for a given family/protocol | -| `procGetSocketListPacket` | Parses `/proc/net/packet` content into `SocketInfoList` | -| `procGetSocketInodeToProcessInfoMap` | Maps socket inodes to owning process via `/proc//fd` | +### Free Functions +| Function | Purpose | +|---|---| | `procGetProcessNamespaces` | Reads namespace inodes from `/proc//ns` | -| `procDecodeAddressFromHex` | Decodes hex-encoded addresses from `/proc/net` files for `AF_INET`/`AF_INET6` | -| `procDecodePortFromHex` | Decodes hex-encoded port numbers | -| `getProcRSS` | Returns RSS memory usage for a process | +| `procReadDescriptor` | Resolves a symlink under `/proc//fd` | +| `procDecodeAddressFromHex` | Decodes a hex-encoded IPv4/IPv6 address | +| `procDecodeUnsignedShortFromHex` | Decodes a hex-encoded port number | +| `procGetSocketList` | Builds `SocketInfoList` from `/proc//net` | +| `procGetSocketListPacket` | Parses `/proc/net/packet` entries | +| `procGetSocketInodeToProcessInfoMap` | Maps socket inodes β†’ owning process/fd | +| `procGetNamespaceInode` | Extracts inode from a namespace symlink | +| `getProcRSS` | Returns resident set size for a process | ### Templates - -- `procEnumerateProcesses` β€” Iterates all PIDs under `/proc`, invoking a callback per process; stops on first `false` return -- `procEnumerateProcessDescriptors` β€” Iterates `/proc//fd`, resolving symlinks and invoking a callback per descriptor +- **`procEnumerateProcesses`** β€” Iterates all numeric entries under `/proc`, invoking a callback per PID; stops on first `false` return +- **`procEnumerateProcessDescriptors`** β€” Iterates `/proc//fd`, resolving each symlink and invoking a callback per descriptor ## Usage Example ```cpp -// Enumerate all processes and collect socket info -struct MyData { SocketInfoList sockets; }; +// Enumerate all running processes and collect their names +struct MyData { std::vector pids; }; MyData data; -procEnumerateProcesses(data, [](const std::string& pid, MyData& d) -> bool { - procGetSocketList(AF_INET, IPPROTO_TCP, 0, pid, d.sockets); - return true; // continue iteration -}); +osquery::procEnumerateProcesses(data, + [](const std::string& pid, MyData& d) -> bool { + d.pids.push_back(pid); + return true; // continue iteration + }); + +// Resolve open file descriptors for a specific PID +struct FdData { std::vector links; }; +FdData fdData; +osquery::procEnumerateProcessDescriptors("1234", fdData, + [](const std::string&, const std::string&, + const std::string& link, FdData& d) -> bool { + d.links.push_back(link); + return true; + }); // Decode a hex address from /proc/net/tcp -std::string addr = procDecodeAddressFromHex("0100007F", AF_INET); +std::string addr = osquery::procDecodeAddressFromHex("0101007F", AF_INET); // addr == "127.0.0.1" - -// Map socket inodes to processes -SocketInodeToProcessInfoMap inodeMap; -procGetSocketInodeToProcessInfoMap("1234", inodeMap); ``` \ No newline at end of file diff --git a/osquery/filesystem/posix/.xattrs.md b/osquery/filesystem/posix/.xattrs.md index e9ae5a43997..a9e35dddc03 100644 --- a/osquery/filesystem/posix/.xattrs.md +++ b/osquery/filesystem/posix/.xattrs.md @@ -1,38 +1,48 @@ -Provides declarations for reading extended attributes (xattrs) from files on supported filesystems, exposing typed error enums and `Expected`-based result types for safe attribute retrieval. +Utility header for reading extended file attributes (xattrs) on Unix-like systems, providing typed error handling and result types for listing and retrieving xattr names and values. ## Key Components ### Error Enums -| Enum | Values | Description | -|------|--------|-------------| -| `XAttrFileError` | `NoLength`, `List`, `SizeChanged` | Errors when listing attribute names | -| `XAttrValueError` | `NoLength`, `Get`, `SizeChanged` | Errors when reading an attribute value | -| `XAttrGetError` | `GenericError`, `NoFile` | Top-level errors for the combined get operation | +- **`XAttrFileError`** β€” Errors when listing attribute names (`NoLength`, `List`, `SizeChanged`) +- **`XAttrValueError`** β€” Errors when reading attribute values (`NoLength`, `Get`, `SizeChanged`) +- **`XAttrGetError`** β€” Top-level errors for full attribute retrieval (`GenericError`, `NoFile`) ### Type Aliases -- **`ExtendedAttributeValue`** β€” `vector` raw byte buffer for an attribute's value -- **`ExtendedAttributeMap`** β€” `unordered_map` mapping attribute names to their values -- **`XAttrGetResult`** / **`XAttrNameListResult`** / **`XAttrValueResult`** β€” `Expected` wrappers for fallible return types +- **`ExtendedAttributeValue`** β€” Raw xattr value as `std::vector` +- **`ExtendedAttributeMap`** β€” Map of attribute name β†’ value (`unordered_map`) +- **`XAttrGetResult`** β€” `Expected` +- **`XAttrNameListResult`** β€” `Expected, XAttrFileError>` +- **`XAttrValueResult`** β€” `Expected` ### Functions -- **`xAttrFileErrorToString`** β€” Converts a `XAttrFileError` to a human-readable string given a file path -- **`xAttrValueErrorToString`** β€” Converts a `XAttrValueError` to a human-readable string given a path and attribute name -- **`getExtendedAttributesNames(int fd)`** β€” Lists all xattr names on an open file descriptor -- **`getExtendedAttributeValue(int fd, name)`** β€” Reads a single xattr value by name from an open file descriptor -- **`getExtendedAttributes(path)`** β€” High-level call: opens the file by path and returns all attributes as a map +- **`xAttrFileErrorToString`** β€” Human-readable message for a file listing error +- **`xAttrValueErrorToString`** β€” Human-readable message for a value retrieval error +- **`getExtendedAttributesNames(fd)`** β€” Lists all xattr names for an open file descriptor +- **`getExtendedAttributeValue(fd, name)`** β€” Reads a single xattr value by name from a file descriptor +- **`getExtendedAttributes(path)`** β€” Retrieves all xattr name/value pairs for a file path ## Usage Example -```c +```cpp +#include "xattrs.h" + auto result = osquery::getExtendedAttributes("/path/to/file"); -if (result.isError()) { - // handle XAttrGetError - return; +if (result.isValue()) { + for (const auto& [name, value] : result.get()) { + // process each attribute name and its raw byte value + } +} else { + auto error = result.getError(); + // handle XAttrGetError::NoFile or XAttrGetError::GenericError } -for (const auto& [name, value] : result.get()) { - // name -> std::string (e.g. "user.comment") - // value -> std::vector +// Per-descriptor usage +int fd = open("/path/to/file", O_RDONLY); +auto names = osquery::getExtendedAttributesNames(fd); +if (names.isValue()) { + for (const auto& name : names.get()) { + auto val = osquery::getExtendedAttributeValue(fd, name); + } } ``` \ No newline at end of file diff --git a/osquery/hashing/.hashing.md b/osquery/hashing/.hashing.md index dc6e457077c..ba00621c51a 100644 --- a/osquery/hashing/.hashing.md +++ b/osquery/hashing/.hashing.md @@ -1,49 +1,45 @@ -Provides hashing utilities for osquery, supporting MD5, SHA1, and SHA256 algorithms with hex or Base64 encoding, for buffers and filesystem paths. +Provides hashing utilities for the osquery framework, supporting MD5, SHA1, and SHA256 algorithms with hex or Base64 encoding for files and in-memory buffers. ## Key Components -### Enums -- **`HashType`** β€” Algorithm selector: `HASH_TYPE_MD5`, `HASH_TYPE_SHA1`, `HASH_TYPE_SHA256` -- **`HashEncodingType`** β€” Output encoding: `HASH_ENCODING_TYPE_HEX`, `HASH_ENCODING_TYPE_BASE64` +**Enums** +- `HashType` β€” Algorithm selector: `HASH_TYPE_MD5`, `HASH_TYPE_SHA1`, `HASH_TYPE_SHA256` +- `HashEncodingType` β€” Output encoding: `HASH_ENCODING_TYPE_HEX`, `HASH_ENCODING_TYPE_BASE64` -### Structs -- **`MultiHashes`** β€” Holds results for simultaneous multi-algorithm hashing (`md5`, `sha1`, `sha256` strings + a bitmask) +**Struct** +- `MultiHashes` β€” Holds concurrent results for MD5, SHA1, and SHA256 alongside a bitmask indicating which were computed -### Class: `Hash` -Non-copyable (move-only) streaming hash context. +**Class** +- `Hash` β€” Stateful, non-copyable (move-only) hasher; supports incremental `update()` calls for streaming large content, finalized via `digest()` -| Method | Description | -|---|---| -| `Hash(HashType)` | Init with algorithm, defaults to hex encoding | -| `Hash(HashType, HashEncodingType)` | Init with algorithm and encoding | -| `update(buffer, size)` | Feed data incrementally | -| `digest()` | Finalize and return encoded hash string | - -### Free Functions - -| Function | Description | -|---|---| -| `hashFromFile(hash_type, path)` | Hash a file, returns hex string | -| `hashMultiFromFile(mask, path)` | Hash a file with multiple algorithms simultaneously | -| `hashFromBuffer(hash_type, buffer, size)` | Hash an in-memory buffer, returns hex string | +**Free Functions** +- `hashFromFile()` β€” Computes a single hash digest from a file path +- `hashMultiFromFile()` β€” Computes multiple hashes from a file in one pass using a bitmask +- `hashFromBuffer()` β€” Computes a single hash digest from a raw memory buffer ## Usage Example ```cpp -// Streaming hash (large file chunks) -osquery::Hash hasher(HASH_TYPE_SHA256, HASH_ENCODING_TYPE_HEX); -hasher.update(chunk_ptr, chunk_size); -std::string result = hasher.digest(); +#include "hashing.h" -// One-shot file hash -std::string md5 = osquery::hashFromFile(HASH_TYPE_MD5, "/etc/hosts"); +// Incremental hashing of chunked data +osquery::Hash hasher(osquery::HASH_TYPE_SHA256); +hasher.update(chunk1, chunk1_size); +hasher.update(chunk2, chunk2_size); +std::string result = hasher.digest(); // hex-encoded SHA256 -// Multiple algorithms in a single file pass -osquery::MultiHashes hashes = osquery::hashMultiFromFile( - HASH_TYPE_MD5 | HASH_TYPE_SHA256, "/var/log/syslog"); +// Hash a file directly +std::string md5 = osquery::hashFromFile(osquery::HASH_TYPE_MD5, "/etc/passwd"); -// Buffer hash -std::string sha1 = osquery::hashFromBuffer( - HASH_TYPE_SHA1, data_ptr, data_size); +// Compute MD5 + SHA1 + SHA256 in a single file read +osquery::MultiHashes hashes = osquery::hashMultiFromFile( + osquery::HASH_TYPE_MD5 | osquery::HASH_TYPE_SHA1 | osquery::HASH_TYPE_SHA256, + "/etc/passwd" +); + +// Hash from a buffer +std::string digest = osquery::hashFromBuffer( + osquery::HASH_TYPE_SHA1, buffer_ptr, buffer_len +); ``` \ No newline at end of file diff --git a/osquery/logger/.logger.md b/osquery/logger/.logger.md index 89cb679344b..e090ec7841c 100644 --- a/osquery/logger/.logger.md +++ b/osquery/logger/.logger.md @@ -1,37 +1,22 @@ -Thin wrapper around `glog` that provides osquery-specific logging macros for consistent verbosity and log-line referencing across the codebase. +Provides logging macros for the osquery framework, extending Google's `glog` library with table-specific and reference-annotated log helpers. ## Key Components -| Macro | Description | -|-------|-------------| -| `TLOG` | Alias for `VLOG(1)` β€” verbose logging intended for table-level parsing events and non-critical edge cases | -| `RLOG(n)` | Prepends a formatted reference tag (e.g., `[Ref #42]`) to a log line for external search and triage | +- **`TLOG`** β€” A verbose logging macro (`VLOG(1)`) intended for table plugin developers. Use this for non-critical parsing issues or expected edge cases rather than warnings/errors. Verbosity can be tuned via config or CLI flags. +- **`RLOG(n)`** β€” A string prefix macro that prepends a formatted reference number (e.g., `[Ref #42]`) to a log message, helping users cross-reference confusing log output with external documentation or issue trackers. ## Usage Example -```c +```cpp #include -// Table-level verbose log (only shown when verbosity >= 1) -TLOG << "Parsed unexpected value in users table, skipping row."; +// Table verbose logging β€” only emitted when VLOG(1) is active +TLOG << "Parsed unexpected column value, skipping row"; -// Log with a reference number for documentation/issue tracking -LOG(WARNING) << RLOG(101) << "Unexpected null pointer in process cache."; +// Reference-tagged log line for a known tricky condition +LOG(WARNING) << RLOG(1234) << "Filesystem mount point inaccessible"; +// Output: [Ref #1234] Filesystem mount point inaccessible ``` -**Output examples:** - -```text -// TLOG output (verbosity level 1): -I0101 12:00:00 table.cpp:42] Parsed unexpected value in users table, skipping row. - -// RLOG output: -W0101 12:00:00 cache.cpp:88] [Ref #101] Unexpected null pointer in process cache. -``` - -## Notes - -- `TLOG` is intended for **table developers** β€” use it for routine parse warnings, not critical errors. -- `RLOG(n)` is purely a string prefix; combine it with any standard `glog` severity macro (`LOG(WARNING)`, `LOG(ERROR)`, etc.). -- Verbosity for `TLOG` is controlled at runtime via CLI flags or osquery config, keeping table noise out of production logs by default. \ No newline at end of file +> **Note:** `TLOG` output is suppressed by default. Enable it by setting the `--verbose` flag or adjusting the logging level in osquery config. Use `RLOG` alongside any standard `glog` severity macro (`LOG(INFO)`, `LOG(WARNING)`, etc.) β€” it is a plain string prefix, not a standalone logging call. \ No newline at end of file diff --git a/osquery/main/.main.md b/osquery/main/.main.md index 9da5b3c6709..de3c91005db 100644 --- a/osquery/main/.main.md +++ b/osquery/main/.main.md @@ -1,32 +1,36 @@ -Platform entry point header that declares service installation/uninstallation functions and the primary osquery startup routine. +Entry point header for the osquery daemon and shell initialization, declaring service management and startup functions within the `osquery` namespace. ## Key Components | Symbol | Type | Description | |--------|------|-------------| -| `installService` | Function | Installs osqueryd as a platform service (currently Windows-only) | -| `uninstallService` | Function | Removes the previously installed osqueryd service | -| `startOsquery` | Function | Platform-agnostic entry point that initializes the shell and daemon | +| `installService` | Function | Installs `osqueryd` as a platform service (currently Windows-only) | +| `uninstallService` | Function | Removes the previously installed `osqueryd` service | +| `startOsquery` | Function | Platform-agnostic entry point to initialize the osquery shell and daemon | ## Usage Example ```c #include "main.h" -// Install osqueryd as a Windows service +int main(int argc, char* argv[]) { + // Start the osquery shell/daemon (handles all platform init) + return osquery::startOsquery(argc, argv); +} + +// Windows-specific service installation osquery::Status status = osquery::installService("C:\\osquery\\osqueryd.exe"); if (!status.ok()) { - // handle error + // Handle installation failure } -// Start the osquery shell/daemon (called from main()) -int result = osquery::startOsquery(argc, argv); +// Remove the service when uninstalling +osquery::Status uninstallStatus = osquery::uninstallService(); ``` ## Notes -- `installService` / `uninstallService` are **Windows-only** at runtime. POSIX platforms rely on an external `osqueryctl` companion script to handle daemon registration (launchd, systemd, init). -- Refactoring POSIX install flows into these methods is a known limitation tracked by the osquery authors. -- Both service functions return `osquery::Status` (from `osquery/utils/status/status.h`), enabling structured error handling. -- `startOsquery` accepts standard `argc`/`argv` and serves as the single cross-platform bootstrap call shared by both the shell and daemon binaries. \ No newline at end of file +- `installService` / `uninstallService` are **Windows-only**; POSIX platforms handle service installation via the external `osqueryctl` companion script using launch daemons, init scripts, or systemd units. +- Both service functions return an `osquery::Status` object (from `osquery/utils/status/status.h`) for error handling. +- Refactoring POSIX install flows into these methods is a known limitation documented in the source. \ No newline at end of file diff --git a/osquery/main/harnesses/.fuzz_utils.md b/osquery/main/harnesses/.fuzz_utils.md index 149e34bba61..483b06a82c0 100644 --- a/osquery/main/harnesses/.fuzz_utils.md +++ b/osquery/main/harnesses/.fuzz_utils.md @@ -1,9 +1,9 @@ -Provides a fuzzing utility interface for osquery, exposing an initialization function that disables core stateful features to create a clean environment for LLVM-based fuzz testing. +Provides a fuzzing utility interface for osquery, offering an initialization function to reduce statefulness during fuzz testing by disabling core features. ## Key Components -- **`osqueryFuzzerInitialize(int* argc, char*** argv)`** β€” Disables core osquery features to reduce statefulness. Intended to be called within `LLVMFuzzerInitialize` before fuzz test execution begins. +- **`osqueryFuzzerInitialize(int* argc, char*** argv)`** β€” Disables core osquery features to minimize statefulness during fuzzing sessions. Intended to be called within `LLVMFuzzerInitialize`. ## Usage Example @@ -20,4 +20,4 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { } ``` -> **Note:** Always call `osqueryFuzzerInitialize` before any fuzz input is processed to ensure core subsystems (logging, eventing, etc.) are suppressed and do not introduce side effects during fuzzing runs. \ No newline at end of file +> **Note:** This header is designed exclusively for use with LLVM's libFuzzer. Always call `osqueryFuzzerInitialize` before executing any fuzz target logic to ensure core subsystems are properly suppressed. \ No newline at end of file diff --git a/osquery/numeric_monitoring/.numeric_monitoring.md b/osquery/numeric_monitoring/.numeric_monitoring.md index 04024910d7d..8d77fa0afd6 100644 --- a/osquery/numeric_monitoring/.numeric_monitoring.md +++ b/osquery/numeric_monitoring/.numeric_monitoring.md @@ -1,26 +1,32 @@ -Defines the public interface for osquery's numeric monitoring subsystem, providing types, constants, and functions for recording time-series numeric data points to a monitoring backend. +Defines the public API for osquery's numeric monitoring system, providing types and functions for recording time-series metric data points with optional pre-aggregation support. ## Key Components ### Structs -- **`RecordKeys`** β€” Field name constants for monitoring record attributes (`path`, `value`, `timestamp`, `pre_aggregation`, `sync`) -- **`HostIdentifierKeys`** β€” Field name constants for host identification (`name`, `scheme`) +- **`RecordKeys`** β€” Field name descriptors (`path`, `value`, `timestamp`, `pre_aggregation`, `sync`) used by monitoring plugins +- **`HostIdentifierKeys`** β€” Host identity descriptors (`name`, `scheme`) for metric attribution ### Type Aliases -- **`Clock`** / **`TimePoint`** β€” `std::chrono::system_clock` aliases for consistent time handling -- **`ValueType`** β€” `long long int`, the numeric type for all monitored values +- **`Clock`** / **`TimePoint`** β€” `std::chrono::system_clock` wrappers for consistent time handling +- **`ValueType`** β€” `long long int` for all numeric metric values -### Enum -- **`PreAggregationType`** β€” Aggregation strategy applied before data is sent: `None`, `Sum`, `Min`, `Max`, `Avg`, `Stddev`, percentiles (`P10`, `P50`, `P95`, `P99`) +### Enum: `PreAggregationType` +Controls how values are aggregated before dispatch: + +| Value | Description | +|-------|-------------| +| `None` | No aggregation | +| `Sum` / `Min` / `Max` / `Avg` / `Stddev` | Statistical aggregations | +| `P10` / `P50` / `P95` / `P99` | Percentile estimates | ### Functions -- **`record()`** β€” Submits a new numeric data point with optional pre-aggregation, sync mode, and timestamp -- **`flush()`** β€” Forces immediate flush of the pre-aggregation buffer -- **`hostIdentifierKeys()`** / **`recordKeys()`** β€” Accessors for canonical field name constants +- **`record()`** β€” Submits a new metric data point with optional aggregation and sync control +- **`flush()`** β€” Forces the pre-aggregation buffer to drain immediately +- **`hostIdentifierKeys()`** / **`recordKeys()`** β€” Accessor functions for key name constants - **`registryName()`** β€” Returns the plugin registry name for this monitoring system -### Conversions +### Conversion Templates - **`to()`** β€” Serializes a `PreAggregationType` to string - **`tryTo()`** β€” Parses a `PreAggregationType` from string, returning `Expected` @@ -29,25 +35,26 @@ Defines the public interface for osquery's numeric monitoring subsystem, providi ```cpp #include "numeric_monitoring.h" -// Record a simple counter with sum pre-aggregation +// Record a simple summed metric (fire-and-forget) osquery::monitoring::record( - "osquery.worker.queries.executed", + "workers.query.executions", 42LL, osquery::monitoring::PreAggregationType::Sum ); -// Record a latency value, forcing immediate delivery +// Record with sync=true to block until the plugin confirms delivery osquery::monitoring::record( - "osquery.worker.query.latency_ms", + "workers.query.latency_ms", 150LL, osquery::monitoring::PreAggregationType::P95, - true /* sync */ + true ); -// Convert aggregation type to/from string -auto str = osquery::to(osquery::monitoring::PreAggregationType::Sum); -// str == "Sum" +// Convert enum to string for logging +std::string label = osquery::to( + osquery::monitoring::PreAggregationType::Sum +); // "Sum" -auto type = osquery::tryTo("P99"); -// type.get() == PreAggregationType::P99 +// Force flush before shutdown +osquery::monitoring::flush(); ``` \ No newline at end of file diff --git a/osquery/numeric_monitoring/.plugin_interface.md b/osquery/numeric_monitoring/.plugin_interface.md index 411756ccf7d..1b155ac426f 100644 --- a/osquery/numeric_monitoring/.plugin_interface.md +++ b/osquery/numeric_monitoring/.plugin_interface.md @@ -1,39 +1,33 @@ -Defines the `NumericMonitoringPlugin` interface, an abstract base class for plugins that integrate with osquery's numeric monitoring system. +Defines the `NumericMonitoringPlugin` interface, an abstract base class for numeric monitoring system plugins within osquery's plugin architecture. ## Key Components -- **`NumericMonitoringPlugin`** β€” Inherits from `Plugin`; provides the entry point for numeric monitoring backend implementations (e.g., filesystem-based storage). - - `call(const PluginRequest&, PluginResponse&)` β€” Overrides the base `Plugin::call` to handle incoming monitoring requests and produce responses. +- **`NumericMonitoringPlugin`** β€” Inherits from `Plugin`; serves as the interface contract for numeric monitoring backend implementations (e.g., filesystem-based monitoring via `NumericMonitoringFilesystemPlugin`). +- **`call()`** β€” Overrides the base `Plugin::call()` method to handle dispatched plugin requests and populate responses. ## Usage Example ```c -#include - -namespace osquery { - +// Implementing a custom numeric monitoring plugin class MyMonitoringPlugin : public NumericMonitoringPlugin { public: Status call(const PluginRequest& request, PluginResponse& response) override { - // Extract monitoring path and value from request - auto path = request.at("path"); - auto value = request.at("value"); - - // Persist or forward the numeric metric - storeMetric(path, value); + // Extract monitoring point data from request + // e.g., request["path"], request["value"], request["timestamp"] + // Write/forward the numeric data point + response.push_back({{"status", "OK"}}); return Status::success(); } }; +// Register the plugin REGISTER(MyMonitoringPlugin, "numeric_monitoring", "my_backend"); - -} // namespace osquery ``` ## Notes -- Concrete implementations must override `call()` and register via `REGISTER`. -- The reference implementation is `NumericMonitoringFilesystemPlugin` in `osquery/numeric_monitoring/plugins/filesystem.h`. -- Relies on `osquery::numeric_monitoring` types for point values and pre-aggregation semantics. \ No newline at end of file +- Concrete implementations must override `call()` to handle `PluginRequest` payloads carrying numeric monitoring data points (path, value, timestamp, aggregation type). +- The built-in reference implementation is `NumericMonitoringFilesystemPlugin` (`osquery/numeric_monitoring/plugins/filesystem.h`). +- Plugin dispatch is managed by osquery's core registry β€” use the `REGISTER` macro to expose your implementation. \ No newline at end of file diff --git a/osquery/numeric_monitoring/.pre_aggregation_cache.md b/osquery/numeric_monitoring/.pre_aggregation_cache.md index 4220542ae73..c0bdcbecf0a 100644 --- a/osquery/numeric_monitoring/.pre_aggregation_cache.md +++ b/osquery/numeric_monitoring/.pre_aggregation_cache.md @@ -4,39 +4,36 @@ Pre-aggregation cache for the osquery numeric monitoring system. Provides in-mem ## Key Components ### `Point` -Represents a single monitoring observation unit with: -- `path_` β€” unique identifier/name for the metric sequence -- `value_` β€” the observed numeric value -- `pre_aggregation_type_` β€” aggregation strategy (e.g., sum, min, max) -- `time_point_` β€” timestamp of observation -- `tryToAggregate(const Point&)` β€” merges a new point into itself if `path_` and `pre_aggregation_type_` match; returns `true` on success, `false` otherwise +Represents a single monitoring observation unit with the following public members: +- `path_` β€” Unique name/identifier for the metric sequence +- `value_` β€” The observed numeric value +- `pre_aggregation_type_` β€” Aggregation strategy (e.g., sum, min, max) +- `time_point_` β€” Timestamp of the observation + +**Key method:** +- `tryToAggregate(const Point& new_point)` β€” Attempts to merge `new_point` into this point. Returns `true` and updates `value_` if `path_` and `pre_aggregation_type_` match; otherwise returns `false` unchanged. ### `PreAggregationCache` -Accumulates `Point` objects and handles deduplication via pre-aggregation: -- `addPoint(Point)` β€” inserts or aggregates a point into the internal store -- `takePoints()` β€” drains and returns all buffered points as a `std::vector` -- `size()` β€” returns current number of buffered points +Manages a collection of `Point` objects, deduplicating and aggregating by path using an internal index. -Internally uses `points_index_` (path β†’ index map) and `points_` (ordered vector) to efficiently locate existing points for aggregation. +| Method | Description | +|--------|-------------| +| `addPoint(Point point)` | Inserts a new point or aggregates it into an existing one with the same path | +| `takePoints()` | Drains and returns all buffered points | +| `size()` | Returns the current number of cached points | ## Usage Example ```cpp -#include "pre_aggregation_cache.h" - using namespace osquery::monitoring; PreAggregationCache cache; -// Add monitoring points +// Add monitoring points - same path will be aggregated cache.addPoint(Point("cpu.usage", 42.0, PreAggregationType::Sum, now())); cache.addPoint(Point("cpu.usage", 18.0, PreAggregationType::Sum, now())); -// Same path + type: values are aggregated (e.g., stored as 60.0) - -// Flush buffered points to the monitoring backend -auto points = cache.takePoints(); -for (const auto& p : points) { - sendToBackend(p.path_, p.value_); -} -// Cache is now empty + +// Flush all aggregated points for dispatch +std::vector points = cache.takePoints(); +// points contains one entry: cpu.usage = 60.0 ``` \ No newline at end of file diff --git a/osquery/registry/.registry_factory.md b/osquery/registry/.registry_factory.md index 908d83f9544..dc9ed367fcf 100644 --- a/osquery/registry/.registry_factory.md +++ b/osquery/registry/.registry_factory.md @@ -1,41 +1,52 @@ -Defines the `RegistryFactory` singleton and supporting registration infrastructure for managing osquery plugin registries, including static call dispatch, broadcast serialization, and extension route tracking. +Singleton factory that manages all osquery plugin registries, providing a unified interface for registering, looking up, and invoking plugins across core and extension boundaries. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `RegistryFactory` | Singleton class | Central manager for all plugin registries; provides add, lookup, call, alias, and broadcast operations | -| `RegistryBroadcast` | Type alias | `std::map` representing all routes from all registries | -| `Registry` | Type alias | Convenience alias for `RegistryFactory` | -| `registries::AR` | Template class | Auto-registers a registry type at startup | -| `registries::AP

` | Template class | Auto-registers a plugin instance into an existing registry | -| `registries::RI` | Template struct | RAII trigger for registry auto-registration | -| `registries::PI

` | Template struct | RAII trigger for plugin auto-registration | -| `CREATE_REGISTRY` | Macro | Registers a registry class under a given name at static init time | -| `CREATE_LAZY_REGISTRY` | Macro | Same as `CREATE_REGISTRY` but marks the registry as lazy (skips `setUp`) | -| `REGISTER` | Macro | Registers a plugin class into a named registry | -| `REGISTER_INTERNAL` | Macro | Same as `REGISTER` but marks the plugin as optional/internal | +### `RegistryFactory` (Singleton) +The central registry manager. Accessible via `RegistryFactory::get()`. + +| Method | Description | +|--------|-------------| +| `call(registry, item, request, response)` | Invoke a named plugin within a registry | +| `getBroadcast()` / `addBroadcast()` / `removeBroadcast()` | Serialize and sync extension routes via `RouteUUID` | +| `registry(name)` | Direct access to a `RegistryInterfaceRef` | +| `setActive()` / `getActive()` | Get/set the active plugin for a registry | +| `addAlias()` / `getAlias()` | Map alternate names to registry items | +| `setExternal()` | Mark registry as extension-mode (forwards internal events to core) | +| `setUp()` | Run `setUp` on all non-lazy registries | +| `exists()`, `count()`, `names()` | Introspection utilities | + +### `registries` Namespace Templates +- **`AR` / `RI`** β€” Auto-register a registry type at static init time +- **`AP

` / `PI

`** β€” Auto-register a plugin into an existing registry at static init time + +### Registration Macros + +| Macro | Purpose | +|-------|---------| +| `CREATE_REGISTRY(class, name)` | Register a new registry type (eager) | +| `CREATE_LAZY_REGISTRY(class, name)` | Register a new registry type (lazy `setUp`) | +| `REGISTER(class, registry, plugin)` | Register a plugin into a registry | +| `REGISTER_INTERNAL(class, registry, plugin)` | Register an optional/internal plugin | ## Usage Example ```cpp // Define and register a custom registry -class MyRegistry : public RegistryPlugin { /* ... */ }; -CREATE_REGISTRY(MyRegistry, "my_registry"); +CREATE_REGISTRY(MyTableRegistry, "my_tables"); // Register a plugin into that registry -class MyPlugin : public Plugin { /* ... */ }; -REGISTER(MyPlugin, "my_registry", "my_plugin"); - -// Call a registered plugin at runtime -PluginRequest request = {{"action", "run"}}; -PluginResponse response; -Status s = RegistryFactory::call("my_registry", "my_plugin", request, response); - -// Check existence and use the active plugin -if (RegistryFactory::get().exists("my_registry", "my_plugin")) { - RegistryFactory::get().setActive("my_registry", "my_plugin"); - RegistryFactory::call("my_registry", request, response); -} -``` \ No newline at end of file +REGISTER(MyTablePlugin, "my_tables", "my_plugin"); + +// Call a plugin at runtime +PluginRequest req = {{"action", "query"}}; +PluginResponse res; +Status s = RegistryFactory::call("my_tables", "my_plugin", req, res); + +// Access the factory singleton directly +auto& factory = RegistryFactory::get(); +auto names = factory.names("my_tables"); +``` + +> **Note:** `using Registry = RegistryFactory` provides a shorter alias for the same singleton. \ No newline at end of file diff --git a/osquery/registry/.registry_interface.md b/osquery/registry/.registry_interface.md index 5d52f255158..b7e18155b44 100644 --- a/osquery/registry/.registry_interface.md +++ b/osquery/registry/.registry_interface.md @@ -1,50 +1,54 @@ -Defines the core plugin registry abstraction layer for osquery, providing interfaces for registering, managing, and routing calls to plugins across both local and external (extension) processes. +Defines the core plugin registry interfaces for osquery's extensible plugin system, providing base classes for managing, routing, and dispatching calls to named plugin instances. ## Key Components ### Type Aliases -- **`RegistryRoutes`** β€” Map of plugin name β†’ `PluginResponse` for broadcasting route info -- **`RouteUUID`** β€” `uint64_t` identifier for extension connections -- **`AddExternalCallback`** / **`RemoveExternalCallback`** β€” Function signatures for extension lifecycle hooks -- **`RegistryInterfaceRef`** β€” `shared_ptr` +- `AddExternalCallback` / `RemoveExternalCallback` β€” Function signatures for managing external extension plugins +- `RegistryRoutes` β€” Map of plugin name β†’ `PluginResponse` for extension broadcasting +- `RouteUUID` β€” `uint64_t` identifier for extension connections +- `RegistryInterfaceRef` β€” Shared pointer alias for `RegistryInterface` ### `RegistryInterface` -Abstract base class (non-copyable) managing plugin lifecycle. Key methods: - -| Method | Description | -|---|---| -| `add()` | Register a plugin (pure virtual) | -| `remove()` | Unregister a plugin by name | -| `call()` | Dispatch a request to a named plugin | -| `addExternal()` / `removeExternal()` | Manage plugins from remote extensions by UUID | -| `getRoutes()` | Broadcast route table for extensions | -| `setUp()` | Initialize all registered plugins | -| `setActive()` | Designate a default plugin for nameless calls | +Abstract, non-copyable base for all registries. Manages: +- Plugin lifecycle (`add`, `remove`, `setUp`) +- Active plugin routing (`setActive`, `getActive`) +- Alias resolution (`addAlias`, `getAlias`) +- Extension route management (`addExternal`, `removeExternal`, `getRoutes`) +- Thread-safe access via `Mutex` + +Pure virtual methods: `add()`, `plugin()`, `addExternalPlugin()`, `removeExternalPlugin()` ### `RegistryType` -Templated concrete registry that downcasts `PluginRef` to the specific `PluginType`. Provides type-safe `add()` and `plugin()` accessors, plus trampoline methods into `PluginType::addExternal` / `removeExternal`. +Typed template subclass of `RegistryInterface`. Provides: +- Type-safe `add()` with `dynamic_pointer_cast` validation +- Concrete `plugin()` lookup returning `nullptr` on miss +- Trampoline implementations wiring `addExternalPlugin`/`removeExternalPlugin` to the static `PluginType::addExternal`/`PluginType::removeExternal` callbacks ### `AutoRegisterInterface` -Static registration utility for auto-loading registries and plugins at startup via `registries()` / `plugins()` collections. Consumed by `registryAndPluginInit()`. +Supports static auto-registration of registries and plugins at startup, accessed via `registries()` and `plugins()` static sets. + +### `registryAndPluginInit()` +Triggers all deferred auto-registration entries. ## Usage Example ```cpp -// Define a typed registry for a custom plugin type -class MyRegistry : public RegistryType { +// Define a typed registry for a custom plugin +class MyPluginType : public Plugin { ... }; + +class MyRegistry : public RegistryType { public: - explicit MyRegistry() : RegistryType("my_registry", true) {} + explicit MyRegistry() : RegistryType("my_registry", true) {} }; -// Add and invoke a plugin +// Add a plugin instance auto registry = std::make_shared(); -registry->add("my_plugin", std::make_shared()); - -PluginRequest request = {{"action", "query"}}; -PluginResponse response; -auto status = registry->call("my_plugin", request, response); +auto plugin = std::make_shared(); +registry->add("my_plugin", plugin, /*internal=*/false); -// Retrieve route info for extension broadcasting -RegistryRoutes routes = registry->getRoutes(); +// Dispatch a call +PluginRequest req = {{"action", "query"}}; +PluginResponse resp; +// Called internally via RegistryFactory::call("my_registry", "my_plugin", req, resp) ``` \ No newline at end of file diff --git a/osquery/remote/.http_client.md b/osquery/remote/.http_client.md index 54fba76226a..b45bab7dcda 100644 --- a/osquery/remote/.http_client.md +++ b/osquery/remote/.http_client.md @@ -1,33 +1,33 @@ -HTTP client interface providing synchronous HTTP/HTTPS request capabilities built on Boost.Beast and OpenSSL, used by osquery's remote communication subsystem. +HTTP client interface for osquery's remote communication layer, providing HTTP/HTTPS request functionality built on Boost.Beast and OpenSSL. ## Key Components ### `Client` -The primary HTTP client class supporting `GET`, `POST`, `PUT`, `HEAD`, and `DELETE` operations over plain or TLS-encrypted connections. +Main HTTP client class supporting GET, POST, PUT, HEAD, and DELETE methods over plain or TLS-encrypted connections. ### `Client::Options` -A fluent builder for configuring client behavior. Key settings include: - -| Option | Description | -|---|---| -| `ssl_connection(bool)` | Enable TLS wrapping | -| `timeout(int)` | Request timeout in seconds | -| `always_verify_peer(bool)` | Enforce peer certificate validation | -| `follow_redirects(bool)` | Auto-follow HTTP redirects | -| `proxy_hostname(string)` | Route through a proxy | -| `openssl_certificate(string)` | Server certificate for pinning | -| `openssl_ciphers(string)` | Allowed TLS cipher suites | +Fluent builder for configuring client behavior. Supports chainable setters for: +- **TLS/SSL** β€” `ssl_connection()`, `openssl_ciphers()`, `openssl_options()`, `openssl_certificate()`, `openssl_private_key_file()`, `always_verify_peer()` +- **Connection** β€” `timeout()`, `keep_alive()`, `follow_redirects()`, `proxy_hostname()`, `remote_hostname()`, `remote_port()` ### `HTTP_Request` / `HTTP_Response` -Template wrappers extending Boost.Beast request/response types with URI parsing. `HTTP_Request` exposes `remoteHost()`, `remotePort()`, `remotePath()`, and `protocol()` helpers. +Template wrappers around Boost.Beast request/response types, extending them with URI parsing helpers: +- `remoteHost()` β€” extracts hostname +- `remotePort()` β€” extracts port +- `remotePath()` β€” builds path + query + fragment +- `protocol()` β€” returns scheme (`http` / `https`) ### Type Aliases -- `Request` β†’ `HTTP_Request` -- `Response` β†’ `HTTP_Response` +```c +typedef HTTP_Request Request; +typedef HTTP_Response Response; +``` ### Constants -- `kInstanceMetadataAuthority` β€” IP `169.254.169.254` for cloud metadata services (EC2, Azure, etc.) +```c +const std::string kInstanceMetadataAuthority = "169.254.169.254"; // EC2/Azure metadata +``` ## Usage Example @@ -36,17 +36,18 @@ Template wrappers extending Boost.Beast request/response types with URI parsing. osquery::http::Client::Options opts; opts.ssl_connection(true) - .always_verify_peer(true) .timeout(10) - .openssl_certificate("/etc/ssl/certs/ca-certificates.crt"); + .always_verify_peer(true) + .openssl_certificate("/etc/ssl/ca-bundle.crt"); osquery::http::Client client(opts); osquery::http::Request req("https://example.com/api/data"); auto response = client.get(req); +// POST with body osquery::http::Request post_req("https://example.com/api/submit"); -auto result = client.post(post_req, R"({"key":"value"})", "application/json"); +auto response = client.post(post_req, "{\"key\":\"value\"}", "application/json"); ``` -> **Note:** SSL2, SSL3, and MD5 are explicitly disabled at compile time. On Windows, Boost.ASIO thread cleanup (`set_terminate_threads`) is initialized once via `std::call_once` to prevent resource leaks. \ No newline at end of file +> **Note:** SSL2, SSL3, MD5, and deprecated OpenSSL APIs are explicitly disabled via preprocessor macros. On Windows, Boost ASIO thread cleanup is handled via `std::call_once` to prevent socket lifecycle issues. \ No newline at end of file diff --git a/osquery/remote/.requests.md b/osquery/remote/.requests.md index 9b517e57595..6865612362b 100644 --- a/osquery/remote/.requests.md +++ b/osquery/remote/.requests.md @@ -1,54 +1,42 @@ -Defines the core abstractions for osquery's remote communication layer, providing abstract base classes for transport and serialization mechanisms along with a templated `Request` class that composes them. +Defines the core abstractions for osquery's remote communication layer, providing base classes for transport and serialization mechanisms along with a templated `Request` class that composes them. ## Key Components -| Component | Type | Description | -|---|---|---| -| `compressString` | Function | GZip-compresses a string; applied post-serialization before transport | -| `Transport` | Abstract class | Base for transport implementations (HTTP, WebSockets, etc.) | -| `Serializer` | Abstract class | Base for serialization formats (JSON, XML, etc.) | -| `Request` | Template class | Composes a transport and serializer to execute remote calls | - -### `Transport` Interface - -| Method | Description | -|---|---| -| `setDestination()` | Sets the remote URI | -| `setSerializer()` | Binds a serializer instance | -| `sendRequest()` | Pure virtual β€” no-param request | -| `sendRequest(params, compress)` | Pure virtual β€” parameterized request with optional GZip | -| `getResponseStatus()` | Returns the response `Status` | -| `getResponseParams()` | Returns response body as `JSON` | -| `setOption(name, value)` | Sets transport-specific options | - -### `Serializer` Interface - -| Method | Description | -|---|---| -| `getContentType()` | Returns HTTP content-type string | -| `serialize(json, out)` | Encodes a `JSON` object to string | -| `deserialize(string, out)` | Decodes a string into a `JSON` object | +- **`compressString()`** β€” GZip-compresses a string; applied post-serialization before transport dispatch +- **`Transport`** (abstract) β€” Base class for network transport implementations (e.g., HTTP/TLS). Subclasses must implement `sendRequest()` (with and without params). Holds destination, serializer reference, response status, response params, and options +- **`Serializer`** (abstract) β€” Base class for data serialization formats (e.g., JSON, XML). Subclasses must implement `getContentType()`, `serialize()`, and `deserialize()` +- **`Request`** β€” Templated class that wires a transport and serializer together for a given destination URI. Exposes `call()`, `call(params)`, `getResponse()`, and `setOption()` ## Usage Example ```cpp -// Instantiate a request with concrete transport and serializer types -osquery::Request req("https://example.com/api"); +#include +#include +#include -// Set options (e.g., enable compression) +using namespace osquery; + +// Instantiate a request using TLS transport and JSON serialization +Request req("https://enrollment.example.com/enroll"); + +// Optional: enable compression req.setOption("compress", true); -// Send a parameterized request -osquery::JSON params; -params.add("key", "value"); -auto status = req.call(params); +// Send request with JSON parameters +JSON params; +params.add("host_identifier", "my-host-001"); -// Retrieve the response -osquery::JSON response; -if (req.getResponse(response).ok()) { - // process response.doc() +Status s = req.call(params); +if (s.ok()) { + JSON response; + s = req.getResponse(response); } -``` -> **Note:** `Request` has a private constructor accepting a custom `TTransport` shared pointer, reserved for unit testing via `FRIEND_TEST` grants to `TLSTransportsTests`. \ No newline at end of file +// Implement a custom transport +class MyTransport : public Transport { + public: + Status sendRequest() override { /* ... */ } + Status sendRequest(const std::string& params, bool compress) override { /* ... */ } +}; +``` \ No newline at end of file diff --git a/osquery/remote/.uri.md b/osquery/remote/.uri.md index 176886f982c..8d1b719b733 100644 --- a/osquery/remote/.uri.md +++ b/osquery/remote/.uri.md @@ -1,25 +1,25 @@ -A URI parsing utility class (derived from folly/uri) that breaks a URI string into its component parts: scheme, authority, host, port, path, query, and fragment. +A URI parsing utility class (adapted from folly/uri) that breaks a URI string into its component parts: scheme, authority, path, query, and fragment. ## Key Components -### `Uri` Class (`osquery` namespace) +**`Uri` class** β€” Core URI representation with the following accessors and methods: | Member | Description | -|--------|-------------| -| `Uri(const std::string& str)` | Constructor β€” parses a URI string; throws `std::invalid_argument` on failure | -| `scheme()` | Returns the lowercased scheme (e.g. `"http"`) | -| `host()` | Returns host, with square brackets for IPv6 (e.g. `"[::1]"`) | -| `hostname()` | Returns host without brackets β€” safe for `getaddrinfo()` and similar APIs | -| `port()` | Returns the port as `uint16_t` | -| `path()` | Returns the URI path component | -| `query()` | Returns the raw query string | -| `fragment()` | Returns the fragment (anchor) | -| `authority()` | Returns the combined authority string (host + port) | -| `setPort(uint16_t)` | Overrides the port and marks authority as present | -| `getQueryParams()` | Parses and returns query string as `vector>` β€” **not thread-safe on first call** | - -> **Note:** Component parts are **not** percent-decoded. Use `uriUnescape()` manually on authority/path, and `uriUnescape(..., UriEscapeMode::QUERY)` on query parameters. +|---|---| +| `Uri(const std::string&)` | Constructor; parses URI string, throws `std::invalid_argument` on failure | +| `scheme()` | Returns lowercased scheme (e.g. `"http"`) | +| `host()` | Returns host, including brackets for IPv6 (e.g. `"[::1]"`) | +| `hostname()` | Returns host without brackets β€” suitable for `getaddrinfo()` and similar APIs | +| `port()` | Returns port as `uint16_t` | +| `path()` | Returns path component (e.g. `"/foo/bar"`) | +| `query()` | Returns raw query string (e.g. `"key=foo"`) | +| `fragment()` | Returns fragment anchor (e.g. `"anchor"`) | +| `authority()` | Returns combined host+port authority string | +| `getQueryParams()` | Returns parsed query parameters as `vector>` | +| `setPort(uint16_t)` | Overrides the port value | + +> **Note:** Component parts are **not** percent-decoded. Use `uriUnescape()` manually when needed. `getQueryParams()` is not thread-safe on first call. ## Usage Example @@ -27,20 +27,18 @@ A URI parsing utility class (derived from folly/uri) that breaks a URI string in #include "uri.h" try { - osquery::Uri uri("http://user:pass@www.example.com:8080/foo/bar?key=val#anchor"); + osquery::Uri uri("http://www.example.com:8080/foo/bar?key=foo&val=bar#anchor"); uri.scheme(); // "http" - uri.hostname(); // "www.example.com" (safe for getaddrinfo) - uri.host(); // "www.example.com" + uri.hostname(); // "www.example.com" uri.port(); // 8080 uri.path(); // "/foo/bar" - uri.query(); // "key=val" uri.fragment(); // "anchor" - // Parse query parameters - for (const auto& [key, value] : uri.getQueryParams()) { - // key="key", value="val" - } + auto params = uri.getQueryParams(); + // params[0] => {"key", "foo"} + // params[1] => {"val", "bar"} + } catch (const std::invalid_argument& e) { // Handle malformed URI } diff --git a/osquery/remote/enroll/.enroll.md b/osquery/remote/enroll/.enroll.md index 556a54703da..d4d3ca6a8cc 100644 --- a/osquery/remote/enroll/.enroll.md +++ b/osquery/remote/enroll/.enroll.md @@ -1,42 +1,43 @@ -Header defining osquery's enrollment plugin interface and utilities for node authentication and secret management in distributed osquery deployments. +Header defining the osquery enrollment plugin interface, providing node authentication and secret management primitives for remote config/logger backends. ## Key Components | Symbol | Type | Description | -|---|---|---| -| `FLAGS_disable_enrollment` | Flag | Boolean flag to disable all enrollment features | -| `kEnrollHostDetails` | `const std::set` | Table names used to populate host identification details sent during enrollment | -| `EnrollPlugin` | Abstract class | Base class for all enrollment plugins; extends `Plugin` | -| `EnrollPlugin::call()` | Method | Routes incoming `PluginRequest` actions to the appropriate handler | -| `EnrollPlugin::enroll()` | Pure virtual | Implemented by subclasses to return a node secret/key | -| `EnrollPlugin::genHostDetails()` | Protected method | Builds a JSON object from `kEnrollHostDetails` table data | -| `getNodeKey()` | Free function | Retrieves a cached node key from RocksDB or triggers enrollment | -| `clearNodeKey()` | Free function | Removes the existing node key from persistent storage | -| `getEnrollSecret()` | Free function | Reads and returns the deployment enrollment secret from disk | +|--------|------|-------------| +| `FLAGS_disable_enrollment` | `bool` flag | Runtime flag to disable all enrollment features | +| `kEnrollHostDetails` | `const std::set` | Table names used to populate host identity data sent during enrollment | +| `EnrollPlugin` | Abstract class | Base class for enrollment plugins; routes `call()` requests and generates host detail payloads | +| `EnrollPlugin::enroll()` | Pure virtual method | Implemented by subclasses to return a node secret/key/identifier | +| `EnrollPlugin::genHostDetails()` | Protected method | Queries `kEnrollHostDetails` tables and serializes first-row results into a JSON object | +| `getNodeKey()` | Free function | Returns a cached node key from RocksDB or triggers enrollment via the named plugin | +| `clearNodeKey()` | Free function | Removes the stored node key from persistent storage | +| `getEnrollSecret()` | Free function | Reads and trims the deployment-wide enrollment secret from disk | ## Usage Example ```cpp -// Implement a custom enroll plugin -class MyEnrollPlugin : public EnrollPlugin { +// Implement a custom enrollment plugin +class MyEnrollPlugin : public osquery::EnrollPlugin { protected: std::string enroll() override { - JSON host_details; - genHostDetails(host_details); // populate host identity info - // call remote endpoint with host_details... - return "my-node-secret-key"; + osquery::JSON host_details; + genHostDetails(host_details); // Populate host identity fields + + // Exchange host_details with your enrollment endpoint + // and return the received node secret + return "my-node-secret-from-server"; } }; -// Retrieve (or request) a node key using a named plugin -std::string node_key = getNodeKey("my_enroll_plugin"); +// Elsewhere: retrieve or enroll to get the node key +std::string key = osquery::getNodeKey("my_enroll_plugin"); -// Clear stored key to force re-enrollment on next run -Status s = clearNodeKey(); +// Reset enrollment (force re-enrollment on next request) +osquery::Status s = osquery::clearNodeKey(); -// Read the shared enterprise enrollment secret -const std::string secret = getEnrollSecret(); +// Read the shared deployment secret from disk +std::string secret = osquery::getEnrollSecret(); ``` -> Enrollment plugins are best used as part of a coordinated **enroll β†’ config β†’ logger** plugin suite. See the [osquery enrollment docs](https://osquery.readthedocs.io/en/stable/deployment/remote/) for full deployment guidance. \ No newline at end of file +> Enrollment plugins are best deployed as part of a coordinated **enroll + config + logger** plugin suite. See the osquery wiki for full authentication flow details. \ No newline at end of file diff --git a/osquery/remote/serializers/.json.md b/osquery/remote/serializers/.json.md index fe919882cd5..c0bb24492d4 100644 --- a/osquery/remote/serializers/.json.md +++ b/osquery/remote/serializers/.json.md @@ -1,39 +1,33 @@ -Defines `JSONSerializer`, a concrete implementation of the `Serializer` interface for serializing and deserializing JSON payloads in osquery remote requests. +Defines the `JSONSerializer` class for serializing and deserializing JSON data in osquery's remote request pipeline. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `JSONSerializer` | Class | Serializes/deserializes `JSON` objects to/from `std::string` | -| `serialize()` | Method | Converts a `JSON` object into a serialized string | -| `deserialize()` | Method | Parses a serialized string back into a `JSON` object | -| `getContentType()` | Method | Returns `"application/json"` as the MIME content type | +- **`JSONSerializer`** β€” Concrete implementation of the `Serializer` interface that handles JSON encoding/decoding for remote communications. + - `serialize(const JSON& json, std::string& serialized)` β€” Converts a `JSON` object into a string representation. + - `deserialize(const std::string& serialized, JSON& json)` β€” Parses a JSON string back into a `JSON` object. + - `getContentType()` β€” Returns `"application/json"` as the MIME content type. ## Usage Example -```c +```cpp #include osquery::JSONSerializer serializer; // Serialize -osquery::JSON json; +osquery::JSON jsonObj; std::string output; -auto status = serializer.serialize(json, output); +auto status = serializer.serialize(jsonObj, output); if (!status.ok()) { - // handle error + LOG(ERROR) << "Serialization failed: " << status.getMessage(); } // Deserialize osquery::JSON parsed; auto status2 = serializer.deserialize(output, parsed); -if (!status2.ok()) { - // handle error -} // Content type for HTTP headers -std::string ct = serializer.getContentType(); // "application/json" -``` - -> `JSONSerializer` is a lightweight adapter β€” all methods delegate to the base `Serializer` contract defined in `osquery/remote/requests.h`. Wire up this class wherever a remote transport requires a `Serializer` instance with JSON encoding. \ No newline at end of file +std::string contentType = serializer.getContentType(); +// => "application/json" +``` \ No newline at end of file diff --git a/osquery/remote/tests/.test_utils.md b/osquery/remote/tests/.test_utils.md index 92a3174fa03..4cc4b5d950b 100644 --- a/osquery/remote/tests/.test_utils.md +++ b/osquery/remote/tests/.test_utils.md @@ -1,48 +1,41 @@ -Provides a singleton test utility class for managing an embedded TLS server process during osquery integration tests, enabling client TLS configuration and lifecycle control. +Provides a singleton test utility class for managing a local TLS server process during osquery integration tests, along with helpers for configuring and restoring TLS-related client flags. ## Key Components ### `TLSServerRunner` (class) -A non-copyable singleton that spawns and manages a local TLS server subprocess for testing purposes. +A non-copyable singleton that spawns and manages a Python/script-based TLS server for testing osquery's TLS client functionality. | Member | Description | |--------|-------------| -| `instance()` | Returns the singleton `TLSServerRunner` instance | -| `start(server_cert, verify_client_cert)` | Launches the TLS server process; returns `false` on failure | +| `instance()` | Returns the singleton instance | +| `start(server_cert, verify_client_cert)` | Starts the TLS server process; returns `false` on failure | | `stop()` | Terminates the server process on exit | -| `port()` | Returns the TCP port the server is bound to | +| `port()` | Returns the TCP port string the server is bound to | | `setClientConfig()` | Applies osquery flags needed for TLS client tests | -| `unsetClientConfig()` | Restores osquery flags after TLS client tests | -| `getListeningPortPid()` | Resolves the PID listening on a given port (empty if not found) | -| `startAndSetScript()` | Internal helper to spawn the server subprocess | +| `unsetClientConfig()` | Restores original flag values after tests | +| `getListeningPortPid(port, pid)` | Resolves the PID bound to a given port; empty on failure | +| `startAndSetScript(port, cert, verify)` | Internal helper to launch the server script and set `server_` handle | -**Private state:** - -- `server_` β€” shared handle to the spawned `PlatformProcess` -- `port_` β€” bound TCP port string -- Saved flag values: `tls_hostname_`, `enroll_tls_endpoint_`, `tls_server_certs_`, `enroll_secret_path_` +**Private state:** `server_` (`PlatformProcess`), `port_`, `tls_hostname_`, `enroll_tls_endpoint_`, `tls_server_certs_`, `enroll_secret_path_` ## Usage Example ```cpp #include "test_utils.h" -// Start TLS server before tests -bool ok = osquery::TLSServerRunner::start("/path/to/server.crt", false); -ASSERT_TRUE(ok); +// In a test fixture SetUp: +bool started = osquery::TLSServerRunner::start("/path/to/server.crt", false); +ASSERT_TRUE(started); -// Configure the osquery client flags to point at the test server osquery::TLSServerRunner::setClientConfig(); -// Access the bound port -std::string port = osquery::TLSServerRunner::port(); - -// Run TLS-dependent tests... +// Run TLS-dependent test logic... +std::string port = osquery::TLSServerRunner::port(); // e.g., "8080" -// Restore flags and shut down +// Teardown: osquery::TLSServerRunner::unsetClientConfig(); osquery::TLSServerRunner::stop(); ``` -> **Note:** `TLSServerRunner` is non-copyable (inherits `boost::noncopyable`). Always access it through the `instance()` singleton or the provided static methods. \ No newline at end of file +> **Note:** `start()` is idempotent β€” calling it multiple times will not spawn duplicate server processes, making it safe to invoke from multiple test cases within the same process. \ No newline at end of file diff --git a/osquery/remote/transports/.tls.md b/osquery/remote/transports/.tls.md index cc1cee8c5ac..04526ee6edf 100644 --- a/osquery/remote/transports/.tls.md +++ b/osquery/remote/transports/.tls.md @@ -1,50 +1,39 @@ -HTTPS/TLS transport layer for osquery remote communications, providing secure HTTP client functionality with configurable certificate handling, cipher suite enforcement, and peer verification. +Defines the `TLSTransport` class and supporting declarations for HTTPS communication within the osquery remote transport layer, including cipher suite configuration, client/server certificate handling, and OpenFrame authorization integration. ## Key Components -### Constants & Flags -- `kTLSCiphers` β€” Hardcoded restrictive cipher suite string (ECDH/DH/RSA with AES/3DES, excludes weak algorithms) -- `tls_client_key` β€” CLI flag: path to TLS client private key -- `tls_client_cert` β€” CLI flag: path to TLS client certificate (PEM) -- `tls_hostname` β€” CLI flag: TLS server hostname - -### Enums -- `HTTPVerb` β€” Defines `HTTP_POST` and `HTTP_PUT` verb selectors - -### Class: `TLSTransport` (extends `Transport`) - -| Method | Description | -|---|---| -| `sendRequest()` | Sends a parameterless HTTPS request | -| `sendRequest(params, compress)` | Sends a request with serialized parameters, optional gzip compression | -| `getInternalOptions()` | Returns strict options (limited ciphers + client certs) for osquery infrastructure | -| `getOptions()` | Returns general options for AWS or public internet endpoints | -| `decorateRequest(r)` | Applies base modifications to an HTTP request object | -| `disableVerifyPeer()` | Test-only: disables peer certificate verification | -| `setClientCertificate(cert, key)` | Configures mutual TLS client authentication | -| `setPeerCertificate(cert)` | Pins a specific server CA/certificate bundle | +- **`kTLSCiphers`** β€” Hardcoded restrictive cipher suite string enforcing best-practice TLS encryption (excludes `aNULL`, `MD5`, `CBC`, `SHA`) +- **`HTTPVerb`** β€” Enum defining supported HTTP methods (`HTTP_POST`, `HTTP_PUT`) +- **`TLSTransport`** β€” Primary class extending `Transport`; handles HTTPS request dispatch with optional compression, client-auth certificates, server pinning, and peer verification + - `sendRequest()` β€” Sends a parameterless HTTPS request + - `sendRequest(params, compress)` β€” Sends a serialized parameter payload, optionally gzip-compressed + - `getInternalOptions()` β€” Returns hardened TLS options (restricted ciphers + client certs) for osquery infrastructure + - `getOptions()` β€” Returns general-purpose TLS options for AWS or public endpoints + - `decorateRequest(r)` β€” Applies base headers/modifications to an outgoing request + - `disableVerifyPeer()` β€” Testing-only method to skip peer certificate validation + +**Flags declared:** +- `tls_client_key` β€” Path to PEM private key for client auth +- `tls_client_cert` β€” Path to PEM certificate for client auth +- `tls_hostname` β€” Target TLS server hostname ## Usage Example ```cpp -#include +#include "tls.h" -// Basic TLS request with internal (strict) options -auto transport = std::make_shared(); +osquery::TLSTransport transport; -// For osquery infrastructure β€” enforces restrictive cipher suite -auto options = transport->getInternalOptions(); +// For osquery infrastructure (strict ciphers, client cert) +auto opts = transport.getInternalOptions(); -// For generic/AWS endpoints β€” uses permissive options -auto generalOptions = transport->getOptions(); +// For general HTTPS endpoints (e.g., AWS) +auto genericOpts = transport.getOptions(); -// Send a POST request with JSON parameters -Status status = transport->sendRequest("{\"key\":\"value\"}", /* compress */ false); +// Send a request with JSON params +osquery::Status status = transport.sendRequest("{\"key\":\"value\"}", true); if (!status.ok()) { - // status.getCode() == 1: connectivity error - // status.getCode() == 2: TLS-specific error + // status.getCode(): 1 = connectivity error, 2 = TLS-specific error } -``` - -> **Return codes:** `sendRequest` returns code `1` for general connectivity failures and code `2` for TLS-specific errors (e.g., handshake failure, certificate mismatch). \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/sql/.dynamic_table_row.md b/osquery/sql/.dynamic_table_row.md index cce2fa01dbc..0b7276eb8fe 100644 --- a/osquery/sql/.dynamic_table_row.md +++ b/osquery/sql/.dynamic_table_row.md @@ -3,44 +3,39 @@ Defines the `DynamicTableRow` and `DynamicTableRowHolder` classes, providing a s ## Key Components -### `DynamicTableRow` -Concrete `TableRow` subclass backed by a `Row` (string map). Supports: -- Construction from `Row&&`, initializer lists, or default empty state -- `get_rowid()` β€” retrieves the SQLite row ID -- `get_column()` β€” populates a SQLite column value from the internal map -- `serialize()` β€” serializes the row to a JSON object -- `clone()` β€” returns a deep copy wrapped in a `TableRowHolder` -- `operator[]` / `count()` β€” map-style key access - -### `DynamicTableRowHolder` -Convenience wrapper that owns a `DynamicTableRow*` alongside its `TableRowHolder`. Simplifies row construction and assignment before strong typing is fully adopted. Implicitly converts to `TableRowHolder&&`. - -### Factory Functions -- `make_table_row()` β€” creates an empty `DynamicTableRowHolder` -- `make_table_row({...})` β€” creates a pre-populated row from an initializer list - -### Free Functions -- `tableRowsFromQueryData(QueryData&&)` β€” converts legacy `QueryData` to `TableRows` -- `deserializeRow(const rapidjson::Value&, DynamicTableRowHolder&)` β€” populates a row from a JSON object +**`DynamicTableRow`** β€” Concrete `TableRow` backed by a `Row` (string map): +- `get_rowid()` β€” returns the row's SQLite row ID +- `get_column()` β€” populates a `sqlite3_context` with column data +- `serialize()` β€” writes the row into a RapidJSON object +- `clone()` β€” returns a heap-allocated copy wrapped in `TableRowHolder` +- `operator[]` / `count()` β€” direct map-style access to row fields + +**`DynamicTableRowHolder`** β€” RAII wrapper pairing a raw `DynamicTableRow*` with its owning `TableRowHolder`, simplifying usage before strongly-typed rows are universally adopted. Implicitly converts to `TableRowHolder&&`. + +**Free functions:** +- `make_table_row()` β€” factory returning an empty `DynamicTableRowHolder` +- `make_table_row(init)` β€” factory from an initializer list of key-value pairs +- `tableRowsFromQueryData()` β€” converts a `QueryData` struct to `TableRows` (intended for generated code) +- `deserializeRow()` β€” populates a `DynamicTableRowHolder` from a RapidJSON value ## Usage Example ```c -// Construct a row with known columns +// Create a row with initial values auto row = make_table_row({ {"pid", "1234"}, - {"name", "launchd"}, - {"path", "/sbin/launchd"} + {"name", "osqueryd"}, + {"path", "/usr/bin/osqueryd"} }); -// Access or modify a field -row["state"] = "S"; +// Access or mutate fields +row["status"] = "running"; -// Convert to TableRowHolder for use in query results +// Convert to TableRowHolder for use in a TableRows result TableRows results; results.push_back(std::move(row)); // Deserialize from JSON -DynamicTableRowHolder deserialized; -Status s = deserializeRow(jsonValue, deserialized); +DynamicTableRowHolder out; +Status s = deserializeRow(jsonValue, out); ``` \ No newline at end of file diff --git a/osquery/sql/.sql.md b/osquery/sql/.sql.md index 101d5f03db5..460f21b866a 100644 --- a/osquery/sql/.sql.md +++ b/osquery/sql/.sql.md @@ -1,56 +1,55 @@ -Defines the core SQL query interface for osquery, providing both a high-level `SQL` class and lower-level utility functions for executing, analyzing, and inspecting SQLite-backed virtual table queries. +Defines the core SQL query execution interface for osquery, providing both a high-level `SQL` class and lower-level utility functions for running queries against osquery's virtual table engine. ## Key Components ### `SQL` Class -The primary interface for executing osquery SQL queries. Inherits `only_movable` (no copy semantics). +The primary interface for executing osquery SQL queries. Inherits `only_movable` (move-only semantics). | Member | Description | -|---|---| -| `SQL(query, use_cache)` | Constructor β€” executes query on instantiation | -| `rows()` | Returns `QueryData` results (const and mutable overloads) | -| `columns()` | Returns `ColumnNames` for result columns | -| `ok()` | Returns `true` if query succeeded | +|--------|-------------| +| `SQL(query, use_cache)` | Constructor β€” executes query immediately on instantiation | +| `rows()` | Returns `QueryData` (const and mutable overloads) | +| `columns()` | Returns `ColumnNames` with column order information | +| `ok()` | Returns `bool` indicating query success | | `getStatus()` | Returns the underlying `Status` object | | `getMessageString()` | Returns human-readable status message | -| `selectAllFrom(table)` | Static: executes `SELECT * FROM ` | -| `selectAllFrom(table, col, op, expr)` | Static: `SELECT *` with a single WHERE constraint | -| `selectFrom(cols, table, col, op, expr)` | Static: `SELECT [cols]` with a single WHERE constraint | +| `selectAllFrom(table)` | Static helper β€” runs `SELECT * FROM
` | +| `selectAllFrom(table, column, op, expr)` | Static helper β€” runs `SELECT *` with a single `WHERE` constraint | +| `selectFrom(columns, table, column, op, expr)` | Static helper β€” runs `SELECT [columns]` with a `WHERE` constraint | ### Free Functions | Function | Description | -|---|---| -| `query(q, results, use_cache)` | Lower-level query execution; prefer `SQL` class | -| `getQueryColumns(q, columns)` | Introspects result column names and types via SQLite | +|----------|-------------| +| `query(q, results, use_cache)` | Lower-level query execution, populates a `QueryData` output parameter | +| `getQueryColumns(q, columns)` | Analyzes result column names and types via SQLite introspection | | `getQueryTables(q, tables)` | Extracts virtual table names referenced in a query | ## Usage Example ```cpp -// High-level usage via SQL class -osquery::SQL sql("SELECT * FROM time"); +// High-level SQL class usage +SQL sql("SELECT * FROM processes WHERE pid = 1"); if (sql.ok()) { - for (const auto& row : sql.rows()) { - for (const auto& col : row) { - LOG(INFO) << col.first << " => " << col.second; + for (const auto& row : sql.rows()) { + for (const auto& col : row) { + LOG(INFO) << col.first << " => " << col.second; + } } - } } else { - LOG(ERROR) << sql.getMessageString(); + LOG(ERROR) << sql.getMessageString(); } // Static helper with constraint -auto results = osquery::SQL::selectAllFrom( - "processes", "pid", osquery::EQUALS, "1234" +QueryData results = SQL::selectAllFrom( + "users", "username", EQUALS, "root" ); // Lower-level query function -osquery::QueryData results; -auto status = osquery::query("SELECT * FROM users;", results); - -// Inspect columns before executing -osquery::TableColumns cols; -osquery::getQueryColumns("SELECT pid, name FROM processes", cols); +QueryData output; +Status s = query("SELECT * FROM time;", output); +if (!s.ok()) { + LOG(ERROR) << s.what(); +} ``` \ No newline at end of file diff --git a/osquery/sql/.sqlite_util.md b/osquery/sql/.sqlite_util.md index 2d7de88818b..588248cbbdd 100644 --- a/osquery/sql/.sqlite_util.md +++ b/osquery/sql/.sqlite_util.md @@ -1,62 +1,48 @@ -Provides SQLite database management utilities for osquery, including RAII database instance wrappers, a singleton manager, query planning, and security authorization controls for safe SQL execution. +Utility header for osquery's internal SQLite database management, providing RAII wrappers, connection management, security authorization, and query planning infrastructure. ## Key Components -### `sqliteAuthorizer` -Callback function enforcing SQLite action allowlists (`kAllowedSQLiteActionCodes`, `kAllowedSQLitePragmas`). Blocks dangerous operations like `SQLITE_ATTACH`. +### Constants +- **`SQLITE_SOFT_HEAP_LIMIT`** β€” 5MB soft heap cap for SQLite memory usage +- **`kAllowedSQLiteActionCodes`** β€” Allowlist of permitted SQLite action codes (blocks `SQLITE_ATTACH` to prevent arbitrary file writes) +- **`kAllowedSQLitePragmas`** β€” Allowlist of permitted SQLite pragma statements +- **`kSQLOpcodes`** β€” Map of SQLite opcodes used for column type inference -### `SQLiteDBInstance` -RAII wrapper around a `sqlite3*` connection. Supports both managed (primary) and transient instances, virtual table cache control, and thread-safe attach locking. +### Classes -| Method | Description | -|---|---| -| `db()` | Access the raw `sqlite3*` pointer | -| `isPrimary()` | Check if using the shared primary DB | -| `useCache(bool)` | Toggle warm query cache | -| `addAffectedTable()` | Track virtual tables used in a query | -| `attachLock()` | Acquire recursive lock for table attachment | +| Class | Purpose | +|-------|---------| +| `SQLiteDBInstance` | RAII wrapper around a `sqlite3*`; manages primary vs. transient connections, virtual table state, and attach locking | +| `SQLiteDBManager` | Singleton resource manager; sole access point for SQLite connections with contention-aware pooling | +| `QueryPlanner` | Lightweight query planner using SQLite `EXPLAIN` to infer column types and scan order | -### `SQLiteDBManager` -Singleton managing SQLite resource lifecycle. Provides connection pooling β€” returns the primary DB when available, otherwise a transient instance. - -| Method | Description | -|---|---| -| `get()` | Return a connection (primary or transient) | -| `getUnique()` | Always return a fresh transient connection | -| `resetPrimary()` | Close and reinitialize the primary DB | -| `isDisabled(name)` | Check if a table is disabled via flags | - -### `QueryPlanner` -Lightweight planner using SQLite `EXPLAIN` output to infer column types and identify scanned tables. - -### `queryInternal` -Execute a raw SQL query against a given `SQLiteDBInstanceRef`. +### Free Functions +- **`sqliteAuthorizer()`** β€” SQLite authorizer callback enforcing action and pragma allowlists +- **`queryInternal()`** β€” Execute a query against a specific `SQLiteDBInstanceRef`, returning typed or untyped results ## Usage Example ```cpp -// Get a DB connection and run a query +// Acquire a managed SQLite connection SQLiteDBInstanceRef db = SQLiteDBManager::get(); -QueryDataTyped results; - -Status status = queryInternal( - "SELECT pid, name FROM processes WHERE uid = 0", - results, - db -); -if (status.ok()) { +// Execute an internal query +QueryDataTyped results; +Status s = queryInternal("SELECT * FROM processes", results, db); +if (s.ok()) { for (const auto& row : results) { - // process each row + // process row... } } -// Use a unique transient connection (e.g., in tests) -SQLiteDBInstanceRef testDb = SQLiteDBManager::getUnique(); - -// Infer column types via query planner -QueryPlanner planner("SELECT * FROM users", db); -TableColumns columns; +// Use QueryPlanner to infer expression column types +TableColumns columns = {{"pid", INTEGER_TYPE, ColumnOptions::DEFAULT}}; +QueryPlanner planner("SELECT pid FROM processes WHERE uid = 0", db); planner.applyTypes(columns); + +// Check if a table is disabled via flags +if (SQLiteDBManager::isDisabled("processes")) { + // table excluded by --disable_tables or --enable_tables +} ``` \ No newline at end of file diff --git a/osquery/sql/.virtual_table.md b/osquery/sql/.virtual_table.md index 49405ccb9cc..ab0d53c39d6 100644 --- a/osquery/sql/.virtual_table.md +++ b/osquery/sql/.virtual_table.md @@ -1,44 +1,38 @@ -Defines the SQLite virtual table interface for osquery, providing the core structures and functions needed to expose osquery table plugins as SQLite virtual tables. +Defines the SQLite virtual table interface for osquery, providing the core structures and functions needed to attach osquery table plugins as SQLite virtual tables in an in-memory database. ## Key Components -### Structures - -| Name | Description | -|------|-------------| -| `BaseCursor` | Tracks SQLite cursor state per query β€” holds rows, position, generator, and current result | -| `VirtualTable` | Wraps a SQLite `sqlite3_vtab` with osquery metadata: table content and the active DB instance | - -### Globals - -- **`kAttachMutex`** β€” `RecursiveMutex` guarding concurrent table attach operations, since SQLite attach is not thread-safe - -### Functions - -| Function | Description | -|----------|-------------| -| `attachTableInternal()` | Attaches a named table plugin to a SQLite in-memory DB instance | -| `detachTableInternal()` | Drops a previously attached virtual table | -| `attachFunctionInternal()` | Registers a custom scalar function with SQLite | -| `attachVirtualTables()` | Bulk-attaches all registered table plugins to a DB instance | -| `registerForeignTables()` | Registers cross-platform schema-only table plugins from the generated amalgamation (excluded in extension builds) | +| Symbol | Type | Description | +|--------|------|-------------| +| `kAttachMutex` | `RecursiveMutex` | Global mutex protecting concurrent table attach operations | +| `BaseCursor` | `struct` | SQLite cursor state β€” holds rows, position, and optional coroutine generator | +| `VirtualTable` | `struct` | SQLite vtab wrapper β€” links the plugin's `VirtualTableContent` metadata and the active `SQLiteDBInstance` | +| `attachTableInternal()` | Function | Attaches a named table plugin to a given SQLite DB instance | +| `detachTableInternal()` | Function | Drops (detaches) a named table from a SQLite DB instance | +| `attachFunctionInternal()` | Function | Registers a custom scalar function into SQLite | +| `attachVirtualTables()` | Function | Bulk-attaches all registered table plugins to a DB instance | +| `registerForeignTables()` | Function | Registers cross-platform schema plugins from the generated amalgamation (non-external builds only) | ## Usage Example -```c -// Attach all tables to a new SQLite instance -SQLiteDBInstanceRef db = SQLiteDBManager::get(); -attachVirtualTables(db); +```cpp +#include -// Attach a single table by name -Status s = attachTableInternal("processes", db, /*is_extension=*/false); +// Attach a single table plugin to a new SQLite instance +auto instance = SQLiteDBManager::getUnique(); +Status s = attachTableInternal("users", instance, false); if (!s.ok()) { - LOG(ERROR) << "Failed to attach: " << s.getMessage(); + LOG(ERROR) << "Failed to attach table: " << s.getMessage(); } -// Detach when done -detachTableInternal("processes", db); +// Attach all registered tables at DB init time +attachVirtualTables(instance); + +// Register a custom SQLite scalar function +attachFunctionInternal("my_func", [](sqlite3_context* ctx, int argc, sqlite3_value** argv) { + sqlite3_result_int(ctx, 42); +}); ``` -> **Note:** All attach operations should be performed while holding `kAttachMutex` to prevent race conditions in concurrent query environments. `BaseCursor` and `VirtualTable` are non-copyable and are only used internally by SQLite virtual table module callbacks. \ No newline at end of file +> Both `BaseCursor` and `VirtualTable` are non-copyable and intended exclusively for use inside SQLite virtual table module callbacks β€” do not instantiate them directly outside of vtab method implementations. \ No newline at end of file diff --git a/osquery/sql/tests/.sql_test_utils.md b/osquery/sql/tests/.sql_test_utils.md index 7aea9eda6d7..608c9806c68 100644 --- a/osquery/sql/tests/.sql_test_utils.md +++ b/osquery/sql/tests/.sql_test_utils.md @@ -1,38 +1,34 @@ -Utility header providing test fixtures and serialization helpers for osquery SQL query testing, including pre-built query results, diff results, and query log item data in both structured and JSON formats. +Utility header providing test fixtures and serialization helpers for osquery SQL unit tests, exposing functions that generate predictable datasets, expected query results, and serialized/deserialized pairs for validating SQL and diff result logic. ## Key Components | Symbol | Type | Description | -|---|---|---| +|--------|------|-------------| | `kTestQuery` | `const std::string` | Standard test query: `SELECT * FROM test_table` | -| `getTestDBResultStream()` | Function | Returns query/result pairs representing incremental mutations on the test database | -| `getTestDBExpectedResults()` | Function | Returns the baseline expected results for `kTestQuery` | -| `getSerializedRowColumnNames()` | Function | Returns test column names, optionally unordered and with duplicates | -| `getSerializedRow()` | Function | Returns a `(JSON, RowTyped)` pair for serialization round-trip testing | -| `getSerializedQueryData()` | Function | Returns a `(JSON, QueryDataTyped)` pair for standard query data | -| `getSerializedQueryDataWithColumnOrder()` | Function | Returns query data with repeated/non-alphabetical columns | -| `getSerializedQueryDataJSON()` | Function | Returns a `(string, QueryDataTyped)` pair in raw JSON string form | -| `getSerializedDiffResults()` | Function | Returns a `(JSON, DiffResults)` pair for diff result testing | -| `getSerializedQueryLogItem()` | Function | Returns a `(JSON, QueryLogItem)` pair for log item testing | +| `getTestDBResultStream()` | Function | Returns query/result pairs representing sequential mutations on the test dataset | +| `getTestDBExpectedResults()` | Function | Returns the baseline `QueryDataTyped` result of `kTestQuery` on the initial test DB | +| `getSerializedRowColumnNames()` | Function | Returns column names in alphabetical or intentionally unordered/repeated form | +| `getSerializedRow()` | Function | Returns a `(JSON, RowTyped)` pair for round-trip serialization testing | +| `getSerializedQueryData()` | Function | Returns a `(JSON, QueryDataTyped)` pair for query data serialization tests | +| `getSerializedQueryDataWithColumnOrder()` | Function | Variant with repeated/non-alphabetical columns | +| `getSerializedQueryDataJSON()` | Function | Returns a `(string, QueryDataTyped)` pair using raw JSON strings | +| `getSerializedDiffResults()` | Function | Returns a `(JSON, DiffResults)` pair for diff result round-trip tests | +| `getSerializedQueryLogItem()` | Function | Returns a `(JSON, QueryLogItem)` pair for log item serialization tests | ## Usage Example -```cpp -#include "sql_test_utils.h" - -// Verify expected query results match baseline -QueryDataTyped expected = osquery::getTestDBExpectedResults(); - -// Round-trip serialization test for a single row +```c +// Validate that a row serializes and deserializes correctly auto [json, row] = osquery::getSerializedRow(); -// Serialize row β†’ compare to json, deserialize json β†’ compare to row +// json should serialize to match row, and row should deserialize from json -// Test diff result serialization -auto [diffJson, diffResult] = osquery::getSerializedDiffResults(); +// Validate expected query results against the test database +auto expected = osquery::getTestDBExpectedResults(); +// Compare expected against results of kTestQuery on createTestDB() -// Iterate over incremental DB mutation stream -for (auto& [query, result] : osquery::getTestDBResultStream()) { - // Apply query, assert result matches expected +// Iterate mutation stream for incremental diff testing +for (auto& [query, expected_result] : osquery::getTestDBResultStream()) { + // Apply query, assert results match expected_result } ``` \ No newline at end of file diff --git a/osquery/system/network/.hostname.md b/osquery/system/network/.hostname.md index 37e6826819d..f201caa9d83 100644 --- a/osquery/system/network/.hostname.md +++ b/osquery/system/network/.hostname.md @@ -1,35 +1,35 @@ -Defines the `HostIdentity` structure used to represent a host machine's identity within the osquery framework, capturing its fully qualified domain name and UUID. +Defines the `HostIdentity` data structure used to represent a host machine's identity within the osquery framework, providing FQDN and UUID fields with schema serialization support. ## Key Components ### `HostIdentity` (class) -A final class encapsulating host identification data within the `osquery` namespace. +A final class encapsulating host identification data with two public fields: +- **`fqdn`** β€” Fully Qualified Domain Name of the host +- **`uuid`** β€” Unique identifier for the host -| Member | Type | Description | -|--------|------|-------------| -| `fqdn` | `std::string` | Fully qualified domain name of the host | -| `uuid` | `std::string` | Unique identifier of the host | - -### `localhost()` (static method) -Factory method that constructs and returns a `HostIdentity` instance populated with the local machine's identity information. - -### `discloseSchema()` (static template method) -Integrates with the `schemer` serialization framework to expose the `fqdn` and `uuid` fields for schema-driven serialization/deserialization (e.g., JSON, config persistence). +**Static Methods:** +- `localhost()` β€” Factory method that constructs a `HostIdentity` populated with the local machine's identity information +- `discloseSchema(Archive& a, ValueType& inst)` β€” Schemer template method enabling serialization/deserialization of `fqdn` and `uuid` fields via the osquery schemer framework ## Usage Example -```cpp -#include -#include "hostname.h" - -// Retrieve identity of the local host -osquery::HostIdentity host = osquery::HostIdentity::localhost(); +```c +// Retrieve identity for the local host +osquery::HostIdentity identity = osquery::HostIdentity::localhost(); // Access identity fields -std::string fqdn = host.fqdn; // e.g. "my-machine.example.com" -std::string uuid = host.uuid; // e.g. "550e8400-e29b-41d4-a716-446655440000" - -// Schema disclosure is handled automatically by the schemer framework -// when serializing HostIdentity instances -``` \ No newline at end of file +std::string fqdn = identity.fqdn; +std::string uuid = identity.uuid; + +// Schema disclosure is called automatically by schemer serializers, +// but can also be invoked manually with a compatible archive type +osquery::HostIdentity inst; +MyArchive archive; +osquery::HostIdentity::discloseSchema(archive, inst); +``` + +## Notes +- Uses `schemer::record` for field registration, making `HostIdentity` compatible with any archive type supported by the osquery schemer utility +- Marked `final` β€” not intended to be subclassed +- Defined within the `osquery` namespace \ No newline at end of file diff --git a/osquery/system/usersgroups/windows/.groups_service.md b/osquery/system/usersgroups/windows/.groups_service.md index b38d06ecef0..924f4a8a98b 100644 --- a/osquery/system/usersgroups/windows/.groups_service.md +++ b/osquery/system/usersgroups/windows/.groups_service.md @@ -1,42 +1,37 @@ -Background service that enumerates and caches local Windows groups by extending the `InternalRunnable` dispatcher interface. +Declares the `GroupsService` class, a background service that enumerates and caches local Windows groups for use in osquery table queries. ## Key Components -| Symbol | Type | Description | -|--------|------|-------------| -| `GroupsService` | Class | Background runnable service that populates and maintains the `GroupsCache` for local Windows groups | -| `GroupsService()` | Constructor | Accepts a `std::promise` (signals cache readiness) and a shared pointer to the `GroupsCache` instance | -| `start()` | Protected method | Entry point called by the dispatcher when the service thread begins execution | -| `processLocalGroups()` | Private method | Iterates over local groups and invokes the provided callback to insert each `Group` into the cache | -| `UpdateGroupFunc` | Type alias | Defines the signature `void(Group)` used for the per-group update callback | +- **`GroupsService`** β€” Extends `InternalRunnable` to run as a background dispatcher thread; responsible for populating and updating a `GroupsCache` with local group data +- **`GroupsService(promise, cache)`** β€” Constructor accepting a `std::promise` (signals cache readiness) and a shared `GroupsCache` instance +- **`start()`** β€” Override of `InternalRunnable::start()`; entry point for the background thread execution +- **`processLocalGroups(update_group_func)`** β€” Enumerates local Windows groups and invokes the provided callback for each discovered `Group` +- **`UpdateGroupFunc`** β€” Type alias for the group update callback signature: `void(Group)` ## Usage Example ```cpp #include -// Create the shared cache and a promise to signal completion +// Create shared cache and promise/future pair auto groups_cache = std::make_shared(); std::promise cache_ready_promise; - -// Retrieve the future before moving the promise -auto cache_ready = cache_ready_promise.get_future(); +auto cache_ready_future = cache_ready_promise.get_future(); // Instantiate and dispatch the service auto groups_svc = std::make_shared( - std::move(cache_ready_promise), groups_cache); - -Dispatcher::addService(groups_svc); + std::move(cache_ready_promise), + groups_cache +); -// Block until the initial cache population is complete -cache_ready.wait(); +// Wait for initial cache population before querying +cache_ready_future.wait(); -// GroupsCache is now populated and safe to query +// groups_cache is now populated with local Windows groups ``` ## Notes -- Windows-only β€” lives under the `windows/` users/groups subsystem path -- The `std::promise` pattern ensures consumers can block until the first full enumeration completes before querying the cache -- Follows the osquery `InternalRunnable` pattern; do not call `start()` directly β€” use `Dispatcher::addService()` \ No newline at end of file +- Windows-only β€” located under the `windows/` path and depends on `users_groups_cache.h` +- The `std::promise` pattern allows callers to block until the initial group enumeration completes, preventing queries against an empty cache \ No newline at end of file diff --git a/osquery/system/usersgroups/windows/.users_groups_cache.md b/osquery/system/usersgroups/windows/.users_groups_cache.md index 429efb17b40..58d64800e96 100644 --- a/osquery/system/usersgroups/windows/.users_groups_cache.md +++ b/osquery/system/usersgroups/windows/.users_groups_cache.md @@ -1,63 +1,54 @@ -Thread-safe Windows user and group caching layer for osquery, providing indexed lookup of system accounts with generation-based cache invalidation. +Thread-safe in-memory cache for Windows users and groups within the osquery framework, providing generation-based staleness tracking and multi-index lookups by UID, GID, SID, and name. ## Key Components ### Structs +- **`User`** β€” Holds user attributes: `uid`, `gid`, `sid`, `username`, `description`, `type`, `directory`, and a `generation` counter for cache lifecycle tracking +- **`Group`** β€” Holds group attributes: `gid`, `sid`, `groupname`, `comment`, and a `generation` counter -| Struct | Fields | Description | -|--------|--------|-------------| -| `User` | `uid`, `gid`, `sid`, `username`, `description`, `type`, `directory`, `generation` | Represents a Windows user account | -| `Group` | `gid`, `sid`, `groupname`, `comment`, `generation` | Represents a Windows group | +### Type Aliases +- `UidCacheIndex` / `GidCacheIndex` β€” `unordered_multimap` for numeric ID β†’ vector index lookups (supports multiple entries per ID) +- `SidCacheIndex` β€” `unordered_map` for SID string β†’ vector index lookups +- `GroupnameCacheIndex` β€” `unordered_map` for group name β†’ vector index lookups -Both structs implement `operator==` for value comparison (excluding `generation`). - -### Cache Index Types - -```c -using UidCacheIndex = std::unordered_multimap; -using GidCacheIndex = std::unordered_multimap; -using SidCacheIndex = std::unordered_map; -using GroupnameCacheIndex = std::unordered_map; -``` - -Indexes map lookup keys to positions in the backing `std::vector`. - -### `UsersCache` +### Classes +**`UsersCache`** | Method | Description | |--------|-------------| | `initializeCache(users)` | Bulk-loads initial user set | -| `updateUser(user)` | Upserts by SID, bumps generation | -| `increaseGeneration()` | Advances cache generation counter | +| `updateUser(user)` | Upserts by SID, advances generation | +| `increaseGeneration()` | Bumps cache generation counter | | `cleanupExpiredUsers()` | Evicts users below current generation | | `getUsersByUid(uid)` | Returns all users matching a UID | -| `getUserBySid(sid)` | Returns a single user by SID | -| `getAllUsers()` | Returns full user list | -| `clear()` | Resets cache state (testing) | +| `getUserBySid(sid)` | Returns single user by SID | +| `getAllUsers()` | Returns full user vector | -### `GroupsCache` - -Mirrors `UsersCache` with additional `getGroupByName(name)` lookup. Internally maintains three indexes: GID, SID, and group name. +**`GroupsCache`** β€” mirrors `UsersCache` with additional `getGroupByName(name)` lookup. ## Usage Example ```cpp -osquery::UsersCache cache; +#include + +osquery::UsersCache users_cache; -// Bulk initialize -cache.initializeCache(fetchAllSystemUsers()); +// Initial population +users_cache.initializeCache(fetchAllSystemUsers()); // Incremental update cycle for (auto& user : getUpdatedUsers()) { - cache.updateUser(user); + users_cache.updateUser(user); } -cache.increaseGeneration(); -cache.cleanupExpiredUsers(); // evicts stale entries +users_cache.increaseGeneration(); +users_cache.cleanupExpiredUsers(); // evicts users not seen this cycle -// Lookups -auto user = cache.getUserBySid("S-1-5-21-..."); -auto users = cache.getUsersByUid(1001); -``` +// Lookup +if (auto user = users_cache.getUserBySid("S-1-5-21-...")) { + std::cout << user->username << "\n"; +} -> **Generation mechanism:** `updateUser` / `updateGroup` stamps entries with the next generation. After a full refresh cycle, call `increaseGeneration()` then `cleanup*()` to evict any accounts removed from the system since the last scan. \ No newline at end of file +// GID/UID lookup (may return multiple β€” e.g. domain + local accounts) +auto matches = users_cache.getUsersByUid(1001); +``` \ No newline at end of file diff --git a/osquery/system/usersgroups/windows/.users_service.md b/osquery/system/usersgroups/windows/.users_service.md index 395cbaeb1d1..cf6eeffb963 100644 --- a/osquery/system/usersgroups/windows/.users_service.md +++ b/osquery/system/usersgroups/windows/.users_service.md @@ -1,36 +1,44 @@ -Declares the `UsersService` class, a background service that populates and maintains a Windows user cache by enumerating local accounts and roaming profiles. +Windows background service that populates and maintains a `UsersCache` by enumerating local accounts and roaming profiles on the system. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `UsersService` | Class | Background `InternalRunnable` that builds the `UsersCache` on Windows | -| `UsersService()` | Constructor | Accepts a `std::promise` (signals cache readiness) and a shared `UsersCache` pointer | -| `start()` | Protected method | Entry point called by the dispatcher to begin user enumeration | -| `processLocalAccounts()` | Private method | Enumerates local Windows accounts and invokes a callback per user; tracks processed SIDs | -| `processRoamingProfiles()` | Private method | Enumerates roaming profile accounts not already covered by local account processing | -| `UpdateUserFunc` | Type alias | `void(User user)` β€” callback signature used to update the cache | +### `UsersService` (class) +Extends `InternalRunnable` to run as a background dispatcher thread. Responsible for the initial and ongoing population of the users cache on Windows. + +**Constructor** +```c +UsersService(std::promise users_cache_promise, + std::shared_ptr users_cache); +``` +Accepts a promise (signaled when the cache is ready) and a shared pointer to the `UsersCache` instance to populate. + +**Protected** +- `start()` β€” Entry point called by the dispatcher when the service thread begins execution. + +**Private** +- `processLocalAccounts(processed_sids, update_user_func)` β€” Enumerates local user accounts, tracking processed SIDs and invoking the provided callback for each user found. +- `processRoamingProfiles(processed_sids, update_user_func)` β€” Enumerates roaming profile users not already covered by local account processing. +- `UpdateUserFunc` β€” Type alias (`void(User user)`) for the update callback passed to both processing methods. ## Usage Example ```cpp #include -#include -// Create the shared cache and a promise to signal when it's ready -auto users_cache = std::make_shared(); +// Create a shared cache and a ready-signal promise +auto users_cache = std::make_shared(); std::promise cache_ready_promise; auto cache_ready_future = cache_ready_promise.get_future(); -// Instantiate and dispatch the service -auto service = std::make_shared( - std::move(cache_ready_promise), users_cache); - -osquery::Dispatcher::addService(service); +// Hand off to the dispatcher β€” UsersService signals the promise once +// the initial cache population is complete +Dispatcher::addService( + std::make_shared( + std::move(cache_ready_promise), users_cache)); -// Block until the initial cache population is complete +// Block until the cache is fully populated cache_ready_future.wait(); ``` -> **Note:** This header is Windows-specific. The `std::promise` pattern ensures that consumers can synchronize against the initial cache population before querying user data. \ No newline at end of file +> **Platform note:** This service is Windows-only, gated under `osquery/system/usersgroups/windows/`. It handles both local SAM accounts and registry-backed roaming profiles as separate enumeration passes. \ No newline at end of file diff --git a/osquery/tables/applications/.jetbrains_plugins.md b/osquery/tables/applications/.jetbrains_plugins.md index 0b2d0d00fb7..01907b8f1d1 100644 --- a/osquery/tables/applications/.jetbrains_plugins.md +++ b/osquery/tables/applications/.jetbrains_plugins.md @@ -1,35 +1,33 @@ -Defines constants and utilities for locating and identifying JetBrains IDE plugin directories across Windows, macOS, and Linux within the osquery tables framework. +Defines constants and utilities for locating and identifying JetBrains IDE plugin directories across Windows, macOS, and Linux platforms within the osquery tables subsystem. ## Key Components -- **`JetBrainsProductType`** β€” Enum class enumerating all supported JetBrains IDEs: CLion, DataGrip, GoLand, IntelliJ IDEA (Ultimate + Community), PhpStorm, PyCharm (Ultimate + Community), ReSharper, Rider, RubyMine, RustRover, and WebStorm. -- **`ProductPathMap`** β€” Type alias for `std::vector>` mapping product types to their relative plugin directory paths. -- **`kWindowsPathList`** β€” Platform path map for Windows (`AppData\Roaming\JetBrains\%\plugins`). -- **`kMacOsPathList`** β€” Platform path map for macOS (`Library/Application Support/JetBrains/%/plugins`). -- **`kLinuxPathList`** β€” Platform path map for Linux (`.local/share/JetBrains/%`). Note: Linux paths omit the trailing `/plugins` segment. -- **`kProductTypeToString`** β€” Lookup map converting `JetBrainsProductType` enum values to lowercase string identifiers (e.g., `"intellij_idea"`). +- **`JetBrainsProductType`** β€” Enum class enumerating 13 supported JetBrains IDEs (CLion, DataGrip, GoLand, IntelliJ IDEA, PhpStorm, PyCharm, Rider, RubyMine, RustRover, WebStorm, and community editions). +- **`ProductPathMap`** β€” Type alias (`std::vector>`) mapping product types to their relative plugin directory paths. +- **`kWindowsPathList`** β€” Per-product plugin paths under `AppData\Roaming\JetBrains\%\plugins`. The `%` acts as a version wildcard. +- **`kMacOsPathList`** β€” Per-product paths under `Library/Application Support/JetBrains/%/plugins`. +- **`kLinuxPathList`** β€” Per-product paths under `.local/share/JetBrains/%`. +- **`kProductTypeToString`** β€” Maps each `JetBrainsProductType` to a lowercase string identifier (e.g., `"intellij_idea"`) used in query results. - **`getProductName()`** β€” Returns the string name for a given `JetBrainsProductType`. -- **`putMoreLikelyPluginJarsFirst()`** β€” Reorders a list of files in a plugin's `lib` directory, prioritizing the most likely primary JAR. -- **`fileNameIsLikeVersionedLibraryName()`** β€” Predicate that checks whether a filename matches the pattern of a versioned library JAR. +- **`putMoreLikelyPluginJarsFirst()`** β€” Reorders a list of JAR files so the most likely plugin JARs appear first, improving lookup efficiency. +- **`fileNameIsLikeVersionedLibraryName()`** β€” Predicate returning `true` if a filename matches the pattern of a versioned library (e.g., `plugin-1.0.0.jar`). ## Usage Example ```c -// Iterate all known Windows JetBrains plugin paths -for (const auto& [product, path] : osquery::tables::kWindowsPathList) { - std::string name = osquery::tables::getProductName(product); - // e.g., name = "clion", path = "AppData\\Roaming\\JetBrains\\CLion%\\plugins" - std::cout << name << " -> " << path << "\n"; +// Iterate Windows plugin paths and print product name + path +for (const auto& [productType, relativePath] : kWindowsPathList) { + const std::string name = getProductName(productType); + std::cout << name << " -> " << relativePath << "\n"; } -// Sort plugin JARs for a plugin directory -std::vector jars = {"plugin-1.0.jar", "plugin.jar", "util.jar"}; -osquery::tables::putMoreLikelyPluginJarsFirst("my-plugin", jars); - -// Check if a filename looks like a versioned library -bool versioned = osquery::tables::fileNameIsLikeVersionedLibraryName("log4j-2.17.jar"); -// versioned == true -``` +// Check if a filename looks like a versioned JAR +if (fileNameIsLikeVersionedLibraryName("myplugin-2.3.1.jar")) { + // prioritize this file during plugin scanning +} -> **Note:** The `%` wildcard in path strings represents a version suffix glob, allowing matching across multiple installed IDE versions. \ No newline at end of file +// Sort JARs so likely plugin JARs appear first +std::vector jars = {"util.jar", "myplugin-1.0.jar", "deps.jar"}; +putMoreLikelyPluginJarsFirst("myplugin", jars); +``` \ No newline at end of file diff --git a/osquery/tables/applications/chrome/.utils.md b/osquery/tables/applications/chrome/.utils.md index 6bfd0fe53d0..bcd4225aa1f 100644 --- a/osquery/tables/applications/chrome/.utils.md +++ b/osquery/tables/applications/chrome/.utils.md @@ -1,5 +1,5 @@ -Utility header for Chrome-based browser extension and profile introspection within the osquery tables subsystem. Defines data structures and helper functions for enumerating browser profiles, parsing extension manifests, and extracting metadata across multiple Chromium-derived browsers. +Utility header for parsing and modeling Chrome-based browser profiles and extensions within the osquery tables framework. ## Key Components @@ -7,48 +7,46 @@ Utility header for Chrome-based browser extension and profile introspection with - **`ChromeBrowserType`** β€” Identifies supported Chromium-based browsers (Chrome, Brave, Edge, Opera, Vivaldi, Arc, etc.) - **`ExtensionKeyError`** β€” Error codes for extension identifier computation failures -### Structs -- **`ChromeProfileSnapshot`** β€” Raw file contents captured from a browser profile directory, including `Preferences`, `Secure Preferences`, and extension manifests -- **`ChromeProfile`** β€” Parsed, structured representation of a profile with its associated `Extension` list and metadata -- **`ContentScriptsEntry`** β€” A `script`/`match` pair parsed from a manifest's `content_scripts` array +### Data Structures +- **`ChromeProfileSnapshot`** β€” Raw file snapshot of a Chrome profile, including preferences and extension file contents +- **`ChromeProfile`** β€” Parsed profile model with named extensions, settings, and manifest data +- **`ContentScriptsEntry`** β€” Represents a `js β†’ match` pair from a manifest's `content_scripts` entry +- **`ChromeProfile::Extension`** β€” Full extension model including properties, manifest hash, content scripts, and computed ID ### Key Functions | Function | Purpose | |---|---| -| `getChromeProfiles()` | Entry point β€” returns all profiles for all supported browsers | -| `getChromeProfilesFromSnapshotList()` | Converts raw snapshots into structured `ChromeProfile` objects | -| `getExtensionFromSnapshot()` | Parses a single extension snapshot into a `ChromeProfile::Extension` | +| `getChromeProfiles()` | Entry point β€” returns all profiles across supported browsers | +| `getChromeProfilesFromSnapshotList()` | Converts raw snapshots into parsed `ChromeProfile` objects | +| `getExtensionFromSnapshot()` | Parses a single extension from its snapshot | | `computeExtensionIdentifier()` | Derives extension ID from the manifest `key` property | | `webkitTimeToUnixTimestamp()` | Converts WebKit epoch timestamps to Unix format | -| `getStringLocalization()` | Resolves `__MSG_*` style localized strings from a parsed locale tree | -| `getExtensionProperty()` | Safe accessor for extension manifest properties with optional defaults | +| `getExtensionProperty()` | Safely reads a property from an extension, with optional fallback | +| `getStringLocalization()` | Resolves localized strings from parsed `_locales` data | ## Usage Example -```cpp -#include "utils.h" - -// Retrieve all Chrome-based browser profiles for a query -osquery::QueryContext ctx; -auto profiles = osquery::tables::getChromeProfiles(ctx); +```c +// Retrieve all Chrome-based browser profiles +QueryContext context; +ChromeProfileList profiles = getChromeProfiles(context); for (const auto& profile : profiles) { - // Print browser name and profile path - std::cout << getChromeBrowserName(profile.type) - << " - " << profile.path << "\n"; - - for (const auto& ext : profile.extension_list) { - // Resolve computed extension identifier - auto id_result = osquery::tables::computeExtensionIdentifier(ext); - if (id_result) { - std::cout << " Extension ID: " << id_result.get() << "\n"; + // Profile metadata + std::cout << getChromeBrowserName(profile.type) + << " | " << profile.name + << " | UID: " << profile.uid << "\n"; + + for (const auto& ext : profile.extension_list) { + // Read a standard manifest property + std::string version = getExtensionProperty(ext, "version", false); + + // Compute the extension identifier from its key + auto id_result = computeExtensionIdentifier(ext); + if (id_result) { + std::cout << " Extension ID: " << id_result.get() << "\n"; + } } - - // Access a manifest property with a fallback default - std::string version = osquery::tables::getExtensionProperty( - ext, "version", /*optional=*/true, "unknown"); - std::cout << " Version: " << version << "\n"; - } } ``` \ No newline at end of file diff --git a/osquery/tables/applications/posix/.prometheus_metrics.md b/osquery/tables/applications/posix/.prometheus_metrics.md index d7ea278e74e..0af4210c172 100644 --- a/osquery/tables/applications/posix/.prometheus_metrics.md +++ b/osquery/tables/applications/posix/.prometheus_metrics.md @@ -1,45 +1,43 @@ -Declares the data structures and core functions for the `prometheus_metrics` osquery table, which scrapes and parses Prometheus-format metrics from HTTP endpoints. +Defines the interface for scraping and parsing Prometheus metrics endpoints within the osquery tables subsystem. ## Key Components -### Constants +**Constants** +- `kColTargetName` β€” Column name for the scraped target URL +- `kColMetric` β€” Column name for the metric name +- `kColValue` β€” Column name for the metric value +- `kColTimeStamp` β€” Column name for the timestamp in milliseconds -| Constant | Value | Description | -|---|---|---| -| `kColTargetName` | `"target_name"` | Column name for the scraped target URL | -| `kColMetric` | `"metric_name"` | Column name for the metric identifier | -| `kColValue` | `"metric_value"` | Column name for the metric value | -| `kColTimeStamp` | `"timestamp_ms"` | Column name for the scrape timestamp | +**Struct** +- `PrometheusResponseData` β€” Holds raw scrape response content (`std::string content`) and the scrape timestamp (`std::chrono::milliseconds timestampMS`) -### Struct - -**`PrometheusResponseData`** β€” Holds the raw HTTP response body (`content`) and the millisecond-precision timestamp (`timestampMS`) recorded at scrape time. - -### Functions - -- **`scrapeTargets(scrapeResults, timeoutS)`** β€” Performs HTTP GET requests against all target URLs provided as keys in `scrapeResults`, writing the response body and timestamp into each corresponding `PrometheusResponseData` value. Default timeout is `1` second. -- **`parseScrapeResults(scrapeResults, rows)`** β€” Converts the raw Prometheus text-format payloads from `scrapeResults` into osquery `QueryData` rows, mapping each metric to the four defined columns. +**Functions** +- `parseScrapeResults()` β€” Parses raw Prometheus exposition-format payloads from a map of target URLs to `PrometheusResponseData`, writing structured rows into `QueryData` +- `scrapeTargets()` β€” HTTP-scrapes one or more Prometheus targets, populating a `PrometheusResponseData` map with response content and timestamps; accepts an optional timeout in seconds (default: `1`) ## Usage Example -```cpp +```c #include "prometheus_metrics.h" -using namespace osquery::tables; - // Define targets to scrape -std::map scrapeResults = { +std::map scrapeResults = { {"http://localhost:9090/metrics", {}}, {"http://localhost:9100/metrics", {}}, }; -// Scrape all targets with a 2-second timeout -scrapeTargets(scrapeResults, 2); +// Scrape all targets with a 5-second timeout +osquery::tables::scrapeTargets(scrapeResults, 5); -// Parse results into osquery rows -QueryData rows; -parseScrapeResults(scrapeResults, rows); +// Parse scraped payloads into osquery QueryData rows +osquery::QueryData rows; +osquery::tables::parseScrapeResults(scrapeResults, rows); -// rows now contains columns: target_name, metric_name, metric_value, timestamp_ms +// Each row contains: target_name, metric_name, metric_value, timestamp_ms +for (const auto& row : rows) { + std::cout << row.at("target_name") << " | " + << row.at("metric_name") << " = " + << row.at("metric_value") << std::endl; +} ``` \ No newline at end of file diff --git a/osquery/tables/events/.event_utils.md b/osquery/tables/events/.event_utils.md index c35eed836e3..e8a6af69a81 100644 --- a/osquery/tables/events/.event_utils.md +++ b/osquery/tables/events/.event_utils.md @@ -1,25 +1,21 @@ -Utility header providing shared helpers for cross-platform file event handling in osquery. +Utility header for decorating file event rows with common columns and optional hashing across platform-specific `file_events` implementations. ## Key Components -- **`kCommonFileColumns`** β€” Extern constant set of column names shared across all platform file event implementations. -- **`decorateFileEvent()`** β€” Decorator function that populates a `Row` with common file metadata and optional hash data from the `file` table. +- **`kCommonFileColumns`** β€” Extern constant set of column names shared across all platform file event implementations (e.g., inotify, FSEvents, FEN). +- **`decorateFileEvent(path, hash, r)`** β€” Populates a `Row` with standard file metadata derived from the given path. Optionally reads and hashes the file contents, mirroring columns from the `file` table. ## Usage Example -```c -#include "event_utils.h" +```cpp +#include -// Populate a row with file event metadata -osquery::Row r; -osquery::decorateFileEvent("/etc/passwd", /* hash= */ true, r); - -// r now contains common file columns (e.g., size, inode, hash) +void handleFileEvent(const std::string& targetPath, Row& eventRow) { + // Decorate the row with common file columns; + // pass true to also compute and add file hashes. + decorateFileEvent(targetPath, /* hash = */ true, eventRow); +} ``` -## Notes - -- Designed to unify file event columns across Linux (inotify), macOS (FSEvents), and Windows platform implementations. -- When `hash` is `true`, the target file is read and its hash is computed and stored in the row. -- Consumers should check `kCommonFileColumns` to determine which columns are guaranteed present after decoration. \ No newline at end of file +> **Note:** The `hash` parameter controls whether the file at `path` is read from disk and hashed at event time. Set to `false` for high-frequency events where I/O overhead is a concern. \ No newline at end of file diff --git a/osquery/tables/events/linux/.apparmor_events.md b/osquery/tables/events/linux/.apparmor_events.md index e183097e9d6..c14a5aa7c3c 100644 --- a/osquery/tables/events/linux/.apparmor_events.md +++ b/osquery/tables/events/linux/.apparmor_events.md @@ -1,46 +1,41 @@ -Brief description of the file's purpose: Declares the `AppArmorEventSubscriber` class, which subscribes to Linux audit events related to AppArmor policy enforcement and processes them into queryable osquery rows. +Defines the `AppArmorEventSubscriber` class, which subscribes to Linux audit events related to AppArmor policy enforcement within the osquery eventing framework. ## Key Components ### `AppArmorEventSubscriber` -A final class inheriting from `EventSubscriber` with the following members: +A final class inheriting from `EventSubscriber` that captures and processes AppArmor-related audit events. -| Member | Type | Description | -|--------|------|-------------| -| `init()` | `Status` | Registers the audit event type subscription on startup | -| `callback()` | `Status` | Fires when a matching kernel audit event is received | -| `processEvents()` | `static Status` | Parses a batch of `AuditEvent` objects into `QueryData` rows | -| `getEventSet()` | `static const std::set&` | Returns the set of audit event type IDs this subscriber handles | +| Method | Description | +|--------|-------------| +| `init()` | Registers the audit event type subscription on startup | +| `callback(ec, sc)` | Fired by the kernel when a matching audit event occurs | +| `processEvents(emitted_row_list, event_list)` | Static method that transforms raw `AuditEvent` records into osquery `QueryData` rows | +| `getEventSet()` | Returns the static set of audit event type codes this subscriber handles | ## Usage Example -```c -// Typical osquery subscriber lifecycle (handled by the framework): +```cpp +// Subscriber is registered automatically via the osquery eventing system. +// Direct usage of processEvents for testing or manual processing: -// 1. Framework calls init() to register subscriptions -AppArmorEventSubscriber subscriber; -subscriber.init(); +std::vector raw_events = /* populated by AuditEventPublisher */; +QueryData rows; -// 2. Kernel audit events fire callback() automatically -// subscriber.callback(ec, sc) is invoked by AuditEventPublisher - -// 3. Manually process a batch of audit events (e.g., in tests) -QueryData results; -std::vector events = getAuditEvents(); - -auto status = AppArmorEventSubscriber::processEvents(results, events); -if (status.ok()) { - for (const auto& row : results) { - // Each row contains AppArmor enforcement data +Status s = AppArmorEventSubscriber::processEvents(rows, raw_events); +if (s.ok()) { + for (const auto& row : rows) { + // Each row contains AppArmor enforcement details: + // e.g., operation, profile, requested_mask, denied_mask, etc. } } -// 4. Inspect handled event type IDs -const auto& eventSet = AppArmorEventSubscriber::getEventSet(); -for (int eventType : eventSet) { - // e.g., AUDIT_APPARMOR_ALLOWED, AUDIT_APPARMOR_DENIED, etc. -} +// Inspect handled event type codes: +const auto& handled = AppArmorEventSubscriber::getEventSet(); ``` -> **Note:** This subscriber is Linux-only and depends on the kernel audit subsystem (`linux/audit.h`). It integrates with osquery's `AuditEventPublisher` to surface AppArmor allow/deny decisions as structured table data. \ No newline at end of file +## Notes + +- Integrates with `AuditEventPublisher` via the Linux kernel audit subsystem (`linux/audit.h`) +- `processEvents` is `noexcept` and static, making it unit-testable without a live subscriber instance +- Resides in the `osquery` namespace; registration into the event framework is handled by osquery's plugin machinery \ No newline at end of file diff --git a/osquery/tables/events/linux/.bpf_process_events.md b/osquery/tables/events/linux/.bpf_process_events.md index 19e0a9c955f..e611abe2700 100644 --- a/osquery/tables/events/linux/.bpf_process_events.md +++ b/osquery/tables/events/linux/.bpf_process_events.md @@ -1,45 +1,42 @@ -Header file defining the `BPFProcessEventSubscriber` class, which subscribes to BPF-based process events on Linux and converts them into osquery table rows. +Header file defining the `BPFProcessEventSubscriber` class, which subscribes to BPF-based process events on Linux and transforms raw kernel events into osquery table rows. ## Key Components ### `BPFProcessEventSubscriber` -An `EventSubscriber` specialized for `BPFEventPublisher` that handles process lifecycle events captured via eBPF. +An `EventSubscriber` specialization bound to `BPFEventPublisher` that captures process lifecycle events via eBPF. | Method | Description | -|---|---| -| `init()` | Initializes the subscriber and registers subscriptions | -| `eventCallback(...)` | Handles incoming BPF events from the publisher | -| `generateRow(...)` | Converts a single `ISystemStateTracker::Event` into a table `Row` | -| `generateRowList(...)` | Batch-converts an `EventList` into a `vector` | -| `generateCmdlineColumn(...)` | Formats argv into a space-separated command line string | -| `generateJsonCmdlineColumn(...)` | Formats argv into a JSON-encoded command line string | +|--------|-------------| +| `init()` | Registers subscriptions with the BPF event publisher | +| `eventCallback()` | Invoked per event; routes data into row generation | +| `generateRow()` | Converts a single `ISystemStateTracker::Event` into a `Row` | +| `generateRowList()` | Batch-converts an `EventList` into a `vector` | +| `generateCmdlineColumn()` | Joins `argv` into a space-separated cmdline string | +| `generateJsonCmdlineColumn()` | Serializes `argv` as a JSON array string | ## Usage Example ```c -// Typical osquery subscriber pattern - registered automatically via macro -// Implementation side (bpf_process_events.cpp): +// Subscriber is auto-registered via osquery's event framework. +// Direct row generation can be tested independently: -Status BPFProcessEventSubscriber::init() { - auto subscription = createSubscriptionContext(); - subscribe(&BPFProcessEventSubscriber::eventCallback, subscription); - return Status::success(); -} - -// Generating a row from a captured process event: -ISystemStateTracker::Event event = /* from BPF publisher */; +ISystemStateTracker::Event evt = /* populated by BPF publisher */; Row row; -if (BPFProcessEventSubscriber::generateRow(row, event)) { - // row is populated with pid, path, cmdline, etc. + +if (BPFProcessEventSubscriber::generateRow(row, evt)) { + // row["pid"], row["cmdline"], row["path"], etc. are now populated } -// Formatting command-line arguments: -std::vector argv = {"/usr/bin/ssh", "-l", "user", "host"}; -auto cmdline = BPFProcessEventSubscriber::generateCmdlineColumn(argv); -// β†’ "/usr/bin/ssh -l user host" -auto json_cmdline = BPFProcessEventSubscriber::generateJsonCmdlineColumn(argv); -// β†’ '["/usr/bin/ssh","-l","user","host"]' +// Build cmdline column from argv +std::vector argv = {"/usr/bin/ls", "-la", "/tmp"}; +std::string cmdline = BPFProcessEventSubscriber::generateCmdlineColumn(argv); +// β†’ "/usr/bin/ls -la /tmp" + +std::string json_cmdline = BPFProcessEventSubscriber::generateJsonCmdlineColumn(argv); +// β†’ ["/usr/bin/ls","-la","/tmp"] ``` -> **Note:** This subscriber is Linux-only and requires eBPF support in the kernel. It feeds the `bpf_process_events` virtual table in osquery. \ No newline at end of file +## Notes +- Linux-only; depends on `BPFEventPublisher` which requires kernel eBPF support. +- Static helpers (`generateRow`, `generateRowList`, `generateCmdlineColumn`, `generateJsonCmdlineColumn`) are unit-testable without a running event loop. \ No newline at end of file diff --git a/osquery/tables/events/linux/.bpf_socket_events.md b/osquery/tables/events/linux/.bpf_socket_events.md index 499edd568be..5ce92760c5f 100644 --- a/osquery/tables/events/linux/.bpf_socket_events.md +++ b/osquery/tables/events/linux/.bpf_socket_events.md @@ -1,37 +1,35 @@ -BPF-based socket event subscriber that listens for socket-related system events via the eBPF event publisher and converts them into osquery table rows. +BPF-based socket event subscriber for osquery that captures and processes Linux socket-related system events via the BPF event publisher pipeline. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `BPFSocketEventSubscriber` | Class | Final subscriber class inheriting from `EventSubscriber` | -| `init()` | Method | Initializes the subscriber and registers subscriptions | -| `eventCallback()` | Method | Handles incoming BPF socket events from the publisher | -| `generateRow()` | Static Method | Converts a single `ISystemStateTracker::Event` into an osquery `Row` | -| `generateRowList()` | Static Method | Batch-converts an `EventList` into a `std::vector` | +- **`BPFSocketEventSubscriber`** β€” Final class extending `EventSubscriber` that subscribes to BPF-sourced socket events on Linux. +- **`init()`** β€” Initializes the subscriber and registers it with the BPF event publisher. +- **`eventCallback()`** β€” Invoked per-event; receives an event context and subscription context for processing incoming socket events. +- **`generateRow()`** β€” Static helper that converts a single `ISystemStateTracker::Event` into an osquery `Row` for table output. +- **`generateRowList()`** β€” Static helper that batch-converts an `ISystemStateTracker::EventList` into a vector of `Row` objects. ## Usage Example ```c -// Subscriber is registered automatically via osquery's event framework. -// Static helpers can be used independently to convert events to rows: +// Subscriber is auto-registered via osquery's event framework. +// generateRow is used internally during event processing: -ISystemStateTracker::Event socket_event = /* ... */; Row row; - -if (BPFSocketEventSubscriber::generateRow(row, socket_event)) { - // row is now populated with socket event data +bool success = BPFSocketEventSubscriber::generateRow(row, event); +if (success) { + // row now contains socket event fields (fd, pid, family, etc.) } -// Batch conversion from an event list -ISystemStateTracker::EventList event_list = /* ... */; -std::vector rows = - BPFSocketEventSubscriber::generateRowList(event_list); +// Batch processing from an event list: +auto rows = BPFSocketEventSubscriber::generateRowList(event_list); +for (const auto& r : rows) { + // emit or store each row +} ``` ## Notes -- Operates on Linux only via the `BPFEventPublisher` pipeline -- The `generateRow` / `generateRowList` static methods are useful for testing row generation independently of the event loop -- Follows osquery's standard subscriber pattern: `init()` β†’ `eventCallback()` β†’ row emission \ No newline at end of file +- Linux-only; depends on `BPFEventPublisher` and `ISystemStateTracker`. +- Follows the standard osquery event subscriber pattern β€” `init()` + `eventCallback()` handle the subscription lifecycle. +- `generateRow` / `generateRowList` are static, making them independently testable without a live subscriber instance. \ No newline at end of file diff --git a/osquery/tables/events/linux/.process_events.md b/osquery/tables/events/linux/.process_events.md index fe10862489c..f57f52c4702 100644 --- a/osquery/tables/events/linux/.process_events.md +++ b/osquery/tables/events/linux/.process_events.md @@ -1,38 +1,40 @@ -Defines the `AuditProcessEventSubscriber` class, an osquery event subscriber that listens to Linux audit events for process-related system calls (e.g., `execve`, `clone`). +Defines the `AuditProcessEventSubscriber` class for subscribing to Linux kernel process-related audit events via the osquery audit event framework. ## Key Components -| Member | Description | -|---|---| -| `init()` | Registers the subscriber's audit event type subscription | -| `Callback()` | Invoked when kernel audit events matching the subscription fire | -| `ProcessEvents()` | Batch-processes a list of `AuditEvent` objects into `Row` output | -| `ProcessExecveEventData()` | Extracts fields specific to `execve`/`execveat` syscall events | -| `GetProcessIDs()` | Parses `pid` and `ppid` from a syscall audit record | -| `IsThreadClone()` | Checks whether a `clone()` syscall includes the `CLONE_THREAD` flag | +**`AuditProcessEventSubscriber`** β€” Final class inheriting from `EventSubscriber` with the following members: + +| Method | Description | +|--------|-------------| +| `init()` | Registers the audit event type subscription | +| `Callback()` | Fires when kernel events match the subscribed type | +| `ProcessEvents()` | Processes batched audit events into emitted rows | +| `ProcessExecveEventData()` | Parses `execve`/`execveat` syscall data into a result row | +| `GetProcessIDs()` | Extracts `pid` and `ppid` from a syscall audit record | +| `IsThreadClone()` | Checks if a `clone()` syscall carries the `CLONE_THREAD` flag | | `GetSyscallName()` | Resolves a syscall number to its string name | -| `GetSyscallNameMap()` | Returns the static syscall number-to-name lookup table | +| `GetSyscallNameMap()` | Returns the static syscall number-to-name lookup map | ## Usage Example ```cpp -// Typical subscriber wiring (handled by osquery internals) -AuditProcessEventSubscriber subscriber; -subscriber.init(); // registers audit event subscription +// Typically registered automatically via osquery's event framework +// Manual usage for testing ProcessEvents: -// Static processing used in unit tests or direct invocation std::vector rows; -std::vector events = getAuditEvents(); +std::vector events = getAuditEventsFromKernel(); auto status = AuditProcessEventSubscriber::ProcessEvents(rows, events); if (!status.ok()) { LOG(ERROR) << "Failed to process audit events: " << status.getMessage(); } -// Check if a clone() call spawned a thread vs. a process -bool is_thread = false; -AuditProcessEventSubscriber::IsThreadClone(is_thread, syscall_nr, record); +// Resolve a syscall number to its name +std::string name; +if (AuditProcessEventSubscriber::GetSyscallName(name, 59 /* execve */)) { + LOG(INFO) << "Syscall: " << name; // "execve" +} ``` -> **Note:** All processing methods are declared `static` and `noexcept`, making them independently testable without a live subscriber instance. This class is Linux-only, depending on `AuditEventPublisher` from `osquery/events/linux/`. \ No newline at end of file +> **Note:** All static processing methods are `noexcept`, making them safe to call without try/catch guards in performance-sensitive audit pipelines. \ No newline at end of file diff --git a/osquery/tables/events/linux/.process_file_events.md b/osquery/tables/events/linux/.process_file_events.md index 0d06d5a5937..4eaf482ef45 100644 --- a/osquery/tables/events/linux/.process_file_events.md +++ b/osquery/tables/events/linux/.process_file_events.md @@ -1,48 +1,45 @@ -Header file defining the Linux audit-based File Integrity Monitoring (FIM) subsystem for osquery, tracking file system events via syscall interception through the Linux audit framework. +Header defining the Linux audit-based File Integrity Monitoring (FIM) subscriber for osquery, tracking file system events via kernel syscall auditing. ## Key Components ### Data Structures -| Struct/Class | Purpose | -|---|---| -| `AuditdFimInodeDescriptor` | Describes a file/folder inode with path and type | -| `AuditdFimFdDescriptor` | Tracks an open file descriptor and its last operation | -| `AuditdFimPathRecordItem` | Aggregates `AUDIT_PATH` records per syscall | -| `AuditdFimSrcDestData` | Holds source/destination for rename/link operations | -| `AuditdFimIOData` | Holds target and type for open/read/write/close/unlink operations | -| `AuditdFimSyscallContext` | Full context for a captured syscall event (PIDs, UIDs, paths, return values) | -| `AuditdFimContext` | Top-level FIM state: included paths, process map, inode map | +| Type | Purpose | +|------|---------| +| `AuditdFimInodeDescriptor` | Maps an inode to a file/folder path with type classification | +| `AuditdFimFdDescriptor` | Tracks a file descriptor's inode and last operation (Open, Read, Write, etc.) | +| `AuditdFimSyscallContext` | Full context for a single audited syscall, including process credentials, paths, and return value | +| `AuditdFimSrcDestData` | Source/destination paths for rename/link syscalls | +| `AuditdFimIOData` | Target path and operation type for read/write/open/close/unlink syscalls | +| `AuditdFimContext` | Top-level FIM state: included path list, process map, inode map | -### Core Classes +### Classes -- **`AuditdFimInodeMap`** β€” Global inode-to-path registry with get/save/remove operations -- **`AuditdFimFdMap`** β€” Per-process file descriptor map supporting open, dup, and close tracking -- **`AuditdFimProcessMap`** β€” Process-level manager over `AuditdFimFdMap` instances; handles `fork`/`clone` via `clone()` and `dup`/`dup2` via `duplicate()` -- **`ProcessFileEventSubscriber`** β€” osquery `EventSubscriber` that consumes `AuditEventPublisher` events and emits rows on read/write activity - -### Supported Syscall Types - -```text -Link Β· Symlink Β· Unlink Β· Rename Β· Open Β· OpenTruncate -Close Β· Dup Β· Read Β· Write Β· Truncate Β· Mmap Β· NameToHandleAt Β· CloneOrFork -``` +- **`AuditdFimInodeMap`** β€” Global inode-to-path registry; supports get-by-reference, take-and-remove, save, and clear operations +- **`AuditdFimFdMap`** β€” Per-process file descriptor map with fd duplication support and untracked-fd warning suppression +- **`AuditdFimProcessMap`** β€” Aggregates `AuditdFimFdMap` instances per PID; handles `fork`/`clone` via `clone()` and `dup`/`dup2` via `duplicate()` +- **`ProcessFileEventSubscriber`** β€” osquery `EventSubscriber` that consumes `AuditEventPublisher` events, maintains FIM context, and emits rows on file activity ## Usage Example ```cpp -// Static entry points on the subscriber +// Core static entry points used by the subscriber internally + +// Process a batch of audit events and collect emitted rows std::vector rows; -AuditdFimContext ctx; +AuditdFimContext fim_context; +std::vector events = /* from publisher */; -// Provide audit events from the publisher -Status s = ProcessFileEventSubscriber::ProcessEvents( - rows, ctx, event_list); +Status status = ProcessFileEventSubscriber::ProcessEvents( + rows, fim_context, events); // Query which syscall numbers this subscriber handles -const std::set& syscalls = - ProcessFileEventSubscriber::GetSyscallSet(); +const std::set& handled = ProcessFileEventSubscriber::GetSyscallSet(); ``` -> **Note:** This header is Linux-only (depends on `` and the Linux audit framework). The subscriber integrates with osquery's event pipeline via `AuditEventPublisher`. \ No newline at end of file +## Notes + +- Syscall coverage includes: `open`, `read`, `write`, `close`, `rename`, `link`, `symlink`, `unlink`, `dup`, `truncate`, `mmap`, `clone`/`fork` +- `SyscallData` uses `boost::variant` to hold either `AuditdFimSrcDestData` or `AuditdFimIOData` depending on syscall type +- Warning log spam for untracked PIDs/FDs is suppressed via time-based filters \ No newline at end of file diff --git a/osquery/tables/events/linux/.seccomp_events.md b/osquery/tables/events/linux/.seccomp_events.md index 0db6a3818e6..1ddcd96ccbf 100644 --- a/osquery/tables/events/linux/.seccomp_events.md +++ b/osquery/tables/events/linux/.seccomp_events.md @@ -1,51 +1,53 @@ -Defines the `SeccompEventSubscriber` class for capturing and parsing Linux seccomp audit events within the osquery event framework. +Header file defining the `SeccompEventSubscriber` class for capturing and processing Linux seccomp audit events within the osquery event framework. ## Key Components -### Preprocessor Definitions -Backward-compatibility constants for kernels older than 4.14: +### Kernel Compatibility Macros -| Constant | Value | Description | +Backfill constants missing from older kernels (pre-4.14): + +| Macro | Value | Description | |---|---|---| | `SECCOMP_RET_KILL_PROCESS` | `0x80000000U` | Kill entire process | -| `SECCOMP_RET_KILL_THREAD` | `SECCOMP_RET_KILL` | Kill calling thread | -| `SECCOMP_RET_LOG` | `0x7ffc0000U` | Allow after logging | +| `SECCOMP_RET_KILL_THREAD` | `SECCOMP_RET_KILL` | Kill offending thread | +| `SECCOMP_RET_LOG` | `0x7ffc0000U` | Allow syscall after logging | ### `SeccompEventSubscriber` -Subscribes to `AuditEventPublisher` events and translates raw audit records into structured query rows. -**Static Lookup Tables** -- `seccomp_actions_map` β€” Maps seccomp return codes to human-readable action names -- `arch_codes_map` β€” Maps architecture codes (from `audit.h`) to architecture name strings -- `syscall_x86_64_map` β€” Maps x86_64 syscall numbers to syscall name strings +Extends `EventSubscriber` to subscribe to Linux audit events related to seccomp violations. + +**Private Members** + +- `seccomp_actions_map` β€” Maps seccomp action codes (`seccomp.h`) to human-readable names +- `arch_codes_map` β€” Maps audit architecture codes (`audit.h`) to architecture names +- `syscall_x86_64_map` β€” Maps x86_64 syscall numbers to syscall names +- `parseEvent()` β€” Extracts fields from a raw `AuditEvent` into a result `Row` -**Methods** +**Public Methods** -| Method | Description | -|---|---| -| `init()` | Registers the audit event type subscription | -| `Callback(ec, sc)` | Fires when a matching kernel seccomp event is received | -| `processEvents(...)` | Batch-processes a list of `AuditEvent` objects into `QueryData` rows | -| `parseEvent(...)` | Internal parser mapping raw `AuditEvent` fields to a `Row` | +- `init()` β€” Registers the audit event type subscription +- `Callback()` β€” Receives matched kernel audit events +- `processEvents()` β€” Batch-processes a list of `AuditEvent` objects into emittable `QueryData` rows ## Usage Example -```c +```cpp // Subscriber is auto-registered via the osquery event framework. -// Emitted rows can be queried via osquery SQL: -// -// SELECT * FROM seccomp_events; -// -// processEvents can be called directly in tests: +// Query seccomp violations directly via osquery SQL: +// SELECT * FROM seccomp_events; + +// Programmatic batch processing (e.g., in tests): QueryData results; -std::vector events = { /* populated by publisher */ }; +std::vector events = getAuditEvents(); Status s = SeccompEventSubscriber::processEvents(results, events); if (s.ok()) { for (const auto& row : results) { - // row["syscall"], row["action"], row["arch"], etc. + // row["syscall"], row["arch"], row["action"], etc. } } -``` \ No newline at end of file +``` + +> **Note:** Requires Linux with audit subsystem enabled. Compatible with kernels 3.x and newer; missing seccomp constants are automatically defined at compile time. \ No newline at end of file diff --git a/osquery/tables/events/linux/.selinux_events.md b/osquery/tables/events/linux/.selinux_events.md index b60e6324b86..c05374953e4 100644 --- a/osquery/tables/events/linux/.selinux_events.md +++ b/osquery/tables/events/linux/.selinux_events.md @@ -1,43 +1,40 @@ -Defines the `SELinuxEventSubscriber` class, an osquery event subscriber that listens for SELinux-related audit events on Linux via the `AuditEventPublisher`. +Defines the `SELinuxEventSubscriber` class for capturing and processing SELinux-related audit events on Linux systems via the osquery audit event framework. ## Key Components -| Symbol | Type | Description | -|--------|------|-------------| -| `SELinuxEventSubscriber` | Class | Event subscriber for SELinux audit events | -| `init()` | Method | Registers the audit event type subscription on startup | -| `Callback()` | Method | Invoked when kernel audit events matching the subscribed type fire | -| `ProcessEvents()` | Static Method | Parses and transforms a batch of `AuditEvent` objects into osquery `Row` results | -| `GetEventSet()` | Static Method | Returns the set of audit event type IDs this subscriber handles | +### `SELinuxEventSubscriber` +An `EventSubscriber` specialization that subscribes to `AuditEventPublisher` events filtered to SELinux audit types. + +| Member | Description | +|--------|-------------| +| `init()` | Registers the subscriber and declares the audit event type subscription | +| `Callback(ec, sc)` | Invoked by the event publisher when a matching kernel audit event fires | +| `ProcessEvents(emitted_row_list, event_list)` | Static method that transforms raw `AuditEvent` objects into osquery `Row` entries | +| `GetEventSet()` | Returns the static set of audit event type codes this subscriber handles | ## Usage Example ```cpp -// ProcessEvents can be called directly for unit testing -// without requiring a live audit publisher +// Typical usage is handled automatically by the osquery event framework. +// ProcessEvents can be tested or called directly: std::vector rows; -std::vector events = GetMockSELinuxEvents(); +std::vector events = getAuditEventsFromKernel(); Status status = SELinuxEventSubscriber::ProcessEvents(rows, events); - if (status.ok()) { for (const auto& row : rows) { - // Each row contains SELinux event fields - // e.g., row["avc_decision"], row["pid"], row["scontext"] + // row contains parsed SELinux audit fields } } -// Inspect which audit event codes this subscriber handles +// Inspect handled event types const std::set& handled = SELinuxEventSubscriber::GetEventSet(); -for (int code : handled) { - LOG(INFO) << "Handles audit event code: " << code; -} ``` ## Notes -- Inherits from `EventSubscriber`, binding it to Linux's audit subsystem -- `ProcessEvents` and `GetEventSet` are `static` and `noexcept`, making them safely testable in isolation -- Lives inside the `osquery` namespace alongside other platform event subscribers \ No newline at end of file +- Marked `final` β€” not intended to be subclassed. +- `ProcessEvents` and `GetEventSet` are `static` and `noexcept`, making them safe for isolated unit testing without a running event loop. +- Subscribes to the Linux `AuditEventPublisher`, so this subscriber is Linux-only. \ No newline at end of file diff --git a/osquery/tables/events/linux/.socket_events.md b/osquery/tables/events/linux/.socket_events.md index bc3b6b8956a..750a5a38eb2 100644 --- a/osquery/tables/events/linux/.socket_events.md +++ b/osquery/tables/events/linux/.socket_events.md @@ -1,22 +1,18 @@ -Defines the `SocketEventSubscriber` class for capturing and processing Linux socket-related audit events via the osquery audit event framework. +Defines the `SocketEventSubscriber` class, an osquery event subscriber that monitors Linux socket-related system calls via the audit event publisher. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `SocketEventSubscriber` | Class | EventSubscriber for `AuditEventPublisher`; handles socket syscall audit events | -| `init()` | Method | Registers audit event type subscriptions on startup | -| `Callback()` | Method | Fires when kernel socket events match the subscribed type | -| `ProcessEvents()` | Static Method | Transforms raw `AuditEvent` list into osquery `Row` results with configurable filtering | -| `GetSyscallSet()` | Static Method | Returns the set of syscall IDs this subscriber handles | -| `parseSockAddr()` | Static Method | Decodes the `saddr` field from `AUDIT_SOCKADDR` records into row columns | +- **`SocketEventSubscriber`** β€” Final class inheriting from `EventSubscriber` that subscribes to kernel-level socket audit events +- **`init()`** β€” Registers the subscriber and declares the audit event type subscription +- **`Callback()`** β€” Handles incoming kernel audit events matching the subscribed socket syscalls +- **`ProcessEvents()`** β€” Static method that transforms raw `AuditEvent` objects into `Row` entries; accepts filtering flags for failed events, UNIX sockets, null accept events, and null accept socket events +- **`GetSyscallSet()`** β€” Returns the set of Linux syscall numbers this subscriber monitors (e.g., `connect`, `accept`, `bind`) +- **`parseSockAddr()`** β€” Parses the `saddr` field from an `AUDIT_SOCKADDR` record, populating a result `Row` and flagging whether the address is a UNIX domain socket ## Usage Example ```cpp -#include - // ProcessEvents is the primary testable interface std::vector rows; std::vector events = getAuditEvents(); @@ -30,17 +26,11 @@ Status status = SocketEventSubscriber::ProcessEvents( /*allow_null_accept_socket_events=*/false ); -// Inspect handled syscalls +// Inspect supported syscalls const auto& syscalls = SocketEventSubscriber::GetSyscallSet(); // Parse a raw saddr field Row row; bool is_unix = false; bool ok = SocketEventSubscriber::parseSockAddr("0200...", row, is_unix); -``` - -## Notes - -- Filtering flags in `ProcessEvents` allow fine-grained control over which socket events are emitted (failed, UNIX domain, null-accept variants) -- `parseSockAddr` sets `unix_socket` to `true` when the address family is `AF_UNIX` -- Part of the osquery Linux audit subsystem; requires `AuditEventPublisher` to be active \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/tables/events/windows/.dns_lookup_events.md b/osquery/tables/events/windows/.dns_lookup_events.md index d14854b274b..9a5e2f23726 100644 --- a/osquery/tables/events/windows/.dns_lookup_events.md +++ b/osquery/tables/events/windows/.dns_lookup_events.md @@ -1,40 +1,35 @@ -Declares the `EtwDNSEventSubscriber` class, an ETW-based Windows event subscriber that captures and processes DNS lookup events via the osquery event system. +Declares the `EtwDNSEventSubscriber` class, an ETW-based event subscriber that captures Windows DNS lookup events via the `EtwPublisherDNS` publisher. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `EtwDNSEventSubscriber` | Class | Final subscriber class templated on `EtwPublisherDNS` | -| `init()` | Method | Initializes the subscriber and registers subscriptions | -| `eventCallback()` | Method | Handles incoming DNS ETW events with event and subscription context | +- **`EtwDNSEventSubscriber`** β€” Final class inheriting from `EventSubscriber`; subscribes to ETW DNS events on Windows + - **`init()`** β€” Initializes the subscriber and registers the event subscription + - **`eventCallback()`** β€” Callback invoked when a DNS event fires; receives the event context (`ECRef`) and subscription context (`SCRef`) for processing ## Usage Example ```c -// Registered automatically via osquery's event framework. -// The subscriber hooks into EtwPublisherDNS at init time. - -class EtwDNSEventSubscriber final : public EventSubscriber { - public: - Status init() override { - // Subscribe to DNS ETW events - auto sc = createSubscriptionContext(); - subscribe(&EtwDNSEventSubscriber::eventCallback, sc); - return Status::success(); - } - - Status eventCallback(const ECRef& event_context, - const SCRef& subscription_context) { - // Process DNS lookup event data from ETW - // e.g., extract queried hostname, response IPs, PID - return Status::success(); - } -}; +// Subscriber is auto-registered via osquery's event framework. +// Typical interaction is through the osquery eventing pipeline: + +namespace osquery { + +Status EtwDNSEventSubscriber::init() { + // Register subscription with the ETW DNS publisher + auto subscription = createSubscriptionContext(); + subscribe(&EtwDNSEventSubscriber::eventCallback, subscription); + return Status::success(); +} + +Status EtwDNSEventSubscriber::eventCallback(const ECRef& event_context, + const SCRef& subscription_context) { + // Process incoming DNS lookup event data + // e.g., extract queried hostname, response IPs, process ID + return Status::success(); +} + +} // namespace osquery ``` -## Notes - -- **Windows-only**: Depends on ETW (Event Tracing for Windows) infrastructure via `etw_publisher_dns.h` -- Follows osquery's pub/sub event model β€” the publisher (`EtwPublisherDNS`) emits raw ETW DNS events; this subscriber consumes and tables them -- Registered into the osquery event framework automatically via `REGISTER` macros (typically in the corresponding `.cpp` file) \ No newline at end of file +> **Note:** This subscriber is Windows-only and relies on the ETW (Event Tracing for Windows) infrastructure. It integrates with osquery's eventing framework and is not instantiated directly β€” the framework handles lifecycle management automatically. \ No newline at end of file diff --git a/osquery/tables/events/windows/.etw_process_events.md b/osquery/tables/events/windows/.etw_process_events.md index 4beed37bb1e..b3117d7d869 100644 --- a/osquery/tables/events/windows/.etw_process_events.md +++ b/osquery/tables/events/windows/.etw_process_events.md @@ -1,35 +1,35 @@ -Defines the `EtwProcessEventSubscriber` class, an ETW (Event Tracing for Windows) event subscriber that listens for process-related events published by `EtwPublisherProcesses`. +Defines the ETW (Event Tracing for Windows) process event subscriber that listens to process-related events published by `EtwPublisherProcesses`. ## Key Components -- **`EtwProcessEventSubscriber`** β€” Final class inheriting from `EventSubscriber`. Subscribes to Windows ETW process events within the osquery event framework. - - **`init()`** β€” Initializes the subscriber and registers its event subscriptions. - - **`eventCallback()`** β€” Called by the event framework when a matching process event fires; receives both the event context (`ECRef`) and subscription context (`SCRef`) for processing. +- **`EtwProcessEventSubscriber`** β€” Final class inheriting from `EventSubscriber`. Subscribes to ETW process events and handles their processing via callback. + - `init()` β€” Initializes the subscriber and registers subscriptions. + - `eventCallback(ECRef, SCRef)` β€” Invoked when a process event is received; handles event processing logic using the event context and subscription context. ## Usage Example ```c -// Subscriber is registered automatically via osquery's plugin/event system. -// Typical initialization flow: +// Subscriber is registered automatically via osquery's event framework. +// Typical pattern for interacting with the subscriber: namespace osquery { Status EtwProcessEventSubscriber::init() { - auto subscription = createSubscriptionContext(); - subscribe(&EtwProcessEventSubscriber::eventCallback, subscription); - return Status::success(); + auto subscription = createSubscriptionContext(); + subscribe(&EtwProcessEventSubscriber::eventCallback, subscription); + return Status::success(); } Status EtwProcessEventSubscriber::eventCallback( const ECRef& event_context, const SCRef& subscription_context) { - // Handle incoming ETW process event data - // e.g., process creation, termination events - return Status::success(); + // Process the ETW event data from event_context + // e.g., log process creation/termination events + return Status::success(); } } // namespace osquery ``` -> **Note:** This subscriber is Windows-only. It integrates with osquery's `EventSubscriber` framework, meaning registration and dispatch are handled by the osquery event publisher/subscriber lifecycle β€” no manual instantiation is required. \ No newline at end of file +> **Note:** This subscriber is part of osquery's Windows ETW event pipeline. It pairs with `EtwPublisherProcesses` to capture process lifecycle events (creation, termination) from the Windows kernel via ETW, feeding data into osquery's evented table infrastructure. \ No newline at end of file diff --git a/osquery/tables/events/windows/.ntfs_journal_events.md b/osquery/tables/events/windows/.ntfs_journal_events.md index b7185618434..5326804fea6 100644 --- a/osquery/tables/events/windows/.ntfs_journal_events.md +++ b/osquery/tables/events/windows/.ntfs_journal_events.md @@ -1,45 +1,40 @@ -Declares the `NTFSEventSubscriber` class and supporting utilities for subscribing to and processing NTFS file system change events on Windows. +Defines the `NTFSEventSubscriber` class and supporting utilities for subscribing to NTFS file system change events on Windows, bridging the NTFS event publisher to osquery's event framework. ## Key Components | Symbol | Type | Description | -|---|---|---| -| `NTFSEventSubscriber` | Class | Event subscriber that receives NTFS file change events from `NTFSEventPublisher` | -| `shouldEmit` | Method | Filters events based on subscription context before emission | -| `generateRowFromEvent` | Method | Converts an `NTFSEventRecord` into an osquery `Row` for table output | -| `init` | Method | Initialization routine called once at startup | -| `configure` | Method | Configuration callback; may be invoked multiple times on config changes | -| `Callback` | Method | Primary entry point for receiving events from the publisher | -| `StringList` | Type alias | Convenience alias for `std::vector` | -| `processConfiguration` | Function | Parses subscriber configuration to populate include/exclude path lists and access categories | +|--------|------|-------------| +| `NTFSEventSubscriber` | Class | Event subscriber that receives, filters, and records NTFS file change events | +| `shouldEmit()` | Method | Filters events based on subscription context before emission | +| `generateRowFromEvent()` | Method | Converts a raw `NTFSEventRecord` into an osquery `Row` for table output | +| `init()` | Method | Initializes the subscriber on startup | +| `configure()` | Method | Applies/reapplies configuration; may be called multiple times | +| `Callback()` | Method | Entry point called by the publisher when a new NTFS event arrives | +| `StringList` | Type alias | `std::vector` convenience alias | +| `processConfiguration()` | Function | Parses config to populate include/exclude path lists and access categories for a subscription context | ## Usage Example -```c -// Typical lifecycle managed by the osquery event framework: - +```cpp +// Typical osquery subscriber wiring (done internally by the framework) NTFSEventSubscriber subscriber; - -// Called once at startup subscriber.init(); - -// Called on config load or reload subscriber.configure(); -// Populate include/exclude paths from config -StringList access_categories = {"write", "delete"}; +// processConfiguration is called during configure() to build path filters +StringList access_categories = {"delete", "write"}; StringList include_paths, exclude_paths; processConfiguration( - subscriptionContext, + subscription_context, access_categories, - include_paths, - exclude_paths + include_paths, // populated from osquery config + exclude_paths // populated from osquery config ); -// Events arrive automatically via: -// subscriber.Callback(eventContext, subscriptionContext); +// The framework invokes Callback() automatically when events arrive +// subscriber.Callback(event_context, subscription_context); ``` -> **Note:** This header is Windows-only and depends on `ntfs_event_publisher.h`. The subscriber integrates with the osquery event framework β€” direct instantiation is typically handled by the framework, not application code. \ No newline at end of file +> **Note:** This header is Windows-only and depends on `ntfs_event_publisher.h`. The subscriber integrates with osquery's `EventSubscriber` template, so registration and dispatch are handled by the osquery event framework β€” direct instantiation is not required in normal usage. \ No newline at end of file diff --git a/osquery/tables/events/windows/.powershell_events.md b/osquery/tables/events/windows/.powershell_events.md index d53bf51e52f..e50565f8dbe 100644 --- a/osquery/tables/events/windows/.powershell_events.md +++ b/osquery/tables/events/windows/.powershell_events.md @@ -1,53 +1,51 @@ -Header file defining the `PowershellEventSubscriber` class, which subscribes to Windows Event Log events to capture and process PowerShell script block logging activity within the osquery framework. +Defines the `PowershellEventSubscriber` class for capturing and processing PowerShell script block execution events on Windows via the `WindowsEventLogPublisher` event pipeline. ## Key Components ### `PowershellEventSubscriber` -Extends `EventSubscriber` to collect PowerShell execution telemetry from Windows Event Log. +An `EventSubscriber` that subscribes to Windows Event Log entries related to PowerShell script block logging (ETW/EVTX events). -- **`init()`** β€” Registers subscriptions with the Windows Event Log publisher -- **`Callback()`** β€” Handles incoming PowerShell event log entries +- **`init()`** β€” Registers subscriptions with the Windows Event Log publisher. +- **`Callback()`** β€” Invoked per event; routes events into context processing. -### `Context` (nested struct) -Holds stateful tracking data across multi-part script block messages: +### `Context` +Internal state struct tracking in-flight and completed script block events: -| Field | Purpose | +| Member | Purpose | |---|---| | `script_state_map` | Maps script block IDs to their partial message fragments | -| `character_frequency_map` | Frequency analysis vector (entropy/obfuscation detection) | -| `row_list` | Accumulated rows ready for osquery table emission | -| `last_event_expiration_time` | Tracks stale event cleanup threshold | +| `character_frequency_map` | Frequency analysis vector for script content scoring | +| `row_list` | Accumulated rows ready for osquery table output | +| `last_event_expiration_time` | Timestamp for expiring stale incomplete events | | `invalid_event_count` / `expired_event_count` | Diagnostic counters | -### Static Processing Methods +### `ScriptMessage` +Holds a single parsed fragment of a (potentially multi-part) PowerShell script block event, including metadata such as `script_path`, `script_name`, `event_time`, and message ordering fields. -| Method | Purpose | -|---|---| -| `parseScriptMessageEvent()` | Extracts a `ScriptMessage` from a raw ptree event | -| `processEventObject()` | Advances context state with a new event | -| `processEventExpiration()` | Evicts incomplete/stale script block entries | -| `generateRow()` | Assembles a final osquery `Row` from completed message fragments | +### Static Utility Methods + +- **`generateRow()`** β€” Assembles a complete osquery `Row` from a list of `ScriptMessage` fragments and the frequency map. +- **`parseScriptMessageEvent()`** β€” Parses a raw `ptree` event into a `ScriptMessage` optional. +- **`processEventObject()`** β€” Merges a parsed event into the running `Context`. +- **`processEventExpiration()`** β€” Evicts stale/incomplete script block entries from `Context`. ## Usage Example ```cpp -// Subscriber is auto-registered via osquery plugin system. -// Manual interaction with static helpers: +// Subscriber is auto-registered via osquery's event framework. +// Manual interaction example for testing: -boost::optional msg; -PowershellEventSubscriber::parseScriptMessageEvent(msg, event_ptree); +PowershellEventSubscriber subscriber; +subscriber.init(); -if (msg) { - PowershellEventSubscriber::Context ctx; - PowershellEventSubscriber::processEventObject(ctx, event_ptree); - PowershellEventSubscriber::processEventExpiration(ctx); +PowershellEventSubscriber::Context ctx; +boost::property_tree::ptree raw_event; // populated from Windows Event Log - Row row; - PowershellEventSubscriber::generateRow( - row, ctx.script_state_map[msg->script_block_id], ctx.character_frequency_map - ); +auto status = PowershellEventSubscriber::processEventObject(ctx, raw_event); +if (status.ok() && !ctx.row_list.empty()) { + for (const auto& row : ctx.row_list) { + // emit row to osquery table results + } } -``` - -> **Note:** PowerShell script block logging must be enabled via Windows Group Policy (`ScriptBlockLogging`) for events to be captured. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/tables/events/windows/.windows_events.md b/osquery/tables/events/windows/.windows_events.md index 7881c43a704..6f50303fe79 100644 --- a/osquery/tables/events/windows/.windows_events.md +++ b/osquery/tables/events/windows/.windows_events.md @@ -1,41 +1,25 @@ -Brief header defining the `WindowsEventSubscriber` class, which subscribes to Windows Event Log events and converts them into osquery table rows. +Defines the `WindowsEventSubscriber` class, an osquery event subscriber that listens to Windows Event Log events and maps them into queryable table rows. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `WindowsEventSubscriber` | Class | Event subscriber for the Windows Event Log publisher | -| `init()` | Method | Initializes the subscriber and registers subscriptions | -| `Callback()` | Method | Handles incoming Windows Event Log events | -| `generateRow()` | Static Method | Converts a `WELEvent` into an osquery `Row` for table output | +- **`WindowsEventSubscriber`** β€” Inherits from `EventSubscriber`; subscribes to Windows Event Log publisher events. +- **`init()`** β€” Initializes the subscriber and registers subscriptions with the publisher. +- **`Callback()`** β€” Invoked per event; receives an event reference (`ECRef`) and subscription context (`SCRef`) for processing. +- **`generateRow()`** β€” Static utility that populates an osquery `Row` from a `WELEvent` struct, transforming raw Windows event data into a table-compatible format. ## Usage Example -```c +```cpp // Subscriber is auto-registered via osquery's event framework. -// Implement callback to process incoming Windows events: +// generateRow can be used independently to convert a WELEvent: -Status WindowsEventSubscriber::Callback(const ECRef& event, - const SCRef& subscription) { - Row row; - WindowsEventSubscriber::generateRow(row, event->windows_event); - add(row); // Adds the row to the osquery results table - return Status::success(); -} +Row row; +WELEvent wel_event = /* populated by WindowsEventLogParser */; +WindowsEventSubscriber::generateRow(row, wel_event); -// generateRow populates an osquery Row from a parsed WELEvent: -void WindowsEventSubscriber::generateRow(Row& row, - const WELEvent& windows_event) { - row["channel"] = windows_event.channel; - row["datetime"] = windows_event.datetime; - row["eventid"] = BIGINT(windows_event.eventId); - // ... additional fields mapped from WELEvent -} +// row now contains mapped fields (e.g., event_id, source, message) +// ready for osquery table emission. ``` -## Dependencies - -- **`WindowsEventLogPublisher`** β€” the publisher this subscriber binds to via the `EventSubscriber` template -- **`WindowsEventLogParser`** β€” provides the `WELEvent` struct populated before `Callback` is invoked -- **`boost::property_tree`** β€” available as namespace alias `pt` for XML/JSON event data parsing \ No newline at end of file +> **Note:** Direct instantiation is not typical. osquery's event subsystem handles lifecycle management β€” `init()` and `Callback()` are invoked automatically when the subscriber is registered and events are published by `WindowsEventLogPublisher`. \ No newline at end of file diff --git a/osquery/tables/networking/linux/.iptc_proxy.md b/osquery/tables/networking/linux/.iptc_proxy.md index 5d0148eb1cd..dd73dcca7cd 100644 --- a/osquery/tables/networking/linux/.iptc_proxy.md +++ b/osquery/tables/networking/linux/.iptc_proxy.md @@ -1,5 +1,5 @@ -A C header providing an abstraction layer over `libiptc` for querying Linux netfilter/iptables rules and chains from osquery table implementations. +Proxy header for accessing Linux `iptables` firewall rules via `libiptc`, providing an opaque handle-based API to enumerate chains and rules from a given filter table. ## Key Components @@ -7,27 +7,31 @@ A C header providing an abstraction layer over `libiptc` for querying Linux netf | Type | Description | |------|-------------| -| `iptcproxy_handle` | Opaque handle for an iptables filter context | -| `iptcproxy_chain` | Represents a chain with its name, policy, and packet/byte counters | -| `iptcproxy_rule` | Represents a firewall rule with target, port match data, and full IP header fields | +| `iptcproxy_handle` | Opaque handle representing an open connection to an iptables filter table | +| `iptcproxy_chain` | Chain name, default policy string, and packet/byte counters | +| `iptcproxy_rule` | Rule entry with target, optional port match data, and full IP header criteria | ### `invflags` Bitmask Constants -| Constant | Value | Description | -|----------|-------|-------------| -| `IPTC_INV_VIA_IN` | `0x01` | Invert inbound interface match | -| `IPTC_INV_VIA_OUT` | `0x02` | Invert outbound interface match | +| Constant | Value | Meaning | +|----------|-------|---------| +| `IPTC_INV_VIA_IN` | `0x01` | Invert input interface match | +| `IPTC_INV_VIA_OUT` | `0x02` | Invert output interface match | | `IPTC_INV_SRCIP` | `0x08` | Invert source IP match | | `IPTC_INV_DSTIP` | `0x10` | Invert destination IP match | | `IPTC_INV_PROTO` | `0x40` | Invert protocol match | -| `IPTC_INV_MASK` | `0x7F` | All flags mask | +| `IPTC_INV_MASK` | `0x7F` | All inversion flags combined | ### Functions -- **`iptcproxy_init(filter)`** β€” Opens a handle for the given iptables table (e.g. `"filter"`) -- **`iptcproxy_free(handle)`** β€” Releases resources for an open handle -- **`iptcproxy_first_chain / iptcproxy_next_chain`** β€” Iterates chains in the table -- **`iptcproxy_first_rule / iptcproxy_next_rule`** β€” Iterates rules within a named chain +| Function | Description | +|----------|-------------| +| `iptcproxy_init(filter)` | Open a handle to the named table (e.g. `"filter"`, `"nat"`) | +| `iptcproxy_free(handle)` | Release all resources associated with a handle | +| `iptcproxy_first_chain(handle)` | Return the first chain in the table | +| `iptcproxy_next_chain(handle)` | Advance to the next chain | +| `iptcproxy_first_rule(chain, handle)` | Return the first rule in the named chain | +| `iptcproxy_next_rule(handle)` | Advance to the next rule | ## Usage Example @@ -35,34 +39,35 @@ A C header providing an abstraction layer over `libiptc` for querying Linux netf #include "iptc_proxy.h" #include -void enumerate_rules(void) { +void dump_rules(void) { const iptcproxy_handle* h = iptcproxy_init("filter"); if (!h) return; const iptcproxy_chain* chain = iptcproxy_first_chain(h); while (chain) { - printf("Chain: %s Policy: %s Packets: %llu\n", + printf("Chain: %s Policy: %s Packets: %llu Bytes: %llu\n", chain->chain, chain->policy, - (unsigned long long)chain->policy_data.pcnt); + (unsigned long long)chain->policy_data.pcnt, + (unsigned long long)chain->policy_data.bcnt); const iptcproxy_rule* rule = iptcproxy_first_rule(chain->chain, h); while (rule) { - printf(" Target: %s Proto: %u\n", + printf(" -> target: %s proto: %u invflags: 0x%02x\n", rule->target, - rule->ip_data.proto); - /* Check inverted source IP flag */ - if (rule->ip_data.invflags & IPTC_INV_SRCIP) { - printf(" (source IP is inverted)\n"); + rule->ip_data.proto, + rule->ip_data.invflags); + + if (rule->match && rule->match_data.valid) { + printf(" sport: %u-%u dport: %u-%u\n", + rule->match_data.spts[0], rule->match_data.spts[1], + rule->match_data.dpts[0], rule->match_data.dpts[1]); } rule = iptcproxy_next_rule(h); } - chain = iptcproxy_next_chain(h); } iptcproxy_free(h); } -``` - -> **Note:** Iteration is stateful per handle. Always call `iptcproxy_first_chain` or `iptcproxy_first_rule` before the corresponding `_next_*` calls. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/tables/networking/posix/.interfaces.md b/osquery/tables/networking/posix/.interfaces.md index c51cb5040e5..5788011a2b2 100644 --- a/osquery/tables/networking/posix/.interfaces.md +++ b/osquery/tables/networking/posix/.interfaces.md @@ -3,7 +3,7 @@ Declares the `genInterfaceDetails` table generation function for querying networ ## Key Components -- **`genInterfaceDetails(QueryContext& context)`** β€” Generates rows of network interface data (e.g., name, MAC address, IP configuration) for the `interface_details` virtual table. +- **`genInterfaceDetails(QueryContext& context)`** β€” Generates rows of network interface metadata (e.g., name, MAC address, IP, flags) for the `interface_details` virtual table. ## Usage Example @@ -15,7 +15,7 @@ namespace tables { QueryData genInterfaceDetails(QueryContext& context) { QueryData results; - // Enumerate network interfaces and populate rows + // Populate rows with network interface data Row r; r["interface"] = "eth0"; r["mac"] = "00:1a:2b:3c:4d:5e"; @@ -29,6 +29,6 @@ QueryData genInterfaceDetails(QueryContext& context) { ## Notes -- Depends on `osquery/core/core.h` and `osquery/core/tables.h`. -- Platform-specific implementations (Linux, macOS, Windows) are expected in corresponding `.cpp` source files. -- Registered as a virtual SQL table via osquery's table plugin system, queryable as `SELECT * FROM interface_details`. \ No newline at end of file +- Depends on `osquery/core/core.h` and `osquery/core/tables.h` for core types and `QueryData`/`QueryContext`. +- Implementation is platform-specific (Linux, macOS, Windows) and lives in a corresponding `.cpp` file. +- Registered as a virtual SQL table, queryable via `SELECT * FROM interface_details;`. \ No newline at end of file diff --git a/osquery/tables/networking/posix/.utils.md b/osquery/tables/networking/posix/.utils.md index 7586a4c1ead..20a33e96432 100644 --- a/osquery/tables/networking/posix/.utils.md +++ b/osquery/tables/networking/posix/.utils.md @@ -1,14 +1,14 @@ -Utility header for network address conversion functions used in osquery's network interface tables. Provides cross-platform helpers to convert raw socket/IP structures into human-readable strings. +Utility header for network interface address formatting within the `osquery::tables` namespace. ## Key Components -- **`AF_INTERFACE` macro** β€” Platform alias for interface address family (`AF_PACKET` on Linux, `AF_LINK` on BSD/macOS) -- **`ipAsString(sockaddr*)`** β€” Converts an IPv4/IPv6 `sockaddr` struct to a string -- **`ipAsString(in_addr*)`** β€” Converts an IPv4 `in_addr` struct to a string -- **`macAsString(ifaddrs*)`** β€” Extracts and formats a MAC address from an `ifaddrs` struct -- **`macAsString(char*)`** β€” Formats a MAC address from a raw byte buffer -- **`netmaskFromIP(sockaddr*)`** β€” Returns the CIDR prefix length from a `sockaddr` netmask +- **`AF_INTERFACE`** β€” Cross-platform macro aliasing `AF_PACKET` (Linux) or `AF_LINK` (BSD/macOS) for interface-level address family detection. +- **`ipAsString(const struct sockaddr *)`** β€” Converts an IPv4 or IPv6 `sockaddr` struct to a human-readable string. +- **`ipAsString(const struct in_addr *)`** β€” Overload for converting an `in_addr` struct directly to a string. +- **`macAsString(const struct ifaddrs *)`** β€” Extracts and formats a MAC address from an `ifaddrs` entry. +- **`macAsString(const char *)`** β€” Overload for formatting a raw MAC address byte buffer. +- **`netmaskFromIP(const struct sockaddr *)`** β€” Derives the CIDR prefix length (netmask bit count) from a `sockaddr`. ## Usage Example @@ -17,21 +17,24 @@ Utility header for network address conversion functions used in osquery's networ #include struct ifaddrs *ifap; -if (getifaddrs(&ifap) == 0) { - for (struct ifaddrs *ifa = ifap; ifa != nullptr; ifa = ifa->ifa_next) { - if (ifa->ifa_addr == nullptr) continue; +getifaddrs(&ifap); - // Get IP address string - std::string ip = osquery::tables::ipAsString(ifa->ifa_addr); +for (struct ifaddrs *ifa = ifap; ifa != nullptr; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == nullptr) continue; - // Get subnet prefix length - int prefix = osquery::tables::netmaskFromIP(ifa->ifa_netmask); + if (ifa->ifa_addr->sa_family == AF_INET) { + // Print IPv4 address and its netmask prefix length + std::string ip = osquery::tables::ipAsString(ifa->ifa_addr); + int prefix = osquery::tables::netmaskFromIP(ifa->ifa_netmask); + printf("%s/%d\n", ip.c_str(), prefix); + } - // Get MAC address (on link-layer entries) - if (ifa->ifa_addr->sa_family == AF_INTERFACE) { - std::string mac = osquery::tables::macAsString(ifa); - } + if (ifa->ifa_addr->sa_family == AF_INTERFACE) { + // Print MAC address + std::string mac = osquery::tables::macAsString(ifa); + printf("MAC: %s\n", mac.c_str()); } - freeifaddrs(ifap); } + +freeifaddrs(ifap); ``` \ No newline at end of file diff --git a/osquery/tables/networking/windows/.interfaces.md b/osquery/tables/networking/windows/.interfaces.md index 666822b762d..7e069371d4f 100644 --- a/osquery/tables/networking/windows/.interfaces.md +++ b/osquery/tables/networking/windows/.interfaces.md @@ -1,25 +1,19 @@ -Declares the `genInterfaceDetails` function for querying network interface details within the osquery tables framework. +Declares the `genInterfaceDetails` function for querying network interface details as part of osquery's tables subsystem. ## Key Components -- **`genInterfaceDetails(QueryContext& context)`** β€” Table generator function that retrieves detailed network interface information, returning a `QueryData` result set for use in the `interfaces` virtual table. +- **`genInterfaceDetails(QueryContext&)`** β€” Table generator function that returns network interface details as `QueryData`. Registered under the `osquery::tables` namespace. ## Usage Example ```c #include "interfaces.h" -namespace osquery { -namespace tables { +osquery::QueryContext context; +osquery::QueryData results = osquery::tables::genInterfaceDetails(context); -QueryData genInterfaceDetails(QueryContext& context) { - QueryData results; - // Populate rows with network interface details - // (IP addresses, MAC, MTU, flags, etc.) - return results; +for (const auto& row : results) { + // Access interface fields, e.g. row.at("interface"), row.at("address") } - -} // namespace tables -} // namespace osquery ``` \ No newline at end of file diff --git a/osquery/tables/networking/windows/.windows_firewall_rules.md b/osquery/tables/networking/windows/.windows_firewall_rules.md index f8aedf7a821..12adc7a7a47 100644 --- a/osquery/tables/networking/windows/.windows_firewall_rules.md +++ b/osquery/tables/networking/windows/.windows_firewall_rules.md @@ -1,46 +1,44 @@ -Defines types and interfaces for querying Windows Firewall rules within the osquery `tables` namespace, providing structured representations of firewall rule data and error states. +Defines data structures and interfaces for querying Windows Firewall rules within the osquery tables subsystem on Windows platforms. ## Key Components ### `WindowsFirewallError` (enum class) -Enumerates all possible failure points when retrieving firewall rules via the Windows Firewall API (`netfw.h`), from policy access errors through individual rule property read failures. +Enumerates all possible failure points when interacting with the Windows Firewall COM API (`INetFwPolicy`, `INetFwRules`, `IEnumVARIANT`), covering policy access, rule enumeration, and individual rule property retrieval errors. ### `WindowsFirewallRule` (struct) -Represents a single Windows Firewall rule with the following fields: +Represents a single parsed firewall rule with the following fields: -| Field | Type | Default | +| Field | Type | Description | |---|---|---| -| `name` | `std::string` | β€” | -| `appName` | `std::string` | β€” | -| `action` | `NET_FW_ACTION` | `NET_FW_ACTION_BLOCK` | -| `enabled` | `bool` | `false` | -| `grouping` | `std::string` | β€” | -| `direction` | `NET_FW_RULE_DIRECTION` | `NET_FW_RULE_DIR_IN` | -| `protocol` | `long` | `0` | -| `localAddresses` / `remoteAddresses` | `std::string` | β€” | -| `localPorts` / `remotePorts` | `std::string` | β€” | -| `icmpTypesCodes` | `std::string` | β€” | -| `profileBitmask` | `long` | `0` | -| `serviceName` | `std::string` | β€” | - -### `WindowsFirewallRules` (type alias) -```cpp -using WindowsFirewallRules = std::vector; -``` +| `name` | `std::string` | Rule display name | +| `appName` | `std::string` | Associated application path | +| `action` | `NET_FW_ACTION` | Allow or block (default: block) | +| `enabled` | `bool` | Whether the rule is active | +| `direction` | `NET_FW_RULE_DIRECTION` | Inbound or outbound (default: inbound) | +| `protocol` | `long` | Protocol number (e.g. TCP/UDP) | +| `localAddresses` / `remoteAddresses` | `std::string` | Address scope | +| `localPorts` / `remotePorts` | `std::string` | Port ranges | +| `icmpTypesCodes` | `std::string` | ICMP type/code filter | +| `profileBitmask` | `long` | Domain/private/public profile flags | +| `serviceName` | `std::string` | Associated Windows service | + +### `WindowsFirewallRules` +Type alias for `std::vector`. ### `renderWindowsFirewallRules()` -Converts a collection of `WindowsFirewallRule` structs into osquery `QueryData` for table output. +Converts a populated `WindowsFirewallRules` collection into osquery `QueryData` for table output. ## Usage Example ```cpp #include "windows_firewall_rules.h" -WindowsFirewallRules rules; +// After populating rules via COM enumeration: +osquery::tables::WindowsFirewallRules rules; -WindowsFirewallRule rule; -rule.name = "Block Inbound Telnet"; +osquery::tables::WindowsFirewallRule rule; +rule.name = "Block Telnet"; rule.action = NET_FW_ACTION_BLOCK; rule.enabled = true; rule.direction = NET_FW_RULE_DIR_IN; @@ -48,5 +46,7 @@ rule.protocol = 6; // TCP rule.localPorts = "23"; rules.push_back(rule); -QueryData result = renderWindowsFirewallRules(rules); +// Render into osquery rows +osquery::QueryData result = + osquery::tables::renderWindowsFirewallRules(rules); ``` \ No newline at end of file diff --git a/osquery/tables/system/.system_utils.md b/osquery/tables/system/.system_utils.md index c881c3304aa..4116ec0b8a4 100644 --- a/osquery/tables/system/.system_utils.md +++ b/osquery/tables/system/.system_utils.md @@ -4,44 +4,30 @@ Utility header providing context-aware helper functions for resolving users and ## Key Components | Function | Description | -|---|---| -| `usersFromContext` | Resolves users from a query context; defaults to current user if none specified | -| `pidsFromContext` | Resolves process IDs from a query context; returns all PIDs by default | +|----------|-------------| +| `usersFromContext()` | Resolves a list of users from a query context; defaults to the current user if none specified | +| `pidsFromContext()` | Resolves a list of process IDs from a query context; returns all PIDs by default | Both functions return a `QueryData` set of rows suitable for direct use in table implementations. ## Usage Example ```c -#include +#include "system_utils.h" namespace osquery { namespace tables { -QueryData genUserFiles(QueryContext& context) { +QueryData genMyTable(QueryContext& context) { QueryData results; - // Resolve users from context (or current user if none provided) + // Resolve users from context (or current user if none specified) QueryData users = usersFromContext(context); - for (const auto& user : users) { - Row r; - r["username"] = user.at("username"); - results.push_back(r); - } - return results; -} + // Resolve all running PIDs regardless of context constraints + QueryData pids = pidsFromContext(context, true); -QueryData genProcessInfo(QueryContext& context) { - // Returns all PIDs by default, or filtered PIDs if context specifies - QueryData pids = pidsFromContext(context); - - QueryData results; - for (const auto& pid : pids) { - Row r; - r["pid"] = pid.at("pid"); - results.push_back(r); - } + // Iterate and populate results... return results; } @@ -51,6 +37,6 @@ QueryData genProcessInfo(QueryContext& context) { ## Notes -- `usersFromContext` defaults `all = false` β€” returns only the current user when no user constraint is present in the query. -- `pidsFromContext` defaults `all = true` β€” enumerates all running processes unless a specific PID constraint is provided. -- Both helpers simplify writing constraint-aware table generators by abstracting context parsing logic. \ No newline at end of file +- `usersFromContext()` defaults `all = false` β€” returns only the current user when no user constraint is present in the context. +- `pidsFromContext()` defaults `all = true` β€” returns all PIDs unless explicitly constrained. +- Both functions accept an optional boolean to override context filtering, useful when a table must enumerate all entities regardless of query predicates. \ No newline at end of file diff --git a/osquery/tables/system/darwin/.asl_utils.md b/osquery/tables/system/darwin/.asl_utils.md index 3bd518f6117..3c1baee63de 100644 --- a/osquery/tables/system/darwin/.asl_utils.md +++ b/osquery/tables/system/darwin/.asl_utils.md @@ -3,37 +3,53 @@ Utility header for bridging osquery query constraints with Apple System Log (ASL ## Key Components -### Compatibility Macros -- **`OLD_ASL_API`** β€” Defined when `ASL_API_VERSION` is absent; enables inline wrappers (`asl_release`, `asl_next`) that map deprecated ASL API calls to their modern equivalents +### Compatibility Shims (`OLD_ASL_API`) +Inline wrappers defined when `ASL_API_VERSION` is absent, normalizing the older ASL API surface: +- `asl_release(aslmsg)` β€” wraps `asl_free()` +- `asl_release(aslresponse)` β€” wraps `aslresponse_free()` +- `asl_next(aslresponse)` β€” wraps `aslresponse_next()` ### Functions | Function | Description | |---|---| -| `addQueryOp()` | Appends a single constraint operation (ANDed) to an ASL query, converting osquery `ConstraintOperator` and `ColumnType` to ASL equivalents | +| `addQueryOp()` | Appends a single constraint operation (ANDed) to an ASL query message, converting osquery `ConstraintOperator` and `ColumnType` to ASL equivalents | | `createAslQuery()` | Builds a complete `aslmsg` query object from an osquery `QueryContext` | -| `readAslRow()` | Reads fields from an ASL message row and populates an osquery `Row` | -| `convertLikeRegex()` | Translates a SQL `LIKE`-style pattern string into a regex for ASL matching | +| `readAslRow()` | Reads fields from an `aslmsg` row into an osquery `Row` map | +| `convertLikeRegex()` | Translates SQL `LIKE` pattern syntax into a regex string for ASL matching | ## Usage Example -```c -// Build an ASL query from an osquery context and iterate results -aslmsg query = createAslQuery(context); +```cpp +#include "asl_utils.h" -asl_object_t client = asl_open(nullptr, nullptr, 0); -aslresponse response = asl_search(client, query); +namespace osquery { +namespace tables { -aslmsg row; -while ((row = asl_next(response)) != nullptr) { +QueryData genAslLogs(QueryContext& context) { + QueryData results; + + // Build an ASL query from the incoming osquery constraints + aslmsg query = createAslQuery(context); + + aslclient client = asl_open(nullptr, nullptr, 0); + aslresponse resp = asl_search(client, query); + asl_release(query); + + aslmsg row; + while ((row = asl_next(resp)) != nullptr) { Row r; - readAslRow(row, r); + readAslRow(row, r); // Populate osquery Row from ASL message results.push_back(r); + } + + asl_release(resp); + asl_close(client); + return results; } -asl_release(response); -asl_release(query); -asl_close(client); +} // namespace tables +} // namespace osquery ``` -> **Note:** This header is macOS-only and depends on ``. The `OLD_ASL_API` compatibility shim handles pre-10.12 macOS SDK differences where `asl_free`/`aslresponse_free` were used instead of the unified `asl_release`. \ No newline at end of file +> **Note:** ASL was deprecated in macOS 10.12. On newer SDK versions, `ASL_API_VERSION` is defined and the compatibility shims are omitted automatically. \ No newline at end of file diff --git a/osquery/tables/system/linux/.apt_sources.md b/osquery/tables/system/linux/.apt_sources.md index 190773b5b21..81c60fd365a 100644 --- a/osquery/tables/system/linux/.apt_sources.md +++ b/osquery/tables/system/linux/.apt_sources.md @@ -1,41 +1,38 @@ -Defines data structures and parsing utilities for APT (Advanced Package Tool) source list entries used by the `apt_sources` osquery table implementation. +Defines data structures and parsing utilities for APT (Advanced Package Tool) source list entries within the osquery tables framework. ## Key Components ### `AptSource` struct Represents a single APT repository source with the following fields: - -| Field | Type | Description | -|---|---|---| -| `base_uri` | `std::string` | Repository base URI (e.g., `http://archive.ubuntu.com/ubuntu`) | -| `name` | `std::string` | Distribution/suite name | -| `cache_file` | `std::vector` | Components used to construct the local cache filename | +- `base_uri` β€” The repository base URI (e.g., `http://archive.ubuntu.com/ubuntu`) +- `name` β€” The distribution/suite name (e.g., `focal`, `focal-updates`) +- `cache_file` β€” Components used to construct the local APT cache filename ### Functions -- **`parseAptSourceLine`** β€” Parses a single line from a traditional `sources.list` format into an `AptSource` struct -- **`parseDeb822Block`** β€” Parses a multi-line [Deb822-format](https://wiki.debian.org/SourcesList#Deb822_format) stanza block into one or more `AptSource` entries -- **`getCacheFilename`** β€” Reconstructs the APT cache filename from the `cache_file` components vector +| Function | Description | +|---|---| +| `parseAptSourceLine()` | Parses a single line from a traditional `sources.list` format into an `AptSource` struct | +| `parseDeb822Block()` | Parses a multi-line [deb822](https://manpages.debian.org/stretch/apt/sources.list.5.en.html) format block into a vector of `AptSource` entries | +| `getCacheFilename()` | Reconstructs the APT cache filename from the `cache_file` components vector | ## Usage Example -```cpp -#include "apt_sources.h" - +```c osquery::tables::AptSource source; -// Parse a classic one-line sources.list entry +// Parse a single sources.list line std::string line = "deb http://archive.ubuntu.com/ubuntu focal main restricted"; -Status s = osquery::tables::parseAptSourceLine(line, source); +Status status = osquery::tables::parseAptSourceLine(line, source); -if (s.ok()) { - std::string cache = osquery::tables::getCacheFilename(source.cache_file); - // cache -> "archive.ubuntu.com_ubuntu_dists_focal_main_binary-amd64_Packages" +if (status.ok()) { + std::string cacheFile = osquery::tables::getCacheFilename(source.cache_file); + // cacheFile -> "archive.ubuntu.com_ubuntu_dists_focal_main_..." } -// Parse a Deb822-format block +// Parse a deb822 format block std::vector sources; -std::string block = "Types: deb\nURIs: http://archive.ubuntu.com/ubuntu\nSuites: focal\nComponents: main"; -Status s2 = osquery::tables::parseDeb822Block(block, sources); +std::string block = "Types: deb\nURIs: http://archive.ubuntu.com/ubuntu\nSuites: focal\n"; +Status s = osquery::tables::parseDeb822Block(block, sources); ``` \ No newline at end of file diff --git a/osquery/tables/system/linux/.md_tables.md b/osquery/tables/system/linux/.md_tables.md index c48dff397b1..d7b3836bcfa 100644 --- a/osquery/tables/system/linux/.md_tables.md +++ b/osquery/tables/system/linux/.md_tables.md @@ -1,5 +1,5 @@ -Header file defining data structures and interfaces for querying Linux MD (Multiple Device) RAID array information within the osquery tables framework. +Header file defining data structures and interfaces for querying Linux MD (Multiple Device) software RAID arrays within the osquery tables framework. ## Key Components @@ -10,45 +10,42 @@ Header file defining data structures and interfaces for querying Linux MD (Multi | `MDDrive` | Represents a single drive in an MD array (name, position) | | `MDAction` | Tracks ongoing array operations (progress, finish time, speed) | | `MDBitmap` | Holds bitmap metadata (memory usage, chunk size, external file path) | -| `MDDevice` | Full MD array descriptor including drives, status, RAID level, and active actions | +| `MDDevice` | Full MD array device descriptor including drives, status, RAID level, and active operations | | `MDStat` | Top-level container parsed from `/proc/mdstat` (personalities, devices, unused line) | ### Classes -- **`MDInterface`** β€” Abstract base class defining the contract for MD driver interaction. All methods are pure virtual, enabling dependency injection and testability. -- **`MD`** β€” Concrete implementation of `MDInterface` that performs actual ioctl-based system calls and file parsing. +**`MDInterface`** β€” Abstract base class defining the contract for MD driver interactions: +- `getDiskInfo()` β€” Retrieves per-disk info via ioctl +- `getArrayInfo()` β€” Retrieves array-level info via ioctl +- `parseMDStat()` β€” Parses `/proc/mdstat` text into `MDStat` +- `getPathByDevName()` β€” Resolves device path from short name (e.g., `md0`) +- `getDevName()` β€” Resolves device name from major/minor numbers +- `getSuperblkVersion()` β€” Returns the array's superblock version -### Key Methods - -| Method | Description | -|--------|-------------| -| `getDiskInfo()` | Retrieves per-disk info via MD ioctl for a named array | -| `getArrayInfo()` | Retrieves array-level info via MD ioctl | -| `parseMDStat()` | Parses `/proc/mdstat` lines into an `MDStat` struct | -| `getPathByDevName()` | Resolves device path from short name (e.g., `md0`) | -| `getDevName()` | Resolves device name from major/minor numbers | -| `getSuperblkVersion()` | Returns the superblock version of an array | +**`MD`** β€” Concrete implementation of `MDInterface` using Linux MD driver syscalls. ### Free Function -- **`getDrivesForArray()`** β€” Populates `QueryData` with all drive information for a given array, using an `MDInterface` instance. +`getDrivesForArray()` β€” Populates `QueryData` with all drive details for a given MD array, used directly by osquery table generation. ## Usage Example -```c -// Instantiate the concrete MD implementation +```cpp +#include "md_tables.h" + osquery::tables::MD md; -osquery::QueryData results; +osquery::tables::MDStat stat; -// Parse /proc/mdstat content +// Parse /proc/mdstat lines into structured data std::vector lines = { /* mdstat file lines */ }; -osquery::tables::MDStat stat; md.parseMDStat(lines, stat); -// Query drives for a specific array +// Query drive info for a specific array +osquery::QueryData results; osquery::tables::getDrivesForArray("md0", md, results); -// Retrieve array-level info via ioctl +// Get array-level info via ioctl mdu_array_info_t arrayInfo; bool ok = md.getArrayInfo("md0", arrayInfo); ``` \ No newline at end of file diff --git a/osquery/tables/system/linux/.pci_devices.md b/osquery/tables/system/linux/.pci_devices.md index a2eb32b7272..d314f44afb3 100644 --- a/osquery/tables/system/linux/.pci_devices.md +++ b/osquery/tables/system/linux/.pci_devices.md @@ -1,54 +1,52 @@ -Defines data structures and a database class for parsing and querying PCI device information from the system `pci.ids` database file. +Defines data structures and a database class for parsing and querying PCI device information from the system `pci.ids` database, used to enrich osquery PCI device table rows with vendor, model, and subsystem descriptions. ## Key Components ### Structs -- **`PciModel`** β€” Holds a PCI device model's ID, description, and a map of subsystem info keyed by `" "` -- **`PciVendor`** β€” Holds a vendor's ID, name, and a map of `PciModel` entries keyed by device model ID +| Struct | Description | +|--------|-------------| +| `PciModel` | Holds a PCI device model ID, description, and a subsystem info map keyed by `" "` | +| `PciVendor` | Holds a vendor ID, name, and a map of `PciModel` instances keyed by device/model ID | ### Class: `PciDB` -Parses a `pci.ids` filestream and exposes lookup methods: +Parses a `pci.ids` file stream and exposes lookup methods: | Method | Description | -|---|---| -| `PciDB(istream&)` | Constructor β€” parses the `pci.ids` stream into an internal map | -| `getVendorName()` | Looks up vendor name by vendor ID | -| `getModel()` | Looks up model description by vendor and model ID | -| `getSubsystemInfo()` | Looks up subsystem description by vendor, model, and subsystem IDs | +|--------|-------------| +| `PciDB(std::istream&)` | Constructor β€” parses the `pci.ids` filestream into the internal `db_` map | +| `getVendorName(vendor_id, vendor)` | Populates vendor name string by vendor ID | +| `getModel(vendor_id, model_id, model)` | Populates model description by vendor + model ID | +| `getSubsystemInfo(vendor_id, model_id, subsystem_vendor_id, subsystem_device_id, subsystem)` | Populates subsystem description using all four IDs | -Private parsing helpers (`parseLine`, `parseVendor`, `parseModel`, `parseSubsystem`) drive the line-by-line ingestion of the database file. +Private parsing methods (`parseLine`, `parseVendor`, `parseModel`, `parseSubsystem`) handle the line-by-line `pci.ids` format internally. ### Free Functions -- **`extractVendorModelFromPciDBIfPresent()`** β€” Populates an osquery `Row` with vendor/model info from sysfs attributes using a `PciDB` instance -- **`extractPCIClassIDAttrs()`** β€” Populates an osquery `Row` with PCI class identifier data from a sysfs attribute string +| Function | Description | +|----------|-------------| +| `extractVendorModelFromPciDBIfPresent` | Enriches an osquery `Row` with vendor/model info from sysfs device and subsystem ID attributes | +| `extractPCIClassIDAttrs` | Enriches an osquery `Row` with PCI class identifier data from a sysfs class attribute string | ## Usage Example ```cpp -#include "pci_devices.h" #include +#include "pci_devices.h" -// Load and query the PCI database -std::ifstream pci_ids("/usr/share/misc/pci.ids"); -osquery::tables::PciDB pcidb(pci_ids); - -std::string vendor_name, model_desc; +std::ifstream pci_ids_file("/usr/share/misc/pci.ids"); +osquery::tables::PciDB pcidb(pci_ids_file); -// Lookup vendor +std::string vendor_name; auto status = pcidb.getVendorName("8086", vendor_name); if (status.ok()) { // vendor_name == "Intel Corporation" } -// Lookup model -status = pcidb.getModel("8086", "1234", model_desc); - -// Populate an osquery row from sysfs +// Enrich an osquery row from sysfs attributes osquery::Row row; osquery::tables::extractVendorModelFromPciDBIfPresent( - row, "0x80861234", "0x80865678", pcidb); + row, "8086:1234", "8086:5678", pcidb); ``` \ No newline at end of file diff --git a/osquery/tables/system/linux/.processes.md b/osquery/tables/system/linux/.processes.md index ba08b30411a..b6dc4fef3f2 100644 --- a/osquery/tables/system/linux/.processes.md +++ b/osquery/tables/system/linux/.processes.md @@ -1,22 +1,22 @@ -Declares a utility function for parsing Linux process cgroup information within the osquery tables subsystem. +Declares a utility function for parsing Linux process cgroup information within the osquery tables namespace. ## Key Components -- **`parseProcCGroup(const std::string& content)`** β€” Parses the raw string content of a Linux `/proc//cgroup` file and returns the extracted cgroup information as a formatted string. +- **`parseProcCGroup(const std::string& content)`** β€” Parses the raw string content of a process's cgroup file (e.g., `/proc//cgroup`) and returns a normalized string representation. ## Usage Example ```c #include "processes.h" -std::string cgroupContent = "12:cpu:/user.slice\n11:memory:/system.slice\n"; +std::string cgroupContent = "12:cpu:/user.slice\n11:memory:/system.slice"; std::string parsed = osquery::tables::parseProcCGroup(cgroupContent); -// parsed contains the processed cgroup hierarchy data +// parsed contains the normalized cgroup information ``` ## Notes -- Lives in the `osquery::tables` namespace, consistent with osquery's table-based query architecture. -- Intended for use in Linux process enumeration tables (e.g., `processes`). -- Input is typically the raw file content read from `/proc//cgroup`. \ No newline at end of file +- Resides in the `osquery::tables` namespace +- Typically used internally by osquery's process-related table implementations on Linux +- Input is expected to be the raw text content read from the `/proc` filesystem cgroup entries \ No newline at end of file diff --git a/osquery/tables/system/linux/dbus/.uniquedbusconnection.md b/osquery/tables/system/linux/dbus/.uniquedbusconnection.md index 40514efbe3f..e85bedaab72 100644 --- a/osquery/tables/system/linux/dbus/.uniquedbusconnection.md +++ b/osquery/tables/system/linux/dbus/.uniquedbusconnection.md @@ -1,19 +1,12 @@ -RAII wrapper for managing D-Bus connection lifetimes in osquery's Linux system tables, ensuring connections are automatically allocated and released. +RAII wrapper for managing D-Bus connection lifetimes in osquery's Linux system tables, ensuring connections are automatically allocated and deallocated. ## Key Components -### `UniqueDbusConnectionAllocator` -A policy class that implements allocation and deallocation strategies for `DBusConnection*` resources. - -| Member | Description | -|---|---| -| `ResourceType` | Alias for `DBusConnection*` | -| `allocate(connection, system)` | Opens a D-Bus connection; `system=true` connects to the system bus, `false` to the session bus | -| `deallocate(connection)` | Closes and unreferences the D-Bus connection | - -### `UniqueDbusConnection` -A type alias combining `UniqueDbusConnectionAllocator` with `UniqueResource<>`, providing automatic RAII lifecycle management for D-Bus connections. +- **`UniqueDbusConnectionAllocator`** β€” Allocator policy class that implements connection lifecycle management: + - `allocate(ResourceType& connection, bool system)` β€” Opens a D-Bus connection; the `bool` parameter selects between the system bus (`true`) and session bus (`false`) + - `deallocate(ResourceType& connection)` β€” Closes and cleans up the `DBusConnection*` +- **`UniqueDbusConnection`** β€” Type alias combining `UniqueResource<>` with the allocator, providing automatic scope-based connection management ## Usage Example @@ -22,21 +15,20 @@ A type alias combining `UniqueDbusConnectionAllocator` with `UniqueResource<>`, namespace osquery { -Status queryDbusSystemBus() { - // Allocate a system bus connection (auto-deallocated on scope exit) +TableRows getDbusData() { + // Acquire a system bus connection; automatically released when out of scope UniqueDbusConnection connection; - // true = system bus, false = session bus - auto status = connection.allocate(true); + auto status = connection.open(/* system= */ true); if (!status.ok()) { - return status; + return {}; } DBusConnection* raw = connection.get(); - // Use raw D-Bus connection for message passing... + // Use raw connection for D-Bus calls... - return Status::success(); - // connection automatically deallocated here + // No manual cleanup needed β€” deallocate() is called automatically + return rows; } } // namespace osquery @@ -44,6 +36,6 @@ Status queryDbusSystemBus() { ## Notes -- Inherits from `UniqueResource<>` β€” see `uniqueresource.h` for the full RAII interface -- The `bool` template parameter maps to the `system` flag passed to `allocate()` -- Part of osquery's Linux-only D-Bus infrastructure under `osquery/tables/system/linux/dbus/` \ No newline at end of file +- Depends on `UniqueResource<>` (see `uniqueresource.h`) for the RAII mechanics +- `ResourceType` is `DBusConnection*` from the `libdbus` C API +- The `bool` template argument to `UniqueResource` maps to the `system` parameter in `allocate`, selecting which bus to connect to \ No newline at end of file diff --git a/osquery/tables/system/linux/dbus/.uniquedbusmessage.md b/osquery/tables/system/linux/dbus/.uniquedbusmessage.md index 0cb0b4a68b4..8aefbb226e2 100644 --- a/osquery/tables/system/linux/dbus/.uniquedbusmessage.md +++ b/osquery/tables/system/linux/dbus/.uniquedbusmessage.md @@ -1,40 +1,44 @@ -RAII wrapper for D-Bus message lifecycle management within the osquery Linux system tables infrastructure. +RAII wrapper for managing `DBusMessage*` lifecycle within osquery's Linux D-Bus table system, ensuring automatic allocation and deallocation of D-Bus method call messages. ## Key Components -### `UniqueDbusMessageAllocator` -A protected allocator class managing `DBusMessage*` resource lifetime: - -- **`allocate()`** β€” Creates a new `DBusMessage*` for a method call, parameterized by destination service, object path, interface, and method name -- **`deallocate()`** β€” Safely releases the `DBusMessage*` resource - -### `UniqueDbusMessage` (type alias) -A `UniqueResource` specialization that combines `UniqueDbusMessageAllocator` with four `const std::string&` constructor arguments, providing automatic cleanup via RAII semantics. +- **`UniqueDbusMessageAllocator`** β€” Allocator policy class providing `allocate()` and `deallocate()` static methods for `DBusMessage*` resources + - `allocate(message, destination, path, iface, method)` β€” Creates a new D-Bus method call message targeting a specific destination, object path, interface, and method + - `deallocate(message)` β€” Releases the underlying `DBusMessage*` resource +- **`UniqueDbusMessage`** β€” Type alias combining `UniqueResource<>` with `UniqueDbusMessageAllocator` and four `const std::string&` constructor parameters (destination, path, interface, method) ## Usage Example ```cpp #include -// Create a scoped D-Bus method call message -UniqueDbusMessage message; -auto status = message.initialize( - "org.freedesktop.NetworkManager", // destination - "/org/freedesktop/NetworkManager", // path - "org.freedesktop.NetworkManager", // interface - "GetDevices" // method -); - -if (!status.ok()) { - // handle error +namespace osquery { + +Status queryDbusMethod() { + UniqueDbusMessage message( + "org.freedesktop.NetworkManager", // destination + "/org/freedesktop/NetworkManager", // object path + "org.freedesktop.NetworkManager", // interface + "GetDevices" // method + ); + + if (!message.isValid()) { + return Status::failure("Failed to create D-Bus message"); + } + + // Use message.get() to access the raw DBusMessage* + DBusMessage* raw = message.get(); + + // Resource is automatically released when `message` goes out of scope + return Status::success(); } -// message is automatically freed when it goes out of scope +} // namespace osquery ``` ## Notes -- Follows the `UniqueResource` pattern used across osquery's D-Bus table helpers for consistent resource safety -- Intended for **Linux only** β€” part of the `osquery/tables/system/linux/dbus/` subsystem -- Dual-licensed under Apache-2.0 or GPL-2.0-only \ No newline at end of file +- Depends on `UniqueResource<>` (RAII base template) from `uniqueresource.h` +- Requires `libdbus` (`dbus/dbus.h`) +- Scoped to the `osquery` namespace; intended for internal use within Linux system table implementations \ No newline at end of file diff --git a/osquery/tables/system/linux/dbus/methods/.getstringproperty.md b/osquery/tables/system/linux/dbus/methods/.getstringproperty.md index d87c29e3df1..e712b531234 100644 --- a/osquery/tables/system/linux/dbus/methods/.getstringproperty.md +++ b/osquery/tables/system/linux/dbus/methods/.getstringproperty.md @@ -1,15 +1,16 @@ -Defines a D-Bus method handler for retrieving a single string property from a systemd D-Bus interface using the `org.freedesktop.DBus.Properties.Get` method. +Defines a D-Bus method handler for retrieving a single string property from a systemd service via the `org.freedesktop.DBus.Properties` interface. ## Key Components -- **`GetStringPropertyMethodHandler`** β€” Handler class implementing the D-Bus `Get` property call against the `org.freedesktop.systemd1` destination. Contains: +- **`GetStringPropertyMethodHandler`** β€” Base handler class with: - `kDestination` β€” Target D-Bus service (`org.freedesktop.systemd1`) - - `kInterface` β€” Property interface (`org.freedesktop.DBus.Properties`) - - `kMethod` β€” D-Bus method name (`Get`) - - `parseReply()` β€” Parses the D-Bus reply message into a `std::string` output, returning an osquery `Status` + - `kInterface` β€” D-Bus interface (`org.freedesktop.DBus.Properties`) + - `kMethod` β€” Method name (`Get`) + - `Output` β€” Type alias for `std::string` (the parsed result) + - `parseReply()` β€” Parses the raw `DBusMessage` reply into a `std::string` output -- **`GetStringPropertyMethod`** β€” Type alias instantiating `DbusMethod<>` with the handler and two `const std::string&` parameters (interface name and property name) +- **`GetStringPropertyMethod`** β€” Concrete `DbusMethod` type alias parameterized with the handler and two `const std::string&` arguments (interface name and property name) ## Usage Example @@ -18,10 +19,10 @@ Defines a D-Bus method handler for retrieving a single string property from a sy using namespace osquery; -// Invoke to retrieve a string property from a systemd unit +// Retrieve the "ActiveState" property from a systemd unit GetStringPropertyMethod method; -std::string result; +std::string result; auto status = method.call( connection, "/org/freedesktop/systemd1/unit/sshd_2eservice", @@ -30,11 +31,11 @@ auto status = method.call( ); if (status.ok()) { - // result contains the property value, e.g. "active" + // result contains e.g. "active", "inactive", "failed" } ``` ## Notes -- Protected constructor enforces use only through the `GetStringPropertyMethod` alias or subclassing -- Designed for use within osquery's Linux system tables that introspect systemd unit properties via D-Bus \ No newline at end of file +- Constructor and destructor are `protected`, enforcing use through the `DbusMethod` template rather than direct instantiation +- The two template arguments to `GetStringPropertyMethod` correspond to the D-Bus **interface** and **property name** passed at call time \ No newline at end of file diff --git a/osquery/tables/system/linux/dbus/methods/.listunitsmethodhandler.md b/osquery/tables/system/linux/dbus/methods/.listunitsmethodhandler.md index a2697f4e24d..3106b4d6e9d 100644 --- a/osquery/tables/system/linux/dbus/methods/.listunitsmethodhandler.md +++ b/osquery/tables/system/linux/dbus/methods/.listunitsmethodhandler.md @@ -1,44 +1,36 @@ -Defines the D-Bus method handler for invoking `ListUnits` on the systemd1 Manager interface, parsing the response into structured unit records for use in osquery's Linux system tables. +Defines the D-Bus method handler for invoking the `ListUnits` method on the systemd D-Bus API, parsing the response into structured unit records for use in osquery's Linux system tables. ## Key Components -### `ListUnitsMethodHandler` -Core handler class that encodes D-Bus targeting constants and reply parsing logic. +**`ListUnitsMethodHandler`** β€” Core handler class with: +- `kDestination`, `kInterface`, `kMethod` β€” D-Bus target constants pointing to `org.freedesktop.systemd1.Manager.ListUnits` +- `Unit` β€” POD struct representing a single systemd unit with fields: `id`, `description`, `load_state`, `active_state`, `sub_state`, `following`, `path`, `job_id`, `job_type`, `job_path` +- `Output` β€” Type alias for `std::vector` +- `parseReply()` β€” Parses a raw `UniqueDbusMessage` reply into the `Output` vector +- `readUnitInformation()` β€” Reads a single unit's fields from a `DBusMessageIter` -| Member | Description | -|---|---| -| `kDestination` | D-Bus destination: `org.freedesktop.systemd1` | -| `kInterface` | Target interface: `org.freedesktop.systemd1.Manager` | -| `kMethod` | Method name: `ListUnits` | -| `Unit` | POD struct holding per-unit fields (id, states, job info, object path) | -| `Output` | Alias for `std::vector` | -| `parseReply()` | Parses a raw `DBusMessage` reply into the `Output` vector | -| `readUnitInformation()` | Reads a single unit's fields from a `DBusMessageIter` | - -### `ListUnitsMethod` -Type alias combining `ListUnitsMethodHandler` with the generic `DbusMethod` template β€” the primary type used by callers. +**`ListUnitsMethod`** β€” Type alias composing `ListUnitsMethodHandler` with the generic `DbusMethod<>` template for dispatch. ## Usage Example -```c +```cpp #include -osquery::ListUnitsMethodHandler::Output units; +osquery::ListUnitsMethod::Output units; +auto method = osquery::ListUnitsMethod::create(connection); -// Invoke via the composed method type -auto conn = /* acquire DBusConnection */; -osquery::ListUnitsMethod method; -auto status = method.call(conn, units); +auto status = method.call(units); +if (!status.ok()) { + LOG(ERROR) << "ListUnits D-Bus call failed: " << status.getMessage(); + return; +} -if (status.ok()) { - for (const auto& unit : units) { - // Access unit.id, unit.active_state, unit.load_state, etc. - } +for (const auto& unit : units) { + LOG(INFO) << unit.id + << " [" << unit.active_state << "/" + << unit.sub_state << "]"; } ``` -## Notes - -- Constructor and destructor are `protected` β€” instantiation is intended only through the `DbusMethod` template. -- `Unit::job_id` defaults to `0U`; a non-zero value indicates a pending systemd job. \ No newline at end of file +The handler is consumed by the systemd units table implementation to populate rows describing active, inactive, and failed systemd units on Linux hosts. \ No newline at end of file diff --git a/osquery/tables/system/posix/.authorized_keys.md b/osquery/tables/system/posix/.authorized_keys.md index 4a9167b2c77..42167265658 100644 --- a/osquery/tables/system/posix/.authorized_keys.md +++ b/osquery/tables/system/posix/.authorized_keys.md @@ -1,39 +1,33 @@ -Header file declaring functions for parsing and retrieving SSH authorized keys for system users within the osquery tables framework. +Declares the interface for parsing SSH `authorized_keys` files within the osquery tables subsystem, exposing functions to enumerate authorized public keys per user. ## Key Components -| Symbol | Type | Description | -|--------|------|-------------| -| `genSSHkeysForUser` | Function | Parses the `~/.ssh/authorized_keys` file for a specific user, appending discovered key entries to the `QueryData` results | -| `getAuthorizedKeys` | Function | Entry point for the `authorized_keys` osquery virtual table; resolves user context and aggregates SSH key data across all eligible users | - -Both functions live inside the `osquery::tables` namespace. +- **`genSSHkeysForUser`** β€” Parses the `authorized_keys` file for a specific user (identified by `uid`, `gid`, and home `directory`), appending discovered key rows to the provided `QueryData` result set using the supplied `Logger`. +- **`getAuthorizedKeys`** β€” Primary osquery table query handler; accepts a `QueryContext` and returns a `QueryData` set of all authorized SSH keys found across users on the system. ## Usage Example -```c +```cpp #include "authorized_keys.h" -// Populate SSH keys for a single user +// Enumerate SSH keys for a single known user osquery::QueryData results; osquery::GlogLogger logger; osquery::tables::genSSHkeysForUser( - "1000", // uid - "1000", // gid - "/home/alice", // home directory + "1001", // uid + "1001", // gid + "/home/alice", // home directory results, logger ); -// Or query via the full table interface (used internally by osquery) +// Or invoke the full table query via osquery context osquery::QueryContext ctx; osquery::QueryData allKeys = osquery::tables::getAuthorizedKeys(ctx); -``` - -## Notes -- `genSSHkeysForUser` is designed to be called in a worker process context, accepting a `Logger&` reference to support osquery's inter-process logging via `GlogLogger`. -- `getAuthorizedKeys` integrates with the osquery table registration system and is typically invoked by the query engine rather than called directly. -- Both functions depend on `osquery/core/tables.h` (`QueryData`, `QueryContext`) and `osquery/logger/logger.h`. \ No newline at end of file +for (const auto& row : allKeys) { + // Each row contains fields: uid, key, key_file, type, comment, etc. +} +``` \ No newline at end of file diff --git a/osquery/tables/system/posix/.known_hosts.md b/osquery/tables/system/posix/.known_hosts.md index a5c383e0a86..af4e9ea0a0d 100644 --- a/osquery/tables/system/posix/.known_hosts.md +++ b/osquery/tables/system/posix/.known_hosts.md @@ -1,32 +1,25 @@ -Declares the interface for parsing SSH `known_hosts` files and enumerating trusted host keys for system users. +Declares the interface for the `known_hosts` osquery table, which parses SSH `known_hosts` files to enumerate trusted host keys for system users. ## Key Components -- **`getKnownHostsKeys(QueryContext& context)`** β€” Primary table generator that queries all known SSH host keys across users on the system. Integrates with the osquery table dispatch system. -- **`impl::genSSHkeysForHosts(...)`** β€” Internal helper that parses a specific user's `known_hosts` file given their `uid`, `gid`, and home `directory`, appending discovered key entries to the provided `QueryData` results set. +- **`getKnownHostsKeys(QueryContext& context)`** β€” Primary table generation function; queries the system for SSH known hosts entries and returns structured `QueryData`. +- **`impl::genSSHkeysForHosts(...)`** β€” Internal implementation helper that processes a specific user's `known_hosts` file given their `uid`, `gid`, home `directory`, and a `results` output buffer. ## Usage Example ```c -// Typical osquery table dispatch usage -namespace osquery { -namespace tables { - -// Called by the osquery table infrastructure -QueryData results = getKnownHostsKeys(context); - -// Internal usage: parse known_hosts for a specific user -QueryData rows; -impl::genSSHkeysForHosts("1000", "1000", "/home/alice", rows); - -// rows now contains entries from /home/alice/.ssh/known_hosts -} -} +// Registering and invoking the table query +QueryContext context; +QueryData results = osquery::tables::getKnownHostsKeys(context); + +// Internal helper usage (within impl namespace) +osquery::tables::impl::genSSHkeysForHosts( + "1000", // uid + "1000", // gid + "/home/username", // home directory + results // output QueryData +); ``` -## Notes - -- The `impl` namespace separates internal parsing logic from the public table interface, allowing unit-testable decomposition. -- Both functions operate within `osquery::tables`, following standard osquery table module conventions. -- The header depends on `osquery/core/query.h` and `osquery/core/tables.h`, which provide `QueryData` and `QueryContext` respectively. \ No newline at end of file +> The `impl` namespace isolates the per-user file parsing logic, keeping the public API clean. The helper is typically called iteratively over all users enumerated from `/etc/passwd` or equivalent sources. \ No newline at end of file diff --git a/osquery/tables/system/posix/.last.md b/osquery/tables/system/posix/.last.md index a29cd4c63be..6fb6de79bc3 100644 --- a/osquery/tables/system/posix/.last.md +++ b/osquery/tables/system/posix/.last.md @@ -1,31 +1,35 @@ -Declares the interface for querying last login/access records from the system's `utmpx` database on Unix-like systems. +Declares the interface for the `last` table plugin, which queries historical login and session records via `utmpx`. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `genLastAccess` | Function | Primary table generator that queries all last login/access records | -| `impl::genLastAccessForRow` | Function | Internal helper that processes a single `utmpx` entry into a result row | +- **`genLastAccess(QueryContext&)`** β€” Main table generator function; retrieves all last-access/login records and returns them as `QueryData`. +- **`impl::genLastAccessForRow(const utmpx&, QueryData&)`** β€” Internal helper that processes a single `utmpx` entry and appends a row to the results set. Separated into `impl` for testability. ## Usage Example -```c +```cpp #include "last.h" -// Generate all last login records -osquery::QueryContext context; -osquery::QueryData results = osquery::tables::genLastAccess(context); +// Register and invoke the table generator +QueryContext ctx; +QueryData results = osquery::tables::genLastAccess(ctx); -// Or process a single utmpx row directly (internal use) -utmpx entry; -// ... populate entry via getutxent() or similar ... -osquery::QueryData rowResults; -osquery::tables::impl::genLastAccessForRow(entry, rowResults); +for (const auto& row : results) { + std::cout << row.at("username") << " - " << row.at("time") << "\n"; +} + +// Unit-test a single utmpx record directly +utmpx entry{}; +strncpy(entry.ut_user, "alice", sizeof(entry.ut_user)); +entry.ut_tv.tv_sec = time(nullptr); + +QueryData singleResult; +osquery::tables::impl::genLastAccessForRow(entry, singleResult); ``` ## Notes -- Depends on `` β€” available on Linux, macOS, and other POSIX systems; **not available on Windows** -- `genLastAccessForRow` lives in the `impl` namespace, signaling it is an internal implementation detail not intended for direct external use -- Feeds the osquery `last` virtual table, which exposes data equivalent to running the `last` command in a terminal \ No newline at end of file +- Depends on POSIX ``; this header (and its implementation) is **Unix/Linux/macOS only**. +- Integrates with the osquery table framework via `QueryData` and `QueryContext` from `osquery/core/tables.h`. +- The `impl` namespace pattern is a common osquery convention for exposing internal logic to unit tests without making it part of the public API. \ No newline at end of file diff --git a/osquery/tables/system/posix/.openssl_utils.md b/osquery/tables/system/posix/.openssl_utils.md index b38f6f48488..c8d40e96e7d 100644 --- a/osquery/tables/system/posix/.openssl_utils.md +++ b/osquery/tables/system/posix/.openssl_utils.md @@ -1,24 +1,24 @@ -Utility header for extracting metadata and attributes from OpenSSL X.509 certificate objects within the `osquery::tables` namespace. +Utility header for extracting metadata and attributes from OpenSSL X.509 certificate objects within the osquery tables framework. ## Key Components -All functions accept a raw `X509*` pointer and return either `boost::optional`, `boost::optional`, or void. +All functions reside in the `osquery::tables` namespace and accept an `X509*` pointer, returning `boost::optional` values (empty on failure). | Function | Returns | Description | -|---|---|---| +|----------|---------|-------------| | `generateCertificateSHA1Digest` | `optional` | SHA1 fingerprint of the certificate | -| `getCertificateAttributes` | `void` | Populates CA and self-signed boolean flags | -| `getCertificateKeyUsage` | `optional` | Key usage extension value | -| `getCertificateSerialNumber` | `optional` | Certificate serial number | +| `getCertificateAttributes` | `void` (out params) | Sets `is_ca` and `is_self_signed` flags | +| `getCertificateKeyUsage` | `optional` | Comma-separated key usage extensions | +| `getCertificateSerialNumber` | `optional` | Hex-encoded serial number | | `getCertificateAuthorityKeyID` | `optional` | Authority Key Identifier extension | | `getCertificateSubjectKeyID` | `optional` | Subject Key Identifier extension | -| `getCertificateIssuerName` | `optional` | Issuer DN (legacy format toggle via flag) | -| `getCertificateSubjectName` | `optional` | Subject DN (legacy format toggle via flag) | -| `getCertificateCommonName` | `optional` | CN field from the subject | +| `getCertificateIssuerName` | `optional` | Issuer DN (with legacy format toggle) | +| `getCertificateSubjectName` | `optional` | Subject DN (with legacy format toggle) | +| `getCertificateCommonName` | `optional` | CN field from subject | | `getCertificateSigningAlgorithm` | `optional` | Signature algorithm (e.g. `sha256WithRSAEncryption`) | | `getCertificateKeyAlgorithm` | `optional` | Public key algorithm | -| `getCertificateKeyStregth` | `optional` | Key size/strength (note: typo in original symbol name) | +| `getCertificateKeyStregth` | `optional` | Key size in bits *(note: typo in symbol name)* | | `getCertificateNotValidBefore` | `optional` | Certificate validity start timestamp | | `getCertificateNotValidAfter` | `optional` | Certificate validity end timestamp | @@ -26,20 +26,19 @@ All functions accept a raw `X509*` pointer and return either `boost::optional -// cert is an X509* obtained from OpenSSL (e.g. via PEM/DER parsing) -X509* cert = /* ... */; - -bool is_ca = false, is_self_signed = false; -osquery::tables::getCertificateAttributes(cert, is_ca, is_self_signed); - +// Assuming `cert` is a loaded X509* object if (auto sha1 = osquery::tables::generateCertificateSHA1Digest(cert)) { std::cout << "Fingerprint: " << *sha1 << "\n"; } +bool is_ca = false, is_self_signed = false; +osquery::tables::getCertificateAttributes(cert, is_ca, is_self_signed); + if (auto expiry = osquery::tables::getCertificateNotValidAfter(cert)) { - std::cout << "Expires: " << std::ctime(&(*expiry)); + // Compare *expiry against std::time(nullptr) to detect expired certs } ``` -> **Note:** `getCertificateKeyStregth` contains a typo in the original symbol name; use as-is to avoid linker errors. \ No newline at end of file +> **Note:** `getCertificateKeyStregth` contains a typo in the original symbol name (`Stregth` vs `Strength`) β€” match exactly when calling. \ No newline at end of file diff --git a/osquery/tables/system/posix/.shell_history.md b/osquery/tables/system/posix/.shell_history.md index 49b8f1564f9..947c860e4c7 100644 --- a/osquery/tables/system/posix/.shell_history.md +++ b/osquery/tables/system/posix/.shell_history.md @@ -1,42 +1,36 @@ -Declares internal helper functions for generating shell history table rows in osquery's `shell_history` virtual table implementation. +Declares internal helper functions for shell history table generation within osquery's `tables` namespace, supporting both standard user history parsing and macOS Bash Sessions. ## Key Components -### Functions +- **`genShellHistoryFromBashSessions`** β€” Parses shell history from macOS Bash Sessions for a given user (`uid`) and home `directory`. Accepts a predicate callback to process each resulting `DynamicTableRowHolder`, enabling testable, injectable row handling. -- **`genShellHistoryFromBashSessions`** β€” Parses shell history from Bash session files within a given directory for a specific user ID. Accepts a predicate callback for row emission, enabling testability without direct table coupling. +- **`genShellHistoryForUser`** β€” Parses standard shell history files for a given user (`uid`, `gid`, `directory`). Also accepts a predicate callback for row processing, decoupling data generation from table population logic. -- **`genShellHistoryForUser`** β€” Enumerates shell history entries for a user identified by `uid`/`gid` and home `directory`. Feeds results through a `DynamicTableRowHolder` predicate rather than returning data directly. +Both functions are designed with testability in mind β€” the predicate pattern allows unit tests to inject custom row consumers without requiring a live osquery table context. ## Usage Example ```c #include "shell_history.h" -// Collect all history rows for a user +// Collect shell history rows for a specific user osquery::tables::genShellHistoryForUser( - "1000", // uid - "1000", // gid + "1000", // uid + "1000", // gid "/home/alice", // home directory [](osquery::DynamicTableRowHolder& row) { - // Process or assert on each emitted history row - std::cout << row["command"] << std::endl; + // Process or assert on each history row + row["username"] = "alice"; } ); -// Collect Bash session history specifically +// Collect history from macOS Bash Sessions osquery::tables::genShellHistoryFromBashSessions( - "1000", - "/home/alice", + "501", + "/Users/alice", [](osquery::DynamicTableRowHolder& row) { - // Handle each session history row + // Handle each session history entry } ); -``` - -## Notes - -- Both functions use a **predicate/callback pattern** (`std::function`) to decouple row generation from table registration, making unit testing straightforward without requiring a live osquery table context. -- Declared within `osquery::tables` namespace, consistent with osquery's virtual table architecture. -- Depends on `osquery/core/tables.h` and `osquery/sql/dynamic_table_row.h`. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/tables/system/posix/.sudoers.md b/osquery/tables/system/posix/.sudoers.md index 129383a3026..ba092ee8f82 100644 --- a/osquery/tables/system/posix/.sudoers.md +++ b/osquery/tables/system/posix/.sudoers.md @@ -1,14 +1,10 @@ -Declares the interface for parsing and querying sudoers configuration files within the osquery tables framework. +Declares the interface for parsing and querying sudoers configuration files within the osquery tables subsystem. ## Key Components -| Symbol | Type | Description | -|--------|------|-------------| -| `genSudoersFile` | Function | Parses a single sudoers file at the given path, recursively handling `#include`/`@include` directives via the `level` depth parameter, and appends results to `QueryData` | -| `genSudoers` | Function | Entry point for the `sudoers` virtual table; dispatches file parsing and returns the full `QueryData` result set | - -Both symbols reside in the `osquery::tables` namespace. +- **`genSudoersFile`** β€” Parses a single sudoers file (or included file) at a given inclusion depth level, appending discovered entries to the provided `QueryData` results set. +- **`genSudoers`** β€” Primary table generator function; entry point called by the osquery runtime to satisfy queries against the `sudoers` virtual table. ## Usage Example @@ -16,17 +12,18 @@ Both symbols reside in the `osquery::tables` namespace. #include "sudoers.h" // Query all sudoers entries via the osquery table interface -osquery::QueryContext ctx; -osquery::QueryData results = osquery::tables::genSudoers(ctx); +osquery::QueryContext context; +osquery::QueryData results = osquery::tables::genSudoers(context); // Or parse a specific sudoers file directly osquery::QueryData fileResults; osquery::tables::genSudoersFile("/etc/sudoers", 0, fileResults); - -for (const auto& row : fileResults) { - // Each row contains parsed sudoers rule fields - // e.g., row.at("source"), row.at("header"), row.at("rule") -} +// level > 0 indicates a file pulled in via #include directive +osquery::tables::genSudoersFile("/etc/sudoers.d/custom", 1, fileResults); ``` -> **Note:** The `level` parameter in `genSudoersFile` guards against circular or deeply nested `#include` directives. Pass `0` for the top-level file; recursive calls increment this value automatically. \ No newline at end of file +## Notes + +- Both functions live under the `osquery::tables` namespace, consistent with all osquery table implementations. +- The `level` parameter in `genSudoersFile` tracks recursive `#include` depth, preventing infinite loops from circular sudoers includes. +- Depends on `osquery/core/query.h` (`QueryData`, `QueryContext`) and `osquery/core/tables.h` for the standard osquery table generation contract. \ No newline at end of file diff --git a/osquery/tables/system/windows/.certificates.md b/osquery/tables/system/windows/.certificates.md index 921161b2584..ba9a07129b0 100644 --- a/osquery/tables/system/windows/.certificates.md +++ b/osquery/tables/system/windows/.certificates.md @@ -1,35 +1,47 @@ -Header for Windows certificate store parsing utilities within osquery's tables module. +Windows certificate store parsing utilities for the osquery tables subsystem. ## Key Components -### Type Aliases -- **`ServiceNameMap`** β€” `std::unordered_map` mapping service names to SIDs for caching lookups +**Type Alias** +- `ServiceNameMap` β€” `std::unordered_map` mapping service names to SIDs -### Constants -| Constant | Value | Description | -|---|---|---| -| `kLocalSystem` | `S-1-5-18` | SID for the Local System account | -| `kLocalService` | `S-1-5-19` | SID for the Local Service account | -| `kNetworkService` | `S-1-5-20` | SID for the Network Service account | +**Constants** -### Functions -- **`parseSystemStoreString`** β€” Parses a wide-character Windows certificate store path string, resolving the store location, service name or user ID, SID, and store name from the system store identifier +| Name | Value | Description | +|------|-------|-------------| +| `kLocalSystem` | `S-1-5-18` | Well-known SID for Local System account | +| `kLocalService` | `S-1-5-19` | Well-known SID for Local Service account | +| `kNetworkService` | `S-1-5-20` | Well-known SID for Network Service account | + +**Function** +- `parseSystemStoreString` β€” Parses a Windows certificate system store path (`LPCWSTR`) into its constituent parts: store location, service name or user ID, SID, and store name. Uses a cache (`service2sidCache`) to avoid redundant service-to-SID lookups. ## Usage Example -```c +```cpp +#include "certificates.h" + osquery::tables::ServiceNameMap sidCache; -std::string serviceNameOrUserId, sid, storeName; +std::string serviceNameOrUserId; +std::string sid; +std::string storeName; osquery::tables::parseSystemStoreString( - L"S-1-5-18\\My", // system store wide string - "LocalMachine", // store location context - sidCache, // service-to-SID cache (mutated) - serviceNameOrUserId, // out: resolved service name or user ID - sid, // out: resolved SID string - storeName // out: certificate store name (e.g. "My") + L"LocalService\\My", + "LocalService", + sidCache, + serviceNameOrUserId, + sid, + storeName ); + +// sid => "S-1-5-19" +// storeName => "My" ``` -> **Note:** This header is Windows-specific. It depends on `LPCWSTR` (Windows API type) and is intended for use within osquery's certificate enumeration table implementation. The `ServiceNameMap` cache avoids redundant service-name-to-SID resolution across multiple store entries. \ No newline at end of file +## Notes + +- Windows-only header (uses `LPCWSTR`) +- Intended for internal use within the `osquery::tables` namespace to enumerate and populate certificate table data +- The `service2sidCache` parameter is mutated in-place to cache resolved SID lookups across multiple calls \ No newline at end of file diff --git a/osquery/tables/system/windows/.programs.md b/osquery/tables/system/windows/.programs.md index 91da9487db6..3b85e55304f 100644 --- a/osquery/tables/system/windows/.programs.md +++ b/osquery/tables/system/windows/.programs.md @@ -1,5 +1,5 @@ -Header file for Windows MSI program registry utilities within the osquery tables subsystem, providing GUID decoding functionality for installed software enumeration. +Header file for Windows MSI program registry utilities within osquery's tables module, providing GUID decoding functionality for installed program enumeration. ## Key Components @@ -7,10 +7,10 @@ Header file for Windows MSI program registry utilities within the osquery tables ```c std::string decodeMsiRegistryGuid(const std::string& encoded); ``` -Converts a registry-encoded MSI GUID (compressed/shuffled format) into a standard human-readable GUID format. +Converts a registry-encoded MSI GUID string into a standard formatted GUID. - **Input:** 32-character encoded GUID string (e.g., `"0D8797326E7E4114DAECB3B66B9CD045"`) -- **Output:** Standard GUID string (e.g., `"{237978D0-E7E6-4114-ADCE-3B6BB6C90D54}"`) or empty string on invalid input +- **Output:** Standard GUID string (e.g., `"{237978D0-E7E6-4114-ADCE-3B6BB6C90D54}"`) or empty string if input is invalid ## Usage Example @@ -21,16 +21,17 @@ Converts a registry-encoded MSI GUID (compressed/shuffled format) into a standar std::string encoded = "0D8797326E7E4114DAECB3B66B9CD045"; std::string guid = osquery::tables::decodeMsiRegistryGuid(encoded); -if (!guid.empty()) { - // guid == "{237978D0-E7E6-4114-ADCE-3B6BB6C90D54}" - // Use guid to look up installed program metadata +if (guid.empty()) { + // Handle invalid input (must be exactly 32 characters) + LOG(WARNING) << "Invalid encoded GUID provided"; } else { - // Handle invalid/malformed encoded input + // guid == "{237978D0-E7E6-4114-ADCE-3B6BB6C90D54}" + LOG(INFO) << "Decoded GUID: " << guid; } ``` ## Notes - Input must be exactly **32 characters** long; any other length returns an empty string -- Used internally by osquery's `programs` table to enumerate installed Windows software from the MSI registry keys under `HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer` -- Namespace: `osquery::tables` \ No newline at end of file +- Used internally by osquery's Windows `programs` table to resolve installed MSI package identifiers from the registry +- Declared within the `osquery::tables` namespace \ No newline at end of file diff --git a/osquery/tables/system/windows/.registry.md b/osquery/tables/system/windows/.registry.md index a7340e9b2dc..7021fbaf288 100644 --- a/osquery/tables/system/windows/.registry.md +++ b/osquery/tables/system/windows/.registry.md @@ -1,29 +1,26 @@ -Windows Registry query utilities for osquery's table implementations, providing helpers to enumerate, expand, and parse registry keys, paths, hives, and COM class identifiers. +Header file providing Windows Registry query utilities for osquery's table implementation, exposing functions to query, expand, and parse registry keys and paths. ## Key Components ### Constants - -| Name | Value | Purpose | -|---|---|---| -| `kRegSep` | `"\\"` | Registry path separator | -| `kDefaultRegName` | `"(Default)"` | Default registry key name | -| `kRegMaxRecursiveDepth` | `32` | Maximum glob expansion depth | +- `kRegSep` β€” Registry path separator (`\`) +- `kDefaultRegName` β€” Default registry key name (`(Default)`) +- `kRegMaxRecursiveDepth` β€” Maximum recursion depth for registry searches (`32`) ### Functions | Function | Description | |---|---| -| `queryKey()` | Queries a single registry key path and populates results | -| `queryMultipleRegistryKeys()` | Batch-queries registry keys using LIKE pattern matching | -| `queryMultipleRegistryPaths()` | Batch-queries full registry paths using LIKE pattern matching | -| `getClassName()` | Resolves a COM Class ID (CLSID) to its human-readable class name | -| `getClassExecutables()` | Returns all executables (`.dll`, `.exe`, etc.) associated with a CLSID | -| `expandRegistryGlobs()` | Expands SQL glob patterns (e.g. `HKLM\%\Microsoft`) into matching key sets | -| `explodeRegistryPath()` | Splits a full registry path into its HIVE and KEY components | -| `getUsernameFromKey()` | Extracts a username from an `HKEY_USERS\\...` path | -| `populateSubkeys()` | Expands a key set with discovered subkeys, optionally replacing the input set | +| `queryKey` | Queries a single registry key path, populating a `QueryData` result set | +| `queryMultipleRegistryKeys` | Batch queries multiple registry keys using LIKE patterns | +| `queryMultipleRegistryPaths` | Batch queries multiple registry paths using LIKE patterns | +| `getClassName` | Resolves a Class ID (CLSID) to its human-readable class name | +| `getClassExecutables` | Retrieves executables (`.dll`, `.exe`, etc.) associated with a CLSID | +| `expandRegistryGlobs` | Expands SQL glob patterns (e.g. `HKEY_LOCAL_MACHINE\%\Microsoft`) into matching key sets | +| `explodeRegistryPath` | Splits a full registry path into its HIVE and KEY components | +| `getUsernameFromKey` | Extracts a username from an `HKEY_USERS` path | +| `populateSubkeys` | Expands a set of registry keys with their discovered subkeys | ## Usage Example @@ -36,21 +33,16 @@ using namespace osquery::tables; QueryData results; Status s = queryKey("HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft", results); -// Expand a glob pattern to matching keys -std::set keys; -expandRegistryGlobs("HKEY_LOCAL_MACHINE\\%\\Microsoft", keys); - -// Split path into hive + key +// Split a full path into hive and key std::string hive, key; explodeRegistryPath("HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft", hive, key); // hive = "HKEY_LOCAL_MACHINE", key = "SOFTWARE\\Microsoft" +// Expand a glob pattern into matching keys +std::set matchedKeys; +expandRegistryGlobs("HKEY_LOCAL_MACHINE\\%\\Microsoft", matchedKeys); + // Resolve a CLSID to its class name std::string className; getClassName("{0000002F-0000-0000-C000-000000000046}", className); - -// Resolve a username from an HKEY_USERS SID path -std::string username; -getUsernameFromKey("HKEY_USERS\\S-1-5-19\\Software", username); -// username = "LOCAL SERVICE" ``` \ No newline at end of file diff --git a/osquery/tables/system/windows/.security_profile_info_utils.md b/osquery/tables/system/windows/.security_profile_info_utils.md index a0906e96118..84821bb4c0b 100644 --- a/osquery/tables/system/windows/.security_profile_info_utils.md +++ b/osquery/tables/system/windows/.security_profile_info_utils.md @@ -1,58 +1,44 @@ -Utility header providing Windows Security Configuration Engine (SCE) RPC client access for retrieving system security profile information via `scecli.dll`. +Utility header for retrieving and managing Windows Security Configuration Engine (SCE) security profile information via the undocumented `scecli.dll` RPC client API. ## Key Components ### `SceClientHelper` (Singleton Class) -Manages runtime dynamic linking to `scecli.dll` and exposes the undocumented SCE RPC Client API. +Manages runtime dynamic linking to `scecli.dll` and exposes SCE RPC client functions. | Member | Description | |---|---| -| `instance()` | Returns the singleton `SceClientHelper` instance | -| `getSceSecurityProfileInfo()` | Calls `SceGetSecurityProfileInfo()` to fetch system security profile data | -| `releaseSceProfileData()` | Calls `SceFreeMemory()` to free SCE-allocated memory | -| `isWow64Process()` | Detects WoW64 (32-bit process on 64-bit Windows) via `IsWow64Process()` | -| `SceProfileInfo` | Raw data structure mapping the SCE RPC protocol response (password policy, audit settings, account settings, log configuration) | +| `SceProfileInfo` | Raw data structure mapping the SCE RPC protocol response (password policy, audit settings, account info) | +| `instance()` | Returns the global singleton instance | +| `getSceSecurityProfileInfo(profileData)` | Calls `SceGetSecurityProfileInfo()` to fetch system security profile | +| `releaseSceProfileData(profileData)` | Calls `SceFreeMemory()` to release allocated profile memory | +| `isWow64Process()` | Detects if the current process runs under WoW64 emulation | ### `SceProfileData` (RAII Wrapper) -Scoped memory manager for `SceProfileInfo` data returned by the SCE RPC server. +Scoped guard for memory returned by the SCE RPC server. | Member | Description | |---|---| -| `getProfileInfo()` | Returns typed pointer to the `SceProfileInfo` buffer | -| `~SceProfileData()` | Automatically calls `SceFreeMemory()` on destruction | -| `getNormalizedInt()` | Normalizes SCE protocol `DWORD` values β€” maps `(DWORD)-1` to `-1` integer, matching `secedit.exe` behavior | +| `getProfileInfo()` | Returns typed pointer to `SceProfileInfo` | +| `~SceProfileData()` | Automatically frees SCE memory on scope exit | +| `getNormalizedInt(input)` | Normalizes `DWORD` `-1` sentinel values to signed `-1` (matches `secedit` behavior) | ## Usage Example ```cpp -#include "security_profile_info_utils.h" - -using namespace osquery::tables; - -void querySecurityProfile() { - // Scoped RAII wrapper handles memory cleanup automatically - SceProfileData profileData; - - // Fetch system security profile via SCE RPC - PVOID rawData = nullptr; - auto& sce = SceClientHelper::instance(); - Status s = sce.getSceSecurityProfileInfo(rawData); - - if (!s.ok()) { - LOG(ERROR) << "SCE call failed: " << s.getMessage(); - return; - } - - // Access typed profile fields - const auto* info = profileData.getProfileInfo(); - if (info != nullptr) { - int minAge = SceProfileData::getNormalizedInt(info->MinPasswdAge); - int maxAge = SceProfileData::getNormalizedInt(info->MaxPasswdAge); - // info->LockoutBadCount, info->AuditLogonEvents, etc. - } - // Memory freed automatically when profileData goes out of scope -} +// Acquire scoped profile data via the singleton helper +SceProfileData profileData; +Status status = SceClientHelper::instance() + .getSceSecurityProfileInfo(profileData.data_); + +if (status.ok()) { + const auto* info = profileData.getProfileInfo(); + if (info != nullptr) { + int maxAge = SceProfileData::getNormalizedInt(info->MaxPasswdAge); + int minLen = SceProfileData::getNormalizedInt(info->MinPasswdLen); + // info->AuditLogonEvents, info->LockoutBadCount, etc. + } +} // profileData destructor calls SceFreeMemory() automatically ``` -> **Note:** This header is Windows-only and depends on undocumented `scecli.dll` exports (`SceGetSecurityProfileInfo`, `SceFreeMemory`) whose prototypes have remained stable since Windows 7. \ No newline at end of file +> **Windows-only.** Relies on undocumented `scecli.dll` exports (`SceGetSecurityProfileInfo`, `SceFreeMemory`) stable since Windows 7. WoW64 detection via `isWow64Process()` guards against incorrect memory layout access on 32-bit processes running on 64-bit Windows. \ No newline at end of file diff --git a/osquery/tables/system/windows/.startup_items.md b/osquery/tables/system/windows/.startup_items.md index 9c2967ef01f..289710b94d7 100644 --- a/osquery/tables/system/windows/.startup_items.md +++ b/osquery/tables/system/windows/.startup_items.md @@ -1,28 +1,34 @@ -Declares the `parseStartupPath` utility function for extracting executable paths and arguments from macOS startup item entries. +Declares a utility function for parsing macOS startup item path entries within the osquery tables framework. ## Key Components -- **`parseStartupPath(const std::string& entry, Row& r)`** β€” Parses a raw startup path string, splitting it into `path` and `args` fields stored in the provided `Row`. Returns `true` on success, `false` if parsing fails. +- **`parseStartupPath(const std::string& entry, Row& r)`** β€” Parses a raw startup path string extracted from a startup entry, populating the `path` and `args` fields of the provided `Row` object. Returns `true` on successful parsing, `false` otherwise. ## Usage Example -```cpp +```c #include "startup_items.h" - -osquery::Row r; -std::string entry = "/usr/bin/myapp --config /etc/myapp.conf"; - -if (osquery::tables::parseStartupPath(entry, r)) { - // r["path"] == "/usr/bin/myapp" - // r["args"] == "--config /etc/myapp.conf" -} else { - // Handle malformed startup entry +#include + +namespace osquery { +namespace tables { + +void processStartupEntry(const std::string& rawEntry) { + Row r; + if (parseStartupPath(rawEntry, r)) { + // r["path"] and r["args"] are now populated + std::string path = r.at("path"); + std::string args = r.at("args"); + } } + +} // namespace tables +} // namespace osquery ``` ## Notes -- Lives within the `osquery::tables` namespace, consistent with osquery's table implementation conventions. -- Intended as a helper for the `startup_items` table implementation, isolating path-parsing logic for testability. -- The `Row` type is defined in `osquery/core/tables.h` and maps column names to string values. \ No newline at end of file +- Resides in the `osquery::tables` namespace, consistent with the osquery table plugin architecture. +- Designed as a parsing helper, intended to be called during startup item enumeration (e.g., processing macOS Login Items or LaunchAgents). +- The `Row` type is defined in `osquery/core/tables.h` and represents a single row in an osquery virtual table result set. \ No newline at end of file diff --git a/osquery/tables/system/windows/.windows_eventlog.md b/osquery/tables/system/windows/.windows_eventlog.md index b4ccb613a64..02d26774acb 100644 --- a/osquery/tables/system/windows/.windows_eventlog.md +++ b/osquery/tables/system/windows/.windows_eventlog.md @@ -1,38 +1,46 @@ -Declares two helper functions for querying and parsing Windows Event Log data within the osquery tables framework. +Header file exposing two helper utilities for querying and parsing Windows Event Log data within osquery's table infrastructure. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `parseWelXml` | Function | Parses a Windows Event Log entry rendered as XML into an osquery `Row` | -| `genXfilterFromConstraints` | Function | Builds an XPath filter string from a `QueryContext` for use with the Windows `EvtQuery` API | +### `parseWelXml` +```text +Status parseWelXml(QueryContext& context, std::wstring& xml_event, Row& row) +``` +Parses a Windows Event Log entry rendered in XML format and populates an osquery table `Row`. Returns a `Status` indicating success or failure. + +### `genXfilterFromConstraints` +```text +void genXfilterFromConstraints(QueryContext& context, std::string& xfilter) +``` +Builds an XPath filter string from the active `QueryContext` constraints. The resulting `xfilter` string is compatible with the Windows `EvtQuery` API for selective event retrieval. ## Usage Example -```c -#include +```cpp +#include namespace osquery { namespace tables { QueryRows genWindowsEventLog(QueryContext& context) { - QueryRows results; - std::string xfilter; + QueryRows results; - // Build an XPath filter from the query constraints (e.g., channel, event ID) - genXfilterFromConstraints(context, xfilter); + // Build XPath filter from query constraints (e.g., WHERE channel='System') + std::string xfilter; + genXfilterFromConstraints(context, xfilter); - // ... open event log channel via EvtQuery using xfilter ... + // ... open event log handle using xfilter via EvtQuery ... - std::wstring xml_event = /* rendered XML from EvtRender */; - Row row; - Status status = parseWelXml(context, xml_event, row); - if (status.ok()) { - results.push_back(row); - } + // For each event retrieved, parse its XML representation + std::wstring xml_event = /* rendered XML from EvtRenderEventXml */; + Row row; + auto status = parseWelXml(context, xml_event, row); + if (status.ok()) { + results.push_back(row); + } - return results; + return results; } } // namespace tables @@ -40,7 +48,6 @@ QueryRows genWindowsEventLog(QueryContext& context) { ``` ## Notes - -- Both functions live in the `osquery::tables` namespace and depend on `osquery/core/core.h` and `osquery/core/tables.h` -- `parseWelXml` accepts a `std::wstring` to handle the wide-character encoding used by the Windows Event Log XML renderer (`EvtRender`) -- `genXfilterFromConstraints` produces an XPath-compatible string, allowing osquery to push SQL `WHERE` clause predicates (channel, event ID, time range, etc.) down to the native Windows API, reducing unnecessary data transfer \ No newline at end of file +- Both functions operate within the `osquery::tables` namespace. +- `parseWelXml` accepts a wide string (`std::wstring`) to handle the UTF-16 encoding used by the Windows Event Log API. +- Intended for internal use by the `windows_event_log` virtual table implementation. \ No newline at end of file diff --git a/osquery/tables/system/windows/.windows_update_history.md b/osquery/tables/system/windows/.windows_update_history.md index 4abe6ae1aca..003840a90b6 100644 --- a/osquery/tables/system/windows/.windows_update_history.md +++ b/osquery/tables/system/windows/.windows_update_history.md @@ -1,47 +1,50 @@ -Header defining types and interfaces for querying Windows Update installation history via the Windows Update Agent (WUA) API. +Declares types and interfaces for querying and rendering Windows Update installation history via the Windows Update Agent (WUA) API within the osquery tables framework. ## Key Components -### `WindowsUpdateHistoryError` (enum class) -Enumerates error conditions that can occur when retrieving update history, covering failures at each stage of the WUA query pipeline β€” from initializing the update searcher through extracting individual entry fields (title, date, description, HResult, etc.). +### Enum: `WindowsUpdateHistoryError` +Enumerates error conditions that can occur during update history retrieval, covering failures at each stage of the WUA query pipeline (searcher initialization, count, query, individual entry field extraction, and identity resolution). -### `WindowsUpdateHistoryEntry` (struct) -Represents a single Windows Update history record with fields mapped directly from the WUA `IUpdateHistoryEntry` COM interface: +### Struct: `WindowsUpdateHistoryEntry` +Represents a single Windows Update history record with fields mapped directly from the WUA `IUpdateHistoryEntry` interface: | Field | Type | Description | -|---|---|---| +|-------|------|-------------| | `clientAppID` | `std::string` | Application that initiated the update | -| `date` | `LONGLONG` | Install/uninstall timestamp | -| `description` | `std::string` | Update description text | -| `hresult` | `LONG` | Win32 result code | +| `date` | `LONGLONG` | Installation timestamp | +| `description` | `std::string` | Update description | +| `hresult` | `LONG` | Operation result code | | `updateOp` | `UpdateOperation` | Install or uninstall operation | | `resultCode` | `OperationResultCode` | Success/failure outcome | -| `serverSelection` | `ServerSelection` | Source server (WSUS, Windows Update, etc.) | -| `updateID` | `std::string` | Unique update GUID | +| `serverSelection` | `ServerSelection` | Source server type (WSUS, Windows Update, etc.) | +| `updateID` | `std::string` | Unique update identifier | | `updateRevision` | `LONG` | Revision number of the update | -### `WindowsUpdateHistory` (type alias) -`std::vector` β€” a collection of update history entries. +### Type Alias: `WindowsUpdateHistory` +```c +using WindowsUpdateHistory = std::vector; +``` +A collection of parsed update history entries. -### `renderWindowsUpdateHistory()` -Converts a `WindowsUpdateHistory` collection into osquery's `QueryData` format for table output. +### Function: `renderWindowsUpdateHistory` +```c +QueryData renderWindowsUpdateHistory(const WindowsUpdateHistory& history); +``` +Converts a `WindowsUpdateHistory` collection into osquery `QueryData` rows for table output. ## Usage Example ```cpp -#include "windows_update_history.h" - -// After populating entries from the WUA COM API: -WindowsUpdateHistory history; -WindowsUpdateHistoryEntry entry; -entry.title = "Security Update KB5012345"; -entry.updateID = "a1b2c3d4-..."; -entry.hresult = S_OK; -entry.updateOp = uoInstallation; -entry.resultCode = orcSucceeded; -history.push_back(entry); - -// Render for osquery table response +WindowsUpdateHistory history = fetchUpdateHistory(); // populated via WUA COM API + +// Render entries into osquery table rows QueryData rows = renderWindowsUpdateHistory(history); -``` \ No newline at end of file + +for (const auto& row : rows) { + // Each row maps column names to string values + LOG(INFO) << row.at("title") << " - " << row.at("result_code"); +} +``` + +> **Note:** This header is Windows-only and depends on `wuapi.h`. It will not compile on non-Windows platforms. \ No newline at end of file diff --git a/osquery/tables/yara/.yara_utils.md b/osquery/tables/yara/.yara_utils.md index e77dd1ea1f2..4e4e8a32938 100644 --- a/osquery/tables/yara/.yara_utils.md +++ b/osquery/tables/yara/.yara_utils.md @@ -1,45 +1,49 @@ -Header file providing YARA integration utilities for osquery, enabling rule compilation, file scanning, and configuration parsing for threat detection workflows. +YARA integration utilities header for osquery, providing RAII wrappers, compiler helpers, and config plugin declarations for scanning files with YARA rules. ## Key Components ### `YaraRulesHandle` -RAII wrapper for `YR_RULES*` that ensures proper cleanup via `yr_rules_destroy()` on destruction. Non-copyable, move-only semantics prevent double-free issues. +RAII wrapper for `YR_RULES*` that automatically calls `yr_rules_destroy()` on destruction. Move-only (copy disabled). + +- `get()` β€” returns the raw `YR_RULES*` pointer ### `YaraCompilerError` / `YaraCompilerResult` -Error enum and `Expected` alias for typed compiler result handling. +Error enum and `Expected` alias for compiler results. + +### Free Functions + +| Function | Purpose | +|----------|---------| +| `yaraInitialize()` | Initialize the YARA library | +| `yaraFinalize()` | Finalize/clean up the YARA library | +| `compileSingleFile(file)` | Compile rules from a `.yar` file path | +| `compileFromString(buffer)` | Compile rules from an in-memory string | +| `handleRuleFiles(category, rule_files, rules)` | Load and compile a ptree of rule file paths into a rules map | +| `yaraShouldSkipFile(path, st_mode)` | Returns `true` for files that may cause hangs (e.g. special device files) | +| `YARACallback(...)` | Scan result callback passed to YARA scanner | +| `YARACompilerCallback(...)` | Compiler diagnostic callback for errors/warnings | ### `YARAConfigParserPlugin` -A `ConfigParserPlugin` that registers a `"yara"` top-level config key. Stores compiled rules grouped by category and maintains a URL allowlist for remote rule sources. +A `ConfigParserPlugin` that parses the top-level `"yara"` config key, compiles referenced rule files, and caches the results. -### Key Functions +- `rules()` β€” returns the compiled rules map (`group β†’ YaraRulesHandle`) +- `url_allow_set()` β€” returns the allowed URL set for remote rule fetching -| Function | Description | -|---|---| -| `yaraInitialize()` / `yaraFinalize()` | Lifecycle management for the YARA library | -| `compileSingleFile(file)` | Compiles a YARA rule from a file path | -| `compileFromString(buffer)` | Compiles YARA rules from an in-memory string | -| `handleRuleFiles(category, rule_files, rules)` | Processes a ptree of rule files into a compiled rules map | -| `yaraShouldSkipFile(path, st_mode)` | Guards against scanning files that may cause hangs | -| `YARACallback(...)` | Scan result callback invoked per YARA match event | -| `YARACompilerCallback(...)` | Compiler diagnostic callback for error/warning reporting | +### Constants +- `kYARAHome` β€” default YARA rules directory (`OSQUERY_HOME/yara/`) ## Usage Example ```c -// Initialize YARA, compile a rule file, and scan -if (yaraInitialize().ok()) { - auto result = compileSingleFile("/etc/osquery/yara/malware.yar"); - if (result) { - YaraRulesHandle handle = std::move(result.take()); - yr_rules_scan_file(handle.get(), "/tmp/suspect", 0, YARACallback, &row, 0); - } - yaraFinalize(); -} +// Initialize YARA, compile a rule file, and check for errors +yaraInitialize(); -// Access parsed config rules via plugin -auto parser = Config::getParser("yara"); -auto& compiledRules = parser->rules(); // map -``` +YaraCompilerResult result = compileSingleFile("/etc/osquery/yara/malware.yar"); +if (result) { + YaraRulesHandle handle = std::move(result.get()); + // Pass handle.get() to yr_rules_scan_file(...) +} -> **Note:** `kYARAHome` defaults to `OSQUERY_HOME/yara/` as the base directory for local rule files. \ No newline at end of file +yaraFinalize(); +``` \ No newline at end of file diff --git a/osquery/utils/.base64.md b/osquery/utils/.base64.md index 586f4812aeb..4f32fdc1305 100644 --- a/osquery/utils/.base64.md +++ b/osquery/utils/.base64.md @@ -1,25 +1,29 @@ -Provides Base64 encoding and decoding utilities within the `osquery::base64` namespace. +Provides base64 encoding and decoding utilities within the `osquery::base64` namespace. ## Key Components | Function | Signature | Description | |----------|-----------|-------------| -| `decode` | `std::string decode(std::string encoded)` | Decodes a Base64-encoded string and returns the original plaintext | -| `encode` | `std::string encode(const std::string_view unencoded)` | Encodes a raw string into its Base64 representation | +| `decode` | `std::string decode(std::string encoded)` | Decodes a base64-encoded string and returns the original plaintext | +| `encode` | `std::string encode(const std::string_view unencoded)` | Encodes a raw string into base64 format | ## Usage Example ```cpp #include -// Encode a plaintext string +// Encode a plain string std::string encoded = osquery::base64::encode("hello world"); // encoded == "aGVsbG8gd29ybGQ=" -// Decode back to original -std::string decoded = osquery::base64::decode(encoded); +// Decode a base64 string +std::string decoded = osquery::base64::decode("aGVsbG8gd29ybGQ="); // decoded == "hello world" ``` -> Both functions operate on standard C++ `std::string` / `std::string_view` types, requiring no external dependencies beyond the STL. \ No newline at end of file +## Notes + +- Both functions operate on standard C++ string types; `encode` accepts a `std::string_view` to avoid unnecessary copies. +- Defined under the nested `osquery::base64` namespace to avoid collisions with other base64 implementations. +- Header is guarded with `#pragma once`. \ No newline at end of file diff --git a/osquery/utils/.chars.md b/osquery/utils/.chars.md index 22888dbe309..6ea8f4e786e 100644 --- a/osquery/utils/.chars.md +++ b/osquery/utils/.chars.md @@ -1,41 +1,41 @@ -Utility header providing character and string encoding helpers within the `osquery` namespace. +Utility header providing character and string encoding helpers for the `osquery` namespace, focused on ASCII printability checks and UTF-8 string handling. ## Key Components | Symbol | Type | Description | -|---|---|---| -| `isPrintable` | Function | Checks whether all characters in a `string_view` are ASCII printable | -| `incUtf8StringIterator` | Template Function | Advances an iterator past the current UTF-8 multi-byte sequence, returning bytes consumed | -| `utf8StringSize` | Function | Returns the logical character count (not byte length) of a UTF-8 encoded string | -| `unescapeUnicode` | Function | Converts unicode escape sequences (e.g. `\uXXXX`) in a string to their UTF-8 representation | +|--------|------|-------------| +| `isPrintable` | Function | Returns `true` if all characters in the given `string_view` are ASCII printable | +| `incUtf8StringIterator` | Template Function | Advances an iterator past a single UTF-8 multi-byte sequence; returns bytes consumed | +| `utf8StringSize` | Function | Returns the character count (not byte count) of a UTF-8 encoded `std::string` | +| `unescapeUnicode` | Function | Converts unicode-escaped ASCII sequences (e.g. `\uXXXX`) into their literal string representation | ## Usage Example ```cpp #include -// Check if a query result value is safe to display -std::string value = "Hello, World!"; -if (osquery::isPrintable(value)) { - // safe to display +// Check printability +std::string raw = "hello\x01world"; +if (!osquery::isPrintable(raw)) { + // contains non-printable bytes } -// Get the display length of a UTF-8 string (character count, not bytes) -std::string utf8str = "caf\xC3\xA9"; // "cafΓ©" -size_t len = osquery::utf8StringSize(utf8str); // returns 4, not 5 - -// Decode a unicode escape sequence -std::string escaped = "\\u0041"; // escaped 'A' -std::string decoded = osquery::unescapeUnicode(escaped); // returns "A" - -// Manually iterate UTF-8 characters using the inline helper -auto it = utf8str.begin(); -auto end = utf8str.end(); -while (it != end) { - size_t bytes = osquery::incUtf8StringIterator(it, end); - // bytes = number of bytes in the current character -} +// Get true character length of a UTF-8 string +std::string utf8 = u8"hΓ©llo"; +size_t charCount = osquery::utf8StringSize(utf8); +// charCount == 5, not 6 (byte length) + +// Decode unicode escapes +std::string escaped = "caf\\u00e9"; +std::string decoded = osquery::unescapeUnicode(escaped); +// decoded == "cafΓ©" + +// Manually advance a UTF-8 iterator +std::string s = u8"ζ—₯本θͺž"; +auto it = s.begin(); +size_t bytes = osquery::incUtf8StringIterator(it, s.end()); +// bytes == 3 (one UTF-8 CJK character) ``` -> **Note:** `utf8StringSize` counts Unicode code points, not bytes. For ASCII-only strings both values are equal, but multi-byte sequences will differ. \ No newline at end of file +> **Note:** `incUtf8StringIterator` is an inline template helper intended primarily as an implementation detail for `utf8StringSize`, not typically called directly. \ No newline at end of file diff --git a/osquery/utils/.only_movable.md b/osquery/utils/.only_movable.md index bd15d7cdf85..3e041c65f93 100644 --- a/osquery/utils/.only_movable.md +++ b/osquery/utils/.only_movable.md @@ -1,20 +1,19 @@ -Defines a base class mixin within the `osquery` namespace that enforces move-only semantics, preventing accidental copying of derived objects while explicitly permitting move construction and assignment. +Defines a base class mixin within the `osquery` namespace that enforces move-only semantics, preventing accidental copying of derived classes while explicitly permitting move operations. ## Key Components ### `only_movable` (class) +A non-instantiable base class analogous to `boost::noncopyable`, designed for inheritance. Enforces move-only semantics by: -A non-copyable, move-enabled abstract base class analogous to `boost::noncopyable`. Designed to be inherited by classes that manage exclusive ownership of resources (e.g., file handles, sockets, threads). - -| Member | Type | Description | -|---|---|---| -| `only_movable()` | `protected default` | Default constructor | +| Member | Modifier | Purpose | +|--------|----------|---------| +| `only_movable()` | `protected default` | Default constructor for subclasses | | `~only_movable()` | `protected default` | Default destructor | -| `only_movable(only_movable&&)` | `protected default` | Move constructor | -| `operator=(only_movable&&)` | `protected default` | Move assignment | -| `only_movable(const only_movable&)` | `public delete` | Copy constructor β€” **disabled** | -| `operator=(const only_movable&)` | `public delete` | Copy assignment β€” **disabled** | +| `only_movable(only_movable&&)` | `protected noexcept default` | Allows move construction in children | +| `operator=(only_movable&&)` | `protected default` | Allows move assignment in children | +| `only_movable(const only_movable&)` | `public delete` | Prevents copy construction | +| `operator=(const only_movable&)` | `public delete` | Prevents copy assignment | ## Usage Example @@ -23,25 +22,24 @@ A non-copyable, move-enabled abstract base class analogous to `boost::noncopyabl namespace osquery { -// Derived class inherits move-only semantics automatically class ResourceHandle : public only_movable { public: ResourceHandle() = default; - // Explicitly default move operations from the base + // Explicitly opt-in to default move semantics inherited from only_movable ResourceHandle(ResourceHandle&&) noexcept = default; ResourceHandle& operator=(ResourceHandle&&) = default; - - // Copy is implicitly deleted via the base class }; -void example() { - ResourceHandle a; - ResourceHandle b = std::move(a); // OK: move allowed - // ResourceHandle c = b; // ERROR: copy deleted -} - } // namespace osquery + +// Allowed: move semantics work as expected +ResourceHandle a; +ResourceHandle b = std::move(a); // OK + +// Compile error: copy is explicitly deleted +// ResourceHandle c = b; // ERROR +// ResourceHandle d(b); // ERROR ``` -> **Design note:** Deleting the copy members in `public` scope (rather than `private`) produces clearer compiler diagnostics β€” the error explicitly states the function is deleted rather than inaccessible. \ No newline at end of file +> Inherit from `only_movable` whenever a class manages exclusive ownership of a resource (e.g., file descriptors, sockets, locks) where copying would be semantically incorrect or unsafe. \ No newline at end of file diff --git a/osquery/utils/.rot13.md b/osquery/utils/.rot13.md index b811ecddd3f..9666b25cd20 100644 --- a/osquery/utils/.rot13.md +++ b/osquery/utils/.rot13.md @@ -1,9 +1,9 @@ -Provides a ROT13 string decoding utility within the `osquery` namespace. +Provides a ROT13 decoding utility within the `osquery` namespace for transforming ROT13-encoded strings back to their original form. ## Key Components -- **`rotDecode(const std::string& rot_string)`** β€” Decodes a ROT13-encoded string and returns the plaintext result. +- **`rotDecode(const std::string& rot_string)`** β€” Decodes a ROT13-encoded string, returning the plaintext result. ## Usage Example @@ -16,4 +16,4 @@ std::string decoded = osquery::rotDecode(encoded); // decoded == "Hello, World!" ``` -> **Note:** ROT13 is a symmetric cipher β€” encoding and decoding use the same operation. Calling `rotDecode` on a plaintext string will also produce the ROT13-encoded output. \ No newline at end of file +> **Note:** ROT13 is a simple symmetric cipher β€” encoding and decoding are the same operation. Calling `rotDecode` on a plaintext string will also produce its ROT13 equivalent. \ No newline at end of file diff --git a/osquery/utils/aws/.aws_util.md b/osquery/utils/aws/.aws_util.md index 239572a168a..ab1e0761c63 100644 --- a/osquery/utils/aws/.aws_util.md +++ b/osquery/utils/aws/.aws_util.md @@ -1,52 +1,55 @@ -Utility header for integrating the AWS SDK into osquery, providing HTTP clients, credential providers, and helpers for instantiating AWS service clients with osquery-specific configuration. +Utility header providing AWS SDK integration for osquery, including custom HTTP client implementations, credential providers, region management, and client factory helpers for EC2, Kinesis, Firehose, and STS services. ## Key Components ### Constants -| Constant | Description | -|---|---| -| `kEc2MetadataUrl` | EC2 instance metadata service URL | -| `kImdsTokenResource` | URL resource for IMDSv2 token requests | -| `kImdsTokenHeader` / `kImdsTokenTtlHeader` | HTTP headers for IMDSv2 API calls | +- `kEc2MetadataUrl` β€” EC2 instance metadata URL +- `kImdsTokenResource`, `kImdsTokenHeader`, `kImdsTokenTtlHeader`, `kImdsTokenTtlDefaultValue` β€” IMDSv2 token request constants ### Classes -- **`AWSRegion`** β€” Validated wrapper around an AWS region string; constructed via `AWSRegion::make()` -- **`OsqueryHttpClientFactory`** / **`OsqueryHttpClient`** β€” Drop-in AWS SDK HTTP client backed by osquery's HTTP layer instead of libcurl -- **`OsqueryFlagsAWSCredentialsProvider`** β€” Reads credentials from osquery config flags (`aws_access_key_id`, `aws_secret_access_key`) -- **`OsquerySTSAWSCredentialsProvider`** β€” Obtains temporary credentials via AWS STS assume-role -- **`OsqueryAWSCredentialsProviderChain`** β€” Ordered credential resolution: STS β†’ flags β†’ profile β†’ env vars β†’ default profile β†’ EC2 IMDS +| Class | Purpose | +|---|---| +| `AWSRegion` | Validated wrapper around an AWS region string | +| `OsqueryHttpClientFactory` | Custom AWS HTTP client factory (avoids libcurl) | +| `OsqueryHttpClient` | osquery's AWS HTTP client implementation | +| `OsqueryFlagsAWSCredentialsProvider` | Credentials from osquery config flags | +| `OsquerySTSAWSCredentialsProvider` | Temporary credentials via STS assume role | +| `OsqueryAWSCredentialsProviderChain` | Priority credential chain (STS β†’ flags β†’ profile β†’ env β†’ EC2 metadata) | ### Free Functions -- **`initAwsSdk()`** β€” One-time SDK initialization using `OsqueryHttpClientFactory` -- **`makeAWSClient()`** β€” Template factory that instantiates Kinesis, Firehose, STS, or EC2 clients with resolved region and credentials -- **`getIMDSToken()`** β€” Fetches an IMDSv2 session token with retry logic -- **`getInstanceIDAndRegion()`** β€” Returns EC2 instance ID and region from metadata service -- **`setAWSProxy()`** β€” Applies proxy config from osquery flags to a `ClientConfiguration` -- **`appendLogTypeToJson()`** β€” Injects a `log_type` key into a JSON log string -- **`enableFIPSInClientConfig()`** β€” Enables FIPS-compliant endpoints for a given service +| Function | Purpose | +|---|---| +| `initAwsSdk()` | One-time SDK initialization with custom HTTP factory | +| `getIMDSToken()` | Fetches IMDSv2 token via HTTP PUT | +| `getInstanceIDAndRegion()` | Returns EC2 instance ID and region from metadata service | +| `setAWSProxy()` | Applies proxy config flags to `ClientConfiguration` | +| `setAwsClientConfig()` | Configures a `ClientConfiguration` for a given service and region | +| `makeAWSClient()` | Template factory instantiating a typed AWS client with osquery credentials | +| `appendLogTypeToJson()` | Mutates a JSON log string to append a `log_type` key | +| `enableFIPSInClientConfig()` | Enables FIPS endpoint on a client configuration | ## Usage Example -```cpp -// Initialize the SDK once at plugin startup +```c +// Initialize AWS SDK once at plugin startup osquery::initAwsSdk(); // Build a validated region auto region_result = osquery::AWSRegion::make("us-east-1"); -if (!region_result) { - return Status::failure("Invalid region"); -} +if (!region_result) { /* handle error */ } -// Instantiate a Kinesis client using osquery credentials + config +// Instantiate a Kinesis client using osquery credentials std::shared_ptr client; -Status s = osquery::makeAWSClient(client, region_result.get()); +osquery::Status s = osquery::makeAWSClient(client, region_result.get()); if (!s.ok()) { - return s; + LOG(ERROR) << "Failed to create Kinesis client: " << s.getMessage(); } -// Append log type metadata before sending to AWS -std::string log = R"({"host":"web01","level":"info"})"; -osquery::appendLogTypeToJson("result", log); -// log β†’ {"host":"web01","level":"info","log_type":"result"} +// Retrieve EC2 instance metadata +auto meta = osquery::getInstanceIDAndRegion(); +if (meta) { + std::string instance_id = meta->first; + std::string region = meta->second; +} ``` \ No newline at end of file diff --git a/osquery/utils/azure/.azure_util.md b/osquery/utils/azure/.azure_util.md index 9978ff40a42..7b4bd5c1975 100644 --- a/osquery/utils/azure/.azure_util.md +++ b/osquery/utils/azure/.azure_util.md @@ -1,42 +1,28 @@ -Utility header for fetching and parsing Azure Instance Metadata Service (IMDS) responses within the osquery framework. +Utility header for fetching and parsing Azure instance metadata within the osquery framework. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `getAzureKey` | Function | Extracts a string value by key from a parsed Azure metadata JSON document | -| `fetchAzureMetadata` | Function | Queries the Azure IMDS endpoint and populates a `JSON` document with the response | +- **`getAzureKey(JSON& doc, const std::string& key)`** β€” Extracts a string value from a parsed Azure metadata JSON document by key name. +- **`fetchAzureMetadata(JSON& doc)`** β€” Queries the Azure Instance Metadata Service (IMDS) and populates the provided `JSON` document with the response. Returns a `Status` indicating success or failure. ## Usage Example ```c #include -namespace osquery { +osquery::JSON doc; +osquery::Status status = osquery::fetchAzureMetadata(doc); -void readAzureInstanceInfo() { - JSON doc; - - // Fetch metadata from Azure IMDS endpoint - Status status = fetchAzureMetadata(doc); - if (!status.ok()) { - // Handle fetch error - return; - } - - // Extract individual fields from the metadata document - std::string vmId = getAzureKey(doc, "vmId"); - std::string region = getAzureKey(doc, "location"); - std::string vmSize = getAzureKey(doc, "vmSize"); +if (status.ok()) { + std::string region = osquery::getAzureKey(doc, "location"); + std::string vmId = osquery::getAzureKey(doc, "vmId"); } - -} // namespace osquery ``` ## Notes -- Both functions live in the `osquery` namespace. -- `fetchAzureMetadata` returns an `osquery::Status`, allowing callers to distinguish between network failures, parse errors, and missing data. -- `getAzureKey` returns an empty `std::string` when the requested key is absent in the document. -- Depends on `osquery/utils/json/json.h` for the `JSON` wrapper type and `osquery/utils/status/status.h` for structured error reporting. \ No newline at end of file +- Both functions operate within the `osquery` namespace. +- `fetchAzureMetadata` is expected to call the Azure IMDS endpoint (`http://169.254.169.254/metadata/instance`) internally. +- Returns an empty string from `getAzureKey` if the key is not present in the document. +- Depends on `osquery/utils/json/json.h` for JSON parsing and `osquery/utils/status/status.h` for error handling. \ No newline at end of file diff --git a/osquery/utils/config/.default_paths.md b/osquery/utils/config/.default_paths.md index 31adc838811..53b663e50bf 100644 --- a/osquery/utils/config/.default_paths.md +++ b/osquery/utils/config/.default_paths.md @@ -5,36 +5,38 @@ Defines platform-specific default filesystem paths used by osquery for configura | Macro | Purpose | |---|---| -| `OSQUERY_HOME` | Base directory for config, flagfile, extensions, and module autoload | -| `OSQUERY_DB_HOME` | RocksDB persistent storage location | -| `OSQUERY_SOCKET` | Unix domain socket (or Windows named pipe) path | -| `OSQUERY_PIDFILE` | Process ID file directory | -| `OSQUERY_LOG_HOME` | Log output directory (used by the filesystem logger plugin) | -| `OSQUERY_CERTS_HOME` | TLS/SSL certificate bundle directory | +| `OSQUERY_HOME` | Base directory for config, flagfiles, extensions, and module autoload | +| `OSQUERY_DB_HOME` | Root path for RocksDB persistent storage | +| `OSQUERY_SOCKET` | Location of the osquery communication socket (or named pipe on Windows) | +| `OSQUERY_PIDFILE` | Directory for the process ID file | +| `OSQUERY_LOG_HOME` | Log output directory when the filesystem plugin is active | +| `OSQUERY_CERTS_HOME` | TLS certificate store location | ## Platform Defaults -| Macro | Linux | Windows | FreeBSD | Other | -|---|---|---|---|---| -| `OSQUERY_HOME` | `/etc/osquery/` | `\Program Files\osquery\` | `/var/db/osquery/` | `/var/osquery/` | -| `OSQUERY_DB_HOME` | `/var/osquery/` | *(same as HOME)* | *(same as HOME)* | *(same as HOME)* | -| `OSQUERY_SOCKET` | `/var/osquery/` | `\\.\pipe\` | `/var/run/` | *(same as DB_HOME)* | -| `OSQUERY_LOG_HOME` | `/var/log/osquery/` | `...\log\` | `/var/log/osquery/` | `/var/log/osquery/` | -| `OSQUERY_CERTS_HOME` | `/opt/osquery/share/osquery/certs/` | `...\certs\` | `/etc/ssl/` | `...certs/` | +```text +Platform OSQUERY_HOME OSQUERY_DB_HOME +--------- ------------------------------ ---------------------- +Linux /etc/osquery/ /var/osquery/ +Windows \Program Files\osquery\ (same as HOME) +FreeBSD /var/db/osquery/ (same as HOME) +Other /var/osquery/ (same as HOME) +``` ## Usage Example ```c #include "default_paths.h" -#include - -int main(void) { - printf("Config home: %s\n", OSQUERY_HOME); - printf("Database home: %s\n", OSQUERY_DB_HOME); - printf("Log directory: %s\n", OSQUERY_LOG_HOME); - printf("Certificates: %s\n", OSQUERY_CERTS_HOME); - return 0; -} + +// Reference compile-time paths directly via macros +const char *config_dir = OSQUERY_HOME; +const char *db_dir = OSQUERY_DB_HOME; +const char *log_dir = OSQUERY_LOG_HOME; +const char *certs_dir = OSQUERY_CERTS_HOME; + +// String concatenation at compile time +const char *flag_file = OSQUERY_HOME "osquery.flags"; +const char *db_path = OSQUERY_DB_HOME "osquery.db"; ``` -> These macros serve as compile-time defaults and are typically overridden at runtime via CLI flags or configuration files. \ No newline at end of file +> All macros are compile-time string literals resolved via `#ifdef` guards, producing zero runtime overhead. No functions or types are declared in this header. \ No newline at end of file diff --git a/osquery/utils/conversions/.split.md b/osquery/utils/conversions/.split.md index 80799ce338d..fc62bdf6027 100644 --- a/osquery/utils/conversions/.split.md +++ b/osquery/utils/conversions/.split.md @@ -1,35 +1,35 @@ -Utility header providing string splitting functions within the `osquery` namespace, offering delimiter-based and whitespace tokenization for `std::string` and `std::string_view` types. +Utility header providing string splitting functions within the `osquery` namespace. ## Key Components | Function | Signature | Description | -|---|---|---| -| `split` | `(string, delim="\t ")` | Splits a string by a delimiter; defaults to whitespace/tab | -| `split` | `(string, char, occurrences)` | Splits a string by a char delimiter up to N times | -| `vsplit` | `(string_view, char)` | High-performance split returning `string_view` tokens (2x+ faster) | +|----------|-----------|-------------| +| `split` | `(string, string)` | Splits a string by a delimiter (defaults to whitespace/tab) | +| `split` | `(string, char, size_t)` | Splits a string by a char delimiter, limited to N occurrences | +| `vsplit` | `(string_view, char)` | Fast split returning `string_view` slices β€” at least 2Γ— faster than `split` | ## Usage Example ```cpp -#include +#include "split.h" -// Whitespace split (default) -auto tokens = osquery::split("foo bar baz"); +// Default whitespace split +std::vector tokens = osquery::split("foo bar baz"); // β†’ ["foo", "bar", "baz"] -// Custom string delimiter -auto parts = osquery::split("a,b,c", ","); -// β†’ ["a", "b", "c"] +// Custom delimiter +std::vector parts = osquery::split("a,b,c,d", ","); +// β†’ ["a", "b", "c", "d"] -// Limited occurrences -auto limited = osquery::split("a:b:c:d", ':', 2); +// Limited occurrences (splits at most 2 times) +std::vector limited = osquery::split("a:b:c:d", ':', 2); // β†’ ["a", "b", "c:d"] -// High-performance string_view split (no heap allocations per token) -std::string_view line = "col1|col2|col3"; -auto views = osquery::vsplit(line, '|'); -// β†’ ["col1", "col2", "col3"] as string_views +// Zero-copy split via string_view (preferred for performance) +std::string_view data = "col1|col2|col3"; +std::vector cols = osquery::vsplit(data, '|'); +// β†’ ["col1", "col2", "col3"] ``` -> **Performance note:** Prefer `vsplit` over `split` when processing large strings or log lines where you need fast tokenization without unnecessary heap allocations. Since it returns `std::string_view` tokens, ensure the source string outlives the result vector. \ No newline at end of file +> **Performance note:** Prefer `vsplit` when parsing large strings or high-frequency paths β€” it avoids heap allocations by returning `string_view` slices into the original buffer. Ensure the source string outlives the returned views. \ No newline at end of file diff --git a/osquery/utils/conversions/.trim.md b/osquery/utils/conversions/.trim.md index a04e925d550..22435563d2b 100644 --- a/osquery/utils/conversions/.trim.md +++ b/osquery/utils/conversions/.trim.md @@ -1,9 +1,9 @@ -Provides a utility function for trimming leading and trailing whitespace from a `std::string_view` within the `osquery` namespace. +Provides a utility function for trimming leading and trailing whitespace from a `string_view` within the `osquery` namespace. ## Key Components -- **`osquery::trim(std::string_view input)`** β€” Returns a `std::string_view` with all leading and trailing spaces removed from the input. Non-owning; no heap allocation. +- **`osquery::trim(std::string_view input)`** β€” Returns a `std::string_view` with all leading and trailing spaces removed. Operates non-destructively on the original string data. ## Usage Example @@ -14,8 +14,8 @@ Provides a utility function for trimming leading and trailing whitespace from a int main() { std::string_view raw = " hello world "; std::string_view trimmed = osquery::trim(raw); - std::cout << trimmed; // "hello world" + std::cout << trimmed; // Output: "hello world" } ``` -> **Note:** The returned `std::string_view` references the original string's memory. Ensure the source string outlives the returned view. \ No newline at end of file +> **Note:** The returned `std::string_view` is a non-owning reference into the original string's memory. Ensure the source string outlives any use of the trimmed view. \ No newline at end of file diff --git a/osquery/utils/conversions/.tryto.md b/osquery/utils/conversions/.tryto.md index 23304fc2f0e..08bbd8e378c 100644 --- a/osquery/utils/conversions/.tryto.md +++ b/osquery/utils/conversions/.tryto.md @@ -1,33 +1,38 @@ -Provides type-safe conversion utilities for transforming between C++ types (string-to-integer, string-to-bool, and identity conversions) using the `Expected` error-handling pattern instead of exceptions. +Provides type-safe conversion utilities for transforming values between compatible types, with structured error handling via the `Expected` monad instead of exceptions. ## Key Components -### `ConversionError` Enum +### `ConversionError` (enum class) Error codes returned on failed conversions: -- `InvalidArgument` β€” input string cannot be parsed +- `InvalidArgument` β€” input string could not be parsed - `OutOfRange` β€” parsed value exceeds the target type's range - `Unknown` β€” unexpected error during conversion -### `tryTo(from, base)` β€” Primary API -Overloaded template function with three specializations: -- **Identity conversion** β€” when `ToType` matches `FromType`, forwards the value directly -- **String β†’ Integer** β€” converts `std::string`/`std::wstring` to any integer type (`int`, `long`, `unsigned long long`, etc.) with configurable base (default: 10) -- **String β†’ Bool** β€” parses human-readable boolean strings (`"1"`, `"yes"`, `"y"`, `"true"`, `"0"`, `"no"`, `"n"`, `"false"`, etc.) +### `tryTo(from)` (function templates) +Three overloads: -### `impl::throwingStringToInt(from, base)` -Internal dispatch layer that selects the correct STL conversion function (`stoi`, `stol`, `stoll`, `stoul`, `stoull`) based on the target integer type. +| Overload | Behavior | +|---|---| +| Same-type identity | Returns the value forwarded as-is | +| `[w]string β†’ integer` | Wraps `std::stoi`/`stol`/`stoll`/`stoul`/`stoull`; accepts optional `base` (default `10`) | +| `string β†’ bool` | Delegates to `impl::stringToBool`; accepts `"1"`, `"y"`, `"yes"`, `"0"`, `"n"`, `"no"`, etc. | + +### `impl::throwingStringToInt` (internal) +SFINAE-dispatched helpers that select the correct `std::sto*` function based on the exact integer target type. ### `operator"" _sz` -User-defined literal converting `unsigned long long` to `uint64_t` for size expressions. +User-defined literal converting `unsigned long long` to `uint64_t` for size expressions (e.g., `4_sz`). ## Usage Example ```cpp +#include + // String to integer auto result = osquery::tryTo("42"); if (result) { - int value = result.get(); // 42 + int value = result.get(); // 42 } // String to integer with base @@ -36,12 +41,12 @@ auto hex = osquery::tryTo("FF", 16); // String to bool auto flag = osquery::tryTo("yes"); if (flag) { - bool val = flag.get(); // true + bool b = flag.get(); // true } // Error handling auto bad = osquery::tryTo("not_a_number"); if (bad.isError()) { - // bad.getError() == ConversionError::InvalidArgument + // bad.getError() == ConversionError::InvalidArgument } ``` \ No newline at end of file diff --git a/osquery/utils/conversions/darwin/.cfdata.md b/osquery/utils/conversions/darwin/.cfdata.md index 41967c69b79..59beec6fcf3 100644 --- a/osquery/utils/conversions/darwin/.cfdata.md +++ b/osquery/utils/conversions/darwin/.cfdata.md @@ -1,13 +1,9 @@ -Utility header for converting Core Foundation `CFDataRef` objects to C++ `std::string` values on macOS/Darwin platforms. +Utility header providing a conversion function from CoreFoundation `CFDataRef` to a C++ `std::string` for use within osquery's macOS-specific code paths. ## Key Components -### Functions - -| Function | Description | -|---|---| -| `stringFromCFData(const CFDataRef& cf_data)` | Converts a Core Foundation `CFDataRef` to a `std::string` | +- **`stringFromCFData(const CFDataRef& cf_data)`** β€” Converts a CoreFoundation `CFDataRef` object into a UTF-8 `std::string`. Declared within the `osquery` namespace. ## Usage Example @@ -15,13 +11,12 @@ Utility header for converting Core Foundation `CFDataRef` objects to C++ `std::s #include "cfdata.h" #include -// Convert a CFDataRef to std::string -CFDataRef rawData = /* obtained from a CF API */; +// Example: Convert CFDataRef to std::string +CFDataRef rawData = ...; // obtained from a CF API call std::string result = osquery::stringFromCFData(rawData); ``` ## Notes -- macOS/Darwin only β€” depends on the `CoreFoundation` framework -- Lives in the `osquery` namespace -- Typically used when consuming binary or raw byte data from macOS system APIs that return `CFDataRef` handles \ No newline at end of file +- macOS/Darwin only β€” depends on `CoreFoundation` framework. +- Useful when bridging Apple's Core Foundation data types with osquery's standard C++ interfaces. \ No newline at end of file diff --git a/osquery/utils/conversions/darwin/.cfdictionary.md b/osquery/utils/conversions/darwin/.cfdictionary.md index a367642cbe1..76af46c10eb 100644 --- a/osquery/utils/conversions/darwin/.cfdictionary.md +++ b/osquery/utils/conversions/darwin/.cfdictionary.md @@ -1,37 +1,28 @@ -Utility header for extracting typed values from Core Foundation `CFDictionaryRef` objects within the osquery macOS/Darwin data collection framework. +Utility header for extracting typed values from Core Foundation `CFDictionaryRef` objects within the osquery macOS subsystem. ## Key Components -### Functions - -- **`getPropertiesFromDictionary(dict, key)`** β€” Looks up a key in a `CFDictionaryRef` and returns the corresponding value as a `std::string`. Relies on the sibling CF conversion helpers (`cfdata.h`, `cfnumber.h`, `cfstring.h`) to handle type coercion. - -### Dependencies - -| Header | Purpose | -|---|---| -| `cfdata.h` | CF `Data` β†’ `std::string` conversion | -| `cfnumber.h` | CF `Number` β†’ `std::string` conversion | -| `cfstring.h` | CF `String` β†’ `std::string` conversion | -| `CoreFoundation/CoreFoundation.h` | Native CF types (`CFDictionaryRef`, etc.) | +- **`getPropertiesFromDictionary`** β€” Looks up a key in a `CFDictionaryRef` and returns the corresponding value as a `std::string`. Internally leverages the sibling CF conversion utilities (`cfdata.h`, `cfnumber.h`, `cfstring.h`) to handle type coercion from Core Foundation types. ## Usage Example ```cpp #include "cfdictionary.h" -#include -// Assume `plistDict` is a CFDictionaryRef obtained from a parsed plist +// Assume `plistDict` is a CFDictionaryRef obtained from a plist or system API CFDictionaryRef plistDict = /* ... */; -std::string version = osquery::getPropertiesFromDictionary(plistDict, "CFBundleVersion"); -std::string bundleId = osquery::getPropertiesFromDictionary(plistDict, "CFBundleIdentifier"); +std::string value = osquery::getPropertiesFromDictionary(plistDict, "CFBundleVersion"); -// Returns an empty string if the key is missing or conversion fails -if (!version.empty()) { - // use version string +if (!value.empty()) { + // Use the extracted string value + LOG(INFO) << "Bundle version: " << value; } ``` -> **Platform note:** This header is macOS/Darwin-only. It depends on `CoreFoundation` and is compiled exclusively for Apple platform targets within osquery. \ No newline at end of file +## Notes + +- macOS/Darwin only β€” depends on `CoreFoundation.framework`. +- Returns an empty `std::string` when the key is absent or the value cannot be converted. +- Part of the `osquery` namespace; include alongside `cfstring.h`, `cfnumber.h`, and `cfdata.h` for full CF type coverage. \ No newline at end of file diff --git a/osquery/utils/conversions/darwin/.cfnumber.md b/osquery/utils/conversions/darwin/.cfnumber.md index b80a7db02a0..3c0c0b712b1 100644 --- a/osquery/utils/conversions/darwin/.cfnumber.md +++ b/osquery/utils/conversions/darwin/.cfnumber.md @@ -1,32 +1,37 @@ -Utility header for converting Apple CoreFoundation `CFNumberRef` values to C++ `std::string` objects within the osquery framework. +Utility header for converting macOS CoreFoundation `CFNumberRef` values to C++ `std::string` objects within the osquery framework. ## Key Components ### Functions -| Function | Description | -|---|---| -| `stringFromCFNumber(const CFDataRef&)` | Converts a `CFNumberRef` to a `std::string`, auto-detecting the number type | -| `stringFromCFNumber(const CFDataRef&, CFNumberType)` | Converts a `CFNumberRef` to a `std::string` using an explicitly specified `CFNumberType` | +- **`stringFromCFNumber(const CFDataRef& cf_number)`** β€” Converts a `CFDataRef` to a `std::string`, automatically inferring the numeric type. +- **`stringFromCFNumber(const CFDataRef& cf_number, CFNumberType type)`** β€” Converts a `CFDataRef` to a `std::string` using an explicitly specified `CFNumberType` (e.g., `kCFNumberIntType`, `kCFNumberDoubleType`). ## Usage Example -```cpp -#include +```c +#include "cfnumber.h" #include -// Auto-detect number type -CFNumberRef cf_num = /* obtained from a CF API */; -std::string value = osquery::stringFromCFNumber( - reinterpret_cast(cf_num) -); +// Auto-inferred type conversion +int value = 42; +CFNumberRef cf_num = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &value); +std::string result = osquery::stringFromCFNumber((CFDataRef)cf_num); +// result == "42" // Explicit type conversion -std::string int_value = osquery::stringFromCFNumber( - reinterpret_cast(cf_num), - kCFNumberIntType -); +double dval = 3.14; +CFNumberRef cf_dbl = CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &dval); +std::string result2 = osquery::stringFromCFNumber((CFDataRef)cf_dbl, kCFNumberDoubleType); +// result2 == "3.14" + +CFRelease(cf_num); +CFRelease(cf_dbl); ``` -> **Note:** This header is macOS-only, as it depends on the CoreFoundation framework. It is intended for use in osquery Darwin-specific table implementations that parse CF property list or registry data structures. \ No newline at end of file +## Notes + +- macOS/Darwin only β€” depends on the `CoreFoundation` framework. +- Both functions are defined in the `osquery` namespace. +- Prefer the explicit-type overload when the `CFNumberType` is known ahead of time to avoid potential type inference mismatches. \ No newline at end of file diff --git a/osquery/utils/conversions/darwin/.cfstring.md b/osquery/utils/conversions/darwin/.cfstring.md index 2cc21378fec..de0e0a12e72 100644 --- a/osquery/utils/conversions/darwin/.cfstring.md +++ b/osquery/utils/conversions/darwin/.cfstring.md @@ -1,9 +1,9 @@ -Utility header for converting Apple CoreFoundation `CFStringRef` objects to standard C++ `std::string` values within the osquery namespace. +Utility header for converting Apple CoreFoundation `CFStringRef` types to C++ `std::string` within the osquery framework. ## Key Components -- **`stringFromCFString(const CFStringRef& cf_string)`** β€” Converts a CoreFoundation `CFStringRef` to a `std::string`. Useful when working with macOS system APIs that return CF types. +- **`stringFromCFString(const CFStringRef& cf_string)`** β€” Converts a CoreFoundation `CFStringRef` to a `std::string`. Declared inside the `osquery` namespace. ## Usage Example @@ -11,15 +11,12 @@ Utility header for converting Apple CoreFoundation `CFStringRef` objects to stan #include "cfstring.h" #include -// Convert a CFStringRef returned by a macOS API to std::string -CFStringRef cf_name = CFStringCreateWithCString( - kCFAllocatorDefault, "example", kCFStringEncodingUTF8); - -std::string name = osquery::stringFromCFString(cf_name); -CFRelease(cf_name); +// Convert a CFStringRef to std::string +CFStringRef cf_str = CFSTR("Hello, macOS!"); +std::string result = osquery::stringFromCFString(cf_str); ``` ## Notes -- macOS only β€” depends on `CoreFoundation`, which is not available on Linux or Windows. -- Caller is responsible for memory management of the `CFStringRef` (follow standard CF ownership rules). \ No newline at end of file +- macOS only β€” depends on `CoreFoundation` framework. +- Handles the encoding bridge between CF's internal string representation and `std::string`. \ No newline at end of file diff --git a/osquery/utils/conversions/darwin/.cftime.md b/osquery/utils/conversions/darwin/.cftime.md index bf5ea0812dd..67a3aea1e23 100644 --- a/osquery/utils/conversions/darwin/.cftime.md +++ b/osquery/utils/conversions/darwin/.cftime.md @@ -1,9 +1,9 @@ -Utility header for converting CoreFoundation `CFAbsoluteTime` values to standard C++ strings on macOS/Apple platforms. +Utility header for converting CoreFoundation `CFAbsoluteTime` values to standard C++ strings within the osquery macOS subsystem. ## Key Components -- **`stringFromCFAbsoluteTime(const CFDataRef& cf_abstime)`** β€” Converts a `CFDataRef` containing a `CFAbsoluteTime` value into a human-readable `std::string`. +- **`stringFromCFAbsoluteTime(const CFDataRef& cf_abstime)`** β€” Converts a `CFDataRef` containing a CoreFoundation absolute time value into a human-readable `std::string`. ## Usage Example @@ -11,15 +11,15 @@ Utility header for converting CoreFoundation `CFAbsoluteTime` values to standard #include "cftime.h" #include -// Assume cf_data is a CFDataRef containing a CFAbsoluteTime -CFDataRef cf_data = /* ... obtain from system API ... */; +// Assume cf_data holds a CFAbsoluteTime encoded as CFDataRef +CFDataRef cf_data = /* ... obtained from a CF API ... */; std::string timeStr = osquery::stringFromCFAbsoluteTime(cf_data); -// timeStr now contains a formatted time string +// timeStr now contains a human-readable timestamp string ``` ## Notes -- macOS/Apple only β€” depends on `CoreFoundation` framework. -- Lives within the `osquery` namespace. -- Typical use case: parsing macOS plist or system data where timestamps are stored as `CFAbsoluteTime` (seconds since Jan 1, 2001). \ No newline at end of file +- macOS/Darwin only β€” depends on `CoreFoundation`, not available on Linux or Windows. +- Lives in the `osquery` namespace. +- Typically used when parsing macOS system data sources (e.g., plist files, keychain entries) that return `CFAbsoluteTime` values needing normalization for osquery table output. \ No newline at end of file diff --git a/osquery/utils/conversions/darwin/.iokit.md b/osquery/utils/conversions/darwin/.iokit.md index 2dfd6cc9bab..b7513aa1f3d 100644 --- a/osquery/utils/conversions/darwin/.iokit.md +++ b/osquery/utils/conversions/darwin/.iokit.md @@ -1,51 +1,40 @@ -Provides IOKit utility functions and constants for querying macOS hardware device information via Apple's IOKit framework. +Provides IOKit utility declarations for querying and parsing macOS hardware device properties via the IOKit framework within the osquery namespace. ## Key Components -### Device Class Name Constants +**Device Class Name Constants** +- `kIOUSBDeviceClassName_` β€” USB devices +- `kIOPCIDeviceClassName_` β€” PCI devices +- `kIOPlatformExpertDeviceClassName_` β€” Platform expert devices +- `kIOACPIPlatformDeviceClassName_` β€” ACPI platform devices +- `kIOPlatformDeviceClassName_` β€” Generic platform devices +- `kAppleARMIODeviceClassName_` β€” Apple ARM I/O devices -String constants identifying IOKit device class types: +**`IOKitPCIProperties` struct** +Holds parsed PCI device metadata: `vendor_id`, `model_id`, `pci_class`, and `driver`. Constructed from the raw `"compatible"` property string returned by IOKit. -| Constant | Device Type | -|---|---| -| `kIOUSBDeviceClassName_` | USB devices | -| `kIOPCIDeviceClassName_` | PCI devices | -| `kIOPlatformExpertDeviceClassName_` | Platform expert devices | -| `kIOACPIPlatformDeviceClassName_` | ACPI platform devices | -| `kIOPlatformDeviceClassName_` | Generic platform devices | -| `kAppleARMIODeviceClassName_` | Apple ARM I/O devices | - -### `IOKitPCIProperties` Struct - -Holds parsed PCI device properties extracted from the IOKit `"compatible"` property string: - -- `vendor_id` β€” PCI vendor identifier -- `model_id` β€” PCI model/device identifier -- `pci_class` β€” PCI device class -- `driver` β€” Associated driver name - -Constructed via `IOKitPCIProperties(const std::string& compatible)` which parses the raw compatible string automatically. - -### Functions - -- **`getIOKitProperty`** β€” Retrieves a string property value from a `CFMutableDictionaryRef` by key -- **`getNumIOKitProperty`** β€” Retrieves a numeric (`long long int`) property value from a `CFMutableDictionaryRef` by key -- **`idToHex`** β€” Converts a device ID string to its hexadecimal representation in-place +**Functions** +- `getIOKitProperty()` β€” Extracts a string value from a `CFMutableDictionaryRef` by key +- `getNumIOKitProperty()` β€” Extracts a numeric (`long long int`) value from a `CFMutableDictionaryRef` by key +- `idToHex()` β€” Converts a device ID string to its hexadecimal representation in-place ## Usage Example ```c -CFMutableDictionaryRef details = getDeviceDetails(); // IOKit registry entry +#include "iokit.h" + +// Retrieve a string property from an IOKit device dictionary +std::string vendor = osquery::getIOKitProperty(deviceDetails, "vendor-id"); -// Read string and numeric properties -std::string vendor = osquery::getIOKitProperty(details, "vendor-id"); -long long int deviceClass = osquery::getNumIOKitProperty(details, "class-code"); +// Retrieve a numeric property +long long int revisionId = osquery::getNumIOKitProperty(deviceDetails, "revision-id"); -// Parse PCI compatible string -osquery::IOKitPCIProperties pci("pci8086,1502"); -// pci.vendor_id == "8086", pci.model_id == "1502" +// Convert a raw ID to hex format +std::string deviceId = "8086"; +osquery::idToHex(deviceId); // modifies in-place -// Convert ID to hex representation -osquery::idToHex(vendor); +// Parse PCI compatible string into structured properties +osquery::IOKitPCIProperties props("pci8086,1502"); +// props.vendor_id == "8086", props.model_id == "1502" ``` \ No newline at end of file diff --git a/osquery/utils/conversions/windows/.strings.md b/osquery/utils/conversions/windows/.strings.md index a1181fe9b65..d4be9dbb52f 100644 --- a/osquery/utils/conversions/windows/.strings.md +++ b/osquery/utils/conversions/windows/.strings.md @@ -4,36 +4,39 @@ Windows-specific string utility header providing conversion functions between na ## Key Components | Function | Description | -|---|---| -| `stringToWstring` | Converts `std::string` β†’ `std::wstring` | +|----------|-------------| +| `stringToWstring()` | Converts `std::string` β†’ `std::wstring` | | `wstringToString(const std::wstring&)` | Converts `std::wstring` β†’ `std::string` | | `wstringToString(const wchar_t*)` | Converts wide C-string β†’ `std::string` | -| `cimDatetimeToUnixtime` | Converts WMI CIM datetime string β†’ Unix timestamp (`LONGLONG`) | -| `bstrToString` | Converts Windows `BSTR` β†’ `std::string` | -| `swapEndianess` | Swaps byte order of a string (little ↔ big endian) | -| `errorDwordToString` | Converts a Windows `DWORD` error code β†’ human-readable `std::string` | - -All functions reside in the `osquery` namespace and are Windows-only (depends on `comutil.h`). +| `cimDatetimeToUnixtime()` | Converts WMI CIM datetime string β†’ Unix timestamp (`LONGLONG`) | +| `bstrToString()` | Converts Windows `BSTR` type β†’ `std::string` | +| `swapEndianess()` | Swaps byte order of a string (little ↔ big endian) | +| `errorDwordToString()` | Converts a Windows `DWORD` error code β†’ human-readable `std::string` | ## Usage Example ```c #include "strings.h" -// Wide/narrow string conversion -std::wstring wide = osquery::stringToWstring("Hello, Windows"); +// Narrow <-> Wide string conversion +std::wstring wide = osquery::stringToWstring("Hello, Windows!"); std::string narrow = osquery::wstringToString(wide); -// WMI CIM datetime to Unix timestamp -LONGLONG ts = osquery::cimDatetimeToUnixtime("20240101120000.000000+000"); +// Convert WMI CIM datetime to Unix timestamp +std::string cimTime = "20240101120000.000000+000"; +LONGLONG unixTime = osquery::cimDatetimeToUnixtime(cimTime); -// Convert BSTR from WMI query result +// Convert Windows BSTR (e.g., from WMI query result) BSTR bstr = SysAllocString(L"WMI Result"); std::string result = osquery::bstrToString(bstr); +SysFreeString(bstr); -// Decode a Windows error code -std::string errMsg = osquery::errorDwordToString(GetLastError()); +// Decode a Windows API error code +DWORD err = GetLastError(); +std::string errMsg = osquery::errorDwordToString(err); -// Swap endianness (e.g., for GUID/UUID processing) +// Swap endianness of a raw byte string std::string swapped = osquery::swapEndianess(rawBytes); -``` \ No newline at end of file +``` + +> **Note:** This header is Windows-only. It depends on `` and links against Windows COM utilities. All functions reside in the `osquery` namespace. \ No newline at end of file diff --git a/osquery/utils/conversions/windows/.windows_time.md b/osquery/utils/conversions/windows/.windows_time.md index dcc19b84842..9860dd7dc62 100644 --- a/osquery/utils/conversions/windows/.windows_time.md +++ b/osquery/utils/conversions/windows/.windows_time.md @@ -1,38 +1,38 @@ -Windows-specific utility header providing timestamp conversion functions that normalize various Windows time formats into Unix epoch values for consistent cross-platform time handling within osquery. +Windows-specific utility header providing timestamp conversion functions that translate various Windows time formats into Unix epoch values. ## Key Components | Function | Input | Description | |---|---|---| -| `filetimeToUnixtime` | `const FILETIME&` | Converts Windows `FILETIME` structure to Unix epoch | -| `longIntToUnixtime` | `LARGE_INTEGER&` | Converts Windows `LARGE_INTEGER` value to Unix epoch | -| `littleEndianToUnixTime` | `const std::string&` | Converts little-endian encoded `FILETIME` (as used in the Windows Registry) to Unix epoch | -| `parseFatTime` | `const std::string&` | Parses and converts FAT/DOS timestamp format to Unix epoch | +| `filetimeToUnixtime` | `const FILETIME&` | Converts a Windows `FILETIME` structure to Unix epoch | +| `longIntToUnixtime` | `LARGE_INTEGER&` | Converts a Windows `LARGE_INTEGER` timestamp to Unix epoch | +| `littleEndianToUnixTime` | `const std::string&` | Converts a little-endian encoded `FILETIME` (as used in the Windows Registry) to Unix epoch | +| `parseFatTime` | `const std::string&` | Parses and converts FAT/DOS filesystem timestamps to Unix epoch | -All functions return `LONGLONG` representing seconds since the Unix epoch. +All functions return a `LONGLONG` representing the resulting Unix epoch timestamp. ## Usage Example ```c -#include +#include -namespace osquery { +// Convert a FILETIME (e.g. from a WIN32_FILE_ATTRIBUTE_DATA) +WIN32_FILE_ATTRIBUTE_DATA fileInfo; +GetFileAttributesEx(L"C:\\file.txt", GetFileExInfoStandard, &fileInfo); +LONGLONG unixTime = osquery::filetimeToUnixtime(fileInfo.ftLastWriteTime); -// Convert a FILETIME (e.g., from file metadata) to Unix epoch -FILETIME ft; -GetSystemTimeAsFileTime(&ft); -LONGLONG unixTime = filetimeToUnixtime(ft); +// Convert a LARGE_INTEGER (e.g. from NtQueryInformationFile) +LARGE_INTEGER li; +LONGLONG unixTime2 = osquery::longIntToUnixtime(li); -// Convert a registry-stored little-endian FILETIME string -std::string regTimeData = getRegistryTimeValue(); -LONGLONG regUnixTime = littleEndianToUnixTime(regTimeData); +// Convert a little-endian Registry FILETIME value +std::string regData = /* raw registry bytes */; +LONGLONG unixTime3 = osquery::littleEndianToUnixTime(regData); -// Convert a FAT/DOS timestamp from a filesystem entry -std::string dosTime = getFatTimestamp(); -LONGLONG fatUnixTime = parseFatTime(dosTime); - -} // namespace osquery +// Convert a FAT/DOS timestamp from a ZIP or FAT filesystem +std::string dosData = /* raw FAT time bytes */; +LONGLONG fatUnix = osquery::parseFatTime(dosData); ``` -> **Note:** This header is Windows-only and guarded by `#pragma once`. It depends on `osquery/utils/system/system.h` for Windows type availability (`LONGLONG`, `FILETIME`, `LARGE_INTEGER`). Use these helpers whenever normalizing Windows-native timestamps to ensure consistent Unix epoch output across osquery tables. \ No newline at end of file +> **Note:** This header is Windows-only and depends on `osquery/utils/system/system.h` for platform types. It is compiled exclusively on Windows targets within the osquery namespace. \ No newline at end of file diff --git a/osquery/utils/info/.platform_type.md b/osquery/utils/info/.platform_type.md index 1c36552b97b..aec275f8b1c 100644 --- a/osquery/utils/info/.platform_type.md +++ b/osquery/utils/info/.platform_type.md @@ -1,18 +1,14 @@ -Defines platform detection types and compile-time constants for identifying the target OS at both build time and runtime within the osquery framework. +Defines platform detection types and constants for runtime and compile-time identification of the operating system environment within the osquery framework. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `PlatformType` | `enum class` | Bitmask enum for platform identification (`TYPE_POSIX`, `TYPE_WINDOWS`, `TYPE_BSD`, `TYPE_LINUX`, `TYPE_OSX`, `TYPE_FREEBSD`) | -| `kPlatformType` | `constexpr` | Compile-time bitmask constant built from preprocessor defines (`POSIX`, `WINDOWS`, `BSD`, `LINUX`, `DARWIN`, `FREEBSD`) | -| `isPlatform()` | Function | Runtime check whether a given `PlatformType` flag is active | -| `operator\|` | Operator | Bitwise OR for combining `PlatformType` values | -| `kSDKPlatform` | `extern const std::string` | String identifier of the build platform, mirrors `OSQUERY_BUILD_PLATFORM` | -| `OSQUERY_PLATFORM` | `#define` | Macro alias for `OSQUERY_BUILD_PLATFORM` | - -> **Note:** `OSQUERY_BUILD_PLATFORM` and `OSQUERY_BUILD_DISTRO` **must** be defined by the build system β€” the header enforces this with `#error` directives. +- **`PlatformType` (enum class)** β€” Bitmask enumeration of supported platforms: `TYPE_POSIX`, `TYPE_WINDOWS`, `TYPE_BSD`, `TYPE_LINUX`, `TYPE_OSX`, `TYPE_FREEBSD` +- **`kPlatformType` (constexpr)** β€” Compile-time constant built by OR-ing active platform flags based on preprocessor defines (`POSIX`, `WINDOWS`, `BSD`, `LINUX`, `DARWIN`, `FREEBSD`) +- **`isPlatform()`** β€” Runtime helper to check if a given `PlatformType` flag is active in a platform mask +- **`operator|`** β€” Bitwise OR operator for combining `PlatformType` values +- **`kSDKPlatform`** β€” Runtime string identifying the build platform (e.g., `"linux"`, `"darwin"`, `"windows"`) +- **`OSQUERY_PLATFORM`** β€” Macro alias for `OSQUERY_BUILD_PLATFORM`; both `OSQUERY_BUILD_PLATFORM` and `OSQUERY_BUILD_DISTRO` must be defined at build time or compilation fails ## Usage Example @@ -24,14 +20,14 @@ if (osquery::isPlatform(osquery::PlatformType::TYPE_LINUX)) { // Linux-specific logic } -// Combining platform flags -osquery::PlatformType combined = - osquery::PlatformType::TYPE_POSIX | osquery::PlatformType::TYPE_LINUX; - -if (osquery::isPlatform(osquery::PlatformType::TYPE_POSIX, combined)) { - // Runs if combined mask includes POSIX +// Combine platform flags for multi-platform checks +auto mask = osquery::PlatformType::TYPE_POSIX | osquery::PlatformType::TYPE_BSD; +if (osquery::isPlatform(osquery::PlatformType::TYPE_BSD, mask)) { + // BSD-specific logic } -// Accessing build-time platform string -std::cout << osquery::kSDKPlatform; // e.g. "linux", "darwin", "windows" -``` \ No newline at end of file +// Access build platform string +std::cout << osquery::kSDKPlatform; // e.g., "darwin" +``` + +> **Note:** Prefer runtime detection via `isPlatform()` over preprocessor `#ifdef` blocks. The platform mask is exposed through the `osquery_info` table β€” avoid changing enum values to preserve compatibility. \ No newline at end of file diff --git a/osquery/utils/info/.tool_type.md b/osquery/utils/info/.tool_type.md index 3e08fd43d30..029792b2982 100644 --- a/osquery/utils/info/.tool_type.md +++ b/osquery/utils/info/.tool_type.md @@ -1,45 +1,44 @@ -Defines the `ToolType` enumeration and related utility functions for identifying the running osquery process type at runtime, enabling conditional behavior across daemons, shells, extensions, and tests. +Defines the `ToolType` enumeration and related utility functions used to identify and query the running osquery process type at runtime. ## Key Components -### Enum: `ToolType` - +### `ToolType` Enum | Value | Description | -|---|---| -| `UNKNOWN` | Default/undetected tool type | -| `SHELL` | Interactive osquery shell | -| `DAEMON` | Background osquery daemon | -| `TEST` | Test harness process | +|-------|-------------| +| `UNKNOWN` | Default undetected type | +| `SHELL` | Interactive osquery shell (`osqueryi`) | +| `DAEMON` | Background daemon (`osqueryd`) | +| `TEST` | Test harness execution | | `EXTENSION` | osquery extension process | | `SHELL_DAEMON` | Combined shell/daemon mode | ### Functions -- **`setToolType(ToolType tool)`** β€” Sets the global tool type, typically called during initialization by the `Initializer` class. -- **`getToolType()`** β€” Returns the currently active `ToolType` for runtime branching. -- **`isDaemon()`** β€” Convenience check; returns `true` if the current tool type is `DAEMON`. -- **`isShell()`** β€” Convenience check; returns `true` if the current tool type is `SHELL`. +- **`setToolType(ToolType tool)`** β€” Sets the global tool type, typically called during `Initializer` startup +- **`getToolType()`** β€” Returns the currently registered `ToolType` +- **`isDaemon()`** β€” Convenience check; returns `true` if tool type is `DAEMON` +- **`isShell()`** β€” Convenience check; returns `true` if tool type is `SHELL` ## Usage Example ```c #include -// Set tool type during startup +// Set during initialization osquery::setToolType(osquery::ToolType::DAEMON); -// Conditional behavior based on tool type +// Branch on runtime tool type if (osquery::isDaemon()) { - // daemon-specific initialization + // daemon-specific logic } else if (osquery::isShell()) { - // interactive shell setup + // interactive shell logic } // Direct enum comparison if (osquery::getToolType() == osquery::ToolType::EXTENSION) { - // extension-specific logic + // extension-specific behavior } ``` -> **Note:** The `ToolType` is typically auto-detected by the `Initializer` class using the process name and compile-time flags β€” manual calls to `setToolType` are generally reserved for tests or special bootstrapping scenarios. \ No newline at end of file +> **Note:** The `ToolType` is typically set automatically by the `Initializer` class using the process name and compile-time flags. Manual calls to `setToolType()` should only be made during early program initialization. \ No newline at end of file diff --git a/osquery/utils/info/.version.md b/osquery/utils/info/.version.md index f1e8e244994..e38eea68b56 100644 --- a/osquery/utils/info/.version.md +++ b/osquery/utils/info/.version.md @@ -1,40 +1,45 @@ -Defines version constants and comparison utilities for the osquery runtime and SDK, enforcing that required version macros are set at build time. +Defines versioning constants and utilities for the osquery framework, exposing the build version, SDK version, and a helper for version comparisons. ## Key Components | Symbol | Type | Description | -|---|---|---| -| `kVersion` | `extern const std::string` | The current osquery runtime version string | +|--------|------|-------------| +| `kVersion` | `extern const std::string` | The current osquery runtime version | | `kSDKVersion` | `extern const std::string` | The osquery SDK version string | | `OSQUERY_SDK_VERSION` | Macro | Stringified form of `OSQUERY_BUILD_SDK_VERSION` | -| `versionAtLeast()` | Function | Compares a version string against the SDK/core version | +| `versionAtLeast()` | Function | Checks if a given version string meets a minimum version threshold | -### Build-time Requirements - -| Macro | Behavior if missing | -|---|---| -| `OSQUERY_VERSION` | **Compile error** β€” must be defined | -| `OSQUERY_BUILD_SDK_VERSION` | **Compile error** β€” must be defined | -| `OSQUERY_BUILD_VERSION` | **Warning** β€” falls back to `1.0.0-unknown` | +**Required build definitions:** +- `OSQUERY_VERSION` β€” must be defined at build time (error if missing) +- `OSQUERY_BUILD_SDK_VERSION` β€” must be defined at build time (error if missing) +- `OSQUERY_BUILD_VERSION` β€” should be defined; falls back to `1.0.0-unknown` with a warning ## Usage Example ```c #include +#include + +namespace osquery { + +void checkCompatibility() { + // Print current versions + std::cout << "osquery version: " << kVersion << std::endl; + std::cout << "SDK version: " << kSDKVersion << std::endl; -// Check if a loaded extension meets the minimum version requirement -void checkExtensionCompatibility(const std::string& extensionVersion) { - if (!osquery::versionAtLeast(extensionVersion)) { - LOG(WARNING) << "Extension version " << extensionVersion - << " is older than core version " << osquery::kVersion; + // Gate logic on a minimum version requirement + if (versionAtLeast("5.0.0")) { + std::cout << "Running osquery 5.0.0 or later" << std::endl; } -} -// Check against a specific version instead of kVersion -bool isAtLeast_5_0(const std::string& v) { - return osquery::versionAtLeast(v, "5.0.0"); + // Check against a specific SDK version instead of kVersion + if (versionAtLeast("4.9.0", kSDKVersion)) { + std::cout << "SDK is at least 4.9.0" << std::endl; + } } + +} // namespace osquery ``` -> Version strings follow `major.minor.patch-commit-hash` format. `versionAtLeast(v)` returns `true` if `v` is equal to or newer than the current osquery version. \ No newline at end of file +> `versionAtLeast()` expects `major.minor.patch[-commit-hash]` format. The optional second argument defaults to `kVersion`, enabling both runtime and SDK compatibility checks. \ No newline at end of file diff --git a/osquery/utils/json/.json.md b/osquery/utils/json/.json.md index 5b88e19c6f2..e75a78761f1 100644 --- a/osquery/utils/json/.json.md +++ b/osquery/utils/json/.json.md @@ -1,50 +1,61 @@ -A wrapper class around RapidJSON that provides safe, memory-leak-resistant construction and manipulation of JSON objects and arrays within the osquery framework. +A thin RAII wrapper around RapidJSON that simplifies safe construction, manipulation, and serialization of JSON objects and arrays within the osquery framework. ## Key Components -### `JSON` class -A move-only (`only_movable`) wrapper around `rapidjson::Document` with the following groups of methods: - -| Method Group | Description | -|---|---| -| `newObject()` / `newArray()` / `newFromValue()` | Static factory constructors | -| `add(key, value, ...)` | Overloaded adders for `string`, `char*`, `int`, `long`, `long long`, `unsigned` variants, `double`, `bool`, `rapidjson::Value` | -| `addCopy()` / `addRef()` | String-specific adders: copy vs. reference semantics | -| `push(value, ...)` | Append values to JSON arrays | -| `pushCopy()` | Append string copies to arrays | -| `copyFrom()` | Deep-copy a JSON value into the document | -| `mergeObject()` / `mergeArray()` | Merge two JSON objects or arrays | -| `toString()` / `toPrettyString()` | Serialize document to string | -| `fromString()` | Parse JSON string (iterative or recursive mode) | -| `getObject()` / `getArray()` | Create new empty sub-documents | -| `doc()` | Direct access to the underlying `rapidjson::Document` | -| `valueToSize()` | Static helper to extract `uint64_t` from a value | - -### `ParseMode` enum -Controls RapidJSON parsing strategy: -- `Iterative` β€” stack-safe, avoids overflow on deep nesting -- `Recursive` β€” default RapidJSON behavior +### `JSON` Class +Move-only wrapper (`only_movable`) around a `rapidjson::Document`, providing memory-safe JSON operations. + +**Factory Methods** +- `newObject()` β€” creates a JSON wrapper for an object (map) +- `newArray()` β€” creates a JSON wrapper for an array (list) +- `newFromValue()` β€” creates a wrapper from an existing `rapidjson::Value` + +**Mutation Methods** +- `add(key, value[, obj])` β€” adds typed key-value pairs (`string`, `char*`, `int`, `long`, `long long`, `unsigned` variants, `double`, `bool`, `rapidjson::Value`) to the document or a sub-object +- `addCopy()` / `addRef()` β€” explicit copy vs. reference semantics for string values +- `push(value[, arr])` β€” appends values to an array (`rapidjson::Value`, `size_t`, `string`) +- `pushCopy()` β€” appends a string copy to an array +- `copyFrom()` β€” deep-copies a value into the document or a target node +- `mergeObject()` / `mergeArray()` β€” merges members from one object/array into another + +**Serialization / Parsing** +- `toString(str)` β€” serializes to compact JSON string +- `toPrettyString(str, indent)` β€” serializes to formatted JSON string +- `fromString(str, ParseMode)` β€” parses JSON from a string; supports `Iterative` (stack-safe) or `Recursive` modes + +**Accessors** +- `doc()` β€” returns reference to the underlying `rapidjson::Document` +- `getObject()` / `getArray()` β€” creates new empty document of the respective type +- `valueToSize()` β€” extracts a `uint64_t` from a value ## Usage Example -```c -// Create a JSON object and serialize it -osquery::JSON json = osquery::JSON::newObject(); -json.add("hostname", "server01"); -json.add("port", 8080); -json.add("active", true); +```cpp +#include + +// Build a JSON object +auto doc = osquery::JSON::newObject(); +doc.add("hostname", "myhost"); +doc.add("port", 8080); +doc.add("enabled", true); +// Serialize to string std::string output; -auto status = json.toString(output); -if (status.ok()) { - // output == {"hostname":"server01","port":8080,"active":true} -} +doc.toString(output); +// output: {"hostname":"myhost","port":8080,"enabled":true} -// Parse JSON from a string +// Parse from string osquery::JSON parsed; -auto s = parsed.fromString( - "{\"key\":\"value\"}", - osquery::JSON::ParseMode::Iterative -); -``` \ No newline at end of file +auto status = parsed.fromString(output, osquery::JSON::ParseMode::Iterative); +if (!status.ok()) { + // handle error +} + +// Build a JSON array +auto arr = osquery::JSON::newArray(); +arr.push(std::size_t{42}); +arr.pushCopy(std::string{"item"}); +``` + +> **Note:** Iterative parse mode (`kParseIterativeFlag`) is set as the default to prevent stack overflows on deeply nested JSON input. \ No newline at end of file diff --git a/osquery/utils/linux/dpkg/.dpkgquery.md b/osquery/utils/linux/dpkg/.dpkgquery.md index 43012c91fdf..d7379201591 100644 --- a/osquery/utils/linux/dpkg/.dpkgquery.md +++ b/osquery/utils/linux/dpkg/.dpkgquery.md @@ -1,43 +1,39 @@ -Concrete implementation of the `IDpkgQuery` interface that queries the dpkg package database on Linux systems to retrieve installed package information. +Concrete implementation of the `IDpkgQuery` interface that queries the dpkg package database on Linux systems using the native `libdpkg` API. ## Key Components -### Class: `DpkgQuery` +### `DpkgQuery` (class) +Final implementation of `IDpkgQuery`, instantiated via the `IDpkgQuery` factory (friend class pattern). -A `final` class inheriting from `IDpkgQuery`, providing the concrete dpkg querying logic. +| Member | Description | +|--------|-------------| +| `getPackageList()` | Returns an `Expected` containing all installed dpkg packages | +| `~DpkgQuery()` | Destructor; cleans up libdpkg state | +| `DpkgQuery(dpkg_admindir)` | Private constructor accepting the dpkg admin directory path (e.g. `/var/lib/dpkg`) | +| `validateAdminDir(path)` | Validates the admin directory exists and is accessible before querying | +| `packageIteratorCallback(...)` | Static callback invoked per-package during `pkg_array` iteration; populates `PackageList` | -| Member | Type | Description | -|---|---|---| -| `getPackageList()` | Public | Returns an `Expected` with all installed dpkg packages | -| `~DpkgQuery()` | Public | Destructor; cleans up dpkg library state | -| `DpkgQuery(admindir)` | Private | Constructor accepting a custom dpkg admin directory path | -| `validateAdminDir(path)` | Private | Validates the provided admin directory exists and is accessible | -| `packageIteratorCallback(...)` | Private Static | Callback invoked per-package during dpkg array iteration; populates `PackageList` | - -### Notable Macros - -- `LIBDPKG_VOLATILE_API` β€” Required define to acknowledge the unstable dpkg C API before including dpkg headers - -### Dependencies - -- `` β€” dpkg C library for package array iteration -- `osquery/utils/linux/idpkgquery.h` β€” Abstract interface defining `IDpkgQuery`, `PackageList`, and `ErrorCode` +### Preprocessor Define +`LIBDPKG_VOLATILE_API` β€” required macro to acknowledge use of the unstable libdpkg API surface before including dpkg headers. ## Usage Example ```cpp -// Instantiation is controlled via the IDpkgQuery factory (friend class) +// DpkgQuery is constructed via the IDpkgQuery factory, not directly auto query = IDpkgQuery::create("/var/lib/dpkg"); -auto result = query->getPackageList(); -if (result.isValue()) { +if (query) { + auto result = query->getPackageList(); + + if (result) { for (const auto& pkg : result.get()) { - // Process each installed package entry + // Access package name, version, arch, etc. } -} else { + } else { // Handle ErrorCode from result.getError() + } } ``` -> **Note:** `DpkgQuery` cannot be instantiated directly β€” the constructor is private and only accessible via the `IDpkgQuery` factory pattern (declared as `friend class IDpkgQuery`). \ No newline at end of file +> **Note:** Direct instantiation is not possible β€” `DpkgQuery`'s constructor is private. Use `IDpkgQuery::create()` which is granted access via the `friend class IDpkgQuery` declaration. \ No newline at end of file diff --git a/osquery/utils/linux/dpkg/.idpkgquery.md b/osquery/utils/linux/dpkg/.idpkgquery.md index bf8f83483dc..c2a581b3d8c 100644 --- a/osquery/utils/linux/dpkg/.idpkgquery.md +++ b/osquery/utils/linux/dpkg/.idpkgquery.md @@ -1,37 +1,19 @@ -Interface definition for querying Debian package (dpkg) information, providing an abstraction layer for accessing installed package metadata from a dpkg admin directory. +Abstract interface for querying dpkg package information from a Debian-based system's package database directory. ## Key Components ### `IDpkgQuery` (Abstract Class) -The primary interface for dpkg package queries within the `osquery` namespace. +- **`ErrorCode`** β€” Enum of failure modes: `MemoryAllocationFailure`, `InvalidAdminDirPath`, `NotADpkgAdminDir`, `NoPackagesFound`, `PermissionError` +- **`Package`** β€” Plain struct holding package metadata fields: `name`, `version`, `source`, `size`, `arch`, `revision`, `status`, `status_string`, `maintainer`, `section`, `priority` +- **`PackageList`** β€” Alias for `std::vector` -**`ErrorCode` Enum** -Represents failure states returned via `Expected<>`: +### Static Methods +- **`create(dpkg_admindir)`** β€” Factory method returning an `Expected`; constructs a concrete implementation pointed to a given dpkg admin directory (typically `/var/lib/dpkg`) +- **`getErrorCodeDescription(error_code)`** β€” Returns a human-readable string for a given `ErrorCode` -| Code | Description | -|---|---| -| `MemoryAllocationFailure` | Failed to allocate required memory | -| `InvalidAdminDirPath` | Provided path is not valid | -| `NotADpkgAdminDir` | Path exists but is not a dpkg admin directory | -| `NoPackagesFound` | Admin dir is valid but contains no packages | -| `PermissionError` | Insufficient permissions to read the directory | - -**`Package` Struct** -Holds metadata for a single installed package: - -| Field | Type | -|---|---| -| `name`, `version`, `source` | `std::string` | -| `size`, `arch`, `revision` | `std::string` | -| `status`, `status_string` | `std::string` | -| `maintainer`, `section`, `priority` | `std::string` | - -**Key Methods** - -- `create(dpkg_admindir)` β€” Static factory returning `Expected` -- `getPackageList()` β€” Pure virtual; returns all installed packages -- `getErrorCodeDescription(error_code)` β€” Human-readable error message +### Virtual Methods +- **`getPackageList()`** β€” Pure virtual; returns `Expected` with all installed packages found in the admin directory ## Usage Example @@ -41,21 +23,22 @@ Holds metadata for a single installed package: auto result = osquery::IDpkgQuery::create("/var/lib/dpkg"); if (result.isError()) { - auto msg = osquery::IDpkgQuery::getErrorCodeDescription( - result.getError() - ); - LOG(ERROR) << "Failed to create dpkg query: " << msg; + auto desc = osquery::IDpkgQuery::getErrorCodeDescription(result.getError()); + LOG(ERROR) << "Failed to initialize dpkg query: " << desc; return; } -auto query = std::move(result.get()); +auto& query = result.get(); + auto packages = query->getPackageList(); +if (packages.isError()) { + LOG(ERROR) << "Failed to list packages"; + return; +} -if (!packages.isError()) { - for (const auto& pkg : packages.get()) { - LOG(INFO) << pkg.name << " @ " << pkg.version; - } +for (const auto& pkg : packages.get()) { + std::cout << pkg.name << " " << pkg.version << " (" << pkg.arch << ")\n"; } ``` -> Non-copyable by design β€” use `IDpkgQuery::Ptr` (`std::unique_ptr`) for ownership transfer. \ No newline at end of file +> **Note:** The class is non-copyable by design. Always interact through the `IDpkgQuery::Ptr` (`unique_ptr`) returned by `create()`. \ No newline at end of file diff --git a/osquery/utils/pidfile/.pidfile.md b/osquery/utils/pidfile/.pidfile.md index d92e72d1534..8b0fc6897ba 100644 --- a/osquery/utils/pidfile/.pidfile.md +++ b/osquery/utils/pidfile/.pidfile.md @@ -1,55 +1,55 @@ -Provides the `Pidfile` class interface for creating, locking, reading, and managing PID files to ensure single-instance process execution within the osquery framework. +Manages creation, locking, reading, and cleanup of PID files to enforce single-instance process execution in osquery. ## Key Components -### `Pidfile` Class -A non-copyable, move-only RAII class managing the lifecycle of a PID file. +**Class: `Pidfile`** +A non-copyable, move-only RAII wrapper for PID file lifecycle management. -### Error Enum -Defines failure states returned via `Expected`: +**Error Enum (`Pidfile::Error`)** +Represents failure states returned via `Expected`: -| Value | Description | -|---|---| +| Value | Meaning | +|-------|---------| | `Busy` | Another instance is already running | -| `NotRunning` | No active process found | -| `AccessDenied` | Insufficient file permissions | -| `MemoryAllocationFailure` | Heap allocation failed | -| `IOError` | Filesystem read/write failure | -| `InvalidProcessID` | PID value is malformed or invalid | -| `Unknown` | Unclassified error | +| `NotRunning` | Process referenced in PID file is gone | +| `AccessDenied` | Insufficient filesystem permissions | +| `IOError` | Read/write failure | +| `InvalidProcessID` | PID file contains a malformed or invalid PID | +| `MemoryAllocationFailure` | Allocation error during operation | -### Static Methods +**Static Methods** | Method | Description | -|---|---| -| `create(path)` | Creates and locks a PID file, writing the current PID | +|--------|-------------| +| `create(path)` | Creates and locks a PID file, writes current PID | | `read(path)` | Reads and returns the PID stored in an existing file | -| `createFile(path)` | Opens/creates the underlying file handle | -| `lockFile(handle)` | Acquires an exclusive lock on the file | -| `readFile(handle)` | Reads raw content from the file handle | -| `writeFile(handle)` | Writes the current process ID to the file | -| `closeFile(handle)` | Releases the file handle | -| `destroyFile(handle, path)` | Closes and deletes the PID file | +| `createFile(path)` | Opens/creates the file, returns a `FileHandle` | +| `lockFile(handle)` | Applies an exclusive lock to the file | +| `readFile(handle)` | Reads raw content from the file | +| `writeFile(handle)` | Writes the current process PID | +| `closeFile(handle)` | Closes the file handle | +| `destroyFile(handle, path)` | Closes and removes the PID file from disk | + +**`operator<<`** β€” Stream-serializes `Pidfile::Error` for logging. ## Usage Example ```cpp -#include - -// Create and lock a PID file for single-instance enforcement -auto pidfile = osquery::Pidfile::create("/var/run/osqueryd.pid"); +// Enforce single-instance at startup +auto pidfile = osquery::Pidfile::create("/var/run/osquery.pid"); if (pidfile.isError()) { - if (pidfile.getError() == osquery::Pidfile::Error::Busy) { - std::cerr << "Another instance is already running.\n"; - } - return 1; + if (pidfile.getError() == osquery::Pidfile::Error::Busy) { + LOG(ERROR) << "osquery is already running"; + } + return 1; } -// Read an existing PID file -auto pid = osquery::Pidfile::read("/var/run/osqueryd.pid"); +// Read PID from an existing PID file +auto pid = osquery::Pidfile::read("/var/run/osquery.pid"); if (pid.isValue()) { - std::cout << "Running PID: " << pid.get() << "\n"; + std::cout << "Running PID: " << pid.get() << std::endl; } -// PID file is automatically released when `pidfile` goes out of scope + +// Pidfile is automatically cleaned up when it goes out of scope (RAII) ``` \ No newline at end of file diff --git a/osquery/utils/status/.status.md b/osquery/utils/status/.status.md index e754c3118db..9c903249217 100644 --- a/osquery/utils/status/.status.md +++ b/osquery/utils/status/.status.md @@ -1,39 +1,45 @@ -Defines the `osquery::Status` class, a lightweight utility for expressing operation outcomes with a numeric code and descriptive message β€” analogous to a simple result type used throughout the osquery codebase. +Defines the `osquery::Status` class, a lightweight utility for expressing operation success or failure with an integer code and descriptive message throughout the osquery codebase. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `Status` | Class | Core result type wrapping an integer code and string message | -| `kSuccessCode` | `static constexpr int` | Success sentinel value (`0`) | -| `ok()` | Method | Returns `true` if code equals `kSuccessCode` | -| `getCode()` | Method | Returns the raw integer status code | -| `getMessage()` / `toString()` / `what()` | Methods | Returns the descriptive message string | -| `Status::success()` | Static factory | Creates a success `Status` | -| `Status::failure(...)` | Static factory | Creates a failure `Status` with optional code and message | -| `to(Expected)` | Template function | Converts an `Expected` to a `Status` | +**Class: `Status`** + +| Member | Description | +|--------|-------------| +| `Status(int c, std::string m)` | Primary constructor β€” code `0` = success, non-zero = failure | +| `Status(const ErrorBase&)` | Constructs a failure status directly from an `ErrorBase` error | +| `ok()` | Returns `true` if code equals `kSuccessCode` (0) | +| `getCode()` | Returns the raw integer status code | +| `getMessage()` / `toString()` / `what()` | Returns the descriptive message string | +| `Status::success()` | Static factory for a successful status | +| `Status::failure(message)` | Static factory for a failed status with message | +| `operator bool()` | Enables direct use in `if` conditions | +| `operator==` / `operator!=` | Equality comparisons, useful with gtest assertions | + +**Free Function: `to(...)`** + +Converts an `Expected` to a `Status`, bridging the two result-type idioms. ## Usage Example ```cpp #include -#include -// Return status from a function +// Returning status from a function osquery::Status doWork() { - if (someCondition()) { + if (workSucceeded()) { return osquery::Status::success(); } - return osquery::Status::failure("Something went wrong"); + return osquery::Status::failure("work failed: resource unavailable"); } -// Check result with ok() +// Checking status auto s = doWork(); if (s.ok()) { LOG(INFO) << "Success"; } else { - LOG(ERROR) << s.getMessage(); // or s.toString() / s.what() + LOG(ERROR) << s.getMessage(); // "work failed: resource unavailable" } // Implicit bool conversion @@ -41,7 +47,7 @@ if (doWork()) { LOG(INFO) << "Also works!"; } -// Convert from Expected<> to Status -Expected result = getExpectedValue(); +// Converting from Expected<> +auto result = someExpectedReturningFn(); osquery::Status status = osquery::to(result); ``` \ No newline at end of file diff --git a/osquery/utils/system/.time.md b/osquery/utils/system/.time.md index 814e95af8ee..da4791c8790 100644 --- a/osquery/utils/system/.time.md +++ b/osquery/utils/system/.time.md @@ -1,38 +1,37 @@ -Cross-platform time utility header providing UNIX epoch and human-readable time conversion functions within the `osquery` namespace. +Cross-platform time utility header for the `osquery` namespace, providing functions to retrieve, convert, and format time values as UNIX epoch integers or human-readable ASCII strings. ## Key Components | Function | Description | |---|---| -| `platformAsctime()` | Converts a `struct tm*` to a C++ `std::string` using the platform's `asctime` equivalent | +| `platformAsctime()` | Converts a `struct tm*` to a `std::string` using a platform-safe `asctime` equivalent | | `toUnixTime()` | Converts a `struct tm*` to a `uint64_t` UNIX epoch timestamp | | `getUnixTime()` | Returns the current time as a `uint64_t` UNIX epoch timestamp | -| `toAsciiTime()` | Converts a UTC `struct tm*` to a human-readable string (`"Wed Sep 21 10:27:52 2011"`) | -| `toAsciiTimeUTC()` | Converts a local `struct tm*` to a UTC human-readable string via epoch + `gmtime()` | -| `getAsciiTime()` | Returns the current date/time as a human-readable string | +| `toAsciiTime()` | Formats a UTC `struct tm*` as a readable string (e.g. `"Wed Sep 21 10:27:52 2011"`) | +| `toAsciiTimeUTC()` | Converts a local `struct tm*` to UTC, then returns it as a readable string | +| `getAsciiTime()` | Returns the current UTC date/time as a readable string | ## Usage Example -```cpp -#include +```c +#include #include // Get current time as UNIX epoch uint64_t epoch = osquery::getUnixTime(); -std::cout << "Epoch: " << epoch << std::endl; -// Get current time as readable string +// Get current time as human-readable string std::string now = osquery::getAsciiTime(); -std::cout << "Now: " << now << std::endl; // "Wed Sep 21 10:27:52 2011" +// Output: "Wed Sep 21 10:27:52 2011" -// Convert a struct tm to UTC ASCII +// Convert an existing struct tm to UTC ASCII struct tm tm_time = {}; -gmtime_r(&some_time_t, &tm_time); -std::string utc = osquery::toAsciiTimeUTC(&tm_time); +// ... populate tm_time ... +std::string utcStr = osquery::toAsciiTimeUTC(&tm_time); // Convert struct tm to epoch uint64_t ts = osquery::toUnixTime(&tm_time); ``` -> **Note:** `toAsciiTime()` expects the `struct tm` to already be in UTC. Use `toAsciiTimeUTC()` when starting from local time, as it internally normalizes via epoch conversion and `gmtime()`. \ No newline at end of file +> **Note:** `toAsciiTime()` expects the input `struct tm` to already be in UTC. Use `toAsciiTimeUTC()` when starting from local time. \ No newline at end of file diff --git a/osquery/utils/system/.uptime.md b/osquery/utils/system/.uptime.md index 79f63b62960..51eb9ad293b 100644 --- a/osquery/utils/system/.uptime.md +++ b/osquery/utils/system/.uptime.md @@ -1,15 +1,14 @@ -Provides a utility interface for retrieving system uptime within the osquery framework. +Declares a single utility function for retrieving system uptime within the `osquery` namespace. ## Key Components -- **`getUptime()`** β€” Returns the system uptime as a `long` value (typically in seconds) within the `osquery` namespace. +- **`getUptime()`** β€” Returns the system uptime as a `long` value (in seconds). ## Usage Example ```c #include "uptime.h" -// Retrieve current system uptime long uptimeSeconds = osquery::getUptime(); ``` \ No newline at end of file diff --git a/osquery/utils/system/linux/.cpu.md b/osquery/utils/system/linux/.cpu.md index 0a76034b4da..7b8d6063941 100644 --- a/osquery/utils/system/linux/.cpu.md +++ b/osquery/utils/system/linux/.cpu.md @@ -1,46 +1,44 @@ -Provides CPU topology utilities for Linux systems, exposing sysfs-based CPU state information as bitmask representations. +Provides CPU topology utilities for Linux systems, exposing sysfs-based CPU state information (online, offline, possible, present) as bitmask representations. ## Key Components **Types & Constants** -- `Error` β€” enum with `IOError` and `IncorrectRange` variants for error handling -- `Mask` β€” alias for `std::bitset<128>`, representing up to 128 CPU cores (aligned with Linux `NR_CPUS`) -- `kMaskSize` β€” constant defining the bitmask width (`128`) +- `Error` β€” enum with `IOError` and `IncorrectRange` variants for failure reporting +- `Mask` β€” `std::bitset<128>` alias representing a CPU bitmask (based on `NR_CPUS`) +- `kMaskSize` β€” constant (`128`) defining bitmask width **Functions** | Function | Description | |---|---| -| `decodeMaskFromString(encoded_str)` | Parses a sysfs CPU mask string into a `Mask` bitset | -| `getOfflineRaw()` / `getOffline()` | Returns hotplugged-off or kernel-limited CPUs | -| `getOnlineRaw()` / `getOnline()` | Returns CPUs currently online and scheduled | -| `getPossibleRaw()` / `getPossible()` | Returns CPUs allocated and bringable online | -| `getPresentRaw()` / `getPresent()` | Returns CPUs physically present in the system | +| `decodeMaskFromString(encoded_str)` | Decodes a string CPU mask into a `Mask` bitset | +| `getOffline()` / `getOfflineRaw()` | CPUs hotplugged off or exceeding kernel config limits | +| `getOnline()` / `getOnlineRaw()` | CPUs currently online and scheduled | +| `getPossible()` / `getPossibleRaw()` | CPUs allocated and bringable online if present | +| `getPresent()` / `getPresentRaw()` | CPUs identified as physically present | -Each query is available in two forms: raw string (`*Raw`) as read from sysfs, or decoded `Mask` bitset. +All functions return `Expected`, where `Raw` variants return the string representation and non-`Raw` variants return a decoded `Mask`. ## Usage Example ```cpp -#include +#include // Get online CPUs as a bitmask -auto result = osquery::cpu::getOnline(); -if (result) { - osquery::cpu::Mask onlineMask = result.get(); - for (std::size_t i = 0; i < osquery::cpu::kMaskSize; ++i) { - if (onlineMask.test(i)) { +auto online = osquery::cpu::getOnline(); +if (online) { + osquery::cpu::Mask mask = online.get(); + for (size_t i = 0; i < osquery::cpu::kMaskSize; ++i) { + if (mask.test(i)) { // CPU i is online } } -} else { - auto err = result.getError(); - // handle Error::IOError or Error::IncorrectRange } -// Decode a mask string directly -auto mask = osquery::cpu::decodeMaskFromString("0000,0000003f"); -``` - -> All functions return `Expected`, following osquery's error-handling pattern β€” always check for errors before accessing the value. \ No newline at end of file +// Decode a raw mask string manually +auto mask = osquery::cpu::decodeMaskFromString("00000000,0000ffff"); +if (!mask) { + // handle mask.getError() == Error::IOError or Error::IncorrectRange +} +``` \ No newline at end of file diff --git a/osquery/utils/system/linux/proc/.proc.md b/osquery/utils/system/linux/proc/.proc.md index 4e5d984393f..5b233cb95a2 100644 --- a/osquery/utils/system/linux/proc/.proc.md +++ b/osquery/utils/system/linux/proc/.proc.md @@ -1,9 +1,9 @@ -Declares a utility function for retrieving the command-line string of a running process by PID on Linux/Unix systems. +Provides a utility interface for querying process information on the system, specifically for retrieving command-line arguments of running processes. ## Key Components -- **`osquery::proc::cmdline(pid_t pid)`** β€” Reads and returns the full command-line string for the given process ID, typically sourced from `/proc//cmdline`. +- **`cmdline(pid_t pid)`** β€” Returns the full command-line string for the process identified by the given `pid`. ## Usage Example @@ -11,8 +11,16 @@ Declares a utility function for retrieving the command-line string of a running #include "proc.h" #include -// Retrieve the command line for a known PID -pid_t target = 1234; -std::string cmd = osquery::proc::cmdline(target); -std::cout << "Command: " << cmd << std::endl; -``` \ No newline at end of file +int main() { + pid_t target_pid = 1234; + std::string cmd = osquery::proc::cmdline(target_pid); + std::cout << "Command line: " << cmd << std::endl; + return 0; +} +``` + +## Notes + +- Lives within the `osquery::proc` namespace. +- Depends on `boost::filesystem` and the standard `` library. +- Typically used in osquery table implementations that expose process metadata (e.g., `processes` table). \ No newline at end of file diff --git a/osquery/utils/system/posix/.errno.md b/osquery/utils/system/posix/.errno.md index 675483fc3a4..f79f58139da 100644 --- a/osquery/utils/system/posix/.errno.md +++ b/osquery/utils/system/posix/.errno.md @@ -1,36 +1,28 @@ -Provides a type-safe C++ wrapper around POSIX `errno` values within the `osquery` namespace, mapping raw integer error codes to a strongly-typed enum. +Provides a strongly-typed C++ wrapper around POSIX `errno` values within the osquery namespace, mapping raw integer error codes to a scoped enumeration for safer error handling. ## Key Components -- **`PosixError` (enum class)** β€” Strongly-typed enumeration of standard POSIX error codes (e.g., `PERM`, `NOENT`, `NOMEM`), mapped directly from system `errno` macros. Includes an `Unknown = 0` sentinel for unrecognized errors. - -- **`impl::toPosixSystemError(int)`** β€” Internal implementation function that converts a raw `errno` integer to its corresponding `PosixError` enum value. - -- **`to(int from_errno)`** β€” Template conversion function, enabled only when `ToType` is `PosixError` (via `std::enable_if`). Provides a clean, generic interface for converting raw errno integers to `PosixError`. +- **`PosixError` enum class** β€” Scoped enumeration mapping all standard POSIX `errno` constants (e.g., `EPERM`, `ENOENT`, `ENOMEM`) to named values, with `Unknown = 0` as a default fallback. +- **`impl::toPosixSystemError(int)`** β€” Internal implementation function that converts a raw `errno` integer into the corresponding `PosixError` enum value. +- **`to(int from_errno)`** β€” Template conversion helper, constrained via `std::enable_if` to only resolve when `ToType` is `PosixError`, providing a clean public API for errno conversion. ## Usage Example ```cpp -#include +#include "errno.h" -// Direct conversion from a raw errno value -int result = open("/nonexistent/path", O_RDONLY); +// Convert a raw errno value to a typed PosixError +int result = ::open("/nonexistent/path", O_RDONLY); if (result < 0) { - PosixError err = osquery::to(errno); + osquery::PosixError err = osquery::to(errno); - switch (err) { - case osquery::PosixError::NOENT: - // Handle file not found - break; - case osquery::PosixError::ACCES: - // Handle permission denied - break; - default: - // Handle unknown/other errors - break; + if (err == osquery::PosixError::NOENT) { + // Handle file-not-found specifically + } else if (err == osquery::PosixError::ACCES) { + // Handle permission denied } } ``` -> **Note:** The `to()` template is SFINAE-constrained, so calling it with a non-`PosixError` type will result in a compile-time error, preventing accidental misuse. \ No newline at end of file +> **Note:** The `to()` template is constrained at compile time β€” passing a non-`PosixError` type will result in a compilation error, enforcing correct usage at the call site. \ No newline at end of file diff --git a/osquery/utils/system/posix/.system.md b/osquery/utils/system/posix/.system.md index 76ea3ea991a..6c1b1bc9df1 100644 --- a/osquery/utils/system/posix/.system.md +++ b/osquery/utils/system/posix/.system.md @@ -1,49 +1,48 @@ -Provides privilege dropping and restoration utilities for osquery, enabling temporary reduction of process permissions to a specified user, group, or path owner. +Defines the `DropPrivileges` RAII class for temporarily lowering process privileges in osquery, supporting drops by path ownership, explicit UID/GID, or username. ## Key Components -### Types -- **`PlatformPidType`** β€” Platform-agnostic alias for `pid_t`, representing a process identifier. +**Type Aliases** +- `PlatformPidType` β€” Platform-agnostic process ID type (`pid_t`) +- `DropPrivilegesRef` β€” `shared_ptr` alias for `DropPrivileges` -### Class: `DropPrivileges` -A non-copyable RAII-style class that temporarily lowers effective process privileges and automatically restores them on destruction. +**`DropPrivileges` Class** +A non-copyable RAII guard that drops and automatically restores effective process privileges on destruction. | Member | Description | -|---|---| -| `static get()` | Factory method returning a `DropPrivilegesRef` (`shared_ptr`) instance | -| `dropToParent(path)` | Drops privileges to the owner of the parent directory of the given path | +|--------|-------------| +| `get()` | Factory method returning a `DropPrivilegesRef` instance | +| `dropToParent(path)` | Drops privileges to match the owner of the parent directory of `path` | | `dropTo(uid, gid)` | Drops to explicit numeric UID/GID | -| `dropTo(uid_str, gid_str)` | Drops to string-specified UID/GID | +| `dropTo(uid_str, gid_str)` | Drops to string-parsed UID/GID | | `dropTo(user)` | Drops to a named user's UID/GID | -| `dropped()` | Returns `true` if effective privileges differ from real privileges | -| `~DropPrivileges()` | Destructor automatically restores original effective permissions | +| `dropped()` | Returns `true` if effective credentials differ from real credentials | +| `~DropPrivileges()` | Restores original effective privileges on destruction | + +**Private State** +- `dropped_` β€” tracks whether a drop is active +- `to_user_`, `to_group_` β€” target credentials +- `original_groups_` / `group_size_` β€” saved supplemental groups for full restoration ## Usage Example ```cpp -#include -#include +#include -// Drop privileges to the owner of a file's parent directory -{ +void readSensitiveFile(const boost::filesystem::path& filePath) { + // Drop privileges to the file's parent directory owner auto dropper = osquery::DropPrivileges::get(); - if (dropper->dropToParent("/etc/osquery/osquery.conf")) { - // Executing with reduced privileges here - readSensitiveFile(); + if (dropper->dropToParent(filePath)) { + // Operate with reduced privileges + // ... } - // Privileges automatically restored when 'dropper' goes out of scope + // Privileges are automatically restored when 'dropper' goes out of scope } -// Drop to a specific user -{ +// Explicit UID/GID drop +void dropExplicit() { auto dropper = osquery::DropPrivileges::get(); - dropper->dropTo("nobody"); - - if (dropper->dropped()) { - // Confirms effective UID/GID differ from real - } + dropper->dropTo(uid_t(65534), gid_t(65534)); // e.g., nobody:nogroup } -``` - -> **Note:** Only one active privilege drop should exist at a time. Attempting a second drop while one is still active will return `false`. The destructor guarantees privilege restoration, making this safe for use in scoped blocks. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/utils/system/windows/.etw_helpers.md b/osquery/utils/system/windows/.etw_helpers.md index 38423f79748..128cd3204b2 100644 --- a/osquery/utils/system/windows/.etw_helpers.md +++ b/osquery/utils/system/windows/.etw_helpers.md @@ -1,38 +1,28 @@ -Windows ETW (Event Tracing for Windows) utility header providing helper functions for extracting metadata from ETW event records within the osquery framework. +Windows ETW (Event Tracing for Windows) utility functions for extracting metadata from ETW event records within the osquery framework. ## Key Components -### Functions - -| Function | Signature | Description | -|---|---|---| -| `sidStringFromEtwRecord` | `std::string(const EVENT_RECORD&)` | Extracts and returns the user SID string from the extended header of an ETW event record | -| `processImagePathFromProcessId` | `std::string(uint32_t processId)` | Resolves and returns the process image file path from a given process ID | +| Function | Description | +|---|---| +| `sidStringFromEtwRecord` | Extracts the user SID string from an ETW `EVENT_RECORD`'s extended header data | +| `processImagePathFromProcessId` | Resolves the full image file path of a process given its PID | ## Usage Example ```c -#include "etw_helpers.h" - -// Extract user SID from an ETW event record -VOID WINAPI MyEventCallback(PEVENT_RECORD pRecord) { - // Get the SID of the user associated with the event - std::string userSid = osquery::sidStringFromEtwRecord(*pRecord); - - // Resolve the image path of the process that generated the event - uint32_t pid = pRecord->EventHeader.ProcessId; - std::string imagePath = osquery::processImagePathFromProcessId(pid); +#include - // Use the extracted metadata - LOG(INFO) << "Event from PID " << pid - << " | SID: " << userSid - << " | Path: " << imagePath; +// Extract SID from an ETW event callback +VOID WINAPI MyEventCallback(PEVENT_RECORD pEventRecord) { + std::string sid = osquery::sidStringFromEtwRecord(*pEventRecord); + // e.g. "S-1-5-21-3623811015-3361044348-30300820-1013" } -``` -## Notes +// Resolve process image path from a known PID +uint32_t pid = 1234; +std::string imagePath = osquery::processImagePathFromProcessId(pid); +// e.g. "C:\Windows\System32\svchost.exe" +``` -- **Windows-only**: Depends on `` and `osquery/utils/system/system.h`, scoped to Windows ETW infrastructure. -- All symbols are declared within the `osquery` namespace. -- SID extraction relies on the **extended data** section of the `EVENT_RECORD` structure; ensure the ETW session is configured to include user SID data. \ No newline at end of file +> **Note:** These helpers are Windows-only and depend on `evntcons.h`. They are intended for use within ETW-based osquery event publishers targeting Windows platforms. \ No newline at end of file diff --git a/osquery/utils/system/windows/.system.md b/osquery/utils/system/windows/.system.md index dea479f326f..1b2e1f1055a 100644 --- a/osquery/utils/system/windows/.system.md +++ b/osquery/utils/system/windows/.system.md @@ -1,19 +1,21 @@ -Windows platform compatibility header that provides POSIX-to-Windows shim definitions and type aliases for the osquery framework, ensuring consistent Windows API configuration across compilation units. +Windows platform compatibility header for osquery that bridges POSIX APIs and signal definitions to their Windows equivalents, ensuring consistent cross-platform behavior. ## Key Components | Symbol | Type | Description | |--------|------|-------------| -| `SIGHUP` | Macro | Defined as `1`, mirroring POSIX convention | +| `SIGHUP` | Macro | Defined as `1`, mirroring POSIX behavior | | `SIGUSR1` | Macro | Mapped to `SIGILL` (Windows has no native `SIGUSR1`) | -| `SIGALRM` | Macro | Aliased to `SIGUSR1` | -| `GetObject` / `GetMessage` | `#undef` | Removes conflicting Win32 global macros that break RapidJSON | -| `gmtime_r` | Function | Thread-safe `gmtime` wrapper matching POSIX signature | -| `localtime_r` | Function | Thread-safe `localtime` wrapper matching POSIX signature | -| `alarm` | Function | No-op stub satisfying POSIX `alarm()` call sites | -| `pid_t` | Type alias | Defined as `unsigned long` (equivalent to Win32 `DWORD`) | -| `PlatformPidType` | Type alias | Defined as `void*` for Windows process handle abstraction | +| `SIGALRM` | Macro | Aliased to `SIGUSR1` / `SIGILL` | +| `NOMINMAX` | Macro | Prevents Windows `min`/`max` macro conflicts | +| `WIN32_LEAN_AND_MEAN` | Macro | Reduces Windows header bloat | +| `GetObject` / `GetMessage` | `#undef` | Removes conflicting Windows global macros (RapidJSON compatibility) | +| `gmtime_r` | Function | POSIX-style thread-safe GMT time conversion wrapper | +| `localtime_r` | Function | POSIX-style thread-safe local time conversion wrapper | +| `alarm` | Function | No-op stub matching POSIX `alarm()` signature | +| `pid_t` | Type alias | Mapped to `unsigned long` (`DWORD` equivalent on Windows) | +| `PlatformPidType` | Type alias | `void*` representing a Windows process handle | ## Usage Example @@ -29,11 +31,9 @@ osquery::gmtime_r(&now, &result); osquery::pid_t pid = GetCurrentProcessId(); // Signal constants work cross-platform -int sig = SIGHUP; // = 1 +if (received_signal == SIGHUP) { + // handle reload +} ``` -## Notes - -- Requires `NTDDI_VERSION` and `_WIN32_WINNT` to be defined at build time (enforced via `#error`); see `tools/build_defs/oss/osquery/cxx.bzl`. -- `WIN32_LEAN_AND_MEAN` and `_WIN32_DCOM` are set here to guarantee consistent Windows API surface across all translation units. -- Warning `C4067` is suppressed locally around `sddl.h` to avoid spurious preprocessor diagnostic noise from the Windows SDK. \ No newline at end of file +> **Note:** `NTDDI_VERSION` and `_WIN32_WINNT` must be defined prior to inclusion, typically via build configuration in `tools/build_defs/oss/osquery/cxx.bzl`. Missing either will produce a compile-time error. \ No newline at end of file diff --git a/osquery/utils/system/windows/.users_groups_helpers.md b/osquery/utils/system/windows/.users_groups_helpers.md index 621b0330895..398fb9bce1e 100644 --- a/osquery/utils/system/windows/.users_groups_helpers.md +++ b/osquery/utils/system/windows/.users_groups_helpers.md @@ -1,56 +1,53 @@ -Windows-specific utility header providing RAII wrappers and helper functions for Windows Net API objects, SID manipulation, and user/group account lookups within the osquery framework. +Windows-specific helper utilities for working with user and group identity primitives (SIDs, RIDs, group memberships) via the Windows Net API, used by osquery's users/groups tables. ## Key Components ### `NetApiObjectPtr` -A move-only RAII smart pointer template that manages Windows Net API buffer lifetimes, automatically calling `NetApiBufferFree()` on destruction. +A move-only RAII smart pointer that wraps Windows Net API buffer pointers. Automatically calls `NetApiBufferFree()` on destruction, preventing resource leaks common with raw `NET_API_STATUS` buffer returns. -| Method | Description | -|--------|-------------| -| `get_new_ptr()` | Returns a raw double pointer for use with Net API calls; frees existing buffer first | -| `get()` | Returns a const raw pointer to the managed buffer | -| `operator->()` | Direct member access to the managed object | +**Type aliases provided:** -### Type Aliases - -```c -user_info_0_ptr // NetApiObjectPtr -user_info_2_ptr // NetApiObjectPtr -user_info_3_ptr // NetApiObjectPtr -user_info_4_ptr // NetApiObjectPtr -localgroup_users_info_0_ptr // NetApiObjectPtr -localgroup_info_1_ptr // NetApiObjectPtr -``` +| Alias | Wrapped Type | +|---|---| +| `user_info_0_ptr` | `USER_INFO_0` | +| `user_info_2_ptr` | `USER_INFO_2` | +| `user_info_3_ptr` | `USER_INFO_3` | +| `user_info_4_ptr` | `USER_INFO_4` | +| `localgroup_users_info_0_ptr` | `LOCALGROUP_USERS_INFO_0` | +| `localgroup_info_1_ptr` | `LOCALGROUP_INFO_1` | ### Free Functions | Function | Description | -|----------|-------------| -| `psidToString(PSID)` | Converts a binary SID struct to its string representation | -| `getSidFromAccountName(...)` | Looks up a SID by account name (wide string or `LPCWSTR`) | -| `getRidFromSid(PSID)` | Extracts the Relative Identifier (RID) from a SID | -| `getGidFromUserSid(PSID)` | Returns optional GID from a user's SID | -| `getGidFromUsername(LPCWSTR)` | Returns optional GID from a username | -| `getGroupSidFromUserSid(PSID)` | Returns group SID string from a user SID | -| `getGroupSidFromUsername(...)` | Returns group SID string from a username | +|---|---| +| `psidToString(PSID)` | Converts a binary SID struct to its string representation (e.g. `S-1-5-21-...`) | +| `getSidFromAccountName(wstring/LPCWSTR)` | Looks up a SID by username, returns heap-allocated `BYTE[]` | +| `getRidFromSid(PSID)` | Extracts the Relative Identifier (RID) from a SID as a `DWORD` | +| `getGidFromUserSid(PSID)` | Returns the primary group ID for a user SID | +| `getGidFromUsername(LPCWSTR)` | Returns the primary group ID for a username | +| `getGroupSidFromUserSid(PSID)` | Returns the group SID string for a user SID | +| `getGroupSidFromUsername(wstring)` | Returns the group SID string for a username | | `getUserHomeDir(string)` | Returns the home directory path for a given SID string | ## Usage Example ```c -// RAII-managed Net API buffer usage -osquery::user_info_4_ptr userInfo; -NetUserGetInfo(nullptr, L"john.doe", 4, - reinterpret_cast(userInfo.get_new_ptr())); +// RAII buffer management for Net API calls +osquery::user_info_4_ptr user_info; +NetUserGetInfo(nullptr, L"jdoe", 4, + reinterpret_cast(user_info.get_new_ptr())); -// SID conversions -std::string sidStr = osquery::psidToString(userInfo->usri4_user_sid); -DWORD rid = osquery::getRidFromSid(userInfo->usri4_user_sid); +// SID β†’ string +std::string sid_str = osquery::psidToString(user_info->usri4_user_sid); +// e.g. "S-1-5-21-3623811015-3361044348-30300820-1013" -// Group lookup -auto gid = osquery::getGidFromUsername(L"john.doe"); +// Lookup GID for a user +auto gid = osquery::getGidFromUsername(L"jdoe"); if (gid.has_value()) { // use gid.value() } + +// Home directory from SID string +std::string home = osquery::getUserHomeDir(sid_str); ``` \ No newline at end of file diff --git a/osquery/utils/windows/.lzxpress.md b/osquery/utils/windows/.lzxpress.md index bcc5e25a6b0..415f9c1ea45 100644 --- a/osquery/utils/windows/.lzxpress.md +++ b/osquery/utils/windows/.lzxpress.md @@ -1,34 +1,32 @@ -Header file exposing a Windows LZ Xpress Huffman decompression utility within the osquery framework. +Header defining the LZXpress Huffman decompression interface for osquery on Windows systems. ## Key Components -| Symbol | Type | Description | -|--------|------|-------------| -| `ExpectedDecompressData` | Type alias | `Expected, ConversionError>` β€” wraps decompressed bytes or a conversion error | -| `decompressLZxpress` | Function | Decompresses LZ Xpress Huffman-encoded data, returning a byte vector or error | +- **`ExpectedDecompressData`** β€” Type alias for `Expected, ConversionError>`, representing either decompressed bytes or a conversion error. +- **`decompressLZxpress()`** β€” Decompresses LZ Xpress Huffman-encoded data, returning the raw bytes or an error. ## Usage Example ```c -#include +#include -std::vector compressed = /* raw compressed bytes from registry/EVTX/etc. */; -unsigned long uncompressed_size = 4096; // known original size +std::vector compressed = getCompressedBlob(); // your compressed input +unsigned long uncompressed_size = 4096; // known output size auto result = osquery::decompressLZxpress(compressed, uncompressed_size); if (result.isError()) { - // Handle ConversionError - LOG(ERROR) << "Decompression failed: " << result.getError().getMessage(); + // handle ConversionError + LOG(ERROR) << result.getError().getMessage(); } else { std::vector data = result.take(); - // Process decompressed bytes... + // use decompressed data } ``` ## Notes -- **Windows-only**: Uses `UCHAR` from ``, which maps to Windows platform types. -- **Caller-provided size**: The `size` parameter must reflect the **total uncompressed output size** in bytes β€” typically sourced from metadata (e.g., registry hive headers or event log records). -- **Error handling**: Returns an `Expected` type (similar to `std::expected`) β€” always check `isError()` before consuming the result to avoid undefined behavior. \ No newline at end of file +- Windows-only: depends on `osquery/utils/system/system.h` which exposes `UCHAR` via the Windows SDK. +- Uses osquery's `Expected` pattern (similar to `std::expected`) for error propagation without exceptions. +- The caller must supply the correct **uncompressed size** (`size`) ahead of time β€” typically read from a metadata header in the compressed stream. \ No newline at end of file diff --git a/osquery/utils/windows/.shellitem.md b/osquery/utils/windows/.shellitem.md index 3a9de17da96..7e8c1b880f0 100644 --- a/osquery/utils/windows/.shellitem.md +++ b/osquery/utils/windows/.shellitem.md @@ -1,56 +1,60 @@ -Windows Shell Item parsing utilities for osquery, providing helper functions to extract structured data from various Windows Shell Item binary formats (LNK files, jump lists, shellbags, etc.). +Windows Shell Item parsing utilities for osquery, providing helper functions to extract metadata from Windows Shell Link (LNK) binary data structures across various shell item types. ## Key Components ### `ShellFileEntryData` Struct -Holds parsed file entry metadata extracted from shell items: +Holds parsed file entry metadata: | Field | Type | Description | -|---|---|---| +|-------|------|-------------| | `path` | `string` | File system path | | `dos_created/accessed/modified` | `long long` | DOS timestamps | -| `mft_entry` / `mft_sequence` | `long long` / `int` | NTFS MFT reference | -| `extension_sig` / `identifier` | `string` | Shell extension data | -| `version` / `string_size` | `int` | Format metadata | +| `version` | `int` | Shell item version | +| `extension_sig` | `string` | Extension signature | +| `identifier` | `string` | Unique identifier | +| `mft_entry` | `long long` | NTFS MFT entry number | +| `mft_sequence` | `int` | NTFS MFT sequence | +| `string_size` | `int` | String data size | ### Parsing Functions | Function | Returns | -|---|---| -| `fileEntry()` | `ShellFileEntryData` struct with full file metadata | +|----------|---------| +| `fileEntry()` | `ShellFileEntryData` struct with file metadata | | `propertyStore()` | Windows Property List GUID name or value | -| `networkShareItem()` | Network share UNC name | -| `zipContentItem()` | Zip archive entry name | -| `rootFolderItem()` | Root folder (e.g., Desktop, My Computer) name | -| `driveLetterItem()` | Drive letter string | -| `controlPanelItem()` / `controlPanelCategoryItem()` | Control panel entry name | -| `ftpItem()` | Vector of FTP hostnames/paths | -| `guidParse()` | Properly-ordered GUID string from little-endian bytes | -| `propertyViewDrive()` | Drive name from property view data | -| `variableGuid()` / `variableFtp()` | GUID name or FTP string from variable-length items | -| `mtpDevice()` / `mtpFolder()` / `mtpRoot()` | MTP (Media Transfer Protocol) device/folder/root names | +| `networkShareItem()` | Network share name | +| `zipContentItem()` | Zip content name | +| `rootFolderItem()` | Root folder name | +| `driveLetterItem()` | Drive letter name | +| `controlPanelCategoryItem()` | Control panel category | +| `controlPanelItem()` | Control panel name | +| `ftpItem()` | Vector of FTP hostnames | +| `guidParse()` | Little-endian GUID reordered to string | +| `propertyViewDrive()` | Property view drive name | +| `variableGuid()` | Variable GUID name or value | +| `variableFtp()` | FTP string | +| `mtpDevice/Folder/Root()` | MTP device/folder/root names | ## Usage Example ```c #include "shellitem.h" -// Parse a raw shell item binary blob -std::string raw_shell_data = /* binary data from shellbag registry key */; +// Parse a file entry shell item from raw binary data +std::string raw_data = /* binary shell item bytes */; +osquery::ShellFileEntryData entry = osquery::fileEntry(raw_data); -// Extract file entry metadata -osquery::ShellFileEntryData entry = osquery::fileEntry(raw_shell_data); +// Access parsed metadata std::cout << "Path: " << entry.path << "\n"; -std::cout << "Created: " << entry.dos_created << "\n"; std::cout << "MFT Entry: " << entry.mft_entry << "\n"; -// Parse a GUID from little-endian bytes -std::string guid = osquery::guidParse(raw_shell_data); +// Parse a network share item +std::string share = osquery::networkShareItem(raw_data); -// Resolve a network share item -std::string share = osquery::networkShareItem(raw_shell_data); +// Parse a little-endian GUID +std::string guid = osquery::guidParse(raw_guid_bytes); -// Enumerate FTP entries -std::vector ftp_paths = osquery::ftpItem(raw_shell_data); +// Parse FTP shell items (returns multiple entries) +std::vector ftp_hosts = osquery::ftpItem(raw_data); ``` \ No newline at end of file diff --git a/osquery/utils/ycloud/.ycloud_util.md b/osquery/utils/ycloud/.ycloud_util.md index fd87fea4287..570da83e7df 100644 --- a/osquery/utils/ycloud/.ycloud_util.md +++ b/osquery/utils/ycloud/.ycloud_util.md @@ -1,36 +1,32 @@ -Utility header for retrieving Yandex Cloud (YCloud) instance metadata within the osquery framework. +Utility header for retrieving Yandex Cloud (YCloud) instance metadata within the osquery framework, providing helpers to parse and fetch cloud-specific configuration values from a JSON document. ## Key Components -| Function | Description | -|---|---| +| Symbol | Description | +|--------|-------------| | `getYCloudKey` | Extracts a named key from a YCloud metadata JSON document | -| `getVendorKey` | Extracts a vendor-specific key from a YCloud metadata JSON document | -| `getYCloudSshKey` | Retrieves the SSH public key from a YCloud metadata document | -| `getSerialPortEnabled` | Reads the serial port enabled flag from a YCloud metadata document | -| `getZoneId` | Parses and returns the zone identifier from a raw zone string | -| `fetchYCloudMetadata` | Fetches YCloud instance metadata from a given endpoint and populates a JSON document | - -All functions reside in the `osquery` namespace and operate on `JSON` document objects (`osquery::JSON`). +| `getVendorKey` | Extracts a vendor-specific key from a YCloud JSON document | +| `getYCloudSshKey` | Retrieves the SSH public key from YCloud instance metadata | +| `getSerialPortEnabled` | Reads the serial port enabled flag from YCloud metadata | +| `getZoneId` | Parses and normalizes a YCloud availability zone string into a zone ID | +| `fetchYCloudMetadata` | Fetches YCloud instance metadata from the given endpoint URL, populating a `JSON` document | ## Usage Example -```c +```cpp #include "ycloud_util.h" +#include osquery::JSON doc; -osquery::Status status = osquery::fetchYCloudMetadata( - doc, "http://169.254.169.254/latest/meta-data/" -); +osquery::Status s = osquery::fetchYCloudMetadata( + doc, "http://169.254.169.254/latest/meta-data/"); -if (status.ok()) { - std::string instanceId = osquery::getYCloudKey(doc, "id"); - std::string vendor = osquery::getVendorKey(doc, "vendor"); - std::string sshKey = osquery::getYCloudSshKey(doc); - std::string serialPort = osquery::getSerialPortEnabled(doc); - std::string zone = osquery::getZoneId("ru-central1-a"); +if (s.ok()) { + std::string sshKey = osquery::getYCloudSshKey(doc); + std::string serialPort = osquery::getSerialPortEnabled(doc); + std::string zone = osquery::getZoneId("ru-central1-a"); } ``` -> **Note:** `fetchYCloudMetadata` returns an `osquery::Status` object β€” always check `status.ok()` before consuming the populated `doc` to handle network or parse failures gracefully. \ No newline at end of file +> **Note:** `fetchYCloudMetadata` communicates with the link-local metadata endpoint (`169.254.169.254`), standard for cloud IMDS services. Callers should always check the returned `Status` before accessing parsed values. \ No newline at end of file diff --git a/osquery/worker/ipc/.table_ipc_json_converter.md b/osquery/worker/ipc/.table_ipc_json_converter.md index 09a49b32dfd..58f937d9e8c 100644 --- a/osquery/worker/ipc/.table_ipc_json_converter.md +++ b/osquery/worker/ipc/.table_ipc_json_converter.md @@ -1,48 +1,48 @@ -Defines the `TableIPCJSONConverter` utility class for bidirectional serialization between osquery's internal data structures and JSON messages exchanged over IPC channels. +Defines the `TableIPCJSONConverter` utility class and `JSONMessageType` enum for serializing and deserializing IPC messages between osquery table processes and the core daemon using JSON. ## Key Components -### Enum: `JSONMessageType` -Identifies the type of JSON message being transmitted over IPC: +**`JSONMessageType` (enum class)** +Identifies the type of an IPC JSON message: +- `None` β€” unrecognized/unset +- `QueryData` β€” table query results +- `Log` β€” log message +- `Job` β€” job/task message -| Value | Description | -|---|---| -| `None` | Unrecognized or unset message type | -| `QueryData` | Contains SQL query result rows | -| `Log` | Contains a log message with priority and type | -| `Job` | Represents a job/task message | - -### Class: `TableIPCJSONConverter` +**`TableIPCJSONConverter` (class)** -All methods are static β€” no instantiation required. +All methods are static: -| Method | Direction | Description | -|---|---|---| -| `JSONToQueryData` | JSON β†’ QueryData | Deserializes JSON into osquery row results | -| `queryDataToJSON` | QueryData β†’ JSON | Serializes query results into JSON | -| `JSONToLogMessage` | JSON β†’ Log fields | Extracts priority, log type, and message string | -| `logMessageToJSON` | Log fields β†’ JSON | Serializes a log entry into JSON | -| `JSONTypeToMessageType` | JSON β†’ enum | Inspects a JSON message and resolves its `JSONMessageType` | +| Method | Description | +|---|---| +| `JSONToQueryData` | Deserializes a JSON object into a `QueryData` result set | +| `queryDataToJSON` | Serializes a `QueryData` result set into a JSON object | +| `JSONToLogMessage` | Deserializes a JSON object into log fields (`priority`, `log_type`, `message`) | +| `logMessageToJSON` | Serializes log fields into a JSON object | +| `JSONTypeToMessageType` | Inspects a JSON object and resolves its `JSONMessageType` | ## Usage Example -```cpp -#include +```c +#include +#include +#include "table_ipc_json_converter.h" + +using namespace osquery; -// Serialize query results for IPC transmission -QueryData results = getQueryResults(); +// Serialize QueryData to JSON for IPC transmission +QueryData results = {/* populated rows */}; JSON json_helper; Status s = TableIPCJSONConverter::queryDataToJSON(results, json_helper); -if (!s.ok()) { /* handle error */ } -// Deserialize received JSON back into QueryData -JSON received_json = receiveFromIPC(); +// Deserialize incoming IPC JSON message +JSON incoming; JSONMessageType msg_type; -TableIPCJSONConverter::JSONTypeToMessageType(received_json, msg_type); +TableIPCJSONConverter::JSONTypeToMessageType(incoming, msg_type); if (msg_type == JSONMessageType::QueryData) { - QueryData data; - TableIPCJSONConverter::JSONToQueryData(received_json, data); + QueryData query_data; + TableIPCJSONConverter::JSONToQueryData(incoming, query_data); } ``` \ No newline at end of file diff --git a/osquery/worker/ipc/linux/.linux_table_container_ipc.md b/osquery/worker/ipc/linux/.linux_table_container_ipc.md index 5dccd728bf4..b7ccf1bcc24 100644 --- a/osquery/worker/ipc/linux/.linux_table_container_ipc.md +++ b/osquery/worker/ipc/linux/.linux_table_container_ipc.md @@ -1,35 +1,32 @@ -Manages IPC communication between the osquery host process and container worker processes on Linux, orchestrating connection lifecycle, query dispatch, and result retrieval across namespace boundaries. +Manages IPC communication between the osquery host process and containerized worker processes on Linux, orchestrating container lifecycle, query dispatch, and result retrieval. ## Key Components ### `LinuxTableContainerIPC` Class -Main class driving container worker lifecycle and query execution over pipe-based IPC. +Main class implementing `TableIPCMessageHandler` that drives the full container query workflow. | Member | Description | -|---|---| +|--------|-------------| | `connectToContainer()` | Spawns/connects to a container worker process for a given table | -| `retrieveQueryDataFromContainer()` | Sends a `QueryContext` and collects `QueryData` results | -| `executeQueryJobs()` | Worker-side loop that processes incoming query jobs (no-return) | -| `stopContainerWorker()` | Terminates the container worker process and cleans up | -| `handleLog()` | Routes GLOG log messages received from the worker | -| `handleJob()` | Executes a table generate function inside the container namespace | +| `retrieveQueryDataFromContainer()` | Sends a query context and collects results from the container | +| `executeQueryJobs()` | Worker-side loop that processes incoming query jobs (noreturn) | +| `stopContainerWorker()` | Terminates the container worker process and cleans up resources | +| `handleLog()` | Handles forwarded GLOG log messages from the worker | +| `handleJob()` | Handles an inbound query job in the worker process | ### `CleanupWorkerOnError` (RAII Guard) -Inner class that automatically calls `stopContainerWorker()` on scope exit unless `dismiss()` is called β€” ensures cleanup on error paths. +Inner class that automatically calls `stopContainerWorker()` on scope exit unless `dismiss()` is called β€” used to safely clean up on error paths. ### Free Functions | Function | Description | -|---|---| -| `hasNamespaceConstraint()` | Returns `true` if `QueryContext` includes a `pid_with_namespace` constraint | -| `generateInNamespace()` | Entry point to run a table generator inside a container namespace via IPC | +|----------|-------------| +| `hasNamespaceConstraint()` | Checks if a `pid_with_namespace` constraint is present in the query context | +| `generateInNamespace()` | Entry point to run a table generator inside a Linux namespace via IPC | -### Type Alias -```c -using TableGeneratePtr = QueryData (*)(QueryContext&, Logger&); -``` -Function pointer type for table generator callbacks passed into the container worker. +### `TableGeneratePtr` +Type alias for a function pointer `QueryData(*)(QueryContext&, Logger&)` β€” the signature of a table's generate function. ## Usage Example @@ -38,16 +35,14 @@ PipeChannelFactory factory; LinuxTableContainerIPC container_ipc(factory); Status s = container_ipc.connectToContainer( - "process_open_files", /*keep_process_open=*/true, &myTableGenerate); + "process_namespaces", /* keep_process_open */ false, &myTableGenerate); if (s.ok()) { + QueryContext ctx; QueryData results; - container_ipc.retrieveQueryDataFromContainer(context, results); + container_ipc.retrieveQueryDataFromContainer(ctx, results); } -``` -For namespace-aware table generation, prefer the free function: - -```cpp -QueryData rows = generateInNamespace(context, "processes", &generateProcesses); +// Or use the high-level helper: +QueryData rows = generateInNamespace(context, "process_namespaces", &myTableGenerate); ``` \ No newline at end of file diff --git a/osquery/worker/ipc/linux/.linux_table_ipc.md b/osquery/worker/ipc/linux/.linux_table_ipc.md index b2371264836..794ad8898df 100644 --- a/osquery/worker/ipc/linux/.linux_table_ipc.md +++ b/osquery/worker/ipc/linux/.linux_table_ipc.md @@ -1,44 +1,52 @@ -Inter-process communication layer for Linux table workers in osquery, providing JSON-over-pipe messaging between parent and child processes that handle table query logic. +Inter-process communication layer for Linux table workers, providing JSON-based messaging over blocking pipes between parent and child processes in osquery's worker architecture. ## Key Components ### `LinuxTableIPC` -Extends `TableIPCBase` (CRTP) to manage blocking pipe-based IPC channels between worker processes. +Extends `TableIPCBase` (CRTP pattern) to manage pipe-based IPC between table worker processes. | Method | Description | -|---|---| -| `sendJSONString` / `recvJSONString` | Low-level JSON string transport over the active pipe channel | -| `processLogMessage` | Deserializes and handles an incoming log message | -| `processJobMessage` | Deserializes and dispatches an incoming query job | -| `processQueryDataMessage` | Deserializes query results into a `QueryData` output param | -| `createChannelTicket` | Allocates a new `PipeChannelTicket` via the factory | -| `connectToChild` / `connectToParent` | Establishes the active channel from either side of the fork | -| `setActiveChannelIfOpen` | Activates an existing channel by table name if available | -| `closeActiveChannel` | Tears down the current pipe connection | -| `getTableNameFromPid` | Reverse-lookup of table name by remote PID | -| `isChannelOpen` / `getRemotePid` / `getTableName` | Channel state inspection helpers | +|--------|-------------| +| `sendJSONString` / `recvJSONString` | Low-level JSON message transport over the active pipe channel | +| `processLogMessage` | Handles incoming log messages from child processes | +| `processJobMessage` | Handles incoming query job dispatches | +| `processQueryDataMessage` | Receives and deserializes `QueryData` results | +| `connectToChild` / `connectToParent` | Establishes pipe channel from parent or child side | +| `setActiveChannelIfOpen` | Switches active channel by table name if the channel is open | +| `createChannelTicket` | Delegates ticket creation to the `PipeChannelFactory` | +| `getTableNameFromPid` | Resolves a table name from a child PID | +| `closeActiveChannel` | Tears down the current active pipe channel | +| `isChannelOpen` / `getRemotePid` / `getTableName` | Channel state accessors | ### `LinuxTableIPCLogger` -Implements the `Logger` interface, forwarding `log()` and `vlog()` calls as serialized log messages over the IPC channel. +Implements the `Logger` interface, forwarding `log()` and `vlog()` calls through `LinuxTableIPC::sendLogMessage` so child worker processes can relay GLOG-compatible log output to the parent. ## Usage Example ```cpp +// Parent side: set up factory, handler, and connect to a spawned child PipeChannelFactory factory; TableIPCMessageHandler handler; LinuxTableIPC ipc(factory, handler); -// Parent side: establish connection to a spawned child auto ticket = ipc.createChannelTicket(); -ipc.connectToParent("some_table", ticket); - -// Send/receive JSON payloads -std::string response; -ipc.sendJSONString(R"({"action":"query"})"); -ipc.recvJSONString(response); - -// Attach logger to route GLOG output through the pipe +pid_t child_pid = fork(); // spawn worker process + +if (child_pid == 0) { + // Child side: connect back to parent + ipc.connectToParent("my_table", ticket); +} else { + // Parent side: connect to child + ipc.connectToChild("my_table", ticket, child_pid); + + if (ipc.isChannelOpen()) { + std::string response; + ipc.recvJSONString(response); + } +} + +// Attach IPC logger in child worker LinuxTableIPCLogger logger(ipc); -logger.log(google::INFO, "Worker ready"); +logger.log(google::INFO, "Worker started"); ``` \ No newline at end of file diff --git a/osquery/worker/ipc/posix/.pipe_channel.md b/osquery/worker/ipc/posix/.pipe_channel.md index 75073f9eae1..062e5e42f90 100644 --- a/osquery/worker/ipc/posix/.pipe_channel.md +++ b/osquery/worker/ipc/posix/.pipe_channel.md @@ -1,36 +1,40 @@ -Implements a pipe-based IPC channel for inter-process communication between osquery worker processes, using file descriptors for bidirectional message passing. +Implements a pipe-based IPC channel for inter-process communication between osquery worker processes and table handlers, using file descriptors for bidirectional message passing. ## Key Components -- **`PipeChannel`** β€” Concrete channel class inheriting from `TableChannelBase` (CRTP pattern) that wraps a pair of Unix pipe file descriptors for sending and receiving string messages between processes. +### `PipeChannel` +A concrete channel class inheriting from `TableChannelBase` (CRTP pattern) that wraps a pair of Unix pipe file descriptors. | Member | Description | -|---|---| -| `PipeChannel(table_name, read_pipe_fd, write_pipe_fd, remote_pid)` | Constructs a channel with dedicated read/write pipe FDs and the remote process PID | +|--------|-------------| +| `PipeChannel(table_name, read_pipe_fd, write_pipe_fd, remote_pid)` | Constructs the channel with explicit read/write FDs and the remote process PID | | `~PipeChannel()` | Automatically closes both pipe FDs on destruction | | `getRemotePid()` | Returns the PID of the remote process on the other end of the pipe | -| `sendStringMessageImpl()` | CRTP-dispatched implementation for writing a string message to the write pipe | -| `recvStringMessageImpl()` | CRTP-dispatched implementation for reading a string message from the read pipe | -| `blockSIGPIPE()` / `restoreSIGPIPE()` | Signal mask helpers that suppress `SIGPIPE` during writes to detect broken pipes gracefully via return codes | - -- **`PipeChannelFactory`** β€” Forward-declared factory (defined elsewhere) responsible for constructing `PipeChannel` instances. +| `sendStringMessageImpl()` | CRTP private implementation for sending a string message over the write pipe | +| `recvStringMessageImpl()` | CRTP private implementation for receiving a string message from the read pipe | +| `blockSIGPIPE()` / `restoreSIGPIPE()` | Temporarily masks `SIGPIPE` signals during writes to handle broken pipe conditions gracefully | ## Usage Example -```cpp +```c // Typically constructed via PipeChannelFactory, not directly. -// Conceptual usage after channel setup: -PipeChannel channel("processes_table", read_fd, write_fd, child_pid); - +// Conceptual direct usage: +PipeChannel channel( + "example_table", + read_fd, // fd opened for reading + write_fd, // fd opened for writing + remote_process_pid +); + +// Send a message to the remote process +Status s = channel.sendStringMessage("{\"query\": \"select * from users\"}"); + +// Receive a response std::string response; -Status recv_status = channel.recvStringMessage(response); - -if (!recv_status.ok()) { - LOG(ERROR) << "IPC recv failed: " << recv_status.getMessage(); -} +Status r = channel.recvStringMessage(response); -// Channel FDs are closed automatically when channel goes out of scope +pid_t peer = channel.getRemotePid(); ``` -> **Note:** `PipeChannel` is non-default-constructible by design. All instances must be created with explicit pipe FDs and a remote PID, enforcing valid channel state at construction time. \ No newline at end of file +> **Note:** Default construction is disabled (`= delete`). Instances are intended to be created through `PipeChannelFactory`. `SIGPIPE` blocking ensures writes to a closed pipe return a `Status` error rather than terminating the process. \ No newline at end of file diff --git a/osquery/worker/ipc/posix/.pipe_channel_factory.md b/osquery/worker/ipc/posix/.pipe_channel_factory.md index a827d7dccdf..61efc0588c8 100644 --- a/osquery/worker/ipc/posix/.pipe_channel_factory.md +++ b/osquery/worker/ipc/posix/.pipe_channel_factory.md @@ -1,46 +1,44 @@ -Manages creation and lifecycle of POSIX pipe-based IPC channels used for inter-process communication between parent and child worker processes in osquery's table extension system. +Manages creation and lifecycle of POSIX pipe-based IPC channels between parent and child worker processes in the osquery table query system. ## Key Components ### `PipeChannelTicket` -A move-only RAII wrapper that holds pre-allocated pipe file descriptors before a channel is fully constructed. Ensures proper cleanup of unused file descriptors and prevents double-use via a `used_` flag. +A move-only RAII wrapper holding two pairs of pipe file descriptors (`read_pipe_fds_`, `write_pipe_fds_`). Ensures safe transfer of ownership to a `PipeChannel` and automatically closes any unclaimed file descriptors on destruction. Can only be created by `PipeChannelFactory` (friend class). -| Method | Description | -|---|---| -| `getAndUseReadFd(index)` | Transfers ownership of a read-end fd (marks it consumed) | -| `getAndUseWriteFd(index)` | Transfers ownership of a write-end fd (marks it consumed) | -| `~PipeChannelTicket()` | Closes any unclaimed file descriptors | +- `getAndUseReadFd(index)` β€” Claims a read-end fd, marking it consumed; returns `-1` if already used +- `getAndUseWriteFd(index)` β€” Claims a write-end fd, marking it consumed; returns `-1` if already used + +### `GetChannelType` (template specialization) +Associates `PipeChannelFactory` with `PipeChannel` as its concrete channel type for the CRTP base. ### `PipeChannelFactory` -Concrete factory extending `TableChannelFactoryBase` that creates `PipeChannel` instances for parent/child process pairs. +Extends `TableChannelFactoryBase` (CRTP). Manages the full lifecycle of pipe channel creation. | Method | Description | -|---|---| -| `createChannelTicket()` | Allocates pipe fd pairs; returns a `PipeChannelTicket` | -| `createChildChannel(name, ticket)` | Builds the child-side `PipeChannel` from a ticket | -| `createParentChannel(name, ticket, pid)` | Builds the parent-side `PipeChannel`, tracking child PID | -| `getTableNameFromPid(pid)` | Looks up the table name associated with a child PID | - -### `GetChannelType` (template specialization) -Maps `PipeChannelFactory` β†’ `PipeChannel` for compile-time type resolution in the factory base. +|--------|-------------| +| `createChannelTicket()` | Allocates and returns a new `PipeChannelTicket` with open pipe fds | +| `createChildChannel(name, ticket)` | Creates a `PipeChannel` for the child worker process side | +| `createParentChannel(name, ticket, pid)` | Creates a `PipeChannel` for the parent process side, associated with the child PID | +| `getTableNameFromPid(pid)` | Looks up the table name associated with a given child PID | ## Usage Example ```cpp PipeChannelFactory factory; -// Before fork: create the ticket (pre-allocates pipe fds) -auto ticket = factory.createChannelTicket(); +// Before fork: create the ticket to pre-allocate pipe fds +PipeChannelTicket ticket = factory.createChannelTicket(); pid_t child_pid = fork(); if (child_pid == 0) { - // Child process - factory.createChildChannel("processes", std::move(ticket)); + // Child process: consume ticket to create its channel end + PipeChannel& child_ch = factory.createChildChannel("process_table", + std::move(ticket)); } else { - // Parent process - factory.createParentChannel("processes", std::move(ticket), child_pid); + // Parent process: consume ticket to create its channel end + PipeChannel& parent_ch = factory.createParentChannel("process_table", + std::move(ticket), + child_pid); } -``` - -> **Note:** `PipeChannelTicket` is move-only and cannot be reused after channel creation. Attempting to move a `used_` ticket throws `std::logic_error`. \ No newline at end of file +``` \ No newline at end of file diff --git a/osquery/worker/logging/glog/.glog_logger.md b/osquery/worker/logging/glog/.glog_logger.md index b0533b1b301..53e5d4b56dd 100644 --- a/osquery/worker/logging/glog/.glog_logger.md +++ b/osquery/worker/logging/glog/.glog_logger.md @@ -1,33 +1,31 @@ -Defines `GLOGLogger`, a singleton logger implementation that bridges osquery's worker logging interface to the GLOG (Google Logging Library) backend. +A singleton logger implementation that bridges osquery's internal worker logging interface to the GLOG (Google Logging) framework. ## Key Components -| Symbol | Type | Description | -|---|---|---| -| `GLOGLogger` | `class` | Final singleton class implementing the `Logger` interface using GLOG | -| `instance()` | `static method` | Returns the singleton `GLOGLogger` instance | -| `log()` | `method` | Logs a message at the specified severity level via GLOG | -| `vlog()` | `method` | Logs a verbose message at the specified severity level via GLOG | +- **`GLOGLogger`** β€” Final class inheriting from `Logger`; routes worker log messages to GLOG's logging backend +- **`instance()`** β€” Static method returning the singleton `GLOGLogger` instance +- **`log(severity, message)`** β€” Logs a message at the specified severity level via GLOG +- **`vlog(severity, message)`** β€” Logs a verbose/diagnostic message at the specified verbosity level via GLOG ## Usage Example ```c #include -// Retrieve the singleton instance and log a message -osquery::GLOGLogger& logger = osquery::GLOGLogger::instance(); +// Retrieve singleton and log a message +auto& logger = osquery::GLOGLogger::instance(); // Standard severity log (e.g., INFO=0, WARNING=1, ERROR=2) -logger.log(0, "Worker process started successfully"); +logger.log(0, "Worker process initialized successfully"); -// Verbose log for debug-level output -logger.vlog(1, "Processing query batch"); +// Verbose log for diagnostic output +logger.vlog(1, "Detailed worker state: task queue size = 42"); ``` ## Notes -- Inherits from `osquery::Logger` (defined in `osquery/worker/logging/logger.h`) -- Marked `final` β€” not intended for further subclassing -- Singleton pattern ensures a single shared GLOG logging instance across worker processes -- Severity integers map to standard GLOG severity levels \ No newline at end of file +- Declared `final` β€” not intended for further subclassing +- Follows the singleton pattern; instantiate only via `instance()`, never directly +- Severity integers align with GLOG conventions (`google::INFO`, `google::WARNING`, etc.) +- Resides in the `osquery` namespace alongside other worker logging utilities \ No newline at end of file diff --git a/plugins/config/.tls_config.md b/plugins/config/.tls_config.md index a6805e17c60..ea7282bad65 100644 --- a/plugins/config/.tls_config.md +++ b/plugins/config/.tls_config.md @@ -1,39 +1,42 @@ -Declares the `TLSConfigPlugin` class, which retrieves osquery configuration over TLS from a remote endpoint. +Declares the `TLSConfigPlugin` class, which fetches osquery configuration over a TLS-secured HTTP connection from a remote config server. ## Key Components ### `TLSConfigPlugin` -Inherits from `ConfigPlugin` and `std::enable_shared_from_this`. Implements TLS-based remote config fetching. +Inherits from `ConfigPlugin` and `std::enable_shared_from_this`. Implements remote TLS-based configuration retrieval. | Member | Type | Description | |--------|------|-------------| -| `setUp()` | `Status` | Initializes the plugin, validates and caches the remote TLS config URL | -| `genConfig()` | `Status` | Fetches and populates the config map from the remote TLS endpoint | -| `uri_` | `std::string` (protected) | Cached URL computed once during setup | +| `setUp()` | `Status` | Initializes the plugin; validates and caches the remote endpoint URL | +| `genConfig()` | `Status` | Fetches configuration data from the TLS endpoint and populates the config map | +| `uri_` | `std::string` (protected) | Cached remote endpoint URL, computed once during `setUp()` | ## Usage Example ```cpp -// Register and use TLSConfigPlugin via the osquery plugin system +#include + +// Typically registered and invoked by the osquery config subsystem. +// Direct usage via the plugin registry: auto plugin = std::make_shared(); -Status s = plugin->setUp(); -if (!s.ok()) { - LOG(ERROR) << "TLS config setup failed: " << s.getMessage(); +osquery::Status status = plugin->setUp(); +if (!status.ok()) { + LOG(ERROR) << "TLS config setup failed: " << status.getMessage(); } std::map config; -Status result = plugin->genConfig(config); -if (result.ok()) { - for (const auto& [source, json] : config) { - // Process each config source JSON blob - } +status = plugin->genConfig(config); +if (status.ok()) { + for (const auto& [key, value] : config) { + // Process retrieved config segments + } } ``` ## Notes -- The `uri_` field is computed once in `setUp()` and reused across `genConfig()` calls to avoid redundant URL resolution. -- `TLSConfigTests` is declared a `friend` class for unit test access to internals. -- Typically registered via osquery's plugin registry rather than instantiated directly. \ No newline at end of file +- The plugin is registered with osquery's `ConfigPlugin` interface and is invoked automatically when `--config_plugin=tls` is set. +- `uri_` is populated during `setUp()` to avoid redundant URL resolution on each `genConfig()` call. +- `TLSConfigTests` is declared a `friend` class to allow white-box unit testing of internal state. \ No newline at end of file diff --git a/plugins/config/parsers/.auto_constructed_tables.md b/plugins/config/parsers/.auto_constructed_tables.md index b338c64b10c..1670f27872f 100644 --- a/plugins/config/parsers/.auto_constructed_tables.md +++ b/plugins/config/parsers/.auto_constructed_tables.md @@ -1,45 +1,47 @@ -Defines the plugin classes for osquery's Auto Table Construction (ATC) system, which dynamically generates queryable SQL tables from user-defined configuration. +Auto Table Construction (ATC) plugin definitions for dynamically creating osquery virtual tables from SQLite databases at runtime via configuration. ## Key Components ### `ATCPlugin` (extends `TablePlugin`) -Represents a dynamically constructed table backed by a SQLite query against an external database path. +A virtual table plugin that wraps a user-defined SQLite query against an external database file. | Member | Description | -|---|---| -| `ATCPlugin(path, tc_columns, sqlite_query)` | Constructor; initializes table in `PENDING` state | -| `generate(context)` | Executes the configured SQLite query and returns rows | +|--------|-------------| +| `ATCPlugin(path, tc_columns, sqlite_query)` | Constructor β€” initializes with DB path, column schema, and query. Starts in `PENDING` state | +| `generate(QueryContext&)` | Executes the SQLite query and returns rows | | `setActive()` | Transitions the table out of `PENDING` state, marking it ready for queries | -| `attributes()` | Returns current `TableAttributes` (e.g., `PENDING` vs active) | +| `attributes()` | Returns current `TableAttributes` (e.g., `PENDING` or active) | +| `columnDefinition()` | Returns SQL column definition string derived from `tc_columns_` | -> **Note:** The `PENDING` state guards against race conditions when the SQL database is reinitialized between table registration and attachment. +> The `PENDING` state guards against race conditions where a table might be attached twice if the SQL database reinitializes between registration and attachment. ### `ATCConfigParserPlugin` (extends `ConfigParserPlugin`) -Parses the `auto_table_construction` section of the osquery config, manages registration and teardown of ATC tables. +Parses the `auto_table_construction` config section and manages the lifecycle of ATC tables. | Member | Description | -|---|---| -| `keys()` | Returns `{"auto_table_construction"}` as the config key | -| `setUp()` | Initializes the parser and restores previously registered ATC tables | -| `update(source, config)` | Processes config changes β€” registers new tables and removes stale ones | -| `removeATCTables(tables)` | Deregisters a set of ATC tables | -| `registeredATCTables()` | Returns the set of currently active ATC table names from the database | +|--------|-------------| +| `keys()` | Returns `{"auto_table_construction"}` β€” the config key this parser handles | +| `setUp()` | Restores previously registered ATC tables on startup | +| `update(source, config)` | Diffs current vs. new ATC config, removes stale tables, registers new ones | +| `removeATCTables(tables)` | Unregisters a set of ATC tables by name | +| `registeredATCTables()` | Returns the set of currently active ATC table names | ## Usage Example -ATC tables are declared in the osquery config: - -```json -{ - "auto_table_construction": { - "my_custom_table": { - "query": "SELECT * FROM my_db_table", - "path": "/path/to/database.db", - "columns": ["col1", "col2"] - } - } -} -``` - -At runtime, `ATCConfigParserPlugin` parses this config and instantiates an `ATCPlugin` per entry, which is then registered and attached as a queryable osquery table. \ No newline at end of file +```c +// Tables are registered automatically via config β€” example ATC config entry: +// osquery.conf: +// { +// "auto_table_construction": { +// "my_custom_table": { +// "query": "SELECT name, value FROM settings", +// "path": "/var/db/myapp.db", +// "columns": ["name", "value"] +// } +// } +// } + +// Resulting virtual table can be queried directly in osquery: +// SELECT * FROM my_custom_table; +``` \ No newline at end of file diff --git a/plugins/config/parsers/.decorators.md b/plugins/config/parsers/.decorators.md index e6398d8ce3b..8461e7d4745 100644 --- a/plugins/config/parsers/.decorators.md +++ b/plugins/config/parsers/.decorators.md @@ -1,46 +1,45 @@ -Defines the decorator subsystem for osquery, which enriches log items with additional key-value metadata before they are dispatched to downstream logging APIs. +Provides the decorator system interface for osquery, enabling dynamic metadata attachment to log items at defined execution points (load, always, interval). ## Key Components -### `DecorationPoint` Enum -Specifies when decorators should execute: +**`DecorationPoint` (enum)** +Defines when decorators execute: +- `DECORATE_LOAD` β€” runs once at config load +- `DECORATE_ALWAYS` β€” runs on every query cycle +- `DECORATE_INTERVAL` β€” runs on a timed interval -| Value | Description | -|---|---| -| `DECORATE_LOAD` | Run once on configuration load | -| `DECORATE_ALWAYS` | Run on every log event | -| `DECORATE_INTERVAL` | Run at a specified time interval | +**`kDecorationPointKeys` (extern map)** +Maps each `DecorationPoint` enum value to its corresponding configuration key string. -### `kDecorationPointKeys` -A constant map linking each `DecorationPoint` enum value to its corresponding configuration key string. +**`runDecorators()`** +Iterates and executes all discovered decorators for a given decoration point. Accepts an optional timestamp for interval-based decorators and an optional source to restrict execution to a specific config source. -### Functions +**`getDecorations()`** +Retrieves the current decoration result set as a `map` of column name to value. Results are appended to log items before dispatch to downstream logging APIs (`logString`, `logSnapshot`, etc.). -| Function | Description | -|---|---| -| `runDecorators(point, time, source)` | Iterates and executes all discovered decorators for the given `DecorationPoint`. Accepts an optional timestamp (for interval-based points) and an optional source filter. | -| `getDecorations(results)` | Populates `results` with the current set of accumulated decoration key-value pairs. | -| `clearDecorations(source)` | Removes all stored decorations associated with a given config source, typically called when that source updates. | +**`clearDecorations()`** +Clears stored decoration results for a specific config source, typically called when that source updates. ## Usage Example -```c -#include +```cpp +#include -// Execute all "always" decorators +// Run decorators that should always execute osquery::runDecorators(osquery::DECORATE_ALWAYS); -// Execute interval decorators at a specific unix timestamp -osquery::runDecorators(osquery::DECORATE_INTERVAL, 1700000000ULL); +// Run interval-based decorators with current epoch time +osquery::runDecorators(osquery::DECORATE_INTERVAL, getCurrentTime()); -// Retrieve accumulated decorations and apply to a log record +// Retrieve accumulated decorations and attach to a log record std::map decorations; osquery::getDecorations(decorations); + for (const auto& kv : decorations) { logRecord[kv.first] = kv.second; } -// Clear decorations when a config source is refreshed +// Clear decorations when a config source refreshes osquery::clearDecorations("filesystem"); ``` \ No newline at end of file diff --git a/plugins/config/parsers/.feature_vectors.md b/plugins/config/parsers/.feature_vectors.md index c3dcda207a4..945a1d939fc 100644 --- a/plugins/config/parsers/.feature_vectors.md +++ b/plugins/config/parsers/.feature_vectors.md @@ -1,27 +1,29 @@ -A `ConfigParserPlugin` implementation that parses and stores feature vector configuration keys from the osquery config system. +Parses and stores feature vector configuration keys from the osquery config system, providing a `ConfigParserPlugin` implementation for feature vector dictionary entries. ## Key Components -- **`kFeatureVectorsRootKey`** β€” External constant defining the root key used to locate feature vector data within the config tree. -- **`FeatureVectorsConfigParserPlugin`** β€” A `ConfigParserPlugin` subclass responsible for extracting feature vector dictionary keys from the osquery configuration. - - **`keys()`** β€” Returns the list of top-level config keys this parser handles. - - **`update()`** β€” Called when the configuration is loaded or refreshed; processes the parsed config data for the given source. +- **`kFeatureVectorsRootKey`** β€” External constant string defining the root config key for feature vector data +- **`FeatureVectorsConfigParserPlugin`** β€” A `ConfigParserPlugin` subclass that handles feature vector dictionary configuration with two overridden methods: + - `keys()` β€” Returns the list of config keys this parser handles + - `update()` β€” Processes config updates from a given source ## Usage Example -```cpp -// Typically registered automatically via osquery's plugin system. -// Access parsed feature vector data through the Config interface: +```c +// Plugin is typically registered and invoked automatically by the config system. +// To access parsed feature vector data: + +auto& config = Config::get(); +auto parser = config.getParser(kFeatureVectorsRootKey); -auto parser = Config::getParser("feature_vectors"); if (parser != nullptr) { - const auto& doc = parser->getData(); - auto it = doc.doc().FindMember(kFeatureVectorsRootKey.c_str()); - if (it != doc.doc().MemberEnd()) { - // Iterate over feature vector keys - } + auto featureParser = + std::dynamic_pointer_cast(parser); + + // Access the parsed feature vector data from the plugin's internal store + const auto& data = featureParser->getData(); } ``` -> **Note:** This parser is registered as part of osquery's config subsystem and is invoked automatically during config load/refresh cycles. Direct instantiation is not typically required β€” rely on `Config::getParser()` to retrieve the active instance. \ No newline at end of file +> The plugin is auto-registered with osquery's config subsystem. Manual instantiation is typically unnecessary β€” the config system calls `update()` whenever configuration changes are detected from any source. \ No newline at end of file diff --git a/plugins/config/parsers/.kafka_topics.md b/plugins/config/parsers/.kafka_topics.md index f5fc4d378e7..e7a05ef3282 100644 --- a/plugins/config/parsers/.kafka_topics.md +++ b/plugins/config/parsers/.kafka_topics.md @@ -1,12 +1,12 @@ -Declares the `KafkaTopicsConfigParserPlugin` class and associated constants for parsing Kafka topic configurations from osquery's configuration system. +Declares the `KafkaTopicsConfigParserPlugin` class and its associated root key constant for parsing Kafka topic configurations from osquery's config system. ## Key Components -- **`kKafkaTopicParserRootKey`** β€” Extern constant string defining the root key used to retrieve Kafka topic configurations from the osquery config. -- **`KafkaTopicsConfigParserPlugin`** β€” A `ConfigParserPlugin` subclass responsible for extracting and updating Kafka topic configurations. - - `keys()` β€” Returns the list of configuration keys this parser handles. - - `update(source, config)` β€” Processes and applies updated Kafka topic configuration from a given source. +- **`kKafkaTopicParserRootKey`** β€” Extern constant string defining the root JSON/config key used to locate Kafka topic configuration entries. +- **`KafkaTopicsConfigParserPlugin`** β€” A `ConfigParserPlugin` subclass responsible for extracting and updating Kafka topic settings from osquery configuration sources. + - `keys()` β€” Returns the list of config keys this parser handles. + - `update(source, config)` β€” Called by the config system when configuration data changes; processes and stores Kafka topic definitions. ## Usage Example @@ -15,20 +15,22 @@ Declares the `KafkaTopicsConfigParserPlugin` class and associated constants for namespace osquery { -// Access the root key for Kafka topic config lookup -std::cout << kKafkaTopicParserRootKey << std::endl; - -// Plugin is registered via osquery's config plugin system -// and invoked automatically on config updates. -// To retrieve parsed config data: -auto& config = Config::get(); -auto parser = config.getParser(kKafkaTopicParserRootKey); -if (parser != nullptr) { - const auto& doc = parser->getData(); - // Access parsed Kafka topic entries from doc +// The parser is typically registered via the plugin registry, +// not instantiated directly. Example of manual use: +KafkaTopicsConfigParserPlugin parser; + +// Retrieve the keys this parser is responsible for +auto handled_keys = parser.keys(); + +// Simulate a config update from a named source +ParserConfig config; // populated by the config subsystem +Status status = parser.update("filesystem", config); + +if (!status.ok()) { + LOG(ERROR) << "Kafka topic config update failed: " << status.getMessage(); } } // namespace osquery ``` -> **Note:** `KafkaTopicsConfigParserPlugin` is intended to be registered with osquery's plugin registry. Direct instantiation is typically handled by the framework during config initialization. \ No newline at end of file +> **Note:** This parser integrates with osquery's plugin registry. In production, register it using `REGISTER` macros so the config subsystem discovers and invokes it automatically during configuration refreshes. \ No newline at end of file diff --git a/plugins/config/parsers/.logger.md b/plugins/config/parsers/.logger.md index b60609a6265..24b4c9ab04d 100644 --- a/plugins/config/parsers/.logger.md +++ b/plugins/config/parsers/.logger.md @@ -1,29 +1,34 @@ -Config parser plugin for osquery logger settings, exposing a `ConfigParserPlugin` implementation that reads and updates logger-related configuration keys. +A configuration parser plugin for osquery's logger subsystem, responsible for parsing and updating logger-related configuration keys. ## Key Components -- **`LoggerConfigParserPlugin`** β€” Extends `ConfigParserPlugin` to handle the logger configuration section. - - **`keys()`** β€” Returns the list of config keys this parser handles (driven by `kLoggerKey`). - - **`update()`** β€” Called when the configuration source changes; applies updated logger settings. - - **`kLoggerKey`** *(private static)* β€” The string key identifying the logger section in the osquery config. +**`LoggerConfigParserPlugin`** β€” Inherits from `ConfigParserPlugin` and handles the logger section of the osquery configuration. + +| Member | Description | +|--------|-------------| +| `keys()` | Returns the list of config keys this parser handles (`kLoggerKey`) | +| `update(source, config)` | Processes config updates from a given source | +| `kLoggerKey` (private, static) | The string key identifying the logger config section | ## Usage Example ```c -// The plugin is registered with osquery's config system and invoked automatically. -// When a config update is received containing the logger key, update() is triggered: - -Status LoggerConfigParserPlugin::update(const std::string& source, - const ParserConfig& config) { - // Access logger config section via kLoggerKey - auto logger_config = config.find(kLoggerKey); - if (logger_config != config.end()) { - // Apply logger settings (e.g., log paths, verbosity) - data_[kLoggerKey] = logger_config->second; - } - return Status::success(); -} +// The plugin is typically registered and invoked automatically +// by the osquery config system. Example of how the key lookup works: + +class LoggerConfigParserPlugin : public ConfigParserPlugin { +public: + std::vector keys() const override { + return {kLoggerKey}; // e.g., "logger" + } + + Status update(const std::string& source, + const ParserConfig& config) override { + // Reads logger config section and applies updates + // Returns Status::success() or failure with message + } +}; ``` -> **Note:** This parser is automatically discovered and invoked by osquery's `Config` subsystem β€” no manual instantiation is required. Register it via the standard plugin registration macros if extending. \ No newline at end of file +The plugin integrates with osquery's `ConfigParserPlugin` framework β€” when the config system loads or refreshes, it calls `update()` on all registered parser plugins matching their declared `keys()`. No manual instantiation is typically required; registration is handled at plugin initialization time. \ No newline at end of file diff --git a/plugins/config/parsers/.prometheus_targets.md b/plugins/config/parsers/.prometheus_targets.md index 098a50d8bb8..7b5edeec558 100644 --- a/plugins/config/parsers/.prometheus_targets.md +++ b/plugins/config/parsers/.prometheus_targets.md @@ -3,27 +3,33 @@ Defines the `PrometheusMetricsConfigParserPlugin` class and associated constants ## Key Components -- **`kPrometheusParserRootKey`** β€” External constant string defining the root configuration key used to identify Prometheus metrics entries in the osquery config. -- **`PrometheusMetricsConfigParserPlugin`** β€” A `ConfigParserPlugin` subclass responsible for extracting and updating Prometheus scrape target configuration from osquery's config sources. - - **`keys()`** β€” Returns the list of config keys this parser handles (typically wrapping `kPrometheusParserRootKey`). - - **`update()`** β€” Called by the config subsystem when a config source changes; processes the parsed Prometheus target data and stores it for use by the Prometheus metrics table. +- **`kPrometheusParserRootKey`** β€” Extern constant string defining the root configuration key used to identify Prometheus metrics sections in the osquery config +- **`PrometheusMetricsConfigParserPlugin`** β€” A `ConfigParserPlugin` subclass responsible for extracting and processing Prometheus scrape target definitions from the osquery configuration + +### Methods + +| Method | Description | +|--------|-------------| +| `keys()` | Returns the list of config keys this parser handles | +| `update(source, config)` | Called when config changes; processes the Prometheus targets section | ## Usage Example ```cpp -// Plugin is registered automatically via osquery's plugin system. -// Access parsed Prometheus targets through the config after registration: - -auto& config = Config::get(); -config.parsers(); // triggers update() on all registered ConfigParserPlugins - -// Retrieve parsed Prometheus config data using the root key: -auto parser = Config::getParser(kPrometheusParserRootKey); -if (parser != nullptr) { - const auto& data = parser->getData(); - // data contains the parsed Prometheus scrape targets - auto targets = data.get(kPrometheusParserRootKey, {}); +// The plugin is typically registered via the osquery plugin system. +// Direct instantiation example: +PrometheusMetricsConfigParserPlugin parser; + +// Retrieve the config keys this parser is responsible for +std::vector handled_keys = parser.keys(); +// handled_keys will contain kPrometheusParserRootKey + +// Config updates are dispatched automatically by the osquery config system +// when a config source containing Prometheus targets is loaded or refreshed. +Status result = parser.update("filesystem", parserConfig); +if (!result.ok()) { + LOG(ERROR) << "Prometheus config parse failed: " << result.getMessage(); } ``` -> **Note:** This header is part of osquery's Prometheus metrics integration. The plugin is typically auto-registered at startup and interacts with the config subsystem rather than being instantiated directly by consumers. \ No newline at end of file +> The plugin integrates with osquery's config subsystem and is invoked automatically when the config is refreshed β€” direct instantiation is typically unnecessary in production usage. \ No newline at end of file diff --git a/plugins/database/.rocksdb.md b/plugins/database/.rocksdb.md index 54d3288da3d..eb0726bb2ec 100644 --- a/plugins/database/.rocksdb.md +++ b/plugins/database/.rocksdb.md @@ -1,59 +1,55 @@ -RocksDB database plugin implementation for osquery, providing persistent key-value storage backed by RocksDB with column family support, corruption detection, and integrated logging via Glog. +RocksDB-backed implementation of osquery's `DatabasePlugin` interface, providing persistent key-value storage with corruption detection, compaction, and structured logging via GLog. ## Key Components ### `GlogRocksDBLogger` -Extends `rocksdb::Logger` to intercept internal RocksDB log events, forwarding warnings and errors to Glog rather than letting them fall through to stderr. +A custom `rocksdb::Logger` subclass that intercepts internal RocksDB log events and forwards them to GLog instead of stderr. + +| Method | Description | +|--------|-------------| +| `Logv(format, ap)` | Captures RocksDB log messages, inspects severity, and routes to GLog | ### `RocksDBDatabasePlugin` -Implements the `DatabasePlugin` interface using RocksDB as the storage backend. Key method groups: - -| Category | Methods | -|---|---| -| **Lifecycle** | `setUp()`, `tearDown()`, `close()`, `flush()` | -| **Read** | `get()` (string & int overloads), `scan()` | -| **Write** | `put()` (string & int overloads), `putBatch()` | -| **Delete** | `remove()`, `removeRange()` | -| **Maintenance** | `compactFiles()`, `repairDB()` | -| **Corruption** | `setCorrupted()`, `isCorrupted()` | - -**Private members of note:** -- `db_` β€” raw RocksDB database handle -- `handles_` β€” column family handle pointers per domain -- `close_mutex_` β€” guards teardown against concurrent access -- `options_` β€” RocksDB connection/tuning options +Concrete implementation of `DatabasePlugin` backed by RocksDB with column family support. + +| Method | Description | +|--------|-------------| +| `get()` | Retrieves a `string` or `int` value by domain + key | +| `put()` | Stores a `string` or `int` value by domain + key | +| `putBatch()` | Batch-writes multiple string values to a domain | +| `remove()` | Deletes a single key from a domain | +| `removeRange()` | Deletes all keys in `[low, high]` within a domain | +| `scan()` | Lists keys in a domain, optionally filtered by prefix and count | +| `setUp()` | Opens the RocksDB instance and initializes column families | +| `tearDown()` | Closes the database and releases resources | +| `setCorrupted()` | Marks the database as corrupted (static, sets global indicator) | +| `isCorrupted()` | Checks if the corruption flag is set | +| `repairDB()` | Attempts best-effort repair of a corrupted database | +| `flush()` | Flushes memtables and triggers compaction | ## Usage Example ```cpp -#include "rocksdb.h" - -osquery::RocksDBDatabasePlugin plugin; +// Plugin is registered and lifecycle-managed by osquery's plugin system +auto db = std::make_shared(); -// Initialize the database -auto status = plugin.setUp(); -if (!status.ok()) { - LOG(ERROR) << "DB init failed: " << status.getMessage(); +Status s = db->setUp(); +if (!s.ok()) { + // handle open failure or corruption } -// Store a value -plugin.put("config", "refresh_interval", "60"); +// Store and retrieve a value +db->put("queries", "last_run", "1700000000"); -// Retrieve a value std::string value; -plugin.get("config", "refresh_interval", value); - -// Scan keys with a prefix -std::vector keys; -plugin.scan("config", keys, "refresh_", 100); +db->get("queries", "last_run", value); // Batch write -osquery::DatabaseStringValueList batch = {{"key1", "val1"}, {"key2", "val2"}}; -plugin.putBatch("config", batch); +DatabaseStringValueList batch = {{"key1", "val1"}, {"key2", "val2"}}; +db->putBatch("cache", batch); -// Cleanup -plugin.tearDown(); +db->tearDown(); ``` -> **Note:** Corruption is tracked via a global indicator (`setCorrupted`/`isCorrupted`). If detected, `repairDB()` is invoked automatically during `setUp()` as a best-effort recovery step. \ No newline at end of file +> **Note:** `setCorrupted()` and `isCorrupted()` are static and may be triggered by `GlogRocksDBLogger` during abnormal RocksDB log events, causing the plugin to enter a safe degraded state. \ No newline at end of file diff --git a/plugins/database/.sqlite.md b/plugins/database/.sqlite.md index 50a8a9c00c3..dc07a6606fd 100644 --- a/plugins/database/.sqlite.md +++ b/plugins/database/.sqlite.md @@ -1,50 +1,40 @@ -Defines the `SQLiteDatabasePlugin` class, providing a SQLite-backed implementation of osquery's `DatabasePlugin` interface for persistent key-value storage. +SQLite-backed implementation of osquery's `DatabasePlugin` interface, providing persistent key-value storage using an embedded SQLite3 database. ## Key Components -### `SQLiteDatabasePlugin` -Concrete plugin class inheriting from `DatabasePlugin` that wraps a long-lived `sqlite3*` handle. Registered internally as the `"sqlite"` database provider. - -| Method | Description | -|---|---| -| `get()` | Retrieves a `string` or `int` value by domain/key | -| `put()` | Stores a `string` or `int` value by domain/key | -| `putBatch()` | Bulk-inserts a list of string key-value pairs | -| `remove()` | Deletes a single key from a domain | -| `removeRange()` | Deletes all keys within a lexicographic range | -| `scan()` | Lists keys in a domain, optionally filtered by prefix and count | -| `setUp()` | Opens and initializes the SQLite database file | -| `tearDown()` / `~SQLiteDatabasePlugin()` | Closes the database safely | - -### Private Members -- `db_` β€” raw `sqlite3*` handle to the open database -- `close_mutex_` β€” `Mutex` guarding safe concurrent teardown -- `close()` β€” internal cleanup helper called by both destructor and `tearDown()` - -### Registration -```cpp -REGISTER_INTERNAL(SQLiteDatabasePlugin, "database", "sqlite"); -``` -Registers the plugin as the internal `"sqlite"` database backend, activated via the `database_path` flag. +- **`SQLiteDatabasePlugin`** β€” Concrete `DatabasePlugin` subclass registered as the internal `"sqlite"` database provider + - `get()` β€” Retrieves string or integer values by domain/key + - `put()` β€” Stores string or integer values by domain/key + - `putBatch()` β€” Bulk-inserts a list of string key-value pairs into a domain + - `remove()` β€” Deletes a single key from a domain + - `removeRange()` β€” Deletes all keys within a lexicographic range in a domain + - `scan()` β€” Lists keys in a domain, optionally filtered by prefix and capped by count + - `setUp()` β€” Opens and initializes the SQLite database file + - `tearDown()` / destructor β€” Safely closes the database via `close()` + +- **`db_`** β€” Raw `sqlite3*` handle to the long-lived database connection +- **`close_mutex_`** β€” Mutex guarding concurrent access during teardown ## Usage Example -```cpp -// Instantiated and managed by the osquery plugin registry. -// Direct interaction is via the DatabasePlugin interface: +```c +// Plugin is auto-registered; interact via the DatabasePlugin interface: +auto db = RegistryFactory::get().plugin("database", "sqlite"); -std::string value; -auto s = db->get("config", "my_key", value); -if (s.ok()) { - // use value -} +// Store a value +db->put("config", "refresh_interval", "60"); -db->put("config", "my_key", "my_value"); +// Retrieve a value +std::string value; +db->get("config", "refresh_interval", value); +// Scan keys with a prefix std::vector keys; -db->scan("config", keys, "prefix_", 100); +db->scan("config", keys, "refresh_", 100); + +// Remove a key +db->remove("config", "refresh_interval"); +``` -db->remove("config", "my_key"); -db->removeRange("config", "a", "z"); -``` \ No newline at end of file +> The plugin is registered internally via `REGISTER_INTERNAL` and uses the path specified by the `--database_path` flag for its SQLite file. \ No newline at end of file diff --git a/plugins/logger/.aws_firehose.md b/plugins/logger/.aws_firehose.md index 75d5cf30a80..0ea802c31b8 100644 --- a/plugins/logger/.aws_firehose.md +++ b/plugins/logger/.aws_firehose.md @@ -1,49 +1,44 @@ -Header file defining the AWS Kinesis Firehose logger plugin and log forwarder for osquery, enabling buffered log streaming to AWS Firehose delivery streams. +Header defining the AWS Kinesis Firehose log forwarding integration for osquery, providing classes to batch and deliver log records to Firehose delivery streams. ## Key Components -### Type Alias: `IFirehoseLogForwarder` -Template instantiation of `AwsLogForwarder` parameterized with Firehose-specific AWS SDK types (`Record`, `FirehoseClient`, `PutRecordBatchOutcome`, `PutRecordBatchResponseEntry`). +**Type Alias: `IFirehoseLogForwarder`** +Template instantiation of `AwsLogForwarder` parameterized with Firehose-specific AWS SDK types (`Record`, `FirehoseClient`, `PutRecordBatchOutcome`, response entries). -### Class: `FirehoseLogForwarder` -Concrete implementation of `IFirehoseLogForwarder` that handles the low-level Firehose batching and transmission logic. +**Class: `FirehoseLogForwarder`** +Concrete forwarder implementing the Firehose-specific batching and sending logic. Key overrides: +- `internalSetup()` β€” initializes the Firehose client +- `internalSend(batch)` β€” submits a batch via `PutRecordBatch` +- `initializeRecord()` β€” populates a Firehose `Record` from a byte buffer +- Limit accessors: `getMaxBytesPerRecord()`, `getMaxRecordsPerBatch()`, `getMaxBytesPerBatch()` +- Retry controls: `getMaxRetryCount()`, `getInitialRetryDelay()` +- `getFailedRecordCount()` / `getResult()` β€” parses outcome for failures -| Method | Purpose | -|---|---| -| `internalSetup()` | Initializes the Firehose client connection | -| `internalSend(batch)` | Transmits a batch of records to Firehose | -| `initializeRecord()` | Populates a Firehose `Record` from a byte buffer | -| `getMax*()` methods | Returns Firehose-specific limits (batch size, record count, retry config) | -| `appendNewlineSeparators()` | Controls newline insertion between log entries | -| `getFailedRecordCount()` / `getResult()` | Parses batch response for failures | +**Class: `FirehoseLoggerPlugin`** +osquery `LoggerPlugin` integration point. Owns a `FirehoseLogForwarder` instance and bridges osquery's logging lifecycle: +- `setUp()` β€” constructs and starts the forwarder +- `logString()` β€” forwards raw string logs +- `logStatus()` β€” forwards structured status log lines +- `usesLogStatus()` β€” returns `true` to opt into status log routing -### Class: `FirehoseLoggerPlugin` -osquery `LoggerPlugin` integration layer that owns a `FirehoseLogForwarder` instance and bridges osquery's logging API to Firehose. - -| Method | Purpose | -|---|---| -| `setUp()` | Configures the plugin and forwarder | -| `logString()` | Forwards string log entries | -| `logStatus()` | Forwards structured status log lines | -| `usesLogStatus()` | Declares status log support (`true`) | - -### Flag: `aws_firehose_period` -Controls the flush interval (in seconds) for the log forwarder background thread. +**Flag: `aws_firehose_period`** +Declared `uint64` gflag controlling the forwarding interval. ## Usage Example ```cpp -// Registration is handled by osquery's plugin registry. -// Configure via osquery flags at startup: -// -// --logger_plugin=aws_firehose -// --aws_firehose_stream=my-delivery-stream -// --aws_firehose_period=10 -// --aws_region=us-east-1 - -// The plugin is instantiated internally; direct usage follows the LoggerPlugin interface: -auto plugin = std::make_shared(); -plugin->setUp(); -plugin->logString("{\"name\":\"osquery\",\"result\":{...}}"); +// Register and use the plugin via osquery's plugin registry +REGISTER(FirehoseLoggerPlugin, "logger", "aws_firehose"); + +// The plugin reads gflags at setup time: +// --aws_firehose_period=10 +// --aws_firehose_stream=my-delivery-stream +// --aws_region=us-east-1 + +// Internally, FirehoseLoggerPlugin::setUp() creates the forwarder: +forwarder_ = std::make_shared( + name, FLAGS_aws_firehose_period, kMaxLines, endpoint, region); +forwarder_->setUp(); +Dispatcher::addService(forwarder_); ``` \ No newline at end of file diff --git a/plugins/logger/.aws_kinesis.md b/plugins/logger/.aws_kinesis.md index ecaf9dcd578..b259da9ea33 100644 --- a/plugins/logger/.aws_kinesis.md +++ b/plugins/logger/.aws_kinesis.md @@ -1,47 +1,48 @@ -Header file defining the AWS Kinesis log forwarding integration for osquery, providing classes to batch and send log records to an Amazon Kinesis stream. +Declares the AWS Kinesis log forwarding plugin for osquery, providing classes to batch and send log records to an Amazon Kinesis stream. ## Key Components -### Type Alias -- **`IKinesisLogForwarder`** β€” Specialization of the generic `AwsLogForwarder` template, configured with Kinesis-specific types (`PutRecordsRequestEntry`, `KinesisClient`, `PutRecordsOutcome`, `PutRecordsResultEntry`) +**Type Alias** +- `IKinesisLogForwarder` β€” Specialization of `AwsLogForwarder` templated with Kinesis-specific AWS SDK types (`PutRecordsRequestEntry`, `KinesisClient`, `PutRecordsOutcome`, `PutRecordsResultEntry`) -### `KinesisLogForwarder` -Concrete forwarder class that implements batched log delivery to Kinesis. Key overrides: +**`KinesisLogForwarder`** (extends `IKinesisLogForwarder`) +- Implements the core batching and transmission logic for Kinesis +- Key overrides: + - `internalSetup()` β€” Initializes the Kinesis client and partition key + - `internalSend(batch)` β€” Submits a batch via `PutRecords` + - `initializeRecord()` β€” Populates a `PutRecordsRequestEntry` with data + - `getMax*()` methods β€” Returns Kinesis-specific limits (batch size, record count, byte limits) + - `getFailedRecordCount()` / `getResult()` β€” Parses outcome for retry logic +- `partition_key_` β€” Private partition key (overridden by `aws_kinesis_random_partition_key` flag) -| Method | Purpose | -|---|---| -| `internalSetup()` | Initializes the Kinesis client and stream configuration | -| `internalSend(batch)` | Sends a batch of records via `PutRecords` API | -| `initializeRecord()` | Populates a `PutRecordsRequestEntry` with a byte buffer | -| `getMax*()` methods | Enforce Kinesis service limits (batch size, record size, retry policy) | -| `getFailedRecordCount()` | Parses partial failures from the outcome | -| `appendNewlineSeparators()` | Controls newline formatting between records | +**`KinesisLoggerPlugin`** (extends `LoggerPlugin`) +- osquery plugin entry point registered with the logger registry +- `setUp()` β€” Configures and starts the `KinesisLogForwarder` +- `logString()` / `logStatus()` β€” Routes log lines and status events to the forwarder +- `usesLogStatus()` β€” Signals that status logs are consumed by this plugin -**Private member:** `partition_key_` β€” used for Kinesis stream sharding unless `aws_kinesis_random_partition_key` flag is set. - -### `KinesisLoggerPlugin` -osquery `LoggerPlugin` implementation that owns a `KinesisLogForwarder` and integrates with the plugin system. - -| Method | Purpose | -|---|---| -| `setUp()` | Bootstraps the forwarder | -| `logString()` | Forwards a raw string log entry | -| `logStatus()` | Forwards ERROR/WARNING/INFO status lines | -| `usesLogStatus()` | Declares status log support | - -### Flag -- **`aws_kinesis_period`** β€” `uint64` gflag controlling the forwarding interval +**Flag** +- `aws_kinesis_period` β€” Declared via `DECLARE_uint64`; controls the flush interval ## Usage Example -```cpp -// Instantiated internally by the plugin system via KinesisLoggerPlugin::setUp() +```c +// Plugin is registered automatically via osquery's plugin registry. +// Configure via gflags before startup: +// +// --aws_kinesis_stream=my-stream +// --aws_kinesis_period=10 +// --aws_kinesis_random_partition_key=true +// --aws_region=us-east-1 +// +// Manual forwarder instantiation (e.g., in tests): auto forwarder = std::make_shared( - "kinesis-forwarder", - /*log_period=*/10, - /*max_lines=*/500, - /*endpoint_override=*/"", - region + "kinesis", // stream name + 10, // log period (seconds) + 1000, // max lines per batch + "", // no endpoint override + AWSRegion{} // default region ); +forwarder->start(); ``` \ No newline at end of file diff --git a/plugins/logger/.buffered.md b/plugins/logger/.buffered.md index b43f2776c88..9c4c657d2f4 100644 --- a/plugins/logger/.buffered.md +++ b/plugins/logger/.buffered.md @@ -1,56 +1,64 @@ -Defines the `BufferedLogForwarder` base class and `iterate` utility for osquery's buffered log forwarding system, providing reliable database-backed buffering of status and result logs with periodic flushing, backoff support, and utilization-aware iteration. +Provides a thread-safe base class and utility for buffering and periodically forwarding osquery log data (results and statuses) to a backing store before sending to remote endpoints. ## Key Components -### `iterate()` +### `iterate()` (inline function) +Iterates over a `std::vector`, applying a predicate to each element. Automatically yields the thread with a 20ms sleep every 100 iterations to prevent CPU thrash. -A free function that iterates a `std::vector`, applying a predicate to each element. Automatically yields the thread (`20ms` sleep) every 100 iterations to prevent CPU thrash. +### `BufferedLogForwarder` (class) +An abstract base class extending `InternalRunnable` that manages periodic, buffered log forwarding. Subclasses must implement `send()`. -### `BufferedLogForwarder` +**Constructors** (protected): +- `(service_name, name)` β€” uses default period and max log lines +- `(service_name, name, log_period)` β€” custom flush interval +- `(service_name, name, log_period, max_log_lines, max_backoff_period)` β€” full control including backoff -An abstract `InternalRunnable` subclass managing the full lifecycle of buffered log forwarding: +**Key Public Methods:** +| Method | Description | +|---|---| +| `start()` | Main run loop; waits and flushes on interval | +| `setUp()` | Initializes buffer count; **must** be called by subclasses | +| `logString()` | Buffers a result log string to the backing store | +| `logStatus()` | Decorates and buffers status log lines | -| Member | Description | +**Key Protected Methods:** +| Method | Description | |---|---| -| `start()` | Runnable entry point; waits and flushes on schedule | -| `setUp()` | Initializes buffer count β€” **must** be called by subclasses | -| `logString()` | Writes a result string to the backing store | -| `logStatus()` | Decorates and writes status lines to the backing store | | `send()` | **Pure virtual** β€” subclasses implement actual log delivery | -| `check()` | Scans buffered logs, sorts by type, forwards via `send()`, then purges | -| `purge()` | Drops oldest entries when buffer exceeds `buffered_log_max` | -| `backoffTick()` | Reduces exponential backoff period on each successful send | - -**Key constants:** - -- `kLogPeriod` β€” Default flush interval (seconds) -- `kMaxLogLines` β€” Maximum lines flushed per `check()` cycle +| `check()` | Scans buffered logs, sorts by type, calls `send()`, then `purge()` | +| `purge()` | Removes oldest logs when buffer exceeds `buffered_log_max` | +| `backoffTick()` | Reduces exponential backoff timer each cycle | ## Usage Example ```cpp -class MyLogForwarder : public BufferedLogForwarder { +class MyForwarder : public BufferedLogForwarder { public: - MyLogForwarder() - : BufferedLogForwarder("my_service", "mylogger", + MyForwarder() + : BufferedLogForwarder("my_service", "my_logger", std::chrono::seconds(30), 500) {} - // Must call base setUp() - Status setUp() override { - return BufferedLogForwarder::setUp(); - } - - protected: - // Implement actual delivery (e.g., HTTP, file, socket) + // Required: implement actual log delivery Status send(std::vector& log_data, const std::string& log_type) override { - for (auto& line : log_data) { - sendToRemote(log_type, line); + for (auto& entry : log_data) { + sendToRemote(log_type, entry); } return Status::success(); } + + Status setUp() override { + // Must call base setUp() first + auto s = BufferedLogForwarder::setUp(); + if (!s.ok()) return s; + // Custom initialization... + return Status::success(); + } }; -``` -> **Note:** Subclasses overriding `setUp()` **must** call `BufferedLogForwarder::setUp()` to correctly initialize the internal buffer count. \ No newline at end of file +// Register and start +auto forwarder = std::make_shared(); +forwarder->setUp(); +Dispatcher::addService(forwarder); +``` \ No newline at end of file diff --git a/plugins/logger/.filesystem_logger.md b/plugins/logger/.filesystem_logger.md index 8ef7b1db21c..503b020c88e 100644 --- a/plugins/logger/.filesystem_logger.md +++ b/plugins/logger/.filesystem_logger.md @@ -1,37 +1,37 @@ -A logger plugin that writes osquery query results, snapshots, and status logs to the local filesystem using separate file paths for each log type. +Header for the `FilesystemLoggerPlugin` β€” an osquery logger plugin that writes query results, snapshots, and status logs to the local filesystem. ## Key Components -| Component | Description | -|---|---| -| `FilesystemLoggerPlugin` | Main logger plugin class extending `LoggerPlugin` | -| `setUp()` | Initializes filesystem paths and prepares log destinations | -| `logString()` | Writes differential query results to a dedicated results file | -| `logSnapshot()` | Writes snapshot query results to a dedicated snapshot file | -| `logStatus()` | Forwards status/health logs to Glog | -| `init()` | Post-startup initialization; may return errors to allow Glog direct file writes | -| `logStringToFile()` | Internal helper that performs the actual file write operation | -| `pimpl_` | Pointer-to-implementation (PIMPL) pattern for private state encapsulation | +| Symbol | Type | Description | +|--------|------|-------------| +| `FilesystemLoggerPlugin` | Class | Concrete `LoggerPlugin` that routes osquery output to filesystem paths | +| `setUp()` | Method | Initializes file handles and validates output paths before logging begins | +| `logString()` | Method | Writes differential query results to a dedicated results file | +| `logSnapshot()` | Method | Writes snapshot query results to a separate snapshot file | +| `init()` | Method | Post-startup initialization; uniquely may return an error to allow Glog direct file writes | +| `logStatus()` | Method | Forwards status/health log lines to Glog | +| `logStringToFile()` | Private method | Core write primitive used by `logString` and `logSnapshot`; accepts an optional `empty` flag | +| `impl` (pimpl) | Private struct | Opaque implementation detail hiding file handles and state (pointer-to-implementation pattern) | +| `REGISTER(...)` | Macro | Registers the plugin under the `"logger"` registry with the key `"filesystem"` | ## Usage Example ```c -// Plugin is auto-registered via the REGISTER macro at load time: -// REGISTER(FilesystemLoggerPlugin, "logger", "filesystem"); - -// Configure via osquery flags (e.g., osquery.flags or CLI): -// --logger_plugin=filesystem -// --logger_path=/var/log/osquery - -// The plugin then writes to separate files: -// /var/log/osquery/osqueryd.results.log <- logString() -// /var/log/osquery/osqueryd.snapshots.log <- logSnapshot() -// /var/log/osquery/osqueryd.INFO <- logStatus() via Glog +// Plugin is auto-registered via the REGISTER macro. +// Activate via osquery flag at startup: +// --logger_plugin=filesystem +// --logger_path=/var/log/osquery + +// Internally, the plugin writes to: +// /var/log/osquery/osqueryd.results.log (logString) +// /var/log/osquery/osqueryd.snapshots.log (logSnapshot) + +// Direct instantiation (e.g., in tests): +osquery::FilesystemLoggerPlugin logger; +logger.setUp(); +logger.logString("{\"name\":\"processes\", \"action\":\"added\"}"); +logger.logSnapshot("{\"name\":\"users\", \"snapshot\":[...]}"); ``` -## Notes - -- Registered globally under the `"logger"` registry with the key `"filesystem"` β€” no manual instantiation required. -- Uses the **PIMPL idiom** (`struct impl`) to hide implementation details and reduce compile-time dependencies. -- Unique among osquery loggers in that `init()` can intentionally return an error, delegating status log file management directly to Glog. \ No newline at end of file +> **Note:** This is the only built-in osquery logger that may intentionally fail during `init()`, allowing Glog to take ownership of file-based status logging directly rather than routing through the plugin. \ No newline at end of file diff --git a/plugins/logger/.kafka_producer.md b/plugins/logger/.kafka_producer.md index 79718d53add..09b72116d0a 100644 --- a/plugins/logger/.kafka_producer.md +++ b/plugins/logger/.kafka_producer.md @@ -1,40 +1,43 @@ -Declares the `KafkaProducerPlugin` class, which integrates osquery's logging pipeline with Apache Kafka by implementing both `LoggerPlugin` and `InternalRunnable` interfaces via librdkafka. +Declares the `KafkaProducerPlugin` class and supporting utilities for publishing osquery log payloads to Apache Kafka brokers via `librdkafka`. ## Key Components -| Symbol | Type | Description | -|--------|------|-------------| -| `kKafkaBaseTopic` | `extern const std::string` | Default Kafka topic used when a payload's `name` field has no configured mapping | -| `getMsgName()` | Free function | Extracts the `"name"` field from a JSON log payload string | -| `KafkaProducerPlugin` | Class | Core plugin β€” produces log messages to Kafka brokers, manages topic routing, and runs a background poll loop | -| `logString()` | Method | Serializes a log string and produces it to the appropriate Kafka topic | -| `init()` | Method | Configures librdkafka producer, registers the OS hostname as `client.id` | -| `start()` / `stop()` | Methods | `InternalRunnable` lifecycle β€” poll loop entry point and graceful flush/shutdown | -| `publishMsg()` | Protected virtual | Low-level publish to a specific `rd_kafka_topic_t*` | -| `flushMessages()` | Protected virtual | Mutex-guarded `rd_kafka_flush` with a 3-second timeout | -| `pollKafka()` | Protected virtual | Mutex-guarded `rd_kafka_poll` to drive delivery callbacks | -| `queryToTopics_` | `std::map` | Routes query names to their corresponding Kafka topic handles | +**Free Functions** +- `getMsgName(payload)` β€” Extracts the `"name"` field from a log payload string to determine the target Kafka topic. +- `kKafkaBaseTopic` β€” Default fallback topic used when a payload name cannot be resolved. + +**`KafkaProducerPlugin`** β€” Dual-inherits `LoggerPlugin` and `InternalRunnable` to act as both an osquery logger and a background thread. + +| Method | Description | +|--------|-------------| +| `init()` | Configures `librdkafka` producer; registers OS hostname as `client.id` | +| `logString(s)` | Publishes a log payload string to the appropriate Kafka topic | +| `start()` | Background thread entry point; polls Kafka in a loop | +| `stop()` | Flushes remaining messages (max 3s wait) and shuts down | +| `publishMsg(topic, payload)` | Low-level publish to a specific `rd_kafka_topic_t*` | +| `flushMessages()` | Mutex-guarded wrapper around `rd_kafka_flush` | +| `pollKafka()` | Mutex-guarded wrapper around `rd_kafka_poll`; triggers delivery callbacks | + +**Key Members** +- `running_` β€” Atomic bool tracking producer state +- `queryToTopics_` β€” Maps query names to resolved `rd_kafka_topic_t*` handles +- `producerMutex_` β€” Guards concurrent access to the producer pointer +- `shutdownFlag_` β€” Ensures `stop()` cleanup runs exactly once ## Usage Example ```cpp -// Registration via osquery plugin system (typical CMake/registry wiring) -REGISTER(KafkaProducerPlugin, "logger", "kafka_producer"); +#include "kafka_producer.h" -// Retrieve and initialize the plugin (osquery runtime handles this internally) -auto plugin = std::make_shared(); -plugin->init("kafka_producer", {}); +osquery::KafkaProducerPlugin producer; -// Log a JSON payload β€” topic is resolved from the payload's "name" field; -// falls back to kKafkaBaseTopic if no matching topic is configured. -Status s = plugin->logString(R"({"name":"pack_query","result":"..."})"); -if (!s.ok()) { - LOG(ERROR) << "Kafka publish failed: " << s.getMessage(); -} +// Initialize with broker config from osquery flags +producer.init("kafka_producer", {}); -// Background dispatcher thread calls start(); shutdown triggers stop() -// which flushes up to 3 seconds of buffered messages before exit. -``` +// Log a JSON payload β€” routes to topic based on "name" field +producer.logString(R"({"name":"process_events","columns":{"pid":"1234"}})"); -> **Note:** `KafkaProducerPlugin` is non-copyable by design. Topic-to-query routing is configured via osquery flags consumed inside `configureTopics()` at `init()` time. \ No newline at end of file +// On shutdown +producer.stop(); +``` \ No newline at end of file diff --git a/plugins/logger/.stdout.md b/plugins/logger/.stdout.md index 428dc446142..b326dc83d35 100644 --- a/plugins/logger/.stdout.md +++ b/plugins/logger/.stdout.md @@ -1,40 +1,34 @@ -Defines the `StdoutLoggerPlugin` class, an osquery logger plugin that outputs log messages directly to standard output (stdout). +Defines the `StdoutLoggerPlugin` class, an osquery logger plugin that outputs log messages and status lines to standard output. ## Key Components -### `StdoutLoggerPlugin` -A concrete implementation of `LoggerPlugin` that writes log output to stdout. +- **`StdoutLoggerPlugin`** β€” Inherits from `LoggerPlugin`; registers itself under the `"stdout"` logger name via the `REGISTER` macro +- **`usesLogStatus()`** β€” Returns `true`, enabling status log forwarding to this plugin +- **`logString()`** β€” Handles plain string log output to stdout +- **`init()`** β€” Initializes the plugin with a name and any buffered status log lines +- **`logStatus()`** β€” Handles structured `StatusLogLine` vector output to stdout -| Member | Type | Description | -|---|---|---| -| `usesLogStatus()` | `bool` | Returns `true`, enabling status log handling | -| `logString()` | `Status` | Writes a plain string message to stdout | -| `init()` | `void` | Initializes the plugin with a name and buffered startup log lines | -| `logStatus()` | `Status` | Writes a batch of structured `StatusLogLine` entries to stdout | +## Usage Example -### Plugin Registration -```c -REGISTER(StdoutLoggerPlugin, "logger", "stdout"); -``` -Registers the plugin under the `"logger"` registry with the key `"stdout"`, making it selectable via osquery configuration. +```cpp +// Register and use the stdout logger via osquery config: +// { "options": { "logger_plugin": "stdout" } } -## Usage Example +// Or instantiate directly in a test context: +StdoutLoggerPlugin plugin; +plugin.init("stdout", {}); -Enable the stdout logger in osquery configuration or via the `--logger_plugin` flag: +Status result = plugin.logString("Hello from osquery"); -```bash -osqueryi --logger_plugin=stdout +std::vector statusLines = { /* ... */ }; +Status statusResult = plugin.logStatus(statusLines); ``` -Or in `osquery.conf`: +The plugin is auto-registered at startup via: -```json -{ - "options": { - "logger_plugin": "stdout" - } -} +```cpp +REGISTER(StdoutLoggerPlugin, "logger", "stdout"); ``` -Log output will be streamed directly to stdout, useful for containerized environments or debugging where log forwarding is handled externally (e.g., Docker log drivers, systemd journal capture). \ No newline at end of file +No manual instantiation is needed in production β€” osquery's registry resolves it by name when `logger_plugin = stdout` is set in configuration. \ No newline at end of file diff --git a/plugins/logger/.syslog_logger.md b/plugins/logger/.syslog_logger.md index 9ccfb0a9dfd..92a5cda8b6c 100644 --- a/plugins/logger/.syslog_logger.md +++ b/plugins/logger/.syslog_logger.md @@ -1,40 +1,31 @@ -Header defining the `SyslogLoggerPlugin` class, which implements osquery's logger plugin interface to route log output to the system syslog facility. +Defines the `SyslogLoggerPlugin` class, an osquery logger plugin that routes query results and status messages to the system syslog daemon. ## Key Components -### `SyslogLoggerPlugin` -Extends `LoggerPlugin` to provide syslog-based logging for osquery. - -| Method | Description | -|---|---| -| `usesLogStatus()` | Returns `true`, enabling status log routing through this plugin | -| `logString(const std::string& s)` | Writes a single log string entry to syslog | -| `init(const std::string& name, const std::vector& log)` | Initializes the plugin and flushes any buffered startup logs | -| `logStatus(const std::vector& log)` | Writes a batch of status log lines to syslog | - -### Plugin Registration -```c -REGISTER(SyslogLoggerPlugin, "logger", "syslog"); -``` -Registers the plugin under the `"logger"` category with the key `"syslog"`, making it selectable via osquery configuration. +- **`SyslogLoggerPlugin`** β€” Extends `LoggerPlugin` to implement syslog-based logging for osquery +- **`usesLogStatus()`** β€” Returns `true`, indicating this plugin handles status log lines +- **`logString()`** β€” Writes a single log string entry to syslog +- **`init()`** β€” Initializes the plugin with a name and any buffered status log lines +- **`logStatus()`** β€” Writes a batch of `StatusLogLine` entries to syslog +- **`REGISTER` macro** β€” Registers the plugin under the `"logger"` registry with the key `"syslog"` ## Usage Example -Enable the syslog logger by setting the `logger_plugin` flag in your osquery configuration: +```c +// Enable the syslog logger via osquery configuration flag: +// --logger_plugin=syslog -```bash -osqueryd --logger_plugin=syslog -``` +// Or programmatically reference the registered plugin: +auto plugin = RegistryFactory::get().plugin("logger", "syslog"); -Or in a config file: +// Once active, osquery routes results and status messages +// through SyslogLoggerPlugin::logString() and logStatus(), +// which write to the system syslog using facilities. -```json -{ - "options": { - "logger_plugin": "syslog" - } -} +// Example syslog output visible via: +// journalctl -t osqueryd +// or: tail -f /var/log/syslog | grep osquery ``` -Once active, osquery query results and status messages are forwarded to the system syslog daemon (typically viewable via `journalctl` or `/var/log/syslog`), using the standard `` interface for log level mapping. \ No newline at end of file +> **Note:** This plugin integrates directly with the host OS syslog daemon (e.g., `rsyslog`, `syslog-ng`, `journald`), making osquery logs available alongside other system services without requiring a separate log file. \ No newline at end of file diff --git a/plugins/logger/.tls_logger.md b/plugins/logger/.tls_logger.md index 814e7188e70..bc3dc38393b 100644 --- a/plugins/logger/.tls_logger.md +++ b/plugins/logger/.tls_logger.md @@ -1,51 +1,47 @@ -TLS logger plugin and forwarder declarations for buffering and transmitting osquery logs to a remote TLS endpoint via a background dispatcher thread. +TLS logger plugin header defining classes for buffering and forwarding osquery logs to a remote TLS endpoint via a background dispatcher thread. ## Key Components -### `TLSLogForwarder` -Extends `BufferedLogForwarder` to flush database-buffered result and status logs to a TLS endpoint. Runs as a `Dispatcher` service when an enrollment key is present. +### `TLSLogForwarder` (extends `BufferedLogForwarder`) +Background thread that flushes buffered result and status logs to a TLS endpoint. -| Member | Description | -|---|---| -| `configuration_mutex` | Guards concurrent reads/writes to configuration state | -| `configuration_updated` | Flag signaling pending config changes | -| `updated_uri`, `updated_log_period`, etc. | Staged config values applied on next `applyNewConfiguration()` | -| `send()` | Transmits a batch of log strings to the TLS endpoint | -| `applyNewConfiguration()` | Applies staged configuration changes atomically | -| `uri_` | Active endpoint URI | +| Member | Type | Description | +|--------|------|-------------| +| `configuration_mutex` | `std::mutex` | Guards concurrent config reads/writes | +| `configuration_updated` | `bool` | Signals pending config changes | +| `updated_uri` | `std::string` | Staged endpoint URI | +| `updated_log_period` | `std::chrono::seconds` | Staged flush interval | +| `updated_max_backoff_period` | `std::chrono::seconds` | Staged max retry backoff | +| `updated_max_log_lines` | `uint64_t` | Staged per-batch line limit | +| `send()` | `Status` | Transmits a batch of logs by type | +| `applyNewConfiguration()` | `void` | Applies staged config values | -### `TLSLoggerPlugin` -Implements `LoggerPlugin` to receive, buffer, and forward osquery logs over TLS. +### `TLSLoggerPlugin` (extends `LoggerPlugin`) +osquery logger plugin that buffers logs locally and delegates forwarding to `TLSLogForwarder`. | Method | Description | -|---|---| -| `init()` | Buffers early status logs captured before logger initialization | +|--------|-------------| +| `init()` | Receives early-boot status logs before logger is fully initialized | | `setUp()` | Validates node key and starts the forwarder dispatcher thread | | `configure()` | Reacts to runtime configuration updates | | `logString()` | Handles snapshot and event result logs | -| `logStatus()` | Handles `ERROR`/`WARNING`/`INFO` status logs | -| `usesLogStatus()` | Returns `true` β€” plugin consumes status log lines | +| `logStatus()` | Handles `ERROR`/`WARNING`/`INFO` status messages | +| `usesLogStatus()` | Returns `true` β€” plugin consumes status logs | ## Usage Example ```cpp -// Plugin is registered and managed by the osquery plugin system. -// Direct instantiation is typically handled by the registry: - -auto plugin = std::make_shared(); - -// Called by the framework at startup with early-buffered status logs: -plugin->init("tls", early_status_logs); - -// Called after node enrollment to start the forwarder thread: -Status s = plugin->setUp(); -if (!s.ok()) { - LOG(ERROR) << "TLS logger setup failed: " << s.getMessage(); -} - -// React to a config change (e.g., updated TLS endpoint URI): -plugin->configure(); -``` - -> **Note:** `TLSLogForwarder` configuration fields (`updated_uri`, `updated_log_period`, etc.) should be written under `configuration_mutex` before setting `configuration_updated = true` to safely stage changes for the next flush cycle. \ No newline at end of file +// Registered automatically via osquery plugin system +// Manual instantiation for testing: +TLSLoggerPlugin plugin; +plugin.setUp(); +plugin.logString("{\"action\":\"added\",\"name\":\"osquery\"}"); + +// Reconfigure the forwarder endpoint at runtime +forwarder->configuration_mutex.lock(); +forwarder->updated_uri = "https://fleet.example.com/api/v1/osquery/log"; +forwarder->updated_log_period = std::chrono::seconds(30); +forwarder->configuration_updated = true; +forwarder->configuration_mutex.unlock(); +``` \ No newline at end of file diff --git a/plugins/logger/.windows_event_log.md b/plugins/logger/.windows_event_log.md index ab21a48c8ad..2a5daecd8af 100644 --- a/plugins/logger/.windows_event_log.md +++ b/plugins/logger/.windows_event_log.md @@ -1,48 +1,41 @@ -Windows Event Log logger plugin interface for osquery, providing integration with the Windows Event Log system via ETW (Event Tracing for Windows). +Windows Event Log logger plugin interface for osquery, providing integration with the Windows Event Log system through the ETW (Event Tracing for Windows) provider mechanism. ## Key Components -### `WindowsEventLoggerPlugin` -Extends `LoggerPlugin` to route osquery logs to the Windows Event Log subsystem. +### `WindowsEventLoggerPlugin` (class) +Extends `LoggerPlugin` to route osquery log output to the Windows Event Log. Manages an ETW provider registration handle and implements the standard logger plugin interface. -| Member | Description | -|---|---| -| `usesLogStatus()` | Returns `true` β€” enables status log routing | -| `logString()` | Writes a plain string message to the event log | -| `init()` | Initializes the plugin with a name and initial log lines | -| `logStatus()` | Writes structured `StatusLogLine` entries to the event log | -| `acquireHandle()` | Registers the process as a Windows Event Log provider | -| `releaseHandle()` | Releases the ETW `REGHANDLE` provider registration | -| `emitLogRecord()` | Emits a single log record with severity, source file, and line number | +**Protected Methods:** +- `logString(const std::string& s)` β€” Logs a plain string message to the event log +- `init(const std::string& name, const std::vector& log)` β€” Initializes the plugin and registers the ETW provider +- `logStatus(const std::vector& log)` β€” Logs structured status lines with severity levels -**Private member:** `registration_handle_` β€” the `REGHANDLE` used for all ETW provider operations. +**Public Static Helpers:** +- `acquireHandle(REGHANDLE&)` β€” Registers the process as a Windows Event Log provider via `evntprov.h` +- `releaseHandle(REGHANDLE&)` β€” Unregisters the ETW provider and releases the handle +- `emitLogRecord(...)` β€” Emits a single event with message, severity, source file, and line number ## Usage Example ```cpp #include "windows_event_log.h" -osquery::WindowsEventLoggerPlugin plugin; +// Acquire a provider handle and emit a log record directly +REGHANDLE handle = 0; +auto status = WindowsEventLoggerPlugin::acquireHandle(handle); -// Acquire an ETW provider handle -REGHANDLE handle; -auto status = osquery::WindowsEventLoggerPlugin::acquireHandle(handle); -if (!status.ok()) { - // handle error -} +if (status.ok()) { + WindowsEventLoggerPlugin::emitLogRecord( + handle, + "Service started successfully", + O_INFO, + __FILE__, + __LINE__ + ); -// Emit a log record manually -osquery::WindowsEventLoggerPlugin::emitLogRecord( - handle, - "osquery service started", - osquery::O_INFO, - __FILE__, - __LINE__ -); - -// Release the handle when done -osquery::WindowsEventLoggerPlugin::releaseHandle(handle); + WindowsEventLoggerPlugin::releaseHandle(handle); +} ``` -> **Note:** This header is Windows-only and depends on `evntprov.h` (Windows SDK). The static `acquireHandle` / `releaseHandle` / `emitLogRecord` methods can be used independently of the plugin lifecycle for direct ETW emission. \ No newline at end of file +> **Note:** This header is Windows-only and depends on `evntprov.h` from the Windows SDK. The plugin is typically registered via osquery's plugin registry rather than instantiated directly. \ No newline at end of file diff --git a/plugins/numeric_monitoring/.filesystem.md b/plugins/numeric_monitoring/.filesystem.md index 80f740c370c..c125cd99d25 100644 --- a/plugins/numeric_monitoring/.filesystem.md +++ b/plugins/numeric_monitoring/.filesystem.md @@ -1,45 +1,42 @@ -Filesystem-backed implementation of the `NumericMonitoringPlugin` interface that writes numeric monitoring data points as delimited text lines to a log file. +Filesystem-based implementation of the `NumericMonitoringPlugin` interface that writes numeric monitoring data as formatted, delimited lines to a log file on disk. ## Key Components -### `NumericMonitoringFilesystemPlugin` -Extends `NumericMonitoringPlugin` to persist monitoring metrics to disk. +**`NumericMonitoringFilesystemPlugin`** β€” Concrete plugin class extending `NumericMonitoringPlugin` with the following members: | Member | Description | -|---|---| +|--------|-------------| | `NumericMonitoringFilesystemPlugin()` | Default constructor using configured log file path | -| `NumericMonitoringFilesystemPlugin(path)` | Constructor with explicit log file path override | -| `call(request, response)` | Handles incoming plugin requests and writes formatted metric lines | -| `setUp()` | Initializes the output file stream; must be called before `call()` | +| `NumericMonitoringFilesystemPlugin(path)` | Constructor accepting an explicit log file path | +| `call(request, response)` | Handles incoming monitoring data, formats and appends a line to the log file | +| `setUp()` | Opens the log file stream and prepares the plugin for use | | `isSetUp()` | Returns whether the plugin has been successfully initialized | -| `formTheLine(line, request)` | Private helper that formats a metric entry using `line_format_` and `separator_` | +| `formTheLine(line, request)` | Private helper that formats a `PluginRequest` into a delimited string using `line_format_` and `separator_` | -**Private fields:** -- `line_format_` β€” ordered list of fields to include per line -- `separator_` β€” single character delimiter between fields -- `log_file_path_` β€” resolved path to the output log file -- `output_file_stream_` β€” open file stream for writing -- `output_file_mutex_` β€” mutex guarding concurrent writes +**Private State:** +- `line_format_` β€” Ordered list of field names defining the output line structure +- `separator_` β€” Character delimiter between fields +- `log_file_path_` β€” Target log file path +- `output_file_stream_` β€” Active file stream handle +- `output_file_mutex_` β€” Mutex guarding concurrent writes ## Usage Example ```cpp -#include +#include "filesystem.h" -// Use explicit path (e.g., in tests or custom setups) -osquery::NumericMonitoringFilesystemPlugin plugin( - boost::filesystem::path("/var/log/osquery/numeric_monitoring.log")); +// Use default configured path +osquery::NumericMonitoringFilesystemPlugin plugin; +plugin.setUp(); -auto status = plugin.setUp(); -if (!plugin.isSetUp()) { - // handle initialization failure -} - -// Plugin is invoked via the registry; direct call example: -osquery::PluginRequest req = {{"path", "cpu.usage"}, {"value", "42.5"}}; -osquery::PluginResponse resp; -plugin.call(req, resp); -``` +// Or specify an explicit path +boost::filesystem::path logPath("/var/log/osquery/monitoring.log"); +osquery::NumericMonitoringFilesystemPlugin plugin(logPath); -> **Note:** `setUp()` must succeed before invoking `call()`. The plugin is thread-safe for concurrent metric writes via `output_file_mutex_`. \ No newline at end of file +if (plugin.setUp().ok() && plugin.isSetUp()) { + osquery::PluginRequest request = {{"path", "cpu.usage"}, {"value", "42.5"}}; + osquery::PluginResponse response; + plugin.call(request, response); +} +``` \ No newline at end of file diff --git a/plugins/remote/enroll/.tls_enroll.md b/plugins/remote/enroll/.tls_enroll.md index 8d8875c6c5a..c1241b1a9a6 100644 --- a/plugins/remote/enroll/.tls_enroll.md +++ b/plugins/remote/enroll/.tls_enroll.md @@ -1,31 +1,34 @@ -TLS enrollment plugin that extends the base `EnrollPlugin` to handle node key acquisition and caching via TLS endpoints. +Declares the `TLSEnrollPlugin` class, which implements TLS-based node enrollment for osquery's remote configuration system by requesting and caching enrollment keys from a TLS endpoint. ## Key Components -- **`TLSEnrollPlugin`** β€” Concrete implementation of `EnrollPlugin` for TLS-based enrollment - - `enroll()` β€” Returns a cached node key if available, otherwise triggers a new key request - - `requestKey(uri, node_key)` β€” Contacts the configured TLS endpoint to obtain a new enrollment key - - `TLSEnrollTests` β€” Declared as a friend class to allow unit test access to private members +### `TLSEnrollPlugin` (extends `EnrollPlugin`) +| Member | Description | +|--------|-------------| +| `enroll()` | Returns a cached enrollment key or triggers a new key request if none is cached | +| `requestKey(uri, node_key)` | Performs the HTTP/TLS request to the enrollment endpoint and populates the node key | +| `TLSEnrollTests` | Declared as a `friend` class to allow unit test access to private members | ## Usage Example -```c -// Plugin is registered and invoked through the osquery enrollment system. -// Direct instantiation is managed by the plugin registry. +```cpp +// TLSEnrollPlugin is typically registered as an osquery plugin +// and invoked through the enrollment subsystem, not directly. -// Enrollment is triggered automatically when the agent starts: -// 1. enroll() checks for a cached node_key -// 2. If none exists, requestKey() contacts the TLS endpoint -// 3. The returned key is cached for subsequent calls +#include -// Configured via osquery flags, e.g.: -// --enroll_tls_endpoint=/api/v1/enroll -// --tls_hostname=fleet.example.com -``` +namespace osquery { + +// Registration example (typically in a plugin init block) +REGISTER(TLSEnrollPlugin, "enroll", "tls"); -## Notes +} // namespace osquery + +// Once registered, the enrollment system calls enroll() automatically +// when a node_key is required for authenticating distributed/config TLS requests. +// The plugin handles key caching internally, only contacting the TLS +// endpoint (--enroll_secret_path / --tls_enroll_uri) when no key is stored. +``` -- Inherits enrollment caching logic from `EnrollPlugin` base class -- `requestKey()` returns a `Status` object β€” callers should check for success before using the populated `node_key` -- All members are private; interaction is through the plugin registry interface defined in `enroll.h` \ No newline at end of file +> **Note:** `requestKey` returns a `Status` object β€” callers should check `Status::ok()` to confirm successful enrollment before using the returned `node_key`. \ No newline at end of file diff --git a/tests/.test_util.md b/tests/.test_util.md index f5eb91f435b..11931e4ed7b 100644 --- a/tests/.test_util.md +++ b/tests/.test_util.md @@ -1,48 +1,41 @@ -Utility header providing shared test infrastructure for osquery unit tests and benchmarks, including process exit codes, path variables, and helper functions used across the test suite. +Shared test utility header providing common constants, path variables, and helper functions used across osquery's test and benchmark suites. ## Key Components ### Constants & Exit Codes - -| Constant | Value | Purpose | -|---|---|---| -| `EXTENSION_SUCCESS_CODE` | `0x45` | Expected exit code for successful extension child process | -| `WORKER_SUCCESS_CODE` | `0x57` | Expected exit code for successful worker child process | -| `ERROR_COMPARE_ARGUMENT` | `-1` | Child process argument comparison failure | -| `ERROR_LAUNCHER_PROCESS` | `-2` | Launcher process error | -| `ERROR_LAUNCHER_MISMATCH` | `-5` | Launcher identity mismatch | - -### Global Path Variables - -- `kTestDataPath` β€” resolved path to test data (source or build directory) -- `kTestWorkingDirectory` β€” working directory for config, logs, and caching during tests -- `kFakeDirectory` β€” root of the fake filesystem tree (`fstree`) used in iterator tests -- `kProcessTestExecPath` β€” path to the currently executing test binary +- `EXTENSION_SUCCESS_CODE` (`0x45`) / `WORKER_SUCCESS_CODE` (`0x57`) β€” expected exit codes for successful child process runs +- `ERROR_COMPARE_ARGUMENT` through `ERROR_LAUNCHER_MISMATCH` β€” error codes returned by child processes on failure + +### Global Test Variables +| Variable | Purpose | +|---|---| +| `kTestDataPath` | Path to test data files (source or build dir) | +| `kTestWorkingDirectory` | Sandbox for config, logs, and cache during tests | +| `kFakeDirectory` | Root of the fake filesystem tree (`fstree`) | +| `kProcessTestExecPath` | Path of the currently executing test binary | +| `kExpectedWorkerArgs` / `kExpectedExtensionArgs` | Expected CLI arguments for worker/extension child processes | ### Functions - -- `initTesting()` β€” initializes osquery runtime for tests and benchmarks -- `shutdownTesting()` β€” tears down the osquery runtime after tests -- `getOsqueryScheduledQuery()` β€” returns a sample `ScheduledQuery` for test use -- `genRows(EventSubscriberPlugin*)` β€” materializes all rows from a generator-based event subscriber table -- `initUsersAndGroupsServices()` / `deinitUsersAndGroupsServices()` *(Windows only)* β€” manages Windows user/group service lifecycle in tests +- `initTesting()` β€” initializes the osquery environment for tests and benchmarks +- `shutdownTesting()` β€” tears down the test environment cleanly +- `getOsqueryScheduledQuery()` β€” returns a sample `ScheduledQuery` object for use in tests +- `genRows(EventSubscriberPlugin*)` β€” exhausts a generator-based table subscriber and returns all `TableRows` +- `initUsersAndGroupsServices` / `deinitUsersAndGroupsServices` *(Windows only)* β€” manages Windows user/group service lifecycle in tests ## Usage Example ```c #include "test_util.h" -int main() { - osquery::initTesting(); +// In a test fixture's SetUp / main(): +osquery::initTesting(); - // Access resolved test data path - std::string dataPath = osquery::kTestDataPath; +// Access shared paths +std::string dataFile = osquery::kTestDataPath + "/sample.conf"; - // Generate rows from an event subscriber - auto rows = osquery::genRows(mySubscriberPlugin); +// Generate rows from an event subscriber +auto rows = osquery::genRows(mySubscriberPlugin); - osquery::shutdownTesting(); - return 0; -} +osquery::shutdownTesting(); ``` \ No newline at end of file diff --git a/tests/integration/tables/.helper.md b/tests/integration/tables/.helper.md index 79409be36c4..72177d558e8 100644 --- a/tests/integration/tables/.helper.md +++ b/tests/integration/tables/.helper.md @@ -1,51 +1,39 @@ -Utility header for osquery table-level integration tests, providing validation primitives, type-flag constants, and helper functions used across SQL table test suites. +Utility header for osquery table integration tests, providing validation primitives and helper types used to verify SQL query results against expected data constraints. ## Key Components ### Validator Classes - -| Class | Purpose | -|---|---| -| `IntMinMaxCheck` | Validates a string-encoded integer falls within `[min, max]` | -| `SpecificValuesCheck` | Validates a string belongs to a predefined allowlist | -| `CronValuesCheck` | Validates cron-style values β€” numeric range or special strings (e.g., `*`, `@reboot`) | +- **`IntMinMaxCheck`** β€” Functor that validates a string value falls within a specified `int64_t` [min, max] range +- **`SpecificValuesCheck`** β€” Functor that validates a string belongs to a predefined set of allowed values +- **`CronValuesCheck`** β€” Functor that validates cron-style values, accepting numeric ranges or named string tokens (e.g., `"MON"`, `"JAN"`) ### Standalone Validators - -- `verifyIpAddress` β€” checks IPv4/IPv6 format -- `verifyEmptyStringOrIpAddress` β€” allows empty string or valid IP -- `verifyMacAddress` β€” validates MAC address format -- `verifyUidGid` β€” validates UNIX UID/GID values -- `is_valid_hex` β€” checks hexadecimal string format - -### Type Flag Enum - -Bitmask flags for column-level validation rules: - -```text -NormalType Β· IntType Β· NonEmpty Β· NonNull Β· NonZero -FileOnDisk Β· DirectoryOnDisk Β· ValidUUID Β· MD5 Β· SHA256 -SHA1 Β· Bool Β· EmptyOk Β· NullOk -Composites: NonNegativeInt Β· NonNegativeOrErrorInt Β· NonEmptyString Β· IntOrEmpty -``` - -### Core Test Helpers - -- `execute_query(query)` β€” runs a raw SQL query against the osquery SQLite engine -- `validate_row(row, map)` β€” validates a single result row against a `ValidationMap` -- `validate_rows(rows, map)` β€” validates all rows in a result set -- `validate_value_using_flags(value, flags)` β€” applies bitmask flag rules to a single value -- `validate_container_rows(table, map, constraints)` β€” validates a full table query with optional SQL constraints -- `setUpEnvironment()` β€” initializes the test environment before test execution - -### Type Aliases - -```cpp -using CustomCheckerType = std::function; -using ValidationDataType = boost::variant; -using ValidationMap = std::unordered_map; -``` +- **`verifyIpAddress`** / **`verifyEmptyStringOrIpAddress`** β€” Validate IPv4/IPv6 address strings +- **`verifyMacAddress`** β€” Validates MAC address format +- **`verifyUidGid`** β€” Validates UID/GID values +- **`is_valid_hex`** β€” Checks if a string is valid hexadecimal + +### Validation Flags (Bitmask Enum) +Composable flags for column-level validation: + +| Flag | Meaning | +|------|---------| +| `IntType` | Value must be an integer | +| `NonEmpty` / `NonNull` | Value must not be empty/null | +| `FileOnDisk` / `DirectoryOnDisk` | Path must exist | +| `ValidUUID`, `MD5`, `SHA256`, `SHA1` | Format validators | +| `NonNegativeInt` | Shorthand: integer, non-empty, non-null | + +### Core Types +- **`ValidationMap`** β€” `unordered_map>` mapping column names to validation rules +- **`CustomCheckerType`** β€” `std::function` for arbitrary validators + +### Test Helpers +- **`execute_query`** β€” Runs a SQL query against the osquery SQLite engine +- **`validate_row`** / **`validate_rows`** β€” Assert all columns in one or more rows pass their `ValidationMap` rules +- **`validate_container_rows`** β€” Validates table output with optional SQL constraints +- **`setUpEnvironment`** β€” Initializes the test environment ## Usage Example @@ -54,14 +42,16 @@ using ValidationMap = std::unordered_map; using namespace osquery::table_tests; -TEST_F(MyTableTest, ValidateColumns) { +TEST_F(MyTableTest, testMyTable) { + setUpEnvironment(); + ValidationMap row_map = { {"pid", NonNegativeInt}, {"name", NonEmptyString}, - {"path", FileOnDisk}, - {"checksum", SHA256}, - {"status", SpecificValuesCheck({"running", "stopped", "idle"})}, - {"port", IntMinMaxCheck(0, 65535)}, + {"state", SpecificValuesCheck({"R", "S", "D", "Z"})}, + {"uid", verifyUidGid}, // CustomCheckerType + {"address", verifyIpAddress}, + {"checksum", IntMinMaxCheck(0, 255)}, }; auto rows = execute_query("SELECT * FROM my_table"); From 1a5151514848d967be7f0545c34826ed5fed8459 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Jun 2026 19:03:58 +0000 Subject: [PATCH 4/7] chore(docs): Remove 1 orphaned inline files [skip ci] --- .doc-pipeline-status.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .doc-pipeline-status.md diff --git a/.doc-pipeline-status.md b/.doc-pipeline-status.md deleted file mode 100644 index 321a7aedce1..00000000000 --- a/.doc-pipeline-status.md +++ /dev/null @@ -1,5 +0,0 @@ -# Documentation Pipeline Started - -Run ID: doc-orchestrator-2026-06-04T17-38-04.379Z -Status: In Progress -Started: 2026-06-04 17:38:28 UTC From c7b63ef2e8bad19ab1cf7f81689fb244b8d02241 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Jun 2026 19:13:17 +0000 Subject: [PATCH 5/7] docs: Stage 2 - Architecture analysis (16 files) [skip ci] --- docs/diagrams/architecture/.gitignore | 8 + docs/diagrams/architecture/README-10.mmd | 4 + docs/diagrams/architecture/README-11.mmd | 12 + docs/diagrams/architecture/README-2.mmd | 10 + docs/diagrams/architecture/README-3.mmd | 7 + docs/diagrams/architecture/README-4.mmd | 8 + docs/diagrams/architecture/README-5.mmd | 5 + docs/diagrams/architecture/README-6.mmd | 5 + docs/diagrams/architecture/README-7.mmd | 5 + docs/diagrams/architecture/README-8.mmd | 4 + docs/diagrams/architecture/README-9.mmd | 7 + docs/diagrams/architecture/README.mmd | 18 + .../architecture/config-and-packs-2.mmd | 8 + .../architecture/config-and-packs-3.mmd | 3 + .../architecture/config-and-packs-4.mmd | 5 + .../architecture/config-and-packs-5.mmd | 4 + .../architecture/config-and-packs-6.mmd | 4 + .../architecture/config-and-packs-7.mmd | 5 + .../architecture/config-and-packs-8.mmd | 6 + .../architecture/config-and-packs-9.mmd | 5 + .../architecture/config-and-packs.mmd | 8 + .../architecture/config-plugins-2.mmd | 6 + .../architecture/config-plugins-3.mmd | 11 + .../architecture/config-plugins-4.mmd | 9 + .../architecture/config-plugins-5.mmd | 5 + .../architecture/config-plugins-6.mmd | 7 + .../architecture/config-plugins-7.mmd | 8 + .../architecture/config-plugins-8.mmd | 10 + docs/diagrams/architecture/config-plugins.mmd | 17 + .../architecture/core-init-and-runtime-2.mmd | 6 + .../architecture/core-init-and-runtime-3.mmd | 6 + .../architecture/core-init-and-runtime-4.mmd | 5 + .../architecture/core-init-and-runtime-5.mmd | 6 + .../architecture/core-init-and-runtime-6.mmd | 10 + .../architecture/core-init-and-runtime-7.mmd | 4 + .../architecture/core-init-and-runtime.mmd | 15 + .../architecture/database-backends-2.mmd | 12 + .../architecture/database-backends-3.mmd | 6 + .../architecture/database-backends.mmd | 11 + .../architecture/distributed-querying-2.mmd | 10 + .../architecture/distributed-querying-3.mmd | 6 + .../architecture/distributed-querying-4.mmd | 13 + .../architecture/distributed-querying.mmd | 10 + docs/diagrams/architecture/events-core-2.mmd | 6 + docs/diagrams/architecture/events-core-3.mmd | 12 + docs/diagrams/architecture/events-core-4.mmd | 4 + docs/diagrams/architecture/events-core-5.mmd | 7 + docs/diagrams/architecture/events-core-6.mmd | 7 + docs/diagrams/architecture/events-core.mmd | 26 ++ .../architecture/extensions-framework-2.mmd | 4 + .../architecture/extensions-framework-3.mmd | 10 + .../architecture/extensions-framework-4.mmd | 5 + .../architecture/extensions-framework-5.mmd | 5 + .../architecture/extensions-framework-6.mmd | 4 + .../architecture/extensions-framework-7.mmd | 5 + .../architecture/extensions-framework-8.mmd | 9 + .../architecture/extensions-framework.mmd | 12 + .../architecture/filesystem-and-fileops-2.mmd | 6 + .../architecture/filesystem-and-fileops-3.mmd | 9 + .../architecture/filesystem-and-fileops-4.mmd | 5 + .../architecture/filesystem-and-fileops-5.mmd | 9 + .../architecture/filesystem-and-fileops-6.mmd | 4 + .../architecture/filesystem-and-fileops.mmd | 15 + docs/diagrams/architecture/hashing-2.mmd | 4 + docs/diagrams/architecture/hashing-3.mmd | 8 + docs/diagrams/architecture/hashing-4.mmd | 17 + docs/diagrams/architecture/hashing-5.mmd | 3 + docs/diagrams/architecture/hashing-6.mmd | 5 + docs/diagrams/architecture/hashing-7.mmd | 5 + docs/diagrams/architecture/hashing-8.mmd | 11 + docs/diagrams/architecture/hashing.mmd | 6 + .../plugin-interfaces-and-logging-2.mmd | 10 + .../plugin-interfaces-and-logging-3.mmd | 3 + .../plugin-interfaces-and-logging-4.mmd | 8 + .../plugin-interfaces-and-logging-5.mmd | 4 + .../plugin-interfaces-and-logging-6.mmd | 4 + .../plugin-interfaces-and-logging-7.mmd | 7 + .../plugin-interfaces-and-logging.mmd | 7 + .../architecture/process-and-profiler-2.mmd | 7 + .../architecture/process-and-profiler-3.mmd | 8 + .../architecture/process-and-profiler.mmd | 22 + .../query-execution-and-logging-10.mmd | 5 + .../query-execution-and-logging-11.mmd | 10 + .../query-execution-and-logging-2.mmd | 12 + .../query-execution-and-logging-3.mmd | 12 + .../query-execution-and-logging-4.mmd | 9 + .../query-execution-and-logging-5.mmd | 5 + .../query-execution-and-logging-6.mmd | 7 + .../query-execution-and-logging-7.mmd | 4 + .../query-execution-and-logging-8.mmd | 5 + .../query-execution-and-logging-9.mmd | 5 + .../query-execution-and-logging.mmd | 9 + .../architecture/remote-http-client-2.mmd | 7 + .../architecture/remote-http-client-3.mmd | 11 + .../architecture/remote-http-client-4.mmd | 8 + .../architecture/remote-http-client-5.mmd | 8 + .../architecture/remote-http-client-6.mmd | 5 + .../architecture/remote-http-client.mmd | 7 + .../sql-core-and-virtual-tables-2.mmd | 10 + .../sql-core-and-virtual-tables-3.mmd | 5 + .../sql-core-and-virtual-tables-4.mmd | 5 + .../sql-core-and-virtual-tables-5.mmd | 10 + .../sql-core-and-virtual-tables.mmd | 12 + .../architecture/system-utilities-2.mmd | 11 + .../architecture/system-utilities.mmd | 22 + docs/reference/architecture/.gitignore | 8 + docs/reference/architecture/README.md | 415 ++++++++++++++++++ .../config-and-packs/config-and-packs.md | 389 ++++++++++++++++ .../config-plugins/config-plugins.md | 369 ++++++++++++++++ .../core-init-and-runtime.md | 380 ++++++++++++++++ .../database-backends/database-backends.md | 225 ++++++++++ .../distributed-querying.md | 369 ++++++++++++++++ .../architecture/events-core/events-core.md | 307 +++++++++++++ .../extensions-framework.md | 373 ++++++++++++++++ .../filesystem-and-fileops.md | 298 +++++++++++++ .../reference/architecture/hashing/hashing.md | 344 +++++++++++++++ .../plugin-interfaces-and-logging.md | 316 +++++++++++++ .../process-and-profiler.md | 283 ++++++++++++ .../query-execution-and-logging.md | 348 +++++++++++++++ .../remote-http-client/remote-http-client.md | 328 ++++++++++++++ .../sql-core-and-virtual-tables.md | 369 ++++++++++++++++ .../system-utilities/system-utilities.md | 298 +++++++++++++ 122 files changed, 6260 insertions(+) create mode 100644 docs/diagrams/architecture/.gitignore create mode 100644 docs/diagrams/architecture/README-10.mmd create mode 100644 docs/diagrams/architecture/README-11.mmd create mode 100644 docs/diagrams/architecture/README-2.mmd create mode 100644 docs/diagrams/architecture/README-3.mmd create mode 100644 docs/diagrams/architecture/README-4.mmd create mode 100644 docs/diagrams/architecture/README-5.mmd create mode 100644 docs/diagrams/architecture/README-6.mmd create mode 100644 docs/diagrams/architecture/README-7.mmd create mode 100644 docs/diagrams/architecture/README-8.mmd create mode 100644 docs/diagrams/architecture/README-9.mmd create mode 100644 docs/diagrams/architecture/README.mmd create mode 100644 docs/diagrams/architecture/config-and-packs-2.mmd create mode 100644 docs/diagrams/architecture/config-and-packs-3.mmd create mode 100644 docs/diagrams/architecture/config-and-packs-4.mmd create mode 100644 docs/diagrams/architecture/config-and-packs-5.mmd create mode 100644 docs/diagrams/architecture/config-and-packs-6.mmd create mode 100644 docs/diagrams/architecture/config-and-packs-7.mmd create mode 100644 docs/diagrams/architecture/config-and-packs-8.mmd create mode 100644 docs/diagrams/architecture/config-and-packs-9.mmd create mode 100644 docs/diagrams/architecture/config-and-packs.mmd create mode 100644 docs/diagrams/architecture/config-plugins-2.mmd create mode 100644 docs/diagrams/architecture/config-plugins-3.mmd create mode 100644 docs/diagrams/architecture/config-plugins-4.mmd create mode 100644 docs/diagrams/architecture/config-plugins-5.mmd create mode 100644 docs/diagrams/architecture/config-plugins-6.mmd create mode 100644 docs/diagrams/architecture/config-plugins-7.mmd create mode 100644 docs/diagrams/architecture/config-plugins-8.mmd create mode 100644 docs/diagrams/architecture/config-plugins.mmd create mode 100644 docs/diagrams/architecture/core-init-and-runtime-2.mmd create mode 100644 docs/diagrams/architecture/core-init-and-runtime-3.mmd create mode 100644 docs/diagrams/architecture/core-init-and-runtime-4.mmd create mode 100644 docs/diagrams/architecture/core-init-and-runtime-5.mmd create mode 100644 docs/diagrams/architecture/core-init-and-runtime-6.mmd create mode 100644 docs/diagrams/architecture/core-init-and-runtime-7.mmd create mode 100644 docs/diagrams/architecture/core-init-and-runtime.mmd create mode 100644 docs/diagrams/architecture/database-backends-2.mmd create mode 100644 docs/diagrams/architecture/database-backends-3.mmd create mode 100644 docs/diagrams/architecture/database-backends.mmd create mode 100644 docs/diagrams/architecture/distributed-querying-2.mmd create mode 100644 docs/diagrams/architecture/distributed-querying-3.mmd create mode 100644 docs/diagrams/architecture/distributed-querying-4.mmd create mode 100644 docs/diagrams/architecture/distributed-querying.mmd create mode 100644 docs/diagrams/architecture/events-core-2.mmd create mode 100644 docs/diagrams/architecture/events-core-3.mmd create mode 100644 docs/diagrams/architecture/events-core-4.mmd create mode 100644 docs/diagrams/architecture/events-core-5.mmd create mode 100644 docs/diagrams/architecture/events-core-6.mmd create mode 100644 docs/diagrams/architecture/events-core.mmd create mode 100644 docs/diagrams/architecture/extensions-framework-2.mmd create mode 100644 docs/diagrams/architecture/extensions-framework-3.mmd create mode 100644 docs/diagrams/architecture/extensions-framework-4.mmd create mode 100644 docs/diagrams/architecture/extensions-framework-5.mmd create mode 100644 docs/diagrams/architecture/extensions-framework-6.mmd create mode 100644 docs/diagrams/architecture/extensions-framework-7.mmd create mode 100644 docs/diagrams/architecture/extensions-framework-8.mmd create mode 100644 docs/diagrams/architecture/extensions-framework.mmd create mode 100644 docs/diagrams/architecture/filesystem-and-fileops-2.mmd create mode 100644 docs/diagrams/architecture/filesystem-and-fileops-3.mmd create mode 100644 docs/diagrams/architecture/filesystem-and-fileops-4.mmd create mode 100644 docs/diagrams/architecture/filesystem-and-fileops-5.mmd create mode 100644 docs/diagrams/architecture/filesystem-and-fileops-6.mmd create mode 100644 docs/diagrams/architecture/filesystem-and-fileops.mmd create mode 100644 docs/diagrams/architecture/hashing-2.mmd create mode 100644 docs/diagrams/architecture/hashing-3.mmd create mode 100644 docs/diagrams/architecture/hashing-4.mmd create mode 100644 docs/diagrams/architecture/hashing-5.mmd create mode 100644 docs/diagrams/architecture/hashing-6.mmd create mode 100644 docs/diagrams/architecture/hashing-7.mmd create mode 100644 docs/diagrams/architecture/hashing-8.mmd create mode 100644 docs/diagrams/architecture/hashing.mmd create mode 100644 docs/diagrams/architecture/plugin-interfaces-and-logging-2.mmd create mode 100644 docs/diagrams/architecture/plugin-interfaces-and-logging-3.mmd create mode 100644 docs/diagrams/architecture/plugin-interfaces-and-logging-4.mmd create mode 100644 docs/diagrams/architecture/plugin-interfaces-and-logging-5.mmd create mode 100644 docs/diagrams/architecture/plugin-interfaces-and-logging-6.mmd create mode 100644 docs/diagrams/architecture/plugin-interfaces-and-logging-7.mmd create mode 100644 docs/diagrams/architecture/plugin-interfaces-and-logging.mmd create mode 100644 docs/diagrams/architecture/process-and-profiler-2.mmd create mode 100644 docs/diagrams/architecture/process-and-profiler-3.mmd create mode 100644 docs/diagrams/architecture/process-and-profiler.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-10.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-11.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-2.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-3.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-4.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-5.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-6.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-7.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-8.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging-9.mmd create mode 100644 docs/diagrams/architecture/query-execution-and-logging.mmd create mode 100644 docs/diagrams/architecture/remote-http-client-2.mmd create mode 100644 docs/diagrams/architecture/remote-http-client-3.mmd create mode 100644 docs/diagrams/architecture/remote-http-client-4.mmd create mode 100644 docs/diagrams/architecture/remote-http-client-5.mmd create mode 100644 docs/diagrams/architecture/remote-http-client-6.mmd create mode 100644 docs/diagrams/architecture/remote-http-client.mmd create mode 100644 docs/diagrams/architecture/sql-core-and-virtual-tables-2.mmd create mode 100644 docs/diagrams/architecture/sql-core-and-virtual-tables-3.mmd create mode 100644 docs/diagrams/architecture/sql-core-and-virtual-tables-4.mmd create mode 100644 docs/diagrams/architecture/sql-core-and-virtual-tables-5.mmd create mode 100644 docs/diagrams/architecture/sql-core-and-virtual-tables.mmd create mode 100644 docs/diagrams/architecture/system-utilities-2.mmd create mode 100644 docs/diagrams/architecture/system-utilities.mmd create mode 100644 docs/reference/architecture/.gitignore create mode 100644 docs/reference/architecture/README.md create mode 100644 docs/reference/architecture/config-and-packs/config-and-packs.md create mode 100644 docs/reference/architecture/config-plugins/config-plugins.md create mode 100644 docs/reference/architecture/core-init-and-runtime/core-init-and-runtime.md create mode 100644 docs/reference/architecture/database-backends/database-backends.md create mode 100644 docs/reference/architecture/distributed-querying/distributed-querying.md create mode 100644 docs/reference/architecture/events-core/events-core.md create mode 100644 docs/reference/architecture/extensions-framework/extensions-framework.md create mode 100644 docs/reference/architecture/filesystem-and-fileops/filesystem-and-fileops.md create mode 100644 docs/reference/architecture/hashing/hashing.md create mode 100644 docs/reference/architecture/plugin-interfaces-and-logging/plugin-interfaces-and-logging.md create mode 100644 docs/reference/architecture/process-and-profiler/process-and-profiler.md create mode 100644 docs/reference/architecture/query-execution-and-logging/query-execution-and-logging.md create mode 100644 docs/reference/architecture/remote-http-client/remote-http-client.md create mode 100644 docs/reference/architecture/sql-core-and-virtual-tables/sql-core-and-virtual-tables.md create mode 100644 docs/reference/architecture/system-utilities/system-utilities.md diff --git a/docs/diagrams/architecture/.gitignore b/docs/diagrams/architecture/.gitignore new file mode 100644 index 00000000000..b4f4c7a6c9c --- /dev/null +++ b/docs/diagrams/architecture/.gitignore @@ -0,0 +1,8 @@ +# CodeWiki temp files (dependency graphs can be 7GB+) +temp/ +dependency_graphs/ + +# JSON intermediate files (except schema/config) +*.json +!*-schema.json +!*-config.json diff --git a/docs/diagrams/architecture/README-10.mmd b/docs/diagrams/architecture/README-10.mmd new file mode 100644 index 00000000000..3c18bf24f21 --- /dev/null +++ b/docs/diagrams/architecture/README-10.mmd @@ -0,0 +1,4 @@ +flowchart LR + Core["Extension Manager"] --> ExtensionA["Extension Process"] + ExtensionA --> Registry["Registry Routes"] + Core --> ExtensionA diff --git a/docs/diagrams/architecture/README-11.mmd b/docs/diagrams/architecture/README-11.mmd new file mode 100644 index 00000000000..49c45ea7085 --- /dev/null +++ b/docs/diagrams/architecture/README-11.mmd @@ -0,0 +1,12 @@ +flowchart TD + Config["Configuration"] --> Scheduler["Scheduler"] + Scheduler --> SQL["SQL Engine"] + SQL --> Diff["Diff Engine"] + Diff --> Persist["Database"] + Diff --> Log["Logger"] + Events["Event Publishers"] --> Subscribers["Event Subscribers"] + Subscribers --> Persist + Distributed["Remote Queries"] --> SQL + SQL --> Distributed + Extensions["Extensions"] --> Registry["Plugin Registry"] + Registry --> SQL diff --git a/docs/diagrams/architecture/README-2.mmd b/docs/diagrams/architecture/README-2.mmd new file mode 100644 index 00000000000..f46501680e9 --- /dev/null +++ b/docs/diagrams/architecture/README-2.mmd @@ -0,0 +1,10 @@ +flowchart TD + Start["Main Entry"] --> Flags["Parse Flags"] + Flags --> Mode["Determine Mode"] + Mode --> RegistryInit["Initialize Registry"] + RegistryInit --> DBInit["Initialize Database"] + DBInit --> ConfigLoad["Load Configuration"] + ConfigLoad --> Plugins["Activate Plugins"] + Plugins --> EventsAttach["Attach Events"] + EventsAttach --> Dispatcher["Run Dispatcher"] + Dispatcher --> Shutdown["Graceful Shutdown"] diff --git a/docs/diagrams/architecture/README-3.mmd b/docs/diagrams/architecture/README-3.mmd new file mode 100644 index 00000000000..872384bbddb --- /dev/null +++ b/docs/diagrams/architecture/README-3.mmd @@ -0,0 +1,7 @@ +flowchart TD + Query["Incoming SQL"] --> SQLite["SQLite Engine"] + SQLite --> VTable["Virtual Table Adapter"] + VTable --> TablePlugin["TablePlugin::generate"] + TablePlugin --> Rows["Row Results"] + Rows --> Diff["DiffResults"] + Diff --> Output["QueryDataTyped"] diff --git a/docs/diagrams/architecture/README-4.mmd b/docs/diagrams/architecture/README-4.mmd new file mode 100644 index 00000000000..157fe7bfae4 --- /dev/null +++ b/docs/diagrams/architecture/README-4.mmd @@ -0,0 +1,8 @@ +flowchart TD + New["New Results"] --> DiffEngine["Diff Engine"] + Old["Previous Results"] --> DiffEngine + DiffEngine --> Added["Added Rows"] + DiffEngine --> Removed["Removed Rows"] + Added --> LogItem["QueryLogItem"] + Removed --> LogItem + LogItem --> Logger["Logger Plugins"] diff --git a/docs/diagrams/architecture/README-5.mmd b/docs/diagrams/architecture/README-5.mmd new file mode 100644 index 00000000000..aed1c9e02e2 --- /dev/null +++ b/docs/diagrams/architecture/README-5.mmd @@ -0,0 +1,5 @@ +flowchart TD + Plugin["ConfigPlugin"] --> Validate["Validate JSON"] + Validate --> Packs["Build Packs"] + Packs --> Schedule["Update Schedule"] + Schedule --> SQL["Scheduled Execution"] diff --git a/docs/diagrams/architecture/README-6.mmd b/docs/diagrams/architecture/README-6.mmd new file mode 100644 index 00000000000..7b027d7be92 --- /dev/null +++ b/docs/diagrams/architecture/README-6.mmd @@ -0,0 +1,5 @@ +flowchart LR + Core["Core Runtime"] --> LoggerInterface["LoggerPlugin"] + SQL["SQL Engine"] --> LoggerInterface + Events["Events"] --> LoggerInterface + LoggerInterface --> Backend["External Sink"] diff --git a/docs/diagrams/architecture/README-7.mmd b/docs/diagrams/architecture/README-7.mmd new file mode 100644 index 00000000000..f6820a0f4d0 --- /dev/null +++ b/docs/diagrams/architecture/README-7.mmd @@ -0,0 +1,5 @@ +flowchart TD + Core --> API["OsqueryDatabase"] + API --> Plugin["Active DB Plugin"] + Plugin --> RocksDB["Persistent Storage"] + Plugin --> Memory["In-Memory Backend"] diff --git a/docs/diagrams/architecture/README-8.mmd b/docs/diagrams/architecture/README-8.mmd new file mode 100644 index 00000000000..b39ab8b7609 --- /dev/null +++ b/docs/diagrams/architecture/README-8.mmd @@ -0,0 +1,4 @@ +flowchart TD + Publisher --> Subscriber + Subscriber --> Storage["Event Backing Store"] + Storage --> SQL["Virtual Table"] diff --git a/docs/diagrams/architecture/README-9.mmd b/docs/diagrams/architecture/README-9.mmd new file mode 100644 index 00000000000..2028ca7423e --- /dev/null +++ b/docs/diagrams/architecture/README-9.mmd @@ -0,0 +1,7 @@ +flowchart TD + Server["Control Plane"] --> TLS["TLS Plugin"] + TLS --> Engine["Distributed Engine"] + Engine --> SQL["Execute SQL"] + SQL --> Engine + Engine --> TLS + TLS --> Server diff --git a/docs/diagrams/architecture/README.mmd b/docs/diagrams/architecture/README.mmd new file mode 100644 index 00000000000..c7c778414dd --- /dev/null +++ b/docs/diagrams/architecture/README.mmd @@ -0,0 +1,18 @@ +flowchart TD + CLI["osqueryi or osqueryd"] --> Core["Core Init And Runtime"] + Core --> Config["Config And Packs"] + Core --> Registry["Plugin Registry"] + Core --> Database["Database Backends"] + Core --> SQL["SQL Core And Virtual Tables"] + Core --> Events["Events Core"] + Core --> Distributed["Distributed Querying"] + Core --> Logger["Plugin Interfaces And Logging"] + Core --> Extensions["Extensions Framework"] + Core --> HTTP["Remote Http Client"] + + Config --> SQL + SQL --> QueryExec["Query Execution And Logging"] + QueryExec --> Logger + Events --> SQL + Distributed --> SQL + Distributed --> HTTP diff --git a/docs/diagrams/architecture/config-and-packs-2.mmd b/docs/diagrams/architecture/config-and-packs-2.mmd new file mode 100644 index 00000000000..d222580ae3b --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs-2.mmd @@ -0,0 +1,8 @@ +flowchart TD + Start["Config Load Triggered"] --> Gen["ConfigPlugin.genConfig()"] + Gen --> Validate["Validate JSON Structure"] + Validate --> Update["Update Sources"] + Update --> Packs["Create / Update Packs"] + Packs --> Parsers["Apply Config Parsers"] + Parsers --> Reconfigure["Reconfigure Registries"] + Reconfigure --> Ready["Schedule Active"] diff --git a/docs/diagrams/architecture/config-and-packs-3.mmd b/docs/diagrams/architecture/config-and-packs-3.mmd new file mode 100644 index 00000000000..accda125c6c --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs-3.mmd @@ -0,0 +1,3 @@ +flowchart TD + LoopStart["Wait refresh_sec seconds"] --> RefreshCall["Config.refresh()"] + RefreshCall --> LoopStart diff --git a/docs/diagrams/architecture/config-and-packs-4.mmd b/docs/diagrams/architecture/config-and-packs-4.mmd new file mode 100644 index 00000000000..1d534aefe3a --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs-4.mmd @@ -0,0 +1,5 @@ +flowchart TD + Pack["Pack"] --> Discovery["Discovery Queries"] + Pack --> Schedule["Scheduled Queries"] + Pack --> Constraints["Platform / Version / Shard"] + Pack --> Stats["PackStats"] diff --git a/docs/diagrams/architecture/config-and-packs-5.mmd b/docs/diagrams/architecture/config-and-packs-5.mmd new file mode 100644 index 00000000000..accfb8c7208 --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs-5.mmd @@ -0,0 +1,4 @@ +flowchart TD + Schedule["Schedule"] --> Packs["Pack List"] + Schedule --> Denylist["Denylisted Queries"] + Schedule --> Executing["Executing Query State"] diff --git a/docs/diagrams/architecture/config-and-packs-6.mmd b/docs/diagrams/architecture/config-and-packs-6.mmd new file mode 100644 index 00000000000..687bcf190a6 --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs-6.mmd @@ -0,0 +1,4 @@ +flowchart TD + Step["Schedule::Step"] --> Check["pack.shouldPackExecute()"] + Check -->|"true"| Include["Included in Iteration"] + Check -->|"false"| Skip["Skipped"] diff --git a/docs/diagrams/architecture/config-and-packs-7.mmd b/docs/diagrams/architecture/config-and-packs-7.mmd new file mode 100644 index 00000000000..c700d9bd6f2 --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs-7.mmd @@ -0,0 +1,5 @@ +flowchart TD + ConfigCore["Config And Packs"] --> Options["Options Parser"] + ConfigCore --> FilePaths["File Paths Parser"] + ConfigCore --> EventsParser["Events Parser"] + ConfigCore --> OtherParsers["Other Parsers"] diff --git a/docs/diagrams/architecture/config-and-packs-8.mmd b/docs/diagrams/architecture/config-and-packs-8.mmd new file mode 100644 index 00000000000..2cc2c2a52c9 --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs-8.mmd @@ -0,0 +1,6 @@ +flowchart TD + ConfigCore["Config And Packs"] --> DB["Persistent Database"] + DB --> Performance["Query Performance"] + DB --> Denylist["Failed Queries"] + DB --> Executing["Executing Query"] + DB --> Backup["Config Backup"] diff --git a/docs/diagrams/architecture/config-and-packs-9.mmd b/docs/diagrams/architecture/config-and-packs-9.mmd new file mode 100644 index 00000000000..53aafa648fc --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs-9.mmd @@ -0,0 +1,5 @@ +flowchart TD + ConfigCore["Config And Packs"] --> Mutex1["Schedule Mutex"] + ConfigCore --> Mutex2["Files Mutex"] + ConfigCore --> Mutex3["Performance Mutex"] + ConfigCore --> Mutex4["Hash Mutex"] diff --git a/docs/diagrams/architecture/config-and-packs.mmd b/docs/diagrams/architecture/config-and-packs.mmd new file mode 100644 index 00000000000..cfa91181de1 --- /dev/null +++ b/docs/diagrams/architecture/config-and-packs.mmd @@ -0,0 +1,8 @@ +flowchart TD + ConfigPlugin["Config Plugin"] --> ConfigCore["Config And Packs"] + ConfigCore --> Schedule["Schedule & Packs"] + ConfigCore --> Parsers["Config Parser Plugins"] + Schedule --> SQLCore["SQL Core & Scheduled Queries"] + SQLCore --> Logging["Query Execution & Logging"] + ConfigCore --> Database["Database Backends"] + ConfigCore --> Events["Events Core"] diff --git a/docs/diagrams/architecture/config-plugins-2.mmd b/docs/diagrams/architecture/config-plugins-2.mmd new file mode 100644 index 00000000000..5594af3c4a2 --- /dev/null +++ b/docs/diagrams/architecture/config-plugins-2.mmd @@ -0,0 +1,6 @@ +flowchart TD + Start["genConfig()"] --> Validate["Validate config_path"] + Validate --> Resolve["Resolve .d drop-in files"] + Resolve --> Sort["Sort files"] + Sort --> Read["Read each file"] + Read --> Merge["Return config map"] diff --git a/docs/diagrams/architecture/config-plugins-3.mmd b/docs/diagrams/architecture/config-plugins-3.mmd new file mode 100644 index 00000000000..8abd01dece7 --- /dev/null +++ b/docs/diagrams/architecture/config-plugins-3.mmd @@ -0,0 +1,11 @@ +flowchart TD + ConfigLoad["Config Update"] --> ParseDecorators["Parse decorators key"] + ParseDecorators --> StoreQueries["Store queries by type"] + StoreQueries --> RunLoad["Run load decorators"] + + ScheduledTick["Scheduler Tick"] --> RunInterval["Run interval decorators"] + QueryExecution["Before Query"] --> RunAlways["Run always decorators"] + + RunLoad --> DecorationStore["Global Decoration Store"] + RunAlways --> DecorationStore + RunInterval --> DecorationStore diff --git a/docs/diagrams/architecture/config-plugins-4.mmd b/docs/diagrams/architecture/config-plugins-4.mmd new file mode 100644 index 00000000000..22321319128 --- /dev/null +++ b/docs/diagrams/architecture/config-plugins-4.mmd @@ -0,0 +1,9 @@ +flowchart TD + ConfigUpdate["Config Update"] --> RemoveOld["Remove old source paths"] + RemoveOld --> ParseStatic["Parse file_paths"] + RemoveOld --> ParseQuery["Parse file_paths_query"] + ParseQuery --> ExecuteSQL["Execute SQL query"] + ExecuteSQL --> ExtractPath["Extract path column"] + ParseStatic --> RegisterFile["Config::addFile()"] + ExtractPath --> RegisterFile + RegisterFile --> EventsFramework["Events Framework"] diff --git a/docs/diagrams/architecture/config-plugins-5.mmd b/docs/diagrams/architecture/config-plugins-5.mmd new file mode 100644 index 00000000000..547715946a1 --- /dev/null +++ b/docs/diagrams/architecture/config-plugins-5.mmd @@ -0,0 +1,5 @@ +flowchart TD + ConfigOptions["options block"] --> ValidateFlag["Validate flag name"] + ValidateFlag --> CheckCLI["Check CLI-only restriction"] + CheckCLI --> UpdateFlag["Flag::updateValue"] + UpdateFlag --> AdjustLogging["setVerboseLevel() if needed"] diff --git a/docs/diagrams/architecture/config-plugins-6.mmd b/docs/diagrams/architecture/config-plugins-6.mmd new file mode 100644 index 00000000000..cc0ef60685a --- /dev/null +++ b/docs/diagrams/architecture/config-plugins-6.mmd @@ -0,0 +1,7 @@ +flowchart TD + ConfigViews["views block"] --> ScanDB["Scan existing views"] + ScanDB --> Compare["Compare with new config"] + Compare --> CreateView["CREATE VIEW"] + Compare --> DropView["DROP VIEW"] + CreateView --> Persist["Store in database"] + DropView --> Persist diff --git a/docs/diagrams/architecture/config-plugins-7.mmd b/docs/diagrams/architecture/config-plugins-7.mmd new file mode 100644 index 00000000000..4ef8ed452c0 --- /dev/null +++ b/docs/diagrams/architecture/config-plugins-7.mmd @@ -0,0 +1,8 @@ +sequenceDiagram + participant Extension + participant Registry + participant CoreConfig + + Extension->>Registry: Config::update() + Registry->>CoreConfig: Route update request + CoreConfig->>CoreConfig: Re-run parser pipeline diff --git a/docs/diagrams/architecture/config-plugins-8.mmd b/docs/diagrams/architecture/config-plugins-8.mmd new file mode 100644 index 00000000000..54945d774eb --- /dev/null +++ b/docs/diagrams/architecture/config-plugins-8.mmd @@ -0,0 +1,10 @@ +flowchart TD + Source["Filesystem or Extension"] --> Retrieve["ConfigPlugin::genConfig"] + Retrieve --> Core["Config Core"] + Core --> Parse["ConfigParserPlugin::update"] + + Parse --> Flags["Runtime Flags"] + Parse --> Views["SQL Views"] + Parse --> Decorators["Log Decorations"] + Parse --> FilePaths["Filesystem Monitoring"] + Parse --> Events["Event Subscriptions"] diff --git a/docs/diagrams/architecture/config-plugins.mmd b/docs/diagrams/architecture/config-plugins.mmd new file mode 100644 index 00000000000..af9d0ebc535 --- /dev/null +++ b/docs/diagrams/architecture/config-plugins.mmd @@ -0,0 +1,17 @@ +flowchart TD + ConfigSource["Configuration Source"] --> FilesystemPlugin["FilesystemConfigPlugin"] + FilesystemPlugin --> CoreConfig["Config Core"] + + CoreConfig --> DecoratorsParser["DecoratorsConfigParserPlugin"] + CoreConfig --> EventsParser["EventsConfigParserPlugin"] + CoreConfig --> FilePathsParser["FilePathsConfigParserPlugin"] + CoreConfig --> OptionsParser["OptionsConfigParserPlugin"] + CoreConfig --> ViewsParser["ViewsConfigParserPlugin"] + + DecoratorsParser --> LoggerSubsystem["Logger Subsystem"] + FilePathsParser --> EventsFramework["Events Framework"] + ViewsParser --> SQLCore["SQL Core"] + OptionsParser --> FlagSystem["Flag System"] + + ExtensionFramework["Extensions Framework"] --> UpdatePlugin["UpdateConfigPlugin"] + UpdatePlugin --> CoreConfig diff --git a/docs/diagrams/architecture/core-init-and-runtime-2.mmd b/docs/diagrams/architecture/core-init-and-runtime-2.mmd new file mode 100644 index 00000000000..091d2a0d0e9 --- /dev/null +++ b/docs/diagrams/architecture/core-init-and-runtime-2.mmd @@ -0,0 +1,6 @@ +flowchart LR + Macro["FLAG Macro"] --> GFlags["DEFINE_type"] + Macro --> Create["Flag::create()"] + Create --> Registry["Internal Flag Map"] + Registry --> Help["Custom Help Output"] + Registry --> RuntimeAccess["Flag::getValue()"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-3.mmd b/docs/diagrams/architecture/core-init-and-runtime-3.mmd new file mode 100644 index 00000000000..9391299f6b5 --- /dev/null +++ b/docs/diagrams/architecture/core-init-and-runtime-3.mmd @@ -0,0 +1,6 @@ +flowchart TD + Start["Initializer Constructor"] --> CheckBinary["Inspect argv[0]"] + CheckBinary -->|"osqueryd"| Daemon["Daemon Mode"] + CheckBinary -->|"osqueryi"| Shell["Shell Mode"] + Shell --> DisableEvents["Disable Events By Default"] + Daemon --> InitWorkDir["Init Work Directories"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-4.mmd b/docs/diagrams/architecture/core-init-and-runtime-4.mmd new file mode 100644 index 00000000000..f005411e95b --- /dev/null +++ b/docs/diagrams/architecture/core-init-and-runtime-4.mmd @@ -0,0 +1,5 @@ +flowchart TD + FlagCheck["openframe_mode Flag"] -->|"true"| Encryption["Encryption Service"] + Encryption --> Extractor["Token Extractor"] + Extractor --> AuthMgr["Authorization Manager"] + Extractor --> Refresher["Token Refresher Thread"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-5.mmd b/docs/diagrams/architecture/core-init-and-runtime-5.mmd new file mode 100644 index 00000000000..c74d3e4400e --- /dev/null +++ b/docs/diagrams/architecture/core-init-and-runtime-5.mmd @@ -0,0 +1,6 @@ +flowchart TD + ExtensionMgr["Start Extension Manager"] --> ConfigPlugin["Activate Config Plugin"] + ConfigPlugin --> RegistrySetup["Registry::setUp()"] + RegistrySetup --> LoggerPlugin["Activate Logger Plugin"] + LoggerPlugin --> DistributedPlugin["Activate Distributed Plugin"] + DistributedPlugin --> Monitoring["Numeric Monitoring Plugin"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-6.mmd b/docs/diagrams/architecture/core-init-and-runtime-6.mmd new file mode 100644 index 00000000000..cdd55a56e41 --- /dev/null +++ b/docs/diagrams/architecture/core-init-and-runtime-6.mmd @@ -0,0 +1,10 @@ +flowchart TD + Signal["SIGTERM or SIGINT"] --> Request["requestShutdown()"] + Request --> FlagSet["requested = true"] + FlagSet --> Notify["Condition Notify"] + Notify --> MainThread["waitForShutdown() Unblocks"] + MainThread --> DispatcherStop["Stop Services"] + DispatcherStop --> Join["Join Threads"] + Join --> EndEvents["EventFactory::end()"] + EndEvents --> DBShutdown["shutdownDatabase()"] + DBShutdown --> Exit["Return Exit Code"] diff --git a/docs/diagrams/architecture/core-init-and-runtime-7.mmd b/docs/diagrams/architecture/core-init-and-runtime-7.mmd new file mode 100644 index 00000000000..ca4e6c134b0 --- /dev/null +++ b/docs/diagrams/architecture/core-init-and-runtime-7.mmd @@ -0,0 +1,4 @@ +flowchart LR + Watcher["Watcher Process"] -->|"Spawns"| Worker["Worker Process"] + Worker -->|"Monitored By"| Watcher + Watcher --> Extensions["Managed Extensions"] diff --git a/docs/diagrams/architecture/core-init-and-runtime.mmd b/docs/diagrams/architecture/core-init-and-runtime.mmd new file mode 100644 index 00000000000..c0386ce47f9 --- /dev/null +++ b/docs/diagrams/architecture/core-init-and-runtime.mmd @@ -0,0 +1,15 @@ +flowchart TD + Main["Main Entry"] --> Initializer["Initializer"] + Initializer --> Flags["Flag Registration"] + Flags --> Mode["Determine Tool Type"] + Mode --> RegistryInit["Registry And Plugin Init"] + RegistryInit --> DatabaseInit["Database Init And Upgrade"] + DatabaseInit --> ExtensionMgr["Extension Manager"] + ExtensionMgr --> ConfigLoad["Config Load"] + ConfigLoad --> LoggerInit["Logger Plugin Init"] + LoggerInit --> DistributedInit["Distributed Plugin Init"] + DistributedInit --> Events["Attach Events"] + Events --> RuntimeLoop["Dispatcher Services"] + RuntimeLoop --> ShutdownReq["Shutdown Requested?"] + ShutdownReq -->|"Yes"| GracefulShutdown["Graceful Shutdown"] + GracefulShutdown --> End["Process Exit"] diff --git a/docs/diagrams/architecture/database-backends-2.mmd b/docs/diagrams/architecture/database-backends-2.mmd new file mode 100644 index 00000000000..cfc4d8ccf85 --- /dev/null +++ b/docs/diagrams/architecture/database-backends-2.mmd @@ -0,0 +1,12 @@ +sequenceDiagram + participant Caller + participant API as "OsqueryDatabase" + participant Helper as "Global Functions" + participant Registry + participant Plugin + + Caller->>API: setDatabaseValue(domain, key, value) + API->>Helper: setDatabaseValue(...) + Helper->>Registry: get active database plugin + Registry->>Plugin: putBatch(...) + Plugin-->>Caller: Status diff --git a/docs/diagrams/architecture/database-backends-3.mmd b/docs/diagrams/architecture/database-backends-3.mmd new file mode 100644 index 00000000000..54478292b8c --- /dev/null +++ b/docs/diagrams/architecture/database-backends-3.mmd @@ -0,0 +1,6 @@ +flowchart TD + Start["Read Current Version"] --> Check{"Version < Target?"} + Check -->|Yes| Migrate["Run Migration Step"] + Migrate --> Persist["Persist New Version"] + Persist --> Check + Check -->|No| End["Migration Complete"] diff --git a/docs/diagrams/architecture/database-backends.mmd b/docs/diagrams/architecture/database-backends.mmd new file mode 100644 index 00000000000..c675ad41833 --- /dev/null +++ b/docs/diagrams/architecture/database-backends.mmd @@ -0,0 +1,11 @@ +flowchart TD + CoreRuntime["Core Runtime"] --> DatabaseAPI["OsqueryDatabase"] + DatabaseAPI --> DBFunctions["Global Database Functions"] + DBFunctions --> Registry["Registry Factory"] + Registry --> ActivePlugin["Active Database Plugin"] + + ActivePlugin --> RocksDBPlugin["RocksDB Plugin"] + ActivePlugin --> EphemeralPlugin["Ephemeral Database Plugin"] + + RocksDBPlugin --> Disk[("Persistent Storage")] + EphemeralPlugin --> Memory[("In-Memory Map")] diff --git a/docs/diagrams/architecture/distributed-querying-2.mmd b/docs/diagrams/architecture/distributed-querying-2.mmd new file mode 100644 index 00000000000..43129964ff8 --- /dev/null +++ b/docs/diagrams/architecture/distributed-querying-2.mmd @@ -0,0 +1,10 @@ +flowchart TD + Start["Start Loop"] --> Pull["pullUpdates()"] + Pull --> Accept["acceptWork()"] + Accept --> Pending{{"Pending Queries?"}} + Pending -->|"Yes"| Run["runQueries()"] + Run --> Exec["Execute via SQL"] + Exec --> AddRes["addResult()"] + AddRes --> Flush["flushCompleted()"] + Flush --> Start + Pending -->|"No"| Start diff --git a/docs/diagrams/architecture/distributed-querying-3.mmd b/docs/diagrams/architecture/distributed-querying-3.mmd new file mode 100644 index 00000000000..85a15ff716b --- /dev/null +++ b/docs/diagrams/architecture/distributed-querying-3.mmd @@ -0,0 +1,6 @@ +flowchart TD + Incoming["Incoming Query"] --> Check["checkAndSetAsRunning()"] + Check --> Deny{{"Within denylist window?"}} + Deny -->|"Yes"| Skip["Skip Execution"] + Deny -->|"No"| Execute["Run SQL"] + Execute --> Clear["setAsNotRunning()"] diff --git a/docs/diagrams/architecture/distributed-querying-4.mmd b/docs/diagrams/architecture/distributed-querying-4.mmd new file mode 100644 index 00000000000..e4ac45b32b2 --- /dev/null +++ b/docs/diagrams/architecture/distributed-querying-4.mmd @@ -0,0 +1,13 @@ +sequenceDiagram + participant Node + participant TLSPlugin + participant Server + + Node->>TLSPlugin: getQueries() + TLSPlugin->>Server: POST read_endpoint + Server-->>TLSPlugin: JSON queries + TLSPlugin-->>Node: JSON payload + + Node->>TLSPlugin: writeResults(json) + TLSPlugin->>Server: POST write_endpoint + Server-->>TLSPlugin: ACK diff --git a/docs/diagrams/architecture/distributed-querying.mmd b/docs/diagrams/architecture/distributed-querying.mmd new file mode 100644 index 00000000000..0e05280ce12 --- /dev/null +++ b/docs/diagrams/architecture/distributed-querying.mmd @@ -0,0 +1,10 @@ +flowchart LR + ControlPlane["Remote Control Plane"] -->|"HTTPS JSON"| TLSPlugin["TLSDistributedPlugin"] + TLSPlugin -->|"getQueries()"| DistributedEngine["Distributed"] + DistributedEngine -->|"Execute SQL"| SQLCore["SQL Core and Virtual Tables"] + SQLCore -->|"QueryData"| DistributedEngine + DistributedEngine -->|"Batch Results"| TLSPlugin + TLSPlugin -->|"writeResults()"| ControlPlane + + DistributedEngine -->|"Record Performance"| QueryPerf["QueryPerformance"] + DistributedEngine -->|"Create Log Items"| QueryLog["QueryLogItem"] diff --git a/docs/diagrams/architecture/events-core-2.mmd b/docs/diagrams/architecture/events-core-2.mmd new file mode 100644 index 00000000000..392f1cf32d7 --- /dev/null +++ b/docs/diagrams/architecture/events-core-2.mmd @@ -0,0 +1,6 @@ +flowchart TD + Start["Register Publisher"] --> Setup["setUp()"] + Setup --> Running["EVENT_RUNNING"] + Running --> Loop["run() loop"] + Loop -->|"isEnding()"| TearDown["tearDown()"] + TearDown --> EndState["EVENT_NONE"] diff --git a/docs/diagrams/architecture/events-core-3.mmd b/docs/diagrams/architecture/events-core-3.mmd new file mode 100644 index 00000000000..97ab215a021 --- /dev/null +++ b/docs/diagrams/architecture/events-core-3.mmd @@ -0,0 +1,12 @@ +sequenceDiagram + participant Publisher + participant Subscriber + participant Database + participant SQL + + Publisher->>Subscriber: EventContext + Subscriber->>Subscriber: Transform to Row + Subscriber->>Database: addBatch(rows) + SQL->>Subscriber: genTable(query context) + Subscriber->>Database: generateRows(time window) + Subscriber->>SQL: Yield rows diff --git a/docs/diagrams/architecture/events-core-4.mmd b/docs/diagrams/architecture/events-core-4.mmd new file mode 100644 index 00000000000..ce07135ef3b --- /dev/null +++ b/docs/diagrams/architecture/events-core-4.mmd @@ -0,0 +1,4 @@ +flowchart LR + Subscriber["EventSubscriberPlugin"] -->|"create()"| SubObj["Subscription"] + SubObj -->|"addSubscription()"| Publisher["EventPublisherPlugin"] + Publisher -->|"invoke callback"| Subscriber diff --git a/docs/diagrams/architecture/events-core-5.mmd b/docs/diagrams/architecture/events-core-5.mmd new file mode 100644 index 00000000000..d970a578832 --- /dev/null +++ b/docs/diagrams/architecture/events-core-5.mmd @@ -0,0 +1,7 @@ +flowchart TD + Delay["EventFactory.delay()"] --> Spawn["Spawn Publisher Threads"] + Spawn --> RunLoop["EventFactory.run(type)"] + RunLoop --> Poll["publisher.run()"] + Poll --> Pause["pause(200ms)"] + Pause --> Poll + Poll -->|"isEnding"| Exit["tearDown()"] diff --git a/docs/diagrams/architecture/events-core-6.mmd b/docs/diagrams/architecture/events-core-6.mmd new file mode 100644 index 00000000000..3a6300f6391 --- /dev/null +++ b/docs/diagrams/architecture/events-core-6.mmd @@ -0,0 +1,7 @@ +flowchart TD + Event["System Event"] --> Publisher + Publisher --> Subscriber + Subscriber --> Store["Persistent Backing Store"] + Store --> Query["SQL Query"] + Query --> ExpireCheck["Expiration Enforcement"] + ExpireCheck --> Result["Filtered Rows"] diff --git a/docs/diagrams/architecture/events-core.mmd b/docs/diagrams/architecture/events-core.mmd new file mode 100644 index 00000000000..5fd2b4b25e0 --- /dev/null +++ b/docs/diagrams/architecture/events-core.mmd @@ -0,0 +1,26 @@ +flowchart TD + Config["Configuration & Schedule"] --> Factory["EventFactory"] + + subgraph publishers["Event Publishers"] + PubA["EventPublisherPlugin A"] + PubB["EventPublisherPlugin B"] + end + + subgraph subscribers["Event Subscribers"] + SubA["EventSubscriberPlugin A"] + SubB["EventSubscriberPlugin B"] + end + + PubA -->|"fires events"| SubA + PubA -->|"fires events"| SubB + PubB -->|"fires events"| SubA + + SubA -->|"addBatch(rows)"| Storage["Event Backing Store"] + SubB -->|"addBatch(rows)"| Storage + + Storage -->|"time filtered rows"| SQL["Virtual Table Query"] + + Factory --> PubA + Factory --> PubB + Factory --> SubA + Factory --> SubB diff --git a/docs/diagrams/architecture/extensions-framework-2.mmd b/docs/diagrams/architecture/extensions-framework-2.mmd new file mode 100644 index 00000000000..0a66a209d71 --- /dev/null +++ b/docs/diagrams/architecture/extensions-framework-2.mmd @@ -0,0 +1,4 @@ +flowchart TD + Request["registerExtension()"] --> Generate["UuidGenerator.getUuid()"] + Generate --> Store["Track UUID in set"] + Store --> Assign["Associate UUID with Registry routes"] diff --git a/docs/diagrams/architecture/extensions-framework-3.mmd b/docs/diagrams/architecture/extensions-framework-3.mmd new file mode 100644 index 00000000000..cd226a32023 --- /dev/null +++ b/docs/diagrams/architecture/extensions-framework-3.mmd @@ -0,0 +1,10 @@ +sequenceDiagram + participant Ext as Extension Process + participant EM as Extension Manager + participant Reg as RegistryFactory + + Ext->>EM: registerExtension(info, registry) + EM->>Reg: addBroadcast(uuid, registry) + EM-->>Ext: return uuid + Ext->>Ext: start Thrift server + EM->>Ext: ping() diff --git a/docs/diagrams/architecture/extensions-framework-4.mmd b/docs/diagrams/architecture/extensions-framework-4.mmd new file mode 100644 index 00000000000..ba99db86ca8 --- /dev/null +++ b/docs/diagrams/architecture/extensions-framework-4.mmd @@ -0,0 +1,5 @@ +flowchart TD + Client["ExtensionClient"] --> Transport["TBufferedTransport"] + Transport --> Socket["UNIX Socket / Named Pipe"] + Socket --> Server["TThreadedServer"] + Server --> Handler["ExtensionHandler / ExtensionManagerHandler"] diff --git a/docs/diagrams/architecture/extensions-framework-5.mmd b/docs/diagrams/architecture/extensions-framework-5.mmd new file mode 100644 index 00000000000..6f5b49dd5be --- /dev/null +++ b/docs/diagrams/architecture/extensions-framework-5.mmd @@ -0,0 +1,5 @@ +flowchart TD + Call["callExtension(uuid, registry, item)"] --> Lookup["Resolve registry alias"] + Lookup --> Client["ExtensionClient.call()"] + Client --> ExtensionServer["ExtensionHandler.call()"] + ExtensionServer --> Registry["RegistryFactory.call()"] diff --git a/docs/diagrams/architecture/extensions-framework-6.mmd b/docs/diagrams/architecture/extensions-framework-6.mmd new file mode 100644 index 00000000000..c363a4812ab --- /dev/null +++ b/docs/diagrams/architecture/extensions-framework-6.mmd @@ -0,0 +1,4 @@ +flowchart LR + Query["SQL Query"] --> External["ExternalSQLPlugin"] + External --> Manager["ExtensionManagerClient.query()"] + Manager --> SQLite["Core SQLite Engine"] diff --git a/docs/diagrams/architecture/extensions-framework-7.mmd b/docs/diagrams/architecture/extensions-framework-7.mmd new file mode 100644 index 00000000000..1c1f68f1ea8 --- /dev/null +++ b/docs/diagrams/architecture/extensions-framework-7.mmd @@ -0,0 +1,5 @@ +flowchart TD + Loop["Periodic Watch"] --> Check["socketExists()"] + Check --> Ping["client.ping()"] + Ping --> Healthy["EXT_SUCCESS?"] + Healthy -->|"No"| Remove["removeBroadcast(uuid)"] diff --git a/docs/diagrams/architecture/extensions-framework-8.mmd b/docs/diagrams/architecture/extensions-framework-8.mmd new file mode 100644 index 00000000000..7d2dc289252 --- /dev/null +++ b/docs/diagrams/architecture/extensions-framework-8.mmd @@ -0,0 +1,9 @@ +sequenceDiagram + participant Core + participant EMRunner + participant EMWatcher + + Core->>EMRunner: start() + Core->>EMWatcher: start() + EMRunner->>Socket: bind + serve + EMWatcher->>Registry: monitor UUID routes diff --git a/docs/diagrams/architecture/extensions-framework.mmd b/docs/diagrams/architecture/extensions-framework.mmd new file mode 100644 index 00000000000..44d7e10be8f --- /dev/null +++ b/docs/diagrams/architecture/extensions-framework.mmd @@ -0,0 +1,12 @@ +flowchart LR + Core["osquery Core"] --> Manager["Extension Manager"] + Manager --> Registry["Registry Factory"] + Manager --> SQLCore["SQL Engine"] + + ExtensionA["Extension Process A"] -->|"registerExtension"| Manager + ExtensionB["Extension Process B"] -->|"registerExtension"| Manager + + Core -->|"callExtension"| ExtensionA + Core -->|"callExtension"| ExtensionB + + ExtensionA -->|"query"| SQLCore diff --git a/docs/diagrams/architecture/filesystem-and-fileops-2.mmd b/docs/diagrams/architecture/filesystem-and-fileops-2.mmd new file mode 100644 index 00000000000..a31464dc296 --- /dev/null +++ b/docs/diagrams/architecture/filesystem-and-fileops-2.mmd @@ -0,0 +1,6 @@ +flowchart LR + ReadCall["read() or write()"] --> NonBlockCheck["Non Blocking Mode?"] + NonBlockCheck -->|"Yes"| Overlap["OVERLAPPED Structure"] + Overlap --> Buffer["Async Buffer"] + Buffer --> Pending["Pending IO State"] + NonBlockCheck -->|"No"| Sync["Synchronous IO"] diff --git a/docs/diagrams/architecture/filesystem-and-fileops-3.mmd b/docs/diagrams/architecture/filesystem-and-fileops-3.mmd new file mode 100644 index 00000000000..cd59e715818 --- /dev/null +++ b/docs/diagrams/architecture/filesystem-and-fileops-3.mmd @@ -0,0 +1,9 @@ +flowchart TD + Client["Caller"] --> Open["PlatformFile Constructor"] + Open --> Handle["PlatformHandle"] + Handle --> Read["read()"] + Handle --> Write["write()"] + Handle --> Seek["seek()"] + Handle --> Size["size()"] + + Read --> AsyncCheck["hasPendingIo()"] diff --git a/docs/diagrams/architecture/filesystem-and-fileops-4.mmd b/docs/diagrams/architecture/filesystem-and-fileops-4.mmd new file mode 100644 index 00000000000..7b8f18bfc43 --- /dev/null +++ b/docs/diagrams/architecture/filesystem-and-fileops-4.mmd @@ -0,0 +1,5 @@ +flowchart TD + ExtensionStartup["Extension Startup"] --> Check["socketExists(remove)"] + Check --> Exists["Socket Exists?"] + Exists -->|"Yes"| Remove["Attempt Removal"] + Exists -->|"No"| Continue["Create Socket"] diff --git a/docs/diagrams/architecture/filesystem-and-fileops-5.mmd b/docs/diagrams/architecture/filesystem-and-fileops-5.mmd new file mode 100644 index 00000000000..4e43e734527 --- /dev/null +++ b/docs/diagrams/architecture/filesystem-and-fileops-5.mmd @@ -0,0 +1,9 @@ +sequenceDiagram + participant Module + participant FS as Filesystem And Fileops + participant OS + + Module->>FS: platformStat(path) + FS->>OS: Native stat or Win API + OS-->>FS: Raw metadata + FS-->>Module: Normalized structure diff --git a/docs/diagrams/architecture/filesystem-and-fileops-6.mmd b/docs/diagrams/architecture/filesystem-and-fileops-6.mmd new file mode 100644 index 00000000000..694956929c2 --- /dev/null +++ b/docs/diagrams/architecture/filesystem-and-fileops-6.mmd @@ -0,0 +1,4 @@ +flowchart TD + FileOpen["Open File"] --> OwnerCheck["isOwnerRoot or isOwnerCurrentUser"] + OwnerCheck --> PermCheck["hasSafePermissions"] + PermCheck --> Decision["Allow or Reject"] diff --git a/docs/diagrams/architecture/filesystem-and-fileops.mmd b/docs/diagrams/architecture/filesystem-and-fileops.mmd new file mode 100644 index 00000000000..967060c0cc6 --- /dev/null +++ b/docs/diagrams/architecture/filesystem-and-fileops.mmd @@ -0,0 +1,15 @@ +flowchart TD + App["Higher Level Modules"] --> SQL["SQL Core And Virtual Tables"] + App --> DB["Database Backends"] + App --> EXT["Extensions Framework"] + App --> CFG["Config Plugins"] + + SQL --> FS["Filesystem And Fileops"] + DB --> FS + EXT --> FS + CFG --> FS + + FS --> PF["PlatformFile"] + FS --> STAT["platformStat and platformLstat"] + FS --> UTIL["Filesystem Utility Functions"] + FS --> ASYNC["AsyncEvent"] diff --git a/docs/diagrams/architecture/hashing-2.mmd b/docs/diagrams/architecture/hashing-2.mmd new file mode 100644 index 00000000000..c9cec189e8e --- /dev/null +++ b/docs/diagrams/architecture/hashing-2.mmd @@ -0,0 +1,4 @@ +flowchart LR + Create["Construct Hash"] --> Stream["update() Calls"] + Stream --> Finalize["digest()"] + Finalize --> Result["Encoded Digest String"] diff --git a/docs/diagrams/architecture/hashing-3.mmd b/docs/diagrams/architecture/hashing-3.mmd new file mode 100644 index 00000000000..e88bd616229 --- /dev/null +++ b/docs/diagrams/architecture/hashing-3.mmd @@ -0,0 +1,8 @@ +flowchart TD + Request["hashFromFile()"] --> OpenFile["Open File"] + OpenFile --> ReadChunk["Read Chunk"] + ReadChunk --> UpdateHash["Hash.update()"] + UpdateHash --> MoreData{"More Data?"} + MoreData -->|"Yes"| ReadChunk + MoreData -->|"No"| Finalize["Hash.digest()"] + Finalize --> Return["Return HEX String"] diff --git a/docs/diagrams/architecture/hashing-4.mmd b/docs/diagrams/architecture/hashing-4.mmd new file mode 100644 index 00000000000..c66b8531a85 --- /dev/null +++ b/docs/diagrams/architecture/hashing-4.mmd @@ -0,0 +1,17 @@ +flowchart TD + Start["File Input"] --> MaskCheck["Check Mask Bits"] + MaskCheck --> MD5Ctx["Initialize MD5 Context"] + MaskCheck --> SHA1Ctx["Initialize SHA1 Context"] + MaskCheck --> SHA256Ctx["Initialize SHA256 Context"] + + MD5Ctx --> StreamData["Stream File Data"] + SHA1Ctx --> StreamData + SHA256Ctx --> StreamData + + StreamData --> FinalMD5["Finalize MD5"] + StreamData --> FinalSHA1["Finalize SHA1"] + StreamData --> FinalSHA256["Finalize SHA256"] + + FinalMD5 --> MultiResult["Populate MultiHashes"] + FinalSHA1 --> MultiResult + FinalSHA256 --> MultiResult diff --git a/docs/diagrams/architecture/hashing-5.mmd b/docs/diagrams/architecture/hashing-5.mmd new file mode 100644 index 00000000000..ef795113043 --- /dev/null +++ b/docs/diagrams/architecture/hashing-5.mmd @@ -0,0 +1,3 @@ +flowchart LR + Fileops["Filesystem and Fileops"] --> Hashing["Hashing"] + Hashing --> DigestResult["Digest String"] diff --git a/docs/diagrams/architecture/hashing-6.mmd b/docs/diagrams/architecture/hashing-6.mmd new file mode 100644 index 00000000000..037512e8313 --- /dev/null +++ b/docs/diagrams/architecture/hashing-6.mmd @@ -0,0 +1,5 @@ +flowchart TD + VirtualTable["Virtual Table"] --> FilePath["File Path"] + FilePath --> HashingModule["Hashing"] + HashingModule --> RowData["Row with Hash Columns"] + RowData --> SQLiteLayer["SQLite Engine"] diff --git a/docs/diagrams/architecture/hashing-7.mmd b/docs/diagrams/architecture/hashing-7.mmd new file mode 100644 index 00000000000..9d4342edb64 --- /dev/null +++ b/docs/diagrams/architecture/hashing-7.mmd @@ -0,0 +1,5 @@ +flowchart TD + ScheduledQuery["Scheduled Query"] --> CollectFile["Collect File Path"] + CollectFile --> ComputeHash["Compute SHA256"] + ComputeHash --> Compare["Compare With Previous Result"] + Compare --> Diff["Generate Diff Results"] diff --git a/docs/diagrams/architecture/hashing-8.mmd b/docs/diagrams/architecture/hashing-8.mmd new file mode 100644 index 00000000000..8bc28c3cfd8 --- /dev/null +++ b/docs/diagrams/architecture/hashing-8.mmd @@ -0,0 +1,11 @@ +flowchart TD + subgraph CoreSystem["Core System"] + SQLModule["SQL Core and Virtual Tables"] + DistributedModule["Distributed Querying"] + QueryLogModule["Query Execution and Logging"] + end + + FileModule["Filesystem and Fileops"] --> HashingModule["Hashing"] + HashingModule --> SQLModule + HashingModule --> DistributedModule + HashingModule --> QueryLogModule diff --git a/docs/diagrams/architecture/hashing.mmd b/docs/diagrams/architecture/hashing.mmd new file mode 100644 index 00000000000..76fec328a9c --- /dev/null +++ b/docs/diagrams/architecture/hashing.mmd @@ -0,0 +1,6 @@ +flowchart TD + HashClass["Hash Class"] --> Algorithm["HashType"] + HashClass --> Encoding["HashEncodingType"] + HashClass --> Context["Internal Context Pointer"] + HashClass --> Update["update(buffer, size)"] + HashClass --> Digest["digest()"] diff --git a/docs/diagrams/architecture/plugin-interfaces-and-logging-2.mmd b/docs/diagrams/architecture/plugin-interfaces-and-logging-2.mmd new file mode 100644 index 00000000000..f6939a9f7cf --- /dev/null +++ b/docs/diagrams/architecture/plugin-interfaces-and-logging-2.mmd @@ -0,0 +1,10 @@ +sequenceDiagram + participant Core + participant Glog + participant Buffer + participant LoggerPlugin + + Core->>Glog: Emit status message + Glog->>Buffer: Store StatusLogLine + Core->>LoggerPlugin: init(binaryName, bufferedLogs) + LoggerPlugin->>LoggerPlugin: Process buffered StatusLogLine entries diff --git a/docs/diagrams/architecture/plugin-interfaces-and-logging-3.mmd b/docs/diagrams/architecture/plugin-interfaces-and-logging-3.mmd new file mode 100644 index 00000000000..af260e016dc --- /dev/null +++ b/docs/diagrams/architecture/plugin-interfaces-and-logging-3.mmd @@ -0,0 +1,3 @@ +flowchart TD + PluginBase["Plugin Base Class"] --> LoggerPlugin["LoggerPlugin"] + LoggerPlugin --> CustomLogger["Custom Logger Implementation"] diff --git a/docs/diagrams/architecture/plugin-interfaces-and-logging-4.mmd b/docs/diagrams/architecture/plugin-interfaces-and-logging-4.mmd new file mode 100644 index 00000000000..547ce73fd4a --- /dev/null +++ b/docs/diagrams/architecture/plugin-interfaces-and-logging-4.mmd @@ -0,0 +1,8 @@ +flowchart TD + Core["Core Subsystem"] --> CheckStatus["Check usesLogStatus"] + CheckStatus -->|"true"| StatusHandler["logStatus"] + CheckStatus -->|"false"| Glog["Default Glog Handling"] + + Core --> CheckEvent["Check usesLogEvent"] + CheckEvent -->|"true"| EventHandler["logEvent or logStringBatch"] + CheckEvent -->|"false"| Database["Event Stored In Database"] diff --git a/docs/diagrams/architecture/plugin-interfaces-and-logging-5.mmd b/docs/diagrams/architecture/plugin-interfaces-and-logging-5.mmd new file mode 100644 index 00000000000..9fb68c3e774 --- /dev/null +++ b/docs/diagrams/architecture/plugin-interfaces-and-logging-5.mmd @@ -0,0 +1,4 @@ +flowchart LR + EventSubscriber["Event Subscriber"] --> EventFactory["Event Factory"] + EventFactory -->|"Event JSON"| LoggerPlugin + LoggerPlugin -->|"logEvent"| ExternalSink diff --git a/docs/diagrams/architecture/plugin-interfaces-and-logging-6.mmd b/docs/diagrams/architecture/plugin-interfaces-and-logging-6.mmd new file mode 100644 index 00000000000..ddf4c3e0615 --- /dev/null +++ b/docs/diagrams/architecture/plugin-interfaces-and-logging-6.mmd @@ -0,0 +1,4 @@ +flowchart TD + ExtensionBinary["Extension Process"] --> ExtensionManager["Extension Manager"] + ExtensionManager --> Registry["Plugin Registry"] + Registry --> LoggerPlugin diff --git a/docs/diagrams/architecture/plugin-interfaces-and-logging-7.mmd b/docs/diagrams/architecture/plugin-interfaces-and-logging-7.mmd new file mode 100644 index 00000000000..834d8963d5d --- /dev/null +++ b/docs/diagrams/architecture/plugin-interfaces-and-logging-7.mmd @@ -0,0 +1,7 @@ +flowchart TD + Start["Process Start"] --> Flags["Parse Flags"] + Flags --> RegistryInit["Initialize Registry"] + RegistryInit --> ExtensionsLoad["Load Extensions"] + ExtensionsLoad --> ConfigLoad["Load Config"] + ConfigLoad --> LoggerInit["LoggerPlugin init"] + LoggerInit --> Runtime["Normal Operation"] diff --git a/docs/diagrams/architecture/plugin-interfaces-and-logging.mmd b/docs/diagrams/architecture/plugin-interfaces-and-logging.mmd new file mode 100644 index 00000000000..6732da95f77 --- /dev/null +++ b/docs/diagrams/architecture/plugin-interfaces-and-logging.mmd @@ -0,0 +1,7 @@ +flowchart LR + CoreRuntime["Core Init And Runtime"] -->|"Buffered Status Logs"| LoggerInterface["LoggerPlugin Interface"] + QueryEngine["Query Execution And Logging"] -->|"QueryLogItem JSON"| LoggerInterface + EventsCore["Events Core"] -->|"Event Records"| LoggerInterface + Extensions["Extensions Framework"] -->|"Registers Logger Plugins"| LoggerInterface + LoggerInterface -->|"logString or logStatus"| LoggerBackend["Concrete Logger Plugin"] + LoggerBackend -->|"Writes To"| ExternalSink["External Logging System"] diff --git a/docs/diagrams/architecture/process-and-profiler-2.mmd b/docs/diagrams/architecture/process-and-profiler-2.mmd new file mode 100644 index 00000000000..db86ee40643 --- /dev/null +++ b/docs/diagrams/architecture/process-and-profiler-2.mmd @@ -0,0 +1,7 @@ +flowchart TD + Parent["Parent Process"] -->|"fork()"| Child["Child Process"] + Child -->|"set environment"| Env["OSQUERY_WORKER or OSQUERY_EXTENSION"] + Child -->|"reset signals via sigaction"| Reset["Default Signal Handlers"] + Child -->|"execve()"| Executable["Worker or Extension Binary"] + Parent -->|"waitpid()"| Status["ProcessState"] + Parent -->|"SIGTERM or SIGKILL"| Child diff --git a/docs/diagrams/architecture/process-and-profiler-3.mmd b/docs/diagrams/architecture/process-and-profiler-3.mmd new file mode 100644 index 00000000000..40790d16ec7 --- /dev/null +++ b/docs/diagrams/architecture/process-and-profiler-3.mmd @@ -0,0 +1,8 @@ +flowchart TD + Start["CodeProfiler Constructor"] --> CaptureStart["call getrusage()"] + CaptureStart --> StoreStart["Store rusage + wall time"] + StoreStart --> Execution["Instrumented Code Executes"] + Execution --> End["Destructor"] + End --> CaptureEnd["call getrusage()"] + CaptureEnd --> Diff["Compute Differences"] + Diff --> Record["Record Metrics"] diff --git a/docs/diagrams/architecture/process-and-profiler.mmd b/docs/diagrams/architecture/process-and-profiler.mmd new file mode 100644 index 00000000000..54573054e65 --- /dev/null +++ b/docs/diagrams/architecture/process-and-profiler.mmd @@ -0,0 +1,22 @@ +flowchart TD + subgraph process_layer["Process Management Layer"] + PlatformProcess["PlatformProcess"] + Signals["POSIX Signals"] + ForkExec["fork() / execve()"] + WaitPid["waitpid()"] + end + + subgraph profiling_layer["Profiling Layer"] + CodeProfiler["CodeProfiler"] + Rusage["getrusage()"] + Timeval["timeval"] + Monitoring["Numeric Monitoring"] + end + + PlatformProcess -->|"spawns"| ForkExec + PlatformProcess -->|"controls"| Signals + PlatformProcess -->|"monitors"| WaitPid + + CodeProfiler -->|"captures"| Rusage + CodeProfiler -->|"converts"| Timeval + CodeProfiler -->|"records"| Monitoring diff --git a/docs/diagrams/architecture/query-execution-and-logging-10.mmd b/docs/diagrams/architecture/query-execution-and-logging-10.mmd new file mode 100644 index 00000000000..2ae64d81aad --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-10.mmd @@ -0,0 +1,5 @@ +flowchart TD + Maintenance["Maintenance Operations"] + Maintenance --> Names["getStoredQueryNames()"] + Maintenance --> Exists["isQueryNameInDatabase()"] + Maintenance --> Current["getCurrentResults()"] diff --git a/docs/diagrams/architecture/query-execution-and-logging-11.mmd b/docs/diagrams/architecture/query-execution-and-logging-11.mmd new file mode 100644 index 00000000000..1de06a89aab --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-11.mmd @@ -0,0 +1,10 @@ +flowchart TD + Config["Configuration"] --> Scheduler["Scheduler"] + Scheduler --> SQL["SQL Engine"] + SQL --> Raw["Raw Results"] + Raw --> QueryObj["Query"] + QueryObj --> Diff["DiffResults"] + QueryObj --> Persist["Persistent Storage"] + Diff --> LogItem["QueryLogItem"] + LogItem --> JSON["Serialization"] + JSON --> Logger["Logger Plugins"] diff --git a/docs/diagrams/architecture/query-execution-and-logging-2.mmd b/docs/diagrams/architecture/query-execution-and-logging-2.mmd new file mode 100644 index 00000000000..a25640dac2e --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-2.mmd @@ -0,0 +1,12 @@ +flowchart TD + QueryLogItem["QueryLogItem"] + QueryLogItem --> IsSnapshot["isSnapshot"] + QueryLogItem --> DiffRes["DiffResults"] + QueryLogItem --> Snapshot["snapshot_results"] + QueryLogItem --> Name["name"] + QueryLogItem --> Identifier["identifier"] + QueryLogItem --> Time["time"] + QueryLogItem --> Epoch["epoch"] + QueryLogItem --> Counter["counter"] + QueryLogItem --> CalendarTime["calendar_time"] + QueryLogItem --> Decorations["decorations"] diff --git a/docs/diagrams/architecture/query-execution-and-logging-3.mmd b/docs/diagrams/architecture/query-execution-and-logging-3.mmd new file mode 100644 index 00000000000..b31c3d8e18b --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-3.mmd @@ -0,0 +1,12 @@ +sequenceDiagram + participant Scheduler + participant QueryObj as Query + participant DB as Database + participant Logger + + Scheduler->>QueryObj: addNewResults(qd, epoch) + QueryObj->>DB: getPreviousQueryResults() + QueryObj->>QueryObj: compute DiffResults + QueryObj->>DB: saveQueryResults(json, epoch) + QueryObj-->>Scheduler: DiffResults + counter + Scheduler->>Logger: serialize QueryLogItem diff --git a/docs/diagrams/architecture/query-execution-and-logging-4.mmd b/docs/diagrams/architecture/query-execution-and-logging-4.mmd new file mode 100644 index 00000000000..d5dd7a6dd3b --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-4.mmd @@ -0,0 +1,9 @@ +flowchart TD + Query["Query"] --> GetPrev["getPreviousQueryResults()"] + Query --> Save["saveQueryResults()"] + Query --> Epoch["getPreviousEpoch()"] + Query --> Counter["getQueryCounter()"] + GetPrev --> DB["OsqueryDatabase"] + Save --> DB + Epoch --> DB + Counter --> DB diff --git a/docs/diagrams/architecture/query-execution-and-logging-5.mmd b/docs/diagrams/architecture/query-execution-and-logging-5.mmd new file mode 100644 index 00000000000..e3e5f7f1e1c --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-5.mmd @@ -0,0 +1,5 @@ +flowchart LR + ConfigChange["Config Change"] --> EpochInc["Epoch Increment"] + EpochInc --> Fresh["Fresh Results"] + Fresh --> CounterReset["Counter Reset"] + NoChange["No Config Change"] --> CounterInc["Counter Increment"] diff --git a/docs/diagrams/architecture/query-execution-and-logging-6.mmd b/docs/diagrams/architecture/query-execution-and-logging-6.mmd new file mode 100644 index 00000000000..dfd8a7a8cb7 --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-6.mmd @@ -0,0 +1,7 @@ +flowchart TD + NewResults["Current Results"] + OldResults["Stored Results"] + NewResults --> DiffEngine["Diff Engine"] + OldResults --> DiffEngine + DiffEngine --> Added["Added Rows"] + DiffEngine --> Removed["Removed Rows"] diff --git a/docs/diagrams/architecture/query-execution-and-logging-7.mmd b/docs/diagrams/architecture/query-execution-and-logging-7.mmd new file mode 100644 index 00000000000..0382f05cfcf --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-7.mmd @@ -0,0 +1,4 @@ +flowchart LR + QueryLogItem["QueryLogItem"] --> Mode{"Serialization Mode"} + Mode -->|"Standard"| Standard["Single JSON Object"] + Mode -->|"Events"| Events["Multiple Event JSON Objects"] diff --git a/docs/diagrams/architecture/query-execution-and-logging-8.mmd b/docs/diagrams/architecture/query-execution-and-logging-8.mmd new file mode 100644 index 00000000000..50193f0270d --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-8.mmd @@ -0,0 +1,5 @@ +flowchart TD + ScheduledQuery["ScheduledQuery"] --> QueryObj["Query"] + QueryObj --> CheckSql["isNewQuerySql()"] + QueryObj --> StatusCheck["getQueryStatus()"] + QueryObj --> CounterLogic["incrementCounter()"] diff --git a/docs/diagrams/architecture/query-execution-and-logging-9.mmd b/docs/diagrams/architecture/query-execution-and-logging-9.mmd new file mode 100644 index 00000000000..49433a79935 --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging-9.mmd @@ -0,0 +1,5 @@ +flowchart LR + EventSubscriber["Event Subscriber"] --> EventRows["GenerateRowsResult"] + EventRows --> QueryObj["Query.addNewEvents()"] + QueryObj --> Diff["DiffResults"] + Diff --> LogItem["QueryLogItem"] diff --git a/docs/diagrams/architecture/query-execution-and-logging.mmd b/docs/diagrams/architecture/query-execution-and-logging.mmd new file mode 100644 index 00000000000..a506663624e --- /dev/null +++ b/docs/diagrams/architecture/query-execution-and-logging.mmd @@ -0,0 +1,9 @@ +flowchart LR + Scheduler["Scheduled Query"] --> SQL["SQL Execution Engine"] + SQL --> Results["QueryDataTyped"] + Results --> QueryObj["Query"] + QueryObj --> Diff["DiffResults"] + QueryObj --> DB["Persistent Storage"] + Diff --> LogItem["QueryLogItem"] + LogItem --> Serializer["JSON Serialization"] + Serializer --> Logger["Logger Plugins"] diff --git a/docs/diagrams/architecture/remote-http-client-2.mmd b/docs/diagrams/architecture/remote-http-client-2.mmd new file mode 100644 index 00000000000..2b7ed63c362 --- /dev/null +++ b/docs/diagrams/architecture/remote-http-client-2.mmd @@ -0,0 +1,7 @@ +flowchart TD + Client["Client"] --> Options["Client::Options"] + Client --> Resolver["TCP Resolver"] + Client --> Socket["TCP Socket"] + Client --> SSLStream["SSL Stream (optional)"] + Client --> Timer["Deadline Timer"] + Client --> IOContext["io_context"] diff --git a/docs/diagrams/architecture/remote-http-client-3.mmd b/docs/diagrams/architecture/remote-http-client-3.mmd new file mode 100644 index 00000000000..c8f80bcfc93 --- /dev/null +++ b/docs/diagrams/architecture/remote-http-client-3.mmd @@ -0,0 +1,11 @@ +flowchart TD + Start["Request Method Invoked"] --> Init["initHTTPRequest()"] + Init --> Connect["createConnection()"] + Connect --> TLSCheck{"TLS Enabled?"} + TLSCheck -->|"Yes"| Encrypt["encryptConnection()"] + TLSCheck -->|"No"| Send["sendHTTPRequest()"] + Encrypt --> Send + Send --> Write["async_write"] + Write --> Read["async_read"] + Read --> Post["postResponseHandler()"] + Post --> End["Response Returned"] diff --git a/docs/diagrams/architecture/remote-http-client-4.mmd b/docs/diagrams/architecture/remote-http-client-4.mmd new file mode 100644 index 00000000000..bbd309e5fb5 --- /dev/null +++ b/docs/diagrams/architecture/remote-http-client-4.mmd @@ -0,0 +1,8 @@ +flowchart TD + TCPConnect["TCP Connect"] --> Wrap["Wrap in SSL Stream"] + Wrap --> Handshake["async_handshake"] + Handshake --> Verify{"Verify Peer?"} + Verify -->|"Yes"| Validate["Certificate Validation"] + Verify -->|"No"| Continue["Proceed"] + Validate --> Continue + Continue --> Ready["Secure Channel Ready"] diff --git a/docs/diagrams/architecture/remote-http-client-5.mmd b/docs/diagrams/architecture/remote-http-client-5.mmd new file mode 100644 index 00000000000..ecfe52c304c --- /dev/null +++ b/docs/diagrams/architecture/remote-http-client-5.mmd @@ -0,0 +1,8 @@ +flowchart TD + AsyncOp["Async Network Operation"] --> Complete{"Completed?"} + Complete -->|"Yes"| CancelTimer["Cancel Timer"] + Complete -->|"No"| Timeout["Timer Fires"] + Timeout --> CancelSocket["Close Socket"] + CancelSocket --> Error["Set error_code"] + CancelTimer --> Return["Return Response or Error"] + Error --> Return diff --git a/docs/diagrams/architecture/remote-http-client-6.mmd b/docs/diagrams/architecture/remote-http-client-6.mmd new file mode 100644 index 00000000000..8392edb3b37 --- /dev/null +++ b/docs/diagrams/architecture/remote-http-client-6.mmd @@ -0,0 +1,5 @@ +flowchart LR + Runtime["Core Runtime"] --> Distributed["Distributed Querying"] + Runtime --> Config["Config and Packs"] + Distributed --> HTTP["Remote Http Client"] + Config --> HTTP diff --git a/docs/diagrams/architecture/remote-http-client.mmd b/docs/diagrams/architecture/remote-http-client.mmd new file mode 100644 index 00000000000..ccd7b3c92f1 --- /dev/null +++ b/docs/diagrams/architecture/remote-http-client.mmd @@ -0,0 +1,7 @@ +flowchart LR + Caller["Distributed Querying or Config Module"] --> Client["Client"] + Client --> Request["HTTP_Request"] + Client --> Response["HTTP_Response"] + Client --> Asio["Boost.Asio io_context"] + Client --> TLS["OpenSSL TLS Layer"] + Client --> Beast["Boost.Beast HTTP Engine"] diff --git a/docs/diagrams/architecture/sql-core-and-virtual-tables-2.mmd b/docs/diagrams/architecture/sql-core-and-virtual-tables-2.mmd new file mode 100644 index 00000000000..d8ccd8f94e7 --- /dev/null +++ b/docs/diagrams/architecture/sql-core-and-virtual-tables-2.mmd @@ -0,0 +1,10 @@ +flowchart LR + SQLite["SQLite Engine"] --> xBestIndex["xBestIndex"] + SQLite --> xFilter["xFilter"] + SQLite --> xNext["xNext"] + SQLite --> xColumn["xColumn"] + + xBestIndex --> ConstraintMap["ConstraintMap"] + xFilter --> QueryContextVT["QueryContext"] + QueryContextVT --> TablePluginImpl["TablePlugin::generate"] + TablePluginImpl --> Rows["TableRows"] diff --git a/docs/diagrams/architecture/sql-core-and-virtual-tables-3.mmd b/docs/diagrams/architecture/sql-core-and-virtual-tables-3.mmd new file mode 100644 index 00000000000..f0891e895e8 --- /dev/null +++ b/docs/diagrams/architecture/sql-core-and-virtual-tables-3.mmd @@ -0,0 +1,5 @@ +flowchart TD + Query["SQL Query"] --> Explain["EXPLAIN"] + Explain --> Opcodes["SQLite Opcodes"] + Opcodes --> TypeMap["Opcode to ColumnType Map"] + TypeMap --> Columns["Resolved TableColumns"] diff --git a/docs/diagrams/architecture/sql-core-and-virtual-tables-4.mmd b/docs/diagrams/architecture/sql-core-and-virtual-tables-4.mmd new file mode 100644 index 00000000000..3f4229156a8 --- /dev/null +++ b/docs/diagrams/architecture/sql-core-and-virtual-tables-4.mmd @@ -0,0 +1,5 @@ +flowchart LR + OldResults["Old QueryData"] --> DiffFunc["diff()"] + NewResults["New QueryData"] --> DiffFunc + DiffFunc --> Added["Added Rows"] + DiffFunc --> Removed["Removed Rows"] diff --git a/docs/diagrams/architecture/sql-core-and-virtual-tables-5.mmd b/docs/diagrams/architecture/sql-core-and-virtual-tables-5.mmd new file mode 100644 index 00000000000..4949558c4d8 --- /dev/null +++ b/docs/diagrams/architecture/sql-core-and-virtual-tables-5.mmd @@ -0,0 +1,10 @@ +flowchart TD + Input["Incoming SQL"] --> SQLInternalExec["SQLInternal"] + SQLInternalExec --> DBConn["SQLiteDBManager::get()"] + DBConn --> Prepare["sqlite3_prepare_v2"] + Prepare --> Authorizer["sqliteAuthorizer"] + Prepare --> VTableCallbacks["Virtual Table Callbacks"] + VTableCallbacks --> TableGen["TablePlugin::generate"] + TableGen --> Results["QueryDataTyped"] + Results --> PostProcess["Escape and Size Calculation"] + PostProcess --> Output["Final Results"] diff --git a/docs/diagrams/architecture/sql-core-and-virtual-tables.mmd b/docs/diagrams/architecture/sql-core-and-virtual-tables.mmd new file mode 100644 index 00000000000..4845d1c95a1 --- /dev/null +++ b/docs/diagrams/architecture/sql-core-and-virtual-tables.mmd @@ -0,0 +1,12 @@ +flowchart TD + UserQuery["SQL Query"] --> SQLInternal["SQLInternal"] + SQLInternal --> SQLiteSQLPlugin["SQLiteSQLPlugin"] + SQLiteSQLPlugin --> DBManager["SQLiteDBManager"] + DBManager --> SQLiteDB["sqlite3 Database"] + + SQLiteDB --> VTableModule["sqlite3_module"] + VTableModule --> VirtualTable["VirtualTable"] + VirtualTable --> TablePlugin["TablePlugin"] + + TablePlugin --> QueryContext["QueryContext"] + QueryContext --> ConstraintList["ConstraintList"] diff --git a/docs/diagrams/architecture/system-utilities-2.mmd b/docs/diagrams/architecture/system-utilities-2.mmd new file mode 100644 index 00000000000..fe080c8879a --- /dev/null +++ b/docs/diagrams/architecture/system-utilities-2.mmd @@ -0,0 +1,11 @@ +flowchart TD + Now["System Clock"] --> GetUnix["getUnixTime()"] + GetUnix --> Scheduler["Scheduled Queries"] + GetUnix --> Distributed["Distributed Expiration"] + GetUnix --> Events["Event Expiration"] + + Scheduler --> ToAscii["toAsciiTime()"] + Distributed --> ToAscii + Events --> ToAscii + + ToAscii --> Logs["Query and Status Logs"] diff --git a/docs/diagrams/architecture/system-utilities.mmd b/docs/diagrams/architecture/system-utilities.mmd new file mode 100644 index 00000000000..6ded1ee7cb1 --- /dev/null +++ b/docs/diagrams/architecture/system-utilities.mmd @@ -0,0 +1,22 @@ +flowchart TD + SystemUtilities["System Utilities"] + + EnumHash["EnumClassHash"] + TimeUtils["Time Utilities"] + + SQLCore["SQL Core and Virtual Tables"] + Scheduler["Scheduled Query Engine"] + Distributed["Distributed Querying"] + Logging["Query Execution and Logging"] + Events["Events Core"] + + SystemUtilities --> EnumHash + SystemUtilities --> TimeUtils + + TimeUtils --> Scheduler + TimeUtils --> Logging + TimeUtils --> Distributed + TimeUtils --> Events + + EnumHash --> SQLCore + EnumHash --> Scheduler diff --git a/docs/reference/architecture/.gitignore b/docs/reference/architecture/.gitignore new file mode 100644 index 00000000000..b4f4c7a6c9c --- /dev/null +++ b/docs/reference/architecture/.gitignore @@ -0,0 +1,8 @@ +# CodeWiki temp files (dependency graphs can be 7GB+) +temp/ +dependency_graphs/ + +# JSON intermediate files (except schema/config) +*.json +!*-schema.json +!*-config.json diff --git a/docs/reference/architecture/README.md b/docs/reference/architecture/README.md new file mode 100644 index 00000000000..303e994cc00 --- /dev/null +++ b/docs/reference/architecture/README.md @@ -0,0 +1,415 @@ +# osquery Repository Overview + +## Purpose + +**osquery** is a cross-platform operating system instrumentation framework that exposes system state as relational tables and allows it to be queried using SQL. + +Instead of writing platform-specific scripts to inspect processes, files, users, network state, or kernel events, osquery provides: + +- A secure embedded SQLite engine +- Virtual tables backed by OS-specific data collectors +- A scheduler for continuous monitoring +- A distributed querying system +- A plugin-based architecture for extensibility +- Event-driven data collection with persistence + +At its core, osquery turns the operating system into a structured, queryable database. + +--- + +# End-to-End Architecture + +At runtime, osquery operates as a layered system composed of orchestration, execution, storage, plugin, networking, and event subsystems. + +## High-Level System Architecture + +```mermaid +flowchart TD + CLI["osqueryi or osqueryd"] --> Core["Core Init And Runtime"] + Core --> Config["Config And Packs"] + Core --> Registry["Plugin Registry"] + Core --> Database["Database Backends"] + Core --> SQL["SQL Core And Virtual Tables"] + Core --> Events["Events Core"] + Core --> Distributed["Distributed Querying"] + Core --> Logger["Plugin Interfaces And Logging"] + Core --> Extensions["Extensions Framework"] + Core --> HTTP["Remote Http Client"] + + Config --> SQL + SQL --> QueryExec["Query Execution And Logging"] + QueryExec --> Logger + Events --> SQL + Distributed --> SQL + Distributed --> HTTP +``` + +### Execution Modes + +- **Daemon mode (`osqueryd`)** – scheduled and distributed queries +- **Interactive shell (`osqueryi`)** – ad-hoc SQL execution +- **Extension processes** – dynamic plugin injection +- **Watcher/worker model** – supervised execution + +--- + +# Core Runtime Spine + +## 1. Core Init And Runtime (`osquery/core`) + +This is the orchestration engine that: + +- Parses flags (`FlagDetail`, `FlagInfo`) +- Determines process mode +- Initializes plugins and registries +- Initializes database backends +- Loads configuration +- Starts event loops +- Supervises workers and extensions +- Coordinates graceful shutdown (`ShutdownData`, `AlarmRunnable`) + +### Runtime Boot Flow + +```mermaid +flowchart TD + Start["Main Entry"] --> Flags["Parse Flags"] + Flags --> Mode["Determine Mode"] + Mode --> RegistryInit["Initialize Registry"] + RegistryInit --> DBInit["Initialize Database"] + DBInit --> ConfigLoad["Load Configuration"] + ConfigLoad --> Plugins["Activate Plugins"] + Plugins --> EventsAttach["Attach Events"] + EventsAttach --> Dispatcher["Run Dispatcher"] + Dispatcher --> Shutdown["Graceful Shutdown"] +``` + +This module defines the lifecycle contract for the entire system. + +--- + +# SQL Execution Layer + +## 2. SQL Core And Virtual Tables (`osquery/sql`) + +Implements: + +- Embedded SQLite engine +- Virtual table integration (`sqlite3_module`) +- Constraint pushdown (`Constraint`, `ConstraintList`) +- Query context (`QueryContext`) +- Diffing (`DiffResults`) +- Scheduled query metadata (`ScheduledQuery`) +- Performance tracking (`QueryPerformance`) + +### SQL Execution Flow + +```mermaid +flowchart TD + Query["Incoming SQL"] --> SQLite["SQLite Engine"] + SQLite --> VTable["Virtual Table Adapter"] + VTable --> TablePlugin["TablePlugin::generate"] + TablePlugin --> Rows["Row Results"] + Rows --> Diff["DiffResults"] + Diff --> Output["QueryDataTyped"] +``` + +Security is enforced via SQLite authorizers and PRAGMA allowlists. + +--- + +# Query State and Logging + +## 3. Query Execution And Logging (`osquery/core`) + +Bridges SQL execution with persistence and logging. + +Key components: + +- `QueryLogItem` +- `Query` (stateful diff engine) +- Epoch and counter management +- Snapshot vs differential logging + +### Differential Logging Model + +```mermaid +flowchart TD + New["New Results"] --> DiffEngine["Diff Engine"] + Old["Previous Results"] --> DiffEngine + DiffEngine --> Added["Added Rows"] + DiffEngine --> Removed["Removed Rows"] + Added --> LogItem["QueryLogItem"] + Removed --> LogItem + LogItem --> Logger["Logger Plugins"] +``` + +Ensures efficient change tracking and structured log emission. + +--- + +# Configuration Control Plane + +## 4. Config And Packs (`osquery/config`) + +Transforms JSON configuration into runtime behavior: + +- Scheduled queries +- Packs +- Discovery queries +- Denylisting +- Performance persistence + +Core components: + +- `ConfigRefreshRunner` +- `Schedule::Step` +- `PackStats` + +### Config Lifecycle + +```mermaid +flowchart TD + Plugin["ConfigPlugin"] --> Validate["Validate JSON"] + Validate --> Packs["Build Packs"] + Packs --> Schedule["Update Schedule"] + Schedule --> SQL["Scheduled Execution"] +``` + +Provides deterministic and safe configuration refresh. + +--- + +# Plugin System + +## 5. Plugin Interfaces And Logging (`osquery/core/plugins`) + +Defines `LoggerPlugin` and `StatusLogLine`. + +Supports: + +- Structured status logs +- Snapshot and diff results +- Event forwarding +- Feature negotiation (`usesLogStatus`, `usesLogEvent`) + +```mermaid +flowchart LR + Core["Core Runtime"] --> LoggerInterface["LoggerPlugin"] + SQL["SQL Engine"] --> LoggerInterface + Events["Events"] --> LoggerInterface + LoggerInterface --> Backend["External Sink"] +``` + +--- + +# Persistent Storage Layer + +## 6. Database Backends (`osquery/database`) + +Provides: + +- Pluggable storage (RocksDB, Ephemeral) +- Domain-based keyspaces +- Migration support +- Thread-safe reset + +Core components: + +- `OsqueryDatabase` +- `EphemeralDatabasePlugin` + +```mermaid +flowchart TD + Core --> API["OsqueryDatabase"] + API --> Plugin["Active DB Plugin"] + Plugin --> RocksDB["Persistent Storage"] + Plugin --> Memory["In-Memory Backend"] +``` + +Used by configuration, events, distributed queries, and performance tracking. + +--- + +# Event Framework + +## 7. Events Core (`osquery/events`) + +Implements a publish/subscribe system: + +- `EventFactory` +- `EventPublisherPlugin` +- `EventSubscriberPlugin` +- `Subscription` + +```mermaid +flowchart TD + Publisher --> Subscriber + Subscriber --> Storage["Event Backing Store"] + Storage --> SQL["Virtual Table"] +``` + +Aligns expiration with scheduled query intervals. + +--- + +# Distributed Querying + +## 8. Distributed Querying (`osquery/distributed`) + +Enables remote orchestration. + +Core structures: + +- `DistributedQueryRequest` +- `DistributedQueryResult` +- `TLSDistributedPlugin` + +```mermaid +flowchart TD + Server["Control Plane"] --> TLS["TLS Plugin"] + TLS --> Engine["Distributed Engine"] + Engine --> SQL["Execute SQL"] + SQL --> Engine + Engine --> TLS + TLS --> Server +``` + +Includes denylisting, concurrency control, and performance tracking. + +--- + +# Extensions Framework + +## 9. Extensions Framework (`osquery/extensions`) + +Enables runtime plugin injection via Thrift IPC. + +Key components: + +- `ExtensionInfo` +- `ExtensionManagerWatcher` +- `ImplExtensionClient` +- `ImplExtensionRunner` + +```mermaid +flowchart LR + Core["Extension Manager"] --> ExtensionA["Extension Process"] + ExtensionA --> Registry["Registry Routes"] + Core --> ExtensionA +``` + +Supports dynamic virtual tables, loggers, config plugins, and distributed transports. + +--- + +# Remote HTTP Layer + +## 10. Remote Http Client (`osquery/remote`) + +Implements: + +- Boost.Asio networking +- TLS with OpenSSL +- Timeout handling +- Proxy support + +Used by: + +- Distributed querying +- TLS logging +- Remote configuration retrieval + +--- + +# Filesystem, Hashing, and System Utilities + +## Filesystem And Fileops (`osquery/filesystem`) +- Cross-platform file abstraction (`PlatformFile`) +- Stat normalization (`WINDOWS_STAT`) +- Globbing +- Socket lifecycle management + +## Hashing (`osquery/hashing`) +- Streaming hash computation +- Multi-algorithm support (`MultiHashes`) +- File and buffer hashing + +## Process And Profiler (`osquery/process`, `osquery/profiler`) +- Worker and extension spawning +- POSIX signal handling +- `CodeProfiler` using `getrusage` + +## System Utilities (`osquery/utils`) +- `EnumClassHash` +- Time conversion utilities + +--- + +# Complete System Data Flow + +```mermaid +flowchart TD + Config["Configuration"] --> Scheduler["Scheduler"] + Scheduler --> SQL["SQL Engine"] + SQL --> Diff["Diff Engine"] + Diff --> Persist["Database"] + Diff --> Log["Logger"] + Events["Event Publishers"] --> Subscribers["Event Subscribers"] + Subscribers --> Persist + Distributed["Remote Queries"] --> SQL + SQL --> Distributed + Extensions["Extensions"] --> Registry["Plugin Registry"] + Registry --> SQL +``` + +--- + +# Repository Core Module References + +| Module | Path | Responsibility | +|--------|------|---------------| +| Core Init And Runtime | `osquery/core` | Lifecycle orchestration | +| Config And Packs | `osquery/config` | Config loading & scheduling | +| Plugin Interfaces And Logging | `osquery/core/plugins` | Logging abstraction | +| Database Backends | `osquery/database` | Persistent state | +| SQL Core And Virtual Tables | `osquery/sql` | Embedded SQL engine | +| Query Execution And Logging | `osquery/core` | Diff & log pipeline | +| Events Core | `osquery/events` | Publish/subscribe events | +| Extensions Framework | `osquery/extensions` | Runtime plugin injection | +| Distributed Querying | `osquery/distributed` | Remote query orchestration | +| Remote Http Client | `osquery/remote` | HTTP/TLS transport | +| Filesystem And Fileops | `osquery/filesystem` | Cross-platform FS layer | +| Hashing | `osquery/hashing` | Cryptographic hashing | +| Process And Profiler | `osquery/process` | Process isolation & profiling | +| System Utilities | `osquery/utils` | Enum hashing & time utilities | +| Config Plugins | `plugins/config` | Retrieval and parsing plugins | + +--- + +# Summary + +The **osquery repository** implements a secure, extensible, SQL-driven operating system instrumentation platform. + +It provides: + +- A hardened embedded SQLite execution engine +- Virtual tables mapping OS state into relational form +- Stateful diff-based logging +- Persistent event storage +- Distributed query orchestration +- Pluggable configuration and logging +- Runtime extension injection +- Secure TLS networking +- Process isolation and profiling + +Architecturally, osquery is built around a **plugin registry and lifecycle-managed runtime**, with SQL as the central abstraction layer and configuration as the control plane. + +It is designed for: + +- Security observability +- Continuous monitoring +- Fleet-wide distributed querying +- Extensibility without recompilation +- Safe, deterministic runtime behavior + +osquery transforms low-level OS primitives into a structured, distributed, SQL-powered telemetry system. \ No newline at end of file diff --git a/docs/reference/architecture/config-and-packs/config-and-packs.md b/docs/reference/architecture/config-and-packs/config-and-packs.md new file mode 100644 index 00000000000..95d93fa1724 --- /dev/null +++ b/docs/reference/architecture/config-and-packs/config-and-packs.md @@ -0,0 +1,389 @@ +# Config And Packs + +The **Config And Packs** module is responsible for loading, validating, parsing, refreshing, and operationalizing osquery configuration data. It transforms raw configuration sources (filesystem, TLS, or other config plugins) into: + +- Scheduled queries +- Query packs +- File path monitors +- Event subscriptions +- Runtime options +- Performance tracking metadata + +This module acts as the *control plane* for query scheduling and runtime behavior. It coordinates with plugins, the database layer, SQL execution, logging, and event systems to ensure configuration changes are safely and dynamically applied. + +--- + +## 1. Architectural Overview + +At a high level, Config And Packs sits between **configuration sources** and **runtime execution systems**. + +```mermaid +flowchart TD + ConfigPlugin["Config Plugin"] --> ConfigCore["Config And Packs"] + ConfigCore --> Schedule["Schedule & Packs"] + ConfigCore --> Parsers["Config Parser Plugins"] + Schedule --> SQLCore["SQL Core & Scheduled Queries"] + SQLCore --> Logging["Query Execution & Logging"] + ConfigCore --> Database["Database Backends"] + ConfigCore --> Events["Events Core"] +``` + +### Key Responsibilities + +1. Retrieve configuration via `ConfigPlugin` +2. Validate and parse JSON configuration safely +3. Build and manage `Pack` objects +4. Maintain a runtime `Schedule` +5. Track query performance and failures +6. Persist state in the database +7. Periodically refresh configuration + +--- + +## 2. Core Components + +This module is primarily implemented across: + +- `ConfigRefreshRunner` +- `Schedule::Step` +- `PackStats` + +Supporting classes include: + +- `Config` +- `Schedule` +- `Pack` +- `ConfigPlugin` +- `ConfigParserPlugin` + +--- + +## 3. Configuration Lifecycle + +The configuration lifecycle follows a deterministic pipeline. + +```mermaid +flowchart TD + Start["Config Load Triggered"] --> Gen["ConfigPlugin.genConfig()"] + Gen --> Validate["Validate JSON Structure"] + Validate --> Update["Update Sources"] + Update --> Packs["Create / Update Packs"] + Packs --> Parsers["Apply Config Parsers"] + Parsers --> Reconfigure["Reconfigure Registries"] + Reconfigure --> Ready["Schedule Active"] +``` + +### 3.1 Loading + +`Config::load()`: + +- Ensures an active config plugin exists +- Sets refresh interval +- Starts `ConfigRefreshRunner` if periodic refresh is enabled +- Performs an initial `refresh()` + +### 3.2 Refreshing + +`ConfigRefreshRunner` runs in a background thread: + +```mermaid +flowchart TD + LoopStart["Wait refresh_sec seconds"] --> RefreshCall["Config.refresh()"] + RefreshCall --> LoopStart +``` + +Behavioral details: + +- Uses accelerated retry interval on failure +- Restores normal interval when config loads successfully +- Supports config backup restore on first failure + +--- + +## 4. JSON Validation & Safety Controls + +To prevent resource exhaustion and malformed configuration: + +- Maximum JSON depth enforced (`kMaxConfigDepth`) +- Maximum size enforced (`kMaxConfigSize`) +- Comment stripping before parsing +- Iterative parsing mode +- Structured validation via `validateConfig()` + +This ensures hostile or malformed config cannot destabilize the system. + +--- + +## 5. Packs and Scheduling + +### 5.1 Pack + +A **Pack** represents a group of scheduled queries and execution constraints. + +Each Pack contains: + +- Discovery queries +- Scheduled queries +- Platform constraints +- Version constraints +- Shard constraints +- Execution state +- Discovery statistics (`PackStats`) + +```mermaid +flowchart TD + Pack["Pack"] --> Discovery["Discovery Queries"] + Pack --> Schedule["Scheduled Queries"] + Pack --> Constraints["Platform / Version / Shard"] + Pack --> Stats["PackStats"] +``` + +### 5.2 Discovery Logic + +A Pack executes only if: + +- Platform matches +- Version matches +- Shard matches +- Discovery queries succeed + +Discovery results are cached to avoid excessive execution. + +### 5.3 PackStats + +`PackStats` tracks discovery efficiency: + +- `total` – total discovery attempts +- `hits` – successful discovery validations +- `misses` – failed discovery attempts + +This supports introspection and debugging of conditional pack activation. + +--- + +## 6. Schedule Management + +The `Schedule` class maintains active packs. + +It: + +- Stores Pack instances +- Filters active packs via `Step` +- Maintains denylisted queries +- Tracks previously executing queries + +```mermaid +flowchart TD + Schedule["Schedule"] --> Packs["Pack List"] + Schedule --> Denylist["Denylisted Queries"] + Schedule --> Executing["Executing Query State"] +``` + +### 6.1 Denylisting + +If a query causes a watchdog failure: + +- It is added to a denylist +- Persisted in the database +- Skipped until expiration + +Denylist entries may expire based on time or configuration override. + +### 6.2 Iteration via Step + +`Schedule::Step` determines if a pack should execute: + +```mermaid +flowchart TD + Step["Schedule::Step"] --> Check["pack.shouldPackExecute()"] + Check -->|"true"| Include["Included in Iteration"] + Check -->|"false"| Skip["Skipped"] +``` + +This ensures inactive packs never contribute scheduled queries. + +--- + +## 7. Query Performance Tracking + +The module tracks execution metrics per scheduled query: + +- Wall time +- CPU user/system time +- Memory usage +- Output size +- Execution count +- Last execution timestamp + +Metrics are persisted in the database and updated via: + +- `recordQueryStart()` +- `recordQueryPerformance()` + +When a query definition changes, performance stats are cleared to avoid mixing incompatible metrics. + +--- + +## 8. Parser Plugin System + +Config parser plugins extend configuration behavior. + +Examples of responsibilities: + +- Parse options +- Parse file paths +- Parse event configuration +- Parse decorators +- Parse views + +```mermaid +flowchart TD + ConfigCore["Config And Packs"] --> Options["Options Parser"] + ConfigCore --> FilePaths["File Paths Parser"] + ConfigCore --> EventsParser["Events Parser"] + ConfigCore --> OtherParsers["Other Parsers"] +``` + +Parsers: + +- Receive filtered JSON sections +- Maintain internal state +- May trigger registry reconfiguration +- Are reset during `Config::reset()` + +The options parser is always applied first since other parsers may depend on flag values. + +--- + +## 9. Database Interaction + +The module heavily interacts with persistent storage for: + +- Query performance stats +- Denylist entries +- Executing query tracking +- Cached splay intervals +- Config backups +- Result expiration + +```mermaid +flowchart TD + ConfigCore["Config And Packs"] --> DB["Persistent Database"] + DB --> Performance["Query Performance"] + DB --> Denylist["Failed Queries"] + DB --> Executing["Executing Query"] + DB --> Backup["Config Backup"] +``` + +### 9.1 Backup & Restore + +If `config_enable_backup` is enabled: + +- Each config update is persisted +- On failure, backup can be restored +- Only triggered on first failed refresh + +--- + +## 10. Hashing and Change Detection + +Each config source is hashed (SHA1). + +If the content hash is unchanged: + +- The update is skipped +- No reconfiguration occurs + +A global configuration hash can be generated by combining per-source hashes. + +This prevents unnecessary reconfiguration cascades. + +--- + +## 11. Purge and Expiration Logic + +`Config::purge()` ensures stale data does not accumulate. + +It: + +- Removes result sets for deleted queries +- Expires results older than one week +- Cleans corrupted timestamps + +This keeps persistent storage bounded and consistent. + +--- + +## 12. Extension-Aware Behavior + +When running inside extensions: + +- Config updates are proxied to core +- External processes do not restore schedule state +- Updates are synchronized through the registry system + +This ensures a single authoritative configuration instance. + +--- + +## 13. Concurrency and Thread Safety + +The module uses: + +- `Mutex` +- `RecursiveMutex` +- `WriteLock` +- `std::atomic` + +Protected areas include: + +- Schedule mutation +- Performance stats +- Config hashing +- Backup operations + +```mermaid +flowchart TD + ConfigCore["Config And Packs"] --> Mutex1["Schedule Mutex"] + ConfigCore --> Mutex2["Files Mutex"] + ConfigCore --> Mutex3["Performance Mutex"] + ConfigCore --> Mutex4["Hash Mutex"] +``` + +This prevents race conditions between: + +- Refresh thread +- Query execution threads +- Extension updates +- Event subscribers + +--- + +## 14. Integration with Other Modules + +Config And Packs coordinates with: + +- SQL scheduling and execution (scheduled queries) +- Database backends (state persistence) +- Event system (event publisher reconfiguration) +- Logger plugins (reconfiguration after update) +- Distributed querying (through config-delivered queries) + +It is the authoritative runtime configuration authority. + +--- + +# Summary + +The **Config And Packs** module transforms raw configuration data into a structured, validated, dynamic execution plan. + +It ensures: + +- Safe parsing +- Deterministic scheduling +- Runtime adaptability +- Performance visibility +- Fault isolation via denylisting +- Efficient change detection + +By bridging configuration sources with scheduling, persistence, and runtime systems, Config And Packs acts as the central orchestration layer of osquery’s operational behavior. \ No newline at end of file diff --git a/docs/reference/architecture/config-plugins/config-plugins.md b/docs/reference/architecture/config-plugins/config-plugins.md new file mode 100644 index 00000000000..139f4d912a9 --- /dev/null +++ b/docs/reference/architecture/config-plugins/config-plugins.md @@ -0,0 +1,369 @@ +# Config Plugins + +The **Config Plugins** module is responsible for retrieving, parsing, transforming, and applying configuration data to the osquery runtime. It acts as the bridge between raw configuration sources (such as files) and the internal runtime subsystems (scheduler, SQL engine, logger, database, events framework, and extensions). + +This module implements: + +- Configuration retrieval via `ConfigPlugin` implementations +- Structured parsing via `ConfigParserPlugin` implementations +- Runtime mutation of flags and options +- View and decorator execution +- File path monitoring configuration +- Controlled config updates from extensions + +It integrates closely with: + +- Core configuration framework (`Config`, `ConfigPlugin`, `ConfigParserPlugin`) +- SQL engine and virtual tables +- Logger subsystem +- Database backends +- Events framework +- Extensions framework + +--- + +## Architecture Overview + +The Config Plugins module follows a two-phase pipeline: + +1. **Configuration Retrieval** β€” A `ConfigPlugin` fetches raw configuration content. +2. **Configuration Parsing** β€” Multiple `ConfigParserPlugin` implementations interpret specific keys and mutate runtime state. + +```mermaid +flowchart TD + ConfigSource["Configuration Source"] --> FilesystemPlugin["FilesystemConfigPlugin"] + FilesystemPlugin --> CoreConfig["Config Core"] + + CoreConfig --> DecoratorsParser["DecoratorsConfigParserPlugin"] + CoreConfig --> EventsParser["EventsConfigParserPlugin"] + CoreConfig --> FilePathsParser["FilePathsConfigParserPlugin"] + CoreConfig --> OptionsParser["OptionsConfigParserPlugin"] + CoreConfig --> ViewsParser["ViewsConfigParserPlugin"] + + DecoratorsParser --> LoggerSubsystem["Logger Subsystem"] + FilePathsParser --> EventsFramework["Events Framework"] + ViewsParser --> SQLCore["SQL Core"] + OptionsParser --> FlagSystem["Flag System"] + + ExtensionFramework["Extensions Framework"] --> UpdatePlugin["UpdateConfigPlugin"] + UpdatePlugin --> CoreConfig +``` + +### Design Principles + +- **Plugin-driven**: Configuration retrieval and parsing are both registry-driven. +- **Key-based parsing**: Each parser owns specific top-level JSON keys. +- **Incremental updates**: Config updates replace only affected sections. +- **Thread-safe decorators and state**: Mutex-protected shared structures. +- **Runtime mutability**: Certain options and views can be modified without restart. + +--- + +# Configuration Retrieval + +## FilesystemConfigPlugin + +**Component:** `osquery.plugins.config.filesystem_config.FilesystemConfigPlugin` + +The Filesystem Config Plugin loads configuration from disk using the `--config_path` flag. + +### Behavior + +- Validates that the primary config file exists +- Loads additional drop-in configuration files from: + +``` +.d/*.conf +``` + +- Sorts drop-in files for deterministic ordering +- Supports pack expansion via `genPack` +- Supports multi-pack loading using wildcard patterns + +### Multi-Pack Assembly + +If the requested pack name is `*`, the plugin: + +1. Resolves a filesystem glob pattern +2. Reads each JSON file +3. Strips comments +4. Merges each pack into a property tree +5. Emits a synthesized JSON pack + +```mermaid +flowchart TD + Start["genConfig()"] --> Validate["Validate config_path"] + Validate --> Resolve["Resolve .d drop-in files"] + Resolve --> Sort["Sort files"] + Sort --> Read["Read each file"] + Read --> Merge["Return config map"] +``` + +### Responsibilities + +- Provide raw configuration payloads +- Support modular configuration layouts +- Provide pack materialization + +--- + +# Config Parser Plugins + +Each parser is registered in the `config_parser` registry and declares ownership of one or more top-level configuration keys. + +## DecoratorsConfigParserPlugin + +**Component:** `osquery.plugins.config.parsers.decorators.DecoratorsConfigParserPlugin` + +Handles the `decorators` key. + +### Purpose + +Decorators append additional columns to: + +- Query results +- Snapshots +- Status logs + +### Decorator Types + +- `load` β€” Run once at config load +- `always` β€” Run before every query +- `interval` β€” Run at configured intervals + +### Execution Model + +Decorators are SQL queries. Only the first row is used. Multiple rows or duplicate column names result in undefined behavior. + +```mermaid +flowchart TD + ConfigLoad["Config Update"] --> ParseDecorators["Parse decorators key"] + ParseDecorators --> StoreQueries["Store queries by type"] + StoreQueries --> RunLoad["Run load decorators"] + + ScheduledTick["Scheduler Tick"] --> RunInterval["Run interval decorators"] + QueryExecution["Before Query"] --> RunAlways["Run always decorators"] + + RunLoad --> DecorationStore["Global Decoration Store"] + RunAlways --> DecorationStore + RunInterval --> DecorationStore +``` + +### Thread Safety + +- Uses mutex-protected global store +- Separates configuration data lock from runtime decoration lock + +### Integration + +- Uses SQL engine to execute decorator queries +- Injects key-value pairs into log items + +--- + +## EventsConfigParserPlugin + +**Component:** `osquery.plugins.config.parsers.events_parser.EventsConfigParserPlugin` + +Handles the `events` key. + +### Purpose + +- Copies event-related configuration into internal JSON storage +- Makes event configuration accessible to the events subsystem + +### Characteristics + +- Minimal transformation +- Purely structural +- Delegates behavior to the events framework + +--- + +## FilePathsConfigParserPlugin + +**Component:** `osquery.plugins.config.parsers.file_paths.FilePathsConfigParserPlugin` + +Handles: + +- `file_paths` +- `file_paths_query` +- `file_accesses` +- `exclude_paths` + +### Responsibilities + +1. Register filesystem patterns for monitoring +2. Execute SQL queries that dynamically produce paths +3. Track categories of file access monitoring +4. Merge exclusion patterns + +```mermaid +flowchart TD + ConfigUpdate["Config Update"] --> RemoveOld["Remove old source paths"] + RemoveOld --> ParseStatic["Parse file_paths"] + RemoveOld --> ParseQuery["Parse file_paths_query"] + ParseQuery --> ExecuteSQL["Execute SQL query"] + ExecuteSQL --> ExtractPath["Extract path column"] + ParseStatic --> RegisterFile["Config::addFile()"] + ExtractPath --> RegisterFile + RegisterFile --> EventsFramework["Events Framework"] +``` + +### Key Features + +- Glob wildcard normalization +- SQL-driven dynamic path resolution +- Category-based access control +- Persistent merging of exclusion rules + +### Integration Points + +- SQL subsystem +- Events framework +- Filesystem utilities + +--- + +## OptionsConfigParserPlugin + +**Component:** `osquery.plugins.config.parsers.options.OptionsConfigParserPlugin` + +Handles the `options` key. + +### Purpose + +Applies runtime flag updates from configuration. + +### Processing Steps + +1. Merge previous and new `options` blocks +2. Validate flag existence +3. Prevent CLI-only flags from being overridden +4. Convert JSON values to string representation +5. Update flags via `Flag::updateValue` +6. Adjust logger verbosity if required + +```mermaid +flowchart TD + ConfigOptions["options block"] --> ValidateFlag["Validate flag name"] + ValidateFlag --> CheckCLI["Check CLI-only restriction"] + CheckCLI --> UpdateFlag["Flag::updateValue"] + UpdateFlag --> AdjustLogging["setVerboseLevel() if needed"] +``` + +### Safety Controls + +- Rejects unknown flags +- Rejects CLI-only flags +- Supports custom flags with `custom_` prefix + +--- + +## ViewsConfigParserPlugin + +**Component:** `osquery.plugins.config.parsers.views.ViewsConfigParserPlugin` + +Handles the `views` key. + +### Purpose + +Dynamically creates and deletes SQL views based on configuration. + +### Behavior + +- Drops outdated views +- Creates new views using SQL +- Stores view definitions in the database +- Avoids redundant recreation except on first load + +```mermaid +flowchart TD + ConfigViews["views block"] --> ScanDB["Scan existing views"] + ScanDB --> Compare["Compare with new config"] + Compare --> CreateView["CREATE VIEW"] + Compare --> DropView["DROP VIEW"] + CreateView --> Persist["Store in database"] + DropView --> Persist +``` + +### Integration + +- SQL execution engine +- Persistent database backend + +--- + +# UpdateConfigPlugin + +**Component:** `osquery.plugins.config.update.UpdateConfigPlugin` + +This is a special-purpose `ConfigPlugin` named `update`. + +### Purpose + +Allows extension plugins to trigger configuration updates in the core process. + +### Behavior + +- Does not generate configuration +- Serves as a routing mechanism for extension-driven updates +- Enables asynchronous config mutation + +```mermaid +sequenceDiagram + participant Extension + participant Registry + participant CoreConfig + + Extension->>Registry: Config::update() + Registry->>CoreConfig: Route update request + CoreConfig->>CoreConfig: Re-run parser pipeline +``` + +### Importance + +This enables distributed and extension-based configuration control without embedding update logic directly into the core. + +--- + +# End-to-End Configuration Flow + +```mermaid +flowchart TD + Source["Filesystem or Extension"] --> Retrieve["ConfigPlugin::genConfig"] + Retrieve --> Core["Config Core"] + Core --> Parse["ConfigParserPlugin::update"] + + Parse --> Flags["Runtime Flags"] + Parse --> Views["SQL Views"] + Parse --> Decorators["Log Decorations"] + Parse --> FilePaths["Filesystem Monitoring"] + Parse --> Events["Event Subscriptions"] +``` + +--- + +# Key Responsibilities Summary + +| Area | Plugin | Runtime Impact | +|------|--------|---------------| +| Retrieval | FilesystemConfigPlugin | Loads JSON configuration | +| Decoration | DecoratorsConfigParserPlugin | Augments logs and results | +| Events | EventsConfigParserPlugin | Supplies event configuration | +| File Monitoring | FilePathsConfigParserPlugin | Registers file paths and exclusions | +| Runtime Flags | OptionsConfigParserPlugin | Updates flags and logging | +| SQL Views | ViewsConfigParserPlugin | Creates/drops SQL views | +| Extension Updates | UpdateConfigPlugin | Enables async config mutation | + +--- + +# Operational Characteristics + +- Fully registry-based plugin model +- Supports dynamic reconfiguration +- Maintains thread safety for shared decorator state +- Integrates deeply with SQL and logging subsystems +- Enables modular configuration layouts + +The Config Plugins module is a central orchestration layer that translates configuration documents into concrete runtime behavior across the entire osquery system. \ No newline at end of file diff --git a/docs/reference/architecture/core-init-and-runtime/core-init-and-runtime.md b/docs/reference/architecture/core-init-and-runtime/core-init-and-runtime.md new file mode 100644 index 00000000000..8decd3f03f9 --- /dev/null +++ b/docs/reference/architecture/core-init-and-runtime/core-init-and-runtime.md @@ -0,0 +1,380 @@ +# Core Init And Runtime + +The **Core Init And Runtime** module is responsible for bootstrapping, configuring, running, and gracefully shutting down the osquery process. It orchestrates flag parsing, plugin activation, database initialization, event lifecycle management, extension startup, and coordinated shutdown. + +This module is the entry point for: + +- Daemon mode (`osqueryd`) +- Interactive shell mode (`osqueryi`) +- Extension processes +- Watcher and worker subprocesses +- OpenFrame-integrated runtime (when enabled) + +It provides the runtime spine that connects configuration, logging, database backends, SQL execution, distributed querying, events, and extensions. + +--- + +## 1. Architectural Overview + +At a high level, Core Init And Runtime performs the following sequence: + +1. Parse and register flags +2. Determine tool mode (daemon, shell, extension) +3. Initialize registries and plugins +4. Initialize database and upgrade schema +5. Load configuration +6. Activate logger and distributed plugins +7. Start event loops and services +8. Block until shutdown request +9. Perform graceful teardown + +### High-Level Runtime Flow + +```mermaid +flowchart TD + Main["Main Entry"] --> Initializer["Initializer"] + Initializer --> Flags["Flag Registration"] + Flags --> Mode["Determine Tool Type"] + Mode --> RegistryInit["Registry And Plugin Init"] + RegistryInit --> DatabaseInit["Database Init And Upgrade"] + DatabaseInit --> ExtensionMgr["Extension Manager"] + ExtensionMgr --> ConfigLoad["Config Load"] + ConfigLoad --> LoggerInit["Logger Plugin Init"] + LoggerInit --> DistributedInit["Distributed Plugin Init"] + DistributedInit --> Events["Attach Events"] + Events --> RuntimeLoop["Dispatcher Services"] + RuntimeLoop --> ShutdownReq["Shutdown Requested?"] + ShutdownReq -->|"Yes"| GracefulShutdown["Graceful Shutdown"] + GracefulShutdown --> End["Process Exit"] +``` + +The Core Init And Runtime module interacts closely with: + +- [Config And Packs](../config-and-packs/config-and-packs.md) +- [Plugin Interfaces And Logging](../plugin-interfaces-and-logging/plugin-interfaces-and-logging.md) +- [Database Backends](../database-backends/database-backends.md) +- [SQL Core And Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) +- [Extensions Framework](../extensions-framework/extensions-framework.md) +- [Events Core](../events-core/events-core.md) +- [Distributed Querying](../distributed-querying/distributed-querying.md) + +This module does not implement business logic for these subsystems, but activates and coordinates them. + +--- + +## 2. Core Components + +The Core Init And Runtime module is primarily composed of: + +- **FlagDetail** and **FlagInfo** – metadata for CLI and configuration flags +- **AlarmRunnable** – forced shutdown watchdog thread +- **ShutdownData** – global shutdown coordination state +- **Initializer** – the orchestration engine (defined across init and shutdown logic) + +--- + +## 3. Flag System + +**Core Components:** +- `osquery.osquery.core.flags.FlagDetail` +- `osquery.osquery.core.flags.FlagInfo` + +The flag system wraps Google GFlags to provide osquery-specific metadata and lifecycle control. + +### FlagDetail + +Describes how a flag behaves: + +- `description` – help text +- `shell` – available only in shell mode +- `external` – available only in extensions +- `cli` – CLI-only (cannot be set in config) +- `hidden` – omitted from help output + +### FlagInfo + +Represents a fully materialized flag: + +- Type (string representation) +- Description +- Default value +- Current value +- Associated `FlagDetail` + +### Flag Lifecycle + +Flags are created using wrapper macros such as: + +- `FLAG` +- `CLI_FLAG` +- `SHELL_FLAG` +- `EXTENSION_FLAG` +- `HIDDEN_FLAG` + +Internally: + +1. A GFlags variable is defined. +2. `Flag::create()` registers metadata. +3. Flags become discoverable through `Flag::flags()`. + +### Flag System Architecture + +```mermaid +flowchart LR + Macro["FLAG Macro"] --> GFlags["DEFINE_type"] + Macro --> Create["Flag::create()"] + Create --> Registry["Internal Flag Map"] + Registry --> Help["Custom Help Output"] + Registry --> RuntimeAccess["Flag::getValue()"] +``` + +This abstraction allows Core Init And Runtime to: + +- Dynamically inspect defaults +- Override values programmatically +- Distinguish CLI-only vs config flags +- Generate structured help output + +--- + +## 4. Initializer: Process Bootstrapping + +The **Initializer** class is the central coordinator of runtime setup. + +It determines the process role and activates subsystems accordingly. + +### Tool Types + +Initializer supports multiple execution contexts: + +- Daemon (`osqueryd`) +- Shell (`osqueryi`) +- Extension +- Watcher (supervisor) +- Worker (child process) + +### Mode Decision Flow + +```mermaid +flowchart TD + Start["Initializer Constructor"] --> CheckBinary["Inspect argv[0]"] + CheckBinary -->|"osqueryd"| Daemon["Daemon Mode"] + CheckBinary -->|"osqueryi"| Shell["Shell Mode"] + Shell --> DisableEvents["Disable Events By Default"] + Daemon --> InitWorkDir["Init Work Directories"] +``` + +### Responsibilities + +Initializer performs: + +- Random seed initialization +- Start time registration +- File descriptor limits adjustment (POSIX) +- Default flagfile resolution +- GFlags parsing +- Registry and plugin initialization +- Signal handler installation +- Logger bootstrap +- OpenFrame initialization (if enabled) + +--- + +## 5. OpenFrame Integration + +When `openframe_mode` is enabled: + +1. Encryption service is created +2. Token extractor reads token from configured path +3. Authorization manager is updated +4. Token refresher thread starts + +This integration modifies authentication and runtime identity behavior without affecting core scheduling. + +```mermaid +flowchart TD + FlagCheck["openframe_mode Flag"] -->|"true"| Encryption["Encryption Service"] + Encryption --> Extractor["Token Extractor"] + Extractor --> AuthMgr["Authorization Manager"] + Extractor --> Refresher["Token Refresher Thread"] +``` + +--- + +## 6. Plugin and Registry Activation + +After flag parsing and base initialization: + +1. Extension manager is started +2. Active config plugin is selected +3. Registry lazy setup is executed +4. Logger plugin is activated +5. Distributed plugin is activated +6. Numeric monitoring plugin (optional) is activated + +This ties into: + +- [Config And Packs](../config-and-packs/config-and-packs.md) +- [Plugin Interfaces And Logging](../plugin-interfaces-and-logging/plugin-interfaces-and-logging.md) +- [Distributed Querying](../distributed-querying/distributed-querying.md) + +### Activation Flow + +```mermaid +flowchart TD + ExtensionMgr["Start Extension Manager"] --> ConfigPlugin["Activate Config Plugin"] + ConfigPlugin --> RegistrySetup["Registry::setUp()"] + RegistrySetup --> LoggerPlugin["Activate Logger Plugin"] + LoggerPlugin --> DistributedPlugin["Activate Distributed Plugin"] + DistributedPlugin --> Monitoring["Numeric Monitoring Plugin"] +``` + +--- + +## 7. Database Initialization + +Before configuration loading completes: + +- Database plugin is initialized +- Schema upgrade is validated +- Persistent state becomes available + +This integrates directly with: + +- [Database Backends](../database-backends/database-backends.md) + +If upgrade fails, shutdown is requested immediately. + +--- + +## 8. Event and Service Runtime + +Once configuration and plugins are ready: + +- Event threads are attached +- Dispatcher services begin execution +- Watcher/worker orchestration begins +- Extension sockets are bound + +Core Init And Runtime does not implement event logic directly; it activates the framework defined in: + +- [Events Core](../events-core/events-core.md) + +--- + +## 9. Graceful Shutdown Mechanism + +**Core Component:** +- `osquery.osquery.core.shutdown.ShutdownData` + +Shutdown is globally coordinated through a shared state structure. + +### ShutdownData Fields + +- `requested` – atomic flag indicating shutdown +- `retcode` – exit code +- `request_signal` – condition variable +- `request_mutex` – synchronization primitive + +### Shutdown Lifecycle + +```mermaid +flowchart TD + Signal["SIGTERM or SIGINT"] --> Request["requestShutdown()"] + Request --> FlagSet["requested = true"] + FlagSet --> Notify["Condition Notify"] + Notify --> MainThread["waitForShutdown() Unblocks"] + MainThread --> DispatcherStop["Stop Services"] + DispatcherStop --> Join["Join Threads"] + Join --> EndEvents["EventFactory::end()"] + EndEvents --> DBShutdown["shutdownDatabase()"] + DBShutdown --> Exit["Return Exit Code"] +``` + +### AlarmRunnable + +**Core Component:** +- `osquery.osquery.core.init.AlarmRunnable` + +This is a safety watchdog thread started during shutdown: + +- Sleeps in 200 ms intervals +- Tracks elapsed time +- If shutdown exceeds `alarm_timeout`, calls `shutdownNow()` +- Forces immediate process termination + +This prevents deadlocks during shutdown. + +--- + +## 10. Watcher and Worker Model + +Core Init And Runtime supports a supervised architecture: + +- Watcher process supervises worker +- Worker runs queries and services +- Watcher restarts worker if needed + +```mermaid +flowchart LR + Watcher["Watcher Process"] -->|"Spawns"| Worker["Worker Process"] + Worker -->|"Monitored By"| Watcher + Watcher --> Extensions["Managed Extensions"] +``` + +The watcher disables logging and persistent database access, acting purely as supervisor. + +--- + +## 11. Signal Handling + +Signals registered: + +- `SIGTERM` +- `SIGINT` +- `SIGUSR1` + +Behavior: + +- `SIGTERM` / `SIGINT` β†’ graceful shutdown +- `SIGUSR1` β†’ mark resource limit hit + +Signal handler always funnels into `Initializer::requestShutdown()`. + +--- + +## 12. Interaction with Other Modules + +Core Init And Runtime is not domain-specific; it is orchestration-centric. + +| Subsystem | Role in Lifecycle | +|------------|-------------------| +| Config And Packs | Loads configuration after plugin activation | +| Database Backends | Persistent state and migrations | +| SQL Core And Virtual Tables | Query engine activated after registry setup | +| Plugin Interfaces And Logging | Status and result logging initialization | +| Extensions Framework | Extension manager startup and socket binding | +| Events Core | Event loop attachment and lifecycle | +| Distributed Querying | Distributed plugin activation | + +Core Init And Runtime ensures correct ordering and isolation between these systems. + +--- + +# Conclusion + +The **Core Init And Runtime** module is the orchestration engine of osquery. + +It provides: + +- Structured flag management +- Deterministic initialization sequencing +- Multi-mode runtime (daemon, shell, extension) +- Watcher/worker supervision +- Plugin activation pipeline +- Coordinated graceful shutdown +- Forced termination safeguards + +Without embedding business logic itself, it guarantees that every subsystem is initialized, monitored, and terminated safely and predictably. + +It forms the foundational lifecycle contract upon which all other modules depend. diff --git a/docs/reference/architecture/database-backends/database-backends.md b/docs/reference/architecture/database-backends/database-backends.md new file mode 100644 index 00000000000..bbcf645288e --- /dev/null +++ b/docs/reference/architecture/database-backends/database-backends.md @@ -0,0 +1,225 @@ +# Database Backends + +The **Database Backends** module provides the persistent and ephemeral storage abstraction layer for osquery. It exposes a unified key–value interface used by higher-level subsystems such as scheduled queries, eventing, distributed querying, logging, and configuration management. + +This module is responsible for: + +- Abstracting database operations behind a plugin interface +- Managing database initialization and lifecycle +- Providing domain-based logical separation of stored data +- Handling database versioning and migrations +- Supporting both persistent (RocksDB) and in-memory (ephemeral) backends + +The core components in this module are: + +- `OsqueryDatabase` – High-level interface implementation used by the rest of the system +- `EphemeralDatabasePlugin` – In-memory database plugin implementation + +--- + +## Architectural Overview + +The Database Backends module sits between the osquery core runtime and concrete storage implementations. + +```mermaid +flowchart TD + CoreRuntime["Core Runtime"] --> DatabaseAPI["OsqueryDatabase"] + DatabaseAPI --> DBFunctions["Global Database Functions"] + DBFunctions --> Registry["Registry Factory"] + Registry --> ActivePlugin["Active Database Plugin"] + + ActivePlugin --> RocksDBPlugin["RocksDB Plugin"] + ActivePlugin --> EphemeralPlugin["Ephemeral Database Plugin"] + + RocksDBPlugin --> Disk[("Persistent Storage")] + EphemeralPlugin --> Memory[("In-Memory Map")] +``` + +### Key Concepts + +1. **Plugin-based architecture** – Database backends are registered via the internal registry under the `database` registry namespace. +2. **Domain isolation** – Logical namespaces (domains) segment stored data. +3. **Thread-safe reset model** – A global read/write mutex protects reset and runtime operations. +4. **Versioned storage** – Migration logic upgrades stored data formats across versions. + +--- + +## Logical Domains + +Data stored in the database is organized into predefined domains: + +- `configurations` +- `queries` +- `events` +- `logs` +- `carves` +- `distributed` +- `distributed_running` +- `query_performance` + +Each domain acts as an isolated keyspace used by other modules such as: + +- Scheduled queries (see SQL core and virtual tables module) +- Eventing subsystem (see Events core module) +- Distributed querying subsystem (see Distributed querying module) + +--- + +## Core Interface: OsqueryDatabase + +The `OsqueryDatabase` class implements `IDatabaseInterface` and delegates all operations to the global database helper functions. + +### Responsibilities + +- Provide a unified API for: + - `getDatabaseValue` + - `setDatabaseValue` + - `setDatabaseBatch` + - `deleteDatabaseValue` + - `deleteDatabaseRange` + - `scanDatabaseKeys` +- Abstract plugin routing and locking logic +- Enforce initialization guarantees + +### Call Flow Example + +```mermaid +sequenceDiagram + participant Caller + participant API as "OsqueryDatabase" + participant Helper as "Global Functions" + participant Registry + participant Plugin + + Caller->>API: setDatabaseValue(domain, key, value) + API->>Helper: setDatabaseValue(...) + Helper->>Registry: get active database plugin + Registry->>Plugin: putBatch(...) + Plugin-->>Caller: Status +``` + +--- + +## Plugin Model + +The Database Backends module uses a registry-based plugin system: + +- Registry name: `database` +- Active plugin selected at runtime +- Default persistent backend: `rocksdb` +- Fallback or testing backend: `ephemeral` + +Initialization logic: + +- If `disable_database` flag is set β†’ use `ephemeral` +- Otherwise β†’ attempt to activate persistent backend +- Retry logic handles startup race conditions + +--- + +## Thread Safety and Lifecycle + +A global mutex protects database resets: + +- **Write lock** – During `reset` operations +- **Read lock** – During standard `get`, `put`, `scan`, `remove` + +Atomic flags manage state: + +- Database initialized +- Database checking state +- Database allow-open control + +Lifecycle functions: + +- `initDatabasePlugin()` +- `resetDatabase()` +- `shutdownDatabase()` +- `upgradeDatabase()` + +--- + +## Database Migration + +The module supports versioned upgrades using a stored key: + +- Version key: `results_version` +- Stored in the `configurations` domain + +Migration flow: + +```mermaid +flowchart TD + Start["Read Current Version"] --> Check{"Version < Target?"} + Check -->|Yes| Migrate["Run Migration Step"] + Migrate --> Persist["Persist New Version"] + Persist --> Check + Check -->|No| End["Migration Complete"] +``` + +Example migrations: + +- V0 β†’ V1: Convert legacy Boost ptree JSON to RapidJSON format +- V1 β†’ V2: Rename legacy audit event keys + +--- + +## Ephemeral Backend + +The in-memory backend is implemented by: + +- [Ephemeral Database Plugin](database_backends/ephemeral/ephemeral.md) + +This backend: + +- Uses nested `std::map` containers +- Stores values as `boost::variant` +- Supports prefix scans and range deletions +- Is primarily used for: + - Testing + - Disabled persistent storage mode + - Controlled environments + +--- + +## Interaction with Other Modules + +The Database Backends module underpins multiple subsystems: + +- Query result storage (SQL core and virtual tables module) +- Event buffering (Events core module) +- Distributed query coordination (Distributed querying module) +- Configuration persistence (Config and packs module) +- Query performance metrics (Query execution and logging module) + +Rather than duplicating logic, higher-level modules rely exclusively on the `IDatabaseInterface` abstraction. + +--- + +## Design Characteristics + +### Advantages + +- Clean separation between storage logic and consumers +- Pluggable backend architecture +- Thread-safe reset and migration model +- Clear domain-based key segmentation + +### Limitations + +- Domain list is statically defined +- Plugin selection is global, not per-domain +- Migration logic is centralized and version-step dependent + +--- + +## Summary + +The **Database Backends** module provides a robust, plugin-driven abstraction layer for all persistent and ephemeral storage in osquery. It ensures: + +- Safe concurrent access +- Flexible backend selection +- Controlled initialization and reset behavior +- Forward-compatible data migration + +It is a foundational infrastructure module enabling reliable state management across the entire osquery runtime. diff --git a/docs/reference/architecture/distributed-querying/distributed-querying.md b/docs/reference/architecture/distributed-querying/distributed-querying.md new file mode 100644 index 00000000000..776888ed2be --- /dev/null +++ b/docs/reference/architecture/distributed-querying/distributed-querying.md @@ -0,0 +1,369 @@ +# Distributed Querying + +The **Distributed Querying** module enables osquery nodes to receive SQL queries from a remote service, execute them locally, and return structured results over a pluggable transport (for example, TLS). + +It acts as a bridge between: + +- The remote query orchestration layer (fleet manager, control plane) +- The local SQL engine and virtual tables +- The plugin framework and remote transport stack +- The query logging and performance monitoring subsystems + +This module is centered around three core concepts: + +- **DistributedQueryRequest** – a unit of remote work +- **DistributedQueryResult** – structured execution output +- **Distributed** – the orchestration engine responsible for pulling, executing, and flushing queries + +--- + +## Architecture Overview + +```mermaid +flowchart LR + ControlPlane["Remote Control Plane"] -->|"HTTPS JSON"| TLSPlugin["TLSDistributedPlugin"] + TLSPlugin -->|"getQueries()"| DistributedEngine["Distributed"] + DistributedEngine -->|"Execute SQL"| SQLCore["SQL Core and Virtual Tables"] + SQLCore -->|"QueryData"| DistributedEngine + DistributedEngine -->|"Batch Results"| TLSPlugin + TLSPlugin -->|"writeResults()"| ControlPlane + + DistributedEngine -->|"Record Performance"| QueryPerf["QueryPerformance"] + DistributedEngine -->|"Create Log Items"| QueryLog["QueryLogItem"] +``` + +At runtime, the flow is: + +1. The plugin retrieves remote work (`getQueries`). +2. The **Distributed** engine parses and queues queries. +3. Each query is executed through the SQL layer. +4. Results are captured, serialized, and flushed. +5. Performance statistics and logging artifacts are recorded. + +--- + +## Core Data Structures + +### DistributedQueryRequest + +**Component:** `osquery.osquery.distributed.distributed.DistributedQueryRequest` + +Represents a single unit of remote work. + +```text +Fields: +- query: string (SQL statement) +- id: string (unique identifier from server) +``` + +Responsibilities: + +- Holds the SQL statement and its server-issued identifier. +- Serialized/deserialized to and from JSON. +- Passed internally through execution and result pipelines. + +Serialization helpers: + +- `serializeDistributedQueryRequest` +- `serializeDistributedQueryRequestJSON` +- `deserializeDistributedQueryRequest` +- `deserializeDistributedQueryRequestJSON` + +These functions ensure consistent transport format across plugins. + +--- + +### DistributedQueryResult + +**Component:** `osquery.osquery.distributed.distributed.DistributedQueryResult` + +Represents the outcome of executing a distributed query. + +```text +Fields: +- request: DistributedQueryRequest +- results: QueryData +- columns: ColumnNames +- status: Status +- message: string +``` + +Responsibilities: + +- Couples execution output with its originating request. +- Captures success/failure status. +- Supports JSON serialization for remote transport. +- Used for batching before flush. + +Related concepts: + +- `QueryData` and `ColumnNames` come from the SQL engine. +- `Status` encapsulates execution outcome. + +--- + +## Distributed Execution Engine + +### Distributed + +**Component:** `osquery.osquery.distributed.distributed.Distributed` + +This class orchestrates the full lifecycle of distributed queries. + +### High-Level Workflow + +```mermaid +flowchart TD + Start["Start Loop"] --> Pull["pullUpdates()"] + Pull --> Accept["acceptWork()"] + Accept --> Pending{{"Pending Queries?"}} + Pending -->|"Yes"| Run["runQueries()"] + Run --> Exec["Execute via SQL"] + Exec --> AddRes["addResult()"] + AddRes --> Flush["flushCompleted()"] + Flush --> Start + Pending -->|"No"| Start +``` + +### Key Responsibilities + +#### 1. Pulling Work + +- `pullUpdates()` invokes the active distributed plugin. +- Receives JSON payload containing queries. +- Delegates parsing to `acceptWork()`. + +Expected remote format: + +```json +{ + "queries": { + "id1": "select * from osquery_info", + "id2": "select * from osquery_schedule" + } +} +``` + +--- + +#### 2. Discovery Query Handling + +`acceptWork()` implements special logic for *discovery queries*: + +- If a query has no discovery requirement β†’ enqueue directly. +- If a discovery query returns rows β†’ enqueue associated query. +- If discovery query returns no rows β†’ skip execution. + +This allows the control plane to conditionally execute work based on host state. + +--- + +#### 3. Denylisting and Concurrency Control + +To prevent repeated failures or runaway loops, Distributed uses a denylist mechanism. + +Key functions: + +- `checkAndSetAsRunning()` +- `setAsNotRunning()` +- `denylistedQueryTimestampExpired()` +- `denylistDuration()` +- `hashQuery()` + +### Denylist Flow + +```mermaid +flowchart TD + Incoming["Incoming Query"] --> Check["checkAndSetAsRunning()"] + Check --> Deny{{"Within denylist window?"}} + Deny -->|"Yes"| Skip["Skip Execution"] + Deny -->|"No"| Execute["Run SQL"] + Execute --> Clear["setAsNotRunning()"] +``` + +Mechanism: + +- Queries are hashed using SHA-256. +- Execution state is tracked in the database backend. +- Expired running queries are cleaned up with `cleanupExpiredRunningQueries()`. + +This protects nodes from repeatedly executing problematic queries. + +--- + +#### 4. SQL Execution and Monitoring + +Execution is performed through: + +- `monitorNonnumeric()` – wraps SQL execution. +- `recordQueryPerformance()` – stores timing and size metrics. + +Performance data is stored in: + +```text +std::map performance_ +``` + +See also: [SQL Core and Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) + +--- + +#### 5. Result Collection and Flushing + +Results are accumulated in: + +```text +std::vector results_ +``` + +Key operations: + +- `addResult()` – enqueue completed result +- `serializeResults()` – convert to JSON +- `flushCompleted()` – send to plugin +- `getCompletedCount()` – monitor batch size + +Flushing delegates to the active DistributedPlugin implementation. + +--- + +## Plugin Interface + +### DistributedPlugin + +**Component:** `osquery.osquery.distributed.distributed.DistributedPlugin` + +Abstract plugin contract for distributed query transport. + +```text +Virtual Methods: +- getQueries(std::string& json) +- writeResults(const std::string& json) +``` + +The plugin is responsible only for transport: + +- Fetching remote queries +- Sending result payloads + +It does not execute SQL or manage internal state. + +--- + +## TLS Transport Implementation + +### TLSDistributedPlugin + +**Component:** `osquery.plugins.distributed.tls_distributed.TLSDistributedPlugin` + +Registered as: + +```text +Registry: "distributed" +Name: "tls" +``` + +### Responsibilities + +- Builds URIs from flags: + - `distributed_tls_read_endpoint` + - `distributed_tls_write_endpoint` +- Performs HTTPS POST requests via `TLSRequestHelper`. +- Retries requests up to `distributed_tls_max_attempts`. + +### TLS Flow + +```mermaid +sequenceDiagram + participant Node + participant TLSPlugin + participant Server + + Node->>TLSPlugin: getQueries() + TLSPlugin->>Server: POST read_endpoint + Server-->>TLSPlugin: JSON queries + TLSPlugin-->>Node: JSON payload + + Node->>TLSPlugin: writeResults(json) + TLSPlugin->>Server: POST write_endpoint + Server-->>TLSPlugin: ACK +``` + +The response from `writeResults()` is intentionally ignored; success/failure is determined by HTTP status. + +--- + +## Integration with Other Modules + +Distributed Querying interacts with multiple subsystems: + +- **SQL Execution** – Executes queries and retrieves `QueryData`. + - See: [SQL Core and Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) + +- **Query Logging** – Converts execution output into `QueryLogItem`. + - See: [Query Execution and Logging](../query-execution-and-logging/query-execution-and-logging.md) + +- **Database Backends** – Stores running/denylisted state. + - See: [Database Backends](../database-backends/database-backends.md) + +- **Remote HTTP Client** – Provides TLS transport primitives. + - See: [Remote HTTP Client](../remote-http-client/remote-http-client.md) + +This layered separation ensures: + +- Transport flexibility +- Safe execution semantics +- Clear separation between orchestration and SQL runtime + +--- + +## Operational Model + +Typical long-running loop: + +```cpp +Distributed dist; + +while (true) { + dist.pullUpdates(); + if (!dist.getPendingQueries().empty()) { + dist.runQueries(); + } +} +``` + +In production, scheduling and execution are integrated with the core runtime and event loop. + +--- + +## Key Design Principles + +### 1. Transport Agnostic + +Execution logic is separated from transport via the plugin interface. + +### 2. Safe Re-Execution Controls + +Denylisting prevents runaway or repeatedly failing queries. + +### 3. Structured Serialization + +All requests and results are strongly typed and serialized via JSON helpers. + +### 4. Performance Visibility + +Each distributed query can record timing and size metrics through `QueryPerformance`. + +--- + +## Summary + +The **Distributed Querying** module enables remote orchestration of osquery nodes by: + +- Receiving SQL tasks from a control plane +- Executing them through the local SQL engine +- Capturing structured results +- Enforcing safety via denylisting +- Sending results back over pluggable transports (e.g., TLS) + +It is a critical component for fleet-wide visibility, remote investigations, and centralized query management across distributed systems. \ No newline at end of file diff --git a/docs/reference/architecture/events-core/events-core.md b/docs/reference/architecture/events-core/events-core.md new file mode 100644 index 00000000000..4738358b1b2 --- /dev/null +++ b/docs/reference/architecture/events-core/events-core.md @@ -0,0 +1,307 @@ +# Events Core + +The **Events Core** module implements osquery’s publish/subscribe event framework. It provides the infrastructure required for: + +- Registering event publishers and subscribers +- Managing event subscriptions +- Persisting event data in a time-indexed backing store +- Exposing event data as virtual tables +- Coordinating lifecycle, threading, and expiration policies + +At its core, Events Core enables real-time system activity (file changes, process activity, kernel notifications, etc.) to be captured by publishers, filtered by subscribers, stored efficiently, and queried through SQL. + +--- + +## Architectural Overview + +Events Core follows a classic publish/subscribe architecture with persistent storage and SQL exposure layered on top. + +```mermaid +flowchart TD + Config["Configuration & Schedule"] --> Factory["EventFactory"] + + subgraph publishers["Event Publishers"] + PubA["EventPublisherPlugin A"] + PubB["EventPublisherPlugin B"] + end + + subgraph subscribers["Event Subscribers"] + SubA["EventSubscriberPlugin A"] + SubB["EventSubscriberPlugin B"] + end + + PubA -->|"fires events"| SubA + PubA -->|"fires events"| SubB + PubB -->|"fires events"| SubA + + SubA -->|"addBatch(rows)"| Storage["Event Backing Store"] + SubB -->|"addBatch(rows)"| Storage + + Storage -->|"time filtered rows"| SQL["Virtual Table Query"] + + Factory --> PubA + Factory --> PubB + Factory --> SubA + Factory --> SubB +``` + +### Key Responsibilities + +- **EventFactory**: Central registry and lifecycle manager +- **EventPublisherPlugin**: Produces raw event contexts +- **EventSubscriberPlugin**: Filters and transforms events into rows +- **Subscription**: Binds a subscriber callback to a publisher +- **Backing Store**: Time-indexed storage for event rows +- **SQL Layer**: Exposes event data via virtual tables + +--- + +## Core Components + +### 1. EventFactory + +Defined in `eventfactory.cpp`, EventFactory is a singleton responsible for: + +- Registering and deregistering publishers and subscribers +- Managing subscriptions +- Running publisher threads +- Coordinating configuration updates +- Forwarding events to logger plugins + +#### Lifecycle Management + +```mermaid +flowchart TD + Start["Register Publisher"] --> Setup["setUp()"] + Setup --> Running["EVENT_RUNNING"] + Running --> Loop["run() loop"] + Loop -->|"isEnding()"| TearDown["tearDown()"] + TearDown --> EndState["EVENT_NONE"] +``` + +Key methods: + +- `registerEventPublisher` +- `registerEventSubscriber` +- `addSubscription` +- `run` +- `delay` +- `end` +- `configUpdate` + +#### Configuration Awareness + +During `configUpdate`, EventFactory: + +1. Scans scheduled queries. +2. Detects queries touching event-backed tables. +3. Computes a minimum expiration window per subscriber. +4. Reconfigures publishers and subscribers. + +This ensures expiration policies align with actual query intervals. + +The internal structure `SubscriberExpirationDetails` tracks: + +- `max_interval`: Longest scheduled query interval +- `query_count`: Number of queries using a subscriber + +--- + +### 2. EventSubscriberPlugin + +Defined in `eventsubscriberplugin.h`, EventSubscriberPlugin: + +- Inherits from `Plugin` and `Eventer` +- Registers subscriptions in `init()` +- Receives event callbacks +- Converts event contexts into table rows +- Stores rows in a persistent backing store + +#### Event Flow + +```mermaid +sequenceDiagram + participant Publisher + participant Subscriber + participant Database + participant SQL + + Publisher->>Subscriber: EventContext + Subscriber->>Subscriber: Transform to Row + Subscriber->>Database: addBatch(rows) + SQL->>Subscriber: genTable(query context) + Subscriber->>Database: generateRows(time window) + Subscriber->>SQL: Yield rows +``` + +#### Storage Model + +Each subscriber maintains a `Context` structure: + +- `database_namespace`: Unique storage namespace +- `event_index`: Time-indexed mapping of events +- `last_event_id`: Monotonic event identifier +- `last_query_time`: Optimization watermark + +Events are stored using: + +- EventID-based keys +- Time-based expiration +- Batch indexing + +#### GenerateRowsResult + +The `GenerateRowsResult` structure reports: + +- `isEnd`: Whether iteration completed +- `last_time`: Last processed event time +- `last_id`: Last processed event identifier + +This supports efficient incremental queries and optimization. + +#### Expiration and Optimization + +Subscribers implement: + +- `getEventsExpiry()` +- `getEventBatchesMax()` +- `shouldOptimize()` +- `expireEventBatches()` +- `removeOverflowingEventBatches()` + +Expiration is influenced by: + +- Global flags +- Subscriber overrides +- Scheduled query intervals + +--- + +### 3. Subscription + +Defined in `subscription.h`, `Subscription` binds: + +- A subscriber name +- A SubscriptionContext +- An EventCallback + +```mermaid +flowchart LR + Subscriber["EventSubscriberPlugin"] -->|"create()"| SubObj["Subscription"] + SubObj -->|"addSubscription()"| Publisher["EventPublisherPlugin"] + Publisher -->|"invoke callback"| Subscriber +``` + +The `EventCallback` signature: + +```text +Status(const EventContextRef&, const SubscriptionContextRef&) +``` + +This allows publishers to: + +- Filter events based on context +- Scope resource usage +- Dispatch events only to relevant subscribers + +--- + +## Threading Model + +Event publishers run in dedicated threads managed by EventFactory. + +```mermaid +flowchart TD + Delay["EventFactory.delay()"] --> Spawn["Spawn Publisher Threads"] + Spawn --> RunLoop["EventFactory.run(type)"] + RunLoop --> Poll["publisher.run()"] + Poll --> Pause["pause(200ms)"] + Pause --> Poll + Poll -->|"isEnding"| Exit["tearDown()"] +``` + +Key properties: + +- Each publisher runs independently +- Restart prevention enforced +- Controlled shutdown via `end()` +- Graceful deregistration support + +--- + +## Data Lifecycle + +```mermaid +flowchart TD + Event["System Event"] --> Publisher + Publisher --> Subscriber + Subscriber --> Store["Persistent Backing Store"] + Store --> Query["SQL Query"] + Query --> ExpireCheck["Expiration Enforcement"] + ExpireCheck --> Result["Filtered Rows"] +``` + +Stages: + +1. Event generated by OS or subsystem +2. Publisher dispatches EventContext +3. Subscriber transforms into Row +4. Row stored with EventTime and EventID +5. Query retrieves rows within time bounds +6. Expiration trims old data + +--- + +## Interaction with Other Modules + +Events Core integrates closely with: + +- **Config and Packs**: Scheduled queries determine expiration policies +- **SQL Core and Virtual Tables**: Exposes event data as tables +- **Database Backends**: Persists event rows and indexes +- **Plugin Interfaces and Logging**: Forwards event data to logger plugins + +EventFactory also supports event forwarding via logger plugins using registry calls. + +--- + +## Design Principles + +### 1. Strict Separation of Concerns + +- Publishers observe and emit +- Subscribers transform and store +- Factory orchestrates +- SQL layer queries + +### 2. Time-Based Query Optimization + +- Incremental queries use watermarks +- Expiration is schedule-aware +- Storage keyed by time and ID + +### 3. Pluggable Architecture + +- Publishers and subscribers are registry plugins +- Subscriptions dynamically configure publishers +- Event system can be globally disabled + +### 4. Controlled Resource Usage + +- Subscription scoping reduces publisher load +- Expiration prevents unbounded growth +- Max batch limits enforce memory control + +--- + +## Summary + +The **Events Core** module is the backbone of osquery’s real-time data collection system. It: + +- Implements a robust publish/subscribe event framework +- Persists event data with time-aware indexing +- Aligns expiration with scheduled query usage +- Exposes event streams as queryable SQL tables +- Provides thread-safe lifecycle and shutdown management + +Through EventFactory, EventSubscriberPlugin, and Subscription, Events Core enables scalable, extensible, and efficient event-driven data collection within the osquery runtime. \ No newline at end of file diff --git a/docs/reference/architecture/extensions-framework/extensions-framework.md b/docs/reference/architecture/extensions-framework/extensions-framework.md new file mode 100644 index 00000000000..4a6a537563d --- /dev/null +++ b/docs/reference/architecture/extensions-framework/extensions-framework.md @@ -0,0 +1,373 @@ +# Extensions Framework + +The **Extensions Framework** enables osquery to be extended at runtime using external processes that register new plugins, virtual tables, loggers, and configuration providers. + +Instead of statically linking all functionality into the core daemon, the Extensions Framework introduces a **manager–extension architecture** based on UNIX domain sockets (or Windows named pipes) and Apache Thrift RPC. This allows: + +- Dynamic plugin registration +- Isolation of extension failures from core +- SDK compatibility enforcement +- Distributed development of custom tables and plugins + +This module acts as the runtime bridge between: + +- The core registry and SQL engine +- External extension processes +- Thrift-based IPC transport +- Plugin routing and health monitoring + +--- + +## High-Level Architecture + +At runtime, osquery operates with a single **Extension Manager** (in core) and zero or more **Extension processes**. + +```mermaid +flowchart LR + Core["osquery Core"] --> Manager["Extension Manager"] + Manager --> Registry["Registry Factory"] + Manager --> SQLCore["SQL Engine"] + + ExtensionA["Extension Process A"] -->|"registerExtension"| Manager + ExtensionB["Extension Process B"] -->|"registerExtension"| Manager + + Core -->|"callExtension"| ExtensionA + Core -->|"callExtension"| ExtensionB + + ExtensionA -->|"query"| SQLCore +``` + +### Core Responsibilities + +| Component | Responsibility | +|------------|---------------| +| Extension Manager | Accepts registrations and routes plugin calls | +| Extension Runner | Hosts Thrift server inside extension process | +| Extension Client | Performs RPC calls over socket | +| Watchers | Monitor health and enforce shutdown policies | +| RegistryFactory | Maps UUIDs to plugin routes | + +--- + +## Key Concepts + +### 1. ExtensionInfo + +Defined in `extensions.h`. + +```text +struct ExtensionInfo { + std::string name; + std::string version; + std::string sdk_version; + std::string min_sdk_version; +}; +``` + +This metadata is exchanged during registration and used to: + +- Enforce SDK compatibility +- Prevent duplicate extension names +- Track active UUIDs + +Each extension is assigned a transient **RouteUUID** at registration time. + +--- + +### 2. UUID Management + +`UuidGenerator` ensures each extension receives a unique 16-bit identifier. + +```mermaid +flowchart TD + Request["registerExtension()"] --> Generate["UuidGenerator.getUuid()"] + Generate --> Store["Track UUID in set"] + Store --> Assign["Associate UUID with Registry routes"] +``` + +When deregistered, the UUID is removed and may be reused later. + +--- + +### 3. Extension Registration Flow + +The lifecycle of an extension: + +```mermaid +sequenceDiagram + participant Ext as Extension Process + participant EM as Extension Manager + participant Reg as RegistryFactory + + Ext->>EM: registerExtension(info, registry) + EM->>Reg: addBroadcast(uuid, registry) + EM-->>Ext: return uuid + Ext->>Ext: start Thrift server + EM->>Ext: ping() +``` + +Steps: + +1. Extension connects to manager socket. +2. Sends metadata and registry routes. +3. Manager checks: + - Duplicate names + - SDK compatibility + - Route conflicts +4. UUID assigned. +5. Registry broadcast stored. +6. Extension starts serving requests. + +--- + +## Thrift Communication Layer + +The Extensions Framework uses Apache Thrift for IPC. + +### Server Components + +- `ExtensionRunnerInterface` +- `ExtensionRunner` +- `ExtensionManagerRunner` +- `ImplExtensionRunner` + +### Client Components + +- `ExtensionClient` +- `ExtensionManagerClient` +- `ImplExtensionClient` + +```mermaid +flowchart TD + Client["ExtensionClient"] --> Transport["TBufferedTransport"] + Transport --> Socket["UNIX Socket / Named Pipe"] + Socket --> Server["TThreadedServer"] + Server --> Handler["ExtensionHandler / ExtensionManagerHandler"] +``` + +### RPC Methods + +From the Thrift IDL (`osquery_types.h`): + +- `ping()` +- `call(registry, item, request)` +- `registerExtension()` +- `deregisterExtension()` +- `query()` +- `getQueryColumns()` + +Extension return codes are defined as: + +```text +EXT_SUCCESS = 0 +EXT_FAILED = 1 +EXT_FATAL = 2 +``` + +--- + +## Registry Integration + +The Extensions Framework integrates directly with the internal Registry system. + +When an extension registers: + +- Its `ExtensionRegistry` (map of registry β†’ routes) is added via `RegistryFactory::addBroadcast()` +- Routes are associated with its UUID + +When a plugin is invoked: + +```mermaid +flowchart TD + Call["callExtension(uuid, registry, item)"] --> Lookup["Resolve registry alias"] + Lookup --> Client["ExtensionClient.call()"] + Client --> ExtensionServer["ExtensionHandler.call()"] + ExtensionServer --> Registry["RegistryFactory.call()"] +``` + +This enables: + +- Virtual tables defined in extensions +- Logger plugins in extensions +- Config plugins in extensions +- Distributed plugins + +See related modules for deeper context: + +- [SQL Core and Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) +- [Config Plugins](../config-plugins/config-plugins.md) +- [Distributed Querying](../distributed-querying/distributed-querying.md) + +--- + +## External SQL Plugin + +The `ExternalSQLPlugin` allows core SQL queries to be routed through the Extension Manager. + +This enables: + +- Extensions to expose virtual tables +- Remote query execution via extensions + +```mermaid +flowchart LR + Query["SQL Query"] --> External["ExternalSQLPlugin"] + External --> Manager["ExtensionManagerClient.query()"] + Manager --> SQLite["Core SQLite Engine"] +``` + +Column metadata retrieval is also proxied via `getQueryColumns()`. + +--- + +## Watchers and Health Monitoring + +Two watcher types maintain system stability: + +### 1. ExtensionWatcher + +Used inside extensions to monitor core. + +- Pings Extension Manager +- Optionally exits if manager disappears +- Ensures extension does not orphan itself + +### 2. ExtensionManagerWatcher + +Used inside core to monitor extensions. + +- Iterates registered UUIDs +- Checks socket existence +- Pings extension +- Removes broadcast routes on failure + +```mermaid +flowchart TD + Loop["Periodic Watch"] --> Check["socketExists()"] + Check --> Ping["client.ping()"] + Ping --> Healthy["EXT_SUCCESS?"] + Healthy -->|"No"| Remove["removeBroadcast(uuid)"] +``` + +This guarantees stale or crashed extensions do not leave broken registry routes. + +--- + +## Autoloading Extensions + +Extensions can be automatically loaded using CLI flags: + +- `--extensions_autoload` +- `--extensions_require` +- `--extensions_timeout` + +Autoload behavior: + +1. Read newline-delimited extension paths +2. Validate file permissions +3. Validate binary extension (.ext, .exe) +4. Spawn extension +5. Wait for registration + +Safety checks include: + +- Directory ownership validation +- File existence +- SDK compatibility + +--- + +## Extension Manager Startup + +The core starts the manager via: + +- `startExtensionManager()` +- `ExtensionManagerRunner` +- `ExtensionManagerWatcher` + +Startup sequence: + +```mermaid +sequenceDiagram + participant Core + participant EMRunner + participant EMWatcher + + Core->>EMRunner: start() + Core->>EMWatcher: start() + EMRunner->>Socket: bind + serve + EMWatcher->>Registry: monitor UUID routes +``` + +If `--disable_extensions` is set, the entire subsystem is bypassed. + +--- + +## Failure Handling Model + +The Extensions Framework isolates failures using: + +- Separate processes +- Thrift transport boundaries +- Watcher-based teardown +- Registry broadcast removal + +Failure scenarios: + +| Scenario | Behavior | +|-----------|----------| +| Extension crash | Manager removes UUID routes | +| Manager crash | ExtensionWatcher triggers shutdown | +| Duplicate name | Registration rejected | +| SDK mismatch | Registration rejected | +| Socket removed | Extension terminated | + +--- + +## Interaction With Other Modules + +The Extensions Framework integrates tightly with: + +- [Core Init and Runtime](../core-init-and-runtime/core-init-and-runtime.md) +- [SQL Core and Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) +- [Config Plugins](../config-plugins/config-plugins.md) +- [Distributed Querying](../distributed-querying/distributed-querying.md) + +It provides the IPC and registration substrate that allows those systems to be extended dynamically. + +--- + +## Design Characteristics + +### βœ… Process Isolation +Extensions run as separate processes. + +### βœ… Runtime Plugin Injection +Plugins can be added without recompiling core. + +### βœ… SDK Version Enforcement +Ensures forward/backward compatibility boundaries. + +### βœ… Health Monitoring +Bidirectional watchers prevent orphaned state. + +### βœ… Transport Abstraction +Thrift implementation encapsulated via PIMPL (`ImplExtensionRunner`, `ImplExtensionClient`). + +--- + +# Summary + +The **Extensions Framework** transforms osquery from a statically compiled system into a dynamically extensible platform. + +It introduces: + +- Extension Manager +- Extension processes +- Thrift-based IPC +- UUID-based route isolation +- Registry broadcast integration +- Health watchers +- SQL and plugin routing + +This module is foundational for enabling custom tables, loggers, config providers, and distributed query systems while preserving process isolation and runtime safety. \ No newline at end of file diff --git a/docs/reference/architecture/filesystem-and-fileops/filesystem-and-fileops.md b/docs/reference/architecture/filesystem-and-fileops/filesystem-and-fileops.md new file mode 100644 index 00000000000..0cfbc72800b --- /dev/null +++ b/docs/reference/architecture/filesystem-and-fileops/filesystem-and-fileops.md @@ -0,0 +1,298 @@ +# Filesystem And Fileops + +The **Filesystem And Fileops** module provides a cross-platform abstraction layer for file system interactions within the osquery runtime. It normalizes differences between POSIX and Windows APIs, offering consistent file metadata access, file I/O primitives, permission handling, globbing, and environment-based path resolution. + +This module is foundational to multiple higher-level components such as: + +- [SQL Core And Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) – for file-backed virtual tables and metadata inspection +- [Database Backends](../database-backends/database-backends.md) – for safe database file access and permission enforcement +- [Extensions Framework](../extensions-framework/extensions-framework.md) – for socket lifecycle management +- [Hashing](../hashing/hashing.md) – for file content hashing +- [Config Plugins](../config-plugins/config-plugins.md) – for filesystem-based configuration loading + +It ensures that file operations behave consistently across platforms while respecting security constraints and OS-specific semantics. + +--- + +## 1. Architectural Overview + +The module is organized around three major responsibilities: + +1. **Platform Abstraction** – Unified interfaces for file descriptors, time, permissions, and stat structures. +2. **File I/O Abstraction** – The `PlatformFile` class encapsulates safe file reading, writing, and seeking. +3. **Filesystem Utilities** – Cross-platform helpers such as globbing, permission validation, temporary directory detection, and socket checks. + +```mermaid +flowchart TD + App["Higher Level Modules"] --> SQL["SQL Core And Virtual Tables"] + App --> DB["Database Backends"] + App --> EXT["Extensions Framework"] + App --> CFG["Config Plugins"] + + SQL --> FS["Filesystem And Fileops"] + DB --> FS + EXT --> FS + CFG --> FS + + FS --> PF["PlatformFile"] + FS --> STAT["platformStat and platformLstat"] + FS --> UTIL["Filesystem Utility Functions"] + FS --> ASYNC["AsyncEvent"] +``` + +The abstraction layer shields upper modules from platform-specific details such as: + +- Windows `HANDLE` vs POSIX file descriptor +- `FILETIME` vs `timeval` +- ACL-based permissions vs POSIX mode bits +- Named pipes vs UNIX sockets + +--- + +## 2. Core Components + +### 2.1 Platform Types + +The module defines normalized platform aliases: + +- `PlatformHandle` – `HANDLE` on Windows, `int` on POSIX +- `PlatformTimeType` – `FILETIME` on Windows, `timeval` on POSIX +- `PlatformTime` – Wrapper storing access and modification timestamps + +These abstractions are used throughout the runtime to avoid scattering conditional compilation across the codebase. + +--- + +### 2.2 WINDOWS_STAT + +`WINDOWS_STAT` is a Windows-specific structure that emulates the POSIX `stat` structure while adding richer Windows metadata. + +It includes: + +- File identifiers (`file_id`, `inode`) +- Ownership (`uid`, `gid`) +- Timestamps (`atime`, `mtime`, `ctime`, `btime`) +- Version metadata (`product_version`, `file_version`) +- NTFS attributes and volume information + +This structure is populated through `platformStat()` and allows SQL tables and logging modules to expose file metadata consistently. + +On POSIX systems, standard `stat` and `lstat` are used via `platformLstat()`. + +--- + +### 2.3 AsyncEvent (Windows) + +`AsyncEvent` simulates POSIX-style non-blocking I/O using Windows asynchronous APIs. + +```mermaid +flowchart LR + ReadCall["read() or write()"] --> NonBlockCheck["Non Blocking Mode?"] + NonBlockCheck -->|"Yes"| Overlap["OVERLAPPED Structure"] + Overlap --> Buffer["Async Buffer"] + Buffer --> Pending["Pending IO State"] + NonBlockCheck -->|"No"| Sync["Synchronous IO"] +``` + +Key fields: + +- `OVERLAPPED overlapped_` +- `std::unique_ptr buffer_` +- `bool is_active_` + +Limitations: + +- Write cancellation may interrupt ongoing reads +- Emulates semantics but cannot fully replicate POSIX guarantees + +This component is internal to `PlatformFile` on Windows. + +--- + +## 3. PlatformFile Class + +`PlatformFile` is the central abstraction for file I/O. + +### Responsibilities + +- Open files with unified flags +- Read and write bytes +- Seek operations +- Inspect size +- Validate ownership and permissions +- Detect executable status +- Provide file times + +```mermaid +flowchart TD + Client["Caller"] --> Open["PlatformFile Constructor"] + Open --> Handle["PlatformHandle"] + Handle --> Read["read()"] + Handle --> Write["write()"] + Handle --> Seek["seek()"] + Handle --> Size["size()"] + + Read --> AsyncCheck["hasPendingIo()"] +``` + +### File Mode Abstraction + +Instead of exposing raw OS flags, the module defines: + +- `PF_READ` +- `PF_WRITE` +- `PF_CREATE_NEW` +- `PF_CREATE_ALWAYS` +- `PF_OPEN_EXISTING` +- `PF_OPEN_ALWAYS` +- `PF_TRUNCATE` +- `PF_NONBLOCK` +- `PF_APPEND` + +These are translated internally into the appropriate POSIX or Windows flags. + +### Ownership and Security Checks + +- `isOwnerRoot()` – Validates administrative ownership +- `isOwnerCurrentUser()` – Confirms ownership +- `hasSafePermissions()` – Ensures restrictive write semantics + +These checks are critical for: + +- Database file safety in [Database Backends](../database-backends/database-backends.md) +- Configuration integrity in [Config Plugins](../config-plugins/config-plugins.md) + +--- + +## 4. Filesystem Utility Functions + +Beyond file descriptors, the module provides higher-level filesystem helpers. + +### 4.1 Permission Handling + +- `platformChmod()` – Cross-platform chmod approximation +- `platformSetSafeDbPerms()` – Enforces restrictive DB permissions +- `platformAccess()` – Abstracted access check +- `describeBSDFileFlags()` – Human-readable BSD flags + +Windows ACL ordering issues are handled explicitly to approximate POSIX semantics. + +--- + +### 4.2 Path and Environment Utilities + +- `getHomeDirectory()` – Multi-strategy resolution (environment variables first) +- `getSystemRoot()` – Returns root path (`/` on POSIX) +- `windowsShortPathToLongPath()` – Expands 8.3 paths +- `windowsGetVersionInfo()` – Retrieves version metadata + +These are frequently used by configuration loading and extension management. + +--- + +### 4.3 Globbing + +`platformGlob()` approximates POSIX glob behavior on Windows. + +Supported features: + +- Tilde expansion (basic) +- Brace expansion (regex-backed) +- Marking behavior + +This is essential for file-based table expansion in [SQL Core And Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md). + +--- + +### 4.4 Socket and Pipe Management + +- `socketExists()` – Checks and optionally removes UNIX sockets or named pipes +- `platformIsTmpDir()` – Detects temporary directory +- `platformIsFileAccessible()` – Unified accessibility check + +These are heavily used by the [Extensions Framework](../extensions-framework/extensions-framework.md) during startup and teardown. + +```mermaid +flowchart TD + ExtensionStartup["Extension Startup"] --> Check["socketExists(remove)"] + Check --> Exists["Socket Exists?"] + Exists -->|"Yes"| Remove["Attempt Removal"] + Exists -->|"No"| Continue["Create Socket"] +``` + +--- + +## 5. Metadata Flow in the System + +When a file is queried or inspected: + +```mermaid +sequenceDiagram + participant Module + participant FS as Filesystem And Fileops + participant OS + + Module->>FS: platformStat(path) + FS->>OS: Native stat or Win API + OS-->>FS: Raw metadata + FS-->>Module: Normalized structure +``` + +This normalized metadata may then: + +- Be hashed via [Hashing](../hashing/hashing.md) +- Be emitted in logs via [Query Execution And Logging](../query-execution-and-logging/query-execution-and-logging.md) +- Be stored by [Database Backends](../database-backends/database-backends.md) + +--- + +## 6. Security Model Integration + +The module enforces security invariants across the runtime: + +- Restrictive DB permissions +- Ownership verification +- Executable bit inspection +- ACL-aware Windows behavior + +```mermaid +flowchart TD + FileOpen["Open File"] --> OwnerCheck["isOwnerRoot or isOwnerCurrentUser"] + OwnerCheck --> PermCheck["hasSafePermissions"] + PermCheck --> Decision["Allow or Reject"] +``` + +This prevents: + +- Unauthorized modification of query packs +- Tampering with database files +- Unsafe extension socket exposure + +--- + +## 7. Cross-Module Relationships + +The Filesystem And Fileops module acts as infrastructure for: + +- **Query evaluation** β†’ file-backed virtual tables +- **Configuration loading** β†’ filesystem-based config plugins +- **Extension IPC** β†’ socket existence and cleanup +- **Database persistence** β†’ safe file permissions +- **Hash computation** β†’ raw file reads + +It is intentionally dependency-light and focused strictly on OS abstraction and safe filesystem manipulation. + +--- + +## 8. Summary + +The **Filesystem And Fileops** module provides: + +- Cross-platform file descriptor abstraction +- Normalized file metadata +- Permission and ACL emulation +- Globbing and path utilities +- Socket lifecycle management +- Safe database permission enforcement + +It is a foundational infrastructure layer that ensures consistent, secure, and portable filesystem behavior across the entire osquery runtime. diff --git a/docs/reference/architecture/hashing/hashing.md b/docs/reference/architecture/hashing/hashing.md new file mode 100644 index 00000000000..ec7094fa713 --- /dev/null +++ b/docs/reference/architecture/hashing/hashing.md @@ -0,0 +1,344 @@ +# Hashing + +## Overview + +The **Hashing** module provides cryptographic digest utilities used across the system to compute file and buffer hashes. It supports multiple algorithms (MD5, SHA1, SHA256) and multiple encodings (HEX, BASE64), and exposes both single-hash and multi-hash interfaces. + +At its core, the module is designed for: + +- File integrity verification +- Change detection for scheduled queries +- Artifact fingerprinting in distributed and extension workflows +- Efficient streaming of large files without loading them fully into memory + +The primary component of this module is `MultiHashes`, supported by the `Hash` class and a set of helper functions: + +- `hashFromFile` +- `hashMultiFromFile` +- `hashFromBuffer` + +This module integrates closely with: + +- [Filesystem and Fileops](../filesystem-and-fileops/filesystem-and-fileops.md) +- [SQL Core and Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) +- [Distributed Querying](../distributed-querying/distributed-querying.md) +- [Query Execution and Logging](../query-execution-and-logging/query-execution-and-logging.md) + +--- + +## Supported Algorithms and Encodings + +### Hash Algorithms + +The module defines a bitmask-based enumeration of supported algorithms: + +- MD5 +- SHA1 +- SHA256 + +Each algorithm is represented as a bit flag, enabling multi-algorithm computation in a single pass. + +### Encoding Types + +Digest output can be encoded as: + +- HEX (default) +- BASE64 + +Encoding is configured at `Hash` construction time. + +--- + +## Core Data Structures + +### MultiHashes + +`MultiHashes` is a result container used when multiple algorithms are computed simultaneously. + +```text +struct MultiHashes { + int mask; + std::string md5; + std::string sha1; + std::string sha256; +} +``` + +- `mask` indicates which algorithms were requested. +- Each digest field is populated only if the corresponding bit is set. + +This structure allows higher-level modules to request several digests efficiently during a single file scan. + +--- + +## Hash Class Architecture + +The `Hash` class is a non-copyable streaming hashing utility. + +### Design Characteristics + +- Non-copyable (inherits from boost noncopyable) +- Move-constructible and move-assignable +- Streaming interface via `update` +- Explicit `digest` finalization +- Internal opaque context pointer + +### Component Structure + +```mermaid +flowchart TD + HashClass["Hash Class"] --> Algorithm["HashType"] + HashClass --> Encoding["HashEncodingType"] + HashClass --> Context["Internal Context Pointer"] + HashClass --> Update["update(buffer, size)"] + HashClass --> Digest["digest()"] +``` + +### Lifecycle + +```mermaid +flowchart LR + Create["Construct Hash"] --> Stream["update() Calls"] + Stream --> Finalize["digest()"] + Finalize --> Result["Encoded Digest String"] +``` + +This streaming model ensures large files can be processed incrementally without high memory usage. + +--- + +## Functional API + +The module exposes three primary helper functions. + +### 1. hashFromFile + +Computes a single digest from file contents. + +```text +std::string hashFromFile(HashType hash_type, const std::string& path); +``` + +Flow: + +```mermaid +flowchart TD + Request["hashFromFile()"] --> OpenFile["Open File"] + OpenFile --> ReadChunk["Read Chunk"] + ReadChunk --> UpdateHash["Hash.update()"] + UpdateHash --> MoreData{"More Data?"} + MoreData -->|"Yes"| ReadChunk + MoreData -->|"No"| Finalize["Hash.digest()"] + Finalize --> Return["Return HEX String"] +``` + +### 2. hashMultiFromFile + +Computes multiple hashes in a single file traversal. + +```text +MultiHashes hashMultiFromFile(int mask, const std::string& path); +``` + +Key advantage: + +- Single disk traversal +- Multiple digest contexts +- Efficient for integrity-heavy workflows + +### 3. hashFromBuffer + +Computes a digest from in-memory data. + +```text +std::string hashFromBuffer(HashType hash_type, const void* buffer, size_t size); +``` + +Used in scenarios such as: + +- Virtual table content hashing +- Distributed result validation +- In-memory artifact fingerprinting + +--- + +## Multi-Algorithm Processing Model + +The bitmask design enables efficient multi-hash computation. + +```mermaid +flowchart TD + Start["File Input"] --> MaskCheck["Check Mask Bits"] + MaskCheck --> MD5Ctx["Initialize MD5 Context"] + MaskCheck --> SHA1Ctx["Initialize SHA1 Context"] + MaskCheck --> SHA256Ctx["Initialize SHA256 Context"] + + MD5Ctx --> StreamData["Stream File Data"] + SHA1Ctx --> StreamData + SHA256Ctx --> StreamData + + StreamData --> FinalMD5["Finalize MD5"] + StreamData --> FinalSHA1["Finalize SHA1"] + StreamData --> FinalSHA256["Finalize SHA256"] + + FinalMD5 --> MultiResult["Populate MultiHashes"] + FinalSHA1 --> MultiResult + FinalSHA256 --> MultiResult +``` + +This design prevents repeated disk reads when multiple digests are required. + +--- + +## Integration Within the System + +### 1. Filesystem Interaction + +The Hashing module depends on file access primitives from: + +- [Filesystem and Fileops](../filesystem-and-fileops/filesystem-and-fileops.md) + +It reads file contents in chunks and processes them incrementally. + +```mermaid +flowchart LR + Fileops["Filesystem and Fileops"] --> Hashing["Hashing"] + Hashing --> DigestResult["Digest String"] +``` + +--- + +### 2. SQL and Virtual Tables + +Hashing is commonly used inside virtual tables that expose file metadata or integrity information. + +Relevant module: + +- [SQL Core and Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) + +Example data flow: + +```mermaid +flowchart TD + VirtualTable["Virtual Table"] --> FilePath["File Path"] + FilePath --> HashingModule["Hashing"] + HashingModule --> RowData["Row with Hash Columns"] + RowData --> SQLiteLayer["SQLite Engine"] +``` + +Hashes may appear as columns such as md5, sha1, or sha256 in result sets. + +--- + +### 3. Distributed Query Validation + +Within distributed query execution: + +- Hashes can verify payload integrity +- Results may include file digests + +Related module: + +- [Distributed Querying](../distributed-querying/distributed-querying.md) + +--- + +### 4. Query Logging and Change Detection + +When scheduled queries monitor file integrity, hash values can be used to detect changes between executions. + +Related module: + +- [Query Execution and Logging](../query-execution-and-logging/query-execution-and-logging.md) + +Conceptual flow: + +```mermaid +flowchart TD + ScheduledQuery["Scheduled Query"] --> CollectFile["Collect File Path"] + CollectFile --> ComputeHash["Compute SHA256"] + ComputeHash --> Compare["Compare With Previous Result"] + Compare --> Diff["Generate Diff Results"] +``` + +--- + +## Memory and Performance Considerations + +### Streaming Design + +The `update` method allows incremental hashing: + +- Large files are processed in chunks +- Memory footprint remains constant +- Suitable for large-scale file scanning + +### Move Semantics + +The class supports move construction and move assignment: + +- Prevents accidental copying of cryptographic state +- Enables efficient transfer of hash contexts + +### Single-Pass Multi-Hashing + +`hashMultiFromFile` ensures: + +- One file open +- One read loop +- Multiple digest computations + +This is critical for performance-sensitive subsystems such as: + +- File integrity monitoring +- Large pack executions in configuration workflows + +--- + +## Security Considerations + +- SHA256 should be preferred for integrity verification. +- MD5 and SHA1 may be used for compatibility but are not collision-resistant. +- Encoding does not affect cryptographic strength, only representation. + +When used in security-sensitive contexts such as distributed validation or extension communication, SHA256 is recommended. + +--- + +## High-Level Architecture Summary + +```mermaid +flowchart TD + subgraph CoreSystem["Core System"] + SQLModule["SQL Core and Virtual Tables"] + DistributedModule["Distributed Querying"] + QueryLogModule["Query Execution and Logging"] + end + + FileModule["Filesystem and Fileops"] --> HashingModule["Hashing"] + HashingModule --> SQLModule + HashingModule --> DistributedModule + HashingModule --> QueryLogModule +``` + +The **Hashing** module acts as a foundational cryptographic utility layer that supports integrity, validation, and change detection across the entire system. + +--- + +## Summary + +The **Hashing** module provides: + +- Streaming cryptographic hashing +- Multi-algorithm single-pass computation +- File and buffer hashing APIs +- Integration with SQL, distributed queries, and logging subsystems + +Its design emphasizes: + +- Performance +- Memory efficiency +- Clear algorithm selection +- Safe ownership semantics + +As a result, Hashing serves as a low-level integrity primitive relied upon by multiple higher-level modules throughout the system. \ No newline at end of file diff --git a/docs/reference/architecture/plugin-interfaces-and-logging/plugin-interfaces-and-logging.md b/docs/reference/architecture/plugin-interfaces-and-logging/plugin-interfaces-and-logging.md new file mode 100644 index 00000000000..e50eb557b02 --- /dev/null +++ b/docs/reference/architecture/plugin-interfaces-and-logging/plugin-interfaces-and-logging.md @@ -0,0 +1,316 @@ +# Plugin Interfaces And Logging + +The Plugin Interfaces And Logging module defines the extensible logging architecture used by osquery. It provides the abstraction layer that allows osquery to forward status logs, query results, snapshots, and events to pluggable backends such as file loggers, TLS loggers, syslog, or custom enterprise pipelines. + +At the center of this module is the `LoggerPlugin` interface and the `StatusLogLine` data structure, which together enable: + +- Structured status logging (INFO, WARNING, ERROR, FATAL) +- Snapshot and differential result logging +- Optional direct event forwarding +- Feature negotiation between the core and logger plugins + +This module acts as a bridge between core execution components and external log consumers. + +--- + +## Architectural Role + +The Plugin Interfaces And Logging module sits between the core runtime and downstream logging implementations. It integrates with: + +- [Core Init And Runtime](core-init-and-runtime/core-init-and-runtime.md) for startup lifecycle and early log buffering +- [Query Execution And Logging](query-execution-and-logging/query-execution-and-logging.md) for structured query result emission +- [Events Core](events-core/events-core.md) for event subscriber forwarding +- [Extensions Framework](extensions-framework/extensions-framework.md) when logger plugins are provided as extensions + +### High-Level Architecture + +```mermaid +flowchart LR + CoreRuntime["Core Init And Runtime"] -->|"Buffered Status Logs"| LoggerInterface["LoggerPlugin Interface"] + QueryEngine["Query Execution And Logging"] -->|"QueryLogItem JSON"| LoggerInterface + EventsCore["Events Core"] -->|"Event Records"| LoggerInterface + Extensions["Extensions Framework"] -->|"Registers Logger Plugins"| LoggerInterface + LoggerInterface -->|"logString or logStatus"| LoggerBackend["Concrete Logger Plugin"] + LoggerBackend -->|"Writes To"| ExternalSink["External Logging System"] +``` + +The LoggerPlugin interface standardizes how log data flows from internal subsystems to an external destination. + +--- + +## Core Data Structure: StatusLogLine + +The `StatusLogLine` structure represents a parsed and normalized status log entry emitted by the internal logging system. + +### Fields + +```text +StatusLogLine + β”œβ”€ severity (StatusLogSeverity) + β”œβ”€ filename (string) + β”œβ”€ line (uint64) + β”œβ”€ message (string) + β”œβ”€ calendar_time (string) + β”œβ”€ time (uint64) + └─ identifier (string) +``` + +### Severity Model + +Severity is mapped using an internal enum compatible with Glog levels: + +- O_INFO +- O_WARNING +- O_ERROR +- O_FATAL + +This ensures that logger plugins can: + +- Preserve original severity +- Map severity to external systems +- Filter or route based on severity + +### Lifecycle of a Status Log + +```mermaid +sequenceDiagram + participant Core + participant Glog + participant Buffer + participant LoggerPlugin + + Core->>Glog: Emit status message + Glog->>Buffer: Store StatusLogLine + Core->>LoggerPlugin: init(binaryName, bufferedLogs) + LoggerPlugin->>LoggerPlugin: Process buffered StatusLogLine entries +``` + +During initialization, buffered status logs are passed into the active logger plugin through the protected `init` method. + +--- + +## LoggerPlugin Interface + +The `LoggerPlugin` class is the extensible abstraction that concrete loggers must implement. + +### Inheritance Model + +```mermaid +flowchart TD + PluginBase["Plugin Base Class"] --> LoggerPlugin["LoggerPlugin"] + LoggerPlugin --> CustomLogger["Custom Logger Implementation"] +``` + +LoggerPlugin inherits from the generic Plugin base and participates in the osquery plugin registry. + +### Mandatory Implementation + +Every logger plugin must implement: + +```text +Status logString(const std::string& s) +``` + +This method receives serialized log payloads (typically JSON) and forwards them to the external system. + +--- + +## Optional Feature Negotiation + +Logger plugins may opt-in to advanced behaviors using feature methods. + +### LoggerFeatures Enumeration + +```text +LOGGER_FEATURE_BLANK +LOGGER_FEATURE_LOGSTATUS +LOGGER_FEATURE_LOGEVENT +``` + +These features correspond to optional capabilities implemented by a logger. + +### Feature Methods + +| Method | Purpose | +|--------|----------| +| usesLogStatus | Take exclusive ownership of status logs | +| usesLogEvent | Receive events directly from subscribers | +| logSnapshot | Handle large snapshot result sets separately | +| logEvent | Handle single event forwarding | +| logStringBatch | Handle batch event forwarding | + +### Feature Flow + +```mermaid +flowchart TD + Core["Core Subsystem"] --> CheckStatus["Check usesLogStatus"] + CheckStatus -->|"true"| StatusHandler["logStatus"] + CheckStatus -->|"false"| Glog["Default Glog Handling"] + + Core --> CheckEvent["Check usesLogEvent"] + CheckEvent -->|"true"| EventHandler["logEvent or logStringBatch"] + CheckEvent -->|"false"| Database["Event Stored In Database"] +``` + +This mechanism allows sophisticated loggers to: + +- Replace default status logging +- Bypass the database for event streaming +- Handle large payloads efficiently + +--- + +## Snapshot and Query Logging + +The Plugin Interfaces And Logging module integrates tightly with [Query Execution And Logging](query-execution-and-logging/query-execution-and-logging.md). + +When scheduled or distributed queries execute, they generate structured log entries that are serialized and passed to `logString` or `logSnapshot`. + +### Snapshot Handling + +If a plugin overrides `logSnapshot`, snapshot query results can be: + +- Written to a separate stream +- Batched or chunked differently +- Routed to a high-throughput pipeline + +Otherwise, snapshot results fall back to `logString`. + +--- + +## Event Forwarding Integration + +This module optionally bypasses database storage defined in [Database Backends](database-backends/database-backends.md) when `usesLogEvent` returns true. + +### Event Processing Flow + +```mermaid +flowchart LR + EventSubscriber["Event Subscriber"] --> EventFactory["Event Factory"] + EventFactory -->|"Event JSON"| LoggerPlugin + LoggerPlugin -->|"logEvent"| ExternalSink +``` + +If disabled, events are first stored in the database and later queried through virtual tables. + +--- + +## Extension-Based Loggers + +Logger plugins may be: + +- Built-in compiled plugins +- Extension-based plugins registered via the [Extensions Framework](extensions-framework/extensions-framework.md) + +### Extension Registration Flow + +```mermaid +flowchart TD + ExtensionBinary["Extension Process"] --> ExtensionManager["Extension Manager"] + ExtensionManager --> Registry["Plugin Registry"] + Registry --> LoggerPlugin +``` + +Because logger plugins can be loaded as extensions, initialization order is critical. The logger is initialized only after: + +- CLI flags are parsed +- Registry items are constructed +- Extensions are discovered +- Configuration is loaded + +This ensures all early logs are preserved and flushed. + +--- + +## Initialization Lifecycle + +The protected `init` method is invoked once the system is ready. + +### Initialization Responsibilities + +A logger plugin should: + +1. Store or process buffered status logs +2. Record the binary process name +3. Establish connections to remote sinks +4. Validate configuration + +### Startup Sequence + +```mermaid +flowchart TD + Start["Process Start"] --> Flags["Parse Flags"] + Flags --> RegistryInit["Initialize Registry"] + RegistryInit --> ExtensionsLoad["Load Extensions"] + ExtensionsLoad --> ConfigLoad["Load Config"] + ConfigLoad --> LoggerInit["LoggerPlugin init"] + LoggerInit --> Runtime["Normal Operation"] +``` + +This design guarantees that no logs are lost during early startup. + +--- + +## Error Handling Model + +All logging methods return a `Status` object, which provides: + +- Success indication +- Error codes +- Failure messages + +Batch logging (`logStringBatch`) tracks individual event failures and returns a consolidated failure status if any entry fails. + +This enables the core to: + +- Detect logger failures +- Emit fallback logs +- Surface operational health issues + +--- + +## Design Characteristics + +### 1. Pluggable by Design +The logging layer is fully abstracted behind the Plugin interface. + +### 2. Backward Compatible +Default behavior relies only on `logString`, ensuring minimal implementation overhead. + +### 3. Feature Opt-In +Advanced capabilities are enabled explicitly via feature methods. + +### 4. Extension Friendly +Logger plugins can run in isolated extension processes. + +### 5. Performance Aware +Supports: + +- Snapshot specialization +- Event batch processing +- Direct event streaming + +--- + +## Relationship to Other Modules + +- Uses lifecycle guarantees from [Core Init And Runtime](core-init-and-runtime/core-init-and-runtime.md) +- Receives structured query output from [Query Execution And Logging](query-execution-and-logging/query-execution-and-logging.md) +- Can bypass storage in [Database Backends](database-backends/database-backends.md) +- May be deployed via [Extensions Framework](extensions-framework/extensions-framework.md) +- Integrates with event pipelines in [Events Core](events-core/events-core.md) + +The Plugin Interfaces And Logging module is therefore the central abstraction for exporting observability data from osquery into external ecosystems. + +--- + +## Summary + +The Plugin Interfaces And Logging module provides: + +- A standardized logging interface (`LoggerPlugin`) +- A structured status log representation (`StatusLogLine`) +- Optional advanced feature negotiation +- Tight integration with query, event, and extension systems + +By decoupling log production from log transport, this module enables flexible deployment scenarios ranging from local file logging to enterprise-scale distributed telemetry pipelines. \ No newline at end of file diff --git a/docs/reference/architecture/process-and-profiler/process-and-profiler.md b/docs/reference/architecture/process-and-profiler/process-and-profiler.md new file mode 100644 index 00000000000..08f10ae5c9a --- /dev/null +++ b/docs/reference/architecture/process-and-profiler/process-and-profiler.md @@ -0,0 +1,283 @@ +# Process And Profiler + +## Overview + +The **Process And Profiler** module provides low-level POSIX process management and runtime profiling capabilities for the osquery runtime. It is responsible for: + +- Spawning and managing worker and extension processes +- Handling process lifecycle events (fork, exec, signals, wait) +- Graceful and forced process termination +- Resource usage measurement (CPU, memory, I/O) +- Wall-clock timing and performance instrumentation + +This module acts as a foundational runtime layer that supports higher-level components such as: + +- [Extensions Framework](../extensions-framework/extensions-framework.md) +- [SQL Core And Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) +- [Query Execution And Logging](../query-execution-and-logging/query-execution-and-logging.md) + +It bridges operating system primitives (signals, `fork`, `exec`, `waitpid`, `getrusage`) with osquery’s internal abstractions. + +--- + +## Architectural Overview + +The module is composed of two main responsibilities: + +1. **Process Lifecycle Management** (POSIX layer) +2. **Runtime Resource Profiling** (CodeProfiler) + +```mermaid +flowchart TD + subgraph process_layer["Process Management Layer"] + PlatformProcess["PlatformProcess"] + Signals["POSIX Signals"] + ForkExec["fork() / execve()"] + WaitPid["waitpid()"] + end + + subgraph profiling_layer["Profiling Layer"] + CodeProfiler["CodeProfiler"] + Rusage["getrusage()"] + Timeval["timeval"] + Monitoring["Numeric Monitoring"] + end + + PlatformProcess -->|"spawns"| ForkExec + PlatformProcess -->|"controls"| Signals + PlatformProcess -->|"monitors"| WaitPid + + CodeProfiler -->|"captures"| Rusage + CodeProfiler -->|"converts"| Timeval + CodeProfiler -->|"records"| Monitoring +``` + +The **PlatformProcess** abstraction encapsulates process operations, while **CodeProfiler** instruments execution scopes to capture resource metrics. + +--- + +## Process Management + +### PlatformProcess + +Core component: + +- `osquery.osquery.process.posix.process.sigaction` + +The `PlatformProcess` class encapsulates a native POSIX process identifier and exposes lifecycle operations. + +### Responsibilities + +- Track process identity (`pid()`) +- Launch workers and extensions +- Send signals (SIGTERM, SIGKILL, SIGUSR1) +- Check process state via `waitpid` +- Reset signal handlers for child processes + +### Process Lifecycle + +```mermaid +flowchart TD + Parent["Parent Process"] -->|"fork()"| Child["Child Process"] + Child -->|"set environment"| Env["OSQUERY_WORKER or OSQUERY_EXTENSION"] + Child -->|"reset signals via sigaction"| Reset["Default Signal Handlers"] + Child -->|"execve()"| Executable["Worker or Extension Binary"] + Parent -->|"waitpid()"| Status["ProcessState"] + Parent -->|"SIGTERM or SIGKILL"| Child +``` + +### Launching Workers + +`launchWorker()`: + +- Calls `fork()` +- Sets `OSQUERY_WORKER` environment variable +- Replaces child image with `execve()` +- Returns a `PlatformProcess` wrapper for the parent + +Workers are used to isolate query execution and runtime logic from the supervising daemon. + +### Launching Extensions + +`launchExtension()`: + +- Forks a child +- Sets `OSQUERY_EXTENSION` environment variable +- Resets all signal handlers using `sigaction` +- Constructs CLI arguments (`--socket`, `--timeout`, `--interval`, `--verbose`) +- Calls `execve()` + +Resetting signals ensures extension processes do not inherit unsafe handlers from the parent. + +This directly supports the [Extensions Framework](../extensions-framework/extensions-framework.md). + +### Process State Monitoring + +`checkStatus()` uses: + +- `waitpid(..., WNOHANG)` +- Exit status macros (`WIFEXITED`, `WIFSIGNALED`, `WEXITSTATUS`) + +It returns a `ProcessState` enum: + +- `PROCESS_STILL_ALIVE` +- `PROCESS_EXITED` +- `PROCESS_ERROR` +- `PROCESS_STATE_CHANGE` + +This mechanism is critical for supervising long-running workers and extensions. + +--- + +## Runtime Profiling + +### CodeProfiler + +Core components: + +- `osquery.osquery.profiler.posix.code_profiler.rusage` +- `osquery.osquery.profiler.posix.code_profiler.timeval` + +`CodeProfiler` is an RAII-based instrumentation utility. It captures system resource usage at construction and computes deltas at destruction. + +### Profiling Model + +```mermaid +flowchart TD + Start["CodeProfiler Constructor"] --> CaptureStart["call getrusage()"] + CaptureStart --> StoreStart["Store rusage + wall time"] + StoreStart --> Execution["Instrumented Code Executes"] + Execution --> End["Destructor"] + End --> CaptureEnd["call getrusage()"] + CaptureEnd --> Diff["Compute Differences"] + Diff --> Record["Record Metrics"] +``` + +The profiler captures: + +- Maximum RSS (`ru_maxrss`) +- RSS increase +- Input blocks (`ru_inblock`) +- Output blocks (`ru_oublock`) +- User CPU time (`ru_utime`) +- System CPU time (`ru_stime`) +- Total CPU time (derived) +- Wall clock duration + +### Rusage Granularity + +On Linux: + +- Uses `RUSAGE_THREAD` for per-thread granularity + +On other POSIX systems: + +- Uses `RUSAGE_SELF` + +This ensures more precise profiling where supported. + +### Time Conversion + +`timeval` structures are converted into milliseconds using: + +- Seconds (`tv_sec`) +- Microseconds (`tv_usec`) +- `std::chrono` duration casting + +The module normalizes all time metrics to millisecond precision for consistent monitoring. + +### Metric Recording + +Metrics are recorded via the numeric monitoring subsystem: + +- Pre-aggregation: `Min` +- Pre-aggregation: `Sum` +- Entity naming: `"."` + +For example: + +- `query.time.user.millis` +- `query.time.wall.millis` +- `query.rss.increase.kb` + +These metrics are consumed by performance dashboards and internal telemetry. + +--- + +## Interaction With Other Modules + +### SQL Core And Virtual Tables + +Query execution paths instrument expensive operations using `CodeProfiler` to measure: + +- Query latency +- CPU usage +- Memory growth + +See: [SQL Core And Virtual Tables](../sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) + +### Query Execution And Logging + +Profiling results are often correlated with structured query logs for performance analysis. + +See: [Query Execution And Logging](../query-execution-and-logging/query-execution-and-logging.md) + +### Extensions Framework + +`PlatformProcess` is responsible for launching and supervising extension binaries. + +See: [Extensions Framework](../extensions-framework/extensions-framework.md) + +--- + +## Error Handling Strategy + +### Process Errors + +- `fork()` failure returns null process wrapper +- `execve()` failure logs error and exits child +- `waitpid()` errors are mapped to `PROCESS_ERROR` + +### Profiling Errors + +- `getrusage()` errors are wrapped in an `Expected` type +- Fatal errors log descriptive messages +- Unsupported fields are traced but not fatal + +Overflow and negative deltas are guarded to prevent invalid metric reporting. + +--- + +## Design Characteristics + +### RAII-Based Instrumentation + +`CodeProfiler` automatically measures duration between construction and destruction, reducing the risk of missing instrumentation cleanup. + +### POSIX Abstraction Layer + +`PlatformProcess` encapsulates raw system calls, allowing the rest of the codebase to operate on a stable abstraction. + +### Isolation-Oriented Runtime Model + +Worker and extension processes are intentionally isolated: + +- Separate address spaces +- Independent signal handling +- Controlled environment variables + +This improves fault tolerance and security. + +--- + +## Summary + +The **Process And Profiler** module provides: + +- Safe and controlled process lifecycle management +- Worker and extension process orchestration +- Precise resource usage measurement +- Millisecond-level wall time tracking +- Integration with osquery’s monitoring infrastructure + +It serves as a foundational runtime component that enables robust execution, isolation, and performance visibility across the osquery system. \ No newline at end of file diff --git a/docs/reference/architecture/query-execution-and-logging/query-execution-and-logging.md b/docs/reference/architecture/query-execution-and-logging/query-execution-and-logging.md new file mode 100644 index 00000000000..4f89fb15e61 --- /dev/null +++ b/docs/reference/architecture/query-execution-and-logging/query-execution-and-logging.md @@ -0,0 +1,348 @@ +# Query Execution And Logging + +The Query Execution And Logging module is responsible for transforming raw SQL execution results into structured, state-aware, and loggable artifacts. It bridges the SQL engine, scheduled query framework, persistent storage layer, and logging plugins by: + +- Tracking historical query results +- Computing differential results between executions +- Managing epochs and execution counters +- Serializing results into structured JSON formats +- Preparing log items for upstream logging systems + +At the center of this module are two primary abstractions: + +- `QueryLogItem` β€” the log-ready representation of a query execution +- `Query` β€” the stateful interface to historical query storage and diff computation + +This module integrates closely with: + +- [SQL Core And Virtual Tables](sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) +- [Database Backends](database-backends/database-backends.md) +- [Plugin Interfaces And Logging](plugin-interfaces-and-logging/plugin-interfaces-and-logging.md) +- [Config And Packs](config-and-packs/config-and-packs.md) +- [Events Core](events-core/events-core.md) + +--- + +## Architectural Overview + +The Query Execution And Logging module sits between query execution and the logging pipeline. + +```mermaid +flowchart LR + Scheduler["Scheduled Query"] --> SQL["SQL Execution Engine"] + SQL --> Results["QueryDataTyped"] + Results --> QueryObj["Query"] + QueryObj --> Diff["DiffResults"] + QueryObj --> DB["Persistent Storage"] + Diff --> LogItem["QueryLogItem"] + LogItem --> Serializer["JSON Serialization"] + Serializer --> Logger["Logger Plugins"] +``` + +### Responsibilities by Stage + +1. **Execution** β€” SQL engine returns raw result rows. +2. **State Management** β€” `Query` retrieves previous results from persistent storage. +3. **Diff Computation** β€” New results are compared with stored results. +4. **Metadata Enrichment** β€” Time, epoch, counter, identifier, and decorations are added. +5. **Serialization** β€” `QueryLogItem` is converted into structured JSON. +6. **Logging** β€” Output is passed to logger plugins for transport. + +--- + +## Core Component: QueryLogItem + +`QueryLogItem` represents a fully-formed query execution log record. + +It encapsulates: + +- Differential or snapshot results +- Query metadata +- Execution timing +- Epoch and invocation counter +- Host identifier +- Custom decorations + +### Data Model + +```mermaid +flowchart TD + QueryLogItem["QueryLogItem"] + QueryLogItem --> IsSnapshot["isSnapshot"] + QueryLogItem --> DiffRes["DiffResults"] + QueryLogItem --> Snapshot["snapshot_results"] + QueryLogItem --> Name["name"] + QueryLogItem --> Identifier["identifier"] + QueryLogItem --> Time["time"] + QueryLogItem --> Epoch["epoch"] + QueryLogItem --> Counter["counter"] + QueryLogItem --> CalendarTime["calendar_time"] + QueryLogItem --> Decorations["decorations"] +``` + +### Snapshot vs Differential Modes + +- **Snapshot mode** (`isSnapshot = true`): + - Entire result set is emitted. + - Used for ad-hoc queries or first executions. + +- **Differential mode** (`isSnapshot = false`): + - Only added and removed rows are logged. + - Reduces log volume and highlights changes. + - Uses `DiffResults` from the SQL core module. + +See [SQL Core And Virtual Tables](sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) for details on `DiffResults`. + +--- + +## Query Class: Historical State And Diff Engine + +The `Query` class manages: + +- Historical storage of query results +- Epoch tracking +- Invocation counters +- Differential computation + +It provides the persistence-backed execution state for scheduled queries. + +### Lifecycle Of A Scheduled Query Execution + +```mermaid +sequenceDiagram + participant Scheduler + participant QueryObj as Query + participant DB as Database + participant Logger + + Scheduler->>QueryObj: addNewResults(qd, epoch) + QueryObj->>DB: getPreviousQueryResults() + QueryObj->>QueryObj: compute DiffResults + QueryObj->>DB: saveQueryResults(json, epoch) + QueryObj-->>Scheduler: DiffResults + counter + Scheduler->>Logger: serialize QueryLogItem +``` + +--- + +## Persistent Storage Integration + +The module relies on the Database Backends module for durable state. + +Key interactions: + +- Retrieve previous results +- Store current results +- Store epoch +- Maintain query execution counters + +```mermaid +flowchart TD + Query["Query"] --> GetPrev["getPreviousQueryResults()"] + Query --> Save["saveQueryResults()"] + Query --> Epoch["getPreviousEpoch()"] + Query --> Counter["getQueryCounter()"] + GetPrev --> DB["OsqueryDatabase"] + Save --> DB + Epoch --> DB + Counter --> DB +``` + +See [Database Backends](database-backends/database-backends.md) for storage engine details. + +--- + +## Epoch And Counter Semantics + +Two mechanisms guarantee correctness and replay safety: + +### Epoch + +- Represents a configuration generation. +- Changes when configuration changes. +- Forces fresh result evaluation. + +### Counter + +- Tracks how many times a query executed within an epoch. +- Automatically increments per execution. +- Resets depending on query reset semantics. + +```mermaid +flowchart LR + ConfigChange["Config Change"] --> EpochInc["Epoch Increment"] + EpochInc --> Fresh["Fresh Results"] + Fresh --> CounterReset["Counter Reset"] + NoChange["No Config Change"] --> CounterInc["Counter Increment"] +``` + +Epoch coordination is driven by configuration updates from: + +- [Config And Packs](config-and-packs/config-and-packs.md) + +--- + +## Differential Result Computation + +When new results are available: + +1. Previous results are retrieved from storage. +2. Both old and new sets are normalized. +3. A diff is computed. +4. Added and removed rows are captured in `DiffResults`. + +```mermaid +flowchart TD + NewResults["Current Results"] + OldResults["Stored Results"] + NewResults --> DiffEngine["Diff Engine"] + OldResults --> DiffEngine + DiffEngine --> Added["Added Rows"] + DiffEngine --> Removed["Removed Rows"] +``` + +The diff logic leverages data structures from the SQL core layer. + +See [SQL Core And Virtual Tables](sql-core-and-virtual-tables/sql-core-and-virtual-tables.md). + +--- + +## Serialization Paths + +The module supports multiple serialization formats. + +### Standard Log Serialization + +- `serializeQueryLogItem` +- `serializeQueryLogItemJSON` + +Produces a JSON object containing: + +- Metadata +- Differential or snapshot results +- Decorations + +### Events-Based Serialization + +- `serializeQueryLogItemAsEvents` +- `serializeQueryLogItemAsEventsJSON` + +Used for event-style queries where each row represents an action. + +```mermaid +flowchart LR + QueryLogItem["QueryLogItem"] --> Mode{"Serialization Mode"} + Mode -->|"Standard"| Standard["Single JSON Object"] + Mode -->|"Events"| Events["Multiple Event JSON Objects"] +``` + +Serialized output is consumed by logging plugins in: + +- [Plugin Interfaces And Logging](plugin-interfaces-and-logging/plugin-interfaces-and-logging.md) + +--- + +## Interaction With Scheduled Queries + +Scheduled queries are defined in the configuration layer and represented by `ScheduledQuery` objects. + +The Query Execution And Logging module: + +- Uses query name as storage key +- Detects SQL changes via `isNewQuerySql()` +- Determines first run via `getQueryStatus()` +- Applies reset semantics + +```mermaid +flowchart TD + ScheduledQuery["ScheduledQuery"] --> QueryObj["Query"] + QueryObj --> CheckSql["isNewQuerySql()"] + QueryObj --> StatusCheck["getQueryStatus()"] + QueryObj --> CounterLogic["incrementCounter()"] +``` + +See [Config And Packs](config-and-packs/config-and-packs.md) for how scheduled queries are defined and refreshed. + +--- + +## Events-Based Queries + +For event-driven queries (from the Events Core module): + +- Results accumulate over time. +- `addNewEvents()` is used instead of `addNewResults()`. +- Diff semantics may differ depending on event expiration. + +```mermaid +flowchart LR + EventSubscriber["Event Subscriber"] --> EventRows["GenerateRowsResult"] + EventRows --> QueryObj["Query.addNewEvents()"] + QueryObj --> Diff["DiffResults"] + Diff --> LogItem["QueryLogItem"] +``` + +See [Events Core](events-core/events-core.md) for event subscription and row generation. + +--- + +## Query Storage Maintenance + +The module provides utilities for: + +- Listing stored query names +- Checking existence in storage +- Retrieving current results + +```mermaid +flowchart TD + Maintenance["Maintenance Operations"] + Maintenance --> Names["getStoredQueryNames()"] + Maintenance --> Exists["isQueryNameInDatabase()"] + Maintenance --> Current["getCurrentResults()"] +``` + +This enables database cleanup and migration logic. + +--- + +## End-To-End Data Flow + +```mermaid +flowchart TD + Config["Configuration"] --> Scheduler["Scheduler"] + Scheduler --> SQL["SQL Engine"] + SQL --> Raw["Raw Results"] + Raw --> QueryObj["Query"] + QueryObj --> Diff["DiffResults"] + QueryObj --> Persist["Persistent Storage"] + Diff --> LogItem["QueryLogItem"] + LogItem --> JSON["Serialization"] + JSON --> Logger["Logger Plugins"] +``` + +This pipeline ensures: + +- Deterministic result tracking +- Efficient differential logging +- Accurate epoch-based resets +- Pluggable logging backends + +--- + +## How This Module Fits In The Overall System + +The Query Execution And Logging module is the stateful boundary between execution and observability. + +- It depends on SQL execution for result generation. +- It depends on database backends for persistent state. +- It feeds logger plugins with structured, serialized output. +- It responds to configuration and epoch changes. + +Without this module: + +- Scheduled queries would lack historical context. +- Logging volume would be excessive (no diffs). +- Epoch resets would not propagate cleanly. +- Distributed and event-based logging would be inconsistent. + +It is therefore a critical coordination layer that ensures correctness, efficiency, and structured output across the entire query lifecycle. \ No newline at end of file diff --git a/docs/reference/architecture/remote-http-client/remote-http-client.md b/docs/reference/architecture/remote-http-client/remote-http-client.md new file mode 100644 index 00000000000..d10cb230a40 --- /dev/null +++ b/docs/reference/architecture/remote-http-client/remote-http-client.md @@ -0,0 +1,328 @@ +# Remote Http Client + +The **Remote Http Client** module provides a general-purpose HTTP and HTTPS client implementation built on top of Boost.Asio, Boost.Beast, and OpenSSL. It is responsible for all outbound network communication that requires HTTP semantics, including secure TLS connections, proxy support, timeout handling, and request/response abstraction. + +This module acts as the networking backbone for features such as: + +- Distributed query transport (see [Distributed Querying](distributed-querying/distributed-querying.md)) +- Remote configuration retrieval (see [Config and Packs](config-and-packs/config-and-packs.md)) +- TLS-based logging plugins (see [Plugin Interfaces and Logging](plugin-interfaces-and-logging/plugin-interfaces-and-logging.md)) + +It encapsulates low-level socket management and asynchronous operations behind a simple request/response API. + +--- + +## 1. Purpose and Responsibilities + +The Remote Http Client module is designed to: + +- Provide HTTP methods: `GET`, `POST`, `PUT`, `HEAD`, `DELETE` +- Support both plain HTTP and HTTPS (TLS) connections +- Enforce configurable TLS behavior and certificate validation +- Manage asynchronous networking with timeouts +- Support proxy connections +- Provide request/response abstractions with URI parsing +- Offer structured access to headers and response metadata + +It ensures that higher-level modules (such as distributed querying or configuration fetching) do not need to interact directly with Boost or OpenSSL primitives. + +--- + +## 2. High-Level Architecture + +The module is centered around three primary abstractions: + +- `Client` – orchestrates connection lifecycle and request execution +- `HTTP_Request` – extends Boost.Beast request with URI parsing and header helpers +- `HTTP_Response` – extends Boost.Beast response with structured accessors + +### 2.1 Component Overview + +```mermaid +flowchart LR + Caller["Distributed Querying or Config Module"] --> Client["Client"] + Client --> Request["HTTP_Request"] + Client --> Response["HTTP_Response"] + Client --> Asio["Boost.Asio io_context"] + Client --> TLS["OpenSSL TLS Layer"] + Client --> Beast["Boost.Beast HTTP Engine"] +``` + +### 2.2 Internal Client Composition + +```mermaid +flowchart TD + Client["Client"] --> Options["Client::Options"] + Client --> Resolver["TCP Resolver"] + Client --> Socket["TCP Socket"] + Client --> SSLStream["SSL Stream (optional)"] + Client --> Timer["Deadline Timer"] + Client --> IOContext["io_context"] +``` + +The `Client` coordinates networking through: + +- `boost::asio::io_context` +- `boost::asio::ip::tcp::resolver` +- `boost::asio::ip::tcp::socket` +- Optional `ssl_stream` wrapping the TCP socket +- `boost::asio::deadline_timer` for request timeouts + +--- + +## 3. Client Class + +### 3.1 Client Options + +The `Client::Options` class drives behavior and security posture. + +Key configuration dimensions: + +- TLS enablement (`ssl_connection_`) +- Certificate validation (`always_verify_peer_`) +- Cipher selection +- Client certificate and private key +- Verify path +- Proxy hostname +- Remote hostname and port overrides +- Timeout configuration +- Redirect following +- Keep-alive support + +Options comparison is implemented via `operator==`, allowing detection of configuration changes and connection reinitialization. + +### 3.2 Connection Lifecycle + +The lifecycle of a typical request is: + +```mermaid +flowchart TD + Start["Request Method Invoked"] --> Init["initHTTPRequest()"] + Init --> Connect["createConnection()"] + Connect --> TLSCheck{"TLS Enabled?"} + TLSCheck -->|"Yes"| Encrypt["encryptConnection()"] + TLSCheck -->|"No"| Send["sendHTTPRequest()"] + Encrypt --> Send + Send --> Write["async_write"] + Write --> Read["async_read"] + Read --> Post["postResponseHandler()"] + Post --> End["Response Returned"] +``` + +### 3.3 Asynchronous Operation Wrapper + +All network operations are wrapped via `callNetworkOperation()`: + +Responsibilities: + +1. Start timeout timer +2. Trigger asynchronous network operation +3. Wait for either completion or timeout +4. Cancel timer and propagate error + +This design ensures: + +- Deterministic timeout enforcement +- Safe error propagation via `boost::system::error_code` +- Centralized network error management + +--- + +## 4. HTTP_Request Abstraction + +`HTTP_Request` extends the underlying Boost.Beast request with: + +- URI parsing via `osquery::Uri` +- Accessors for host, port, path, and protocol +- Header helper via `operator<<` + +### 4.1 URI Handling + +The request stores a parsed `Uri` and exposes: + +- `remoteHost()` +- `remotePort()` +- `remotePath()` +- `protocol()` + +This separation allows: + +- Clean redirect handling +- Transparent switching between HTTP and HTTPS +- Automatic host and path resolution + +### 4.2 Header Helper + +The nested `Header` struct enables intuitive header insertion: + +```text +Request req("https://example.com/api"); +req << Request::Header("Content-Type", "application/json"); +``` + +Internally this calls the underlying Beast `set()` method. + +--- + +## 5. HTTP_Response Abstraction + +`HTTP_Response` enhances the Beast response with: + +- `status()` – numeric HTTP status +- `body()` – response body +- `headers()` – iterable headers container + +### 5.1 Header Iteration + +The `Headers` and `Iterator` helper classes allow structured access: + +```text +for (const auto& header : resp.headers()) { + header.first; // name + header.second; // value +} +``` + +This abstraction shields higher-level modules from Boost-specific iterator types. + +--- + +## 6. TLS and Security Model + +The Remote Http Client enforces strong security defaults: + +- SSLv2 and SSLv3 disabled +- MD5 disabled +- Deprecated OpenSSL APIs disabled +- Optional peer verification +- Configurable cipher suites +- Optional client certificate authentication + +### 6.1 TLS Flow + +```mermaid +flowchart TD + TCPConnect["TCP Connect"] --> Wrap["Wrap in SSL Stream"] + Wrap --> Handshake["async_handshake"] + Handshake --> Verify{"Verify Peer?"} + Verify -->|"Yes"| Validate["Certificate Validation"] + Verify -->|"No"| Continue["Proceed"] + Validate --> Continue + Continue --> Ready["Secure Channel Ready"] +``` + +Special handling exists for TLS short-read conditions where a remote peer does not properly shut down the TLS session. + +--- + +## 7. Timeout and Error Handling + +Timeouts are enforced using `boost::asio::deadline_timer`. + +Error flow: + +```mermaid +flowchart TD + AsyncOp["Async Network Operation"] --> Complete{"Completed?"} + Complete -->|"Yes"| CancelTimer["Cancel Timer"] + Complete -->|"No"| Timeout["Timer Fires"] + Timeout --> CancelSocket["Close Socket"] + CancelSocket --> Error["Set error_code"] + CancelTimer --> Return["Return Response or Error"] + Error --> Return +``` + +This ensures: + +- No hung connections +- Proper resource cleanup +- Deterministic behavior under network instability + +--- + +## 8. Integration with Other Modules + +### 8.1 Distributed Querying + +The [Distributed Querying](distributed-querying/distributed-querying.md) module uses the Remote Http Client to: + +- Fetch remote queries +- Submit query results +- Communicate with TLS endpoints + +The `TLSDistributedPlugin` depends on secure HTTP transport provided by this module. + +### 8.2 Config and Packs + +The [Config and Packs](config-and-packs/config-and-packs.md) module may retrieve configuration over HTTPS using this client. + +The HTTP abstraction ensures: + +- Clean retry behavior +- TLS verification +- Controlled timeouts + +### 8.3 Logging Plugins + +TLS-based logging (see [Plugin Interfaces and Logging](plugin-interfaces-and-logging/plugin-interfaces-and-logging.md)) relies on the client to: + +- Deliver logs to remote collectors +- Enforce TLS certificate verification + +--- + +## 9. Interaction with Core Runtime + +The module operates within the broader runtime managed by [Core Init and Runtime](core-init-and-runtime/core-init-and-runtime.md). + +```mermaid +flowchart LR + Runtime["Core Runtime"] --> Distributed["Distributed Querying"] + Runtime --> Config["Config and Packs"] + Distributed --> HTTP["Remote Http Client"] + Config --> HTTP +``` + +The HTTP client itself does not manage threading beyond its `io_context`, making it safe to use from higher-level schedulers and execution engines. + +--- + +## 10. Design Principles + +### 10.1 Encapsulation of Networking Complexity + +All Boost and OpenSSL details are isolated inside this module. Higher-level modules interact only with: + +- `Request` +- `Response` +- `Client` + +### 10.2 Security-First Defaults + +- Legacy TLS protocols disabled +- Strong cipher configuration supported +- Certificate validation configurable but explicit + +### 10.3 Deterministic Resource Management + +- Explicit socket closing +- Destructor calls `closeSocket()` +- Centralized timeout logic + +### 10.4 Extensibility + +Template-based `HTTP_Request` and `HTTP_Response` allow reuse with different Beast message types while maintaining abstraction. + +--- + +## 11. Summary + +The **Remote Http Client** module is the secure, asynchronous networking foundation of the system. It: + +- Abstracts Boost.Asio and Boost.Beast +- Provides structured HTTP request/response APIs +- Enforces TLS security +- Handles timeouts and connection lifecycle +- Enables distributed querying, remote configuration, and TLS logging + +By isolating HTTP and TLS complexity into a single, well-defined module, the system achieves strong separation of concerns, improved maintainability, and consistent security guarantees across all outbound communications. diff --git a/docs/reference/architecture/sql-core-and-virtual-tables/sql-core-and-virtual-tables.md b/docs/reference/architecture/sql-core-and-virtual-tables/sql-core-and-virtual-tables.md new file mode 100644 index 00000000000..e967c7a353a --- /dev/null +++ b/docs/reference/architecture/sql-core-and-virtual-tables/sql-core-and-virtual-tables.md @@ -0,0 +1,369 @@ +# Sql Core And Virtual Tables + +## Overview + +The **Sql Core And Virtual Tables** module implements the internal SQL execution engine for osquery. It provides: + +- An embedded SQLite runtime with strict authorization controls +- A virtual table abstraction over osquery TablePlugin implementations +- Query planning and column type inference +- Scheduled query metadata and performance tracking +- Result diffing utilities for stateful query comparison + +This module is the foundation of how osquery turns system data into relational tables and executes SQL queries against them. + +--- + +## High-Level Architecture + +At runtime, the module layers osquery-specific abstractions on top of SQLite: + +```mermaid +flowchart TD + UserQuery["SQL Query"] --> SQLInternal["SQLInternal"] + SQLInternal --> SQLiteSQLPlugin["SQLiteSQLPlugin"] + SQLiteSQLPlugin --> DBManager["SQLiteDBManager"] + DBManager --> SQLiteDB["sqlite3 Database"] + + SQLiteDB --> VTableModule["sqlite3_module"] + VTableModule --> VirtualTable["VirtualTable"] + VirtualTable --> TablePlugin["TablePlugin"] + + TablePlugin --> QueryContext["QueryContext"] + QueryContext --> ConstraintList["ConstraintList"] +``` + +### Key Layers + +1. **SQLInternal / SQLiteSQLPlugin** – Entry points for executing queries. +2. **SQLiteDBManager / SQLiteDBInstance** – Lifecycle and concurrency control for SQLite connections. +3. **VirtualTable / sqlite3_module** – SQLite virtual table glue code. +4. **TablePlugin / QueryContext** – Table implementations and constraint-aware row generation. +5. **DiffResults / QueryPerformance / ScheduledQuery** – Query lifecycle and result management. + +--- + +## Core Components + +### SQLInternal + +`SQLInternal` executes a query directly against the internal SQLite engine and exposes: + +- Typed result rows (`QueryDataTyped`) +- Execution status +- Event-based table detection +- Result size computation +- Output escaping via `StringEscaperVisitor` + +It is typically used by higher-level systems such as the scheduler or distributed query executor. + +--- + +### SQLiteSQLPlugin + +`SQLiteSQLPlugin` is the internal SQL registry provider. It: + +- Executes SQL queries +- Attaches and detaches virtual tables +- Retrieves query column metadata +- Enumerates tables used in a query via `QueryPlanner` + +It acts as the bridge between osquery's plugin system and SQLite. + +--- + +### SQLiteDBManager and SQLiteDBInstance + +These classes manage SQLite lifecycle and concurrency. + +#### SQLiteDBManager + +- Owns the primary in-memory SQLite database +- Attaches all registered virtual tables +- Handles enable/disable table flags +- Provides transient connections under contention + +#### SQLiteDBInstance + +- RAII wrapper around `sqlite3*` +- Tracks affected tables per query +- Controls warm query cache usage +- Applies a strict SQLite authorizer + +Security is enforced through an allowlist of: + +- SQLite action codes +- SQLite PRAGMA statements + +Any non-allowlisted action is denied at prepare time. + +--- + +## Virtual Table Integration + +The Virtual Table layer connects SQLite to osquery TablePlugin implementations. + +```mermaid +flowchart LR + SQLite["SQLite Engine"] --> xBestIndex["xBestIndex"] + SQLite --> xFilter["xFilter"] + SQLite --> xNext["xNext"] + SQLite --> xColumn["xColumn"] + + xBestIndex --> ConstraintMap["ConstraintMap"] + xFilter --> QueryContextVT["QueryContext"] + QueryContextVT --> TablePluginImpl["TablePlugin::generate"] + TablePluginImpl --> Rows["TableRows"] +``` + +### VirtualTable + +`VirtualTable` wraps: + +- SQLite `sqlite3_vtab` +- Shared `VirtualTableContent` +- Associated `SQLiteDBInstance` + +### BaseCursor + +`BaseCursor` manages: + +- Row iteration +- Generator-based row streaming +- Cursor position and state + +### sqlite3_module + +Defined in `virtual_table.cpp`, this structure provides SQLite callbacks: + +- `xCreate` +- `xBestIndex` +- `xFilter` +- `xNext` +- `xColumn` +- `xRowid` +- `xUpdate` (for extension-backed writable tables) + +These callbacks translate SQLite operations into `TablePlugin` calls. + +--- + +## Constraint and QueryContext Model + +The constraint system enables predicate pushdown and efficient table scanning. + +### Constraint + +Represents a single operator-expression pair: + +- Operator (e.g., EQUALS, LIKE, GREATER_THAN) +- Expression string + +### ConstraintList + +For each column: + +- Stores multiple constraints +- Evaluates matching expressions +- Supports operator filtering +- Handles affinity-aware comparisons + +### QueryContext + +`QueryContext` is passed to every table generator. It contains: + +- Column-to-constraint mappings +- Used column tracking +- Cache control flags +- Table-level metadata + +It enables: + +- Required constraint enforcement +- Selective column population +- Efficient index usage +- Constraint expansion (e.g., globbing) + +--- + +## Query Planning and Type Inference + +### QueryPlanner + +`QueryPlanner` executes: + +- `EXPLAIN QUERY PLAN` +- `EXPLAIN` + +It extracts: + +- Tables referenced +- SQLite opcodes +- Inferred result column types + +A map of specific SQLite opcodes to result types enables type inference for expression-based columns. + +```mermaid +flowchart TD + Query["SQL Query"] --> Explain["EXPLAIN"] + Explain --> Opcodes["SQLite Opcodes"] + Opcodes --> TypeMap["Opcode to ColumnType Map"] + TypeMap --> Columns["Resolved TableColumns"] +``` + +This avoids treating expression columns as UNKNOWN whenever possible. + +--- + +## Scheduled Query Support + +### ScheduledQuery + +Represents metadata for a scheduled query: + +- Pack name +- Query name +- SQL string +- Execution interval +- Startup priority +- Snapshot mode +- Removed row reporting option + +Helper methods determine: + +- Whether the query is snapshot-based +- Whether removed rows should be reported + +This structure is used by the scheduler in conjunction with the query execution and logging subsystems. + +--- + +## Result Diffing + +### DiffResults + +`DiffResults` represents the difference between: + +- Previous query results +- Current query results + +It contains: + +- `added` rows +- `removed` rows + +It supports: + +- Equality comparison +- Empty diff detection +- JSON serialization + +```mermaid +flowchart LR + OldResults["Old QueryData"] --> DiffFunc["diff()"] + NewResults["New QueryData"] --> DiffFunc + DiffFunc --> Added["Added Rows"] + DiffFunc --> Removed["Removed Rows"] +``` + +This is critical for scheduled queries that operate in differential mode. + +--- + +## Query Performance Tracking + +### QueryPerformance + +Tracks execution metrics across runs: + +- Execution count +- Last execution timestamp +- Wall time (total and last) +- User and system CPU time +- Memory usage +- Output size + +It supports: + +- CSV serialization +- Equality comparison + +This enables persistent performance analysis and tuning. + +--- + +## Caching Model + +The module implements two levels of caching: + +1. **Table-level caching** via `TablePlugin`: + - Controlled by scheduled interval + - Avoids recomputation for identical intervals + +2. **QueryContext in-memory cache**: + - Per-query temporary cache + - Optimizes repeated filter operations + +Cache usage is coordinated via: + +- `SQLiteDBInstance::useCache` +- `QueryContext::useCache` +- Table attribute flags (e.g., CACHEABLE) + +--- + +## Security Model + +The Sql Core And Virtual Tables module enforces strict SQLite restrictions: + +- Only allowlisted action codes are permitted +- `SQLITE_ATTACH` is explicitly disallowed +- Only allowlisted PRAGMA statements are permitted +- Virtual tables are attached under controlled mutex locks + +This prevents arbitrary file writes, unsafe pragmas, and unintended schema manipulation. + +--- + +## End-to-End Execution Flow + +```mermaid +flowchart TD + Input["Incoming SQL"] --> SQLInternalExec["SQLInternal"] + SQLInternalExec --> DBConn["SQLiteDBManager::get()"] + DBConn --> Prepare["sqlite3_prepare_v2"] + Prepare --> Authorizer["sqliteAuthorizer"] + Prepare --> VTableCallbacks["Virtual Table Callbacks"] + VTableCallbacks --> TableGen["TablePlugin::generate"] + TableGen --> Results["QueryDataTyped"] + Results --> PostProcess["Escape and Size Calculation"] + PostProcess --> Output["Final Results"] +``` + +--- + +## Role Within the Overall System + +The Sql Core And Virtual Tables module: + +- Powers the interactive shell +- Executes scheduled queries +- Serves distributed query results +- Bridges extension-provided tables into the core SQL engine +- Enforces security boundaries for query execution + +It acts as the central data execution engine that transforms system state into relational query results. + +--- + +## Summary + +The **Sql Core And Virtual Tables** module combines: + +- A secured embedded SQLite engine +- A high-performance virtual table abstraction +- Constraint-aware query execution +- Query planning and type inference +- Result diffing and performance tracking + +It is the core execution layer that enables osquery to behave like a distributed, secure, and extensible SQL-based operating system instrumentation platform. diff --git a/docs/reference/architecture/system-utilities/system-utilities.md b/docs/reference/architecture/system-utilities/system-utilities.md new file mode 100644 index 00000000000..e6f3ebf60f5 --- /dev/null +++ b/docs/reference/architecture/system-utilities/system-utilities.md @@ -0,0 +1,298 @@ +# System Utilities + +The **System Utilities** module provides low-level helper primitives used across the osquery codebase. While small in scope, it plays a foundational role in enabling consistent behavior for: + +- Enum hashing compatibility across standard libraries +- Time conversion and formatting +- Cross-platform time normalization + +These utilities are intentionally lightweight and dependency-minimal so they can be safely reused across core systems such as SQL execution, scheduling, distributed querying, logging, and extensions. + +--- + +## Purpose and Design Principles + +The System Utilities module exists to: + +1. Provide safe, reusable abstractions over platform-specific behavior +2. Normalize time handling across the runtime +3. Offer compatibility fixes for C++ standard library inconsistencies +4. Avoid introducing heavy dependencies into core systems + +Because these utilities are used by multiple subsystems (SQL, scheduler, events, distributed querying, logging), correctness and determinism are critical. + +--- + +## Architecture Overview + +Although small, this module supports multiple high-level subsystems. + +```mermaid +flowchart TD + SystemUtilities["System Utilities"] + + EnumHash["EnumClassHash"] + TimeUtils["Time Utilities"] + + SQLCore["SQL Core and Virtual Tables"] + Scheduler["Scheduled Query Engine"] + Distributed["Distributed Querying"] + Logging["Query Execution and Logging"] + Events["Events Core"] + + SystemUtilities --> EnumHash + SystemUtilities --> TimeUtils + + TimeUtils --> Scheduler + TimeUtils --> Logging + TimeUtils --> Distributed + TimeUtils --> Events + + EnumHash --> SQLCore + EnumHash --> Scheduler +``` + +### Key Observations + +- **EnumClassHash** enables safe use of strongly-typed enums in hash-based containers. +- **Time utilities** ensure consistent UNIX epoch and ASCII time formatting across the system. +- The module introduces no runtime state and is purely functional. + +--- + +# Core Components + +## 1. EnumClassHash + +**Component:** `osquery.osquery.utils.enum_class_hash.EnumClassHash` + +### Problem Addressed + +Historically, certain versions of `libc++` and `libstdc++` did not properly support hashing of `enum class` types in unordered containers. + +This utility provides a small compatibility layer allowing `enum class` values to be used in: + +- `std::unordered_map` +- `std::unordered_set` +- Other hash-based STL containers + +### Implementation Strategy + +The implementation uses SFINAE (`std::enable_if`) and `std::is_enum` to restrict hashing only to enum types: + +```cpp +struct EnumClassHash { + template + typename std::enable_if::value, std::size_t>::type + operator()(EnumClassType t) const { + return static_cast(t); + } +}; +``` + +### Design Characteristics + +- Compile-time type restriction +- Zero runtime overhead +- No external dependencies +- Safe conversion via `static_cast` + +### Typical Usage Pattern + +```cpp +std::unordered_map map; +``` + +### Why This Matters + +Strongly-typed enums are widely used in: + +- SQL opcode handling +- Scheduler state machines +- Extension lifecycle states +- Event subscription types + +Without a stable hash implementation, these systems would require unsafe workarounds. + +--- + +## 2. Time Utilities + +**Component:** `osquery.osquery.utils.system.time.tm` + +This group of functions provides consistent time handling across platforms. + +### Responsibilities + +- Convert `struct tm` to UNIX epoch time +- Format time into human-readable ASCII +- Normalize local time to UTC +- Provide current system time + +--- + +## Time Utility Functions + +### platformAsctime + +Returns the ASCII version of a `struct tm` as a C++ string. + +Purpose: +- Wrap platform-specific `asctime` behavior +- Provide safe string-based output + +--- + +### toUnixTime + +Converts a `struct tm` into UNIX epoch time. + +```text +Input: struct tm +Output: uint64_t (seconds since UNIX epoch) +``` + +Used in: +- Query scheduling timestamps +- Event expiration logic +- Distributed result expiration + +--- + +### getUnixTime + +Returns the current time as seconds since the UNIX epoch. + +This is critical for: + +- Scheduled query execution +- Performance measurement +- Logging timestamps +- Expiration tracking + +--- + +### toAsciiTime + +Converts a UTC `struct tm` into a human-readable string. + +Format example: + +```text +Wed Sep 21 10:27:52 2011 +``` + +Used in: +- Query logs +- Status logs +- Debug output + +--- + +### toAsciiTimeUTC + +Converts a local `struct tm` into UTC ASCII format by: + +1. Converting to epoch +2. Applying `gmtime()` +3. Formatting into ASCII + +This guarantees consistent UTC logging across distributed environments. + +--- + +### getAsciiTime + +Returns the current time in human-readable ASCII format. + +Often used for: + +- Log lines +- Status reporting +- Diagnostics + +--- + +## Time Flow Across the System + +```mermaid +flowchart TD + Now["System Clock"] --> GetUnix["getUnixTime()"] + GetUnix --> Scheduler["Scheduled Queries"] + GetUnix --> Distributed["Distributed Expiration"] + GetUnix --> Events["Event Expiration"] + + Scheduler --> ToAscii["toAsciiTime()"] + Distributed --> ToAscii + Events --> ToAscii + + ToAscii --> Logs["Query and Status Logs"] +``` + +### Guarantees Provided + +- Unified epoch time source +- Deterministic formatting +- UTC normalization when required +- Platform-independent behavior + +--- + +# How System Utilities Fits Into the Overall Architecture + +The System Utilities module underpins multiple runtime layers: + +| Subsystem | Dependency on System Utilities | +|-----------|--------------------------------| +| SQL Core | Enum hashing for opcodes and states | +| Scheduler | UNIX time for interval execution | +| Distributed Querying | Timestamp-based result validity | +| Events Core | Expiration and event time tracking | +| Logging | Human-readable timestamps | +| Extensions Framework | Enum state hashing | + +It intentionally: + +- Does not manage state +- Does not perform I/O +- Does not depend on higher-level modules + +This ensures it can be safely included anywhere in the codebase. + +--- + +# Design Constraints and Trade-offs + +### Minimalism + +The module avoids abstractions that could: + +- Introduce dynamic allocation +- Create circular dependencies +- Increase compile-time coupling + +### Cross-Platform Safety + +Time conversion and formatting behave consistently across: + +- Linux +- macOS +- Windows + +### Backward Compatibility + +Enum hashing is implemented as a compatibility shim and may be removed once all standard libraries fully conform. + +--- + +# Summary + +The **System Utilities** module is a foundational, low-level component that ensures: + +- Reliable enum hashing across platforms +- Deterministic time conversion and formatting +- Consistent timestamp handling throughout osquery + +While small in size, it is deeply integrated into core subsystems including scheduling, SQL execution, logging, distributed querying, and events. + +Its simplicity, statelessness, and platform-neutral behavior make it a critical infrastructure layer supporting the entire runtime. From 47821fdc75d2fa3a376d6a2eec270eb688903b94 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Jun 2026 19:20:02 +0000 Subject: [PATCH 6/7] docs: Stage 3 - Tutorial generation (11 files) [skip ci] --- docs/development/.gitignore | 7 + docs/development/README.md | 115 ++++++++ docs/development/architecture/README.md | 233 ++++++++++++++++ docs/development/contributing/guidelines.md | 280 +++++++++++++++++++ docs/development/security/README.md | 258 ++++++++++++++++++ docs/development/setup/environment.md | 230 ++++++++++++++++ docs/development/setup/local-development.md | 273 +++++++++++++++++++ docs/development/testing/README.md | 283 ++++++++++++++++++++ docs/getting-started/.gitignore | 7 + docs/getting-started/first-steps.md | 248 +++++++++++++++++ docs/getting-started/introduction.md | 126 +++++++++ docs/getting-started/prerequisites.md | 159 +++++++++++ docs/getting-started/quick-start.md | 193 +++++++++++++ 13 files changed, 2412 insertions(+) create mode 100644 docs/development/.gitignore create mode 100644 docs/development/README.md create mode 100644 docs/development/architecture/README.md create mode 100644 docs/development/contributing/guidelines.md create mode 100644 docs/development/security/README.md create mode 100644 docs/development/setup/environment.md create mode 100644 docs/development/setup/local-development.md create mode 100644 docs/development/testing/README.md create mode 100644 docs/getting-started/.gitignore create mode 100644 docs/getting-started/first-steps.md create mode 100644 docs/getting-started/introduction.md create mode 100644 docs/getting-started/prerequisites.md create mode 100644 docs/getting-started/quick-start.md diff --git a/docs/development/.gitignore b/docs/development/.gitignore new file mode 100644 index 00000000000..a5d01be5236 --- /dev/null +++ b/docs/development/.gitignore @@ -0,0 +1,7 @@ +# VoltAgent temp files +temp/ + +# JSON intermediate files (except schema/config) +*.json +!*-schema.json +!*-config.json diff --git a/docs/development/README.md b/docs/development/README.md new file mode 100644 index 00000000000..f4ba310b324 --- /dev/null +++ b/docs/development/README.md @@ -0,0 +1,115 @@ +# Development Documentation + +Welcome to the osquery (with OpenFrame integration) development documentation. This section covers everything you need to contribute to, build upon, and understand the internals of the project. + +--- + +## Overview + +osquery is a C++ application built with CMake. The codebase spans cross-platform OS instrumentation, an embedded SQLite engine, a plugin registry, an event pub/sub framework, and the OpenFrame integration layer. Understanding the architecture before diving into code will save you significant time. + +--- + +## Documentation Index + +| Document | Description | +|----------|-------------| +| [Environment Setup](setup/environment.md) | IDE recommendations, tools, and editor extensions | +| [Local Development](setup/local-development.md) | Clone, build, run, and debug locally | +| [Architecture Overview](architecture/README.md) | System design, components, and data flows | +| [Security Guidelines](security/README.md) | Auth patterns, secrets management, and secure coding | +| [Testing Guide](testing/README.md) | Test structure, running tests, and writing new tests | +| [Contributing Guidelines](contributing/guidelines.md) | Code style, PRs, commits, and review process | + +--- + +## Technology Stack + +| Layer | Technology | +|-------|-----------| +| **Language** | C++17 | +| **Build System** | CMake 3.21+ | +| **SQL Engine** | Embedded SQLite with virtual table extensions | +| **IPC / Extensions** | Apache Thrift over UNIX domain sockets / Windows named pipes | +| **Networking** | Boost.Asio + Boost.Beast + OpenSSL | +| **Storage** | RocksDB (persistent) / In-memory ephemeral backend | +| **Event Publishers** | Platform-specific: BPF (Linux), inotify, EndpointSecurity (macOS), ETW (Windows) | +| **OpenFrame Auth** | Custom token extractor + refresher with Boost threading | + +--- + +## Project Layout + +```text +osquery/ +β”œβ”€β”€ osquery/ # Core library modules +β”‚ β”œβ”€β”€ core/ # Runtime initialization, flags, shutdown +β”‚ β”œβ”€β”€ sql/ # SQLite engine and virtual table integration +β”‚ β”œβ”€β”€ events/ # Publish/subscribe event framework +β”‚ β”œβ”€β”€ config/ # Configuration loading and packs +β”‚ β”œβ”€β”€ database/ # RocksDB and ephemeral storage backends +β”‚ β”œβ”€β”€ distributed/ # Distributed query engine +β”‚ β”œβ”€β”€ extensions/ # Apache Thrift-based extension framework +β”‚ β”œβ”€β”€ remote/ # HTTP client (Boost.Beast + OpenSSL) +β”‚ β”œβ”€β”€ hashing/ # MD5, SHA1, SHA256 utilities +β”‚ └── filesystem/ # Cross-platform file operations +β”œβ”€β”€ openframe/ # OpenFrame authentication integration +β”‚ β”œβ”€β”€ openframe_authorization_manager.* +β”‚ β”œβ”€β”€ openframe_encryption_service.* +β”‚ β”œβ”€β”€ openframe_token_extractor.* +β”‚ └── openframe_token_refresher.* +β”œβ”€β”€ plugins/ # Config, logger, database, distributed plugins +β”œβ”€β”€ tables/ # Virtual table implementations (300+ tables) +β”œβ”€β”€ libraries/ # Vendored third-party libraries (CMake-managed) +β”œβ”€β”€ tools/ # Code generation, CI scripts, formatting tools +β”œβ”€β”€ tests/ # Integration and unit tests +└── external/ # Extension SDK examples +``` + +--- + +## Quick Developer Commands + +```bash +# Configure build (Debug mode with tests) +cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DOSQUERY_BUILD_TESTS=ON + +# Build everything +cmake --build build -j$(nproc) + +# Build specific target +cmake --build build --target osqueryi -j$(nproc) + +# Run tests +cd build && ctest --output-on-failure + +# Format code +cmake --build build --target clang-format +``` + +--- + +## Architecture Quick Reference + +```mermaid +flowchart TD + CLI["osqueryi / osqueryd"] --> Core["Core Runtime"] + Core --> SQL["SQL Engine"] + Core --> Events["Events Framework"] + Core --> Config["Config & Packs"] + Core --> DB["Database (RocksDB)"] + Core --> Ext["Extensions (Thrift)"] + Core --> OF["OpenFrame Integration"] + SQL --> Tables["300+ Virtual Tables"] + Events --> Publishers["Platform Event Publishers"] + Config --> Schedule["Query Scheduler"] +``` + +--- + +## Community + +All discussion, questions, and collaboration happen on the **OpenMSP Slack**: + +- [Join OpenMSP Slack](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) +- Community website: [openmsp.ai](https://www.openmsp.ai/) diff --git a/docs/development/architecture/README.md b/docs/development/architecture/README.md new file mode 100644 index 00000000000..0b8efd002da --- /dev/null +++ b/docs/development/architecture/README.md @@ -0,0 +1,233 @@ +# Architecture Overview + +This document describes the high-level architecture of osquery with the OpenFrame integration. For deep-dive reference documentation, see the individual module docs in `docs/reference/architecture/`. + +--- + +## System Architecture + +osquery is a layered, plugin-driven C++ application. At its core is an embedded SQLite engine that exposes OS state through virtual tables. An event framework captures real-time activity, a distributed query system handles remote work, and the OpenFrame integration provides authenticated connectivity to the Flamingo MSP platform. + +### High-Level Architecture Diagram + +```mermaid +flowchart TD + CLI["osqueryi / osqueryd"] --> Core["Core Init And Runtime"] + + Core --> Config["Config And Packs"] + Core --> Registry["Plugin Registry"] + Core --> Database["Database Backends (RocksDB)"] + Core --> SQL["SQL Core And Virtual Tables"] + Core --> Events["Events Core (pub/sub)"] + Core --> Distributed["Distributed Querying"] + Core --> Logger["Plugin Interfaces And Logging"] + Core --> Extensions["Extensions Framework (Thrift)"] + Core --> HTTP["Remote HTTP Client (Boost.Beast)"] + Core --> OpenFrame["OpenFrame Integration"] + + Config --> SQL + SQL --> QueryExec["Query Execution And Logging"] + QueryExec --> Logger + Events --> SQL + Distributed --> SQL + Distributed --> HTTP + OpenFrame --> AuthMgr["Authorization Manager"] + OpenFrame --> Refresher["Token Refresher Thread"] +``` + +--- + +## Core Components + +| Module | Location | Responsibility | +|--------|----------|---------------| +| **Core Init And Runtime** | `osquery/core/` | Bootstrapping, flag parsing, lifecycle | +| **SQL Core And Virtual Tables** | `osquery/sql/` | SQLite engine, virtual table integration, constraint pushdown | +| **Events Core** | `osquery/events/` | Publish/subscribe framework for real-time OS events | +| **Config And Packs** | `osquery/config/` | Configuration loading, query scheduling, pack management | +| **Database Backends** | `osquery/database/` | RocksDB (persistent) and ephemeral key-value storage | +| **Distributed Querying** | `osquery/distributed/` | Pull-execute-push model for remote query orchestration | +| **Extensions Framework** | `osquery/extensions/` | Apache Thrift-based runtime plugin injection | +| **Remote HTTP Client** | `osquery/remote/` | Boost.Asio + Boost.Beast HTTP/HTTPS with TLS | +| **Hashing** | `osquery/hashing/` | MD5, SHA1, SHA256 for integrity verification | +| **Filesystem And Fileops** | `osquery/filesystem/` | Cross-platform file access primitives | +| **Plugin Interfaces And Logging** | `osquery/core/plugins/` | Logger plugin interface, status log routing | +| **OpenFrame Integration** | `openframe/` | Token lifecycle, encryption, and auth management | + +--- + +## Runtime Boot Flow + +The `Initializer` class in Core Init And Runtime orchestrates the entire startup sequence: + +```mermaid +flowchart TD + Main["Main Entry"] --> Initializer["Initializer"] + Initializer --> Flags["Parse Flags"] + Flags --> Mode["Determine Tool Type (daemon/shell/extension)"] + Mode --> RegistryInit["Registry And Plugin Init"] + RegistryInit --> DBInit["Database Init And Upgrade"] + DBInit --> ExtMgr["Extension Manager Start"] + ExtMgr --> ConfigLoad["Config Load"] + ConfigLoad --> LoggerInit["Logger Plugin Init"] + LoggerInit --> DistInit["Distributed Plugin Init"] + DistInit --> Events["Attach Events"] + Events --> Dispatcher["Dispatcher Services (scheduler loop)"] + Dispatcher --> ShutdownReq{"Shutdown Requested?"} + ShutdownReq -->|"Yes"| GracefulShutdown["Graceful Shutdown"] + GracefulShutdown --> End["Process Exit"] +``` + +--- + +## SQL Execution Data Flow + +Every SQL query follows this path from user input to result rows: + +```mermaid +sequenceDiagram + participant User + participant SQLInternal + participant SQLiteDBManager + participant VirtualTable + participant TablePlugin + + User->>SQLInternal: Execute SQL + SQLInternal->>SQLiteDBManager: Get DB Instance + SQLiteDBManager->>SQLiteDBManager: Apply authorizer + SQLiteDBManager->>VirtualTable: xBestIndex / xFilter + VirtualTable->>TablePlugin: generate(QueryContext) + TablePlugin-->>VirtualTable: TableRows + VirtualTable-->>SQLInternal: Result Set + SQLInternal-->>User: QueryDataTyped +``` + +The key security feature here is the SQLite **authorizer** β€” it enforces an allowlist of permitted SQL actions and PRAGMA statements. Any non-allowlisted operation is denied at prepare time. + +--- + +## Event Framework Architecture + +The Events Core implements a publish/subscribe pattern with persistent backing storage: + +```mermaid +flowchart TD + Config["Configuration"] --> Factory["EventFactory (singleton)"] + + subgraph publishers["Event Publishers (platform-specific)"] + PubLinux["BPF / inotify (Linux)"] + PubMac["EndpointSecurity (macOS)"] + PubWin["ETW (Windows)"] + end + + subgraph subscribers["Event Subscribers"] + SubFile["File Events Subscriber"] + SubProc["Process Events Subscriber"] + SubSock["Socket Events Subscriber"] + end + + PubLinux -->|"fires events"| SubFile + PubLinux -->|"fires events"| SubProc + PubMac -->|"fires events"| SubFile + PubWin -->|"fires events"| SubProc + + SubFile -->|"addBatch(rows)"| Store["Time-indexed Backing Store (RocksDB)"] + SubProc -->|"addBatch(rows)"| Store + SubSock -->|"addBatch(rows)"| Store + + Store -->|"genTable(time window)"| SQL["Virtual Table Query"] + + Factory --> PubLinux + Factory --> PubMac + Factory --> PubWin +``` + +--- + +## OpenFrame Integration Architecture + +The OpenFrame integration is an authentication layer that runs alongside the core runtime when `openframe_mode` is enabled: + +```mermaid +flowchart TD + Flag["--openframe_mode=true"] --> Encryption["OpenframeEncryptionService"] + Encryption --> Extractor["OpenframeTokenExtractor"] + Extractor --> AuthMgr["OpenframeAuthorizationManager (singleton)"] + Extractor --> Refresher["OpenframeTokenRefresher (background thread)"] + AuthMgr --> APICall["Authenticated OpenFrame API Calls"] + Refresher -->|"periodic re-read"| Extractor +``` + +The `OpenframeAuthorizationManager` is a singleton that uses the **provider pattern** β€” only `OpenframeAuthorizationManagerProvider` can construct or destroy it, separating token lifecycle from token access. + +--- + +## Plugin Registry + +The plugin registry (`osquery/registry/`) is the dependency injection backbone of the entire system. Every major subsystem β€” loggers, config sources, databases, SQL engines, distributed plugins β€” is registered and resolved through it. + +```mermaid +flowchart LR + RegistryFactory["RegistryFactory"] --> LoggerReg["logger registry"] + RegistryFactory --> ConfigReg["config registry"] + RegistryFactory --> DatabaseReg["database registry"] + RegistryFactory --> TableReg["table registry"] + RegistryFactory --> DistributedReg["distributed registry"] + + LoggerReg --> TLSLogger["TLS Logger Plugin"] + LoggerReg --> FSLogger["Filesystem Logger Plugin"] + ConfigReg --> TLSConfig["TLS Config Plugin"] + DatabaseReg --> RocksDB["RocksDB Plugin"] + TableReg --> ProcessesTable["processes virtual table"] +``` + +--- + +## Extensions Framework + +Extensions allow external processes to add virtual tables, loggers, and config sources at runtime: + +```mermaid +sequenceDiagram + participant Ext as Extension Process + participant EM as Extension Manager + participant Reg as RegistryFactory + participant SQL as SQL Engine + + Ext->>EM: registerExtension(info, registry) + EM->>Reg: addBroadcast(uuid, routes) + EM-->>Ext: return UUID + Ext->>Ext: Start Thrift server + SQL->>EM: callExtension(uuid, table, query) + EM->>Ext: call() via Thrift + Ext-->>SQL: TableRows +``` + +--- + +## Key Design Decisions + +| Decision | Rationale | +|----------|-----------| +| **Embedded SQLite** | No external database dependency; strict authorizer provides security | +| **Plugin Registry** | All major components are swappable; enables testing with stubs | +| **RocksDB for events** | High write throughput for real-time event ingestion | +| **Apache Thrift for extensions** | Language-agnostic IPC; extensions can be written in any language | +| **Boost.Beast for HTTP** | Production-grade async networking; TLS enforced by default | +| **Provider pattern for OpenFrame auth** | Strict access control to token lifecycle prevents accidental mutation | + +--- + +## Reference Documentation + +For detailed module-level documentation, see: + +- [./reference/architecture/core-init-and-runtime/core-init-and-runtime.md](./reference/architecture/core-init-and-runtime/core-init-and-runtime.md) +- [./reference/architecture/sql-core-and-virtual-tables/sql-core-and-virtual-tables.md](./reference/architecture/sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) +- [./reference/architecture/events-core/events-core.md](./reference/architecture/events-core/events-core.md) +- [./reference/architecture/config-and-packs/config-and-packs.md](./reference/architecture/config-and-packs/config-and-packs.md) +- [./reference/architecture/database-backends/database-backends.md](./reference/architecture/database-backends/database-backends.md) +- [./reference/architecture/distributed-querying/distributed-querying.md](./reference/architecture/distributed-querying/distributed-querying.md) +- [./reference/architecture/extensions-framework/extensions-framework.md](./reference/architecture/extensions-framework/extensions-framework.md) +- [./reference/architecture/remote-http-client/remote-http-client.md](./reference/architecture/remote-http-client/remote-http-client.md) +- [./reference/architecture/hashing/hashing.md](./reference/architecture/hashing/hashing.md) diff --git a/docs/development/contributing/guidelines.md b/docs/development/contributing/guidelines.md new file mode 100644 index 00000000000..90674f16a11 --- /dev/null +++ b/docs/development/contributing/guidelines.md @@ -0,0 +1,280 @@ +# Contributing Guidelines + +Thank you for contributing to osquery with the OpenFrame integration! This document covers code style, branching, commit messages, pull requests, and the review process. + +--- + +## Community First + +All discussion, questions, bug reports, and feature requests are handled via the **OpenMSP Slack** community. We do not use GitHub Issues or GitHub Discussions. + +- **Slack:** [Join OpenMSP](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) +- **Community site:** [openmsp.ai](https://www.openmsp.ai/) + +Before opening a pull request, please discuss the change in Slack to ensure it aligns with the project roadmap. + +--- + +## Code Style and Conventions + +### C++ Style + +The project follows the **Google C++ Style Guide** with some osquery-specific modifications. `clang-format` enforces formatting β€” configuration is in `.clang-format` at the project root. + +**Before every commit, format your changes:** + +```bash +# Format all staged C++ files +git diff --cached --name-only | grep -E '\.(cpp|h)$' | xargs clang-format -i + +# Format all modified files +git diff --name-only | grep -E '\.(cpp|h)$' | xargs clang-format -i +``` + +### Key Conventions + +| Convention | Rule | +|------------|------| +| **Namespacing** | All code lives in the `osquery` namespace | +| **File naming** | `snake_case.cpp` / `snake_case.h` | +| **Class naming** | `PascalCase` | +| **Function naming** | `camelCase` | +| **Constant naming** | `kConstantName` prefix | +| **Member variables** | Trailing underscore: `member_` | +| **Include guards** | `#pragma once` (not `#ifndef` guards) | +| **Smart pointers** | Prefer `std::shared_ptr` and `std::unique_ptr` over raw pointers | +| **Error handling** | Use `osquery::Status` or `osquery::Expected` | + +### Example: Correct Style + +```cpp +#pragma once + +#include +#include + +namespace osquery { + +/// Brief description of what this class does. +class MyNewFeature { + public: + explicit MyNewFeature(std::shared_ptr dep); + ~MyNewFeature(); + + /// Returns the current feature value, or an error status. + Expected getValue() const; + + private: + std::shared_ptr dependency_; + std::string cached_value_; +}; + +} // namespace osquery +``` + +--- + +## Branch Naming + +| Branch Type | Pattern | Example | +|-------------|---------|---------| +| Feature | `feature/` | `feature/openframe-token-rotation` | +| Bug fix | `fix/` | `fix/sql-authorizer-pragma-list` | +| Documentation | `docs/` | `docs/extension-api-guide` | +| Refactor | `refactor/` | `refactor/database-interface` | +| Hotfix | `hotfix/` | `hotfix/token-refresher-crash` | + +```bash +# Create a feature branch from main +git checkout main +git pull origin main +git checkout -b feature/my-new-virtual-table +``` + +--- + +## Commit Message Format + +Use the **Conventional Commits** specification: + +```text +(): + +[Optional body β€” explain WHY, not WHAT] + +[Optional footer β€” breaking changes, references] +``` + +### Types + +| Type | When to Use | +|------|-------------| +| `feat` | New feature or virtual table | +| `fix` | Bug fix | +| `docs` | Documentation changes only | +| `refactor` | Code change that neither fixes a bug nor adds a feature | +| `test` | Adding or modifying tests | +| `perf` | Performance improvement | +| `ci` | CI/CD configuration changes | +| `chore` | Build system, dependency updates | + +### Scope Examples + +```text +feat(sql): add constraint pushdown for inet_diag table +fix(events): prevent duplicate subscription on config reload +docs(openframe): add token rotation guide +refactor(database): extract IDatabaseInterface for testability +test(core): add flag override tests for daemon mode +perf(events): use batch DB writes in EventSubscriberPlugin +``` + +### Full Example + +```text +feat(openframe): add configurable token refresh interval + +Previously the token refresher used a hardcoded 5-minute interval. +This change reads the interval from the --openframe_refresh_interval +flag, allowing operators to tune refresh frequency based on their +token expiry policies. + +Closes: FLAMINGO-1234 +``` + +--- + +## Pull Request Process + +### Before Opening a PR + +1. **Discuss in Slack** β€” confirm the change is wanted before investing time +2. **Branch from `main`** β€” never commit directly to `main` +3. **Run tests** β€” all tests must pass locally +4. **Format code** β€” run `clang-format` on all changed files +5. **Update docs** β€” if you add a new module or change behavior, update or add inline docs + +```bash +# Full pre-PR checklist +git checkout -b feature/my-change +# ... make changes ... + +# 1. Format +git diff --name-only | grep -E '\.(cpp|h)$' | xargs clang-format -i + +# 2. Build +cmake --build build -j$(nproc) + +# 3. Test +cd build && ctest --output-on-failure + +# 4. Commit +git add -A +git commit -m "feat(sql): add my new virtual table" + +# 5. Push +git push origin feature/my-change +``` + +### PR Description Template + +```markdown +## Summary +Brief description of what this PR does and why. + +## Changes +- Added `MyNewTable` virtual table in `osquery/tables/system/` +- Added unit tests in `tests/integration/tables/my_new_table.cpp` +- Updated CMakeLists.txt to include new table + +## Testing +- [ ] All existing tests pass (`ctest --output-on-failure`) +- [ ] New tests added for new functionality +- [ ] Manually tested on Linux x86_64 +- [ ] Manually tested on macOS (if applicable) + +## Breaking Changes +None / Describe any breaking changes here. +``` + +--- + +## Code Review Checklist + +Reviewers should verify: + +### Correctness +- [ ] Logic is correct and handles edge cases +- [ ] Error paths return appropriate `Status` or `Expected<>` values +- [ ] No resource leaks (file handles, DB connections, threads) + +### Security +- [ ] No hardcoded secrets or credentials +- [ ] Query constraints from `QueryContext` are validated before use +- [ ] New network code uses the `Remote HTTP Client` module +- [ ] Thread-shared state is properly synchronized + +### Style and Quality +- [ ] Follows naming conventions (see above) +- [ ] `clang-format` applied to all changed files +- [ ] Public APIs have doc comments +- [ ] No dead code or commented-out blocks + +### Tests +- [ ] New functionality has tests +- [ ] Tests are independent (use ephemeral DB, no shared state) +- [ ] Tests cover both happy path and error paths + +### Documentation +- [ ] Inline docs updated if behavior changes +- [ ] New modules have doc comments on public API + +--- + +## Adding a New Virtual Table + +Virtual tables are the primary contribution type. Here is the pattern: + +1. **Create the table spec** in `osquery/tables/` (appropriate subdirectory) +2. **Implement `generate()`** β€” use `QueryContext` for constraint pushdown +3. **Register the table** in the appropriate `CMakeLists.txt` +4. **Add an integration test** in `tests/integration/tables/` +5. **Test on all target platforms** where the table is supported + +```cpp +// osquery/tables/system/linux/my_new_table.cpp +#include +#include + +namespace osquery { +namespace tables { + +TableRows genMyNewTable(QueryContext& context) { + TableRows results; + + // Validate constraints before use + auto paths = context.constraints["path"].getAll(EQUALS); + for (const auto& path : paths) { + if (path.empty()) continue; + + Row r; + r["path"] = path; + r["size"] = INTEGER(0); // populate from OS API + results.push_back(r); + } + + return results; +} + +} // namespace tables +} // namespace osquery +``` + +--- + +## Getting Help + +Stuck on something? The community is here to help: + +- **Slack:** [OpenMSP Community](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) +- **Platform:** [flamingo.run](https://flamingo.run) | [openframe.ai](https://openframe.ai) diff --git a/docs/development/security/README.md b/docs/development/security/README.md new file mode 100644 index 00000000000..bf4882649e6 --- /dev/null +++ b/docs/development/security/README.md @@ -0,0 +1,258 @@ +# Security Guidelines + +This document covers security best practices for developing, deploying, and extending osquery with the OpenFrame integration. + +--- + +## Authentication and Authorization + +### OpenFrame Token Lifecycle + +The OpenFrame integration uses a **provider pattern** to enforce strict access control over authentication tokens: + +```mermaid +flowchart TD + Provider["OpenframeAuthorizationManagerProvider"] -->|"sole factory"| Manager["OpenframeAuthorizationManager"] + Manager --> UpdateToken["updateToken(token)"] + Manager --> GetToken["getToken()"] + Refresher["OpenframeTokenRefresher"] -->|"periodic update"| UpdateToken + APIClient["OpenFrame API Client"] -->|"read-only access"| GetToken +``` + +**Key rules:** +- `OpenframeAuthorizationManager` is **non-copyable** (inherits `boost::noncopyable`) +- Constructor and destructor are `private` β€” only `OpenframeAuthorizationManagerProvider` can create or destroy instances +- Token access is thread-safe via internal synchronization +- Tokens are refreshed by a dedicated background thread; never cached in calling code + +### SQLite Authorizer + +The embedded SQLite engine enforces a strict **allowlist** of permitted operations. Any SQL action or PRAGMA not on the allowlist is rejected at prepare time: + +```mermaid +flowchart TD + Query["Incoming SQL"] --> Prepare["sqlite3_prepare()"] + Prepare --> Authorizer["SQLite Authorizer Callback"] + Authorizer --> Check{"Action in allowlist?"} + Check -->|"Yes"| Execute["Execute Query"] + Check -->|"No"| Deny["SQLITE_DENY β†’ Query Rejected"] +``` + +**Never bypass the authorizer** in custom table implementations. All `generate()` methods receive a sanitized `QueryContext` β€” do not execute raw SQLite calls outside the virtual table framework. + +--- + +## Secrets and Environment Variables Management + +### Token Storage + +OpenFrame tokens are read from a file path configured at startup: + +```bash +./build/osqueryd \ + --openframe_mode=true \ + --openframe_token_path=/etc/osquery/openframe_token +``` + +**Best practices for token files:** + +```bash +# Set restrictive permissions β€” only the osquery service user should read this +sudo chown osquery:osquery /etc/osquery/openframe_token +sudo chmod 600 /etc/osquery/openframe_token + +# Verify permissions +ls -la /etc/osquery/openframe_token +# Expected: -rw------- 1 osquery osquery +``` + +### Never Commit Secrets + +Add token files and credential stores to `.gitignore`: + +```text +# .gitignore β€” secrets and runtime state +/etc/osquery/openframe_token +*.token +*.pem +*.key +osquery.db/ +osquery-db/ +/var/osquery/ +/tmp/osquery*/ +``` + +### Environment Variables vs. Flag Files + +Prefer **flag files** over environment variables for osquery configuration to avoid token exposure in `ps` output or shell history: + +```bash +# Create a flagfile (not visible in process arguments) +cat > /etc/osquery/osquery.flags << 'EOF' +--openframe_mode=true +--openframe_token_path=/etc/osquery/openframe_token +--config_path=/etc/osquery/osquery.conf +--logger_path=/var/log/osquery +EOF + +# Run with flagfile +sudo ./build/osqueryd --flagfile=/etc/osquery/osquery.flags +``` + +--- + +## TLS and Network Security + +### Remote HTTP Client Security Defaults + +The `Remote HTTP Client` module (`osquery/remote/`) enforces strong TLS by default: + +| Security Control | Default | +|-----------------|---------| +| SSLv2 / SSLv3 | **Disabled** | +| MD5 cipher suites | **Disabled** | +| Deprecated OpenSSL APIs | **Disabled** | +| Peer certificate verification | Configurable (verify by default in production) | + +**Always enable peer verification for TLS endpoints:** + +```bash +# In your osquery config or flagfile +--tls_server_certs=/etc/ssl/certs/ca-certificates.crt +--verify_peer=true +``` + +### TLS Configuration for Distributed Queries + +```json +{ + "options": { + "tls_server_certs": "/etc/ssl/certs/ca-bundle.crt", + "tls_client_cert": "/etc/osquery/certs/client.pem", + "tls_client_key": "/etc/osquery/certs/client.key", + "verify_peer": true + } +} +``` + +--- + +## Input Validation and Sanitization + +### Query Denylisting + +The Config And Packs module automatically denylists queries that cause watchdog failures: + +- Denylisted queries are persisted in the database +- They are skipped until the denylist entry expires +- This prevents runaway queries from destabilizing the agent + +**In custom table implementations**, validate and sanitize all inputs from `QueryContext`: + +```cpp +// GOOD: validate constraint before use +if (context.constraints.count("path") > 0) { + auto paths = context.constraints.at("path").getAll(EQUALS); + for (const auto& path : paths) { + // Validate the path before opening + if (path.empty() || path.find('\0') != std::string::npos) { + continue; // Skip invalid paths + } + // ... process path + } +} +``` + +### Config Size and Depth Limits + +The configuration parser enforces: + +| Limit | Value | Purpose | +|-------|-------|---------| +| Maximum JSON depth | `kMaxConfigDepth` | Prevent stack overflow | +| Maximum config size | `kMaxConfigSize` | Prevent memory exhaustion | + +Do not accept configuration from untrusted sources without these limits in place. + +--- + +## Common Vulnerabilities and Mitigations + +| Vulnerability | Mitigation in osquery | +|--------------|----------------------| +| SQL injection via virtual tables | SQLite authorizer + constraint-only query context | +| Path traversal in file tables | Validate paths from `QueryContext` before filesystem access | +| Privilege escalation | Run `osqueryd` as a dedicated low-privilege service user | +| Token leakage in logs | `OpenframeAuthorizationManager` never logs token values | +| Unsafe config from TLS server | JSON depth and size limits enforced before parsing | +| Extension process hijacking | UUID validation + SDK version check at registration | +| Memory exhaustion from events | `getEventsExpiry()` and `removeOverflowingEventBatches()` enforce limits | + +--- + +## Least Privilege β€” Running osquery Safely + +### Linux β€” Dedicated Service User + +```bash +# Create a dedicated system user +sudo useradd --system --no-create-home --shell /usr/sbin/nologin osquery + +# Set ownership of config and data directories +sudo chown -R osquery:osquery /etc/osquery /var/osquery /var/log/osquery +sudo chmod 750 /etc/osquery /var/osquery /var/log/osquery + +# Run as service user (example with systemd) +# In /etc/systemd/system/osqueryd.service: +# User=osquery +# Group=osquery +``` + +### macOS β€” LaunchDaemon + +```xml + +UserName +_osquery +``` + +--- + +## Security Testing and Code Review + +### Pre-Commit Checklist + +Before submitting a pull request, verify: + +- [ ] No secrets, tokens, or API keys in source code or test fixtures +- [ ] New virtual tables sanitize all `QueryContext` constraints before use +- [ ] New network-facing code uses the `Remote HTTP Client` module (not raw sockets) +- [ ] All file paths received from queries are validated +- [ ] New config parameters have documented size/depth limits if they accept user data +- [ ] Thread-shared state uses appropriate synchronization primitives + +### Static Analysis + +```bash +# Run clang-tidy on changed files +git diff --name-only HEAD~1 | grep -E '\.(cpp|h)$' | \ + xargs clang-tidy -p build --checks='cert-*,clang-analyzer-security.*' + +# Run AddressSanitizer build +cmake -S . -B build-asan \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-fsanitize=address" +cmake --build build-asan --target osqueryi -j$(nproc) +./build-asan/osqueryi "SELECT * FROM processes;" +``` + +--- + +## Reporting Security Issues + +Do **not** open GitHub Issues for security vulnerabilities. + +Report security concerns via the **OpenMSP Slack** community: + +- [Join OpenMSP Slack](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) +- Community site: [openmsp.ai](https://www.openmsp.ai/) diff --git a/docs/development/setup/environment.md b/docs/development/setup/environment.md new file mode 100644 index 00000000000..4363820016d --- /dev/null +++ b/docs/development/setup/environment.md @@ -0,0 +1,230 @@ +# Development Environment Setup + +This guide covers IDE recommendations, required development tools, and editor configuration for working on osquery with the OpenFrame integration. + +--- + +## IDE Recommendations + +### Visual Studio Code (Recommended β€” All Platforms) + +VS Code with the CMake and C++ extensions provides the best cross-platform development experience. + +**Required Extensions:** + +| Extension | Publisher | Purpose | +|-----------|-----------|---------| +| C/C++ | Microsoft | IntelliSense, debugging | +| CMake Tools | Microsoft | CMake configure/build integration | +| CMake | twxs | CMake syntax highlighting | +| clangd | LLVM | Advanced code completion and navigation | +| GitLens | GitKraken | Enhanced Git integration | + +Install all at once: + +```bash +code --install-extension ms-vscode.cpptools +code --install-extension ms-vscode.cmake-tools +code --install-extension twxs.cmake +code --install-extension llvm-vs-code-extensions.vscode-clangd +code --install-extension eamodio.gitlens +``` + +**Recommended VS Code settings (`.vscode/settings.json`):** + +```json +{ + "cmake.buildDirectory": "${workspaceFolder}/build", + "cmake.configureArgs": [ + "-DCMAKE_BUILD_TYPE=Debug", + "-DOSQUERY_BUILD_TESTS=ON", + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON" + ], + "cmake.generator": "Ninja", + "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", + "editor.formatOnSave": true, + "editor.tabSize": 2, + "files.trimTrailingWhitespace": true +} +``` + +--- + +### CLion (Recommended β€” Linux / macOS) + +JetBrains CLion has native CMake support and powerful C++ refactoring tools. + +**Setup steps:** + +1. Open the project root (where `CMakeLists.txt` is located) +2. CLion auto-detects CMake configuration +3. Set CMake options in **Settings β†’ Build β†’ CMake**: + +```text +-DCMAKE_BUILD_TYPE=Debug +-DOSQUERY_BUILD_TESTS=ON +-DCMAKE_EXPORT_COMPILE_COMMANDS=ON +``` + +**Recommended plugins:** + +- **File Watchers** β€” auto-format on save +- **GitToolBox** β€” enhanced Git annotations + +--- + +### Xcode (macOS Only) + +For macOS-specific development or debugging macOS-specific event publishers: + +```bash +# Generate Xcode project +cmake -S . -B build-xcode -G Xcode \ + -DCMAKE_BUILD_TYPE=Debug \ + -DOSQUERY_BUILD_TESTS=ON + +# Open in Xcode +open build-xcode/osquery.xcodeproj +``` + +--- + +### Visual Studio 2022 (Windows) + +Visual Studio 2022 has native CMake support. Open the folder directly (File β†’ Open β†’ Folder) and VS will detect `CMakeLists.txt` automatically. + +--- + +## Required Development Tools + +### Core Toolchain + +```bash +# Linux (Ubuntu/Debian) +sudo apt-get install -y \ + clang-14 lldb-14 clang-format-14 clang-tidy-14 \ + cmake ninja-build ccache \ + valgrind gdb + +# macOS +brew install llvm cmake ninja ccache +``` + +### ccache β€” Compiler Cache (Strongly Recommended) + +ccache dramatically speeds up incremental builds. Enable it in CMake: + +```bash +cmake -S . -B build \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Debug +``` + +### compile_commands.json β€” IntelliSense Database + +Always generate this for IDE support: + +```bash +cmake -S . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +# Optionally symlink to the root for clangd +ln -sf build/compile_commands.json ./compile_commands.json +``` + +--- + +## Environment Variables for Development + +| Variable | Purpose | Example Value | +|----------|---------|---------------| +| `CC` | C compiler override | `clang-14` | +| `CXX` | C++ compiler override | `clang++-14` | +| `CCACHE_DIR` | ccache storage directory | `/tmp/ccache` | +| `OSQUERY_BUILD_TESTS` | Enable test targets | `ON` | +| `CMAKE_BUILD_TYPE` | Build mode | `Debug` or `Release` | + +Set in your shell profile (`~/.bashrc`, `~/.zshrc`): + +```bash +export CC=clang-14 +export CXX=clang++-14 +export CCACHE_DIR=/tmp/osquery-ccache +``` + +--- + +## Code Formatting + +The project uses **clang-format** for C++ code style enforcement. Configuration is in `.clang-format` at the project root. + +```bash +# Format all modified files +git diff --name-only | grep -E '\.(cpp|h)$' | xargs clang-format -i + +# Or use the CMake target (if configured) +cmake --build build --target clang-format +``` + +**Editor integration** β€” VS Code with the C++ extension will auto-format on save if you set: + +```json +{ + "editor.formatOnSave": true, + "C_Cpp.clang_format_fallbackStyle": "file" +} +``` + +--- + +## Debug Symbols and Sanitizers + +For debugging crashes or memory issues: + +```bash +# Debug build with AddressSanitizer +cmake -S . -B build-asan \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-fsanitize=address -fno-omit-frame-pointer" \ + -DCMAKE_LINKER_FLAGS="-fsanitize=address" + +cmake --build build-asan --target osqueryi -j$(nproc) +``` + +```bash +# Debug build with ThreadSanitizer +cmake -S . -B build-tsan \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-fsanitize=thread" +``` + +--- + +## GDB / LLDB Quick Reference + +```bash +# Debug with GDB (Linux) +gdb ./build/osqueryi +(gdb) run "SELECT * FROM processes;" +(gdb) bt # print backtrace + +# Debug with LLDB (macOS) +lldb ./build/osqueryi +(lldb) run "SELECT * FROM processes;" +(lldb) bt # print backtrace +``` + +--- + +## Checking Your Environment + +```bash +# Verify clang-format is available +clang-format --version + +# Verify ccache is working +ccache --version +ccache --show-stats + +# Verify compile_commands.json was generated +ls -la build/compile_commands.json +``` diff --git a/docs/development/setup/local-development.md b/docs/development/setup/local-development.md new file mode 100644 index 00000000000..b11f27b97b2 --- /dev/null +++ b/docs/development/setup/local-development.md @@ -0,0 +1,273 @@ +# Local Development Guide + +This guide explains how to clone, configure, build, run, and debug osquery locally for development. + +--- + +## Clone and Initial Setup + +```bash +# Clone the repository +git clone https://github.com/flamingo-stack/osquery.git +cd osquery + +# Initialize and update all submodules +git submodule update --init --recursive +``` + +> **Note:** The `libraries/` directory contains vendored third-party libraries managed via CMake's FetchContent and submodules. The initial clone with submodules can take several minutes depending on your connection speed. + +--- + +## Configure the Build + +### Development (Debug) Build + +```bash +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DOSQUERY_BUILD_TESTS=ON \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -G Ninja +``` + +### Release Build + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DOSQUERY_BUILD_TESTS=OFF \ + -DOSQUERY_NO_DEBUG_SYMBOLS=ON \ + -G Ninja +``` + +### CMake Configuration Reference + +| Option | Description | Development Value | +|--------|-------------|------------------| +| `CMAKE_BUILD_TYPE` | Optimization level | `Debug` | +| `OSQUERY_BUILD_TESTS` | Compile test targets | `ON` | +| `CMAKE_EXPORT_COMPILE_COMMANDS` | Generate IntelliSense DB | `ON` | +| `OSQUERY_NO_DEBUG_SYMBOLS` | Strip debug symbols | `OFF` | +| `OSQUERY_BUILD_EXTENSIONS` | Build extension support | `ON` | +| `OSQUERY_ENABLE_ASAN` | Address Sanitizer | `OFF` (enable as needed) | + +--- + +## Build Targets + +```bash +# Build the interactive shell only +cmake --build build --target osqueryi -j$(nproc) + +# Build the daemon only +cmake --build build --target osqueryd -j$(nproc) + +# Build everything (all targets including tests) +cmake --build build -j$(nproc) + +# Build a specific module's tests +cmake --build build --target osquery_sql_tests -j$(nproc) +``` + +> On macOS, replace `$(nproc)` with `$(sysctl -n hw.logicalcpu)`. + +--- + +## Running Locally + +### Interactive Shell (osqueryi) + +The fastest way to test SQL queries and table implementations: + +```bash +./build/osqueryi +``` + +Or run a single query directly: + +```bash +./build/osqueryi "SELECT name, pid FROM processes LIMIT 5;" +``` + +Run with verbose logging: + +```bash +./build/osqueryi --verbose --minloglevel=0 +``` + +### Daemon Mode (osqueryd) + +Create a minimal dev config: + +```json +{ + "options": { + "logger_path": "/tmp/osquery-dev/logs", + "database_path": "/tmp/osquery-dev/osquery.db", + "disable_events": false + }, + "schedule": { + "test_query": { + "query": "SELECT * FROM system_info;", + "interval": 30 + } + } +} +``` + +Run the daemon: + +```bash +mkdir -p /tmp/osquery-dev/logs +./build/osqueryd \ + --config_path /tmp/osquery-dev/osquery.conf \ + --verbose \ + --minloglevel=0 \ + --disable_watchdog +``` + +--- + +## Watch Mode β€” Iterative Development + +For rapid iteration on virtual table implementations, use a shell loop to auto-rebuild on file changes: + +```bash +# Linux with inotifywait +while inotifywait -e modify osquery/sql/*.cpp osquery/sql/*.h; do + cmake --build build --target osqueryi -j$(nproc) && \ + echo "Build succeeded" || echo "Build FAILED" +done +``` + +Or use `entr` for a cleaner experience: + +```bash +# Install entr first: apt-get install entr / brew install entr +find osquery/sql -name "*.cpp" -o -name "*.h" | \ + entr -r cmake --build build --target osqueryi -j$(nproc) +``` + +--- + +## Debug Configuration + +### VS Code Debug Launch Config + +Create `.vscode/launch.json`: + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug osqueryi", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/osqueryi", + "args": ["--verbose", "--minloglevel=0"], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "Debug osqueryd", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/osqueryd", + "args": [ + "--config_path=/tmp/osquery-dev/osquery.conf", + "--verbose", + "--disable_watchdog" + ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "MIMode": "gdb" + } + ] +} +``` + +### Adding Log Verbosity + +```bash +# Maximum verbosity β€” shows all internal log messages +./build/osqueryi \ + --verbose \ + --minloglevel=0 \ + --stderrthreshold=0 \ + --logger_plugin=stdout +``` + +### Useful Debug Flags + +| Flag | Description | +|------|-------------| +| `--verbose` | Enable verbose output | +| `--minloglevel=0` | Show all log levels (INFO, WARNING, ERROR) | +| `--stderrthreshold=0` | Mirror logs to stderr | +| `--disable_watchdog` | Disable the watcher process (easier debugging) | +| `--disable_events` | Disable event publishers (faster startup) | +| `--database_path=:memory:` | Use ephemeral in-memory database | + +--- + +## Testing During Development + +```bash +# Run all unit tests +cd build && ctest --output-on-failure + +# Run a specific test binary +./build/osquery_sql_tests + +# Run with verbose test output +./build/osquery_sql_tests --gtest_verbose=1 + +# Run a specific test case +./build/osquery_core_tests --gtest_filter="FlagsTests.*" +``` + +--- + +## Working with the OpenFrame Integration + +The OpenFrame integration lives in `openframe/`. To develop and test it: + +```bash +# Build with OpenFrame support explicitly enabled +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DOSQUERY_OPENFRAME=ON + +# Run with OpenFrame mode enabled (requires a token file) +echo "your-bearer-token-here" > /tmp/openframe_token +./build/osqueryd \ + --openframe_mode=true \ + --openframe_token_path=/tmp/openframe_token \ + --verbose +``` + +The `OpenframeTokenRefresher` will start a background thread that re-reads the token file at its configured interval. + +--- + +## Cleaning Up + +```bash +# Clean build artifacts +cmake --build build --target clean + +# Full rebuild from scratch +rm -rf build && cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug +``` diff --git a/docs/development/testing/README.md b/docs/development/testing/README.md new file mode 100644 index 00000000000..da38d76c923 --- /dev/null +++ b/docs/development/testing/README.md @@ -0,0 +1,283 @@ +# Testing Guide + +This document describes the test structure, how to run tests, and how to write new tests for osquery with the OpenFrame integration. + +--- + +## Test Structure and Organization + +osquery uses **Google Test (gtest)** as its primary unit testing framework. Tests are co-located with the source modules they test: + +```text +osquery/ +β”œβ”€β”€ core/ +β”‚ β”œβ”€β”€ core.cpp +β”‚ └── tests/ +β”‚ β”œβ”€β”€ flags_tests.cpp +β”‚ β”œβ”€β”€ process_tests.cpp +β”‚ β”œβ”€β”€ query_tests.cpp +β”‚ └── tables_tests.cpp +β”œβ”€β”€ sql/ +β”‚ β”œβ”€β”€ sql.cpp +β”‚ └── tests/ +β”‚ β”œβ”€β”€ sql.cpp +β”‚ └── sqlite_util_tests.cpp +β”œβ”€β”€ events/ +β”‚ β”œβ”€β”€ eventer.cpp +β”‚ └── tests/ +β”‚ β”œβ”€β”€ events_tests.cpp +β”‚ └── linux/ +β”‚ β”œβ”€β”€ audit_tests.cpp +β”‚ └── inotify_tests.cpp +└── database/ + β”œβ”€β”€ database.cpp + └── tests/ + └── database.cpp +``` + +Integration tests live in a separate top-level directory: + +```text +tests/integration/tables/ +β”œβ”€β”€ processes.cpp +β”œβ”€β”€ listening_ports.cpp +β”œβ”€β”€ system_info.cpp +└── ... (one file per virtual table) +``` + +--- + +## Running Tests + +### Run All Tests + +```bash +# From the build directory +cd build +ctest --output-on-failure + +# With parallel execution +ctest --output-on-failure -j$(nproc) +``` + +### Run a Specific Test Suite + +```bash +# Run a specific test binary directly +./build/osquery_sql_tests +./build/osquery_core_tests +./build/osquery_events_tests +./build/osquery_database_tests +./build/osquery_filesystem_tests +``` + +### Run a Specific Test Case + +```bash +# Google Test filter syntax +./build/osquery_sql_tests --gtest_filter="SQLTests.*" +./build/osquery_core_tests --gtest_filter="FlagsTests.testCustomFlags" + +# Run tests matching a pattern +./build/osquery_sql_tests --gtest_filter="*SQLiteEncoding*" +``` + +### Run Tests with Verbose Output + +```bash +./build/osquery_sql_tests --gtest_verbose=1 +# or +ctest --output-on-failure --verbose +``` + +--- + +## Building Test Targets + +```bash +# Configure with tests enabled +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DOSQUERY_BUILD_TESTS=ON + +# Build all test targets +cmake --build build --target all -j$(nproc) + +# Build a specific test target +cmake --build build --target osquery_sql_tests -j$(nproc) +cmake --build build --target osquery_core_tests -j$(nproc) +cmake --build build --target osquery_events_tests -j$(nproc) +``` + +--- + +## Writing New Tests + +### Unit Test β€” Basic Structure + +Create a new test file co-located with the source: + +```cpp +// osquery/sql/tests/my_feature_tests.cpp + +#include +#include +#include + +namespace osquery { + +class MyFeatureTests : public testing::Test { + protected: + void SetUp() override { + // Use ephemeral database for tests β€” never persistent + setDatabaseAllowOpen(); + initDatabasePlugin(); + } + + void TearDown() override { + shutdownDatabase(); + } +}; + +TEST_F(MyFeatureTests, testBasicQuery) { + auto results = SQL::selectAllFrom("osquery_info"); + EXPECT_GT(results.size(), 0U); +} + +TEST_F(MyFeatureTests, testEmptyResult) { + auto results = SQL::selectAllFrom("processes", "pid", EQUALS, "999999999"); + EXPECT_EQ(results.size(), 0U); +} + +} // namespace osquery +``` + +### Integration Test β€” Virtual Table + +Integration tests verify a virtual table returns expected results on the real OS: + +```cpp +// tests/integration/tables/my_new_table.cpp + +#include +#include "helper.h" + +namespace osquery::table_tests { + +class MyNewTableTest : public testing::Test {}; + +TEST_F(MyNewTableTest, test_implementation) { + auto const data = execute_query("SELECT * FROM my_new_table;"); + ASSERT_GE(data.size(), 0U); + ValidateSchema(data, { + {"column_a", INTEGER_TYPE}, + {"column_b", TEXT_TYPE}, + }); +} + +} // namespace osquery::table_tests +``` + +### Test with Ephemeral Database + +Most tests need a database. Always use the ephemeral backend in tests: + +```cpp +#include + +class DatabaseTest : public testing::Test { + protected: + void SetUp() override { + // Force ephemeral (in-memory) database for test isolation + FLAGS_disable_database = true; + setDatabaseAllowOpen(); + initDatabasePlugin(); + } + + void TearDown() override { + shutdownDatabase(); + } +}; +``` + +--- + +## Test Coverage Areas + +| Module | Test Binary | Key Test Files | +|--------|-------------|---------------| +| Core Flags | `osquery_core_tests` | `core/tests/flags_tests.cpp` | +| SQL Engine | `osquery_sql_tests` | `sql/tests/sql.cpp` | +| Virtual Tables | `osquery_sql_tests` | `sql/tests/virtual_table.cpp` | +| Events | `osquery_events_tests` | `events/tests/events_tests.cpp` | +| Database | `osquery_database_tests` | `database/tests/database.cpp` | +| Config | `osquery_config_tests` | `config/tests/config_tests.cpp` | +| Hashing | `osquery_hashing_tests` | `hashing/tests/hashing.cpp` | +| Filesystem | `osquery_filesystem_tests` | `filesystem/tests/filesystem.cpp` | +| Extensions | `osquery_extensions_tests` | `extensions/tests/extensions.cpp` | +| Distributed | `osquery_distributed_tests` | `distributed/tests/distributed_tests.cpp` | + +--- + +## Benchmarks + +Performance-sensitive modules have benchmark suites using **Google Benchmark**: + +```bash +# Build benchmarks +cmake -S . -B build \ + -DOSQUERY_BUILD_BENCHMARKS=ON \ + -DCMAKE_BUILD_TYPE=Release + +# Run SQL benchmarks +./build/osquery_sql_benchmarks + +# Run events benchmarks +./build/osquery_events_benchmarks + +# Run database benchmarks +./build/osquery_database_benchmarks +``` + +--- + +## Testing the OpenFrame Integration + +The OpenFrame modules (`openframe/`) should be tested with mock token files: + +```bash +# Create a test token file +echo "Bearer test-token-value" > /tmp/test_openframe_token +chmod 600 /tmp/test_openframe_token + +# Run osqueryi with OpenFrame mode to verify token loading +./build/osqueryi \ + --openframe_mode=true \ + --openframe_token_path=/tmp/test_openframe_token \ + --verbose +``` + +--- + +## CI Test Matrix + +Tests are run across the following configurations: + +| Platform | Compiler | Build Type | +|----------|----------|------------| +| Linux x86_64 | GCC 11 | Debug + Release | +| Linux aarch64 | GCC 11 | Debug | +| macOS x86_64 | AppleClang | Debug + Release | +| macOS aarch64 | AppleClang | Debug + Release | +| Windows x86_64 | MSVC 2022 | Debug + Release | + +--- + +## Common Test Failures and Fixes + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `database not initialized` | Missing `initDatabasePlugin()` in `SetUp()` | Add database init to test fixture | +| `permission denied` on `/proc` | Running tests as non-root | Some Linux tables require root for full data | +| `table not found` | Virtual table not registered in test binary | Ensure the table plugin is linked | +| `gtest: test binary not found` | `OSQUERY_BUILD_TESTS=OFF` | Reconfigure CMake with `OSQUERY_BUILD_TESTS=ON` | diff --git a/docs/getting-started/.gitignore b/docs/getting-started/.gitignore new file mode 100644 index 00000000000..a5d01be5236 --- /dev/null +++ b/docs/getting-started/.gitignore @@ -0,0 +1,7 @@ +# VoltAgent temp files +temp/ + +# JSON intermediate files (except schema/config) +*.json +!*-schema.json +!*-config.json diff --git a/docs/getting-started/first-steps.md b/docs/getting-started/first-steps.md new file mode 100644 index 00000000000..76993348d10 --- /dev/null +++ b/docs/getting-started/first-steps.md @@ -0,0 +1,248 @@ +# First Steps + +Congratulations on getting osquery running! This guide walks you through the first 5 things to do after a successful build to start getting value from the platform. + +--- + +## 1. Explore the Interactive Shell + +Launch `osqueryi` and get familiar with the built-in meta-commands: + +```bash +./build/osqueryi +``` + +Inside the shell: + +```sql +-- List all available tables +.tables + +-- Show the schema for a specific table +.schema processes + +-- Show help +.help + +-- Enable output timing +.timer on + +-- Pretty print results +.mode column +.headers on +``` + +### Useful Discovery Queries + +```sql +-- How many tables are available on this platform? +SELECT count(*) FROM osquery_registry WHERE active = 1 AND registry = 'table'; + +-- What osquery version am I running? +SELECT version, build_platform, build_distro FROM osquery_info; + +-- What flags are configured? +SELECT name, value, type, description FROM osquery_flags LIMIT 20; +``` + +--- + +## 2. Query System Information + +Get a complete picture of the system you are monitoring: + +```sql +-- Hardware and OS overview +SELECT hostname, cpu_brand, cpu_physical_cores, + physical_memory, hardware_vendor, hardware_model +FROM system_info; + +-- OS version +SELECT name, version, major, minor, patch, build FROM os_version; + +-- Uptime +SELECT days, hours, minutes FROM uptime; + +-- Disk usage +SELECT path, type, blocks_size, blocks_available, blocks +FROM mounts WHERE path = '/'; +``` + +--- + +## 3. Write Your First Scheduled Configuration + +Create a configuration file to schedule continuous monitoring: + +```json +{ + "options": { + "logger_path": "/var/log/osquery", + "database_path": "/var/osquery/osquery.db", + "schedule_splay_percent": 10 + }, + "schedule": { + "system_info": { + "query": "SELECT hostname, cpu_brand, physical_memory FROM system_info;", + "interval": 3600, + "description": "Capture system hardware info hourly" + }, + "processes": { + "query": "SELECT name, pid, state, user_time FROM processes;", + "interval": 60, + "description": "Track running processes every minute" + }, + "listening_ports": { + "query": "SELECT pid, port, protocol, family, address FROM listening_ports;", + "interval": 60, + "description": "Monitor open network ports" + }, + "users": { + "query": "SELECT uid, gid, username, description, shell FROM users;", + "interval": 3600, + "description": "Audit user accounts hourly" + } + } +} +``` + +Run the daemon with this config: + +```bash +sudo ./build/osqueryd \ + --config_path /etc/osquery/osquery.conf \ + --logger_path /var/log/osquery \ + --verbose +``` + +--- + +## 4. Explore Event-Driven Monitoring + +osquery's Events Core provides real-time monitoring of system activity. Enable file integrity monitoring (FIM): + +```json +{ + "file_paths": { + "etc": ["/etc/%%"], + "bin": ["/usr/bin/%%", "/usr/sbin/%%"] + }, + "schedule": { + "file_events": { + "query": "SELECT target_path, action, sha256, md5, time FROM file_events;", + "interval": 30 + } + } +} +``` + +On Linux, process events are available via BPF or inotify: + +```sql +-- See recent process events (requires event publisher running) +SELECT pid, parent, path, cmdline, time FROM process_events LIMIT 10; + +-- Socket events +SELECT action, local_address, remote_address, local_port, remote_port +FROM socket_events LIMIT 10; +``` + +--- + +## 5. Use Query Packs + +Query packs let you bundle related queries together. Download community packs or write your own: + +```json +{ + "packs": { + "incident-response": "/etc/osquery/packs/incident-response.conf", + "vuln-management": "/etc/osquery/packs/vuln-management.conf" + } +} +``` + +Example pack file: + +```json +{ + "queries": { + "shell_history": { + "query": "SELECT uid, time, command, history_file FROM shell_history;", + "interval": 3600, + "platform": "posix", + "description": "Collect shell command history" + }, + "sudoers": { + "query": "SELECT header, rule_details FROM sudoers;", + "interval": 3600, + "description": "Audit sudoers configuration" + }, + "ssh_keys": { + "query": "SELECT uid, path, key_type, comment FROM user_ssh_keys;", + "interval": 86400, + "description": "Enumerate SSH keys" + } + } +} +``` + +--- + +## Key SQL Patterns to Know + +### Differential Queries (Change Detection) + +osquery logs the _difference_ between query runs by default. This means you only see rows that were added or removed since the last execution β€” ideal for detecting changes. + +```sql +-- Find new processes since last check (differential mode is automatic) +SELECT name, pid, path, cmdline FROM processes; +``` + +### Joining Tables + +```sql +-- Map listening ports to process names and users +SELECT p.name AS process, u.username, lp.port, lp.protocol +FROM listening_ports lp +JOIN processes p ON lp.pid = p.pid +JOIN users u ON p.uid = u.uid +WHERE lp.protocol = 6 +ORDER BY lp.port; +``` + +### Using Subqueries + +```sql +-- Find processes running as root that are listening on the network +SELECT name, pid, path +FROM processes +WHERE pid IN ( + SELECT pid FROM listening_ports WHERE address != '127.0.0.1' +) AND uid = 0; +``` + +--- + +## Where to Get Help + +| Resource | Link | +|----------|------| +| OpenMSP Slack Community | [Join here](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) | +| Community Site | [openmsp.ai](https://www.openmsp.ai/) | +| Flamingo Platform | [flamingo.run](https://flamingo.run) | +| OpenFrame Platform | [openframe.ai](https://openframe.ai) | + +--- + +## Quick Reference β€” Shell Commands + +```text +.tables List all virtual tables +.schema
Show table column definitions +.mode column Columnar output format +.headers on Show column headers +.timer on Show query execution time +.quit Exit the shell +``` diff --git a/docs/getting-started/introduction.md b/docs/getting-started/introduction.md new file mode 100644 index 00000000000..5546ec37bcb --- /dev/null +++ b/docs/getting-started/introduction.md @@ -0,0 +1,126 @@ +# Introduction to osquery with OpenFrame + +**osquery** is a cross-platform operating system instrumentation framework that exposes system state as relational tables and allows it to be queried using SQL. Instead of writing platform-specific scripts to inspect processes, files, users, network state, or kernel events, osquery gives you a unified SQL interface to your entire fleet. + +> **Part of the OpenFrame Platform:** This repository includes the **OpenFrame integration** for osquery β€” enabling authentication, token management, and secure runtime identity for OpenFrame-connected deployments on the [Flamingo](https://flamingo.run) MSP platform. + +--- + +## What Is osquery? + +osquery turns your operating system into a queryable relational database. Want to know which processes are listening on which ports? Just write a SQL query. Want to monitor file integrity changes across thousands of endpoints? Schedule a query pack. + +```sql +-- Find all listening TCP ports and the processes behind them +SELECT p.name, p.pid, lp.port, lp.protocol +FROM listening_ports lp +JOIN processes p ON lp.pid = p.pid +WHERE lp.protocol = 6; +``` + +osquery handles the complexity of OS APIs so you never have to. + +--- + +## Key Features + +| Feature | Description | +|---------|-------------| +| **SQL Interface** | Query OS state with standard SQL across all platforms | +| **Virtual Tables** | 300+ built-in tables covering processes, network, files, users, and more | +| **Event-Driven Monitoring** | Real-time capture of file changes, process activity, network events | +| **Scheduled Queries** | Continuously run queries and log differential results | +| **Distributed Querying** | Push ad-hoc queries to remote nodes from a control plane | +| **Extensible Plugin System** | Add custom tables, loggers, and config sources via extensions | +| **Cross-Platform** | Linux, macOS, and Windows support | +| **OpenFrame Integration** | Secure token-based authentication for MSP fleet management | + +--- + +## Target Audience + +osquery is designed for: + +- **Security Engineers** building endpoint detection and response (EDR) workflows +- **SREs and DevOps Teams** monitoring fleet health and configuration drift +- **MSP Technicians** using Flamingo/OpenFrame for unified IT operations +- **Platform Engineers** extending osquery with custom tables and plugins + +--- + +## Architecture Overview + +At a high level, osquery is a layered system with a SQL execution engine at its core, an event framework for real-time monitoring, and a distributed querying system for fleet-wide operations. + +```mermaid +flowchart TD + CLI["osqueryi / osqueryd"] --> Core["Core Init And Runtime"] + Core --> Config["Config And Packs"] + Core --> Registry["Plugin Registry"] + Core --> Database["Database Backends"] + Core --> SQL["SQL Core And Virtual Tables"] + Core --> Events["Events Core"] + Core --> Distributed["Distributed Querying"] + Core --> Logger["Plugin Interfaces And Logging"] + Core --> Extensions["Extensions Framework"] + Core --> HTTP["Remote HTTP Client"] + + Config --> SQL + SQL --> QueryExec["Query Execution And Logging"] + QueryExec --> Logger + Events --> SQL + Distributed --> SQL + Distributed --> HTTP +``` + +### Execution Modes + +- **Daemon mode (`osqueryd`)** β€” scheduled and distributed queries running continuously +- **Interactive shell (`osqueryi`)** β€” ad-hoc SQL exploration +- **Extension processes** β€” dynamically inject custom plugins at runtime +- **Watcher/Worker model** β€” supervised execution for stability + +--- + +## OpenFrame Integration + +This fork of osquery includes the **OpenFrame** integration layer (`openframe/` directory), which provides: + +- **Encryption Service** β€” secure token storage and retrieval +- **Token Extractor** β€” reads authentication tokens from configured paths +- **Authorization Manager** β€” thread-safe token lifecycle management +- **Token Refresher** β€” background thread keeping credentials fresh + +When `openframe_mode` is enabled at startup, osquery seamlessly integrates with the [OpenFrame](https://openframe.ai) unified MSP platform. + +```mermaid +flowchart TD + FlagCheck["openframe_mode = true"] --> Encryption["Encryption Service"] + Encryption --> Extractor["Token Extractor"] + Extractor --> AuthMgr["Authorization Manager"] + Extractor --> Refresher["Token Refresher Thread"] + AuthMgr --> OutboundCalls["Authenticated OpenFrame API Calls"] +``` + +--- + +## YouTube β€” osquery Overview + +[![osquery Introduction](https://img.youtube.com/vi/KbGH9IBELWA/hqdefault.jpg)](https://www.youtube.com/watch?v=KbGH9IBELWA) + +--- + +## Getting Started + +| Next Step | Description | +|-----------|-------------| +| [Prerequisites](prerequisites.md) | Required tools, hardware, and environment setup | +| [Quick Start](quick-start.md) | Build and run osquery in 5 minutes | +| [First Steps](first-steps.md) | Explore key features after your first successful run | + +--- + +## Community and Support + +- **Slack:** [OpenMSP Community](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) β€” `https://www.openmsp.ai/` +- **Platform:** [Flamingo](https://flamingo.run) | [OpenFrame](https://openframe.ai) diff --git a/docs/getting-started/prerequisites.md b/docs/getting-started/prerequisites.md new file mode 100644 index 00000000000..5032ba24adc --- /dev/null +++ b/docs/getting-started/prerequisites.md @@ -0,0 +1,159 @@ +# Prerequisites + +Before building or running osquery with the OpenFrame integration, ensure your system meets the following requirements. + +--- + +## Hardware Requirements + +| Tier | RAM | CPU Cores | Disk Space | +|------|-----|-----------|------------| +| **Minimum** | 24 GB | 6 cores | 50 GB | +| **Recommended** | 32 GB | 12 cores | 100 GB | + +> **Note:** The CMake build system compiles a large number of bundled libraries. Builds with fewer than 6 CPU cores will be significantly slower. + +--- + +## Supported Operating Systems + +| Platform | Versions | +|----------|---------| +| **Linux** | Ubuntu 20.04+, Debian 11+, CentOS 8+, RHEL 8+, Fedora 36+ | +| **macOS** | 12 (Monterey)+, Apple Silicon (aarch64) and Intel (x86_64) | +| **Windows** | Windows 10/11, Windows Server 2019+ | + +--- + +## Required Software + +| Tool | Minimum Version | Purpose | +|------|----------------|---------| +| **CMake** | 3.21+ | Build system | +| **Clang** or **GCC** | Clang 12+ / GCC 10+ | C++ compiler (C++17 required) | +| **Git** | 2.30+ | Source control | +| **Python 3** | 3.8+ | Code generation scripts | +| **Bash** | 5.0+ | Build helper scripts | +| **Ninja** | 1.10+ | Recommended build backend (optional) | + +### Linux-Specific Dependencies + +```bash +# Ubuntu / Debian +sudo apt-get update && sudo apt-get install -y \ + build-essential cmake git python3 python3-pip \ + libssl-dev libgflags-dev libgoogle-glog-dev \ + librocksdb-dev libbz2-dev libsnappy-dev \ + libboost-all-dev liblzma-dev libthrift-dev + +# Fedora / RHEL / CentOS +sudo dnf install -y \ + gcc-c++ cmake git python3 \ + openssl-devel gflags-devel glog-devel \ + boost-devel xz-devel +``` + +### macOS-Specific Dependencies + +```bash +# Homebrew is required +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + +# Install dependencies +brew install cmake git python3 openssl gflags glog boost thrift +``` + +### Windows-Specific Dependencies + +Download and install the following: + +- [Visual Studio 2022](https://visualstudio.microsoft.com/) (with C++ Desktop workload) +- [CMake for Windows](https://cmake.org/download/) +- [Git for Windows](https://git-scm.com/download/win) +- [Python 3 for Windows](https://www.python.org/downloads/windows/) + +--- + +## OpenFrame Integration Requirements + +If you are building with OpenFrame mode enabled, you additionally need: + +| Requirement | Details | +|-------------|---------| +| **OpenFrame Account** | An active account at [openframe.ai](https://openframe.ai) | +| **Authentication Token** | A valid OpenFrame bearer token | +| **Network Access** | Outbound HTTPS to OpenFrame API endpoints | + +--- + +## Environment Variables + +The following environment variables may be needed during build and runtime: + +| Variable | Purpose | Example | +|----------|---------|---------| +| `OPENSSL_ROOT_DIR` | Path to OpenSSL installation | Set if CMake cannot auto-detect | +| `BOOST_ROOT` | Path to Boost installation | Set if CMake cannot auto-detect | +| `OSQUERY_TOOLCHAIN_SYSROOT` | Cross-compilation sysroot | Optional, for cross-compile targets | + +> Inside code blocks, variables like `$OPENSSL_ROOT_DIR` are referenced directly. In prose, always use backtick notation to avoid ambiguity. + +--- + +## Verification Commands + +Run these commands to verify your environment is ready before building: + +```bash +# Verify CMake version +cmake --version + +# Verify compiler +c++ --version + +# Verify Git +git --version + +# Verify Python 3 +python3 --version + +# Verify available memory (Linux) +free -h + +# Verify available memory (macOS) +sysctl hw.memsize | awk '{print $2/1073741824 " GB"}' + +# Verify disk space +df -h . +``` + +Expected output example: + +```text +cmake version 3.28.1 +g++ (Ubuntu 12.3.0) 12.3.0 +git version 2.43.0 +Python 3.11.6 + total used free +Mem: 31G 4.2G 27G +Disk: 95G available +``` + +--- + +## Build Tools Compatibility Matrix + +| CMake Version | Clang Version | GCC Version | Status | +|--------------|--------------|------------|--------| +| 3.21+ | 12+ | 10+ | βœ… Supported | +| 3.18-3.20 | 11 | 9 | ⚠️ May work, not tested | +| < 3.18 | < 11 | < 9 | ❌ Not supported | + +--- + +## Community Support + +If you run into dependency issues, join the community: + +- **Slack:** [OpenMSP Community](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) +- **Platform:** [openmsp.ai](https://www.openmsp.ai/) diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md new file mode 100644 index 00000000000..da0bc532e3a --- /dev/null +++ b/docs/getting-started/quick-start.md @@ -0,0 +1,193 @@ +# Quick Start + +Get osquery (with OpenFrame integration) built and running in under 5 minutes. + +--- + +## TL;DR β€” 4 Steps to Your First Query + +```bash +# 1. Clone the repository +git clone https://github.com/flamingo-stack/osquery.git +cd osquery + +# 2. Configure the build with CMake +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DOSQUERY_BUILD_TESTS=OFF + +# 3. Build (use -j with your CPU core count) +cmake --build build --target osqueryi -j$(nproc) + +# 4. Run your first query +./build/osqueryi "SELECT name, pid, user_time FROM processes LIMIT 5;" +``` + +> **Tip:** On macOS replace `$(nproc)` with `$(sysctl -n hw.logicalcpu)`. +> On Windows, use the Visual Studio Developer Command Prompt and CMake's `--parallel` flag. + +--- + +## Windows Quick Start + +Download the Windows AMD64 CLI installer: + +- [openframe-cli_windows_amd64.zip](https://github.com/flamingo-stack/openframe-cli/releases/latest/download/openframe-cli_windows_amd64.zip) + +Then run the installer the same way as other platforms after extracting. + +--- + +## Step-by-Step Walkthrough + +### Step 1: Clone the Repository + +```bash +git clone https://github.com/flamingo-stack/osquery.git +cd osquery +``` + +### Step 2: Configure the CMake Build + +```bash +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DOSQUERY_BUILD_TESTS=OFF \ + -DOSQUERY_NO_DEBUG_SYMBOLS=ON +``` + +Key CMake options: + +| Option | Default | Description | +|--------|---------|-------------| +| `CMAKE_BUILD_TYPE` | `Debug` | Use `Release` for production | +| `OSQUERY_BUILD_TESTS` | `ON` | Set `OFF` to skip test compilation | +| `OSQUERY_NO_DEBUG_SYMBOLS` | `OFF` | Set `ON` to reduce binary size | +| `OSQUERY_BUILD_EXTENSIONS` | `ON` | Build the extensions framework | + +### Step 3: Build osqueryi (Interactive Shell) + +```bash +cmake --build build --target osqueryi -j$(nproc) +``` + +Or build the full daemon: + +```bash +cmake --build build --target osqueryd -j$(nproc) +``` + +### Step 4: Run Your First Query + +```bash +./build/osqueryi +``` + +You should see the interactive prompt: + +```text +Using a virtual database. Need help, type '.help' +osquery> +``` + +--- + +## "Hello World" β€” Your First Queries + +Once inside `osqueryi`, try these queries to verify everything is working: + +```sql +-- System info +SELECT hostname, cpu_type, cpu_brand, physical_memory FROM system_info; + +-- Running processes +SELECT name, pid, state FROM processes LIMIT 10; + +-- Network interfaces +SELECT interface, address, mask FROM interface_addresses; + +-- Listening ports +SELECT port, protocol, pid FROM listening_ports WHERE protocol = 6 LIMIT 5; +``` + +### Expected Output + +```text +osquery> SELECT hostname, cpu_type FROM system_info; ++--------------------+---------+ +| hostname | cpu_type| ++--------------------+---------+ +| my-workstation | x86_64 | ++--------------------+---------+ +``` + +--- + +## Running as a Daemon (osqueryd) + +Create a minimal configuration file: + +```json +{ + "options": { + "logger_path": "/var/log/osquery", + "database_path": "/var/osquery/osquery.db" + }, + "schedule": { + "system_info": { + "query": "SELECT * FROM system_info;", + "interval": 3600 + }, + "listening_ports": { + "query": "SELECT * FROM listening_ports;", + "interval": 60 + } + } +} +``` + +Run the daemon: + +```bash +sudo ./build/osqueryd --config_path /etc/osquery/osquery.conf --verbose +``` + +--- + +## Enable OpenFrame Mode + +If you have an OpenFrame token, enable the integration at startup: + +```bash +./build/osqueryd \ + --openframe_mode=true \ + --openframe_token_path=/etc/osquery/openframe_token \ + --config_path=/etc/osquery/osquery.conf +``` + +The token refresher will start automatically in the background, keeping your authentication credentials up to date. + +--- + +## Verify the Build Succeeded + +```bash +# Check the binary exists and is executable +ls -la build/osqueryi build/osqueryd + +# Confirm osquery version +./build/osqueryi --version + +# Run a quick self-check +./build/osqueryi "SELECT * FROM osquery_info;" +``` + +--- + +## Next Steps + +After your first successful run: + +- Review the [Prerequisites](prerequisites.md) if you encounter build issues +- Work through [First Steps](first-steps.md) to explore key features and configuration +- Join the [OpenMSP Slack community](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) for help From 5664a951d18aec774ada55f8ad1f4efd3c322f13 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Jun 2026 19:22:45 +0000 Subject: [PATCH 7/7] docs: Stage 4 - Repository documentation (4 files) [skip ci] --- CONTRIBUTING.md | 437 +++++++++++++++++++++++------------------------- README.md | 289 +++++++++++++++----------------- docs/README.md | 139 +++++++++++++++ 3 files changed, 480 insertions(+), 385 deletions(-) create mode 100644 docs/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5e5092f04d5..23221c70aec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,365 +1,348 @@ # Contributing to osquery β€” OpenFrame Edition -Thank you for contributing to osquery with OpenFrame! This guide covers everything you need to get started: code style, branching conventions, commit format, testing requirements, and the pull request process. +Thank you for your interest in contributing to osquery with the OpenFrame integration! This document covers everything you need: code style, branching conventions, commit messages, pull requests, and the review process. --- -## Community First +## πŸ’¬ Community First -All collaboration happens on the **OpenMSP Slack community** β€” not GitHub Issues or GitHub Discussions. +All discussion, questions, bug reports, and feature requests are handled via the **OpenMSP Slack** community. We do not use GitHub Issues or GitHub Discussions. -| Resource | Link | -|---|---| -| πŸ’¬ OpenMSP Community Slack | [Join here](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) | -| 🌐 OpenMSP Website | [https://www.openmsp.ai/](https://www.openmsp.ai/) | +- **Slack:** [Join OpenMSP](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) +- **Community site:** [openmsp.ai](https://www.openmsp.ai/) -Before starting significant work, please discuss your changes in Slack to align with the team's roadmap. +**Before opening a pull request**, please discuss the change in Slack to ensure it aligns with the project roadmap. --- -## Development Setup +## πŸ› οΈ Development Setup ### Hardware Requirements -| Tier | RAM | CPU Cores | Disk | -|---|---|---|---| +| Tier | RAM | CPU Cores | Disk Space | +|------|-----|-----------|------------| | **Minimum** | 24 GB | 6 cores | 50 GB | | **Recommended** | 32 GB | 12 cores | 100 GB | -### Required Tools - -| Tool | Minimum Version | Purpose | -|---|---|---| -| CMake | 3.21+ | Build system generator | -| Python | 3.8+ | Code generation scripts | -| Git | 2.x | Source control | -| C++ Compiler | GCC 9+ / Clang 10+ / MSVC 2019+ | C++17 compilation | -| Ninja | 1.10+ | Fast parallel builds | -| OpenSSL | 1.1.1+ | TLS + AES-256-GCM (OpenFrame layer) | -| clang-format | β€” | Code formatting (CI enforced) | - -### Quick Setup +### Quick Developer Commands ```bash -# Clone +# Clone the repository git clone https://github.com/flamingo-stack/osquery.git cd osquery +git submodule update --init --recursive -# Configure (Debug build for development) +# Configure build (Debug mode with tests) cmake -S . -B build \ - -G Ninja \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DOSQUERY_BUILD_TESTS=ON + -DOSQUERY_BUILD_TESTS=ON \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -# Build -cmake --build build --parallel $(nproc) +# Build everything +cmake --build build -j$(nproc) + +# Run tests +cd build && ctest --output-on-failure -# Run -./build/osquery/osqueryi +# Format code +cmake --build build --target clang-format ``` -For full environment setup instructions see the [Development Documentation](./docs/development/README.md). +See [Development Documentation](./docs/README.md) for full environment setup, local development, testing, and security guides. --- -## Code Style and Conventions +## ✏️ Code Style and Conventions -### C++ Standards +The project follows the **Google C++ Style Guide** with osquery-specific modifications. `clang-format` enforces formatting β€” configuration is in `.clang-format` at the project root. -- Use **C++17** features where appropriate -- Follow the existing code style in each file you modify -- All new code must pass `clang-format` with the repository's `.clang-format` config - -### Formatting - -osquery enforces `clang-format`. Run it before every commit: +**Before every commit, format your changes:** ```bash -# Format a single file -clang-format -i path/to/your/file.cpp - -# Format all changed files (compared to main branch) -git diff --name-only main | grep -E '\.(cpp|h)$' | xargs clang-format -i +# Format all staged C++ files +git diff --cached --name-only | grep -E '\.(cpp|h)$' | xargs clang-format -i -# Check without modifying -clang-format --dry-run --Werror path/to/your/file.cpp +# Format all modified files +git diff --name-only | grep -E '\.(cpp|h)$' | xargs clang-format -i ``` -### Naming Conventions +### Key Conventions -| Item | Convention | Example | -|---|---|---| -| Classes | `PascalCase` | `EventSubscriberPlugin` | -| Methods | `camelCase` | `generateRows()` | -| Member variables | `snake_case_` (trailing underscore) | `running_` | -| Constants | `kPascalCase` | `kSQLOpcodes` | -| Macros | `UPPER_SNAKE_CASE` | `DECLARE_FLAG` | -| Namespaces | `snake_case` | `osquery` | -| Files | `snake_case.cpp` / `snake_case.h` | `event_subscriber.cpp` | +| Convention | Rule | +|------------|------| +| **Namespacing** | All code lives in the `osquery` namespace | +| **File naming** | `snake_case.cpp` / `snake_case.h` | +| **Class naming** | `PascalCase` | +| **Function naming** | `camelCase` | +| **Constant naming** | `kConstantName` prefix | +| **Member variables** | Trailing underscore: `member_` | +| **Include guards** | `#pragma once` (not `#ifndef` guards) | +| **Smart pointers** | Prefer `std::shared_ptr` and `std::unique_ptr` over raw pointers | +| **Error handling** | Use `osquery::Status` or `osquery::Expected` | -### Include Order - -Follow this include order with blank lines between groups: +### Example: Correct Style ```cpp -// 1. Standard library +#pragma once + #include #include -#include -// 2. Third-party libraries -#include -#include +namespace osquery { -// 3. osquery headers -#include "osquery/core/core.h" -#include "osquery/sql/sql.h" +/// Brief description of what this class does. +class MyNewFeature { + public: + explicit MyNewFeature(std::shared_ptr dep); + ~MyNewFeature(); -// 4. Local headers (same directory) -#include "my_local_header.h" -``` + /// Returns the current feature value, or an error status. + Expected getValue() const; -### Code Organization + private: + std::shared_ptr dependency_; + std::string cached_value_; +}; -- Keep headers (`*.h`) minimal β€” forward declare where possible -- Use the `osquery` namespace for all production code -- Place tests in `tests/` subdirectories alongside the source -- New virtual tables go in `osquery/tables//` +} // namespace osquery +``` --- -## Branch Naming +## 🌿 Branch Naming -Always branch from the latest `main`: +| Branch Type | Pattern | Example | +|-------------|---------|---------| +| Feature | `feature/` | `feature/openframe-token-rotation` | +| Bug fix | `fix/` | `fix/sql-authorizer-pragma-list` | +| Documentation | `docs/` | `docs/extension-api-guide` | +| Refactor | `refactor/` | `refactor/database-interface` | +| Hotfix | `hotfix/` | `hotfix/token-refresher-crash` | ```bash +# Create a feature branch from main git checkout main git pull origin main -git checkout -b feature/my-new-feature +git checkout -b feature/my-new-virtual-table ``` -| Type | Pattern | Example | -|---|---|---| -| Feature | `feature/` | `feature/bpf-socket-events` | -| Bug fix | `fix/` | `fix/config-refresh-race` | -| OpenFrame integration | `openframe/` | `openframe/token-refresh-retry` | -| Documentation | `docs/` | `docs/virtual-table-guide` | -| Refactor | `refactor/` | `refactor/sql-authorizer` | -| Test | `test/` | `test/events-integration` | -| Release | `release/v` | `release/v5.13.0` | - --- -## Commit Message Format +## πŸ“ Commit Message Format + +Use the **Conventional Commits** specification: ```text (): - +[Optional body β€” explain WHY, not WHAT] - +[Optional footer β€” breaking changes, references] ``` ### Types | Type | When to Use | -|---|---| -| `feat` | New feature or capability | +|------|-------------| +| `feat` | New feature or virtual table | | `fix` | Bug fix | | `docs` | Documentation changes only | -| `style` | Code formatting, no logic change | -| `refactor` | Code refactoring without behavior change | -| `test` | Adding or fixing tests | +| `refactor` | Code change that neither fixes a bug nor adds a feature | +| `test` | Adding or modifying tests | | `perf` | Performance improvement | -| `chore` | Build, CI, tooling changes | -| `openframe` | OpenFrame platform-specific changes | - -### Scopes - -| Scope | Area | -|---|---| -| `core` | Core init and runtime | -| `sql` | SQL engine and virtual tables | -| `config` | Configuration and packs | -| `events` | Eventing framework | -| `logger` | Logging and observability | -| `db` | Database and storage | -| `distributed` | Distributed querying | -| `extensions` | Extension IPC | -| `http` | Remote HTTP client | -| `openframe` | OpenFrame auth layer | -| `tables` | Virtual table implementations | +| `ci` | CI/CD configuration changes | +| `chore` | Build system, dependency updates | ### Examples ```text -feat(events): add BPF socket event publisher for Linux - -Implements a new BPF-based publisher that captures socket connect/accept -events and exposes them via the bpf_socket_events virtual table. +feat(sql): add constraint pushdown for inet_diag table +fix(events): prevent duplicate subscription on config reload +docs(openframe): add token rotation guide +refactor(database): extract IDatabaseInterface for testability +test(core): add flag override tests for daemon mode +perf(events): use batch DB writes in EventSubscriberPlugin ``` -```text -fix(openframe): handle token refresh failure with exponential backoff - -When OpenframeTokenRefresher encounters a network error, it now retries -with exponential backoff instead of immediately stopping the refresh loop. -``` +### Full Commit Example ```text -test(config): add pack discovery query unit tests -``` - ---- - -## Testing +feat(openframe): add configurable token refresh interval -Tests use **Google Test** and **Google Mock**. Always build with `-DOSQUERY_BUILD_TESTS=ON`: +Previously the token refresher used a hardcoded 5-minute interval. +This change reads the interval from the --openframe_refresh_interval +flag, allowing operators to tune refresh frequency based on their +token expiry policies. -```bash -cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DOSQUERY_BUILD_TESTS=ON -cmake --build build --parallel $(nproc) - -# Run all tests -cd build && ctest --output-on-failure - -# Run in parallel -cd build && ctest --output-on-failure --parallel $(nproc) - -# Run a specific suite -cd build && ctest -R "osquery_sql_tests" --output-on-failure +Closes: FLAMINGO-1234 ``` -### Test Categories +--- -| Category | Location | Description | -|---|---|---| -| Unit Tests | `osquery/*/tests/` | Fast, isolated, no OS dependencies | -| Integration Tests | `tests/integration/tables/` | Live table queries against the real OS | -| Extension Tests | `osquery/extensions/tests/` | IPC and Thrift round-trips | -| Plugin Tests | `plugins/*/tests/` | Logger, config, and database plugins | +## πŸ”„ Pull Request Process -When contributing a new virtual table, a corresponding integration test in `tests/integration/tables/` is expected. +### Before Opening a PR ---- +1. **Discuss in Slack** β€” confirm the change is wanted before investing time +2. **Branch from `main`** β€” never commit directly to `main` +3. **Run tests** β€” all tests must pass locally +4. **Format code** β€” run `clang-format` on all changed files +5. **Update docs** β€” if you add a new module or change behavior, update or add inline docs -## Pull Request Process +```bash +# Full pre-PR checklist +git checkout -b feature/my-change +# ... make changes ... -### Before Submitting +# 1. Format +git diff --name-only | grep -E '\.(cpp|h)$' | xargs clang-format -i -- [ ] Branch is up-to-date with `main` -- [ ] All tests pass: `cd build && ctest --output-on-failure` -- [ ] Code is formatted: `clang-format --dry-run --Werror` -- [ ] Copyright headers are present on new files -- [ ] New virtual tables have integration tests in `tests/integration/tables/` -- [ ] OpenFrame-specific changes include updated documentation -- [ ] Discussed in OpenMSP Slack (for significant changes) +# 2. Build +cmake --build build -j$(nproc) -### PR Title Format +# 3. Test +cd build && ctest --output-on-failure -Use the same format as commit messages: +# 4. Commit +git add -A +git commit -m "feat(sql): add my new virtual table" -```text -feat(sql): add query result caching for repeated virtual table scans +# 5. Push +git push origin feature/my-change ``` ### PR Description Template ```markdown ## Summary - +Brief description of what this PR does and why. ## Changes - +- Added `MyNewTable` virtual table in `osquery/tables/system/` +- Added unit tests in `tests/integration/tables/my_new_table.cpp` +- Updated CMakeLists.txt to include new table ## Testing - - -## Platform Support - +- [ ] All existing tests pass (`ctest --output-on-failure`) +- [ ] New tests added for new functionality +- [ ] Manually tested on Linux x86_64 +- [ ] Manually tested on macOS (if applicable) -## Checklist -- [ ] Tests pass (ctest) -- [ ] clang-format applied -- [ ] Documentation updated (if applicable) -- [ ] Discussed in OpenMSP Slack (if significant change) +## Breaking Changes +None / Describe any breaking changes here. ``` --- -## Copyright Headers +## βœ… Code Review Checklist -All new source files must include a copyright header: +Reviewers should verify: -```cpp -/** - * Copyright (c) 2014-present, The osquery authors - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ -``` +### Correctness +- [ ] Logic is correct and handles edge cases +- [ ] Error paths return appropriate `Status` or `Expected<>` values +- [ ] No resource leaks (file handles, DB connections, threads) + +### Security +- [ ] No hardcoded secrets or credentials +- [ ] Query constraints from `QueryContext` are validated before use +- [ ] New network code uses the `Remote HTTP Client` module +- [ ] Thread-shared state is properly synchronized + +### Style and Quality +- [ ] Follows naming conventions (see above) +- [ ] `clang-format` applied to all changed files +- [ ] Public APIs have doc comments +- [ ] No dead code or commented-out blocks -The CI script `tools/ci/scripts/check_copyright_headers.py` enforces this on all pull requests. +### Tests +- [ ] New functionality has tests +- [ ] Tests are independent (use ephemeral DB, no shared state) +- [ ] Tests cover both happy path and error paths + +### Documentation +- [ ] Inline docs updated if behavior changes +- [ ] New modules have doc comments on public API --- -## Adding a New Virtual Table +## πŸ—ƒοΈ Adding a New Virtual Table -1. Define the table schema in `osquery/tables//.table` -2. Run the code generator: +Virtual tables are the most common contribution type. Follow this pattern: -```bash -python3 tools/codegen/gentable.py osquery/tables//.table -``` +1. **Create the table spec** in `osquery/tables/` (appropriate subdirectory) +2. **Implement `generate()`** β€” use `QueryContext` for constraint pushdown +3. **Register the table** in the appropriate `CMakeLists.txt` +4. **Add an integration test** in `tests/integration/tables/` +5. **Test on all target platforms** where the table is supported -3. Implement the `generate()` method in `.cpp` -4. Register in the CMakefile for your category -5. Add an integration test in `tests/integration/tables/.cpp` -6. Test locally: +```cpp +// osquery/tables/system/linux/my_new_table.cpp +#include +#include -```bash -cmake --build build --target osqueryi -./build/osquery/osqueryi -osquery> SELECT * FROM ; +namespace osquery { +namespace tables { + +TableRows genMyNewTable(QueryContext& context) { + TableRows results; + + // Validate constraints before use + auto paths = context.constraints["path"].getAll(EQUALS); + for (const auto& path : paths) { + if (path.empty()) continue; + + Row r; + r["path"] = path; + r["size"] = INTEGER(0); // populate from OS API + results.push_back(r); + } + + return results; +} + +} // namespace tables +} // namespace osquery ``` --- -## Security Guidelines +## πŸ”’ Security Guidelines + +Before submitting a pull request, verify: -- **Never** hardcode secrets, credentials, or tokens in source code -- **Never** log JWT token values, even at debug level -- SQL inputs must pass through the SQLite authorizer β€” do not bypass it -- AES-GCM nonces must be generated freshly (never reused for the same key) -- TLS peer verification must **not** be disabled in production code -- New config keys must include size/depth validation -- Thread-shared state must use proper synchronization primitives +- [ ] No secrets, tokens, or API keys in source code or test fixtures +- [ ] New virtual tables sanitize all `QueryContext` constraints before use +- [ ] New network-facing code uses the `Remote HTTP Client` module (not raw sockets) +- [ ] All file paths received from queries are validated +- [ ] New config parameters have documented size/depth limits if they accept user data +- [ ] Thread-shared state uses appropriate synchronization primitives -Report security vulnerabilities directly to the Flamingo team via the **OpenMSP Slack community** β€” do not open public GitHub Issues for security issues. +To report a security vulnerability, **do not open a GitHub Issue**. Contact the team via the [OpenMSP Slack](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA). --- -## Reviewer Checklist +## πŸ§ͺ CI Test Matrix -When reviewing a PR: +Tests are run across the following configurations: -- [ ] Logic is correct and all error paths are handled -- [ ] No secrets or credentials in source -- [ ] SQL inputs are validated through the authorizer -- [ ] Thread safety considered for shared state -- [ ] Platform-specific code is properly guarded with `#ifdef` -- [ ] Tests added for new functionality -- [ ] No debug/temporary code left in -- [ ] Commit messages follow format convention -- [ ] Performance impact considered for hot paths (scheduler, SQL engine) +| Platform | Compiler | Build Type | +|----------|----------|------------| +| Linux x86_64 | GCC 11 | Debug + Release | +| Linux aarch64 | GCC 11 | Debug | +| macOS x86_64 | AppleClang | Debug + Release | +| macOS aarch64 | AppleClang | Debug + Release | +| Windows x86_64 | MSVC 2022 | Debug + Release | --- -## Code of Conduct +## πŸ†˜ Getting Help + +Stuck on something? The community is here to help: -Be respectful, collaborative, and constructive. All contributors are expected to maintain a professional and welcoming environment in both Slack and code reviews. +- **Slack:** [OpenMSP Community](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) +- **Community:** [openmsp.ai](https://www.openmsp.ai/) +- **Platform:** [flamingo.run](https://flamingo.run) | [openframe.ai](https://openframe.ai) --- diff --git a/README.md b/README.md index a707452a43e..ac28dde6cbc 100644 --- a/README.md +++ b/README.md @@ -12,158 +12,138 @@ # osquery β€” OpenFrame Edition -**osquery** is a cross-platform operating system instrumentation framework that exposes live system state and activity as relational data β€” enabling you to query your operating system using SQL. +**osquery** is a cross-platform operating system instrumentation framework that exposes system state as relational tables and allows it to be queried using SQL. This repository includes the **OpenFrame integration** β€” enabling authenticated, token-managed runtime identity for [OpenFrame](https://openframe.ai)-connected deployments on the [Flamingo](https://flamingo.run) MSP platform. -Built and extended by the [Flamingo / OpenFrame](https://openframe.ai) platform, this distribution integrates osquery's powerful SQL telemetry engine with OpenFrame's AI-driven MSP automation infrastructure. Every host becomes a **SQL-queryable telemetry node** β€” no proprietary log parsers, no fragile scripts, just standard SQL. +Instead of writing platform-specific scripts to inspect processes, files, users, network state, or kernel events, osquery gives you a unified SQL interface to your entire fleet. ```sql --- Which processes are listening on ports? -SELECT pid, name, port, protocol FROM listening_ports JOIN processes USING (pid); - --- What Chrome extensions are installed? -SELECT u.username, e.name, e.version, e.permissions -FROM users u, chrome_extensions e -WHERE e.uid = u.uid; +-- Find all listening TCP ports and the processes behind them +SELECT p.name, p.pid, lp.port, lp.protocol +FROM listening_ports lp +JOIN processes p ON lp.pid = p.pid +WHERE lp.protocol = 6; ``` -## Features +--- + +## ▢️ Watch: osquery Overview -- **SQL Querying** β€” Query live OS data with standard SQL via an embedded, hardened SQLite engine -- **300+ Virtual Tables** β€” Platform-specific tables across Linux, macOS, and Windows covering processes, sockets, packages, users, registry, hardware, and more -- **Event Monitoring** β€” inotify, BPF, FSEvents, ETW, and OpenBSM events exposed as queryable tables -- **Scheduled Query Packs** β€” Configuration-driven query scheduling with differential result tracking (only changed rows are logged) -- **Distributed Fleet Queries** β€” Remote SQL orchestration across entire fleets via TLS -- **Extension System** β€” Runtime plugin model via Apache Thrift IPC β€” add custom tables, loggers, and config plugins as external processes -- **OpenFrame Auth Layer** β€” AES-256-GCM encrypted JWT token management (`OpenframeAuthorizationManager`, `OpenframeEncryptionService`, `OpenframeTokenRefresher`) for seamless integration with the Flamingo/OpenFrame MSP platform -- **Cross-Platform** β€” Linux (x86\_64, aarch64), macOS (Intel + Apple Silicon), and Windows x86\_64 -- **Security-First** β€” SQLite authorizer allowlists only safe opcodes, Watcher/Worker process isolation, peer-verified TLS everywhere +[![osquery Introduction](https://img.youtube.com/vi/KbGH9IBELWA/hqdefault.jpg)](https://www.youtube.com/watch?v=KbGH9IBELWA) --- -## Architecture +## ✨ Features + +| Feature | Description | +|---------|-------------| +| **SQL Interface** | Query OS state with standard SQL across all platforms | +| **Virtual Tables** | 300+ built-in tables covering processes, network, files, users, and more | +| **Event-Driven Monitoring** | Real-time capture of file changes, process activity, and network events | +| **Scheduled Queries** | Continuously run queries and log differential results | +| **Distributed Querying** | Push ad-hoc queries to remote nodes from a control plane | +| **Extensible Plugin System** | Add custom tables, loggers, and config sources via extensions | +| **Cross-Platform** | Linux, macOS, and Windows support | +| **OpenFrame Integration** | Secure token-based authentication for MSP fleet management | + +--- + +## πŸ—οΈ Architecture + +osquery is a layered, plugin-driven C++ application. At its core is an embedded SQLite engine that exposes OS state through virtual tables. An event framework captures real-time activity, a distributed query system handles remote work, and the OpenFrame integration provides authenticated connectivity to the Flamingo MSP platform. ```mermaid -graph TD +flowchart TD CLI["osqueryi / osqueryd"] --> Core["Core Init And Runtime"] - Core --> Config["Configuration And Packs"] - Core --> SQL["SQL Engine And Virtual Tables"] - Core --> Events["Eventing Framework"] - Core --> Logging["Logging And Observability"] - Core --> DB["Database And Storage Plugins"] - Core --> Dist["Distributed Querying"] - Core --> Ext["Extensions And IPC"] - Core --> HTTP["Remote HTTP Client"] - Core --> OF["OpenFrame Auth Layer"] + Core --> Config["Config And Packs"] + Core --> Registry["Plugin Registry"] + Core --> Database["Database Backends (RocksDB)"] + Core --> SQL["SQL Core And Virtual Tables"] + Core --> Events["Events Core (pub/sub)"] + Core --> Distributed["Distributed Querying"] + Core --> Logger["Plugin Interfaces And Logging"] + Core --> Extensions["Extensions Framework (Thrift)"] + Core --> HTTP["Remote HTTP Client (Boost.Beast)"] + Core --> OpenFrame["OpenFrame Integration"] Config --> SQL - Config --> Events - SQL --> Logging - Events --> DB - Dist --> SQL - Dist --> HTTP - Ext --> SQL - Ext --> DB - OF --> HTTP + SQL --> QueryExec["Query Execution And Logging"] + QueryExec --> Logger + Events --> SQL + Distributed --> SQL + Distributed --> HTTP + OpenFrame --> AuthMgr["Authorization Manager"] + OpenFrame --> Refresher["Token Refresher Thread"] ``` -| Module | Location | Responsibility | -|---|---|---| -| Core Init And Runtime | `osquery/core/` | Process lifecycle, flags, watcher/worker model | -| SQL Engine And Virtual Tables | `osquery/sql/` | SQLite engine, authorizer, virtual table binding | -| Configuration And Packs | `osquery/config/` | Scheduled queries, packs, decorators | -| Eventing Framework | `osquery/events/` | Publisher/subscriber event system | -| Logging And Observability | `osquery/logger/` | Differential logging, JSON serialization | -| Database And Storage Plugins | `osquery/database/` | RocksDB persistent + ephemeral in-memory stores | -| Distributed Querying | `osquery/distributed/` | Remote fleet orchestration | -| Extensions And IPC | `osquery/extensions/` | Apache Thrift-based runtime extensions | -| Remote HTTP Client | `osquery/remote/` | Boost.Asio/Beast HTTPS with strict TLS | -| OpenFrame Auth Layer | `openframe/` | JWT token management + AES-256-GCM encryption | +### Execution Modes + +- **Daemon mode (`osqueryd`)** β€” scheduled and distributed queries running continuously +- **Interactive shell (`osqueryi`)** β€” ad-hoc SQL exploration +- **Extension processes** β€” dynamically inject custom plugins at runtime +- **Watcher/Worker model** β€” supervised execution for stability --- -## Technology Stack +## πŸ› οΈ Technology Stack | Layer | Technology | -|---|---| -| Language | C++17 | -| Build System | CMake 3.21+ with Ninja | -| SQL Engine | SQLite (embedded, in-memory) | -| IPC | Apache Thrift (UNIX sockets / named pipes) | -| Encryption | OpenSSL (AES-256-GCM) | -| Networking | Boost.Asio + Boost.Beast | -| Event Systems | inotify, BPF, FSEvents, ETW, OpenBSM | -| Database | RocksDB (persistent), ephemeral in-memory | -| Testing | Google Test + Google Mock | +|-------|-----------| +| **Language** | C++17 | +| **Build System** | CMake 3.21+ | +| **SQL Engine** | Embedded SQLite with virtual table extensions | +| **IPC / Extensions** | Apache Thrift over UNIX domain sockets / Windows named pipes | +| **Networking** | Boost.Asio + Boost.Beast + OpenSSL | +| **Storage** | RocksDB (persistent) / In-memory ephemeral backend | +| **Event Publishers** | BPF / inotify (Linux), EndpointSecurity (macOS), ETW (Windows) | +| **OpenFrame Auth** | Custom token extractor + refresher with Boost threading | --- -## Hardware Requirements +## πŸ”§ Hardware Requirements -| Tier | RAM | CPU Cores | Disk | -|---|---|---|---| +| Tier | RAM | CPU Cores | Disk Space | +|------|-----|-----------|------------| | **Minimum** | 24 GB | 6 cores | 50 GB | | **Recommended** | 32 GB | 12 cores | 100 GB | -> Building from source is resource-intensive. The recommended configuration significantly reduces build times and prevents out-of-memory failures during compilation. - ---- - -## Supported Platforms - -| Platform | Architecture | Notes | -|---|---|---| -| Linux | x86\_64, aarch64 | Ubuntu 20.04+, RHEL 8+, Debian 11+ | -| macOS | x86\_64, aarch64 | macOS 12+ (Intel + Apple Silicon) | -| Windows | x86\_64 | Windows 10/11, Server 2019+ | - --- -## Quick Start +## πŸš€ Quick Start -### 1. Install Prerequisites - -**Linux (Debian/Ubuntu):** +### 1. Clone the Repository ```bash -sudo apt-get update -sudo apt-get install -y \ - build-essential cmake ninja-build python3 python3-pip \ - git openssl libssl-dev clang-format ccache +git clone https://github.com/flamingo-stack/osquery.git +cd osquery ``` -**macOS:** +### 2. Configure the Build ```bash -xcode-select --install -brew install cmake ninja python3 openssl git ccache +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DOSQUERY_BUILD_TESTS=OFF ``` -**Windows:** - -1. Install [Visual Studio 2022](https://visualstudio.microsoft.com/) with the **Desktop development with C++** workload -2. Install [CMake 3.21+](https://cmake.org/download/), [Git](https://git-scm.com/download/win), [Python 3.8+](https://www.python.org/downloads/windows/) -3. Download the CLI binary directly: [openframe-cli\_windows\_amd64.zip](https://github.com/flamingo-stack/openframe-cli/releases/latest/download/openframe-cli_windows_amd64.zip) - -### 2. Clone and Build +### 3. Build ```bash -# Clone -git clone https://github.com/flamingo-stack/osquery.git -cd osquery +# Linux +cmake --build build --target osqueryi -j$(nproc) -# Configure (Linux/macOS) -cmake -S . -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo -G Ninja +# macOS +cmake --build build --target osqueryi -j$(sysctl -n hw.logicalcpu) +``` -# Configure (Windows) -cmake -S . -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo -G "Visual Studio 17 2022" +### 4. Run Your First Query -# Build β€” uses all available CPU cores -cmake --build build --parallel $(nproc) +```bash +./build/osqueryi "SELECT name, pid, user_time FROM processes LIMIT 5;" ``` -### 3. Run Your First Query +Or launch the interactive shell: ```bash -./build/osquery/osqueryi +./build/osqueryi ``` ```text @@ -171,77 +151,70 @@ Using a virtual database. Need help, type '.help' osquery> ``` -```sql -osquery> SELECT hostname, cpu_brand, physical_memory FROM system_info; -osquery> SELECT pid, name, port, protocol FROM listening_ports LIMIT 10; -osquery> SELECT uid, username, shell FROM users; -osquery> .tables -``` +### Windows Quick Start -### 4. Run as a Daemon +Download the Windows AMD64 CLI installer: -```bash -sudo mkdir -p /etc/osquery -sudo tee /etc/osquery/osquery.conf <<'EOF' -{ - "options": { - "logger_plugin": "filesystem", - "schedule_splay_percent": 10 - }, - "schedule": { - "system_info": { - "query": "SELECT hostname, cpu_brand, physical_memory FROM system_info;", - "interval": 3600 - } - } -} -EOF - -sudo ./build/osquery/osqueryd --config_path=/etc/osquery/osquery.conf -``` +- [openframe-cli_windows_amd64.zip](https://github.com/flamingo-stack/openframe-cli/releases/latest/download/openframe-cli_windows_amd64.zip) + +Extract and run the installer the same way as other platforms. --- -## OpenFrame Integration +## ⚑ Enable OpenFrame Mode -This distribution extends osquery with a secure authentication and encryption layer for the [OpenFrame platform](https://www.flamingo.run/openframe): +If you have an OpenFrame token, enable the integration at startup: -| Component | Description | -|---|---| -| `OpenframeAuthorizationManager` | Lifecycle-controlled JWT token singleton (non-copyable) | -| `OpenframeAuthorizationManagerProvider` | Sole factory and owner of the token manager | -| `OpenframeEncryptionService` | AES-256-GCM symmetric encryption via OpenSSL | -| `OpenframeTokenExtractor` | Token acquisition from OpenFrame services | -| `OpenframeTokenRefresher` | Background thread for seamless token renewal | +```bash +./build/osqueryd \ + --openframe_mode=true \ + --openframe_token_path=/etc/osquery/openframe_token \ + --config_path=/etc/osquery/osquery.conf +``` -Obtain your OpenFrame credentials from your platform administrator. Token lifecycle is fully automated once configured. Refer to your environment configuration for connection details. +The token refresher starts automatically in the background, keeping credentials fresh and authentication seamless. --- -## Documentation +## πŸ—‚οΈ Project Layout + +```text +osquery/ +β”œβ”€β”€ osquery/ # Core library modules +β”‚ β”œβ”€β”€ core/ # Runtime initialization, flags, shutdown +β”‚ β”œβ”€β”€ sql/ # SQLite engine and virtual table integration +β”‚ β”œβ”€β”€ events/ # Publish/subscribe event framework +β”‚ β”œβ”€β”€ config/ # Configuration loading and packs +β”‚ β”œβ”€β”€ database/ # RocksDB and ephemeral storage backends +β”‚ β”œβ”€β”€ distributed/ # Distributed query engine +β”‚ β”œβ”€β”€ extensions/ # Apache Thrift-based extension framework +β”‚ β”œβ”€β”€ remote/ # HTTP client (Boost.Beast + OpenSSL) +β”‚ β”œβ”€β”€ hashing/ # MD5, SHA1, SHA256 utilities +β”‚ └── filesystem/ # Cross-platform file operations +β”œβ”€β”€ openframe/ # OpenFrame authentication integration +β”œβ”€β”€ plugins/ # Config, logger, database, distributed plugins +β”œβ”€β”€ tables/ # Virtual table implementations (300+ tables) +β”œβ”€β”€ libraries/ # Vendored third-party libraries (CMake-managed) +β”œβ”€β”€ tools/ # Code generation, CI scripts, formatting tools +β”œβ”€β”€ tests/ # Integration and unit tests +└── external/ # Extension SDK examples +``` + +--- -πŸ“š See the [Documentation](./docs/README.md) for comprehensive guides. +## πŸ“š Documentation -- [Introduction](./docs/getting-started/introduction.md) β€” What is osquery + OpenFrame? -- [Prerequisites](./docs/getting-started/prerequisites.md) β€” System and software requirements -- [Quick Start](./docs/getting-started/quick-start.md) β€” Clone, build, and run -- [First Steps](./docs/getting-started/first-steps.md) β€” Explore tables, packs, FIM, extensions -- [Architecture Overview](./docs/development/architecture/README.md) β€” Module deep-dives -- [Local Development](./docs/development/setup/local-development.md) β€” Build configurations and debugging -- [Contributing Guidelines](./docs/development/contributing/guidelines.md) β€” Code style and PR process +See the [Documentation](./docs/README.md) for comprehensive guides covering architecture, development setup, testing, security, and API reference. --- -## Community and Support +## πŸ’¬ Community & Support -> We do **not** use GitHub Issues or GitHub Discussions. All support and collaboration happens on the **OpenMSP Slack community**. +All discussion, questions, and contributions are managed via the **OpenMSP Slack** community. We do not use GitHub Issues or GitHub Discussions. -| Resource | Link | -|---|---| -| πŸ’¬ OpenMSP Community Slack | [Join here](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) | -| 🌐 OpenMSP Website | [https://www.openmsp.ai/](https://www.openmsp.ai/) | -| πŸš€ OpenFrame Platform | [https://openframe.ai](https://openframe.ai) | -| 🦩 Flamingo | [https://flamingo.run](https://flamingo.run) | +- **Slack:** [Join OpenMSP](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) +- **Community:** [openmsp.ai](https://www.openmsp.ai/) +- **Platform:** [flamingo.run](https://flamingo.run) | [OpenFrame](https://openframe.ai) --- diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000000..29fe9c02851 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,139 @@ +# osquery β€” OpenFrame Edition: Documentation + +Welcome to the documentation for **osquery with the OpenFrame integration** β€” the SQL-powered OS instrumentation framework for the [Flamingo](https://flamingo.run) MSP platform. + +--- + +## πŸ“š Table of Contents + +- [Getting Started](#-getting-started) +- [Development](#-development) +- [Reference Architecture](#-reference-architecture) +- [Architecture Diagrams](#-architecture-diagrams) +- [Quick Links](#-quick-links) + +--- + +## πŸš€ Getting Started + +New to osquery? Start here for a guided introduction, prerequisites, and your first running query. + +| Guide | Description | +|-------|-------------| +| [Introduction](./getting-started/introduction.md) | What osquery is, key features, and the OpenFrame integration overview | +| [Prerequisites](./getting-started/prerequisites.md) | Hardware requirements, supported OS, required tools, and environment setup | +| [Quick Start](./getting-started/quick-start.md) | Build and run osquery in under 5 minutes | +| [First Steps](./getting-started/first-steps.md) | Explore key features, scheduled queries, events, and SQL patterns after your first build | + +--- + +## πŸ› οΈ Development + +Guides for contributors and platform engineers working on the codebase. + +| Guide | Description | +|-------|-------------| +| [Development Overview](./development/README.md) | Technology stack, project layout, quick developer commands, and community links | +| [Environment Setup](./development/setup/environment.md) | IDE recommendations (VS Code, CLion, Xcode, Visual Studio), toolchain setup, ccache, and sanitizers | +| [Local Development](./development/setup/local-development.md) | Clone, configure, build targets, run, watch-mode iteration, and debug configuration | +| [Architecture Overview](./development/architecture/README.md) | System architecture diagrams, boot flow, SQL execution data flow, and key design decisions | +| [Security Guidelines](./development/security/README.md) | Auth patterns, token storage, TLS configuration, input validation, and least-privilege setup | +| [Testing Guide](./development/testing/README.md) | Test structure, running tests, writing unit and integration tests, benchmarks, and CI matrix | +| [Contributing Guidelines](./development/contributing/guidelines.md) | Code style, branching, commit messages, pull request process, and review checklist | + +--- + +## πŸ“– Reference Architecture + +Deep-dive technical reference documentation for every major module, generated from source code analysis. + +### Core Runtime + +| Module | Description | +|--------|-------------| +| [Core Init And Runtime](./reference/architecture/core-init-and-runtime/core-init-and-runtime.md) | Lifecycle orchestration β€” flag parsing, plugin activation, watcher/worker model, graceful shutdown | +| [Config And Packs](./reference/architecture/config-and-packs/config-and-packs.md) | Configuration loading, query scheduling, pack management, and config refresh lifecycle | +| [Config Plugins](./reference/architecture/config-plugins/config-plugins.md) | Configuration retrieval and parsing plugin implementations | + +### SQL and Query Engine + +| Module | Description | +|--------|-------------| +| [SQL Core And Virtual Tables](./reference/architecture/sql-core-and-virtual-tables/sql-core-and-virtual-tables.md) | Embedded SQLite engine, virtual table integration, constraint pushdown, result diffing | +| [Query Execution And Logging](./reference/architecture/query-execution-and-logging/query-execution-and-logging.md) | Differential logging model, query state persistence, epoch and counter management | + +### Storage and Events + +| Module | Description | +|--------|-------------| +| [Database Backends](./reference/architecture/database-backends/database-backends.md) | RocksDB persistent storage, ephemeral in-memory backend, domain keyspaces, migrations | +| [Events Core](./reference/architecture/events-core/events-core.md) | Publish/subscribe event framework, event publishers, subscriber plugins, backing store | + +### Distributed and Networking + +| Module | Description | +|--------|-------------| +| [Distributed Querying](./reference/architecture/distributed-querying/distributed-querying.md) | Remote query orchestration, TLS distributed plugin, denylisting, concurrency control | +| [Remote HTTP Client](./reference/architecture/remote-http-client/remote-http-client.md) | Boost.Asio + OpenSSL HTTP/HTTPS transport, timeout handling, proxy support | + +### Extensions and Plugins + +| Module | Description | +|--------|-------------| +| [Extensions Framework](./reference/architecture/extensions-framework/extensions-framework.md) | Apache Thrift IPC, runtime plugin injection, extension manager, SDK | +| [Plugin Interfaces And Logging](./reference/architecture/plugin-interfaces-and-logging/plugin-interfaces-and-logging.md) | LoggerPlugin interface, status log routing, snapshot and diff result forwarding | + +### Utilities + +| Module | Description | +|--------|-------------| +| [Filesystem And Fileops](./reference/architecture/filesystem-and-fileops/filesystem-and-fileops.md) | Cross-platform file abstraction, stat normalization, globbing, socket lifecycle | +| [Hashing](./reference/architecture/hashing/hashing.md) | Streaming hash computation, multi-algorithm support (MD5, SHA1, SHA256) | +| [Process And Profiler](./reference/architecture/process-and-profiler/process-and-profiler.md) | Worker and extension spawning, POSIX signal handling, code profiler | +| [System Utilities](./reference/architecture/system-utilities/system-utilities.md) | `EnumClassHash`, time conversion utilities | + +--- + +## πŸ—ΊοΈ Architecture Diagrams + +Visual architecture documentation is available as Mermaid diagram files in the diagrams directory: + +```text +docs/diagrams/architecture/ +``` + +Diagrams cover all major modules including: + +- System overview and end-to-end data flow (`README.mmd`) +- Core Init And Runtime boot flow (`core-init-and-runtime.mmd`) +- SQL Core And Virtual Tables execution (`sql-core-and-virtual-tables.mmd`) +- Query Execution And Logging pipeline (`query-execution-and-logging.mmd`) +- Events Core publish/subscribe model (`events-core.mmd`) +- Distributed Querying orchestration (`distributed-querying.mmd`) +- Extensions Framework IPC (`extensions-framework.mmd`) +- Config And Packs lifecycle (`config-and-packs.mmd`) +- Database Backends storage layer (`database-backends.mmd`) +- Remote HTTP Client transport (`remote-http-client.mmd`) +- Hashing module (`hashing.mmd`) +- Filesystem And Fileops (`filesystem-and-fileops.mmd`) +- Process And Profiler (`process-and-profiler.mmd`) +- Plugin Interfaces And Logging (`plugin-interfaces-and-logging.mmd`) +- Config Plugins (`config-plugins.mmd`) +- System Utilities (`system-utilities.mmd`) + +--- + +## πŸ”— Quick Links + +| Resource | Link | +|----------|------| +| [Project README](../README.md) | Main project overview, quick start, and feature summary | +| [Contributing Guide](../CONTRIBUTING.md) | How to contribute β€” code style, branches, commits, PR process | +| [OpenMSP Slack](https://join.slack.com/t/openmsp/shared_invite/zt-36bl7mx0h-3~U2nFH6nqHqoTPXMaHEHA) | Community discussion and support (no GitHub Issues) | +| [openmsp.ai](https://www.openmsp.ai/) | OpenMSP community website | +| [flamingo.run](https://flamingo.run) | Flamingo MSP platform | +| [openframe.ai](https://openframe.ai) | OpenFrame unified MSP platform | + +--- + +*Documentation generated by [OpenFrame Doc Orchestrator](https://github.com/flamingo-stack/openframe-oss-tenant)*