diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 1ca22c5c49a..00000000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,3 +0,0 @@ -# These are supported funding model platforms - -community_bridge: osquery diff --git a/.github/ISSUE_TEMPLATE/Blueprint.md b/.github/ISSUE_TEMPLATE/Blueprint.md deleted file mode 100644 index 8b30625804d..00000000000 --- a/.github/ISSUE_TEMPLATE/Blueprint.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: Blueprint -about: You're planning on implementing a change -title: '' -labels: '' -assignees: '' - ---- - - - -# Blueprint - - diff --git a/.github/ISSUE_TEMPLATE/Bug_Report.md b/.github/ISSUE_TEMPLATE/Bug_Report.md deleted file mode 100644 index a98105b9ac5..00000000000 --- a/.github/ISSUE_TEMPLATE/Bug_Report.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: Bug Report -about: Something is not working as expected -title: '' -labels: '' -assignees: '' - ---- - - - -# Bug report - - - -### What operating system and version are you using? - - - -### What version of osquery are you using? - - - -### What steps did you take to reproduce the issue? - -### What did you expect to see? - -### What did you see instead? diff --git a/.github/ISSUE_TEMPLATE/Feature_Request.md b/.github/ISSUE_TEMPLATE/Feature_Request.md deleted file mode 100644 index 9fffce2797e..00000000000 --- a/.github/ISSUE_TEMPLATE/Feature_Request.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: Feature Request -about: You want the team to implement a new feature -title: '' -labels: '' -assignees: '' - ---- - - - -# Feature request - - - -### What new feature do you want? - - - -### How is this new feature useful? - - - -### How can this be implemented? - - diff --git a/.github/ISSUE_TEMPLATE/Free_Form.md b/.github/ISSUE_TEMPLATE/Free_Form.md deleted file mode 100644 index 7bcf75a0205..00000000000 --- a/.github/ISSUE_TEMPLATE/Free_Form.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: Task -about: A generic task or TODO item -title: '' -labels: '' -assignees: '' - ---- - - - -# Task - - diff --git a/.github/ISSUE_TEMPLATE/New_Release.md b/.github/ISSUE_TEMPLATE/New_Release.md deleted file mode 100644 index b16ece99279..00000000000 --- a/.github/ISSUE_TEMPLATE/New_Release.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: New Release -about: Checklist of Release actions -title: '' -labels: '' -assignees: '' - ---- - - - - - -# New release checklist - -## Before creating a GitHub tag - -- [ ] Review the [milestones](https://github.com/osquery/osquery/milestones), move unfinished issues to the next release, close the milestone. -- [ ] Ask for a Technical Steering Committee member to make a GitHub tag. -- [ ] Trigger [code signing workflow](https://github.com/osquery/osquery-codesign/actions/workflows/release-generator.yml) -- [ ] Assure that testable packages are available on the Releases page, and announce to the `#core` channel on Slack. - -## After creating a GitHub tag - -- [ ] If the `CHANGELOG.md` hasn't already been updated, update it in a new Pull Request, review and merge it. It should reflect everything done up to the tagged version commit. -- [ ] Assure that the ChangeLog shows up in the [tag release notes](https://github.com/osquery/osquery/tags). - -## Promoting to Stable -- [ ] Use [code signing workflow](https://github.com/osquery/osquery-codesign/actions/workflows/release-generator.yml) to build packages and upload to S3 -- [ ] Update the website with the new release and schema. -- [ ] Publish the website changes. -- [ ] Publish the new packages into the hosted repos. -- [ ] Bump or ping the [Homebrew Cask](https://github.com/Homebrew/homebrew-cask/blob/master/Casks/osquery.rb). -- [ ] Advertise the new release. :) diff --git a/.github/ISSUE_TEMPLATE/Question.md b/.github/ISSUE_TEMPLATE/Question.md deleted file mode 100644 index 5547b2c3461..00000000000 --- a/.github/ISSUE_TEMPLATE/Question.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: Question -about: You have a question about osquery -title: '' -labels: '' -assignees: '' - ---- - - - -# Question - -**We are happy to help, however please DO NOT submit issues for questions about osquery.** - -We only use issues to track bugs, feature requests or blueprints. For everything else, check out Slack or the other channels below: - -Slack : https://osquery.slack.com - -Slack newcomers, first use this invite link: https://join.slack.com/t/osquery/shared_invite/zt-1wipcuc04-DBXmo51zYJKBu3_EP3xZPA - -Reddit : https://www.reddit.com/r/osquery/ - -Stack Overflow : https://stackoverflow.com/tags/osquery - - -**If you go ahead and submit an issue with a question it will not be answered.** diff --git a/.github/steps/sign-macos-package/action.yml b/.github/steps/sign-macos-package/action.yml new file mode 100644 index 00000000000..8c8318c22e7 --- /dev/null +++ b/.github/steps/sign-macos-package/action.yml @@ -0,0 +1,105 @@ +name: Sign MacOS package +description: Sign built universal MacOS package and Notarize it +inputs: + apple_certificate_p12: + description: 'Base64 encoded Apple Developer certificate (.p12)' + required: true + apple_certificate_password: + description: 'Password for the .p12 certificate' + required: true + apple_developer_id: + description: 'Apple Developer ID string' + required: true + apple_id_username: + description: 'Apple ID username for notarization' + required: true + apple_id_password: + description: 'App-specific password for Apple ID' + required: true + apple_team_id: + description: 'Apple Developer ID' + required: true + binary_name: + description: 'Name of built MacOS binary (e.g. "rmmagent")' + required: true + +runs: + using: "composite" + steps: + - name: Setup and export all needed vars + shell: bash + run: | + BINARY_NAME="${{ inputs.binary_name }}" + BINARY_PATH="./artifacts/$BINARY_NAME" + DEVELOPER_ID="${{ inputs.apple_developer_id }}" + + echo "BINARY_NAME=$BINARY_NAME" >> $GITHUB_ENV + echo "BINARY_PATH=$BINARY_PATH" >> $GITHUB_ENV + echo "DEVELOPER_ID=$DEVELOPER_ID" >> $GITHUB_ENV + + - name: Setup Apple Code Signing + shell: bash + run: | + CERTIFICATE_PATH="$HOME/certificate.p12" + KEYCHAIN_PATH="$HOME/signing.keychain-db" + + # Decode certificate and create keychain + echo "${{ inputs.apple_certificate_p12 }}" | base64 --decode > "$CERTIFICATE_PATH" + security create-keychain -p "${{ inputs.apple_certificate_password }}" "$KEYCHAIN_PATH" + security default-keychain -s "$KEYCHAIN_PATH" + security unlock-keychain -p "${{ inputs.apple_certificate_password }}" "$KEYCHAIN_PATH" + + # Import certificate and set key partition list + security import "$CERTIFICATE_PATH" -P "${{ inputs.apple_certificate_password }}" -A -t cert -f pkcs12 -k "$KEYCHAIN_PATH" + security set-key-partition-list -S apple-tool:,apple: -s -k "${{ inputs.apple_certificate_password }}" "$KEYCHAIN_PATH" + + # Verify setup, cleanup initial cert and export path + security find-identity -v -p codesigning "$KEYCHAIN_PATH" + rm "$CERTIFICATE_PATH" + echo "KEYCHAIN_PATH=$KEYCHAIN_PATH" >> $GITHUB_ENV + + - name: Code Sign Standalone binary + shell: bash + run: | + chmod +x "$BINARY_PATH" + + # Sign the binary and verify + codesign --sign "$DEVELOPER_ID" --timestamp --options runtime --deep --force "$BINARY_PATH" + + # Verify signature + codesign --verify --deep --strict --verbose=2 "$BINARY_PATH" + codesign -dv --verbose=4 "$BINARY_PATH" + echo "Binary signed successfully" + + - name: Create archive and submit for Notarization + shell: bash + run: | + zip "${BINARY_NAME}.zip" -j "$BINARY_PATH" + + echo "Submitting binary for notarization..." + xcrun notarytool submit "${BINARY_NAME}.zip" \ + --apple-id "${{ inputs.apple_id_username }}" \ + --password "${{ inputs.apple_id_password }}" \ + --team-id "${{ inputs.apple_team_id }}" \ + --wait \ + --timeout 30m + + # Cleanup zip + rm "${BINARY_NAME}.zip" + echo "Notarization completed successfully" + + - name: Verify Final Binary + shell: bash + run: | + # Verify code signature and notarization + codesign --verify --deep --strict --verbose=2 "$BINARY_PATH" + spctl --assess --type open --context context:primary-signature --verbose "$BINARY_PATH" || true + echo "Binary is signed, notarized, and ready for distribution" + + - name: Cleanup Keychain + if: always() + shell: bash + run: | + if [[ -n "$KEYCHAIN_PATH" ]] && [[ -f "$KEYCHAIN_PATH" ]]; then + security delete-keychain "$KEYCHAIN_PATH" + fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..80ab4893a8c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,264 @@ +name: OSQuery Build and Push Latest + +permissions: + contents: write + packages: write + actions: write + pull-requests: write + attestations: write + id-token: write + +on: + push: + branches: [master] + paths-ignore: + - "**/*.md" + - '.github/**' + workflow_dispatch: + inputs: + tag: + description: "Tag to release (e.g., v1.2.3)" + required: true + type: string + +env: + REGISTRY: "ghcr.io" + ORGANISATION: ${{ github.repository_owner }} + REPOSITORY: ${{ github.event.repository.name }} + CMAKE_OSX_DEPLOYMENT_TARGET: "15.0" + CMAKE_BUILD_TYPE: Release + BINARY_NAME: osqueryd + +# ============================================================================= +# JOBS +# ============================================================================= + +jobs: + build_macos: + name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" + runs-on: ${{ matrix.os }} + if: | + github.event_name == 'push' || github.event_name == 'workflow_dispatch' + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - name: "macOS x86-64" + os: macos-13 + os_arch: x86-64 + cmake_arch: x86_64 + artifact_name: osquery-macos-x64 + - name: "macOS ARM64" + os: macos-latest + os_arch: arm64 + cmake_arch: arm64 + artifact_name: osquery-macos-arm64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install Xcode Command Line Tools + shell: bash + run: | + # Install Xcode if not already installed + if ! xcode-select -p >/dev/null 2>&1; then + echo "Installing Xcode Command Line Tools..." + xcode-select --install + # Wait for installation to complete + while ! xcode-select -p >/dev/null 2>&1; do + echo "Waiting for Xcode Command Line Tools installation..." + sleep 10 + done + fi + + - name: Cache Homebrew + uses: actions/cache@v4 + with: + path: | + ~/Library/Caches/Homebrew + /usr/local/Homebrew + /opt/homebrew + key: ${{ runner.os }}-${{ runner.arch }}-homebrew-${{ hashFiles('.github/workflows/*.yml') }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-homebrew- + + - name: Install macOS Dependencies + run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck + shell: bash + + - name: Cache CMake Build for faster rebuilds + uses: actions/cache@v4 + with: + path: | + build/ + ~/.ccache + key: ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} + ${{ runner.os }}-${{ matrix.os_arch }}-cmake- + + - name: Setup Build Environment + run: | + echo "Setting up build environment for ${{ matrix.os_arch }}..." + export SDKROOT=$(xcrun --show-sdk-path) + export CFLAGS="-isysroot $SDKROOT" + export CXXFLAGS="-isysroot $SDKROOT" + + echo "SDKROOT=$SDKROOT" >> $GITHUB_ENV + echo "CFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + echo "CXXFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + + - name: Configure CMake + run: | + echo "Creating build directory and configuring..." + mkdir -p build && cd build + + cmake \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ + -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_OSX_SYSROOT=$SDKROOT \ + -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ + -DOSQUERY_VERSION="5.9.1" \ + .. + + echo "CMake configuration completed" + + - name: Build Project + run: | + echo "Starting build for ${{ matrix.os_arch }}..." + cd build + + CPU_COUNT=$(sysctl -n hw.ncpu) + echo "Building with $CPU_COUNT parallel jobs" + time cmake --build . -j $CPU_COUNT + + echo "Build completed successfully" + + - name: Upload client artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: build/osquery/${{ env.BINARY_NAME }} + if-no-files-found: warn + retention-days: 30 + compression-level: 9 + + create_universal_macos: + name: "Create Universal macOS Binary" + needs: [build_macos] + runs-on: macos-latest + if: | + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && + !failure() && !cancelled() + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Download macOS x86-64 Binary + uses: actions/download-artifact@v4 + with: + name: osquery-macos-x64 + path: x64-artifacts/ + + - name: Download macOS ARM64 Binary + uses: actions/download-artifact@v4 + with: + name: osquery-macos-arm64 + path: arm64-artifacts/ + + - name: Create Universal Binary + run: | + echo "Creating universal macOS binary..." + X64_BINARY="x64-artifacts/${{ env.BINARY_NAME }}" + ARM64_BINARY="arm64-artifacts/${{ env.BINARY_NAME }}" + + echo "Creating universal binary with lipo..." + mkdir -p artifacts + lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/${{ env.BINARY_NAME }} + + # Verify universal binary + echo "Universal binary info:" + lipo -info ./artifacts/${{ env.BINARY_NAME }} + file ./artifacts/${{ env.BINARY_NAME }} + + - name: Sign MacOS package + uses: ./.github/steps/sign-macos-package + with: + apple_certificate_p12: ${{ secrets.APPLE_CERTIFICATE_P12 }} + apple_certificate_password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + apple_developer_id: ${{ secrets.APPLE_DEVELOPER_ID }} + apple_id_username: ${{ secrets.APPLE_ID_USERNAME }} + apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} + apple_team_id: ${{ secrets.APPLE_TEAM_ID }} + binary_name: ${{ env.BINARY_NAME }} + + - name: Upload Universal Binary + uses: actions/upload-artifact@v4 + with: + name: osquery-macos-universal + path: artifacts/ + retention-days: 30 + compression-level: 9 + + build_windows: + name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" + runs-on: ${{ matrix.os }} + if: | + github.event_name == 'push' || github.event_name == 'workflow_dispatch' + defaults: + run: + shell: cmd + strategy: + fail-fast: false + matrix: + include: + - name: "Windows x64" + os: windows-latest + os_arch: x64 + artifact_name: osquery-windows-x64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Setup MSBuild + uses: microsoft/setup-msbuild@v2 + + - name: Setup Visual Studio Build Tools + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.os_arch }} + + - name: Configure CMake for ${{ matrix.os }} + run: | + mkdir build + cd build + cmake -G "Visual Studio 17 2022" -A ${{ matrix.os_arch }} -DOSQUERY_VERSION="5.9.1" .. + + - name: Build OSQuery for Windows ${{ matrix.os_arch }} with MSBuild + run: | + cd build + cmake --build . --config RelWithDebInfo -j10 + + - name: Upload Windows OSQuery artifact + if: runner.os == 'Windows' + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: "build/osquery/RelWithDebInfo/${{ env.BINARY_NAME }}${{ matrix.os == 'windows-latest' && '.exe' || '' }}" + if-no-files-found: warn + retention-days: 30 + compression-level: 9 diff --git a/.github/workflows/cleanup_ec2_runners.yml b/.github/workflows/cleanup_ec2_runners.yml deleted file mode 100644 index 8e7e608cbda..00000000000 --- a/.github/workflows/cleanup_ec2_runners.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright (c) 2014-present, The osquery authors -# -# This source code is licensed as defined by the LICENSE file found in the -# root directory of this source tree. -# -# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) - - -# Cleanup any lingering ec2 runners -name: cleanup_ec2_runners - -on: - schedule: - - cron: "0 */1 * * *" - workflow_dispatch: - -jobs: - cleanup: - runs-on: ubuntu-latest - timeout-minutes: 5 - - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} - - - name: Find Instances - run: | - cutoff=$(date -u --date="-2 hours" "+%Y-%m-%dT%H:%M") - - aws ec2 describe-instances \ - --filters Name=instance-state-name,Values=running Name=instance.group-id,Values=sg-0757a3162c999331b \ - --query "Reservations[].Instances[?LaunchTime<=\`$cutoff\`][].{id: InstanceId, type: InstanceType, launched: LaunchTime}" \ - | tee instances.json - - - name: Terminate Instances - run: | - # This is a for loop, so that any failures don't stop all the instances from terminating - for i in $(jq -r '.[].id' instances.json); do - aws ec2 terminate-instances --instance-ids "$i" - done diff --git a/.github/workflows/cve_scan_runner.yml b/.github/workflows/cve_scan_runner.yml deleted file mode 100644 index d4d401c0d95..00000000000 --- a/.github/workflows/cve_scan_runner.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: cve-scan-runner - -on: - workflow_dispatch: - - schedule: - - cron: "17 23 * * *" - -env: - # To see the script output in real time - PYTHONUNBUFFERED: 1 - -jobs: - scan-and-open-issues: - runs-on: ubuntu-22.04 - timeout-minutes: 20 - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - - - name: Install python pre-requisites - run: | - pip3 install -r ./tools/ci/scripts/cve/requirements.txt - - - name: Scan CVEs and open issues - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NIST_API_KEY: ${{ secrets.NVD_API_KEY }} - - run: | - ./tools/ci/scripts/cve/validate_manifest_libraries_versions.py --manifest libraries/third_party_libraries_manifest.json \ - --repository . - - ./tools/ci/scripts/cve/third_party_libraries_cves_scanner.py --manifest libraries/third_party_libraries_manifest.json \ - --create-issues \ - --debug diff --git a/.github/workflows/hosted_runners.yml b/.github/workflows/hosted_runners.yml deleted file mode 100644 index 2a738239cd6..00000000000 --- a/.github/workflows/hosted_runners.yml +++ /dev/null @@ -1,1393 +0,0 @@ -# -# Copyright (c) 2014-present, The osquery authors -# -# This source code is licensed as defined by the LICENSE file found in the -# root directory of this source tree. -# -# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) -# - -# Due to a limitation in how GitHub Actions works, we can't reference -# jobs in another file inside the `needs` statement. -# -# This configuration file takes care of the Windows, macOS and Linux -# builds on the x86 platform. -name: build_x86 - -on: - # Run this workflow once every day against the master branch - schedule: - - cron: "0 7 * * *" - - push: - branches: - - 'master' - - 'main' - - tags: - - '*' - - pull_request: - branches: - - '*' - -# Cancel old jobs of a PR if a new job is started. -# Fallback on using the run id if it's not a PR, which is unique, so no job canceling. -concurrency: - group: hosted-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true - -# Please remember to update values for both x86 and aarch64 workflows. -env: - PACKAGING_REPO: https://github.com/osquery/osquery-packaging - PACKAGING_COMMIT: c089fb2d3d796d976e3b2fbea7ee69a1616b9576 - SUBMODULE_CACHE_VERSION: 3 - -# If the initial code sanity checks are passing, then one job -# per [`platform` * `build_type`] will start, building osquery -# and generating packages that are later attached to the commit -# (or PR) as build artifacts. -jobs: - # This job performs basic source code check, looking for formatting - # issues and missing copyright headers - check_code_style: - runs-on: ubuntu-22.04 - - container: - image: osquery/builder20.04:7e9ee0339 - options: --user 1001 - - steps: - - name: Setup the build paths - shell: bash - id: build_paths - run: | - rel_build_path="workspace/build" - rel_source_path="workspace/src" - - mkdir -p "${rel_build_path}" - mkdir -p "${rel_source_path}" - - echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - # This script makes sure that the copyright headers have been correctly - # placed on all the source code files - - name: Check the copyright headers - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - ./tools/ci/scripts/check_copyright_headers.py - - - name: Configure the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DOSQUERY_ENABLE_FORMAT_ONLY=ON \ - "${{ steps.build_paths.outputs.SOURCE }}" - - # Formatting is tested against the clang-format binary we ship - # with the osquery-toolchain, so this job is only performed once on - # a Linux machine. - - name: Check code formatting - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: - cmake --build . --target format_check - - - - # This jobs checks that the third party libraries manifest has the correct format - # and that is up to date compared to the current state of the repository - check_libraries_manifest: - runs-on: ubuntu-22.04 - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - - - name: Install python pre-requisites - run: | - pip3 install -r ./tools/ci/scripts/cve/requirements.txt - - - name: Verify the third party libraries manifest - run: | - ./tools/ci/scripts/cve/validate_manifest_libraries_versions.py --manifest libraries/third_party_libraries_manifest.json \ - --repository . - - - # Test generating the website json - check_genwebsitejson: - needs: [check_code_style, check_libraries_manifest] - runs-on: ubuntu-22.04 - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - - - name: genwebsitejson.py - run: python3 tools/codegen/genwebsitejson.py --specs=specs/ - - - # This job runs source code analysis tools (currently, just cppcheck) - check_source_code: - needs: [check_code_style, check_libraries_manifest] - - runs-on: ${{ matrix.os }} - - container: - image: osquery/builder20.04:7e9ee0339 - options: --user 1001 - - strategy: - matrix: - os: [ubuntu-22.04] - - steps: - - name: Setup the build paths - shell: bash - id: build_paths - run: | - rel_build_path="workspace/build" - rel_source_path="workspace/src" - rel_install_path="workspace/install" - - mkdir -p ${rel_build_path} \ - ${rel_source_path} \ - ${rel_install_path} - - echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - echo "REL_BINARY=${rel_build_path}" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules - - key: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the git submodules - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - git submodule sync --recursive - - - name: Configure the project (Release) - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DOSQUERY_BUILD_TESTS=ON \ - -DOSQUERY_BUILD_ROOT_TESTS=ON \ - "${{ steps.build_paths.outputs.SOURCE }}" - - - name: Initialize the project (Release) - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake --build . --target prepare_for_ide - - - name: Run cppcheck (Release) - shell: bash - id: release_cppcheck_runner - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake --build . --target cppcheck 2>&1 | tee cppcheck_release.txt - - - name: Store the cppcheck log (Release) - uses: actions/upload-artifact@v4.4.0 - with: - name: cppcheck-release - path: ${{ steps.build_paths.outputs.REL_BINARY }}/cppcheck_release.txt - - - name: Configure the project (Debug) - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DCMAKE_BUILD_TYPE:STRING=Debug \ - -DOSQUERY_BUILD_TESTS=ON \ - -DOSQUERY_BUILD_ROOT_TESTS=ON \ - "${{ steps.build_paths.outputs.SOURCE }}" - - - name: Initialize the project (Debug) - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake --build . --target prepare_for_ide - - - name: Run cppcheck (Debug) - shell: bash - id: debug_cppcheck_runner - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake --build . --target cppcheck 2>&1 | tee cppcheck_debug.txt - - - name: Store the cppcheck log (Debug) - uses: actions/upload-artifact@v4.4.0 - with: - name: cppcheck-debug - path: ${{ steps.build_paths.outputs.REL_BINARY }}/cppcheck_debug.txt - - - - - # The Linux build will only start once we know that the code - # has been properly formatted - build_linux: - needs: [check_code_style, check_libraries_manifest] - - runs-on: ${{ matrix.os }} - - container: - image: osquery/builder20.04:7e9ee0339 - options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock --pid=host --user 1001 - - strategy: - matrix: - build_type: [Release, RelWithDebInfo, Debug] - os: [ubuntu-22.04] - - steps: - - name: Make space uninstalling packages - shell: bash - run: | - run_on_host="sudo nsenter -t 1 -m -u -n -i" - packages_to_remove=$($run_on_host dpkg-query -f '${Package}\n' -W | grep "^clang-.*\|^llvm-.*\|^php.*\|^mono-.*\|^mongodb-.*\ - \|^libmono-.*\|^temurin-8-jdk\|^temurin-11-jdk\|^temurin-17-jdk\|^dotnet-.*\|^google-chrome-stable\|^microsoft-edge-stable\|^google-cloud-sdk\|^firefox\|^hhvm\|^snapd") - $run_on_host apt purge $packages_to_remove - - # Due to how the RPM packaging tools work, we have to adhere to some - # character count requirements in the build path vs source path. - # - # Failing to do so, will break the debuginfo RPM package. - - name: Setup the build paths - id: build_paths - run: | - rel_build_path="workspace/usr/src/debug/osquery/build" - rel_src_path="workspace/padding-required-by-rpm-packages/src" - rel_ccache_path="workspace/ccache" - rel_package_data_path="workspace/package_data" - rel_packaging_path="workspace/osquery-packaging" - rel_package_build_path="workspace/package-build" - - mkdir -p ${rel_build_path} \ - ${rel_src_path} \ - ${rel_ccache_path} \ - ${rel_src_path} \ - ${rel_package_data_path} \ - ${rel_package_build_path} - - echo "SOURCE=$(realpath ${rel_src_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - echo "CCACHE=$(realpath ${rel_ccache_path})" >> $GITHUB_OUTPUT - echo "PACKAGING=$(realpath ${rel_packaging_path})" >> $GITHUB_OUTPUT - echo "PACKAGE_DATA=$(realpath ${rel_package_data_path})" >> $GITHUB_OUTPUT - echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT - echo "PACKAGE_BUILD=$(realpath ${rel_package_build_path})" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - - name: Select the build job count - shell: bash - id: build_job_count - run: | - echo "VALUE=$(($(nproc) + 1))" >> $GITHUB_OUTPUT - - - name: Select the build options for the tests - shell: bash - id: tests_build_settings - run: | - if [[ "${{ matrix.build_type }}" == "RelWithDebInfo" ]] ; then - echo "VALUE=OFF" >> $GITHUB_OUTPUT - else - echo "VALUE=ON" >> $GITHUB_OUTPUT - fi - - - name: Get runner hardware info - shell: bash - id: runner_stats - run: | - df -h - echo "Cores: $(nproc)" - free -m - - # We don't have enough space on the worker to actually generate all - # the debug symbols (osquery + dependencies), so we have a flag to - # disable them when running a Debug build - - name: Select the debug symbols options - shell: bash - id: debug_symbols_settings - run: | - if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then - echo "VALUE=ON" >> $GITHUB_OUTPUT - else - echo "VALUE=OFF" >> $GITHUB_OUTPUT - fi - - - name: Clone the osquery-packaging repository - run: | - git clone ${{ env.PACKAGING_REPO }} \ - ${{ steps.build_paths.outputs.PACKAGING }} - cd ${{ steps.build_paths.outputs.PACKAGING }} - git checkout ${{ env.PACKAGING_COMMIT }} - - - name: Update the cache (ccache) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.CCACHE }} - - key: | - ccache_${{ matrix.os }}_${{ matrix.build_type }}_${{ github.sha }} - - restore-keys: | - ccache_${{ matrix.os }}_${{ matrix.build_type }} - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules - - key: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the git submodules - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - git submodule sync --recursive - - - name: Configure the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ - -DOSQUERY_BUILD_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ - -DOSQUERY_BUILD_ROOT_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ - "${{ steps.build_paths.outputs.SOURCE }}" - - - name: Build the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Disk space information - shell: bash - id: disk_space_info_post_build - run: | - df -h - du -sh ${{ steps.build_paths.outputs.BINARY }} - - # Only run the tests on Debug and Release configurations; skip RelWithDebInfo - - name: Run the tests as normal user - working-directory: ${{ steps.build_paths.outputs.BINARY }} - if: matrix.build_type != 'RelWithDebInfo' - run: | - # Allow access to the docker socket - sudo chmod 666 /var/run/docker.sock - ctest --build-nocmake -LE "root-required" -V - - - name: Run the tests as root user - working-directory: ${{ steps.build_paths.outputs.BINARY }} - if: matrix.build_type != 'RelWithDebInfo' - run: | - sudo -u root ctest --build-nocmake -L "root-required" -V - - - name: Run the install target - if: matrix.build_type == 'RelWithDebInfo' - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - run: | - cmake \ - --build . \ - --target install \ - -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Create the packages - if: matrix.build_type == 'RelWithDebInfo' - working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} - - shell: bash - - run: | - osquery_version=$(cd ${{ steps.build_paths.outputs.SOURCE }} && git describe --tags --always ) - - tar pcvzf package_data.tar.gz \ - ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - package_format_list=( "DEB" "RPM" "TGZ" ) - - for package_format in "${package_format_list[@]}" ; do - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCPACK_GENERATOR=${package_format} \ - -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ - -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} \ - -DOSQUERY_SOURCE_DIRECTORY_LIST="${{ steps.build_paths.outputs.SOURCE }};${{ steps.build_paths.outputs.BINARY }}" \ - ${{ steps.build_paths.outputs.PACKAGING }} - - cmake --build . \ - --target package - done - - - name: Locate the packages - if: matrix.build_type == 'RelWithDebInfo' - id: packages - shell: bash - run: | - echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.deb)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_DEBUG_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.ddeb)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-?.*.rpm)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_DEBUG_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-debuginfo-*.rpm)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*linux_x86_64.tar.gz)" >> $GITHUB_OUTPUT - - - name: Store the unsigned release package data artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_release_package_data - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} - - - name: Store the unsigned release DEB artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_release_deb - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_DEB_PATH }} - - - name: Store the unsigned debug DEB artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_debug_deb - path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_DEB_PATH }} - - - name: Store the unsigned release RPM artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_release_rpm - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_RPM_PATH }} - - - name: Store the unsigned debug RPM artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_debug_rpm - path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_RPM_PATH }} - - - name: Store the unsigned release TGZ artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_release_tgz - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} - - # Before we terminate this job, delete the build folder. The cache - # actions will require the disk space to create the archives. - - name: Reclaim disk space - run: | - rm -rf ${{ steps.build_paths.outputs.BINARY }} - - - - - # The macOS build will only start once we know that the code - # has been properly formatted - build_macos: - needs: [check_code_style, check_libraries_manifest] - - runs-on: ${{ matrix.os }} - - strategy: - matrix: - build_type: [Release, Debug] - architecture: [x86_64, arm64] - os: [macos-14, macos-15] - - steps: - - name: Select the build job count - shell: bash - id: build_job_count - run: | - echo "VALUE=$(($(sysctl -n hw.logicalcpu) + 1))" >> $GITHUB_OUTPUT - - - name: Setup the build paths - shell: bash - id: build_paths - run: | - rel_build_path="workspace/build" - rel_src_path="workspace/src" - rel_ccache_path="workspace/ccache" - rel_downloads_path="workspace/downloads" - rel_install_path="workspace/install" - rel_package_data_path="workspace/package_data" - rel_packaging_path="workspace/osquery-packaging" - rel_package_build_path="workspace/package-build" - - mkdir -p ${rel_build_path} \ - ${rel_ccache_path} \ - ${rel_downloads_path} \ - ${rel_install_path} \ - ${rel_package_data_path} \ - ${rel_package_build_path} - - echo "SOURCE=$(pwd)/${rel_src_path}" >> $GITHUB_OUTPUT - echo "REL_SOURCE=${rel_src_path}" >> $GITHUB_OUTPUT - echo "BINARY=$(pwd)/${rel_build_path}" >> $GITHUB_OUTPUT - echo "CCACHE=$(pwd)/${rel_ccache_path}" >> $GITHUB_OUTPUT - echo "DOWNLOADS=$(pwd)/${rel_downloads_path}" >> $GITHUB_OUTPUT - echo "INSTALL=$(pwd)/${rel_install_path}" >> $GITHUB_OUTPUT - echo "PACKAGING=$(pwd)/${rel_packaging_path}" >> $GITHUB_OUTPUT - echo "PACKAGE_DATA=$(pwd)/${rel_package_data_path}" >> $GITHUB_OUTPUT - echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT - echo "PACKAGE_BUILD=$(pwd)/${rel_package_build_path}" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - path: ${{ steps.build_paths.outputs.REL_SOURCE }} - - - name: Get runner hardware info - shell: bash - id: runner_stats - run: | - df -h - echo "Cores: $(sysctl -n hw.logicalcpu)" - vm_stat - - - name: Update the cache (ccache) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.CCACHE }} - - key: | - ccache_${{ matrix.os }}_${{ matrix.architecture }}_${{ matrix.build_type }}_${{ github.sha }} - - restore-keys: | - ccache_${{ matrix.os }}_${{ matrix.architecture }}_${{ matrix.build_type }} - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules - - key: | - gitmodules_${{ matrix.os }}_${{ matrix.architecture }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.os }}_${{ matrix.architecture }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the cache (downloads) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.DOWNLOADS }} - - key: | - downloads_${{ matrix.os }}_${{ matrix.architecture }}_${{ github.sha }} - - restore-keys: | - downloads_${{ matrix.os }}_${{ matrix.architecture }} - - - name: Update the git submodules - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - git submodule sync --recursive - - - name: Install build dependencies - run: | - brew install \ - ccache \ - flex \ - bison \ - coreutils \ - gnu-sed - - - name: Setup python - uses: actions/setup-python@v5 - with: - # Pin an exact python version to avoid mismatches between this builder and the test_older_macos runner - python-version: '3.13.1' - architecture: arm64 - - - name: Install tests dependencies - id: install_test_deps - run: | - pip3 install setuptools \ - pexpect==3.3 \ - psutil \ - timeout_decorator \ - six \ - thrift==0.11.0 \ - osquery - - - name: Install CMake - shell: bash - run: | - ${{ steps.build_paths.outputs.REL_SOURCE }}/tools/ci/scripts/macos/install_cmake.sh \ - "${{ steps.build_paths.outputs.DOWNLOADS }}" \ - "${{ steps.build_paths.outputs.INSTALL }}" \ - "3.21.4" - - - name: Select the Xcode version - shell: bash - id: xcode_selector - run: | - if [[ "${{ matrix.os }}" == "macos-14" ]]; then - xcode_path="/Applications/Xcode_15.4.app/Contents/Developer" - elif [[ "${{ matrix.os }}" == "macos-15" ]]; then - xcode_path="/Applications/Xcode_16.app/Contents/Developer" - else - echo "Invalid matrix.os: ${{ matrix.os }}" - exit 1 - fi - - echo "PATH=${path}" >> $GITHUB_OUTPUT - - sudo xcode-select -s "${xcode_path}" - - echo "DEPLOYMENT_TARGET=10.15" >> $GITHUB_OUTPUT - - # We don't have enough space on the worker to actually generate all - # the debug symbols (osquery + dependencies), so we have a flag to - # disable them when running a Debug build - - name: Select the debug symbols options - shell: bash - id: debug_symbols_settings - run: | - if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then - echo "VALUE=ON" >> $GITHUB_OUTPUT - else - echo "VALUE=OFF" >> $GITHUB_OUTPUT - fi - - - name: Configure the project - shell: bash - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake -G "Unix Makefiles" \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_OSX_ARCHITECTURES="${{ matrix.architecture }}" \ - -DCMAKE_OSX_DEPLOYMENT_TARGET="${{ steps.xcode_selector.outputs.DEPLOYMENT_TARGET }}" \ - -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ - -DOSQUERY_BUILD_TESTS=ON \ - -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ - ${{ steps.build_paths.outputs.SOURCE }} - - - name: Build the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Disk space information - shell: bash - id: disk_space_info_post_build - run: | - df -h - du -sh ${{ steps.build_paths.outputs.BINARY }} - - - name: Run the tests - if: matrix.architecture == 'arm64' - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - ctest --build-nocmake -V - - - name: Run the install target - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - run: | - cmake \ - --build . \ - --target install \ - -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Create the package data - working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} - run: | - tar pcvzf package_data.tar.gz \ - -C ${{ steps.build_paths.outputs.PACKAGE_DATA }} \ - . - - - name: Locate the package data - if: matrix.build_type == 'Release' - id: packages - shell: bash - run: | - echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz)" >> $GITHUB_OUTPUT - - - name: Store the ${{ matrix.architecture }} unsigned release package data artifact - if: matrix.build_type == 'Release' && matrix.os == 'macos-14' - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_unsigned_release_package_data_${{ matrix.architecture }} - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} - - - name: Package the tests for older x86_64 macOS workers - if: matrix.architecture == 'x86_64' && matrix.os == 'macos-14' - working-directory: ${{ github.workspace }}/workspace - run: | - mkdir macos_tests_${{ matrix.build_type }} - ${{ steps.build_paths.outputs.SOURCE }}/tools/ci/scripts/macos/package_tests.sh build macos_tests_${{ matrix.build_type }} ${{ matrix.architecture }} - - - name: Store the packaged tests for older x86_64 macOS workers - if: matrix.architecture == 'x86_64' && matrix.os == 'macos-14' - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_tests_${{ matrix.build_type }} - path: workspace/macos_tests_${{ matrix.build_type }}.tar.gz - - # Before we terminate this job, delete the build folder. The cache - # actions will require the disk space to create the archives. - - name: Reclaim disk space - run: | - rm -rf ${{ steps.build_paths.outputs.BINARY }} - - - - - # This job takes the packaged tests (Release + Debug) from the latest macos - # builder and runs them on the previous available macOS versions - test_older_macos: - - needs: build_macos - - runs-on: ${{ matrix.os }} - - strategy: - matrix: - python_architecture: [x64] - os: [macos-13] - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/download-artifact@v4.1.7 - with: - name: macos_tests_Release - - - uses: actions/download-artifact@v4.1.7 - with: - name: macos_tests_Debug - - - name: Setup python - uses: actions/setup-python@v5 - with: - # Pin an exact python version to avoid mismatches between the builder and this test_older_macos runner - python-version: '3.13.1' - architecture: ${{ matrix.python_architecture }} - - - name: Install tests dependencies - id: install_test_deps - run: | - pip3 install setuptools \ - pexpect==3.3 \ - psutil \ - timeout_decorator \ - six \ - thrift==0.11.0 \ - osquery - - - name: Install CMake - shell: bash - run: | - mkdir -p "workspace/downloads" \ - "workspace/install" - - ./tools/ci/scripts/macos/install_cmake.sh \ - "workspace/downloads" \ - "workspace/install" \ - "3.21.4" - - - name: Extract the tests - run: | - tar xzf macos_tests_Release.tar.gz - tar xzf macos_tests_Debug.tar.gz - - - name: Run the Debug tests - run: | - ( cd macos_tests_Debug && ./run.sh ) - - - name: Run the Release tests - run: | - ( cd macos_tests_Release && ./run.sh ) - - - - - # This job builds the universal macOS artifacts - build_universal_macos_artifacts: - needs: test_older_macos - - runs-on: macos-14 - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install CMake - shell: bash - run: | - mkdir -p "workspace/downloads" \ - "workspace/install" - - ./tools/ci/scripts/macos/install_cmake.sh \ - "workspace/downloads" \ - "workspace/install" \ - "3.21.4" - - - uses: actions/download-artifact@v4.1.7 - with: - name: macos_unsigned_release_package_data_x86_64 - path: macos_unsigned_release_package_data_x86_64 - - - uses: actions/download-artifact@v4.1.7 - with: - name: macos_unsigned_release_package_data_arm64 - path: macos_unsigned_release_package_data_arm64 - - - name: Create the universal package data - run: | - tools/ci/scripts/macos/build_universal_package_data.sh - - - name: Store the universal unsigned release package data artifact - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_unsigned_release_package_data_universal - path: package_data.tar.gz - - - name: Clone the osquery-packaging repository - run: | - git clone ${{ env.PACKAGING_REPO }} osquery-packaging - - cd osquery-packaging - git checkout ${{ env.PACKAGING_COMMIT }} - - - name: Create the packages - shell: bash - run: | - osquery_version=$(git describe --tags --always ) - package_format_list=( "productbuild" "TGZ" ) - - for package_format in "${package_format_list[@]}" ; do - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCPACK_GENERATOR=${package_format} \ - -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ - -DOSQUERY_DATA_PATH=$(pwd)/universal \ - -S osquery-packaging \ - -B package_build - - cmake --build package_build \ - --target package - done - - - name: Locate the packages - id: packages - shell: bash - run: | - echo "REL_UNSIGNED_RELEASE_PKG_PATH=$(ls package_build/*.pkg)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls package_build/*.tar.gz)" >> $GITHUB_OUTPUT - - - name: Store the PKG unsigned release packages - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_unsigned_pkg_universal - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PKG_PATH }} - - - name: Store the TGZ unsigned release packages - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_unsigned_tgz_universal - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} - - - - - # The Windows build will only start once we know that the code - # has been properly formatted - build_windows: - needs: [check_code_style, check_libraries_manifest] - - runs-on: ${{ matrix.os }} - - strategy: - matrix: - build_type: [Release] - bitness: [64, arm64] - os: [windows-2022] - - steps: - - name: Select the build job count - shell: powershell - id: build_job_count - run: | - $cores=(Get-CIMInstance Win32_ComputerSystem).NumberOfLogicalProcessors + 1 - echo "VALUE=$cores" >> $env:GITHUB_OUTPUT - - - name: Setup the build paths - shell: powershell - id: build_paths - run: | - $rel_src_path = "w\src" - $rel_build_path = "w\build" - $rel_sccache_path = "w\sccache" - $rel_downloads_path = "w\downloads" - $rel_install_path = "w\install" - $rel_package_data_path = "w\package_data" - $rel_packaging_path = "w\osquery-packaging" - $rel_packaging_build = "w\package-build" - - New-Item -ItemType Directory -Force -Path $rel_build_path - New-Item -ItemType Directory -Force -Path $rel_sccache_path - New-Item -ItemType Directory -Force -Path $rel_downloads_path - New-Item -ItemType Directory -Force -Path $rel_install_path - New-Item -ItemType Directory -Force -Path $rel_package_data_path - New-Item -ItemType Directory -Force -Path $rel_packaging_build - - $base_dir = (Get-Item .).FullName - - echo "SOURCE=$base_dir\$rel_src_path" >> $env:GITHUB_OUTPUT - echo "REL_SOURCE=$rel_src_path" >> $env:GITHUB_OUTPUT - echo "BINARY=$base_dir\$rel_build_path" >> $env:GITHUB_OUTPUT - echo "SCCACHE=$base_dir\$rel_sccache_path" >> $env:GITHUB_OUTPUT - echo "DOWNLOADS=$base_dir\$rel_downloads_path" >> $env:GITHUB_OUTPUT - echo "INSTALL=$base_dir\$rel_install_path" >> $env:GITHUB_OUTPUT - echo "PACKAGING=$base_dir\$rel_packaging_path" >> $env:GITHUB_OUTPUT - echo "PACKAGE_DATA=$base_dir\$rel_package_data_path" >> $env:GITHUB_OUTPUT - echo "PACKAGE_BUILD=$base_dir\$rel_packaging_build" >> $env:GITHUB_OUTPUT - - - name: Setup the VC arch - shell: powershell - id: vc_arch - run: | - $arch = switch ("${{ matrix.bitness }}") - { - "64" { "x64" } - "arm64" { "x64_arm64" } - } - echo "VC_ARCH=$arch" >> $env:GITHUB_OUTPUT - - # Symbolic links are supported by default on Linux and macOS. On - # Windows, we have to enable them explicitly. They are used to - # handle the include header namespace support that came with - # the initial Buck build system refactor - - name: Configure git - run: | - git config --global core.autocrlf false - git config --global core.symlinks true - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - path: ${{ steps.build_paths.outputs.REL_SOURCE }} - - - name: Get runner hardware info - id: runner_stats - shell: powershell - run: | - Get-CIMInstance Win32_LogicalDisk - Get-CIMInstance Win32_ComputerSystem | Select-Object -Property NumberOfLogicalProcessors - Get-CIMInstance Win32_ComputerSystem | Select-Object -Property TotalPhysicalMemory - - - name: Detect the osquery version - shell: powershell - id: osquery_version - run: | - cd ${{ steps.build_paths.outputs.SOURCE }} - $osquery_version=$(git describe --tags --abbrev=0) - - echo "VALUE=$osquery_version" >> $env:GITHUB_OUTPUT - - - name: Clone the osquery-packaging repository - run: | - cd w - git clone ${{ env.PACKAGING_REPO }} - cd ${{ steps.build_paths.outputs.PACKAGING }} - git checkout ${{ env.PACKAGING_COMMIT }} - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}\.git\modules - - key: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the cache (downloads) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.DOWNLOADS }} - - key: | - downloads_${{ matrix.os }}_${{ github.sha }} - - restore-keys: | - downloads_${{ matrix.os }} - - - name: Initialize the Python 3 installation - uses: actions/setup-python@v5 - with: - python-version: "3.13.x" - architecture: x64 - - # The runners will likely have both the x86 and x64 versions of - # Python but we always need the 64-bit one regardless of which - # architecture we are building for. - # - # The setup-python action should have put the right Python version - # in the PATH variable for us, so locate the installation directory - # so we can use it as a hint when we configure the project with - # CMake - - name: Locate the Python root directory - id: python_root_directory - shell: powershell - run: | - $python_executable_path = $(Get-Command python.exe | Select-Object -ExpandProperty Definition) - $python_root_directory = (Get-Item $python_executable_path).Directory.FullName - - echo "VALUE=$python_root_directory" >> $env:GITHUB_OUTPUT - - # Install the Python dependencies needed for our testing framework - - name: Install tests prerequisites - run: | - python -m pip install --upgrade pip - - python -m pip install wheel ` - setuptools ` - psutil ` - timeout_decorator ` - thrift==0.11.0 ` - osquery ` - pywin32 - - - name: Install Strawberry Perl - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - shell: powershell - run: | - tools\ci\scripts\install_openssl_formula_dependencies.ps1 - - # TODO: Implement a cache so we don't keep hitting the server at each run - - name: Install CMake - working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} - shell: powershell - run: | - $long_cmake_ver = "3.21.4" - $short_cmake_ver = $($long_cmake_ver.split(".")[0] + "." + $long_cmake_ver.split(".")[1]) - - $folder_name = $("cmake-" + $long_cmake_ver + "-windows-x86_64") - $archive_name = $($folder_name + ".zip") - - $url = $("https://cmake.org/files/v" + $short_cmake_ver + "/" + $archive_name) - - (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) - 7z x -o${{ steps.build_paths.outputs.INSTALL }} -y $archive_name - - echo "${{ steps.build_paths.outputs.INSTALL }}\$folder_name\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - # TODO: Implement a cache so we don't keep hitting the server at each run - - name: Install sccache - working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} - shell: powershell - run: | - $long_version = "0.0.1" - - $archive_name = $("sccache-" + $long_version + "-windows.7z") - $url = $("https://github.com/osquery/sccache/releases/download/" + $long_version + "-osquery/" + $archive_name) - - (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) - 7z x -o${{ steps.build_paths.outputs.INSTALL }}\sccache -y $archive_name - - echo "${{ steps.build_paths.outputs.INSTALL }}\sccache" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - # TODO: Implement a cache so we don't keep hitting the server at each run - - name: Install Ninja - working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} - shell: powershell - run: | - $long_version = "1.10.0" - - $archive_name = "ninja-win.zip" - $url = $("https://github.com/ninja-build/ninja/releases/download/v" + $long_version + "/" + $archive_name) - - (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) - 7z x -o${{ steps.build_paths.outputs.INSTALL }}\ninja -y $archive_name - - echo "${{ steps.build_paths.outputs.INSTALL }}\ninja" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Configure the project - shell: cmd - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - SCCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} - SCCACHE_CACHE_SIZE: "5G" - - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} - @echo on - - set cross_compilation= - if ${{ matrix.bitness }}==arm64 ( - echo set (CMAKE_SYSTEM_NAME Windows^) > cross.cmake - echo set (CMAKE_SYSTEM_PROCESSOR ARM64^) >> cross.cmake - set cross_compilation=-DCMAKE_TOOLCHAIN_FILE=cross.cmake - ) - - cmake -G Ninja ^ - -DCMAKE_C_COMPILER=cl.exe ^ - -DCMAKE_CXX_COMPILER=cl.exe ^ - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ - -DOSQUERY_BUILD_TESTS=ON ^ - -DCMAKE_C_COMPILER_LAUNCHER="sccache.exe" ^ - -DCMAKE_CXX_COMPILER_LAUNCHER="sccache.exe" ^ - -DPython3_ROOT_DIR=${{ steps.python_root_directory.outputs.VALUE }} ^ - ${{ steps.build_paths.outputs.SOURCE }} ^ - %cross_compilation% - - - name: Determine compiler version - id: determine_compiler_version - shell: pwsh - run: | - $compiler = (Get-Content "${{ steps.build_paths.outputs.BINARY }}\CMakeCache.txt" | Select-String -Pattern "CMAKE_CXX_COMPILER:STRING=(.*)").Matches[0].Groups[1].Value - - echo "Compiler configured by CMake is $compiler" - - if ($compiler -eq $null || $compiler -eq "") { - Write-Error "Could not find the configured compiler" -ErrorAction Stop - } - - <# - We run the compiler help option; the compiler will write its version in stderr. - Due to how powershell works, we have to go through some hoops to extract the stderr to a variable - and also avoid it considering the command as failed because stderr contains messages. - The expression runs the compiler in a subshell, discards its stdout, then the stderr of the subshell is redirected - to the stdout of the parent shell. - #> - $ErrorActionPreference = 'Continue' - $erroutput = $( & "$compiler" /? 1>$null ) 2>&1 - $ErrorActionPreference = 'Stop' - - if ($erroutput -eq $null || $erroutput -eq "") { - Write-Error "Failed to run the compiler at $compiler" -ErrorAction Stop - } - - $version = ($erroutput | Select-String -Pattern "Compiler Version (.*) for").Matches[0].Groups[1].Value.Replace(".", "") - - if ($version -eq $null || $version -eq "") { - Write-Error "Failed to determine compiler version for $compiler and output $erroutput" -ErrorAction Stop - } - - echo "Found compiler version $version" - - echo "COMPILER_VERSION=$version" >> $env:GITHUB_OUTPUT - - - name: Update the cache (sccache) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SCCACHE }} - - key: | - sccache_${{ matrix.os }}_${{ matrix.bitness }}_${{ matrix.build_type }}_${{ steps.determine_compiler_version.outputs.COMPILER_VERSION }}_${{ github.sha }} - - restore-keys: | - sccache_${{ matrix.os }}_${{ matrix.bitness }}_${{ matrix.build_type }}_${{ steps.determine_compiler_version.outputs.COMPILER_VERSION }} - - - name: Build the project - shell: cmd - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - SCCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} - SCCACHE_CACHE_SIZE: "5G" - - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} - @echo on - - cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} - - if %errorlevel% neq 0 exit /b %errorlevel% - sccache.exe --stop-server - - - name: Disk space information - id: disk_space_info_post_build - shell: powershell - run: | - Get-CIMInstance Win32_LogicalDisk - $bindir_size = (Get-ChildItem -Path ${{ steps.build_paths.outputs.BINARY }} -Recurse | Measure-Object -Property Length -Sum).sum / 1MB - echo "Binary dir size: $bindir_size MBs" - - - name: Run the tests - # arm64 tests can't be run on x64 machine - if: matrix.bitness != 'arm64' - working-directory: ${{ steps.build_paths.outputs.BINARY }} - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} - @echo on - ctest --build-nocmake -C Release -V - - - name: Run the install target - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} - DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - run: | - cmake --build . --target install -j 3 - - - name: Create the packages - working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} - shell: cmd - - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} - @echo on - - 7z ^ - a windows_package_data.zip ^ - ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - if %errorlevel% neq 0 exit /b %errorlevel% - - if ${{ matrix.bitness }}==arm64 ( - echo Do not create other packages for arm64 - exit 0 - ) - - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ - -DCPACK_GENERATOR=WIX ^ - -DOSQUERY_PACKAGE_VERSION=${{ steps.osquery_version.outputs.VALUE }} ^ - -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} ^ - -DOSQUERY_BITNESS=${{ matrix.bitness }} ^ - ${{ steps.build_paths.outputs.PACKAGING }} - - if %errorlevel% neq 0 exit /b %errorlevel% - - cmake --build . ^ - --config Release ^ - --target package - - if %errorlevel% neq 0 exit /b %errorlevel% - - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ - -DCPACK_GENERATOR=NuGet ^ - -DOSQUERY_PACKAGE_VERSION=${{ steps.osquery_version.outputs.VALUE }} ^ - -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} ^ - -DOSQUERY_BITNESS=${{ matrix.bitness }} ^ - ${{ steps.build_paths.outputs.PACKAGING }} - - if %errorlevel% neq 0 exit /b %errorlevel% - - cmake --build . ^ - --config Release ^ - --target package - - if %errorlevel% neq 0 exit /b %errorlevel% - - - name: Locate the packages - id: packages - shell: powershell - run: | - echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=$((ls ${{ steps.build_paths.outputs.PACKAGE_BUILD }}\*.zip).FullName)" >> $env:GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_MSI_PATH=$((ls ${{ steps.build_paths.outputs.PACKAGE_BUILD }}\*.msi).FullName)" >> $env:GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_NUPKG_PATH=$((ls ${{ steps.build_paths.outputs.PACKAGE_BUILD }}\*.nupkg).FullName)" >> $env:GITHUB_OUTPUT - - - name: Store the unsigned release package data artifact - uses: actions/upload-artifact@v4.4.0 - with: - name: windows${{ matrix.bitness }}_unsigned_release_package_data - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} - - - name: Store the unsigned release MSI artifact - if: matrix.bitness != 'arm64' - uses: actions/upload-artifact@v4.4.0 - with: - name: windows${{ matrix.bitness }}_unsigned_release_msi - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_MSI_PATH }} - - - name: Store the unsigned release NUPKG artifact - if: matrix.bitness != 'arm64' - uses: actions/upload-artifact@v4.4.0 - with: - name: windows${{ matrix.bitness }}_unsigned_release_nupkg - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_NUPKG_PATH }} - - # Before we terminate this job, delete the build folder. The cache - # actions will require the disk space to create the archives. - - name: Reclaim disk space - shell: powershell - run: | - rm -r -Force ${{ steps.build_paths.outputs.BINARY }} - - # This job is here as a github status check -- it allows us to move the dependency from - # being on all the jobs to this single one. - mergeable: - runs-on: ubuntu-latest - steps: - - run: true - needs: - - build_macos - - build_linux - - build_windows - - build_universal_macos_artifacts - - check_source_code diff --git a/.github/workflows/self_hosted_runners.yml b/.github/workflows/self_hosted_runners.yml deleted file mode 100644 index 7b327b2eaf5..00000000000 --- a/.github/workflows/self_hosted_runners.yml +++ /dev/null @@ -1,456 +0,0 @@ -# -# Copyright (c) 2014-present, The osquery authors -# -# This source code is licensed as defined by the LICENSE file found in the -# root directory of this source tree. -# -# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) -# - -# Due to a limitation in how GitHub Actions works, we can't reference -# jobs in another file inside the `needs` statement. -# -# This configuration file takes care of the AArch64 Linux build -name: build_aarch64 - -on: - # Run this workflow once every day against the master branch - schedule: - - cron: "0 7 * * *" - - # Do not run this on pull requests - push: - branches: - - '**' - - tags: - - '**' - -# Please remember to update values for both x86 and aarch64 workflows. -env: - PACKAGING_REPO: https://github.com/osquery/osquery-packaging - PACKAGING_COMMIT: c089fb2d3d796d976e3b2fbea7ee69a1616b9576 - SUBMODULE_CACHE_VERSION: 3 - -# If the initial code sanity checks are passing, then one job -# per [`platform` * `build_type`] will start, building osquery -# and generating packages that are later attached to the commit -# (or PR) as build artifacts. -jobs: - # This job performs basic source code check, looking for formatting - # issues and missing copyright headers - check_code_style: - runs-on: ubuntu-22.04 - - container: - image: osquery/builder20.04:7e9ee0339 - options: --user 1001 - - steps: - - name: Setup the build paths - shell: bash - id: build_paths - run: | - rel_build_path="workspace/build" - rel_source_path="workspace/src" - - mkdir -p "${rel_build_path}" - mkdir -p "${rel_source_path}" - - echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - # This script makes sure that the copyright headers have been correctly - # placed on all the source code files - - name: Check the copyright headers - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - ./tools/ci/scripts/check_copyright_headers.py - - - name: Configure the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DOSQUERY_ENABLE_FORMAT_ONLY=ON \ - "${{ steps.build_paths.outputs.SOURCE }}" - - # Formatting is tested against the clang-format binary we ship - # with the osquery-toolchain, so this job is only performed once on - # a Linux machine. - - name: Check code formatting - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: - cmake --build . --target format_check - - - - - # NOTE: Here we manually create two different steps and outputs because there's no easy way - # to dynamically access variables per matrix job. Having outputs with different names - # lets us easily select the correct one later. - - # Starts self-hosted runner. - start_ec2_runners: - needs: check_code_style - runs-on: ubuntu-latest - - outputs: - label-Release: ${{ steps.start_ec2_runner-Release.outputs.label }} - ec2-instance-id-Release: ${{ steps.start_ec2_runner-Release.outputs.ec2-instance-id }} - label-RelWithDebInfo: ${{ steps.start_ec2_runner-RelWithDebInfo.outputs.label }} - ec2-instance-id-RelWithDebInfo: ${{ steps.start_ec2_runner-RelWithDebInfo.outputs.ec2-instance-id }} - - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} - - - name: Start aarch64 EC2 runner for a Release build - id: start_ec2_runner-Release - uses: osquery/ec2-github-runner@v2.3.6 - with: - mode: start - github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} - ec2-image-id: ami-063d99d9ce1bebbe8 - ec2-instance-type: c6g.4xlarge - subnet-id: subnet-06390365e72579736 - security-group-id: sg-0757a3162c999331b - - - name: Start aarch64 EC2 runner for a RelWithDebInfo build - id: start_ec2_runner-RelWithDebInfo - uses: osquery/ec2-github-runner@v2.3.6 - with: - mode: start - github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} - ec2-image-id: ami-063d99d9ce1bebbe8 - ec2-instance-type: c6g.4xlarge - subnet-id: subnet-06390365e72579736 - security-group-id: sg-0757a3162c999331b - - - - - # Stops self-hosted runners. - stop_ec2_runners: - needs: - - start_ec2_runners - - build_linux - - if: always() - - runs-on: ubuntu-latest - - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} - - - name: Stop aarch64 EC2 runner for Release mode build - uses: osquery/ec2-github-runner@v2.3.6 - with: - mode: stop - github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start_ec2_runners.outputs.label-Release }} - ec2-instance-id: ${{ needs.start_ec2_runners.outputs.ec2-instance-id-Release }} - - - name: Stop aarch64 EC2 runner for RelWithDebInfo mode build - uses: osquery/ec2-github-runner@v2.3.6 - with: - mode: stop - github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start_ec2_runners.outputs.label-RelWithDebInfo }} - ec2-instance-id: ${{ needs.start_ec2_runners.outputs.ec2-instance-id-RelWithDebInfo }} - - - - - # The Linux build will only start once we know that the code - # has been properly formatted - build_linux: - needs: start_ec2_runners - - runs-on: ${{ matrix.os }} - - container: - image: osquery/builder20.04:7e9ee0339 - options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock --pid=host --user root - - strategy: - matrix: - include: - - os: ${{ needs.start_ec2_runners.outputs.label-Release }} - build_type: Release - cache_key: ubuntu-20.04_aarch64 - - - os: ${{ needs.start_ec2_runners.outputs.label-RelWithDebInfo }} - build_type: RelWithDebInfo - cache_key: ubuntu-20.04_aarch64 - - steps: - # Due to how the RPM packaging tools work, we have to adhere to some - # character count requirements in the build path vs source path. - # - # Failing to do so, will break the debuginfo RPM package. - - name: Setup the build paths - id: build_paths - run: | - # The group for the docker socket doesn't have gid docker - sudo chown root:docker /var/run/docker.sock - - rel_build_path="workspace/usr/src/debug/osquery/build" - rel_src_path="workspace/padding-required-by-rpm-packages/src" - rel_ccache_path="workspace/ccache" - rel_package_data_path="workspace/package_data" - rel_packaging_path="workspace/osquery-packaging" - rel_package_build_path="workspace/package-build" - - mkdir -p ${rel_build_path} \ - ${rel_src_path} \ - ${rel_ccache_path} \ - ${rel_src_path} \ - ${rel_package_data_path} \ - ${rel_package_build_path} - - echo "SOURCE=$(realpath ${rel_src_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - echo "CCACHE=$(realpath ${rel_ccache_path})" >> $GITHUB_OUTPUT - echo "PACKAGING=$(realpath ${rel_packaging_path})" >> $GITHUB_OUTPUT - echo "PACKAGE_DATA=$(realpath ${rel_package_data_path})" >> $GITHUB_OUTPUT - echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT - echo "PACKAGE_BUILD=$(realpath ${rel_package_build_path})" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - - name: Select the build job count - shell: bash - id: build_job_count - run: | - echo "VALUE=$(($(nproc) + 1))" >> $GITHUB_OUTPUT - - - name: Select the build options for the tests - shell: bash - id: tests_build_settings - run: | - if [[ "${{ matrix.build_type }}" == "RelWithDebInfo" ]] ; then - echo "VALUE=OFF" >> $GITHUB_OUTPUT - else - echo "VALUE=ON" >> $GITHUB_OUTPUT - fi - - # try to gather some info about build space - - name: Disk Space Information - shell: bash - id: disk_space_info - run: | - df -h - - # We don't have enough space on the worker to actually generate all - # the debug symbols (osquery + dependencies), so we have a flag to - # disable them when running a Debug build - - name: Select the debug symbols options - shell: bash - id: debug_symbols_settings - run: | - if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then - echo "VALUE=ON" >> $GITHUB_OUTPUT - else - echo "VALUE=OFF" >> $GITHUB_OUTPUT - fi - - - name: Clone the osquery-packaging repository - run: | - git clone ${{ env.PACKAGING_REPO }} \ - ${{ steps.build_paths.outputs.PACKAGING }} - cd ${{ steps.build_paths.outputs.PACKAGING }} - git checkout ${{ env.PACKAGING_COMMIT }} - - - name: Update the cache (ccache) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.CCACHE }} - - key: | - ccache_${{ matrix.cache_key }}_${{ matrix.build_type }}_${{ github.sha }} - - restore-keys: | - ccache_${{ matrix.cache_key }}_${{ matrix.build_type }} - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules - - key: | - gitmodules_${{ matrix.cache_key }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.cache_key }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the git submodules - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - git submodule sync --recursive - - - name: Configure the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ - -DOSQUERY_BUILD_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ - -DOSQUERY_BUILD_ROOT_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ - "${{ steps.build_paths.outputs.SOURCE }}" - - - name: Build the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Disk Space Information - shell: bash - id: disk_space_info_post_build - run: | - df -h - du -sh ${{ steps.build_paths.outputs.BINARY }} - - - name: Run the tests as normal user - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - sudo chown -R runner:runner . - sudo -u runner ctest --build-nocmake -LE "root-required" -V - - - name: Run the tests as root user - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - sudo -u root ctest --build-nocmake -L "root-required" -V - - - name: Run the install target - if: matrix.build_type == 'RelWithDebInfo' - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - run: | - cmake \ - --build . \ - --target install \ - -j ${{ steps.build_job_count.outputs.VALUE }} - - # Do some targeted cleanups to get additional free space before installing dependencies, - # we are unfortunately out of space otherwise. - find . -name "*.a" -exec rm {} \; - find . -name "*.o" -exec rm {} \; - - - name: Create the packages - if: matrix.build_type == 'RelWithDebInfo' - working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} - - shell: bash - - run: | - osquery_version=$(cd ${{ steps.build_paths.outputs.SOURCE }} && git describe --tags --always ) - - tar pcvzf package_data.tar.gz \ - ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - package_format_list=( "DEB" "RPM" "TGZ" ) - - for package_format in "${package_format_list[@]}" ; do - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCPACK_GENERATOR=${package_format} \ - -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ - -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} \ - -DOSQUERY_SOURCE_DIRECTORY_LIST="${{ steps.build_paths.outputs.SOURCE }};${{ steps.build_paths.outputs.BINARY }}" \ - ${{ steps.build_paths.outputs.PACKAGING }} - - cmake --build . \ - --target package - done - - - name: Locate the packages - if: matrix.build_type == 'RelWithDebInfo' - id: packages - shell: bash - run: | - echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.deb)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_DEBUG_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.ddeb)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-?.*.rpm)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_DEBUG_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-debuginfo-*.rpm)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*linux_aarch64.tar.gz)" >> $GITHUB_OUTPUT - - - name: Store the unsigned release package data artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_release_package_data_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} - - - name: Store the unsigned release DEB artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_release_deb_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_DEB_PATH }} - - - name: Store the unsigned debug DEB artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_debug_deb_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_DEB_PATH }} - - - name: Store the unsigned release RPM artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_release_rpm_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_RPM_PATH }} - - - name: Store the unsigned debug RPM artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_debug_rpm_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_RPM_PATH }} - - - name: Store the unsigned release TGZ artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_release_tgz_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} - - # Before we terminate this job, delete the build folder. The cache - # actions will require the disk space to create the archives. - - name: Reclaim disk space - run: | - rm -rf ${{ steps.build_paths.outputs.BINARY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..bcff0fac4d8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,179 @@ +name: OSQuery Integration Tests + +permissions: + contents: write + packages: write + actions: write + pull-requests: write + attestations: write + id-token: write + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: [master] + paths-ignore: + - "**/*.md" + +concurrency: + group: pr-${{ github.head_ref || github.ref }} + cancel-in-progress: false + +env: + REGISTRY: "ghcr.io" + ORGANISATION: ${{ github.repository_owner }} + REPOSITORY: ${{ github.event.repository.name }} + CMAKE_OSX_DEPLOYMENT_TARGET: "15.0" + CMAKE_BUILD_TYPE: Release + BINARY_NAME: osqueryd + +# ============================================================================= +# JOBS +# ============================================================================= + +jobs: + build_macos: + name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" + runs-on: ${{ matrix.os }} + if: github.event_name == 'pull_request' && !github.event.pull_request.draft + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - name: "macOS x86-64" + os: macos-13 + os_arch: x86-64 + cmake_arch: x86_64 + artifact_name: osquery-macos-x64 + - name: "macOS ARM64" + os: macos-latest + os_arch: arm64 + cmake_arch: arm64 + artifact_name: osquery-macos-arm64 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install Xcode Command Line Tools + shell: bash + run: | + # Install Xcode if not already installed + if ! xcode-select -p >/dev/null 2>&1; then + echo "Installing Xcode Command Line Tools..." + xcode-select --install + # Wait for installation to complete + while ! xcode-select -p >/dev/null 2>&1; do + echo "Waiting for Xcode Command Line Tools installation..." + sleep 10 + done + fi + + - name: Cache Homebrew + uses: actions/cache@v4 + with: + path: | + ~/Library/Caches/Homebrew + /usr/local/Homebrew + /opt/homebrew + key: ${{ runner.os }}-${{ runner.arch }}-homebrew-${{ hashFiles('.github/workflows/*.yml') }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-homebrew- + + - name: Install macOS Dependencies + run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck + shell: bash + + - name: Cache CMake Build for faster rebuilds + uses: actions/cache@v4 + with: + path: | + build/ + ~/.ccache + key: ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} + ${{ runner.os }}-${{ matrix.os_arch }}-cmake- + + - name: Setup Build Environment + run: | + echo "Setting up build environment for ${{ matrix.os_arch }}..." + export SDKROOT=$(xcrun --show-sdk-path) + export CFLAGS="-isysroot $SDKROOT" + export CXXFLAGS="-isysroot $SDKROOT" + + echo "SDKROOT=$SDKROOT" >> $GITHUB_ENV + echo "CFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + echo "CXXFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + + - name: Configure CMake + run: | + echo "Creating build directory and configuring..." + mkdir -p build && cd build + + cmake \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ + -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_OSX_SYSROOT=$SDKROOT \ + -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ + -DOSQUERY_VERSION="5.9.1" \ + .. + + echo "CMake configuration completed" + + - name: Build Project + run: | + echo "Starting build for ${{ matrix.os_arch }}..." + cd build + + CPU_COUNT=$(sysctl -n hw.ncpu) + echo "Building with $CPU_COUNT parallel jobs" + time cmake --build . -j $CPU_COUNT + + echo "Build completed successfully" + + build_windows: + name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" + runs-on: ${{ matrix.os }} + if: github.event_name == 'pull_request' && !github.event.pull_request.draft + defaults: + run: + shell: cmd + strategy: + fail-fast: false + matrix: + include: + - name: "Windows x64" + os: windows-latest + os_arch: x64 + artifact_name: osquery-windows-x64 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Setup MSBuild + uses: microsoft/setup-msbuild@v2 + + - name: Setup Visual Studio Build Tools + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.os_arch }} + + - name: Configure CMake for ${{ matrix.os }} + run: | + mkdir build + cd build + cmake -G "Visual Studio 17 2022" -A ${{ matrix.os_arch }} -DOSQUERY_VERSION="5.9.1" .. + + - name: Build OSQuery for Windows ${{ matrix.os_arch }} with MSBuild + run: | + cd build + cmake --build . --config RelWithDebInfo -j10 diff --git a/CMakeLists.txt b/CMakeLists.txt index 17d61942baa..1af548840d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,12 @@ function(main) (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")) message(STATUS "Warning: the selected C or C++ compiler is not clang/clang++. Compilation may fail") endif() + + # Fix for macOS zlib fdopen macro conflict + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_UNISTD_H=1 -DHAVE_FDOPEN=1") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_UNISTD_H=1 -DHAVE_FDOPEN=1") + add_definitions(-D_DARWIN_C_SOURCE=1) + elseif(NOT DEFINED PLATFORM_WINDOWS) if(NOT "${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") diff --git a/libraries/cmake/source/zlib/src b/libraries/cmake/source/zlib/src index 51b7f2abdad..5a82f71ed1d 160000 --- a/libraries/cmake/source/zlib/src +++ b/libraries/cmake/source/zlib/src @@ -1 +1 @@ -Subproject commit 51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf +Subproject commit 5a82f71ed1dfc0bec044d9702463dbdf84ea3b71