From ae0a29f9d68854f7f27b1c05fcac1ca905ff2176 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Tue, 23 Sep 2025 18:03:10 +0300 Subject: [PATCH 01/16] test github workflow --- .github/workflows/hosted_runners.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/hosted_runners.yml b/.github/workflows/hosted_runners.yml index 2a738239cd6..3e3853d61b0 100644 --- a/.github/workflows/hosted_runners.yml +++ b/.github/workflows/hosted_runners.yml @@ -23,6 +23,7 @@ on: branches: - 'master' - 'main' + - 'feature/agent-ci' tags: - '*' From a9d0918013eaabc2e8159f0d1caf3773ee0a2b27 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Fri, 26 Sep 2025 15:01:50 +0300 Subject: [PATCH 02/16] try macOS build --- .github/FUNDING.yml | 3 - .github/ISSUE_TEMPLATE/Blueprint.md | 16 - .github/ISSUE_TEMPLATE/Bug_Report.md | 39 - .github/ISSUE_TEMPLATE/Feature_Request.md | 29 - .github/ISSUE_TEMPLATE/Free_Form.md | 17 - .github/ISSUE_TEMPLATE/New_Release.md | 34 - .github/ISSUE_TEMPLATE/Question.md | 27 - .github/steps/sign-macos-package/action.yml | 169 +++ .github/workflows/build.yml | 236 ++++ .github/workflows/cleanup_ec2_runners.yml | 45 - .github/workflows/cve_scan_runner.yml | 37 - .github/workflows/hosted_runners.yml | 1394 ------------------- .github/workflows/self_hosted_runners.yml | 456 ------ CMakeLists.txt | 6 + 14 files changed, 411 insertions(+), 2097 deletions(-) delete mode 100644 .github/FUNDING.yml delete mode 100644 .github/ISSUE_TEMPLATE/Blueprint.md delete mode 100644 .github/ISSUE_TEMPLATE/Bug_Report.md delete mode 100644 .github/ISSUE_TEMPLATE/Feature_Request.md delete mode 100644 .github/ISSUE_TEMPLATE/Free_Form.md delete mode 100644 .github/ISSUE_TEMPLATE/New_Release.md delete mode 100644 .github/ISSUE_TEMPLATE/Question.md create mode 100644 .github/steps/sign-macos-package/action.yml create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/cleanup_ec2_runners.yml delete mode 100644 .github/workflows/cve_scan_runner.yml delete mode 100644 .github/workflows/hosted_runners.yml delete mode 100644 .github/workflows/self_hosted_runners.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 1ca22c5c49a..00000000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,3 +0,0 @@ -# These are supported funding model platforms - -community_bridge: osquery diff --git a/.github/ISSUE_TEMPLATE/Blueprint.md b/.github/ISSUE_TEMPLATE/Blueprint.md deleted file mode 100644 index 8b30625804d..00000000000 --- a/.github/ISSUE_TEMPLATE/Blueprint.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: Blueprint -about: You're planning on implementing a change -title: '' -labels: '' -assignees: '' - ---- - - - -# Blueprint - - diff --git a/.github/ISSUE_TEMPLATE/Bug_Report.md b/.github/ISSUE_TEMPLATE/Bug_Report.md deleted file mode 100644 index a98105b9ac5..00000000000 --- a/.github/ISSUE_TEMPLATE/Bug_Report.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: Bug Report -about: Something is not working as expected -title: '' -labels: '' -assignees: '' - ---- - - - -# Bug report - - - -### What operating system and version are you using? - - - -### What version of osquery are you using? - - - -### What steps did you take to reproduce the issue? - -### What did you expect to see? - -### What did you see instead? diff --git a/.github/ISSUE_TEMPLATE/Feature_Request.md b/.github/ISSUE_TEMPLATE/Feature_Request.md deleted file mode 100644 index 9fffce2797e..00000000000 --- a/.github/ISSUE_TEMPLATE/Feature_Request.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: Feature Request -about: You want the team to implement a new feature -title: '' -labels: '' -assignees: '' - ---- - - - -# Feature request - - - -### What new feature do you want? - - - -### How is this new feature useful? - - - -### How can this be implemented? - - diff --git a/.github/ISSUE_TEMPLATE/Free_Form.md b/.github/ISSUE_TEMPLATE/Free_Form.md deleted file mode 100644 index 7bcf75a0205..00000000000 --- a/.github/ISSUE_TEMPLATE/Free_Form.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: Task -about: A generic task or TODO item -title: '' -labels: '' -assignees: '' - ---- - - - -# Task - - diff --git a/.github/ISSUE_TEMPLATE/New_Release.md b/.github/ISSUE_TEMPLATE/New_Release.md deleted file mode 100644 index b16ece99279..00000000000 --- a/.github/ISSUE_TEMPLATE/New_Release.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: New Release -about: Checklist of Release actions -title: '' -labels: '' -assignees: '' - ---- - - - - - -# New release checklist - -## Before creating a GitHub tag - -- [ ] Review the [milestones](https://github.com/osquery/osquery/milestones), move unfinished issues to the next release, close the milestone. -- [ ] Ask for a Technical Steering Committee member to make a GitHub tag. -- [ ] Trigger [code signing workflow](https://github.com/osquery/osquery-codesign/actions/workflows/release-generator.yml) -- [ ] Assure that testable packages are available on the Releases page, and announce to the `#core` channel on Slack. - -## After creating a GitHub tag - -- [ ] If the `CHANGELOG.md` hasn't already been updated, update it in a new Pull Request, review and merge it. It should reflect everything done up to the tagged version commit. -- [ ] Assure that the ChangeLog shows up in the [tag release notes](https://github.com/osquery/osquery/tags). - -## Promoting to Stable -- [ ] Use [code signing workflow](https://github.com/osquery/osquery-codesign/actions/workflows/release-generator.yml) to build packages and upload to S3 -- [ ] Update the website with the new release and schema. -- [ ] Publish the website changes. -- [ ] Publish the new packages into the hosted repos. -- [ ] Bump or ping the [Homebrew Cask](https://github.com/Homebrew/homebrew-cask/blob/master/Casks/osquery.rb). -- [ ] Advertise the new release. :) diff --git a/.github/ISSUE_TEMPLATE/Question.md b/.github/ISSUE_TEMPLATE/Question.md deleted file mode 100644 index 5547b2c3461..00000000000 --- a/.github/ISSUE_TEMPLATE/Question.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: Question -about: You have a question about osquery -title: '' -labels: '' -assignees: '' - ---- - - - -# Question - -**We are happy to help, however please DO NOT submit issues for questions about osquery.** - -We only use issues to track bugs, feature requests or blueprints. For everything else, check out Slack or the other channels below: - -Slack : https://osquery.slack.com - -Slack newcomers, first use this invite link: https://join.slack.com/t/osquery/shared_invite/zt-1wipcuc04-DBXmo51zYJKBu3_EP3xZPA - -Reddit : https://www.reddit.com/r/osquery/ - -Stack Overflow : https://stackoverflow.com/tags/osquery - - -**If you go ahead and submit an issue with a question it will not be answered.** diff --git a/.github/steps/sign-macos-package/action.yml b/.github/steps/sign-macos-package/action.yml new file mode 100644 index 00000000000..b731153288b --- /dev/null +++ b/.github/steps/sign-macos-package/action.yml @@ -0,0 +1,169 @@ +name: Sign MacOS package +description: Sign built MacOS package and Notarize it +inputs: + apple_certificate_p12: + description: 'Base64 encoded Apple Developer certificate (.p12)' + required: true + apple_certificate_password: + description: 'Password for the .p12 certificate' + required: true + apple_developer_id: + description: 'Apple Developer ID string' + required: true + apple_id_username: + description: 'Apple ID username for notarization' + required: true + apple_id_password: + description: 'App-specific password for Apple ID' + required: true + apple_team_id: + description: 'Apple Developer ID' + required: true + binary_name: + description: 'Name of built MacOS binary (e.g. "rmmagent-mac-universal")' + required: true + app_name: + description: 'Name of MacOS App targets to add (e.g. "RMMAgent")' + required: true + +runs: + using: "composite" + steps: + - name: Setup and export all needed vars + shell: bash + run: | + BINARY_NAME="${{ inputs.binary_name }}" + APP_NAME="${{ inputs.app_name }}" + APP_BUNDLE="${APP_NAME}.app" + BINARY_PATH="./artifacts/$BINARY_NAME" + DEVELOPER_ID="${{ inputs.apple_developer_id }}" + + echo "BINARY_NAME=$BINARY_NAME" >> $GITHUB_ENV + echo "APP_NAME=$APP_NAME" >> $GITHUB_ENV + echo "APP_BUNDLE=$APP_BUNDLE" >> $GITHUB_ENV + echo "BINARY_PATH=$BINARY_PATH" >> $GITHUB_ENV + echo "DEVELOPER_ID=$DEVELOPER_ID" >> $GITHUB_ENV + + - name: Setup Apple Code Signing + shell: bash + run: | + CERTIFICATE_PATH="$HOME/certificate.p12" + KEYCHAIN_PATH="$HOME/signing.keychain-db" + + # Decode certificate and create keychain + echo "${{ inputs.apple_certificate_p12 }}" | base64 --decode > "$CERTIFICATE_PATH" + security create-keychain -p "${{ inputs.apple_certificate_password }}" "$KEYCHAIN_PATH" + security default-keychain -s "$KEYCHAIN_PATH" + security unlock-keychain -p "${{ inputs.apple_certificate_password }}" "$KEYCHAIN_PATH" + + # Import certificate and set key partition list + security import "$CERTIFICATE_PATH" -P "${{ inputs.apple_certificate_password }}" -A -t cert -f pkcs12 -k "$KEYCHAIN_PATH" + security set-key-partition-list -S apple-tool:,apple: -s -k "${{ inputs.apple_certificate_password }}" "$KEYCHAIN_PATH" + + # Verify setup, cleanup initial cert and export path + security find-identity -v -p codesigning "$KEYCHAIN_PATH" + rm "$CERTIFICATE_PATH" + echo "KEYCHAIN_PATH=$KEYCHAIN_PATH" >> $GITHUB_ENV + + - name: Code Sign Binary + shell: bash + run: | + to_lower() { + # Lowercase regardless of input + printf '%s' "$*" | tr '[:upper:]' '[:lower:]' + } + APP_NAME_LOWER="$(to_lower $APP_NAME)" + BUNDLE_ID="cx.flamingo.$APP_NAME_LOWER" + + # Clean up any existing bundle and create new app bundle structure + rm -rf "$APP_BUNDLE" "${APP_BUNDLE}.zip" + mkdir -p "${APP_BUNDLE}/Contents/MacOS" + mkdir -p "${APP_BUNDLE}/Contents/Resources" + cp "$BINARY_PATH" "${APP_BUNDLE}/Contents/MacOS/${APP_NAME}" + chmod +x "${APP_BUNDLE}/Contents/MacOS/${APP_NAME}" + + # Create Info.plist and entitlements + cat > "${APP_BUNDLE}/Contents/Info.plist" << EOF + + + + + CFBundleExecutable + ${APP_NAME} + CFBundleIdentifier + ${BUNDLE_ID} + CFBundleName + ${APP_NAME} + CFBundleDisplayName + ${APP_NAME} + CFBundleVersion + 1.0.0 + CFBundleShortVersionString + 1.0.0 + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + APPL + CFBundleSignature + ???? + LSMinimumSystemVersion + 10.15 + NSHighResolutionCapable + + LSUIElement + + LSBackgroundOnly + + + + EOF + + cat > "entitlements.plist" << EOF + + + + + com.apple.security.cs.disable-library-validation + com.apple.developer.endpoint-security.client + + + EOF + + # Sign the binary and verify + codesign --sign "$DEVELOPER_ID" --timestamp --options runtime --entitlements "entitlements.plist" --deep --force "$APP_BUNDLE" + + # Verify signature + codesign --verify --deep --strict --verbose=2 "$APP_BUNDLE" + echo "Binary signed successfully" + + - name: Create archive and submit for Notarization + shell: bash + run: | + zip -r "${APP_BUNDLE}.zip" "$APP_BUNDLE" + + xcrun notarytool submit "${APP_BUNDLE}.zip" \ + --apple-id "${{ inputs.apple_id_username }}" \ + --password "${{ inputs.apple_id_password }}" \ + --team-id "${{ inputs.apple_team_id }}" \ + --wait \ + --timeout 30m + + - name: Staple Notarization Ticket and Verify Final Binary + shell: bash + run: | + xcrun stapler staple "$APP_BUNDLE" + xcrun stapler validate "$APP_BUNDLE" + echo "Notarization completed successfully" + + # Verify code signature and notarization + codesign --verify --deep --strict --verbose=2 "$APP_BUNDLE" + spctl --assess --type execute --verbose "$APP_BUNDLE" + echo "Binary verification completed" + + - name: Cleanup Keychain + if: always() + shell: bash + run: | + if [[ -n "$KEYCHAIN_PATH" ]] && [[ -f "$KEYCHAIN_PATH" ]]; then + security delete-keychain "$KEYCHAIN_PATH" + fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..0864234d3a8 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,236 @@ +name: OSQuery Build and Push Latest + +permissions: + contents: write + packages: write + actions: write + pull-requests: write + attestations: write + id-token: write + +on: + push: + branches: [master, feature/agent-ci] + paths-ignore: + - "**/*.md" + workflow_dispatch: + inputs: + tag: + description: "Tag to release (e.g., v1.2.3)" + required: true + type: string + +env: + REGISTRY: "ghcr.io" + ORGANISATION: ${{ github.repository_owner }} + REPOSITORY: ${{ github.event.repository.name }} + CMAKE_OSX_DEPLOYMENT_TARGET: "15.0" + CMAKE_BUILD_TYPE: Release + +# ============================================================================= +# JOBS +# ============================================================================= + +jobs: + build_macos: + name: "Build C Client (${{ matrix.os }})" + runs-on: ${{ matrix.os }} + if: | + github.event_name == 'push' || github.event_name == 'workflow_dispatch' + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - name: "macOS x86-64" + os: macos-13 + os_arch: x86-64 + cmake_arch: x86_64 + artifact_name: osquery-macos-x64 + - name: "macOS ARM64" + os: macos-latest + os_arch: arm64 + cmake_arch: arm64 + artifact_name: osquery-macos-arm64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install Xcode Command Line Tools + shell: bash + run: | + # Install Xcode if not already installed + if ! xcode-select -p >/dev/null 2>&1; then + echo "Installing Xcode Command Line Tools..." + xcode-select --install + # Wait for installation to complete + while ! xcode-select -p >/dev/null 2>&1; do + echo "Waiting for Xcode Command Line Tools installation..." + sleep 10 + done + fi + + - name: Cache Homebrew + uses: actions/cache@v4 + with: + path: | + ~/Library/Caches/Homebrew + /usr/local/Homebrew + /opt/homebrew + key: ${{ runner.os }}-${{ runner.arch }}-homebrew-${{ hashFiles('.github/workflows/*.yml') }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-homebrew- + + - name: Install macOS Dependencies + run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck + shell: bash + + - name: Setup ccache + run: | + echo "Setting up ccache..." + ccache --version + ccache --zero-stats + ccache --show-stats + + # Configure ccache + ccache --set-config max_size=2G + ccache --set-config compression=true + + - name: Cache CMake Build for faster rebuilds + uses: actions/cache@v4 + with: + path: | + build/ + ~/.ccache + key: ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} + ${{ runner.os }}-${{ matrix.os_arch }}-cmake- + + - name: Setup Build Environment + run: | + echo "Setting up build environment for ${{ matrix.os_arch }}..." + export SDKROOT=$(xcrun --show-sdk-path) + export CFLAGS="-isysroot $SDKROOT" + export CXXFLAGS="-isysroot $SDKROOT" + + echo "SDKROOT=$SDKROOT" >> $GITHUB_ENV + echo "CFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + echo "CXXFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + + - name: Configure CMake + run: | + echo "Creating build directory and configuring..." + mkdir -p build && cd build + + cmake \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ + -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_OSX_SYSROOT=$SDKROOT \ + -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + .. + + echo "CMake configuration completed" + + # Build the project + - name: Build Project + run: | + echo "Starting build for ${{ matrix.os_arch }}..." + cd build + + CPU_COUNT=$(sysctl -n hw.ncpu) + echo "Building with $CPU_COUNT parallel jobs" + time cmake --build . -j $CPU_COUNT + + echo "Build completed successfully" + +# - name: Build OSQuery for macOS ${{ matrix.os_arch }} +# run: | +# echo "Building OSQuery for macOS ${{ matrix.os_arch }}" +# make macos ARCHID=${{ matrix.archid }} + + - name: Prepare Artifacts + run: | + mkdir -p artifacts + cp build/osquery/osqueryd "artifacts/${{ matrix.artifact_name }}" + + - name: Upload client artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: | + artifacts/ + if-no-files-found: warn + retention-days: 30 + compression-level: 9 + + create_universal_macos: + name: "Create Universal macOS Binary" + needs: [build_macos] + runs-on: macos-latest + if: | + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && + !failure() && !cancelled() + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Download macOS x86-64 Binary + uses: actions/download-artifact@v4 + with: + name: osquery-macos-x64 + path: x64-artifacts/ + + - name: Download macOS ARM64 Binary + uses: actions/download-artifact@v4 + with: + name: osquery-macos-arm64 + path: arm64-artifacts/ + + - name: Create Universal Binary + run: | + echo "Creating universal macOS binary..." + X64_BINARY="x64-artifacts/osquery-macos-x64" + ARM64_BINARY="arm64-artifacts/osquery-macos-arm64" + + echo "Creating universal binary with lipo..." + mkdir -p artifacts + lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/osquery-macos-universal + + # Verify universal binary + echo "Universal binary info:" + lipo -info ./artifacts/osquery-macos-universal + file ./artifacts/osquery-macos-universal + + - name: Sign MacOS package + uses: ./.github/steps/sign-macos-package + with: + apple_certificate_p12: ${{ secrets.APPLE_CERTIFICATE_P12 }} + apple_certificate_password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + apple_developer_id: ${{ secrets.APPLE_DEVELOPER_ID }} + apple_id_username: ${{ secrets.APPLE_ID_USERNAME }} + apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} + apple_team_id: ${{ secrets.APPLE_TEAM_ID }} + binary_name: osquery-macos-universal + app_name: OSQuery + + + - name: Upload Universal Binary + uses: actions/upload-artifact@v4 + with: + name: osquery-macos-universal + path: artifacts/ + retention-days: 30 + compression-level: 9 \ No newline at end of file diff --git a/.github/workflows/cleanup_ec2_runners.yml b/.github/workflows/cleanup_ec2_runners.yml deleted file mode 100644 index 8e7e608cbda..00000000000 --- a/.github/workflows/cleanup_ec2_runners.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright (c) 2014-present, The osquery authors -# -# This source code is licensed as defined by the LICENSE file found in the -# root directory of this source tree. -# -# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) - - -# Cleanup any lingering ec2 runners -name: cleanup_ec2_runners - -on: - schedule: - - cron: "0 */1 * * *" - workflow_dispatch: - -jobs: - cleanup: - runs-on: ubuntu-latest - timeout-minutes: 5 - - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} - - - name: Find Instances - run: | - cutoff=$(date -u --date="-2 hours" "+%Y-%m-%dT%H:%M") - - aws ec2 describe-instances \ - --filters Name=instance-state-name,Values=running Name=instance.group-id,Values=sg-0757a3162c999331b \ - --query "Reservations[].Instances[?LaunchTime<=\`$cutoff\`][].{id: InstanceId, type: InstanceType, launched: LaunchTime}" \ - | tee instances.json - - - name: Terminate Instances - run: | - # This is a for loop, so that any failures don't stop all the instances from terminating - for i in $(jq -r '.[].id' instances.json); do - aws ec2 terminate-instances --instance-ids "$i" - done diff --git a/.github/workflows/cve_scan_runner.yml b/.github/workflows/cve_scan_runner.yml deleted file mode 100644 index d4d401c0d95..00000000000 --- a/.github/workflows/cve_scan_runner.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: cve-scan-runner - -on: - workflow_dispatch: - - schedule: - - cron: "17 23 * * *" - -env: - # To see the script output in real time - PYTHONUNBUFFERED: 1 - -jobs: - scan-and-open-issues: - runs-on: ubuntu-22.04 - timeout-minutes: 20 - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - - - name: Install python pre-requisites - run: | - pip3 install -r ./tools/ci/scripts/cve/requirements.txt - - - name: Scan CVEs and open issues - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NIST_API_KEY: ${{ secrets.NVD_API_KEY }} - - run: | - ./tools/ci/scripts/cve/validate_manifest_libraries_versions.py --manifest libraries/third_party_libraries_manifest.json \ - --repository . - - ./tools/ci/scripts/cve/third_party_libraries_cves_scanner.py --manifest libraries/third_party_libraries_manifest.json \ - --create-issues \ - --debug diff --git a/.github/workflows/hosted_runners.yml b/.github/workflows/hosted_runners.yml deleted file mode 100644 index 3e3853d61b0..00000000000 --- a/.github/workflows/hosted_runners.yml +++ /dev/null @@ -1,1394 +0,0 @@ -# -# Copyright (c) 2014-present, The osquery authors -# -# This source code is licensed as defined by the LICENSE file found in the -# root directory of this source tree. -# -# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) -# - -# Due to a limitation in how GitHub Actions works, we can't reference -# jobs in another file inside the `needs` statement. -# -# This configuration file takes care of the Windows, macOS and Linux -# builds on the x86 platform. -name: build_x86 - -on: - # Run this workflow once every day against the master branch - schedule: - - cron: "0 7 * * *" - - push: - branches: - - 'master' - - 'main' - - 'feature/agent-ci' - - tags: - - '*' - - pull_request: - branches: - - '*' - -# Cancel old jobs of a PR if a new job is started. -# Fallback on using the run id if it's not a PR, which is unique, so no job canceling. -concurrency: - group: hosted-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true - -# Please remember to update values for both x86 and aarch64 workflows. -env: - PACKAGING_REPO: https://github.com/osquery/osquery-packaging - PACKAGING_COMMIT: c089fb2d3d796d976e3b2fbea7ee69a1616b9576 - SUBMODULE_CACHE_VERSION: 3 - -# If the initial code sanity checks are passing, then one job -# per [`platform` * `build_type`] will start, building osquery -# and generating packages that are later attached to the commit -# (or PR) as build artifacts. -jobs: - # This job performs basic source code check, looking for formatting - # issues and missing copyright headers - check_code_style: - runs-on: ubuntu-22.04 - - container: - image: osquery/builder20.04:7e9ee0339 - options: --user 1001 - - steps: - - name: Setup the build paths - shell: bash - id: build_paths - run: | - rel_build_path="workspace/build" - rel_source_path="workspace/src" - - mkdir -p "${rel_build_path}" - mkdir -p "${rel_source_path}" - - echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - # This script makes sure that the copyright headers have been correctly - # placed on all the source code files - - name: Check the copyright headers - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - ./tools/ci/scripts/check_copyright_headers.py - - - name: Configure the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DOSQUERY_ENABLE_FORMAT_ONLY=ON \ - "${{ steps.build_paths.outputs.SOURCE }}" - - # Formatting is tested against the clang-format binary we ship - # with the osquery-toolchain, so this job is only performed once on - # a Linux machine. - - name: Check code formatting - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: - cmake --build . --target format_check - - - - # This jobs checks that the third party libraries manifest has the correct format - # and that is up to date compared to the current state of the repository - check_libraries_manifest: - runs-on: ubuntu-22.04 - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - - - name: Install python pre-requisites - run: | - pip3 install -r ./tools/ci/scripts/cve/requirements.txt - - - name: Verify the third party libraries manifest - run: | - ./tools/ci/scripts/cve/validate_manifest_libraries_versions.py --manifest libraries/third_party_libraries_manifest.json \ - --repository . - - - # Test generating the website json - check_genwebsitejson: - needs: [check_code_style, check_libraries_manifest] - runs-on: ubuntu-22.04 - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - - - name: genwebsitejson.py - run: python3 tools/codegen/genwebsitejson.py --specs=specs/ - - - # This job runs source code analysis tools (currently, just cppcheck) - check_source_code: - needs: [check_code_style, check_libraries_manifest] - - runs-on: ${{ matrix.os }} - - container: - image: osquery/builder20.04:7e9ee0339 - options: --user 1001 - - strategy: - matrix: - os: [ubuntu-22.04] - - steps: - - name: Setup the build paths - shell: bash - id: build_paths - run: | - rel_build_path="workspace/build" - rel_source_path="workspace/src" - rel_install_path="workspace/install" - - mkdir -p ${rel_build_path} \ - ${rel_source_path} \ - ${rel_install_path} - - echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - echo "REL_BINARY=${rel_build_path}" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules - - key: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the git submodules - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - git submodule sync --recursive - - - name: Configure the project (Release) - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DOSQUERY_BUILD_TESTS=ON \ - -DOSQUERY_BUILD_ROOT_TESTS=ON \ - "${{ steps.build_paths.outputs.SOURCE }}" - - - name: Initialize the project (Release) - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake --build . --target prepare_for_ide - - - name: Run cppcheck (Release) - shell: bash - id: release_cppcheck_runner - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake --build . --target cppcheck 2>&1 | tee cppcheck_release.txt - - - name: Store the cppcheck log (Release) - uses: actions/upload-artifact@v4.4.0 - with: - name: cppcheck-release - path: ${{ steps.build_paths.outputs.REL_BINARY }}/cppcheck_release.txt - - - name: Configure the project (Debug) - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DCMAKE_BUILD_TYPE:STRING=Debug \ - -DOSQUERY_BUILD_TESTS=ON \ - -DOSQUERY_BUILD_ROOT_TESTS=ON \ - "${{ steps.build_paths.outputs.SOURCE }}" - - - name: Initialize the project (Debug) - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake --build . --target prepare_for_ide - - - name: Run cppcheck (Debug) - shell: bash - id: debug_cppcheck_runner - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake --build . --target cppcheck 2>&1 | tee cppcheck_debug.txt - - - name: Store the cppcheck log (Debug) - uses: actions/upload-artifact@v4.4.0 - with: - name: cppcheck-debug - path: ${{ steps.build_paths.outputs.REL_BINARY }}/cppcheck_debug.txt - - - - - # The Linux build will only start once we know that the code - # has been properly formatted - build_linux: - needs: [check_code_style, check_libraries_manifest] - - runs-on: ${{ matrix.os }} - - container: - image: osquery/builder20.04:7e9ee0339 - options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock --pid=host --user 1001 - - strategy: - matrix: - build_type: [Release, RelWithDebInfo, Debug] - os: [ubuntu-22.04] - - steps: - - name: Make space uninstalling packages - shell: bash - run: | - run_on_host="sudo nsenter -t 1 -m -u -n -i" - packages_to_remove=$($run_on_host dpkg-query -f '${Package}\n' -W | grep "^clang-.*\|^llvm-.*\|^php.*\|^mono-.*\|^mongodb-.*\ - \|^libmono-.*\|^temurin-8-jdk\|^temurin-11-jdk\|^temurin-17-jdk\|^dotnet-.*\|^google-chrome-stable\|^microsoft-edge-stable\|^google-cloud-sdk\|^firefox\|^hhvm\|^snapd") - $run_on_host apt purge $packages_to_remove - - # Due to how the RPM packaging tools work, we have to adhere to some - # character count requirements in the build path vs source path. - # - # Failing to do so, will break the debuginfo RPM package. - - name: Setup the build paths - id: build_paths - run: | - rel_build_path="workspace/usr/src/debug/osquery/build" - rel_src_path="workspace/padding-required-by-rpm-packages/src" - rel_ccache_path="workspace/ccache" - rel_package_data_path="workspace/package_data" - rel_packaging_path="workspace/osquery-packaging" - rel_package_build_path="workspace/package-build" - - mkdir -p ${rel_build_path} \ - ${rel_src_path} \ - ${rel_ccache_path} \ - ${rel_src_path} \ - ${rel_package_data_path} \ - ${rel_package_build_path} - - echo "SOURCE=$(realpath ${rel_src_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - echo "CCACHE=$(realpath ${rel_ccache_path})" >> $GITHUB_OUTPUT - echo "PACKAGING=$(realpath ${rel_packaging_path})" >> $GITHUB_OUTPUT - echo "PACKAGE_DATA=$(realpath ${rel_package_data_path})" >> $GITHUB_OUTPUT - echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT - echo "PACKAGE_BUILD=$(realpath ${rel_package_build_path})" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - - name: Select the build job count - shell: bash - id: build_job_count - run: | - echo "VALUE=$(($(nproc) + 1))" >> $GITHUB_OUTPUT - - - name: Select the build options for the tests - shell: bash - id: tests_build_settings - run: | - if [[ "${{ matrix.build_type }}" == "RelWithDebInfo" ]] ; then - echo "VALUE=OFF" >> $GITHUB_OUTPUT - else - echo "VALUE=ON" >> $GITHUB_OUTPUT - fi - - - name: Get runner hardware info - shell: bash - id: runner_stats - run: | - df -h - echo "Cores: $(nproc)" - free -m - - # We don't have enough space on the worker to actually generate all - # the debug symbols (osquery + dependencies), so we have a flag to - # disable them when running a Debug build - - name: Select the debug symbols options - shell: bash - id: debug_symbols_settings - run: | - if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then - echo "VALUE=ON" >> $GITHUB_OUTPUT - else - echo "VALUE=OFF" >> $GITHUB_OUTPUT - fi - - - name: Clone the osquery-packaging repository - run: | - git clone ${{ env.PACKAGING_REPO }} \ - ${{ steps.build_paths.outputs.PACKAGING }} - cd ${{ steps.build_paths.outputs.PACKAGING }} - git checkout ${{ env.PACKAGING_COMMIT }} - - - name: Update the cache (ccache) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.CCACHE }} - - key: | - ccache_${{ matrix.os }}_${{ matrix.build_type }}_${{ github.sha }} - - restore-keys: | - ccache_${{ matrix.os }}_${{ matrix.build_type }} - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules - - key: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the git submodules - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - git submodule sync --recursive - - - name: Configure the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ - -DOSQUERY_BUILD_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ - -DOSQUERY_BUILD_ROOT_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ - "${{ steps.build_paths.outputs.SOURCE }}" - - - name: Build the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Disk space information - shell: bash - id: disk_space_info_post_build - run: | - df -h - du -sh ${{ steps.build_paths.outputs.BINARY }} - - # Only run the tests on Debug and Release configurations; skip RelWithDebInfo - - name: Run the tests as normal user - working-directory: ${{ steps.build_paths.outputs.BINARY }} - if: matrix.build_type != 'RelWithDebInfo' - run: | - # Allow access to the docker socket - sudo chmod 666 /var/run/docker.sock - ctest --build-nocmake -LE "root-required" -V - - - name: Run the tests as root user - working-directory: ${{ steps.build_paths.outputs.BINARY }} - if: matrix.build_type != 'RelWithDebInfo' - run: | - sudo -u root ctest --build-nocmake -L "root-required" -V - - - name: Run the install target - if: matrix.build_type == 'RelWithDebInfo' - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - run: | - cmake \ - --build . \ - --target install \ - -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Create the packages - if: matrix.build_type == 'RelWithDebInfo' - working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} - - shell: bash - - run: | - osquery_version=$(cd ${{ steps.build_paths.outputs.SOURCE }} && git describe --tags --always ) - - tar pcvzf package_data.tar.gz \ - ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - package_format_list=( "DEB" "RPM" "TGZ" ) - - for package_format in "${package_format_list[@]}" ; do - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCPACK_GENERATOR=${package_format} \ - -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ - -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} \ - -DOSQUERY_SOURCE_DIRECTORY_LIST="${{ steps.build_paths.outputs.SOURCE }};${{ steps.build_paths.outputs.BINARY }}" \ - ${{ steps.build_paths.outputs.PACKAGING }} - - cmake --build . \ - --target package - done - - - name: Locate the packages - if: matrix.build_type == 'RelWithDebInfo' - id: packages - shell: bash - run: | - echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.deb)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_DEBUG_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.ddeb)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-?.*.rpm)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_DEBUG_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-debuginfo-*.rpm)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*linux_x86_64.tar.gz)" >> $GITHUB_OUTPUT - - - name: Store the unsigned release package data artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_release_package_data - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} - - - name: Store the unsigned release DEB artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_release_deb - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_DEB_PATH }} - - - name: Store the unsigned debug DEB artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_debug_deb - path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_DEB_PATH }} - - - name: Store the unsigned release RPM artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_release_rpm - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_RPM_PATH }} - - - name: Store the unsigned debug RPM artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_debug_rpm - path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_RPM_PATH }} - - - name: Store the unsigned release TGZ artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4.4.0 - with: - name: linux_unsigned_release_tgz - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} - - # Before we terminate this job, delete the build folder. The cache - # actions will require the disk space to create the archives. - - name: Reclaim disk space - run: | - rm -rf ${{ steps.build_paths.outputs.BINARY }} - - - - - # The macOS build will only start once we know that the code - # has been properly formatted - build_macos: - needs: [check_code_style, check_libraries_manifest] - - runs-on: ${{ matrix.os }} - - strategy: - matrix: - build_type: [Release, Debug] - architecture: [x86_64, arm64] - os: [macos-14, macos-15] - - steps: - - name: Select the build job count - shell: bash - id: build_job_count - run: | - echo "VALUE=$(($(sysctl -n hw.logicalcpu) + 1))" >> $GITHUB_OUTPUT - - - name: Setup the build paths - shell: bash - id: build_paths - run: | - rel_build_path="workspace/build" - rel_src_path="workspace/src" - rel_ccache_path="workspace/ccache" - rel_downloads_path="workspace/downloads" - rel_install_path="workspace/install" - rel_package_data_path="workspace/package_data" - rel_packaging_path="workspace/osquery-packaging" - rel_package_build_path="workspace/package-build" - - mkdir -p ${rel_build_path} \ - ${rel_ccache_path} \ - ${rel_downloads_path} \ - ${rel_install_path} \ - ${rel_package_data_path} \ - ${rel_package_build_path} - - echo "SOURCE=$(pwd)/${rel_src_path}" >> $GITHUB_OUTPUT - echo "REL_SOURCE=${rel_src_path}" >> $GITHUB_OUTPUT - echo "BINARY=$(pwd)/${rel_build_path}" >> $GITHUB_OUTPUT - echo "CCACHE=$(pwd)/${rel_ccache_path}" >> $GITHUB_OUTPUT - echo "DOWNLOADS=$(pwd)/${rel_downloads_path}" >> $GITHUB_OUTPUT - echo "INSTALL=$(pwd)/${rel_install_path}" >> $GITHUB_OUTPUT - echo "PACKAGING=$(pwd)/${rel_packaging_path}" >> $GITHUB_OUTPUT - echo "PACKAGE_DATA=$(pwd)/${rel_package_data_path}" >> $GITHUB_OUTPUT - echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT - echo "PACKAGE_BUILD=$(pwd)/${rel_package_build_path}" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - path: ${{ steps.build_paths.outputs.REL_SOURCE }} - - - name: Get runner hardware info - shell: bash - id: runner_stats - run: | - df -h - echo "Cores: $(sysctl -n hw.logicalcpu)" - vm_stat - - - name: Update the cache (ccache) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.CCACHE }} - - key: | - ccache_${{ matrix.os }}_${{ matrix.architecture }}_${{ matrix.build_type }}_${{ github.sha }} - - restore-keys: | - ccache_${{ matrix.os }}_${{ matrix.architecture }}_${{ matrix.build_type }} - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules - - key: | - gitmodules_${{ matrix.os }}_${{ matrix.architecture }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.os }}_${{ matrix.architecture }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the cache (downloads) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.DOWNLOADS }} - - key: | - downloads_${{ matrix.os }}_${{ matrix.architecture }}_${{ github.sha }} - - restore-keys: | - downloads_${{ matrix.os }}_${{ matrix.architecture }} - - - name: Update the git submodules - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - git submodule sync --recursive - - - name: Install build dependencies - run: | - brew install \ - ccache \ - flex \ - bison \ - coreutils \ - gnu-sed - - - name: Setup python - uses: actions/setup-python@v5 - with: - # Pin an exact python version to avoid mismatches between this builder and the test_older_macos runner - python-version: '3.13.1' - architecture: arm64 - - - name: Install tests dependencies - id: install_test_deps - run: | - pip3 install setuptools \ - pexpect==3.3 \ - psutil \ - timeout_decorator \ - six \ - thrift==0.11.0 \ - osquery - - - name: Install CMake - shell: bash - run: | - ${{ steps.build_paths.outputs.REL_SOURCE }}/tools/ci/scripts/macos/install_cmake.sh \ - "${{ steps.build_paths.outputs.DOWNLOADS }}" \ - "${{ steps.build_paths.outputs.INSTALL }}" \ - "3.21.4" - - - name: Select the Xcode version - shell: bash - id: xcode_selector - run: | - if [[ "${{ matrix.os }}" == "macos-14" ]]; then - xcode_path="/Applications/Xcode_15.4.app/Contents/Developer" - elif [[ "${{ matrix.os }}" == "macos-15" ]]; then - xcode_path="/Applications/Xcode_16.app/Contents/Developer" - else - echo "Invalid matrix.os: ${{ matrix.os }}" - exit 1 - fi - - echo "PATH=${path}" >> $GITHUB_OUTPUT - - sudo xcode-select -s "${xcode_path}" - - echo "DEPLOYMENT_TARGET=10.15" >> $GITHUB_OUTPUT - - # We don't have enough space on the worker to actually generate all - # the debug symbols (osquery + dependencies), so we have a flag to - # disable them when running a Debug build - - name: Select the debug symbols options - shell: bash - id: debug_symbols_settings - run: | - if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then - echo "VALUE=ON" >> $GITHUB_OUTPUT - else - echo "VALUE=OFF" >> $GITHUB_OUTPUT - fi - - - name: Configure the project - shell: bash - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake -G "Unix Makefiles" \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_OSX_ARCHITECTURES="${{ matrix.architecture }}" \ - -DCMAKE_OSX_DEPLOYMENT_TARGET="${{ steps.xcode_selector.outputs.DEPLOYMENT_TARGET }}" \ - -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ - -DOSQUERY_BUILD_TESTS=ON \ - -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ - ${{ steps.build_paths.outputs.SOURCE }} - - - name: Build the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Disk space information - shell: bash - id: disk_space_info_post_build - run: | - df -h - du -sh ${{ steps.build_paths.outputs.BINARY }} - - - name: Run the tests - if: matrix.architecture == 'arm64' - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - ctest --build-nocmake -V - - - name: Run the install target - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - run: | - cmake \ - --build . \ - --target install \ - -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Create the package data - working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} - run: | - tar pcvzf package_data.tar.gz \ - -C ${{ steps.build_paths.outputs.PACKAGE_DATA }} \ - . - - - name: Locate the package data - if: matrix.build_type == 'Release' - id: packages - shell: bash - run: | - echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz)" >> $GITHUB_OUTPUT - - - name: Store the ${{ matrix.architecture }} unsigned release package data artifact - if: matrix.build_type == 'Release' && matrix.os == 'macos-14' - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_unsigned_release_package_data_${{ matrix.architecture }} - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} - - - name: Package the tests for older x86_64 macOS workers - if: matrix.architecture == 'x86_64' && matrix.os == 'macos-14' - working-directory: ${{ github.workspace }}/workspace - run: | - mkdir macos_tests_${{ matrix.build_type }} - ${{ steps.build_paths.outputs.SOURCE }}/tools/ci/scripts/macos/package_tests.sh build macos_tests_${{ matrix.build_type }} ${{ matrix.architecture }} - - - name: Store the packaged tests for older x86_64 macOS workers - if: matrix.architecture == 'x86_64' && matrix.os == 'macos-14' - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_tests_${{ matrix.build_type }} - path: workspace/macos_tests_${{ matrix.build_type }}.tar.gz - - # Before we terminate this job, delete the build folder. The cache - # actions will require the disk space to create the archives. - - name: Reclaim disk space - run: | - rm -rf ${{ steps.build_paths.outputs.BINARY }} - - - - - # This job takes the packaged tests (Release + Debug) from the latest macos - # builder and runs them on the previous available macOS versions - test_older_macos: - - needs: build_macos - - runs-on: ${{ matrix.os }} - - strategy: - matrix: - python_architecture: [x64] - os: [macos-13] - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/download-artifact@v4.1.7 - with: - name: macos_tests_Release - - - uses: actions/download-artifact@v4.1.7 - with: - name: macos_tests_Debug - - - name: Setup python - uses: actions/setup-python@v5 - with: - # Pin an exact python version to avoid mismatches between the builder and this test_older_macos runner - python-version: '3.13.1' - architecture: ${{ matrix.python_architecture }} - - - name: Install tests dependencies - id: install_test_deps - run: | - pip3 install setuptools \ - pexpect==3.3 \ - psutil \ - timeout_decorator \ - six \ - thrift==0.11.0 \ - osquery - - - name: Install CMake - shell: bash - run: | - mkdir -p "workspace/downloads" \ - "workspace/install" - - ./tools/ci/scripts/macos/install_cmake.sh \ - "workspace/downloads" \ - "workspace/install" \ - "3.21.4" - - - name: Extract the tests - run: | - tar xzf macos_tests_Release.tar.gz - tar xzf macos_tests_Debug.tar.gz - - - name: Run the Debug tests - run: | - ( cd macos_tests_Debug && ./run.sh ) - - - name: Run the Release tests - run: | - ( cd macos_tests_Release && ./run.sh ) - - - - - # This job builds the universal macOS artifacts - build_universal_macos_artifacts: - needs: test_older_macos - - runs-on: macos-14 - - steps: - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install CMake - shell: bash - run: | - mkdir -p "workspace/downloads" \ - "workspace/install" - - ./tools/ci/scripts/macos/install_cmake.sh \ - "workspace/downloads" \ - "workspace/install" \ - "3.21.4" - - - uses: actions/download-artifact@v4.1.7 - with: - name: macos_unsigned_release_package_data_x86_64 - path: macos_unsigned_release_package_data_x86_64 - - - uses: actions/download-artifact@v4.1.7 - with: - name: macos_unsigned_release_package_data_arm64 - path: macos_unsigned_release_package_data_arm64 - - - name: Create the universal package data - run: | - tools/ci/scripts/macos/build_universal_package_data.sh - - - name: Store the universal unsigned release package data artifact - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_unsigned_release_package_data_universal - path: package_data.tar.gz - - - name: Clone the osquery-packaging repository - run: | - git clone ${{ env.PACKAGING_REPO }} osquery-packaging - - cd osquery-packaging - git checkout ${{ env.PACKAGING_COMMIT }} - - - name: Create the packages - shell: bash - run: | - osquery_version=$(git describe --tags --always ) - package_format_list=( "productbuild" "TGZ" ) - - for package_format in "${package_format_list[@]}" ; do - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCPACK_GENERATOR=${package_format} \ - -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ - -DOSQUERY_DATA_PATH=$(pwd)/universal \ - -S osquery-packaging \ - -B package_build - - cmake --build package_build \ - --target package - done - - - name: Locate the packages - id: packages - shell: bash - run: | - echo "REL_UNSIGNED_RELEASE_PKG_PATH=$(ls package_build/*.pkg)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls package_build/*.tar.gz)" >> $GITHUB_OUTPUT - - - name: Store the PKG unsigned release packages - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_unsigned_pkg_universal - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PKG_PATH }} - - - name: Store the TGZ unsigned release packages - uses: actions/upload-artifact@v4.4.0 - with: - name: macos_unsigned_tgz_universal - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} - - - - - # The Windows build will only start once we know that the code - # has been properly formatted - build_windows: - needs: [check_code_style, check_libraries_manifest] - - runs-on: ${{ matrix.os }} - - strategy: - matrix: - build_type: [Release] - bitness: [64, arm64] - os: [windows-2022] - - steps: - - name: Select the build job count - shell: powershell - id: build_job_count - run: | - $cores=(Get-CIMInstance Win32_ComputerSystem).NumberOfLogicalProcessors + 1 - echo "VALUE=$cores" >> $env:GITHUB_OUTPUT - - - name: Setup the build paths - shell: powershell - id: build_paths - run: | - $rel_src_path = "w\src" - $rel_build_path = "w\build" - $rel_sccache_path = "w\sccache" - $rel_downloads_path = "w\downloads" - $rel_install_path = "w\install" - $rel_package_data_path = "w\package_data" - $rel_packaging_path = "w\osquery-packaging" - $rel_packaging_build = "w\package-build" - - New-Item -ItemType Directory -Force -Path $rel_build_path - New-Item -ItemType Directory -Force -Path $rel_sccache_path - New-Item -ItemType Directory -Force -Path $rel_downloads_path - New-Item -ItemType Directory -Force -Path $rel_install_path - New-Item -ItemType Directory -Force -Path $rel_package_data_path - New-Item -ItemType Directory -Force -Path $rel_packaging_build - - $base_dir = (Get-Item .).FullName - - echo "SOURCE=$base_dir\$rel_src_path" >> $env:GITHUB_OUTPUT - echo "REL_SOURCE=$rel_src_path" >> $env:GITHUB_OUTPUT - echo "BINARY=$base_dir\$rel_build_path" >> $env:GITHUB_OUTPUT - echo "SCCACHE=$base_dir\$rel_sccache_path" >> $env:GITHUB_OUTPUT - echo "DOWNLOADS=$base_dir\$rel_downloads_path" >> $env:GITHUB_OUTPUT - echo "INSTALL=$base_dir\$rel_install_path" >> $env:GITHUB_OUTPUT - echo "PACKAGING=$base_dir\$rel_packaging_path" >> $env:GITHUB_OUTPUT - echo "PACKAGE_DATA=$base_dir\$rel_package_data_path" >> $env:GITHUB_OUTPUT - echo "PACKAGE_BUILD=$base_dir\$rel_packaging_build" >> $env:GITHUB_OUTPUT - - - name: Setup the VC arch - shell: powershell - id: vc_arch - run: | - $arch = switch ("${{ matrix.bitness }}") - { - "64" { "x64" } - "arm64" { "x64_arm64" } - } - echo "VC_ARCH=$arch" >> $env:GITHUB_OUTPUT - - # Symbolic links are supported by default on Linux and macOS. On - # Windows, we have to enable them explicitly. They are used to - # handle the include header namespace support that came with - # the initial Buck build system refactor - - name: Configure git - run: | - git config --global core.autocrlf false - git config --global core.symlinks true - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - path: ${{ steps.build_paths.outputs.REL_SOURCE }} - - - name: Get runner hardware info - id: runner_stats - shell: powershell - run: | - Get-CIMInstance Win32_LogicalDisk - Get-CIMInstance Win32_ComputerSystem | Select-Object -Property NumberOfLogicalProcessors - Get-CIMInstance Win32_ComputerSystem | Select-Object -Property TotalPhysicalMemory - - - name: Detect the osquery version - shell: powershell - id: osquery_version - run: | - cd ${{ steps.build_paths.outputs.SOURCE }} - $osquery_version=$(git describe --tags --abbrev=0) - - echo "VALUE=$osquery_version" >> $env:GITHUB_OUTPUT - - - name: Clone the osquery-packaging repository - run: | - cd w - git clone ${{ env.PACKAGING_REPO }} - cd ${{ steps.build_paths.outputs.PACKAGING }} - git checkout ${{ env.PACKAGING_COMMIT }} - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}\.git\modules - - key: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the cache (downloads) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.DOWNLOADS }} - - key: | - downloads_${{ matrix.os }}_${{ github.sha }} - - restore-keys: | - downloads_${{ matrix.os }} - - - name: Initialize the Python 3 installation - uses: actions/setup-python@v5 - with: - python-version: "3.13.x" - architecture: x64 - - # The runners will likely have both the x86 and x64 versions of - # Python but we always need the 64-bit one regardless of which - # architecture we are building for. - # - # The setup-python action should have put the right Python version - # in the PATH variable for us, so locate the installation directory - # so we can use it as a hint when we configure the project with - # CMake - - name: Locate the Python root directory - id: python_root_directory - shell: powershell - run: | - $python_executable_path = $(Get-Command python.exe | Select-Object -ExpandProperty Definition) - $python_root_directory = (Get-Item $python_executable_path).Directory.FullName - - echo "VALUE=$python_root_directory" >> $env:GITHUB_OUTPUT - - # Install the Python dependencies needed for our testing framework - - name: Install tests prerequisites - run: | - python -m pip install --upgrade pip - - python -m pip install wheel ` - setuptools ` - psutil ` - timeout_decorator ` - thrift==0.11.0 ` - osquery ` - pywin32 - - - name: Install Strawberry Perl - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - shell: powershell - run: | - tools\ci\scripts\install_openssl_formula_dependencies.ps1 - - # TODO: Implement a cache so we don't keep hitting the server at each run - - name: Install CMake - working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} - shell: powershell - run: | - $long_cmake_ver = "3.21.4" - $short_cmake_ver = $($long_cmake_ver.split(".")[0] + "." + $long_cmake_ver.split(".")[1]) - - $folder_name = $("cmake-" + $long_cmake_ver + "-windows-x86_64") - $archive_name = $($folder_name + ".zip") - - $url = $("https://cmake.org/files/v" + $short_cmake_ver + "/" + $archive_name) - - (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) - 7z x -o${{ steps.build_paths.outputs.INSTALL }} -y $archive_name - - echo "${{ steps.build_paths.outputs.INSTALL }}\$folder_name\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - # TODO: Implement a cache so we don't keep hitting the server at each run - - name: Install sccache - working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} - shell: powershell - run: | - $long_version = "0.0.1" - - $archive_name = $("sccache-" + $long_version + "-windows.7z") - $url = $("https://github.com/osquery/sccache/releases/download/" + $long_version + "-osquery/" + $archive_name) - - (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) - 7z x -o${{ steps.build_paths.outputs.INSTALL }}\sccache -y $archive_name - - echo "${{ steps.build_paths.outputs.INSTALL }}\sccache" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - # TODO: Implement a cache so we don't keep hitting the server at each run - - name: Install Ninja - working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} - shell: powershell - run: | - $long_version = "1.10.0" - - $archive_name = "ninja-win.zip" - $url = $("https://github.com/ninja-build/ninja/releases/download/v" + $long_version + "/" + $archive_name) - - (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) - 7z x -o${{ steps.build_paths.outputs.INSTALL }}\ninja -y $archive_name - - echo "${{ steps.build_paths.outputs.INSTALL }}\ninja" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Configure the project - shell: cmd - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - SCCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} - SCCACHE_CACHE_SIZE: "5G" - - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} - @echo on - - set cross_compilation= - if ${{ matrix.bitness }}==arm64 ( - echo set (CMAKE_SYSTEM_NAME Windows^) > cross.cmake - echo set (CMAKE_SYSTEM_PROCESSOR ARM64^) >> cross.cmake - set cross_compilation=-DCMAKE_TOOLCHAIN_FILE=cross.cmake - ) - - cmake -G Ninja ^ - -DCMAKE_C_COMPILER=cl.exe ^ - -DCMAKE_CXX_COMPILER=cl.exe ^ - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ - -DOSQUERY_BUILD_TESTS=ON ^ - -DCMAKE_C_COMPILER_LAUNCHER="sccache.exe" ^ - -DCMAKE_CXX_COMPILER_LAUNCHER="sccache.exe" ^ - -DPython3_ROOT_DIR=${{ steps.python_root_directory.outputs.VALUE }} ^ - ${{ steps.build_paths.outputs.SOURCE }} ^ - %cross_compilation% - - - name: Determine compiler version - id: determine_compiler_version - shell: pwsh - run: | - $compiler = (Get-Content "${{ steps.build_paths.outputs.BINARY }}\CMakeCache.txt" | Select-String -Pattern "CMAKE_CXX_COMPILER:STRING=(.*)").Matches[0].Groups[1].Value - - echo "Compiler configured by CMake is $compiler" - - if ($compiler -eq $null || $compiler -eq "") { - Write-Error "Could not find the configured compiler" -ErrorAction Stop - } - - <# - We run the compiler help option; the compiler will write its version in stderr. - Due to how powershell works, we have to go through some hoops to extract the stderr to a variable - and also avoid it considering the command as failed because stderr contains messages. - The expression runs the compiler in a subshell, discards its stdout, then the stderr of the subshell is redirected - to the stdout of the parent shell. - #> - $ErrorActionPreference = 'Continue' - $erroutput = $( & "$compiler" /? 1>$null ) 2>&1 - $ErrorActionPreference = 'Stop' - - if ($erroutput -eq $null || $erroutput -eq "") { - Write-Error "Failed to run the compiler at $compiler" -ErrorAction Stop - } - - $version = ($erroutput | Select-String -Pattern "Compiler Version (.*) for").Matches[0].Groups[1].Value.Replace(".", "") - - if ($version -eq $null || $version -eq "") { - Write-Error "Failed to determine compiler version for $compiler and output $erroutput" -ErrorAction Stop - } - - echo "Found compiler version $version" - - echo "COMPILER_VERSION=$version" >> $env:GITHUB_OUTPUT - - - name: Update the cache (sccache) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SCCACHE }} - - key: | - sccache_${{ matrix.os }}_${{ matrix.bitness }}_${{ matrix.build_type }}_${{ steps.determine_compiler_version.outputs.COMPILER_VERSION }}_${{ github.sha }} - - restore-keys: | - sccache_${{ matrix.os }}_${{ matrix.bitness }}_${{ matrix.build_type }}_${{ steps.determine_compiler_version.outputs.COMPILER_VERSION }} - - - name: Build the project - shell: cmd - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - SCCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} - SCCACHE_CACHE_SIZE: "5G" - - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} - @echo on - - cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} - - if %errorlevel% neq 0 exit /b %errorlevel% - sccache.exe --stop-server - - - name: Disk space information - id: disk_space_info_post_build - shell: powershell - run: | - Get-CIMInstance Win32_LogicalDisk - $bindir_size = (Get-ChildItem -Path ${{ steps.build_paths.outputs.BINARY }} -Recurse | Measure-Object -Property Length -Sum).sum / 1MB - echo "Binary dir size: $bindir_size MBs" - - - name: Run the tests - # arm64 tests can't be run on x64 machine - if: matrix.bitness != 'arm64' - working-directory: ${{ steps.build_paths.outputs.BINARY }} - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} - @echo on - ctest --build-nocmake -C Release -V - - - name: Run the install target - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} - DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - run: | - cmake --build . --target install -j 3 - - - name: Create the packages - working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} - shell: cmd - - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} - @echo on - - 7z ^ - a windows_package_data.zip ^ - ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - if %errorlevel% neq 0 exit /b %errorlevel% - - if ${{ matrix.bitness }}==arm64 ( - echo Do not create other packages for arm64 - exit 0 - ) - - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ - -DCPACK_GENERATOR=WIX ^ - -DOSQUERY_PACKAGE_VERSION=${{ steps.osquery_version.outputs.VALUE }} ^ - -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} ^ - -DOSQUERY_BITNESS=${{ matrix.bitness }} ^ - ${{ steps.build_paths.outputs.PACKAGING }} - - if %errorlevel% neq 0 exit /b %errorlevel% - - cmake --build . ^ - --config Release ^ - --target package - - if %errorlevel% neq 0 exit /b %errorlevel% - - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ - -DCPACK_GENERATOR=NuGet ^ - -DOSQUERY_PACKAGE_VERSION=${{ steps.osquery_version.outputs.VALUE }} ^ - -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} ^ - -DOSQUERY_BITNESS=${{ matrix.bitness }} ^ - ${{ steps.build_paths.outputs.PACKAGING }} - - if %errorlevel% neq 0 exit /b %errorlevel% - - cmake --build . ^ - --config Release ^ - --target package - - if %errorlevel% neq 0 exit /b %errorlevel% - - - name: Locate the packages - id: packages - shell: powershell - run: | - echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=$((ls ${{ steps.build_paths.outputs.PACKAGE_BUILD }}\*.zip).FullName)" >> $env:GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_MSI_PATH=$((ls ${{ steps.build_paths.outputs.PACKAGE_BUILD }}\*.msi).FullName)" >> $env:GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_NUPKG_PATH=$((ls ${{ steps.build_paths.outputs.PACKAGE_BUILD }}\*.nupkg).FullName)" >> $env:GITHUB_OUTPUT - - - name: Store the unsigned release package data artifact - uses: actions/upload-artifact@v4.4.0 - with: - name: windows${{ matrix.bitness }}_unsigned_release_package_data - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} - - - name: Store the unsigned release MSI artifact - if: matrix.bitness != 'arm64' - uses: actions/upload-artifact@v4.4.0 - with: - name: windows${{ matrix.bitness }}_unsigned_release_msi - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_MSI_PATH }} - - - name: Store the unsigned release NUPKG artifact - if: matrix.bitness != 'arm64' - uses: actions/upload-artifact@v4.4.0 - with: - name: windows${{ matrix.bitness }}_unsigned_release_nupkg - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_NUPKG_PATH }} - - # Before we terminate this job, delete the build folder. The cache - # actions will require the disk space to create the archives. - - name: Reclaim disk space - shell: powershell - run: | - rm -r -Force ${{ steps.build_paths.outputs.BINARY }} - - # This job is here as a github status check -- it allows us to move the dependency from - # being on all the jobs to this single one. - mergeable: - runs-on: ubuntu-latest - steps: - - run: true - needs: - - build_macos - - build_linux - - build_windows - - build_universal_macos_artifacts - - check_source_code diff --git a/.github/workflows/self_hosted_runners.yml b/.github/workflows/self_hosted_runners.yml deleted file mode 100644 index 7b327b2eaf5..00000000000 --- a/.github/workflows/self_hosted_runners.yml +++ /dev/null @@ -1,456 +0,0 @@ -# -# Copyright (c) 2014-present, The osquery authors -# -# This source code is licensed as defined by the LICENSE file found in the -# root directory of this source tree. -# -# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) -# - -# Due to a limitation in how GitHub Actions works, we can't reference -# jobs in another file inside the `needs` statement. -# -# This configuration file takes care of the AArch64 Linux build -name: build_aarch64 - -on: - # Run this workflow once every day against the master branch - schedule: - - cron: "0 7 * * *" - - # Do not run this on pull requests - push: - branches: - - '**' - - tags: - - '**' - -# Please remember to update values for both x86 and aarch64 workflows. -env: - PACKAGING_REPO: https://github.com/osquery/osquery-packaging - PACKAGING_COMMIT: c089fb2d3d796d976e3b2fbea7ee69a1616b9576 - SUBMODULE_CACHE_VERSION: 3 - -# If the initial code sanity checks are passing, then one job -# per [`platform` * `build_type`] will start, building osquery -# and generating packages that are later attached to the commit -# (or PR) as build artifacts. -jobs: - # This job performs basic source code check, looking for formatting - # issues and missing copyright headers - check_code_style: - runs-on: ubuntu-22.04 - - container: - image: osquery/builder20.04:7e9ee0339 - options: --user 1001 - - steps: - - name: Setup the build paths - shell: bash - id: build_paths - run: | - rel_build_path="workspace/build" - rel_source_path="workspace/src" - - mkdir -p "${rel_build_path}" - mkdir -p "${rel_source_path}" - - echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - # This script makes sure that the copyright headers have been correctly - # placed on all the source code files - - name: Check the copyright headers - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - ./tools/ci/scripts/check_copyright_headers.py - - - name: Configure the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DOSQUERY_ENABLE_FORMAT_ONLY=ON \ - "${{ steps.build_paths.outputs.SOURCE }}" - - # Formatting is tested against the clang-format binary we ship - # with the osquery-toolchain, so this job is only performed once on - # a Linux machine. - - name: Check code formatting - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: - cmake --build . --target format_check - - - - - # NOTE: Here we manually create two different steps and outputs because there's no easy way - # to dynamically access variables per matrix job. Having outputs with different names - # lets us easily select the correct one later. - - # Starts self-hosted runner. - start_ec2_runners: - needs: check_code_style - runs-on: ubuntu-latest - - outputs: - label-Release: ${{ steps.start_ec2_runner-Release.outputs.label }} - ec2-instance-id-Release: ${{ steps.start_ec2_runner-Release.outputs.ec2-instance-id }} - label-RelWithDebInfo: ${{ steps.start_ec2_runner-RelWithDebInfo.outputs.label }} - ec2-instance-id-RelWithDebInfo: ${{ steps.start_ec2_runner-RelWithDebInfo.outputs.ec2-instance-id }} - - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} - - - name: Start aarch64 EC2 runner for a Release build - id: start_ec2_runner-Release - uses: osquery/ec2-github-runner@v2.3.6 - with: - mode: start - github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} - ec2-image-id: ami-063d99d9ce1bebbe8 - ec2-instance-type: c6g.4xlarge - subnet-id: subnet-06390365e72579736 - security-group-id: sg-0757a3162c999331b - - - name: Start aarch64 EC2 runner for a RelWithDebInfo build - id: start_ec2_runner-RelWithDebInfo - uses: osquery/ec2-github-runner@v2.3.6 - with: - mode: start - github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} - ec2-image-id: ami-063d99d9ce1bebbe8 - ec2-instance-type: c6g.4xlarge - subnet-id: subnet-06390365e72579736 - security-group-id: sg-0757a3162c999331b - - - - - # Stops self-hosted runners. - stop_ec2_runners: - needs: - - start_ec2_runners - - build_linux - - if: always() - - runs-on: ubuntu-latest - - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} - - - name: Stop aarch64 EC2 runner for Release mode build - uses: osquery/ec2-github-runner@v2.3.6 - with: - mode: stop - github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start_ec2_runners.outputs.label-Release }} - ec2-instance-id: ${{ needs.start_ec2_runners.outputs.ec2-instance-id-Release }} - - - name: Stop aarch64 EC2 runner for RelWithDebInfo mode build - uses: osquery/ec2-github-runner@v2.3.6 - with: - mode: stop - github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start_ec2_runners.outputs.label-RelWithDebInfo }} - ec2-instance-id: ${{ needs.start_ec2_runners.outputs.ec2-instance-id-RelWithDebInfo }} - - - - - # The Linux build will only start once we know that the code - # has been properly formatted - build_linux: - needs: start_ec2_runners - - runs-on: ${{ matrix.os }} - - container: - image: osquery/builder20.04:7e9ee0339 - options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock --pid=host --user root - - strategy: - matrix: - include: - - os: ${{ needs.start_ec2_runners.outputs.label-Release }} - build_type: Release - cache_key: ubuntu-20.04_aarch64 - - - os: ${{ needs.start_ec2_runners.outputs.label-RelWithDebInfo }} - build_type: RelWithDebInfo - cache_key: ubuntu-20.04_aarch64 - - steps: - # Due to how the RPM packaging tools work, we have to adhere to some - # character count requirements in the build path vs source path. - # - # Failing to do so, will break the debuginfo RPM package. - - name: Setup the build paths - id: build_paths - run: | - # The group for the docker socket doesn't have gid docker - sudo chown root:docker /var/run/docker.sock - - rel_build_path="workspace/usr/src/debug/osquery/build" - rel_src_path="workspace/padding-required-by-rpm-packages/src" - rel_ccache_path="workspace/ccache" - rel_package_data_path="workspace/package_data" - rel_packaging_path="workspace/osquery-packaging" - rel_package_build_path="workspace/package-build" - - mkdir -p ${rel_build_path} \ - ${rel_src_path} \ - ${rel_ccache_path} \ - ${rel_src_path} \ - ${rel_package_data_path} \ - ${rel_package_build_path} - - echo "SOURCE=$(realpath ${rel_src_path})" >> $GITHUB_OUTPUT - echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT - echo "CCACHE=$(realpath ${rel_ccache_path})" >> $GITHUB_OUTPUT - echo "PACKAGING=$(realpath ${rel_packaging_path})" >> $GITHUB_OUTPUT - echo "PACKAGE_DATA=$(realpath ${rel_package_data_path})" >> $GITHUB_OUTPUT - echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT - echo "PACKAGE_BUILD=$(realpath ${rel_package_build_path})" >> $GITHUB_OUTPUT - - - name: Clone the osquery repository - uses: actions/checkout@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }} - fetch-depth: 0 - - - name: Select the build job count - shell: bash - id: build_job_count - run: | - echo "VALUE=$(($(nproc) + 1))" >> $GITHUB_OUTPUT - - - name: Select the build options for the tests - shell: bash - id: tests_build_settings - run: | - if [[ "${{ matrix.build_type }}" == "RelWithDebInfo" ]] ; then - echo "VALUE=OFF" >> $GITHUB_OUTPUT - else - echo "VALUE=ON" >> $GITHUB_OUTPUT - fi - - # try to gather some info about build space - - name: Disk Space Information - shell: bash - id: disk_space_info - run: | - df -h - - # We don't have enough space on the worker to actually generate all - # the debug symbols (osquery + dependencies), so we have a flag to - # disable them when running a Debug build - - name: Select the debug symbols options - shell: bash - id: debug_symbols_settings - run: | - if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then - echo "VALUE=ON" >> $GITHUB_OUTPUT - else - echo "VALUE=OFF" >> $GITHUB_OUTPUT - fi - - - name: Clone the osquery-packaging repository - run: | - git clone ${{ env.PACKAGING_REPO }} \ - ${{ steps.build_paths.outputs.PACKAGING }} - cd ${{ steps.build_paths.outputs.PACKAGING }} - git checkout ${{ env.PACKAGING_COMMIT }} - - - name: Update the cache (ccache) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.CCACHE }} - - key: | - ccache_${{ matrix.cache_key }}_${{ matrix.build_type }}_${{ github.sha }} - - restore-keys: | - ccache_${{ matrix.cache_key }}_${{ matrix.build_type }} - - - name: Update the cache (git submodules) - uses: actions/cache@v4 - with: - path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules - - key: | - gitmodules_${{ matrix.cache_key }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} - - restore-keys: | - gitmodules_${{ matrix.cache_key }}_${{env.SUBMODULE_CACHE_VERSION}} - - - name: Update the git submodules - working-directory: ${{ steps.build_paths.outputs.SOURCE }} - run: | - git submodule sync --recursive - - - name: Configure the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake -G "Unix Makefiles" \ - -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ - -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ - -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ - -DOSQUERY_BUILD_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ - -DOSQUERY_BUILD_ROOT_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ - "${{ steps.build_paths.outputs.SOURCE }}" - - - name: Build the project - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - - run: | - cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} - - - name: Disk Space Information - shell: bash - id: disk_space_info_post_build - run: | - df -h - du -sh ${{ steps.build_paths.outputs.BINARY }} - - - name: Run the tests as normal user - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - sudo chown -R runner:runner . - sudo -u runner ctest --build-nocmake -LE "root-required" -V - - - name: Run the tests as root user - working-directory: ${{ steps.build_paths.outputs.BINARY }} - run: | - sudo -u root ctest --build-nocmake -L "root-required" -V - - - name: Run the install target - if: matrix.build_type == 'RelWithDebInfo' - working-directory: ${{ steps.build_paths.outputs.BINARY }} - - env: - CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} - DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - run: | - cmake \ - --build . \ - --target install \ - -j ${{ steps.build_job_count.outputs.VALUE }} - - # Do some targeted cleanups to get additional free space before installing dependencies, - # we are unfortunately out of space otherwise. - find . -name "*.a" -exec rm {} \; - find . -name "*.o" -exec rm {} \; - - - name: Create the packages - if: matrix.build_type == 'RelWithDebInfo' - working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} - - shell: bash - - run: | - osquery_version=$(cd ${{ steps.build_paths.outputs.SOURCE }} && git describe --tags --always ) - - tar pcvzf package_data.tar.gz \ - ${{ steps.build_paths.outputs.PACKAGE_DATA }} - - package_format_list=( "DEB" "RPM" "TGZ" ) - - for package_format in "${package_format_list[@]}" ; do - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCPACK_GENERATOR=${package_format} \ - -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ - -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} \ - -DOSQUERY_SOURCE_DIRECTORY_LIST="${{ steps.build_paths.outputs.SOURCE }};${{ steps.build_paths.outputs.BINARY }}" \ - ${{ steps.build_paths.outputs.PACKAGING }} - - cmake --build . \ - --target package - done - - - name: Locate the packages - if: matrix.build_type == 'RelWithDebInfo' - id: packages - shell: bash - run: | - echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.deb)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_DEBUG_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.ddeb)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-?.*.rpm)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_DEBUG_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-debuginfo-*.rpm)" >> $GITHUB_OUTPUT - echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*linux_aarch64.tar.gz)" >> $GITHUB_OUTPUT - - - name: Store the unsigned release package data artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_release_package_data_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} - - - name: Store the unsigned release DEB artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_release_deb_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_DEB_PATH }} - - - name: Store the unsigned debug DEB artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_debug_deb_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_DEB_PATH }} - - - name: Store the unsigned release RPM artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_release_rpm_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_RPM_PATH }} - - - name: Store the unsigned debug RPM artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_debug_rpm_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_RPM_PATH }} - - - name: Store the unsigned release TGZ artifact - if: matrix.build_type == 'RelWithDebInfo' - uses: actions/upload-artifact@v4 - with: - name: linux_unsigned_release_tgz_aarch64 - path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} - - # Before we terminate this job, delete the build folder. The cache - # actions will require the disk space to create the archives. - - name: Reclaim disk space - run: | - rm -rf ${{ steps.build_paths.outputs.BINARY }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 17d61942baa..1af548840d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,12 @@ function(main) (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")) message(STATUS "Warning: the selected C or C++ compiler is not clang/clang++. Compilation may fail") endif() + + # Fix for macOS zlib fdopen macro conflict + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_UNISTD_H=1 -DHAVE_FDOPEN=1") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_UNISTD_H=1 -DHAVE_FDOPEN=1") + add_definitions(-D_DARWIN_C_SOURCE=1) + elseif(NOT DEFINED PLATFORM_WINDOWS) if(NOT "${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") From bd015bad8ed73d8ecae53e787f16372d216e36aa Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Fri, 26 Sep 2025 15:07:21 +0300 Subject: [PATCH 03/16] patch zlib src --- libraries/cmake/source/zlib/src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/cmake/source/zlib/src b/libraries/cmake/source/zlib/src index 51b7f2abdad..09155eaa2f9 160000 --- a/libraries/cmake/source/zlib/src +++ b/libraries/cmake/source/zlib/src @@ -1 +1 @@ -Subproject commit 51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf +Subproject commit 09155eaa2f9270dc4ed1fa13e2b4b2613e6e4851 From 66698633d3457acc3ce0af7ccea50c8fe12676c0 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Fri, 26 Sep 2025 15:10:30 +0300 Subject: [PATCH 04/16] fixes on the way --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0864234d3a8..20c39f03484 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -130,7 +130,6 @@ jobs: cmake \ -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ - -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_OSX_SYSROOT=$SDKROOT \ From a8262b8a5e8c7e2074b11e930dc62559fcfea251 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Fri, 26 Sep 2025 15:16:31 +0300 Subject: [PATCH 05/16] moar fixes on the way --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20c39f03484..f4aa4612c92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -130,12 +130,14 @@ jobs: cmake \ -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ + -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_OSX_SYSROOT=$SDKROOT \ -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DOSQUERY_VERSION="5.9.1" \ .. echo "CMake configuration completed" From 7a738c668a3153cc7b2385807d405f5674b0c92a Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Fri, 26 Sep 2025 15:25:02 +0300 Subject: [PATCH 06/16] even moar fixes on the way --- .github/workflows/build.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4aa4612c92..6ca73a26b95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,17 +90,6 @@ jobs: run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck shell: bash - - name: Setup ccache - run: | - echo "Setting up ccache..." - ccache --version - ccache --zero-stats - ccache --show-stats - - # Configure ccache - ccache --set-config max_size=2G - ccache --set-config compression=true - - name: Cache CMake Build for faster rebuilds uses: actions/cache@v4 with: @@ -135,8 +124,6 @@ jobs: -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_OSX_SYSROOT=$SDKROOT \ -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DOSQUERY_VERSION="5.9.1" \ .. From e44cdc4b4d3c436760c8d240a77f0de63e1b5530 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Fri, 26 Sep 2025 17:15:38 +0300 Subject: [PATCH 07/16] force update zlib --- .github/workflows/build.yml | 8 ++++++++ libraries/cmake/source/zlib/src | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ca73a26b95..0df7fd20631 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,6 +61,14 @@ jobs: with: fetch-depth: 1 +# - name: Fix zlib fdopen macro conflict on macOS +# run: | +# # TEMPORARY Comment out the problematic fdopen macro definition on macOS +# if [[ "$RUNNER_OS" == "macOS" ]]; then +# sed -i '' 's/# define fdopen(fd,mode) NULL \/\* No fdopen() \*\/\/\/# define fdopen(fd,mode) NULL \/\* No fdopen() \*\/ \/\* Disabled for macOS \*\//' \ +# libraries/cmake/source/zlib/src/zutil.h +# fi + - name: Install Xcode Command Line Tools shell: bash run: | diff --git a/libraries/cmake/source/zlib/src b/libraries/cmake/source/zlib/src index 09155eaa2f9..5a82f71ed1d 160000 --- a/libraries/cmake/source/zlib/src +++ b/libraries/cmake/source/zlib/src @@ -1 +1 @@ -Subproject commit 09155eaa2f9270dc4ed1fa13e2b4b2613e6e4851 +Subproject commit 5a82f71ed1dfc0bec044d9702463dbdf84ea3b71 From b8db8faeca9eb2bd2900f26cf036534aec033919 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Mon, 29 Sep 2025 19:56:21 +0300 Subject: [PATCH 08/16] test osquery windows x64 build --- .github/workflows/build.yml | 334 +++++++++++++++++++++--------------- 1 file changed, 198 insertions(+), 136 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0df7fd20631..6ba0c7f799d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,142 +32,142 @@ env: # ============================================================================= jobs: - build_macos: - name: "Build C Client (${{ matrix.os }})" - runs-on: ${{ matrix.os }} - if: | - github.event_name == 'push' || github.event_name == 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - fail-fast: false - matrix: - include: - - name: "macOS x86-64" - os: macos-13 - os_arch: x86-64 - cmake_arch: x86_64 - artifact_name: osquery-macos-x64 - - name: "macOS ARM64" - os: macos-latest - os_arch: arm64 - cmake_arch: arm64 - artifact_name: osquery-macos-arm64 - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 1 - -# - name: Fix zlib fdopen macro conflict on macOS -# run: | -# # TEMPORARY Comment out the problematic fdopen macro definition on macOS -# if [[ "$RUNNER_OS" == "macOS" ]]; then -# sed -i '' 's/# define fdopen(fd,mode) NULL \/\* No fdopen() \*\/\/\/# define fdopen(fd,mode) NULL \/\* No fdopen() \*\/ \/\* Disabled for macOS \*\//' \ -# libraries/cmake/source/zlib/src/zutil.h -# fi - - - name: Install Xcode Command Line Tools - shell: bash - run: | - # Install Xcode if not already installed - if ! xcode-select -p >/dev/null 2>&1; then - echo "Installing Xcode Command Line Tools..." - xcode-select --install - # Wait for installation to complete - while ! xcode-select -p >/dev/null 2>&1; do - echo "Waiting for Xcode Command Line Tools installation..." - sleep 10 - done - fi - - - name: Cache Homebrew - uses: actions/cache@v4 - with: - path: | - ~/Library/Caches/Homebrew - /usr/local/Homebrew - /opt/homebrew - key: ${{ runner.os }}-${{ runner.arch }}-homebrew-${{ hashFiles('.github/workflows/*.yml') }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-homebrew- - - - name: Install macOS Dependencies - run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck - shell: bash - - - name: Cache CMake Build for faster rebuilds - uses: actions/cache@v4 - with: - path: | - build/ - ~/.ccache - key: ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} - ${{ runner.os }}-${{ matrix.os_arch }}-cmake- - - - name: Setup Build Environment - run: | - echo "Setting up build environment for ${{ matrix.os_arch }}..." - export SDKROOT=$(xcrun --show-sdk-path) - export CFLAGS="-isysroot $SDKROOT" - export CXXFLAGS="-isysroot $SDKROOT" - - echo "SDKROOT=$SDKROOT" >> $GITHUB_ENV - echo "CFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV - echo "CXXFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV - - - name: Configure CMake - run: | - echo "Creating build directory and configuring..." - mkdir -p build && cd build - - cmake \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ - -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_OSX_SYSROOT=$SDKROOT \ - -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ - -DOSQUERY_VERSION="5.9.1" \ - .. - - echo "CMake configuration completed" - - # Build the project - - name: Build Project - run: | - echo "Starting build for ${{ matrix.os_arch }}..." - cd build - - CPU_COUNT=$(sysctl -n hw.ncpu) - echo "Building with $CPU_COUNT parallel jobs" - time cmake --build . -j $CPU_COUNT - - echo "Build completed successfully" - -# - name: Build OSQuery for macOS ${{ matrix.os_arch }} -# run: | -# echo "Building OSQuery for macOS ${{ matrix.os_arch }}" -# make macos ARCHID=${{ matrix.archid }} - - - name: Prepare Artifacts - run: | - mkdir -p artifacts - cp build/osquery/osqueryd "artifacts/${{ matrix.artifact_name }}" - - - name: Upload client artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.artifact_name }} - path: | - artifacts/ - if-no-files-found: warn - retention-days: 30 - compression-level: 9 +## build_macos: +## name: "Build C Client (${{ matrix.os }})" +## runs-on: ${{ matrix.os }} +## if: | +## github.event_name == 'push' || github.event_name == 'workflow_dispatch' +## defaults: +## run: +## shell: bash +## strategy: +## fail-fast: false +## matrix: +## include: +## - name: "macOS x86-64" +## os: macos-13 +## os_arch: x86-64 +## cmake_arch: x86_64 +## artifact_name: osquery-macos-x64 +## - name: "macOS ARM64" +## os: macos-latest +## os_arch: arm64 +## cmake_arch: arm64 +## artifact_name: osquery-macos-arm64 +## +## steps: +## - name: Checkout +## uses: actions/checkout@v4 +## with: +## fetch-depth: 1 +## +### - name: Fix zlib fdopen macro conflict on macOS +### run: | +### # TEMPORARY Comment out the problematic fdopen macro definition on macOS +### if [[ "$RUNNER_OS" == "macOS" ]]; then +### sed -i '' 's/# define fdopen(fd,mode) NULL \/\* No fdopen() \*\/\/\/# define fdopen(fd,mode) NULL \/\* No fdopen() \*\/ \/\* Disabled for macOS \*\//' \ +### libraries/cmake/source/zlib/src/zutil.h +### fi +## +## - name: Install Xcode Command Line Tools +## shell: bash +## run: | +## # Install Xcode if not already installed +## if ! xcode-select -p >/dev/null 2>&1; then +## echo "Installing Xcode Command Line Tools..." +## xcode-select --install +## # Wait for installation to complete +## while ! xcode-select -p >/dev/null 2>&1; do +## echo "Waiting for Xcode Command Line Tools installation..." +## sleep 10 +## done +## fi +## +## - name: Cache Homebrew +## uses: actions/cache@v4 +## with: +## path: | +## ~/Library/Caches/Homebrew +## /usr/local/Homebrew +## /opt/homebrew +## key: ${{ runner.os }}-${{ runner.arch }}-homebrew-${{ hashFiles('.github/workflows/*.yml') }} +## restore-keys: | +## ${{ runner.os }}-${{ runner.arch }}-homebrew- +## +## - name: Install macOS Dependencies +## run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck +## shell: bash +## +## - name: Cache CMake Build for faster rebuilds +## uses: actions/cache@v4 +## with: +## path: | +## build/ +## ~/.ccache +## key: ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }}-${{ github.sha }} +## restore-keys: | +## ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} +## ${{ runner.os }}-${{ matrix.os_arch }}-cmake- +## +## - name: Setup Build Environment +## run: | +## echo "Setting up build environment for ${{ matrix.os_arch }}..." +## export SDKROOT=$(xcrun --show-sdk-path) +## export CFLAGS="-isysroot $SDKROOT" +## export CXXFLAGS="-isysroot $SDKROOT" +## +## echo "SDKROOT=$SDKROOT" >> $GITHUB_ENV +## echo "CFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV +## echo "CXXFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV +## +## - name: Configure CMake +## run: | +## echo "Creating build directory and configuring..." +## mkdir -p build && cd build +## +## cmake \ +## -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ +## -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ +## -DCMAKE_C_COMPILER=clang \ +## -DCMAKE_CXX_COMPILER=clang++ \ +## -DCMAKE_OSX_SYSROOT=$SDKROOT \ +## -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ +## -DOSQUERY_VERSION="5.9.1" \ +## .. +## +## echo "CMake configuration completed" +## +## # Build the project +## - name: Build Project +## run: | +## echo "Starting build for ${{ matrix.os_arch }}..." +## cd build +## +## CPU_COUNT=$(sysctl -n hw.ncpu) +## echo "Building with $CPU_COUNT parallel jobs" +## time cmake --build . -j $CPU_COUNT +## +## echo "Build completed successfully" +## +### - name: Build OSQuery for macOS ${{ matrix.os_arch }} +### run: | +### echo "Building OSQuery for macOS ${{ matrix.os_arch }}" +### make macos ARCHID=${{ matrix.archid }} +## +## - name: Prepare Artifacts +## run: | +## mkdir -p artifacts +## cp build/osquery/osqueryd "artifacts/${{ matrix.artifact_name }}" +## +## - name: Upload client artifact +## uses: actions/upload-artifact@v4 +## with: +## name: ${{ matrix.artifact_name }} +## path: | +## artifacts/ +## if-no-files-found: warn +## retention-days: 30 +## compression-level: 9 create_universal_macos: name: "Create Universal macOS Binary" @@ -229,4 +229,66 @@ jobs: name: osquery-macos-universal path: artifacts/ retention-days: 30 + compression-level: 9 + + build_windows: + name: "Build C Client for (${{ matrix.os }})" + runs-on: ${{ matrix.os }} + if: | + github.event_name == 'push' || github.event_name == 'workflow_dispatch' + defaults: + run: + shell: cmd + strategy: + fail-fast: false + matrix: + include: + - name: "Windows x64" + os: windows-latest + os_arch: x64 + artifact_name: osquery-windows-x64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Setup MSBuild + uses: microsoft/setup-msbuild@v2 + + - name: Setup Visual Studio Build Tools + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.os_arch }} + + - name: Configure CMake for ${{ matrix.os }} + run: | + mkdir build + cd build + cmake -G "Visual Studio 17 2022" -A ${{ matrix.os_arch }} .. + + - name: Build OSQuery for Windows ${{ matrix.os_arch }} with MSBuild + run: | + cd build + cmake --build . --config RelWithDebInfo -j10 + + - name: Prepare Windows Artifacts + if: runner.os == 'Windows' + shell: bash + run: | + mkdir -p artifacts + find ./ type f -name "osqueryd*" + cp build/osquery/osqueryd* artifacts/ + # List what we're uploading + ls -la artifacts/ + + - name: Upload client artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: | + artifacts/ + if-no-files-found: warn + retention-days: 30 compression-level: 9 \ No newline at end of file From 329772d1d39b0af2d77315a1e5ac7e758c3cc26a Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Mon, 29 Sep 2025 19:58:27 +0300 Subject: [PATCH 09/16] myte macOS jobs to test osquery windows x64 build --- .github/workflows/build.yml | 124 ++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ba0c7f799d..25151cd3715 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -168,68 +168,68 @@ jobs: ## if-no-files-found: warn ## retention-days: 30 ## compression-level: 9 - - create_universal_macos: - name: "Create Universal macOS Binary" - needs: [build_macos] - runs-on: macos-latest - if: | - (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && - !failure() && !cancelled() - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Download macOS x86-64 Binary - uses: actions/download-artifact@v4 - with: - name: osquery-macos-x64 - path: x64-artifacts/ - - - name: Download macOS ARM64 Binary - uses: actions/download-artifact@v4 - with: - name: osquery-macos-arm64 - path: arm64-artifacts/ - - - name: Create Universal Binary - run: | - echo "Creating universal macOS binary..." - X64_BINARY="x64-artifacts/osquery-macos-x64" - ARM64_BINARY="arm64-artifacts/osquery-macos-arm64" - - echo "Creating universal binary with lipo..." - mkdir -p artifacts - lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/osquery-macos-universal - - # Verify universal binary - echo "Universal binary info:" - lipo -info ./artifacts/osquery-macos-universal - file ./artifacts/osquery-macos-universal - - - name: Sign MacOS package - uses: ./.github/steps/sign-macos-package - with: - apple_certificate_p12: ${{ secrets.APPLE_CERTIFICATE_P12 }} - apple_certificate_password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} - apple_developer_id: ${{ secrets.APPLE_DEVELOPER_ID }} - apple_id_username: ${{ secrets.APPLE_ID_USERNAME }} - apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} - apple_team_id: ${{ secrets.APPLE_TEAM_ID }} - binary_name: osquery-macos-universal - app_name: OSQuery - - - - name: Upload Universal Binary - uses: actions/upload-artifact@v4 - with: - name: osquery-macos-universal - path: artifacts/ - retention-days: 30 - compression-level: 9 +## +## create_universal_macos: +## name: "Create Universal macOS Binary" +## needs: [build_macos] +## runs-on: macos-latest +## if: | +## (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && +## !failure() && !cancelled() +## +## steps: +## - name: Checkout +## uses: actions/checkout@v4 +## with: +## fetch-depth: 1 +## +## - name: Download macOS x86-64 Binary +## uses: actions/download-artifact@v4 +## with: +## name: osquery-macos-x64 +## path: x64-artifacts/ +## +## - name: Download macOS ARM64 Binary +## uses: actions/download-artifact@v4 +## with: +## name: osquery-macos-arm64 +## path: arm64-artifacts/ +## +## - name: Create Universal Binary +## run: | +## echo "Creating universal macOS binary..." +## X64_BINARY="x64-artifacts/osquery-macos-x64" +## ARM64_BINARY="arm64-artifacts/osquery-macos-arm64" +## +## echo "Creating universal binary with lipo..." +## mkdir -p artifacts +## lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/osquery-macos-universal +## +## # Verify universal binary +## echo "Universal binary info:" +## lipo -info ./artifacts/osquery-macos-universal +## file ./artifacts/osquery-macos-universal +## +## - name: Sign MacOS package +## uses: ./.github/steps/sign-macos-package +## with: +## apple_certificate_p12: ${{ secrets.APPLE_CERTIFICATE_P12 }} +## apple_certificate_password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} +## apple_developer_id: ${{ secrets.APPLE_DEVELOPER_ID }} +## apple_id_username: ${{ secrets.APPLE_ID_USERNAME }} +## apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} +## apple_team_id: ${{ secrets.APPLE_TEAM_ID }} +## binary_name: osquery-macos-universal +## app_name: OSQuery +## +## +## - name: Upload Universal Binary +## uses: actions/upload-artifact@v4 +## with: +## name: osquery-macos-universal +## path: artifacts/ +## retention-days: 30 +## compression-level: 9 build_windows: name: "Build C Client for (${{ matrix.os }})" From 2a7a216b7fd62be8c5796484a98d481140421caf Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Mon, 29 Sep 2025 20:02:57 +0300 Subject: [PATCH 10/16] add version to test osquery windows x64 build --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25151cd3715..fc05d6914ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -232,7 +232,7 @@ jobs: ## compression-level: 9 build_windows: - name: "Build C Client for (${{ matrix.os }})" + name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" runs-on: ${{ matrix.os }} if: | github.event_name == 'push' || github.event_name == 'workflow_dispatch' @@ -266,7 +266,7 @@ jobs: run: | mkdir build cd build - cmake -G "Visual Studio 17 2022" -A ${{ matrix.os_arch }} .. + cmake -G "Visual Studio 17 2022" -A ${{ matrix.os_arch }} -DOSQUERY_VERSION="5.9.1" .. - name: Build OSQuery for Windows ${{ matrix.os_arch }} with MSBuild run: | From d1e78227781dd637076ff50d534d677810d671ee Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Mon, 29 Sep 2025 23:44:13 +0300 Subject: [PATCH 11/16] fix upload of osquery windows x64 artifact --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc05d6914ed..7126023057d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -283,12 +283,12 @@ jobs: # List what we're uploading ls -la artifacts/ - - name: Upload client artifact + - name: Upload Windows OSQuery artifact + if: runner.os == 'Windows' uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact_name }} - path: | - artifacts/ + path: "build/osquery/RelWithDebInfo/osqueryd${{ matrix.os == 'windows-latest' && '.exe' || '' }}" if-no-files-found: warn retention-days: 30 - compression-level: 9 \ No newline at end of file + compression-level: 9 From db649088944cfae30844c20d99d1b04bb2695963 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Tue, 30 Sep 2025 01:08:45 +0300 Subject: [PATCH 12/16] pre-PR final check --- .github/steps/sign-macos-package/action.yml | 100 +---- .github/workflows/build.yml | 384 +++++++++----------- 2 files changed, 194 insertions(+), 290 deletions(-) diff --git a/.github/steps/sign-macos-package/action.yml b/.github/steps/sign-macos-package/action.yml index b731153288b..8c8318c22e7 100644 --- a/.github/steps/sign-macos-package/action.yml +++ b/.github/steps/sign-macos-package/action.yml @@ -1,5 +1,5 @@ name: Sign MacOS package -description: Sign built MacOS package and Notarize it +description: Sign built universal MacOS package and Notarize it inputs: apple_certificate_p12: description: 'Base64 encoded Apple Developer certificate (.p12)' @@ -20,10 +20,7 @@ inputs: description: 'Apple Developer ID' required: true binary_name: - description: 'Name of built MacOS binary (e.g. "rmmagent-mac-universal")' - required: true - app_name: - description: 'Name of MacOS App targets to add (e.g. "RMMAgent")' + description: 'Name of built MacOS binary (e.g. "rmmagent")' required: true runs: @@ -33,14 +30,10 @@ runs: shell: bash run: | BINARY_NAME="${{ inputs.binary_name }}" - APP_NAME="${{ inputs.app_name }}" - APP_BUNDLE="${APP_NAME}.app" BINARY_PATH="./artifacts/$BINARY_NAME" DEVELOPER_ID="${{ inputs.apple_developer_id }}" echo "BINARY_NAME=$BINARY_NAME" >> $GITHUB_ENV - echo "APP_NAME=$APP_NAME" >> $GITHUB_ENV - echo "APP_BUNDLE=$APP_BUNDLE" >> $GITHUB_ENV echo "BINARY_PATH=$BINARY_PATH" >> $GITHUB_ENV echo "DEVELOPER_ID=$DEVELOPER_ID" >> $GITHUB_ENV @@ -65,100 +58,43 @@ runs: rm "$CERTIFICATE_PATH" echo "KEYCHAIN_PATH=$KEYCHAIN_PATH" >> $GITHUB_ENV - - name: Code Sign Binary + - name: Code Sign Standalone binary shell: bash run: | - to_lower() { - # Lowercase regardless of input - printf '%s' "$*" | tr '[:upper:]' '[:lower:]' - } - APP_NAME_LOWER="$(to_lower $APP_NAME)" - BUNDLE_ID="cx.flamingo.$APP_NAME_LOWER" - - # Clean up any existing bundle and create new app bundle structure - rm -rf "$APP_BUNDLE" "${APP_BUNDLE}.zip" - mkdir -p "${APP_BUNDLE}/Contents/MacOS" - mkdir -p "${APP_BUNDLE}/Contents/Resources" - cp "$BINARY_PATH" "${APP_BUNDLE}/Contents/MacOS/${APP_NAME}" - chmod +x "${APP_BUNDLE}/Contents/MacOS/${APP_NAME}" - - # Create Info.plist and entitlements - cat > "${APP_BUNDLE}/Contents/Info.plist" << EOF - - - - - CFBundleExecutable - ${APP_NAME} - CFBundleIdentifier - ${BUNDLE_ID} - CFBundleName - ${APP_NAME} - CFBundleDisplayName - ${APP_NAME} - CFBundleVersion - 1.0.0 - CFBundleShortVersionString - 1.0.0 - CFBundleInfoDictionaryVersion - 6.0 - CFBundlePackageType - APPL - CFBundleSignature - ???? - LSMinimumSystemVersion - 10.15 - NSHighResolutionCapable - - LSUIElement - - LSBackgroundOnly - - - - EOF - - cat > "entitlements.plist" << EOF - - - - - com.apple.security.cs.disable-library-validation - com.apple.developer.endpoint-security.client - - - EOF + chmod +x "$BINARY_PATH" # Sign the binary and verify - codesign --sign "$DEVELOPER_ID" --timestamp --options runtime --entitlements "entitlements.plist" --deep --force "$APP_BUNDLE" + codesign --sign "$DEVELOPER_ID" --timestamp --options runtime --deep --force "$BINARY_PATH" # Verify signature - codesign --verify --deep --strict --verbose=2 "$APP_BUNDLE" + codesign --verify --deep --strict --verbose=2 "$BINARY_PATH" + codesign -dv --verbose=4 "$BINARY_PATH" echo "Binary signed successfully" - name: Create archive and submit for Notarization shell: bash run: | - zip -r "${APP_BUNDLE}.zip" "$APP_BUNDLE" + zip "${BINARY_NAME}.zip" -j "$BINARY_PATH" - xcrun notarytool submit "${APP_BUNDLE}.zip" \ + echo "Submitting binary for notarization..." + xcrun notarytool submit "${BINARY_NAME}.zip" \ --apple-id "${{ inputs.apple_id_username }}" \ --password "${{ inputs.apple_id_password }}" \ --team-id "${{ inputs.apple_team_id }}" \ --wait \ --timeout 30m - - name: Staple Notarization Ticket and Verify Final Binary + # Cleanup zip + rm "${BINARY_NAME}.zip" + echo "Notarization completed successfully" + + - name: Verify Final Binary shell: bash run: | - xcrun stapler staple "$APP_BUNDLE" - xcrun stapler validate "$APP_BUNDLE" - echo "Notarization completed successfully" - # Verify code signature and notarization - codesign --verify --deep --strict --verbose=2 "$APP_BUNDLE" - spctl --assess --type execute --verbose "$APP_BUNDLE" - echo "Binary verification completed" + codesign --verify --deep --strict --verbose=2 "$BINARY_PATH" + spctl --assess --type open --context context:primary-signature --verbose "$BINARY_PATH" || true + echo "Binary is signed, notarized, and ready for distribution" - name: Cleanup Keychain if: always() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7126023057d..036881e318f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,204 +32,182 @@ env: # ============================================================================= jobs: -## build_macos: -## name: "Build C Client (${{ matrix.os }})" -## runs-on: ${{ matrix.os }} -## if: | -## github.event_name == 'push' || github.event_name == 'workflow_dispatch' -## defaults: -## run: -## shell: bash -## strategy: -## fail-fast: false -## matrix: -## include: -## - name: "macOS x86-64" -## os: macos-13 -## os_arch: x86-64 -## cmake_arch: x86_64 -## artifact_name: osquery-macos-x64 -## - name: "macOS ARM64" -## os: macos-latest -## os_arch: arm64 -## cmake_arch: arm64 -## artifact_name: osquery-macos-arm64 -## -## steps: -## - name: Checkout -## uses: actions/checkout@v4 -## with: -## fetch-depth: 1 -## -### - name: Fix zlib fdopen macro conflict on macOS -### run: | -### # TEMPORARY Comment out the problematic fdopen macro definition on macOS -### if [[ "$RUNNER_OS" == "macOS" ]]; then -### sed -i '' 's/# define fdopen(fd,mode) NULL \/\* No fdopen() \*\/\/\/# define fdopen(fd,mode) NULL \/\* No fdopen() \*\/ \/\* Disabled for macOS \*\//' \ -### libraries/cmake/source/zlib/src/zutil.h -### fi -## -## - name: Install Xcode Command Line Tools -## shell: bash -## run: | -## # Install Xcode if not already installed -## if ! xcode-select -p >/dev/null 2>&1; then -## echo "Installing Xcode Command Line Tools..." -## xcode-select --install -## # Wait for installation to complete -## while ! xcode-select -p >/dev/null 2>&1; do -## echo "Waiting for Xcode Command Line Tools installation..." -## sleep 10 -## done -## fi -## -## - name: Cache Homebrew -## uses: actions/cache@v4 -## with: -## path: | -## ~/Library/Caches/Homebrew -## /usr/local/Homebrew -## /opt/homebrew -## key: ${{ runner.os }}-${{ runner.arch }}-homebrew-${{ hashFiles('.github/workflows/*.yml') }} -## restore-keys: | -## ${{ runner.os }}-${{ runner.arch }}-homebrew- -## -## - name: Install macOS Dependencies -## run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck -## shell: bash -## -## - name: Cache CMake Build for faster rebuilds -## uses: actions/cache@v4 -## with: -## path: | -## build/ -## ~/.ccache -## key: ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }}-${{ github.sha }} -## restore-keys: | -## ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} -## ${{ runner.os }}-${{ matrix.os_arch }}-cmake- -## -## - name: Setup Build Environment -## run: | -## echo "Setting up build environment for ${{ matrix.os_arch }}..." -## export SDKROOT=$(xcrun --show-sdk-path) -## export CFLAGS="-isysroot $SDKROOT" -## export CXXFLAGS="-isysroot $SDKROOT" -## -## echo "SDKROOT=$SDKROOT" >> $GITHUB_ENV -## echo "CFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV -## echo "CXXFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV -## -## - name: Configure CMake -## run: | -## echo "Creating build directory and configuring..." -## mkdir -p build && cd build -## -## cmake \ -## -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ -## -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ -## -DCMAKE_C_COMPILER=clang \ -## -DCMAKE_CXX_COMPILER=clang++ \ -## -DCMAKE_OSX_SYSROOT=$SDKROOT \ -## -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ -## -DOSQUERY_VERSION="5.9.1" \ -## .. -## -## echo "CMake configuration completed" -## -## # Build the project -## - name: Build Project -## run: | -## echo "Starting build for ${{ matrix.os_arch }}..." -## cd build -## -## CPU_COUNT=$(sysctl -n hw.ncpu) -## echo "Building with $CPU_COUNT parallel jobs" -## time cmake --build . -j $CPU_COUNT -## -## echo "Build completed successfully" -## -### - name: Build OSQuery for macOS ${{ matrix.os_arch }} -### run: | -### echo "Building OSQuery for macOS ${{ matrix.os_arch }}" -### make macos ARCHID=${{ matrix.archid }} -## -## - name: Prepare Artifacts -## run: | -## mkdir -p artifacts -## cp build/osquery/osqueryd "artifacts/${{ matrix.artifact_name }}" -## -## - name: Upload client artifact -## uses: actions/upload-artifact@v4 -## with: -## name: ${{ matrix.artifact_name }} -## path: | -## artifacts/ -## if-no-files-found: warn -## retention-days: 30 -## compression-level: 9 -## -## create_universal_macos: -## name: "Create Universal macOS Binary" -## needs: [build_macos] -## runs-on: macos-latest -## if: | -## (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && -## !failure() && !cancelled() -## -## steps: -## - name: Checkout -## uses: actions/checkout@v4 -## with: -## fetch-depth: 1 -## -## - name: Download macOS x86-64 Binary -## uses: actions/download-artifact@v4 -## with: -## name: osquery-macos-x64 -## path: x64-artifacts/ -## -## - name: Download macOS ARM64 Binary -## uses: actions/download-artifact@v4 -## with: -## name: osquery-macos-arm64 -## path: arm64-artifacts/ -## -## - name: Create Universal Binary -## run: | -## echo "Creating universal macOS binary..." -## X64_BINARY="x64-artifacts/osquery-macos-x64" -## ARM64_BINARY="arm64-artifacts/osquery-macos-arm64" -## -## echo "Creating universal binary with lipo..." -## mkdir -p artifacts -## lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/osquery-macos-universal -## -## # Verify universal binary -## echo "Universal binary info:" -## lipo -info ./artifacts/osquery-macos-universal -## file ./artifacts/osquery-macos-universal -## -## - name: Sign MacOS package -## uses: ./.github/steps/sign-macos-package -## with: -## apple_certificate_p12: ${{ secrets.APPLE_CERTIFICATE_P12 }} -## apple_certificate_password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} -## apple_developer_id: ${{ secrets.APPLE_DEVELOPER_ID }} -## apple_id_username: ${{ secrets.APPLE_ID_USERNAME }} -## apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} -## apple_team_id: ${{ secrets.APPLE_TEAM_ID }} -## binary_name: osquery-macos-universal -## app_name: OSQuery -## -## -## - name: Upload Universal Binary -## uses: actions/upload-artifact@v4 -## with: -## name: osquery-macos-universal -## path: artifacts/ -## retention-days: 30 -## compression-level: 9 + build_macos: + name: "Build C Client (${{ matrix.os }})" + runs-on: ${{ matrix.os }} + if: | + github.event_name == 'push' || github.event_name == 'workflow_dispatch' + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - name: "macOS x86-64" + os: macos-13 + os_arch: x86-64 + cmake_arch: x86_64 + artifact_name: osquery-macos-x64 + - name: "macOS ARM64" + os: macos-latest + os_arch: arm64 + cmake_arch: arm64 + artifact_name: osquery-macos-arm64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install Xcode Command Line Tools + shell: bash + run: | + # Install Xcode if not already installed + if ! xcode-select -p >/dev/null 2>&1; then + echo "Installing Xcode Command Line Tools..." + xcode-select --install + # Wait for installation to complete + while ! xcode-select -p >/dev/null 2>&1; do + echo "Waiting for Xcode Command Line Tools installation..." + sleep 10 + done + fi + + - name: Cache Homebrew + uses: actions/cache@v4 + with: + path: | + ~/Library/Caches/Homebrew + /usr/local/Homebrew + /opt/homebrew + key: ${{ runner.os }}-${{ runner.arch }}-homebrew-${{ hashFiles('.github/workflows/*.yml') }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-homebrew- + + - name: Install macOS Dependencies + run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck + shell: bash + + - name: Cache CMake Build for faster rebuilds + uses: actions/cache@v4 + with: + path: | + build/ + ~/.ccache + key: ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} + ${{ runner.os }}-${{ matrix.os_arch }}-cmake- + + - name: Setup Build Environment + run: | + echo "Setting up build environment for ${{ matrix.os_arch }}..." + export SDKROOT=$(xcrun --show-sdk-path) + export CFLAGS="-isysroot $SDKROOT" + export CXXFLAGS="-isysroot $SDKROOT" + + echo "SDKROOT=$SDKROOT" >> $GITHUB_ENV + echo "CFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + echo "CXXFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + + - name: Configure CMake + run: | + echo "Creating build directory and configuring..." + mkdir -p build && cd build + + cmake \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ + -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_OSX_SYSROOT=$SDKROOT \ + -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ + -DOSQUERY_VERSION="5.9.1" \ + .. + + echo "CMake configuration completed" + + - name: Build Project + run: | + echo "Starting build for ${{ matrix.os_arch }}..." + cd build + + CPU_COUNT=$(sysctl -n hw.ncpu) + echo "Building with $CPU_COUNT parallel jobs" + time cmake --build . -j $CPU_COUNT + + echo "Build completed successfully" + + - name: Upload client artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: build/osquery/osqueryd + if-no-files-found: warn + retention-days: 30 + compression-level: 9 + + create_universal_macos: + name: "Create Universal macOS Binary" + needs: [build_macos] + runs-on: macos-latest + if: | + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && + !failure() && !cancelled() + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Download macOS x86-64 Binary + uses: actions/download-artifact@v4 + with: + name: osquery-macos-x64 + path: x64-artifacts/ + + - name: Download macOS ARM64 Binary + uses: actions/download-artifact@v4 + with: + name: osquery-macos-arm64 + path: arm64-artifacts/ + + - name: Create Universal Binary + run: | + echo "Creating universal macOS binary..." + X64_BINARY="x64-artifacts/osqueryd" + ARM64_BINARY="arm64-artifacts/osqueryd" + + echo "Creating universal binary with lipo..." + mkdir -p artifacts + lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/osqueryd + + # Verify universal binary + echo "Universal binary info:" + lipo -info ./artifacts/osqueryd + file ./artifacts/osqueryd + + - name: Sign MacOS package + uses: ./.github/steps/sign-macos-package + with: + apple_certificate_p12: ${{ secrets.APPLE_CERTIFICATE_P12 }} + apple_certificate_password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + apple_developer_id: ${{ secrets.APPLE_DEVELOPER_ID }} + apple_id_username: ${{ secrets.APPLE_ID_USERNAME }} + apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} + apple_team_id: ${{ secrets.APPLE_TEAM_ID }} + binary_name: osqueryd + + - name: Upload Universal Binary + uses: actions/upload-artifact@v4 + with: + name: osquery-macos-universal + path: artifacts/ + retention-days: 30 + compression-level: 9 build_windows: name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" @@ -273,16 +251,6 @@ jobs: cd build cmake --build . --config RelWithDebInfo -j10 - - name: Prepare Windows Artifacts - if: runner.os == 'Windows' - shell: bash - run: | - mkdir -p artifacts - find ./ type f -name "osqueryd*" - cp build/osquery/osqueryd* artifacts/ - # List what we're uploading - ls -la artifacts/ - - name: Upload Windows OSQuery artifact if: runner.os == 'Windows' uses: actions/upload-artifact@v4 From 824da9715bf89657e0c964c55c6f3504e04a55f9 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Tue, 30 Sep 2025 16:18:58 +0300 Subject: [PATCH 13/16] added tests, pre-PR cleanup --- .github/workflows/build.yml | 21 +++-- .github/workflows/test.yml | 179 ++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 036881e318f..fc53616a6d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ permissions: on: push: - branches: [master, feature/agent-ci] + branches: [master] paths-ignore: - "**/*.md" workflow_dispatch: @@ -26,6 +26,7 @@ env: REPOSITORY: ${{ github.event.repository.name }} CMAKE_OSX_DEPLOYMENT_TARGET: "15.0" CMAKE_BUILD_TYPE: Release + BINARY_NAME: osqueryd # ============================================================================= # JOBS @@ -33,7 +34,7 @@ env: jobs: build_macos: - name: "Build C Client (${{ matrix.os }})" + name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" runs-on: ${{ matrix.os }} if: | github.event_name == 'push' || github.event_name == 'workflow_dispatch' @@ -144,7 +145,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact_name }} - path: build/osquery/osqueryd + path: build/osquery/$BINARY_NAME if-no-files-found: warn retention-days: 30 compression-level: 9 @@ -178,17 +179,17 @@ jobs: - name: Create Universal Binary run: | echo "Creating universal macOS binary..." - X64_BINARY="x64-artifacts/osqueryd" - ARM64_BINARY="arm64-artifacts/osqueryd" + X64_BINARY="x64-artifacts/$BINARY_NAME" + ARM64_BINARY="arm64-artifacts/$BINARY_NAME" echo "Creating universal binary with lipo..." mkdir -p artifacts - lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/osqueryd + lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/$BINARY_NAME # Verify universal binary echo "Universal binary info:" - lipo -info ./artifacts/osqueryd - file ./artifacts/osqueryd + lipo -info ./artifacts/$BINARY_NAME + file ./artifacts/$BINARY_NAME - name: Sign MacOS package uses: ./.github/steps/sign-macos-package @@ -199,7 +200,7 @@ jobs: apple_id_username: ${{ secrets.APPLE_ID_USERNAME }} apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} apple_team_id: ${{ secrets.APPLE_TEAM_ID }} - binary_name: osqueryd + binary_name: $BINARY_NAME - name: Upload Universal Binary uses: actions/upload-artifact@v4 @@ -256,7 +257,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact_name }} - path: "build/osquery/RelWithDebInfo/osqueryd${{ matrix.os == 'windows-latest' && '.exe' || '' }}" + path: "build/osquery/RelWithDebInfo/$BINARY_NAME${{ matrix.os == 'windows-latest' && '.exe' || '' }}" if-no-files-found: warn retention-days: 30 compression-level: 9 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..49f9e592592 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,179 @@ +name: OSQuery Build and Push Latest + +permissions: + contents: write + packages: write + actions: write + pull-requests: write + attestations: write + id-token: write + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: [master] + paths-ignore: + - "**/*.md" + +concurrency: + group: pr-${{ github.head_ref || github.ref }} + cancel-in-progress: false + +env: + REGISTRY: "ghcr.io" + ORGANISATION: ${{ github.repository_owner }} + REPOSITORY: ${{ github.event.repository.name }} + CMAKE_OSX_DEPLOYMENT_TARGET: "15.0" + CMAKE_BUILD_TYPE: Release + BINARY_NAME: osqueryd + +# ============================================================================= +# JOBS +# ============================================================================= + +jobs: + build_macos: + name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" + runs-on: ${{ matrix.os }} + if: github.event_name == 'pull_request' && !github.event.pull_request.draft + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - name: "macOS x86-64" + os: macos-13 + os_arch: x86-64 + cmake_arch: x86_64 + artifact_name: osquery-macos-x64 + - name: "macOS ARM64" + os: macos-latest + os_arch: arm64 + cmake_arch: arm64 + artifact_name: osquery-macos-arm64 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install Xcode Command Line Tools + shell: bash + run: | + # Install Xcode if not already installed + if ! xcode-select -p >/dev/null 2>&1; then + echo "Installing Xcode Command Line Tools..." + xcode-select --install + # Wait for installation to complete + while ! xcode-select -p >/dev/null 2>&1; do + echo "Waiting for Xcode Command Line Tools installation..." + sleep 10 + done + fi + + - name: Cache Homebrew + uses: actions/cache@v4 + with: + path: | + ~/Library/Caches/Homebrew + /usr/local/Homebrew + /opt/homebrew + key: ${{ runner.os }}-${{ runner.arch }}-homebrew-${{ hashFiles('.github/workflows/*.yml') }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-homebrew- + + - name: Install macOS Dependencies + run: brew install ccache git git-lfs cmake python clang-format flex bison cppcheck + shell: bash + + - name: Cache CMake Build for faster rebuilds + uses: actions/cache@v4 + with: + path: | + build/ + ~/.ccache + key: ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ matrix.os_arch }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} + ${{ runner.os }}-${{ matrix.os_arch }}-cmake- + + - name: Setup Build Environment + run: | + echo "Setting up build environment for ${{ matrix.os_arch }}..." + export SDKROOT=$(xcrun --show-sdk-path) + export CFLAGS="-isysroot $SDKROOT" + export CXXFLAGS="-isysroot $SDKROOT" + + echo "SDKROOT=$SDKROOT" >> $GITHUB_ENV + echo "CFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + echo "CXXFLAGS=-isysroot $SDKROOT" >> $GITHUB_ENV + + - name: Configure CMake + run: | + echo "Creating build directory and configuring..." + mkdir -p build && cd build + + cmake \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.CMAKE_OSX_DEPLOYMENT_TARGET }} \ + -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_OSX_SYSROOT=$SDKROOT \ + -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} \ + -DOSQUERY_VERSION="5.9.1" \ + .. + + echo "CMake configuration completed" + + - name: Build Project + run: | + echo "Starting build for ${{ matrix.os_arch }}..." + cd build + + CPU_COUNT=$(sysctl -n hw.ncpu) + echo "Building with $CPU_COUNT parallel jobs" + time cmake --build . -j $CPU_COUNT + + echo "Build completed successfully" + + build_windows: + name: "Build C Client for (${{ matrix.os }} ${{ matrix.os_arch }})" + runs-on: ${{ matrix.os }} + if: github.event_name == 'pull_request' && !github.event.pull_request.draft + defaults: + run: + shell: cmd + strategy: + fail-fast: false + matrix: + include: + - name: "Windows x64" + os: windows-latest + os_arch: x64 + artifact_name: osquery-windows-x64 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Setup MSBuild + uses: microsoft/setup-msbuild@v2 + + - name: Setup Visual Studio Build Tools + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.os_arch }} + + - name: Configure CMake for ${{ matrix.os }} + run: | + mkdir build + cd build + cmake -G "Visual Studio 17 2022" -A ${{ matrix.os_arch }} -DOSQUERY_VERSION="5.9.1" .. + + - name: Build OSQuery for Windows ${{ matrix.os_arch }} with MSBuild + run: | + cd build + cmake --build . --config RelWithDebInfo -j10 From 6b642a6d4570c50fefb0825c13ae35ff568b6db9 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Tue, 30 Sep 2025 16:20:17 +0300 Subject: [PATCH 14/16] correct naming for test workflow --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 49f9e592592..bcff0fac4d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: OSQuery Build and Push Latest +name: OSQuery Integration Tests permissions: contents: write From c7a97de59f85a54ebf58a3ba2d6636a672286a04 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Tue, 30 Sep 2025 16:26:09 +0300 Subject: [PATCH 15/16] add .github to ignored paths for build workflow --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc53616a6d9..0b26f08620e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,6 +13,7 @@ on: branches: [master] paths-ignore: - "**/*.md" + - '.github/**' workflow_dispatch: inputs: tag: From 74b9b39afbaf1d63ffd6f49ffb4ae472957b2a22 Mon Sep 17 00:00:00 2001 From: Ivan Liesnov Date: Tue, 30 Sep 2025 16:46:34 +0300 Subject: [PATCH 16/16] fix env usage --- .github/workflows/build.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0b26f08620e..80ab4893a8c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -146,7 +146,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact_name }} - path: build/osquery/$BINARY_NAME + path: build/osquery/${{ env.BINARY_NAME }} if-no-files-found: warn retention-days: 30 compression-level: 9 @@ -180,17 +180,17 @@ jobs: - name: Create Universal Binary run: | echo "Creating universal macOS binary..." - X64_BINARY="x64-artifacts/$BINARY_NAME" - ARM64_BINARY="arm64-artifacts/$BINARY_NAME" + X64_BINARY="x64-artifacts/${{ env.BINARY_NAME }}" + ARM64_BINARY="arm64-artifacts/${{ env.BINARY_NAME }}" echo "Creating universal binary with lipo..." mkdir -p artifacts - lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/$BINARY_NAME + lipo -create "$X64_BINARY" "$ARM64_BINARY" -output ./artifacts/${{ env.BINARY_NAME }} # Verify universal binary echo "Universal binary info:" - lipo -info ./artifacts/$BINARY_NAME - file ./artifacts/$BINARY_NAME + lipo -info ./artifacts/${{ env.BINARY_NAME }} + file ./artifacts/${{ env.BINARY_NAME }} - name: Sign MacOS package uses: ./.github/steps/sign-macos-package @@ -201,7 +201,7 @@ jobs: apple_id_username: ${{ secrets.APPLE_ID_USERNAME }} apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} apple_team_id: ${{ secrets.APPLE_TEAM_ID }} - binary_name: $BINARY_NAME + binary_name: ${{ env.BINARY_NAME }} - name: Upload Universal Binary uses: actions/upload-artifact@v4 @@ -258,7 +258,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact_name }} - path: "build/osquery/RelWithDebInfo/$BINARY_NAME${{ matrix.os == 'windows-latest' && '.exe' || '' }}" + path: "build/osquery/RelWithDebInfo/${{ env.BINARY_NAME }}${{ matrix.os == 'windows-latest' && '.exe' || '' }}" if-no-files-found: warn retention-days: 30 compression-level: 9