ci.yml

name: CI

on:
  push:
  pull_request:
  schedule:
    # Run daily at 00:00 so we get notified if CI is broken before a pull request
    # is submitted.
    - cron:  '0 0 * * *'

env:
  DOTNET_CLI_TELEMETRY_OPTOUT: true
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
  PUBLISH_RELEASE: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && !github.event.repository.fork }}

permissions:
  contents: read

jobs:

  check-format:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    name: Check format
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
      - name: Setup .NET SDK v10.0.x
        uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
        with:
          dotnet-version: 10.0.x
      - name: Code formating check
        run: |
          dotnet tool restore
          dotnet jb cleanupcode "csharp" "csharp.test" "csharp.benchmark" "csharp.config.benchmarks" --profile="Built-in: Reformat Code" --settings="ParquetSharp.DotSettings" --verbosity=WARN
          files=($(git diff --name-only))
          if [ ${#files[@]} -gt 0 ]
          then
            for file in $files; do echo "::error file=$file::Code format check failed"; done
            exit 1
          fi

  # Build everything on all platorms (thus testing the developer workflow).
  # Upload the native shared libraries as artifacts.
  build-native:
    # Do not run this job for pull requests where both branches are from the same repo.
    # Other jobs will be skipped too, as they depend on this one.
    # This prevents duplicate CI runs for our own pull requests, whilst preserving the ability to
    # run the CI for each branch push to a fork, and for each pull request originating from a fork.
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    strategy:
      matrix:
        include:
        - os: ubuntu-22.04-arm
          arch: arm64
        - os: ubuntu-22.04
          arch: x64
        - os: macos-14
          arch: arm64
        - os: macos-15-intel
          arch: x64
        - os: windows-2022
          arch: x64
        - os: windows-11-arm
          arch: arm64
      fail-fast: false
    name: Build native ${{ matrix.arch }} library (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    steps:
    - name: Checkout
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd

    # Compute vcpkg triplet and root
    - name: Compute vcpkg triplet and root
      id: vcpkg-info
      run: |
        triplet="${{ matrix.arch }}-"
        case ${{ runner.os }} in
          Linux)
            triplet+="linux"
            ;;
          macOS)
            triplet+="osx"
            ;;
          Windows)
            triplet+="windows-static"
            ;;
        esac
        echo "triplet=$triplet" >> $GITHUB_OUTPUT
        echo "root=$VCPKG_INSTALLATION_ROOT" >> $GITHUB_OUTPUT
      shell: bash

    # Ensure vcpkg builtin registry is up-to-date
    - name: Update vcpkg builtin registry
      working-directory: ${{ steps.vcpkg-info.outputs.root }}
      run: |
        git reset --hard
        git pull

    # We may need to re-bootstrap vcpkg after updating
    - name: Bootstrap vcpkg (Unix)
      if: runner.os != 'Windows'
      working-directory: ${{ steps.vcpkg-info.outputs.root }}
      run: |
        ./bootstrap-vcpkg.sh

    - name: Bootstrap vcpkg (Windows)
      if: runner.os == 'Windows'
      working-directory: ${{ steps.vcpkg-info.outputs.root }}
      run: |
        ./bootstrap-vcpkg.bat

    # Setup a CentOS 7 container to build on Linux for backwards compatibility.
    # We are using Docker layer caching to avoid rebuilding on every CI run.
    - name: Set up Docker Buildx
      if: runner.os == 'Linux'
      uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
    - name: Build CI container image
      if: runner.os == 'Linux'
      uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
      with:
        file: .github/ci-build/Dockerfile
        context: .github/ci-build
        build-args: MANYLINUX_ARCH=${{ matrix.arch == 'arm64' && 'aarch64' || 'x86_64' }}
        load: true
        tags: manylinux2014-parquetsharp-ci-build:latest
        cache-from: type=gha,scope=ci-build-${{ matrix.arch }}
        cache-to: type=gha,scope=ci-build-${{ matrix.arch }},mode=max
    - name: Start CI build container
      if: runner.os == 'Linux'
      run: |
        docker run -d --name centos --entrypoint tail --net host \
                   -v $PWD:$PWD -v $VCPKG_INSTALLATION_ROOT:$VCPKG_INSTALLATION_ROOT \
                   manylinux2014-parquetsharp-ci-build:latest -f /dev/null

    # Install vcpkg dependencies
    - name: Install vcpkg build dependencies (macOS)
      if: runner.os == 'macOS'
      run: brew install bison

    # .NET Setup (and also MSBuild for Windows).
    - name: Setup .NET SDK v10.0.x
      uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
      with:
        dotnet-version: 10.0.x
    - name: Setup MSBuild
      if: runner.os == 'Windows'
      uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57

    # Setup vcpkg caching
    - name: Setup vcpkg caching
      uses: G-Research/vcpkg-cache-action@b3c4717d2ae112029977d05c4392c7ad8e84d639

    # Compile ParquetSharp and C++ dependencies (and upload the native library as an artifact).
    - name: Compile native ParquetSharp library (Linux)
      if: runner.os == 'Linux'
      run: |
        docker exec -w $PWD \
          -e GITHUB_ACTIONS -e VCPKG_BINARY_SOURCES -e VCPKG_INSTALLATION_ROOT -e VCPKG_FORCE_SYSTEM_BINARIES=1 \
          centos \
          scl enable devtoolset-10 rh-git227 httpd24 -- \
          sh -c 'PATH=$(uv tool dir --bin):$PATH ./build_unix.sh'
    - name: Compile native ParquetSharp library (macOS)
      if: runner.os == 'macOS'
      run: ./build_unix.sh
    - name: Compile native ParquetSharp library (Windows)
      if: runner.os == 'Windows'
      run: ./build_windows.ps1
    - name: Upload vcpkg arrow logs
      if: success() || failure()
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
      with:
        name: ${{ steps.vcpkg-info.outputs.triplet }}-vcpkg-arrow-logs
        path: ${{ steps.vcpkg-info.outputs.root }}/buildtrees/arrow/*.log
    - name: Build .NET benchmarks & unit tests
      run: |
        dotnet build csharp.benchmark --configuration=Release -p:OSArchitecture=${{ matrix.arch }}
        dotnet build csharp.test --configuration=Release -p:OSArchitecture=${{ matrix.arch }}
        dotnet build fsharp.test --configuration=Release -p:OSArchitecture=${{ matrix.arch }}
    - name: Upload native ParquetSharp library
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
      with:
        name: ${{ steps.vcpkg-info.outputs.triplet }}-native-library
        path: bin

    - name: Stop CentOS container
      if: always() && runner.os == 'Linux'
      run: docker rm -f centos

  # Download all native shared libraries and create the nuget package.
  # Upload nuget package as an artifact.
  build-nuget:
    name: Build NuGet package
    runs-on: ubuntu-latest
    needs: build-native
    outputs:
      version: ${{ steps.get-version.outputs.version }}
    steps:
    - name: Checkout
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
    - name: Download all artifacts
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      with:
        path: artifacts
    - name: Copy native ParquetSharp libraries
      run: |
        mkdir bin
        cp -rv artifacts/*-native-library/* bin/
    - name: Setup .NET SDK v10.0.x
      uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
      with:
        dotnet-version: 10.0.x
    - name: Get version
      id: get-version
      shell: pwsh
      run: |
        $version_prefix=$((Select-Xml -Path ./csharp/ParquetSharp.csproj -XPath '/Project/PropertyGroup/VersionPrefix/text()').node.Value)
        if ( "${env:PUBLISH_RELEASE}" -eq "true") {
          $version_suffix=""
          $version="${version_prefix}"
        } else {
          $version_suffix="sha$(git rev-parse --short HEAD)"
          $version="${version_prefix}-${version_suffix}"
        }
        echo "version=${version}"
        echo "version=${version}" >> $env:GITHUB_OUTPUT
        echo "version_suffix=${version_suffix}" >> $env:GITHUB_OUTPUT
    - name: Build NuGet package
      run: dotnet build csharp --configuration=Release --version-suffix "${{ steps.get-version.outputs.version_suffix }}"
    - name: Upload NuGet artifact
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
      with:
        name: nuget-package
        path: nuget

  # Run .NET unit tests with the nuget package on all platforms and all supported .NET runtimes (thus testing the user workflow).
  test-nuget:
    strategy:
      matrix:
        os:
        - ubuntu-22.04
        - ubuntu-22.04-arm
        - ubuntu-24.04
        - ubuntu-24.04-arm
        - macos-14
        - macos-15
        - macos-15-intel
        - windows-2022
        - windows-2025
        - windows-11-arm
        dotnet: [net8.0, net9.0, net10.0]
        include:
        - os: windows-2022
          dotnet: net472
        - os: windows-2025
          dotnet: net472
        - os: windows-11-arm
          dotnet: net472
      fail-fast: false
    name: Test NuGet package (${{ matrix.dotnet }} on ${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    needs: build-nuget
    steps:
    - name: Checkout
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
    - name: Download NuGet artifact
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      with:
        name: nuget-package
        path: nuget
    - name: Setup .NET SDK v8.0.x
      if: matrix.dotnet == 'net8.0'
      uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
      with:
        dotnet-version: 8.0.x
    - name: Setup .NET SDK v9.0.x
      if: matrix.dotnet == 'net9.0'
      uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
      with:
        dotnet-version: 9.0.x
    - name: Setup .NET SDK v10.0.x
      uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
      with:
        dotnet-version: 10.0.x
    - name: Add local NuGet feed
      run: |
        dotnet new nugetconfig
        dotnet nuget add source -n local $PWD/nuget
    - name: Change test project references to use local NuGet package
      run: |
        dotnet remove csharp.test reference csharp/ParquetSharp.csproj
        dotnet add csharp.test package ParquetSharp -v ${{ needs.build-nuget.outputs.version }}
        dotnet remove fsharp.test reference csharp/ParquetSharp.csproj
        dotnet add fsharp.test package ParquetSharp -v ${{ needs.build-nuget.outputs.version }}
    - name: Build & Run C# unit tests
      run: dotnet test csharp.test --configuration=Release --framework ${{ matrix.dotnet }}
    - name: Build & Run F# unit tests
      run: dotnet test fsharp.test --configuration=Release --framework ${{ matrix.dotnet }}

  smoke-test-distroless:
    # https://github.com/dotnet/dotnet-docker/blob/main/documentation/distroless.md
    strategy:
      matrix:
        include:
        - os: ubuntu-24.04
          arch: x64
        - os: ubuntu-24.04-arm
          arch: arm64
      fail-fast: false
    name: Smoke-test NuGet package (chiseled linux-${{ matrix.arch }})
    runs-on: ${{ matrix.os }}
    needs: build-nuget
    steps:
    - name: Checkout
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
    - name: Download NuGet artifact
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      with:
        name: nuget-package
        path: nuget
    - name: Setup .NET SDK v10.0.x
      uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
      with:
        dotnet-version: 10.0.x
    - name: Add local NuGet feed
      run: |
        dotnet new nugetconfig
        dotnet nuget add source -n local $PWD/nuget
    - name: Add ParquetSharp package reference
      run: dotnet add csharp.smoketest package ParquetSharp -v ${{ needs.build-nuget.outputs.version }}
    - name: Publish self-contained linux-${{ matrix.arch }} smoke test app
      run: |
        dotnet publish csharp.smoketest \
          --configuration Release \
          --framework net10.0 \
          --runtime linux-${{ matrix.arch }} \
          --self-contained true \
          -o publish
    - name: Run smoke test app in chiseled runtime-deps container
      run: |
        docker run --rm \
          -v "$PWD/publish:/app:ro" \
          mcr.microsoft.com/dotnet/runtime-deps:10.0-noble-chiseled \
          /app/ParquetSharp.SmokeTest

  test-benchmarks:
    name: Run benchmarks in check mode
    runs-on: ubuntu-latest
    needs: build-nuget
    steps:
    - name: Checkout
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd

    - name: Download NuGet artifact
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      with:
        name: nuget-package
        path: nuget

    - name: Setup .NET SDK v10.0.x
      uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
      with:
        dotnet-version: 10.0.x

    - name: Add local NuGet feed
      run: |
        dotnet new nugetconfig
        dotnet nuget add source -n local $PWD/nuget

    - name: Change test project references to use local NuGet package
      run: |
        dotnet remove csharp.benchmark reference csharp/ParquetSharp.csproj
        dotnet add csharp.benchmark package ParquetSharp -v ${{ needs.build-nuget.outputs.version }}

    - name: Build & Run C# benchmarks in check mode
      run: dotnet run --project=csharp.benchmark --configuration=Release -- --check --small

  # Virtual job that can be configured as a required check before a PR can be merged.
  # As GitHub considers a check as successful if it is skipped, we need to check its status in
  # another workflow (check-required.yml) and create a check there.
  all-required-checks-done:
    name: All required checks done
    needs:
      - check-format
      - test-nuget
      - test-benchmarks
      - smoke-test-distroless
    runs-on: ubuntu-latest
    steps:
      - run: echo "All required checks done"

  # Create a GitHub release and publish the NuGet packages to nuget.org when a tag is pushed.
  publish-release:
    # This should match env.PUBLISH_RELEASE (which we can't access in an if condition)
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && !github.event.repository.fork
    name: Publish release
    runs-on: ubuntu-latest
    permissions:
      contents: write
    needs: [build-nuget,all-required-checks-done]
    steps:
    - name: Checkout
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
    - name: Check version
      id: check-version
      shell: pwsh
      run: |
        $version = "${{ needs.build-nuget.outputs.version }}"
        $tag = "${{ github.ref }}".SubString(10)
        if (-not ($tag -eq $version)) {
          echo "::error ::There is a mismatch between the project version ($version) and the tag ($tag)"
          exit 1
        }
    - name: Check for unshipped API updates
      shell: pwsh
      run: |
        $version = "${{ needs.build-nuget.outputs.version }}"
        if (-not ($version -like "*-*")) {
            $unshipped = Get-Content "./csharp/PublicAPI/*/PublicAPI.Unshipped.txt" | Select-String -notMatch "^#" | % { $_.Line }
            if ($unshipped) {
                echo "::error ::Unshipped API changes must be empty before publishing a stable release"
                exit 1
            }
        }
    - name: Download NuGet artifact
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
      with:
        name: nuget-package
        path: nuget
    - name: Setup .NET SDK v10.0.x
      uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2
      with:
        dotnet-version: 10.0.x
    # if version contains "-" treat it as pre-release
    # example: 1.0.0-beta1
    - name: Create release
      uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda
      with:
        name: ParquetSharp ${{ needs.build-nuget.outputs.version }}
        draft: true
        prerelease: ${{ contains(needs.build-nuget.outputs.version, '-') }}
        files: |
          nuget/ParquetSharp.${{ needs.build-nuget.outputs.version }}.nupkg
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    - name: Publish to NuGet
      run: dotnet nuget push nuget/ParquetSharp.${{ needs.build-nuget.outputs.version }}.nupkg --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json