diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index a792b88021..715936c43f 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -50,7 +50,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-15-intel", "macos-latest", "ubuntu-latest"] + os: ["macos-15-intel", "macos-latest", "ubuntu-latest", "windows-latest"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/dev/release/verify-release-candidate.ps1 b/dev/release/verify-release-candidate.ps1 index 894d5dc59b..66b9f72127 100755 --- a/dev/release/verify-release-candidate.ps1 +++ b/dev/release/verify-release-candidate.ps1 @@ -51,9 +51,15 @@ Assumes Mamba is set up and available on the path. exit 1 } +function Show-Header { + echo "" + echo "============================================================" + echo $args[0] + echo "============================================================" +} + $ArrowDistUrl = "https://dist.apache.org/repos/dist/dev/arrow" $DistName = "apache-arrow-adbc-$($Version)" - function Download-Dist-File { $DistUrl = "$($ArrowDistUrl)/$($DistName)-rc$($RcNumber)/$($args[0])" $DistPath = Join-Path $ArrowTempDir $args[0] @@ -67,13 +73,49 @@ function Download-Dist-File { } } -function Show-Header { - echo "" - echo "============================================================" - echo $args[0] - echo "============================================================" +# ============================================================ +# What to test +# ============================================================ + +function Get-Bool($envname, [bool] $default) { + if (-not (Test-Path -Path "Env:$envname")) { + return $default + } + + $value = (Get-Item -Path "Env:$envname").Value + switch ($value) { + "1" { return $true } + "0" { return $false } + "true" { return $true } + "false" { return $false } + "on" { return $true } + "off" { return $false } + "yes" { return $true } + "no" { return $false } + default { throw "Invalid boolean value for $($envname): $($value)" } + } +} + +$TestDefault = Get-Bool "TEST_DEFAULT" $true +$TestSource = Get-Bool "TEST_SOURCE" $TestDefault +$TestBinaries = Get-Bool "TEST_BINARIES" $TestDefault +$TestBinaries = $TestBinaries -and ($SourceKind -eq "tarball") +$TestJars = Get-Bool "TEST_JARS" $TestBinaries + +if (-not $TestSource -and -not $TestBinaries) { + echo "Nothing to test, exiting" + exit 1 } +echo "Default: $($TestDefault)" +echo "Source: $($TestSource)" +echo "Binaries: $($TestBinaries)" +echo "- JARs: $($TestJars)" + +# ============================================================ +# Set up common artifacts +# ============================================================ + Show-Header "Create Temporary Directory" if ($env:ARROW_TMPDIR -eq $null) { $ArrowTempDir = New-TemporaryFile | % { $_.FullName } @@ -82,81 +124,193 @@ if ($env:ARROW_TMPDIR -eq $null) { $ArrowTempDir = $env:ARROW_TMPDIR } New-Item -ItemType Directory -Force -Path $ArrowTempDir | Out-Null - echo "Using $($ArrowTempDir)" -Show-Header "Ensure Source Directory" - -if ($SourceKind -eq "local") { - $ArrowSourceDir = Join-Path $PSScriptRoot "..\.." | Resolve-Path | % { $_.Path } +Show-Header "Clone apache/arrow" +$ArrowSourceDir = Join-Path $ArrowTempDir "arrow" +$StampFile = Join-Path $ArrowSourceDir "stamp.txt" +if (-not (Test-Path -Path $StampFile)) { + git clone --depth 1 https://github.com/apache/arrow $ArrowSourceDir + if (-not $?) { throw "Failed to clone apache/arrow" } + New-Item -ItemType File -Force -Path $StampFile | Out-Null } else { - $ArrowSourceDir = Join-Path $ArrowTempDir $DistName - New-Item -ItemType Directory -Path $ArrowSourceDir -Force - # Convert to an absolute now that it should exist - $ArrowSourceDir = $ArrowSourceDir | Resolve-Path | % { $_.Path } + echo "Using cached $($ArrowSourceDir)" +} + +$BinaryDir = Join-Path $ArrowTempDir "binaries" +if ($TestBinaries) { + Show-Header "Download binary artifacts" + + $StampFile = Join-Path $BinaryDir "stamp.txt" + if (-not (Test-Path -Path $StampFile)) { + python "$ArrowSourceDir/dev/release/download_rc_binaries.py" ` + $Version $RcNumber ` + --dest="$BinaryDir" ` + --num_parallel 4 ` + --package_type=github ` + --repository="apache/arrow-adbc" ` + --tag="apache-arrow-adbc-$($Version)-rc$($RcNumber)" + if (-not $?) { throw "Failed to download binary artifacts" } + New-Item -ItemType File -Force -Path $StampFile | Out-Null + } else { + echo "Using cached $($BinaryDir)" + } +} - Download-Dist-File "$($DistName).tar.gz" - Download-Dist-File "$($DistName).tar.gz.sha512" - $DistPath = Join-Path $ArrowTempDir "$($DistName).tar.gz" - $Sha512Path = Join-Path $ArrowTempDir "$($DistName).tar.gz.sha512" +# ============================================================ +# Test release +# ============================================================ +if ($TestSource) { + Show-Header "Ensure Source Directory" - $ExpectedSha512 = (Get-Content $Sha512Path).Split(" ")[0] - if (-not ((Get-FileHash -Algorithm SHA512 $DistPath).Hash -eq $ExpectedSha512)) { - echo "SHA512 hash mismatch" - exit 1 + if ($SourceKind -eq "local") { + $ArrowSourceDir = Join-Path $PSScriptRoot "..\.." | Resolve-Path | % { $_.Path } + } else { + $ArrowSourceDir = Join-Path $ArrowTempDir $DistName + New-Item -ItemType Directory -Path $ArrowSourceDir -Force + # Convert to an absolute now that it should exist + $ArrowSourceDir = $ArrowSourceDir | Resolve-Path | % { $_.Path } + + Download-Dist-File "$($DistName).tar.gz" + Download-Dist-File "$($DistName).tar.gz.sha512" + + $DistPath = Join-Path $ArrowTempDir "$($DistName).tar.gz" + $Sha512Path = Join-Path $ArrowTempDir "$($DistName).tar.gz.sha512" + + $ExpectedSha512 = (Get-Content $Sha512Path).Split(" ")[0] + if (-not ((Get-FileHash -Algorithm SHA512 $DistPath).Hash -eq $ExpectedSha512)) { + echo "SHA512 hash mismatch" + exit 1 + } + + tar -C $ArrowSourceDir --strip-components 1 -xf $DistPath } - tar -C $ArrowSourceDir --strip-components 1 -xf $DistPath -} + echo "Using $($ArrowSourceDir)" -echo "Using $($ArrowSourceDir)" + Show-Header "Create Conda Environment" -Show-Header "Create Conda Environment" + mamba create -c conda-forge --yes --prefix $(Join-Path $ArrowTempDir conda-env) ` + --file $(Join-Path $ArrowSourceDir ci\conda_env_cpp.txt) ` + --file $(Join-Path $ArrowSourceDir ci\conda_env_python.txt) ` + go ` + m2w64-gcc -mamba create -c conda-forge --yes --prefix $(Join-Path $ArrowTempDir conda-env) ` - --file $(Join-Path $ArrowSourceDir ci\conda_env_cpp.txt) ` - --file $(Join-Path $ArrowSourceDir ci\conda_env_python.txt) ` - go ` - m2w64-gcc + Invoke-Expression $(conda shell.powershell hook | Out-String) + conda activate $(Join-Path $ArrowTempDir conda-env) + # XXX: force bundled gtest as the conda-forge version appears to require you + # to exactly match the MSVC version it was compiled with. Uninstalling also + # removes a bunch of other things, so force-remove instead + # (https://github.com/conda-forge/libprotobuf-feedstock/issues/186) + # Use conda, mamba appears to ignore --force + conda remove -y --force gtest -Invoke-Expression $(conda shell.powershell hook | Out-String) -conda activate $(Join-Path $ArrowTempDir conda-env) -# XXX: force bundled gtest as the conda-forge version appears to require you -# to exactly match the MSVC version it was compiled with. Uninstalling also -# removes a bunch of other things, so force-remove instead -# (https://github.com/conda-forge/libprotobuf-feedstock/issues/186) -# Use conda, mamba appears to ignore --force -conda remove -y --force gtest + # Activating doesn't appear to set GOROOT + $env:GOROOT = $(Join-Path $ArrowTempDir $(Join-Path conda-env go)) -# Activating doesn't appear to set GOROOT -$env:GOROOT = $(Join-Path $ArrowTempDir conda-env go) + Show-Header "Verify C/C++ Sources" -Show-Header "Verify C/C++ Sources" + $CppBuildDir = Join-Path $ArrowTempDir cpp-build + New-Item -ItemType Directory -Force -Path $CppBuildDir | Out-Null -$CppBuildDir = Join-Path $ArrowTempDir cpp-build -New-Item -ItemType Directory -Force -Path $CppBuildDir | Out-Null + $env:_ADBC_IS_CONDA = "1" + # XXX(apache/arrow-adbc#634): not working on Windows due to it picking + # up MSVC as the C compiler, which then blows up when /Werror gets + # passed in by some package + $env:BUILD_DRIVER_FLIGHTSQL = "0" -$env:_ADBC_IS_CONDA = "1" -# XXX(apache/arrow-adbc#634): not working on Windows due to it picking -# up MSVC as the C compiler, which then blows up when /Werror gets -# passed in by some package -$env:BUILD_DRIVER_FLIGHTSQL = "0" + & $(Join-Path $ArrowSourceDir ci\scripts\cpp_build.ps1) $ArrowSourceDir $CppBuildDir + if (-not $?) { exit 1 } -& $(Join-Path $ArrowSourceDir ci\scripts\cpp_build.ps1) $ArrowSourceDir $CppBuildDir -if (-not $?) { exit 1 } + $env:BUILD_DRIVER_POSTGRESQL = "0" + & $(Join-Path $ArrowSourceDir ci\scripts\cpp_test.ps1) $CppBuildDir + if (-not $?) { exit 1 } + $env:BUILD_DRIVER_POSTGRESQL = "1" -$env:BUILD_DRIVER_POSTGRESQL = "0" -& $(Join-Path $ArrowSourceDir ci\scripts\cpp_test.ps1) $CppBuildDir -if (-not $?) { exit 1 } -$env:BUILD_DRIVER_POSTGRESQL = "1" + Show-Header "Verify Python Sources" -Show-Header "Verify Python Sources" + & $(Join-Path $ArrowSourceDir ci\scripts\python_build.ps1) $ArrowSourceDir $CppBuildDir + if (-not $?) { exit 1 } -& $(Join-Path $ArrowSourceDir ci\scripts\python_build.ps1) $ArrowSourceDir $CppBuildDir -if (-not $?) { exit 1 } + & $(Join-Path $ArrowSourceDir ci\scripts\python_test.ps1) $ArrowSourceDir $CppBuildDir + if (-not $?) { exit 1 } +} -& $(Join-Path $ArrowSourceDir ci\scripts\python_test.ps1) $ArrowSourceDir $CppBuildDir -if (-not $?) { exit 1 } +if ($TestBinaries) { + Show-Header "Verify Binary Distribution" + + if ($TestJars) { + Show-Header "Verify Java JARs" + if ($env:JAVA_HOME -eq $null) { + # Work around PowerShell < 7. Temporarily set ErrorActionPreference + # to continue to avoid the redirect below from stopping the script. + $PreviousErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $env:JAVA_HOME = & java -XshowSettings:properties -version 2>&1 | Select-String "java.home" | ForEach-Object { $_.ToString().Split("=")[1].Trim() } + $ErrorActionPreference = $PreviousErrorActionPreference + } + echo "JAVA_HOME: $($env:JAVA_HOME)" + + $RootPoms = @(Get-ChildItem -Path $BinaryDir -Filter "arrow-adbc-java-root-*.pom") + if ($RootPoms.Count -ne 1) { + throw "Expected exactly one Arrow ADBC Java root POM, found $($RootPoms.Count)" + } + + $RootPomPath = $RootPoms[0].FullName + [xml] $RootPom = Get-Content -Raw $RootPomPath + $JavaVersion = $RootPom.project.version + echo "ADBC version: $($JavaVersion)" + + $MavenRepository = Join-Path $ArrowTempDir "maven-repository" + New-Item -ItemType Directory -Force -Path $MavenRepository | Out-Null + $MavenRepositoryArgument = "-Dmaven.repo.local=$($MavenRepository)" + + mvn -B install:install-file ` + $MavenRepositoryArgument ` + "-Dfile=$($RootPomPath)" ` + "-DpomFile=$($RootPomPath)" ` + "-Dpackaging=pom" + if (-not $?) { exit 1 } + + $Artifacts = @( + "adbc-core", + "adbc-driver-flight-sql", + "adbc-driver-jdbc", + "adbc-driver-jni", + "adbc-driver-manager", + "adbc-sql" + ) + foreach ($Artifact in $Artifacts) { + $ArtifactBase = Join-Path $BinaryDir "$($Artifact)-$($JavaVersion)" + $JarPath = "$($ArtifactBase).jar" + $PomPath = "$($ArtifactBase).pom" + $SourcesPath = "$($ArtifactBase)-sources.jar" + $JavadocPath = "$($ArtifactBase)-javadoc.jar" + foreach ($Path in @($JarPath, $PomPath, $SourcesPath, $JavadocPath)) { + if (-not (Test-Path -Path $Path -PathType Leaf)) { + throw "Missing Java artifact: $($Path)" + } + } + + mvn -B install:install-file ` + $MavenRepositoryArgument ` + "-Dfile=$($JarPath)" ` + "-DpomFile=$($PomPath)" ` + "-Dsources=$($SourcesPath)" ` + "-Djavadoc=$($JavadocPath)" + if (-not $?) { exit 1 } + } + + $JavaVerificationPom = Join-Path $PSScriptRoot "verify\java\pom.xml" + mvn -B test ` + $MavenRepositoryArgument ` + "-Dadbc.version=$($JavaVersion)" ` + -f $JavaVerificationPom + if (-not $?) { exit 1 } + } else { + Show-Header "Skipping Java JARs" + } +} Show-Header "Release candidate looks good!" diff --git a/dev/release/verify/java/README.md b/dev/release/verify/java/README.md new file mode 100644 index 0000000000..cf932e7afc --- /dev/null +++ b/dev/release/verify/java/README.md @@ -0,0 +1,21 @@ + + +A minimal Java project used to verify precompiled Java JARs during +the release verification process. diff --git a/dev/release/verify/java/pom.xml b/dev/release/verify/java/pom.xml new file mode 100644 index 0000000000..eef3ae12aa --- /dev/null +++ b/dev/release/verify/java/pom.xml @@ -0,0 +1,107 @@ + + + + 4.0.0 + + org.apache.arrow.adbc.verify + adbc-java-verification + 1.0-SNAPSHOT + Apache Arrow ADBC Java Verification + + + + 0.0.0 + ${surefireArgLine} --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + 11 + UTF-8 + + + + + + + org.junit + junit-bom + 5.14.4 + pom + import + + + + + + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + 3.27.7 + test + + + + + org.apache.arrow.adbc + adbc-core + ${adbc.version} + + + org.apache.arrow.adbc + adbc-driver-flight-sql + ${adbc.version} + + + org.apache.arrow.adbc + adbc-driver-jdbc + ${adbc.version} + + + org.apache.arrow.adbc + adbc-driver-jni + ${adbc.version} + + + org.apache.arrow.adbc + adbc-driver-manager + ${adbc.version} + + + org.apache.arrow.adbc + adbc-sql + ${adbc.version} + + + + + + java-25+ + + [25,) + + + --sun-misc-unsafe-memory-access=allow + + + + diff --git a/dev/release/verify/java/src/main/java/org/apache/arrow/adbc/verify/App.java b/dev/release/verify/java/src/main/java/org/apache/arrow/adbc/verify/App.java new file mode 100644 index 0000000000..b394528385 --- /dev/null +++ b/dev/release/verify/java/src/main/java/org/apache/arrow/adbc/verify/App.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.arrow.adbc.verify; + +public final class App { + private App() {} + + public static void main(String[] args) {} +} diff --git a/dev/release/verify/java/src/test/java/org/apache/arrow/adbc/verify/VerifyReleaseCandidateTest.java b/dev/release/verify/java/src/test/java/org/apache/arrow/adbc/verify/VerifyReleaseCandidateTest.java new file mode 100644 index 0000000000..c52a4f755c --- /dev/null +++ b/dev/release/verify/java/src/test/java/org/apache/arrow/adbc/verify/VerifyReleaseCandidateTest.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.arrow.adbc.verify; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcDatabase; +import org.apache.arrow.adbc.core.AdbcException; +import org.apache.arrow.adbc.driver.flightsql.FlightSqlDriver; +import org.apache.arrow.adbc.driver.jdbc.JdbcDriver; +import org.apache.arrow.adbc.driver.jni.JniDriver; +import org.apache.arrow.adbc.drivermanager.AdbcDriverManager; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Briefly test that shipped JARs actually work. */ +class VerifyReleaseCandidateTest { + BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + @Test + void testDriverFlightSql() throws Exception { + var driver = new FlightSqlDriver(allocator); + Map params = new HashMap<>(); + FlightSqlDriver.PARAM_URI.set(params, "grpc://localhost:1212"); + try (AdbcDatabase database = driver.open(params)) { + assertThrows( + AdbcException.class, + () -> { + //noinspection EmptyTryBlock + try (AdbcConnection ignored = database.connect()) {} + }); + } + } + + @Test + void testDriverJdbcAdapter() throws Exception { + var driver = new JdbcDriver(allocator); + Map params = new HashMap<>(); + params.put(JdbcDriver.PARAM_URI, "jdbc:driverdoesnotexist://"); + try (AdbcDatabase database = driver.open(params)) { + assertThrows( + AdbcException.class, + () -> { + //noinspection EmptyTryBlock + try (AdbcConnection ignored = database.connect()) {} + }); + } + } + + @Test + void testDriverJni() { + // smoke test: try to load a driver that does not exist; should fail with a proper error + // (not something like a linker error, which would imply the JNI shim doesn't work) + var driver = new JniDriver(allocator); + AdbcException exception = + assertThrows(AdbcException.class, () -> driver.load().driver("nonexistent").open()); + assertThat(exception).hasMessageContaining("Could not load `nonexistent`"); + } + + @Test + void testManagedDriverManager() { + // Not the JNI/C driver manager. + var manager = AdbcDriverManager.getInstance(); + assertThat(manager).isNotNull(); + + for (var factory : + List.of( + "org.apache.arrow.adbc.driver.flightsql.FlightSqlDriverFactory", + "org.apache.arrow.adbc.driver.jdbc.JdbcDriverFactory", + "org.apache.arrow.adbc.driver.jni.JniDriverFactory")) { + assertThat( + assertThrows( + IllegalStateException.class, + () -> manager.registerDriver(factory, (allocator) -> null))) + .hasMessageContaining("Driver factory already registered"); + } + } +}