Add Android native artifacts and CI coverage

.github/workflows/main.yml

@@ -8,12 +8,20 @@ on:
     paths-ignore:
     - "**/README.md"
 jobs:
-  test:
-    name: Test on ${{ matrix.os }}
+  native:
+    name: Build native on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, windows-2022, macos-15-intel, macos-14]
+        include:
+        - os: ubuntu-22.04
+          targets: x86_64-linux-gnu,aarch64-linux-gnu
+        - os: windows-2022
+          targets: x86_64-windows
+        - os: macos-15-intel
+          targets: x86_64-macos
+        - os: macos-14
+          targets: aarch64-macos
     env:
       ORG_GRADLE_PROJECT_ossrhUsername: na
       ORG_GRADLE_PROJECT_ossrhPassword: na
@@ -26,8 +34,13 @@ jobs:
         distribution: 'corretto'
     - name: Setup Gradle
       uses: gradle/actions/setup-gradle@v4.2.1
-    - name: Test with Gradle
-      run: ./gradlew test --stacktrace
+    - name: Build native libraries with Gradle
+      run: >
+        ./gradlew
+        buildNative
+        -PtreeSitterTargets=${{ matrix.targets }}
+        --max-workers=1
+        --stacktrace
 
   android-native:
     name: Build Android native library
@@ -51,4 +64,4 @@ jobs:
       run: |
         "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "ndk;${ANDROID_NDK_VERSION}"
     - name: Build Android native library
-      run: ./gradlew :tree-sitter:buildNative -PtreeSitterTargets=aarch64-linux-android --stacktrace
+      run: ./gradlew buildNative -PtreeSitterTargets=aarch64-linux-android --max-workers=1 --stacktrace

THIRD_PARTY_NOTICES.md

@@ -0,0 +1,59 @@
+# Third-party notices
+
+This repository includes a small number of bundled third-party files that are
+used for native builds and runtime loading.
+
+## OpenJDK 11 JNI headers
+
+The JNI, JAWT, and Windows AccessBridge header files under `include/jni` are
+copied from OpenJDK 11. Their license texts are included under
+`third_party_licenses/openjdk-11`.
+
+## Gradle wrapper
+
+The Gradle wrapper files are provided by Gradle. The full Apache License 2.0
+text is included under `third_party_licenses/gradle`.
+
+## Unicode ConvertUTF
+
+`tree-sitter/src/main/c/ConvertUTF.c` and
+`tree-sitter/src/main/c/ConvertUTF.h` are Unicode conversion source files. The
+Unicode license text is included under `third_party_licenses/unicode`.
+
+## Zig
+
+The native build downloads Zig to compile the bundled native artifacts. The Zig
+license text is included under `third_party_licenses/zig`.
+
+## Minisign
+
+The native build downloads Minisign to verify Zig release signatures. The
+Minisign license text is included under `third_party_licenses/minisign`.
+
+## Android NDK
+
+The Android native build can use or download Android NDK r27c. The Android SDK
+license text referenced by the NDK package metadata is included under
+`third_party_licenses/android-ndk`.
+
+The Android NDK Clang/LLVM toolchain is based on the LLVM project. The upstream
+LLVM license text, including the Apache License 2.0 with LLVM exceptions, is
+included under `third_party_licenses/android-ndk/LLVM-CLANG-NOTICE`. It is
+copied from
+https://raw.githubusercontent.com/llvm/llvm-project/main/llvm/LICENSE.TXT.
+
+## tree-sitter core
+
+The core `tree-sitter` native library is built from the tree-sitter source
+archive. Its license text is included under
+`third_party_licenses/tree-sitter-core`.
+
+The tree-sitter core source also includes Unicode data/code under
+`lib/src/unicode`; that license text is included under
+`third_party_licenses/tree-sitter-core/lib-src-unicode`.
+
+## Grammar sources
+
+The checked-in grammar native artifacts are compiled from downloaded
+tree-sitter grammar source archives. Each grammar license text is included under
+`third_party_licenses/grammars/<grammar-project>/LICENSE`.

build.gradle

@@ -103,19 +103,25 @@ subprojects {
         useJUnitPlatform()
     }
 
-    tasks.register('downloadSource', DownloadSourceTask)
+    if (project.hasProperty('libVersion')) {
+        tasks.register('downloadSource', DownloadSourceTask)
 
-    if (project.name != 'tree-sitter' && project.name != 'tree-sitter-demo' && project.name != 'tree-sitter-tests') {
-        tasks.named('test') {
-            dependsOn tasks.named('downloadSource')
+        tasks.register('buildNative', BuildNativeTask) {
+            zigExe = rootProject.downloadZig.zigExe
+            dependsOn downloadSource, rootProject.downloadZig
+            if (((String) rootProject.properties.get("treeSitterTargets")).contains("android")) {
+                dependsOn rootProject.downloadAndroidNdk
+            }
         }
-    }
 
-    tasks.register('buildNative', BuildNativeTask) {
-        zigExe = rootProject.downloadZig.zigExe
-        dependsOn downloadSource, rootProject.downloadZig
-        if (((String) rootProject.properties.get("treeSitterTargets")).contains("android")) {
-            dependsOn rootProject.downloadAndroidNdk
+        tasks.named('processResources') {
+            dependsOn tasks.named('buildNative')
+        }
+
+        if (project.name != 'tree-sitter') {
+            tasks.named('test') {
+                dependsOn tasks.named('downloadSource')
+            }
         }
     }
 

buildSrc/src/main/groovy/BuildNativeTask.groovy

@@ -181,10 +181,10 @@ class BuildNativeTask extends DefaultTask{
             cmd.addAll(jniCFiles)
             cmd.addAll(parserCFiles)
             cmd.addAll(additionalCFiles)
-            project.exec{
+            project.providers.exec {
                 workingDir jniCDir
                 commandLine(cmd)
-            }
+            }.result.get()
         }
         this.removeWindowsDebugFiles()
     }
@@ -220,6 +220,12 @@ class BuildNativeTask extends DefaultTask{
         }
 
         def sdkDir = System.getenv("ANDROID_HOME") ?: System.getenv("ANDROID_SDK_ROOT")
+        if (!sdkDir) {
+            def defaultSdk = new File(System.getProperty("user.home"), "Library/Android/sdk")
+            if (defaultSdk.exists()) {
+                sdkDir = defaultSdk.absolutePath
+            }
+        }
         if (sdkDir) {
             def ndkRoot = new File(sdkDir, "ndk")
             def ndks = ndkRoot.listFiles()?.findAll { it.isDirectory() }?.sort { a, b -> b.name <=> a.name }
@@ -237,7 +243,7 @@ class BuildNativeTask extends DefaultTask{
         throw new GradleException("Android NDK not found. Set ANDROID_NDK_HOME, ANDROID_NDK_ROOT, ANDROID_HOME, or ANDROID_SDK_ROOT, or run the downloadAndroidNdk task.")
     }
 
-    private void removeWindowsDebugFiles(){
+    void removeWindowsDebugFiles(){
         def files = jniOutDir.asFileTree.matching {
             include("**/*.pdb")
             include("**/*.lib")

buildSrc/src/main/groovy/DownloadAndroidNdkTask.groovy

@@ -104,6 +104,12 @@ class DownloadAndroidNdkTask extends DefaultTask {
         }
 
         def sdkDir = System.getenv("ANDROID_HOME") ?: System.getenv("ANDROID_SDK_ROOT")
+        if (!sdkDir) {
+            def defaultSdk = new File(System.getProperty("user.home"), "Library/Android/sdk")
+            if (defaultSdk.exists()) {
+                sdkDir = defaultSdk.absolutePath
+            }
+        }
         if (sdkDir) {
             def ndkRoot = new File(sdkDir, "ndk")
             def ndks = ndkRoot.listFiles()?.findAll { it.isDirectory() }?.sort { a, b -> b.name <=> a.name }

buildSrc/src/main/groovy/DownloadZigTask.groovy

@@ -152,31 +152,42 @@ class DownloadZigTask extends DefaultTask{
                 logger.lifecycle("Downloading Zig from mirror: $zigZipUrl")
                 Utils.downloadFile(zigZipUrl, zigZipDest)
                 Utils.downloadFile(new URL(zigSignatureUrl(zigZipUrl.toString())), zigSignatureFileDest)
-                return
+                if (verifyZigArchive()) {
+                    return
+                }
+                logger.error("Zig archive signature does not match for mirror: $zigZipUrl")
             }catch(Exception e){
                 logger.error("Failed to download Zig from mirror: $zigZipUrl", e)
             }
+            zigZipDest.delete()
+            zigSignatureFileDest.delete()
         }
         throw new GradleException("Failed to download Zig from any mirror")
     }
 
-    @TaskAction
-    downloadZig(){
-        def mirrorUrls = mirrorUrls()
-        downloadZigFromMirrors(mirrorUrls, zigZipFile.asFile, zigSignatureFile.asFile)
+    boolean verifyZigArchive() {
         miniSignExe.asFile.setExecutable(true, true)
-        def zipVerified = project.exec {
+        return project.providers.exec {
             ignoreExitValue = true
             workingDir zigDir.asFile
             commandLine miniSignExe,
                            "-qVm",
                            zigZipFile,
                            "-P",
                            zigPubKey
+        }.result.get().exitValue == 0
+    }
+
+    @TaskAction
+    downloadZig(){
+        if (zigExe.asFile.exists()) {
+            logger.lifecycle("Using downloaded Zig: {}", zigExe.asFile)
+            zigExe.asFile.setExecutable(true, true)
+            return
         }
-        if(!zipVerified) {
-            throw new GradleException("$zigZipFile signature does not match!")
-        }
+        def mirrorUrls = mirrorUrls()
+        mirrorUrls.add("https://ziglang.org/download/$zigVersion")
+        downloadZigFromMirrors(mirrorUrls, zigZipFile.asFile, zigSignatureFile.asFile)
         Utils.unzipArchive(zigZipFile.asFile, zigDir.asFile)
         zigExe.asFile.setExecutable(true, true)
     }

buildSrc/src/main/groovy/org/treesitter/build/Utils.groovy

@@ -146,17 +146,32 @@ abstract class Utils {
     }
 
     static void downloadFile(URL url, File dest, String accept = "application/zip"){
-        url.openConnection().with { conn ->
-            conn.setRequestProperty("User-Agent", "Mozilla/5.0")
-            conn.setRequestProperty("Accept", accept)
-            conn.setConnectTimeout(30000)
-            conn.setReadTimeout(30000)
-            dest.withOutputStream { output ->
-                conn.inputStream.withCloseable { input ->
-                    output << input
+        int maxAttempts = 6
+        Exception lastException = null
+        for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+            try {
+                url.openConnection().with { conn ->
+                    conn.setRequestProperty("User-Agent", "Mozilla/5.0")
+                    conn.setRequestProperty("Accept", accept)
+                    conn.setConnectTimeout(30000)
+                    conn.setReadTimeout(30000)
+                    dest.withOutputStream { output ->
+                        conn.inputStream.withCloseable { input ->
+                            output << input
+                        }
+                    }
+                }
+                return
+            } catch (IOException e) {
+                lastException = e
+                dest.delete()
+                if (attempt == maxAttempts) {
+                    break
                 }
+                Thread.sleep(2000L * attempt)
             }
         }
+        throw lastException
     }
 
     static String fetchUrl(URL url){

include/README.md

@@ -1,2 +1,7 @@
 # About
-C header files copied from OpenJDK 11
+C header files copied from OpenJDK 11.
+
+The OpenJDK 11 license files are included in:
+
+- `third_party_licenses/openjdk-11/LICENSE`
+- `third_party_licenses/openjdk-11/ADDITIONAL_LICENSE_INFO`