From 5e0e70d102f301adf1e0569dbca3fd2fb5b4954c Mon Sep 17 00:00:00 2001 From: Tomasz Godzik Date: Tue, 4 Aug 2026 19:19:43 +0200 Subject: [PATCH 1/3] improvement: Add an option to cache turbine results This priduces a jar in .metals directory that is later read by metals into turbine. --- .../scala/meta/internal/metals/Configs.scala | 21 ++ .../meta/internal/metals/Directories.scala | 2 + .../internal/metals/MetalsLspService.scala | 1 + .../internal/metals/UserConfiguration.scala | 6 + .../mbt/MbtWorkspaceSymbolProvider.scala | 10 + .../internal/metals/mbt/TurbineCache.scala | 177 ++++++++++++++++ .../internal/metals/mbt/TurbineCompiler.scala | 64 +++++- .../scala/tests/mbt/TurbineCacheSuite.scala | 198 ++++++++++++++++++ 8 files changed, 468 insertions(+), 11 deletions(-) create mode 100644 metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCache.scala create mode 100644 tests/unit/src/test/scala/tests/mbt/TurbineCacheSuite.scala diff --git a/metals/src/main/scala/scala/meta/internal/metals/Configs.scala b/metals/src/main/scala/scala/meta/internal/metals/Configs.scala index 272608b8094b..a11e5502cbba 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/Configs.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/Configs.scala @@ -1036,4 +1036,25 @@ object Configs { } } } + + /** + * Configuration for turbine cache. When enabled, turbine compilation results + * are persisted to disk and restored on startup to avoid recompiling unchanged + * sources. + * + * @param enabled Whether caching is enabled + */ + final case class TurbineCacheConfig(enabled: Boolean) + + object TurbineCacheConfig { + val default: TurbineCacheConfig = TurbineCacheConfig(enabled = false) + val enabled: TurbineCacheConfig = TurbineCacheConfig(enabled = true) + + def fromConfig(value: Option[Boolean]): TurbineCacheConfig = { + value match { + case Some(enabled) => TurbineCacheConfig(enabled) + case None => default + } + } + } } diff --git a/metals/src/main/scala/scala/meta/internal/metals/Directories.scala b/metals/src/main/scala/scala/meta/internal/metals/Directories.scala index f2f95de86142..5cde74246752 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/Directories.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/Directories.scala @@ -40,6 +40,8 @@ object Directories { RelativePath(".metals").resolve("rules") def explainedDiagnostics: RelativePath = RelativePath(".metals").resolve("explained-diagnostics") + def turbineCache: RelativePath = + RelativePath(".metals").resolve("turbine-cache.jar") val stacktraceFilename = "stacktrace.scala" val dependenciesName = "dependencies" diff --git a/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala b/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala index dd342497254e..05bd7419c672 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala @@ -360,6 +360,7 @@ abstract class MetalsLspService( fallbackClasspaths = () => compilers.fallbackClasspaths, sleeper = sleeper, turbineRecompileDelay = () => userConfig.javaTurbineRecompileDelay, + turbineCacheConfig = () => userConfig.javaTurbineCache, indexFilters = MbtIndexFilter.allFilters, protobufLspConfig = () => userConfig.protobufLspConfig, metalsOutDir = Some(embedded.targetDir), diff --git a/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala b/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala index 37721afbeb0c..1855174dd3b9 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala @@ -25,6 +25,7 @@ import scala.meta.internal.metals.Configs.ProtobufLspConfig import scala.meta.internal.metals.Configs.RangeFormattingProviders import scala.meta.internal.metals.Configs.ReferenceProviderConfig import scala.meta.internal.metals.Configs.ScalaImportsPlacementConfig +import scala.meta.internal.metals.Configs.TurbineCacheConfig import scala.meta.internal.metals.Configs.TurbineRecompileDelayConfig import scala.meta.internal.metals.Configs.WorkspaceSymbolProviderConfig import scala.meta.internal.metals.JsonParser.XtensionSerializedAsOption @@ -106,6 +107,7 @@ case class UserConfiguration( javaSymbolLoader: JavaSymbolLoaderConfig = JavaSymbolLoaderConfig.default, javaTurbineRecompileDelay: TurbineRecompileDelayConfig = TurbineRecompileDelayConfig.default, + javaTurbineCache: TurbineCacheConfig = TurbineCacheConfig.default, javacServicesOverrides: JavacServicesOverrides = JavacServicesOverrides.default, compilerProgress: CompilerProgressConfig = CompilerProgressConfig.default, @@ -1393,6 +1395,9 @@ object UserConfiguration { val javaTurbineRecompileDelay = TurbineRecompileDelayConfig.fromConfig( getStringKey("java-turbine-recompile-delay") ) + val javaTurbineCache = TurbineCacheConfig.fromConfig( + getBooleanKey("java-turbine-cache") + ) val javacServicesOverrides = getKey( "javac-services-overrides", @@ -1525,6 +1530,7 @@ object UserConfiguration { protoOutlineProvider, javaSymbolLoader, javaTurbineRecompileDelay, + javaTurbineCache, javacServicesOverrides, compilerProgress, referenceProvider, diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala index 07a5bf34bc21..5a1b8b9e939c 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala @@ -38,6 +38,7 @@ import scala.meta.internal.metals.BaseWorkDoneProgress import scala.meta.internal.metals.Buffers import scala.meta.internal.metals.Configs.JavaSymbolLoaderConfig import scala.meta.internal.metals.Configs.ProtobufLspConfig +import scala.meta.internal.metals.Configs.TurbineCacheConfig import scala.meta.internal.metals.Configs.TurbineRecompileDelayConfig import scala.meta.internal.metals.Configs.WorkspaceSymbolProviderConfig import scala.meta.internal.metals.Directories @@ -106,6 +107,8 @@ class MbtWorkspaceSymbolProvider( sleeper: Sleeper = Sleeper.TestingSleeper, turbineRecompileDelay: () => TurbineRecompileDelayConfig = () => TurbineRecompileDelayConfig.fromConfig(None), + turbineCacheConfig: () => TurbineCacheConfig = () => + TurbineCacheConfig.default, indexFilters: List[MbtIndexFilter] = MbtIndexFilter.allFilters, protobufLspConfig: () => ProtobufLspConfig = () => ProtobufLspConfig.default, @@ -140,6 +143,12 @@ class MbtWorkspaceSymbolProvider( def protoJavaOutlines(file: AbsolutePath): Seq[VirtualTextDocument] = documents.get(file).toSeq.flatMap(protobufWorkspace.allJavaOutlines) + private val turbineCache = new TurbineCache( + workspace.resolve(Directories.turbineCache).toNIO, + turbineCacheConfig, + turbineRecompileDelay, + time, + ) private val turbineCompiler: TurbineCompiler[AbsolutePath] = new TurbineCompiler[AbsolutePath]( () => documentsKeys, @@ -180,6 +189,7 @@ class MbtWorkspaceSymbolProvider( onIndexingDone = onIndexingDone, onNewProjectClasspath = classpath => protobufWorkspace.onNewProjectClasspath(classpath), + turbineCache = Some(turbineCache), ) // NOTE: runs unconditionally even if the user config is not mbt-v2 for usage diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCache.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCache.scala new file mode 100644 index 000000000000..dfcb4333c984 --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCache.scala @@ -0,0 +1,177 @@ +package scala.meta.internal.metals.mbt + +import java.io.BufferedOutputStream +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import java.time.LocalDateTime +import java.util.jar.JarEntry +import java.util.jar.JarOutputStream +import java.util.zip.ZipEntry + +import scala.util.Using +import scala.util.control.NonFatal + +import scala.meta.internal.jdk.CollectionConverters._ +import scala.meta.internal.metals.Configs.TurbineCacheConfig +import scala.meta.internal.metals.Configs.TurbineRecompileDelayConfig +import scala.meta.internal.metals.Time +import scala.meta.internal.metals.Timer + +import com.google.common.collect.ImmutableMap +import com.google.common.collect.ImmutableSet +import com.google.common.hash.Hashing +import com.google.turbine.binder.ClassPathBinder +import com.google.turbine.binder.sym.ClassSymbol +import com.google.turbine.lower.Lower +import com.google.turbine.zip.Zip + +/** + * Handles caching of Turbine compilation results to disk. + * + * The cache is stored as a JAR file containing the compiled class files. + * Each class file is stored under its binary name with a .class extension. + * + * @param cachePath Path to the cache JAR file + * @param cacheConfig Configuration for caching behavior + * @param recompileDelayConfig Configuration for recompile delay (to check if turbine is disabled) + */ +class TurbineCache( + cachePath: Path, + cacheConfig: () => TurbineCacheConfig, + recompileDelayConfig: () => TurbineRecompileDelayConfig, + time: Time, +) { + + // we need to always compile on start + private def isCacheEnabled: Boolean = { + val config = cacheConfig() + val recompileConfig = recompileDelayConfig() + config.enabled && !recompileConfig.isEffectivelyDisabled + } + + /** + * Writes the Turbine compilation result to the cache file. + * + * @param result The compilation result to cache + */ + def writeCache(result: TurbineCompileResult): Unit = { + if (!isCacheEnabled) return + + val timer = new Timer(time) + try { + val bytes = result.lowered.bytes() + if (bytes.isEmpty()) { + scribe.debug("turbine-cache: skipping write, no classes to cache") + return + } + + Files.createDirectories(cachePath.getParent()) + + Using.resource( + new JarOutputStream( + new BufferedOutputStream( + Files.newOutputStream( + cachePath, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + ) + ) + ) + ) { jos => + bytes.forEach { (binaryName, classBytes) => + addEntry(jos, binaryName + ".class", classBytes) + } + } + + scribe.info( + s"turbine-cache: wrote ${result.lowered.symbols().size()} classes in ${timer.elapsedMillis}ms" + ) + } catch { + case NonFatal(e) => + scribe.warn(s"turbine-cache: failed to write cache: ${e.getMessage}") + } + } + + /** + * Reads the cached Turbine compilation result from disk. + * + * @return The cached result, or None if cache doesn't exist or is invalid + */ + def readCache(classpath: Seq[Path]): Option[TurbineCompileResult] = { + if (!isCacheEnabled) { + scribe.debug("turbine-cache: caching is disabled") + None + } else if (!Files.exists(cachePath)) { + scribe.debug("turbine-cache: no cache file found") + None + } else { + val timer = new Timer(time) + try { + val bytesBuilder = ImmutableMap.builder[String, Array[Byte]]() + val symbolsBuilder = ImmutableSet.builder[ClassSymbol]() + + Using.resource(new Zip.ZipIterable(cachePath)) { zipIterable => + zipIterable.forEach { entry => + val name = entry.name() + if (name.endsWith(".class")) { + val binaryName = name.stripSuffix(".class") + val sym = new ClassSymbol(binaryName) + symbolsBuilder.add(sym) + bytesBuilder.put(binaryName, entry.data()) + } + } + } + + val lowered = Lower.Lowered.create( + bytesBuilder.build(), + symbolsBuilder.build(), + ) + // Bind the project classpath (libraries) so dependency symbols remain + // discoverable when serving classes from the cached lowered output. + val classPath = ClassPathBinder.bindClasspath(classpath.asJava) + val result = TurbineCompileResult(classPath, lowered) + + scribe.info( + s"turbine-cache: loaded ${lowered.symbols().size()} classes in ${timer.elapsedMillis}ms" + ) + Some(result) + } catch { + case NonFatal(e) => + scribe.warn(s"turbine-cache: failed to read cache: ${e.getMessage}") + deleteCache() + None + } + } + } + + /** + * Deletes the cache file if it exists. + */ + def deleteCache(): Unit = { + try { + Files.deleteIfExists(cachePath) + scribe.debug("turbine-cache: deleted cache file") + } catch { + case NonFatal(e) => + scribe.warn(s"turbine-cache: failed to delete cache: ${e.getMessage}") + } + } + + private val DEFAULT_TIMESTAMP: LocalDateTime = + LocalDateTime.of(2010, 1, 1, 0, 0, 0) + + private def addEntry( + jos: JarOutputStream, + name: String, + bytes: Array[Byte], + ): Unit = { + val entry = new JarEntry(name) + entry.setTimeLocal(DEFAULT_TIMESTAMP) + entry.setMethod(ZipEntry.STORED) + entry.setSize(bytes.length) + entry.setCrc(Hashing.crc32().hashBytes(bytes).padToLong()) + jos.putNextEntry(entry) + jos.write(bytes) + } +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala index 83a5c819f2b9..003442d400f7 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala @@ -109,10 +109,10 @@ object TurbineCompiler { ) TurbineCompileResult(boundClasspath, lowered) } - private def validClasspaths(classpath: Seq[Path]): Seq[Path] = { + private[mbt] def validClasspaths(classpath: Seq[Path]): Seq[Path] = { classpath.filter(isJarFile) } - private def isJarFile(path: Path): Boolean = { + private[mbt] def isJarFile(path: Path): Boolean = { Files.isRegularFile(path) && path.getFileName().toString().endsWith(".jar") } @@ -134,6 +134,7 @@ class TurbineCompiler[T]( sleeper: Sleeper, onIndexingDone: () => Unit, onNewProjectClasspath: ClassPath => Unit, + turbineCache: Option[TurbineCache] = None, )(implicit ec: ExecutionContext, rc: ReportContext) { private val sourcepathByPackageName = TrieMap.empty[String, ju.concurrent.ConcurrentLinkedDeque[ @@ -157,6 +158,7 @@ class TurbineCompiler[T]( private def isRecompilationDisabled: Boolean = debounceDelay.toMillis >= 3600000 + private val isFirstCompile = new AtomicBoolean(true) private val doCompile = BatchedFunction.fromFuture[Unit, TurbineCompileResult]( _ => { @@ -184,16 +186,56 @@ class TurbineCompiler[T]( } var result = TurbineCompiler.emptyResult + + /** + * Attempts to load compilation results from cache. + * Should be called during initialization before any compilation. + * + * @return true if cache was loaded successfully, false otherwise + */ + def loadFromCache(classpath: Seq[Path]): Option[TurbineCompileResult] = { + turbineCache match { + case Some(cache) => + cache.readCache(classpath) match { + case Some(cachedResult) => + scribe.info( + s"Loaded turbine cache with ${cachedResult.lowered.symbols().size()} symbols" + ) + Some(cachedResult) + case None => + None + } + case None => + None + } + } + def doCompileNow(): TurbineCompileResult = { - result = TurbineCompiler.compileClassfiles( - allCompilationUnits(), - parseUnit, - classpath(), - progressBars, - ) - cleanup() - // Clear deleted binary names after recompile - they are no longer in the compiled output - deletedBinaryNames.clear() + + def compile() = { + result = TurbineCompiler.compileClassfiles( + allCompilationUnits(), + parseUnit, + classpath(), + progressBars, + ) + cleanup() + // Clear deleted binary names after recompile - they are no longer in the compiled output + deletedBinaryNames.clear() + // Write to cache after successful compilation + turbineCache.foreach(_.writeCache(result)) + } + + if (isFirstCompile.getAndSet(false)) { + loadFromCache(TurbineCompiler.validClasspaths(classpath())) match { + case Some(cachedResult) => + result = cachedResult + case None => + compile() + } + } else { + compile() + } onIndexingDone() result } diff --git a/tests/unit/src/test/scala/tests/mbt/TurbineCacheSuite.scala b/tests/unit/src/test/scala/tests/mbt/TurbineCacheSuite.scala new file mode 100644 index 000000000000..38976ecfec35 --- /dev/null +++ b/tests/unit/src/test/scala/tests/mbt/TurbineCacheSuite.scala @@ -0,0 +1,198 @@ +package tests.mbt + +import java.nio.file.Files + +import scala.meta.internal.metals.AutoImportBuildKind +import scala.meta.internal.metals.Configs +import scala.meta.internal.metals.Configs.ReferenceProviderConfig +import scala.meta.internal.metals.Configs.WorkspaceSymbolProviderConfig +import scala.meta.internal.metals.Directories +import scala.meta.internal.metals.UserConfiguration +import scala.meta.internal.metals.mbt.MbtBuildServer + +import tests.BaseLspSuite +import tests.BuildInfo +import tests.MbtJsonBuilder +import tests.TestHovers + +class TurbineCacheSuite extends BaseLspSuite("turbine-cache") with TestHovers { + + override def userConfig: UserConfiguration = super.userConfig.copy( + fallbackScalaVersion = Some(BuildInfo.scalaVersion), + presentationCompilerDiagnostics = true, + buildOnChange = false, + buildOnFocus = false, + workspaceSymbolProvider = WorkspaceSymbolProviderConfig.mbt, + referenceProvider = ReferenceProviderConfig.mbt, + javaTurbineCache = Configs.TurbineCacheConfig.enabled, + preferredBuildServer = Some(MbtBuildServer.name), + automaticImportBuild = AutoImportBuildKind.All, + ) + + override def initializeGitRepo: Boolean = true + + private val testName = "turbine-cache-persists-across-restarts" + + private val javaFile = "src/com/example/Hello.java" + private val scalaFile = "src/com/example/Main.scala" + + private val javaContents = + """|package com.example; + | + |import com.google.common.collect.ImmutableList; + | + |public class Hello { + | public static class Person { + | public final String name; + | + | public Person(String name) { + | this.name = name; + | } + | } + | + | public static String greet(String name) { + | return "Hello, " + name + "!"; + | } + | + | public static ImmutableList names() { + | return ImmutableList.of("Alice", "Bob"); + | } + | + | public static Person person(String name) { + | return new Person(name); + | } + |} + |""".stripMargin + + private val scalaContents = + """|package com.example + | + |object Main { + | def run(): String = Hello.greet("World") + | def names = Hello.names() + | def person: Hello.Person = Hello.person("Alice") + |} + |""".stripMargin + + private val immutableListHover = + """|```java + |public abstract class com.google.common.collect.ImmutableList extends com.google.common.collect.ImmutableCollection implements java.util.List, java.util.RandomAccess + |``` + |""".stripMargin + + private val greetHover = + """|```scala + |def greet(name: String): String + |``` + |""".stripMargin.hover + + private val personTypeHover = + """|```scala + |class Person: Hello.Person + |``` + |""".stripMargin.hover + + private val personInnerHover = + """|```java + |public static class com.example.Hello.Person + |``` + |""".stripMargin + + private def assertHovers() = + for { + _ <- server.assertHover( + scalaFile, + """|package com.example + | + |object Main { + | def run(): String = Hello.gre@@et("World") + | def names = Hello.names() + | def person: Hello.Person = Hello.person("Alice") + |} + |""".stripMargin, + greetHover, + ) + // Verify the library dependency is discoverable via turbine classpath. + _ <- server.assertHover( + javaFile, + javaContents.replace("ImmutableList.of", "Immutabl@@eList.of"), + immutableListHover, + ) + // Verify nested/inner classes resolve from Scala and Java. + _ <- server.assertHover( + scalaFile, + """|package com.example + | + |object Main { + | def run(): String = Hello.greet("World") + | def names = Hello.names() + | def person: Hello.Per@@son = Hello.person("Alice") + |} + |""".stripMargin, + personTypeHover, + ) + _ <- server.assertHover( + javaFile, + javaContents.replace( + "public static Person person", + "public static Per@@son person", + ), + personInnerHover, + ) + } yield () + + test(testName) { + cleanWorkspace() + // Fetch Guava first, then prepend scala-library (addJavaDependency replaces the list). + val mbtJson = new MbtJsonBuilder(BuildInfo.scalaVersion) + .addJavaDependency("com.google.guava", "guava", "33.5.0-jre") + .addScalaLibrary() + .addNamespace("core", List("src/**")) + .build() + + for { + _ <- initialize( + s"""|/.metals/mbt.json + |$mbtJson + |/$javaFile + |$javaContents + |/$scalaFile + |$scalaContents + |""".stripMargin + ) + + _ <- server.didOpen(javaFile) + _ <- server.didOpen(scalaFile) + _ = assertNoDiagnostics() + _ <- assertHovers() + + cachePath = workspace.resolve(Directories.turbineCache) + _ = assert( + Files.exists(cachePath.toNIO), + s"Turbine cache file should exist at $cachePath after compilation", + ) + + cacheSize = Files.size(cachePath.toNIO) + _ = assert(cacheSize > 0, "Turbine cache file should not be empty") + + _ = cancelServer() + _ = newServer(testName) + + _ <- server.initialize() + _ <- server.initialized() + _ <- server.didChangeConfiguration(userConfig.toString) + _ = server.assertBuildServerConnection() + + _ <- server.didOpen(javaFile) + _ <- server.didOpen(scalaFile) + _ = assertNoDiagnostics() + // Symbols (including library + inner classes) must still resolve after cache load. + _ <- assertHovers() + + _ = assert( + Files.exists(cachePath.toNIO), + "Turbine cache file should still exist after restart", + ) + } yield () + } +} From 07461aaca38df49f5099898acca024de286fdeb6 Mon Sep 17 00:00:00 2001 From: Tomasz Godzik Date: Thu, 6 Aug 2026 16:46:06 +0200 Subject: [PATCH 2/3] improvement: Use git hash forchecking if changed --- .../internal/metals/UserConfiguration.scala | 6 + .../meta/internal/metals/mbt/GitVCS.scala | 24 +++ .../mbt/MbtWorkspaceSymbolProvider.scala | 2 +- .../internal/metals/mbt/TurbineCache.scala | 152 +++++++++++------- .../internal/metals/mbt/TurbineCompiler.scala | 4 +- .../scala/tests/UserConfigurationSuite.scala | 1 + 6 files changed, 132 insertions(+), 57 deletions(-) diff --git a/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala b/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala index 1855174dd3b9..d98c2c1eb1ee 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala @@ -301,6 +301,12 @@ case class UserConfiguration( javaTurbineRecompileDelay.duration.toString(), ) ), + Some( + ( + "javaTurbineCache", + javaTurbineCache.enabled, + ) + ), Some( ( "javacServicesOverrides", diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/GitVCS.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/GitVCS.scala index 9c89b942dd47..9f8085c046cd 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/GitVCS.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/GitVCS.scala @@ -21,6 +21,30 @@ import scala.meta.io.AbsolutePath object GitVCS { + /** + * Gets the current git HEAD hash for the given workspace. + * Returns None if git is not available or the workspace is not a git repo. + */ + def getHeadHash(workspace: AbsolutePath): Option[String] = { + try { + var result: Option[String] = None + val logger = ProcessLogger { line => + if (result.isEmpty && line.trim.nonEmpty) { + result = Some(line.trim) + } + } + val exitCode = Process( + List("git", "rev-parse", "HEAD"), + cwd = workspace.toFile, + ).!(logger) + if (exitCode == 0) result else None + } catch { + case NonFatal(e) => + scribe.debug(s"GitVCS.getHeadHash failed: ${e.getMessage}") + None + } + } + /** * Runs `git status --porcelain --untracked-files=all` and returns a list of * absolute paths to the relevant files. diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala index 5a1b8b9e939c..c712e06c5f9f 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala @@ -144,7 +144,7 @@ class MbtWorkspaceSymbolProvider( documents.get(file).toSeq.flatMap(protobufWorkspace.allJavaOutlines) private val turbineCache = new TurbineCache( - workspace.resolve(Directories.turbineCache).toNIO, + workspace, turbineCacheConfig, turbineRecompileDelay, time, diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCache.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCache.scala index dfcb4333c984..5aabd07267a5 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCache.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCache.scala @@ -1,6 +1,7 @@ package scala.meta.internal.metals.mbt import java.io.BufferedOutputStream +import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption @@ -12,11 +13,13 @@ import java.util.zip.ZipEntry import scala.util.Using import scala.util.control.NonFatal -import scala.meta.internal.jdk.CollectionConverters._ import scala.meta.internal.metals.Configs.TurbineCacheConfig import scala.meta.internal.metals.Configs.TurbineRecompileDelayConfig +import scala.meta.internal.metals.Directories +import scala.meta.internal.metals.MetalsEnrichments._ import scala.meta.internal.metals.Time import scala.meta.internal.metals.Timer +import scala.meta.io.AbsolutePath import com.google.common.collect.ImmutableMap import com.google.common.collect.ImmutableSet @@ -31,17 +34,21 @@ import com.google.turbine.zip.Zip * * The cache is stored as a JAR file containing the compiled class files. * Each class file is stored under its binary name with a .class extension. + * The cache is keyed by the current git HEAD hash to ensure it's invalidated + * when the source revision changes. * * @param cachePath Path to the cache JAR file * @param cacheConfig Configuration for caching behavior * @param recompileDelayConfig Configuration for recompile delay (to check if turbine is disabled) */ class TurbineCache( - cachePath: Path, + workspace: AbsolutePath, cacheConfig: () => TurbineCacheConfig, recompileDelayConfig: () => TurbineRecompileDelayConfig, time: Time, ) { + private val cachePath = workspace.resolve(Directories.turbineCache) + private val CacheKeyEntry = "META-INF/turbine-cache-key" // we need to always compile on start private def isCacheEnabled: Boolean = { @@ -52,69 +59,89 @@ class TurbineCache( /** * Writes the Turbine compilation result to the cache file. + * Uses the current git hash as the cache key. * * @param result The compilation result to cache */ - def writeCache(result: TurbineCompileResult): Unit = { - if (!isCacheEnabled) return - - val timer = new Timer(time) - try { - val bytes = result.lowered.bytes() - if (bytes.isEmpty()) { - scribe.debug("turbine-cache: skipping write, no classes to cache") - return - } - - Files.createDirectories(cachePath.getParent()) - - Using.resource( - new JarOutputStream( - new BufferedOutputStream( - Files.newOutputStream( - cachePath, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - ) - ) - ) - ) { jos => - bytes.forEach { (binaryName, classBytes) => - addEntry(jos, binaryName + ".class", classBytes) + def writeCache(result: TurbineCompileResult): Unit = + if (isCacheEnabled) { + val timer = new Timer(time) + try { + val bytes = result.lowered.bytes() + if (bytes.isEmpty()) { + scribe.debug("turbine-cache: skipping write, no classes to cache") + } else { + GitVCS.getHeadHash(workspace) match { + case Some(gitHash) => + cachePath.parent.createDirectories() + Using.resource( + new JarOutputStream( + new BufferedOutputStream( + Files.newOutputStream( + cachePath.toNIO, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + ) + ) + ) + ) { jos => + // Store the git hash as cache key + addEntry( + jos, + CacheKeyEntry, + gitHash.getBytes(StandardCharsets.UTF_8), + ) + bytes.forEach { (binaryName, classBytes) => + addEntry(jos, binaryName + ".class", classBytes) + } + } + + scribe.info( + s"turbine-cache: wrote ${result.lowered.symbols().size()} classes in ${timer.elapsedMillis}ms (git: ${gitHash.take(8)})" + ) + case None => + scribe.debug( + "turbine-cache: skipping write, not in a git repository" + ) + } } + } catch { + case NonFatal(e) => + scribe.warn(s"turbine-cache: failed to write cache: ${e.getMessage}") } - - scribe.info( - s"turbine-cache: wrote ${result.lowered.symbols().size()} classes in ${timer.elapsedMillis}ms" - ) - } catch { - case NonFatal(e) => - scribe.warn(s"turbine-cache: failed to write cache: ${e.getMessage}") } - } /** * Reads the cached Turbine compilation result from disk. + * Validates that the stored git hash matches the current HEAD. * - * @return The cached result, or None if cache doesn't exist or is invalid + * @param classpath The classpath to bind for dependency resolution + * @return The cached result, or None if cache doesn't exist, is invalid, or git hash mismatches */ def readCache(classpath: Seq[Path]): Option[TurbineCompileResult] = { + lazy val expectedHash = GitVCS.getHeadHash(workspace) if (!isCacheEnabled) { scribe.debug("turbine-cache: caching is disabled") None - } else if (!Files.exists(cachePath)) { + } else if (!cachePath.exists) { scribe.debug("turbine-cache: no cache file found") None + } else if (expectedHash.isEmpty) { + scribe.warn("turbine-cache: not in a git repository, skipping cache") + None } else { val timer = new Timer(time) try { val bytesBuilder = ImmutableMap.builder[String, Array[Byte]]() val symbolsBuilder = ImmutableSet.builder[ClassSymbol]() - - Using.resource(new Zip.ZipIterable(cachePath)) { zipIterable => + var storedHash: Option[String] = None + Using.resource(new Zip.ZipIterable(cachePath.toNIO)) { zipIterable => zipIterable.forEach { entry => val name = entry.name() - if (name.endsWith(".class")) { + if (name == CacheKeyEntry) { + storedHash = + Some(new String(entry.data(), StandardCharsets.UTF_8)) + } else if (name.endsWith(".class")) { val binaryName = name.stripSuffix(".class") val sym = new ClassSymbol(binaryName) symbolsBuilder.add(sym) @@ -123,19 +150,36 @@ class TurbineCache( } } - val lowered = Lower.Lowered.create( - bytesBuilder.build(), - symbolsBuilder.build(), - ) - // Bind the project classpath (libraries) so dependency symbols remain - // discoverable when serving classes from the cached lowered output. - val classPath = ClassPathBinder.bindClasspath(classpath.asJava) - val result = TurbineCompileResult(classPath, lowered) + // Validate the git hash + storedHash match { + case Some(hash) if hash == expectedHash.get => + val lowered = Lower.Lowered.create( + bytesBuilder.build(), + symbolsBuilder.build(), + ) + // Bind the project classpath (libraries) so dependency symbols remain + // discoverable when serving classes from the cached lowered output. + val classPath = ClassPathBinder.bindClasspath(classpath.asJava) + val result = TurbineCompileResult(classPath, lowered) + + scribe.info( + s"turbine-cache: loaded ${lowered.symbols().size()} classes in ${timer.elapsedMillis}ms (git: ${expectedHash.get.take(8)})" + ) + Some(result) + case Some(hash) => + scribe.info( + s"turbine-cache: git hash mismatch, invalidating cache (stored=${hash.take(8)}, current=${expectedHash.get.take(8)})" + ) + deleteCache() + None + case None => + scribe.info( + "turbine-cache: no git hash found in cache, invalidating" + ) + deleteCache() + None + } - scribe.info( - s"turbine-cache: loaded ${lowered.symbols().size()} classes in ${timer.elapsedMillis}ms" - ) - Some(result) } catch { case NonFatal(e) => scribe.warn(s"turbine-cache: failed to read cache: ${e.getMessage}") @@ -150,7 +194,7 @@ class TurbineCache( */ def deleteCache(): Unit = { try { - Files.deleteIfExists(cachePath) + cachePath.deleteIfExists() scribe.debug("turbine-cache: deleted cache file") } catch { case NonFatal(e) => diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala index 003442d400f7..5838215ac882 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala @@ -193,10 +193,10 @@ class TurbineCompiler[T]( * * @return true if cache was loaded successfully, false otherwise */ - def loadFromCache(classpath: Seq[Path]): Option[TurbineCompileResult] = { + def loadFromCache(classpathPaths: Seq[Path]): Option[TurbineCompileResult] = { turbineCache match { case Some(cache) => - cache.readCache(classpath) match { + cache.readCache(classpathPaths) match { case Some(cachedResult) => scribe.info( s"Loaded turbine cache with ${cachedResult.lowered.symbols().size()} symbols" diff --git a/tests/unit/src/test/scala/tests/UserConfigurationSuite.scala b/tests/unit/src/test/scala/tests/UserConfigurationSuite.scala index 7a32c01c1737..d8bc3936a9d6 100644 --- a/tests/unit/src/test/scala/tests/UserConfigurationSuite.scala +++ b/tests/unit/src/test/scala/tests/UserConfigurationSuite.scala @@ -498,6 +498,7 @@ class UserConfigurationSuite extends BaseSuite { "protoOutlineProvider": "v1", "javaSymbolLoader": "turbine-classpath", "javaTurbineRecompileDelay": "100 milliseconds", + "javaTurbineCache": false, "javacServicesOverrides": { "names": false, "attr": true, From b7e1914ea0b6db4ac1b9e13e9975a399e1907044 Mon Sep 17 00:00:00 2001 From: Tomasz Godzik Date: Fri, 7 Aug 2026 16:59:00 +0200 Subject: [PATCH 3/3] improvement: Put dirty files into sourcepath --- .../mbt/MbtWorkspaceSymbolProvider.scala | 49 +++++- .../internal/metals/mbt/TurbineCompiler.scala | 29 +++- .../scala/tests/mbt/TurbineCacheSuite.scala | 140 ++++++++++++++++++ 3 files changed, 214 insertions(+), 4 deletions(-) diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala index c712e06c5f9f..dfe90270c9a5 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala @@ -13,6 +13,7 @@ import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger import java.{util => ju} import javax.tools.JavaFileManager +import javax.tools.JavaFileObject import javax.tools.StandardJavaFileManager import scala.collection.concurrent.TrieMap @@ -149,6 +150,45 @@ class MbtWorkspaceSymbolProvider( turbineRecompileDelay, time, ) + + /** + * Returns dirty Java files (uncommitted changes) with their package names. + * Used to populate the sourcepath when loading from cache so that + * changed files take precedence over cached compiled classes. + */ + private def getDirtyJavaFiles(): Seq[(String, JavaFileObject)] = { + val result = for { + status <- GitVCS.status(workspace) + if !status.isDeleted && status.file.isJava + } yield { + try { + // Derive IndexedDocument from current source to handle: + // 1. Untracked Java files (not yet in documents map) + // 2. Package relocations (stale metadata in existing document) + val doc = IndexedDocument.fromFile( + status.file, + mtags(), + buffers, + dialects.Scala3, + ) + for { + input <- toInput(status.file) + pkg <- doc.semanticdbPackages.headOption + } yield { + val packageName = normalizePackageName(pkg) + val compilationUnit: JavaFileObject = + doc.toSemanticdbCompilationUnit(input) + (packageName, compilationUnit) + } + } catch { + case NonFatal(e) => + scribe.debug(s"mbt-v2: error indexing dirty file ${status.file}: $e") + None + } + } + result.flatten.toSeq + } + private val turbineCompiler: TurbineCompiler[AbsolutePath] = new TurbineCompiler[AbsolutePath]( () => documentsKeys, @@ -190,6 +230,7 @@ class MbtWorkspaceSymbolProvider( onNewProjectClasspath = classpath => protobufWorkspace.onNewProjectClasspath(classpath), turbineCache = Some(turbineCache), + getDirtyJavaFiles = getDirtyJavaFiles, ) // NOTE: runs unconditionally even if the user config is not mbt-v2 for usage @@ -464,7 +505,7 @@ class MbtWorkspaceSymbolProvider( // Add empty file to SOURCE_PATH so javac parses it and doesn't find the class doc.semanticdbPackages.headOption match { case Some(pkg) => - val packageName = pkg.stripSuffix("/").replace("/", ".") + val packageName = normalizePackageName(pkg) val emptyCompilationUnit = VirtualTextDocument( SourceJavaFileObject.makeRelativeURI(file.toURI), pc.Language.JAVA, @@ -982,7 +1023,7 @@ class MbtWorkspaceSymbolProvider( doc.semanticdbPackages.headOption match { case Some(pkg) => val input = file.toInputFromBuffers(buffers) - val packageName = pkg.stripSuffix("/").replace("/", ".") + val packageName = normalizePackageName(pkg) val compilationUnit = doc.toSemanticdbCompilationUnit(input) turbineCompiler .onDidChange(packageName, compilationUnit) @@ -1082,6 +1123,10 @@ class MbtWorkspaceSymbolProvider( newValue } + private def normalizePackageName(packageName: String): String = { + packageName.stripSuffix("/").replace("/", ".") + } + // Reads .metals/index.mbt, which is a serialized Mbt.Index protobuf payload, // into memory and converts it into TrieMap[AbsolutePath, IndexedDocument]. // For a very large repo (>100k Scala/Java files), this file still only takes diff --git a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala index 5838215ac882..4be7fb5d3772 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala @@ -135,6 +135,7 @@ class TurbineCompiler[T]( onIndexingDone: () => Unit, onNewProjectClasspath: ClassPath => Unit, turbineCache: Option[TurbineCache] = None, + getDirtyJavaFiles: () => Seq[(String, JavaFileObject)] = () => Seq.empty, )(implicit ec: ExecutionContext, rc: ReportContext) { private val sourcepathByPackageName = TrieMap.empty[String, ju.concurrent.ConcurrentLinkedDeque[ @@ -230,6 +231,8 @@ class TurbineCompiler[T]( loadFromCache(TurbineCompiler.validClasspaths(classpath())) match { case Some(cachedResult) => result = cachedResult + // Add dirty files to sourcepath so they take precedence over cached classes + addDirtyFilesToSourcepath() case None => compile() } @@ -279,6 +282,30 @@ class TurbineCompiler[T]( packageName: String, javaFileObject: JavaFileObject, ): Future[TurbineCompileResult] = { + addToSourcepath(packageName, javaFileObject) + doCompile(()) + } + + /** + * Add dirty Java files to the sourcepath so they take precedence over cached classes. + * This is called after loading from cache to ensure uncommitted changes are properly handled. + */ + private def addDirtyFilesToSourcepath(): Unit = { + val dirtyFiles = getDirtyJavaFiles() + if (dirtyFiles.nonEmpty) { + scribe.info( + s"turbine: adding ${dirtyFiles.size} dirty files to sourcepath" + ) + for ((packageName, javaFileObject) <- dirtyFiles) { + addToSourcepath(packageName, javaFileObject) + } + } + } + + private def addToSourcepath( + packageName: String, + javaFileObject: JavaFileObject, + ): Unit = { require( !packageName.endsWith("/"), s"package name '$packageName' cannot end with '/'. It should be a javac dot-separate package name like 'com.foo'", @@ -295,8 +322,6 @@ class TurbineCompiler[T]( item.ne(obj) && item.javaFileObject.getName() == obj.javaFileObject.getName() ) - - doCompile(()) } def createFileManager( diff --git a/tests/unit/src/test/scala/tests/mbt/TurbineCacheSuite.scala b/tests/unit/src/test/scala/tests/mbt/TurbineCacheSuite.scala index 38976ecfec35..790e0a2d8277 100644 --- a/tests/unit/src/test/scala/tests/mbt/TurbineCacheSuite.scala +++ b/tests/unit/src/test/scala/tests/mbt/TurbineCacheSuite.scala @@ -195,4 +195,144 @@ class TurbineCacheSuite extends BaseLspSuite("turbine-cache") with TestHovers { ) } yield () } + + private val dirtyFilesTestName = "turbine-cache-handles-dirty-files" + + private val javaContentsWithNewMethod = + """|package com.example; + | + |import com.google.common.collect.ImmutableList; + | + |public class Hello { + | public static class Person { + | public final String name; + | + | public Person(String name) { + | this.name = name; + | } + | } + | + | public static String greet(String name) { + | return "Hello, " + name + "!"; + | } + | + | public static ImmutableList names() { + | return ImmutableList.of("Alice", "Bob"); + | } + | + | public static Person person(String name) { + | return new Person(name); + | } + | + | public static String farewell(String name) { + | return "Goodbye, " + name + "!"; + | } + |} + |""".stripMargin + + private val scalaContentsWithFarewell = + """|package com.example + | + |object Main { + | def run(): String = Hello.greet("World") + | def names = Hello.names() + | def person: Hello.Person = Hello.person("Alice") + | def bye(): String = Hello.farewell("World") + |} + |""".stripMargin + + private val farewellHover = + """|```scala + |def farewell(name: String): String + |``` + |""".stripMargin.hover + + test(dirtyFilesTestName) { + cleanWorkspace() + val mbtJson = new MbtJsonBuilder(BuildInfo.scalaVersion) + .addJavaDependency("com.google.guava", "guava", "33.5.0-jre") + .addScalaLibrary() + .addNamespace("core", List("src/**")) + .build() + + for { + // Step 1: Initialize workspace and create cache + _ <- initialize( + s"""|/.metals/mbt.json + |$mbtJson + |/$javaFile + |$javaContents + |/$scalaFile + |$scalaContents + |""".stripMargin + ) + + _ <- server.didOpen(javaFile) + _ <- server.didOpen(scalaFile) + _ = assertNoDiagnostics() + _ <- assertHovers() + + cachePath = workspace.resolve(Directories.turbineCache) + _ = assert( + Files.exists(cachePath.toNIO), + s"Turbine cache file should exist at $cachePath after compilation", + ) + + // Step 2: Restart server + _ = cancelServer() + _ = newServer(dirtyFilesTestName) + + // Step 3: Modify the Java file on disk BEFORE starting the server + // This simulates uncommitted changes (dirty files) + _ = Files.writeString( + workspace.resolve(javaFile).toNIO, + javaContentsWithNewMethod, + ) + _ = Files.writeString( + workspace.resolve(scalaFile).toNIO, + scalaContentsWithFarewell, + ) + + _ <- server.initialize() + _ <- server.initialized() + _ <- server.didChangeConfiguration(userConfig.toString) + _ = server.assertBuildServerConnection() + + _ <- server.didOpen(javaFile) + _ <- server.didOpen(scalaFile) + _ = assertNoDiagnostics() + + // Step 4: Verify that the NEW method is visible via hover + // This proves that dirty files are added to sourcepath and take precedence + // over the cached compiled classes + _ <- server.assertHover( + scalaFile, + """|package com.example + | + |object Main { + | def run(): String = Hello.greet("World") + | def names = Hello.names() + | def person: Hello.Person = Hello.person("Alice") + | def bye(): String = Hello.fare@@well("World") + |} + |""".stripMargin, + farewellHover, + ) + + // Original methods should still work + _ <- server.assertHover( + scalaFile, + """|package com.example + | + |object Main { + | def run(): String = Hello.gre@@et("World") + | def names = Hello.names() + | def person: Hello.Person = Hello.person("Alice") + | def bye(): String = Hello.farewell("World") + |} + |""".stripMargin, + greetHover, + ) + } yield () + } }