Add file system support for jars

metals/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider

@@ -0,0 +1 @@
+scala.meta.internal.metals.filesystem.MetalsFileSystemProvider

metals/src/main/scala/scala/meta/internal/decorations/SyntheticsDecorationProvider.scala

@@ -282,12 +282,16 @@ final class SyntheticsDecorationProvider(
       textDocument: Option[s.TextDocument],
       path: AbsolutePath
   ): Option[TextDocument] = {
-    for {
-      doc <- textDocument
-      source <- fingerprints.loadLastValid(path, doc.md5, charset)
-      docWithText = doc.withText(source)
-      _ = Document.set(docWithText)
-    } yield docWithText
+    // TODO what does `enrichWithText` do and should it be run on readOnly files?  fingerprints causes slurping ;-(
+    if (path.isReadOnly)
+      textDocument
+    else
+      for {
+        doc <- textDocument
+        source <- fingerprints.loadLastValid(path, doc.md5, charset)
+        docWithText = doc.withText(source)
+        _ = Document.set(docWithText)
+      } yield docWithText
   }
 
   private def localSymbolName(symbol: String, textDoc: TextDocument): String = {

metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala

@@ -395,11 +395,16 @@ final class ImplementationProvider(
     def isDefinitionOccurrence(occ: SymbolOccurrence) =
       occ.role.isDefinition && occ.symbol == symbol
 
+    val input =
+      if (source.isReadOnly)
+        source.toInputFromBuffers(buffer)
+      else
+        source.toInput
     semanticDb.occurrences
       .find(isDefinitionOccurrence)
       .orElse(
         Mtags
-          .allToplevels(source.toInput, scalaVersionSelector.getDialect(source))
+          .allToplevels(input, scalaVersionSelector.getDialect(source))
           .occurrences
           .find(isDefinitionOccurrence)
       )

metals/src/main/scala/scala/meta/internal/metals/BuildTargetInfo.scala

@@ -1,5 +1,6 @@
 package scala.meta.internal.metals
 
+import java.nio.file.Files
 import java.nio.file.Path
 
 import scala.collection.mutable.ListBuffer
@@ -154,7 +155,7 @@ class BuildTargetInfo(buildTargets: BuildTargets) {
     val sourceJarName = jarName.replace(".jar", "-sources.jar")
     buildTargets
       .sourceJarFile(sourceJarName)
-      .exists(_.toFile.exists())
+      .exists(path => path.exists)
   }
 
   private def getSingleClassPathInfo(
@@ -164,7 +165,7 @@ class BuildTargetInfo(buildTargets: BuildTargets) {
   ): String = {
     val filename = shortPath.toString()
     val padding = " " * (maxFilenameSize - filename.size)
-    val status = if (path.toFile.exists) {
+    val status = if (Files.exists(path)) {
       val blankWarning = " " * 9
       if (path.toFile().isDirectory() || jarHasSource(filename))
         blankWarning

metals/src/main/scala/scala/meta/internal/metals/BuildTargets.scala

@@ -13,6 +13,7 @@ import scala.util.control.NonFatal
 
 import scala.meta.internal.io.PathIO
 import scala.meta.internal.metals.MetalsEnrichments._
+import scala.meta.internal.metals.filesystem.MetalsPath
 import scala.meta.internal.mtags.Symbol
 import scala.meta.io.AbsolutePath
 
@@ -93,6 +94,9 @@ final class BuildTargets() {
   def allJava: Iterator[JavaTarget] =
     data.fromIterators(_.allJava)
 
+  def allJDKs: Iterator[String] =
+    data.fromIterators(_.allJDKs).distinct
+
   def info(id: BuildTargetIdentifier): Option[BuildTarget] =
     data.fromOptions(_.info(id))
 
@@ -267,7 +271,8 @@ final class BuildTargets() {
     } yield imports
 
   /**
-   * Tries to guess what build target this readonly file belongs to from the symbols it defines.
+   * Tries to guess what build target this metalsfs or readonly file belongs
+   * to from the symbols it defines.
    *
    * By default, we rely on carefully recording what build target produced what
    * files in the `.metals/readonly/` directory. This approach has the problem
@@ -276,51 +281,53 @@ final class BuildTargets() {
    * - a new metals feature forgot to record the build target
    * - a user removes `.metals/metals.h2.db`
    *
-   * When encountering an unknown `readonly/` file we do the following steps to
+   * When encountering an unknown `readonly/` or metalfs file we do the following steps to
    * infer what build target it belongs to:
    *
    * - check if file is in `.metals/readonly/dependencies/${source-jar-name}`
+   * - check if the file is a jdk file
    * - find the build targets that have a sourceDependency with that name
-   *
-   * Otherwise if it's a jar file we find a build target it belongs to.
+   * - find the build targets that have a workspace jar with that name as class files can be viewed
    *
    * This approach is not glamorous but it seems to work reasonably well.
    */
   def inferBuildTarget(
       source: AbsolutePath
   ): Option[BuildTargetIdentifier] = {
-    if (source.isJarFileSystem) {
-      for {
-        jarName <- source.jarPath.map(_.filename)
-        sourceJarFile <- sourceJarFile(jarName)
-        buildTargetId <- inverseDependencySource(sourceJarFile).headOption
-      } yield buildTargetId
-    } else {
-      val readonly = workspace.resolve(Directories.readonly)
-      source.toRelativeInside(readonly) match {
-        case Some(rel) =>
-          val names = rel.toNIO.iterator().asScala.toList.map(_.filename)
-          names match {
-            case Directories.dependenciesName :: jarName :: _ =>
-              // match build target by source jar name
-              sourceJarFile(jarName)
-                .flatMap(inverseDependencySource(_).headOption)
-            case _ => None
-          }
-        case None =>
-          // else it can be a source file inside a jar
-          val fromJar = jarPath(source)
-            .flatMap { jar =>
-              allBuildTargetIdsInternal.find { case (_, id) =>
-                targetJarClasspath(id).exists(_.contains(jar))
-              }
-            }
-          fromJar.map { case (data0, id) =>
-            data0.addSourceItem(source, id)
-            id
-          }
-      }
+    // source could be on metalFS or in readonly area
+    val metalsPath = source.toNIO match {
+      case metalsPath: MetalsPath => Some(metalsPath)
+      case _ =>
+        val readonly = workspace.resolve(Directories.readonly)
+        source.toRelativeInside(readonly) match {
+          case Some(rel) => Some(MetalsPath.fromReadOnly(rel))
+          case None => None
+        }
     }
+    metalsPath.flatMap(metalsPath => {
+      // check JDK - any build target is OK
+      if (metalsPath.isJDK)
+        allBuildTargetIdsInternal.headOption.map(_._2)
+      else {
+        // check source jars
+        for {
+          jarName <- metalsPath.jarName
+          jarPath <- sourceJarFile(jarName)
+          buildTargetId <- inverseDependencySource(jarPath).headOption
+        } yield buildTargetId
+      }.orElse({
+        // check workspace jars
+        // TODO speed this up - we're iterating over all build targets and all classpath entries - create inverseDependencySource equivalent
+        val targetIds = for {
+          targetId <- allBuildTargetIds
+          classpathEntries <- targetJarClasspath(targetId).toList
+          classpathEntry <- classpathEntries
+          jarName <- metalsPath.jarName
+          if classpathEntry.filename == jarName
+        } yield targetId
+        targetIds.headOption
+      })
+    })
   }
 
   def findByDisplayName(name: String): Option[BuildTarget] = {
@@ -329,14 +336,6 @@ final class BuildTargets() {
       .find(_.getDisplayName() == name)
   }
 
-  private def jarPath(source: AbsolutePath): Option[AbsolutePath] = {
-    source.jarPath.map { sourceJarPath =>
-      sourceJarPath.parent.resolve(
-        source.filename.replace("-sources.jar", ".jar")
-      )
-    }
-  }
-
   /**
    * Returns meta build target for `*.sbt` or `*.scala`  files.
    * It selects build target by directory of its connection

metals/src/main/scala/scala/meta/internal/metals/ClientCommands.scala

@@ -241,6 +241,15 @@ object ClientCommands {
       |""".stripMargin
   )
 
+  val CreateLibraryFileSystem = new ParametrizedCommand[String](
+    "metals-create-library-filesystem",
+    "Create Library FS",
+    """|Notifies the client that it should create an empty
+       |filesystem to navigate jar dependencies
+       |""".stripMargin,
+    arguments = """`string`, the URI root of the filesystem.""".stripMargin
+  )
+
   val RefreshModel = new Command(
     "metals-model-refresh",
     "Refresh model",
@@ -325,6 +334,7 @@ object ClientCommands {
       FocusDiagnostics,
       GotoLocation,
       EchoCommand,
+      CreateLibraryFileSystem,
       RefreshModel,
       ShowStacktrace,
       CopyWorksheetOutput,

metals/src/main/scala/scala/meta/internal/metals/ClientConfiguration.scala

@@ -57,6 +57,9 @@ case class ClientConfiguration(initialConfig: MetalsServerConfig) {
   def isVirtualDocumentSupported(): Boolean =
     initializationOptions.isVirtualDocumentSupported.getOrElse(false)
 
+  def isLibraryFileSystemSupported(): Boolean = true
+  // initializationOptions.isLibraryFileSystemSupported.getOrElse(false)
+
   def icons(): Icons =
     initializationOptions.icons
       .map(Icons.fromString)

metals/src/main/scala/scala/meta/internal/metals/DefinitionProvider.scala

@@ -1,5 +1,6 @@
 package scala.meta.internal.metals
 
+import java.nio.file.Paths
 import java.util.Collections
 import java.{util => ju}
 
@@ -8,6 +9,7 @@ import scala.concurrent.Future
 
 import scala.meta.inputs.Input
 import scala.meta.internal.metals.MetalsEnrichments._
+import scala.meta.internal.metals.filesystem.MetalsFileSystem
 import scala.meta.internal.mtags.GlobalSymbolIndex
 import scala.meta.internal.mtags.Mtags
 import scala.meta.internal.mtags.Semanticdbs
@@ -16,9 +18,11 @@ import scala.meta.internal.mtags.SymbolDefinition
 import scala.meta.internal.parsing.TokenEditDistance
 import scala.meta.internal.parsing.Trees
 import scala.meta.internal.remotels.RemoteLanguageServer
+import scala.meta.internal.semanticdb.Language
 import scala.meta.internal.semanticdb.Scala._
 import scala.meta.internal.semanticdb.SymbolOccurrence
 import scala.meta.internal.semanticdb.TextDocument
+import scala.meta.internal.tvp.ClasspathSymbols
 import scala.meta.io.AbsolutePath
 import scala.meta.pc.CancelToken
 
@@ -265,6 +269,7 @@ final class DefinitionProvider(
   }
 
   private def fromMtags(source: AbsolutePath, dirtyPos: Position) = {
+    // TODO why does a writable file use `source.toInput` instead of `source.toInputFromBuffers(buffers)`
     Mtags
       .allToplevels(source.toInput, scalaVersionSelector.getDialect(source))
       .occurrences
@@ -316,12 +321,28 @@ class DestinationProvider(
   private def bestTextDocument(
       symbolDefinition: SymbolDefinition
   ): TextDocument = {
-    val defnRevisedInput = symbolDefinition.path.toInput
+    val decodedClassfile =
+      if (symbolDefinition.path.isClassfile)
+        MetalsFileSystem.metalsFS
+          .decodeCFRFromClassFile(symbolDefinition.path.toNIO)
+          .map(text =>
+            Input.VirtualFile(symbolDefinition.path.toURI.toString(), text)
+          )
+      else None
+    val defnRevisedInput = decodedClassfile.getOrElse(
+      if (symbolDefinition.path.isReadOnly)
+        symbolDefinition.path.toInputFromBuffers(buffers)
+      else
+        symbolDefinition.path.toInput
+    )
     // Read text file from disk instead of editor buffers because the file
     // on disk is more likely to parse.
+    val language =
+      if (symbolDefinition.path.isClassfile) Language.JAVA
+      else symbolDefinition.path.toLanguage
     lazy val parsed =
       mtags.index(
-        symbolDefinition.path.toLanguage,
+        language,
         defnRevisedInput,
         symbolDefinition.dialect
       )
@@ -346,23 +367,54 @@ class DestinationProvider(
     definition(symbol, targets)
   }
 
+  private val classpathSymbols = new ClasspathSymbols(
+    isStatisticsEnabled = false
+  )
+
   def definition(
       symbol: String,
       allowedBuildTargets: Set[BuildTargetIdentifier]
   ): Option[SymbolDefinition] = {
-    val definitions = index.definitions(Symbol(symbol)).filter(_.path.exists)
+    val querySymbol: Symbol = Symbol(symbol)
+    val definitions = index.definitions(querySymbol).filter(_.path.exists)
+    val extraDefinitions = if (definitions.isEmpty) {
+      // search in all classpath, i.e. jars with no source
+      buildTargets
+        .inferBuildTarget(List(querySymbol.toplevel))
+        .flatMap(buildTarget => {
+          val metalsFSJar =
+            MetalsFileSystem.metalsFS
+              .getOrElseUpdateWorkspaceJar(buildTarget.jar)
+          classpathSymbols
+            .symbols(metalsFSJar, symbol)
+            .filter(_.symbol == symbol)
+            .flatMap(_.uri)
+            .headOption
+            .map(symbolFullURI => {
+              val fullJarPath = AbsolutePath(Paths.get(symbolFullURI))
+              SymbolDefinition(
+                querySymbol,
+                querySymbol,
+                fullJarPath,
+                scala.meta.dialects.Scala212Source3,
+                None
+              )
+            })
+        })
+        .toList
+    } else definitions
     if (allowedBuildTargets.isEmpty)
-      definitions.headOption
+      extraDefinitions.headOption
     else {
-      val matched = definitions.find { defn =>
+      val matched = extraDefinitions.find { defn =>
         sourceBuildTargets(defn.path).exists(id =>
           allowedBuildTargets.contains(id)
         )
       }
       // Fallback to any definition - it's needed for worksheets
       // They might have dynamic `import $dep` and these sources jars
       // aren't registered in buildTargets
-      matched.orElse(definitions.headOption)
+      matched.orElse(extraDefinitions.headOption)
     }
   }