From 10bf91b3b18065016889133cffdd15c0e700f1ad Mon Sep 17 00:00:00 2001 From: rochala Date: Fri, 17 Oct 2025 00:48:49 +0200 Subject: [PATCH] Implement definition provider, atm without range support for Scala 2 and Java symbols --- build.mill | 16 +- .../scala/abusers/sls/BspStateManager.scala | 57 ++++++-- .../abusers/sls/ClassFileSourceResolver.scala | 137 ++++++++++++++++++ .../abusers/sls/DefinitionProvider.scala | 105 ++++++++++++++ .../org/scala/abusers/sls/ServerImpl.scala | 30 ++-- .../org/scala/abusers/sls/UnzipUtils.scala | 75 ++++++++++ 6 files changed, 377 insertions(+), 43 deletions(-) create mode 100644 sls/src/org/scala/abusers/sls/ClassFileSourceResolver.scala create mode 100644 sls/src/org/scala/abusers/sls/DefinitionProvider.scala create mode 100644 sls/src/org/scala/abusers/sls/UnzipUtils.scala diff --git a/build.mill b/build.mill index 63adf4b..0ce371e 100644 --- a/build.mill +++ b/build.mill @@ -42,7 +42,7 @@ object slsSmithy extends CommonScalaModule with Smithy4sModule { ) } -object profilingRuntime extends CommonScalaModule with PublishModule { +object profilingRuntime extends CommonScalaModule { def ivyDeps = Agg( ivy"io.pyroscope:agent:2.1.2", ivy"org.typelevel::cats-effect:3.6.3", @@ -51,19 +51,6 @@ object profilingRuntime extends CommonScalaModule with PublishModule { ivy"org.typelevel::otel4s-experimental-metrics:0.7.0", ivy"org.typelevel::otel4s-instrumentation-metrics:0.13.1" ) - - def pomSettings: T[PomSettings] = PomSettings( - description = "A runtime with pyroscope profiling and OpenTelemetry tracing", - url = "", - licenses = Seq(License.`MPL-2.0`), - organization = "org.scala.abusers", - versionControl = VersionControl.github("simple-scala-tooling", "simple-language-server"), - developers = Seq( - Developer("rochala", "Jędrzej Rochala", url = "https://github.com/rochala") - ) - ) - - def publishVersion: T[String] = "0.0.1-SNAPSHOT" } object sls extends CommonScalaModule { @@ -89,6 +76,7 @@ object sls extends CommonScalaModule { ivy"com.evolution::scache:5.1.2", ivy"org.typelevel::cats-parse:1.1.0", ivy"io.get-coursier:interface:1.0.28", + ivy"org.ow2.asm:asm:9.6", ) def javacOptions = Seq( diff --git a/sls/src/org/scala/abusers/sls/BspStateManager.scala b/sls/src/org/scala/abusers/sls/BspStateManager.scala index 1fba201..51260ac 100644 --- a/sls/src/org/scala/abusers/sls/BspStateManager.scala +++ b/sls/src/org/scala/abusers/sls/BspStateManager.scala @@ -10,10 +10,14 @@ import cats.effect.std.AtomicCell import cats.effect.IO import org.scala.abusers.pc.ScalaVersion import org.scala.abusers.sls.LoggingUtils.* +import bsp.DependencyModule.DependencyModuleMavenDependencyModule import java.net.URI +import bsp.DependencyModulesParams +import bsp.BuildTargetIdentifier +import bsp.DependencyModule -type ScalaBuildTargetInformation = (scalacOptions: ScalacOptionsItem, buildTarget: BuildTargetScalaBuildTarget) +type ScalaBuildTargetInformation = (scalacOptions: ScalacOptionsItem, buildTarget: BuildTargetScalaBuildTarget, dependencies: List[DependencyModule]) object ScalaBuildTargetInformation { extension (buildTargetInformation: ScalaBuildTargetInformation) { @@ -68,10 +72,10 @@ class BspStateManager( private def getBuildInformation(bspServer: BuildServer): IO[Set[ScalaBuildTargetInformation]] = for { workspaceBuildTargets <- bspServer.generic.workspaceBuildTargets() - scalacOptions <- bspServer.scala.buildTargetScalacOptions( - ScalacOptionsParams(targets = workspaceBuildTargets.targets.map(_.id)) - ) // - } yield buildTargetToScalaTargets(workspaceBuildTargets, scalacOptions) + buildTargets = workspaceBuildTargets.targets.map(_.id) + scalacOptions <- bspServer.scala.buildTargetScalacOptions(ScalacOptionsParams(buildTargets)) + dependencyInformations <- bspServer.generic.buildTargetDependencyModules(DependencyModulesParams(buildTargets)) + } yield toScalaBuildTargetInformation(workspaceBuildTargets, scalacOptions, dependencyInformations) .groupMapReduce(_.buildTarget.id)(identity)(byScalaVersion.max) .values .toSet @@ -85,25 +89,56 @@ class BspStateManager( ) yield inverseSources.targets - private def buildTargetToScalaTargets( + private def toScalaBuildTargetInformation( targets: bsp.WorkspaceBuildTargetsResult, scalacOptions: bsp.scala_.ScalacOptionsResult, + dependencyInformations: bsp.DependencyModulesResult, ): Set[ScalaBuildTargetInformation] = { - val scalacOptions0 = scalacOptions.items.map(item => item.target -> item).toMap + val scalacOptions0 = scalacOptions.items.groupBy(_.target) + val dependencyInformations0 = dependencyInformations.items.groupBy(_.target) val (mismatchedTargets, zippedTargets) = targets.targets.partitionMap { target => - scalacOptions0.get(target.id) match { - case Some(scalacOptionsItem) if target.project.scala.isDefined => - Right(scalacOptions = scalacOptionsItem, buildTarget = target.project.scala.get) + val withScalacOptions = scalacOptions0.get(target.id) match { + case Some(scalacOptionsItems) if target.project.scala.isDefined => + val scalacOptions0 = scalacOptionsItems + .headOption + .getOrElse(sys.error(s"There should be exactly one ScalacOptionsItem for ${target} but was ${scalacOptionsItems}")) + Right(scalacOptions = scalacOptions0, buildTarget = target.project.scala.get) case _ => Left(target.id) } + + val withDependencies: Either[BuildTargetIdentifier, ScalaBuildTargetInformation] = withScalacOptions.flatMap { info => + dependencyInformations0.get(target.id) match { + case Some(dependencies) => + val dependencies0 = dependencies + .headOption + .map(_.modules) + .getOrElse(sys.error(s"There should be exactly one DependencyModulesItem for ${target} but was ${dependencies}")) + Right(info ++ (dependencies = dependencies0)) + case _ => + Left(target.id) + } + } + withDependencies } if mismatchedTargets.nonEmpty then throw new IllegalStateException( - s"Mismatched targets to ScalacOptionsResult probably caused due to existance of java scopes. ${mismatchedTargets.mkString(", ")}" + s"Mismatched options (*probably caused due to existance of java scopes). ${mismatchedTargets.mkString(", ")}" ) else zippedTargets.toSet } + def getDependencyInfo(uri: URI)(using SynchronizedState): IO[Option[DependencyModuleMavenDependencyModule]] = { + val queriedUri = bsp.URI(uri.toString) + + targets.get.map { targets => + targets.iterator.flatMap { + _.dependencies + .collect { case m: DependencyModuleMavenDependencyModule => m } + .find(_.data.artifacts.find(_.uri == queriedUri).isDefined) + }.nextOption() + } + } + /** didOpen / didChange is always a first request in sequence e.g didChange -> completion -> semanticTokens * * We want to fail fast if this is not the case because it is a way bigger problem that we may hide diff --git a/sls/src/org/scala/abusers/sls/ClassFileSourceResolver.scala b/sls/src/org/scala/abusers/sls/ClassFileSourceResolver.scala new file mode 100644 index 0000000..939b95e --- /dev/null +++ b/sls/src/org/scala/abusers/sls/ClassFileSourceResolver.scala @@ -0,0 +1,137 @@ +package org.scala.abusers.sls + +import cats.data.OptionT +import cats.effect.IO +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.Opcodes + +import java.nio.file.Files +import java.nio.file.Path +import scala.jdk.CollectionConverters.* + +/** Fallback algorithm to map classfile paths to source directories. + * + * Implements the algorithm described in JVM specification (JVMS §4.7.10) + * to resolve source file locations from classfile information when direct + * source path information is unavailable. + * + * Since we control compilation through BSP, the SourceFile attribute (JVMS §4.7.10) + * will always be present and includes the full filename with extension. + */ +class ClassFileSourceResolver(rootPath: Path, lspClient: SlsLanguageClient[IO]) { + + private def extractSourceFileName(classFilePath: Path): OptionT[IO, String] = OptionT.fromOption { + val bytes = Files.readAllBytes(classFilePath) + val reader = ClassReader(bytes) + + var sourceFileName: Option[String] = None + + // SourceFile attribute (JVMS §4.7.10): contains source filename with extension + val visitor = new ClassVisitor(Opcodes.ASM9) { + override def visitSource(source: String, debug: String): Unit = { + sourceFileName = Option(source) + } + } + + reader.accept(visitor, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES) + sourceFileName + } + + private def sourceNameFromFullyQualifiedPath(fullyQualifiedPath: String, isJava: Boolean): String = { + val simpleClassName = fullyQualifiedPath.split('.').last + val baseName = simpleClassName.takeWhile(_ != '$') + val ext = if (isJava) "java" else "scala" + s"$baseName.$ext" + } + + private def extractPackageName(fullyQualifiedClassName: String): String = + fullyQualifiedClassName.split('.').dropRight(1).mkString(".") + + /** Find all source files with the given filename + * + * Current implementation: Recursively searches the workspace directory tree. + * + * Enhancement opportunity: We could extract the build target/project name from the + * classfile directory structure (e.g., "out/myproject/compile/...") and query only + * sources from that specific build target via BSP, significantly reducing the search space + * and improving accuracy. + * + * @param sourceFileName The name of the source file to find (e.g., "MyClass.scala") + * @return IO containing list of paths matching the filename + */ + private def findSourcesByName(sourceFileName: String): IO[List[Path]] = IO { + import java.nio.file.{Files, FileVisitOption} + import scala.jdk.StreamConverters.* + + Files.walk(rootPath, FileVisitOption.FOLLOW_LINKS) + .toScala(List) + .filter { path => + Files.isRegularFile(path) && path.getFileName.toString == sourceFileName + } + } + + /** Refine candidate paths using package structure matching + * + * Matches package components in reverse order (innermost to outermost) as per + * the reference algorithm. Prefers paths with the best package alignment. + * + * @param candidates List of candidate source paths + * @param expectedPackage The expected package path (e.g., "com.example") + * @return List of best-matched paths based on package structure + */ + private def refineByPackageStructure( + candidates: List[Path], + expectedPackage: String, + ): List[Path] = candidates match { + case Nil => Nil + case single :: Nil => List(single) + case multiple => + val packageParts = if (expectedPackage.nonEmpty) expectedPackage.split('.').toList.reverse else Nil + + val scored = multiple.map { path => + val pathParts = path.iterator.asScala.map(_.toString).toList.reverse.tail + // Match package components from innermost to outermost + val score = packageParts.zip(pathParts).count { case (pkg, dir) => pkg == dir } + (path, score) + } + + val maxScore = scored.map(_._2).maxOption.getOrElse(0) + + if (maxScore > 0) { + // Return paths with the best package alignment + scored.filter(_._2 == maxScore).map(_._1) + } else { + // If no package match, prefer shorter paths (fewer directory components) + val minDepth = multiple.map(_.getNameCount).min + multiple.filter(_.getNameCount == minDepth) + } + } + + def resolveSourceFromClassFile( + classFilePath: String, + fullyQualifiedClassName: String, + isJava: Boolean + ): OptionT[IO, Path] = OptionT( + for { + path <- IO.pure(Path.of(classFilePath)) + sourceFileName <- extractSourceFileName(path) + .getOrElse(sourceNameFromFullyQualifiedPath(fullyQualifiedClassName, isJava)) + packageName = extractPackageName(fullyQualifiedClassName) + candidates <- findSourcesByName(sourceFileName) + refined = refineByPackageStructure(candidates, packageName) + } yield refined match { + case single :: Nil => Some(single) + case _ => None + } + ) +} + +object ClassFileSourceResolver { + def instance( + rootPath: Path, + lspClient: SlsLanguageClient[IO], + ): IO[ClassFileSourceResolver] = { + IO.pure(new ClassFileSourceResolver(rootPath, lspClient)) + } +} diff --git a/sls/src/org/scala/abusers/sls/DefinitionProvider.scala b/sls/src/org/scala/abusers/sls/DefinitionProvider.scala new file mode 100644 index 0000000..553d9fc --- /dev/null +++ b/sls/src/org/scala/abusers/sls/DefinitionProvider.scala @@ -0,0 +1,105 @@ +package org.scala.abusers.sls + +import cats.effect.IO +import cats.syntax.all.* +import fs2.io.file.Files +import io.scalaland.chimney.dsl._ +import java.net.URI +import scala.meta.pc.SymbolSource +import scala.meta.pc.SymbolSource.ExternalTastySymbolSource +import scala.meta.pc.SymbolSource.InternalTastySymbolSource +import scala.meta.pc.SymbolSource.InternalClassFileSymbolSource +import scala.meta.pc.SymbolSource.ExternalClassFileSymbolSource +import scala.meta.pc.SymbolSource.ScalaFileSymbolSource +import cats.data.OptionT + +class DefinitionProvider(rootPath: os.Path, lspClient: SlsLanguageClient[IO], bspStateManager: BspStateManager) { + val cacheDir = fs2.io.file.Path.fromNioPath((rootPath / ".sls" / "sources").toNIO) + + def definition(symbolSource: SymbolSource)(using SynchronizedState): IO[List[lsp.Location]] = { + symbolSource match { + case tastySource: ExternalTastySymbolSource => + val range = tastySource.getRange().into[lsp.Range].enableBeanGetters.transform + findLocationsFromTasty(tastySource.getTastyJarPath(), tastySource.getInJarTastyFilePath(), range).value.map(_.toList.flatten) + case tastySource: InternalTastySymbolSource => + val range = tastySource.getRange().into[lsp.Range].enableBeanGetters.transform + findScalaSource(tastySource.getSourcePath(), range).value.map(_.toList.flatten) + case classFileSource: ExternalClassFileSymbolSource => + findLocationsFromClassFile(classFileSource.getClassFileJarPath(), classFileSource.getInJarClassFilePath(), classFileSource.isJava()) + case classFileSource: InternalClassFileSymbolSource => + findLocationFromInternalClassFile(classFileSource.getClassFilePath(), classFileSource.getFullyQualifiedPath(), classFileSource.isJava()) + case scalaSymbolSource: ScalaFileSymbolSource => + List( + lsp.Location( + uri = scalaSymbolSource.getSourcePath(), + range = scalaSymbolSource.getRange().into[lsp.Range].enableBeanGetters.transform + ) + ).pure + } + } + + def fromURI(uri: String): OptionT[IO, fs2.io.file.Path] = { + OptionT.pure[IO](URI.create(uri).getPath).map(fs2.io.file.Path.apply) + } + + def findLocationsFromTasty(tastyJarPath: String, inTastyJarPath: String, range: lsp.Range)(using SynchronizedState): OptionT[IO, List[lsp.Location]] = { + for { + tastyJarPath <- fromURI(tastyJarPath).filterF(Files[IO].exists) + sourceJarPath <- findSourceJar(tastyJarPath) + thisJarCacheDir = cacheDir.resolve(sourceJarPath.fileName) + _ <- OptionT.liftF(UnzipUtils.unzipJarFromPath(sourceJarPath, thisJarCacheDir)) + } yield { + lsp.Location( + uri = thisJarCacheDir.resolve(inTastyJarPath.stripSuffix(".tasty") + ".scala").toString, + range = range + ) :: Nil + } + } + + def findLocationFromInternalClassFile(classFilePath: String, fullyQualifiedPath: String, isJava: Boolean): IO[List[lsp.Location]] = + ClassFileSourceResolver(rootPath.toNIO, lspClient) + .resolveSourceFromClassFile(classFilePath, fullyQualifiedPath, isJava).value + .map(maybeSource => maybeSource.map { sourcePath => + lsp.Location( + uri = sourcePath.toUri.toString, + range = lsp.Range(lsp.Position(0, 0), lsp.Position(0, 0)) // TODO: Extract range via treesitter / scalameta / JDT + ) + }.toList) + + def findLocationsFromClassFile(classJarPath: String, inClassJarPath: String, isJava: Boolean)(using SynchronizedState): IO[List[lsp.Location]] = { + (for { + classJarPath <- fromURI(classJarPath).filterF(Files[IO].exists) + sourceJarPath <- findSourceJar(classJarPath) + thisJarCacheDir = cacheDir.resolve(sourceJarPath.fileName) + _ <- OptionT.liftF(UnzipUtils.unzipJarFromPath(sourceJarPath, thisJarCacheDir)) + } yield { + val suffix = if isJava then ".java" else ".scala" + lsp.Location( + uri = thisJarCacheDir.resolve(inClassJarPath.stripSuffix(".class") + suffix).toString, + range = lsp.Range(lsp.Position(0, 0), lsp.Position(0, 0)) // TODO: Extract range via treesitter / scalameta / JDT + ) :: Nil + }).value.map(_.getOrElse(Nil)) + } + + def findSourceJar(tastyJarPath: fs2.io.file.Path)(using SynchronizedState): OptionT[IO, fs2.io.file.Path] = { + import org.scala.abusers.sls.NioConverter.asNio + for { + dependencyInfo <- OptionT(bspStateManager.getDependencyInfo(tastyJarPath.toNioPath.toUri)) + sourceJar <- dependencyInfo.data.artifacts.find(_.classifier.contains("sources")).map { artifact => + fs2.io.file.Path(artifact.uri.asNio.getPath()) + }.toOptionT[IO].filterF(Files[IO].exists) + } yield (sourceJar) + } + + + def findScalaSource(path: String, range: lsp.Range): OptionT[IO, List[lsp.Location]] = { + fromURI(path) + .filterF(Files[IO].exists) + .map { fs2Path => + lsp.Location( + uri = fs2Path.toNioPath.toUri.toString, + range = range + ) :: Nil + } + } +} diff --git a/sls/src/org/scala/abusers/sls/ServerImpl.scala b/sls/src/org/scala/abusers/sls/ServerImpl.scala index 599d3eb..a4bfbab 100644 --- a/sls/src/org/scala/abusers/sls/ServerImpl.scala +++ b/sls/src/org/scala/abusers/sls/ServerImpl.scala @@ -24,21 +24,18 @@ import smithy4s.schema.Schema import util.chaining.* import java.net.URI -import java.util.concurrent.CompletableFuture import scala.concurrent.duration.* import scala.jdk.CollectionConverters.* import scala.jdk.OptionConverters.* import scala.meta.pc.CancelToken import scala.meta.pc.InlayHintsParams import scala.meta.pc.OffsetParams -import scala.meta.pc.PresentationCompiler import scala.meta.pc.VirtualFileParams import LoggingUtils.* import ScalaBuildTargetInformation.* import scala.meta.pc.RawPresentationCompiler -import lsp.CompletionTriggerKind.TRIGGER_CHARACTER -import lsp.CompletionTriggerKind.TRIGGER_FOR_INCOMPLETE_COMPLETIONS +import scala.meta.pc.SymbolSource class ServerImpl( pcProvider: PresentationCompilerProvider, @@ -53,6 +50,8 @@ class ServerImpl( )(using Tracer[IO], Meter[IO]) extends SlsLanguageServer[IO] { + val rootPathDeferred = Deferred.unsafe[IO, os.Path] + /* There can only be one client for one language-server */ def initializeOp(params: lsp.InitializeParams): IO[lsp.InitializeOpOutput] = { @@ -60,6 +59,7 @@ class ServerImpl( val rootPath = os.Path(java.net.URI.create(rootUri).getPath()) (for { _ <- lspClient.logMessage("Ready to initialise!") + _ <- rootPathDeferred.complete(rootPath) _ <- importMillBsp(rootPath) bspClient <- connectWithBloop(steward, diagnosticManager) _ <- lspClient.logMessage("Connection with bloop estabilished") @@ -166,20 +166,14 @@ class ServerImpl( } } - def handleDefinition(params: lsp.DefinitionParams)(using SynchronizedState) = - { - offsetParamsRequest(params)(_.definition).map { result => - lsp.TextDocumentDefinitionOpOutput( - result - .locations() - .asScala - .headOption - .map(definition => - convert[lsp4j.Location, lsp.DefinitionOrListOfDefinitionLink](definition) - ) // FIXME: missing completion on lsp.TextDocumentDefinitionOpOutput - ) - } - } + def handleDefinition(params: lsp.DefinitionParams)(using SynchronizedState): IO[lsp.TextDocumentDefinitionOpOutput] = + ( + for { + symbolSources <- offsetParamsRequest(params)(_.symbolSource).map(_.asScala.toList) + rootPath <- rootPathDeferred.get + locations <- symbolSources.parFlatTraverse(DefinitionProvider(rootPath, lspClient, bspStateManager).definition) + } yield lsp.DefinitionOrListOfDefinitionLink.Case0Case(lsp.Definition.Case1Case(locations)).some + ).map(lsp.TextDocumentDefinitionOpOutput.apply) def virtualFileParams(uri0: URI, content: String, token0: CancelToken): VirtualFileParams = new VirtualFileParams { override def text(): String = content diff --git a/sls/src/org/scala/abusers/sls/UnzipUtils.scala b/sls/src/org/scala/abusers/sls/UnzipUtils.scala new file mode 100644 index 0000000..07754e3 --- /dev/null +++ b/sls/src/org/scala/abusers/sls/UnzipUtils.scala @@ -0,0 +1,75 @@ +package org.scala.abusers.sls + +import cats.effect.IO +import fs2.Stream +import fs2.io.file.{Files, Path} +import java.io.InputStream +import java.util.zip.{ZipEntry, ZipInputStream} + +object UnzipUtils { + + def unzipJar(jarInputStream: InputStream, targetDirectory: Path): IO[Unit] = + Files[IO].createDirectories(targetDirectory) *> + Stream + .bracket(IO(new ZipInputStream(jarInputStream)))(zis => IO(zis.close())) + .flatMap { zipInputStream => + Stream + .unfoldEval(zipInputStream) { zis => + IO.blocking { // Check to wrap this in fs2 streams ? + Option(zis.getNextEntry()).map { entry => + val entryData = readEntryBytes(zis, entry) + ((entry, entryData), zis) + } + } + } + .evalMap { case (entry, data) => + processZipEntry(entry, data, targetDirectory) + } + } + .compile + .drain + + def unzipJarFromPath(jarPath: Path, targetDirectory: Path): IO[Unit] = + Files[IO].exists(targetDirectory).flatMap { + case true => + cats.effect.std.Console[IO].errorln(s"Target directory $targetDirectory already exists. Aborting unzip operation.") *> IO.unit + case false => Files[IO] + .readAll(jarPath) + .through(fs2.io.toInputStream[IO]) + .evalMap(unzipJar(_, targetDirectory)) + .compile + .drain + } + + + private def readEntryBytes(zis: ZipInputStream, entry: ZipEntry): Array[Byte] = { + if (entry.isDirectory) { + Array.empty[Byte] + } else { + val buffer = new Array[Byte](8192) + val result = scala.collection.mutable.ArrayBuffer[Byte]() + var bytesRead = zis.read(buffer) + while (bytesRead != -1) { + result ++= buffer.take(bytesRead) + bytesRead = zis.read(buffer) + } + result.toArray + } + } + + private def processZipEntry(entry: ZipEntry, data: Array[Byte], targetDirectory: Path): IO[Unit] = { + val entryPath = targetDirectory / Path(entry.getName) + + if (entry.isDirectory) { + Files[IO].createDirectories(entryPath) + } else { + // Ensure parent directories exist + Files[IO].createDirectories(entryPath.parent.getOrElse(targetDirectory)) *> + Stream + .emits(data) + .through(Files[IO].writeAll(entryPath)) + .compile + .drain + } + } +}