LSP support

buildSrc/src/main/kotlin/pklFatJar.gradle.kts

@@ -69,6 +69,10 @@ val relocations =
 
     // pkl-codegen-kotlin dependencies
     "com.squareup.kotlinpoet." to "org.pkl.thirdparty.kotlinpoet.",
+
+    // pkl-lsp dependencies
+    "com.google.gson" to "org.pkl.thirdparty.gson",
+    "org.eclipse.lsp4j" to "org.pkl.thirdparty.lsp4j",
   )
 
 val nonRelocations = listOf("com/oracle/truffle/")
@@ -87,6 +91,9 @@ tasks.shadowJar {
   // org.antlr.v4.runtime.misc.RuleDependencyProcessor
   exclude("META-INF/services/javax.annotation.processing.Processor")
 
+  // org.eclipse.lsp4j
+  exclude("about.html")
+
   exclude("module-info.*")
 
   for ((from, to) in relocations) {

gradle/libs.versions.toml

@@ -38,6 +38,7 @@ ktfmt = "0.44"
 # replaces nuValidator's log4j dependency
 # something related to log4j-1.2-api is apparently broken in 2.17.2
 log4j = "2.17.1"
+lsp4j = "0.23.1"
 msgpack = "0.9.0"
 nexusPublishPlugin = "1.3.0"
 nuValidator = "20.+"
@@ -82,6 +83,7 @@ kotlinStdLib = { group = "org.jetbrains.kotlin", name = "kotlin-stdlib-jdk8", ve
 kotlinxHtml = { group = "org.jetbrains.kotlinx", name = "kotlinx-html-jvm", version.ref = "kotlinxHtml" }
 kotlinxSerializationJson = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerialization" }
 log4j12Api = { group = "org.apache.logging.log4j", name = "log4j-1.2-api", version.ref = "log4j" }
+lsp4j = { group = "org.eclipse.lsp4j", name = "org.eclipse.lsp4j", version.ref = "lsp4j" }
 msgpack = { group = "org.msgpack", name = "msgpack-core", version.ref = "msgpack" }
 nuValidator = { group = "nu.validator", name = "validator", version.ref = "nuValidator" }
 # to be replaced with https://github.com/usethesource/capsule or https://github.com/lacuna/bifurcan

pkl-cli/gradle.lockfile

@@ -12,6 +12,7 @@ com.github.ajalt.clikt:clikt-jvm:3.5.1=compileClasspath,default,runtimeClasspath
 com.github.ajalt.clikt:clikt:3.5.1=apiDependenciesMetadata,compileClasspath,default,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.github.jknack:handlebars-helpers:4.3.1=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.github.jknack:handlebars:4.3.1=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+com.google.code.gson:gson:2.10.1=runtimeClasspath,testRuntimeClasspath
 com.google.errorprone:error_prone_annotations:2.26.1=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.google.guava:failureaccess:1.0.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.google.guava:guava:33.2.0-jre=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
@@ -54,6 +55,8 @@ org.eclipse.jetty:jetty-servlets:11.0.20=testCompileClasspath,testImplementation
 org.eclipse.jetty:jetty-util:11.0.20=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.eclipse.jetty:jetty-webapp:11.0.20=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.eclipse.jetty:jetty-xml:11.0.20=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.eclipse.lsp4j:org.eclipse.lsp4j.jsonrpc:0.22.0=runtimeClasspath,testRuntimeClasspath
+org.eclipse.lsp4j:org.eclipse.lsp4j:0.22.0=runtimeClasspath,testRuntimeClasspath
 org.fusesource.jansi:jansi:2.4.0=default
 org.fusesource.jansi:jansi:2.4.1=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.graalvm.compiler:compiler:23.0.2=compileClasspath,compileOnlyDependenciesMetadata

pkl-cli/pkl-cli.gradle.kts

@@ -62,6 +62,7 @@ dependencies {
   implementation(libs.jlineTerminal)
   implementation(libs.jlineTerminalJansi)
   implementation(projects.pklServer)
+  implementation(projects.pklLsp)
   implementation(libs.clikt) {
     // force clikt to use our version of the kotlin stdlib
     exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib-jdk8")

pkl-cli/src/main/kotlin/org/pkl/cli/Main.kt

@@ -34,7 +34,8 @@ internal fun main(args: Array<String>) {
         ServerCommand(helpLink),
         TestCommand(helpLink),
         ProjectCommand(helpLink),
-        DownloadPackageCommand(helpLink)
+        DownloadPackageCommand(helpLink),
+        LspCommand(helpLink)
       )
       .main(args)
   }

pkl-cli/src/main/kotlin/org/pkl/cli/commands/LspCommand.kt

@@ -0,0 +1,39 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.cli.commands
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.options.flag
+import com.github.ajalt.clikt.parameters.options.option
+import org.pkl.commons.cli.commands.single
+import org.pkl.lsp.PklLSP
+
+class LspCommand(helpLink: String) :
+  CliktCommand(
+    name = "lsp",
+    help = "Run a Language Server Protocol server that communicates over standard input/output",
+    epilog = "For more information, visit $helpLink"
+  ) {
+
+  private val verbose: Boolean by
+    option(names = arrayOf("--verbose"), help = "Send debug information to the client")
+      .single()
+      .flag(default = false)
+
+  override fun run() {
+    PklLSP.run(verbose)
+  }
+}

pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java

@@ -2721,7 +2721,7 @@ private String getCommonIndent(MultiLineStringPartContext lastPart, Token endQuo
         .build();
   }
 
-  private static boolean isIndentChars(Token token) {
+  public static boolean isIndentChars(Token token) {
     var text = token.getText();
 
     for (var i = 0; i < text.length(); i++) {
@@ -2732,7 +2732,7 @@ private static boolean isIndentChars(Token token) {
     return true;
   }
 
-  private static String getLeadingIndent(Token token) {
+  public static String getLeadingIndent(Token token) {
     var text = token.getText();
 
     for (var i = 0; i < text.length(); i++) {

pkl-lsp/gradle.lockfile

@@ -0,0 +1,38 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+com.google.code.gson:gson:2.10.1=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+com.tunnelvisionlabs:antlr4-runtime:4.9.0=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+net.bytebuddy:byte-buddy:1.14.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+net.java.dev.jna:jna:5.6.0=kotlinCompilerClasspath,kotlinKlibCommonizerClasspath
+org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeOnlyDependenciesMetadata
+org.assertj:assertj-core:3.25.3=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.eclipse.lsp4j:org.eclipse.lsp4j.jsonrpc:0.23.1=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.eclipse.lsp4j:org.eclipse.lsp4j:0.23.1=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.graalvm.sdk:graal-sdk:22.3.3=runtimeClasspath,testRuntimeClasspath
+org.graalvm.truffle:truffle-api:22.3.3=runtimeClasspath,testRuntimeClasspath
+org.jetbrains.intellij.deps:trove4j:1.0.20200330=kotlinCompilerClasspath,kotlinKlibCommonizerClasspath
+org.jetbrains.kotlin:kotlin-compiler-embeddable:1.7.10=kotlinCompilerClasspath,kotlinKlibCommonizerClasspath
+org.jetbrains.kotlin:kotlin-daemon-embeddable:1.7.10=kotlinCompilerClasspath,kotlinKlibCommonizerClasspath
+org.jetbrains.kotlin:kotlin-klib-commonizer-embeddable:1.7.10=kotlinKlibCommonizerClasspath
+org.jetbrains.kotlin:kotlin-reflect:1.7.10=kotlinCompilerClasspath,kotlinKlibCommonizerClasspath
+org.jetbrains.kotlin:kotlin-script-runtime:1.7.10=kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath
+org.jetbrains.kotlin:kotlin-scripting-common:1.7.10=kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest
+org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable:1.7.10=kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest
+org.jetbrains.kotlin:kotlin-scripting-compiler-impl-embeddable:1.7.10=kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest
+org.jetbrains.kotlin:kotlin-scripting-jvm:1.7.10=kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest
+org.jetbrains.kotlin:kotlin-stdlib-common:1.7.10=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.7.10=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.7.10=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib:1.7.10=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.jetbrains:annotations:13.0=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-api:5.10.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit.jupiter:junit-jupiter-engine:5.10.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit.jupiter:junit-jupiter-params:5.10.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.junit.platform:junit-platform-commons:1.10.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit.platform:junit-platform-engine:1.10.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.junit:junit-bom:5.10.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
+org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath
+empty=annotationProcessor,compileOnlyDependenciesMetadata,intransitiveDependenciesMetadata,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDef,kotlinScriptDefExtensions,runtimeOnlyDependenciesMetadata,sourcesJar,testAnnotationProcessor,testApiDependenciesMetadata,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDef,testKotlinScriptDefExtensions

pkl-lsp/pkl-lsp.gradle.kts

@@ -0,0 +1,27 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+  pklAllProjects
+  pklKotlinLibrary
+}
+
+dependencies {
+  implementation(projects.pklCore)
+  implementation(libs.antlrRuntime)
+  implementation(libs.lsp4j)
+}
+
+tasks.test {}

pkl-lsp/src/main/kotlin/org/pkl/lsp/Builder.kt

@@ -0,0 +1,129 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.lsp
+
+import java.io.File
+import java.io.IOException
+import java.net.URI
+import java.util.concurrent.CompletableFuture
+import org.eclipse.lsp4j.Diagnostic
+import org.eclipse.lsp4j.DiagnosticSeverity
+import org.eclipse.lsp4j.PublishDiagnosticsParams
+import org.pkl.core.parser.LexParseException
+import org.pkl.core.parser.Parser
+import org.pkl.lsp.LSPUtil.toRange
+import org.pkl.lsp.analyzers.*
+import org.pkl.lsp.ast.Node
+import org.pkl.lsp.ast.PklModule
+import org.pkl.lsp.ast.PklModuleImpl
+import org.pkl.lsp.ast.Span
+
+class Builder(private val server: PklLSPServer) {
+  private val runningBuild: MutableMap<String, CompletableFuture<PklModule?>> = mutableMapOf()
+  private val successfulBuilds: MutableMap<String, PklModule> = mutableMapOf()
+
+  private val parser = Parser()
+
+  private val analyzers: List<Analyzer> =
+    listOf(ModifierAnalyzer(server), AnnotationAnalyzer(server), SyntaxAnalyzer(server))
+
+  fun runningBuild(uri: String): CompletableFuture<PklModule?> =
+    runningBuild[uri] ?: CompletableFuture.supplyAsync(::noop)
+
+  fun requestBuild(uri: URI, vfile: VirtualFile, change: String): CompletableFuture<PklModule?> {
+    val build = CompletableFuture.supplyAsync { build(uri, vfile, change) }
+    runningBuild[uri.toString()] = build
+    return build
+  }
+
+  fun lastSuccessfulBuild(uri: String): PklModule? = successfulBuilds[uri]
+
+  private fun build(file: URI, vfile: VirtualFile, change: String): PklModule? {
+    return try {
+      server.logger().log("building $file")
+      val moduleCtx = parser.parseModule(change)
+      val module = PklModuleImpl(moduleCtx, file, vfile)
+      val diagnostics = analyze(module)
+      makeDiagnostics(file, diagnostics)
+      successfulBuilds[file.toString()] = module
+      return module
+    } catch (e: LexParseException) {
+      server.logger().error("Parser Error building $file: ${e.message}")
+      makeParserDiagnostics(file, listOf(toParserError(e)))
+      null
+    } catch (e: Exception) {
+      server.logger().error("Error building $file: ${e.message} ${e.stackTraceToString()}")
+      null
+    }
+  }
+
+  private fun analyze(node: Node): List<Diagnostic> {
+    return buildList<PklDiagnostic> {
+      for (analyzer in analyzers) {
+        analyzer.analyze(node, this)
+      }
+    }
+  }
+
+  private fun makeParserDiagnostics(file: URI, errors: List<ParseError>) {
+    val diags =
+      errors.map { err ->
+        val msg = ErrorMessages.create(err.errorType, *err.args)
+        val diag = Diagnostic(err.span.toRange(), "$msg\n\n")
+        diag.severity = DiagnosticSeverity.Error
+        diag.source = "Pkl Language Server"
+        server.logger().log("diagnostic: $msg at ${err.span}")
+        diag
+      }
+    makeDiagnostics(file, diags)
+  }
+
+  private fun makeDiagnostics(file: URI, diags: List<Diagnostic>) {
+    server.logger().log("Found ${diags.size} diagnostic errors for $file")
+    val params = PublishDiagnosticsParams(file.toString(), diags)
+    // Have to publish diagnostics even if there are no errors, so we clear previous problems
+    server.client().publishDiagnostics(params)
+  }
+
+  companion object {
+    private fun noop(): PklModule? {
+      return null
+    }
+
+    fun fileToModule(file: File, virtualFile: VirtualFile): PklModule? {
+      if (!file.exists() || file.isDirectory) return null
+      val change = file.readText()
+      return fileToModule(change, file.normalize().toURI(), virtualFile)
+    }
+
+    fun fileToModule(contents: String, uri: URI, virtualFile: VirtualFile): PklModule? {
+      val parser = Parser()
+      try {
+        val moduleCtx = parser.parseModule(contents)
+        return PklModuleImpl(moduleCtx, uri, virtualFile)
+      } catch (_: IOException) {
+        return null
+      }
+    }
+
+    private fun toParserError(ex: LexParseException): ParseError {
+      val span = Span(ex.line, ex.column, ex.line, ex.column + ex.length)
+      return ParseError(ex.message ?: "Parser error", span)
+    }
+  }
+}
+
+class ParseError(val errorType: String, val span: Span, vararg val args: Any)

pkl-lsp/src/main/kotlin/org/pkl/lsp/CacheManager.kt

@@ -0,0 +1,55 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.lsp
+
+import java.net.URI
+import java.net.URLEncoder
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.*
+import org.pkl.lsp.ast.PklModule
+
+/** Manages all Pkl files that are not local to the file system: http(s), packages. */
+object CacheManager {
+  val pklDir = Path.of(System.getProperty("user.home")).resolve(".pkl")
+  val pklCacheDir = pklDir.resolve(".cache")
+
+  val lspCacheDir = Files.createTempDirectory("pkl-lsp-cache")
+
+  private fun findHttpPath(uri: URI): Path? {
+    if (!uri.scheme.equals("https", ignoreCase = true)) return null
+    val encoded = URLEncoder.encode(uri.toString(), Charsets.UTF_8)
+    return lspCacheDir.resolve(encoded)
+  }
+
+  fun findHttpContent(uri: URI): String? {
+    val path = findHttpPath(uri) ?: return null
+    return if (path.exists() && path.isRegularFile() && path.isReadable()) {
+      // uri is cached
+      path.readText()
+    } else {
+      val content = uri.toURL().readText()
+      path.writeText(content)
+      content
+    }
+  }
+
+  fun findHttpModule(uri: URI): PklModule? {
+    return findHttpContent(uri)?.let { contents ->
+      Builder.fileToModule(contents, uri, HttpsFile(uri))
+    }
+  }
+}