From 4448751feccb4bb01927f415663599c9a4a8a773 Mon Sep 17 00:00:00 2001 From: Cottand Date: Sun, 16 Jun 2024 15:21:47 +0100 Subject: [PATCH 01/19] boilerplate for manipulator, fix bad unescaping of indStrings --- .../idea/psi/impl/AbstractIndNixString.kt | 60 ++++++++++++ .../idea/psi/impl/AbstractNixPsiElement.java | 2 +- .../org/nixos/idea/util/NixIndStringUtil.kt | 34 +++++++ .../org/nixos/idea/util/NixStringUtil.java | 4 + src/main/lang/Nix.bnf | 4 + .../nixos/idea/util/NixIndStringUtilTest.java | 95 +++++++++++++++++++ .../nixos/idea/util/NixStringUtilTest.java | 3 + 7 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/nixos/idea/psi/impl/AbstractIndNixString.kt create mode 100644 src/main/java/org/nixos/idea/util/NixIndStringUtil.kt create mode 100644 src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractIndNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractIndNixString.kt new file mode 100644 index 00000000..f899392e --- /dev/null +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractIndNixString.kt @@ -0,0 +1,60 @@ +package org.nixos.idea.psi.impl + +import com.intellij.lang.ASTNode +import com.intellij.openapi.util.TextRange +import com.intellij.psi.AbstractElementManipulator +import com.intellij.psi.ElementManipulators +import com.intellij.psi.LiteralTextEscaper +import com.intellij.psi.PsiLanguageInjectionHost +import org.intellij.grammar.psi.impl.BnfStringImpl +import org.intellij.grammar.psi.impl.BnfStringManipulator.getStringTokenRange +import org.nixos.idea.util.NixStringUtil + + +abstract class AbstractNixString(node: ASTNode) : PsiLanguageInjectionHost, AbstractNixPsiElement(node) { + + override fun isValidHost() = true + + override fun updateText(text: String): PsiLanguageInjectionHost = + ElementManipulators.handleContentChange(this, text) + + override fun createLiteralTextEscaper() = NixStringLiteralEscaper(this) +} + +class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { + + override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean { + val sb = StringBuilder() + NixStringUtil.escape(sb, rangeInsideHost.toString()) + outChars.append(sb, rangeInsideHost.startOffset, rangeInsideHost.endOffset) + + return true + } + + override fun getOffsetInHost(offsetInDecoded: Int, rangeInsideHost: TextRange): Int = with(rangeInsideHost) { + // todo implement proper java-like string escapes support + val offset = offsetInDecoded + startOffset + return when { + offset < startOffset -> startOffset + offset > endOffset -> endOffset + else -> offset + } + } + + override fun isOneLine(): Boolean = false +} + +class IndStringManipulator : AbstractElementManipulator() { + override fun handleContentChange( + element: NixStringImpl, + range: TextRange, + newContent: String? + ): NixStringImpl? { + + TODO("Not yet implemented") + } + + fun getRangeInElement(element: NixStringImpl?): TextRange { + TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java b/src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java index bc33025c..6bf3191e 100644 --- a/src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java @@ -5,7 +5,7 @@ import org.jetbrains.annotations.NotNull; import org.nixos.idea.psi.NixPsiElement; -abstract class AbstractNixPsiElement extends ASTWrapperPsiElement implements NixPsiElement { +abstract public class AbstractNixPsiElement extends ASTWrapperPsiElement implements NixPsiElement { AbstractNixPsiElement(@NotNull ASTNode node) { super(node); diff --git a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt new file mode 100644 index 00000000..bf350f76 --- /dev/null +++ b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt @@ -0,0 +1,34 @@ +package org.nixos.idea.util + +import java.lang.StringBuilder + +object NixIndStringUtil { + /** + * Escapes the given string for use in a double-quoted string expression in the Nix Expression Language. + * + * See [Nix docs](https://nix.dev/manual/nix/2.22/language/values.html#type-string) for the logic, which + * is non-trivial. + * + * For example, `''` can be used to escape `'`, which means `''` is not the string + * terminator + * ``` + * $ nix eval --expr " '' ''' '' " + * "'' " + * ``` + */ + fun escape(sb: StringBuilder, chars: CharSequence): Unit = sb.run { + for ((index, c) in chars.withIndex()) { + fun prevChar() = chars.getOrNull(index - 1) + fun prev2Chars(): String? { + val prev = prevChar() ?: return null + val prevPrev = chars.getOrNull(index - 2) ?: return null + return "${prevPrev}${prev}" + } + + when (c) { +// '\'' -> if (prevChar != '\'') append(c) + '$' -> if (prev2Chars() == "''") append('$') + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/nixos/idea/util/NixStringUtil.java b/src/main/java/org/nixos/idea/util/NixStringUtil.java index 90b73518..30a454d5 100644 --- a/src/main/java/org/nixos/idea/util/NixStringUtil.java +++ b/src/main/java/org/nixos/idea/util/NixStringUtil.java @@ -113,6 +113,10 @@ private static void parse(@NotNull StringBuilder builder, @NotNull ASTNode token } else if (type == NixTypes.IND_STR_ESCAPE) { assert text.length() == 3 && ("''$".contentEquals(text) || "'''".contentEquals(text)) || text.length() == 4 && "''\\".contentEquals(text.subSequence(0, 3)) : text; + if ("'''".contentEquals(text)){ + builder.append("''"); + return; + } char c = text.charAt(text.length() - 1); builder.append(unescape(c)); } else { diff --git a/src/main/lang/Nix.bnf b/src/main/lang/Nix.bnf index 5aba71c7..8b074a10 100644 --- a/src/main/lang/Nix.bnf +++ b/src/main/lang/Nix.bnf @@ -24,6 +24,10 @@ // no one really needs to know that + - * / are expected at any offset consumeTokenMethod("expr_op.*")="consumeTokenFast" + // make IndStrings language injection Hosts + implements("string")="com.intellij.psi.PsiLanguageInjectionHost" + mixin("string")="org.nixos.idea.psi.impl.AbstractNixString" + tokens = [ // This list does not contain all tokens. This list only defines the debug // names for tokens which have a distinct text representation. Other tokens diff --git a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java new file mode 100644 index 00000000..10d2c400 --- /dev/null +++ b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java @@ -0,0 +1,95 @@ +package org.nixos.idea.util; + +import com.intellij.openapi.project.Project; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.nixos.idea._testutil.WithIdeaPlatform; +import org.nixos.idea.psi.NixElementFactory; +import org.nixos.idea.psi.NixString; +import org.nixos.idea.psi.NixStringPart; +import org.nixos.idea.psi.NixStringText; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +final class NixIndStringUtilTest { +// @ParameterizedTest(name = "[{index}] {0} -> {1}") +// @CsvSource(textBlock = """ +// '' , "" +// abc , "abc" +// " , "\\\"" +// \\ , "\\\\" +// \\x , "\\\\x" +// a${b}c , "a\\${b}c" +// '\n' , "\\n" +// '\r' , "\\r" +// '\t' , "\\t" +// # supplementary character, i.e. character form a supplementary plane, +// # which needs a surrogate pair to be represented in UTF-16 +// \uD83C\uDF09 , "\uD83C\uDF09" +// """) +// void quote(String unescaped, String expectedResult) { +// assertEquals(expectedResult, NixStringUtil.quote(unescaped)); +// } + + @ParameterizedTest(name = "[{index}] {0} -> {1}") + @CsvSource(textBlock = """ + '' , '' + abc , abc + " , \\" + \\ , \\\\ + \\x , \\\\x + a${b}c , a\\${b}c + '\n' , \\n + '\r' , \\r + '\t' , \\t + # supplementary character, i.e. character form a supplementary plane, + # which needs a surrogate pair to be represented in UTF-16 + \uD83C\uDF09 , \uD83C\uDF09 + """) + void escape(String unescaped, String expectedResult) { + var sb = new StringBuilder(); + NixIndStringUtil.INSTANCE.escape(sb, unescaped); + assertEquals(expectedResult, sb.toString()); + } + + @ParameterizedTest(name = "[{index}] {0} -> {1}") + @CsvSource(quoteCharacter = '|', textBlock = """ + "" , || + "x" , x + ''abc'' , abc + "\\"" , " + "\\\\" , \\ + "\\\\x" , \\x + ''\\"'' , \\" + ''\\\\'' , \\\\ + ''\\\\x'' , \\\\x + ''''\\"'' , " + ''''\\\\'' , \\ + ''''\\\\x'' , \\x + '' ''' '' , | '''| + "''\\"" , ''" + "a\\${b}c" , a${b}c + ''a''${b}c'' , a${b}c + ''a''\\${b}c'' , a${b}c + "a$${b}c" , a$${b}c + ''a$${b}c'' , a$${b}c + |"\n"| , |\n| + |"\r"| , |\r| + |"\t"| , |\t| + # supplementary character, i.e. character form a supplementary plane, + # which needs a surrogate pair to be represented in UTF-16 + "\uD83C\uDF09" , \uD83C\uDF09 + ''\uD83C\uDF09'', \uD83C\uDF09 + """) + @WithIdeaPlatform.OnEdt + void parse(String code, String expectedResult, Project project) { + NixString string = NixElementFactory.createString(project, code); + List parts = string.getStringParts(); + assert parts.isEmpty() || parts.size() == 1; + for (NixStringPart part : parts) { + assertEquals(expectedResult, NixStringUtil.parse((NixStringText) part)); + } + } +} diff --git a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java index 0ff66c98..90a15934 100644 --- a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java @@ -68,6 +68,7 @@ void escape(String unescaped, String expectedResult) { ''''\\"'' , " ''''\\\\'' , \\ ''''\\\\x'' , \\x + ''''''' , |''| "''\\"" , ''" "a\\${b}c" , a${b}c ''a''${b}c'' , a${b}c @@ -81,6 +82,8 @@ void escape(String unescaped, String expectedResult) { # which needs a surrogate pair to be represented in UTF-16 "\uD83C\uDF09" , \uD83C\uDF09 ''\uD83C\uDF09'', \uD83C\uDF09 + # TODO implement indentation (the one below fails) + # '' a '' , |a | """) @WithIdeaPlatform.OnEdt void parse(String code, String expectedResult, Project project) { From 02610cb912ace0dd6e88aba20d4ed094f63fbb82 Mon Sep 17 00:00:00 2001 From: Cottand Date: Sun, 16 Jun 2024 17:29:41 +0100 Subject: [PATCH 02/19] split files, get highlighting working --- .../nixos/idea/psi/NixStringLiteralEscaper.kt | 30 ++++++++++ .../nixos/idea/psi/NixStringManipulator.kt | 38 ++++++++++++ .../idea/psi/impl/AbstractIndNixString.kt | 60 ------------------- .../nixos/idea/psi/impl/AbstractNixString.kt | 24 ++++++++ .../org/nixos/idea/util/NixIndStringUtil.kt | 31 ++++++++-- .../org/nixos/idea/util/NixStringUtil.java | 3 +- src/main/resources/META-INF/plugin.xml | 4 ++ .../nixos/idea/util/NixIndStringUtilTest.java | 44 +------------- .../nixos/idea/util/NixStringUtilTest.java | 2 +- 9 files changed, 126 insertions(+), 110 deletions(-) create mode 100644 src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt create mode 100644 src/main/java/org/nixos/idea/psi/NixStringManipulator.kt delete mode 100644 src/main/java/org/nixos/idea/psi/impl/AbstractIndNixString.kt create mode 100644 src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt new file mode 100644 index 00000000..18702b34 --- /dev/null +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -0,0 +1,30 @@ +package org.nixos.idea.psi + +import com.intellij.openapi.util.TextRange +import com.intellij.psi.LiteralTextEscaper +import com.intellij.psi.PsiLanguageInjectionHost +import org.nixos.idea.psi.impl.AbstractNixString +import org.nixos.idea.util.NixIndStringUtil + +class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { + + override fun isOneLine(): Boolean = false + + + override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean { + val subText: String = rangeInsideHost.substring(myHost.text) + val escaped = NixIndStringUtil.escape(subText) + outChars.append(escaped) + return true + } + + override fun getOffsetInHost(offsetInDecoded: Int, rangeInsideHost: TextRange): Int { + // TODO: Implement proper String back-feed support. + // this involves keeping track of text offsets between decoded + // and encoded Nix text. See how Terraform does it here: + // https://github.com/JetBrains/intellij-plugins/blob/master/terraform/src/org/intellij/terraform/hcl/psi/impl/HCLStringLiteralTextEscaper.kt + val offsetInHost = offsetInDecoded + rangeInsideHost.startOffset + return (offsetInHost).coerceIn(rangeInsideHost.startOffset..rangeInsideHost.endOffset) + } + +} \ No newline at end of file diff --git a/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt b/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt new file mode 100644 index 00000000..ba9be5aa --- /dev/null +++ b/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt @@ -0,0 +1,38 @@ +package org.nixos.idea.psi + +import com.intellij.openapi.util.TextRange +import com.intellij.psi.AbstractElementManipulator +import com.intellij.refactoring.suggested.startOffset +import org.nixos.idea.psi.impl.AbstractNixString +import org.nixos.idea.util.NixIndStringUtil +import org.nixos.idea.util.NixStringUtil + +class NixStringManipulator : AbstractElementManipulator() { + + /** + * This function's result is in fact unused because + * [AbstractNixString.updateText] does not do anything yet, + */ + override fun handleContentChange( + element: NixString, + range: TextRange, + newContent: String + ): NixString? { + var replacement = "" + for (part in element.stringParts) { + val parsed = NixStringUtil.parse(part) + replacement = range.replace(element.text, parsed) + } + return (element as? AbstractNixString)?.updateText(replacement) + } + + override fun getRangeInElement(element: NixString): TextRange = + when { + element.textLength == 0 -> TextRange.EMPTY_RANGE + element is NixIndString && element.textLength < 4 -> TextRange(0, element.textLength) + element is NixIndString -> TextRange(2, element.textLength - 2) + // element is not IndString, so it must be StdString + element.textLength == 1 -> TextRange(0, 1) + else -> TextRange(1, element.textLength - 1) + } +} \ No newline at end of file diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractIndNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractIndNixString.kt deleted file mode 100644 index f899392e..00000000 --- a/src/main/java/org/nixos/idea/psi/impl/AbstractIndNixString.kt +++ /dev/null @@ -1,60 +0,0 @@ -package org.nixos.idea.psi.impl - -import com.intellij.lang.ASTNode -import com.intellij.openapi.util.TextRange -import com.intellij.psi.AbstractElementManipulator -import com.intellij.psi.ElementManipulators -import com.intellij.psi.LiteralTextEscaper -import com.intellij.psi.PsiLanguageInjectionHost -import org.intellij.grammar.psi.impl.BnfStringImpl -import org.intellij.grammar.psi.impl.BnfStringManipulator.getStringTokenRange -import org.nixos.idea.util.NixStringUtil - - -abstract class AbstractNixString(node: ASTNode) : PsiLanguageInjectionHost, AbstractNixPsiElement(node) { - - override fun isValidHost() = true - - override fun updateText(text: String): PsiLanguageInjectionHost = - ElementManipulators.handleContentChange(this, text) - - override fun createLiteralTextEscaper() = NixStringLiteralEscaper(this) -} - -class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { - - override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean { - val sb = StringBuilder() - NixStringUtil.escape(sb, rangeInsideHost.toString()) - outChars.append(sb, rangeInsideHost.startOffset, rangeInsideHost.endOffset) - - return true - } - - override fun getOffsetInHost(offsetInDecoded: Int, rangeInsideHost: TextRange): Int = with(rangeInsideHost) { - // todo implement proper java-like string escapes support - val offset = offsetInDecoded + startOffset - return when { - offset < startOffset -> startOffset - offset > endOffset -> endOffset - else -> offset - } - } - - override fun isOneLine(): Boolean = false -} - -class IndStringManipulator : AbstractElementManipulator() { - override fun handleContentChange( - element: NixStringImpl, - range: TextRange, - newContent: String? - ): NixStringImpl? { - - TODO("Not yet implemented") - } - - fun getRangeInElement(element: NixStringImpl?): TextRange { - TODO("Not yet implemented") - } -} \ No newline at end of file diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt new file mode 100644 index 00000000..1bddb92e --- /dev/null +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt @@ -0,0 +1,24 @@ +package org.nixos.idea.psi.impl + +import com.intellij.lang.ASTNode +import com.intellij.psi.PsiLanguageInjectionHost +import com.intellij.psi.impl.source.tree.LeafElement +import org.nixos.idea.psi.NixString +import org.nixos.idea.psi.NixStringLiteralEscaper + + +abstract class AbstractNixString(node: ASTNode) : PsiLanguageInjectionHost, AbstractNixPsiElement(node), NixString { + + override fun isValidHost() = true + + override fun updateText(text: String): NixString { + // TODO implement. This is called when you edit the injected file in + // order for the final Nix string to get updated + // It is not necessary for syntax highlighting in injections + // but is a nice to have + return this + } + + override fun createLiteralTextEscaper() = NixStringLiteralEscaper(this) +} + diff --git a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt index bf350f76..70937b41 100644 --- a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt +++ b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt @@ -1,7 +1,5 @@ package org.nixos.idea.util -import java.lang.StringBuilder - object NixIndStringUtil { /** * Escapes the given string for use in a double-quoted string expression in the Nix Expression Language. @@ -15,19 +13,40 @@ object NixIndStringUtil { * $ nix eval --expr " '' ''' '' " * "'' " * ``` + * + * This function does not erase string interpolations, because + * they are hard to parse in a loop without a proper grammar. For example: + * ```nix + * '' ${someNixFunc "${foo "}}" }" } '' + * ``` */ - fun escape(sb: StringBuilder, chars: CharSequence): Unit = sb.run { + fun escape(chars: CharSequence) = buildString { for ((index, c) in chars.withIndex()) { fun prevChar() = chars.getOrNull(index - 1) fun prev2Chars(): String? { val prev = prevChar() ?: return null - val prevPrev = chars.getOrNull(index - 2) ?: return null + val prevPrev = chars.getOrNull(index - 2) ?: return null return "${prevPrev}${prev}" } + fun prev3Chars(): String? { + val prev2 = prev2Chars() ?: return null + val prevPrev2 = chars.getOrNull(index - 3) ?: return null + return "${prevPrev2}${prev2}" + } + when (c) { -// '\'' -> if (prevChar != '\'') append(c) - '$' -> if (prev2Chars() == "''") append('$') + // ''' is escaped to '' + // '' is the string delimiter + '\'' -> when { + prev2Chars() == "''" -> append("''") + prevChar() == '\'' -> continue + } + // ''\ escapes any character, but we can only cover known ones in advance: + 'r' -> if (prev3Chars() == "''\\") append('\r') + 'n' -> if (prev3Chars() == "''\\") append('\n') + 't' -> if (prev3Chars() == "''\\") append('\t') + else -> append(c) } } } diff --git a/src/main/java/org/nixos/idea/util/NixStringUtil.java b/src/main/java/org/nixos/idea/util/NixStringUtil.java index 30a454d5..0e55673b 100644 --- a/src/main/java/org/nixos/idea/util/NixStringUtil.java +++ b/src/main/java/org/nixos/idea/util/NixStringUtil.java @@ -4,6 +4,7 @@ import com.intellij.psi.tree.IElementType; import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; +import org.nixos.idea.psi.NixStringPart; import org.nixos.idea.psi.NixStringText; import org.nixos.idea.psi.NixTypes; @@ -93,7 +94,7 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence * @param textNode A part of a string. * @return The resulting string after resolving all escape sequences. */ - public static @NotNull String parse(@NotNull NixStringText textNode) { + public static @NotNull String parse(@NotNull NixStringPart textNode) { StringBuilder builder = new StringBuilder(); for (ASTNode child = textNode.getNode().getFirstChildNode(); child != null; child = child.getTreeNext()) { parse(builder, child); diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml index 159b1e7f..bdabdb6c 100644 --- a/src/main/resources/META-INF/plugin.xml +++ b/src/main/resources/META-INF/plugin.xml @@ -59,6 +59,10 @@ + + + diff --git a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java index 10d2c400..6eb0d759 100644 --- a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java @@ -49,47 +49,7 @@ final class NixIndStringUtilTest { \uD83C\uDF09 , \uD83C\uDF09 """) void escape(String unescaped, String expectedResult) { - var sb = new StringBuilder(); - NixIndStringUtil.INSTANCE.escape(sb, unescaped); - assertEquals(expectedResult, sb.toString()); - } - - @ParameterizedTest(name = "[{index}] {0} -> {1}") - @CsvSource(quoteCharacter = '|', textBlock = """ - "" , || - "x" , x - ''abc'' , abc - "\\"" , " - "\\\\" , \\ - "\\\\x" , \\x - ''\\"'' , \\" - ''\\\\'' , \\\\ - ''\\\\x'' , \\\\x - ''''\\"'' , " - ''''\\\\'' , \\ - ''''\\\\x'' , \\x - '' ''' '' , | '''| - "''\\"" , ''" - "a\\${b}c" , a${b}c - ''a''${b}c'' , a${b}c - ''a''\\${b}c'' , a${b}c - "a$${b}c" , a$${b}c - ''a$${b}c'' , a$${b}c - |"\n"| , |\n| - |"\r"| , |\r| - |"\t"| , |\t| - # supplementary character, i.e. character form a supplementary plane, - # which needs a surrogate pair to be represented in UTF-16 - "\uD83C\uDF09" , \uD83C\uDF09 - ''\uD83C\uDF09'', \uD83C\uDF09 - """) - @WithIdeaPlatform.OnEdt - void parse(String code, String expectedResult, Project project) { - NixString string = NixElementFactory.createString(project, code); - List parts = string.getStringParts(); - assert parts.isEmpty() || parts.size() == 1; - for (NixStringPart part : parts) { - assertEquals(expectedResult, NixStringUtil.parse((NixStringText) part)); - } + var str = NixIndStringUtil.INSTANCE.escape(unescaped); + assertEquals(expectedResult, str); } } diff --git a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java index 90a15934..0a1c13d0 100644 --- a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java @@ -91,7 +91,7 @@ void parse(String code, String expectedResult, Project project) { List parts = string.getStringParts(); assert parts.isEmpty() || parts.size() == 1; for (NixStringPart part : parts) { - assertEquals(expectedResult, NixStringUtil.parse((NixStringText) part)); + assertEquals(expectedResult, NixStringUtil.parse(part)); } } } From 1848120e80c63fa5c354e1951ed8c501a5f4915b Mon Sep 17 00:00:00 2001 From: Cottand Date: Sun, 16 Jun 2024 20:20:58 +0100 Subject: [PATCH 03/19] fix bad escaping of whitespace chars, handle one line string --- .../nixos/idea/psi/NixStringLiteralEscaper.kt | 9 ++++++--- .../org/nixos/idea/psi/NixStringManipulator.kt | 17 ++++++++--------- .../nixos/idea/psi/impl/AbstractNixString.kt | 1 - .../org/nixos/idea/util/NixIndStringUtil.kt | 9 +++++---- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index 18702b34..0e0ba43c 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -5,16 +5,19 @@ import com.intellij.psi.LiteralTextEscaper import com.intellij.psi.PsiLanguageInjectionHost import org.nixos.idea.psi.impl.AbstractNixString import org.nixos.idea.util.NixIndStringUtil +import org.nixos.idea.util.NixStringUtil class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { override fun isOneLine(): Boolean = false - override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean { val subText: String = rangeInsideHost.substring(myHost.text) - val escaped = NixIndStringUtil.escape(subText) - outChars.append(escaped) + if (myHost is NixIndString) { + outChars.append(NixIndStringUtil.escape(subText)) + } else { + NixStringUtil.escape(outChars, subText) + } return true } diff --git a/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt b/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt index ba9be5aa..18f2a7fc 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt @@ -26,13 +26,12 @@ class NixStringManipulator : AbstractElementManipulator() { return (element as? AbstractNixString)?.updateText(replacement) } - override fun getRangeInElement(element: NixString): TextRange = - when { - element.textLength == 0 -> TextRange.EMPTY_RANGE - element is NixIndString && element.textLength < 4 -> TextRange(0, element.textLength) - element is NixIndString -> TextRange(2, element.textLength - 2) - // element is not IndString, so it must be StdString - element.textLength == 1 -> TextRange(0, 1) - else -> TextRange(1, element.textLength - 1) - } + override fun getRangeInElement(element: NixString): TextRange = when { + element.textLength == 0 -> TextRange.EMPTY_RANGE + element is NixIndString && element.textLength < 4 -> TextRange(0, element.textLength) + element is NixIndString -> TextRange(2, element.textLength - 2) + // element is not IndString, so it must be StdString + element.textLength == 1 -> TextRange(0, 1) + else -> TextRange(1, element.textLength - 1) + } } \ No newline at end of file diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt index 1bddb92e..42cab13c 100644 --- a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt @@ -2,7 +2,6 @@ package org.nixos.idea.psi.impl import com.intellij.lang.ASTNode import com.intellij.psi.PsiLanguageInjectionHost -import com.intellij.psi.impl.source.tree.LeafElement import org.nixos.idea.psi.NixString import org.nixos.idea.psi.NixStringLiteralEscaper diff --git a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt index 70937b41..4b5e9d45 100644 --- a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt +++ b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt @@ -20,7 +20,7 @@ object NixIndStringUtil { * '' ${someNixFunc "${foo "}}" }" } '' * ``` */ - fun escape(chars: CharSequence) = buildString { + fun escape(chars: CharSequence): String = buildString { for ((index, c) in chars.withIndex()) { fun prevChar() = chars.getOrNull(index - 1) fun prev2Chars(): String? { @@ -41,11 +41,12 @@ object NixIndStringUtil { '\'' -> when { prev2Chars() == "''" -> append("''") prevChar() == '\'' -> continue + else -> append(c) } // ''\ escapes any character, but we can only cover known ones in advance: - 'r' -> if (prev3Chars() == "''\\") append('\r') - 'n' -> if (prev3Chars() == "''\\") append('\n') - 't' -> if (prev3Chars() == "''\\") append('\t') + 'r' -> if (prev3Chars() == "''\\") append('\r') else append(c) + 'n' -> if (prev3Chars() == "''\\") append('\n') else append(c) + 't' -> if (prev3Chars() == "''\\") append('\t') else append(c) else -> append(c) } } From 6b6df0e5883306ad08af4c0aa744defe9d48e591 Mon Sep 17 00:00:00 2001 From: Cottand Date: Sun, 16 Jun 2024 20:40:45 +0100 Subject: [PATCH 04/19] more testing for escaping --- .../org/nixos/idea/util/NixIndStringUtil.kt | 27 +++++++++--- .../nixos/idea/util/NixIndStringUtilTest.java | 43 +++++++------------ 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt index 4b5e9d45..805f8e9a 100644 --- a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt +++ b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt @@ -36,18 +36,31 @@ object NixIndStringUtil { } when (c) { - // ''' is escaped to '' - // '' is the string delimiter + // ''\ escapes any character, but we can only cover known ones in advance: '\'' -> when { + // ''' is escaped to '' prev2Chars() == "''" -> append("''") + // '' is the string delimiter + else -> continue + } + + '\\' -> when { + prev2Chars() == "''" -> continue prevChar() == '\'' -> continue else -> append(c) } - // ''\ escapes any character, but we can only cover known ones in advance: - 'r' -> if (prev3Chars() == "''\\") append('\r') else append(c) - 'n' -> if (prev3Chars() == "''\\") append('\n') else append(c) - 't' -> if (prev3Chars() == "''\\") append('\t') else append(c) - else -> append(c) + + '$' -> if (prevChar() == '$') append(c) else continue + '{' -> if (prevChar() == '$') append("\${") else append(c) + + else -> if (prev3Chars() == "''\\") when (c) { + 'r' -> if (prev3Chars() == "''\\") append('\r') else append(c) + 'n' -> if (prev3Chars() == "''\\") append('\n') else append(c) + 't' -> if (prev3Chars() == "''\\") append('\t') else append(c) + else -> append("''\\").append(c) + } else { + append(c) + } } } } diff --git a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java index 6eb0d759..f4c99a24 100644 --- a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java @@ -14,41 +14,28 @@ import static org.junit.jupiter.api.Assertions.assertEquals; final class NixIndStringUtilTest { -// @ParameterizedTest(name = "[{index}] {0} -> {1}") -// @CsvSource(textBlock = """ -// '' , "" -// abc , "abc" -// " , "\\\"" -// \\ , "\\\\" -// \\x , "\\\\x" -// a${b}c , "a\\${b}c" -// '\n' , "\\n" -// '\r' , "\\r" -// '\t' , "\\t" -// # supplementary character, i.e. character form a supplementary plane, -// # which needs a surrogate pair to be represented in UTF-16 -// \uD83C\uDF09 , "\uD83C\uDF09" -// """) -// void quote(String unescaped, String expectedResult) { -// assertEquals(expectedResult, NixStringUtil.quote(unescaped)); -// } - @ParameterizedTest(name = "[{index}] {0} -> {1}") - @CsvSource(textBlock = """ - '' , '' + @CsvSource(quoteCharacter = '|', textBlock = """ + || , || abc , abc - " , \\" - \\ , \\\\ - \\x , \\\\x - a${b}c , a\\${b}c - '\n' , \\n - '\r' , \\r - '\t' , \\t + " , " + \\ , \\ + \\x , \\x + a${b}c , a${b}c + |\n| , |\n| + |\r| , |\r| + |\t| , |\t| + |''\\t| , |\t| + |''\\r| , |\r| + |''\\n| , |\n| + |'''| , |''| + $$ , $ # supplementary character, i.e. character form a supplementary plane, # which needs a surrogate pair to be represented in UTF-16 \uD83C\uDF09 , \uD83C\uDF09 """) void escape(String unescaped, String expectedResult) { + NixIndStringUtil.INSTANCE.escape("''\\t"); var str = NixIndStringUtil.INSTANCE.escape(unescaped); assertEquals(expectedResult, str); } From a44290483a8013cf6e0d2c52e45996729f1c2d6a Mon Sep 17 00:00:00 2001 From: Cottand Date: Mon, 17 Jun 2024 19:31:13 +0100 Subject: [PATCH 05/19] optimise imports in NixIndStringUtilTest --- .../java/org/nixos/idea/util/NixIndStringUtilTest.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java index f4c99a24..d9abdedc 100644 --- a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java @@ -1,15 +1,7 @@ package org.nixos.idea.util; -import com.intellij.openapi.project.Project; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.nixos.idea._testutil.WithIdeaPlatform; -import org.nixos.idea.psi.NixElementFactory; -import org.nixos.idea.psi.NixString; -import org.nixos.idea.psi.NixStringPart; -import org.nixos.idea.psi.NixStringText; - -import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; From d70911451039dab5c680ec5ae274b46c39704870 Mon Sep 17 00:00:00 2001 From: Cottand Date: Mon, 17 Jun 2024 20:01:30 +0100 Subject: [PATCH 06/19] address review comments: change wording in comment undo change signature of NixStringUtil.parse remove remnant in NixIndStringUtilTest make NixIndStringUtil.escape JvmStatic --- src/main/java/org/nixos/idea/util/NixIndStringUtil.kt | 5 +++-- src/main/java/org/nixos/idea/util/NixStringUtil.java | 2 +- src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java | 3 +-- src/test/java/org/nixos/idea/util/NixStringUtilTest.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt index 805f8e9a..c7a83cd3 100644 --- a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt +++ b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt @@ -7,8 +7,8 @@ object NixIndStringUtil { * See [Nix docs](https://nix.dev/manual/nix/2.22/language/values.html#type-string) for the logic, which * is non-trivial. * - * For example, `''` can be used to escape `'`, which means `''` is not the string - * terminator + * For example, `'` can be used to escape `''`, which means `'''` does not contain + * a string terminator * ``` * $ nix eval --expr " '' ''' '' " * "'' " @@ -20,6 +20,7 @@ object NixIndStringUtil { * '' ${someNixFunc "${foo "}}" }" } '' * ``` */ + @JvmStatic fun escape(chars: CharSequence): String = buildString { for ((index, c) in chars.withIndex()) { fun prevChar() = chars.getOrNull(index - 1) diff --git a/src/main/java/org/nixos/idea/util/NixStringUtil.java b/src/main/java/org/nixos/idea/util/NixStringUtil.java index 0e55673b..33fe05f3 100644 --- a/src/main/java/org/nixos/idea/util/NixStringUtil.java +++ b/src/main/java/org/nixos/idea/util/NixStringUtil.java @@ -94,7 +94,7 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence * @param textNode A part of a string. * @return The resulting string after resolving all escape sequences. */ - public static @NotNull String parse(@NotNull NixStringPart textNode) { + public static @NotNull String parse(@NotNull NixStringText textNode) { StringBuilder builder = new StringBuilder(); for (ASTNode child = textNode.getNode().getFirstChildNode(); child != null; child = child.getTreeNext()) { parse(builder, child); diff --git a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java index d9abdedc..d5407389 100644 --- a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java @@ -27,8 +27,7 @@ final class NixIndStringUtilTest { \uD83C\uDF09 , \uD83C\uDF09 """) void escape(String unescaped, String expectedResult) { - NixIndStringUtil.INSTANCE.escape("''\\t"); - var str = NixIndStringUtil.INSTANCE.escape(unescaped); + var str = NixIndStringUtil.escape(unescaped); assertEquals(expectedResult, str); } } diff --git a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java index 0a1c13d0..90a15934 100644 --- a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java @@ -91,7 +91,7 @@ void parse(String code, String expectedResult, Project project) { List parts = string.getStringParts(); assert parts.isEmpty() || parts.size() == 1; for (NixStringPart part : parts) { - assertEquals(expectedResult, NixStringUtil.parse(part)); + assertEquals(expectedResult, NixStringUtil.parse((NixStringText) part)); } } } From 88c8dced13f9a46b67760aef139e9cba7c66209f Mon Sep 17 00:00:00 2001 From: Cottand Date: Sat, 6 Jul 2024 20:14:13 +0100 Subject: [PATCH 07/19] remove leftover type mismatch --- .../java/org/nixos/idea/psi/NixStringManipulator.kt | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt b/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt index 18f2a7fc..d056a5aa 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringManipulator.kt @@ -10,20 +10,15 @@ import org.nixos.idea.util.NixStringUtil class NixStringManipulator : AbstractElementManipulator() { /** - * This function's result is in fact unused because - * [AbstractNixString.updateText] does not do anything yet, + * This function's result changes the original text in the host language + * when the fragment in the guest language changes */ override fun handleContentChange( element: NixString, range: TextRange, newContent: String - ): NixString? { - var replacement = "" - for (part in element.stringParts) { - val parsed = NixStringUtil.parse(part) - replacement = range.replace(element.text, parsed) - } - return (element as? AbstractNixString)?.updateText(replacement) + ): NixString { + return element } override fun getRangeInElement(element: NixString): TextRange = when { From 9bc2749b0c8e60f54291b11c0df3ab7b7a00f0ac Mon Sep 17 00:00:00 2001 From: Cottand Date: Sat, 6 Jul 2024 23:30:27 +0100 Subject: [PATCH 08/19] first iteration of offset tracking --- .../nixos/idea/psi/NixStringLiteralEscaper.kt | 129 ++++++++++++++++-- .../org/nixos/idea/util/NixIndStringUtil.kt | 4 +- .../nixos/idea/util/NixIndStringUtilTest.java | 4 +- 3 files changed, 119 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index 0e0ba43c..63ddd4a4 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -4,30 +4,131 @@ import com.intellij.openapi.util.TextRange import com.intellij.psi.LiteralTextEscaper import com.intellij.psi.PsiLanguageInjectionHost import org.nixos.idea.psi.impl.AbstractNixString -import org.nixos.idea.util.NixIndStringUtil -import org.nixos.idea.util.NixStringUtil class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { override fun isOneLine(): Boolean = false + private var outSourceOffsets: IntArray? = null + + override fun getRelevantTextRange(): TextRange { + if (myHost.textLength <= 4) return TextRange.EMPTY_RANGE + return TextRange.create(2, myHost.textLength - 2) + } + override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean { + // only indented strings supported for now + if (myHost !is NixIndString) return false + val subText: String = rangeInsideHost.substring(myHost.text) - if (myHost is NixIndString) { - outChars.append(NixIndStringUtil.escape(subText)) - } else { - NixStringUtil.escape(outChars, subText) - } - return true + + + val array = IntArray(subText.length + 1) + val success = unescapeAndDecode(subText, outChars, array, interpolations = false) + outSourceOffsets = array + return success } override fun getOffsetInHost(offsetInDecoded: Int, rangeInsideHost: TextRange): Int { - // TODO: Implement proper String back-feed support. - // this involves keeping track of text offsets between decoded - // and encoded Nix text. See how Terraform does it here: - // https://github.com/JetBrains/intellij-plugins/blob/master/terraform/src/org/intellij/terraform/hcl/psi/impl/HCLStringLiteralTextEscaper.kt - val offsetInHost = offsetInDecoded + rangeInsideHost.startOffset - return (offsetInHost).coerceIn(rangeInsideHost.startOffset..rangeInsideHost.endOffset) + val offsets = outSourceOffsets ?: throw IllegalStateException("#decode was not called") + val result = if (offsetInDecoded < offsets.size) offsets[offsetInDecoded] else -1 + println("gotOffsetInHost decoded=${offsetInDecoded} rangeInsideHost=${rangeInsideHost} result=$result") + return result.coerceIn(2..rangeInsideHost.length) + rangeInsideHost.startOffset + } + + companion object { + fun unescapeAndDecode( + chars: String, + outChars: StringBuilder, + sourceOffsets: IntArray?, + interpolations: Boolean + ): Boolean { + assert(sourceOffsets == null || sourceOffsets.size == chars.length + 1) + + var index = 0 + val outOffset = outChars.length + var braces = 0 + + + while (index < chars.length) { + fun updateOffsets(index: Int) { + if (sourceOffsets != null) { + sourceOffsets[outChars.length - outOffset] = index - 1 + sourceOffsets[outChars.length - outOffset + 1] = index + } + } + var c = chars[index++] + + updateOffsets(index) + + + if (braces > 0) { + if (c == '{') braces++ + else if (c == '}') braces-- + outChars.append(c) + continue + } + + if (c == '\'') { + if (index == chars.length) return false + c = chars[index++] + + if (c != '\'') { + // if what follows isn't another ' then we are not escaping anything, + // so we can continue + outChars.append("\'") + updateOffsets(index - 1) + outChars.append(c) + continue + } + + if (index == chars.length) return false + c = chars[index++] + + when(c) { + // '' can be escaped by prefixing it with ', i.e., '''. + '\'' -> { + outChars.append("\'") + updateOffsets(index - 1) + outChars.append(c) + } + // $ can be escaped by prefixing it with '' (that is, two single quotes), i.e., ''$. + '$' -> outChars.append(c) + // Linefeed, carriage-return and tab characters can + // be written as ''\n, ''\r, ''\t, and ''\ escapes any other character. + 'a' -> outChars.append(0x07.toChar()) + 'b' -> outChars.append('\b') + 'f' -> outChars.append(0x0c.toChar()) + 'n' -> outChars.append('\n') + 't' -> outChars.append('\t') + 'r' -> outChars.append('\r') + 'v' -> outChars.append(0x0b.toChar()) + else -> return false + } + } + +// // $ removes any special meaning from the following $. +// if (c == '$') { +// if (index == chars.length) return false +// c = chars[index++] +// if (c != '$') { +// // if what follows isn't another ' then we are not escaping anything, +// // so we can continue +// outChars.append('$') +// updateOffsets(index - 1) +// outChars.append(c) +// continue +// } +// // what here?? +// } + + + if (sourceOffsets != null) { + sourceOffsets[outChars.length - outOffset] = index + } + } + return true + } } } \ No newline at end of file diff --git a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt index c7a83cd3..903c8a57 100644 --- a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt +++ b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt @@ -2,7 +2,7 @@ package org.nixos.idea.util object NixIndStringUtil { /** - * Escapes the given string for use in a double-quoted string expression in the Nix Expression Language. + * Unescapes the given string for use in a double-quoted string expression in the Nix Expression Language. * * See [Nix docs](https://nix.dev/manual/nix/2.22/language/values.html#type-string) for the logic, which * is non-trivial. @@ -21,7 +21,7 @@ object NixIndStringUtil { * ``` */ @JvmStatic - fun escape(chars: CharSequence): String = buildString { + fun unescape(chars: CharSequence): String = buildString { for ((index, c) in chars.withIndex()) { fun prevChar() = chars.getOrNull(index - 1) fun prev2Chars(): String? { diff --git a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java index d5407389..63d18881 100644 --- a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java @@ -26,8 +26,8 @@ final class NixIndStringUtilTest { # which needs a surrogate pair to be represented in UTF-16 \uD83C\uDF09 , \uD83C\uDF09 """) - void escape(String unescaped, String expectedResult) { - var str = NixIndStringUtil.escape(unescaped); + void unescape(String unescaped, String expectedResult) { + var str = NixIndStringUtil.unescape(unescaped); assertEquals(expectedResult, str); } } From 82d8b4c47bc325bd9d43fd0577b10a9a227c5b36 Mon Sep 17 00:00:00 2001 From: Cottand Date: Sat, 6 Jul 2024 23:40:45 +0100 Subject: [PATCH 09/19] fix not covering happy path that has no escaping --- .../nixos/idea/psi/NixStringLiteralEscaper.kt | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index 63ddd4a4..2e7ea0cf 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -94,17 +94,28 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper outChars.append(c) - // Linefeed, carriage-return and tab characters can - // be written as ''\n, ''\r, ''\t, and ''\ escapes any other character. - 'a' -> outChars.append(0x07.toChar()) - 'b' -> outChars.append('\b') - 'f' -> outChars.append(0x0c.toChar()) - 'n' -> outChars.append('\n') - 't' -> outChars.append('\t') - 'r' -> outChars.append('\r') - 'v' -> outChars.append(0x0b.toChar()) + '\\' -> { + if (index == chars.length) return false + c = chars[index++] + when(c) { + // Linefeed, carriage-return and tab characters can + // be written as ''\n, ''\r, ''\t, and ''\ escapes any other character. + 'a' -> outChars.append(0x07.toChar()) + 'b' -> outChars.append('\b') + 'f' -> outChars.append(0x0c.toChar()) + 'n' -> outChars.append('\n') + 't' -> outChars.append('\t') + 'r' -> outChars.append('\r') + 'v' -> outChars.append(0x0b.toChar()) + else -> return false + } + } else -> return false } + if (sourceOffsets != null) { + sourceOffsets[outChars.length - outOffset] = index + } + continue } // // $ removes any special meaning from the following $. @@ -122,10 +133,7 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper Date: Sun, 7 Jul 2024 08:14:19 +0100 Subject: [PATCH 10/19] add test case for $ escapings --- .../nixos/idea/psi/NixStringLiteralEscaper.kt | 33 ++++--------------- .../nixos/idea/psi/impl/AbstractNixString.kt | 22 +++++++++---- .../nixos/idea/util/NixIndStringUtilTest.java | 10 ++++-- 3 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index 2e7ea0cf..ff3144cb 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -17,14 +17,14 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper Date: Sun, 7 Jul 2024 08:55:38 +0100 Subject: [PATCH 11/19] fix not being able to inject in-fragment --- src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt | 2 +- src/main/java/org/nixos/idea/psi/NixStringManipulator.kt | 6 ++++-- src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index ff3144cb..bfa434ac 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -32,7 +32,7 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper() { element: NixString, range: TextRange, newContent: String - ): NixString { - return element + ): NixString? { + val escaped = newContent + val replacement = range.replace(element.text, escaped) + return element.updateText(replacement) as? NixString } override fun getRangeInElement(element: NixString): TextRange = when { diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt index ce541265..abe45bd9 100644 --- a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt @@ -4,6 +4,7 @@ import com.intellij.lang.ASTNode import com.intellij.openapi.diagnostic.Logger import com.intellij.psi.PsiLanguageInjectionHost import com.intellij.psi.impl.source.tree.LeafElement +import com.intellij.psi.impl.source.tree.LeafPsiElement import com.intellij.util.IncorrectOperationException import org.nixos.idea.psi.NixIndString import org.nixos.idea.psi.NixString @@ -20,7 +21,8 @@ abstract class AbstractNixString(private val astNode: ASTNode) : PsiLanguageInje LOG.info("not a nix ind string") return this } - (astNode.firstChildNode as LeafElement).replaceWithText(s) + (astNode.firstChildNode.treeNext.firstChildNode as? LeafPsiElement) + ?.replaceWithText(s) return this } From f97fee31897dd82e173c82c969972e19e557bfdd Mon Sep 17 00:00:00 2001 From: Cottand Date: Sun, 7 Jul 2024 09:01:01 +0100 Subject: [PATCH 12/19] fix bad replacing where quotes where duplicated --- src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt index abe45bd9..972dac28 100644 --- a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt @@ -3,9 +3,7 @@ package org.nixos.idea.psi.impl import com.intellij.lang.ASTNode import com.intellij.openapi.diagnostic.Logger import com.intellij.psi.PsiLanguageInjectionHost -import com.intellij.psi.impl.source.tree.LeafElement import com.intellij.psi.impl.source.tree.LeafPsiElement -import com.intellij.util.IncorrectOperationException import org.nixos.idea.psi.NixIndString import org.nixos.idea.psi.NixString import org.nixos.idea.psi.NixStringLiteralEscaper @@ -21,8 +19,9 @@ abstract class AbstractNixString(private val astNode: ASTNode) : PsiLanguageInje LOG.info("not a nix ind string") return this } + val withoutQuotes = s.substring(2..(s.lastIndex - 2)) (astNode.firstChildNode.treeNext.firstChildNode as? LeafPsiElement) - ?.replaceWithText(s) + ?.replaceWithText(withoutQuotes) return this } From 43664e2053c24897340cbd3b25d8213b31056b8e Mon Sep 17 00:00:00 2001 From: Cottand Date: Sun, 7 Jul 2024 10:42:51 +0100 Subject: [PATCH 13/19] support indent in fragment editor --- .../nixos/idea/psi/NixStringLiteralEscaper.kt | 51 +++++++++++++------ .../nixos/idea/psi/impl/AbstractNixString.kt | 26 ++++++++-- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index bfa434ac..9fce9a03 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -10,6 +10,7 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper { outChars.append("\'") @@ -91,9 +111,9 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper outChars.append(c) '\\' -> { - if (index == chars.length) return false + if (index == chars.length) return null c = chars[index++] - when(c) { + when (c) { // Linefeed, carriage-return and tab characters can // be written as ''\n, ''\r, ''\t, and ''\ escapes any other character. 'a' -> outChars.append(0x07.toChar()) @@ -103,10 +123,11 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper outChars.append('\t') 'r' -> outChars.append('\r') 'v' -> outChars.append(0x0b.toChar()) - else -> return false + else -> return null } } - else -> return false + + else -> return null } if (sourceOffsets != null) { sourceOffsets[outChars.length - outOffset] = index @@ -116,7 +137,7 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper + if (index != 0) leadingSpace + line else line + } + + originalNode?.replaceWithText(withoutQuotesWithIndent) return this } From d675347b5f789a23219b243e077167c80a3db176 Mon Sep 17 00:00:00 2001 From: Cottand Date: Sun, 7 Jul 2024 11:41:13 +0100 Subject: [PATCH 14/19] fix backtracking with ' and remove minIndent parsing in decode --- .../nixos/idea/psi/NixStringLiteralEscaper.kt | 26 +++++++++---------- .../nixos/idea/psi/impl/AbstractNixString.kt | 17 ++++++------ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index 9fce9a03..b8241889 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -3,6 +3,7 @@ package org.nixos.idea.psi import com.intellij.openapi.util.TextRange import com.intellij.psi.LiteralTextEscaper import com.intellij.psi.PsiLanguageInjectionHost +import org.intellij.lang.annotations.Language import org.nixos.idea.psi.impl.AbstractNixString class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { @@ -10,7 +11,6 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper outChars.append(c) '\\' -> { - if (index == chars.length) return null + if (index == chars.length) return false c = chars[index++] when (c) { // Linefeed, carriage-return and tab characters can @@ -123,11 +121,11 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper outChars.append('\t') 'r' -> outChars.append('\r') 'v' -> outChars.append(0x0b.toChar()) - else -> return null + else -> return false } } - else -> return null + else -> return false } if (sourceOffsets != null) { sourceOffsets[outChars.length - outOffset] = index @@ -137,7 +135,7 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper - if (index != 0) leadingSpace + line else line - } + .map { (index, line) -> if (index != 0) leadingSpace + line else line } + + // if the first line was removed in the fragment, add it back to preserve a multiline string + val withLeadingBlankLine = if (lines.first().isNotEmpty()) listOf("") + withIndent else withIndent - originalNode?.replaceWithText(withoutQuotesWithIndent) + originalNode?.replaceWithText(withLeadingBlankLine.joinToString(separator = System.lineSeparator())) return this } From ade4a64c54c3c6ed053c90a4b28c729b4796cc18 Mon Sep 17 00:00:00 2001 From: Cottand Date: Sat, 13 Jul 2024 18:03:29 +0100 Subject: [PATCH 15/19] address review: - fix outdated javadoc - fix outdated valid injection host check --- src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt | 2 -- src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index b8241889..9454450a 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -40,8 +40,6 @@ class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper Date: Sat, 13 Jul 2024 18:25:21 +0100 Subject: [PATCH 16/19] replace system line separator with '`n' --- src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt index 9f572eb1..657f4dba 100644 --- a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt @@ -2,6 +2,7 @@ package org.nixos.idea.psi.impl import com.intellij.lang.ASTNode import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.vfs.VirtualFile import com.intellij.psi.PsiLanguageInjectionHost import com.intellij.psi.impl.source.tree.LeafPsiElement import org.nixos.idea.psi.NixIndString @@ -38,7 +39,7 @@ abstract class AbstractNixString(private val astNode: ASTNode) : PsiLanguageInje // if the first line was removed in the fragment, add it back to preserve a multiline string val withLeadingBlankLine = if (lines.first().isNotEmpty()) listOf("") + withIndent else withIndent - originalNode?.replaceWithText(withLeadingBlankLine.joinToString(separator = System.lineSeparator())) + originalNode?.replaceWithText(withLeadingBlankLine.joinToString(separator = "\n")) return this } From f47a40329bcb6dc4bb3d3d2916a15589869fc025 Mon Sep 17 00:00:00 2001 From: Johannes Spangenberg Date: Tue, 16 Jul 2024 00:49:23 +0200 Subject: [PATCH 17/19] Use lexer to help detect indentation of strings --- .../highlighter/NixSyntaxHighlighter.java | 6 +- .../org/nixos/idea/util/NixStringUtil.java | 78 ++++++++++++++++--- src/main/lang/Nix.bnf | 2 +- src/main/lang/Nix.flex | 46 +++++++---- .../nixos/idea/util/NixStringUtilTest.java | 76 +++++++++++++++++- .../StringWithMultipleLines.lexer.txt | 11 ++- .../ParsingTest/StringWithMultipleLines.txt | 13 +++- 7 files changed, 197 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java b/src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java index 805139eb..54e14e3a 100644 --- a/src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java +++ b/src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java @@ -71,13 +71,15 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase { entry(NixTypes.URI, NixTextAttributes.URI), // String literals entry(NixTypes.STR, NixTextAttributes.STRING), + entry(NixTypes.STR_ESCAPE, NixTextAttributes.STRING_ESCAPE), entry(NixTypes.STRING_CLOSE, NixTextAttributes.STRING), entry(NixTypes.STRING_OPEN, NixTextAttributes.STRING), entry(NixTypes.IND_STR, NixTextAttributes.STRING), + entry(NixTypes.IND_STR_LF, NixTextAttributes.STRING), + entry(NixTypes.IND_STR_INDENT, NixTextAttributes.STRING), + entry(NixTypes.IND_STR_ESCAPE, NixTextAttributes.STRING_ESCAPE), entry(NixTypes.IND_STRING_CLOSE, NixTextAttributes.STRING), entry(NixTypes.IND_STRING_OPEN, NixTextAttributes.STRING), - entry(NixTypes.STR_ESCAPE, NixTextAttributes.STRING_ESCAPE), - entry(NixTypes.IND_STR_ESCAPE, NixTextAttributes.STRING_ESCAPE), // Other entry(NixTypes.SCOMMENT, NixTextAttributes.LINE_COMMENT), entry(NixTypes.MCOMMENT, NixTextAttributes.BLOCK_COMMENT), diff --git a/src/main/java/org/nixos/idea/util/NixStringUtil.java b/src/main/java/org/nixos/idea/util/NixStringUtil.java index 33fe05f3..3d72c512 100644 --- a/src/main/java/org/nixos/idea/util/NixStringUtil.java +++ b/src/main/java/org/nixos/idea/util/NixStringUtil.java @@ -4,12 +4,16 @@ import com.intellij.psi.tree.IElementType; import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; +import org.nixos.idea.psi.NixAntiquotation; +import org.nixos.idea.psi.NixIndString; +import org.nixos.idea.psi.NixStdString; +import org.nixos.idea.psi.NixString; import org.nixos.idea.psi.NixStringPart; import org.nixos.idea.psi.NixStringText; import org.nixos.idea.psi.NixTypes; /** - * Utilities for strings in the Nix Expression Language. + * Utilities for encoding and decoding strings in the Nix Expression Language. */ public final class NixStringUtil { @@ -87,6 +91,43 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence } } + /** + * Detects the maximal amount of characters removed from the start of the lines. + * May return {@link Integer#MAX_VALUE} if the content of the string is blank. + * + * @param string the string from which to get the indentation + * @return the detected indentation, or {@link Integer#MAX_VALUE} + */ + public static int detectMaxIndent(@NotNull NixString string) { + if (string instanceof NixStdString) { + return 0; + } else if (string instanceof NixIndString) { + int result = Integer.MAX_VALUE; + int preliminary = 0; + for (NixStringPart part : string.getStringParts()) { + if (part instanceof NixStringText textNode) { + for (ASTNode token = textNode.getNode().getFirstChildNode(); token != null; token = token.getTreeNext()) { + IElementType type = token.getElementType(); + if (type == NixTypes.IND_STR_INDENT) { + preliminary = Math.min(result, token.getTextLength()); + } else if (type == NixTypes.IND_STR_LF) { + preliminary = 0; + } else { + assert type == NixTypes.IND_STR || type == NixTypes.IND_STR_ESCAPE : type; + result = preliminary; + } + } + } else { + assert part instanceof NixAntiquotation : part.getClass(); + result = preliminary; + } + } + return result; + } else { + throw new IllegalStateException("Unexpected subclass of NixString: " + string.getClass()); + } + } + /** * Returns the content of the given part of a string in the Nix Expression Language. * All escape sequences are resolved. @@ -95,31 +136,46 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence * @return The resulting string after resolving all escape sequences. */ public static @NotNull String parse(@NotNull NixStringText textNode) { + int maxIndent = detectMaxIndent((NixString) textNode.getParent()); StringBuilder builder = new StringBuilder(); for (ASTNode child = textNode.getNode().getFirstChildNode(); child != null; child = child.getTreeNext()) { - parse(builder, child); + parse(builder, child, maxIndent); } return builder.toString(); } - private static void parse(@NotNull StringBuilder builder, @NotNull ASTNode token) { + private static void parse(@NotNull StringBuilder builder, @NotNull ASTNode token, int maxIndent) { CharSequence text = token.getChars(); IElementType type = token.getElementType(); - if (type == NixTypes.STR || type == NixTypes.IND_STR) { + if (type == NixTypes.STR || type == NixTypes.IND_STR || type == NixTypes.IND_STR_LF) { builder.append(text); + } else if (type == NixTypes.IND_STR_INDENT) { + int end = text.length(); + if (end > maxIndent) { + CharSequence remain = text.subSequence(maxIndent, end); + builder.append(remain); + } } else if (type == NixTypes.STR_ESCAPE) { assert text.length() == 2 && text.charAt(0) == '\\' : text; char c = text.charAt(1); builder.append(unescape(c)); } else if (type == NixTypes.IND_STR_ESCAPE) { - assert text.length() == 3 && ("''$".contentEquals(text) || "'''".contentEquals(text)) || - text.length() == 4 && "''\\".contentEquals(text.subSequence(0, 3)) : text; - if ("'''".contentEquals(text)){ - builder.append("''"); - return; + switch (text.charAt(2)) { + case '$' -> { + assert "''$".contentEquals(text) : text; + builder.append("$"); + } + case '\'' -> { + assert "'''".contentEquals(text) : text; + builder.append("''"); + } + case '\\' -> { + assert text.length() == 4 && "''\\".contentEquals(text.subSequence(0, 3)) : text; + char c = text.charAt(3); + builder.append(unescape(c)); + } + default -> throw new IllegalStateException("Unknown escape sequence: " + text); } - char c = text.charAt(text.length() - 1); - builder.append(unescape(c)); } else { throw new IllegalStateException("Unexpected token in string: " + token); } diff --git a/src/main/lang/Nix.bnf b/src/main/lang/Nix.bnf index 8b074a10..dffa4db7 100644 --- a/src/main/lang/Nix.bnf +++ b/src/main/lang/Nix.bnf @@ -211,7 +211,7 @@ string_text ::= string_token+ antiquotation ::= DOLLAR LCURLY expr recover_antiquotation RCURLY { pin=1 } private recover_antiquotation ::= { recoverWhile=curly_recover } private string_part_recover ::= !(DOLLAR | STRING_CLOSE | IND_STRING_CLOSE | string_token) -private string_token ::= STR | IND_STR | STR_ESCAPE | IND_STR_ESCAPE +private string_token ::= STR | STR_ESCAPE | IND_STR | IND_STR_INDENT | IND_STR_ESCAPE | IND_STR_LF ;{ extends("bind_attr|bind_inherit")=bind } bind ::= bind_attr | bind_inherit diff --git a/src/main/lang/Nix.flex b/src/main/lang/Nix.flex index af5107e6..517e2c3a 100644 --- a/src/main/lang/Nix.flex +++ b/src/main/lang/Nix.flex @@ -13,26 +13,27 @@ import static org.nixos.idea.psi.NixTypes.*; private final AbstractIntList states = new IntArrayList(); private void pushState(int newState) { - if (newState == YYINITIAL){ - throw new IllegalStateException("Pusing YYINITIAL is not supported"); - } + assert newState != YYINITIAL : "Pusing YYINITIAL is not supported"; // store current state on the stack to allow restoring it in popState(...) states.push(yystate()); yybegin(newState); } private void popState(int expectedState) { - if (states.isEmpty()){ - throw new IllegalStateException("Popping an empty stack of states. Expected: " + expectedState); - } + assert !states.isEmpty() : "Popping an empty stack of states. Expected: " + expectedState; // safe-guard, because we always know which state we're currently in in the rules below - if (yystate() != expectedState) { - throw new IllegalStateException(String.format("Unexpected state. Current: %d, expected: %d", yystate(), expectedState)); - } + assert yystate() == expectedState : String.format("Unexpected state. Current: %d, expected: %d", yystate(), expectedState); // start the lexer with the previous state, which was stored by pushState(...) yybegin(states.popInt()); } + private void replaceState(int expectedState, int newState) { + assert newState != YYINITIAL : "Pusing YYINITIAL is not supported"; + // safe-guard, because we always know which state we're currently in in the rules below + assert yystate() == expectedState : String.format("Unexpected state. Current: %d, expected: %d", yystate(), expectedState); + yybegin(newState); + } + protected void onReset() { states.clear(); } @@ -44,7 +45,9 @@ import static org.nixos.idea.psi.NixTypes.*; %function advance %type IElementType %unicode -%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION PATH +%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION +%xstate IND_STRING_START IND_STRING_INDENT PATH +%suppress empty-match ANY=[^] ID=[a-zA-Z_][a-zA-Z0-9_'-]* @@ -71,8 +74,19 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ \" { popState(STRING); return STRING_CLOSE; } } + { + // The first line is ignored in case it is empty + [\ ]*\n { replaceState(IND_STRING_START, IND_STRING_INDENT); return com.intellij.psi.TokenType.WHITE_SPACE; } +} + + { + [\ ]+ { replaceState(yystate(), IND_STRING); return IND_STR_INDENT; } + "" { replaceState(yystate(), IND_STRING); } +} + { - [^\$\']+ { return IND_STR; } + \n { replaceState(IND_STRING, IND_STRING_INDENT); return IND_STR_LF; } + [^\$\'\n]+ { return IND_STR; } "$"|"$$"|"'" { return IND_STR; } "''$"|"'''" { return IND_STR_ESCAPE; } "''"\\{ANY} { return IND_STR_ESCAPE; } @@ -83,7 +97,7 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ { // '$' and '{' must be two separate tokens to make NixBraceMatcher work // correctly with Grammar-Kit. - "{" { popState(ANTIQUOTATION_START); pushState(ANTIQUOTATION); return LCURLY; } + "{" { replaceState(ANTIQUOTATION_START, ANTIQUOTATION); return LCURLY; } } { @@ -98,10 +112,10 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ "$"/"{" { pushState(ANTIQUOTATION_START); return DOLLAR; } {PATH_SEG} { return PATH_SEGMENT; } {PATH_CHAR}+ { return PATH_SEGMENT; } - // anything else, e.g. whitespace, stops lexing of a PATH + // anything else, e.g. a whitespace, stops lexing of a PATH // we're delegating back to the parent state // PATH_END is an empty-length token to signal the end of the path - [^] { popState(PATH); yypushback(yylength()); return PATH_END; } + "" { popState(PATH); return PATH_END; } } { @@ -152,7 +166,7 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ "->" { return IMPL; } \" { pushState(STRING); return STRING_OPEN; } - \'\' { pushState(IND_STRING); return IND_STRING_OPEN; } + \'\' { pushState(IND_STRING_START); return IND_STRING_OPEN; } // Note that `true`, `false` and `null` are built-in variables but not // keywords. Therefore, they are not listed here. @@ -171,5 +185,5 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ {WHITE_SPACE} { return com.intellij.psi.TokenType.WHITE_SPACE; } } -// matched by all %state states +// matched by inclusive states (%state), but not by exclusive states (%xstate) [^] { return com.intellij.psi.TokenType.BAD_CHARACTER; } diff --git a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java index 90a15934..fe0002ac 100644 --- a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java @@ -13,6 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +@SuppressWarnings("UnnecessaryStringEscape") final class NixStringUtilTest { @ParameterizedTest(name = "[{index}] {0} -> {1}") @CsvSource(textBlock = """ @@ -54,6 +55,51 @@ void escape(String unescaped, String expectedResult) { assertEquals(expectedResult, stringBuilder.toString()); } + @ParameterizedTest(name = "[{index}] {0} -> {1}") + @CsvSource(quoteCharacter = '|', textBlock = """ + # Non-indented strings always return the empty string + |""| , 0 + |" a"| , 0 + |" a\n b"| , 0 + # When there are only spaces, we return Integer.MAX_VALUE + |''''| , 2147483647 + |'' ''| , 2147483647 + |''\n \n ''| , 2147483647 + # The smallest indentation counts + |''\n a\n b''| , 1 + |''\n a\n b''| , 1 + |''\n a\n b''| , 2 + |''\n a\n ${b}''| , 1 + |''\n a\n ''\\b''| , 1 + # First line counts + |''a\n b''| , 0 + |''${a}\n b''| , 0 + |''''\\a\n b''| , 0 + # But only the first token in a line counts + |'' a${b}''| , 2 + |'' a''\\b''| , 2 + |'' ${a}b''| , 2 + |'' ${a}${b}''| , 2 + |'' ${a}''\\b''| , 2 + |'' ''\\ab''| , 2 + |'' ''\\a${b}''| , 2 + |'' ''\\a''\\b''| , 2 + # Tab and CR are treated as normal characters, not as spaces + # See NixOS/nix#2911 and NixOS/nix#3759 + |''\t''| , 0 + |''\n \t''| , 2 + |''\r\n''| , 0 + |''\n \r\n''| , 2 + # Indentation within interpolations is ignored + |'' ${\n"a"}''| , 2 + |'' ${\n''a''}''| , 2 + """) + @WithIdeaPlatform.OnEdt + void detectMaxIndent(String code, int expectedResult, Project project) { + NixString string = NixElementFactory.createString(project, code); + assertEquals(expectedResult, NixStringUtil.detectMaxIndent(string)); + } + @ParameterizedTest(name = "[{index}] {0} -> {1}") @CsvSource(quoteCharacter = '|', textBlock = """ "" , || @@ -62,7 +108,9 @@ void escape(String unescaped, String expectedResult) { "\\"" , " "\\\\" , \\ "\\\\x" , \\x + ''"'' , " ''\\"'' , \\" + ''\\x'' , \\x ''\\\\'' , \\\\ ''\\\\x'' , \\\\x ''''\\"'' , " @@ -78,12 +126,36 @@ void escape(String unescaped, String expectedResult) { |"\n"| , |\n| |"\r"| , |\r| |"\t"| , |\t| + |"\\n"| , |\n| + |"\\r"| , |\r| + |"\\t"| , |\t| + |''_\n''| , |_\n| + |''\r''| , |\r| + |''\t''| , |\t| + |''''\\n''| , |\n| + |''''\\r''| , |\r| + |''''\\t''| , |\t| # supplementary character, i.e. character form a supplementary plane, # which needs a surrogate pair to be represented in UTF-16 "\uD83C\uDF09" , \uD83C\uDF09 ''\uD83C\uDF09'', \uD83C\uDF09 - # TODO implement indentation (the one below fails) - # '' a '' , |a | + # Remove common indentation in indented strings + |'' a ''| , |a | + |'' a ''| , |a | + |'' a\n b\n''| , |a\nb\n| + |'' a\n b\n''| , |a\n b\n| + # But don't remove indentation when there is one line without it + |'' a\nb\n c''| , | a\nb\n c| + |''a\n b\n c''| , |a\n b\n c| + |'' a\n\tb''|, | a\n\tb| + |''\ta\n b''|, |\ta\n b| + # Even when the line is blank + |'' a\n ''| , |a\n | + # Ignore indentation of empty lines + |'' a\n\n b\n''|, |a\n\nb\n| + # Remove initial line break in indented strings + |''\n a''| , |a| + |'' \n a''| , |a| """) @WithIdeaPlatform.OnEdt void parse(String code, String expectedResult, Project project) { diff --git a/src/test/testData/ParsingTest/StringWithMultipleLines.lexer.txt b/src/test/testData/ParsingTest/StringWithMultipleLines.lexer.txt index 010624ea..e41d77a6 100644 --- a/src/test/testData/ParsingTest/StringWithMultipleLines.lexer.txt +++ b/src/test/testData/ParsingTest/StringWithMultipleLines.lexer.txt @@ -5,7 +5,16 @@ STR ('\n first\n second\n third\n') STRING_CLOSE ('"') WHITE_SPACE ('\n') IND_STRING_OPEN ('''') -IND_STR ('\n first\n second\n third\n') +WHITE_SPACE ('\n') +IND_STR_INDENT (' ') +IND_STR ('first') +IND_STR_LF ('\n') +IND_STR_INDENT (' ') +IND_STR ('second') +IND_STR_LF ('\n') +IND_STR_INDENT (' ') +IND_STR ('third') +IND_STR_LF ('\n') IND_STRING_CLOSE ('''') WHITE_SPACE ('\n') ] (']') diff --git a/src/test/testData/ParsingTest/StringWithMultipleLines.txt b/src/test/testData/ParsingTest/StringWithMultipleLines.txt index 6f121793..84694972 100644 --- a/src/test/testData/ParsingTest/StringWithMultipleLines.txt +++ b/src/test/testData/ParsingTest/StringWithMultipleLines.txt @@ -10,8 +10,17 @@ Nix File(0,63) PsiWhiteSpace('\n')(30,31) NixIndStringImpl(IND_STRING)(31,61) PsiElement(IND_STRING_OPEN)('''')(31,33) - NixStringTextImpl(STRING_TEXT)(33,59) - PsiElement(IND_STR)('\n first\n second\n third\n')(33,59) + PsiWhiteSpace('\n')(33,34) + NixStringTextImpl(STRING_TEXT)(34,59) + PsiElement(IND_STR_INDENT)(' ')(34,36) + PsiElement(IND_STR)('first')(36,41) + PsiElement(IND_STR_LF)('\n')(41,42) + PsiElement(IND_STR_INDENT)(' ')(42,44) + PsiElement(IND_STR)('second')(44,50) + PsiElement(IND_STR_LF)('\n')(50,51) + PsiElement(IND_STR_INDENT)(' ')(51,53) + PsiElement(IND_STR)('third')(53,58) + PsiElement(IND_STR_LF)('\n')(58,59) PsiElement(IND_STRING_CLOSE)('''')(59,61) PsiWhiteSpace('\n')(61,62) PsiElement(])(']')(62,63) From 328d64404eae6e8b33a79bc5bd5461e4ad7784d3 Mon Sep 17 00:00:00 2001 From: Johannes Spangenberg Date: Tue, 16 Jul 2024 01:03:25 +0200 Subject: [PATCH 18/19] Use NixStringUtil in NixStringLiteralEscaper --- .../nixos/idea/psi/NixStringLiteralEscaper.kt | 166 ++++++------------ .../org/nixos/idea/util/NixStringUtil.java | 59 +++++-- .../nixos/idea/util/NixIndStringUtilTest.java | 37 ---- 3 files changed, 99 insertions(+), 163 deletions(-) delete mode 100644 src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index 9454450a..c6e5bb07 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -2,139 +2,85 @@ package org.nixos.idea.psi import com.intellij.openapi.util.TextRange import com.intellij.psi.LiteralTextEscaper -import com.intellij.psi.PsiLanguageInjectionHost -import org.intellij.lang.annotations.Language import org.nixos.idea.psi.impl.AbstractNixString +import org.nixos.idea.util.NixStringUtil -class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { +class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { override fun isOneLine(): Boolean = false private var outSourceOffsets: IntArray? = null override fun getRelevantTextRange(): TextRange { - if (myHost.textLength <= 4) return TextRange.EMPTY_RANGE - return TextRange.create(2, myHost.textLength - 2) + val parts = myHost.stringParts + return if (parts.isEmpty()) TextRange.EMPTY_RANGE + else TextRange.create(parts.first().startOffsetInParent, parts.last().textRangeInParent.endOffset) } override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean { - // TODO issue #81 only indented strings supported for now - // single line strings require a new decode function because - // it uses different escaping mechanisms - if (myHost !is NixIndString) return false - + val maxIndent = NixStringUtil.detectMaxIndent(myHost) val subText: String = rangeInsideHost.substring(myHost.text) + val outOffset = outChars.length val array = IntArray(subText.length + 1) - val success = unescapeAndDecode(subText, outChars, array) - outSourceOffsets = array - return success - } - - override fun getOffsetInHost(offsetInDecoded: Int, rangeInsideHost: TextRange): Int { - val offsets = outSourceOffsets ?: throw IllegalStateException("#decode was not called") - val result = if (offsetInDecoded < offsets.size) offsets[offsetInDecoded] else -1 - return result.coerceIn(0..rangeInsideHost.length) + rangeInsideHost.startOffset - } - - companion object { - /** - * Does not consider interpolations so that - * they do appear in the guest language and remain when we end up converting back to Nix. - */ - fun unescapeAndDecode(chars: String, outChars: StringBuilder, sourceOffsets: IntArray?): Boolean { - assert(sourceOffsets == null || sourceOffsets.size == chars.length + 1) - - var index = 0 - val outOffset = outChars.length - var braces = 0 - var indentSoFar = 0 - val minIndent = chars.lines() - .filterNot { it.isEmpty() } - .minOfOrNull { it.takeWhile(Char::isWhitespace).count() } ?: 0 - - - while (index < chars.length) { - fun updateOffsets(index: Int) { - if (sourceOffsets != null) { - sourceOffsets[outChars.length - outOffset] = index - 1 - sourceOffsets[outChars.length - outOffset + 1] = index - } - } - - var c = chars[index++] - updateOffsets(index) - - - if (braces > 0) { - if (c == '{') braces++ - else if (c == '}') braces-- - outChars.append(c) - continue - } + var success = true + + for (part in myHost.stringParts) { + assert(part.parent == myHost) + val partRange = part.textRangeInParent + if (partRange.startOffset >= rangeInsideHost.endOffset) { + break + } else if (partRange.endOffset < rangeInsideHost.startOffset) { + continue + } - if (c == '\n' && index < chars.length - 1) { - // we know that the next n chars are going to be whitespace indent - index += minIndent - outChars.append(c) - if (sourceOffsets != null) { - sourceOffsets[outChars.length - outOffset] = index + fun addText(text: CharSequence, offset: Int): Boolean { + val start = partRange.startOffset + offset + for (i in text.indices) { + if (start + i >= rangeInsideHost.startOffset) { + array[outChars.length - outOffset] = start + i + outChars.append(text[i]) + } else if (start + i >= rangeInsideHost.endOffset) { + return false } - continue } + return true + } - if (c == '\'') { - if (index == chars.length) return false - c = chars[index++] - - if (c != '\'') { - // if what follows isn't another ' then we are not escaping anything, - // so we can backtrace and continue - outChars.append("\'") - index-- - continue + if (part is NixStringText) { + NixStringUtil.visit(object : NixStringUtil.StringVisitor { + override fun text(text: CharSequence, offset: Int): Boolean { + return addText(text, offset) } - if (index == chars.length) return false - c = chars[index++] - - when (c) { - // '' can be escaped by prefixing it with ', i.e., '''. - '\'' -> { - outChars.append("\'") - updateOffsets(index - 1) - outChars.append(c) - } - // $ can be escaped by prefixing it with '' (that is, two single quotes), i.e., ''$. - '$' -> outChars.append(c) - '\\' -> { - if (index == chars.length) return false - c = chars[index++] - when (c) { - // Linefeed, carriage-return and tab characters can - // be written as ''\n, ''\r, ''\t, and ''\ escapes any other character. - 'a' -> outChars.append(0x07.toChar()) - 'b' -> outChars.append('\b') - 'f' -> outChars.append(0x0c.toChar()) - 'n' -> outChars.append('\n') - 't' -> outChars.append('\t') - 'r' -> outChars.append('\r') - 'v' -> outChars.append(0x0b.toChar()) - else -> return false + override fun escapeSequence(text: String, offset: Int, escapeSequence: CharSequence): Boolean { + val start = partRange.startOffset + offset + val end = start + escapeSequence.length + return if (start < rangeInsideHost.startOffset || end > rangeInsideHost.endOffset) { + success = false + false + } else { + for (i in escapeSequence.indices) { + array[outChars.length - outOffset + i] = start } + outChars.append(text) + true } - - else -> return false - } - if (sourceOffsets != null) { - sourceOffsets[outChars.length - outOffset] = index } - continue - } - - outChars.append(c) + }, part, maxIndent) + } else { + assert(part is NixAntiquotation) + addText(part.text, 0) } - return true } + + outSourceOffsets = array + return success + } + + override fun getOffsetInHost(offsetInDecoded: Int, rangeInsideHost: TextRange): Int { + val offsets = outSourceOffsets ?: throw IllegalStateException("#decode was not called") + val result = if (offsetInDecoded < offsets.size) offsets[offsetInDecoded] else -1 + return result.coerceIn(0..rangeInsideHost.length) + rangeInsideHost.startOffset } -} \ No newline at end of file +} diff --git a/src/main/java/org/nixos/idea/util/NixStringUtil.java b/src/main/java/org/nixos/idea/util/NixStringUtil.java index 3d72c512..ecff671c 100644 --- a/src/main/java/org/nixos/idea/util/NixStringUtil.java +++ b/src/main/java/org/nixos/idea/util/NixStringUtil.java @@ -138,55 +138,82 @@ public static int detectMaxIndent(@NotNull NixString string) { public static @NotNull String parse(@NotNull NixStringText textNode) { int maxIndent = detectMaxIndent((NixString) textNode.getParent()); StringBuilder builder = new StringBuilder(); + visit(new StringVisitor() { + @Override + public boolean text(@NotNull CharSequence text, int offset) { + builder.append(text); + return true; + } + + @Override + public boolean escapeSequence(@NotNull String text, int offset, @NotNull CharSequence escapeSequence) { + builder.append(text); + return true; + } + }, textNode, maxIndent); + return builder.toString(); + } + + public static void visit(@NotNull StringVisitor visitor, @NotNull NixStringText textNode, int maxIndent) { + int offset = 0; for (ASTNode child = textNode.getNode().getFirstChildNode(); child != null; child = child.getTreeNext()) { - parse(builder, child, maxIndent); + if (!parse(visitor, child, offset, maxIndent)) { + break; + } + offset += child.getTextLength(); } - return builder.toString(); } - private static void parse(@NotNull StringBuilder builder, @NotNull ASTNode token, int maxIndent) { + private static boolean parse(@NotNull StringVisitor visitor, @NotNull ASTNode token, int offset, int maxIndent) { CharSequence text = token.getChars(); IElementType type = token.getElementType(); if (type == NixTypes.STR || type == NixTypes.IND_STR || type == NixTypes.IND_STR_LF) { - builder.append(text); + return visitor.text(text, offset); } else if (type == NixTypes.IND_STR_INDENT) { int end = text.length(); if (end > maxIndent) { CharSequence remain = text.subSequence(maxIndent, end); - builder.append(remain); + return visitor.text(remain, offset + maxIndent); } + return true; } else if (type == NixTypes.STR_ESCAPE) { assert text.length() == 2 && text.charAt(0) == '\\' : text; char c = text.charAt(1); - builder.append(unescape(c)); + return visitor.escapeSequence(unescape(c), offset, text); } else if (type == NixTypes.IND_STR_ESCAPE) { - switch (text.charAt(2)) { + return switch (text.charAt(2)) { case '$' -> { assert "''$".contentEquals(text) : text; - builder.append("$"); + yield visitor.escapeSequence("$", offset, text); } case '\'' -> { assert "'''".contentEquals(text) : text; - builder.append("''"); + yield visitor.escapeSequence("''", offset, text); } case '\\' -> { assert text.length() == 4 && "''\\".contentEquals(text.subSequence(0, 3)) : text; char c = text.charAt(3); - builder.append(unescape(c)); + yield visitor.escapeSequence(unescape(c), offset, text); } default -> throw new IllegalStateException("Unknown escape sequence: " + text); - } + }; } else { throw new IllegalStateException("Unexpected token in string: " + token); } } - private static char unescape(char c) { + private static @NotNull String unescape(char c) { return switch (c) { - case 'n' -> '\n'; - case 'r' -> '\r'; - case 't' -> '\t'; - default -> c; + case 'n' -> "\n"; + case 'r' -> "\r"; + case 't' -> "\t"; + default -> String.valueOf(c); }; } + + public interface StringVisitor { + boolean text(@NotNull CharSequence text, int offset); + + boolean escapeSequence(@NotNull String text, int offset, @NotNull CharSequence escapeSequence); + } } diff --git a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java deleted file mode 100644 index 846c05d7..00000000 --- a/src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.nixos.idea.util; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.nixos.idea.psi.NixStringLiteralEscaper; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -final class NixIndStringUtilTest { - @ParameterizedTest(name = "[{index}] {0} -> {1}") - @CsvSource(quoteCharacter = '|', textBlock = """ - || , || - abc , abc - " , " - \\ , \\ - \\x , \\x - a${b}c , a${b}c - |\n| , |\n| - |\r| , |\r| - |\t| , |\t| - |''\\t| , |\t| - |''\\r| , |\r| - |''\\n| , |\n| - |'''| , |''| - $$ , $$ - ''$ , $ - # supplementary character, i.e. character form a supplementary plane, - # which needs a surrogate pair to be represented in UTF-16 - \uD83C\uDF09 , \uD83C\uDF09 - """) - void unescape(String escaped, String expectedResult) { - var sb = new StringBuilder(); - NixStringLiteralEscaper.Companion.unescapeAndDecode(escaped, sb, null); - var str = sb.toString(); - assertEquals(expectedResult, str); - } -} From ce8ee544158f7278aad6ee4cb1aef26e0148b4d1 Mon Sep 17 00:00:00 2001 From: Johannes Spangenberg Date: Sun, 28 Jul 2024 20:22:03 +0200 Subject: [PATCH 19/19] Refactor to use NixInjectionPerformer (still buggy) --- .../org/nixos/idea/psi/NixElementFactory.java | 8 + .../nixos/idea/psi/NixInjectionPerformer.java | 92 ++++++++++ .../nixos/idea/psi/NixStringLiteralEscaper.kt | 82 ++++----- .../nixos/idea/psi/NixStringManipulator.kt | 26 ++- .../nixos/idea/psi/impl/AbstractNixString.kt | 80 +++++---- .../org/nixos/idea/util/NixStringUtil.java | 72 +++++++- src/main/lang/Nix.bnf | 14 +- src/main/resources/META-INF/plugin.xml | 10 +- .../nixos/idea/psi/NixElementFactoryTest.java | 28 +++ .../idea/psi/NixInjectionPerformerTest.java | 166 ++++++++++++++++++ .../nixos/idea/util/NixStringUtilTest.java | 42 ++++- 11 files changed, 502 insertions(+), 118 deletions(-) create mode 100644 src/main/java/org/nixos/idea/psi/NixInjectionPerformer.java create mode 100644 src/test/java/org/nixos/idea/psi/NixInjectionPerformerTest.java diff --git a/src/main/java/org/nixos/idea/psi/NixElementFactory.java b/src/main/java/org/nixos/idea/psi/NixElementFactory.java index 6ff85992..1614d7d6 100644 --- a/src/main/java/org/nixos/idea/psi/NixElementFactory.java +++ b/src/main/java/org/nixos/idea/psi/NixElementFactory.java @@ -20,6 +20,14 @@ private NixElementFactory() {} // Cannot be instantiated return createElement(project, NixString.class, "", code, ""); } + public static @NotNull NixStringText createStdStringText(@NotNull Project project, @NotNull String code) { + return createElement(project, NixStringText.class, "\"", code, "\""); + } + + public static @NotNull NixStringText createIndStringText(@NotNull Project project, @NotNull String code) { + return createElement(project, NixStringText.class, "''\n", code, "''"); + } + public static @NotNull NixAttr createAttr(@NotNull Project project, @NotNull String code) { return createElement(project, NixAttr.class, "x.", code, ""); } diff --git a/src/main/java/org/nixos/idea/psi/NixInjectionPerformer.java b/src/main/java/org/nixos/idea/psi/NixInjectionPerformer.java new file mode 100644 index 00000000..a04ccd13 --- /dev/null +++ b/src/main/java/org/nixos/idea/psi/NixInjectionPerformer.java @@ -0,0 +1,92 @@ +package org.nixos.idea.psi; + +import com.intellij.lang.Language; +import com.intellij.lang.injection.MultiHostRegistrar; +import com.intellij.lang.injection.general.Injection; +import com.intellij.lang.injection.general.LanguageInjectionPerformer; +import com.intellij.openapi.fileTypes.LanguageFileType; +import com.intellij.psi.LiteralTextEscaper; +import com.intellij.psi.PsiElement; +import com.intellij.psi.util.PsiTreeUtil; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +public class NixInjectionPerformer implements LanguageInjectionPerformer { + + @Override + public boolean isPrimary() { + return true; + } + + @Override + public boolean performInjection(@NotNull MultiHostRegistrar registrar, @NotNull Injection injection, @NotNull PsiElement context) { + + // TODO Support injection on quotes? + // It is currently not possible to press Alt+Enter on the quotes to enable language injections. + // TODO Add InjectedLanguageManager.FRANKENSTEIN_INJECTION when there are interpolations? + // This seems to be used by various languages to disable certain error checks as when part of the source code is unavailable. + // TODO JetBrains implementations of LanguageInjectionPerformer do more stuff, + // like calling InjectorUtils.registerSupport(...). Not sure whether this is relevant. + // TODO Adding new interpolations after enabling a language injection has strange effect. + // TODO What about LanguageInjectionSupport? https://plugins.jetbrains.com/docs/intellij/language-injection.html + + NixString string = PsiTreeUtil.getParentOfType(context, NixString.class, false); + if (string == null) { + return false; + } + Language injectedLanguage = injection.getInjectedLanguage(); + if (injectedLanguage == null) { + return false; + } + + LanguageFileType injectedFileType = injectedLanguage.getAssociatedFileType(); + String injectedFileExtension = injectedFileType == null ? null : injectedFileType.getDefaultExtension(); + + registrar.startInjecting(injectedLanguage, injectedFileExtension); + for (Place place : collectPlaces(string, injection.getPrefix(), injection.getSuffix())) { + LiteralTextEscaper escaper = place.item().createLiteralTextEscaper(); + registrar.addPlace( + place.prefix().toString(), place.suffix().toString(), + place.item(), escaper.getRelevantTextRange() + ); + } + registrar.doneInjecting(); + return true; + } + + private record Place(@NotNull NixStringText item, @NotNull CharSequence prefix, @NotNull CharSequence suffix) {} + + private static List collectPlaces(@NotNull NixString string, @NotNull String prefix, @NotNull String suffix) { + int interpolations = 0; + StringBuilder prevSuffix = null; + List result = new ArrayList<>(); + for (NixStringPart stringPart : string.getStringParts()) { + // Note: The first and last part is always of type NixStringText, the list is never empty + if (stringPart instanceof NixStringText text) { + prevSuffix = new StringBuilder(); + result.add(new Place(text, prefix, prevSuffix)); + prefix = ""; + } else { + assert stringPart instanceof NixAntiquotation : stringPart.getClass(); + assert prevSuffix != null; + prevSuffix.append(interpolationPlaceholder(interpolations++)); + } + } + assert prevSuffix != null; + prevSuffix.append(suffix); + return result; + } + + /** + * Arbitrary code used in the guest language in place for string interpolations. + * This code is not visible for the user in the IDE as IDEA will visualize source code of the interpolations instead. + * This method returns a different string for each interpolation to prevent tooling of the guess language + * from assuming that all interpolations generate the same result. + */ + private static String interpolationPlaceholder(int index) { + // TODO What should I return here? Would an empty string work? What do other plugins use as placeholders? + return "(__interpolation" + index + "__)"; + } +} diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt index c6e5bb07..9c334fc3 100644 --- a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -5,72 +5,54 @@ import com.intellij.psi.LiteralTextEscaper import org.nixos.idea.psi.impl.AbstractNixString import org.nixos.idea.util.NixStringUtil -class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { +class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { override fun isOneLine(): Boolean = false private var outSourceOffsets: IntArray? = null - override fun getRelevantTextRange(): TextRange { - val parts = myHost.stringParts - return if (parts.isEmpty()) TextRange.EMPTY_RANGE - else TextRange.create(parts.first().startOffsetInParent, parts.last().textRangeInParent.endOffset) - } - override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean { - val maxIndent = NixStringUtil.detectMaxIndent(myHost) + val maxIndent = NixStringUtil.detectMaxIndent(myHost.parent as NixString) val subText: String = rangeInsideHost.substring(myHost.text) val outOffset = outChars.length val array = IntArray(subText.length + 1) var success = true - for (part in myHost.stringParts) { - assert(part.parent == myHost) - val partRange = part.textRangeInParent - if (partRange.startOffset >= rangeInsideHost.endOffset) { - break - } else if (partRange.endOffset < rangeInsideHost.startOffset) { - continue - } - - fun addText(text: CharSequence, offset: Int): Boolean { - val start = partRange.startOffset + offset - for (i in text.indices) { - if (start + i >= rangeInsideHost.startOffset) { - array[outChars.length - outOffset] = start + i - outChars.append(text[i]) - } else if (start + i >= rangeInsideHost.endOffset) { - return false - } + fun addText(text: CharSequence, offset: Int): Boolean { + for (i in text.indices) { + if (offset + i >= rangeInsideHost.startOffset) { + array[outChars.length - outOffset] = offset + i + outChars.append(text[i]) + } else if (offset + i >= rangeInsideHost.endOffset) { + return false } - return true } + return true + } - if (part is NixStringText) { - NixStringUtil.visit(object : NixStringUtil.StringVisitor { - override fun text(text: CharSequence, offset: Int): Boolean { - return addText(text, offset) - } + NixStringUtil.visit(object : NixStringUtil.StringVisitor { + override fun text(text: CharSequence, offset: Int): Boolean { + return addText(text, offset) + } - override fun escapeSequence(text: String, offset: Int, escapeSequence: CharSequence): Boolean { - val start = partRange.startOffset + offset - val end = start + escapeSequence.length - return if (start < rangeInsideHost.startOffset || end > rangeInsideHost.endOffset) { - success = false - false - } else { - for (i in escapeSequence.indices) { - array[outChars.length - outOffset + i] = start - } - outChars.append(text) - true - } + override fun escapeSequence(text: String, offset: Int, escapeSequence: CharSequence): Boolean { + val end = offset + escapeSequence.length + return if (offset < rangeInsideHost.startOffset || end > rangeInsideHost.endOffset) { + success = false + false + } else { + for (i in escapeSequence.indices) { + array[outChars.length - outOffset + i] = offset } - }, part, maxIndent) - } else { - assert(part is NixAntiquotation) - addText(part.text, 0) + outChars.append(text) + true + } } + }, myHost, maxIndent) + // TODO Fix ArrayIndexOutOfBoundsException in the following line. (Not sure how to reproduce it.) + array[outChars.length - outOffset] = rangeInsideHost.endOffset + for (i in (outChars.length - outOffset + 1)..() { +class NixStringManipulator : AbstractElementManipulator() { /** * This function's result changes the original text in the host language * when the fragment in the guest language changes */ override fun handleContentChange( - element: NixString, + element: NixStringText, range: TextRange, newContent: String - ): NixString? { + ): NixStringText { + // TODO This implementation is wrong as escaped and non-escaped strings a mixed together. + // The variable `replacement` is not supposed to be escaped, + // but `element.text` is from the Nix source code, and therefore escaped. + // We probably have to switch the call dependency and let `updateText` call this more generic method. val escaped = newContent val replacement = range.replace(element.text, escaped) - return element.updateText(replacement) as? NixString + return element.updateText(replacement) as NixStringText } - override fun getRangeInElement(element: NixString): TextRange = when { + override fun getRangeInElement(element: NixStringText): TextRange = when { element.textLength == 0 -> TextRange.EMPTY_RANGE - element is NixIndString && element.textLength < 4 -> TextRange(0, element.textLength) - element is NixIndString -> TextRange(2, element.textLength - 2) - // element is not IndString, so it must be StdString - element.textLength == 1 -> TextRange(0, 1) - else -> TextRange(1, element.textLength - 1) + else -> TextRange.from(0, element.textLength) } -} \ No newline at end of file +} diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt index 657f4dba..bd71556a 100644 --- a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt @@ -1,52 +1,62 @@ package org.nixos.idea.psi.impl import com.intellij.lang.ASTNode -import com.intellij.openapi.diagnostic.Logger -import com.intellij.openapi.vfs.VirtualFile import com.intellij.psi.PsiLanguageInjectionHost -import com.intellij.psi.impl.source.tree.LeafPsiElement +import org.nixos.idea.psi.NixElementFactory import org.nixos.idea.psi.NixIndString -import org.nixos.idea.psi.NixString +import org.nixos.idea.psi.NixStdString import org.nixos.idea.psi.NixStringLiteralEscaper +import org.nixos.idea.psi.NixStringText +import org.nixos.idea.util.NixStringUtil abstract class AbstractNixString(private val astNode: ASTNode) : PsiLanguageInjectionHost, - AbstractNixPsiElement(astNode), NixString { - - override fun isValidHost() = this is NixIndString - - override fun updateText(s: String): NixString { - // TODO issue #81 also support single-line strings - if (this !is NixIndString) { - LOG.info("not a nix ind string") - return this + AbstractNixPsiElement(astNode), NixStringText { + + override fun isValidHost() = true + + override fun updateText(s: String): NixStringText { + val project = project + val replacement = when (val string = parent) { + is NixStdString -> { + val escaped = buildString { NixStringUtil.escapeStd(this, s) } + NixElementFactory.createStdStringText(project, escaped) + } + + is NixIndString -> { + val indent = indentForNewText(string) + val indentStart = prevSibling == null + val indentEnd = if (nextSibling == null) trailingIndent(text) ?: baseIndent(string) else indent + val escaped = buildString { NixStringUtil.escapeInd(this, s, indent, indentStart, indentEnd) } + NixElementFactory.createIndStringText(project, escaped) + } + + else -> throw IllegalStateException("Unexpected parent: " + parent.javaClass) } - val originalNode = astNode.firstChildNode.treeNext.firstChildNode as? LeafPsiElement - val minIndentInOriginal = originalNode?.text?.lines() - ?.filterNot { it.isEmpty() } - ?.minOfOrNull { it.takeWhile(Char::isWhitespace).count() } ?: 0 - - val leadingSpace = buildString { repeat(minIndentInOriginal) { append(' ') } } - - val lines = s.substring(2..(s.lastIndex - 2)) // remove quotes - .lines() - - // restore indent - val withIndent = lines - .withIndex() - .map { (index, line) -> if (index != 0) leadingSpace + line else line } - - // if the first line was removed in the fragment, add it back to preserve a multiline string - val withLeadingBlankLine = if (lines.first().isNotEmpty()) listOf("") + withIndent else withIndent - - originalNode?.replaceWithText(withLeadingBlankLine.joinToString(separator = "\n")) - return this + return replace(replacement) as NixStringText } override fun createLiteralTextEscaper() = NixStringLiteralEscaper(this) companion object { - val LOG = Logger.getInstance(AbstractNixString::class.java) + private fun indentForNewText(string: NixIndString): Int { + return NixStringUtil.detectMaxIndent(string).takeIf { it != Int.MAX_VALUE } ?: (baseIndent(string) + 2) + } + + private fun baseIndent(string: NixIndString): Int { + // TODO Detect indent of string + // This should be the indent of the line where the string starts. + return 0 + } + + private fun trailingIndent(str: String): Int? { + val lastLineFeed = str.lastIndexOf('\n') + val lastLine = if (lastLineFeed != -1) str.substring(lastLineFeed + 1) else null + return if (lastLine != null && lastLine.all { it == ' ' }) { + lastLine.length + } else { + null + } + } } } - diff --git a/src/main/java/org/nixos/idea/util/NixStringUtil.java b/src/main/java/org/nixos/idea/util/NixStringUtil.java index ecff671c..ebc9d306 100644 --- a/src/main/java/org/nixos/idea/util/NixStringUtil.java +++ b/src/main/java/org/nixos/idea/util/NixStringUtil.java @@ -36,7 +36,7 @@ private NixStringUtil() {} // Cannot be instantiated public static @NotNull String quote(@NotNull CharSequence unescaped) { StringBuilder builder = new StringBuilder(); builder.append('"'); - escape(builder, unescaped); + escapeStd(builder, unescaped); builder.append('"'); return builder.toString(); } @@ -47,19 +47,20 @@ private NixStringUtil() {} // Cannot be instantiated * For example, the following code would generate a broken result. * 
{@code
      *     StringBuilder b1 = new StringBuilder(), b2 = new StringBuilder();
-     *     NixStringUtil.escape(b1, "$");
-     *     NixStringUtil.escape(b2, "{''}");
+     *     NixStringUtil.escapeStd(b1, "$");
+     *     NixStringUtil.escapeStd(b2, "{''}");
      *     System.out.println(b1.toString() + b2.toString());
      * }
* The result would be the following broken Nix code. * 
-     *     "${''}"
+     *     ${''}
      *
* * @param builder The target string builder. The result will be appended to the given string builder. * @param unescaped The raw string which shall be escaped. */ - public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence unescaped) { + public static void escapeStd(@NotNull StringBuilder builder, @NotNull CharSequence unescaped) { + boolean potentialInterpolation = false; for (int charIndex = 0; charIndex < unescaped.length(); charIndex++) { char nextChar = unescaped.charAt(charIndex); switch (nextChar) { @@ -68,7 +69,7 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence builder.append('\\').append(nextChar); break; case '{': - if (builder.charAt(builder.length() - 1) == '$') { + if (potentialInterpolation) { builder.setCharAt(builder.length() - 1, '\\'); builder.append('$').append('{'); } else { @@ -88,6 +89,65 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence builder.append(nextChar); break; } + potentialInterpolation = nextChar == '$' && !potentialInterpolation; + } + } + + /** + * Escapes the given string for use in an indented string expression in the Nix Expression Language. + * Note that it is not safe to concat the result of two calls of this method. + * + * @param builder The target string builder. The result will be appended to the given string builder. + * @param unescaped The raw string which shall be escaped. + * @param indent The number as spaces used for indentation + * @param indentStart Whether the start of the string needs to be indented + * @param indentEnd The number as spaces used for indentation in the last line + */ + public static void escapeInd(@NotNull StringBuilder builder, @NotNull CharSequence unescaped, int indent, boolean indentStart, int indentEnd) { + String indentStr = " ".repeat(indent); + boolean potentialInterpolation = false; + boolean potentialClosing = false; + for (int charIndex = 0; charIndex < unescaped.length(); charIndex++) { + char nextChar = unescaped.charAt(charIndex); + if (indentStart && nextChar != '\n') { + builder.append(indentStr); + indentStart = false; + } + switch (nextChar) { + case '\'': + // Convert `''` to `'''` + if (potentialClosing) { + builder.append('\''); + } + builder.append('\''); + break; + case '{': + // Convert `${` to `''${`, but leave `$${` untouched + if (potentialInterpolation) { + builder.setLength(builder.length() - 1); + builder.append("''${"); + } else { + builder.append('{'); + } + break; + case '\r': + builder.append("''\\r"); + break; + case '\t': + builder.append("''\\t"); + break; + case '\n': + indentStart = true; + // fallthrough + default: + builder.append(nextChar); + break; + } + potentialInterpolation = nextChar == '$' && !potentialInterpolation; + potentialClosing = nextChar == '\'' && !potentialClosing; + } + if (indentStart) { + builder.append(" ".repeat(Math.min(indent, indentEnd))); } } diff --git a/src/main/lang/Nix.bnf b/src/main/lang/Nix.bnf index dffa4db7..4a965407 100644 --- a/src/main/lang/Nix.bnf +++ b/src/main/lang/Nix.bnf @@ -25,8 +25,8 @@ consumeTokenMethod("expr_op.*")="consumeTokenFast" // make IndStrings language injection Hosts - implements("string")="com.intellij.psi.PsiLanguageInjectionHost" - mixin("string")="org.nixos.idea.psi.impl.AbstractNixString" + implements("string_text")="com.intellij.psi.PsiLanguageInjectionHost" + mixin("string_text")="org.nixos.idea.psi.impl.AbstractNixString" tokens = [ // This list does not contain all tokens. This list only defines the debug @@ -202,15 +202,15 @@ private set_recover ::= curly_recover !bind private list_recover ::= brac_recover !expr_select ;{ extends(".*_string")="string" } -std_string ::= STRING_OPEN string_part* STRING_CLOSE { pin=1 } -ind_string ::= IND_STRING_OPEN string_part* IND_STRING_CLOSE { pin=1 } +std_string ::= STRING_OPEN string_parts STRING_CLOSE { pin=1 } +ind_string ::= IND_STRING_OPEN string_parts IND_STRING_CLOSE { pin=1 } fake string ::= string_part* { methods=[ string_parts="string_part" ] } +private string_parts ::= string_text ( antiquotation string_text )* ;{ extends("string_text|antiquotation")=string_part } -string_part ::= string_text | antiquotation { recoverWhile=string_part_recover } -string_text ::= string_token+ +fake string_part ::= +string_text ::= string_token* antiquotation ::= DOLLAR LCURLY expr recover_antiquotation RCURLY { pin=1 } private recover_antiquotation ::= { recoverWhile=curly_recover } -private string_part_recover ::= !(DOLLAR | STRING_CLOSE | IND_STRING_CLOSE | string_token) private string_token ::= STR | STR_ESCAPE | IND_STR | IND_STR_INDENT | IND_STR_ESCAPE | IND_STR_LF ;{ extends("bind_attr|bind_inherit")=bind } diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml index bdabdb6c..8f49cbf5 100644 --- a/src/main/resources/META-INF/plugin.xml +++ b/src/main/resources/META-INF/plugin.xml @@ -59,9 +59,13 @@ - - + + + diff --git a/src/test/java/org/nixos/idea/psi/NixElementFactoryTest.java b/src/test/java/org/nixos/idea/psi/NixElementFactoryTest.java index 47347287..048b08d5 100644 --- a/src/test/java/org/nixos/idea/psi/NixElementFactoryTest.java +++ b/src/test/java/org/nixos/idea/psi/NixElementFactoryTest.java @@ -33,6 +33,34 @@ void createStringFail(String code) { () -> NixElementFactory.createString(myProject, code)); } + @ParameterizedTest + @ValueSource(strings = {"", "x", "abc", " ", "\\n", "\\${x}", "$\\{x}"}) + void createStdStringText(String code) { + NixStringText result = NixElementFactory.createStdStringText(myProject, code); + assertEquals(code, result.getText()); + } + + @ParameterizedTest + @ValueSource(strings = {"\"", "\"x\"", "${x}", "\\${\"42\"}"}) + void createStdStringTextFail(String code) { + assertThrows(RuntimeException.class, + () -> NixElementFactory.createStdStringText(myProject, code)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "x", "abc", " ", "\n", "''\\n", "''${x}", "$${x}"}) + void createIndStringText(String code) { + NixStringText result = NixElementFactory.createIndStringText(myProject, code); + assertEquals(code, result.getText()); + } + + @ParameterizedTest + @ValueSource(strings = {"''", "''x''", "${x}", "$${''42''}"}) + void createIndStringTextFail(String code) { + assertThrows(RuntimeException.class, + () -> NixElementFactory.createIndStringText(myProject, code)); + } + @ParameterizedTest @ValueSource(strings = {"x", "\"x\"", "${x}", "${\"x\"}", "\"x${y}z\""}) void createAttr(String code) { diff --git a/src/test/java/org/nixos/idea/psi/NixInjectionPerformerTest.java b/src/test/java/org/nixos/idea/psi/NixInjectionPerformerTest.java new file mode 100644 index 00000000..6c8ea26b --- /dev/null +++ b/src/test/java/org/nixos/idea/psi/NixInjectionPerformerTest.java @@ -0,0 +1,166 @@ +package org.nixos.idea.psi; + +import com.intellij.lang.injection.InjectedLanguageManager; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.testFramework.EdtTestUtil; +import com.intellij.testFramework.fixtures.CodeInsightTestFixture; +import com.intellij.testFramework.fixtures.IdeaProjectTestFixture; +import com.intellij.testFramework.fixtures.IdeaTestExecutionPolicy; +import com.intellij.testFramework.fixtures.IdeaTestFixtureFactory; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Test; + +import java.util.Objects; + +final class NixInjectionPerformerTest { + + @Test + void trivial_ind_string() throws Exception { + doTest( + "''first ''${} line''", + "first ${} line" + ); + } + + @Test + void trivial_std_string() throws Exception { + doTest( + "\"first \\${} line\"", + "first ${} line" + ); + } + + @Test + void trim_initial_line() throws Exception { + doTest( + "''\nfirst line''", + "first line" + ); + } + + @Test + void trim_indentation() throws Exception { + doTest( + """ + '' + first line + + second line + \s + third line + '' + """, + """ + first line + + second line + \s + third line + """ + ); + } + + @Test + void keep_indentation_of_closing_quotes() throws Exception { + doTest( + """ + '' + first line + '' + """, + """ + first line + """ + ); + } + + @Test + void insert_into_empty_line() throws Exception { + // TODO remove spaces from empty line in source code. + // This test is supposed to test a scenario where the edited line misses the indent in the host language + doTest( + """ + '' + first line + + second line + '' + """, + """ + first line + + second line + """ + ); + } + + @Test + void insert_into_string() throws Exception { + // TODO remove spaces from empty line in source code. + // This test is supposed to test a scenario where the edited line misses the indent in the host language + doTest( + """ + '' + + '' + """, + """ + first line + + second line + """ + ); + } + + @Test + void insert_before_interpolation() throws Exception { + doTest( + "''${x}''", + "" + ); + } + + @Test + void insert_after_interpolation() throws Exception { + doTest( + "''${x}''", + "" + ); + } + + @Test + void insert_between_interpolations() throws Exception { + doTest( + "''${x}${y}''", + "" + ); + } + + private void doTest(@NotNull String sourceCode, @NotNull String injected) throws Exception { + IdeaTestFixtureFactory factory = IdeaTestFixtureFactory.getFixtureFactory(); + IdeaTestExecutionPolicy policy = Objects.requireNonNull(IdeaTestExecutionPolicy.current()); + IdeaProjectTestFixture baseFixture = factory.createLightFixtureBuilder("test").getFixture(); + CodeInsightTestFixture fixture = factory.createCodeInsightFixture(baseFixture, policy.createTempDirTestFixture()); + fixture.setTestDataPath(FileUtil.toSystemIndependentName(policy.getHomePath())); + fixture.setUp(); + try { + // TODO Implement test. This test is supposed to + // 1. Enable a language injection. + // 2. Open the injected language in the editor. + // 3. Insert text at in the editor of the injected language + // 4. Remove text at again + // 5. Insert another text at which needs to be escaped + // 6. Verify that during step 3 to 5, the text in the editor of the host language is adjusted appropriately + // How to enable language injection using code? + // There doesn't seem to be an action I could use via fixture.performEditorAction(IdeActions...). + // Maybe I can use TemporaryPlacesRegistry.addHostWithUndo directly, but it is not available in the classpath right now. + InjectedLanguageManager instance = InjectedLanguageManager.getInstance(fixture.getProject()); + // instance.getNonEditableFragments(); This seems like it could be used to detect the interpolations + int pos = sourceCode.indexOf(""); + fixture.setCaresAboutInjection(true); // This doesn't do anything as true is the default, but I left it here for now. + fixture.configureByText("test.nix", sourceCode.replace("", "")); + } finally { + EdtTestUtil.runInEdtAndWait(fixture::tearDown); + } + } +} diff --git a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java index fe0002ac..d936b1da 100644 --- a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java @@ -23,6 +23,7 @@ final class NixStringUtilTest { \\ , "\\\\" \\x , "\\\\x" a${b}c , "a\\${b}c" + a$${b}c , "a$${b}c" '\n' , "\\n" '\r' , "\\r" '\t' , "\\t" @@ -42,6 +43,7 @@ void quote(String unescaped, String expectedResult) { \\ , \\\\ \\x , \\\\x a${b}c , a\\${b}c + a$${b}c , a$${b}c '\n' , \\n '\r' , \\r '\t' , \\t @@ -49,9 +51,44 @@ void quote(String unescaped, String expectedResult) { # which needs a surrogate pair to be represented in UTF-16 \uD83C\uDF09 , \uD83C\uDF09 """) - void escape(String unescaped, String expectedResult) { + void escapeStd(String unescaped, String expectedResult) { StringBuilder stringBuilder = new StringBuilder(); - NixStringUtil.escape(stringBuilder, unescaped); + NixStringUtil.escapeStd(stringBuilder, unescaped); + assertEquals(expectedResult, stringBuilder.toString()); + } + + @ParameterizedTest(name = "[{index}] {0} -> {1}") + @CsvSource(quoteCharacter = '|', textBlock = """ + # Indent non-empty lines + || , 4, false, 2, || + || , 4, true , 2, || + |a| , 4, false, 2, |a| + |a| , 4, true , 2, | a| + |\n\n| , 4, false, 2, |\n| + |\n\n| , 4, true , 2, |\n| + | \n \n| , 4, false, 2, | \n \n | + | \n \n | , 4, false, 2, | \n \n | + | \n \n| , 4, true , 2, | \n \n | + | \n \n | , 4, true , 2, | \n \n | + # Should be be escaped + |''| , 2, false, 0, |'''| + |'''| , 2, false, 0, |''''| + |''''| , 2, false, 0, |''''''| + |${| , 2, false, 0, |''${| + |\r| , 2, false, 0, |''\\r| + |\t| , 2, false, 0, |''\\t| + # Should not be escaped + |\\| , 2, false, 0, |\\| + |\\x| , 2, false, 0, |\\x| + |$${| , 2, false, 0, |$${| + |\nx| , 2, false, 0, |\n x| + # supplementary character, i.e. character form a supplementary plane, + # which needs a surrogate pair to be represented in UTF-16 + |\uD83C\uDF09| , 2, false, 0, |\uD83C\uDF09| + """) + void escapeInd(String unescaped, int indent, boolean indentStart, int indentEnd, String expectedResult) { + StringBuilder stringBuilder = new StringBuilder(); + NixStringUtil.escapeInd(stringBuilder, unescaped, indent, indentStart, indentEnd); assertEquals(expectedResult, stringBuilder.toString()); } @@ -140,6 +177,7 @@ void detectMaxIndent(String code, int expectedResult, Project project) { "\uD83C\uDF09" , \uD83C\uDF09 ''\uD83C\uDF09'', \uD83C\uDF09 # Remove common indentation in indented strings + |'' ''| , || |'' a ''| , |a | |'' a ''| , |a | |'' a\n b\n''| , |a\nb\n|