diff --git a/src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java b/src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java index 805139eb..54e14e3a 100644 --- a/src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java +++ b/src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java @@ -71,13 +71,15 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase { entry(NixTypes.URI, NixTextAttributes.URI), // String literals entry(NixTypes.STR, NixTextAttributes.STRING), + entry(NixTypes.STR_ESCAPE, NixTextAttributes.STRING_ESCAPE), entry(NixTypes.STRING_CLOSE, NixTextAttributes.STRING), entry(NixTypes.STRING_OPEN, NixTextAttributes.STRING), entry(NixTypes.IND_STR, NixTextAttributes.STRING), + entry(NixTypes.IND_STR_LF, NixTextAttributes.STRING), + entry(NixTypes.IND_STR_INDENT, NixTextAttributes.STRING), + entry(NixTypes.IND_STR_ESCAPE, NixTextAttributes.STRING_ESCAPE), entry(NixTypes.IND_STRING_CLOSE, NixTextAttributes.STRING), entry(NixTypes.IND_STRING_OPEN, NixTextAttributes.STRING), - entry(NixTypes.STR_ESCAPE, NixTextAttributes.STRING_ESCAPE), - entry(NixTypes.IND_STR_ESCAPE, NixTextAttributes.STRING_ESCAPE), // Other entry(NixTypes.SCOMMENT, NixTextAttributes.LINE_COMMENT), entry(NixTypes.MCOMMENT, NixTextAttributes.BLOCK_COMMENT), diff --git a/src/main/java/org/nixos/idea/psi/NixElementFactory.java b/src/main/java/org/nixos/idea/psi/NixElementFactory.java index 6ff85992..1614d7d6 100644 --- a/src/main/java/org/nixos/idea/psi/NixElementFactory.java +++ b/src/main/java/org/nixos/idea/psi/NixElementFactory.java @@ -20,6 +20,14 @@ private NixElementFactory() {} // Cannot be instantiated return createElement(project, NixString.class, "", code, ""); } + public static @NotNull NixStringText createStdStringText(@NotNull Project project, @NotNull String code) { + return createElement(project, NixStringText.class, "\"", code, "\""); + } + + public static @NotNull NixStringText createIndStringText(@NotNull Project project, @NotNull String code) { + return createElement(project, NixStringText.class, "''\n", code, "''"); + } + public static @NotNull NixAttr createAttr(@NotNull Project project, @NotNull String code) { return createElement(project, NixAttr.class, "x.", code, ""); } diff --git a/src/main/java/org/nixos/idea/psi/NixInjectionPerformer.java b/src/main/java/org/nixos/idea/psi/NixInjectionPerformer.java new file mode 100644 index 00000000..a04ccd13 --- /dev/null +++ b/src/main/java/org/nixos/idea/psi/NixInjectionPerformer.java @@ -0,0 +1,92 @@ +package org.nixos.idea.psi; + +import com.intellij.lang.Language; +import com.intellij.lang.injection.MultiHostRegistrar; +import com.intellij.lang.injection.general.Injection; +import com.intellij.lang.injection.general.LanguageInjectionPerformer; +import com.intellij.openapi.fileTypes.LanguageFileType; +import com.intellij.psi.LiteralTextEscaper; +import com.intellij.psi.PsiElement; +import com.intellij.psi.util.PsiTreeUtil; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +public class NixInjectionPerformer implements LanguageInjectionPerformer { + + @Override + public boolean isPrimary() { + return true; + } + + @Override + public boolean performInjection(@NotNull MultiHostRegistrar registrar, @NotNull Injection injection, @NotNull PsiElement context) { + + // TODO Support injection on quotes? + // It is currently not possible to press Alt+Enter on the quotes to enable language injections. + // TODO Add InjectedLanguageManager.FRANKENSTEIN_INJECTION when there are interpolations? + // This seems to be used by various languages to disable certain error checks as when part of the source code is unavailable. + // TODO JetBrains implementations of LanguageInjectionPerformer do more stuff, + // like calling InjectorUtils.registerSupport(...). Not sure whether this is relevant. + // TODO Adding new interpolations after enabling a language injection has strange effect. + // TODO What about LanguageInjectionSupport? https://plugins.jetbrains.com/docs/intellij/language-injection.html + + NixString string = PsiTreeUtil.getParentOfType(context, NixString.class, false); + if (string == null) { + return false; + } + Language injectedLanguage = injection.getInjectedLanguage(); + if (injectedLanguage == null) { + return false; + } + + LanguageFileType injectedFileType = injectedLanguage.getAssociatedFileType(); + String injectedFileExtension = injectedFileType == null ? null : injectedFileType.getDefaultExtension(); + + registrar.startInjecting(injectedLanguage, injectedFileExtension); + for (Place place : collectPlaces(string, injection.getPrefix(), injection.getSuffix())) { + LiteralTextEscaper escaper = place.item().createLiteralTextEscaper(); + registrar.addPlace( + place.prefix().toString(), place.suffix().toString(), + place.item(), escaper.getRelevantTextRange() + ); + } + registrar.doneInjecting(); + return true; + } + + private record Place(@NotNull NixStringText item, @NotNull CharSequence prefix, @NotNull CharSequence suffix) {} + + private static List collectPlaces(@NotNull NixString string, @NotNull String prefix, @NotNull String suffix) { + int interpolations = 0; + StringBuilder prevSuffix = null; + List result = new ArrayList<>(); + for (NixStringPart stringPart : string.getStringParts()) { + // Note: The first and last part is always of type NixStringText, the list is never empty + if (stringPart instanceof NixStringText text) { + prevSuffix = new StringBuilder(); + result.add(new Place(text, prefix, prevSuffix)); + prefix = ""; + } else { + assert stringPart instanceof NixAntiquotation : stringPart.getClass(); + assert prevSuffix != null; + prevSuffix.append(interpolationPlaceholder(interpolations++)); + } + } + assert prevSuffix != null; + prevSuffix.append(suffix); + return result; + } + + /** + * Arbitrary code used in the guest language in place for string interpolations. + * This code is not visible for the user in the IDE as IDEA will visualize source code of the interpolations instead. + * This method returns a different string for each interpolation to prevent tooling of the guess language + * from assuming that all interpolations generate the same result. + */ + private static String interpolationPlaceholder(int index) { + // TODO What should I return here? Would an empty string work? What do other plugins use as placeholders? + return "(__interpolation" + index + "__)"; + } +} diff --git a/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt new file mode 100644 index 00000000..9c334fc3 --- /dev/null +++ b/src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt @@ -0,0 +1,68 @@ +package org.nixos.idea.psi + +import com.intellij.openapi.util.TextRange +import com.intellij.psi.LiteralTextEscaper +import org.nixos.idea.psi.impl.AbstractNixString +import org.nixos.idea.util.NixStringUtil + +class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper(host) { + + override fun isOneLine(): Boolean = false + + private var outSourceOffsets: IntArray? = null + + override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean { + val maxIndent = NixStringUtil.detectMaxIndent(myHost.parent as NixString) + val subText: String = rangeInsideHost.substring(myHost.text) + val outOffset = outChars.length + val array = IntArray(subText.length + 1) + var success = true + + fun addText(text: CharSequence, offset: Int): Boolean { + for (i in text.indices) { + if (offset + i >= rangeInsideHost.startOffset) { + array[outChars.length - outOffset] = offset + i + outChars.append(text[i]) + } else if (offset + i >= rangeInsideHost.endOffset) { + return false + } + } + return true + } + + NixStringUtil.visit(object : NixStringUtil.StringVisitor { + override fun text(text: CharSequence, offset: Int): Boolean { + return addText(text, offset) + } + + override fun escapeSequence(text: String, offset: Int, escapeSequence: CharSequence): Boolean { + val end = offset + escapeSequence.length + return if (offset < rangeInsideHost.startOffset || end > rangeInsideHost.endOffset) { + success = false + false + } else { + for (i in escapeSequence.indices) { + array[outChars.length - outOffset + i] = offset + } + outChars.append(text) + true + } + } + }, myHost, maxIndent) + // TODO Fix ArrayIndexOutOfBoundsException in the following line. (Not sure how to reproduce it.) + array[outChars.length - outOffset] = rangeInsideHost.endOffset + for (i in (outChars.length - outOffset + 1)..() { + + /** + * This function's result changes the original text in the host language + * when the fragment in the guest language changes + */ + override fun handleContentChange( + element: NixStringText, + range: TextRange, + newContent: String + ): NixStringText { + // TODO This implementation is wrong as escaped and non-escaped strings a mixed together. + // The variable `replacement` is not supposed to be escaped, + // but `element.text` is from the Nix source code, and therefore escaped. + // We probably have to switch the call dependency and let `updateText` call this more generic method. + val escaped = newContent + val replacement = range.replace(element.text, escaped) + return element.updateText(replacement) as NixStringText + } + + override fun getRangeInElement(element: NixStringText): TextRange = when { + element.textLength == 0 -> TextRange.EMPTY_RANGE + else -> TextRange.from(0, element.textLength) + } +} diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java b/src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java index bc33025c..6bf3191e 100644 --- a/src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java @@ -5,7 +5,7 @@ import org.jetbrains.annotations.NotNull; import org.nixos.idea.psi.NixPsiElement; -abstract class AbstractNixPsiElement extends ASTWrapperPsiElement implements NixPsiElement { +abstract public class AbstractNixPsiElement extends ASTWrapperPsiElement implements NixPsiElement { AbstractNixPsiElement(@NotNull ASTNode node) { super(node); diff --git a/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt new file mode 100644 index 00000000..bd71556a --- /dev/null +++ b/src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt @@ -0,0 +1,62 @@ +package org.nixos.idea.psi.impl + +import com.intellij.lang.ASTNode +import com.intellij.psi.PsiLanguageInjectionHost +import org.nixos.idea.psi.NixElementFactory +import org.nixos.idea.psi.NixIndString +import org.nixos.idea.psi.NixStdString +import org.nixos.idea.psi.NixStringLiteralEscaper +import org.nixos.idea.psi.NixStringText +import org.nixos.idea.util.NixStringUtil + + +abstract class AbstractNixString(private val astNode: ASTNode) : PsiLanguageInjectionHost, + AbstractNixPsiElement(astNode), NixStringText { + + override fun isValidHost() = true + + override fun updateText(s: String): NixStringText { + val project = project + val replacement = when (val string = parent) { + is NixStdString -> { + val escaped = buildString { NixStringUtil.escapeStd(this, s) } + NixElementFactory.createStdStringText(project, escaped) + } + + is NixIndString -> { + val indent = indentForNewText(string) + val indentStart = prevSibling == null + val indentEnd = if (nextSibling == null) trailingIndent(text) ?: baseIndent(string) else indent + val escaped = buildString { NixStringUtil.escapeInd(this, s, indent, indentStart, indentEnd) } + NixElementFactory.createIndStringText(project, escaped) + } + + else -> throw IllegalStateException("Unexpected parent: " + parent.javaClass) + } + return replace(replacement) as NixStringText + } + + override fun createLiteralTextEscaper() = NixStringLiteralEscaper(this) + + companion object { + private fun indentForNewText(string: NixIndString): Int { + return NixStringUtil.detectMaxIndent(string).takeIf { it != Int.MAX_VALUE } ?: (baseIndent(string) + 2) + } + + private fun baseIndent(string: NixIndString): Int { + // TODO Detect indent of string + // This should be the indent of the line where the string starts. + return 0 + } + + private fun trailingIndent(str: String): Int? { + val lastLineFeed = str.lastIndexOf('\n') + val lastLine = if (lastLineFeed != -1) str.substring(lastLineFeed + 1) else null + return if (lastLine != null && lastLine.all { it == ' ' }) { + lastLine.length + } else { + null + } + } + } +} diff --git a/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt new file mode 100644 index 00000000..903c8a57 --- /dev/null +++ b/src/main/java/org/nixos/idea/util/NixIndStringUtil.kt @@ -0,0 +1,68 @@ +package org.nixos.idea.util + +object NixIndStringUtil { + /** + * Unescapes the given string for use in a double-quoted string expression in the Nix Expression Language. + * + * See [Nix docs](https://nix.dev/manual/nix/2.22/language/values.html#type-string) for the logic, which + * is non-trivial. + * + * For example, `'` can be used to escape `''`, which means `'''` does not contain + * a string terminator + * ``` + * $ nix eval --expr " '' ''' '' " + * "'' " + * ``` + * + * This function does not erase string interpolations, because + * they are hard to parse in a loop without a proper grammar. For example: + * ```nix + * '' ${someNixFunc "${foo "}}" }" } '' + * ``` + */ + @JvmStatic + fun unescape(chars: CharSequence): String = buildString { + for ((index, c) in chars.withIndex()) { + fun prevChar() = chars.getOrNull(index - 1) + fun prev2Chars(): String? { + val prev = prevChar() ?: return null + val prevPrev = chars.getOrNull(index - 2) ?: return null + return "${prevPrev}${prev}" + } + + fun prev3Chars(): String? { + val prev2 = prev2Chars() ?: return null + val prevPrev2 = chars.getOrNull(index - 3) ?: return null + return "${prevPrev2}${prev2}" + } + + when (c) { + // ''\ escapes any character, but we can only cover known ones in advance: + '\'' -> when { + // ''' is escaped to '' + prev2Chars() == "''" -> append("''") + // '' is the string delimiter + else -> continue + } + + '\\' -> when { + prev2Chars() == "''" -> continue + prevChar() == '\'' -> continue + else -> append(c) + } + + '$' -> if (prevChar() == '$') append(c) else continue + '{' -> if (prevChar() == '$') append("\${") else append(c) + + else -> if (prev3Chars() == "''\\") when (c) { + 'r' -> if (prev3Chars() == "''\\") append('\r') else append(c) + 'n' -> if (prev3Chars() == "''\\") append('\n') else append(c) + 't' -> if (prev3Chars() == "''\\") append('\t') else append(c) + else -> append("''\\").append(c) + } else { + append(c) + } + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/nixos/idea/util/NixStringUtil.java b/src/main/java/org/nixos/idea/util/NixStringUtil.java index 90b73518..ebc9d306 100644 --- a/src/main/java/org/nixos/idea/util/NixStringUtil.java +++ b/src/main/java/org/nixos/idea/util/NixStringUtil.java @@ -4,11 +4,16 @@ import com.intellij.psi.tree.IElementType; import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; +import org.nixos.idea.psi.NixAntiquotation; +import org.nixos.idea.psi.NixIndString; +import org.nixos.idea.psi.NixStdString; +import org.nixos.idea.psi.NixString; +import org.nixos.idea.psi.NixStringPart; import org.nixos.idea.psi.NixStringText; import org.nixos.idea.psi.NixTypes; /** - * Utilities for strings in the Nix Expression Language. + * Utilities for encoding and decoding strings in the Nix Expression Language. */ public final class NixStringUtil { @@ -31,7 +36,7 @@ private NixStringUtil() {} // Cannot be instantiated public static @NotNull String quote(@NotNull CharSequence unescaped) { StringBuilder builder = new StringBuilder(); builder.append('"'); - escape(builder, unescaped); + escapeStd(builder, unescaped); builder.append('"'); return builder.toString(); } @@ -42,19 +47,20 @@ private NixStringUtil() {} // Cannot be instantiated * For example, the following code would generate a broken result. * 
{@code
      *     StringBuilder b1 = new StringBuilder(), b2 = new StringBuilder();
-     *     NixStringUtil.escape(b1, "$");
-     *     NixStringUtil.escape(b2, "{''}");
+     *     NixStringUtil.escapeStd(b1, "$");
+     *     NixStringUtil.escapeStd(b2, "{''}");
      *     System.out.println(b1.toString() + b2.toString());
      * }
* The result would be the following broken Nix code. * 
-     *     "${''}"
+     *     ${''}
      *
* * @param builder The target string builder. The result will be appended to the given string builder. * @param unescaped The raw string which shall be escaped. */ - public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence unescaped) { + public static void escapeStd(@NotNull StringBuilder builder, @NotNull CharSequence unescaped) { + boolean potentialInterpolation = false; for (int charIndex = 0; charIndex < unescaped.length(); charIndex++) { char nextChar = unescaped.charAt(charIndex); switch (nextChar) { @@ -63,7 +69,7 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence builder.append('\\').append(nextChar); break; case '{': - if (builder.charAt(builder.length() - 1) == '$') { + if (potentialInterpolation) { builder.setCharAt(builder.length() - 1, '\\'); builder.append('$').append('{'); } else { @@ -83,6 +89,102 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence builder.append(nextChar); break; } + potentialInterpolation = nextChar == '$' && !potentialInterpolation; + } + } + + /** + * Escapes the given string for use in an indented string expression in the Nix Expression Language. + * Note that it is not safe to concat the result of two calls of this method. + * + * @param builder The target string builder. The result will be appended to the given string builder. + * @param unescaped The raw string which shall be escaped. + * @param indent The number as spaces used for indentation + * @param indentStart Whether the start of the string needs to be indented + * @param indentEnd The number as spaces used for indentation in the last line + */ + public static void escapeInd(@NotNull StringBuilder builder, @NotNull CharSequence unescaped, int indent, boolean indentStart, int indentEnd) { + String indentStr = " ".repeat(indent); + boolean potentialInterpolation = false; + boolean potentialClosing = false; + for (int charIndex = 0; charIndex < unescaped.length(); charIndex++) { + char nextChar = unescaped.charAt(charIndex); + if (indentStart && nextChar != '\n') { + builder.append(indentStr); + indentStart = false; + } + switch (nextChar) { + case '\'': + // Convert `''` to `'''` + if (potentialClosing) { + builder.append('\''); + } + builder.append('\''); + break; + case '{': + // Convert `${` to `''${`, but leave `$${` untouched + if (potentialInterpolation) { + builder.setLength(builder.length() - 1); + builder.append("''${"); + } else { + builder.append('{'); + } + break; + case '\r': + builder.append("''\\r"); + break; + case '\t': + builder.append("''\\t"); + break; + case '\n': + indentStart = true; + // fallthrough + default: + builder.append(nextChar); + break; + } + potentialInterpolation = nextChar == '$' && !potentialInterpolation; + potentialClosing = nextChar == '\'' && !potentialClosing; + } + if (indentStart) { + builder.append(" ".repeat(Math.min(indent, indentEnd))); + } + } + + /** + * Detects the maximal amount of characters removed from the start of the lines. + * May return {@link Integer#MAX_VALUE} if the content of the string is blank. + * + * @param string the string from which to get the indentation + * @return the detected indentation, or {@link Integer#MAX_VALUE} + */ + public static int detectMaxIndent(@NotNull NixString string) { + if (string instanceof NixStdString) { + return 0; + } else if (string instanceof NixIndString) { + int result = Integer.MAX_VALUE; + int preliminary = 0; + for (NixStringPart part : string.getStringParts()) { + if (part instanceof NixStringText textNode) { + for (ASTNode token = textNode.getNode().getFirstChildNode(); token != null; token = token.getTreeNext()) { + IElementType type = token.getElementType(); + if (type == NixTypes.IND_STR_INDENT) { + preliminary = Math.min(result, token.getTextLength()); + } else if (type == NixTypes.IND_STR_LF) { + preliminary = 0; + } else { + assert type == NixTypes.IND_STR || type == NixTypes.IND_STR_ESCAPE : type; + result = preliminary; + } + } + } else { + assert part instanceof NixAntiquotation : part.getClass(); + result = preliminary; + } + } + return result; + } else { + throw new IllegalStateException("Unexpected subclass of NixString: " + string.getClass()); } } @@ -94,38 +196,84 @@ public static void escape(@NotNull StringBuilder builder, @NotNull CharSequence * @return The resulting string after resolving all escape sequences. */ public static @NotNull String parse(@NotNull NixStringText textNode) { + int maxIndent = detectMaxIndent((NixString) textNode.getParent()); StringBuilder builder = new StringBuilder(); + visit(new StringVisitor() { + @Override + public boolean text(@NotNull CharSequence text, int offset) { + builder.append(text); + return true; + } + + @Override + public boolean escapeSequence(@NotNull String text, int offset, @NotNull CharSequence escapeSequence) { + builder.append(text); + return true; + } + }, textNode, maxIndent); + return builder.toString(); + } + + public static void visit(@NotNull StringVisitor visitor, @NotNull NixStringText textNode, int maxIndent) { + int offset = 0; for (ASTNode child = textNode.getNode().getFirstChildNode(); child != null; child = child.getTreeNext()) { - parse(builder, child); + if (!parse(visitor, child, offset, maxIndent)) { + break; + } + offset += child.getTextLength(); } - return builder.toString(); } - private static void parse(@NotNull StringBuilder builder, @NotNull ASTNode token) { + private static boolean parse(@NotNull StringVisitor visitor, @NotNull ASTNode token, int offset, int maxIndent) { CharSequence text = token.getChars(); IElementType type = token.getElementType(); - if (type == NixTypes.STR || type == NixTypes.IND_STR) { - builder.append(text); + if (type == NixTypes.STR || type == NixTypes.IND_STR || type == NixTypes.IND_STR_LF) { + return visitor.text(text, offset); + } else if (type == NixTypes.IND_STR_INDENT) { + int end = text.length(); + if (end > maxIndent) { + CharSequence remain = text.subSequence(maxIndent, end); + return visitor.text(remain, offset + maxIndent); + } + return true; } else if (type == NixTypes.STR_ESCAPE) { assert text.length() == 2 && text.charAt(0) == '\\' : text; char c = text.charAt(1); - builder.append(unescape(c)); + return visitor.escapeSequence(unescape(c), offset, text); } else if (type == NixTypes.IND_STR_ESCAPE) { - assert text.length() == 3 && ("''$".contentEquals(text) || "'''".contentEquals(text)) || - text.length() == 4 && "''\\".contentEquals(text.subSequence(0, 3)) : text; - char c = text.charAt(text.length() - 1); - builder.append(unescape(c)); + return switch (text.charAt(2)) { + case '$' -> { + assert "''$".contentEquals(text) : text; + yield visitor.escapeSequence("$", offset, text); + } + case '\'' -> { + assert "'''".contentEquals(text) : text; + yield visitor.escapeSequence("''", offset, text); + } + case '\\' -> { + assert text.length() == 4 && "''\\".contentEquals(text.subSequence(0, 3)) : text; + char c = text.charAt(3); + yield visitor.escapeSequence(unescape(c), offset, text); + } + default -> throw new IllegalStateException("Unknown escape sequence: " + text); + }; } else { throw new IllegalStateException("Unexpected token in string: " + token); } } - private static char unescape(char c) { + private static @NotNull String unescape(char c) { return switch (c) { - case 'n' -> '\n'; - case 'r' -> '\r'; - case 't' -> '\t'; - default -> c; + case 'n' -> "\n"; + case 'r' -> "\r"; + case 't' -> "\t"; + default -> String.valueOf(c); }; } + + public interface StringVisitor { + boolean text(@NotNull CharSequence text, int offset); + + boolean escapeSequence(@NotNull String text, int offset, @NotNull CharSequence escapeSequence); + } } diff --git a/src/main/lang/Nix.bnf b/src/main/lang/Nix.bnf index 5aba71c7..4a965407 100644 --- a/src/main/lang/Nix.bnf +++ b/src/main/lang/Nix.bnf @@ -24,6 +24,10 @@ // no one really needs to know that + - * / are expected at any offset consumeTokenMethod("expr_op.*")="consumeTokenFast" + // make IndStrings language injection Hosts + implements("string_text")="com.intellij.psi.PsiLanguageInjectionHost" + mixin("string_text")="org.nixos.idea.psi.impl.AbstractNixString" + tokens = [ // This list does not contain all tokens. This list only defines the debug // names for tokens which have a distinct text representation. Other tokens @@ -198,16 +202,16 @@ private set_recover ::= curly_recover !bind private list_recover ::= brac_recover !expr_select ;{ extends(".*_string")="string" } -std_string ::= STRING_OPEN string_part* STRING_CLOSE { pin=1 } -ind_string ::= IND_STRING_OPEN string_part* IND_STRING_CLOSE { pin=1 } +std_string ::= STRING_OPEN string_parts STRING_CLOSE { pin=1 } +ind_string ::= IND_STRING_OPEN string_parts IND_STRING_CLOSE { pin=1 } fake string ::= string_part* { methods=[ string_parts="string_part" ] } +private string_parts ::= string_text ( antiquotation string_text )* ;{ extends("string_text|antiquotation")=string_part } -string_part ::= string_text | antiquotation { recoverWhile=string_part_recover } -string_text ::= string_token+ +fake string_part ::= +string_text ::= string_token* antiquotation ::= DOLLAR LCURLY expr recover_antiquotation RCURLY { pin=1 } private recover_antiquotation ::= { recoverWhile=curly_recover } -private string_part_recover ::= !(DOLLAR | STRING_CLOSE | IND_STRING_CLOSE | string_token) -private string_token ::= STR | IND_STR | STR_ESCAPE | IND_STR_ESCAPE +private string_token ::= STR | STR_ESCAPE | IND_STR | IND_STR_INDENT | IND_STR_ESCAPE | IND_STR_LF ;{ extends("bind_attr|bind_inherit")=bind } bind ::= bind_attr | bind_inherit diff --git a/src/main/lang/Nix.flex b/src/main/lang/Nix.flex index af5107e6..517e2c3a 100644 --- a/src/main/lang/Nix.flex +++ b/src/main/lang/Nix.flex @@ -13,26 +13,27 @@ import static org.nixos.idea.psi.NixTypes.*; private final AbstractIntList states = new IntArrayList(); private void pushState(int newState) { - if (newState == YYINITIAL){ - throw new IllegalStateException("Pusing YYINITIAL is not supported"); - } + assert newState != YYINITIAL : "Pusing YYINITIAL is not supported"; // store current state on the stack to allow restoring it in popState(...) states.push(yystate()); yybegin(newState); } private void popState(int expectedState) { - if (states.isEmpty()){ - throw new IllegalStateException("Popping an empty stack of states. Expected: " + expectedState); - } + assert !states.isEmpty() : "Popping an empty stack of states. Expected: " + expectedState; // safe-guard, because we always know which state we're currently in in the rules below - if (yystate() != expectedState) { - throw new IllegalStateException(String.format("Unexpected state. Current: %d, expected: %d", yystate(), expectedState)); - } + assert yystate() == expectedState : String.format("Unexpected state. Current: %d, expected: %d", yystate(), expectedState); // start the lexer with the previous state, which was stored by pushState(...) yybegin(states.popInt()); } + private void replaceState(int expectedState, int newState) { + assert newState != YYINITIAL : "Pusing YYINITIAL is not supported"; + // safe-guard, because we always know which state we're currently in in the rules below + assert yystate() == expectedState : String.format("Unexpected state. Current: %d, expected: %d", yystate(), expectedState); + yybegin(newState); + } + protected void onReset() { states.clear(); } @@ -44,7 +45,9 @@ import static org.nixos.idea.psi.NixTypes.*; %function advance %type IElementType %unicode -%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION PATH +%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION +%xstate IND_STRING_START IND_STRING_INDENT PATH +%suppress empty-match ANY=[^] ID=[a-zA-Z_][a-zA-Z0-9_'-]* @@ -71,8 +74,19 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ \" { popState(STRING); return STRING_CLOSE; } } + { + // The first line is ignored in case it is empty + [\ ]*\n { replaceState(IND_STRING_START, IND_STRING_INDENT); return com.intellij.psi.TokenType.WHITE_SPACE; } +} + + { + [\ ]+ { replaceState(yystate(), IND_STRING); return IND_STR_INDENT; } + "" { replaceState(yystate(), IND_STRING); } +} + { - [^\$\']+ { return IND_STR; } + \n { replaceState(IND_STRING, IND_STRING_INDENT); return IND_STR_LF; } + [^\$\'\n]+ { return IND_STR; } "$"|"$$"|"'" { return IND_STR; } "''$"|"'''" { return IND_STR_ESCAPE; } "''"\\{ANY} { return IND_STR_ESCAPE; } @@ -83,7 +97,7 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ { // '$' and '{' must be two separate tokens to make NixBraceMatcher work // correctly with Grammar-Kit. - "{" { popState(ANTIQUOTATION_START); pushState(ANTIQUOTATION); return LCURLY; } + "{" { replaceState(ANTIQUOTATION_START, ANTIQUOTATION); return LCURLY; } } { @@ -98,10 +112,10 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ "$"/"{" { pushState(ANTIQUOTATION_START); return DOLLAR; } {PATH_SEG} { return PATH_SEGMENT; } {PATH_CHAR}+ { return PATH_SEGMENT; } - // anything else, e.g. whitespace, stops lexing of a PATH + // anything else, e.g. a whitespace, stops lexing of a PATH // we're delegating back to the parent state // PATH_END is an empty-length token to signal the end of the path - [^] { popState(PATH); yypushback(yylength()); return PATH_END; } + "" { popState(PATH); return PATH_END; } } { @@ -152,7 +166,7 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ "->" { return IMPL; } \" { pushState(STRING); return STRING_OPEN; } - \'\' { pushState(IND_STRING); return IND_STRING_OPEN; } + \'\' { pushState(IND_STRING_START); return IND_STRING_OPEN; } // Note that `true`, `false` and `null` are built-in variables but not // keywords. Therefore, they are not listed here. @@ -171,5 +185,5 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/ {WHITE_SPACE} { return com.intellij.psi.TokenType.WHITE_SPACE; } } -// matched by all %state states +// matched by inclusive states (%state), but not by exclusive states (%xstate) [^] { return com.intellij.psi.TokenType.BAD_CHARACTER; } diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml index 159b1e7f..8f49cbf5 100644 --- a/src/main/resources/META-INF/plugin.xml +++ b/src/main/resources/META-INF/plugin.xml @@ -59,6 +59,14 @@ + + + + diff --git a/src/test/java/org/nixos/idea/psi/NixElementFactoryTest.java b/src/test/java/org/nixos/idea/psi/NixElementFactoryTest.java index 47347287..048b08d5 100644 --- a/src/test/java/org/nixos/idea/psi/NixElementFactoryTest.java +++ b/src/test/java/org/nixos/idea/psi/NixElementFactoryTest.java @@ -33,6 +33,34 @@ void createStringFail(String code) { () -> NixElementFactory.createString(myProject, code)); } + @ParameterizedTest + @ValueSource(strings = {"", "x", "abc", " ", "\\n", "\\${x}", "$\\{x}"}) + void createStdStringText(String code) { + NixStringText result = NixElementFactory.createStdStringText(myProject, code); + assertEquals(code, result.getText()); + } + + @ParameterizedTest + @ValueSource(strings = {"\"", "\"x\"", "${x}", "\\${\"42\"}"}) + void createStdStringTextFail(String code) { + assertThrows(RuntimeException.class, + () -> NixElementFactory.createStdStringText(myProject, code)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "x", "abc", " ", "\n", "''\\n", "''${x}", "$${x}"}) + void createIndStringText(String code) { + NixStringText result = NixElementFactory.createIndStringText(myProject, code); + assertEquals(code, result.getText()); + } + + @ParameterizedTest + @ValueSource(strings = {"''", "''x''", "${x}", "$${''42''}"}) + void createIndStringTextFail(String code) { + assertThrows(RuntimeException.class, + () -> NixElementFactory.createIndStringText(myProject, code)); + } + @ParameterizedTest @ValueSource(strings = {"x", "\"x\"", "${x}", "${\"x\"}", "\"x${y}z\""}) void createAttr(String code) { diff --git a/src/test/java/org/nixos/idea/psi/NixInjectionPerformerTest.java b/src/test/java/org/nixos/idea/psi/NixInjectionPerformerTest.java new file mode 100644 index 00000000..6c8ea26b --- /dev/null +++ b/src/test/java/org/nixos/idea/psi/NixInjectionPerformerTest.java @@ -0,0 +1,166 @@ +package org.nixos.idea.psi; + +import com.intellij.lang.injection.InjectedLanguageManager; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.testFramework.EdtTestUtil; +import com.intellij.testFramework.fixtures.CodeInsightTestFixture; +import com.intellij.testFramework.fixtures.IdeaProjectTestFixture; +import com.intellij.testFramework.fixtures.IdeaTestExecutionPolicy; +import com.intellij.testFramework.fixtures.IdeaTestFixtureFactory; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Test; + +import java.util.Objects; + +final class NixInjectionPerformerTest { + + @Test + void trivial_ind_string() throws Exception { + doTest( + "''first ''${} line''", + "first ${} line" + ); + } + + @Test + void trivial_std_string() throws Exception { + doTest( + "\"first \\${} line\"", + "first ${} line" + ); + } + + @Test + void trim_initial_line() throws Exception { + doTest( + "''\nfirst line''", + "first line" + ); + } + + @Test + void trim_indentation() throws Exception { + doTest( + """ + '' + first line + + second line + \s + third line + '' + """, + """ + first line + + second line + \s + third line + """ + ); + } + + @Test + void keep_indentation_of_closing_quotes() throws Exception { + doTest( + """ + '' + first line + '' + """, + """ + first line + """ + ); + } + + @Test + void insert_into_empty_line() throws Exception { + // TODO remove spaces from empty line in source code. + // This test is supposed to test a scenario where the edited line misses the indent in the host language + doTest( + """ + '' + first line + + second line + '' + """, + """ + first line + + second line + """ + ); + } + + @Test + void insert_into_string() throws Exception { + // TODO remove spaces from empty line in source code. + // This test is supposed to test a scenario where the edited line misses the indent in the host language + doTest( + """ + '' + + '' + """, + """ + first line + + second line + """ + ); + } + + @Test + void insert_before_interpolation() throws Exception { + doTest( + "''${x}''", + "" + ); + } + + @Test + void insert_after_interpolation() throws Exception { + doTest( + "''${x}''", + "" + ); + } + + @Test + void insert_between_interpolations() throws Exception { + doTest( + "''${x}${y}''", + "" + ); + } + + private void doTest(@NotNull String sourceCode, @NotNull String injected) throws Exception { + IdeaTestFixtureFactory factory = IdeaTestFixtureFactory.getFixtureFactory(); + IdeaTestExecutionPolicy policy = Objects.requireNonNull(IdeaTestExecutionPolicy.current()); + IdeaProjectTestFixture baseFixture = factory.createLightFixtureBuilder("test").getFixture(); + CodeInsightTestFixture fixture = factory.createCodeInsightFixture(baseFixture, policy.createTempDirTestFixture()); + fixture.setTestDataPath(FileUtil.toSystemIndependentName(policy.getHomePath())); + fixture.setUp(); + try { + // TODO Implement test. This test is supposed to + // 1. Enable a language injection. + // 2. Open the injected language in the editor. + // 3. Insert text at in the editor of the injected language + // 4. Remove text at again + // 5. Insert another text at which needs to be escaped + // 6. Verify that during step 3 to 5, the text in the editor of the host language is adjusted appropriately + // How to enable language injection using code? + // There doesn't seem to be an action I could use via fixture.performEditorAction(IdeActions...). + // Maybe I can use TemporaryPlacesRegistry.addHostWithUndo directly, but it is not available in the classpath right now. + InjectedLanguageManager instance = InjectedLanguageManager.getInstance(fixture.getProject()); + // instance.getNonEditableFragments(); This seems like it could be used to detect the interpolations + int pos = sourceCode.indexOf(""); + fixture.setCaresAboutInjection(true); // This doesn't do anything as true is the default, but I left it here for now. + fixture.configureByText("test.nix", sourceCode.replace("", "")); + } finally { + EdtTestUtil.runInEdtAndWait(fixture::tearDown); + } + } +} diff --git a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java index 0ff66c98..d936b1da 100644 --- a/src/test/java/org/nixos/idea/util/NixStringUtilTest.java +++ b/src/test/java/org/nixos/idea/util/NixStringUtilTest.java @@ -13,6 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +@SuppressWarnings("UnnecessaryStringEscape") final class NixStringUtilTest { @ParameterizedTest(name = "[{index}] {0} -> {1}") @CsvSource(textBlock = """ @@ -22,6 +23,7 @@ final class NixStringUtilTest { \\ , "\\\\" \\x , "\\\\x" a${b}c , "a\\${b}c" + a$${b}c , "a$${b}c" '\n' , "\\n" '\r' , "\\r" '\t' , "\\t" @@ -41,6 +43,7 @@ void quote(String unescaped, String expectedResult) { \\ , \\\\ \\x , \\\\x a${b}c , a\\${b}c + a$${b}c , a$${b}c '\n' , \\n '\r' , \\r '\t' , \\t @@ -48,12 +51,92 @@ void quote(String unescaped, String expectedResult) { # which needs a surrogate pair to be represented in UTF-16 \uD83C\uDF09 , \uD83C\uDF09 """) - void escape(String unescaped, String expectedResult) { + void escapeStd(String unescaped, String expectedResult) { StringBuilder stringBuilder = new StringBuilder(); - NixStringUtil.escape(stringBuilder, unescaped); + NixStringUtil.escapeStd(stringBuilder, unescaped); assertEquals(expectedResult, stringBuilder.toString()); } + @ParameterizedTest(name = "[{index}] {0} -> {1}") + @CsvSource(quoteCharacter = '|', textBlock = """ + # Indent non-empty lines + || , 4, false, 2, || + || , 4, true , 2, || + |a| , 4, false, 2, |a| + |a| , 4, true , 2, | a| + |\n\n| , 4, false, 2, |\n| + |\n\n| , 4, true , 2, |\n| + | \n \n| , 4, false, 2, | \n \n | + | \n \n | , 4, false, 2, | \n \n | + | \n \n| , 4, true , 2, | \n \n | + | \n \n | , 4, true , 2, | \n \n | + # Should be be escaped + |''| , 2, false, 0, |'''| + |'''| , 2, false, 0, |''''| + |''''| , 2, false, 0, |''''''| + |${| , 2, false, 0, |''${| + |\r| , 2, false, 0, |''\\r| + |\t| , 2, false, 0, |''\\t| + # Should not be escaped + |\\| , 2, false, 0, |\\| + |\\x| , 2, false, 0, |\\x| + |$${| , 2, false, 0, |$${| + |\nx| , 2, false, 0, |\n x| + # supplementary character, i.e. character form a supplementary plane, + # which needs a surrogate pair to be represented in UTF-16 + |\uD83C\uDF09| , 2, false, 0, |\uD83C\uDF09| + """) + void escapeInd(String unescaped, int indent, boolean indentStart, int indentEnd, String expectedResult) { + StringBuilder stringBuilder = new StringBuilder(); + NixStringUtil.escapeInd(stringBuilder, unescaped, indent, indentStart, indentEnd); + assertEquals(expectedResult, stringBuilder.toString()); + } + + @ParameterizedTest(name = "[{index}] {0} -> {1}") + @CsvSource(quoteCharacter = '|', textBlock = """ + # Non-indented strings always return the empty string + |""| , 0 + |" a"| , 0 + |" a\n b"| , 0 + # When there are only spaces, we return Integer.MAX_VALUE + |''''| , 2147483647 + |'' ''| , 2147483647 + |''\n \n ''| , 2147483647 + # The smallest indentation counts + |''\n a\n b''| , 1 + |''\n a\n b''| , 1 + |''\n a\n b''| , 2 + |''\n a\n ${b}''| , 1 + |''\n a\n ''\\b''| , 1 + # First line counts + |''a\n b''| , 0 + |''${a}\n b''| , 0 + |''''\\a\n b''| , 0 + # But only the first token in a line counts + |'' a${b}''| , 2 + |'' a''\\b''| , 2 + |'' ${a}b''| , 2 + |'' ${a}${b}''| , 2 + |'' ${a}''\\b''| , 2 + |'' ''\\ab''| , 2 + |'' ''\\a${b}''| , 2 + |'' ''\\a''\\b''| , 2 + # Tab and CR are treated as normal characters, not as spaces + # See NixOS/nix#2911 and NixOS/nix#3759 + |''\t''| , 0 + |''\n \t''| , 2 + |''\r\n''| , 0 + |''\n \r\n''| , 2 + # Indentation within interpolations is ignored + |'' ${\n"a"}''| , 2 + |'' ${\n''a''}''| , 2 + """) + @WithIdeaPlatform.OnEdt + void detectMaxIndent(String code, int expectedResult, Project project) { + NixString string = NixElementFactory.createString(project, code); + assertEquals(expectedResult, NixStringUtil.detectMaxIndent(string)); + } + @ParameterizedTest(name = "[{index}] {0} -> {1}") @CsvSource(quoteCharacter = '|', textBlock = """ "" , || @@ -62,12 +145,15 @@ void escape(String unescaped, String expectedResult) { "\\"" , " "\\\\" , \\ "\\\\x" , \\x + ''"'' , " ''\\"'' , \\" + ''\\x'' , \\x ''\\\\'' , \\\\ ''\\\\x'' , \\\\x ''''\\"'' , " ''''\\\\'' , \\ ''''\\\\x'' , \\x + ''''''' , |''| "''\\"" , ''" "a\\${b}c" , a${b}c ''a''${b}c'' , a${b}c @@ -77,10 +163,37 @@ void escape(String unescaped, String expectedResult) { |"\n"| , |\n| |"\r"| , |\r| |"\t"| , |\t| + |"\\n"| , |\n| + |"\\r"| , |\r| + |"\\t"| , |\t| + |''_\n''| , |_\n| + |''\r''| , |\r| + |''\t''| , |\t| + |''''\\n''| , |\n| + |''''\\r''| , |\r| + |''''\\t''| , |\t| # supplementary character, i.e. character form a supplementary plane, # which needs a surrogate pair to be represented in UTF-16 "\uD83C\uDF09" , \uD83C\uDF09 ''\uD83C\uDF09'', \uD83C\uDF09 + # Remove common indentation in indented strings + |'' ''| , || + |'' a ''| , |a | + |'' a ''| , |a | + |'' a\n b\n''| , |a\nb\n| + |'' a\n b\n''| , |a\n b\n| + # But don't remove indentation when there is one line without it + |'' a\nb\n c''| , | a\nb\n c| + |''a\n b\n c''| , |a\n b\n c| + |'' a\n\tb''|, | a\n\tb| + |''\ta\n b''|, |\ta\n b| + # Even when the line is blank + |'' a\n ''| , |a\n | + # Ignore indentation of empty lines + |'' a\n\n b\n''|, |a\n\nb\n| + # Remove initial line break in indented strings + |''\n a''| , |a| + |'' \n a''| , |a| """) @WithIdeaPlatform.OnEdt void parse(String code, String expectedResult, Project project) { diff --git a/src/test/testData/ParsingTest/StringWithMultipleLines.lexer.txt b/src/test/testData/ParsingTest/StringWithMultipleLines.lexer.txt index 010624ea..e41d77a6 100644 --- a/src/test/testData/ParsingTest/StringWithMultipleLines.lexer.txt +++ b/src/test/testData/ParsingTest/StringWithMultipleLines.lexer.txt @@ -5,7 +5,16 @@ STR ('\n first\n second\n third\n') STRING_CLOSE ('"') WHITE_SPACE ('\n') IND_STRING_OPEN ('''') -IND_STR ('\n first\n second\n third\n') +WHITE_SPACE ('\n') +IND_STR_INDENT (' ') +IND_STR ('first') +IND_STR_LF ('\n') +IND_STR_INDENT (' ') +IND_STR ('second') +IND_STR_LF ('\n') +IND_STR_INDENT (' ') +IND_STR ('third') +IND_STR_LF ('\n') IND_STRING_CLOSE ('''') WHITE_SPACE ('\n') ] (']') diff --git a/src/test/testData/ParsingTest/StringWithMultipleLines.txt b/src/test/testData/ParsingTest/StringWithMultipleLines.txt index 6f121793..84694972 100644 --- a/src/test/testData/ParsingTest/StringWithMultipleLines.txt +++ b/src/test/testData/ParsingTest/StringWithMultipleLines.txt @@ -10,8 +10,17 @@ Nix File(0,63) PsiWhiteSpace('\n')(30,31) NixIndStringImpl(IND_STRING)(31,61) PsiElement(IND_STRING_OPEN)('''')(31,33) - NixStringTextImpl(STRING_TEXT)(33,59) - PsiElement(IND_STR)('\n first\n second\n third\n')(33,59) + PsiWhiteSpace('\n')(33,34) + NixStringTextImpl(STRING_TEXT)(34,59) + PsiElement(IND_STR_INDENT)(' ')(34,36) + PsiElement(IND_STR)('first')(36,41) + PsiElement(IND_STR_LF)('\n')(41,42) + PsiElement(IND_STR_INDENT)(' ')(42,44) + PsiElement(IND_STR)('second')(44,50) + PsiElement(IND_STR_LF)('\n')(50,51) + PsiElement(IND_STR_INDENT)(' ')(51,53) + PsiElement(IND_STR)('third')(53,58) + PsiElement(IND_STR_LF)('\n')(58,59) PsiElement(IND_STRING_CLOSE)('''')(59,61) PsiWhiteSpace('\n')(61,62) PsiElement(])(']')(62,63)