Implement basic one-off language inections

src/main/java/org/nixos/idea/psi/NixStringLiteralEscaper.kt

@@ -0,0 +1,142 @@
+package org.nixos.idea.psi
+
+import com.intellij.openapi.util.TextRange
+import com.intellij.psi.LiteralTextEscaper
+import com.intellij.psi.PsiLanguageInjectionHost
+import org.intellij.lang.annotations.Language
+import org.nixos.idea.psi.impl.AbstractNixString
+
+class NixStringLiteralEscaper(host: AbstractNixString) : LiteralTextEscaper<PsiLanguageInjectionHost>(host) {
+
+    override fun isOneLine(): Boolean = false
+
+    private var outSourceOffsets: IntArray? = null
+
+    override fun getRelevantTextRange(): TextRange {
+        if (myHost.textLength <= 4) return TextRange.EMPTY_RANGE
+        return TextRange.create(2, myHost.textLength - 2)
+    }
+
+    override fun decode(rangeInsideHost: TextRange, outChars: StringBuilder): Boolean {
+        // TODO issue #81 only indented strings supported for now
+        // single line strings require a new decode function because
+        // it uses different escaping mechanisms
+        if (myHost !is NixIndString) return false
+
+        val subText: String = rangeInsideHost.substring(myHost.text)
+        val array = IntArray(subText.length + 1)
+        val success = unescapeAndDecode(subText, outChars, array)
+        outSourceOffsets = array
+        return success
+    }
+
+    override fun getOffsetInHost(offsetInDecoded: Int, rangeInsideHost: TextRange): Int {
+        val offsets = outSourceOffsets ?: throw IllegalStateException("#decode was not called")
+        val result = if (offsetInDecoded < offsets.size) offsets[offsetInDecoded] else -1
+        return result.coerceIn(0..rangeInsideHost.length) + rangeInsideHost.startOffset
+    }
+
+    companion object {
+        /**
+         * Does not consider interpolations so that
+         * they do appear in the guest language and remain when we end up converting back to Nix.
+         *
+         * @returns the minIndent of the string if successful, or null if unsuccessful.
+         */
+        fun unescapeAndDecode(chars: String, outChars: StringBuilder, sourceOffsets: IntArray?): Boolean {
+            assert(sourceOffsets == null || sourceOffsets.size == chars.length + 1)
+
+            var index = 0
+            val outOffset = outChars.length
+            var braces = 0
+            var indentSoFar = 0
+            val minIndent = chars.lines()
+                .filterNot { it.isEmpty() }
+                .minOfOrNull { it.takeWhile(Char::isWhitespace).count() } ?: 0
+
+
+            while (index < chars.length) {
+                fun updateOffsets(index: Int) {
+                    if (sourceOffsets != null) {
+                        sourceOffsets[outChars.length - outOffset] = index - 1
+                        sourceOffsets[outChars.length - outOffset + 1] = index
+                    }
+                }
+
+                var c = chars[index++]
+                updateOffsets(index)
+
+
+                if (braces > 0) {
+                    if (c == '{') braces++
+                    else if (c == '}') braces--
+                    outChars.append(c)
+                    continue
+                }
+
+                if (c == '\n' && index < chars.length - 1) {
+                    // we know that the next n chars are going to be whitespace indent
+                    index += minIndent
+                    outChars.append(c)
+                    if (sourceOffsets != null) {
+                        sourceOffsets[outChars.length - outOffset] = index
+                    }
+                    continue
+                }
+
+                if (c == '\'') {
+                    if (index == chars.length) return false
+                    c = chars[index++]
+
+                    if (c != '\'') {
+                        // if what follows isn't another ' then we are not escaping anything,
+                        // so we can backtrace and continue
+                        outChars.append("\'")
+                        index--
+                        continue
+                    }
+
+                    if (index == chars.length) return false
+                    c = chars[index++]
+
+                    when (c) {
+                        // '' can be escaped by prefixing it with ', i.e., '''.
+                        '\'' -> {
+                            outChars.append("\'")
+                            updateOffsets(index - 1)
+                            outChars.append(c)
+                        }
+                        //  $ can be escaped by prefixing it with '' (that is, two single quotes), i.e., ''$.
+                        '$' -> outChars.append(c)
+                        '\\' -> {
+                            if (index == chars.length) return false
+                            c = chars[index++]
+                            when (c) {
+                                // Linefeed, carriage-return and tab characters can
+                                // be written as ''\n, ''\r, ''\t, and ''\ escapes any other character.
+                                'a' -> outChars.append(0x07.toChar())
+                                'b' -> outChars.append('\b')
+                                'f' -> outChars.append(0x0c.toChar())
+                                'n' -> outChars.append('\n')
+                                't' -> outChars.append('\t')
+                                'r' -> outChars.append('\r')
+                                'v' -> outChars.append(0x0b.toChar())
+                                else -> return false
+                            }
+                        }
+
+                        else -> return false
+                    }
+                    if (sourceOffsets != null) {
+                        sourceOffsets[outChars.length - outOffset] = index
+                    }
+                    continue
+                }
+
+                outChars.append(c)
+            }
+            return true
+        }
+    }
+
+}

src/main/java/org/nixos/idea/psi/NixStringManipulator.kt

@@ -0,0 +1,34 @@
+package org.nixos.idea.psi
+
+import com.intellij.openapi.util.TextRange
+import com.intellij.psi.AbstractElementManipulator
+import com.intellij.refactoring.suggested.startOffset
+import org.nixos.idea.psi.impl.AbstractNixString
+import org.nixos.idea.util.NixIndStringUtil
+import org.nixos.idea.util.NixStringUtil
+
+class NixStringManipulator : AbstractElementManipulator<NixString>() {
+
+    /**
+     * This function's result changes the original text in the host language
+     * when the fragment in the guest language changes
+     */
+    override fun handleContentChange(
+        element: NixString,
+        range: TextRange,
+        newContent: String
+    ): NixString? {
+        val escaped = newContent
+        val replacement = range.replace(element.text, escaped)
+        return element.updateText(replacement) as? NixString
+    }
+
+    override fun getRangeInElement(element: NixString): TextRange = when {
+        element.textLength == 0 -> TextRange.EMPTY_RANGE
+        element is NixIndString && element.textLength < 4 -> TextRange(0, element.textLength)
+        element is NixIndString -> TextRange(2, element.textLength - 2)
+        // element is not IndString, so it must be StdString
+        element.textLength == 1 -> TextRange(0, 1)
+        else -> TextRange(1, element.textLength - 1)
+    }
+}

src/main/java/org/nixos/idea/psi/impl/AbstractNixPsiElement.java

@@ -5,7 +5,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.nixos.idea.psi.NixPsiElement;
 
-abstract class AbstractNixPsiElement extends ASTWrapperPsiElement implements NixPsiElement {
+abstract public class AbstractNixPsiElement extends ASTWrapperPsiElement implements NixPsiElement {
 
     AbstractNixPsiElement(@NotNull ASTNode node) {
         super(node);

src/main/java/org/nixos/idea/psi/impl/AbstractNixString.kt

@@ -0,0 +1,51 @@
+package org.nixos.idea.psi.impl
+
+import com.intellij.lang.ASTNode
+import com.intellij.openapi.diagnostic.Logger
+import com.intellij.psi.PsiLanguageInjectionHost
+import com.intellij.psi.impl.source.tree.LeafPsiElement
+import org.nixos.idea.psi.NixIndString
+import org.nixos.idea.psi.NixString
+import org.nixos.idea.psi.NixStringLiteralEscaper
+
+
+abstract class AbstractNixString(private val astNode: ASTNode) : PsiLanguageInjectionHost,
+    AbstractNixPsiElement(astNode), NixString {
+
+    override fun isValidHost() = true
+
+    override fun updateText(s: String): NixString {
+        // TODO issue #81 also support single-line strings
+        if (this !is NixIndString) {
+            LOG.info("not a nix ind string")
+            return this
+        }
+        val originalNode = astNode.firstChildNode.treeNext.firstChildNode as? LeafPsiElement
+        val minIndentInOriginal = originalNode?.text?.lines()
+            ?.filterNot { it.isEmpty() }
+            ?.minOfOrNull { it.takeWhile(Char::isWhitespace).count() } ?: 0
+
+        val leadingSpace = buildString { repeat(minIndentInOriginal) { append(' ') } }
+
+        val lines = s.substring(2..(s.lastIndex - 2)) // remove quotes
+            .lines()
+
+        // restore indent
+        val withIndent = lines
+            .withIndex()
+            .map { (index, line) -> if (index != 0) leadingSpace + line else line }
+
+        // if the first line was removed in the fragment, add it back to preserve a multiline string
+        val withLeadingBlankLine = if (lines.first().isNotEmpty()) listOf("") + withIndent else withIndent
+
+        originalNode?.replaceWithText(withLeadingBlankLine.joinToString(separator = System.lineSeparator()))
+        return this
+    }
+
+    override fun createLiteralTextEscaper() = NixStringLiteralEscaper(this)
+
+    companion object {
+        val LOG = Logger.getInstance(AbstractNixString::class.java)
+    }
+}
+

src/main/java/org/nixos/idea/util/NixIndStringUtil.kt

@@ -0,0 +1,68 @@
+package org.nixos.idea.util
+
+object NixIndStringUtil {
+    /**
+     * Unescapes the given string for use in a double-quoted string expression in the Nix Expression Language.
+     *
+     * See [Nix docs](https://nix.dev/manual/nix/2.22/language/values.html#type-string) for the logic, which
+     * is non-trivial.
+     *
+     * For example, `'` can be used to escape `''`, which means `'''` does not contain
+     * a string terminator
+     * ```
+     * $ nix eval --expr " ''   '''   '' "
+     *  "''   "
+     * ```
+     *
+     * This function does not erase string interpolations, because
+     * they are hard to parse in a loop without a proper grammar. For example:
+     * ```nix
+     * '' ${someNixFunc "${foo "}}" }" } ''
+     * ```
+     */
+    @JvmStatic
+    fun unescape(chars: CharSequence): String = buildString {
+        for ((index, c) in chars.withIndex()) {
+            fun prevChar() = chars.getOrNull(index - 1)
+            fun prev2Chars(): String? {
+                val prev = prevChar() ?: return null
+                val prevPrev = chars.getOrNull(index - 2) ?: return null
+                return "${prevPrev}${prev}"
+            }
+
+            fun prev3Chars(): String? {
+                val prev2 = prev2Chars() ?: return null
+                val prevPrev2 = chars.getOrNull(index - 3) ?: return null
+                return "${prevPrev2}${prev2}"
+            }
+
+            when (c) {
+                // ''\ escapes any character, but we can only cover known ones in advance:
+                '\'' -> when {
+                    // ''' is escaped to ''
+                    prev2Chars() == "''" -> append("''")
+                    // ''  is the string delimiter
+                    else -> continue
+                }
+
+                '\\' -> when {
+                    prev2Chars() == "''" -> continue
+                    prevChar() == '\'' -> continue
+                    else -> append(c)
+                }
+
+                '$' -> if (prevChar() == '$') append(c) else continue
+                '{' -> if (prevChar() == '$') append("\${") else append(c)
+
+                else -> if (prev3Chars() == "''\\") when (c) {
+                    'r' -> if (prev3Chars() == "''\\") append('\r') else append(c)
+                    'n' -> if (prev3Chars() == "''\\") append('\n') else append(c)
+                    't' -> if (prev3Chars() == "''\\") append('\t') else append(c)
+                    else -> append("''\\").append(c)
+                } else {
+                    append(c)
+                }
+            }
+        }
+    }
+}

src/main/java/org/nixos/idea/util/NixStringUtil.java

@@ -4,6 +4,7 @@
 import com.intellij.psi.tree.IElementType;
 import org.jetbrains.annotations.Contract;
 import org.jetbrains.annotations.NotNull;
+import org.nixos.idea.psi.NixStringPart;
 import org.nixos.idea.psi.NixStringText;
 import org.nixos.idea.psi.NixTypes;
 
@@ -113,6 +114,10 @@ private static void parse(@NotNull StringBuilder builder, @NotNull ASTNode token
         } else if (type == NixTypes.IND_STR_ESCAPE) {
             assert text.length() == 3 && ("''$".contentEquals(text) || "'''".contentEquals(text)) ||
                     text.length() == 4 && "''\\".contentEquals(text.subSequence(0, 3)) : text;
+            if ("'''".contentEquals(text)){
+                builder.append("''");
+                return;
+            }
             char c = text.charAt(text.length() - 1);
             builder.append(unescape(c));
         } else {

src/main/lang/Nix.bnf

@@ -24,6 +24,10 @@
   // no one really needs to know that + - * / are expected at any offset
   consumeTokenMethod("expr_op.*")="consumeTokenFast"
 
+   // make IndStrings language injection Hosts
+   implements("string")="com.intellij.psi.PsiLanguageInjectionHost"
+   mixin("string")="org.nixos.idea.psi.impl.AbstractNixString"
+
   tokens = [
     // This list does not contain all tokens. This list only defines the debug
     // names for tokens which have a distinct text representation. Other tokens

src/main/resources/META-INF/plugin.xml

@@ -59,6 +59,10 @@
 
         <formattingService implementation="org.nixos.idea.format.NixExternalFormatter"/>
 
+        <!-- Manipulator for injections support -->
+        <lang.elementManipulator forClass="org.nixos.idea.psi.NixString"
+                                 implementationClass="org.nixos.idea.psi.NixStringManipulator"/>
+
         <notificationGroup id="NixIDEA" displayType="BALLOON"/>
     </extensions>
 

src/test/java/org/nixos/idea/util/NixIndStringUtilTest.java

@@ -0,0 +1,37 @@
+package org.nixos.idea.util;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.nixos.idea.psi.NixStringLiteralEscaper;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+final class NixIndStringUtilTest {
+    @ParameterizedTest(name = "[{index}] {0} -> {1}")
+    @CsvSource(quoteCharacter = '|', textBlock = """
+            ||              , ||
+            abc             , abc
+            "               , "
+            \\              , \\
+            \\x             , \\x
+            a${b}c          , a${b}c
+            |\n|            , |\n|
+            |\r|            , |\r|
+            |\t|            , |\t|
+            |''\\t|         , |\t|
+            |''\\r|         , |\r|
+            |''\\n|         , |\n|
+            |'''|           , |''|
+            $$              , $$
+            ''$             , $
+            # supplementary character, i.e. character form a supplementary plane,
+            # which needs a surrogate pair to be represented in UTF-16
+            \uD83C\uDF09    , \uD83C\uDF09
+            """)
+    void unescape(String escaped, String expectedResult) {
+        var sb = new StringBuilder();
+        NixStringLiteralEscaper.Companion.unescapeAndDecode(escaped, sb, null);
+        var str = sb.toString();
+        assertEquals(expectedResult, str);
+    }
+}