#2846 Fixed a problem with heredoc inside regex sublexing

Previously, we could find the heredoc terminator behind the end of the sublexed token and capture heredoc body longer than we should have.

Fixes #2846

See also: #2905

plugin/core/src/main/java/com/perl5/lang/perl/lexer/PerlBaseLexer.java

@@ -1046,7 +1046,7 @@ protected void startHeredocCapture() {
 
     while( offset < bufferEnd){
       int markerStartOffset = StringUtil.indexOf(buffer, closeMarker, offset);
-      if( markerStartOffset < 0){
+      if (markerStartOffset < 0 || markerStartOffset >= bufferEnd) {
         // no end marker text ahead
         break;
       }

plugin/src/test/java/unit/perl/parser/PerlHighlightingLexerTest.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2015-2024 Alexandr Evstigneev
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package unit.perl.parser
+
+import base.PerlLightTestCase
+import com.perl5.lang.perl.lexer.PerlLexer
+import com.perl5.lang.perl.lexer.PerlLexingContext
+import com.perl5.lang.perl.lexer.adapters.PerlMergingLexerAdapter
+import org.jetbrains.annotations.NonNls
+import org.junit.Test
+import java.lang.reflect.Modifier
+
+class PerlHighlightingLexerTest : PerlLightTestCase() {
+  val lexerStates = PerlLexer::class.java.declaredFields
+    .filter { Modifier.isStatic(it.modifiers) && it.type == Int::class.java && it.canAccess(null) }
+    .associate { it.getInt(null) to it.name }
+
+  override fun getBaseDataPath(): @NonNls String? = "unit/perl/lexer"
+
+  @Test
+  fun testHeredocInRegexp() {
+    doTestLexer();
+  }
+
+  @Test
+  fun testHeredocInRegexpSublexed() {
+    doTestLexer("heredocInRegexp", true);
+  }
+
+  private fun doTestLexer(sourceName: String? = null, forceSubLexing: Boolean = false) {
+    val testFileText = loadFileContent("${sourceName ?: getTestName(true)}${realDataFileExtension}")
+    val lexer = PerlMergingLexerAdapter(PerlLexingContext.create(project).withEnforcedSublexing(forceSubLexing));
+    lexer.start(testFileText)
+    val sb = StringBuilder()
+    while (lexer.tokenType != null) {
+      sb.append("[").append(lexer.tokenType).append("] ").append(lexer.tokenStart).append("-").append(lexer.tokenEnd).append(" ")
+        .append(lexerStates[lexer.state] ?: lexer.state).append("\n")
+      if (lexer.tokenStart <= lexer.tokenEnd) {
+        sb.append(protectSpaces(testFileText.substring(lexer.tokenStart, lexer.tokenEnd))).append("\n")
+      } else {
+        sb.append("************** BAD TOKEN RANGE *************").append("\n")
+      }
+      lexer.advance()
+    }
+    assertSameLinesWithFile(getTestResultsFilePath(), sb.toString())
+  }
+}

plugin/src/test/java/unit/perl/parser/PerlParserLikeTest.java

@@ -26,6 +26,9 @@ protected String getBaseDataPath() {
     return "unit/perl/parser";
   }
 
+  @Test
+  public void testHeredocInRegexp() { doTest(false); }
+
   @Test
   public void testSubSignatureDefault() { doTest(); }
 

plugin/src/test/resources/highlighting/perl/syntax/heredocInRegexp.pl.txt

@@ -0,0 +1,23 @@
+s
+    PERL_KEYWORD => DEFAULT_KEYWORD
+/
+    PERL_REGEX_QUOTE => DEFAULT_BRACKETS
+test
+    PERL_REGEX_TOKEN => DEFAULT_STRING
+/
+    PERL_REGEX_QUOTE => DEFAULT_BRACKETS
+EOM
+    PERL_SQ_STRING => DEFAULT_STRING
+
+
+    PERL_DQ_STRING => DEFAULT_STRING
+/
+    PERL_REGEX_QUOTE => DEFAULT_BRACKETS
+e
+    PERL_KEYWORD => DEFAULT_KEYWORD
+;
+    PERL_SEMICOLON => DEFAULT_SEMICOLON
+failing
+    PERL_SUB => DEFAULT_FUNCTION_CALL => DEFAULT_IDENTIFIER => TEXT
+EOM
+    PERL_SUB => DEFAULT_FUNCTION_CALL => DEFAULT_IDENTIFIER => TEXT

plugin/src/test/resources/unit/perl/lexer/heredocInRegexp.code

@@ -0,0 +1,5 @@
+s/test/<<EOM
+
+/e;
+failing
+EOM

plugin/src/test/resources/unit/perl/lexer/heredocInRegexp.pl.txt

@@ -0,0 +1,34 @@
+[Perl5: s] 0-1 YYINITIAL
+s
+[Perl5: r{] 1-2 65535
+/
+[Perl5: regex] 2-6 PREPARSED_ITEMS
+test
+[Perl5: re/] 6-7 65535
+/
+[Perl5: heredoc<<] 7-9 PREPARSED_ITEMS
+<<
+[Perl5: STRING_CONTENT] 9-12 PREPARSED_ITEMS
+EOM
+[WHITE_SPACE] 12-13 PREPARSED_ITEMS
+
+
+[Perl5: HEREDOC_QQ] 13-14 PREPARSED_ITEMS
+
+
+[Perl5: r}] 14-15 65535
+/
+[Perl5: /m] 15-16 65535
+e
+[Perl5: ;] 16-17 AFTER_VALUE
+;
+[WHITE_SPACE] 17-18 YYINITIAL
+
+
+[Perl5: subname] 18-25 YYINITIAL
+failing
+[WHITE_SPACE] 25-26 AFTER_IDENTIFIER
+
+
+[Perl5: subname] 26-29 AFTER_IDENTIFIER
+EOM

plugin/src/test/resources/unit/perl/lexer/heredocInRegexpSublexed.pl.txt

@@ -0,0 +1,34 @@
+[Perl5: s] 0-1 YYINITIAL
+s
+[Perl5: r{] 1-2 65535
+/
+[Perl5: regex] 2-6 PREPARSED_ITEMS
+test
+[Perl5: re/] 6-7 65535
+/
+[Perl5: heredoc<<] 7-9 PREPARSED_ITEMS
+<<
+[Perl5: STRING_CONTENT] 9-12 PREPARSED_ITEMS
+EOM
+[WHITE_SPACE] 12-13 PREPARSED_ITEMS
+
+
+[Perl5: STRING_CONTENT_QQ] 13-14 PREPARSED_ITEMS
+
+
+[Perl5: r}] 14-15 65535
+/
+[Perl5: /m] 15-16 65535
+e
+[Perl5: ;] 16-17 AFTER_VALUE
+;
+[WHITE_SPACE] 17-18 YYINITIAL
+
+
+[Perl5: subname] 18-25 YYINITIAL
+failing
+[WHITE_SPACE] 25-26 AFTER_IDENTIFIER
+
+
+[Perl5: subname] 26-29 AFTER_IDENTIFIER
+EOM

plugin/src/test/resources/unit/perl/parser/heredocInRegexp.code

@@ -0,0 +1,5 @@
+s/test/<<EOM
+
+/e;
+failing
+EOM

plugin/src/test/resources/unit/perl/parser/heredocInRegexp.txt

@@ -0,0 +1,34 @@
+Subtree: Perl5 (Perl5)
+Perl5
+  PsiPerlStatementImpl(Perl5: STATEMENT)
+    PsiPerlReplacementRegexImpl(Perl5: REPLACEMENT_REGEX)
+      PsiElement(Perl5: s)('s')
+      PsiElement(Perl5: r{)('/')
+      PsiPerlPerlRegexImpl(Perl5: PERL_REGEX)
+        PsiElement(Perl5: regex)('test')
+      PsiElement(Perl5: re/)('/')
+      PsiPerlBlockBracelessImpl(Perl5: BLOCK_BRACELESS)
+        PsiPerlStatementImpl(Perl5: STATEMENT)
+          PsiPerlHeredocOpenerImpl(Perl5: HEREDOC_OPENER)
+            PsiElement(Perl5: heredoc<<)('<<')
+            PsiPerlStringBareImpl(Perl5: STRING_BARE)
+              PerlStringContentElementImpl(Perl5: STRING_CONTENT)('EOM')
+        PerlHeredocElementImpl(Perl5: HEREDOC_QQ)
+          PerlStringContentElementImpl(Perl5: STRING_CONTENT_QQ)('\n')
+      PsiElement(Perl5: r})('/')
+      PsiPerlPerlRegexModifiersImpl(Perl5: PERL_REGEX_MODIFIERS)
+        PsiElement(Perl5: /m)('e')
+    PsiElement(Perl5: ;)(';')
+  PsiPerlStatementImpl(Perl5: STATEMENT)
+    PsiPerlSubCallImpl(SUB_CALL)
+      PsiPerlMethodImpl(Perl5: METHOD)
+        PerlSubNameElementImpl(Perl5: subname)('failing')
+      PsiPerlCallArgumentsImpl(Perl5: CALL_ARGUMENTS)
+        PsiPerlSubCallImpl(SUB_CALL)
+          PsiPerlMethodImpl(Perl5: METHOD)
+            PerlSubNameElementImpl(Perl5: subname)('EOM')
+--------------------------------------------------------------------------------
+Subtree: Perl5 POD (Perl5 POD)
+POD file
+  PsiElement(Perl5 POD: POD_OUTER)('s/test/<<EOM\n\n/e;\nfailing\nEOM')
+--------------------------------------------------------------------------------

plugin/src/testFixtures/java/base/PerlLightTestCaseBase.java

@@ -512,7 +512,11 @@ protected String getRealDataFileExtension() {
   }
 
   public void initWithFile(String targetFileName, String targetFileExtension, String sourceFileNameWithExtension) {
-    initWithFileContent(targetFileName, targetFileExtension, loadFile(new File(getTestDataPath(), sourceFileNameWithExtension)));
+    initWithFileContent(targetFileName, targetFileExtension, loadFileContent(sourceFileNameWithExtension));
+  }
+
+  protected @NotNull String loadFileContent(@NotNull String sourceFileNameWithExtension) {
+    return loadFile(new File(getTestDataPath(), sourceFileNameWithExtension));
   }
 
   public static @NotNull String loadFile(@NotNull File fileToLoad) {