From 47b48fec4b2ca82ef9ede3f6bbd7fd3c99f8246a Mon Sep 17 00:00:00 2001 From: Alexandr Evstigneev Date: Sat, 14 Sep 2024 11:22:43 +0400 Subject: [PATCH] #2846 Fixed a problem with heredoc inside regex sublexing Previously, we could find the heredoc terminator behind the end of the sublexed token and capture heredoc body longer than we should have. Fixes #2846 See also: #2905 --- .../perl5/lang/perl/lexer/PerlBaseLexer.java | 2 +- .../perl/parser/PerlHighlightingLexerTest.kt | 61 +++++++++++++++++++ .../unit/perl/parser/PerlParserLikeTest.java | 3 + .../perl/syntax/heredocInRegexp.pl.txt | 23 +++++++ .../unit/perl/lexer/heredocInRegexp.code | 5 ++ .../unit/perl/lexer/heredocInRegexp.pl.txt | 34 +++++++++++ .../perl/lexer/heredocInRegexpSublexed.pl.txt | 34 +++++++++++ .../unit/perl/parser/heredocInRegexp.code | 5 ++ .../unit/perl/parser/heredocInRegexp.txt | 34 +++++++++++ .../java/base/PerlLightTestCaseBase.java | 6 +- 10 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 plugin/src/test/java/unit/perl/parser/PerlHighlightingLexerTest.kt create mode 100644 plugin/src/test/resources/highlighting/perl/syntax/heredocInRegexp.pl.txt create mode 100644 plugin/src/test/resources/unit/perl/lexer/heredocInRegexp.code create mode 100644 plugin/src/test/resources/unit/perl/lexer/heredocInRegexp.pl.txt create mode 100644 plugin/src/test/resources/unit/perl/lexer/heredocInRegexpSublexed.pl.txt create mode 100644 plugin/src/test/resources/unit/perl/parser/heredocInRegexp.code create mode 100644 plugin/src/test/resources/unit/perl/parser/heredocInRegexp.txt diff --git a/plugin/core/src/main/java/com/perl5/lang/perl/lexer/PerlBaseLexer.java b/plugin/core/src/main/java/com/perl5/lang/perl/lexer/PerlBaseLexer.java index df60115eac..ebb99f63f3 100644 --- a/plugin/core/src/main/java/com/perl5/lang/perl/lexer/PerlBaseLexer.java +++ b/plugin/core/src/main/java/com/perl5/lang/perl/lexer/PerlBaseLexer.java @@ -1046,7 +1046,7 @@ protected void startHeredocCapture() { while( offset < bufferEnd){ int markerStartOffset = StringUtil.indexOf(buffer, closeMarker, offset); - if( markerStartOffset < 0){ + if (markerStartOffset < 0 || markerStartOffset >= bufferEnd) { // no end marker text ahead break; } diff --git a/plugin/src/test/java/unit/perl/parser/PerlHighlightingLexerTest.kt b/plugin/src/test/java/unit/perl/parser/PerlHighlightingLexerTest.kt new file mode 100644 index 0000000000..7aeb487d3c --- /dev/null +++ b/plugin/src/test/java/unit/perl/parser/PerlHighlightingLexerTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2015-2024 Alexandr Evstigneev + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package unit.perl.parser + +import base.PerlLightTestCase +import com.perl5.lang.perl.lexer.PerlLexer +import com.perl5.lang.perl.lexer.PerlLexingContext +import com.perl5.lang.perl.lexer.adapters.PerlMergingLexerAdapter +import org.jetbrains.annotations.NonNls +import org.junit.Test +import java.lang.reflect.Modifier + +class PerlHighlightingLexerTest : PerlLightTestCase() { + val lexerStates = PerlLexer::class.java.declaredFields + .filter { Modifier.isStatic(it.modifiers) && it.type == Int::class.java && it.canAccess(null) } + .associate { it.getInt(null) to it.name } + + override fun getBaseDataPath(): @NonNls String? = "unit/perl/lexer" + + @Test + fun testHeredocInRegexp() { + doTestLexer(); + } + + @Test + fun testHeredocInRegexpSublexed() { + doTestLexer("heredocInRegexp", true); + } + + private fun doTestLexer(sourceName: String? = null, forceSubLexing: Boolean = false) { + val testFileText = loadFileContent("${sourceName ?: getTestName(true)}${realDataFileExtension}") + val lexer = PerlMergingLexerAdapter(PerlLexingContext.create(project).withEnforcedSublexing(forceSubLexing)); + lexer.start(testFileText) + val sb = StringBuilder() + while (lexer.tokenType != null) { + sb.append("[").append(lexer.tokenType).append("] ").append(lexer.tokenStart).append("-").append(lexer.tokenEnd).append(" ") + .append(lexerStates[lexer.state] ?: lexer.state).append("\n") + if (lexer.tokenStart <= lexer.tokenEnd) { + sb.append(protectSpaces(testFileText.substring(lexer.tokenStart, lexer.tokenEnd))).append("\n") + } else { + sb.append("************** BAD TOKEN RANGE *************").append("\n") + } + lexer.advance() + } + assertSameLinesWithFile(getTestResultsFilePath(), sb.toString()) + } +} \ No newline at end of file diff --git a/plugin/src/test/java/unit/perl/parser/PerlParserLikeTest.java b/plugin/src/test/java/unit/perl/parser/PerlParserLikeTest.java index 2eac94624a..8a7418a4e5 100644 --- a/plugin/src/test/java/unit/perl/parser/PerlParserLikeTest.java +++ b/plugin/src/test/java/unit/perl/parser/PerlParserLikeTest.java @@ -26,6 +26,9 @@ protected String getBaseDataPath() { return "unit/perl/parser"; } + @Test + public void testHeredocInRegexp() { doTest(false); } + @Test public void testSubSignatureDefault() { doTest(); } diff --git a/plugin/src/test/resources/highlighting/perl/syntax/heredocInRegexp.pl.txt b/plugin/src/test/resources/highlighting/perl/syntax/heredocInRegexp.pl.txt new file mode 100644 index 0000000000..e86295fd3d --- /dev/null +++ b/plugin/src/test/resources/highlighting/perl/syntax/heredocInRegexp.pl.txt @@ -0,0 +1,23 @@ +s + PERL_KEYWORD => DEFAULT_KEYWORD +/ + PERL_REGEX_QUOTE => DEFAULT_BRACKETS +test + PERL_REGEX_TOKEN => DEFAULT_STRING +/ + PERL_REGEX_QUOTE => DEFAULT_BRACKETS +EOM + PERL_SQ_STRING => DEFAULT_STRING + + + PERL_DQ_STRING => DEFAULT_STRING +/ + PERL_REGEX_QUOTE => DEFAULT_BRACKETS +e + PERL_KEYWORD => DEFAULT_KEYWORD +; + PERL_SEMICOLON => DEFAULT_SEMICOLON +failing + PERL_SUB => DEFAULT_FUNCTION_CALL => DEFAULT_IDENTIFIER => TEXT +EOM + PERL_SUB => DEFAULT_FUNCTION_CALL => DEFAULT_IDENTIFIER => TEXT \ No newline at end of file diff --git a/plugin/src/test/resources/unit/perl/lexer/heredocInRegexp.code b/plugin/src/test/resources/unit/perl/lexer/heredocInRegexp.code new file mode 100644 index 0000000000..774b9da057 --- /dev/null +++ b/plugin/src/test/resources/unit/perl/lexer/heredocInRegexp.code @@ -0,0 +1,5 @@ +s/test/<