#2763 implemented psi-based words scanning for the perl files

Fixes #2763

plugin/core/src/main/java/com/perl5/lang/perl/idea/findusages/PerlIdIndexer.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2015-2023 Alexandr Evstigneev
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.perl5.lang.perl.idea.findusages;
+
+import com.intellij.lang.cacheBuilder.WordOccurrence;
+import com.intellij.lang.cacheBuilder.WordsScanner;
+import com.intellij.psi.impl.cache.impl.id.IdDataConsumer;
+import com.intellij.psi.impl.cache.impl.id.IdIndexEntry;
+import com.intellij.psi.impl.cache.impl.id.ScanningIdIndexer;
+import com.intellij.psi.search.UsageSearchContext;
+import com.intellij.util.Processor;
+import com.intellij.util.indexing.FileContent;
+import com.intellij.util.text.CharArrayUtil;
+import com.perl5.lang.perl.PerlLanguage;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Map;
+
+public class PerlIdIndexer extends ScanningIdIndexer {
+  @Override
+  protected PerlWordsScanner createScanner() {
+    return new PerlWordsScanner();
+  }
+
+  @Override
+  public @NotNull Map<IdIndexEntry, Integer> map(@NotNull FileContent inputData) {
+    final IdDataConsumer consumer = new IdDataConsumer();
+    var psiFile = inputData.getPsiFile();
+    var perlPsi = psiFile.getViewProvider().getPsi(PerlLanguage.INSTANCE);
+    if (perlPsi != null) {
+      //noinspection deprecation
+      createScanner().processWordsUsingPsi(psiFile, createProcessor(inputData.getContentAsText(), consumer));
+    }
+    return consumer.getResult();
+  }
+
+  /**
+   * @deprecated available in IDEA since 2023.3
+   * @see com.intellij.psi.impl.cache.impl.id.ScanningIdIndexer#createProcessor
+   */
+  private static @NotNull Processor<WordOccurrence> createProcessor(@NotNull CharSequence chars, @NotNull IdDataConsumer consumer) {
+    final char[] charsArray = CharArrayUtil.fromSequenceWithoutCopying(chars);
+    return new Processor<>() {
+      @Override
+      public boolean process(final WordOccurrence t) {
+        if (charsArray != null && t.getBaseText() == chars) {
+          consumer.addOccurrence(charsArray, t.getStart(), t.getEnd(), convertToMask(t.getKind()));
+        }
+        else {
+          consumer.addOccurrence(t.getBaseText(), t.getStart(), t.getEnd(), convertToMask(t.getKind()));
+        }
+        return true;
+      }
+
+      private int convertToMask(final WordOccurrence.Kind kind) {
+        if (kind == null) {
+          return UsageSearchContext.ANY;
+        }
+        if (kind == WordOccurrence.Kind.CODE) return UsageSearchContext.IN_CODE;
+        if (kind == WordOccurrence.Kind.COMMENTS) return UsageSearchContext.IN_COMMENTS;
+        if (kind == WordOccurrence.Kind.LITERALS) return UsageSearchContext.IN_STRINGS;
+        if (kind == WordOccurrence.Kind.FOREIGN_LANGUAGE) return UsageSearchContext.IN_FOREIGN_LANGUAGES;
+        return 0;
+      }
+    };
+  }
+}

plugin/core/src/main/java/com/perl5/lang/perl/idea/findusages/PerlWordsScanner.java

@@ -17,17 +17,22 @@
 package com.perl5.lang.perl.idea.findusages;
 
 import com.intellij.lang.cacheBuilder.DefaultWordsScanner;
+import com.intellij.lang.cacheBuilder.WordOccurrence;
+import com.intellij.psi.PsiFile;
+import com.intellij.psi.impl.source.tree.TreeUtil;
+import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.tree.TokenSet;
+import com.intellij.util.Processor;
 import com.perl5.lang.perl.PerlParserDefinition;
-import com.perl5.lang.perl.lexer.PerlElementTypes;
 import com.perl5.lang.perl.lexer.PerlLexingContext;
 import com.perl5.lang.perl.lexer.adapters.PerlMergingLexerAdapter;
 import org.jetbrains.annotations.NotNull;
 
+import static com.perl5.lang.perl.lexer.PerlElementTypesGenerated.*;
 import static com.perl5.lang.perl.lexer.PerlTokenSets.*;
 
 
-public class PerlWordsScanner extends DefaultWordsScanner implements PerlElementTypes {
+public class PerlWordsScanner extends DefaultWordsScanner implements PsiBasedWordScanner {
   private static final TokenSet IGNORE_CODE_TOKENSET = TokenSet.orSet(
     SIGILS, SPECIAL_STRING_TOKENS, KEYWORDS_TOKENSET, OPERATORS_TOKENSET,
     QUOTE_OPEN_ANY, QUOTE_CLOSE_FIRST_ANY,
@@ -38,17 +43,79 @@ public class PerlWordsScanner extends DefaultWordsScanner implements PerlElement
       OPERATOR_FILETEST, BUILTIN_ARGUMENTLESS, BUILTIN_UNARY, BUILTIN_LIST,
       NUMBER_VERSION, VERSION_ELEMENT, SUB_PROTOTYPE_TOKEN
     ));
-  private static final @NotNull TokenSet COMMENTS = TokenSet.andNot(
+  private static final @NotNull TokenSet COMMENTS_TOKENSET = TokenSet.andNot(
     TokenSet.orSet(PerlParserDefinition.COMMENTS, TokenSet.create(POD)),
     TokenSet.create(HEREDOC_END)
   );
+  private static final TokenSet IDENTIFIERS_TOKENSET = PerlParserDefinition.IDENTIFIERS;
+  private static final TokenSet LITERALS_TOKENSET = PerlParserDefinition.LITERALS;
+
 
   public PerlWordsScanner() {
     super(new PerlMergingLexerAdapter(PerlLexingContext.create(null).withEnforcedSublexing(true)),
-          PerlParserDefinition.IDENTIFIERS,
-          COMMENTS,
-          PerlParserDefinition.LITERALS,
-          IGNORE_CODE_TOKENSET
+          IDENTIFIERS_TOKENSET, COMMENTS_TOKENSET, LITERALS_TOKENSET, IGNORE_CODE_TOKENSET
     );
   }
+
+  private static @NotNull PerlMergingLexerAdapter createLexer() {
+    return new PerlMergingLexerAdapter(PerlLexingContext.create(null).withEnforcedSublexing(true));
+  }
+
+  @Override
+  public void processWordsUsingPsi(@NotNull PsiFile psiFile, @NotNull Processor<? super WordOccurrence> processor) {
+    var fileNode = psiFile.getNode();
+    var run = TreeUtil.findFirstLeaf(fileNode);
+    if (run == null) {
+      return;
+    }
+    var fileText = psiFile.getText();
+    WordOccurrence occurrence = new WordOccurrence(fileText, 0, 0, WordOccurrence.Kind.CODE); // shared occurrence
+    while (run != null) {
+      if (!processToken(fileText, processor, run.getElementType(), run.getStartOffset(), run.getStartOffset() + run.getTextLength(),
+                        occurrence)) {
+        return;
+      }
+      run = TreeUtil.nextLeaf(run);
+    }
+  }
+
+  @Override
+  public void processWords(final @NotNull CharSequence fileText, final @NotNull Processor<? super WordOccurrence> processor) {
+    var lexer = createLexer();
+    lexer.start(fileText);
+    WordOccurrence occurrence = new WordOccurrence(fileText, 0, 0, WordOccurrence.Kind.CODE); // shared occurrence
+
+    while (lexer.getTokenType() != null) {
+      if (!processToken(fileText, processor, lexer.getTokenType(), lexer.getTokenStart(), lexer.getTokenEnd(), occurrence)) {
+        return;
+      }
+      lexer.advance();
+    }
+  }
+
+  @SuppressWarnings("BooleanMethodIsAlwaysInverted")
+  protected static boolean processToken(@NotNull CharSequence fileText,
+                                        @NotNull Processor<? super WordOccurrence> processor,
+                                        @NotNull IElementType type,
+                                        int start,
+                                        int end,
+                                        @NotNull WordOccurrence occurrence) {
+    WordOccurrence.Kind kind;
+    if (IDENTIFIERS_TOKENSET.contains(type)) {
+      kind = WordOccurrence.Kind.CODE;
+    }
+    else if (COMMENTS_TOKENSET.contains(type)) {
+      kind = WordOccurrence.Kind.COMMENTS;
+    }
+    else if (LITERALS_TOKENSET.contains(type)) {
+      kind = WordOccurrence.Kind.LITERALS;
+    }
+    else if (!IGNORE_CODE_TOKENSET.contains(type)) {
+      kind = WordOccurrence.Kind.CODE;
+    }
+    else {
+      return true;
+    }
+    return stripWords(processor, fileText, start, end, kind, occurrence, false);
+  }
 }

plugin/core/src/main/java/com/perl5/lang/perl/idea/findusages/PsiBasedWordScanner.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2015-2023 Alexandr Evstigneev
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.perl5.lang.perl.idea.findusages;
+
+import com.intellij.lang.cacheBuilder.WordOccurrence;
+import com.intellij.lang.cacheBuilder.WordsScanner;
+import com.intellij.psi.PsiFile;
+import com.intellij.util.Processor;
+import org.jetbrains.annotations.NotNull;
+
+public interface PsiBasedWordScanner extends WordsScanner {
+  void processWordsUsingPsi(@NotNull PsiFile psiFile, @NotNull Processor<? super WordOccurrence> processor);
+}

plugin/src/main/resources/META-INF/plugin.xml

@@ -212,6 +212,10 @@
     <targetElementEvaluator language="Perl5"
                             implementationClass="com.perl5.lang.perl.psi.references.PerlTargetElementEvaluatorEx2"/>
     <highlightUsagesHandlerFactory id="mainone" implementation="com.perl5.lang.perl.idea.findusages.PerlHighlightUsagesHandlerFactory"/>
+    <idIndexer filetype="Perl5 Script" implementationClass="com.perl5.lang.perl.idea.findusages.PerlIdIndexer"/>
+    <idIndexer filetype="Perl5 Package" implementationClass="com.perl5.lang.perl.idea.findusages.PerlIdIndexer"/>
+    <idIndexer filetype="Cpanfile" implementationClass="com.perl5.lang.perl.idea.findusages.PerlIdIndexer"/>
+    <idIndexer filetype="Perl5 Test" implementationClass="com.perl5.lang.perl.idea.findusages.PerlIdIndexer"/>
     <highlightUsagesHandlerFactory order="before mainone"
                                    implementation="com.perl5.lang.perl.idea.codeInsight.highlighting.PerlHighlightExitPointsHandlerFactory"/>
     <usageTypeProvider implementation="com.perl5.lang.perl.idea.findusages.PerlUsageTypeProvider"/>

plugin/src/test/resources/wordScanner/perl/controller.pl.txt

@@ -1,5 +1,5 @@
 Statistics:
-	CODE: 697 occurrences
+	CODE: 699 occurrences
 	COMMENTS: 140 occurrences
 	LITERALS: 135 occurrences
 ================================
@@ -155,6 +155,7 @@ Tokens by type:
 		type
 		URL
 		app
+		has
 		isa
 		log
 		msg
@@ -381,8 +382,8 @@ use <CODE>Mojolicious</CODE>::<CODE>Routes</CODE>::<CODE>Match</CODE>;
 use <CODE>Scalar</CODE>::<CODE>Util</CODE> ();
 use <CODE>Time</CODE>::<CODE>HiRes</CODE>  ();
 
-has [qw(<LITERALS>app</LITERALS> <LITERALS>tx</LITERALS>)];
-has <LITERALS>match</LITERALS> =>
+<CODE>has</CODE> [qw(<LITERALS>app</LITERALS> <LITERALS>tx</LITERALS>)];
+<CODE>has</CODE> <LITERALS>match</LITERALS> =>
   sub { <CODE>Mojolicious</CODE>::<CODE>Routes</CODE>::<CODE>Match</CODE>-><CODE>new</CODE>(<LITERALS>root</LITERALS> => shift-><CODE>app</CODE>-><CODE>routes</CODE>) };
 
 # <COMMENTS>Reserved</COMMENTS> <COMMENTS>stash</COMMENTS> <COMMENTS>values</COMMENTS>