Bound pathological addFile() cost by truncating REFS tokenization

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/JFlexTokenizer.java

@@ -43,6 +43,9 @@ public class JFlexTokenizer extends Tokenizer
 
     private final ScanningSymbolMatcher matcher;
     private boolean didSetAttribsValues;
+    private int maxEmittedTokens;
+    private int emittedTokenCount;
+    private boolean tokenLimitReached;
 
     /**
      * Initialize an instance, passing a {@link ScanningSymbolMatcher} which
@@ -72,6 +75,8 @@ public void reset() throws IOException {
         matcher.yyreset(input);
         matcher.reset();
         clearAttributesEtc();
+        emittedTokenCount = 0;
+        tokenLimitReached = false;
     }
 
     /**
@@ -98,14 +103,46 @@ public final void close() throws IOException {
      */
     @Override
     public final boolean incrementToken() throws IOException {
+        if (tokenLimitReached) {
+            clearAttributesEtc();
+            return false;
+        }
+
         boolean notAtEOF;
         do {
             clearAttributesEtc();
             notAtEOF = matcher.yylex() != matcher.getYYEOF();
         } while (!didSetAttribsValues && notAtEOF);
+
+        if (didSetAttribsValues) {
+            emittedTokenCount++;
+            if (maxEmittedTokens > 0 && emittedTokenCount >= maxEmittedTokens) {
+                tokenLimitReached = true;
+            }
+        }
+
         return notAtEOF;
     }
 
+    /**
+     * Sets the maximum number of emitted tokens. Non-positive values disable
+     * the limit.
+     *
+     * @param maxEmittedTokens maximum number of emitted tokens
+     */
+    public void setMaxEmittedTokens(int maxEmittedTokens) {
+        this.maxEmittedTokens = maxEmittedTokens;
+    }
+
+    /**
+     * Gets whether the configured token limit has been reached.
+     *
+     * @return {@code true} if the limit has been reached, {@code false} otherwise
+     */
+    public boolean isTokenLimitReached() {
+        return tokenLimitReached;
+    }
+
     /**
      * Calls {@link #setAttribs(String, long, long)} on the publishing
      * of a {@link SymbolMatchedEvent}.

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/plain/PlainAnalyzer.java

@@ -57,6 +57,12 @@
  */
 public class PlainAnalyzer extends TextAnalyzer {
 
+    /**
+     * Conservative safeguard to stop pathological symbol streams from
+     * monopolizing indexing work while preserving normal indexing behavior.
+     */
+    static final int MAX_REFS_TOKENS_PER_FILE = 100_000;
+
     /**
      * Creates a new instance of PlainAnalyzer.
      * @param factory defined instance for the analyzer
@@ -130,6 +136,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOExc
          * work around #1376: symbols search works like full text search.
          */
         JFlexTokenizer symbolTokenizer = symbolTokenizerFactory.get();
+        symbolTokenizer.setMaxEmittedTokens(getRefsTokenLimit());
         OGKTextField ref = new OGKTextField(QueryBuilder.REFS, symbolTokenizer);
         symbolTokenizer.setReader(getReader(src.getStream()));
         doc.add(ref);
@@ -207,4 +214,14 @@ private void tryAddingDefs(Document doc, Definitions defs, StreamSource src) thr
     private Reader wrapReader(Reader reader) {
         return ExpandTabsReader.wrap(reader, project);
     }
+
+    /**
+     * Gets the maximum number of tokens emitted into the REFS field for a
+     * single file.
+     *
+     * @return maximum REFS token count, or non-positive for unlimited
+     */
+    protected int getRefsTokenLimit() {
+        return MAX_REFS_TOKENS_PER_FILE;
+    }
 }

opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/JFlexTokenizerTest.java

@@ -26,6 +26,8 @@
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.junit.jupiter.api.Test;
@@ -164,4 +166,182 @@ void truncatedUuencodedFile() throws IOException {
         // This call used to hang forever.
         assertFalse(tokenizer.incrementToken());
     }
+
+    @Test
+    void tokenLimitStopsFurtherScanningAndResetRestoresTokenizer() throws Exception {
+        FakeMatcher matcher = new FakeMatcher(List.of("alpha", "beta", "gamma", "delta"));
+        JFlexTokenizer tokenizer = new JFlexTokenizer(matcher);
+        tokenizer.setMaxEmittedTokens(2);
+        tokenizer.setReader(new StringReader("unused"));
+        tokenizer.reset();
+
+        CharTermAttribute term = tokenizer.addAttribute(CharTermAttribute.class);
+        List<String> seen = new ArrayList<>();
+        while (tokenizer.incrementToken()) {
+            seen.add(term.toString());
+        }
+
+        assertEquals(List.of("alpha", "beta"), seen);
+        assertTrue(tokenizer.isTokenLimitReached());
+        assertEquals(2, matcher.getYylexCalls());
+
+        tokenizer.close();
+        assertTrue(matcher.isClosed());
+
+        matcher.resetYylexCalls();
+        tokenizer.setReader(new StringReader("unused-again"));
+        tokenizer.reset();
+        tokenizer.setMaxEmittedTokens(0);
+        seen.clear();
+        while (tokenizer.incrementToken()) {
+            seen.add(term.toString());
+        }
+
+        assertEquals(List.of("alpha", "beta", "gamma", "delta"), seen);
+        assertFalse(tokenizer.isTokenLimitReached());
+        assertEquals(5, matcher.getYylexCalls());
+    }
+
+    private static final class FakeMatcher implements ScanningSymbolMatcher {
+        private static final int TOKEN = 1;
+        private static final int YYEOF = -1;
+
+        private final List<String> tokens;
+        private int cursor;
+        private int yylexCalls;
+        private Reader reader;
+        private SymbolMatchedListener symbolMatchedListener;
+        private NonSymbolMatchedListener nonSymbolMatchedListener;
+        private boolean closed;
+
+        private FakeMatcher(List<String> tokens) {
+            this.tokens = tokens;
+        }
+
+        int getYylexCalls() {
+            return yylexCalls;
+        }
+
+        void resetYylexCalls() {
+            yylexCalls = 0;
+        }
+
+        boolean isClosed() {
+            return closed;
+        }
+
+        @Override
+        public void setSymbolMatchedListener(SymbolMatchedListener l) {
+            symbolMatchedListener = l;
+        }
+
+        @Override
+        public void clearSymbolMatchedListener() {
+            symbolMatchedListener = null;
+        }
+
+        @Override
+        public void setNonSymbolMatchedListener(NonSymbolMatchedListener l) {
+            nonSymbolMatchedListener = l;
+        }
+
+        @Override
+        public void clearNonSymbolMatchedListener() {
+            nonSymbolMatchedListener = null;
+        }
+
+        @Override
+        public void reset() {
+            cursor = 0;
+            closed = false;
+        }
+
+        @Override
+        public void yypush(int newState) {
+        }
+
+        @Override
+        public void yypop() {
+        }
+
+        @Override
+        public long getYYCHAR() {
+            return cursor;
+        }
+
+        @Override
+        public int getYYEOF() {
+            return YYEOF;
+        }
+
+        @Override
+        public int getLineNumber() {
+            return 1;
+        }
+
+        @Override
+        public boolean emptyStack() {
+            return true;
+        }
+
+        @Override
+        public String yytext() {
+            return cursor == 0 ? "" : tokens.get(Math.min(cursor - 1, tokens.size() - 1));
+        }
+
+        @Override
+        public int yylength() {
+            return yytext().length();
+        }
+
+        @Override
+        public char yycharat(int pos) {
+            return yytext().charAt(pos);
+        }
+
+        @Override
+        public void yyclose() {
+            closed = true;
+        }
+
+        @Override
+        public void yyreset(Reader reader) {
+            this.reader = reader;
+            reset();
+        }
+
+        @Override
+        public int yystate() {
+            return 0;
+        }
+
+        @Override
+        public void yybegin(int lexicalState) {
+        }
+
+        @Override
+        public void yypushback(int number) {
+        }
+
+        @Override
+        public int yylex() throws IOException {
+            if (reader == null) {
+                throw new IOException("reader not set");
+            }
+
+            yylexCalls++;
+            if (cursor >= tokens.size()) {
+                return YYEOF;
+            }
+
+            String token = tokens.get(cursor);
+            cursor++;
+            if (symbolMatchedListener != null) {
+                long start = cursor * 10L;
+                symbolMatchedListener.symbolMatched(
+                        new SymbolMatchedEvent(this, token, start, start + token.length()));
+            }
+            return TOKEN;
+        }
+    }
 }

opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/javascript/JavaScriptSymbolTokenizerTest.java

@@ -23,15 +23,24 @@
  */
 package org.opengrok.indexer.analysis.javascript;
 
+import java.io.InputStream;
+import java.io.Reader;
+import java.util.List;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.document.Document;
 import org.junit.jupiter.api.Test;
+import org.opengrok.indexer.analysis.OGKTextField;
+import org.opengrok.indexer.search.QueryBuilder;
+import org.opengrok.indexer.util.StreamUtils;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.opengrok.indexer.util.CustomAssertions.assertSymbolStream;
 import static org.opengrok.indexer.util.StreamUtils.readSampleSymbols;
 
-import java.io.InputStream;
-import java.util.List;
-
 /**
  * Tests the {@link JavaScriptSymbolTokenizer} class.
  */
@@ -68,4 +77,64 @@ private void testSymbols(String codeResource, String symbolsResource) throws Exc
         List<String> expectedSymbols = readSampleSymbols(symres);
         assertSymbolStream(JavaScriptSymbolTokenizer.class, jsres, expectedSymbols);
     }
+
+    @Test
+    void largeJavaScriptFixtureTruncatesRefsButKeepsFullField() throws Exception {
+        int tokenLimit = 1;
+        LimitedJavaScriptAnalyzer analyzer = new LimitedJavaScriptAnalyzer(tokenLimit);
+        Document doc = new Document();
+
+        analyzer.analyze(doc, StreamUtils.sourceFromEmbedded("sources/javascript/testlong.js"), null);
+
+        OGKTextField refsField = (OGKTextField) doc.getField(QueryBuilder.REFS);
+        assertNotNull(refsField);
+        assertEquals(tokenLimit, countTokens(refsField.tokenStreamValue()));
+
+        OGKTextField fullField = (OGKTextField) doc.getField(QueryBuilder.FULL);
+        assertNotNull(fullField);
+        try (TokenStream fullStream = analyzer.tokenStream(
+                QueryBuilder.FULL, (Reader) fullField.readerValue())) {
+            CharTermAttribute term = fullStream.addAttribute(CharTermAttribute.class);
+            fullStream.reset();
+            assertTrue(fullStream.incrementToken());
+            assertEquals("beforelongline", term.toString());
+            assertTrue(countAtLeast(fullStream, tokenLimit + 1));
+            fullStream.end();
+        }
+    }
+
+    private static int countTokens(TokenStream tokenStream) throws Exception {
+        int count = 0;
+        try (TokenStream stream = tokenStream) {
+            stream.addAttribute(CharTermAttribute.class);
+            stream.reset();
+            while (stream.incrementToken()) {
+                count++;
+            }
+            stream.end();
+        }
+        return count;
+    }
+
+    private static boolean countAtLeast(TokenStream tokenStream, int minimumTokenCount) throws Exception {
+        int count = 1;
+        while (count < minimumTokenCount && tokenStream.incrementToken()) {
+            count++;
+        }
+        return count >= minimumTokenCount;
+    }
+
+    private static final class LimitedJavaScriptAnalyzer extends JavaScriptAnalyzer {
+        private final int refsTokenLimit;
+
+        private LimitedJavaScriptAnalyzer(int refsTokenLimit) {
+            super(new JavaScriptAnalyzerFactory());
+            this.refsTokenLimit = refsTokenLimit;
+        }
+
+        @Override
+        protected int getRefsTokenLimit() {
+            return refsTokenLimit;
+        }
+    }
 }