Fix token detection for single letters

mgod · mgod · commit b71b7a0f7c93 · 2019-03-19T11:15:08.000-04:00
diff --git a/example/src/main/java/com/tokenautocompleteexample/TagCompletionView.java b/example/src/main/java/com/tokenautocompleteexample/TagCompletionView.java
@@ -2,6 +2,7 @@
 
 import android.app.Activity;
 import android.content.Context;
+import android.os.Parcelable;
 import android.util.AttributeSet;
 import android.view.LayoutInflater;
 import android.view.View;
@@ -25,6 +26,10 @@ protected View getViewForObject(Tag object) {
 
     @Override
     protected Tag defaultObject(String completionText) {
-        return new Tag(completionText.charAt(0), completionText.substring(1, completionText.length()));
+        if (completionText.length() == 1) {
+            return null;
+        } else {
+            return new Tag(completionText.charAt(0), completionText.substring(1, completionText.length()));
+        }
     }
 }
diff --git a/example/src/main/java/com/tokenautocompleteexample/TokenActivity.java b/example/src/main/java/com/tokenautocompleteexample/TokenActivity.java
@@ -39,6 +39,7 @@ protected void onCreate(Bundle savedInstanceState) {
 
         completionView = (ContactsCompletionView)findViewById(R.id.searchView);
         completionView.setAdapter(adapter);
+        completionView.setThreshold(1);
         completionView.setTokenListener(this);
         completionView.setTokenClickStyle(TokenCompleteTextView.TokenClickStyle.Select);
         completionView.addTextChangedListener(new TextWatcher() {
@@ -92,6 +93,7 @@ public void onClick(View v) {
         tagView.setTokenizer(new TagTokenizer(Arrays.asList('@', '#')));
         tagView.setAdapter(new TagAdapter(this, R.layout.tag_layout, Tag.sampleTags()));
         tagView.setTokenClickStyle(TokenCompleteTextView.TokenClickStyle.Select);
+        tagView.setThreshold(1);
 
         final TextView taggedContentPreview = findViewById(R.id.composedValue);
 
diff --git a/library/src/main/java/com/tokenautocomplete/CharacterTokenizer.java b/library/src/main/java/com/tokenautocomplete/CharacterTokenizer.java
@@ -50,7 +50,7 @@ public List<Range> findTokenRanges(CharSequence charSequence, int start, int end
         int tokenStart = start;
 
         for (int cursor = start; cursor < end; ++cursor) {
-            Character character = charSequence.charAt(cursor);
+            char character = charSequence.charAt(cursor);
 
             //Avoid including leading whitespace, tokenStart will match the cursor as long as we're at the start
             if (tokenStart == cursor && Character.isWhitespace(character)) {
@@ -59,7 +59,12 @@ public List<Range> findTokenRanges(CharSequence charSequence, int start, int end
 
             //Either this is a split character, or we contain some content and are at the end of input
             if (splitChar.contains(character) || cursor == end - 1) {
-                if (cursor > tokenStart) {
+                boolean hasTokenContent =
+                        //There is token content befor the current character
+                        cursor > tokenStart ||
+                        //If the current single character is valid token content, not a split char or whitespace
+                        (cursor == tokenStart && !splitChar.contains(character));
+                if (hasTokenContent) {
                     //There is some token content
                     //Add one to range end as the end of the ranges is not inclusive
                     result.add(new Range(tokenStart, cursor + 1));
diff --git a/library/src/main/java/com/tokenautocomplete/TagTokenizer.java b/library/src/main/java/com/tokenautocomplete/TagTokenizer.java
@@ -52,19 +52,13 @@ public List<Range> findTokenRanges(CharSequence charSequence, int start, int end
         int tokenStart = Integer.MAX_VALUE;
 
         for (int cursor = start; cursor < end; ++cursor) {
-            Character character = charSequence.charAt(cursor);
+            char character = charSequence.charAt(cursor);
 
             //Either this is a terminator, or we contain some content and are at the end of input
-            if (isTokenTerminator(character) || cursor == end - 1) {
+            if (isTokenTerminator(character)) {
                 //Is there some token content? Might just be two terminators in a row
                 if (cursor - 1 > tokenStart) {
-                    if (isTokenTerminator(character)) {
-                        result.add(new Range(tokenStart, cursor));
-                    } else {
-                        //If we're here and the character isn't a token terminator, make sure we include
-                        //the last character of the input
-                        result.add(new Range(tokenStart, cursor + 1));
-                    }
+                    result.add(new Range(tokenStart, cursor));
                 }
 
                 //mark that we don't have a candidate token start any more
@@ -77,6 +71,11 @@ public List<Range> findTokenRanges(CharSequence charSequence, int start, int end
             }
         }
 
+        if (end > tokenStart) {
+            //There was unterminated text after a start of token
+            result.add(new Range(tokenStart, end));
+        }
+
         return result;
     }
 
diff --git a/library/src/main/java/com/tokenautocomplete/TokenCompleteTextView.java b/library/src/main/java/com/tokenautocomplete/TokenCompleteTextView.java
@@ -508,7 +508,7 @@ private Range getCurrentCandidateTokenRange() {
                 candidateStringStart = spanEnd;
             }
             int spanStart = editable.getSpanStart(span);
-            if (candidateStringEnd < spanStart && cursorEndPosition <= spanEnd) {
+            if (candidateStringEnd > spanStart && cursorEndPosition <= spanEnd) {
                 candidateStringEnd = spanStart;
             }
         }
diff --git a/library/src/test/java/com/tokenautocomplete/CharacterTokenizerTest.java b/library/src/test/java/com/tokenautocomplete/CharacterTokenizerTest.java
@@ -1,12 +1,15 @@
 package com.tokenautocomplete;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertEquals;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.RandomAccess;
 
 /**
  * Make sure the tokenizer finds the right boundaries
@@ -31,6 +34,7 @@ public void handleWhiteSpaceWithCommaTokens() {
         assertEquals("ponies", text.subSequence(ranges.get(1).start, ranges.get(1).end));
 
         ranges = tokenizer.findTokenRanges(text, 5, text.length());
+        assertEquals(", ponies", text.substring(5));
         assertEquals(Collections.singletonList(new Range(7, 13)), ranges);
 
         ranges = tokenizer.findTokenRanges(text, 1, text.length());
@@ -85,4 +89,40 @@ public void handleLotsOfWhitespaceWithWhitespaceTokenizer() {
         List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
         assertEquals(Arrays.asList(new Range(0, 7), new Range(12, 19), new Range(23, 31)), ranges);
     }
+
+    @Test
+    public void allowsOneCharacterCandidateRangeMatches() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = "a";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Collections.singletonList(new Range(0,1)), ranges);
+    }
+
+    @Test
+    public void allowsOneCharacterCandidateRangeMatchesWithWhitespace() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = " a";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Collections.singletonList(new Range(1,2)), ranges);
+    }
+
+    @Test
+    public void doesntMatchWhitespaceAsCandidateTokenRange() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = "test, ";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Collections.singletonList(new Range(0, 5)), ranges);
+    }
+
+    @Test
+    public void matchesSingleLetterTokens() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = "t,r, a,,b";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Arrays.asList(new Range(0, 2), new Range(2,4), new Range(5,7), new Range(8,9)), ranges);
+    }
 }
diff --git a/library/src/test/java/com/tokenautocomplete/TagTokenizerTest.java b/library/src/test/java/com/tokenautocomplete/TagTokenizerTest.java
@@ -30,7 +30,7 @@ public void testBasicTagDetection() {
     @Test
     public void testSequentialTagDetection() {
         String test = "@@bears#tokens#";
-        assertEquals(Arrays.asList(new Range(1,7), new Range(7,14)),
+        assertEquals(Arrays.asList(new Range(1,7), new Range(7,14), new Range(14, 15)),
                 tokenizer.findTokenRanges(test, 0, test.length()));
     }
 
@@ -47,4 +47,28 @@ public void testMissingTokenContentInput() {
         assertEquals(Arrays.asList(new Range(0,6), new Range(25, 36)),
                 tokenizer.findTokenRanges(test, 0, test.length()));
     }
+
+    @Test
+    public void allowsOneCharacterCandidateRangeMatches() {
+        String text = "#";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Collections.singletonList(new Range(0,1)), ranges);
+    }
+
+    @Test
+    public void allowsOneCharacterCandidateRangeMatchesWithWhitespace() {
+        String text = " #";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Collections.singletonList(new Range(1,2)), ranges);
+    }
+
+    @Test
+    public void matchesSingleLetterTokens() {
+        String text = "#t#r #a##b";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Arrays.asList(new Range(0, 2), new Range(2,4), new Range(5,7), new Range(8,10)), ranges);
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -508,7 +508,7 @@ private Range getCurrentCandidateTokenRange() {`
`508`	`508`	`candidateStringStart = spanEnd;`
`509`	`509`	`}`
`510`	`510`	`int spanStart = editable.getSpanStart(span);`
`511`		`- if (candidateStringEnd < spanStart && cursorEndPosition <= spanEnd) {`
	`511`	`+ if (candidateStringEnd > spanStart && cursorEndPosition <= spanEnd) {`
`512`	`512`	`candidateStringEnd = spanStart;`
`513`	`513`	`}`
`514`	`514`	`}`