diff --git a/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
index 1acf39ced..adcc0a598 100644
--- a/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
+++ b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
@@ -6,6 +6,13 @@
import org.commonmark.parser.SourceLines;
import org.commonmark.text.CharMatcher;
+/**
+ * A cursor over source lines for parsing inline content.
+ *
+ * Lines are scanned as one sequence of characters with a {@code \n} yielded between lines but
+ * not after the last line. As usual for Java strings, positions and the counts returned by the
+ * methods are in {@code char}s.
+ */
public class Scanner {
/**
@@ -13,7 +20,8 @@ public class Scanner {
* "previous" methods).
*
*
Note that we can use NULL to represent this because CommonMark does not allow those in the
- * input (we replace them in the beginning of parsing).
+ * input (we replace them in the beginning of parsing). It is not a searchable source character:
+ * {@link #find(char)} and {@link #findPrev(char, Position)} skip it instead of matching it.
*/
public static final char END = '\0';
@@ -40,10 +48,22 @@ public class Scanner {
}
}
+ /**
+ * Create a scanner positioned at the beginning of {@code lines}.
+ *
+ * @param lines the source lines to scan
+ * @return a new scanner
+ */
public static Scanner of(SourceLines lines) {
return new Scanner(lines.getLines(), 0, 0);
}
+ /**
+ * Return the character at the current position without advancing.
+ *
+ * @return the current character, {@code \n} at the end of a line that is not the last one, or
+ * {@link #END} at the end of input
+ */
public char peek() {
if (index < lineLength) {
return line.getContent().charAt(index);
@@ -57,6 +77,13 @@ public char peek() {
}
}
+ /**
+ * Return the code point at the current position without advancing. If the current character is
+ * a high surrogate followed by a low surrogate on the same line, the pair is combined.
+ *
+ * @return the current code point, {@code \n} at the end of a line that is not the last one, or
+ * {@link #END} at the end of input
+ */
public int peekCodePoint() {
if (index < lineLength) {
char c = line.getContent().charAt(index);
@@ -77,6 +104,13 @@ public int peekCodePoint() {
}
}
+ /**
+ * Return the code point before the current position without moving. If the previous character
+ * is a low surrogate preceded by a high surrogate on the same line, the pair is combined.
+ *
+ * @return the previous code point, {@code \n} at the beginning of a line that is not the first
+ * one, or {@link #END} at the beginning of input
+ */
public int peekPreviousCodePoint() {
if (index > 0) {
int prev = index - 1;
@@ -97,6 +131,10 @@ public int peekPreviousCodePoint() {
}
}
+ /**
+ * @return true if there is a character at the current position (including a {@code \n} between
+ * lines), false at the end of input
+ */
public boolean hasNext() {
if (index < lineLength) {
return true;
@@ -106,6 +144,7 @@ public boolean hasNext() {
}
}
+ /** Advance by one character, moving to the next line after the {@code \n} between lines. */
public void next() {
index++;
if (index > lineLength) {
@@ -119,6 +158,24 @@ public void next() {
}
}
+ // Move back by one character, reversing next(). From index zero the position moves to the
+ // previous line's length (the newline between lines). At the beginning of the input, including
+ // an empty input, the position is left unchanged.
+ private void prev() {
+ if (index > 0) {
+ index--;
+ } else if (lineIndex > 0) {
+ lineIndex--;
+ if (lineIndex < lines.size()) {
+ setLine(lines.get(lineIndex));
+ } else {
+ // Landed past the end (after next() at EOF); same representation as next().
+ setLine(SourceLine.of("", null));
+ }
+ index = lineLength;
+ }
+ }
+
/**
* Check if the specified char is next and advance the position.
*
@@ -156,6 +213,12 @@ public boolean next(String content) {
}
}
+ /**
+ * Advance past consecutive occurrences of {@code c}.
+ *
+ * @param c the character to match (including a newline)
+ * @return the number of characters matched, 0 if the current character is not {@code c}
+ */
public int matchMultiple(char c) {
int count = 0;
while (peek() == c) {
@@ -165,6 +228,13 @@ public int matchMultiple(char c) {
return count;
}
+ /**
+ * Advance past consecutive characters matched by {@code matcher}.
+ *
+ * @param matcher matcher applied to each character, including {@code \n} between lines and
+ * {@link #END} at the end of input
+ * @return the number of characters matched
+ */
public int match(CharMatcher matcher) {
int count = 0;
while (matcher.matches(peek())) {
@@ -174,6 +244,12 @@ public int match(CharMatcher matcher) {
return count;
}
+ /**
+ * Advance past consecutive whitespace characters: space, tab, newline, line tabulation, form
+ * feed and carriage return.
+ *
+ * @return the number of characters skipped
+ */
public int whitespace() {
int count = 0;
while (true) {
@@ -193,6 +269,15 @@ public int whitespace() {
}
}
+ /**
+ * Find the next occurrence of {@code c} at or after the current position.
+ *
+ *
If found, the scanner is left at the match. Otherwise it is left at the end of input and
+ * {@code -1} is returned. {@link #END} is never matched.
+ *
+ * @param c character to find (including a newline)
+ * @return the number of characters moved forward, or {@code -1} if not found
+ */
public int find(char c) {
int count = 0;
while (true) {
@@ -207,6 +292,15 @@ public int find(char c) {
}
}
+ /**
+ * Find the next character matched by {@code matcher} at or after the current position.
+ *
+ *
See {@link #find(char)} for where the scanner is left. {@link #END} is not passed to
+ * {@code matcher}.
+ *
+ * @param matcher matcher applied to each character
+ * @return the number of characters moved forward, or {@code -1} if not found
+ */
public int find(CharMatcher matcher) {
int count = 0;
while (true) {
@@ -221,12 +315,92 @@ public int find(CharMatcher matcher) {
}
}
+ /**
+ * Find the nearest occurrence of {@code c} at or before the current position and at or after
+ * {@code bound}.
+ *
+ *
The current character is included, matching {@link #find(char)}, so an immediate match
+ * returns {@code 0}. From the end of input a match on the character immediately before the end
+ * returns {@code 1}. A newline between lines can be matched. {@link #END} is skipped and never
+ * matches.
+ *
+ *
{@code bound} is inclusive: the character there is examined, and the search never moves
+ * before it. If {@code bound} is the current position, only that character is checked. If there
+ * is no match, the scanner is left at {@code bound} and {@code -1} is returned. On an empty
+ * input the search returns {@code -1}; the position at the beginning of that input is a valid
+ * bound.
+ *
+ *
The bound is checked before the scanner moves. It must have valid coordinates for this
+ * scanner and must not be after the current position. As with {@link #setPosition(Position)},
+ * positions are expected to come from the same scanner.
+ *
+ * @param c character to find (including a newline)
+ * @param bound inclusive lower bound
+ * @return the number of characters moved backward, or {@code -1} if not found
+ * @throws IllegalArgumentException if {@code bound} is after the current position or outside
+ * the input
+ */
+ public int findPrev(char c, Position bound) {
+ checkLowerBound(bound);
+ return findPrev(c, null, bound.lineIndex, bound.index);
+ }
+
+ /**
+ * Find the nearest character matched by {@code matcher} at or before the current position and
+ * at or after {@code bound}.
+ *
+ *
See {@link #findPrev(char, Position)} for the bound, {@link #END}, and a search with no
+ * match. {@link #END} is not passed to {@code matcher}.
+ *
+ * @param matcher matcher applied to each character
+ * @param bound inclusive lower bound
+ * @return the number of characters moved backward, or {@code -1} if not found
+ * @throws IllegalArgumentException if {@code bound} is after the current position or outside
+ * the input
+ */
+ public int findPrev(CharMatcher matcher, Position bound) {
+ checkLowerBound(bound);
+ return findPrev(END, matcher, bound.lineIndex, bound.index);
+ }
+
+ // matcher == null matches c exactly, so the char overload allocates nothing per character.
+ private int findPrev(char c, CharMatcher matcher, int boundLine, int boundIndex) {
+ int count = 0;
+ while (true) {
+ char cur = peek();
+ if (cur != END && (matcher != null ? matcher.matches(cur) : cur == c)) {
+ return count;
+ }
+ if (lineIndex == boundLine && index == boundIndex) {
+ return -1;
+ }
+ int lineBefore = lineIndex;
+ int indexBefore = index;
+ prev();
+ // Beginning of input: prev() is a no-op, so stop instead of spinning.
+ if (lineIndex == lineBefore && index == indexBefore) {
+ return -1;
+ }
+ count++;
+ }
+ }
+
// Don't expose the int index, because it would be good if we could switch input to a
// List of lines later instead of one contiguous String.
+ /**
+ * @return the current position, for use with {@link #setPosition(Position)}, {@link
+ * #getSource(Position, Position)} and {@link #findPrev(char, Position)}
+ */
public Position position() {
return new Position(lineIndex, index);
}
+ /**
+ * Move to a position previously returned by {@link #position()} on this scanner.
+ *
+ * @param position the position to move to
+ * @throws IllegalArgumentException if {@code position} is outside the input
+ */
public void setPosition(Position position) {
checkPosition(position.lineIndex, position.index);
this.lineIndex = position.lineIndex;
@@ -236,6 +410,14 @@ public void setPosition(Position position) {
// For cases where the caller appends the result to a StringBuilder, we could offer another
// method to avoid some unnecessary copying.
+ /**
+ * Get the source between two positions of this scanner, with {@code begin} inclusive and {@code
+ * end} exclusive. A {@code \n} between lines is represented by splitting into separate lines.
+ *
+ * @param begin the start position
+ * @param end the end position, not before {@code begin}
+ * @return the source lines, including their source spans if available
+ */
public SourceLines getSource(Position begin, Position end) {
if (begin.lineIndex == end.lineIndex) {
// Shortcut for common case of text from a single line
@@ -280,4 +462,35 @@ private void checkPosition(int lineIndex, int index) {
"Index " + index + " out of range, line length: " + line.getContent().length());
}
}
+
+ private void checkLowerBound(Position position) {
+ // Empty input has no lines; its only position is the canonical (0, 0) cursor.
+ if (lines.isEmpty()) {
+ if (position.lineIndex != 0) {
+ throw new IllegalArgumentException(
+ "Line index "
+ + position.lineIndex
+ + " out of range, number of lines: "
+ + lines.size());
+ }
+ if (position.index != 0) {
+ throw new IllegalArgumentException(
+ "Index " + position.index + " out of range, line length: 0");
+ }
+ } else {
+ checkPosition(position.lineIndex, position.index);
+ }
+ if (position.lineIndex > lineIndex
+ || (position.lineIndex == lineIndex && position.index > index)) {
+ throw new IllegalArgumentException(
+ "Bound line "
+ + position.lineIndex
+ + " index "
+ + position.index
+ + " is after the current position line "
+ + lineIndex
+ + " index "
+ + index);
+ }
+ }
}
diff --git a/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java b/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java
index 167f74f25..8907adcd1 100644
--- a/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java
+++ b/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java
@@ -9,6 +9,8 @@
import org.commonmark.parser.beta.InlineParserState;
import org.commonmark.parser.beta.ParsedInline;
import org.commonmark.test.Nodes;
+import org.commonmark.text.AsciiMatcher;
+import org.commonmark.text.CharMatcher;
import org.junit.jupiter.api.Test;
class InlineContentParserTest {
@@ -55,6 +57,47 @@ void bangInlineContentParser() {
assertThat(((Text) image.getNext().getNext().getNext()).getLiteral()).isEqualTo("notimage");
}
+ @Test
+ void backwardSearchInlineContentParser() {
+ var parser =
+ Parser.builder()
+ .customInlineContentParserFactory(new DestinationInlineParser.Factory())
+ .build();
+
+ var doc = parser.parse("img(a-link) more text");
+ var inline = Nodes.find(doc, DestinationInline.class);
+ assertThat(((Text) inline.getPrevious()).getLiteral()).isEqualTo("img");
+ assertThat(inline.getDestination()).isEqualTo("a-link");
+ assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text");
+
+ doc = parser.parse("img(a-link-)more-text) more text");
+ inline = Nodes.find(doc, DestinationInline.class);
+ assertThat(((Text) inline.getPrevious()).getLiteral()).isEqualTo("img");
+ assertThat(inline.getDestination()).isEqualTo("a-link-)more-text");
+ assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text");
+
+ doc = parser.parse("img(a-link-)more-text)");
+ inline = Nodes.find(doc, DestinationInline.class);
+ assertThat(inline.getDestination()).isEqualTo("a-link-)more-text");
+ assertThat(inline.getNext()).isNull();
+
+ doc = parser.parse("img() more text");
+ inline = Nodes.find(doc, DestinationInline.class);
+ assertThat(inline.getDestination()).isEqualTo("");
+ assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text");
+
+ doc = parser.parse("img(not-a-link more text");
+ assertThat(Nodes.tryFind(doc, DestinationInline.class)).isNull();
+ assertThat(((Text) doc.getFirstChild().getFirstChild()).getLiteral())
+ .isEqualTo("img(not-a-link more text");
+
+ // The ')' before the saved opening position is outside the inclusive bound.
+ doc = parser.parse("before)img(not-a-link more text");
+ assertThat(Nodes.tryFind(doc, DestinationInline.class)).isNull();
+ assertThat(((Text) doc.getFirstChild().getFirstChild()).getLiteral())
+ .isEqualTo("before)img(not-a-link more text");
+ }
+
private static class DollarInline extends CustomNode {
private final String literal;
private final int index;
@@ -105,6 +148,55 @@ public InlineContentParser create() {
}
}
+ /**
+ * Custom image-like syntax {@code (destination)} whose destination may itself contain {@code
+ * )}. The closer is the last {@code )} before whitespace or the end of input, and it must not
+ * be before the opening parenthesis.
+ */
+ private static class DestinationInline extends CustomNode {
+ private final String destination;
+
+ public DestinationInline(String destination) {
+ this.destination = destination;
+ }
+
+ public String getDestination() {
+ return destination;
+ }
+ }
+
+ private static class DestinationInlineParser implements InlineContentParser {
+
+ private static final CharMatcher WHITESPACE =
+ AsciiMatcher.builder().anyOf(" \t\n\u000B\f\r").build();
+
+ @Override
+ public ParsedInline tryParse(InlineParserState inlineParserState) {
+ var scanner = inlineParserState.scanner();
+ scanner.next();
+ var contentStart = scanner.position();
+ scanner.find(WHITESPACE);
+ if (scanner.findPrev(')', contentStart) == -1) {
+ return ParsedInline.none();
+ }
+ var destination = scanner.getSource(contentStart, scanner.position()).getContent();
+ scanner.next();
+ return ParsedInline.of(new DestinationInline(destination), scanner.position());
+ }
+
+ static class Factory implements InlineContentParserFactory {
+ @Override
+ public Set getTriggerCharacters() {
+ return Set.of('(');
+ }
+
+ @Override
+ public InlineContentParser create() {
+ return new DestinationInlineParser();
+ }
+ }
+ }
+
private static class BangInline extends CustomNode {}
private static class BangInlineParser implements InlineContentParser {
diff --git a/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
index 437812866..5ee743972 100644
--- a/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
+++ b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
@@ -1,11 +1,13 @@
package org.commonmark.parser.beta;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.List;
import org.commonmark.node.SourceSpan;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.SourceLines;
+import org.commonmark.text.CharMatcher;
import org.junit.jupiter.api.Test;
class ScannerTest {
@@ -144,6 +146,453 @@ private void assertSourceLines(
assertThat(sourceLines.getSourceSpans()).isEqualTo(List.of(expectedSourceSpans));
}
+ @Test
+ void testFindPrevSingleSteps() {
+ Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("abc", null)));
+ Position start = scanner.position();
+ scanner.next();
+ Position b = scanner.position();
+ scanner.next();
+ Position c = scanner.position();
+ scanner.next();
+ Position eof = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ assertThat(scanner.hasNext()).isFalse();
+
+ assertThat(scanner.findPrev('c', start)).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(c);
+ assertThat(scanner.findPrev('b', start)).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(b);
+ assertThat(scanner.findPrev('a', start)).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(start);
+
+ // A miss at the beginning stays there.
+ assertThat(scanner.findPrev('z', start)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.peek()).isEqualTo('a');
+
+ // Stepping back from the end and forward again returns to the end.
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('c', start)).isEqualTo(1);
+ scanner.next();
+ assertThat(scanner.position()).isEqualTo(eof);
+ assertThat(scanner.peek()).isEqualTo('\0');
+
+ Scanner emptyLine = Scanner.of(SourceLines.of(SourceLine.of("", null)));
+ Position emptyLinePos = emptyLine.position();
+ assertThat(emptyLine.peek()).isEqualTo('\0');
+ assertThat(emptyLine.hasNext()).isFalse();
+ assertThat(emptyLine.findPrev(ch -> true, emptyLinePos)).isEqualTo(-1);
+ assertThat(emptyLine.position()).isEqualTo(emptyLinePos);
+ assertThat(emptyLine.peek()).isEqualTo('\0');
+ }
+
+ @Test
+ void testFindPrevAcrossLines() {
+ Scanner scanner =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("ab", null), SourceLine.of("c", null))));
+ Position start = scanner.position();
+ scanner.next();
+ scanner.next();
+ Position newline = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('\n');
+ scanner.next();
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ assertThat(scanner.hasNext()).isFalse();
+
+ assertThat(scanner.findPrev('c', start)).isEqualTo(1);
+ assertThat(scanner.findPrev('\n', start)).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(newline);
+ assertThat(scanner.findPrev('b', start)).isEqualTo(1);
+ assertThat(scanner.peek()).isEqualTo('b');
+ scanner.next();
+ assertThat(scanner.position()).isEqualTo(newline);
+ assertThat(scanner.findPrev('a', start)).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.findPrev('z', start)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(start);
+
+ // Unequal lengths: cached line content/length must follow the line we step onto.
+ Scanner uneven =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("a", null), SourceLine.of("bbb", null))));
+ Position unevenStart = uneven.position();
+ uneven.next();
+ uneven.next();
+ assertThat(uneven.peek()).isEqualTo('b');
+ assertThat(uneven.findPrev('\n', unevenStart)).isEqualTo(1);
+ assertThat(uneven.findPrev('a', unevenStart)).isEqualTo(1);
+ uneven.next();
+ assertThat(uneven.peek()).isEqualTo('\n');
+ uneven.next();
+ uneven.next();
+ uneven.next();
+ uneven.next();
+ assertThat(uneven.peek()).isEqualTo('\0');
+ assertThat(uneven.findPrev('b', unevenStart)).isEqualTo(1);
+ assertThat(uneven.peek()).isEqualTo('b');
+
+ // Consecutive empty lines, including an empty final line: no newline after the last line.
+ Scanner blanks =
+ Scanner.of(
+ SourceLines.of(
+ List.of(
+ SourceLine.of("", null),
+ SourceLine.of("", null),
+ SourceLine.of("", null))));
+ Position blanksStart = blanks.position();
+ assertThat(blanks.peek()).isEqualTo('\n');
+ blanks.next();
+ Position secondNewline = blanks.position();
+ assertThat(blanks.peek()).isEqualTo('\n');
+ blanks.next();
+ Position blanksEof = blanks.position();
+ assertThat(blanks.peek()).isEqualTo('\0');
+ assertThat(blanks.hasNext()).isFalse();
+ assertThat(blanks.findPrev('\n', blanksStart)).isEqualTo(1);
+ assertThat(blanks.position()).isEqualTo(secondNewline);
+ blanks.setPosition(blanksEof);
+ assertThat(blanks.findPrev('x', blanksStart)).isEqualTo(-1);
+ assertThat(blanks.position()).isEqualTo(blanksStart);
+ assertThat(blanks.peek()).isEqualTo('\n');
+ blanks.next();
+ blanks.next();
+ assertThat(blanks.peek()).isEqualTo('\0');
+
+ // Empty final line steps back onto the preceding newline, not a made-up terminal one.
+ Scanner trailingBlank =
+ Scanner.of(
+ SourceLines.of(List.of(SourceLine.of("a", null), SourceLine.of("", null))));
+ Position trailingStart = trailingBlank.position();
+ trailingBlank.next();
+ Position precedingNewline = trailingBlank.position();
+ assertThat(trailingBlank.peek()).isEqualTo('\n');
+ trailingBlank.next();
+ assertThat(trailingBlank.peek()).isEqualTo('\0');
+ assertThat(trailingBlank.hasNext()).isFalse();
+ assertThat(trailingBlank.findPrev(ch -> true, trailingStart)).isEqualTo(1);
+ assertThat(trailingBlank.position()).isEqualTo(precedingNewline);
+ assertThat(trailingBlank.peek()).isEqualTo('\n');
+ trailingBlank.next();
+ assertThat(trailingBlank.peek()).isEqualTo('\0');
+ assertThat(trailingBlank.findPrev('a', trailingStart)).isEqualTo(2);
+ assertThat(trailingBlank.position()).isEqualTo(trailingStart);
+ assertThat(trailingBlank.peek()).isEqualTo('a');
+ }
+
+ @Test
+ void testFindPrev() {
+ Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("abXcdX", null)));
+ Position start = scanner.position();
+ assertThat(scanner.findPrev('a', start)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.findPrev(c -> c == 'a', start)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(start);
+
+ scanner.next();
+ scanner.next();
+ Position firstX = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('X');
+ assertThat(scanner.findPrev('X', start)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(firstX);
+
+ scanner.next();
+ scanner.next();
+ // Between the two matches: the nearer (earlier) X, not one after the cursor.
+ assertThat(scanner.peek()).isEqualTo('d');
+ assertThat(scanner.findPrev('X', start)).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(firstX);
+
+ scanner.next();
+ scanner.next();
+ scanner.next();
+ Position secondX = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('X');
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ // Match immediately before EOF counts the step off the sentinel.
+ assertThat(scanner.findPrev('X', start)).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(secondX);
+ assertThat(scanner.peek()).isEqualTo('X');
+
+ scanner.next();
+ assertThat(scanner.findPrev(c -> c == 'X', start)).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(secondX);
+
+ assertThat(scanner.findPrev('z', start)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.peek()).isEqualTo('a');
+
+ scanner.setPosition(secondX);
+ scanner.next();
+ assertThat(scanner.findPrev(Scanner.END, start)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(start);
+ scanner.setPosition(secondX);
+ scanner.next();
+ assertThat(scanner.findPrev(c -> c == Scanner.END, start)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(start);
+
+ CharMatcher any = c -> true;
+ scanner.setPosition(secondX);
+ assertThat(scanner.findPrev(any, start)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(secondX);
+ scanner.next();
+ assertThat(scanner.findPrev(any, start)).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(secondX);
+ assertThat(scanner.peek()).isEqualTo('X');
+
+ Scanner lines =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("ab", null), SourceLine.of("c", null))));
+ Position linesStart = lines.position();
+ lines.next();
+ lines.next();
+ Position newline = lines.position();
+ lines.next();
+ lines.next();
+ assertThat(lines.peek()).isEqualTo('\0');
+ assertThat(lines.findPrev('\n', linesStart)).isEqualTo(2);
+ assertThat(lines.position()).isEqualTo(newline);
+ assertThat(lines.peek()).isEqualTo('\n');
+ assertThat(lines.findPrev(c -> c == '\n', linesStart)).isEqualTo(0);
+ assertThat(lines.position()).isEqualTo(newline);
+
+ Scanner empty = Scanner.of(SourceLines.empty());
+ Position emptyPos = empty.position();
+ assertThat(empty.findPrev('a', emptyPos)).isEqualTo(-1);
+ assertThat(empty.findPrev(Scanner.END, emptyPos)).isEqualTo(-1);
+ assertThat(empty.findPrev(any, emptyPos)).isEqualTo(-1);
+ assertThat(empty.findPrev(c -> c == Scanner.END, emptyPos)).isEqualTo(-1);
+ assertThat(empty.position()).isEqualTo(emptyPos);
+ assertThat(empty.peek()).isEqualTo('\0');
+
+ Scanner text = Scanner.of(SourceLines.of(SourceLine.of("a", null)));
+ Position textStart = text.position();
+ text.next();
+ assertThat(text.findPrev(any, textStart)).isEqualTo(1);
+ assertThat(text.peek()).isEqualTo('a');
+ }
+
+ @Test
+ void testFindPrevBounds() {
+ Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("aXbYc", null)));
+ Position start = scanner.position();
+ scanner.next();
+ Position x = scanner.position();
+ scanner.next();
+ scanner.next();
+ Position y = scanner.position();
+ scanner.next();
+ scanner.next();
+ Position eof = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('\0');
+
+ // Inclusive bound on the same line, including a match sitting on the bound.
+ assertThat(scanner.findPrev('Y', x)).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(y);
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('Y', y)).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(y);
+ assertThat(scanner.peek()).isEqualTo('Y');
+
+ // A match before the bound is excluded; a miss stays on the bound.
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('X', y)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(y);
+ assertThat(scanner.peek()).isEqualTo('Y');
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('z', y)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(y);
+
+ // Start and bound are the same position: only that character is checked.
+ assertThat(scanner.findPrev('Y', y)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(y);
+ assertThat(scanner.findPrev('X', y)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(y);
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('c', eof)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(eof);
+ assertThat(scanner.peek()).isEqualTo('\0');
+
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev(c -> c == 'Y', x)).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(y);
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev(c -> c == 'X', y)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(y);
+ assertThat(scanner.findPrev(c -> c == 'Y', y)).isEqualTo(0);
+
+ // Across lines the bound is still inclusive and stops the search.
+ Scanner lines =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("aXb", null), SourceLine.of("cYd", null))));
+ Position lineStart = lines.position();
+ lines.next();
+ Position lineX = lines.position();
+ lines.next();
+ lines.next();
+ Position newline = lines.position();
+ assertThat(lines.peek()).isEqualTo('\n');
+ lines.next();
+ Position secondStart = lines.position();
+ lines.next();
+ Position lineY = lines.position();
+ lines.next();
+ lines.next();
+ Position linesEof = lines.position();
+
+ assertThat(lines.findPrev('Y', newline)).isEqualTo(2);
+ assertThat(lines.position()).isEqualTo(lineY);
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev('\n', lineStart)).isEqualTo(4);
+ assertThat(lines.position()).isEqualTo(newline);
+ assertThat(lines.peek()).isEqualTo('\n');
+
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev('X', newline)).isEqualTo(-1);
+ assertThat(lines.position()).isEqualTo(newline);
+ assertThat(lines.peek()).isEqualTo('\n');
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev('a', secondStart)).isEqualTo(-1);
+ assertThat(lines.position()).isEqualTo(secondStart);
+
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev('a', lineStart)).isEqualTo(7);
+ assertThat(lines.position()).isEqualTo(lineStart);
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev(c -> c == '\n', newline)).isEqualTo(4);
+ assertThat(lines.position()).isEqualTo(newline);
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev(c -> c == 'X', secondStart)).isEqualTo(-1);
+ assertThat(lines.position()).isEqualTo(secondStart);
+
+ // Future and out-of-range bounds throw before the cursor moves.
+ lines.setPosition(lineX);
+ assertThat(lines.peek()).isEqualTo('X');
+ assertThatThrownBy(() -> lines.findPrev('a', lineY))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev(c -> true, linesEof))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev('a', new Position(-1, 0)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev('a', new Position(0, -1)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev('a', new Position(0, 4)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev('a', new Position(2, 0)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThat(lines.position()).isEqualTo(lineX);
+ assertThat(lines.peek()).isEqualTo('X');
+
+ Scanner empty = Scanner.of(SourceLines.empty());
+ Position origin = empty.position();
+ assertThat(empty.findPrev('x', origin)).isEqualTo(-1);
+ assertThat(empty.findPrev(c -> true, origin)).isEqualTo(-1);
+ assertThat(empty.findPrev(Scanner.END, origin)).isEqualTo(-1);
+ assertThat(empty.position()).isEqualTo(origin);
+ assertThatThrownBy(() -> empty.findPrev('x', new Position(0, 1)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> empty.findPrev(c -> true, new Position(1, 0)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThat(empty.position()).isEqualTo(origin);
+ assertThat(empty.peek()).isEqualTo('\0');
+
+ // Same-line bound equal to the start still accepts the character there.
+ scanner.setPosition(start);
+ assertThat(scanner.findPrev('a', start)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(start);
+ }
+
+ @Test
+ void testCodeUnitMovement() {
+ Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("A\uD83D\uDE0AZ", null)));
+ Position start = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('A');
+ assertThat(scanner.peekCodePoint()).isEqualTo('A');
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\0');
+
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\uD83D');
+ assertThat(scanner.peekCodePoint()).isEqualTo(128522);
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A');
+
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\uD83D');
+
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('Z');
+ // Previous code point reassembles the surrogate pair behind this code unit.
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo(128522);
+ scanner.next();
+ assertThat(scanner.hasNext()).isFalse();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ assertThat(scanner.peekCodePoint()).isEqualTo('\0');
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('Z');
+
+ assertThat(scanner.findPrev('Z', start)).isEqualTo(1);
+ assertThat(scanner.peek()).isEqualTo('Z');
+ assertThat(scanner.findPrev('\uDE0A', start)).isEqualTo(1);
+ assertThat(scanner.peek()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\uD83D');
+ assertThat(scanner.findPrev('\uD83D', start)).isEqualTo(1);
+ assertThat(scanner.peek()).isEqualTo('\uD83D');
+ assertThat(scanner.peekCodePoint()).isEqualTo(128522);
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A');
+
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\uDE0A');
+ assertThat(scanner.findPrev('\uD83D', start)).isEqualTo(1);
+ assertThat(scanner.peek()).isEqualTo('\uD83D');
+ assertThat(scanner.peekCodePoint()).isEqualTo(128522);
+
+ while (scanner.hasNext()) {
+ scanner.next();
+ }
+ assertThat(scanner.findPrev('\uDE0A', start)).isEqualTo(2);
+ assertThat(scanner.peek()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A');
+ while (scanner.hasNext()) {
+ scanner.next();
+ }
+ assertThat(scanner.findPrev('\uD83D', start)).isEqualTo(3);
+ assertThat(scanner.peek()).isEqualTo('\uD83D');
+ assertThat(scanner.peekCodePoint()).isEqualTo(128522);
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A');
+ }
+
+ @Test
+ void testFindPrevSource() {
+ Scanner scanner =
+ new Scanner(
+ List.of(
+ SourceLine.of("ab", SourceSpan.of(10, 3, 13, 2)),
+ SourceLine.of("cde", SourceSpan.of(11, 4, 20, 3))),
+ 0,
+ 0);
+ Position start = scanner.position();
+ while (scanner.hasNext()) {
+ scanner.next();
+ }
+ Position end = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ assertThat(scanner.findPrev('b', start)).isEqualTo(5);
+ assertThat(scanner.peek()).isEqualTo('b');
+ assertSourceLines(
+ scanner.getSource(scanner.position(), end),
+ "b\ncde",
+ SourceSpan.of(10, 4, 14, 1),
+ SourceSpan.of(11, 4, 20, 3));
+ }
+
@Test
void nextString() {
Scanner scanner =