/** * Return the next monitored token. Test the token following the monitored token. If following is * another monitored token, save it for the next invocation of nextToken (like a single lookahead * token) and return it then. If following is unmonitored, nondiscarded (hidden) channel token, * add it to the monitored token. * * <p>Note: EOF must be a monitored Token. */ public Token nextToken() throws TokenStreamException { // handle an initial condition; don't want to get lookahead // token of this splitter until first call to nextToken if (LA(1) == null) { consumeFirst(); } // we always consume hidden tokens after monitored, thus, // upon entry LA(1) is a monitored token. CommonHiddenStreamToken monitored = LA(1); // point to hidden tokens found during last invocation monitored.setHiddenBefore(lastHiddenToken); lastHiddenToken = null; // Look for hidden tokens, hook them into list emanating // from the monitored tokens. consume(); CommonHiddenStreamToken p = monitored; // while hidden or discarded scarf tokens while (hideMask.member(LA(1).getType()) || discardMask.member(LA(1).getType())) { if (hideMask.member(LA(1).getType())) { // attach the hidden token to the monitored in a chain // link forwards p.setHiddenAfter(LA(1)); // link backwards if (p != monitored) { // hidden cannot point to monitored tokens LA(1).setHiddenBefore(p); } p = lastHiddenToken = LA(1); } consume(); } return monitored; }
/** * Make sure current lookahead symbol matches the given set Throw an exception upon mismatch, * which is catch by either the error handler or by the syntactic predicate. */ public void match(BitSet b) throws MismatchedTokenException, TokenStreamException { if (!b.member(LA(1))) throw new MismatchedTokenException(tokenNames, LT(1), b, false, getFilename()); else // mark token as consumed -- fetch next token deferred until LA/LT consume(); }
private void consumeFirst() throws TokenStreamException { consume(); // get first token of input stream // Handle situation where hidden or discarded tokens // appear first in input stream CommonHiddenStreamToken p = null; // while hidden or discarded scarf tokens while (hideMask.member(LA(1).getType()) || discardMask.member(LA(1).getType())) { if (hideMask.member(LA(1).getType())) { if (p == null) { p = LA(1); } else { p.setHiddenAfter(LA(1)); LA(1).setHiddenBefore(p); // double-link p = LA(1); } lastHiddenToken = p; if (firstHidden == null) { firstHidden = p; // record hidden token if first } } consume(); } }
/** Consume tokens until one matches the given token set */ public void consumeUntil(BitSet set) throws TokenStreamException { while (LA(1) != Token.EOF_TYPE && !set.member(LA(1))) { consume(); } }