/** * Return the next monitored token. Test the token following the monitored token. If following is * another monitored token, save it for the next invocation of nextToken (like a single lookahead * token) and return it then. If following is unmonitored, nondiscarded (hidden) channel token, * add it to the monitored token. * * <p>Note: EOF must be a monitored Token. */ public Token nextToken() throws TokenStreamException { // handle an initial condition; don't want to get lookahead // token of this splitter until first call to nextToken if (LA(1) == null) { consumeFirst(); } // we always consume hidden tokens after monitored, thus, // upon entry LA(1) is a monitored token. CommonHiddenStreamToken monitored = LA(1); // point to hidden tokens found during last invocation monitored.setHiddenBefore(lastHiddenToken); lastHiddenToken = null; // Look for hidden tokens, hook them into list emanating // from the monitored tokens. consume(); CommonHiddenStreamToken p = monitored; // while hidden or discarded scarf tokens while (hideMask.member(LA(1).getType()) || discardMask.member(LA(1).getType())) { if (hideMask.member(LA(1).getType())) { // attach the hidden token to the monitored in a chain // link forwards p.setHiddenAfter(LA(1)); // link backwards if (p != monitored) { // hidden cannot point to monitored tokens LA(1).setHiddenBefore(p); } p = lastHiddenToken = LA(1); } consume(); } return monitored; }
private void consumeFirst() throws TokenStreamException { consume(); // get first token of input stream // Handle situation where hidden or discarded tokens // appear first in input stream CommonHiddenStreamToken p = null; // while hidden or discarded scarf tokens while (hideMask.member(LA(1).getType()) || discardMask.member(LA(1).getType())) { if (hideMask.member(LA(1).getType())) { if (p == null) { p = LA(1); } else { p.setHiddenAfter(LA(1)); LA(1).setHiddenBefore(p); // double-link p = LA(1); } lastHiddenToken = p; if (firstHidden == null) { firstHidden = p; // record hidden token if first } } consume(); } }
/** Return a ptr to the hidden token appearing immediately before token t in the input stream. */ public CommonHiddenStreamToken getHiddenBefore(CommonHiddenStreamToken t) { return t.getHiddenBefore(); }
/** Return a ptr to the hidden token appearing immediately after token t in the input stream. */ public CommonHiddenStreamToken getHiddenAfter(CommonHiddenStreamToken t) { return t.getHiddenAfter(); }