public void testWriteLineBreak() throws Exception {
StringWriter string = new StringWriter();
TokenFilter writer = new TokenFilter(Collections.EMPTY_MAP);
writer.writeLine(string, "");
assertEquals(System.getProperty("line.separator"), string.toString());
}
/* This is for debugging. */
public static void main(String[] args) throws Exception {
TokenFilter st = new PorterStemmer(null, null);
String str;
for (int i = 0; i < args.length; ++i) {
str = st.apply(args[i]);
if (str != null) System.out.println(str);
}
}
public TokenStream ProcessTokenonFilterTypes(TokenStream ts) {
TokenFilter tfforauth = null;
TokenFilterFactory tff = TokenFilterFactory.getInstance();
TokenFilterAccents tfa = (TokenFilterAccents) tff.getFilterByType(TokenFilterType.ACCENT, ts);
tfforauth = tfa.accentsProcessing(ts);
if (tfforauth != null) ts = tfforauth.getStream();
TokenFilter.AnalyzerType = 2;
return ts;
}
public void testWriteLineReplaceNoTokens() throws Exception {
StringWriter string = new StringWriter();
Map toks = new HashMap();
toks.put("a", "b");
TokenFilter writer = new TokenFilter(toks);
writer.writeLine(string, "xxx");
assertEquals("xxx" + System.getProperty("line.separator"), string.toString());
}
public void testWriteLineReplaceLongTokensShorReplacements() throws Exception {
StringWriter string = new StringWriter();
Map toks = new HashMap();
toks.put("@aaaa@", "b");
TokenFilter writer = new TokenFilter(toks);
writer.writeLine(string, "@aaaa@ @aaaa@ sss");
assertEquals("b b sss" + System.getProperty("line.separator"), string.toString());
}
/**
* Returns a map of public static final integer fields in the specified classes, to their String
* representations. An optional filter can be specified to only include specific fields. The
* target map may be null, in which case a new map is allocated and returned.
*
* <p>This method is useful when debugging to quickly identify values returned from the AL/GL/CL
* APIs.
*
* @param filter the filter to use (optional)
* @param target the target map (optional)
* @param tokenClasses the classes to get tokens from
* @return the token map
*/
public static Map<Integer, String> getClassTokens(
final TokenFilter filter, Map<Integer, String> target, final Iterable<Class> tokenClasses) {
if (target == null) target = new HashMap<Integer, String>();
final int TOKEN_MODIFIERS = Modifier.PUBLIC | Modifier.STATIC | Modifier.FINAL;
for (final Class tokenClass : tokenClasses) {
for (final Field field : tokenClass.getDeclaredFields()) {
// Get only <public static final int> fields.
if ((field.getModifiers() & TOKEN_MODIFIERS) == TOKEN_MODIFIERS
&& field.getType() == int.class) {
try {
final int value = field.getInt(null);
if (filter != null && !filter.accept(field, value)) continue;
if (target.containsKey(value)) // Print colliding tokens in their hex representation.
target.put(value, toHexString(value));
else target.put(value, field.getName());
} catch (IllegalAccessException e) {
// Ignore
}
}
}
}
return target;
}
/** {@inheritDoc} */
@Override
public void reset() throws IOException {
super.reset();
hyphenated.setLength(0);
savedState = null;
}
@Override
public void reset() throws IOException {
super.reset();
this.numSeen = 0;
}
@Override
public void reset() throws IOException {
super.reset();
currentPrefix = null;
}
@Override
public void end() throws IOException {
super.end();
posIncAtt.setPositionIncrement(pendingPosInc + posIncAtt.getPositionIncrement());
}
@Override
public void reset() throws IOException {
super.reset();
pendingPosInc = 0;
}
/** Method called when a new potentially included context is found. */
protected final JsonToken _nextTokenWithBuffering(final TokenFilterContext buffRoot)
throws IOException {
main_loop:
while (true) {
JsonToken t = delegate.nextToken();
if (t == null) { // is this even legal?
return t;
}
TokenFilter f;
// One simplification here: we know for a fact that the item filter is
// neither null nor 'include all', for most cases; the only exception
// being FIELD_NAME handling
switch (t.id()) {
case ID_START_ARRAY:
f = _headContext.checkValue(_itemFilter);
if (f == null) {
delegate.skipChildren();
continue main_loop;
}
if (f != TokenFilter.INCLUDE_ALL) {
f = f.filterStartArray();
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildArrayContext(f, true);
return _nextBuffered(buffRoot);
}
_headContext = _headContext.createChildArrayContext(f, false);
continue main_loop;
case ID_START_OBJECT:
f = _itemFilter;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildObjectContext(f, true);
return t;
}
if (f == null) { // does this occur?
delegate.skipChildren();
continue main_loop;
}
// Otherwise still iffy, need to check
f = _headContext.checkValue(f);
if (f == null) {
delegate.skipChildren();
continue main_loop;
}
if (f != TokenFilter.INCLUDE_ALL) {
f = f.filterStartObject();
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildObjectContext(f, true);
return _nextBuffered(buffRoot);
}
_headContext = _headContext.createChildObjectContext(f, false);
continue main_loop;
case ID_END_ARRAY:
case ID_END_OBJECT:
{
// Unlike with other loops, here we know that content was NOT
// included (won't get this far otherwise)
f = _headContext.getFilter();
if ((f != null) && (f != TokenFilter.INCLUDE_ALL)) {
f.filterFinishArray();
}
boolean gotEnd = (_headContext == buffRoot);
boolean returnEnd = gotEnd && _headContext.isStartHandled();
_headContext = _headContext.getParent();
_itemFilter = _headContext.getFilter();
if (returnEnd) {
return t;
}
// Hmmh. Do we need both checks, or should above suffice?
if (gotEnd || (_headContext == buffRoot)) {
return null;
}
}
continue main_loop;
case ID_FIELD_NAME:
{
final String name = delegate.getCurrentName();
f = _headContext.setFieldName(name);
if (f == TokenFilter.INCLUDE_ALL) {
_itemFilter = f;
return _nextBuffered(buffRoot);
}
if (f == null) { // filter out the value
delegate.nextToken();
delegate.skipChildren();
continue main_loop;
}
f = f.includeProperty(name);
if (f == null) { // filter out the value
delegate.nextToken();
delegate.skipChildren();
continue main_loop;
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
return _nextBuffered(buffRoot);
}
}
continue main_loop;
default: // scalar value
f = _itemFilter;
if (f == TokenFilter.INCLUDE_ALL) {
return _nextBuffered(buffRoot);
}
if (f != null) {
f = _headContext.checkValue(f);
if ((f == TokenFilter.INCLUDE_ALL) || ((f != null) && f.includeValue(delegate))) {
return _nextBuffered(buffRoot);
}
}
// Otherwise not included (leaves must be explicitly included)
continue main_loop;
}
}
}
/**
* Offlined handling for cases where there was no buffered token to return, and the token read
* next could not be returned as-is, at least not yet, but where we have not yet established that
* buffering is needed.
*/
protected final JsonToken _nextToken2() throws IOException {
main_loop:
while (true) {
JsonToken t = delegate.nextToken();
if (t == null) { // is this even legal?
return (_currToken = t);
}
TokenFilter f;
switch (t.id()) {
case ID_START_ARRAY:
f = _itemFilter;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildArrayContext(f, true);
return (_currToken = t);
}
if (f == null) { // does this occur?
delegate.skipChildren();
continue main_loop;
}
// Otherwise still iffy, need to check
f = _headContext.checkValue(f);
if (f == null) {
delegate.skipChildren();
continue main_loop;
}
if (f != TokenFilter.INCLUDE_ALL) {
f = f.filterStartArray();
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildArrayContext(f, true);
return (_currToken = t);
}
_headContext = _headContext.createChildArrayContext(f, false);
// but if we didn't figure it out yet, need to buffer possible events
if (_includePath) {
t = _nextTokenWithBuffering(_headContext);
if (t != null) {
_currToken = t;
return t;
}
}
continue main_loop;
case ID_START_OBJECT:
f = _itemFilter;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildObjectContext(f, true);
return (_currToken = t);
}
if (f == null) { // does this occur?
delegate.skipChildren();
continue main_loop;
}
// Otherwise still iffy, need to check
f = _headContext.checkValue(f);
if (f == null) {
delegate.skipChildren();
continue main_loop;
}
if (f != TokenFilter.INCLUDE_ALL) {
f = f.filterStartObject();
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildObjectContext(f, true);
return (_currToken = t);
}
_headContext = _headContext.createChildObjectContext(f, false);
if (_includePath) {
t = _nextTokenWithBuffering(_headContext);
if (t != null) {
_currToken = t;
return t;
}
}
continue main_loop;
case ID_END_ARRAY:
case ID_END_OBJECT:
{
boolean returnEnd = _headContext.isStartHandled();
f = _headContext.getFilter();
if ((f != null) && (f != TokenFilter.INCLUDE_ALL)) {
f.filterFinishArray();
}
_headContext = _headContext.getParent();
_itemFilter = _headContext.getFilter();
if (returnEnd) {
return (_currToken = t);
}
}
continue main_loop;
case ID_FIELD_NAME:
{
final String name = delegate.getCurrentName();
f = _headContext.setFieldName(name);
if (f == TokenFilter.INCLUDE_ALL) {
_itemFilter = f;
return (_currToken = t);
}
if (f == null) { // filter out the value
delegate.nextToken();
delegate.skipChildren();
continue main_loop;
}
f = f.includeProperty(name);
if (f == null) { // filter out the value
delegate.nextToken();
delegate.skipChildren();
continue main_loop;
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
if (_includePath) {
return (_currToken = t);
}
// if (_includeImmediateParent) { ...
continue main_loop;
}
if (_includePath) {
t = _nextTokenWithBuffering(_headContext);
if (t != null) {
_currToken = t;
return t;
}
}
}
continue main_loop;
default: // scalar value
f = _itemFilter;
if (f == TokenFilter.INCLUDE_ALL) {
return (_currToken = t);
}
if (f != null) {
f = _headContext.checkValue(f);
if ((f == TokenFilter.INCLUDE_ALL) || ((f != null) && f.includeValue(delegate))) {
return (_currToken = t);
}
}
// Otherwise not included (leaves must be explicitly included)
break;
}
}
}
@Override
public JsonToken nextToken() throws IOException {
// Anything buffered?
TokenFilterContext ctxt = _exposedContext;
if (ctxt != null) {
while (true) {
JsonToken t = ctxt.nextTokenToRead();
if (t != null) {
_currToken = t;
return t;
}
// all done with buffered stuff?
if (ctxt == _headContext) {
_exposedContext = null;
if (ctxt.inArray()) {
t = delegate.getCurrentToken();
// Is this guaranteed to work without further checks?
// if (t != JsonToken.START_ARRAY) {
_currToken = t;
return t;
}
// Almost! Most likely still have the current token;
// with the sole exception of
/*
t = delegate.getCurrentToken();
if (t != JsonToken.FIELD_NAME) {
_currToken = t;
return t;
}
*/
break;
}
// If not, traverse down the context chain
ctxt = _headContext.findChildOf(ctxt);
_exposedContext = ctxt;
if (ctxt == null) { // should never occur
throw _constructError("Unexpected problem: chain of filtered context broken");
}
}
}
// If not, need to read more. If we got any:
JsonToken t = delegate.nextToken();
if (t == null) {
// no strict need to close, since we have no state here
return (_currToken = t);
}
// otherwise... to include or not?
TokenFilter f;
switch (t.id()) {
case ID_START_ARRAY:
f = _itemFilter;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildArrayContext(f, true);
return (_currToken = t);
}
if (f == null) { // does this occur?
delegate.skipChildren();
break;
}
// Otherwise still iffy, need to check
f = _headContext.checkValue(f);
if (f == null) {
delegate.skipChildren();
break;
}
if (f != TokenFilter.INCLUDE_ALL) {
f = f.filterStartArray();
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildArrayContext(f, true);
return (_currToken = t);
}
_headContext = _headContext.createChildArrayContext(f, false);
// Also: only need buffering if parent path to be included
if (_includePath) {
t = _nextTokenWithBuffering(_headContext);
if (t != null) {
_currToken = t;
return t;
}
}
break;
case ID_START_OBJECT:
f = _itemFilter;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildObjectContext(f, true);
return (_currToken = t);
}
if (f == null) { // does this occur?
delegate.skipChildren();
break;
}
// Otherwise still iffy, need to check
f = _headContext.checkValue(f);
if (f == null) {
delegate.skipChildren();
break;
}
if (f != TokenFilter.INCLUDE_ALL) {
f = f.filterStartObject();
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
_headContext = _headContext.createChildObjectContext(f, true);
return (_currToken = t);
}
_headContext = _headContext.createChildObjectContext(f, false);
// Also: only need buffering if parent path to be included
if (_includePath) {
t = _nextTokenWithBuffering(_headContext);
if (t != null) {
_currToken = t;
return t;
}
}
// note: inclusion of surrounding Object handled separately via
// FIELD_NAME
break;
case ID_END_ARRAY:
case ID_END_OBJECT:
{
boolean returnEnd = _headContext.isStartHandled();
f = _headContext.getFilter();
if ((f != null) && (f != TokenFilter.INCLUDE_ALL)) {
f.filterFinishArray();
}
_headContext = _headContext.getParent();
_itemFilter = _headContext.getFilter();
if (returnEnd) {
return (_currToken = t);
}
}
break;
case ID_FIELD_NAME:
{
final String name = delegate.getCurrentName();
// note: this will also set 'needToHandleName'
f = _headContext.setFieldName(name);
if (f == TokenFilter.INCLUDE_ALL) {
_itemFilter = f;
if (!_includePath) {
// Minor twist here: if parent NOT included, may need to induce output of
// surrounding START_OBJECT/END_OBJECT
if (_includeImmediateParent && !_headContext.isStartHandled()) {
t =
_headContext
.nextTokenToRead(); // returns START_OBJECT but also marks it handled
_exposedContext = _headContext;
}
}
return (_currToken = t);
}
if (f == null) {
delegate.nextToken();
delegate.skipChildren();
break;
}
f = f.includeProperty(name);
if (f == null) {
delegate.nextToken();
delegate.skipChildren();
break;
}
_itemFilter = f;
if (f == TokenFilter.INCLUDE_ALL) {
if (_includePath) {
return (_currToken = t);
}
}
if (_includePath) {
t = _nextTokenWithBuffering(_headContext);
if (t != null) {
_currToken = t;
return t;
}
}
break;
}
default: // scalar value
f = _itemFilter;
if (f == TokenFilter.INCLUDE_ALL) {
return (_currToken = t);
}
if (f != null) {
f = _headContext.checkValue(f);
if ((f == TokenFilter.INCLUDE_ALL) || ((f != null) && f.includeValue(delegate))) {
return (_currToken = t);
}
}
// Otherwise not included (leaves must be explicitly included)
break;
}
// We get here if token was not yet found; offlined handling
return _nextToken2();
}
