@Test
public void testTokenStream() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
ContextSuggestField field =
new ContextSuggestField("field", "input", 1, "context1", "context2");
BytesRef surfaceForm = new BytesRef("input");
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
output.writeVInt(surfaceForm.length);
output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
output.writeVInt(1 + 1);
output.writeByte(ContextSuggestField.TYPE);
}
BytesRef payload = new BytesRef(byteArrayOutputStream.toByteArray());
String[] expectedOutputs = new String[2];
CharsRefBuilder builder = new CharsRefBuilder();
builder.append("context1");
builder.append(((char) ContextSuggestField.CONTEXT_SEPARATOR));
builder.append(((char) CompletionAnalyzer.SEP_LABEL));
builder.append("input");
expectedOutputs[0] = builder.toCharsRef().toString();
builder.clear();
builder.append("context2");
builder.append(((char) ContextSuggestField.CONTEXT_SEPARATOR));
builder.append(((char) CompletionAnalyzer.SEP_LABEL));
builder.append("input");
expectedOutputs[1] = builder.toCharsRef().toString();
TokenStream stream =
new CompletionTokenStreamTest.PayloadAttrToTypeAttrFilter(
field.tokenStream(analyzer, null));
assertTokenStreamContents(
stream,
expectedOutputs,
null,
null,
new String[] {payload.utf8ToString(), payload.utf8ToString()},
new int[] {1, 1},
null,
null);
CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer);
stream =
new CompletionTokenStreamTest.PayloadAttrToTypeAttrFilter(
field.tokenStream(completionAnalyzer, null));
assertTokenStreamContents(
stream,
expectedOutputs,
null,
null,
new String[] {payload.utf8ToString(), payload.utf8ToString()},
new int[] {1, 1},
null,
null);
}
@Test
public void testTokenStream() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
SuggestField suggestField = new SuggestField("field", "input", 1);
BytesRef surfaceForm = new BytesRef("input");
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
output.writeVInt(surfaceForm.length);
output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
output.writeVInt(1 + 1);
output.writeByte(SuggestField.TYPE);
}
BytesRef payload = new BytesRef(byteArrayOutputStream.toByteArray());
TokenStream stream =
new CompletionTokenStreamTest.PayloadAttrToTypeAttrFilter(
suggestField.tokenStream(analyzer, null));
assertTokenStreamContents(
stream,
new String[] {"input"},
null,
null,
new String[] {payload.utf8ToString()},
new int[] {1},
null,
null);
CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer);
stream =
new CompletionTokenStreamTest.PayloadAttrToTypeAttrFilter(
suggestField.tokenStream(completionAnalyzer, null));
assertTokenStreamContents(
stream,
new String[] {"input"},
null,
null,
new String[] {payload.utf8ToString()},
new int[] {1},
null,
null);
}
@Override
public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload)
throws IOException {
if (weight < -1 || weight > Integer.MAX_VALUE) {
throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE");
}
for (int i = 0; i < surfaceForm.length; i++) {
if (surfaceForm.bytes[i] == UNIT_SEPARATOR) {
throw new IllegalArgumentException(
"surface form cannot contain unit separator character U+001F; this character is reserved");
}
}
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream);
output.writeVLong(weight + 1);
output.writeVInt(surfaceForm.length);
output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
output.writeVInt(payload.length);
output.writeBytes(payload.bytes, 0, payload.length);
output.close();
return new BytesRef(byteArrayOutputStream.toByteArray());
}
