コード例 #1
0
  @Test
  public void testTokenStream() throws Exception {
    Analyzer analyzer = new MockAnalyzer(random());
    ContextSuggestField field =
        new ContextSuggestField("field", "input", 1, "context1", "context2");
    BytesRef surfaceForm = new BytesRef("input");
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
      output.writeVInt(surfaceForm.length);
      output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
      output.writeVInt(1 + 1);
      output.writeByte(ContextSuggestField.TYPE);
    }
    BytesRef payload = new BytesRef(byteArrayOutputStream.toByteArray());
    String[] expectedOutputs = new String[2];
    CharsRefBuilder builder = new CharsRefBuilder();
    builder.append("context1");
    builder.append(((char) ContextSuggestField.CONTEXT_SEPARATOR));
    builder.append(((char) CompletionAnalyzer.SEP_LABEL));
    builder.append("input");
    expectedOutputs[0] = builder.toCharsRef().toString();
    builder.clear();
    builder.append("context2");
    builder.append(((char) ContextSuggestField.CONTEXT_SEPARATOR));
    builder.append(((char) CompletionAnalyzer.SEP_LABEL));
    builder.append("input");
    expectedOutputs[1] = builder.toCharsRef().toString();
    TokenStream stream =
        new CompletionTokenStreamTest.PayloadAttrToTypeAttrFilter(
            field.tokenStream(analyzer, null));
    assertTokenStreamContents(
        stream,
        expectedOutputs,
        null,
        null,
        new String[] {payload.utf8ToString(), payload.utf8ToString()},
        new int[] {1, 1},
        null,
        null);

    CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer);
    stream =
        new CompletionTokenStreamTest.PayloadAttrToTypeAttrFilter(
            field.tokenStream(completionAnalyzer, null));
    assertTokenStreamContents(
        stream,
        expectedOutputs,
        null,
        null,
        new String[] {payload.utf8ToString(), payload.utf8ToString()},
        new int[] {1, 1},
        null,
        null);
  }
コード例 #2
0
  @Test
  public void testTokenStream() throws Exception {
    Analyzer analyzer = new MockAnalyzer(random());
    SuggestField suggestField = new SuggestField("field", "input", 1);
    BytesRef surfaceForm = new BytesRef("input");
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
      output.writeVInt(surfaceForm.length);
      output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
      output.writeVInt(1 + 1);
      output.writeByte(SuggestField.TYPE);
    }
    BytesRef payload = new BytesRef(byteArrayOutputStream.toByteArray());
    TokenStream stream =
        new CompletionTokenStreamTest.PayloadAttrToTypeAttrFilter(
            suggestField.tokenStream(analyzer, null));
    assertTokenStreamContents(
        stream,
        new String[] {"input"},
        null,
        null,
        new String[] {payload.utf8ToString()},
        new int[] {1},
        null,
        null);

    CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer);
    stream =
        new CompletionTokenStreamTest.PayloadAttrToTypeAttrFilter(
            suggestField.tokenStream(completionAnalyzer, null));
    assertTokenStreamContents(
        stream,
        new String[] {"input"},
        null,
        null,
        new String[] {payload.utf8ToString()},
        new int[] {1},
        null,
        null);
  }
コード例 #3
0
    @Override
    public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload)
        throws IOException {
      if (weight < -1 || weight > Integer.MAX_VALUE) {
        throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE");
      }
      for (int i = 0; i < surfaceForm.length; i++) {
        if (surfaceForm.bytes[i] == UNIT_SEPARATOR) {
          throw new IllegalArgumentException(
              "surface form cannot contain unit separator character U+001F; this character is reserved");
        }
      }
      ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
      OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream);
      output.writeVLong(weight + 1);
      output.writeVInt(surfaceForm.length);
      output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
      output.writeVInt(payload.length);
      output.writeBytes(payload.bytes, 0, payload.length);

      output.close();
      return new BytesRef(byteArrayOutputStream.toByteArray());
    }