public void addItem(IToken token) throws IOException {
if (!fixedSize) {
offsets.add(outputStorage.getLength());
}
numberOfItems++;
token.serializeToken(outputStorage);
}
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
accessor.reset(buffer);
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
short numTokens = 0;
tokenizer.reset(
accessor.getBuffer().array(),
accessor.getTupleStartOffset(i)
+ accessor.getFieldSlotsLength()
+ accessor.getFieldStartOffset(i, docField),
accessor.getFieldLength(i, docField));
if (addNumTokensKey) {
// Get the total number of tokens.
numTokens = tokenizer.getTokensCount();
}
// Write token and data into frame by following the order specified
// in the writeKeyFieldsFirst field.
while (tokenizer.hasNext()) {
tokenizer.next();
builder.reset();
// Writing Order: token, number of token, keyfield1 ... n
if (!writeKeyFieldsFirst) {
try {
IToken token = tokenizer.getToken();
token.serializeToken(builderData);
builder.addFieldEndOffset();
// Add number of tokens if requested.
if (addNumTokensKey) {
builder.getDataOutput().writeShort(numTokens);
builder.addFieldEndOffset();
}
} catch (IOException e) {
throw new HyracksDataException(e.getMessage());
}
for (int k = 0; k < keyFields.length; k++) {
builder.addField(accessor, i, keyFields[k]);
}
}
// Writing Order: keyfield1 ... n, token, number of token
else {
for (int k = 0; k < keyFields.length; k++) {
builder.addField(accessor, i, keyFields[k]);
}
try {
IToken token = tokenizer.getToken();
token.serializeToken(builderData);
builder.addFieldEndOffset();
// Add number of tokens if requested.
if (addNumTokensKey) {
builder.getDataOutput().writeShort(numTokens);
builder.addFieldEndOffset();
}
} catch (IOException e) {
throw new HyracksDataException(e.getMessage());
}
}
FrameUtils.appendToWriter(
writer,
appender,
builder.getFieldEndOffsets(),
builder.getByteArray(),
0,
builder.getSize());
}
}
}