public void addItem(IToken token) throws IOException { if (!fixedSize) { offsets.add(outputStorage.getLength()); } numberOfItems++; token.serializeToken(outputStorage); }
@Override public void nextFrame(ByteBuffer buffer) throws HyracksDataException { accessor.reset(buffer); int tupleCount = accessor.getTupleCount(); for (int i = 0; i < tupleCount; i++) { short numTokens = 0; tokenizer.reset( accessor.getBuffer().array(), accessor.getTupleStartOffset(i) + accessor.getFieldSlotsLength() + accessor.getFieldStartOffset(i, docField), accessor.getFieldLength(i, docField)); if (addNumTokensKey) { // Get the total number of tokens. numTokens = tokenizer.getTokensCount(); } // Write token and data into frame by following the order specified // in the writeKeyFieldsFirst field. while (tokenizer.hasNext()) { tokenizer.next(); builder.reset(); // Writing Order: token, number of token, keyfield1 ... n if (!writeKeyFieldsFirst) { try { IToken token = tokenizer.getToken(); token.serializeToken(builderData); builder.addFieldEndOffset(); // Add number of tokens if requested. if (addNumTokensKey) { builder.getDataOutput().writeShort(numTokens); builder.addFieldEndOffset(); } } catch (IOException e) { throw new HyracksDataException(e.getMessage()); } for (int k = 0; k < keyFields.length; k++) { builder.addField(accessor, i, keyFields[k]); } } // Writing Order: keyfield1 ... n, token, number of token else { for (int k = 0; k < keyFields.length; k++) { builder.addField(accessor, i, keyFields[k]); } try { IToken token = tokenizer.getToken(); token.serializeToken(builderData); builder.addFieldEndOffset(); // Add number of tokens if requested. if (addNumTokensKey) { builder.getDataOutput().writeShort(numTokens); builder.addFieldEndOffset(); } } catch (IOException e) { throw new HyracksDataException(e.getMessage()); } } FrameUtils.appendToWriter( writer, appender, builder.getFieldEndOffsets(), builder.getByteArray(), 0, builder.getSize()); } } }