ByteBuffer makeAFrame(int capacity, int count, int deletedBytes) throws HyracksDataException {
ByteBuffer buffer = ByteBuffer.allocate(capacity);
int metaOffset = capacity - 4;
buffer.putInt(metaOffset, deletedBytes);
metaOffset -= 4;
buffer.putInt(metaOffset, count);
metaOffset -= 4;
for (int i = 0; i < count; i++, metaOffset -= 4) {
makeARecord(builder, i);
for (int x = 0; x < builder.getFieldEndOffsets().length; x++) {
buffer.putInt(builder.getFieldEndOffsets()[x]);
}
buffer.put(builder.getByteArray(), 0, builder.getSize());
assert (metaOffset > buffer.position());
buffer.putInt(metaOffset, buffer.position());
}
return buffer;
}
static void prepareData(
IHyracksTaskContext ctx,
List<IFrame> frameList,
int minDataSize,
int minRecordSize,
int maxRecordSize,
Map<Integer, String> specialData,
Map<Integer, String> keyValuePair)
throws HyracksDataException {
ArrayTupleBuilder tb = new ArrayTupleBuilder(RecordDesc.getFieldCount());
FrameTupleAppender appender = new FrameTupleAppender();
int datasize = 0;
if (specialData != null) {
for (Map.Entry<Integer, String> entry : specialData.entrySet()) {
tb.reset();
tb.addField(IntegerSerializerDeserializer.INSTANCE, entry.getKey());
tb.addField(UTF8StringSerializerDeserializer.INSTANCE, entry.getValue());
VSizeFrame frame =
new VSizeFrame(
ctx,
FrameHelper.calcAlignedFrameSizeToStore(
tb.getFieldEndOffsets().length, tb.getSize(), ctx.getInitialFrameSize()));
appender.reset(frame, true);
assertTrue(appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize()));
frameList.add(frame);
datasize += frame.getFrameSize();
}
keyValuePair.putAll(specialData);
}
VSizeFrame frame = new VSizeFrame(ctx, ctx.getInitialFrameSize());
appender.reset(frame, true);
while (datasize < minDataSize) {
tb.reset();
int key = GRandom.nextInt(minDataSize + 1);
if (!keyValuePair.containsKey(key)) {
String value = generateRandomRecord(minRecordSize, maxRecordSize);
tb.addField(IntegerSerializerDeserializer.INSTANCE, key);
tb.addField(UTF8StringSerializerDeserializer.INSTANCE, value);
if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
frameList.add(frame);
datasize += frame.getFrameSize();
frame =
new VSizeFrame(
ctx,
FrameHelper.calcAlignedFrameSizeToStore(
tb.getFieldEndOffsets().length, tb.getSize(), ctx.getInitialFrameSize()));
appender.reset(frame, true);
assertTrue(appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize()));
}
keyValuePair.put(key, value);
}
}
if (appender.getTupleCount() > 0) {
frameList.add(frame);
}
}
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
accessor.reset(buffer);
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
short numTokens = 0;
tokenizer.reset(
accessor.getBuffer().array(),
accessor.getTupleStartOffset(i)
+ accessor.getFieldSlotsLength()
+ accessor.getFieldStartOffset(i, docField),
accessor.getFieldLength(i, docField));
if (addNumTokensKey) {
// Get the total number of tokens.
numTokens = tokenizer.getTokensCount();
}
// Write token and data into frame by following the order specified
// in the writeKeyFieldsFirst field.
while (tokenizer.hasNext()) {
tokenizer.next();
builder.reset();
// Writing Order: token, number of token, keyfield1 ... n
if (!writeKeyFieldsFirst) {
try {
IToken token = tokenizer.getToken();
token.serializeToken(builderData);
builder.addFieldEndOffset();
// Add number of tokens if requested.
if (addNumTokensKey) {
builder.getDataOutput().writeShort(numTokens);
builder.addFieldEndOffset();
}
} catch (IOException e) {
throw new HyracksDataException(e.getMessage());
}
for (int k = 0; k < keyFields.length; k++) {
builder.addField(accessor, i, keyFields[k]);
}
}
// Writing Order: keyfield1 ... n, token, number of token
else {
for (int k = 0; k < keyFields.length; k++) {
builder.addField(accessor, i, keyFields[k]);
}
try {
IToken token = tokenizer.getToken();
token.serializeToken(builderData);
builder.addFieldEndOffset();
// Add number of tokens if requested.
if (addNumTokensKey) {
builder.getDataOutput().writeShort(numTokens);
builder.addFieldEndOffset();
}
} catch (IOException e) {
throw new HyracksDataException(e.getMessage());
}
}
FrameUtils.appendToWriter(
writer,
appender,
builder.getFieldEndOffsets(),
builder.getByteArray(),
0,
builder.getSize());
}
}
}