private void writeSearchResults(IFrameTupleAccessor leftAccessor, int tIndex) throws Exception {
while (cursor.hasNext()) {
tb.reset();
cursor.next();
ITupleReference frameTuple = cursor.getTuple();
for (int i = 0; i < inputRecDesc.getFields().length; i++) {
int tupleStart = leftAccessor.getTupleStartOffset(tIndex);
int fieldStart = leftAccessor.getFieldStartOffset(tIndex, i);
int offset = leftAccessor.getFieldSlotsLength() + tupleStart + fieldStart;
int len = leftAccessor.getFieldEndOffset(tIndex, i) - fieldStart;
dos.write(leftAccessor.getBuffer().array(), offset, len);
tb.addFieldEndOffset();
}
for (int i = 0; i < frameTuple.getFieldCount(); i++) {
dos.write(
frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
tb.addFieldEndOffset();
}
if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
FrameUtils.flushFrame(writeBuffer, writer);
appender.reset(writeBuffer, true);
if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
throw new IllegalStateException();
}
}
}
}
@Override
public void open() throws HyracksDataException {
accessor = new FrameTupleAccessor(treeIndexOpHelper.getTaskContext().getFrameSize(), recDesc);
try {
treeIndexOpHelper.open();
index = (ITreeIndex) treeIndexOpHelper.getIndexInstance();
writer.open();
int lowKeySearchFields = index.getComparatorFactories().length;
int highKeySearchFields = index.getComparatorFactories().length;
if (lowKey != null) lowKeySearchFields = lowKey.getFieldCount();
if (highKey != null) highKeySearchFields = highKey.getFieldCount();
IBinaryComparator[] lowKeySearchComparators = new IBinaryComparator[lowKeySearchFields];
for (int i = 0; i < lowKeySearchFields; i++) {
lowKeySearchComparators[i] = index.getComparatorFactories()[i].createBinaryComparator();
}
lowKeySearchCmp = new MultiComparator(lowKeySearchComparators);
if (lowKeySearchFields == highKeySearchFields) {
highKeySearchCmp = lowKeySearchCmp;
} else {
IBinaryComparator[] highKeySearchComparators = new IBinaryComparator[highKeySearchFields];
for (int i = 0; i < highKeySearchFields; i++) {
highKeySearchComparators[i] = index.getComparatorFactories()[i].createBinaryComparator();
}
highKeySearchCmp = new MultiComparator(highKeySearchComparators);
}
rangePred =
new RangePredicate(
null, null, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp, highKeySearchCmp);
writeBuffer = treeIndexOpHelper.getTaskContext().allocateFrame();
tb = new ArrayTupleBuilder(inputRecDesc.getFields().length + index.getFieldCount());
dos = tb.getDataOutput();
appender = new FrameTupleAppender(treeIndexOpHelper.getTaskContext().getFrameSize());
appender.reset(writeBuffer, true);
indexAccessor =
index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
setCursor();
} catch (Exception e) {
treeIndexOpHelper.close();
throw new HyracksDataException(e);
}
}
/** write the left result */
private void writeLeftResults(IFrameTupleAccessor leftAccessor, int tIndex) throws Exception {
tb.reset();
for (int i = 0; i < inputRecDesc.getFields().length; i++) {
int tupleStart = leftAccessor.getTupleStartOffset(tIndex);
int fieldStart = leftAccessor.getFieldStartOffset(tIndex, i);
int offset = leftAccessor.getFieldSlotsLength() + tupleStart + fieldStart;
int len = leftAccessor.getFieldEndOffset(tIndex, i) - fieldStart;
dos.write(leftAccessor.getBuffer().array(), offset, len);
tb.addFieldEndOffset();
}
if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
FrameUtils.flushFrame(writeBuffer, writer);
appender.reset(writeBuffer, true);
if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
throw new IllegalStateException();
}
}
}
public class FrameFixedFieldTupleAppenderTest {
static final int INPUT_BUFFER_SIZE = 4096;
static final int TEST_FRAME_SIZE = 256;
FrameFixedFieldAppender appender;
static ISerializerDeserializer[] fields =
new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE,
UTF8StringSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE,
UTF8StringSerializerDeserializer.INSTANCE,
};
static RecordDescriptor recordDescriptor = new RecordDescriptor(fields);
static ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(recordDescriptor.getFieldCount());
class SequetialDataVerifier implements IFrameWriter {
private final IFrameTupleAccessor accessor;
private IFrameTupleAccessor innerAccessor;
private int tid;
public SequetialDataVerifier(IFrameTupleAccessor accessor) {
this.accessor = accessor;
this.innerAccessor = new FrameTupleAccessor(recordDescriptor);
}
@Override
public void open() throws HyracksDataException {
this.tid = 0;
}
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
innerAccessor.reset(buffer);
for (int i = 0; i < innerAccessor.getTupleCount(); ++i) {
validate(innerAccessor, i);
}
}
private void validate(IFrameTupleAccessor innerAccessor, int i) {
assertTrue(tid < accessor.getTupleCount());
assertEquals(accessor.getTupleLength(tid), innerAccessor.getTupleLength(i));
assertArrayEquals(
Arrays.copyOfRange(
accessor.getBuffer().array(),
accessor.getTupleStartOffset(tid),
accessor.getTupleEndOffset(tid)),
Arrays.copyOfRange(
innerAccessor.getBuffer().array(),
innerAccessor.getTupleStartOffset(i),
innerAccessor.getTupleEndOffset(i)));
tid++;
}
@Override
public void fail() throws HyracksDataException {
assert false;
}
@Override
public void close() throws HyracksDataException {
assertEquals(accessor.getTupleCount(), tid);
}
}
@Before
public void createAppender() throws HyracksDataException {
appender = new FrameFixedFieldAppender(fields.length);
FrameManager manager = new FrameManager(TEST_FRAME_SIZE);
IFrame frame = new VSizeFrame(manager);
appender.reset(frame, true);
}
private void testProcess(IFrameTupleAccessor accessor) throws HyracksDataException {
IFrameWriter writer = prepareValidator(accessor);
writer.open();
for (int tid = 0; tid < accessor.getTupleCount(); tid++) {
for (int fid = 0; fid < fields.length; fid++) {
if (!appender.appendField(accessor, tid, fid)) {
appender.flush(writer, true);
if (!appender.appendField(accessor, tid, fid)) {}
}
}
}
appender.flush(writer, true);
writer.close();
}
@Test
public void testAppendFieldShouldSucceed() throws HyracksDataException {
IFrameTupleAccessor accessor = prepareData(DATA_TYPE.NORMAL_RECORD);
testProcess(accessor);
}
@Test
public void testResetShouldWork() throws HyracksDataException {
testAppendFieldShouldSucceed();
appender.reset(new VSizeFrame(new FrameManager(TEST_FRAME_SIZE)), true);
testAppendFieldShouldSucceed();
}
private IFrameWriter prepareValidator(IFrameTupleAccessor accessor) throws HyracksDataException {
return new SequetialDataVerifier(accessor);
}
enum DATA_TYPE {
NORMAL_RECORD,
ONE_FIELD_LONG,
ONE_RECORD_LONG,
}
private IFrameTupleAccessor prepareData(DATA_TYPE type) throws HyracksDataException {
IFrameTupleAccessor accessor = new FrameTupleAccessor(recordDescriptor);
IFrameTupleAppender appender =
new FrameTupleAppender(new VSizeFrame(new FrameManager(INPUT_BUFFER_SIZE)), true);
int i = 0;
do {
switch (type) {
case NORMAL_RECORD:
makeATuple(tupleBuilder, i++);
break;
case ONE_FIELD_LONG:
makeASizeUpTuple(tupleBuilder, i++);
break;
case ONE_RECORD_LONG:
makeABigObjectTuple(tupleBuilder, i++);
break;
}
} while (appender.append(
tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0, tupleBuilder.getSize()));
accessor.reset(appender.getBuffer());
return accessor;
}
private void makeATuple(ArrayTupleBuilder tupleBuilder, int i) throws HyracksDataException {
tupleBuilder.reset();
tupleBuilder.addField(fields[0], i);
tupleBuilder.addField(fields[1], String.valueOf(i));
tupleBuilder.addField(fields[2], -i);
tupleBuilder.addField(fields[3], String.valueOf(-i));
}
private String makeALongString(int length, char ch) {
char[] array = new char[length];
Arrays.fill(array, ch);
return new String(array);
}
private void makeASizeUpTuple(ArrayTupleBuilder tupleBuilder, int i) throws HyracksDataException {
tupleBuilder.reset();
tupleBuilder.addField(fields[0], i);
tupleBuilder.addField(
fields[1], makeALongString(Math.min(Math.abs(1 << i), INPUT_BUFFER_SIZE), (char) i));
tupleBuilder.addField(fields[2], -i);
tupleBuilder.addField(fields[3], String.valueOf(-i));
}
private void makeABigObjectTuple(ArrayTupleBuilder tupleBuilder, int i)
throws HyracksDataException {
tupleBuilder.reset();
tupleBuilder.addField(fields[0], i);
tupleBuilder.addField(fields[1], makeALongString(Math.min(i * 20, TEST_FRAME_SIZE), (char) i));
tupleBuilder.addField(fields[2], -i);
tupleBuilder.addField(fields[3], makeALongString(Math.min(i * 20, TEST_FRAME_SIZE), (char) i));
}
@Test
public void testAppendLargeFieldShouldSucceed() throws HyracksDataException {
IFrameTupleAccessor accessor = prepareData(DATA_TYPE.ONE_FIELD_LONG);
testProcess(accessor);
}
@Test
public void testAppendSmallFieldButLargeObjectWithShouldSucceed() throws HyracksDataException {
IFrameTupleAccessor accessor = prepareData(DATA_TYPE.ONE_RECORD_LONG);
testProcess(accessor);
}
}