private void writeSearchResults(IFrameTupleAccessor leftAccessor, int tIndex) throws Exception { while (cursor.hasNext()) { tb.reset(); cursor.next(); ITupleReference frameTuple = cursor.getTuple(); for (int i = 0; i < inputRecDesc.getFields().length; i++) { int tupleStart = leftAccessor.getTupleStartOffset(tIndex); int fieldStart = leftAccessor.getFieldStartOffset(tIndex, i); int offset = leftAccessor.getFieldSlotsLength() + tupleStart + fieldStart; int len = leftAccessor.getFieldEndOffset(tIndex, i) - fieldStart; dos.write(leftAccessor.getBuffer().array(), offset, len); tb.addFieldEndOffset(); } for (int i = 0; i < frameTuple.getFieldCount(); i++) { dos.write( frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i)); tb.addFieldEndOffset(); } if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { FrameUtils.flushFrame(writeBuffer, writer); appender.reset(writeBuffer, true); if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { throw new IllegalStateException(); } } } }
@Override public void open() throws HyracksDataException { accessor = new FrameTupleAccessor(treeIndexOpHelper.getTaskContext().getFrameSize(), recDesc); try { treeIndexOpHelper.open(); index = (ITreeIndex) treeIndexOpHelper.getIndexInstance(); writer.open(); int lowKeySearchFields = index.getComparatorFactories().length; int highKeySearchFields = index.getComparatorFactories().length; if (lowKey != null) lowKeySearchFields = lowKey.getFieldCount(); if (highKey != null) highKeySearchFields = highKey.getFieldCount(); IBinaryComparator[] lowKeySearchComparators = new IBinaryComparator[lowKeySearchFields]; for (int i = 0; i < lowKeySearchFields; i++) { lowKeySearchComparators[i] = index.getComparatorFactories()[i].createBinaryComparator(); } lowKeySearchCmp = new MultiComparator(lowKeySearchComparators); if (lowKeySearchFields == highKeySearchFields) { highKeySearchCmp = lowKeySearchCmp; } else { IBinaryComparator[] highKeySearchComparators = new IBinaryComparator[highKeySearchFields]; for (int i = 0; i < highKeySearchFields; i++) { highKeySearchComparators[i] = index.getComparatorFactories()[i].createBinaryComparator(); } highKeySearchCmp = new MultiComparator(highKeySearchComparators); } rangePred = new RangePredicate( null, null, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp, highKeySearchCmp); writeBuffer = treeIndexOpHelper.getTaskContext().allocateFrame(); tb = new ArrayTupleBuilder(inputRecDesc.getFields().length + index.getFieldCount()); dos = tb.getDataOutput(); appender = new FrameTupleAppender(treeIndexOpHelper.getTaskContext().getFrameSize()); appender.reset(writeBuffer, true); indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); setCursor(); } catch (Exception e) { treeIndexOpHelper.close(); throw new HyracksDataException(e); } }
/** write the left result */ private void writeLeftResults(IFrameTupleAccessor leftAccessor, int tIndex) throws Exception { tb.reset(); for (int i = 0; i < inputRecDesc.getFields().length; i++) { int tupleStart = leftAccessor.getTupleStartOffset(tIndex); int fieldStart = leftAccessor.getFieldStartOffset(tIndex, i); int offset = leftAccessor.getFieldSlotsLength() + tupleStart + fieldStart; int len = leftAccessor.getFieldEndOffset(tIndex, i) - fieldStart; dos.write(leftAccessor.getBuffer().array(), offset, len); tb.addFieldEndOffset(); } if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { FrameUtils.flushFrame(writeBuffer, writer); appender.reset(writeBuffer, true); if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { throw new IllegalStateException(); } } }
public class FrameFixedFieldTupleAppenderTest { static final int INPUT_BUFFER_SIZE = 4096; static final int TEST_FRAME_SIZE = 256; FrameFixedFieldAppender appender; static ISerializerDeserializer[] fields = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE, }; static RecordDescriptor recordDescriptor = new RecordDescriptor(fields); static ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(recordDescriptor.getFieldCount()); class SequetialDataVerifier implements IFrameWriter { private final IFrameTupleAccessor accessor; private IFrameTupleAccessor innerAccessor; private int tid; public SequetialDataVerifier(IFrameTupleAccessor accessor) { this.accessor = accessor; this.innerAccessor = new FrameTupleAccessor(recordDescriptor); } @Override public void open() throws HyracksDataException { this.tid = 0; } @Override public void nextFrame(ByteBuffer buffer) throws HyracksDataException { innerAccessor.reset(buffer); for (int i = 0; i < innerAccessor.getTupleCount(); ++i) { validate(innerAccessor, i); } } private void validate(IFrameTupleAccessor innerAccessor, int i) { assertTrue(tid < accessor.getTupleCount()); assertEquals(accessor.getTupleLength(tid), innerAccessor.getTupleLength(i)); assertArrayEquals( Arrays.copyOfRange( accessor.getBuffer().array(), accessor.getTupleStartOffset(tid), accessor.getTupleEndOffset(tid)), Arrays.copyOfRange( innerAccessor.getBuffer().array(), innerAccessor.getTupleStartOffset(i), innerAccessor.getTupleEndOffset(i))); tid++; } @Override public void fail() throws HyracksDataException { assert false; } @Override public void close() throws HyracksDataException { assertEquals(accessor.getTupleCount(), tid); } } @Before public void createAppender() throws HyracksDataException { appender = new FrameFixedFieldAppender(fields.length); FrameManager manager = new FrameManager(TEST_FRAME_SIZE); IFrame frame = new VSizeFrame(manager); appender.reset(frame, true); } private void testProcess(IFrameTupleAccessor accessor) throws HyracksDataException { IFrameWriter writer = prepareValidator(accessor); writer.open(); for (int tid = 0; tid < accessor.getTupleCount(); tid++) { for (int fid = 0; fid < fields.length; fid++) { if (!appender.appendField(accessor, tid, fid)) { appender.flush(writer, true); if (!appender.appendField(accessor, tid, fid)) {} } } } appender.flush(writer, true); writer.close(); } @Test public void testAppendFieldShouldSucceed() throws HyracksDataException { IFrameTupleAccessor accessor = prepareData(DATA_TYPE.NORMAL_RECORD); testProcess(accessor); } @Test public void testResetShouldWork() throws HyracksDataException { testAppendFieldShouldSucceed(); appender.reset(new VSizeFrame(new FrameManager(TEST_FRAME_SIZE)), true); testAppendFieldShouldSucceed(); } private IFrameWriter prepareValidator(IFrameTupleAccessor accessor) throws HyracksDataException { return new SequetialDataVerifier(accessor); } enum DATA_TYPE { NORMAL_RECORD, ONE_FIELD_LONG, ONE_RECORD_LONG, } private IFrameTupleAccessor prepareData(DATA_TYPE type) throws HyracksDataException { IFrameTupleAccessor accessor = new FrameTupleAccessor(recordDescriptor); IFrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(new FrameManager(INPUT_BUFFER_SIZE)), true); int i = 0; do { switch (type) { case NORMAL_RECORD: makeATuple(tupleBuilder, i++); break; case ONE_FIELD_LONG: makeASizeUpTuple(tupleBuilder, i++); break; case ONE_RECORD_LONG: makeABigObjectTuple(tupleBuilder, i++); break; } } while (appender.append( tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())); accessor.reset(appender.getBuffer()); return accessor; } private void makeATuple(ArrayTupleBuilder tupleBuilder, int i) throws HyracksDataException { tupleBuilder.reset(); tupleBuilder.addField(fields[0], i); tupleBuilder.addField(fields[1], String.valueOf(i)); tupleBuilder.addField(fields[2], -i); tupleBuilder.addField(fields[3], String.valueOf(-i)); } private String makeALongString(int length, char ch) { char[] array = new char[length]; Arrays.fill(array, ch); return new String(array); } private void makeASizeUpTuple(ArrayTupleBuilder tupleBuilder, int i) throws HyracksDataException { tupleBuilder.reset(); tupleBuilder.addField(fields[0], i); tupleBuilder.addField( fields[1], makeALongString(Math.min(Math.abs(1 << i), INPUT_BUFFER_SIZE), (char) i)); tupleBuilder.addField(fields[2], -i); tupleBuilder.addField(fields[3], String.valueOf(-i)); } private void makeABigObjectTuple(ArrayTupleBuilder tupleBuilder, int i) throws HyracksDataException { tupleBuilder.reset(); tupleBuilder.addField(fields[0], i); tupleBuilder.addField(fields[1], makeALongString(Math.min(i * 20, TEST_FRAME_SIZE), (char) i)); tupleBuilder.addField(fields[2], -i); tupleBuilder.addField(fields[3], makeALongString(Math.min(i * 20, TEST_FRAME_SIZE), (char) i)); } @Test public void testAppendLargeFieldShouldSucceed() throws HyracksDataException { IFrameTupleAccessor accessor = prepareData(DATA_TYPE.ONE_FIELD_LONG); testProcess(accessor); } @Test public void testAppendSmallFieldButLargeObjectWithShouldSucceed() throws HyracksDataException { IFrameTupleAccessor accessor = prepareData(DATA_TYPE.ONE_RECORD_LONG); testProcess(accessor); } }