private void writeSearchResults(IFrameTupleAccessor leftAccessor, int tIndex) throws Exception { while (cursor.hasNext()) { tb.reset(); cursor.next(); ITupleReference frameTuple = cursor.getTuple(); for (int i = 0; i < inputRecDesc.getFields().length; i++) { int tupleStart = leftAccessor.getTupleStartOffset(tIndex); int fieldStart = leftAccessor.getFieldStartOffset(tIndex, i); int offset = leftAccessor.getFieldSlotsLength() + tupleStart + fieldStart; int len = leftAccessor.getFieldEndOffset(tIndex, i) - fieldStart; dos.write(leftAccessor.getBuffer().array(), offset, len); tb.addFieldEndOffset(); } for (int i = 0; i < frameTuple.getFieldCount(); i++) { dos.write( frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i)); tb.addFieldEndOffset(); } if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { FrameUtils.flushFrame(writeBuffer, writer); appender.reset(writeBuffer, true); if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { throw new IllegalStateException(); } } } }
/** write the right result */ private void writeRightResults(ITupleReference frameTuple) throws Exception { tb.reset(); for (int i = 0; i < frameTuple.getFieldCount(); i++) { dos.write( frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i)); tb.addFieldEndOffset(); } if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { FrameUtils.flushFrame(writeBuffer, writer); appender.reset(writeBuffer, true); if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { throw new IllegalStateException(); } } }
@Override public void open() throws HyracksDataException { /** open the function */ functionProxy.functionOpen(); accessor = new FrameTupleAccessor(treeIndexHelper.getTaskContext().getFrameSize(), recDesc); try { treeIndexHelper.open(); btree = (BTree) treeIndexHelper.getIndexInstance(); cursorFrame = btree.getLeafFrameFactory().createFrame(); setCursor(); // Construct range predicate. lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getComparatorFactories(), lowKey); highKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getComparatorFactories(), highKey); rangePred = new RangePredicate( null, null, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp, highKeySearchCmp); writeBuffer = treeIndexHelper.getTaskContext().allocateFrame(); tb = new ArrayTupleBuilder(btree.getFieldCount()); dos = tb.getDataOutput(); appender = new FrameTupleAppender(treeIndexHelper.getTaskContext().getFrameSize()); appender.reset(writeBuffer, true); indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); cloneUpdateTb = new ArrayTupleBuilder(btree.getFieldCount()); updateBuffer.setFieldCount(btree.getFieldCount()); } catch (Exception e) { treeIndexHelper.close(); throw new HyracksDataException(e); } }
/** write the left result */ private void writeLeftResults(IFrameTupleAccessor leftAccessor, int tIndex) throws Exception { tb.reset(); for (int i = 0; i < inputRecDesc.getFields().length; i++) { int tupleStart = leftAccessor.getTupleStartOffset(tIndex); int fieldStart = leftAccessor.getFieldStartOffset(tIndex, i); int offset = leftAccessor.getFieldSlotsLength() + tupleStart + fieldStart; int len = leftAccessor.getFieldEndOffset(tIndex, i) - fieldStart; dos.write(leftAccessor.getBuffer().array(), offset, len); tb.addFieldEndOffset(); } if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { FrameUtils.flushFrame(writeBuffer, writer); appender.reset(writeBuffer, true); if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { throw new IllegalStateException(); } } }
public void probe(ByteBuffer buffer, IFrameWriter writer) throws HyracksDataException { accessorProbe.reset(buffer); int tupleCount = accessorProbe.getTupleCount(); boolean print = false; if (print) { accessorProbe.prettyPrint(); } if (numOfSpilledParts == 0) { inMemJoiner.join(buffer, writer); return; } for (int i = 0; i < tupleCount; ++i) { int pid = probeHpc.partition(accessorProbe, i, numOfPartitions); if (buildPSizeInTups[pid] > 0) { // Tuple has potential match from previous phase if (pStatus.get(pid)) { // pid is Spilled boolean needToClear = false; ByteBuffer buff = sPartBuffs[curPBuff[pid]]; while (true) { probeTupAppenderToSpilled.reset(buff, needToClear); if (probeTupAppenderToSpilled.append(accessorProbe, i)) { break; } probeWrite(pid, buff); buff.clear(); needToClear = true; } } else { // pid is Resident while (true) { if (probeTupAppenderToResident.append(accessorProbe, i)) { break; } inMemJoiner.join(probeResBuff, writer); probeTupAppenderToResident.reset(probeResBuff, true); } } probePSizeInTups[pid]++; } } }
private void processTuple(int tid, int pid) throws HyracksDataException { ByteBuffer partition = memBuffs[curPBuff[pid]]; // Getting current buffer for the target partition if (!pStatus.get(pid)) { // resident partition buildTupAppender.reset(partition, false); while (true) { if (buildTupAppender.append( accessorBuild, tid)) { // Tuple added to resident partition successfully break; } // partition does not have enough room int newBuffIx = allocateFreeBuffer(pid); if (newBuffIx == NO_MORE_FREE_BUFFER) { // Spill one partition int pidToSpill = selectPartitionToSpill(); if (pidToSpill == -1) { // No more partition to spill throw new HyracksDataException( "not enough memory for Hash Join (Allocation exceeds the limit)"); } spillPartition(pidToSpill); buildTupAppender.reset(memBuffs[pidToSpill], true); processTuple(tid, pid); break; } // New Buffer allocated successfully partition = memBuffs[ curPBuff[ pid]]; // Current Buffer for the partition is now updated by // allocateFreeBuffer() call above buildTupAppender.reset(partition, true); if (!buildTupAppender.append(accessorBuild, tid)) { throw new HyracksDataException( "Invalid State (Can not append to newly allocated buffer)"); } buildPSizeInFrames[pid]++; break; } } else { // spilled partition boolean needClear = false; while (true) { buildTupAppender.reset(partition, needClear); if (buildTupAppender.append(accessorBuild, tid)) { break; } // Dedicated in-memory buffer for the partition is full, needed to be flushed first buildWrite(pid, partition); partition.clear(); needClear = true; buildPSizeInFrames[pid]++; } } }
@Override public void close() throws HyracksDataException { try { if (appender.getTupleCount() > 0) { FrameUtils.flushFrame(writeBuffer, writer); } writer.close(); try { cursor.close(); } catch (Exception e) { throw new HyracksDataException(e); } } finally { treeIndexOpHelper.close(); } }
@Override public void open() throws HyracksDataException { accessor = new FrameTupleAccessor(treeIndexOpHelper.getTaskContext().getFrameSize(), recDesc); try { treeIndexOpHelper.open(); index = (ITreeIndex) treeIndexOpHelper.getIndexInstance(); writer.open(); int lowKeySearchFields = index.getComparatorFactories().length; int highKeySearchFields = index.getComparatorFactories().length; if (lowKey != null) lowKeySearchFields = lowKey.getFieldCount(); if (highKey != null) highKeySearchFields = highKey.getFieldCount(); IBinaryComparator[] lowKeySearchComparators = new IBinaryComparator[lowKeySearchFields]; for (int i = 0; i < lowKeySearchFields; i++) { lowKeySearchComparators[i] = index.getComparatorFactories()[i].createBinaryComparator(); } lowKeySearchCmp = new MultiComparator(lowKeySearchComparators); if (lowKeySearchFields == highKeySearchFields) { highKeySearchCmp = lowKeySearchCmp; } else { IBinaryComparator[] highKeySearchComparators = new IBinaryComparator[highKeySearchFields]; for (int i = 0; i < highKeySearchFields; i++) { highKeySearchComparators[i] = index.getComparatorFactories()[i].createBinaryComparator(); } highKeySearchCmp = new MultiComparator(highKeySearchComparators); } rangePred = new RangePredicate( null, null, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp, highKeySearchCmp); writeBuffer = treeIndexOpHelper.getTaskContext().allocateFrame(); tb = new ArrayTupleBuilder(inputRecDesc.getFields().length + index.getFieldCount()); dos = tb.getDataOutput(); appender = new FrameTupleAppender(treeIndexOpHelper.getTaskContext().getFrameSize()); appender.reset(writeBuffer, true); indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); setCursor(); } catch (Exception e) { treeIndexOpHelper.close(); throw new HyracksDataException(e); } }
@Override public void open() throws HyracksDataException { accessor = new FrameTupleAccessor(treeIndexOpHelper.getTaskContext().getFrameSize(), recDesc); try { treeIndexOpHelper.open(); btree = (BTree) treeIndexOpHelper.getIndexInstance(); cursorFrame = btree.getLeafFrameFactory().createFrame(); setCursor(); writer.open(); rangePred = new RangePredicate(null, null, true, true, null, null); int lowKeySearchFields = btree.getComparatorFactories().length; IBinaryComparator[] lowKeySearchComparators = new IBinaryComparator[lowKeySearchFields]; for (int i = 0; i < lowKeySearchFields; i++) { lowKeySearchComparators[i] = btree.getComparatorFactories()[i].createBinaryComparator(); } lowKeySearchCmp = new MultiComparator(lowKeySearchComparators); writeBuffer = treeIndexOpHelper.getTaskContext().allocateFrame(); tb = new ArrayTupleBuilder(btree.getFieldCount()); dos = tb.getDataOutput(); appender = new FrameTupleAppender(treeIndexOpHelper.getTaskContext().getFrameSize()); appender.reset(writeBuffer, true); indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); /** set the search cursor */ rangePred.setLowKey(null, true); rangePred.setHighKey(null, true); cursor.reset(); indexAccessor.search(cursor, rangePred); /** set up current top tuple */ if (cursor.hasNext()) { cursor.next(); currentTopTuple = cursor.getTuple(); match = false; } } catch (Exception e) { treeIndexOpHelper.close(); throw new HyracksDataException(e); } }
public void initProbe() { sPartBuffs = new ByteBuffer[numOfSpilledParts]; for (int i = 0; i < numOfSpilledParts; i++) { sPartBuffs[i] = ctx.allocateFrame(); } curPBuff = new int[numOfPartitions]; int nextBuffIxToAlloc = 0; /* We only need to allocate one frame per spilled partition. * Resident partitions do not need frames in probe, as their tuples join * immediately with the resident build tuples using the inMemoryHashJoin */ for (int i = 0; i < numOfPartitions; i++) { curPBuff[i] = (pStatus.get(i)) ? nextBuffIxToAlloc++ : BUFFER_FOR_RESIDENT_PARTS; } probePSizeInTups = new int[numOfPartitions]; probeRFWriters = new RunFileWriter[numOfPartitions]; probeResBuff = ctx.allocateFrame(); probeTupAppenderToResident = new FrameTupleAppender(ctx.getFrameSize()); probeTupAppenderToResident.reset(probeResBuff, true); probeTupAppenderToSpilled = new FrameTupleAppender(ctx.getFrameSize()); }