public void initBuild() { memBuffs = new ByteBuffer[memForJoin]; curPBuff = new int[numOfPartitions]; nextBuff = new int[memForJoin]; pStatus = new BitSet(numOfPartitions); buildPSizeInTups = new int[numOfPartitions]; buildPSizeInFrames = new int[numOfPartitions]; freeFramesCounter = memForJoin - numOfPartitions; for (int i = 0; i < numOfPartitions; i++) { // Allocating one buffer per partition and setting as the head of the chain of // buffers for that partition memBuffs[i] = ctx.allocateFrame(); curPBuff[i] = i; nextBuff[i] = -1; buildPSizeInFrames[i] = 1; // The dedicated initial buffer } nextFreeBuffIx = ((numOfPartitions < memForJoin) ? numOfPartitions : NO_MORE_FREE_BUFFER); // Setting the chain of unallocated frames for (int i = numOfPartitions; i < memBuffs.length; i++) { nextBuff[i] = UNALLOCATED_FRAME; } buildTupAppender = new FrameTupleAppender(ctx.getFrameSize()); }
private void probeWrite(int pid, ByteBuffer buff) throws HyracksDataException { RunFileWriter pWriter = probeRFWriters[pid]; if (pWriter == null) { FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(rel1Name); pWriter = new RunFileWriter(file, ctx.getIOManager()); pWriter.open(); probeRFWriters[pid] = pWriter; } pWriter.nextFrame(buff); }
private void buildWrite(int pid, ByteBuffer buff) throws HyracksDataException { RunFileWriter writer = buildRFWriters[pid]; if (writer == null) { FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(rel0Name); writer = new RunFileWriter(file, ctx.getIOManager()); writer.open(); buildRFWriters[pid] = writer; } writer.nextFrame(buff); }
private void createInMemoryJoiner(int inMemTupCount) throws HyracksDataException { ISerializableTable table = new SerializableHashTable(inMemTupCount, ctx); this.inMemJoiner = new InMemoryHashJoin( ctx, inMemTupCount, new FrameTupleAccessor(ctx.getFrameSize(), probeRd), probeHpc, new FrameTupleAccessor(ctx.getFrameSize(), buildRd), buildHpc, new FrameTuplePairComparator(probeKeys, buildKeys, comparators), isLeftOuter, nullWriters1, table); }
public OptimizedHybridHashJoin( IHyracksTaskContext ctx, int memForJoin, int numOfPartitions, String rel0Name, String rel1Name, int[] keys0, int[] keys1, IBinaryComparator[] comparators, RecordDescriptor buildRd, RecordDescriptor probeRd, ITuplePartitionComputer probeHpc, ITuplePartitionComputer buildHpc) { this.ctx = ctx; this.memForJoin = memForJoin; this.buildRd = buildRd; this.probeRd = probeRd; this.buildHpc = buildHpc; this.probeHpc = probeHpc; this.buildKeys = keys1; this.probeKeys = keys0; this.comparators = comparators; this.rel0Name = rel0Name; this.rel1Name = rel1Name; this.numOfPartitions = numOfPartitions; this.buildRFWriters = new RunFileWriter[numOfPartitions]; this.probeRFWriters = new RunFileWriter[numOfPartitions]; this.accessorBuild = new FrameTupleAccessor(ctx.getFrameSize(), buildRd); this.accessorProbe = new FrameTupleAccessor(ctx.getFrameSize(), probeRd); this.isLeftOuter = false; this.nullWriters1 = null; }
private void loadPartitionInMem(int pid, RunFileWriter wr, int[] buffs) throws HyracksDataException { RunFileReader r = wr.createReader(); r.open(); int counter = 0; ByteBuffer mBuff = null; reloadBuffer.clear(); while (r.nextFrame(reloadBuffer)) { mBuff = memBuffs[buffs[counter]]; if (mBuff == null) { mBuff = ctx.allocateFrame(); memBuffs[buffs[counter]] = mBuff; } FrameUtils.copy(reloadBuffer, mBuff); counter++; reloadBuffer.clear(); } int curNext = nextBuff[buffs[buffs.length - 1]]; nextBuff[buffs[buffs.length - 1]] = END_OF_PARTITION; nextFreeBuffIx = curNext; r.close(); pStatus.set(pid, false); buildRFWriters[pid] = null; }
private void partitionTune() throws HyracksDataException { reloadBuffer = ctx.allocateFrame(); ArrayList<Integer> reloadSet = selectPartitionsToReload(); for (int i = 0; i < reloadSet.size(); i++) { int pid = reloadSet.get(i); int[] buffsToLoad = new int[buildPSizeInFrames[pid]]; for (int j = 0; j < buffsToLoad.length; j++) { buffsToLoad[j] = nextFreeBuffIx; int oldNext = nextBuff[nextFreeBuffIx]; if (oldNext == UNALLOCATED_FRAME) { nextFreeBuffIx++; if (nextFreeBuffIx == memForJoin) { // No more free buffer nextFreeBuffIx = NO_MORE_FREE_BUFFER; } } else { nextFreeBuffIx = oldNext; } } curPBuff[pid] = buffsToLoad[0]; for (int k = 1; k < buffsToLoad.length; k++) { nextBuff[buffsToLoad[k - 1]] = buffsToLoad[k]; } loadPartitionInMem(pid, buildRFWriters[pid], buffsToLoad); } reloadSet.clear(); reloadSet = null; }
private int allocateFreeBuffer(int pid) { if (nextFreeBuffIx != NO_MORE_FREE_BUFFER) { if (memBuffs[nextFreeBuffIx] == null) { memBuffs[nextFreeBuffIx] = ctx.allocateFrame(); } int curPartBuffIx = curPBuff[pid]; curPBuff[pid] = nextFreeBuffIx; int oldNext = nextBuff[nextFreeBuffIx]; nextBuff[nextFreeBuffIx] = curPartBuffIx; if (oldNext == UNALLOCATED_FRAME) { nextFreeBuffIx++; if (nextFreeBuffIx == memForJoin) { // No more free buffer nextFreeBuffIx = NO_MORE_FREE_BUFFER; } } else { nextFreeBuffIx = oldNext; } (memBuffs[curPBuff[pid]]).clear(); freeFramesCounter--; return (curPBuff[pid]); } else { return NO_MORE_FREE_BUFFER; // A partitions needs to be spilled (if feasible) } }
public FeedMessageOperatorNodePushable( IHyracksTaskContext ctx, FeedConnectionId connectionId, IFeedMessage feedMessage, int partition, int nPartitions) { this.connectionId = connectionId; this.message = feedMessage; this.partition = partition; IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject(); this.feedManager = runtimeCtx.getFeedManager(); }
public void initProbe() { sPartBuffs = new ByteBuffer[numOfSpilledParts]; for (int i = 0; i < numOfSpilledParts; i++) { sPartBuffs[i] = ctx.allocateFrame(); } curPBuff = new int[numOfPartitions]; int nextBuffIxToAlloc = 0; /* We only need to allocate one frame per spilled partition. * Resident partitions do not need frames in probe, as their tuples join * immediately with the resident build tuples using the inMemoryHashJoin */ for (int i = 0; i < numOfPartitions; i++) { curPBuff[i] = (pStatus.get(i)) ? nextBuffIxToAlloc++ : BUFFER_FOR_RESIDENT_PARTS; } probePSizeInTups = new int[numOfPartitions]; probeRFWriters = new RunFileWriter[numOfPartitions]; probeResBuff = ctx.allocateFrame(); probeTupAppenderToResident = new FrameTupleAppender(ctx.getFrameSize()); probeTupAppenderToResident.reset(probeResBuff, true); probeTupAppenderToSpilled = new FrameTupleAppender(ctx.getFrameSize()); }
@SuppressWarnings("unchecked") public static RecordDescriptor getRecordDescriptorFromKeyValueClasses( IHyracksTaskContext ctx, Configuration conf, String className1, String className2) throws HyracksException { RecordDescriptor recordDescriptor = null; try { recordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor( (Class<? extends Writable>) ctx.getJobletContext().loadClass(className1), (Class<? extends Writable>) ctx.getJobletContext().loadClass(className2), conf); } catch (Exception cnfe) { throw new HyracksException(cnfe); } return recordDescriptor; }
public OptimizedHybridHashJoin( IHyracksTaskContext ctx, int memForJoin, int numOfPartitions, String rel0Name, String rel1Name, int[] keys0, int[] keys1, IBinaryComparator[] comparators, RecordDescriptor buildRd, RecordDescriptor probeRd, ITuplePartitionComputer probeHpc, ITuplePartitionComputer buildHpc, boolean isLeftOuter, INullWriterFactory[] nullWriterFactories1) { this.ctx = ctx; this.memForJoin = memForJoin; this.buildRd = buildRd; this.probeRd = probeRd; this.buildHpc = buildHpc; this.probeHpc = probeHpc; this.buildKeys = keys1; this.probeKeys = keys0; this.comparators = comparators; this.rel0Name = rel0Name; this.rel1Name = rel1Name; this.numOfPartitions = numOfPartitions; this.buildRFWriters = new RunFileWriter[numOfPartitions]; this.probeRFWriters = new RunFileWriter[numOfPartitions]; this.accessorBuild = new FrameTupleAccessor(ctx.getFrameSize(), buildRd); this.accessorProbe = new FrameTupleAccessor(ctx.getFrameSize(), probeRd); this.isLeftOuter = isLeftOuter; this.nullWriters1 = isLeftOuter ? new INullWriter[nullWriterFactories1.length] : null; if (isLeftOuter) { for (int i = 0; i < nullWriterFactories1.length; i++) { nullWriters1[i] = nullWriterFactories1[i].createNullWriter(); } } }
@SuppressWarnings({"unchecked", "rawtypes"}) public static RecordDescriptor getRecordDescriptorFromWritableClasses( IHyracksTaskContext ctx, Configuration conf, String... classNames) throws HyracksException { RecordDescriptor recordDescriptor = null; ISerializerDeserializer[] serdes = new ISerializerDeserializer[classNames.length]; try { int i = 0; for (String className : classNames) { Class<? extends Writable> c = (Class<? extends Writable>) ctx.getJobletContext().loadClass(className); serdes[i++] = DatatypeHelper.createSerializerDeserializer(c, conf, ctx); // System.out.println("thread " + Thread.currentThread().getId() + " after creating serde " // + c.getClassLoader()); } } catch (Exception cnfe) { throw new HyracksException(cnfe); } recordDescriptor = new RecordDescriptor(serdes); return recordDescriptor; }
public static RuntimeContext get(IHyracksTaskContext ctx) { return (RuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject(); }