public void initBuild() {
memBuffs = new ByteBuffer[memForJoin];
curPBuff = new int[numOfPartitions];
nextBuff = new int[memForJoin];
pStatus = new BitSet(numOfPartitions);
buildPSizeInTups = new int[numOfPartitions];
buildPSizeInFrames = new int[numOfPartitions];
freeFramesCounter = memForJoin - numOfPartitions;
for (int i = 0;
i < numOfPartitions;
i++) { // Allocating one buffer per partition and setting as the head of the chain of
// buffers for that partition
memBuffs[i] = ctx.allocateFrame();
curPBuff[i] = i;
nextBuff[i] = -1;
buildPSizeInFrames[i] = 1; // The dedicated initial buffer
}
nextFreeBuffIx =
((numOfPartitions < memForJoin)
? numOfPartitions
: NO_MORE_FREE_BUFFER); // Setting the chain of unallocated frames
for (int i = numOfPartitions; i < memBuffs.length; i++) {
nextBuff[i] = UNALLOCATED_FRAME;
}
buildTupAppender = new FrameTupleAppender(ctx.getFrameSize());
}
private void probeWrite(int pid, ByteBuffer buff) throws HyracksDataException {
RunFileWriter pWriter = probeRFWriters[pid];
if (pWriter == null) {
FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(rel1Name);
pWriter = new RunFileWriter(file, ctx.getIOManager());
pWriter.open();
probeRFWriters[pid] = pWriter;
}
pWriter.nextFrame(buff);
}
private void buildWrite(int pid, ByteBuffer buff) throws HyracksDataException {
RunFileWriter writer = buildRFWriters[pid];
if (writer == null) {
FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(rel0Name);
writer = new RunFileWriter(file, ctx.getIOManager());
writer.open();
buildRFWriters[pid] = writer;
}
writer.nextFrame(buff);
}
private void createInMemoryJoiner(int inMemTupCount) throws HyracksDataException {
ISerializableTable table = new SerializableHashTable(inMemTupCount, ctx);
this.inMemJoiner =
new InMemoryHashJoin(
ctx,
inMemTupCount,
new FrameTupleAccessor(ctx.getFrameSize(), probeRd),
probeHpc,
new FrameTupleAccessor(ctx.getFrameSize(), buildRd),
buildHpc,
new FrameTuplePairComparator(probeKeys, buildKeys, comparators),
isLeftOuter,
nullWriters1,
table);
}
public OptimizedHybridHashJoin(
IHyracksTaskContext ctx,
int memForJoin,
int numOfPartitions,
String rel0Name,
String rel1Name,
int[] keys0,
int[] keys1,
IBinaryComparator[] comparators,
RecordDescriptor buildRd,
RecordDescriptor probeRd,
ITuplePartitionComputer probeHpc,
ITuplePartitionComputer buildHpc) {
this.ctx = ctx;
this.memForJoin = memForJoin;
this.buildRd = buildRd;
this.probeRd = probeRd;
this.buildHpc = buildHpc;
this.probeHpc = probeHpc;
this.buildKeys = keys1;
this.probeKeys = keys0;
this.comparators = comparators;
this.rel0Name = rel0Name;
this.rel1Name = rel1Name;
this.numOfPartitions = numOfPartitions;
this.buildRFWriters = new RunFileWriter[numOfPartitions];
this.probeRFWriters = new RunFileWriter[numOfPartitions];
this.accessorBuild = new FrameTupleAccessor(ctx.getFrameSize(), buildRd);
this.accessorProbe = new FrameTupleAccessor(ctx.getFrameSize(), probeRd);
this.isLeftOuter = false;
this.nullWriters1 = null;
}
private void loadPartitionInMem(int pid, RunFileWriter wr, int[] buffs)
throws HyracksDataException {
RunFileReader r = wr.createReader();
r.open();
int counter = 0;
ByteBuffer mBuff = null;
reloadBuffer.clear();
while (r.nextFrame(reloadBuffer)) {
mBuff = memBuffs[buffs[counter]];
if (mBuff == null) {
mBuff = ctx.allocateFrame();
memBuffs[buffs[counter]] = mBuff;
}
FrameUtils.copy(reloadBuffer, mBuff);
counter++;
reloadBuffer.clear();
}
int curNext = nextBuff[buffs[buffs.length - 1]];
nextBuff[buffs[buffs.length - 1]] = END_OF_PARTITION;
nextFreeBuffIx = curNext;
r.close();
pStatus.set(pid, false);
buildRFWriters[pid] = null;
}
private void partitionTune() throws HyracksDataException {
reloadBuffer = ctx.allocateFrame();
ArrayList<Integer> reloadSet = selectPartitionsToReload();
for (int i = 0; i < reloadSet.size(); i++) {
int pid = reloadSet.get(i);
int[] buffsToLoad = new int[buildPSizeInFrames[pid]];
for (int j = 0; j < buffsToLoad.length; j++) {
buffsToLoad[j] = nextFreeBuffIx;
int oldNext = nextBuff[nextFreeBuffIx];
if (oldNext == UNALLOCATED_FRAME) {
nextFreeBuffIx++;
if (nextFreeBuffIx == memForJoin) { // No more free buffer
nextFreeBuffIx = NO_MORE_FREE_BUFFER;
}
} else {
nextFreeBuffIx = oldNext;
}
}
curPBuff[pid] = buffsToLoad[0];
for (int k = 1; k < buffsToLoad.length; k++) {
nextBuff[buffsToLoad[k - 1]] = buffsToLoad[k];
}
loadPartitionInMem(pid, buildRFWriters[pid], buffsToLoad);
}
reloadSet.clear();
reloadSet = null;
}
private int allocateFreeBuffer(int pid) {
if (nextFreeBuffIx != NO_MORE_FREE_BUFFER) {
if (memBuffs[nextFreeBuffIx] == null) {
memBuffs[nextFreeBuffIx] = ctx.allocateFrame();
}
int curPartBuffIx = curPBuff[pid];
curPBuff[pid] = nextFreeBuffIx;
int oldNext = nextBuff[nextFreeBuffIx];
nextBuff[nextFreeBuffIx] = curPartBuffIx;
if (oldNext == UNALLOCATED_FRAME) {
nextFreeBuffIx++;
if (nextFreeBuffIx == memForJoin) { // No more free buffer
nextFreeBuffIx = NO_MORE_FREE_BUFFER;
}
} else {
nextFreeBuffIx = oldNext;
}
(memBuffs[curPBuff[pid]]).clear();
freeFramesCounter--;
return (curPBuff[pid]);
} else {
return NO_MORE_FREE_BUFFER; // A partitions needs to be spilled (if feasible)
}
}
public FeedMessageOperatorNodePushable(
IHyracksTaskContext ctx,
FeedConnectionId connectionId,
IFeedMessage feedMessage,
int partition,
int nPartitions) {
this.connectionId = connectionId;
this.message = feedMessage;
this.partition = partition;
IAsterixAppRuntimeContext runtimeCtx =
(IAsterixAppRuntimeContext)
ctx.getJobletContext().getApplicationContext().getApplicationObject();
this.feedManager = runtimeCtx.getFeedManager();
}
public void initProbe() {
sPartBuffs = new ByteBuffer[numOfSpilledParts];
for (int i = 0; i < numOfSpilledParts; i++) {
sPartBuffs[i] = ctx.allocateFrame();
}
curPBuff = new int[numOfPartitions];
int nextBuffIxToAlloc = 0;
/* We only need to allocate one frame per spilled partition.
* Resident partitions do not need frames in probe, as their tuples join
* immediately with the resident build tuples using the inMemoryHashJoin */
for (int i = 0; i < numOfPartitions; i++) {
curPBuff[i] = (pStatus.get(i)) ? nextBuffIxToAlloc++ : BUFFER_FOR_RESIDENT_PARTS;
}
probePSizeInTups = new int[numOfPartitions];
probeRFWriters = new RunFileWriter[numOfPartitions];
probeResBuff = ctx.allocateFrame();
probeTupAppenderToResident = new FrameTupleAppender(ctx.getFrameSize());
probeTupAppenderToResident.reset(probeResBuff, true);
probeTupAppenderToSpilled = new FrameTupleAppender(ctx.getFrameSize());
}
@SuppressWarnings("unchecked")
public static RecordDescriptor getRecordDescriptorFromKeyValueClasses(
IHyracksTaskContext ctx, Configuration conf, String className1, String className2)
throws HyracksException {
RecordDescriptor recordDescriptor = null;
try {
recordDescriptor =
DatatypeHelper.createKeyValueRecordDescriptor(
(Class<? extends Writable>) ctx.getJobletContext().loadClass(className1),
(Class<? extends Writable>) ctx.getJobletContext().loadClass(className2),
conf);
} catch (Exception cnfe) {
throw new HyracksException(cnfe);
}
return recordDescriptor;
}
public OptimizedHybridHashJoin(
IHyracksTaskContext ctx,
int memForJoin,
int numOfPartitions,
String rel0Name,
String rel1Name,
int[] keys0,
int[] keys1,
IBinaryComparator[] comparators,
RecordDescriptor buildRd,
RecordDescriptor probeRd,
ITuplePartitionComputer probeHpc,
ITuplePartitionComputer buildHpc,
boolean isLeftOuter,
INullWriterFactory[] nullWriterFactories1) {
this.ctx = ctx;
this.memForJoin = memForJoin;
this.buildRd = buildRd;
this.probeRd = probeRd;
this.buildHpc = buildHpc;
this.probeHpc = probeHpc;
this.buildKeys = keys1;
this.probeKeys = keys0;
this.comparators = comparators;
this.rel0Name = rel0Name;
this.rel1Name = rel1Name;
this.numOfPartitions = numOfPartitions;
this.buildRFWriters = new RunFileWriter[numOfPartitions];
this.probeRFWriters = new RunFileWriter[numOfPartitions];
this.accessorBuild = new FrameTupleAccessor(ctx.getFrameSize(), buildRd);
this.accessorProbe = new FrameTupleAccessor(ctx.getFrameSize(), probeRd);
this.isLeftOuter = isLeftOuter;
this.nullWriters1 = isLeftOuter ? new INullWriter[nullWriterFactories1.length] : null;
if (isLeftOuter) {
for (int i = 0; i < nullWriterFactories1.length; i++) {
nullWriters1[i] = nullWriterFactories1[i].createNullWriter();
}
}
}
@SuppressWarnings({"unchecked", "rawtypes"})
public static RecordDescriptor getRecordDescriptorFromWritableClasses(
IHyracksTaskContext ctx, Configuration conf, String... classNames) throws HyracksException {
RecordDescriptor recordDescriptor = null;
ISerializerDeserializer[] serdes = new ISerializerDeserializer[classNames.length];
try {
int i = 0;
for (String className : classNames) {
Class<? extends Writable> c =
(Class<? extends Writable>) ctx.getJobletContext().loadClass(className);
serdes[i++] = DatatypeHelper.createSerializerDeserializer(c, conf, ctx);
// System.out.println("thread " + Thread.currentThread().getId() + " after creating serde "
// + c.getClassLoader());
}
} catch (Exception cnfe) {
throw new HyracksException(cnfe);
}
recordDescriptor = new RecordDescriptor(serdes);
return recordDescriptor;
}
public static RuntimeContext get(IHyracksTaskContext ctx) {
return (RuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject();
}