public void initBuild() {
    memBuffs = new ByteBuffer[memForJoin];
    curPBuff = new int[numOfPartitions];
    nextBuff = new int[memForJoin];
    pStatus = new BitSet(numOfPartitions);
    buildPSizeInTups = new int[numOfPartitions];

    buildPSizeInFrames = new int[numOfPartitions];
    freeFramesCounter = memForJoin - numOfPartitions;

    for (int i = 0;
        i < numOfPartitions;
        i++) { // Allocating one buffer per partition and setting as the head of the chain of
               // buffers for that partition
      memBuffs[i] = ctx.allocateFrame();
      curPBuff[i] = i;
      nextBuff[i] = -1;
      buildPSizeInFrames[i] = 1; // The dedicated initial buffer
    }

    nextFreeBuffIx =
        ((numOfPartitions < memForJoin)
            ? numOfPartitions
            : NO_MORE_FREE_BUFFER); // Setting the chain of unallocated frames
    for (int i = numOfPartitions; i < memBuffs.length; i++) {
      nextBuff[i] = UNALLOCATED_FRAME;
    }

    buildTupAppender = new FrameTupleAppender(ctx.getFrameSize());
  }
 private void probeWrite(int pid, ByteBuffer buff) throws HyracksDataException {
   RunFileWriter pWriter = probeRFWriters[pid];
   if (pWriter == null) {
     FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(rel1Name);
     pWriter = new RunFileWriter(file, ctx.getIOManager());
     pWriter.open();
     probeRFWriters[pid] = pWriter;
   }
   pWriter.nextFrame(buff);
 }
 private void buildWrite(int pid, ByteBuffer buff) throws HyracksDataException {
   RunFileWriter writer = buildRFWriters[pid];
   if (writer == null) {
     FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(rel0Name);
     writer = new RunFileWriter(file, ctx.getIOManager());
     writer.open();
     buildRFWriters[pid] = writer;
   }
   writer.nextFrame(buff);
 }
 private void createInMemoryJoiner(int inMemTupCount) throws HyracksDataException {
   ISerializableTable table = new SerializableHashTable(inMemTupCount, ctx);
   this.inMemJoiner =
       new InMemoryHashJoin(
           ctx,
           inMemTupCount,
           new FrameTupleAccessor(ctx.getFrameSize(), probeRd),
           probeHpc,
           new FrameTupleAccessor(ctx.getFrameSize(), buildRd),
           buildHpc,
           new FrameTuplePairComparator(probeKeys, buildKeys, comparators),
           isLeftOuter,
           nullWriters1,
           table);
 }
  public OptimizedHybridHashJoin(
      IHyracksTaskContext ctx,
      int memForJoin,
      int numOfPartitions,
      String rel0Name,
      String rel1Name,
      int[] keys0,
      int[] keys1,
      IBinaryComparator[] comparators,
      RecordDescriptor buildRd,
      RecordDescriptor probeRd,
      ITuplePartitionComputer probeHpc,
      ITuplePartitionComputer buildHpc) {
    this.ctx = ctx;
    this.memForJoin = memForJoin;
    this.buildRd = buildRd;
    this.probeRd = probeRd;
    this.buildHpc = buildHpc;
    this.probeHpc = probeHpc;
    this.buildKeys = keys1;
    this.probeKeys = keys0;
    this.comparators = comparators;
    this.rel0Name = rel0Name;
    this.rel1Name = rel1Name;

    this.numOfPartitions = numOfPartitions;
    this.buildRFWriters = new RunFileWriter[numOfPartitions];
    this.probeRFWriters = new RunFileWriter[numOfPartitions];

    this.accessorBuild = new FrameTupleAccessor(ctx.getFrameSize(), buildRd);
    this.accessorProbe = new FrameTupleAccessor(ctx.getFrameSize(), probeRd);

    this.isLeftOuter = false;
    this.nullWriters1 = null;
  }
  private void loadPartitionInMem(int pid, RunFileWriter wr, int[] buffs)
      throws HyracksDataException {
    RunFileReader r = wr.createReader();
    r.open();
    int counter = 0;
    ByteBuffer mBuff = null;
    reloadBuffer.clear();
    while (r.nextFrame(reloadBuffer)) {
      mBuff = memBuffs[buffs[counter]];
      if (mBuff == null) {
        mBuff = ctx.allocateFrame();
        memBuffs[buffs[counter]] = mBuff;
      }
      FrameUtils.copy(reloadBuffer, mBuff);
      counter++;
      reloadBuffer.clear();
    }

    int curNext = nextBuff[buffs[buffs.length - 1]];
    nextBuff[buffs[buffs.length - 1]] = END_OF_PARTITION;
    nextFreeBuffIx = curNext;

    r.close();
    pStatus.set(pid, false);
    buildRFWriters[pid] = null;
  }
 private void partitionTune() throws HyracksDataException {
   reloadBuffer = ctx.allocateFrame();
   ArrayList<Integer> reloadSet = selectPartitionsToReload();
   for (int i = 0; i < reloadSet.size(); i++) {
     int pid = reloadSet.get(i);
     int[] buffsToLoad = new int[buildPSizeInFrames[pid]];
     for (int j = 0; j < buffsToLoad.length; j++) {
       buffsToLoad[j] = nextFreeBuffIx;
       int oldNext = nextBuff[nextFreeBuffIx];
       if (oldNext == UNALLOCATED_FRAME) {
         nextFreeBuffIx++;
         if (nextFreeBuffIx == memForJoin) { // No more free buffer
           nextFreeBuffIx = NO_MORE_FREE_BUFFER;
         }
       } else {
         nextFreeBuffIx = oldNext;
       }
     }
     curPBuff[pid] = buffsToLoad[0];
     for (int k = 1; k < buffsToLoad.length; k++) {
       nextBuff[buffsToLoad[k - 1]] = buffsToLoad[k];
     }
     loadPartitionInMem(pid, buildRFWriters[pid], buffsToLoad);
   }
   reloadSet.clear();
   reloadSet = null;
 }
  private int allocateFreeBuffer(int pid) {
    if (nextFreeBuffIx != NO_MORE_FREE_BUFFER) {
      if (memBuffs[nextFreeBuffIx] == null) {
        memBuffs[nextFreeBuffIx] = ctx.allocateFrame();
      }
      int curPartBuffIx = curPBuff[pid];
      curPBuff[pid] = nextFreeBuffIx;
      int oldNext = nextBuff[nextFreeBuffIx];
      nextBuff[nextFreeBuffIx] = curPartBuffIx;
      if (oldNext == UNALLOCATED_FRAME) {
        nextFreeBuffIx++;
        if (nextFreeBuffIx == memForJoin) { // No more free buffer
          nextFreeBuffIx = NO_MORE_FREE_BUFFER;
        }
      } else {
        nextFreeBuffIx = oldNext;
      }
      (memBuffs[curPBuff[pid]]).clear();

      freeFramesCounter--;
      return (curPBuff[pid]);
    } else {
      return NO_MORE_FREE_BUFFER; // A partitions needs to be spilled (if feasible)
    }
  }
 public FeedMessageOperatorNodePushable(
     IHyracksTaskContext ctx,
     FeedConnectionId connectionId,
     IFeedMessage feedMessage,
     int partition,
     int nPartitions) {
   this.connectionId = connectionId;
   this.message = feedMessage;
   this.partition = partition;
   IAsterixAppRuntimeContext runtimeCtx =
       (IAsterixAppRuntimeContext)
           ctx.getJobletContext().getApplicationContext().getApplicationObject();
   this.feedManager = runtimeCtx.getFeedManager();
 }
  public void initProbe() {

    sPartBuffs = new ByteBuffer[numOfSpilledParts];
    for (int i = 0; i < numOfSpilledParts; i++) {
      sPartBuffs[i] = ctx.allocateFrame();
    }
    curPBuff = new int[numOfPartitions];
    int nextBuffIxToAlloc = 0;
    /* We only need to allocate one frame per spilled partition.
     * Resident partitions do not need frames in probe, as their tuples join
     * immediately with the resident build tuples using the inMemoryHashJoin */
    for (int i = 0; i < numOfPartitions; i++) {
      curPBuff[i] = (pStatus.get(i)) ? nextBuffIxToAlloc++ : BUFFER_FOR_RESIDENT_PARTS;
    }
    probePSizeInTups = new int[numOfPartitions];
    probeRFWriters = new RunFileWriter[numOfPartitions];

    probeResBuff = ctx.allocateFrame();

    probeTupAppenderToResident = new FrameTupleAppender(ctx.getFrameSize());
    probeTupAppenderToResident.reset(probeResBuff, true);

    probeTupAppenderToSpilled = new FrameTupleAppender(ctx.getFrameSize());
  }
Beispiel #11
0
 @SuppressWarnings("unchecked")
 public static RecordDescriptor getRecordDescriptorFromKeyValueClasses(
     IHyracksTaskContext ctx, Configuration conf, String className1, String className2)
     throws HyracksException {
   RecordDescriptor recordDescriptor = null;
   try {
     recordDescriptor =
         DatatypeHelper.createKeyValueRecordDescriptor(
             (Class<? extends Writable>) ctx.getJobletContext().loadClass(className1),
             (Class<? extends Writable>) ctx.getJobletContext().loadClass(className2),
             conf);
   } catch (Exception cnfe) {
     throw new HyracksException(cnfe);
   }
   return recordDescriptor;
 }
  public OptimizedHybridHashJoin(
      IHyracksTaskContext ctx,
      int memForJoin,
      int numOfPartitions,
      String rel0Name,
      String rel1Name,
      int[] keys0,
      int[] keys1,
      IBinaryComparator[] comparators,
      RecordDescriptor buildRd,
      RecordDescriptor probeRd,
      ITuplePartitionComputer probeHpc,
      ITuplePartitionComputer buildHpc,
      boolean isLeftOuter,
      INullWriterFactory[] nullWriterFactories1) {
    this.ctx = ctx;
    this.memForJoin = memForJoin;
    this.buildRd = buildRd;
    this.probeRd = probeRd;
    this.buildHpc = buildHpc;
    this.probeHpc = probeHpc;
    this.buildKeys = keys1;
    this.probeKeys = keys0;
    this.comparators = comparators;
    this.rel0Name = rel0Name;
    this.rel1Name = rel1Name;

    this.numOfPartitions = numOfPartitions;
    this.buildRFWriters = new RunFileWriter[numOfPartitions];
    this.probeRFWriters = new RunFileWriter[numOfPartitions];

    this.accessorBuild = new FrameTupleAccessor(ctx.getFrameSize(), buildRd);
    this.accessorProbe = new FrameTupleAccessor(ctx.getFrameSize(), probeRd);

    this.isLeftOuter = isLeftOuter;

    this.nullWriters1 = isLeftOuter ? new INullWriter[nullWriterFactories1.length] : null;
    if (isLeftOuter) {
      for (int i = 0; i < nullWriterFactories1.length; i++) {
        nullWriters1[i] = nullWriterFactories1[i].createNullWriter();
      }
    }
  }
Beispiel #13
0
 @SuppressWarnings({"unchecked", "rawtypes"})
 public static RecordDescriptor getRecordDescriptorFromWritableClasses(
     IHyracksTaskContext ctx, Configuration conf, String... classNames) throws HyracksException {
   RecordDescriptor recordDescriptor = null;
   ISerializerDeserializer[] serdes = new ISerializerDeserializer[classNames.length];
   try {
     int i = 0;
     for (String className : classNames) {
       Class<? extends Writable> c =
           (Class<? extends Writable>) ctx.getJobletContext().loadClass(className);
       serdes[i++] = DatatypeHelper.createSerializerDeserializer(c, conf, ctx);
       // System.out.println("thread " + Thread.currentThread().getId() + " after creating serde "
       // + c.getClassLoader());
     }
   } catch (Exception cnfe) {
     throw new HyracksException(cnfe);
   }
   recordDescriptor = new RecordDescriptor(serdes);
   return recordDescriptor;
 }
 public static RuntimeContext get(IHyracksTaskContext ctx) {
   return (RuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject();
 }