Exemplo n.º 1
0
 private boolean bytesEquals(VectorHashKeyWrapper keyThat) {
   // By the time we enter here the byteValues.lentgh and isNull must have already been compared
   for (int i = 0; i < byteValues.length; ++i) {
     // the byte comparison is potentially expensive so is better to branch on null
     if (!isNull[longValues.length + doubleValues.length + i]) {
       if (0
           != StringExpr.compare(
               byteValues[i],
               byteStarts[i],
               byteLengths[i],
               keyThat.byteValues[i],
               keyThat.byteStarts[i],
               keyThat.byteLengths[i])) {
         return false;
       }
     }
   }
   return true;
 }
  @Override
  public void process(Object row, int tag) throws HiveException {

    try {
      VectorizedRowBatch batch = (VectorizedRowBatch) row;

      alias = (byte) tag;

      if (needCommonSetup) {
        // Our one time process method initialization.
        commonSetup(batch);

        /*
         * Initialize Single-Column String members for this specialized class.
         */

        singleJoinColumn = bigTableKeyColumnMap[0];

        needCommonSetup = false;
      }

      if (needHashTableSetup) {
        // Setup our hash table specialization.  It will be the first time the process
        // method is called, or after a Hybrid Grace reload.

        /*
         * Get our Single-Column String hash set information for this specialized class.
         */

        hashSet = (VectorMapJoinBytesHashSet) vectorMapJoinHashTable;

        needHashTableSetup = false;
      }

      batchCounter++;

      // Do the per-batch setup for an left semi join.

      // (Currently none)
      // leftSemiPerBatchSetup(batch);

      // For left semi joins, we may apply the filter(s) now.
      for (VectorExpression ve : bigTableFilterExpressions) {
        ve.evaluate(batch);
      }

      final int inputLogicalSize = batch.size;

      if (inputLogicalSize == 0) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(CLASS_NAME + " batch #" + batchCounter + " empty");
        }
        return;
      }

      // Perform any key expressions.  Results will go into scratch columns.
      if (bigTableKeyExpressions != null) {
        for (VectorExpression ve : bigTableKeyExpressions) {
          ve.evaluate(batch);
        }
      }

      /*
       * Single-Column String specific declarations.
       */

      // The one join column for this specialized class.
      BytesColumnVector joinColVector = (BytesColumnVector) batch.cols[singleJoinColumn];
      byte[][] vector = joinColVector.vector;
      int[] start = joinColVector.start;
      int[] length = joinColVector.length;

      /*
       * Single-Column Long check for repeating.
       */

      // Check single column for repeating.
      boolean allKeyInputColumnsRepeating = joinColVector.isRepeating;

      if (allKeyInputColumnsRepeating) {

        /*
         * Repeating.
         */

        // All key input columns are repeating.  Generate key once.  Lookup once.
        // Since the key is repeated, we must use entry 0 regardless of selectedInUse.

        /*
         * Single-Column String specific repeated lookup.
         */

        byte[] keyBytes = vector[0];
        int keyStart = start[0];
        int keyLength = length[0];
        JoinUtil.JoinResult joinResult =
            hashSet.contains(keyBytes, keyStart, keyLength, hashSetResults[0]);

        /*
         * Common repeated join result processing.
         */

        if (LOG.isDebugEnabled()) {
          LOG.debug(
              CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
        }
        finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]);
      } else {

        /*
         * NOT Repeating.
         */

        if (LOG.isDebugEnabled()) {
          LOG.debug(CLASS_NAME + " batch #" + batchCounter + " non-repeated");
        }

        // We remember any matching rows in matchs / matchSize.  At the end of the loop,
        // selected / batch.size will represent both matching and non-matching rows for outer join.
        // Only deferred rows will have been removed from selected.
        int selected[] = batch.selected;
        boolean selectedInUse = batch.selectedInUse;

        int hashSetResultCount = 0;
        int allMatchCount = 0;
        int spillCount = 0;

        /*
         * Single-Column String specific variables.
         */

        int saveKeyBatchIndex = -1;

        // We optimize performance by only looking up the first key in a series of equal keys.
        boolean haveSaveKey = false;
        JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH;

        // Logical loop over the rows in the batch since the batch may have selected in use.
        for (int logical = 0; logical < inputLogicalSize; logical++) {
          int batchIndex = (selectedInUse ? selected[logical] : logical);

          /*
           * Single-Column String get key.
           */

          // Implicit -- use batchIndex.

          /*
           * Equal key series checking.
           */

          if (!haveSaveKey
              || StringExpr.compare(
                      vector[saveKeyBatchIndex],
                      start[saveKeyBatchIndex],
                      length[saveKeyBatchIndex],
                      vector[batchIndex],
                      start[batchIndex],
                      length[batchIndex])
                  != 0) {

            // New key.

            if (haveSaveKey) {
              // Move on with our counts.
              switch (saveJoinResult) {
                case MATCH:
                  // We have extracted the existence from the hash set result, so we don't keep it.
                  break;
                case SPILL:
                  // We keep the hash set result for its spill information.
                  hashSetResultCount++;
                  break;
                case NOMATCH:
                  break;
              }
            }

            // Regardless of our matching result, we keep that information to make multiple use
            // of it for a possible series of equal keys.
            haveSaveKey = true;

            /*
             * Single-Column String specific save key and lookup.
             */

            saveKeyBatchIndex = batchIndex;

            /*
             * Single-Column String specific lookup key.
             */

            byte[] keyBytes = vector[batchIndex];
            int keyStart = start[batchIndex];
            int keyLength = length[batchIndex];
            saveJoinResult =
                hashSet.contains(keyBytes, keyStart, keyLength, hashSetResults[hashSetResultCount]);

            /*
             * Common left-semi join result processing.
             */

            switch (saveJoinResult) {
              case MATCH:
                allMatchs[allMatchCount++] = batchIndex;
                // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH
                // isSingleValue " + equalKeySeriesIsSingleValue[equalKeySeriesCount] + " currentKey
                // " + currentKey);
                break;

              case SPILL:
                spills[spillCount] = batchIndex;
                spillHashMapResultIndices[spillCount] = hashSetResultCount;
                spillCount++;
                break;

              case NOMATCH:
                // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH"
                // + " currentKey " + currentKey);
                break;
            }
          } else {
            // Series of equal keys.

            switch (saveJoinResult) {
              case MATCH:
                allMatchs[allMatchCount++] = batchIndex;
                // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH
                // duplicate");
                break;

              case SPILL:
                spills[spillCount] = batchIndex;
                spillHashMapResultIndices[spillCount] = hashSetResultCount;
                spillCount++;
                break;

              case NOMATCH:
                // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH
                // duplicate");
                break;
            }
          }
        }

        if (haveSaveKey) {
          // Update our counts for the last key.
          switch (saveJoinResult) {
            case MATCH:
              // We have extracted the existence from the hash set result, so we don't keep it.
              break;
            case SPILL:
              // We keep the hash set result for its spill information.
              hashSetResultCount++;
              break;
            case NOMATCH:
              break;
          }
        }

        if (LOG.isDebugEnabled()) {
          LOG.debug(
              CLASS_NAME
                  + " allMatchs "
                  + intArrayToRangesString(allMatchs, allMatchCount)
                  + " spills "
                  + intArrayToRangesString(spills, spillCount)
                  + " spillHashMapResultIndices "
                  + intArrayToRangesString(spillHashMapResultIndices, spillCount)
                  + " hashMapResults "
                  + Arrays.toString(Arrays.copyOfRange(hashSetResults, 0, hashSetResultCount)));
        }

        finishLeftSemi(
            batch, allMatchCount, spillCount, (VectorMapJoinHashTableResult[]) hashSetResults);
      }

      if (batch.size > 0) {
        // Forward any remaining selected rows.
        forwardBigTableBatch(batch);
      }

    } catch (IOException e) {
      throw new HiveException(e);
    } catch (Exception e) {
      throw new HiveException(e);
    }
  }
  @Override
  public void process(Object row, int tag) throws HiveException {

    try {
      VectorizedRowBatch batch = (VectorizedRowBatch) row;

      alias = (byte) tag;

      if (needCommonSetup) {
        // Our one time process method initialization.
        commonSetup(batch);

        /*
         * Initialize Single-Column String members for this specialized class.
         */

        singleJoinColumn = bigTableKeyColumnMap[0];

        needCommonSetup = false;
      }

      if (needHashTableSetup) {
        // Setup our hash table specialization.  It will be the first time the process
        // method is called, or after a Hybrid Grace reload.

        /*
         * Get our Single-Column String hash map information for this specialized class.
         */

        hashMap = (VectorMapJoinBytesHashMap) vectorMapJoinHashTable;

        needHashTableSetup = false;
      }

      batchCounter++;

      // Do the per-batch setup for an outer join.

      outerPerBatchSetup(batch);

      // For outer join, DO NOT apply filters yet.  It is incorrect for outer join to
      // apply the filter before hash table matching.

      final int inputLogicalSize = batch.size;

      if (inputLogicalSize == 0) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(CLASS_NAME + " batch #" + batchCounter + " empty");
        }
        return;
      }

      // Perform any key expressions.  Results will go into scratch columns.
      if (bigTableKeyExpressions != null) {
        for (VectorExpression ve : bigTableKeyExpressions) {
          ve.evaluate(batch);
        }
      }

      // We rebuild in-place the selected array with rows destine to be forwarded.
      int numSel = 0;

      /*
       * Single-Column String specific declarations.
       */

      // The one join column for this specialized class.
      BytesColumnVector joinColVector = (BytesColumnVector) batch.cols[singleJoinColumn];
      byte[][] vector = joinColVector.vector;
      int[] start = joinColVector.start;
      int[] length = joinColVector.length;

      /*
       * Single-Column String check for repeating.
       */

      // Check single column for repeating.
      boolean allKeyInputColumnsRepeating = joinColVector.isRepeating;

      if (allKeyInputColumnsRepeating) {

        /*
         * Repeating.
         */

        // All key input columns are repeating.  Generate key once.  Lookup once.
        // Since the key is repeated, we must use entry 0 regardless of selectedInUse.

        /*
         * Single-Column String specific repeated lookup.
         */

        JoinUtil.JoinResult joinResult;
        if (!joinColVector.noNulls && joinColVector.isNull[0]) {
          // Null key is no match for whole batch.
          joinResult = JoinUtil.JoinResult.NOMATCH;
        } else {
          // Handle *repeated* join key, if found.
          byte[] keyBytes = vector[0];
          int keyStart = start[0];
          int keyLength = length[0];
          joinResult = hashMap.lookup(keyBytes, keyStart, keyLength, hashMapResults[0]);
        }

        /*
         * Common repeated join result processing.
         */

        if (LOG.isDebugEnabled()) {
          LOG.debug(
              CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
        }
        numSel = finishOuterRepeated(batch, joinResult, hashMapResults[0], scratch1);
      } else {

        /*
         * NOT Repeating.
         */

        if (LOG.isDebugEnabled()) {
          LOG.debug(CLASS_NAME + " batch #" + batchCounter + " non-repeated");
        }

        int selected[] = batch.selected;
        boolean selectedInUse = batch.selectedInUse;

        // For outer join we must apply the filter after match and cause some matches to become
        // non-matches, we do not track non-matches here.  Instead we remember all non spilled rows
        // and compute non matches later in finishOuter.
        int hashMapResultCount = 0;
        int matchCount = 0;
        int nonSpillCount = 0;
        int spillCount = 0;

        /*
         * Single-Column String specific variables.
         */

        int saveKeyBatchIndex = -1;

        // We optimize performance by only looking up the first key in a series of equal keys.
        boolean haveSaveKey = false;
        JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH;

        // Logical loop over the rows in the batch since the batch may have selected in use.
        for (int logical = 0; logical < inputLogicalSize; logical++) {
          int batchIndex = (selectedInUse ? selected[logical] : logical);

          /*
           * Single-Column String outer null detection.
           */

          boolean isNull = !joinColVector.noNulls && joinColVector.isNull[batchIndex];

          if (isNull) {

            // Have that the NULL does not interfere with the current equal key series, if there
            // is one. We do not set saveJoinResult.
            //
            //    Let a current MATCH equal key series keep going, or
            //    Let a current SPILL equal key series keep going, or
            //    Let a current NOMATCH keep not matching.

            // Remember non-matches for Outer Join.
            nonSpills[nonSpillCount++] = batchIndex;
            // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + "
            // NULL");
          } else {

            /*
             * Single-Column String outer get key.
             */

            // Implicit -- use batchIndex.

            /*
             * Equal key series checking.
             */

            if (!haveSaveKey
                || StringExpr.compare(
                        vector[saveKeyBatchIndex],
                        start[saveKeyBatchIndex],
                        length[saveKeyBatchIndex],
                        vector[batchIndex],
                        start[batchIndex],
                        length[batchIndex])
                    != 0) {
              // New key.

              if (haveSaveKey) {
                // Move on with our count(s).
                switch (saveJoinResult) {
                  case MATCH:
                  case SPILL:
                    hashMapResultCount++;
                    break;
                  case NOMATCH:
                    break;
                }
              }

              // Regardless of our matching result, we keep that information to make multiple use
              // of it for a possible series of equal keys.
              haveSaveKey = true;

              /*
               * Single-Column String specific save key.
               */

              saveKeyBatchIndex = batchIndex;

              /*
               * Single-Column Long specific lookup key.
               */

              byte[] keyBytes = vector[batchIndex];
              int keyStart = start[batchIndex];
              int keyLength = length[batchIndex];

              saveJoinResult =
                  hashMap.lookup(keyBytes, keyStart, keyLength, hashMapResults[hashMapResultCount]);
              // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " New
              // Key " + saveJoinResult.name());
            } else {
              // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key
              // Continues " + saveJoinResult.name());
            }

            /*
             * Common outer join result processing.
             */

            switch (saveJoinResult) {
              case MATCH:
                matchs[matchCount] = batchIndex;
                matchHashMapResultIndices[matchCount] = hashMapResultCount;
                matchCount++;
                nonSpills[nonSpillCount++] = batchIndex;
                break;

              case SPILL:
                spills[spillCount] = batchIndex;
                spillHashMapResultIndices[spillCount] = hashMapResultCount;
                spillCount++;
                break;

              case NOMATCH:
                nonSpills[nonSpillCount++] = batchIndex;
                // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH
                // duplicate");
                break;
            }
          }
        }

        if (haveSaveKey) {
          // Account for last equal key sequence.
          switch (saveJoinResult) {
            case MATCH:
            case SPILL:
              hashMapResultCount++;
              break;
            case NOMATCH:
              break;
          }
        }

        if (LOG.isDebugEnabled()) {
          LOG.debug(
              CLASS_NAME
                  + " batch #"
                  + batchCounter
                  + " matchs "
                  + intArrayToRangesString(matchs, matchCount)
                  + " matchHashMapResultIndices "
                  + intArrayToRangesString(matchHashMapResultIndices, matchCount)
                  + " nonSpills "
                  + intArrayToRangesString(nonSpills, nonSpillCount)
                  + " spills "
                  + intArrayToRangesString(spills, spillCount)
                  + " spillHashMapResultIndices "
                  + intArrayToRangesString(spillHashMapResultIndices, spillCount)
                  + " hashMapResults "
                  + Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount)));
        }

        // We will generate results for all matching and non-matching rows.
        // Note that scratch1 is undefined at this point -- it's preallocated storage.
        numSel =
            finishOuter(
                batch,
                matchs,
                matchHashMapResultIndices,
                matchCount,
                nonSpills,
                nonSpillCount,
                spills,
                spillHashMapResultIndices,
                spillCount,
                hashMapResults,
                hashMapResultCount,
                scratch1);
      }

      batch.selectedInUse = true;
      batch.size = numSel;

      if (batch.size > 0) {
        // Forward any remaining selected rows.
        forwardBigTableBatch(batch);
      }

    } catch (IOException e) {
      throw new HiveException(e);
    } catch (Exception e) {
      throw new HiveException(e);
    }
  }