private boolean bytesEquals(VectorHashKeyWrapper keyThat) { // By the time we enter here the byteValues.lentgh and isNull must have already been compared for (int i = 0; i < byteValues.length; ++i) { // the byte comparison is potentially expensive so is better to branch on null if (!isNull[longValues.length + doubleValues.length + i]) { if (0 != StringExpr.compare( byteValues[i], byteStarts[i], byteLengths[i], keyThat.byteValues[i], keyThat.byteStarts[i], keyThat.byteLengths[i])) { return false; } } } return true; }
@Override public void process(Object row, int tag) throws HiveException { try { VectorizedRowBatch batch = (VectorizedRowBatch) row; alias = (byte) tag; if (needCommonSetup) { // Our one time process method initialization. commonSetup(batch); /* * Initialize Single-Column String members for this specialized class. */ singleJoinColumn = bigTableKeyColumnMap[0]; needCommonSetup = false; } if (needHashTableSetup) { // Setup our hash table specialization. It will be the first time the process // method is called, or after a Hybrid Grace reload. /* * Get our Single-Column String hash set information for this specialized class. */ hashSet = (VectorMapJoinBytesHashSet) vectorMapJoinHashTable; needHashTableSetup = false; } batchCounter++; // Do the per-batch setup for an left semi join. // (Currently none) // leftSemiPerBatchSetup(batch); // For left semi joins, we may apply the filter(s) now. for (VectorExpression ve : bigTableFilterExpressions) { ve.evaluate(batch); } final int inputLogicalSize = batch.size; if (inputLogicalSize == 0) { if (LOG.isDebugEnabled()) { LOG.debug(CLASS_NAME + " batch #" + batchCounter + " empty"); } return; } // Perform any key expressions. Results will go into scratch columns. if (bigTableKeyExpressions != null) { for (VectorExpression ve : bigTableKeyExpressions) { ve.evaluate(batch); } } /* * Single-Column String specific declarations. */ // The one join column for this specialized class. BytesColumnVector joinColVector = (BytesColumnVector) batch.cols[singleJoinColumn]; byte[][] vector = joinColVector.vector; int[] start = joinColVector.start; int[] length = joinColVector.length; /* * Single-Column Long check for repeating. */ // Check single column for repeating. boolean allKeyInputColumnsRepeating = joinColVector.isRepeating; if (allKeyInputColumnsRepeating) { /* * Repeating. */ // All key input columns are repeating. Generate key once. Lookup once. // Since the key is repeated, we must use entry 0 regardless of selectedInUse. /* * Single-Column String specific repeated lookup. */ byte[] keyBytes = vector[0]; int keyStart = start[0]; int keyLength = length[0]; JoinUtil.JoinResult joinResult = hashSet.contains(keyBytes, keyStart, keyLength, hashSetResults[0]); /* * Common repeated join result processing. */ if (LOG.isDebugEnabled()) { LOG.debug( CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name()); } finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]); } else { /* * NOT Repeating. */ if (LOG.isDebugEnabled()) { LOG.debug(CLASS_NAME + " batch #" + batchCounter + " non-repeated"); } // We remember any matching rows in matchs / matchSize. At the end of the loop, // selected / batch.size will represent both matching and non-matching rows for outer join. // Only deferred rows will have been removed from selected. int selected[] = batch.selected; boolean selectedInUse = batch.selectedInUse; int hashSetResultCount = 0; int allMatchCount = 0; int spillCount = 0; /* * Single-Column String specific variables. */ int saveKeyBatchIndex = -1; // We optimize performance by only looking up the first key in a series of equal keys. boolean haveSaveKey = false; JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH; // Logical loop over the rows in the batch since the batch may have selected in use. for (int logical = 0; logical < inputLogicalSize; logical++) { int batchIndex = (selectedInUse ? selected[logical] : logical); /* * Single-Column String get key. */ // Implicit -- use batchIndex. /* * Equal key series checking. */ if (!haveSaveKey || StringExpr.compare( vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex], vector[batchIndex], start[batchIndex], length[batchIndex]) != 0) { // New key. if (haveSaveKey) { // Move on with our counts. switch (saveJoinResult) { case MATCH: // We have extracted the existence from the hash set result, so we don't keep it. break; case SPILL: // We keep the hash set result for its spill information. hashSetResultCount++; break; case NOMATCH: break; } } // Regardless of our matching result, we keep that information to make multiple use // of it for a possible series of equal keys. haveSaveKey = true; /* * Single-Column String specific save key and lookup. */ saveKeyBatchIndex = batchIndex; /* * Single-Column String specific lookup key. */ byte[] keyBytes = vector[batchIndex]; int keyStart = start[batchIndex]; int keyLength = length[batchIndex]; saveJoinResult = hashSet.contains(keyBytes, keyStart, keyLength, hashSetResults[hashSetResultCount]); /* * Common left-semi join result processing. */ switch (saveJoinResult) { case MATCH: allMatchs[allMatchCount++] = batchIndex; // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH // isSingleValue " + equalKeySeriesIsSingleValue[equalKeySeriesCount] + " currentKey // " + currentKey); break; case SPILL: spills[spillCount] = batchIndex; spillHashMapResultIndices[spillCount] = hashSetResultCount; spillCount++; break; case NOMATCH: // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH" // + " currentKey " + currentKey); break; } } else { // Series of equal keys. switch (saveJoinResult) { case MATCH: allMatchs[allMatchCount++] = batchIndex; // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH // duplicate"); break; case SPILL: spills[spillCount] = batchIndex; spillHashMapResultIndices[spillCount] = hashSetResultCount; spillCount++; break; case NOMATCH: // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH // duplicate"); break; } } } if (haveSaveKey) { // Update our counts for the last key. switch (saveJoinResult) { case MATCH: // We have extracted the existence from the hash set result, so we don't keep it. break; case SPILL: // We keep the hash set result for its spill information. hashSetResultCount++; break; case NOMATCH: break; } } if (LOG.isDebugEnabled()) { LOG.debug( CLASS_NAME + " allMatchs " + intArrayToRangesString(allMatchs, allMatchCount) + " spills " + intArrayToRangesString(spills, spillCount) + " spillHashMapResultIndices " + intArrayToRangesString(spillHashMapResultIndices, spillCount) + " hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashSetResults, 0, hashSetResultCount))); } finishLeftSemi( batch, allMatchCount, spillCount, (VectorMapJoinHashTableResult[]) hashSetResults); } if (batch.size > 0) { // Forward any remaining selected rows. forwardBigTableBatch(batch); } } catch (IOException e) { throw new HiveException(e); } catch (Exception e) { throw new HiveException(e); } }
@Override public void process(Object row, int tag) throws HiveException { try { VectorizedRowBatch batch = (VectorizedRowBatch) row; alias = (byte) tag; if (needCommonSetup) { // Our one time process method initialization. commonSetup(batch); /* * Initialize Single-Column String members for this specialized class. */ singleJoinColumn = bigTableKeyColumnMap[0]; needCommonSetup = false; } if (needHashTableSetup) { // Setup our hash table specialization. It will be the first time the process // method is called, or after a Hybrid Grace reload. /* * Get our Single-Column String hash map information for this specialized class. */ hashMap = (VectorMapJoinBytesHashMap) vectorMapJoinHashTable; needHashTableSetup = false; } batchCounter++; // Do the per-batch setup for an outer join. outerPerBatchSetup(batch); // For outer join, DO NOT apply filters yet. It is incorrect for outer join to // apply the filter before hash table matching. final int inputLogicalSize = batch.size; if (inputLogicalSize == 0) { if (LOG.isDebugEnabled()) { LOG.debug(CLASS_NAME + " batch #" + batchCounter + " empty"); } return; } // Perform any key expressions. Results will go into scratch columns. if (bigTableKeyExpressions != null) { for (VectorExpression ve : bigTableKeyExpressions) { ve.evaluate(batch); } } // We rebuild in-place the selected array with rows destine to be forwarded. int numSel = 0; /* * Single-Column String specific declarations. */ // The one join column for this specialized class. BytesColumnVector joinColVector = (BytesColumnVector) batch.cols[singleJoinColumn]; byte[][] vector = joinColVector.vector; int[] start = joinColVector.start; int[] length = joinColVector.length; /* * Single-Column String check for repeating. */ // Check single column for repeating. boolean allKeyInputColumnsRepeating = joinColVector.isRepeating; if (allKeyInputColumnsRepeating) { /* * Repeating. */ // All key input columns are repeating. Generate key once. Lookup once. // Since the key is repeated, we must use entry 0 regardless of selectedInUse. /* * Single-Column String specific repeated lookup. */ JoinUtil.JoinResult joinResult; if (!joinColVector.noNulls && joinColVector.isNull[0]) { // Null key is no match for whole batch. joinResult = JoinUtil.JoinResult.NOMATCH; } else { // Handle *repeated* join key, if found. byte[] keyBytes = vector[0]; int keyStart = start[0]; int keyLength = length[0]; joinResult = hashMap.lookup(keyBytes, keyStart, keyLength, hashMapResults[0]); } /* * Common repeated join result processing. */ if (LOG.isDebugEnabled()) { LOG.debug( CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name()); } numSel = finishOuterRepeated(batch, joinResult, hashMapResults[0], scratch1); } else { /* * NOT Repeating. */ if (LOG.isDebugEnabled()) { LOG.debug(CLASS_NAME + " batch #" + batchCounter + " non-repeated"); } int selected[] = batch.selected; boolean selectedInUse = batch.selectedInUse; // For outer join we must apply the filter after match and cause some matches to become // non-matches, we do not track non-matches here. Instead we remember all non spilled rows // and compute non matches later in finishOuter. int hashMapResultCount = 0; int matchCount = 0; int nonSpillCount = 0; int spillCount = 0; /* * Single-Column String specific variables. */ int saveKeyBatchIndex = -1; // We optimize performance by only looking up the first key in a series of equal keys. boolean haveSaveKey = false; JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH; // Logical loop over the rows in the batch since the batch may have selected in use. for (int logical = 0; logical < inputLogicalSize; logical++) { int batchIndex = (selectedInUse ? selected[logical] : logical); /* * Single-Column String outer null detection. */ boolean isNull = !joinColVector.noNulls && joinColVector.isNull[batchIndex]; if (isNull) { // Have that the NULL does not interfere with the current equal key series, if there // is one. We do not set saveJoinResult. // // Let a current MATCH equal key series keep going, or // Let a current SPILL equal key series keep going, or // Let a current NOMATCH keep not matching. // Remember non-matches for Outer Join. nonSpills[nonSpillCount++] = batchIndex; // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " // NULL"); } else { /* * Single-Column String outer get key. */ // Implicit -- use batchIndex. /* * Equal key series checking. */ if (!haveSaveKey || StringExpr.compare( vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex], vector[batchIndex], start[batchIndex], length[batchIndex]) != 0) { // New key. if (haveSaveKey) { // Move on with our count(s). switch (saveJoinResult) { case MATCH: case SPILL: hashMapResultCount++; break; case NOMATCH: break; } } // Regardless of our matching result, we keep that information to make multiple use // of it for a possible series of equal keys. haveSaveKey = true; /* * Single-Column String specific save key. */ saveKeyBatchIndex = batchIndex; /* * Single-Column Long specific lookup key. */ byte[] keyBytes = vector[batchIndex]; int keyStart = start[batchIndex]; int keyLength = length[batchIndex]; saveJoinResult = hashMap.lookup(keyBytes, keyStart, keyLength, hashMapResults[hashMapResultCount]); // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " New // Key " + saveJoinResult.name()); } else { // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key // Continues " + saveJoinResult.name()); } /* * Common outer join result processing. */ switch (saveJoinResult) { case MATCH: matchs[matchCount] = batchIndex; matchHashMapResultIndices[matchCount] = hashMapResultCount; matchCount++; nonSpills[nonSpillCount++] = batchIndex; break; case SPILL: spills[spillCount] = batchIndex; spillHashMapResultIndices[spillCount] = hashMapResultCount; spillCount++; break; case NOMATCH: nonSpills[nonSpillCount++] = batchIndex; // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH // duplicate"); break; } } } if (haveSaveKey) { // Account for last equal key sequence. switch (saveJoinResult) { case MATCH: case SPILL: hashMapResultCount++; break; case NOMATCH: break; } } if (LOG.isDebugEnabled()) { LOG.debug( CLASS_NAME + " batch #" + batchCounter + " matchs " + intArrayToRangesString(matchs, matchCount) + " matchHashMapResultIndices " + intArrayToRangesString(matchHashMapResultIndices, matchCount) + " nonSpills " + intArrayToRangesString(nonSpills, nonSpillCount) + " spills " + intArrayToRangesString(spills, spillCount) + " spillHashMapResultIndices " + intArrayToRangesString(spillHashMapResultIndices, spillCount) + " hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount))); } // We will generate results for all matching and non-matching rows. // Note that scratch1 is undefined at this point -- it's preallocated storage. numSel = finishOuter( batch, matchs, matchHashMapResultIndices, matchCount, nonSpills, nonSpillCount, spills, spillHashMapResultIndices, spillCount, hashMapResults, hashMapResultCount, scratch1); } batch.selectedInUse = true; batch.size = numSel; if (batch.size > 0) { // Forward any remaining selected rows. forwardBigTableBatch(batch); } } catch (IOException e) { throw new HiveException(e); } catch (Exception e) { throw new HiveException(e); } }