private boolean bytesEquals(VectorHashKeyWrapper keyThat) {
// By the time we enter here the byteValues.lentgh and isNull must have already been compared
for (int i = 0; i < byteValues.length; ++i) {
// the byte comparison is potentially expensive so is better to branch on null
if (!isNull[longValues.length + doubleValues.length + i]) {
if (0
!= StringExpr.compare(
byteValues[i],
byteStarts[i],
byteLengths[i],
keyThat.byteValues[i],
keyThat.byteStarts[i],
keyThat.byteLengths[i])) {
return false;
}
}
}
return true;
}
@Override
public void process(Object row, int tag) throws HiveException {
try {
VectorizedRowBatch batch = (VectorizedRowBatch) row;
alias = (byte) tag;
if (needCommonSetup) {
// Our one time process method initialization.
commonSetup(batch);
/*
* Initialize Single-Column String members for this specialized class.
*/
singleJoinColumn = bigTableKeyColumnMap[0];
needCommonSetup = false;
}
if (needHashTableSetup) {
// Setup our hash table specialization. It will be the first time the process
// method is called, or after a Hybrid Grace reload.
/*
* Get our Single-Column String hash set information for this specialized class.
*/
hashSet = (VectorMapJoinBytesHashSet) vectorMapJoinHashTable;
needHashTableSetup = false;
}
batchCounter++;
// Do the per-batch setup for an left semi join.
// (Currently none)
// leftSemiPerBatchSetup(batch);
// For left semi joins, we may apply the filter(s) now.
for (VectorExpression ve : bigTableFilterExpressions) {
ve.evaluate(batch);
}
final int inputLogicalSize = batch.size;
if (inputLogicalSize == 0) {
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " empty");
}
return;
}
// Perform any key expressions. Results will go into scratch columns.
if (bigTableKeyExpressions != null) {
for (VectorExpression ve : bigTableKeyExpressions) {
ve.evaluate(batch);
}
}
/*
* Single-Column String specific declarations.
*/
// The one join column for this specialized class.
BytesColumnVector joinColVector = (BytesColumnVector) batch.cols[singleJoinColumn];
byte[][] vector = joinColVector.vector;
int[] start = joinColVector.start;
int[] length = joinColVector.length;
/*
* Single-Column Long check for repeating.
*/
// Check single column for repeating.
boolean allKeyInputColumnsRepeating = joinColVector.isRepeating;
if (allKeyInputColumnsRepeating) {
/*
* Repeating.
*/
// All key input columns are repeating. Generate key once. Lookup once.
// Since the key is repeated, we must use entry 0 regardless of selectedInUse.
/*
* Single-Column String specific repeated lookup.
*/
byte[] keyBytes = vector[0];
int keyStart = start[0];
int keyLength = length[0];
JoinUtil.JoinResult joinResult =
hashSet.contains(keyBytes, keyStart, keyLength, hashSetResults[0]);
/*
* Common repeated join result processing.
*/
if (LOG.isDebugEnabled()) {
LOG.debug(
CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]);
} else {
/*
* NOT Repeating.
*/
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " non-repeated");
}
// We remember any matching rows in matchs / matchSize. At the end of the loop,
// selected / batch.size will represent both matching and non-matching rows for outer join.
// Only deferred rows will have been removed from selected.
int selected[] = batch.selected;
boolean selectedInUse = batch.selectedInUse;
int hashSetResultCount = 0;
int allMatchCount = 0;
int spillCount = 0;
/*
* Single-Column String specific variables.
*/
int saveKeyBatchIndex = -1;
// We optimize performance by only looking up the first key in a series of equal keys.
boolean haveSaveKey = false;
JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH;
// Logical loop over the rows in the batch since the batch may have selected in use.
for (int logical = 0; logical < inputLogicalSize; logical++) {
int batchIndex = (selectedInUse ? selected[logical] : logical);
/*
* Single-Column String get key.
*/
// Implicit -- use batchIndex.
/*
* Equal key series checking.
*/
if (!haveSaveKey
|| StringExpr.compare(
vector[saveKeyBatchIndex],
start[saveKeyBatchIndex],
length[saveKeyBatchIndex],
vector[batchIndex],
start[batchIndex],
length[batchIndex])
!= 0) {
// New key.
if (haveSaveKey) {
// Move on with our counts.
switch (saveJoinResult) {
case MATCH:
// We have extracted the existence from the hash set result, so we don't keep it.
break;
case SPILL:
// We keep the hash set result for its spill information.
hashSetResultCount++;
break;
case NOMATCH:
break;
}
}
// Regardless of our matching result, we keep that information to make multiple use
// of it for a possible series of equal keys.
haveSaveKey = true;
/*
* Single-Column String specific save key and lookup.
*/
saveKeyBatchIndex = batchIndex;
/*
* Single-Column String specific lookup key.
*/
byte[] keyBytes = vector[batchIndex];
int keyStart = start[batchIndex];
int keyLength = length[batchIndex];
saveJoinResult =
hashSet.contains(keyBytes, keyStart, keyLength, hashSetResults[hashSetResultCount]);
/*
* Common left-semi join result processing.
*/
switch (saveJoinResult) {
case MATCH:
allMatchs[allMatchCount++] = batchIndex;
// VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH
// isSingleValue " + equalKeySeriesIsSingleValue[equalKeySeriesCount] + " currentKey
// " + currentKey);
break;
case SPILL:
spills[spillCount] = batchIndex;
spillHashMapResultIndices[spillCount] = hashSetResultCount;
spillCount++;
break;
case NOMATCH:
// VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH"
// + " currentKey " + currentKey);
break;
}
} else {
// Series of equal keys.
switch (saveJoinResult) {
case MATCH:
allMatchs[allMatchCount++] = batchIndex;
// VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH
// duplicate");
break;
case SPILL:
spills[spillCount] = batchIndex;
spillHashMapResultIndices[spillCount] = hashSetResultCount;
spillCount++;
break;
case NOMATCH:
// VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH
// duplicate");
break;
}
}
}
if (haveSaveKey) {
// Update our counts for the last key.
switch (saveJoinResult) {
case MATCH:
// We have extracted the existence from the hash set result, so we don't keep it.
break;
case SPILL:
// We keep the hash set result for its spill information.
hashSetResultCount++;
break;
case NOMATCH:
break;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
CLASS_NAME
+ " allMatchs "
+ intArrayToRangesString(allMatchs, allMatchCount)
+ " spills "
+ intArrayToRangesString(spills, spillCount)
+ " spillHashMapResultIndices "
+ intArrayToRangesString(spillHashMapResultIndices, spillCount)
+ " hashMapResults "
+ Arrays.toString(Arrays.copyOfRange(hashSetResults, 0, hashSetResultCount)));
}
finishLeftSemi(
batch, allMatchCount, spillCount, (VectorMapJoinHashTableResult[]) hashSetResults);
}
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
}
} catch (IOException e) {
throw new HiveException(e);
} catch (Exception e) {
throw new HiveException(e);
}
}
@Override
public void process(Object row, int tag) throws HiveException {
try {
VectorizedRowBatch batch = (VectorizedRowBatch) row;
alias = (byte) tag;
if (needCommonSetup) {
// Our one time process method initialization.
commonSetup(batch);
/*
* Initialize Single-Column String members for this specialized class.
*/
singleJoinColumn = bigTableKeyColumnMap[0];
needCommonSetup = false;
}
if (needHashTableSetup) {
// Setup our hash table specialization. It will be the first time the process
// method is called, or after a Hybrid Grace reload.
/*
* Get our Single-Column String hash map information for this specialized class.
*/
hashMap = (VectorMapJoinBytesHashMap) vectorMapJoinHashTable;
needHashTableSetup = false;
}
batchCounter++;
// Do the per-batch setup for an outer join.
outerPerBatchSetup(batch);
// For outer join, DO NOT apply filters yet. It is incorrect for outer join to
// apply the filter before hash table matching.
final int inputLogicalSize = batch.size;
if (inputLogicalSize == 0) {
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " empty");
}
return;
}
// Perform any key expressions. Results will go into scratch columns.
if (bigTableKeyExpressions != null) {
for (VectorExpression ve : bigTableKeyExpressions) {
ve.evaluate(batch);
}
}
// We rebuild in-place the selected array with rows destine to be forwarded.
int numSel = 0;
/*
* Single-Column String specific declarations.
*/
// The one join column for this specialized class.
BytesColumnVector joinColVector = (BytesColumnVector) batch.cols[singleJoinColumn];
byte[][] vector = joinColVector.vector;
int[] start = joinColVector.start;
int[] length = joinColVector.length;
/*
* Single-Column String check for repeating.
*/
// Check single column for repeating.
boolean allKeyInputColumnsRepeating = joinColVector.isRepeating;
if (allKeyInputColumnsRepeating) {
/*
* Repeating.
*/
// All key input columns are repeating. Generate key once. Lookup once.
// Since the key is repeated, we must use entry 0 regardless of selectedInUse.
/*
* Single-Column String specific repeated lookup.
*/
JoinUtil.JoinResult joinResult;
if (!joinColVector.noNulls && joinColVector.isNull[0]) {
// Null key is no match for whole batch.
joinResult = JoinUtil.JoinResult.NOMATCH;
} else {
// Handle *repeated* join key, if found.
byte[] keyBytes = vector[0];
int keyStart = start[0];
int keyLength = length[0];
joinResult = hashMap.lookup(keyBytes, keyStart, keyLength, hashMapResults[0]);
}
/*
* Common repeated join result processing.
*/
if (LOG.isDebugEnabled()) {
LOG.debug(
CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
numSel = finishOuterRepeated(batch, joinResult, hashMapResults[0], scratch1);
} else {
/*
* NOT Repeating.
*/
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " non-repeated");
}
int selected[] = batch.selected;
boolean selectedInUse = batch.selectedInUse;
// For outer join we must apply the filter after match and cause some matches to become
// non-matches, we do not track non-matches here. Instead we remember all non spilled rows
// and compute non matches later in finishOuter.
int hashMapResultCount = 0;
int matchCount = 0;
int nonSpillCount = 0;
int spillCount = 0;
/*
* Single-Column String specific variables.
*/
int saveKeyBatchIndex = -1;
// We optimize performance by only looking up the first key in a series of equal keys.
boolean haveSaveKey = false;
JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH;
// Logical loop over the rows in the batch since the batch may have selected in use.
for (int logical = 0; logical < inputLogicalSize; logical++) {
int batchIndex = (selectedInUse ? selected[logical] : logical);
/*
* Single-Column String outer null detection.
*/
boolean isNull = !joinColVector.noNulls && joinColVector.isNull[batchIndex];
if (isNull) {
// Have that the NULL does not interfere with the current equal key series, if there
// is one. We do not set saveJoinResult.
//
// Let a current MATCH equal key series keep going, or
// Let a current SPILL equal key series keep going, or
// Let a current NOMATCH keep not matching.
// Remember non-matches for Outer Join.
nonSpills[nonSpillCount++] = batchIndex;
// LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + "
// NULL");
} else {
/*
* Single-Column String outer get key.
*/
// Implicit -- use batchIndex.
/*
* Equal key series checking.
*/
if (!haveSaveKey
|| StringExpr.compare(
vector[saveKeyBatchIndex],
start[saveKeyBatchIndex],
length[saveKeyBatchIndex],
vector[batchIndex],
start[batchIndex],
length[batchIndex])
!= 0) {
// New key.
if (haveSaveKey) {
// Move on with our count(s).
switch (saveJoinResult) {
case MATCH:
case SPILL:
hashMapResultCount++;
break;
case NOMATCH:
break;
}
}
// Regardless of our matching result, we keep that information to make multiple use
// of it for a possible series of equal keys.
haveSaveKey = true;
/*
* Single-Column String specific save key.
*/
saveKeyBatchIndex = batchIndex;
/*
* Single-Column Long specific lookup key.
*/
byte[] keyBytes = vector[batchIndex];
int keyStart = start[batchIndex];
int keyLength = length[batchIndex];
saveJoinResult =
hashMap.lookup(keyBytes, keyStart, keyLength, hashMapResults[hashMapResultCount]);
// LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " New
// Key " + saveJoinResult.name());
} else {
// LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key
// Continues " + saveJoinResult.name());
}
/*
* Common outer join result processing.
*/
switch (saveJoinResult) {
case MATCH:
matchs[matchCount] = batchIndex;
matchHashMapResultIndices[matchCount] = hashMapResultCount;
matchCount++;
nonSpills[nonSpillCount++] = batchIndex;
break;
case SPILL:
spills[spillCount] = batchIndex;
spillHashMapResultIndices[spillCount] = hashMapResultCount;
spillCount++;
break;
case NOMATCH:
nonSpills[nonSpillCount++] = batchIndex;
// VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH
// duplicate");
break;
}
}
}
if (haveSaveKey) {
// Account for last equal key sequence.
switch (saveJoinResult) {
case MATCH:
case SPILL:
hashMapResultCount++;
break;
case NOMATCH:
break;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
CLASS_NAME
+ " batch #"
+ batchCounter
+ " matchs "
+ intArrayToRangesString(matchs, matchCount)
+ " matchHashMapResultIndices "
+ intArrayToRangesString(matchHashMapResultIndices, matchCount)
+ " nonSpills "
+ intArrayToRangesString(nonSpills, nonSpillCount)
+ " spills "
+ intArrayToRangesString(spills, spillCount)
+ " spillHashMapResultIndices "
+ intArrayToRangesString(spillHashMapResultIndices, spillCount)
+ " hashMapResults "
+ Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount)));
}
// We will generate results for all matching and non-matching rows.
// Note that scratch1 is undefined at this point -- it's preallocated storage.
numSel =
finishOuter(
batch,
matchs,
matchHashMapResultIndices,
matchCount,
nonSpills,
nonSpillCount,
spills,
spillHashMapResultIndices,
spillCount,
hashMapResults,
hashMapResultCount,
scratch1);
}
batch.selectedInUse = true;
batch.size = numSel;
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
}
} catch (IOException e) {
throw new HiveException(e);
} catch (Exception e) {
throw new HiveException(e);
}
}
