/**
* Convenience method that allows running tests on various {@link ValueVector vector} instances.
*
* @param test test function to execute
*/
private void testVectors(VectorVerifier test) throws Exception {
final MaterializedField[] fields = {
MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE),
MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE),
MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE),
MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE),
MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE),
MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE),
MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE)
};
final ValueVector[] vectors = {
new UInt4Vector(fields[0], allocator),
new BitVector(fields[1], allocator),
new VarCharVector(fields[2], allocator),
new NullableVarCharVector(fields[3], allocator),
new RepeatedListVector(fields[4], allocator, null),
new MapVector(fields[5], allocator, null),
new RepeatedMapVector(fields[6], allocator, null)
};
try {
for (final ValueVector vector : vectors) {
test.verify(vector);
}
} finally {
AutoCloseables.close(vectors);
}
}
@Test
public void testVVInitialCapacity() throws Exception {
final MaterializedField[] fields = new MaterializedField[9];
final ValueVector[] valueVectors = new ValueVector[9];
fields[0] = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
fields[1] = MaterializedField.create(EMPTY_SCHEMA_PATH, IntHolder.TYPE);
fields[2] = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
fields[3] = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVar16CharHolder.TYPE);
fields[4] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedFloat4Holder.TYPE);
fields[5] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedVarBinaryHolder.TYPE);
fields[6] = MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE);
fields[6].addChild(fields[0] /*bit*/);
fields[6].addChild(fields[2] /*varchar*/);
fields[7] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE);
fields[7].addChild(fields[1] /*int*/);
fields[7].addChild(fields[3] /*optional var16char*/);
fields[8] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE);
fields[8].addChild(fields[1] /*int*/);
final int initialCapacity = 1024;
try {
for (int i = 0; i < valueVectors.length; i++) {
valueVectors[i] = TypeHelper.getNewVector(fields[i], allocator);
valueVectors[i].setInitialCapacity(initialCapacity);
valueVectors[i].allocateNew();
}
for (int i = 0; i < valueVectors.length; i++) {
final ValueVector vv = valueVectors[i];
final int vvCapacity = vv.getValueCapacity();
assertEquals(
String.format("Incorrect value capacity for %s [%d]", vv.getField(), vvCapacity),
initialCapacity,
vvCapacity);
}
} finally {
AutoCloseables.close(valueVectors);
}
}
@Override
public void close() throws Exception {
try {
if (!closed) {
List<AutoCloseable> toClose = Lists.newArrayList();
// TODO(DRILL-1942) the new allocator has this capability built-in, so we can remove
// bufferManager and
// allocator from the toClose list.
toClose.add(bufferManager);
toClose.add(allocator);
toClose.add(schemaTreeProvider);
AutoCloseables.close(toClose);
}
} finally {
closed = true;
}
}
public BatchGroup mergeAndSpill(LinkedList<BatchGroup> batchGroups) throws SchemaChangeException {
logger.debug("Copier allocator current allocation {}", copierAllocator.getAllocatedMemory());
logger.debug(
"mergeAndSpill: starting total size in memory = {}", oAllocator.getAllocatedMemory());
VectorContainer outputContainer = new VectorContainer();
List<BatchGroup> batchGroupList = Lists.newArrayList();
int batchCount = batchGroups.size();
for (int i = 0; i < batchCount / 2; i++) {
if (batchGroups.size() == 0) {
break;
}
BatchGroup batch = batchGroups.pollLast();
assert batch != null : "Encountered a null batch during merge and spill operation";
batchGroupList.add(batch);
}
if (batchGroupList.size() == 0) {
return null;
}
int estimatedRecordSize = 0;
for (VectorWrapper<?> w : batchGroupList.get(0)) {
try {
estimatedRecordSize += TypeHelper.getSize(w.getField().getType());
} catch (UnsupportedOperationException e) {
estimatedRecordSize += 50;
}
}
int targetRecordCount = Math.max(1, COPIER_BATCH_MEM_LIMIT / estimatedRecordSize);
VectorContainer hyperBatch = constructHyperBatch(batchGroupList);
createCopier(hyperBatch, batchGroupList, outputContainer, true);
int count = copier.next(targetRecordCount);
assert count > 0;
logger.debug(
"mergeAndSpill: estimated record size = {}, target record count = {}",
estimatedRecordSize,
targetRecordCount);
// 1 output container is kept in memory, so we want to hold on to it and transferClone
// allows keeping ownership
VectorContainer c1 = VectorContainer.getTransferClone(outputContainer, oContext);
c1.buildSchema(BatchSchema.SelectionVectorMode.NONE);
c1.setRecordCount(count);
String spillDir = dirs.next();
Path currSpillPath = new Path(Joiner.on("/").join(spillDir, fileName));
currSpillDirs.add(currSpillPath);
String outputFile = Joiner.on("/").join(currSpillPath, spillCount++);
try {
fs.deleteOnExit(currSpillPath);
} catch (IOException e) {
// since this is meant to be used in a batches's spilling, we don't propagate the exception
logger.warn("Unable to mark spill directory " + currSpillPath + " for deleting on exit", e);
}
stats.setLongStat(Metric.SPILL_COUNT, spillCount);
BatchGroup newGroup = new BatchGroup(c1, fs, outputFile, oContext);
try (AutoCloseable a = AutoCloseables.all(batchGroupList)) {
logger.info("Merging and spilling to {}", outputFile);
while ((count = copier.next(targetRecordCount)) > 0) {
outputContainer.buildSchema(BatchSchema.SelectionVectorMode.NONE);
outputContainer.setRecordCount(count);
// note that addBatch also clears the outputContainer
newGroup.addBatch(outputContainer);
}
injector.injectChecked(
context.getExecutionControls(), INTERRUPTION_WHILE_SPILLING, IOException.class);
newGroup.closeOutputStream();
} catch (Throwable e) {
// we only need to cleanup newGroup if spill failed
try {
AutoCloseables.close(e, newGroup);
} catch (Throwable t) {
/* close() may hit the same IO issue; just ignore */
}
throw UserException.resourceError(e)
.message("External Sort encountered an error while spilling to disk")
.addContext(e.getMessage() /* more detail */)
.build(logger);
} finally {
hyperBatch.clear();
}
logger.debug("mergeAndSpill: final total size in memory = {}", oAllocator.getAllocatedMemory());
logger.info("Completed spilling to {}", outputFile);
return newGroup;
}