private VectorContainer getBatch() throws IOException {
assert fs != null;
assert path != null;
if (inputStream == null) {
inputStream = fs.open(path);
}
VectorAccessibleSerializable vas = new VectorAccessibleSerializable(allocator);
Stopwatch watch = Stopwatch.createStarted();
vas.readFromStream(inputStream);
VectorContainer c = vas.get();
if (schema != null) {
c = SchemaUtil.coerceContainer(c, schema, context);
}
// logger.debug("Took {} us to read {} records", watch.elapsed(TimeUnit.MICROSECONDS),
// c.getRecordCount());
spilledBatches--;
currentContainer.zeroVectors();
Iterator<VectorWrapper<?>> wrapperIterator = c.iterator();
for (VectorWrapper w : currentContainer) {
TransferPair pair =
wrapperIterator.next().getValueVector().makeTransferPair(w.getValueVector());
pair.transfer();
}
currentContainer.setRecordCount(c.getRecordCount());
c.zeroVectors();
return c;
}
private FieldWriter promoteToUnion() {
String name = vector.getField().getLastName();
TransferPair tp =
vector.getTransferPair(
vector.getField().getType().getMinorType().name().toLowerCase(), vector.getAllocator());
tp.transfer();
if (parentContainer != null) {
unionVector =
parentContainer.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class);
} else if (listVector != null) {
unionVector = listVector.promoteToUnion();
}
unionVector.addVector(tp.getTo());
writer = new UnionWriter(unionVector);
writer.setPosition(idx());
for (int i = 0; i < idx(); i++) {
unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
}
vector = null;
state = State.UNION;
return writer;
}
private void doTransfers() {
for (TransferPair t : transfers) {
t.transfer();
}
}