示例#1
0
 private VectorContainer getBatch() throws IOException {
   assert fs != null;
   assert path != null;
   if (inputStream == null) {
     inputStream = fs.open(path);
   }
   VectorAccessibleSerializable vas = new VectorAccessibleSerializable(allocator);
   Stopwatch watch = Stopwatch.createStarted();
   vas.readFromStream(inputStream);
   VectorContainer c = vas.get();
   if (schema != null) {
     c = SchemaUtil.coerceContainer(c, schema, context);
   }
   //    logger.debug("Took {} us to read {} records", watch.elapsed(TimeUnit.MICROSECONDS),
   // c.getRecordCount());
   spilledBatches--;
   currentContainer.zeroVectors();
   Iterator<VectorWrapper<?>> wrapperIterator = c.iterator();
   for (VectorWrapper w : currentContainer) {
     TransferPair pair =
         wrapperIterator.next().getValueVector().makeTransferPair(w.getValueVector());
     pair.transfer();
   }
   currentContainer.setRecordCount(c.getRecordCount());
   c.zeroVectors();
   return c;
 }
示例#2
0
 private FieldWriter promoteToUnion() {
   String name = vector.getField().getLastName();
   TransferPair tp =
       vector.getTransferPair(
           vector.getField().getType().getMinorType().name().toLowerCase(), vector.getAllocator());
   tp.transfer();
   if (parentContainer != null) {
     unionVector =
         parentContainer.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class);
   } else if (listVector != null) {
     unionVector = listVector.promoteToUnion();
   }
   unionVector.addVector(tp.getTo());
   writer = new UnionWriter(unionVector);
   writer.setPosition(idx());
   for (int i = 0; i < idx(); i++) {
     unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
   }
   vector = null;
   state = State.UNION;
   return writer;
 }
 private void doTransfers() {
   for (TransferPair t : transfers) {
     t.transfer();
   }
 }