@Override
  public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
    accessor.reset(buffer);
    int tupleCount = accessor.getTupleCount();

    for (int i = 0; i < tupleCount; i++) {
      short numTokens = 0;

      tokenizer.reset(
          accessor.getBuffer().array(),
          accessor.getTupleStartOffset(i)
              + accessor.getFieldSlotsLength()
              + accessor.getFieldStartOffset(i, docField),
          accessor.getFieldLength(i, docField));

      if (addNumTokensKey) {
        // Get the total number of tokens.
        numTokens = tokenizer.getTokensCount();
      }

      // Write token and data into frame by following the order specified
      // in the writeKeyFieldsFirst field.
      while (tokenizer.hasNext()) {

        tokenizer.next();

        builder.reset();

        // Writing Order: token, number of token, keyfield1 ... n
        if (!writeKeyFieldsFirst) {
          try {
            IToken token = tokenizer.getToken();
            token.serializeToken(builderData);

            builder.addFieldEndOffset();
            // Add number of tokens if requested.
            if (addNumTokensKey) {
              builder.getDataOutput().writeShort(numTokens);
              builder.addFieldEndOffset();
            }
          } catch (IOException e) {
            throw new HyracksDataException(e.getMessage());
          }

          for (int k = 0; k < keyFields.length; k++) {
            builder.addField(accessor, i, keyFields[k]);
          }

        }
        // Writing Order: keyfield1 ... n, token, number of token
        else {

          for (int k = 0; k < keyFields.length; k++) {
            builder.addField(accessor, i, keyFields[k]);
          }

          try {
            IToken token = tokenizer.getToken();
            token.serializeToken(builderData);

            builder.addFieldEndOffset();
            // Add number of tokens if requested.
            if (addNumTokensKey) {
              builder.getDataOutput().writeShort(numTokens);
              builder.addFieldEndOffset();
            }
          } catch (IOException e) {
            throw new HyracksDataException(e.getMessage());
          }
        }

        FrameUtils.appendToWriter(
            writer,
            appender,
            builder.getFieldEndOffsets(),
            builder.getByteArray(),
            0,
            builder.getSize());
      }
    }
  }
コード例 #2
0
 @SuppressWarnings("rawtypes")
 public void testSchemaful() {
   try {
     File file = new File("target/classad-wtih-temporals.adm");
     File expected =
         new File(getClass().getResource("/results/classad-with-temporals.adm").toURI().getPath());
     FileUtils.deleteQuietly(file);
     PrintStream printStream = new PrintStream(Files.newOutputStream(Paths.get(file.toURI())));
     String[] recordFieldNames = {
       "GlobalJobId",
       "Owner",
       "ClusterId",
       "ProcId",
       "RemoteWallClockTime",
       "CompletionDate",
       "QDate",
       "JobCurrentStartDate",
       "JobStartDate",
       "JobCurrentStartExecutingDate"
     };
     IAType[] recordFieldTypes = {
       BuiltinType.ASTRING,
       BuiltinType.ASTRING,
       BuiltinType.AINT32,
       BuiltinType.AINT32,
       BuiltinType.ADURATION,
       BuiltinType.ADATETIME,
       BuiltinType.ADATETIME,
       BuiltinType.ADATETIME,
       BuiltinType.ADATETIME,
       BuiltinType.ADATETIME
     };
     ARecordType recordType = new ARecordType("value", recordFieldNames, recordFieldTypes, true);
     int numOfTupleFields = 1;
     ISerializerDeserializer[] serdes = new ISerializerDeserializer[1];
     serdes[0] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(recordType);
     IPrinterFactory[] printerFactories = new IPrinterFactory[1];
     printerFactories[0] = AqlADMPrinterFactoryProvider.INSTANCE.getPrinterFactory(recordType);
     // create output descriptor
     IPrinter[] printers = new IPrinter[printerFactories.length];
     for (int i = 0; i < printerFactories.length; i++) {
       printers[i] = printerFactories[i].createPrinter();
     }
     ClassAdObjectPool objectPool = new ClassAdObjectPool();
     String[] files = new String[] {"/classad-with-temporals.classads"};
     ClassAdParser parser =
         new ClassAdParser(recordType, false, false, false, null, null, null, objectPool);
     ArrayTupleBuilder tb = new ArrayTupleBuilder(numOfTupleFields);
     for (String path : files) {
       List<Path> paths = new ArrayList<>();
       paths.add(Paths.get(getClass().getResource(path).toURI()));
       FileSystemWatcher watcher = new FileSystemWatcher(paths, null, false);
       LocalFSInputStream in = new LocalFSInputStream(watcher);
       SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader(in, "[", "]");
       while (recordReader.hasNext()) {
         tb.reset();
         IRawRecord<char[]> record = recordReader.next();
         parser.parse(record, tb.getDataOutput());
         tb.addFieldEndOffset();
         printTuple(tb, printers, printStream);
       }
       recordReader.close();
       printStream.close();
       Assert.assertTrue(FileUtils.contentEquals(file, expected));
     }
   } catch (Throwable th) {
     System.err.println("TEST FAILED");
     th.printStackTrace();
     Assert.assertTrue(false);
   }
   System.err.println("TEST PASSED");
 }