@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
accessor.reset(buffer);
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
short numTokens = 0;
tokenizer.reset(
accessor.getBuffer().array(),
accessor.getTupleStartOffset(i)
+ accessor.getFieldSlotsLength()
+ accessor.getFieldStartOffset(i, docField),
accessor.getFieldLength(i, docField));
if (addNumTokensKey) {
// Get the total number of tokens.
numTokens = tokenizer.getTokensCount();
}
// Write token and data into frame by following the order specified
// in the writeKeyFieldsFirst field.
while (tokenizer.hasNext()) {
tokenizer.next();
builder.reset();
// Writing Order: token, number of token, keyfield1 ... n
if (!writeKeyFieldsFirst) {
try {
IToken token = tokenizer.getToken();
token.serializeToken(builderData);
builder.addFieldEndOffset();
// Add number of tokens if requested.
if (addNumTokensKey) {
builder.getDataOutput().writeShort(numTokens);
builder.addFieldEndOffset();
}
} catch (IOException e) {
throw new HyracksDataException(e.getMessage());
}
for (int k = 0; k < keyFields.length; k++) {
builder.addField(accessor, i, keyFields[k]);
}
}
// Writing Order: keyfield1 ... n, token, number of token
else {
for (int k = 0; k < keyFields.length; k++) {
builder.addField(accessor, i, keyFields[k]);
}
try {
IToken token = tokenizer.getToken();
token.serializeToken(builderData);
builder.addFieldEndOffset();
// Add number of tokens if requested.
if (addNumTokensKey) {
builder.getDataOutput().writeShort(numTokens);
builder.addFieldEndOffset();
}
} catch (IOException e) {
throw new HyracksDataException(e.getMessage());
}
}
FrameUtils.appendToWriter(
writer,
appender,
builder.getFieldEndOffsets(),
builder.getByteArray(),
0,
builder.getSize());
}
}
}
@SuppressWarnings("rawtypes")
public void testSchemaful() {
try {
File file = new File("target/classad-wtih-temporals.adm");
File expected =
new File(getClass().getResource("/results/classad-with-temporals.adm").toURI().getPath());
FileUtils.deleteQuietly(file);
PrintStream printStream = new PrintStream(Files.newOutputStream(Paths.get(file.toURI())));
String[] recordFieldNames = {
"GlobalJobId",
"Owner",
"ClusterId",
"ProcId",
"RemoteWallClockTime",
"CompletionDate",
"QDate",
"JobCurrentStartDate",
"JobStartDate",
"JobCurrentStartExecutingDate"
};
IAType[] recordFieldTypes = {
BuiltinType.ASTRING,
BuiltinType.ASTRING,
BuiltinType.AINT32,
BuiltinType.AINT32,
BuiltinType.ADURATION,
BuiltinType.ADATETIME,
BuiltinType.ADATETIME,
BuiltinType.ADATETIME,
BuiltinType.ADATETIME,
BuiltinType.ADATETIME
};
ARecordType recordType = new ARecordType("value", recordFieldNames, recordFieldTypes, true);
int numOfTupleFields = 1;
ISerializerDeserializer[] serdes = new ISerializerDeserializer[1];
serdes[0] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(recordType);
IPrinterFactory[] printerFactories = new IPrinterFactory[1];
printerFactories[0] = AqlADMPrinterFactoryProvider.INSTANCE.getPrinterFactory(recordType);
// create output descriptor
IPrinter[] printers = new IPrinter[printerFactories.length];
for (int i = 0; i < printerFactories.length; i++) {
printers[i] = printerFactories[i].createPrinter();
}
ClassAdObjectPool objectPool = new ClassAdObjectPool();
String[] files = new String[] {"/classad-with-temporals.classads"};
ClassAdParser parser =
new ClassAdParser(recordType, false, false, false, null, null, null, objectPool);
ArrayTupleBuilder tb = new ArrayTupleBuilder(numOfTupleFields);
for (String path : files) {
List<Path> paths = new ArrayList<>();
paths.add(Paths.get(getClass().getResource(path).toURI()));
FileSystemWatcher watcher = new FileSystemWatcher(paths, null, false);
LocalFSInputStream in = new LocalFSInputStream(watcher);
SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader(in, "[", "]");
while (recordReader.hasNext()) {
tb.reset();
IRawRecord<char[]> record = recordReader.next();
parser.parse(record, tb.getDataOutput());
tb.addFieldEndOffset();
printTuple(tb, printers, printStream);
}
recordReader.close();
printStream.close();
Assert.assertTrue(FileUtils.contentEquals(file, expected));
}
} catch (Throwable th) {
System.err.println("TEST FAILED");
th.printStackTrace();
Assert.assertTrue(false);
}
System.err.println("TEST PASSED");
}