@Override public void nextFrame(ByteBuffer buffer) throws HyracksDataException { accessor.reset(buffer); int tupleCount = accessor.getTupleCount(); for (int i = 0; i < tupleCount; i++) { short numTokens = 0; tokenizer.reset( accessor.getBuffer().array(), accessor.getTupleStartOffset(i) + accessor.getFieldSlotsLength() + accessor.getFieldStartOffset(i, docField), accessor.getFieldLength(i, docField)); if (addNumTokensKey) { // Get the total number of tokens. numTokens = tokenizer.getTokensCount(); } // Write token and data into frame by following the order specified // in the writeKeyFieldsFirst field. while (tokenizer.hasNext()) { tokenizer.next(); builder.reset(); // Writing Order: token, number of token, keyfield1 ... n if (!writeKeyFieldsFirst) { try { IToken token = tokenizer.getToken(); token.serializeToken(builderData); builder.addFieldEndOffset(); // Add number of tokens if requested. if (addNumTokensKey) { builder.getDataOutput().writeShort(numTokens); builder.addFieldEndOffset(); } } catch (IOException e) { throw new HyracksDataException(e.getMessage()); } for (int k = 0; k < keyFields.length; k++) { builder.addField(accessor, i, keyFields[k]); } } // Writing Order: keyfield1 ... n, token, number of token else { for (int k = 0; k < keyFields.length; k++) { builder.addField(accessor, i, keyFields[k]); } try { IToken token = tokenizer.getToken(); token.serializeToken(builderData); builder.addFieldEndOffset(); // Add number of tokens if requested. if (addNumTokensKey) { builder.getDataOutput().writeShort(numTokens); builder.addFieldEndOffset(); } } catch (IOException e) { throw new HyracksDataException(e.getMessage()); } } FrameUtils.appendToWriter( writer, appender, builder.getFieldEndOffsets(), builder.getByteArray(), 0, builder.getSize()); } } }
@SuppressWarnings("rawtypes") public void testSchemaful() { try { File file = new File("target/classad-wtih-temporals.adm"); File expected = new File(getClass().getResource("/results/classad-with-temporals.adm").toURI().getPath()); FileUtils.deleteQuietly(file); PrintStream printStream = new PrintStream(Files.newOutputStream(Paths.get(file.toURI()))); String[] recordFieldNames = { "GlobalJobId", "Owner", "ClusterId", "ProcId", "RemoteWallClockTime", "CompletionDate", "QDate", "JobCurrentStartDate", "JobStartDate", "JobCurrentStartExecutingDate" }; IAType[] recordFieldTypes = { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.AINT32, BuiltinType.AINT32, BuiltinType.ADURATION, BuiltinType.ADATETIME, BuiltinType.ADATETIME, BuiltinType.ADATETIME, BuiltinType.ADATETIME, BuiltinType.ADATETIME }; ARecordType recordType = new ARecordType("value", recordFieldNames, recordFieldTypes, true); int numOfTupleFields = 1; ISerializerDeserializer[] serdes = new ISerializerDeserializer[1]; serdes[0] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(recordType); IPrinterFactory[] printerFactories = new IPrinterFactory[1]; printerFactories[0] = AqlADMPrinterFactoryProvider.INSTANCE.getPrinterFactory(recordType); // create output descriptor IPrinter[] printers = new IPrinter[printerFactories.length]; for (int i = 0; i < printerFactories.length; i++) { printers[i] = printerFactories[i].createPrinter(); } ClassAdObjectPool objectPool = new ClassAdObjectPool(); String[] files = new String[] {"/classad-with-temporals.classads"}; ClassAdParser parser = new ClassAdParser(recordType, false, false, false, null, null, null, objectPool); ArrayTupleBuilder tb = new ArrayTupleBuilder(numOfTupleFields); for (String path : files) { List<Path> paths = new ArrayList<>(); paths.add(Paths.get(getClass().getResource(path).toURI())); FileSystemWatcher watcher = new FileSystemWatcher(paths, null, false); LocalFSInputStream in = new LocalFSInputStream(watcher); SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader(in, "[", "]"); while (recordReader.hasNext()) { tb.reset(); IRawRecord<char[]> record = recordReader.next(); parser.parse(record, tb.getDataOutput()); tb.addFieldEndOffset(); printTuple(tb, printers, printStream); } recordReader.close(); printStream.close(); Assert.assertTrue(FileUtils.contentEquals(file, expected)); } } catch (Throwable th) { System.err.println("TEST FAILED"); th.printStackTrace(); Assert.assertTrue(false); } System.err.println("TEST PASSED"); }