/** return a mapping of expected keys -> records */
private HashMap<String, Record> createTextSequenceFile(File file, int numRecords)
throws IOException {
HashMap<String, Record> map = new HashMap<String, Record>();
SequenceFile.Metadata metadata = new SequenceFile.Metadata(getMetadataForSequenceFile());
FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(file), null);
SequenceFile.Writer writer = null;
try {
writer =
SequenceFile.createWriter(
new Configuration(),
out,
Text.class,
Text.class,
SequenceFile.CompressionType.NONE,
null,
metadata);
for (int i = 0; i < numRecords; ++i) {
Text key = new Text("key" + i);
Text value = new Text("value" + i);
writer.append(key, value);
Record record = new Record();
record.put("key", key);
record.put("value", value);
map.put(key.toString(), record);
}
} finally {
Closeables.closeQuietly(writer);
}
return map;
}
@Override
protected boolean doProcess(Record inputRecord, final InputStream in) throws IOException {
SequenceFile.Metadata sequenceFileMetaData = null;
SequenceFile.Reader reader = null;
try {
reader =
new SequenceFile.Reader(
conf,
SequenceFile.Reader.stream(new FSDataInputStream(new ForwardOnlySeekable(in))));
if (includeMetaData) {
sequenceFileMetaData = reader.getMetadata();
}
Class keyClass = reader.getKeyClass();
Class valueClass = reader.getValueClass();
Record template = inputRecord.copy();
removeAttachments(template);
while (true) {
Writable key = (Writable) ReflectionUtils.newInstance(keyClass, conf);
Writable val = (Writable) ReflectionUtils.newInstance(valueClass, conf);
try {
if (!reader.next(key, val)) {
break;
}
} catch (EOFException ex) {
// SequenceFile.Reader will throw an EOFException after reading
// all the data, if it doesn't know the length. Since we are
// passing in an InputStream, we hit this case;
LOG.trace("Received expected EOFException", ex);
break;
}
incrementNumRecords();
Record outputRecord = template.copy();
outputRecord.put(keyField, key);
outputRecord.put(valueField, val);
outputRecord.put(Fields.ATTACHMENT_MIME_TYPE, OUTPUT_MEDIA_TYPE);
if (includeMetaData && sequenceFileMetaData != null) {
outputRecord.put(SEQUENCE_FILE_META_DATA, sequenceFileMetaData);
}
// pass record to next command in chain:
if (!getChild().process(outputRecord)) {
return false;
}
}
} finally {
Closeables.closeQuietly(reader);
}
return true;
}
@Test
public void testBasic() throws Exception {
morphline = createMorphline("test-morphlines/startReportingMetricsToHTTP");
Record record = new Record();
String msg = "foo";
record.put(Fields.MESSAGE, msg);
Record expected = new Record();
expected.put(Fields.MESSAGE, msg);
processAndVerifySuccess(record, expected);
if ("true".equals(System.getProperty("HttpMetricsMorphlineTest.isDemo"))) {
// wait forever so user can browse to http://localhost:8080/ and interactively explore the
// features
Thread.sleep(Long.MAX_VALUE);
}
verifyServing(8080);
verifyServing(8081);
verifyShutdown(8080);
verifyShutdown(8081);
}
@Override
public void process(String line, Emitter<T> emitter) {
record.removeAll(Fields.ATTACHMENT_BODY);
record.put(Fields.ATTACHMENT_BODY, new ByteArrayInputStream(line.toString().getBytes()));
if (!morphline.process(record)) {
LOGGER.error("Unable to process record: {}", line);
return;
}
// the process command above parses the record
// and stores it into the temporaryRecord
emitter.emit(temporaryRecord);
}
@Override
protected boolean doProcess(Record inputRecord) {
Record outputRecord = inputRecord.copy();
AbstractParser.removeAttachments(outputRecord);
ByteArrayOutputStream bout = new ByteArrayOutputStream(1024);
if (format == Format.container) {
writeContainer(inputRecord, bout);
} else {
writeContainerless(inputRecord, bout);
}
outputRecord.put(Fields.ATTACHMENT_BODY, bout.toByteArray());
// pass record to next command in chain:
return super.doProcess(outputRecord);
}
/**
* Test that Solr queries on a parsed SequenceFile document return the expected content and
* fields. Don't pass in our own parser via the context.
*/
@Test
public void testSequenceFileContentSimple() throws Exception {
morphline = createMorphline("test-morphlines/sequenceFileMorphlineSimple");
String path = RESOURCES_DIR;
File sequenceFile = new File(path, "testSequenceFileContentSimple.seq");
int numRecords = 5;
HashMap<String, Record> expected = createTextSequenceFile(sequenceFile, numRecords);
InputStream in = new FileInputStream(sequenceFile.getAbsolutePath());
Record record = new Record();
record.put(Fields.ATTACHMENT_BODY, in);
startSession();
assertEquals(1, collector.getNumStartEvents());
assertTrue(morphline.process(record));
assertTrue(areFieldsEqual(expected, collector.getRecords()));
}
@Test
public void testDiscardRecord() {
final MorphlineContext context = new MorphlineContext.Builder().build();
Collector collectorParent = new Collector();
Collector collectorChild = new Collector();
final Command command =
new TimeFilterBuilder().build(config, collectorParent, collectorChild, context);
Record record = new Record();
record.put("createdAt", "21/01/2015");
command.process(record);
List<Record> records = collectorChild.getRecords();
assertThat(records.size()).isEqualTo(0);
}