public FileSystemViewKeyInputFormat(FileSystemDataset<E> dataset, Configuration conf) {
this.dataset = dataset;
LOG.debug("Dataset: {}", dataset);
Format format = dataset.getDescriptor().getFormat();
boolean isSpecific = SpecificRecord.class.isAssignableFrom(dataset.getType());
if (Formats.AVRO.equals(format)) {
setModel.invoke(conf, DataModelUtil.getDataModelForType(dataset.getType()).getClass());
// Use the reader's schema type if provided.
if (isSpecific) {
conf.set(AVRO_SCHEMA_INPUT_KEY, SpecificData.get().getSchema(dataset.getType()).toString());
}
} else if (Formats.PARQUET.equals(format)) {
// Use the reader's schema type if provided.
if (isSpecific) {
AvroReadSupport.setAvroReadSchema(conf, SpecificData.get().getSchema(dataset.getType()));
}
}
}
@Test
public void testDeepCopy() {
// Set all non-default fields in an Interop instance:
Interop.Builder interopBuilder = Interop.newBuilder();
interopBuilder.setArrayField(Arrays.asList(new Double[] {1.1, 1.2, 1.3, 1.4}));
interopBuilder.setBoolField(true);
interopBuilder.setBytesField(ByteBuffer.wrap(new byte[] {1, 2, 3, 4}));
interopBuilder.setDoubleField(3.14d);
interopBuilder.setEnumField(Kind.B);
interopBuilder.setFixedField(
new MD5(new byte[] {4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1}));
interopBuilder.setFloatField(6.022f);
interopBuilder.setIntField(32);
interopBuilder.setLongField(64L);
Map<java.lang.String, org.apache.avro.Foo> map =
new HashMap<java.lang.String, org.apache.avro.Foo>(1);
map.put("foo", Foo.newBuilder().setLabel("bar").build());
interopBuilder.setMapField(map);
interopBuilder.setNullField(null);
Node.Builder rootBuilder = Node.newBuilder().setLabel("/");
Node.Builder homeBuilder = Node.newBuilder().setLabel("home");
homeBuilder.setChildren(new ArrayList<Node>(0));
rootBuilder.setChildren(Arrays.asList(new Node[] {homeBuilder.build()}));
interopBuilder.setRecordField(rootBuilder.build());
interopBuilder.setStringField("Hello");
interopBuilder.setUnionField(true);
Interop interop = interopBuilder.build();
// Verify that deepCopy works for all fields:
for (Field field : Interop.SCHEMA$.getFields()) {
// Original field and deep copy should be equivalent:
if (interop.get(field.pos()) instanceof ByteBuffer) {
assertTrue(
Arrays.equals(
((ByteBuffer) interop.get(field.pos())).array(),
((ByteBuffer) GenericData.get().deepCopy(field.schema(), interop.get(field.pos())))
.array()));
} else {
assertEquals(
interop.get(field.pos()),
SpecificData.get().deepCopy(field.schema(), interop.get(field.pos())));
}
// Original field and deep copy should be different instances:
if ((field.schema().getType() != Type.ENUM)
&& (field.schema().getType() != Type.NULL)
&& (field.schema().getType() != Type.STRING)) {
assertFalse(
"Field " + field.name() + " is same instance in deep copy",
interop.get(field.pos())
== GenericData.get().deepCopy(field.schema(), interop.get(field.pos())));
}
}
}
private void run() {
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
// Hard coding single threaded consumer
topicCountMap.put(inputTopic, 1);
Properties props = createConsumerConfig(zookeeper, groupId, url);
VerifiableProperties vProps = new VerifiableProperties(props);
// Create decoders for key and value
KafkaAvroDecoder avroDecoder = new KafkaAvroDecoder(vProps);
StringDecoder stringDecoder = new StringDecoder(new VerifiableProperties());
KafkaStream stream =
consumer
.createMessageStreams(topicCountMap, stringDecoder, avroDecoder)
.get(inputTopic)
.get(0);
ConsumerIterator it = stream.iterator();
System.out.println("Ready to start iterating wih properties: " + props.toString());
System.out.println("Reading topic:" + inputTopic);
while (it.hasNext()) {
MessageAndMetadata messageAndMetadata = it.next();
String ip = (String) messageAndMetadata.key();
// Once we release a new version of the avro deserializer that can return SpecificData, the
// deep copy will be unnecessary
GenericRecord genericEvent = (GenericRecord) messageAndMetadata.message();
LogLine event = (LogLine) SpecificData.get().deepCopy(LogLine.SCHEMA$, genericEvent);
SessionState oldState = state.get(ip);
int sessionId = 0;
if (oldState == null) {
state.put(ip, new SessionState(event.getTimestamp(), 0));
} else {
sessionId = oldState.getSessionId();
// if the old timestamp is more than 30 minutes older than new one, we have a new session
if (oldState.getLastConnection() < event.getTimestamp() - sessionLengthMs)
sessionId = sessionId + 1;
SessionState newState = new SessionState(event.getTimestamp(), sessionId);
state.put(ip, newState);
}
event.setSessionid(sessionId);
System.out.println(event.toString());
ProducerRecord<String, LogLine> record =
new ProducerRecord<String, LogLine>(outputTopic, event.getIp().toString(), event);
producer.send(record);
}
}
@Override
public Object respond(Protocol.Message message, Object request) throws Exception {
Object response;
int numParams = message.getRequest().getFields().size();
Object[] params = new Object[numParams];
Class<?>[] paramTypes = new Class[numParams];
int i = 0;
for (Schema.Field param : message.getRequest().getFields()) {
params[i] = ((GenericRecord) request).get(param.name());
paramTypes[i] = SpecificData.get().getClass(param.schema());
i++;
}
Exchange exchange = consumer.getEndpoint().createExchange(message, params);
try {
consumer.getProcessor().process(exchange);
} catch (Throwable e) {
consumer.getExceptionHandler().handleException(e);
}
if (ExchangeHelper.isOutCapable(exchange)) {
response = exchange.getOut().getBody();
} else {
response = null;
}
boolean failed = exchange.isFailed();
if (failed) {
if (exchange.getException() != null) {
response = exchange.getException();
} else {
// failed and no exception, must be a fault
response = exchange.getOut().getBody();
}
}
return response;
}