public FileSystemViewKeyInputFormat(FileSystemDataset<E> dataset, Configuration conf) {
    this.dataset = dataset;
    LOG.debug("Dataset: {}", dataset);

    Format format = dataset.getDescriptor().getFormat();

    boolean isSpecific = SpecificRecord.class.isAssignableFrom(dataset.getType());

    if (Formats.AVRO.equals(format)) {
      setModel.invoke(conf, DataModelUtil.getDataModelForType(dataset.getType()).getClass());

      // Use the reader's schema type if provided.
      if (isSpecific) {

        conf.set(AVRO_SCHEMA_INPUT_KEY, SpecificData.get().getSchema(dataset.getType()).toString());
      }
    } else if (Formats.PARQUET.equals(format)) {

      // Use the reader's schema type if provided.
      if (isSpecific) {

        AvroReadSupport.setAvroReadSchema(conf, SpecificData.get().getSchema(dataset.getType()));
      }
    }
  }
示例#2
0
  @Test
  public void testDeepCopy() {
    // Set all non-default fields in an Interop instance:
    Interop.Builder interopBuilder = Interop.newBuilder();
    interopBuilder.setArrayField(Arrays.asList(new Double[] {1.1, 1.2, 1.3, 1.4}));
    interopBuilder.setBoolField(true);
    interopBuilder.setBytesField(ByteBuffer.wrap(new byte[] {1, 2, 3, 4}));
    interopBuilder.setDoubleField(3.14d);
    interopBuilder.setEnumField(Kind.B);
    interopBuilder.setFixedField(
        new MD5(new byte[] {4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1}));
    interopBuilder.setFloatField(6.022f);
    interopBuilder.setIntField(32);
    interopBuilder.setLongField(64L);

    Map<java.lang.String, org.apache.avro.Foo> map =
        new HashMap<java.lang.String, org.apache.avro.Foo>(1);
    map.put("foo", Foo.newBuilder().setLabel("bar").build());
    interopBuilder.setMapField(map);

    interopBuilder.setNullField(null);

    Node.Builder rootBuilder = Node.newBuilder().setLabel("/");
    Node.Builder homeBuilder = Node.newBuilder().setLabel("home");
    homeBuilder.setChildren(new ArrayList<Node>(0));
    rootBuilder.setChildren(Arrays.asList(new Node[] {homeBuilder.build()}));
    interopBuilder.setRecordField(rootBuilder.build());

    interopBuilder.setStringField("Hello");
    interopBuilder.setUnionField(true);

    Interop interop = interopBuilder.build();

    // Verify that deepCopy works for all fields:
    for (Field field : Interop.SCHEMA$.getFields()) {
      // Original field and deep copy should be equivalent:
      if (interop.get(field.pos()) instanceof ByteBuffer) {
        assertTrue(
            Arrays.equals(
                ((ByteBuffer) interop.get(field.pos())).array(),
                ((ByteBuffer) GenericData.get().deepCopy(field.schema(), interop.get(field.pos())))
                    .array()));
      } else {
        assertEquals(
            interop.get(field.pos()),
            SpecificData.get().deepCopy(field.schema(), interop.get(field.pos())));
      }

      // Original field and deep copy should be different instances:
      if ((field.schema().getType() != Type.ENUM)
          && (field.schema().getType() != Type.NULL)
          && (field.schema().getType() != Type.STRING)) {
        assertFalse(
            "Field " + field.name() + " is same instance in deep copy",
            interop.get(field.pos())
                == GenericData.get().deepCopy(field.schema(), interop.get(field.pos())));
      }
    }
  }
  private void run() {
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();

    // Hard coding single threaded consumer
    topicCountMap.put(inputTopic, 1);

    Properties props = createConsumerConfig(zookeeper, groupId, url);
    VerifiableProperties vProps = new VerifiableProperties(props);

    // Create decoders for key and value
    KafkaAvroDecoder avroDecoder = new KafkaAvroDecoder(vProps);
    StringDecoder stringDecoder = new StringDecoder(new VerifiableProperties());

    KafkaStream stream =
        consumer
            .createMessageStreams(topicCountMap, stringDecoder, avroDecoder)
            .get(inputTopic)
            .get(0);
    ConsumerIterator it = stream.iterator();
    System.out.println("Ready to start iterating wih properties: " + props.toString());
    System.out.println("Reading topic:" + inputTopic);

    while (it.hasNext()) {
      MessageAndMetadata messageAndMetadata = it.next();
      String ip = (String) messageAndMetadata.key();

      // Once we release a new version of the avro deserializer that can return SpecificData, the
      // deep copy will be unnecessary
      GenericRecord genericEvent = (GenericRecord) messageAndMetadata.message();
      LogLine event = (LogLine) SpecificData.get().deepCopy(LogLine.SCHEMA$, genericEvent);

      SessionState oldState = state.get(ip);
      int sessionId = 0;
      if (oldState == null) {
        state.put(ip, new SessionState(event.getTimestamp(), 0));
      } else {
        sessionId = oldState.getSessionId();

        // if the old timestamp is more than 30 minutes older than new one, we have a new session
        if (oldState.getLastConnection() < event.getTimestamp() - sessionLengthMs)
          sessionId = sessionId + 1;

        SessionState newState = new SessionState(event.getTimestamp(), sessionId);
        state.put(ip, newState);
      }
      event.setSessionid(sessionId);
      System.out.println(event.toString());
      ProducerRecord<String, LogLine> record =
          new ProducerRecord<String, LogLine>(outputTopic, event.getIp().toString(), event);
      producer.send(record);
    }
  }
示例#4
0
  @Override
  public Object respond(Protocol.Message message, Object request) throws Exception {
    Object response;
    int numParams = message.getRequest().getFields().size();
    Object[] params = new Object[numParams];
    Class<?>[] paramTypes = new Class[numParams];
    int i = 0;
    for (Schema.Field param : message.getRequest().getFields()) {
      params[i] = ((GenericRecord) request).get(param.name());
      paramTypes[i] = SpecificData.get().getClass(param.schema());
      i++;
    }
    Exchange exchange = consumer.getEndpoint().createExchange(message, params);

    try {
      consumer.getProcessor().process(exchange);
    } catch (Throwable e) {
      consumer.getExceptionHandler().handleException(e);
    }

    if (ExchangeHelper.isOutCapable(exchange)) {
      response = exchange.getOut().getBody();
    } else {
      response = null;
    }

    boolean failed = exchange.isFailed();
    if (failed) {
      if (exchange.getException() != null) {
        response = exchange.getException();
      } else {
        // failed and no exception, must be a fault
        response = exchange.getOut().getBody();
      }
    }
    return response;
  }