Пример #1
0
 @Override
 public void sink(FlowProcess<JobConf> fp, SinkCall<Object[], OutputCollector> sink)
     throws IOException {
   TupleEntry tuple = sink.getOutgoingEntry();
   OutputCollector outputCollector = sink.getOutput();
   outputCollector.collect(null, tuple);
 }
Пример #2
0
  @Override
  public void sink(
      FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall)
      throws IOException {
    TupleEntry tupleEntry = sinkCall.getOutgoingEntry();

    Iterable<String> strings = tupleEntry.asIterableOf(String.class);

    delimitedParser.joinLine(strings, sinkCall.getContext());

    sinkCall.getContext().println();
  }
Пример #3
0
  @Override
  public void sinkPrepare(
      FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall) {
    sinkCall.setContext(createOutput(sinkCall.getOutput()));

    if (writeHeader) {
      Fields fields = sinkCall.getOutgoingEntry().getFields();
      delimitedParser.joinFirstLine(fields, sinkCall.getContext());

      sinkCall.getContext().println();
    }
  }
Пример #4
0
  /**
   * @param flowProcess
   * @param sinkCall
   * @throws IOException
   */
  @Override
  public void sink(
      FlowProcess<JobConf> flowProcess, SinkCall<BSONWritable[], OutputCollector> sinkCall)
      throws IOException {
    TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
    OutputCollector outputCollector = sinkCall.getOutput();

    String keyFieldName = this.fieldMappings.get(this.keyColumnName);
    Object key;

    // if fieldMappings doesn't have keyColumnName ("_id") field, then use new ObjectId() as key
    if (keyFieldName == null) {
      key = new ObjectId();
    } else {
      key = tupleEntry.selectTuple(new Fields(keyFieldName)).get(0);
    }
    // Object key = tupleEntry.selectTuple(new
    // Fields(this.fieldMappings.get(this.keyColumnName))).get(0);

    BasicDBObject dbObject = new BasicDBObject();

    for (String columnFieldName : columnFieldNames) {
      String columnFieldMapping = fieldMappings.get(columnFieldName);
      Object tupleEntryValue = null;

      try {
        if (columnFieldMapping != null) {
          // columnFieldMapping is null if no corresponding field name defined in Mappings.
          // only write the field value back to mongo if the field also defined in Mappings (ie. not
          // null)
          tupleEntryValue = tupleEntry.get(columnFieldMapping);
        }
      } catch (FieldsResolverException e) {
        logger.error("Couldn't resolve field: {}", columnFieldName);
      }

      if (tupleEntryValue != null && columnFieldName != keyColumnName) {
        // logger.info("Putting for output: {} {}", columnFieldName, tupleEntryValue);
        dbObject.put(columnFieldName, tupleEntryValue);
      }
    }
    logger.info("Putting key for output: {} {}", key, dbObject);
    // outputCollector.collect(new ObjectId(), dbObject);
    outputCollector.collect(key, dbObject);
  }
    static Tuple coerceToString(SinkCall<?, ?> sinkCall) {
      TupleEntry entry = sinkCall.getOutgoingEntry();
      Fields fields = entry.getFields();
      Tuple tuple = entry.getTuple();

      if (fields.hasTypes()) {
        Type types[] = new Type[fields.size()];
        for (int index = 0; index < fields.size(); index++) {
          Type type = fields.getType(index);
          if (type instanceof CoercibleType<?>) {
            types[index] = String.class;
          } else {
            types[index] = type;
          }
        }

        tuple = entry.getCoercedTuple(types);
      }
      return tuple;
    }
 static Tuple coerceToString(SinkCall<?, ?> sinkCall) {
   return sinkCall.getOutgoingEntry().getTuple();
 }
Пример #7
0
 @Override
 public void sinkCleanup(
     FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall) {
   sinkCall.getContext().flush();
   sinkCall.setContext(null);
 }