Example #1
0
  /**
   * @param flowProcess
   * @param sourceCall
   * @return
   * @throws IOException
   */
  @Override
  public boolean source(
      FlowProcess<JobConf> flowProcess, SourceCall<BSONWritable[], RecordReader> sourceCall)
      throws IOException {
    Tuple result = new Tuple();

    BSONWritable key = sourceCall.getContext()[0];
    BSONWritable value = sourceCall.getContext()[1];

    if (!sourceCall.getInput().next(key, value)) {
      logger.info("Nothing left to read, exiting");
      return false;
    }

    for (String columnFieldName : columnFieldNames) {
      Object tupleEntry = value.get(columnFieldName);
      if (tupleEntry != null) {
        result.add(tupleEntry);
      } else if (columnFieldName != this.keyColumnName) {
        result.add("");
      }
    }

    sourceCall.getIncomingEntry().setTuple(result);
    return true;
  }
Example #2
0
  /**
   * @param flowProcess
   * @param sourceCall
   */
  @Override
  public void sourcePrepare(
      FlowProcess<JobConf> flowProcess, SourceCall<BSONWritable[], RecordReader> sourceCall) {
    sourceCall.setContext(new BSONWritable[2]);

    sourceCall.getContext()[0] = (BSONWritable) sourceCall.getInput().createKey();
    sourceCall.getContext()[1] = (BSONWritable) sourceCall.getInput().createValue();
  }
Example #3
0
  @Override
  public void sourcePrepare(
      FlowProcess<? extends Properties> flowProcess,
      SourceCall<LineNumberReader, InputStream> sourceCall)
      throws IOException {
    sourceCall.setContext(createInput(sourceCall.getInput()));

    sourceCall.getIncomingEntry().setTuple(TupleViews.createObjectArray());
  }
Example #4
0
 @Override
 public void sourceCleanup(
     FlowProcess<? extends Properties> flowProcess,
     SourceCall<LineNumberReader, InputStream> sourceCall)
     throws IOException {
   sourceCall.setContext(null);
 }
  @SuppressWarnings("unchecked")
  @Override
  public boolean source(FlowProcess<JobConf> fp, SourceCall<Object[], RecordReader> sc)
      throws IOException {
    Container<Tuple> value = (Container<Tuple>) sc.getInput().createValue();
    boolean hasNext = sc.getInput().next(null, value);
    if (!hasNext) {
      return false;
    }

    // Skip nulls
    if (value == null) {
      return true;
    }

    sc.getIncomingEntry().setTuple(value.get());
    return true;
  }
Example #6
0
  @Override
  public boolean source(
      FlowProcess<? extends Properties> flowProcess,
      SourceCall<LineNumberReader, InputStream> sourceCall)
      throws IOException {
    String line = sourceCall.getContext().readLine();

    if (line == null) return false;

    if (skipHeader && sourceCall.getContext().getLineNumber() == 1) // todo: optimize this away
    line = sourceCall.getContext().readLine();

    if (line == null) return false;

    Object[] split = delimitedParser.parseLine(line);

    // assumption it is better to re-use than to construct new
    Tuple tuple = sourceCall.getIncomingEntry().getTuple();

    TupleViews.reset(tuple, split);

    return true;
  }