Пример #1
0
    @Override
    public void reduce(
        LongWritable key,
        Iterator<NullWritable> values,
        OutputCollector<NullWritable, Text> output,
        Reporter reporter)
        throws IOException {

      output.collect(NullWritable.get(), new Text(key.toString()));
    }
Пример #2
0
 public void reduce(WritablePoint key, Iterable<LongWritable> values, Context context)
     throws IOException, InterruptedException {
   String ids = new String("");
   for (LongWritable val : values) {
     ids += val.toString() + ", ";
   }
   Text text = new Text(ids);
   int x = part / grid;
   int y = part % grid;
   out.write(key, text, x + "/" + y);
 }
Пример #3
0
 public boolean nextKeyValue() {
   value = new Text();
   if (key == null) {
     key = new LongWritable();
   }
   if (finish < count) {
     key.set(start + finish);
     value.set(key.toString());
     finish += 1;
     return true;
   } else {
     return false;
   }
 }
Пример #4
0
  private void splitRead(LongWritable key, String line, Writer writer) throws IOException {
    String[] fields = line.split("\t");

    if (fields[1].length() != fields[3].length()) {
      logger.warn("Warning; mismatching seq and qual lengths in record " + key.toString() + "!");
      logger.warn("Seq:");
      logger.warn(fields[1]);
      logger.warn("Qual:");
      logger.warn(fields[3]);
      logger.warn("DONE WARNING");
    }
    writer.write(fields[0] + "\n");
    writer.write(fields[1] + "\n");
    writer.write(fields[2] + "\n");
    writer.write(fields[3] + "\n");
  }
  @Override
  public void map(LongWritable key, Text value, Context context)
      throws IOException, InterruptedException {

    String className = key.toString();
    String rawRecord = value.toString();

    // get the class
    Class<? extends USPSRecord> clazz = null;
    try {
      clazz = uspsUtils.getClassForRawRecord(rawRecord.substring(0, 1), myRecordLength);
      if (clazz == null) {
        throw new IOException("NO CLASS 4 RAW RECORD:|" + rawRecord + "|");
      }
    } catch (Exception e) {
      throw new IOException("NO CLASS 4 RAW RECORD:|" + rawRecord + "|");
      //	throw new IOException("Could not determine target USPSRecord class for record of type: "
      // + rawRecord.substring(0,1) + " len: " + myRecordLength);
    }

    // create a new instance to populate
    USPSRecord record = null;
    try {
      record = clazz.newInstance();
    } catch (Exception e) {
      throw new IOException(
          "Could not create target USPSRecord instance for record of type: "
              + clazz.getName()
              + " "
              + e.getMessage());
    }

    // apply the line to the record
    try {
      lineParser.applyRawRecord(rawRecord, record);
      record.setIdentifier(idGenerator.generateId(record));

    } catch (Exception e) {
      throw new IOException(
          "Could not apply record data to target USPSRecord instance for record of type: "
              + rawRecord
              + " "
              + myRecordLength
              + " "
              + clazz.getName()
              + " "
              + e.getMessage());
    }

    // get the record ID
    String id = record.getIdentifier();

    // if a copyright, update this as our latest one...
    // if the copyright has changed.., we need to flush all
    // previous records that were loaded after the copyrightToApply
    // before updating our copyrightToApply which will be applied
    // to all subsequent records
    if (record instanceof Copyright) {
      Copyright tmpC = (Copyright) record;
      if (copyrightToApply == null
          || !copyrightToApply.getIdentifier().equals(tmpC.getIdentifier())) {
        flushBatch();
        copyrightToApply = (Copyright) record;
      }
    }

    // add to batch
    batch.add(record);

    // commit?
    if (batch.size() >= processorBatchSize) {
      flushBatch();
    }

    // or the old version was Text,Text
    // context.write(new Text(id), new WritableUSPSRecord(record)); // json output

  }
Пример #6
0
 @Override
 protected void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   Text outKey = new Text(key.toString());
   context.write(outKey, value);
 }