Exemplo n.º 1
0
 @Override
 public void write(DataOutput out) throws IOException {
   date.write(out);
   type.write(out);
   group.write(out);
   begin.write(out);
   end.write(out);
   source.write(out);
   target.write(out);
 }
Exemplo n.º 2
0
 @Override
 public void write(DataOutput out) throws IOException {
   Text text = new Text();
   text.set(this.jobId);
   text.write(out);
   text.set(this.taskId);
   text.write(out);
   out.writeLong(this.startTime);
   out.writeLong(this.endTime);
   out.writeLong(this.duration);
   super.write(out);
 }
Exemplo n.º 3
0
 @Override
 public void write(DataOutput out) throws IOException {
   out.writeInt(tuple.size());
   Text value = new Text();
   for (String entry : tuple) {
     value.set(entry);
     value.write(out);
   }
 }
Exemplo n.º 4
0
 /** Implement write of Writable */
 public void write(DataOutput out) throws IOException {
   out.writeLong(offset);
   out.writeLong(length);
   out.writeInt(names.length);
   for (int i = 0; i < names.length; i++) {
     Text name = new Text(names[i]);
     name.write(out);
   }
   out.writeInt(hosts.length);
   for (int i = 0; i < hosts.length; i++) {
     Text host = new Text(hosts[i]);
     host.write(out);
   }
   out.writeInt(topologyPaths.length);
   for (int i = 0; i < topologyPaths.length; i++) {
     Text host = new Text(topologyPaths[i]);
     host.write(out);
   }
 }
Exemplo n.º 5
0
 @Override
 public void write(DataOutput out) throws IOException {
   for (int i = 0; i < values.size(); i++) {
     Text value = values.get(i);
     value.write(out);
     if (!isLastIndex(i)) {
       this.fieldTerminator.write(out);
     }
   }
   if (this.fileFormat != Table.FILE_FORMAT.TEXTFILE) {
     this.lineTerminator.write(out);
   }
 }
Exemplo n.º 6
0
 @Override
 public void write(DataOutput output) throws IOException {
   element.write(output);
   neighbour.write(output);
 }
Exemplo n.º 7
0
 @Override
 public void write(DataOutput out) throws IOException {
   first.write(out);
   second.write(out);
 }
  /**
   * Use the input splits to take samples of the input and generate sample keys. By default reads
   * 100,000 keys from 10 locations in the input, sorts them and picks N-1 keys to generate N
   * equally sized partitions.
   *
   * @param job the job to sample
   * @param partFile where to write the output file to
   * @throws IOException if something goes wrong
   */
  public static void writePartitionFile(final JobContext job, Path partFile)
      throws IOException, InterruptedException {
    long t1 = System.currentTimeMillis();
    Configuration conf = job.getConfiguration();
    final TeraEncryptedInputFormat inFormat = new TeraEncryptedInputFormat();
    final TextSampler sampler = new TextSampler();
    int partitions = job.getNumReduceTasks();
    long sampleSize = conf.getLong(SAMPLE_SIZE, 100000);
    final List<InputSplit> splits = inFormat.getSplits(job);
    long t2 = System.currentTimeMillis();
    System.out.println("Computing input splits took " + (t2 - t1) + "ms");
    int samples = Math.min(conf.getInt(NUM_PARTITIONS, 10), splits.size());
    System.out.println("Sampling " + samples + " splits of " + splits.size());
    final long recordsPerSample = sampleSize / samples;
    final int sampleStep = splits.size() / samples;
    Thread[] samplerReader = new Thread[samples];
    // take N samples from different parts of the input
    for (int i = 0; i < samples; ++i) {
      final int idx = i;
      samplerReader[i] =
          new Thread("Sampler Reader " + idx) {
            {
              setDaemon(true);
            }

            public void run() {
              long records = 0;
              try {
                TaskAttemptContext context =
                    new TaskAttemptContextImpl(job.getConfiguration(), new TaskAttemptID());
                RecordReader<Text, Text> reader =
                    inFormat.createRecordReader(splits.get(sampleStep * idx), context);
                reader.initialize(splits.get(sampleStep * idx), context);
                while (reader.nextKeyValue()) {
                  sampler.addKey(new Text(reader.getCurrentKey()));
                  records += 1;
                  if (recordsPerSample <= records) {
                    break;
                  }
                }
              } catch (IOException ie) {
                System.err.println(
                    "Got an exception while reading splits " + StringUtils.stringifyException(ie));
                System.exit(-1);
              } catch (InterruptedException e) {

              }
            }
          };
      samplerReader[i].start();
    }
    FileSystem outFs = partFile.getFileSystem(conf);
    DataOutputStream writer =
        outFs.create(partFile, true, 64 * 1024, (short) 10, outFs.getDefaultBlockSize());
    for (int i = 0; i < samples; i++) {
      try {
        samplerReader[i].join();
      } catch (InterruptedException e) {
      }
    }
    for (Text split : sampler.createPartitions(partitions)) {
      split.write(writer);
    }
    writer.close();
    long t3 = System.currentTimeMillis();
    System.out.println("Computing parititions took " + (t3 - t2) + "ms");
  }
 @Override
 public void write(DataOutput out) throws IOException {
   docId.write(out);
   out.writeDouble(distance.get());
   vector.write(out);
 }
Exemplo n.º 10
0
 @Override
 public void write(DataOutput out) throws IOException {
   __term.write(out);
   __documentId.write(out);
 }
Exemplo n.º 11
0
 @Override
 public void write(DataOutput out) throws IOException {
   key.write(out);
   value.write(out);
 }
 public void write(DataOutput out) throws IOException {
   keyWritable.write(out);
   typeWritable.write(out);
 }