@Override public void write(DataOutput out) throws IOException { date.write(out); type.write(out); group.write(out); begin.write(out); end.write(out); source.write(out); target.write(out); }
@Override public void write(DataOutput out) throws IOException { Text text = new Text(); text.set(this.jobId); text.write(out); text.set(this.taskId); text.write(out); out.writeLong(this.startTime); out.writeLong(this.endTime); out.writeLong(this.duration); super.write(out); }
@Override public void write(DataOutput out) throws IOException { out.writeInt(tuple.size()); Text value = new Text(); for (String entry : tuple) { value.set(entry); value.write(out); } }
/** Implement write of Writable */ public void write(DataOutput out) throws IOException { out.writeLong(offset); out.writeLong(length); out.writeInt(names.length); for (int i = 0; i < names.length; i++) { Text name = new Text(names[i]); name.write(out); } out.writeInt(hosts.length); for (int i = 0; i < hosts.length; i++) { Text host = new Text(hosts[i]); host.write(out); } out.writeInt(topologyPaths.length); for (int i = 0; i < topologyPaths.length; i++) { Text host = new Text(topologyPaths[i]); host.write(out); } }
@Override public void write(DataOutput out) throws IOException { for (int i = 0; i < values.size(); i++) { Text value = values.get(i); value.write(out); if (!isLastIndex(i)) { this.fieldTerminator.write(out); } } if (this.fileFormat != Table.FILE_FORMAT.TEXTFILE) { this.lineTerminator.write(out); } }
@Override public void write(DataOutput output) throws IOException { element.write(output); neighbour.write(output); }
@Override public void write(DataOutput out) throws IOException { first.write(out); second.write(out); }
/** * Use the input splits to take samples of the input and generate sample keys. By default reads * 100,000 keys from 10 locations in the input, sorts them and picks N-1 keys to generate N * equally sized partitions. * * @param job the job to sample * @param partFile where to write the output file to * @throws IOException if something goes wrong */ public static void writePartitionFile(final JobContext job, Path partFile) throws IOException, InterruptedException { long t1 = System.currentTimeMillis(); Configuration conf = job.getConfiguration(); final TeraEncryptedInputFormat inFormat = new TeraEncryptedInputFormat(); final TextSampler sampler = new TextSampler(); int partitions = job.getNumReduceTasks(); long sampleSize = conf.getLong(SAMPLE_SIZE, 100000); final List<InputSplit> splits = inFormat.getSplits(job); long t2 = System.currentTimeMillis(); System.out.println("Computing input splits took " + (t2 - t1) + "ms"); int samples = Math.min(conf.getInt(NUM_PARTITIONS, 10), splits.size()); System.out.println("Sampling " + samples + " splits of " + splits.size()); final long recordsPerSample = sampleSize / samples; final int sampleStep = splits.size() / samples; Thread[] samplerReader = new Thread[samples]; // take N samples from different parts of the input for (int i = 0; i < samples; ++i) { final int idx = i; samplerReader[i] = new Thread("Sampler Reader " + idx) { { setDaemon(true); } public void run() { long records = 0; try { TaskAttemptContext context = new TaskAttemptContextImpl(job.getConfiguration(), new TaskAttemptID()); RecordReader<Text, Text> reader = inFormat.createRecordReader(splits.get(sampleStep * idx), context); reader.initialize(splits.get(sampleStep * idx), context); while (reader.nextKeyValue()) { sampler.addKey(new Text(reader.getCurrentKey())); records += 1; if (recordsPerSample <= records) { break; } } } catch (IOException ie) { System.err.println( "Got an exception while reading splits " + StringUtils.stringifyException(ie)); System.exit(-1); } catch (InterruptedException e) { } } }; samplerReader[i].start(); } FileSystem outFs = partFile.getFileSystem(conf); DataOutputStream writer = outFs.create(partFile, true, 64 * 1024, (short) 10, outFs.getDefaultBlockSize()); for (int i = 0; i < samples; i++) { try { samplerReader[i].join(); } catch (InterruptedException e) { } } for (Text split : sampler.createPartitions(partitions)) { split.write(writer); } writer.close(); long t3 = System.currentTimeMillis(); System.out.println("Computing parititions took " + (t3 - t2) + "ms"); }
@Override public void write(DataOutput out) throws IOException { docId.write(out); out.writeDouble(distance.get()); vector.write(out); }
@Override public void write(DataOutput out) throws IOException { __term.write(out); __documentId.write(out); }
@Override public void write(DataOutput out) throws IOException { key.write(out); value.write(out); }
public void write(DataOutput out) throws IOException { keyWritable.write(out); typeWritable.write(out); }