@Override
public void write(DataOutput out) throws IOException {
date.write(out);
type.write(out);
group.write(out);
begin.write(out);
end.write(out);
source.write(out);
target.write(out);
}
@Override
public void write(DataOutput out) throws IOException {
Text text = new Text();
text.set(this.jobId);
text.write(out);
text.set(this.taskId);
text.write(out);
out.writeLong(this.startTime);
out.writeLong(this.endTime);
out.writeLong(this.duration);
super.write(out);
}
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(tuple.size());
Text value = new Text();
for (String entry : tuple) {
value.set(entry);
value.write(out);
}
}
/** Implement write of Writable */
public void write(DataOutput out) throws IOException {
out.writeLong(offset);
out.writeLong(length);
out.writeInt(names.length);
for (int i = 0; i < names.length; i++) {
Text name = new Text(names[i]);
name.write(out);
}
out.writeInt(hosts.length);
for (int i = 0; i < hosts.length; i++) {
Text host = new Text(hosts[i]);
host.write(out);
}
out.writeInt(topologyPaths.length);
for (int i = 0; i < topologyPaths.length; i++) {
Text host = new Text(topologyPaths[i]);
host.write(out);
}
}
@Override
public void write(DataOutput out) throws IOException {
for (int i = 0; i < values.size(); i++) {
Text value = values.get(i);
value.write(out);
if (!isLastIndex(i)) {
this.fieldTerminator.write(out);
}
}
if (this.fileFormat != Table.FILE_FORMAT.TEXTFILE) {
this.lineTerminator.write(out);
}
}
@Override
public void write(DataOutput output) throws IOException {
element.write(output);
neighbour.write(output);
}
@Override
public void write(DataOutput out) throws IOException {
first.write(out);
second.write(out);
}
/**
* Use the input splits to take samples of the input and generate sample keys. By default reads
* 100,000 keys from 10 locations in the input, sorts them and picks N-1 keys to generate N
* equally sized partitions.
*
* @param job the job to sample
* @param partFile where to write the output file to
* @throws IOException if something goes wrong
*/
public static void writePartitionFile(final JobContext job, Path partFile)
throws IOException, InterruptedException {
long t1 = System.currentTimeMillis();
Configuration conf = job.getConfiguration();
final TeraEncryptedInputFormat inFormat = new TeraEncryptedInputFormat();
final TextSampler sampler = new TextSampler();
int partitions = job.getNumReduceTasks();
long sampleSize = conf.getLong(SAMPLE_SIZE, 100000);
final List<InputSplit> splits = inFormat.getSplits(job);
long t2 = System.currentTimeMillis();
System.out.println("Computing input splits took " + (t2 - t1) + "ms");
int samples = Math.min(conf.getInt(NUM_PARTITIONS, 10), splits.size());
System.out.println("Sampling " + samples + " splits of " + splits.size());
final long recordsPerSample = sampleSize / samples;
final int sampleStep = splits.size() / samples;
Thread[] samplerReader = new Thread[samples];
// take N samples from different parts of the input
for (int i = 0; i < samples; ++i) {
final int idx = i;
samplerReader[i] =
new Thread("Sampler Reader " + idx) {
{
setDaemon(true);
}
public void run() {
long records = 0;
try {
TaskAttemptContext context =
new TaskAttemptContextImpl(job.getConfiguration(), new TaskAttemptID());
RecordReader<Text, Text> reader =
inFormat.createRecordReader(splits.get(sampleStep * idx), context);
reader.initialize(splits.get(sampleStep * idx), context);
while (reader.nextKeyValue()) {
sampler.addKey(new Text(reader.getCurrentKey()));
records += 1;
if (recordsPerSample <= records) {
break;
}
}
} catch (IOException ie) {
System.err.println(
"Got an exception while reading splits " + StringUtils.stringifyException(ie));
System.exit(-1);
} catch (InterruptedException e) {
}
}
};
samplerReader[i].start();
}
FileSystem outFs = partFile.getFileSystem(conf);
DataOutputStream writer =
outFs.create(partFile, true, 64 * 1024, (short) 10, outFs.getDefaultBlockSize());
for (int i = 0; i < samples; i++) {
try {
samplerReader[i].join();
} catch (InterruptedException e) {
}
}
for (Text split : sampler.createPartitions(partitions)) {
split.write(writer);
}
writer.close();
long t3 = System.currentTimeMillis();
System.out.println("Computing parititions took " + (t3 - t2) + "ms");
}
@Override
public void write(DataOutput out) throws IOException {
docId.write(out);
out.writeDouble(distance.get());
vector.write(out);
}
@Override
public void write(DataOutput out) throws IOException {
__term.write(out);
__documentId.write(out);
}
@Override
public void write(DataOutput out) throws IOException {
key.write(out);
value.write(out);
}
public void write(DataOutput out) throws IOException {
keyWritable.write(out);
typeWritable.write(out);
}