@Override
public void serialize(Object o, ImmutableBytesWritable bytesWritable) throws IOException {
byte[] bytesToWriteIn = bytesWritable.get();
int offset = bytesWritable.getOffset();
if (o == null) {
if (fixedPrefixLength > 0)
throw new IllegalStateException(
"excepted at least " + fixedPrefixLength + " bytes to write");
else if (terminate()) {
// write one (masked) null byte
bytesToWriteIn[offset] = mask(NULL);
RowKeyUtils.seek(bytesWritable, 1);
}
} else {
final BytesWritable input = (BytesWritable) o;
if (fixedPrefixLength > input.getLength())
throw new IllegalStateException(
"excepted at least " + fixedPrefixLength + " bytes to write");
else {
encodeFixedPrefix(input.getBytes(), bytesWritable);
encodedCustomizedReversedPackedBcd(
toStringRepresentation(
input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength),
bytesWritable);
}
}
}
@Override
public void map(ImmutableHexWritable key, FsEntry value, Context context)
throws IOException, InterruptedException {
if (Extensions.contains(value.extension())) {
FullPath.set(value.fullPath());
Ext.set(value.extension());
encodeHex(Sha, value, "sha1");
encodeHex(Md5, value, "md5");
if (value.isContentHDFS()) {
Vid.setSize(0);
HdfsPath.set(value.getContentHdfsPath());
} else {
final byte[] buf = value.getContentBuffer();
if (buf == null) {
LOG.warn(value.fullPath() + " didn't have a content buffer, skipping.");
return;
}
Vid.set(buf, 0, buf.length);
HdfsPath.set("");
}
byte[] keybytes = key.get();
OutKey.set(keybytes, 0, keybytes.length);
context.write(OutKey, Fields);
}
}
public void reduce(
BytesWritable key,
Iterator<IntWritable> values,
OutputCollector<BytesWritable, IntWritable> output,
Reporter reporter)
throws IOException {
int ones = 0;
int twos = 0;
while (values.hasNext()) {
IntWritable count = values.next();
if (count.equals(sortInput)) {
++ones;
} else if (count.equals(sortOutput)) {
++twos;
} else {
throw new IOException(
"Invalid 'value' of " + count.get() + " for (key,value): " + key.toString());
}
}
// Check to ensure there are equal no. of ones and twos
if (ones != twos) {
throw new IOException(
"Illegal ('one', 'two'): ("
+ ones
+ ", "
+ twos
+ ") for (key, value): "
+ key.toString());
}
}
public KeyAndPartitionWritable(Domain domain, BytesWritable key) {
this.key = key;
int partition =
domain
.getPartitioner()
.partition(ByteBuffer.wrap(key.getBytes(), 0, key.getLength()), domain.getNumParts());
this.partition = new IntWritable(partition);
}
@Override
public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) {
BytesWritable writable = new BytesWritable();
ByteBuffer bb = (ByteBuffer) value;
writable.set(bb.array(), 0, bb.array().length);
size.value = bb.array().length;
return writable;
}
@Override
public boolean next(BytesWritable k, BytesWritable v) throws IOException {
if (nextKeyValue()) {
k.set(new BytesWritable(getCurrentKey().getBytes()));
v.set(new BytesWritable(getCurrentValue()));
return true;
}
return false;
}
@Override
public synchronized void write(BytesWritable key, BytesWritable value) throws IOException {
try {
mWriter.put(key.getBytes(), value.getBytes());
// Send a progress to the job manager to inform it that the task is still running.
mProgress.progress();
} catch (TachyonException e) {
throw new IOException(e);
}
}
@Override
public int getSerializedLength(Object o) throws IOException {
if (o == null) return terminate() ? fixedPrefixLength + 1 : fixedPrefixLength;
final BytesWritable input = (BytesWritable) o;
return fixedPrefixLength
+ getSerializedLength(
toStringRepresentation(
input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength));
}
@Override
@SuppressWarnings("unchecked")
public void run(final JobConf job, final TaskUmbilicalProtocol umbilical) throws IOException {
final Reporter reporter = getReporter(umbilical);
// start thread that will handle communication with parent
startCommunicationThread(umbilical);
int numReduceTasks = conf.getNumReduceTasks();
LOG.info("numReduceTasks: " + numReduceTasks);
MapOutputCollector collector = null;
if (numReduceTasks > 0) {
collector = new MapOutputBuffer(umbilical, job, reporter);
} else {
collector = new DirectMapOutputCollector(umbilical, job, reporter);
}
// reinstantiate the split
try {
instantiatedSplit =
(InputSplit) ReflectionUtils.newInstance(job.getClassByName(splitClass), job);
} catch (ClassNotFoundException exp) {
IOException wrap = new IOException("Split class " + splitClass + " not found");
wrap.initCause(exp);
throw wrap;
}
DataInputBuffer splitBuffer = new DataInputBuffer();
splitBuffer.reset(split.get(), 0, split.getSize());
instantiatedSplit.readFields(splitBuffer);
// if it is a file split, we can give more details
if (instantiatedSplit instanceof FileSplit) {
FileSplit fileSplit = (FileSplit) instantiatedSplit;
job.set("map.input.file", fileSplit.getPath().toString());
job.setLong("map.input.start", fileSplit.getStart());
job.setLong("map.input.length", fileSplit.getLength());
}
RecordReader rawIn = // open input
job.getInputFormat().getRecordReader(instantiatedSplit, job, reporter);
RecordReader in = new TrackedRecordReader(rawIn, getCounters());
MapRunnable runner = (MapRunnable) ReflectionUtils.newInstance(job.getMapRunnerClass(), job);
try {
runner.run(in, collector, reporter);
collector.flush();
} finally {
// close
in.close(); // close input
collector.close();
}
done(umbilical);
}
static Rename[] readRenameOptions(DataInputStream in) throws IOException {
BytesWritable writable = new BytesWritable();
writable.readFields(in);
byte[] bytes = writable.getBytes();
Rename[] options = new Rename[bytes.length];
for (int i = 0; i < bytes.length; i++) {
options[i] = Rename.valueOf(bytes[i]);
}
return options;
}
/** Convert bytes to SHA-1 */
public Text evaluate(BytesWritable b) {
if (b == null) {
return null;
}
digest.reset();
digest.update(b.getBytes(), 0, b.getLength());
byte[] shaBytes = digest.digest();
String shaHex = Hex.encodeHexString(shaBytes);
result.set(shaHex);
return result;
}
@Override
public void reduce(
BytesWritable topkRollupKey, Iterable<BytesWritable> timeSeriesIterable, Context context)
throws IOException, InterruptedException {
TopKRollupPhaseOneMapOutputKey wrapper =
TopKRollupPhaseOneMapOutputKey.fromBytes(topkRollupKey.getBytes());
LOGGER.info(
"DimensionName {} DimensionValue {}",
wrapper.getDimensionName(),
wrapper.getDimensionValue());
MetricTimeSeries aggregateSeries = new MetricTimeSeries(metricSchema);
for (BytesWritable writable : timeSeriesIterable) {
MetricTimeSeries series = MetricTimeSeries.fromBytes(writable.copyBytes(), metricSchema);
aggregateSeries.aggregate(series);
}
Map<String, Long> metricValues = new HashMap<String, Long>();
for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
metricValues.put(metricSpec.getName(), 0L);
}
for (Long time : aggregateSeries.getTimeWindowSet()) {
for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
String metricName = metricSpec.getName();
long metricValue = aggregateSeries.get(time, metricName).longValue();
metricValues.put(metricName, metricValues.get(metricName) + metricValue);
}
}
boolean aboveThreshold = true;
for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
String metricName = metricSpec.getName();
long metricValue = metricValues.get(metricName);
long metricSum = metricSums.get(metricName);
double metricThreshold = metricThresholds.get(metricName);
LOGGER.info("metricValue : {} metricSum : {}", metricValue, metricSum);
if (metricValue < (metricThreshold / 100) * metricSum) {
aboveThreshold = false;
break;
}
}
if (aboveThreshold) {
LOGGER.info("Passed threshold");
valWritable.set(aggregateSeries.toBytes(), 0, aggregateSeries.toBytes().length);
context.write(topkRollupKey, valWritable);
}
}
private Object deserializeValue(BytesWritable valueWritable, byte tag) throws HiveException {
try {
return inputValueDeserializer[tag].deserialize(valueWritable);
} catch (SerDeException e) {
throw new HiveException(
"Error: Unable to deserialize reduce input value (tag="
+ tag
+ ") from "
+ Utilities.formatBinaryString(valueWritable.getBytes(), 0, valueWritable.getLength())
+ " with properties "
+ valueTableDesc[tag].getProperties(),
e);
}
}
/* decrypt, then compute
*/
@Override
public void map(BytesWritable key, BytesWritable val, Context context)
throws IOException, InterruptedException {
byte[] iv = key.copyBytes();
byte[] ct = val.copyBytes();
// String[] ss = new String(crypto.decrypt_word_rnd(ct, iv)).split("\\s+");
String text = new String(crypto.decrypt_word_rnd(ct, iv));
Matcher matcher = pattern.matcher(text);
while (matcher.find()) {
context.write(new Text(matcher.group(0)), new LongWritable(1));
}
}
@Override
protected void map(
final Text key,
BytesWritable value,
final Mapper<Text, BytesWritable, Text, BytesWritable>.Context context)
throws java.io.IOException, InterruptedException {
if (wordlist.contains(key.toString())) {
IOUtils.deserialize(
value.getBytes(),
new ReadableListBinary<Object>(new ArrayList<Object>()) {
@Override
protected Object readValue(DataInput in) throws IOException {
WordDFIDF idf = new WordDFIDF();
idf.readBinary(in);
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
IOUtils.writeBinary(baos, idf);
context.write(key, new BytesWritable(baos.toByteArray()));
} catch (InterruptedException e) {
throw new IOException("");
}
return NullWritable.get();
}
});
}
};
@Override
public void map(
final Text key,
BytesWritable value,
final Mapper<Text, BytesWritable, Text, LongWritable>.Context context)
throws InterruptedException {
try {
final long[] largest = new long[] {0};
final boolean[] anyDayOverLimit = new boolean[] {false};
IOUtils.deserialize(
value.getBytes(),
new ReadableListBinary<Object>(new ArrayList<Object>()) {
@Override
protected Object readValue(DataInput in) throws IOException {
WordDFIDF idf = new WordDFIDF();
idf.readBinary(in);
if (idf.wf > wordTimeCountThresh) {
anyDayOverLimit[0] = true;
}
if (largest[0] < idf.Twf) largest[0] = idf.Twf;
return new Object();
}
});
if (anyDayOverLimit[0]) // One of the days was over the day limit
context.write(key, new LongWritable(largest[0]));
} catch (IOException e) {
System.err.println("Couldnt read word: " + key);
}
}
@Override
public void write(Writable w) throws IOException {
// Get input data
byte[] input;
int inputLength;
if (w instanceof Text) {
input = ((Text) w).getBytes();
inputLength = ((Text) w).getLength();
} else {
assert (w instanceof BytesWritable);
input = ((BytesWritable) w).get();
inputLength = ((BytesWritable) w).getSize();
}
// Add signature
byte[] wrapped = new byte[signature.length + inputLength];
for (int i = 0; i < signature.length; i++) {
wrapped[i] = signature[i];
}
for (int i = 0; i < inputLength; i++) {
wrapped[i + signature.length] = input[i];
}
// Encode
byte[] output = base64.encode(wrapped);
bytesWritable.set(output, 0, output.length);
writer.write(bytesWritable);
}
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
Text.writeString(out, splitClass);
split.write(out);
split = null;
}
private static ArrayListWritable<PairOfInts> deserializePosting(BytesWritable inputBytes) {
ArrayListWritable<PairOfInts> posting = new ArrayListWritable<PairOfInts>();
DataInputStream dataIn = new DataInputStream(new ByteArrayInputStream(inputBytes.getBytes()));
int prevDocID = 0;
try {
while (true) {
int left = WritableUtils.readVInt(dataIn);
int right = WritableUtils.readVInt(dataIn);
if (right != 0) {
posting.add(new PairOfInts(left + prevDocID, right));
prevDocID += left;
}
}
} catch (EOFException e) {
} catch (IOException e) {
}
try {
dataIn.close();
} catch (IOException e) {
}
return posting;
}
public BytesWritable toBytes() // throws IOException
{
sbuffer[0] = (byte) (m_isRC ? 1 : 0);
sbuffer[1] = (byte) ((m_refID & 0xFF000000) >> 24);
sbuffer[2] = (byte) ((m_refID & 0x00FF0000) >> 16);
sbuffer[3] = (byte) ((m_refID & 0x0000FF00) >> 8);
sbuffer[4] = (byte) ((m_refID & 0x000000FF));
sbuffer[5] = (byte) ((m_refStart & 0xFF000000) >> 24);
sbuffer[6] = (byte) ((m_refStart & 0x00FF0000) >> 16);
sbuffer[7] = (byte) ((m_refStart & 0x0000FF00) >> 8);
sbuffer[8] = (byte) ((m_refStart & 0x000000FF));
sbuffer[9] = (byte) ((m_refEnd & 0xFF000000) >> 24);
sbuffer[10] = (byte) ((m_refEnd & 0x00FF0000) >> 16);
sbuffer[11] = (byte) ((m_refEnd & 0x0000FF00) >> 8);
sbuffer[12] = (byte) ((m_refEnd & 0x000000FF));
sbuffer[13] = (byte) ((m_differences & 0xFF000000) >> 24);
sbuffer[14] = (byte) ((m_differences & 0x00FF0000) >> 16);
sbuffer[15] = (byte) ((m_differences & 0x0000FF00) >> 8);
sbuffer[16] = (byte) ((m_differences & 0x000000FF));
bytes.set(sbuffer, 0, 17);
return bytes;
}
private static byte[] getBytes(BytesWritable val) {
byte[] buffer = val.getBytes();
/*
* FIXME: remove the following part once the below jira is fixed
* https://issues.apache.org/jira/browse/HADOOP-6298
*/
long len = val.getLength();
byte[] bytes = buffer;
if (len < buffer.length) {
bytes = new byte[(int) len];
System.arraycopy(buffer, 0, bytes, 0, (int) len);
}
return bytes;
}
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
FileSystem fileSystem = FileSystem.get(configuration);
if (fileSystem.isDirectory(split.getPath())) {
return false;
}
if (fileProcessed) {
return false;
}
int fileLength = (int) split.getLength();
byte[] result = new byte[fileLength];
FSDataInputStream inputStream = null;
try {
inputStream = fileSystem.open(split.getPath());
IOUtils.readFully(inputStream, result, 0, fileLength);
currentValue.set(result, 0, fileLength);
} finally {
IOUtils.closeStream(inputStream);
}
fileProcessed = true;
return true;
}
@SuppressWarnings("unchecked")
protected void setValue(long length) throws IOException {
ByteArrayOutputStream baos;
if (length > 0) {
baos = new ByteArrayOutputStream((int) length);
} else {
baos = new ByteArrayOutputStream();
}
int size;
while ((size = zipIn.read(buf, 0, buf.length)) != -1) {
baos.write(buf, 0, size);
}
if (value instanceof Text) {
((Text) value).set(baos.toString(encoding));
} else if (value instanceof BytesWritable) {
if (batchSize > 1) {
// Copy data since XCC won't do it when Content is created.
value = (VALUEIN) new BytesWritable();
}
((BytesWritable) value).set(baos.toByteArray(), 0, baos.size());
} else {
String error = "Unsupported input value class: " + value.getClass();
LOG.error(error, new UnsupportedOperationException(error));
key = null;
}
baos.close();
}
@Override
public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) {
InputStream is = null;
try {
is = typeConverter.convertTo(InputStream.class, value);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
IOUtils.copyBytes(is, bos, HdfsConstants.DEFAULT_BUFFERSIZE, false);
BytesWritable writable = new BytesWritable();
writable.set(bos.toByteArray(), 0, bos.toByteArray().length);
size.value = bos.toByteArray().length;
return writable;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
} finally {
IOHelper.close(is);
}
}
@Override
public Object deserialize(Writable wr) throws SerDeException {
LOG.debug("Entry SdbSerDe::deserialize");
if (!(wr instanceof BytesWritable)) {
throw new SerDeException("Expected BSONWritable, received " + wr.getClass().getName());
}
BytesWritable record = (BytesWritable) wr;
ByteArrayRef bytes = new ByteArrayRef();
bytes.setData(record.getBytes());
row.init(bytes, 0, record.getLength());
return row;
}
@Override
public Object deserialize(Writable blob) throws SerDeException {
BytesWritable data = (BytesWritable) blob;
inputByteBuffer.reset(data.getBytes(), 0, data.getLength());
try {
for (int i = 0; i < columnNames.size(); i++) {
row.set(
i,
deserialize(inputByteBuffer, columnTypes.get(i), columnSortOrderIsDesc[i], row.get(i)));
}
} catch (IOException e) {
throw new SerDeException(e);
}
return row;
}
public BytesWritable readBytes(BytesWritable bw) throws IOException {
byte[] bytes = in.readBytes();
if (bw == null) {
bw = new BytesWritable(bytes);
} else {
bw.set(bytes, 0, bytes.length);
}
return bw;
}
@Override
public ByteArrayRef create(BytesWritable bw) {
if (null == bw) {
return null;
}
ByteArrayRef ba = new ByteArrayRef();
ba.setData(bw.getBytes());
return ba;
}
@Override
public void commit() throws IOException {
target.setSize(uncommittedLength);
if (compressGaps) {
writeGap(gamma + 1);
} else {
writeGapUncompressed(gamma + 1);
}
}
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
valueBytesWritable.setSize(0);
if (!scanner.advance()) {
value = null;
return false;
}
TFile.Reader.Scanner.Entry entry = scanner.entry();
// populate key, value
entry.getKey(keyBytesWritable);
StringBuilder k = new StringBuilder();
// split path contains the machine name. Create the key as splitPath + realKey
k.append(splitPath.getName()).append(":").append(new String(keyBytesWritable.getBytes()));
key.set(k.toString());
entry.getValue(valueBytesWritable);
value.set(valueBytesWritable.getBytes());
return true;
}
