/** Returns true if <code>o</code> is a MyWritable with the same values. */
@Override
public boolean equals(Object o) {
if (!(o instanceof MyWritable)) return false;
MyWritable other = (MyWritable) o;
return field1.equals(other.field1) && field2.equals(other.field2);
}
public void set(VLongWritable fld1, VLongWritable fld2) {
// make sure the smaller field is always put as field1
if (fld1.get() <= fld2.get()) {
this.field1 = fld1;
this.field2 = fld2;
} else {
this.field1 = fld2;
this.field2 = fld1;
}
}
/**
* This reducer is only useful to Kneser-Ney language models as a combiner. It is included for
* later implementation of techniques such as stupid backoff.
*/
@Override
public void reduce(CountOfCountsInfo key, Iterable<VLongWritable> values, Context context) {
try {
long sum = 0;
for (VLongWritable count : values) {
sum += count.get();
}
writable.set(sum);
context.write(key, writable);
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public VLongWritable readVLong(VLongWritable lw) throws IOException {
if (lw == null) {
lw = new VLongWritable();
}
lw.set(in.readLong());
return lw;
}
public static VLongWritable getWritable(final Value v) {
if (null == v) {
throw new IllegalArgumentException("Value cannot be null");
}
ByteArrayInputStream bais = new ByteArrayInputStream(v.get());
DataInputStream in = new DataInputStream(bais);
VLongWritable writable = new VLongWritable();
try {
writable.readFields(in);
} catch (IOException e) {
// If this ever happens, some seriously screwed up is happening or someone subclasses Value
// and made it do crazy stuff.
throw new RuntimeException(e);
}
return writable;
}
public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context)
throws IOException, InterruptedException {
int defCount = 0;
refs.clear();
for (VLongWritable type : values) {
if (type.get() == -1) {
defCount++;
} else {
refs.add(type.get());
}
}
// TODO check for more than one def, should not happen
if (defCount == 0 && refs.size() > 0) {
// this is bad, found a node that is referenced but not defined. It must have been lost,
// emit some info about this node for debugging purposes.
StringBuilder sb = new StringBuilder();
String comma = "";
for (Long ref : refs) {
sb.append(comma);
comma = ",";
sb.append(String.format("%016x", ref));
}
context.write(new Text(String.format("%016x", key.get())), new Text(sb.toString()));
context.getCounter(Counts.UNDEFINED).increment(1);
} else if (defCount > 0 && refs.size() == 0) {
// node is defined but not referenced
context.getCounter(Counts.UNREFERENCED).increment(1);
} else {
// node is defined and referenced
context.getCounter(Counts.REFERENCED).increment(1);
}
}
public static Value getValue(final VLongWritable w) {
if (w == null) {
throw new IllegalArgumentException("Writable cannot be null");
}
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(byteStream);
// We could also close it, but we know that VLongWritable and BAOS don't need it.
try {
w.write(out);
} catch (IOException e) {
// If this ever happens, some seriously screwed up is happening or someone subclasses
// VLongWritable
// and made it do crazy stuff.
throw new RuntimeException(e);
}
return new Value(byteStream.toByteArray());
}
@SuppressWarnings("unchecked")
@Override
public Vertex<VLongWritable, Text, FloatWritable, DoubleWritable> getCurrentVertex()
throws IOException, InterruptedException {
if (vertex == null) {
vertex = BspUtils.createVertex(getContext().getConfiguration());
}
vertex.getMsgList().clear();
vertex.getEdges().clear();
vertex.reset();
/** set the src vertex id */
vertexId.set(currentId++);
vertex.setVertexId(vertexId);
/** set the vertex value */
vertex.setVertexValue(new Text("aaa"));
return vertex;
}
@Override
protected void map(LongWritable key, cidynamonode node, Context context)
throws IOException, InterruptedException {
if (flushed != null) {
Long count = flushed.get(node.getClient());
if (count == null || node.getCount() >= count) {
context.getCounter(Counts.IGNORED).increment(1);
return;
}
}
row.set(key.get());
context.write(row, DEF);
if (node.getPrev() >= 0) {
ref.set((long) node.getPrev());
vrow.set(key.get());
context.write(ref, vrow);
}
}
@Override
public String toString() {
return field1.toString() + "\t" + field2.toString();
}
@Override
public int hashCode() {
return field1.hashCode() * 163 + field2.hashCode();
}
@Override
public void readFields(DataInput in) throws IOException {
field1.readFields(in);
field2.readFields(in);
}
// How to write and read MyWritable fields from DataOutput and DataInput stream
@Override
public void write(DataOutput out) throws IOException {
field1.write(out);
field2.write(out);
}
@Override
public int compare(VLongWritable o1, VLongWritable o2) {
// TODO Auto-generated method stub
return ((o1.get() / 10) < (o2.get() / 10) ? -1 : ((o1.get() / 10) == (o2.get() / 10) ? 0 : 1));
}
