Esempio n. 1
0
  /** Returns true if <code>o</code> is a MyWritable with the same values. */
  @Override
  public boolean equals(Object o) {
    if (!(o instanceof MyWritable)) return false;

    MyWritable other = (MyWritable) o;
    return field1.equals(other.field1) && field2.equals(other.field2);
  }
Esempio n. 2
0
  public void set(VLongWritable fld1, VLongWritable fld2) {
    // make sure the smaller field is always put as field1
    if (fld1.get() <= fld2.get()) {
      this.field1 = fld1;
      this.field2 = fld2;
    } else {

      this.field1 = fld2;
      this.field2 = fld1;
    }
  }
Esempio n. 3
0
 /**
  * This reducer is only useful to Kneser-Ney language models as a combiner. It is included for
  * later implementation of techniques such as stupid backoff.
  */
 @Override
 public void reduce(CountOfCountsInfo key, Iterable<VLongWritable> values, Context context) {
   try {
     long sum = 0;
     for (VLongWritable count : values) {
       sum += count.get();
     }
     writable.set(sum);
     context.write(key, writable);
   } catch (InterruptedException e) {
     throw new RuntimeException(e);
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
 public VLongWritable readVLong(VLongWritable lw) throws IOException {
   if (lw == null) {
     lw = new VLongWritable();
   }
   lw.set(in.readLong());
   return lw;
 }
  public static VLongWritable getWritable(final Value v) {
    if (null == v) {
      throw new IllegalArgumentException("Value cannot be null");
    }

    ByteArrayInputStream bais = new ByteArrayInputStream(v.get());
    DataInputStream in = new DataInputStream(bais);

    VLongWritable writable = new VLongWritable();
    try {
      writable.readFields(in);
    } catch (IOException e) {
      // If this ever happens, some seriously screwed up is happening or someone subclasses Value
      // and made it do crazy stuff.
      throw new RuntimeException(e);
    }

    return writable;
  }
Esempio n. 6
0
    public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context)
        throws IOException, InterruptedException {

      int defCount = 0;

      refs.clear();
      for (VLongWritable type : values) {
        if (type.get() == -1) {
          defCount++;
        } else {
          refs.add(type.get());
        }
      }

      // TODO check for more than one def, should not happen

      if (defCount == 0 && refs.size() > 0) {
        // this is bad, found a node that is referenced but not defined. It must have been lost,
        // emit some info about this node for debugging purposes.

        StringBuilder sb = new StringBuilder();
        String comma = "";
        for (Long ref : refs) {
          sb.append(comma);
          comma = ",";
          sb.append(String.format("%016x", ref));
        }

        context.write(new Text(String.format("%016x", key.get())), new Text(sb.toString()));
        context.getCounter(Counts.UNDEFINED).increment(1);

      } else if (defCount > 0 && refs.size() == 0) {
        // node is defined but not referenced
        context.getCounter(Counts.UNREFERENCED).increment(1);
      } else {
        // node is defined and referenced
        context.getCounter(Counts.REFERENCED).increment(1);
      }
    }
  public static Value getValue(final VLongWritable w) {
    if (w == null) {
      throw new IllegalArgumentException("Writable cannot be null");
    }
    ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(byteStream);

    // We could also close it, but we know that VLongWritable and BAOS don't need it.
    try {
      w.write(out);
    } catch (IOException e) {
      // If this ever happens, some seriously screwed up is happening or someone subclasses
      // VLongWritable
      // and made it do crazy stuff.
      throw new RuntimeException(e);
    }

    return new Value(byteStream.toByteArray());
  }
  @SuppressWarnings("unchecked")
  @Override
  public Vertex<VLongWritable, Text, FloatWritable, DoubleWritable> getCurrentVertex()
      throws IOException, InterruptedException {
    if (vertex == null) {
      vertex = BspUtils.createVertex(getContext().getConfiguration());
    }
    vertex.getMsgList().clear();
    vertex.getEdges().clear();

    vertex.reset();

    /** set the src vertex id */
    vertexId.set(currentId++);
    vertex.setVertexId(vertexId);

    /** set the vertex value */
    vertex.setVertexValue(new Text("aaa"));
    return vertex;
  }
Esempio n. 9
0
    @Override
    protected void map(LongWritable key, cidynamonode node, Context context)
        throws IOException, InterruptedException {
      if (flushed != null) {
        Long count = flushed.get(node.getClient());
        if (count == null || node.getCount() >= count) {
          context.getCounter(Counts.IGNORED).increment(1);
          return;
        }
      }

      row.set(key.get());
      context.write(row, DEF);

      if (node.getPrev() >= 0) {
        ref.set((long) node.getPrev());
        vrow.set(key.get());
        context.write(ref, vrow);
      }
    }
Esempio n. 10
0
 @Override
 public String toString() {
   return field1.toString() + "\t" + field2.toString();
 }
Esempio n. 11
0
  @Override
  public int hashCode() {

    return field1.hashCode() * 163 + field2.hashCode();
  }
Esempio n. 12
0
  @Override
  public void readFields(DataInput in) throws IOException {

    field1.readFields(in);
    field2.readFields(in);
  }
Esempio n. 13
0
  // How to write and read MyWritable fields from DataOutput and DataInput stream
  @Override
  public void write(DataOutput out) throws IOException {

    field1.write(out);
    field2.write(out);
  }
Esempio n. 14
0
 @Override
 public int compare(VLongWritable o1, VLongWritable o2) {
   // TODO Auto-generated method stub
   return ((o1.get() / 10) < (o2.get() / 10) ? -1 : ((o1.get() / 10) == (o2.get() / 10) ? 0 : 1));
 }