Beispiel #1
0
/** @author Marko A. Rodriguez (http://markorodriguez.com) */
public class ScriptMap {

  public static final String CLASS = Tokens.makeNamespace(ScriptMap.class) + ".class";
  public static final String SCRIPT_PATH = Tokens.makeNamespace(ScriptMap.class) + ".scriptPath";
  public static final String SCRIPT_ARGS = Tokens.makeNamespace(ScriptMap.class) + ".scriptArgs";

  private static final String ARGS = "args";
  private static final String V = "v";
  private static final String SETUP_ARGS = "setup(args)";
  private static final String MAP_V_ARGS = "map(v,args)";
  private static final String CLEANUP_ARGS = "cleanup(args)";

  public static Configuration createConfiguration(final String scriptUri, final String... args) {
    Configuration configuration = new EmptyConfiguration();
    configuration.set(SCRIPT_PATH, scriptUri);
    configuration.setStrings(SCRIPT_ARGS, args);
    return configuration;
  }

  public static class Map extends Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex> {

    private final ScriptEngine engine = new FaunusGremlinScriptEngine();
    private SafeMapperOutputs outputs;
    private Text textWritable = new Text();

    @Override
    public void setup(final Mapper.Context context) throws IOException, InterruptedException {
      final FileSystem fs = FileSystem.get(context.getConfiguration());
      try {
        this.engine.eval(
            new InputStreamReader(fs.open(new Path(context.getConfiguration().get(SCRIPT_PATH)))));
        this.engine.put(ARGS, context.getConfiguration().getStrings(SCRIPT_ARGS));
        this.engine.eval(SETUP_ARGS);
      } catch (Exception e) {
        throw new InterruptedException(e.getMessage());
      }
      this.outputs = new SafeMapperOutputs(context);
    }

    @Override
    public void map(
        final NullWritable key,
        final FaunusVertex value,
        final Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex>.Context context)
        throws IOException, InterruptedException {
      if (value.hasPaths()) {
        final Object result;
        try {
          this.engine.put(V, value);
          result = engine.eval(MAP_V_ARGS);
        } catch (Exception e) {
          throw new InterruptedException(e.getMessage());
        }
        this.textWritable.set((null == result) ? Tokens.NULL : result.toString());
        this.outputs.write(Tokens.SIDEEFFECT, NullWritable.get(), this.textWritable);
      }
      this.outputs.write(Tokens.GRAPH, NullWritable.get(), value);
    }

    @Override
    public void cleanup(
        final Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex>.Context context)
        throws IOException, InterruptedException {
      try {
        this.engine.eval(CLEANUP_ARGS);
      } catch (Exception e) {
        throw new InterruptedException(e.getMessage());
      }
      this.outputs.close();
    }
  }
}
/** @author Marko A. Rodriguez (http://markorodriguez.com) */
public class ValueGroupCountMapReduce {

  public static final String PROPERTY =
      Tokens.makeNamespace(ValueGroupCountMapReduce.class) + ".property";
  public static final String CLASS =
      Tokens.makeNamespace(ValueGroupCountMapReduce.class) + ".class";
  public static final String TYPE = Tokens.makeNamespace(ValueGroupCountMapReduce.class) + ".type";

  public enum Counters {
    PROPERTIES_COUNTED
  }

  public static Configuration createConfiguration(
      final Class<? extends Element> klass,
      final String key,
      final Class<? extends Writable> type) {
    final Configuration configuration = new Configuration();
    configuration.setClass(CLASS, klass, Element.class);
    configuration.set(PROPERTY, key);
    configuration.setClass(TYPE, type, Writable.class);
    return configuration;
  }

  public static class Map
      extends Mapper<NullWritable, FaunusVertex, WritableComparable, LongWritable> {

    private String property;
    private WritableHandler handler;
    private boolean isVertex;
    // making use of in-map aggregation/combiner
    private CounterMap<Object> map;

    private SafeMapperOutputs outputs;

    @Override
    public void setup(final Mapper.Context context) throws IOException, InterruptedException {
      this.map = new CounterMap<Object>();
      this.property = context.getConfiguration().get(PROPERTY);
      this.isVertex =
          context
              .getConfiguration()
              .getClass(CLASS, Element.class, Element.class)
              .equals(Vertex.class);
      this.handler =
          new WritableHandler(
              context.getConfiguration().getClass(TYPE, Text.class, WritableComparable.class));
      this.outputs = new SafeMapperOutputs(context);
    }

    @Override
    public void map(
        final NullWritable key,
        final FaunusVertex value,
        final Mapper<NullWritable, FaunusVertex, WritableComparable, LongWritable>.Context context)
        throws IOException, InterruptedException {

      if (this.isVertex) {
        if (value.hasPaths()) {
          this.map.incr(ElementPicker.getProperty(value, this.property), value.pathCount());
          context.getCounter(Counters.PROPERTIES_COUNTED).increment(1l);
        }
      } else {
        for (final Edge e : value.getEdges(Direction.OUT)) {
          final FaunusEdge edge = (FaunusEdge) e;
          if (edge.hasPaths()) {
            this.map.incr(ElementPicker.getProperty(edge, this.property), edge.pathCount());
            context.getCounter(Counters.PROPERTIES_COUNTED).increment(1l);
          }
        }
      }

      // protected against memory explosion
      if (this.map.size() > Tokens.MAP_SPILL_OVER) {
        this.dischargeMap(context);
      }

      this.outputs.write(Tokens.GRAPH, NullWritable.get(), value);
    }

    private final LongWritable longWritable = new LongWritable();

    public void dischargeMap(
        final Mapper<NullWritable, FaunusVertex, WritableComparable, LongWritable>.Context context)
        throws IOException, InterruptedException {
      for (final java.util.Map.Entry<Object, Long> entry : this.map.entrySet()) {
        this.longWritable.set(entry.getValue());
        context.write(this.handler.set(entry.getKey()), this.longWritable);
      }
      this.map.clear();
    }

    @Override
    public void cleanup(
        final Mapper<NullWritable, FaunusVertex, WritableComparable, LongWritable>.Context context)
        throws IOException, InterruptedException {
      this.dischargeMap(context);
      this.outputs.close();
    }
  }

  public static class Combiner
      extends Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable> {

    private final LongWritable longWritable = new LongWritable();

    @Override
    public void reduce(
        final WritableComparable key,
        final Iterable<LongWritable> values,
        final Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable>.Context
            context)
        throws IOException, InterruptedException {
      long totalCount = 0;
      for (final LongWritable token : values) {
        totalCount = totalCount + token.get();
      }
      this.longWritable.set(totalCount);
      context.write(key, this.longWritable);
    }
  }

  public static class Reduce
      extends Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable> {

    private SafeReducerOutputs outputs;

    @Override
    public void setup(final Reducer.Context context) throws IOException, InterruptedException {
      this.outputs = new SafeReducerOutputs(context);
    }

    private final LongWritable longWritable = new LongWritable();

    @Override
    public void reduce(
        final WritableComparable key,
        final Iterable<LongWritable> values,
        final Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable>.Context
            context)
        throws IOException, InterruptedException {
      long totalCount = 0;
      for (final LongWritable token : values) {
        totalCount = totalCount + token.get();
      }
      this.longWritable.set(totalCount);
      this.outputs.write(Tokens.SIDEEFFECT, key, this.longWritable);
    }

    @Override
    public void cleanup(
        final Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable>.Context
            context)
        throws IOException, InterruptedException {
      this.outputs.close();
    }
  }
}
/** @author Marko A. Rodriguez (http://markorodriguez.com) */
public class CyclicPathFilterMap {

  public static final String CLASS = Tokens.makeNamespace(CyclicPathFilterMap.class) + ".class";

  public enum Counters {
    PATHS_FILTERED
  }

  public static Configuration createConfiguration(final Class<? extends Element> klass) {
    final Configuration configuration = new Configuration();
    configuration.setClass(CLASS, klass, Element.class);
    return configuration;
  }

  public static class Map extends Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex> {

    private boolean isVertex;
    private HashSet set = new HashSet();

    @Override
    public void setup(final Mapper.Context context) throws IOException, InterruptedException {
      this.isVertex =
          context
              .getConfiguration()
              .getClass(CLASS, Element.class, Element.class)
              .equals(Vertex.class);
    }

    @Override
    public void map(
        final NullWritable key,
        final FaunusVertex value,
        final Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex>.Context context)
        throws IOException, InterruptedException {

      long pathsFiltered = 0l;
      if (this.isVertex) {
        if (value.hasPaths()) {
          final Iterator<List<MicroElement>> itty = value.getPaths().iterator();
          while (itty.hasNext()) {
            final List<MicroElement> path = itty.next();
            this.set.clear();
            this.set.addAll(path);
            if (path.size() != this.set.size()) {
              itty.remove();
              pathsFiltered++;
            }
          }
        }
      } else {
        for (final Edge e : value.getEdges(Direction.BOTH)) {
          final FaunusEdge edge = (FaunusEdge) e;
          if (edge.hasPaths()) {
            final Iterator<List<MicroElement>> itty = edge.getPaths().iterator();
            while (itty.hasNext()) {
              final List<MicroElement> path = itty.next();
              this.set.clear();
              this.set.addAll(path);
              if (path.size() != this.set.size()) {
                itty.remove();
                pathsFiltered++;
              }
            }
          }
        }
      }

      context.getCounter(Counters.PATHS_FILTERED).increment(pathsFiltered);
      context.write(NullWritable.get(), value);
    }
  }
}