/** @author Marko A. Rodriguez (http://markorodriguez.com) */
public class ScriptMap {
public static final String CLASS = Tokens.makeNamespace(ScriptMap.class) + ".class";
public static final String SCRIPT_PATH = Tokens.makeNamespace(ScriptMap.class) + ".scriptPath";
public static final String SCRIPT_ARGS = Tokens.makeNamespace(ScriptMap.class) + ".scriptArgs";
private static final String ARGS = "args";
private static final String V = "v";
private static final String SETUP_ARGS = "setup(args)";
private static final String MAP_V_ARGS = "map(v,args)";
private static final String CLEANUP_ARGS = "cleanup(args)";
public static Configuration createConfiguration(final String scriptUri, final String... args) {
Configuration configuration = new EmptyConfiguration();
configuration.set(SCRIPT_PATH, scriptUri);
configuration.setStrings(SCRIPT_ARGS, args);
return configuration;
}
public static class Map extends Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex> {
private final ScriptEngine engine = new FaunusGremlinScriptEngine();
private SafeMapperOutputs outputs;
private Text textWritable = new Text();
@Override
public void setup(final Mapper.Context context) throws IOException, InterruptedException {
final FileSystem fs = FileSystem.get(context.getConfiguration());
try {
this.engine.eval(
new InputStreamReader(fs.open(new Path(context.getConfiguration().get(SCRIPT_PATH)))));
this.engine.put(ARGS, context.getConfiguration().getStrings(SCRIPT_ARGS));
this.engine.eval(SETUP_ARGS);
} catch (Exception e) {
throw new InterruptedException(e.getMessage());
}
this.outputs = new SafeMapperOutputs(context);
}
@Override
public void map(
final NullWritable key,
final FaunusVertex value,
final Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex>.Context context)
throws IOException, InterruptedException {
if (value.hasPaths()) {
final Object result;
try {
this.engine.put(V, value);
result = engine.eval(MAP_V_ARGS);
} catch (Exception e) {
throw new InterruptedException(e.getMessage());
}
this.textWritable.set((null == result) ? Tokens.NULL : result.toString());
this.outputs.write(Tokens.SIDEEFFECT, NullWritable.get(), this.textWritable);
}
this.outputs.write(Tokens.GRAPH, NullWritable.get(), value);
}
@Override
public void cleanup(
final Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex>.Context context)
throws IOException, InterruptedException {
try {
this.engine.eval(CLEANUP_ARGS);
} catch (Exception e) {
throw new InterruptedException(e.getMessage());
}
this.outputs.close();
}
}
}
/** @author Marko A. Rodriguez (http://markorodriguez.com) */
public class ValueGroupCountMapReduce {
public static final String PROPERTY =
Tokens.makeNamespace(ValueGroupCountMapReduce.class) + ".property";
public static final String CLASS =
Tokens.makeNamespace(ValueGroupCountMapReduce.class) + ".class";
public static final String TYPE = Tokens.makeNamespace(ValueGroupCountMapReduce.class) + ".type";
public enum Counters {
PROPERTIES_COUNTED
}
public static Configuration createConfiguration(
final Class<? extends Element> klass,
final String key,
final Class<? extends Writable> type) {
final Configuration configuration = new Configuration();
configuration.setClass(CLASS, klass, Element.class);
configuration.set(PROPERTY, key);
configuration.setClass(TYPE, type, Writable.class);
return configuration;
}
public static class Map
extends Mapper<NullWritable, FaunusVertex, WritableComparable, LongWritable> {
private String property;
private WritableHandler handler;
private boolean isVertex;
// making use of in-map aggregation/combiner
private CounterMap<Object> map;
private SafeMapperOutputs outputs;
@Override
public void setup(final Mapper.Context context) throws IOException, InterruptedException {
this.map = new CounterMap<Object>();
this.property = context.getConfiguration().get(PROPERTY);
this.isVertex =
context
.getConfiguration()
.getClass(CLASS, Element.class, Element.class)
.equals(Vertex.class);
this.handler =
new WritableHandler(
context.getConfiguration().getClass(TYPE, Text.class, WritableComparable.class));
this.outputs = new SafeMapperOutputs(context);
}
@Override
public void map(
final NullWritable key,
final FaunusVertex value,
final Mapper<NullWritable, FaunusVertex, WritableComparable, LongWritable>.Context context)
throws IOException, InterruptedException {
if (this.isVertex) {
if (value.hasPaths()) {
this.map.incr(ElementPicker.getProperty(value, this.property), value.pathCount());
context.getCounter(Counters.PROPERTIES_COUNTED).increment(1l);
}
} else {
for (final Edge e : value.getEdges(Direction.OUT)) {
final FaunusEdge edge = (FaunusEdge) e;
if (edge.hasPaths()) {
this.map.incr(ElementPicker.getProperty(edge, this.property), edge.pathCount());
context.getCounter(Counters.PROPERTIES_COUNTED).increment(1l);
}
}
}
// protected against memory explosion
if (this.map.size() > Tokens.MAP_SPILL_OVER) {
this.dischargeMap(context);
}
this.outputs.write(Tokens.GRAPH, NullWritable.get(), value);
}
private final LongWritable longWritable = new LongWritable();
public void dischargeMap(
final Mapper<NullWritable, FaunusVertex, WritableComparable, LongWritable>.Context context)
throws IOException, InterruptedException {
for (final java.util.Map.Entry<Object, Long> entry : this.map.entrySet()) {
this.longWritable.set(entry.getValue());
context.write(this.handler.set(entry.getKey()), this.longWritable);
}
this.map.clear();
}
@Override
public void cleanup(
final Mapper<NullWritable, FaunusVertex, WritableComparable, LongWritable>.Context context)
throws IOException, InterruptedException {
this.dischargeMap(context);
this.outputs.close();
}
}
public static class Combiner
extends Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable> {
private final LongWritable longWritable = new LongWritable();
@Override
public void reduce(
final WritableComparable key,
final Iterable<LongWritable> values,
final Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable>.Context
context)
throws IOException, InterruptedException {
long totalCount = 0;
for (final LongWritable token : values) {
totalCount = totalCount + token.get();
}
this.longWritable.set(totalCount);
context.write(key, this.longWritable);
}
}
public static class Reduce
extends Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable> {
private SafeReducerOutputs outputs;
@Override
public void setup(final Reducer.Context context) throws IOException, InterruptedException {
this.outputs = new SafeReducerOutputs(context);
}
private final LongWritable longWritable = new LongWritable();
@Override
public void reduce(
final WritableComparable key,
final Iterable<LongWritable> values,
final Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable>.Context
context)
throws IOException, InterruptedException {
long totalCount = 0;
for (final LongWritable token : values) {
totalCount = totalCount + token.get();
}
this.longWritable.set(totalCount);
this.outputs.write(Tokens.SIDEEFFECT, key, this.longWritable);
}
@Override
public void cleanup(
final Reducer<WritableComparable, LongWritable, WritableComparable, LongWritable>.Context
context)
throws IOException, InterruptedException {
this.outputs.close();
}
}
}
/** @author Marko A. Rodriguez (http://markorodriguez.com) */
public class CyclicPathFilterMap {
public static final String CLASS = Tokens.makeNamespace(CyclicPathFilterMap.class) + ".class";
public enum Counters {
PATHS_FILTERED
}
public static Configuration createConfiguration(final Class<? extends Element> klass) {
final Configuration configuration = new Configuration();
configuration.setClass(CLASS, klass, Element.class);
return configuration;
}
public static class Map extends Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex> {
private boolean isVertex;
private HashSet set = new HashSet();
@Override
public void setup(final Mapper.Context context) throws IOException, InterruptedException {
this.isVertex =
context
.getConfiguration()
.getClass(CLASS, Element.class, Element.class)
.equals(Vertex.class);
}
@Override
public void map(
final NullWritable key,
final FaunusVertex value,
final Mapper<NullWritable, FaunusVertex, NullWritable, FaunusVertex>.Context context)
throws IOException, InterruptedException {
long pathsFiltered = 0l;
if (this.isVertex) {
if (value.hasPaths()) {
final Iterator<List<MicroElement>> itty = value.getPaths().iterator();
while (itty.hasNext()) {
final List<MicroElement> path = itty.next();
this.set.clear();
this.set.addAll(path);
if (path.size() != this.set.size()) {
itty.remove();
pathsFiltered++;
}
}
}
} else {
for (final Edge e : value.getEdges(Direction.BOTH)) {
final FaunusEdge edge = (FaunusEdge) e;
if (edge.hasPaths()) {
final Iterator<List<MicroElement>> itty = edge.getPaths().iterator();
while (itty.hasNext()) {
final List<MicroElement> path = itty.next();
this.set.clear();
this.set.addAll(path);
if (path.size() != this.set.size()) {
itty.remove();
pathsFiltered++;
}
}
}
}
}
context.getCounter(Counters.PATHS_FILTERED).increment(pathsFiltered);
context.write(NullWritable.get(), value);
}
}
}