/**
* A basic example showing the use of the org.apache.abdera2.common.misc.MapRed lightweight
* MapReduce functionality. This class is designed to provide a general purpose, *lightweight*,
* *basic*, *simple* MapReduce capability designed around working with relatively *small* data sets
* for basic analysis operations. It is NOT designed to provide an alternative to a full MapReduce
* implementation such as Hadoop.
*
* <p>The MapRed class has been tightly integrated with the Guava Libraries Function interface to
* make it possible to encapsulate a mapreduce operation within a single Function object.
*
* <p>In this example, we first pull an Atom feed and convert that into an Activity Streams. Second,
* we prepare the input data for the MapReduce operation which is composed statically and stored in
* a final static Function variable. Third, we invoke the Function with the input data
* asynchronously using an ExecutorService. The main thread waits for the operation to complete,
* then iterates the output, which, in this case, is a summarizaton of the total number of posts per
* author in the original Atom feed.
*/
public class MapRedExample {
// Prepare the various functions and store them as static final variables
private static final Function<
Iterable<Pair<Void, Activity>>, Iterable<Pair<String, Iterable<Integer>>>>
f1 = compose(new MyMapper(), MapRed.<String, ASObject>countingReducer());
private static final ReducerFunction<String, Integer, Integer, String> f2 =
asFunction(MapRed.<String, Integer>invertReducer(), Collections.<Integer>reverseOrder());
private static final Function<
Iterable<Pair<Void, Activity>>, Iterable<Pair<Integer, Iterable<String>>>>
f3 = Functions.compose(f2, f1);
private static final ExecutorService exec =
getExitingExecutorService((ThreadPoolExecutor) Executors.newCachedThreadPool());
private static final Function<
Iterable<Pair<Void, Activity>>, Future<Iterable<Pair<Integer, Iterable<String>>>>>
ff =
MoreFunctions
.<Iterable<Pair<Void, Activity>>, Iterable<Pair<Integer, Iterable<String>>>>
futureFunction(f3, exec);
public static void main(String... args) throws Exception {
// Read an Atom Feed... this part isn't required.. the mapred stuff
// works on any activity stream source, this just gives us some
// interesting input material
Abdera abdera = Abdera.getInstance();
URL url = new URL("http://planet.intertwingly.net/atom.xml");
Parser parser = abdera.getParser();
ParserOptions options = parser.makeDefaultParserOptions().charset("UTF-8").get();
Document<Feed> doc = abdera.getParser().parse(url.openStream(), url.toString(), options);
Feed feed = doc.getRoot();
ByteArrayOutputStream out = new ByteArrayOutputStream();
feed.writeTo("activity", out);
// Convert it to an Activity Stream
String r = new String(out.toByteArray(), "UTF-8");
Collection<Activity> col = IO.get().readCollection(new StringReader(r));
// Prepare the input data.. here's where the interesting bit starts...
// this first step indexes the collection of activities into a Iterable
// of Pair objects. A Pair object is essentially a tuple with two elements,
// called first() and second(). The first() is used as the key in the
// Map function, while second() is used as the value. In this particular
// case, we're using a null key on the input...
PairBuilder<Void, Activity> gen =
Pair.<Void, Activity>make().index(MoreFunctions.<Activity>alwaysVoid(), col.getItems());
// The Function ff is asynchronous... we apply it, then call get on
// the returned Future to wait for the result. The mapreduce operation
// occurs in a different thread and sets the value of the Future
// when it is complete... once it does, we iterate through the collection
// of Pairs it kicks out.. which in this case, is a listing of actors
// in the stream sorted by number of activities each.
for (Pair<Integer, Iterable<String>> entry : ff.apply(gen).get())
System.out.println(entry.first() + "=" + entry.second());
}
static class MyMapper implements Mapper<Void, Activity, String, ASObject> {
public void map(Void key, Activity val, Collector<String, ASObject> context) {
String ot = val.getActor().getDisplayName();
context.collect(ot != null ? ot : "", val.getActor());
}
}
}
public class MiscTest {
@Test
public void multiIteratorTest() {
Set<String> a1 = new LinkedHashSet<String>();
a1.add("a");
a1.add("b");
a1.add("c");
Set<String> a2 = new LinkedHashSet<String>();
a2.add("d");
a2.add("e");
a2.add("f");
Iterator<String> mi = Iterators.concat(a1.iterator(), a2.iterator());
assertEquals("a", mi.next());
assertEquals("b", mi.next());
assertEquals("c", mi.next());
assertTrue(mi.hasNext());
assertEquals("d", mi.next());
assertEquals("e", mi.next());
assertEquals("f", mi.next());
assertFalse(mi.hasNext());
}
@Test
public void arrayBuilderTest() {
// Set array builder (no duplicates)
ArrayBuilder<String> ab = ArrayBuilder.set(String.class);
String[] array = ab.add("a").add("a").add("b").add("c").build();
assertEquals(3, array.length);
assertEquals("a", array[0]);
assertEquals("b", array[1]);
assertEquals("c", array[2]);
// List array (duplicates)
ab = ArrayBuilder.list(String.class);
array = ab.add("a").add("a").add("b").add("c").build();
assertEquals(4, array.length);
assertEquals("a", array[0]);
assertEquals("a", array[1]);
assertEquals("b", array[2]);
assertEquals("c", array[3]);
}
@Test
public void chainTest() {
final AtomicInteger i = new AtomicInteger(0);
ArrayBuilder<Task<String, String>> tasks = ArrayBuilder.list(Task.class);
for (int n = 0; n < 10; n++)
tasks.add(
new Task<String, String>() {
public String apply(String input, Chain<String, String> flow) {
i.getAndIncrement();
return flow.next(input);
}
});
Chain<String, String> chain =
Chain.<String, String>make()
.via(tasks.build())
.to(
new Function<String, String>() {
public String apply(String input) {
i.getAndIncrement();
return input.toUpperCase(Locale.US);
}
})
.get();
String v = chain.apply("a");
assertEquals("A", v);
assertEquals(11, i.get());
}
@Test
public void comparisonTest() {
assertTrue(Comparisons.<String>bothApply(Predicates.<String>notNull()).apply("A", "B"));
assertFalse(Comparisons.<String>bothApply(Predicates.<String>notNull()).apply("A", null));
assertFalse(Comparisons.<String>bothApply(Predicates.<String>notNull()).apply(null, "B"));
assertTrue(Comparisons.<String>bothAreNull().apply(null, null));
assertFalse(Comparisons.<String>bothAreNull().apply("A", null));
assertFalse(Comparisons.<String>bothAreNull().apply(null, "B"));
assertTrue(Comparisons.<String>onlyFirstIsNull().apply(null, "A"));
assertFalse(Comparisons.<String>onlyFirstIsNull().apply(null, null));
assertFalse(Comparisons.<String>onlyFirstIsNull().apply("A", null));
assertFalse(Comparisons.<String>onlyFirstIsNull().apply("A", "B"));
assertTrue(Comparisons.<String>onlySecondIsNull().apply("A", null));
assertFalse(Comparisons.<String>onlySecondIsNull().apply(null, null));
assertFalse(Comparisons.<String>onlySecondIsNull().apply(null, "B"));
assertFalse(Comparisons.<String>onlySecondIsNull().apply("A", "B"));
assertTrue(Comparisons.<String>onlyOneIsNull().apply("A", null));
assertFalse(Comparisons.<String>onlyOneIsNull().apply(null, null));
assertTrue(Comparisons.<String>onlyOneIsNull().apply(null, "B"));
assertFalse(Comparisons.<String>onlyOneIsNull().apply("A", "B"));
assertFalse(Comparisons.<String>neitherIsNull().apply("A", null));
assertFalse(Comparisons.<String>neitherIsNull().apply(null, null));
assertFalse(Comparisons.<String>neitherIsNull().apply(null, "B"));
assertTrue(Comparisons.<String>neitherIsNull().apply("A", "B"));
assertFalse(Comparisons.eitherApply(Predicates.isNull()).apply("A", "B"));
assertTrue(Comparisons.eitherApply(Predicates.isNull()).apply(null, "B"));
assertTrue(Comparisons.eitherApply(Predicates.isNull()).apply("A", null));
assertTrue(Comparisons.eitherApply(Predicates.isNull()).apply(null, null));
Comparison<String> equalsIgnoreCase =
new Comparison<String>() {
public boolean apply(String r1, String r2) {
return r1.equalsIgnoreCase(r2);
}
};
assertTrue(equalsIgnoreCase.apply("A", "a"));
Comparison<String> c = Comparisons.<String>neitherIsNull().and(equalsIgnoreCase);
assertTrue(c.apply("A", "a"));
assertFalse(c.apply(null, "a"));
assertFalse(c.apply("a", null));
}
@Test
public void testMoreFunctions() {
// Up first... Choice
Function<String, Integer> choice =
MoreFunctions.<String, Integer>choice()
.of(Comparisons.equalsIgnoreCase().predicateFor("A"), 1)
.of(Comparisons.equalsIgnoreCase().predicateFor("B"), 2)
.otherwise(3)
.get();
assertEquals(Integer.valueOf(1), choice.apply("a"));
assertEquals(Integer.valueOf(2), choice.apply("b"));
assertEquals(Integer.valueOf(3), choice.apply("c"));
// Test createInstance
Foo foo = MoreFunctions.createInstance(Foo.class).apply(null);
assertNotNull(foo);
foo = MoreFunctions.createInstance(Foo.class, String.class).apply(MoreFunctions.array("A"));
assertNotNull(foo);
assertEquals("A", foo.a());
// Test each
String[] array = MoreFunctions.array("a", "b", "c", "d");
array =
MoreFunctions.eachArray(
new Function<String, String>() {
public String apply(String input) {
return input.toUpperCase();
}
},
String.class)
.apply(array);
assertEquals("A", array[0]);
assertEquals("B", array[1]);
assertEquals("C", array[2]);
assertEquals("D", array[3]);
// Test firstNonNull...
assertEquals("A", MoreFunctions.firstNonNull(null, null, null, "A"));
// Test futureFunction... executes the function in a separate
// thread using the passed in ExecutorService
Future<String> future =
MoreFunctions.<String, String>futureFunction(
new Function<String, String>() {
public String apply(String input) {
System.out.println("Thread sleeping....");
try {
Thread.sleep(10 * 1000);
} catch (Throwable t) {
}
return input;
}
},
MoreExecutors2.getExitingExecutor())
.apply("A");
try {
System.out.println("Waiting for return...");
assertEquals("A", future.get());
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
@Test
public void testPair() {
Pair<String, String> pair = Pair.of("A", "B");
assertEquals("A", pair.first());
assertEquals("B", pair.second());
Map<String, String> map = new HashMap<String, String>();
map.put("A", "B");
map.put("B", "C");
Iterable<Pair<String, String>> pairs = Pair.from(map);
assertEquals(2, Iterables.size(pairs));
assertEquals("A", Iterables.get(pairs, 0).first());
assertEquals("B", Iterables.get(pairs, 0).second());
assertEquals("B", Iterables.get(pairs, 1).first());
assertEquals("C", Iterables.get(pairs, 1).second());
}
private static final Function<
Iterable<Pair<Void, Activity>>, Iterable<Pair<String, Iterable<Integer>>>>
f1 = compose(new MyMapper(), MapRed.<String, ASObject>countingReducer());
private static final ReducerFunction<String, Integer, Integer, String> f2 =
asFunction(MapRed.<String, Integer>invertReducer(), Collections.<Integer>reverseOrder());
private static final Function<
Iterable<Pair<Void, Activity>>, Iterable<Pair<Integer, Iterable<String>>>>
f3 = Functions.compose(f2, f1);
private static final ExecutorService exec = MoreExecutors2.getExitingExecutor();
private static final Function<
Collection<Activity>, Future<Iterable<Pair<Integer, Iterable<String>>>>>
ff =
Functions.compose(
MoreFunctions
.<Iterable<Pair<Void, Activity>>, Iterable<Pair<Integer, Iterable<String>>>>
futureFunction(f3, exec),
Extra.<Activity>pairIndexer());
private Activity getActivity(String name, int n) {
return Activity.makeActivity()
.actor(PersonObject.makePerson(name))
.id(String.format("urn:%s:%s", name, n))
.get();
}
@Test
public void testMapRed() throws Exception {
Collection<Activity> col =
Collection.<Activity>makeCollection()
.item(getActivity("Joe", 1))
.item(getActivity("Joe", 2))
.item(getActivity("Mark", 3))
.item(getActivity("Mark", 4))
.item(getActivity("Sally", 5))
.get();
// This is basically MapReduce contained within a Function,
// Runs asynch using exiting executorservice... call to
// ff.apply(gen).get() hides all the magic...
// this particular function looks at the activity stream
// and counts the number of activities per actor
Iterable<Pair<Integer, Iterable<String>>> ret = ff.apply(col).get();
Pair<Integer, Iterable<String>> first = Iterables.get(ret, 0);
Pair<Integer, Iterable<String>> second = Iterables.get(ret, 1);
assertEquals(Integer.valueOf(2), first.first());
assertThat(first.second(), hasItems("Joe", "Mark"));
assertEquals(Integer.valueOf(1), second.first());
assertThat(second.second(), hasItems("Sally"));
}
static class MyMapper implements Mapper<Void, Activity, String, ASObject> {
public void map(Void key, Activity val, Collector<String, ASObject> context) {
String ot = val.getActor().getDisplayName();
context.collect(ot != null ? ot : "", val.getActor());
}
}
public static class Foo {
private final String a;
public Foo() {
this.a = null;
}
public Foo(String a) {
this.a = a;
}
public String a() {
return a;
}
}
}
