Esempio n. 1
0
  public static void main(String... args) throws Exception {
    // Read an Atom Feed... this part isn't required.. the mapred stuff
    // works on any activity stream source, this just gives us some
    // interesting input material
    Abdera abdera = Abdera.getInstance();
    URL url = new URL("http://planet.intertwingly.net/atom.xml");
    Parser parser = abdera.getParser();
    ParserOptions options = parser.makeDefaultParserOptions().charset("UTF-8").get();
    Document<Feed> doc = abdera.getParser().parse(url.openStream(), url.toString(), options);
    Feed feed = doc.getRoot();
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    feed.writeTo("activity", out);

    // Convert it to an Activity Stream
    String r = new String(out.toByteArray(), "UTF-8");
    Collection<Activity> col = IO.get().readCollection(new StringReader(r));

    // Prepare the input data.. here's where the interesting bit starts...
    // this first step indexes the collection of activities into a Iterable
    // of Pair objects. A Pair object is essentially a tuple with two elements,
    // called first() and second(). The first() is used as the key in the
    // Map function, while second() is used as the value. In this particular
    // case, we're using a null key on the input...
    PairBuilder<Void, Activity> gen =
        Pair.<Void, Activity>make().index(MoreFunctions.<Activity>alwaysVoid(), col.getItems());

    // The Function ff is asynchronous... we apply it, then call get on
    // the returned Future to wait for the result. The mapreduce operation
    // occurs in a different thread and sets the value of the Future
    // when it is complete... once it does, we iterate through the collection
    // of Pairs it kicks out.. which in this case, is a listing of actors
    // in the stream sorted by number of activities each.
    for (Pair<Integer, Iterable<String>> entry : ff.apply(gen).get())
      System.out.println(entry.first() + "=" + entry.second());
  }
Esempio n. 2
0
 public int hashCode() {
   return MoreFunctions.genHashCode(1, name.toLowerCase());
 }
Esempio n. 3
0
public class MiscTest {

  @Test
  public void multiIteratorTest() {
    Set<String> a1 = new LinkedHashSet<String>();
    a1.add("a");
    a1.add("b");
    a1.add("c");
    Set<String> a2 = new LinkedHashSet<String>();
    a2.add("d");
    a2.add("e");
    a2.add("f");
    Iterator<String> mi = Iterators.concat(a1.iterator(), a2.iterator());
    assertEquals("a", mi.next());
    assertEquals("b", mi.next());
    assertEquals("c", mi.next());
    assertTrue(mi.hasNext());
    assertEquals("d", mi.next());
    assertEquals("e", mi.next());
    assertEquals("f", mi.next());
    assertFalse(mi.hasNext());
  }

  @Test
  public void arrayBuilderTest() {
    // Set array builder (no duplicates)
    ArrayBuilder<String> ab = ArrayBuilder.set(String.class);
    String[] array = ab.add("a").add("a").add("b").add("c").build();
    assertEquals(3, array.length);
    assertEquals("a", array[0]);
    assertEquals("b", array[1]);
    assertEquals("c", array[2]);

    // List array (duplicates)
    ab = ArrayBuilder.list(String.class);
    array = ab.add("a").add("a").add("b").add("c").build();
    assertEquals(4, array.length);
    assertEquals("a", array[0]);
    assertEquals("a", array[1]);
    assertEquals("b", array[2]);
    assertEquals("c", array[3]);
  }

  @Test
  public void chainTest() {
    final AtomicInteger i = new AtomicInteger(0);
    ArrayBuilder<Task<String, String>> tasks = ArrayBuilder.list(Task.class);
    for (int n = 0; n < 10; n++)
      tasks.add(
          new Task<String, String>() {
            public String apply(String input, Chain<String, String> flow) {
              i.getAndIncrement();
              return flow.next(input);
            }
          });
    Chain<String, String> chain =
        Chain.<String, String>make()
            .via(tasks.build())
            .to(
                new Function<String, String>() {
                  public String apply(String input) {
                    i.getAndIncrement();
                    return input.toUpperCase(Locale.US);
                  }
                })
            .get();
    String v = chain.apply("a");
    assertEquals("A", v);
    assertEquals(11, i.get());
  }

  @Test
  public void comparisonTest() {
    assertTrue(Comparisons.<String>bothApply(Predicates.<String>notNull()).apply("A", "B"));
    assertFalse(Comparisons.<String>bothApply(Predicates.<String>notNull()).apply("A", null));
    assertFalse(Comparisons.<String>bothApply(Predicates.<String>notNull()).apply(null, "B"));

    assertTrue(Comparisons.<String>bothAreNull().apply(null, null));
    assertFalse(Comparisons.<String>bothAreNull().apply("A", null));
    assertFalse(Comparisons.<String>bothAreNull().apply(null, "B"));

    assertTrue(Comparisons.<String>onlyFirstIsNull().apply(null, "A"));
    assertFalse(Comparisons.<String>onlyFirstIsNull().apply(null, null));
    assertFalse(Comparisons.<String>onlyFirstIsNull().apply("A", null));
    assertFalse(Comparisons.<String>onlyFirstIsNull().apply("A", "B"));

    assertTrue(Comparisons.<String>onlySecondIsNull().apply("A", null));
    assertFalse(Comparisons.<String>onlySecondIsNull().apply(null, null));
    assertFalse(Comparisons.<String>onlySecondIsNull().apply(null, "B"));
    assertFalse(Comparisons.<String>onlySecondIsNull().apply("A", "B"));

    assertTrue(Comparisons.<String>onlyOneIsNull().apply("A", null));
    assertFalse(Comparisons.<String>onlyOneIsNull().apply(null, null));
    assertTrue(Comparisons.<String>onlyOneIsNull().apply(null, "B"));
    assertFalse(Comparisons.<String>onlyOneIsNull().apply("A", "B"));

    assertFalse(Comparisons.<String>neitherIsNull().apply("A", null));
    assertFalse(Comparisons.<String>neitherIsNull().apply(null, null));
    assertFalse(Comparisons.<String>neitherIsNull().apply(null, "B"));
    assertTrue(Comparisons.<String>neitherIsNull().apply("A", "B"));

    assertFalse(Comparisons.eitherApply(Predicates.isNull()).apply("A", "B"));
    assertTrue(Comparisons.eitherApply(Predicates.isNull()).apply(null, "B"));
    assertTrue(Comparisons.eitherApply(Predicates.isNull()).apply("A", null));
    assertTrue(Comparisons.eitherApply(Predicates.isNull()).apply(null, null));

    Comparison<String> equalsIgnoreCase =
        new Comparison<String>() {
          public boolean apply(String r1, String r2) {
            return r1.equalsIgnoreCase(r2);
          }
        };
    assertTrue(equalsIgnoreCase.apply("A", "a"));

    Comparison<String> c = Comparisons.<String>neitherIsNull().and(equalsIgnoreCase);
    assertTrue(c.apply("A", "a"));
    assertFalse(c.apply(null, "a"));
    assertFalse(c.apply("a", null));
  }

  @Test
  public void testMoreFunctions() {
    // Up first... Choice
    Function<String, Integer> choice =
        MoreFunctions.<String, Integer>choice()
            .of(Comparisons.equalsIgnoreCase().predicateFor("A"), 1)
            .of(Comparisons.equalsIgnoreCase().predicateFor("B"), 2)
            .otherwise(3)
            .get();
    assertEquals(Integer.valueOf(1), choice.apply("a"));
    assertEquals(Integer.valueOf(2), choice.apply("b"));
    assertEquals(Integer.valueOf(3), choice.apply("c"));

    // Test createInstance
    Foo foo = MoreFunctions.createInstance(Foo.class).apply(null);
    assertNotNull(foo);

    foo = MoreFunctions.createInstance(Foo.class, String.class).apply(MoreFunctions.array("A"));
    assertNotNull(foo);
    assertEquals("A", foo.a());

    // Test each
    String[] array = MoreFunctions.array("a", "b", "c", "d");
    array =
        MoreFunctions.eachArray(
                new Function<String, String>() {
                  public String apply(String input) {
                    return input.toUpperCase();
                  }
                },
                String.class)
            .apply(array);
    assertEquals("A", array[0]);
    assertEquals("B", array[1]);
    assertEquals("C", array[2]);
    assertEquals("D", array[3]);

    // Test firstNonNull...
    assertEquals("A", MoreFunctions.firstNonNull(null, null, null, "A"));

    // Test futureFunction... executes the function in a separate
    // thread using the passed in ExecutorService
    Future<String> future =
        MoreFunctions.<String, String>futureFunction(
                new Function<String, String>() {
                  public String apply(String input) {
                    System.out.println("Thread sleeping....");
                    try {
                      Thread.sleep(10 * 1000);
                    } catch (Throwable t) {
                    }
                    return input;
                  }
                },
                MoreExecutors2.getExitingExecutor())
            .apply("A");
    try {
      System.out.println("Waiting for return...");
      assertEquals("A", future.get());
    } catch (Throwable t) {
      throw new RuntimeException(t);
    }
  }

  @Test
  public void testPair() {
    Pair<String, String> pair = Pair.of("A", "B");
    assertEquals("A", pair.first());
    assertEquals("B", pair.second());

    Map<String, String> map = new HashMap<String, String>();
    map.put("A", "B");
    map.put("B", "C");
    Iterable<Pair<String, String>> pairs = Pair.from(map);
    assertEquals(2, Iterables.size(pairs));
    assertEquals("A", Iterables.get(pairs, 0).first());
    assertEquals("B", Iterables.get(pairs, 0).second());
    assertEquals("B", Iterables.get(pairs, 1).first());
    assertEquals("C", Iterables.get(pairs, 1).second());
  }

  private static final Function<
          Iterable<Pair<Void, Activity>>, Iterable<Pair<String, Iterable<Integer>>>>
      f1 = compose(new MyMapper(), MapRed.<String, ASObject>countingReducer());

  private static final ReducerFunction<String, Integer, Integer, String> f2 =
      asFunction(MapRed.<String, Integer>invertReducer(), Collections.<Integer>reverseOrder());

  private static final Function<
          Iterable<Pair<Void, Activity>>, Iterable<Pair<Integer, Iterable<String>>>>
      f3 = Functions.compose(f2, f1);

  private static final ExecutorService exec = MoreExecutors2.getExitingExecutor();

  private static final Function<
          Collection<Activity>, Future<Iterable<Pair<Integer, Iterable<String>>>>>
      ff =
          Functions.compose(
              MoreFunctions
                  .<Iterable<Pair<Void, Activity>>, Iterable<Pair<Integer, Iterable<String>>>>
                      futureFunction(f3, exec),
              Extra.<Activity>pairIndexer());

  private Activity getActivity(String name, int n) {
    return Activity.makeActivity()
        .actor(PersonObject.makePerson(name))
        .id(String.format("urn:%s:%s", name, n))
        .get();
  }

  @Test
  public void testMapRed() throws Exception {
    Collection<Activity> col =
        Collection.<Activity>makeCollection()
            .item(getActivity("Joe", 1))
            .item(getActivity("Joe", 2))
            .item(getActivity("Mark", 3))
            .item(getActivity("Mark", 4))
            .item(getActivity("Sally", 5))
            .get();

    // This is basically MapReduce contained within a Function,
    // Runs asynch using exiting executorservice... call to
    // ff.apply(gen).get() hides all the magic...
    // this particular function looks at the activity stream
    // and counts the number of activities per actor

    Iterable<Pair<Integer, Iterable<String>>> ret = ff.apply(col).get();

    Pair<Integer, Iterable<String>> first = Iterables.get(ret, 0);
    Pair<Integer, Iterable<String>> second = Iterables.get(ret, 1);
    assertEquals(Integer.valueOf(2), first.first());
    assertThat(first.second(), hasItems("Joe", "Mark"));
    assertEquals(Integer.valueOf(1), second.first());
    assertThat(second.second(), hasItems("Sally"));
  }

  static class MyMapper implements Mapper<Void, Activity, String, ASObject> {
    public void map(Void key, Activity val, Collector<String, ASObject> context) {
      String ot = val.getActor().getDisplayName();
      context.collect(ot != null ? ot : "", val.getActor());
    }
  }

  public static class Foo {
    private final String a;

    public Foo() {
      this.a = null;
    }

    public Foo(String a) {
      this.a = a;
    }

    public String a() {
      return a;
    }
  }
}
Esempio n. 4
0
/**
 * A basic example showing the use of the org.apache.abdera2.common.misc.MapRed lightweight
 * MapReduce functionality. This class is designed to provide a general purpose, *lightweight*,
 * *basic*, *simple* MapReduce capability designed around working with relatively *small* data sets
 * for basic analysis operations. It is NOT designed to provide an alternative to a full MapReduce
 * implementation such as Hadoop.
 *
 * <p>The MapRed class has been tightly integrated with the Guava Libraries Function interface to
 * make it possible to encapsulate a mapreduce operation within a single Function object.
 *
 * <p>In this example, we first pull an Atom feed and convert that into an Activity Streams. Second,
 * we prepare the input data for the MapReduce operation which is composed statically and stored in
 * a final static Function variable. Third, we invoke the Function with the input data
 * asynchronously using an ExecutorService. The main thread waits for the operation to complete,
 * then iterates the output, which, in this case, is a summarizaton of the total number of posts per
 * author in the original Atom feed.
 */
public class MapRedExample {
  // Prepare the various functions and store them as static final variables

  private static final Function<
          Iterable<Pair<Void, Activity>>, Iterable<Pair<String, Iterable<Integer>>>>
      f1 = compose(new MyMapper(), MapRed.<String, ASObject>countingReducer());

  private static final ReducerFunction<String, Integer, Integer, String> f2 =
      asFunction(MapRed.<String, Integer>invertReducer(), Collections.<Integer>reverseOrder());

  private static final Function<
          Iterable<Pair<Void, Activity>>, Iterable<Pair<Integer, Iterable<String>>>>
      f3 = Functions.compose(f2, f1);

  private static final ExecutorService exec =
      getExitingExecutorService((ThreadPoolExecutor) Executors.newCachedThreadPool());

  private static final Function<
          Iterable<Pair<Void, Activity>>, Future<Iterable<Pair<Integer, Iterable<String>>>>>
      ff =
          MoreFunctions
              .<Iterable<Pair<Void, Activity>>, Iterable<Pair<Integer, Iterable<String>>>>
                  futureFunction(f3, exec);

  public static void main(String... args) throws Exception {
    // Read an Atom Feed... this part isn't required.. the mapred stuff
    // works on any activity stream source, this just gives us some
    // interesting input material
    Abdera abdera = Abdera.getInstance();
    URL url = new URL("http://planet.intertwingly.net/atom.xml");
    Parser parser = abdera.getParser();
    ParserOptions options = parser.makeDefaultParserOptions().charset("UTF-8").get();
    Document<Feed> doc = abdera.getParser().parse(url.openStream(), url.toString(), options);
    Feed feed = doc.getRoot();
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    feed.writeTo("activity", out);

    // Convert it to an Activity Stream
    String r = new String(out.toByteArray(), "UTF-8");
    Collection<Activity> col = IO.get().readCollection(new StringReader(r));

    // Prepare the input data.. here's where the interesting bit starts...
    // this first step indexes the collection of activities into a Iterable
    // of Pair objects. A Pair object is essentially a tuple with two elements,
    // called first() and second(). The first() is used as the key in the
    // Map function, while second() is used as the value. In this particular
    // case, we're using a null key on the input...
    PairBuilder<Void, Activity> gen =
        Pair.<Void, Activity>make().index(MoreFunctions.<Activity>alwaysVoid(), col.getItems());

    // The Function ff is asynchronous... we apply it, then call get on
    // the returned Future to wait for the result. The mapreduce operation
    // occurs in a different thread and sets the value of the Future
    // when it is complete... once it does, we iterate through the collection
    // of Pairs it kicks out.. which in this case, is a listing of actors
    // in the stream sorted by number of activities each.
    for (Pair<Integer, Iterable<String>> entry : ff.apply(gen).get())
      System.out.println(entry.first() + "=" + entry.second());
  }

  static class MyMapper implements Mapper<Void, Activity, String, ASObject> {
    public void map(Void key, Activity val, Collector<String, ASObject> context) {
      String ot = val.getActor().getDisplayName();
      context.collect(ot != null ? ot : "", val.getActor());
    }
  }
}
Esempio n. 5
0
  @Test
  public void testMoreFunctions() {
    // Up first... Choice
    Function<String, Integer> choice =
        MoreFunctions.<String, Integer>choice()
            .of(Comparisons.equalsIgnoreCase().predicateFor("A"), 1)
            .of(Comparisons.equalsIgnoreCase().predicateFor("B"), 2)
            .otherwise(3)
            .get();
    assertEquals(Integer.valueOf(1), choice.apply("a"));
    assertEquals(Integer.valueOf(2), choice.apply("b"));
    assertEquals(Integer.valueOf(3), choice.apply("c"));

    // Test createInstance
    Foo foo = MoreFunctions.createInstance(Foo.class).apply(null);
    assertNotNull(foo);

    foo = MoreFunctions.createInstance(Foo.class, String.class).apply(MoreFunctions.array("A"));
    assertNotNull(foo);
    assertEquals("A", foo.a());

    // Test each
    String[] array = MoreFunctions.array("a", "b", "c", "d");
    array =
        MoreFunctions.eachArray(
                new Function<String, String>() {
                  public String apply(String input) {
                    return input.toUpperCase();
                  }
                },
                String.class)
            .apply(array);
    assertEquals("A", array[0]);
    assertEquals("B", array[1]);
    assertEquals("C", array[2]);
    assertEquals("D", array[3]);

    // Test firstNonNull...
    assertEquals("A", MoreFunctions.firstNonNull(null, null, null, "A"));

    // Test futureFunction... executes the function in a separate
    // thread using the passed in ExecutorService
    Future<String> future =
        MoreFunctions.<String, String>futureFunction(
                new Function<String, String>() {
                  public String apply(String input) {
                    System.out.println("Thread sleeping....");
                    try {
                      Thread.sleep(10 * 1000);
                    } catch (Throwable t) {
                    }
                    return input;
                  }
                },
                MoreExecutors2.getExitingExecutor())
            .apply("A");
    try {
      System.out.println("Waiting for return...");
      assertEquals("A", future.get());
    } catch (Throwable t) {
      throw new RuntimeException(t);
    }
  }