コード例 #1
0
  @Override
  public void prepare(Object configurationObject) {

    mapper = StreamsJacksonMapper.getInstance();

    uriBuilder =
        new URIBuilder()
            .setScheme(this.configuration.getProtocol())
            .setHost(this.configuration.getHostname())
            .setPath(this.configuration.getResourcePath());

    if (!Strings.isNullOrEmpty(configuration.getAccessToken()))
      uriBuilder = uriBuilder.addParameter("access_token", configuration.getAccessToken());
    if (!Strings.isNullOrEmpty(configuration.getUsername())
        && !Strings.isNullOrEmpty(configuration.getPassword())) {
      StringBuilder stringBuilder = new StringBuilder();
      stringBuilder.append(configuration.getUsername());
      stringBuilder.append(":");
      stringBuilder.append(configuration.getPassword());
      String string = stringBuilder.toString();
      authHeader = Base64.encodeBase64String(string.getBytes());
    }
    httpclient = HttpClients.createDefault();
  }
コード例 #2
0
public class FacebookUserstreamProvider implements StreamsProvider, Serializable {

  public static final String STREAMS_ID = "FacebookUserstreamProvider";
  private static final Logger LOGGER = LoggerFactory.getLogger(FacebookUserstreamProvider.class);

  private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  private static final String ALL_PERMISSIONS = "read_stream";
  private FacebookUserstreamConfiguration configuration;

  private Class klass;
  protected final ReadWriteLock lock = new ReentrantReadWriteLock();

  protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<StreamsDatum>();

  public FacebookUserstreamConfiguration getConfig() {
    return configuration;
  }

  public void setConfig(FacebookUserstreamConfiguration config) {
    this.configuration = config;
  }

  protected ListeningExecutorService executor;

  protected DateTime start;
  protected DateTime end;

  protected final AtomicBoolean running = new AtomicBoolean();

  private DatumStatusCounter countersCurrent = new DatumStatusCounter();
  private DatumStatusCounter countersTotal = new DatumStatusCounter();

  protected Facebook client;

  private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
    return new ThreadPoolExecutor(
        nThreads,
        nThreads,
        5000L,
        TimeUnit.MILLISECONDS,
        new ArrayBlockingQueue<Runnable>(queueSize, true),
        new ThreadPoolExecutor.CallerRunsPolicy());
  }

  public FacebookUserstreamProvider() {
    Config config = StreamsConfigurator.config.getConfig("facebook");
    FacebookUserInformationConfiguration facebookUserInformationConfiguration;
    try {
      facebookUserInformationConfiguration =
          mapper.readValue(
              config.root().render(ConfigRenderOptions.concise()),
              FacebookUserInformationConfiguration.class);
    } catch (IOException e) {
      e.printStackTrace();
      return;
    }
  }

  public FacebookUserstreamProvider(FacebookUserstreamConfiguration config) {
    this.configuration = config;
  }

  public FacebookUserstreamProvider(Class klass) {
    Config config = StreamsConfigurator.config.getConfig("facebook");
    FacebookUserInformationConfiguration facebookUserInformationConfiguration;
    try {
      facebookUserInformationConfiguration =
          mapper.readValue(
              config.root().render(ConfigRenderOptions.concise()),
              FacebookUserInformationConfiguration.class);
    } catch (IOException e) {
      e.printStackTrace();
      return;
    }
    this.klass = klass;
  }

  public FacebookUserstreamProvider(FacebookUserstreamConfiguration config, Class klass) {
    this.configuration = config;
    this.klass = klass;
  }

  public Queue<StreamsDatum> getProviderQueue() {
    return this.providerQueue;
  }

  @Override
  public void startStream() {

    client = getFacebookClient();

    if (configuration.getInfo() != null && configuration.getInfo().size() > 0) {
      for (String id : configuration.getInfo()) {
        executor.submit(new FacebookFeedPollingTask(this, id));
      }
      running.set(true);
    } else {
      try {
        String id = client.getMe().getId();
        executor.submit(new FacebookFeedPollingTask(this, id));
        running.set(true);
      } catch (FacebookException e) {
        LOGGER.error(e.getMessage());
        running.set(false);
      }
    }
  }

  public StreamsResultSet readCurrent() {

    StreamsResultSet current;

    synchronized (FacebookUserstreamProvider.class) {
      current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(providerQueue));
      current.setCounter(new DatumStatusCounter());
      current.getCounter().add(countersCurrent);
      countersTotal.add(countersCurrent);
      countersCurrent = new DatumStatusCounter();
      providerQueue.clear();
    }

    return current;
  }

  public StreamsResultSet readNew(BigInteger sequence) {
    LOGGER.debug("{} readNew", STREAMS_ID);
    throw new NotImplementedException();
  }

  public StreamsResultSet readRange(DateTime start, DateTime end) {
    LOGGER.debug("{} readRange", STREAMS_ID);
    this.start = start;
    this.end = end;
    readCurrent();
    StreamsResultSet result = (StreamsResultSet) providerQueue.iterator();
    return result;
  }

  @Override
  public boolean isRunning() {
    return running.get();
  }

  void shutdownAndAwaitTermination(ExecutorService pool) {
    pool.shutdown(); // Disable new tasks from being submitted
    try {
      // Wait a while for existing tasks to terminate
      if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
        pool.shutdownNow(); // Cancel currently executing tasks
        // Wait a while for tasks to respond to being cancelled
        if (!pool.awaitTermination(10, TimeUnit.SECONDS))
          System.err.println("Pool did not terminate");
      }
    } catch (InterruptedException ie) {
      // (Re-)Cancel if current thread also interrupted
      pool.shutdownNow();
      // Preserve interrupt status
      Thread.currentThread().interrupt();
    }
  }

  @Override
  public void prepare(Object o) {

    executor = MoreExecutors.listeningDecorator(newFixedThreadPoolWithQueueSize(5, 20));

    Preconditions.checkNotNull(providerQueue);
    Preconditions.checkNotNull(this.klass);
    Preconditions.checkNotNull(configuration.getOauth().getAppId());
    Preconditions.checkNotNull(configuration.getOauth().getAppSecret());
    Preconditions.checkNotNull(configuration.getOauth().getUserAccessToken());

    client = getFacebookClient();

    if (configuration.getInfo() != null && configuration.getInfo().size() > 0) {

      List<String> ids = new ArrayList<String>();
      List<String[]> idsBatches = new ArrayList<String[]>();

      for (String s : configuration.getInfo()) {
        if (s != null) {
          ids.add(s);

          if (ids.size() >= 100) {
            // add the batch
            idsBatches.add(ids.toArray(new String[ids.size()]));
            // reset the Ids
            ids = new ArrayList<String>();
          }
        }
      }
    }
  }

  protected Facebook getFacebookClient() {
    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true)
        .setOAuthAppId(configuration.getOauth().getAppId())
        .setOAuthAppSecret(configuration.getOauth().getAppSecret())
        .setOAuthAccessToken(configuration.getOauth().getUserAccessToken())
        .setOAuthPermissions(ALL_PERMISSIONS)
        .setJSONStoreEnabled(true);

    FacebookFactory ff = new FacebookFactory(cb.build());
    Facebook facebook = ff.getInstance();

    return facebook;
  }

  @Override
  public void cleanUp() {
    shutdownAndAwaitTermination(executor);
  }

  private class FacebookFeedPollingTask implements Runnable {

    FacebookUserstreamProvider provider;
    Facebook client;
    String id;

    private Set<Post> priorPollResult = Sets.newHashSet();

    public FacebookFeedPollingTask(FacebookUserstreamProvider facebookUserstreamProvider) {
      this.provider = facebookUserstreamProvider;
    }

    public FacebookFeedPollingTask(
        FacebookUserstreamProvider facebookUserstreamProvider, String id) {
      this.provider = facebookUserstreamProvider;
      this.client = provider.client;
      this.id = id;
    }

    @Override
    public void run() {
      while (provider.isRunning()) {
        ResponseList<Post> postResponseList;
        try {
          postResponseList = client.getFeed(id);

          Set<Post> update = Sets.newHashSet(postResponseList);
          Set<Post> repeats = Sets.intersection(priorPollResult, Sets.newHashSet(update));
          Set<Post> entrySet = Sets.difference(update, repeats);
          LOGGER.debug(
              this.id
                  + " response: "
                  + update.size()
                  + " previous: "
                  + repeats.size()
                  + " new: "
                  + entrySet.size());
          for (Post item : entrySet) {
            String json = DataObjectFactory.getRawJSON(item);
            org.apache.streams.facebook.Post post =
                mapper.readValue(json, org.apache.streams.facebook.Post.class);
            try {
              lock.readLock().lock();
              ComponentUtils.offerUntilSuccess(new StreamsDatum(post), providerQueue);
              countersCurrent.incrementAttempt();
            } finally {
              lock.readLock().unlock();
            }
          }
          priorPollResult = update;
        } catch (Exception e) {
          e.printStackTrace();
        } finally {
          try {
            Thread.sleep(configuration.getPollIntervalMillis());
          } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
          }
        }
      }
    }
  }
}
コード例 #3
0
/**
 * Test for
 *
 * @see {@link org.apache.streams.components.http.persist.SimpleHTTPPostPersistWriter}
 */
@RunWith(PowerMockRunner.class)
@PrepareForTest({HttpClients.class, CloseableHttpResponse.class, CloseableHttpResponse.class})
public class SimpleHTTPPostPersistWriterTest {

  private ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  /** test port. */
  private static final int PORT = 18080;

  /** test hosts. */
  private static final String HOSTNAME = "localhost";

  /** test protocol. */
  private static final String PROTOCOL = "http";

  /** HttpClients mock. */
  private HttpClients httpClients;

  /** CloseableHttpClient mock. */
  private CloseableHttpClient client;

  /** CloseableHttpClient mock. */
  private CloseableHttpResponse response = Mockito.mock(CloseableHttpResponse.class);

  /** Our output. */
  private ByteArrayOutputStream output;

  /** Our input. */
  private ByteArrayInputStream input;

  /** Instance under tests. */
  private SimpleHTTPPostPersistWriter writer;

  @Before
  public void setUp() throws Exception {
    this.httpClients = PowerMockito.mock(HttpClients.class);
    this.client = PowerMockito.mock(CloseableHttpClient.class);

    PowerMockito.mockStatic(HttpClients.class);

    PowerMockito.when(HttpClients.createDefault()).thenReturn(client);

    PowerMockito.when(client.execute(any(HttpUriRequest.class))).thenReturn(response);

    Mockito.when(response.getEntity()).thenReturn(null);
    Mockito.doNothing().when(response).close();
  }

  @Test
  public void testPersist() throws Exception {
    HttpPersistWriterConfiguration configuration = new HttpPersistWriterConfiguration();
    configuration.setProtocol(PROTOCOL);
    configuration.setHostname(HOSTNAME);
    configuration.setPort(new Long(PORT));
    configuration.setResourcePath("/");

    this.writer = new SimpleHTTPPostPersistWriter(configuration);

    this.writer.prepare(null);

    StreamsDatum testDatum =
        new StreamsDatum(mapper.readValue("{\"message\":\"ping\"}", ObjectNode.class));

    this.writer.write(testDatum);

    Mockito.verify(this.client).execute(any(HttpUriRequest.class));

    Mockito.verify(this.response).close();
  }
}
コード例 #4
0
/** Test copying documents between two indexes on same cluster */
@ElasticsearchIntegrationTest.ClusterScope(
    scope = ElasticsearchIntegrationTest.Scope.TEST,
    numNodes = 1)
public class MongoElasticsearchSyncIT extends ElasticsearchIntegrationTest {

  private static final Logger LOGGER = LoggerFactory.getLogger(MongoElasticsearchSyncIT.class);

  ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();

  MongoElasticsearchSyncConfiguration syncConfiguration;

  int srcCount = 0;

  @Before
  public void prepareTest() throws Exception {

    syncConfiguration =
        MAPPER.readValue(
            MongoElasticsearchSyncIT.class.getResourceAsStream("/testSync.json"),
            MongoElasticsearchSyncConfiguration.class);

    syncConfiguration.getDestination().setClusterName(cluster().getClusterName());

    MongoPersistWriter setupWriter = new MongoPersistWriter(syncConfiguration.getSource());

    setupWriter.prepare(null);

    InputStream testActivityFolderStream =
        MongoElasticsearchSyncIT.class.getClassLoader().getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, Charsets.UTF_8);

    for (String file : files) {
      LOGGER.info("File: " + file);
      InputStream testActivityFileStream =
          MongoElasticsearchSyncIT.class.getClassLoader().getResourceAsStream("activities/" + file);
      Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
      activity.getAdditionalProperties().remove("$license");
      StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
      setupWriter.write(datum);
      LOGGER.info("Wrote: " + activity.getVerb());
      srcCount++;
    }

    setupWriter.cleanUp();
  }

  @Test
  public void testSync() throws Exception {

    assert srcCount > 0;

    MongoElasticsearchSync sync = new MongoElasticsearchSync(syncConfiguration);

    Thread reindexThread = new Thread(sync);
    reindexThread.start();
    reindexThread.join();

    flushAndRefresh();

    assert (indexExists("destination"));

    long destCount =
        client().count(client().prepareCount("destination").request()).get().getCount();
    assert srcCount == destCount;
  }
}
コード例 #5
0
public class MongoPersistWriter implements StreamsPersistWriter, Runnable {

  public static final String STREAMS_ID = "MongoPersistWriter";

  private static final Logger LOGGER = LoggerFactory.getLogger(MongoPersistWriter.class);

  private static final long MAX_WRITE_LATENCY = 1000;

  protected volatile Queue<StreamsDatum> persistQueue;

  private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
  private volatile AtomicLong lastWrite = new AtomicLong(System.currentTimeMillis());
  private ScheduledExecutorService backgroundFlushTask =
      Executors.newSingleThreadScheduledExecutor();

  private MongoConfiguration config;

  protected DB client;
  protected DBAddress dbaddress;
  protected DBCollection collection;

  protected List<DBObject> insertBatch = Lists.newArrayList();

  protected final ReadWriteLock lock = new ReentrantReadWriteLock();

  public MongoPersistWriter() {
    this(MongoConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("mongo")));
  }

  public MongoPersistWriter(MongoConfiguration config) {
    this.config = config;
  }

  public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
    this.persistQueue = persistQueue;
  }

  public Queue<StreamsDatum> getPersistQueue() {
    return persistQueue;
  }

  @Override
  public String getId() {
    return STREAMS_ID;
  }

  @Override
  public void write(StreamsDatum streamsDatum) {

    DBObject dbObject = prepareObject(streamsDatum);
    if (dbObject != null) {
      addToBatch(dbObject);
      flushIfNecessary();
    }
  }

  public void flush() throws IOException {
    try {
      LOGGER.debug("Attempting to flush {} items to mongo", insertBatch.size());
      lock.writeLock().lock();
      collection.insert(insertBatch);
      lastWrite.set(System.currentTimeMillis());
      insertBatch = Lists.newArrayList();
    } finally {
      lock.writeLock().unlock();
    }
  }

  public synchronized void close() throws IOException {
    client.cleanCursors(true);
    backgroundFlushTask.shutdownNow();
  }

  public void start() {
    connectToMongo();
    backgroundFlushTask.scheduleAtFixedRate(
        new Runnable() {
          @Override
          public void run() {
            flushIfNecessary();
          }
        },
        0,
        MAX_WRITE_LATENCY * 2,
        TimeUnit.MILLISECONDS);
  }

  public void stop() {

    try {
      flush();
    } catch (IOException e) {
      LOGGER.error("Error flushing", e);
    }
    try {
      close();
    } catch (IOException e) {
      LOGGER.error("Error closing", e);
    }
    try {
      backgroundFlushTask.shutdown();
      // Wait a while for existing tasks to terminate
      if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
        backgroundFlushTask.shutdownNow(); // Cancel currently executing tasks
        // Wait a while for tasks to respond to being cancelled
        if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
          LOGGER.error("Stream did not terminate");
        }
      }
    } catch (InterruptedException ie) {
      // (Re-)Cancel if current thread also interrupted
      backgroundFlushTask.shutdownNow();
      // Preserve interrupt status
      Thread.currentThread().interrupt();
    }
  }

  @Override
  public void run() {

    while (true) {
      if (persistQueue.peek() != null) {
        try {
          StreamsDatum entry = persistQueue.remove();
          write(entry);
        } catch (Exception e) {
          e.printStackTrace();
        }
      }
      try {
        Thread.sleep(new Random().nextInt(1));
      } catch (InterruptedException e) {
      }
    }
  }

  @Override
  public void prepare(Object configurationObject) {
    this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
    start();
  }

  @Override
  public void cleanUp() {
    stop();
  }

  protected void flushIfNecessary() {
    long lastLatency = System.currentTimeMillis() - lastWrite.get();
    // Flush iff the size > 0 AND the size is divisible by 100 or the time between now and the last
    // flush is greater
    // than the maximum desired latency
    if (insertBatch.size() > 0
        && (insertBatch.size() % 100 == 0 || lastLatency > MAX_WRITE_LATENCY)) {
      try {
        flush();
      } catch (IOException e) {
        LOGGER.error("Error writing to Mongo", e);
      }
    }
  }

  protected void addToBatch(DBObject dbObject) {
    try {
      lock.readLock().lock();
      insertBatch.add(dbObject);
    } finally {
      lock.readLock().unlock();
    }
  }

  protected DBObject prepareObject(StreamsDatum streamsDatum) {
    DBObject dbObject = null;
    if (streamsDatum.getDocument() instanceof String) {
      dbObject = (DBObject) JSON.parse((String) streamsDatum.getDocument());
    } else {
      try {
        ObjectNode node = mapper.valueToTree(streamsDatum.getDocument());
        dbObject = (DBObject) JSON.parse(node.toString());
      } catch (Exception e) {
        e.printStackTrace();
        LOGGER.error("Unsupported type: " + streamsDatum.getDocument().getClass(), e);
      }
    }
    return dbObject;
  }

  private synchronized void connectToMongo() {

    try {
      client = new MongoClient(config.getHost(), config.getPort().intValue()).getDB(config.getDb());
    } catch (UnknownHostException e) {
      e.printStackTrace();
      return;
    }

    if (!Strings.isNullOrEmpty(config.getUser()) && !Strings.isNullOrEmpty(config.getPassword()))
      client.authenticate(config.getUser(), config.getPassword().toCharArray());

    if (!client.collectionExists(config.getCollection())) {
      client.createCollection(config.getCollection(), null);
    }
    ;

    collection = client.getCollection(config.getCollection());
  }
}