@Override
public void prepare(Object configurationObject) {
mapper = StreamsJacksonMapper.getInstance();
uriBuilder =
new URIBuilder()
.setScheme(this.configuration.getProtocol())
.setHost(this.configuration.getHostname())
.setPath(this.configuration.getResourcePath());
if (!Strings.isNullOrEmpty(configuration.getAccessToken()))
uriBuilder = uriBuilder.addParameter("access_token", configuration.getAccessToken());
if (!Strings.isNullOrEmpty(configuration.getUsername())
&& !Strings.isNullOrEmpty(configuration.getPassword())) {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(configuration.getUsername());
stringBuilder.append(":");
stringBuilder.append(configuration.getPassword());
String string = stringBuilder.toString();
authHeader = Base64.encodeBase64String(string.getBytes());
}
httpclient = HttpClients.createDefault();
}
public class FacebookUserstreamProvider implements StreamsProvider, Serializable {
public static final String STREAMS_ID = "FacebookUserstreamProvider";
private static final Logger LOGGER = LoggerFactory.getLogger(FacebookUserstreamProvider.class);
private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
private static final String ALL_PERMISSIONS = "read_stream";
private FacebookUserstreamConfiguration configuration;
private Class klass;
protected final ReadWriteLock lock = new ReentrantReadWriteLock();
protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<StreamsDatum>();
public FacebookUserstreamConfiguration getConfig() {
return configuration;
}
public void setConfig(FacebookUserstreamConfiguration config) {
this.configuration = config;
}
protected ListeningExecutorService executor;
protected DateTime start;
protected DateTime end;
protected final AtomicBoolean running = new AtomicBoolean();
private DatumStatusCounter countersCurrent = new DatumStatusCounter();
private DatumStatusCounter countersTotal = new DatumStatusCounter();
protected Facebook client;
private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
return new ThreadPoolExecutor(
nThreads,
nThreads,
5000L,
TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(queueSize, true),
new ThreadPoolExecutor.CallerRunsPolicy());
}
public FacebookUserstreamProvider() {
Config config = StreamsConfigurator.config.getConfig("facebook");
FacebookUserInformationConfiguration facebookUserInformationConfiguration;
try {
facebookUserInformationConfiguration =
mapper.readValue(
config.root().render(ConfigRenderOptions.concise()),
FacebookUserInformationConfiguration.class);
} catch (IOException e) {
e.printStackTrace();
return;
}
}
public FacebookUserstreamProvider(FacebookUserstreamConfiguration config) {
this.configuration = config;
}
public FacebookUserstreamProvider(Class klass) {
Config config = StreamsConfigurator.config.getConfig("facebook");
FacebookUserInformationConfiguration facebookUserInformationConfiguration;
try {
facebookUserInformationConfiguration =
mapper.readValue(
config.root().render(ConfigRenderOptions.concise()),
FacebookUserInformationConfiguration.class);
} catch (IOException e) {
e.printStackTrace();
return;
}
this.klass = klass;
}
public FacebookUserstreamProvider(FacebookUserstreamConfiguration config, Class klass) {
this.configuration = config;
this.klass = klass;
}
public Queue<StreamsDatum> getProviderQueue() {
return this.providerQueue;
}
@Override
public void startStream() {
client = getFacebookClient();
if (configuration.getInfo() != null && configuration.getInfo().size() > 0) {
for (String id : configuration.getInfo()) {
executor.submit(new FacebookFeedPollingTask(this, id));
}
running.set(true);
} else {
try {
String id = client.getMe().getId();
executor.submit(new FacebookFeedPollingTask(this, id));
running.set(true);
} catch (FacebookException e) {
LOGGER.error(e.getMessage());
running.set(false);
}
}
}
public StreamsResultSet readCurrent() {
StreamsResultSet current;
synchronized (FacebookUserstreamProvider.class) {
current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(providerQueue));
current.setCounter(new DatumStatusCounter());
current.getCounter().add(countersCurrent);
countersTotal.add(countersCurrent);
countersCurrent = new DatumStatusCounter();
providerQueue.clear();
}
return current;
}
public StreamsResultSet readNew(BigInteger sequence) {
LOGGER.debug("{} readNew", STREAMS_ID);
throw new NotImplementedException();
}
public StreamsResultSet readRange(DateTime start, DateTime end) {
LOGGER.debug("{} readRange", STREAMS_ID);
this.start = start;
this.end = end;
readCurrent();
StreamsResultSet result = (StreamsResultSet) providerQueue.iterator();
return result;
}
@Override
public boolean isRunning() {
return running.get();
}
void shutdownAndAwaitTermination(ExecutorService pool) {
pool.shutdown(); // Disable new tasks from being submitted
try {
// Wait a while for existing tasks to terminate
if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
pool.shutdownNow(); // Cancel currently executing tasks
// Wait a while for tasks to respond to being cancelled
if (!pool.awaitTermination(10, TimeUnit.SECONDS))
System.err.println("Pool did not terminate");
}
} catch (InterruptedException ie) {
// (Re-)Cancel if current thread also interrupted
pool.shutdownNow();
// Preserve interrupt status
Thread.currentThread().interrupt();
}
}
@Override
public void prepare(Object o) {
executor = MoreExecutors.listeningDecorator(newFixedThreadPoolWithQueueSize(5, 20));
Preconditions.checkNotNull(providerQueue);
Preconditions.checkNotNull(this.klass);
Preconditions.checkNotNull(configuration.getOauth().getAppId());
Preconditions.checkNotNull(configuration.getOauth().getAppSecret());
Preconditions.checkNotNull(configuration.getOauth().getUserAccessToken());
client = getFacebookClient();
if (configuration.getInfo() != null && configuration.getInfo().size() > 0) {
List<String> ids = new ArrayList<String>();
List<String[]> idsBatches = new ArrayList<String[]>();
for (String s : configuration.getInfo()) {
if (s != null) {
ids.add(s);
if (ids.size() >= 100) {
// add the batch
idsBatches.add(ids.toArray(new String[ids.size()]));
// reset the Ids
ids = new ArrayList<String>();
}
}
}
}
}
protected Facebook getFacebookClient() {
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true)
.setOAuthAppId(configuration.getOauth().getAppId())
.setOAuthAppSecret(configuration.getOauth().getAppSecret())
.setOAuthAccessToken(configuration.getOauth().getUserAccessToken())
.setOAuthPermissions(ALL_PERMISSIONS)
.setJSONStoreEnabled(true);
FacebookFactory ff = new FacebookFactory(cb.build());
Facebook facebook = ff.getInstance();
return facebook;
}
@Override
public void cleanUp() {
shutdownAndAwaitTermination(executor);
}
private class FacebookFeedPollingTask implements Runnable {
FacebookUserstreamProvider provider;
Facebook client;
String id;
private Set<Post> priorPollResult = Sets.newHashSet();
public FacebookFeedPollingTask(FacebookUserstreamProvider facebookUserstreamProvider) {
this.provider = facebookUserstreamProvider;
}
public FacebookFeedPollingTask(
FacebookUserstreamProvider facebookUserstreamProvider, String id) {
this.provider = facebookUserstreamProvider;
this.client = provider.client;
this.id = id;
}
@Override
public void run() {
while (provider.isRunning()) {
ResponseList<Post> postResponseList;
try {
postResponseList = client.getFeed(id);
Set<Post> update = Sets.newHashSet(postResponseList);
Set<Post> repeats = Sets.intersection(priorPollResult, Sets.newHashSet(update));
Set<Post> entrySet = Sets.difference(update, repeats);
LOGGER.debug(
this.id
+ " response: "
+ update.size()
+ " previous: "
+ repeats.size()
+ " new: "
+ entrySet.size());
for (Post item : entrySet) {
String json = DataObjectFactory.getRawJSON(item);
org.apache.streams.facebook.Post post =
mapper.readValue(json, org.apache.streams.facebook.Post.class);
try {
lock.readLock().lock();
ComponentUtils.offerUntilSuccess(new StreamsDatum(post), providerQueue);
countersCurrent.incrementAttempt();
} finally {
lock.readLock().unlock();
}
}
priorPollResult = update;
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
Thread.sleep(configuration.getPollIntervalMillis());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
}
}
/**
* Test for
*
* @see {@link org.apache.streams.components.http.persist.SimpleHTTPPostPersistWriter}
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest({HttpClients.class, CloseableHttpResponse.class, CloseableHttpResponse.class})
public class SimpleHTTPPostPersistWriterTest {
private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
/** test port. */
private static final int PORT = 18080;
/** test hosts. */
private static final String HOSTNAME = "localhost";
/** test protocol. */
private static final String PROTOCOL = "http";
/** HttpClients mock. */
private HttpClients httpClients;
/** CloseableHttpClient mock. */
private CloseableHttpClient client;
/** CloseableHttpClient mock. */
private CloseableHttpResponse response = Mockito.mock(CloseableHttpResponse.class);
/** Our output. */
private ByteArrayOutputStream output;
/** Our input. */
private ByteArrayInputStream input;
/** Instance under tests. */
private SimpleHTTPPostPersistWriter writer;
@Before
public void setUp() throws Exception {
this.httpClients = PowerMockito.mock(HttpClients.class);
this.client = PowerMockito.mock(CloseableHttpClient.class);
PowerMockito.mockStatic(HttpClients.class);
PowerMockito.when(HttpClients.createDefault()).thenReturn(client);
PowerMockito.when(client.execute(any(HttpUriRequest.class))).thenReturn(response);
Mockito.when(response.getEntity()).thenReturn(null);
Mockito.doNothing().when(response).close();
}
@Test
public void testPersist() throws Exception {
HttpPersistWriterConfiguration configuration = new HttpPersistWriterConfiguration();
configuration.setProtocol(PROTOCOL);
configuration.setHostname(HOSTNAME);
configuration.setPort(new Long(PORT));
configuration.setResourcePath("/");
this.writer = new SimpleHTTPPostPersistWriter(configuration);
this.writer.prepare(null);
StreamsDatum testDatum =
new StreamsDatum(mapper.readValue("{\"message\":\"ping\"}", ObjectNode.class));
this.writer.write(testDatum);
Mockito.verify(this.client).execute(any(HttpUriRequest.class));
Mockito.verify(this.response).close();
}
}
/** Test copying documents between two indexes on same cluster */
@ElasticsearchIntegrationTest.ClusterScope(
scope = ElasticsearchIntegrationTest.Scope.TEST,
numNodes = 1)
public class MongoElasticsearchSyncIT extends ElasticsearchIntegrationTest {
private static final Logger LOGGER = LoggerFactory.getLogger(MongoElasticsearchSyncIT.class);
ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
MongoElasticsearchSyncConfiguration syncConfiguration;
int srcCount = 0;
@Before
public void prepareTest() throws Exception {
syncConfiguration =
MAPPER.readValue(
MongoElasticsearchSyncIT.class.getResourceAsStream("/testSync.json"),
MongoElasticsearchSyncConfiguration.class);
syncConfiguration.getDestination().setClusterName(cluster().getClusterName());
MongoPersistWriter setupWriter = new MongoPersistWriter(syncConfiguration.getSource());
setupWriter.prepare(null);
InputStream testActivityFolderStream =
MongoElasticsearchSyncIT.class.getClassLoader().getResourceAsStream("activities");
List<String> files = IOUtils.readLines(testActivityFolderStream, Charsets.UTF_8);
for (String file : files) {
LOGGER.info("File: " + file);
InputStream testActivityFileStream =
MongoElasticsearchSyncIT.class.getClassLoader().getResourceAsStream("activities/" + file);
Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
activity.getAdditionalProperties().remove("$license");
StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
setupWriter.write(datum);
LOGGER.info("Wrote: " + activity.getVerb());
srcCount++;
}
setupWriter.cleanUp();
}
@Test
public void testSync() throws Exception {
assert srcCount > 0;
MongoElasticsearchSync sync = new MongoElasticsearchSync(syncConfiguration);
Thread reindexThread = new Thread(sync);
reindexThread.start();
reindexThread.join();
flushAndRefresh();
assert (indexExists("destination"));
long destCount =
client().count(client().prepareCount("destination").request()).get().getCount();
assert srcCount == destCount;
}
}
public class MongoPersistWriter implements StreamsPersistWriter, Runnable {
public static final String STREAMS_ID = "MongoPersistWriter";
private static final Logger LOGGER = LoggerFactory.getLogger(MongoPersistWriter.class);
private static final long MAX_WRITE_LATENCY = 1000;
protected volatile Queue<StreamsDatum> persistQueue;
private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
private volatile AtomicLong lastWrite = new AtomicLong(System.currentTimeMillis());
private ScheduledExecutorService backgroundFlushTask =
Executors.newSingleThreadScheduledExecutor();
private MongoConfiguration config;
protected DB client;
protected DBAddress dbaddress;
protected DBCollection collection;
protected List<DBObject> insertBatch = Lists.newArrayList();
protected final ReadWriteLock lock = new ReentrantReadWriteLock();
public MongoPersistWriter() {
this(MongoConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("mongo")));
}
public MongoPersistWriter(MongoConfiguration config) {
this.config = config;
}
public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
this.persistQueue = persistQueue;
}
public Queue<StreamsDatum> getPersistQueue() {
return persistQueue;
}
@Override
public String getId() {
return STREAMS_ID;
}
@Override
public void write(StreamsDatum streamsDatum) {
DBObject dbObject = prepareObject(streamsDatum);
if (dbObject != null) {
addToBatch(dbObject);
flushIfNecessary();
}
}
public void flush() throws IOException {
try {
LOGGER.debug("Attempting to flush {} items to mongo", insertBatch.size());
lock.writeLock().lock();
collection.insert(insertBatch);
lastWrite.set(System.currentTimeMillis());
insertBatch = Lists.newArrayList();
} finally {
lock.writeLock().unlock();
}
}
public synchronized void close() throws IOException {
client.cleanCursors(true);
backgroundFlushTask.shutdownNow();
}
public void start() {
connectToMongo();
backgroundFlushTask.scheduleAtFixedRate(
new Runnable() {
@Override
public void run() {
flushIfNecessary();
}
},
0,
MAX_WRITE_LATENCY * 2,
TimeUnit.MILLISECONDS);
}
public void stop() {
try {
flush();
} catch (IOException e) {
LOGGER.error("Error flushing", e);
}
try {
close();
} catch (IOException e) {
LOGGER.error("Error closing", e);
}
try {
backgroundFlushTask.shutdown();
// Wait a while for existing tasks to terminate
if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
backgroundFlushTask.shutdownNow(); // Cancel currently executing tasks
// Wait a while for tasks to respond to being cancelled
if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
LOGGER.error("Stream did not terminate");
}
}
} catch (InterruptedException ie) {
// (Re-)Cancel if current thread also interrupted
backgroundFlushTask.shutdownNow();
// Preserve interrupt status
Thread.currentThread().interrupt();
}
}
@Override
public void run() {
while (true) {
if (persistQueue.peek() != null) {
try {
StreamsDatum entry = persistQueue.remove();
write(entry);
} catch (Exception e) {
e.printStackTrace();
}
}
try {
Thread.sleep(new Random().nextInt(1));
} catch (InterruptedException e) {
}
}
}
@Override
public void prepare(Object configurationObject) {
this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
start();
}
@Override
public void cleanUp() {
stop();
}
protected void flushIfNecessary() {
long lastLatency = System.currentTimeMillis() - lastWrite.get();
// Flush iff the size > 0 AND the size is divisible by 100 or the time between now and the last
// flush is greater
// than the maximum desired latency
if (insertBatch.size() > 0
&& (insertBatch.size() % 100 == 0 || lastLatency > MAX_WRITE_LATENCY)) {
try {
flush();
} catch (IOException e) {
LOGGER.error("Error writing to Mongo", e);
}
}
}
protected void addToBatch(DBObject dbObject) {
try {
lock.readLock().lock();
insertBatch.add(dbObject);
} finally {
lock.readLock().unlock();
}
}
protected DBObject prepareObject(StreamsDatum streamsDatum) {
DBObject dbObject = null;
if (streamsDatum.getDocument() instanceof String) {
dbObject = (DBObject) JSON.parse((String) streamsDatum.getDocument());
} else {
try {
ObjectNode node = mapper.valueToTree(streamsDatum.getDocument());
dbObject = (DBObject) JSON.parse(node.toString());
} catch (Exception e) {
e.printStackTrace();
LOGGER.error("Unsupported type: " + streamsDatum.getDocument().getClass(), e);
}
}
return dbObject;
}
private synchronized void connectToMongo() {
try {
client = new MongoClient(config.getHost(), config.getPort().intValue()).getDB(config.getDb());
} catch (UnknownHostException e) {
e.printStackTrace();
return;
}
if (!Strings.isNullOrEmpty(config.getUser()) && !Strings.isNullOrEmpty(config.getPassword()))
client.authenticate(config.getUser(), config.getPassword().toCharArray());
if (!client.collectionExists(config.getCollection())) {
client.createCollection(config.getCollection(), null);
}
;
collection = client.getCollection(config.getCollection());
}
}
