@Override
public void run() {
while (provider.isRunning()) {
ResponseList<Post> postResponseList;
try {
postResponseList = client.getFeed(id);
Set<Post> update = Sets.newHashSet(postResponseList);
Set<Post> repeats = Sets.intersection(priorPollResult, Sets.newHashSet(update));
Set<Post> entrySet = Sets.difference(update, repeats);
LOGGER.debug(
this.id
+ " response: "
+ update.size()
+ " previous: "
+ repeats.size()
+ " new: "
+ entrySet.size());
for (Post item : entrySet) {
String json = DataObjectFactory.getRawJSON(item);
org.apache.streams.facebook.Post post =
mapper.readValue(json, org.apache.streams.facebook.Post.class);
try {
lock.readLock().lock();
ComponentUtils.offerUntilSuccess(new StreamsDatum(post), providerQueue);
countersCurrent.incrementAttempt();
} finally {
lock.readLock().unlock();
}
}
priorPollResult = update;
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
Thread.sleep(configuration.getPollIntervalMillis());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
public static String searchPosts(List<String> keywords) {
int max_posts = 10000;
Facebook facebook = FacebookUtil.getFacebookInstance();
for (String key : keywords) {
boolean fileSave = false;
String home = System.getProperty("user.home");
try {
// write data into local file
File file = new File(home + File.separator + "f_" + key.toLowerCase() + ".txt");
if (file.exists()) {
log.info(
"File "
+ home
+ File.separator
+ "f_"
+ key.toLowerCase()
+ ".txt"
+ " already exists");
file.delete();
log.info(
"File " + home + File.separator + "f_" + key.toLowerCase() + ".txt" + " deleted");
}
if (!file.exists()) {
file.createNewFile();
log.info(
"File " + home + File.separator + "f_" + key.toLowerCase() + ".txt" + " created");
}
BufferedWriter writer = new BufferedWriter(new FileWriter(file));
ResponseList<Post> results = facebook.getFeed(key, new Reading().limit(max_posts));
for (Post post : results) {
writer.write(post.getMessage() + "\n");
}
writer.flush();
writer.close();
fileSave = true;
Thread.sleep(65000);
} catch (FacebookException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
if (fileSave) log.info(home + File.separator + key + " .txt saved sucessfully");
else log.error("Failed to save " + home + File.separator + key + ".txt");
}
}
FileSystem hdfs = null;
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
String home = System.getProperty("user.home");
try {
hdfs = FileSystem.get(new URI(filesPath.HDFS_URL), conf);
for (String key : keywords) {
Path localFilePath = new Path(home + File.separator + "f_" + key + ".txt");
Path modelPath = new Path("/user/dev11" + File.separator + "f_" + key + ".txt");
if (hdfs.exists(modelPath)) {
hdfs.delete(new Path("/user/dev11" + File.separator + "f_" + key + ".txt"), true);
}
hdfs.copyFromLocalFile(localFilePath, modelPath);
}
} catch (IOException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
return "";
}