예제 #1
0
 public static List<ReviewObjectWrapper> getReviewObjectWrapper(
     Map<String, Review> reviewPerUser, final Facebook facebook) {
   final List<ReviewObjectWrapper> reviewsWrapperList = new ArrayList<ReviewObjectWrapper>();
   for (final String facebookId : reviewPerUser.keySet()) {
     try {
       final String url = facebook.getPictureURL(160, 160).toString();
       final ReviewObjectWrapper reviewObjectWrapper = new ReviewObjectWrapper();
       reviewObjectWrapper.setDpUrl(url);
       reviewObjectWrapper.setReview(reviewPerUser.get(facebookId));
       reviewObjectWrapper.setFacebookId(facebookId);
       reviewsWrapperList.add(reviewObjectWrapper);
     } catch (FacebookException e) {
       // TODO Auto-generated catch block
       e.printStackTrace();
     }
   }
   return reviewsWrapperList;
 }
  public static String getUserTimeline(List<String> ids) {
    Facebook facebook = FacebookUtil.getFacebookInstance();
    int max_posts = 10;
    for (String key : ids) {
      boolean fileSave = false;
      String home = System.getProperty("user.home");
      try {
        // write data into local file
        File file = new File(home + File.separator + "f_" + key.toLowerCase() + ".txt");
        if (file.exists()) {
          log.info(
              "File "
                  + home
                  + File.separator
                  + "f_"
                  + key.toLowerCase()
                  + ".txt"
                  + " already exists");
          file.delete();
          log.info(
              "File " + home + File.separator + "f_" + key.toLowerCase() + ".txt" + " deleted");
        }
        if (!file.exists()) {
          file.createNewFile();
          log.info(
              "File " + home + File.separator + "f_" + key.toLowerCase() + ".txt" + " created");
        }
        BufferedWriter writer = new BufferedWriter(new FileWriter(file));

        User user = facebook.getUser(key, new Reading().fields("email"));
        System.out.println(" user " + user.getUsername());
        ResponseList<Post> feeds = facebook.getPosts(key, new Reading().limit(max_posts));

        // For all 25 feeds...
        for (int i = 0; i < feeds.size(); i++) {
          // Get post.
          Post post = feeds.get(i);
          // Get (string) message.
          String message = post.getMessage();
          // Print out the message.
          System.out.println(message);

          // Get more stuff...
          PagableList<Comment> comments = post.getComments();
          String date = post.getCreatedTime().toString();
          String name = post.getFrom().getName();
          String id = post.getId();
        }

      } catch (FacebookException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } finally {
        if (fileSave) log.info(home + File.separator + key + " .txt saved sucessfully");
        else log.error("Failed to save " + home + File.separator + key + ".txt");
      }
    }
    return "hello";
  }
  public static String searchPosts(List<String> keywords) {
    int max_posts = 10000;
    Facebook facebook = FacebookUtil.getFacebookInstance();
    for (String key : keywords) {
      boolean fileSave = false;
      String home = System.getProperty("user.home");
      try {
        // write data into local file
        File file = new File(home + File.separator + "f_" + key.toLowerCase() + ".txt");
        if (file.exists()) {
          log.info(
              "File "
                  + home
                  + File.separator
                  + "f_"
                  + key.toLowerCase()
                  + ".txt"
                  + " already exists");
          file.delete();
          log.info(
              "File " + home + File.separator + "f_" + key.toLowerCase() + ".txt" + " deleted");
        }
        if (!file.exists()) {
          file.createNewFile();
          log.info(
              "File " + home + File.separator + "f_" + key.toLowerCase() + ".txt" + " created");
        }
        BufferedWriter writer = new BufferedWriter(new FileWriter(file));

        ResponseList<Post> results = facebook.getFeed(key, new Reading().limit(max_posts));
        for (Post post : results) {
          writer.write(post.getMessage() + "\n");
        }
        writer.flush();
        writer.close();
        fileSave = true;
        Thread.sleep(65000);
      } catch (FacebookException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } finally {
        if (fileSave) log.info(home + File.separator + key + " .txt saved sucessfully");
        else log.error("Failed to save " + home + File.separator + key + ".txt");
      }
    }
    FileSystem hdfs = null;
    org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
    String home = System.getProperty("user.home");
    try {
      hdfs = FileSystem.get(new URI(filesPath.HDFS_URL), conf);
      for (String key : keywords) {
        Path localFilePath = new Path(home + File.separator + "f_" + key + ".txt");
        Path modelPath = new Path("/user/dev11" + File.separator + "f_" + key + ".txt");
        if (hdfs.exists(modelPath)) {
          hdfs.delete(new Path("/user/dev11" + File.separator + "f_" + key + ".txt"), true);
        }
        hdfs.copyFromLocalFile(localFilePath, modelPath);
      }
    } catch (IOException e) {
      e.printStackTrace();
    } catch (URISyntaxException e) {
      e.printStackTrace();
    }

    return "";
  }
예제 #4
0
  public static void main(String[] args) throws IOException {

    Properties props = new Properties();

    props.put("metadata.broker.list", "localhost:9092");

    props.put("serializer.class", "kafka.serializer.StringEncoder");

    props.put("partitioner.class", "test.SimplePartitioner");

    /* props.put("message.max.bytes", "" + 1024 * 1024 * 40);*/
    props.put("message.max.bytes", "1037626");

    props.put("request.required.acks", "1");

    props.put("retry.backoff.ms", "150");

    props.put("message.send.max.retries", "10");

    props.put("topic.metadata.refresh.interval.ms", "0");

    ProducerConfig config = new ProducerConfig(props);

    final Producer<String, String> producer = new Producer<String, String>(config);
    String output = null;
    KeyedMessage<String, String> fbdata = new KeyedMessage<String, String>("facebook", output);

    // Generate facebook instance.
    Facebook facebook = new FacebookFactory().getInstance();
    // Use default values for oauth app id.
    facebook.setOAuthAppId("1238270156199618", "177cef157d0c8c006d0067b49b4bde32");

    AccessToken accessTokenString;
    try {
      accessTokenString = facebook.getOAuthAppAccessToken();
      facebook.setOAuthAccessToken(accessTokenString);
      /// BrandBazaarr,rakulpreetsinghs
      // AnushkaShetty//SachinTendulkar//narendramodi
      while (true) {
        // File text = new File("/home/storm/Desktop/test/input.csv");
        // Creating Scanner instnace to read File in Java
        // Scanner scnr = new Scanner(text);
        Scanner scnr = new Scanner(new File("/home/storm/Desktop/test/input.csv"));
        // Reading each line of file using Scanner class
        int lineNumber = 1;
        scnr.useDelimiter(",");
        while (scnr.hasNext()) {
          //  String line = scnr.nextLine();
          String line = scnr.next();
          System.out.println("line " + lineNumber + " :" + line);
          // lineNumber++;
          String fbquery =
              line
                  + "/?fields=posts.limit(1).since(2015).until(now){id,message,name,type,picture,link,caption,description,icon,application,shares,updated_time,source,comments.summary(true){comment_count,message,can_remove,id,created_time,can_like,like_count,comments{comment_count,comments{comment_count}}},place,object_id,privacy,status_type,created_time,story,parent_id,story_tags,full_picture,likes.summary(true){id,name,username}},id,hometown,website,about,location,birthday,name,tagged{message_tags},category,category_list,talking_about_count,likes";
          try {
            RawAPIResponse rawresponse = facebook.callGetAPI(fbquery);
            JSONObject jsonobjmain = rawresponse.asJSONObject();

            output = jsonobjmain.toString();

            String postlike;
            String commentnext;
            JSONObject posts = jsonobjmain.getJSONObject("posts");

            JSONArray postdata = posts.getJSONArray("data");

            JSONObject postpaging = posts.getJSONObject("paging");

            String postnext = postpaging.getString("next");

            int count = 1;
            // int postlikecount=1;
            // int commentscount =1;
            JSONArray commetsarry;
            JSONArray likesdata;

            while (postnext != null) {
              count++;

              URL post_oracle = new URL(postnext);

              URLConnection yc = post_oracle.openConnection();

              BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
              String post_inputLine;

              JSONObject post_obj = new JSONObject();

              while ((post_inputLine = in.readLine()) != null) {
                post_obj = new JSONObject(post_inputLine);

                JSONArray addposts = post_obj.getJSONArray("data");

                for (int i = 0; i < addposts.length(); i++) {
                  JSONObject addspostobj = addposts.getJSONObject(i);

                  postdata.put(addspostobj);
                  output = jsonobjmain.toString();
                  // adding up the postlikes
                  System.out.println("ADDED POSTS CHANGED TO-STRING");

                  // ************************************************************* likes
                  // ****************************************************************
                  JSONObject likes = addspostobj.getJSONObject("likes");

                  likesdata = likes.getJSONArray("data"); // likes of the a post object

                  try {

                    JSONObject paging = likes.getJSONObject("paging");

                    postlike = paging.getString("next");
                    int postlikecount = 1;
                    while (postlike != null) {

                      // System.out.println("*********************************8");
                      postlikecount++;
                      URL oraclepostlike = new URL(postlike);
                      URLConnection oraclepostlikeyc = oraclepostlike.openConnection();
                      BufferedReader oraclepostlikeycin =
                          new BufferedReader(
                              new InputStreamReader(oraclepostlikeyc.getInputStream()));
                      String postlikeinputLine;
                      JSONObject postlikeadd = new JSONObject();
                      while ((postlikeinputLine = oraclepostlikeycin.readLine()) != null) {

                        postlikeadd = new JSONObject(postlikeinputLine);
                        // System.out.println(postlikeadd);
                        JSONArray postaddlikes = postlikeadd.getJSONArray("data");
                        for (int like = 0; like < postaddlikes.length(); like++) {
                          JSONObject addslikobj = postaddlikes.getJSONObject(like);
                          likesdata.put(addslikobj);
                          output = jsonobjmain.toString();
                          System.out.println("ADDED LIKES CHANGED TO STRING");
                          // System.out.println(likesdata);
                        } // for close
                      }

                      try {
                        JSONObject likesnullmake = postlikeadd.getJSONObject("paging");
                        postlike = likesnullmake.getString("next");
                        System.out.println("POST LIKES ENTERED LOOP COUNT" + postlikecount);

                      } catch (Exception e) {
                        postlike = null;
                        System.out.println("there is no next in likes paging");
                      }

                      oraclepostlikeycin.close();
                    }

                    // System.out.println("POST LIKEZS INCREMENT " +postlikecount);

                  } // try  close
                  catch (Exception e) {

                    System.out.println("there is no next in likespg");
                  }

                  // ********************************************************** likes end
                  // ********************************************************************

                  // ********************************************************** comments
                  // *******************************************************************

                  // comments
                  JSONObject comments = addspostobj.getJSONObject("comments");
                  commetsarry = comments.getJSONArray("data");
                  try {

                    JSONObject commentspg = comments.getJSONObject("paging");
                    commentnext = commentspg.getString("next");
                    int commentscount = 1;
                    while (commentnext != null) {

                      System.out.println("*********************************8");
                      commentscount++;
                      URL oraclecomments = new URL(commentnext);
                      URLConnection commentsyc = oraclecomments.openConnection();
                      BufferedReader commentsin =
                          new BufferedReader(new InputStreamReader(commentsyc.getInputStream()));
                      String commentsinputLine;
                      JSONObject commentsobj = new JSONObject();
                      while ((commentsinputLine = commentsin.readLine()) != null) {

                        commentsobj = new JSONObject(commentsinputLine);

                        JSONArray commentsadd = commentsobj.getJSONArray("data");
                        for (int comentsinc = 0; comentsinc < commentsadd.length(); comentsinc++) {

                          JSONObject commentsaddobj = commentsadd.getJSONObject(i);
                          commetsarry.put(commentsaddobj);
                          output = jsonobjmain.toString();
                          System.out.println("ADDED COMMENTS CHANGESD TO STRING");
                        }
                        System.out.println("COMMENTS ENTERED LOOP COUNT" + commentscount);
                      } // comments readline while close

                      try {
                        JSONObject commentssnullmake = commentsobj.getJSONObject("paging");
                        commentnext = commentssnullmake.getString("next");
                      } catch (Exception e) {
                        commentnext = null;
                        System.out.println("there is no comments next");
                      }

                      commentsin.close();
                    }

                  } // comments try close
                  catch (Exception e) {
                    commentnext = null;
                    System.out.println("there is no comments next");
                  } // comments catch close
                } // for loop end

                //  String output=jsonobjmain.toString();

                try {
                  producer.send(fbdata);
                  System.out.println("sent done");
                } catch (Exception e) {
                  PrintStream out =
                      new PrintStream(new FileOutputStream("/home/storm/Videos/erroroutput.txt"));
                  System.setErr(out);
                  System.out.println("system exception returned to a file");
                }

                BufferedWriter writer = null;
                try {
                  writer = new BufferedWriter(new FileWriter("/home/storm/Videos/posts.json"));
                  writer.write(jsonobjmain.toString());
                  System.out.println("Done writing");

                } catch (IOException e) {
                  System.out.println("vanaja");
                } finally {
                  try {
                    if (writer != null) writer.close();
                  } catch (IOException e) {
                    System.out.println("dilip");
                  }
                }
              }
              try {
                JSONObject jo = post_obj.getJSONObject("paging");
                postnext = jo.getString("next");
                // System.out.println("try "+postnext);
              } catch (Exception e) {
                postnext = null;
                System.out.println("there is no post next");
              }

              in.close();
              System.out.println("LOOP COUNT STARTS WITH ONE  " + count);
            }

          } catch (FacebookException e) {

            e.printStackTrace();
            System.out.println("The great  " + e);
          } // raw response try close
        } // read line while close
      } // read file while close

    } catch (Exception e) {

      System.out.println("This error is from first try   " + e);
    } // starting catch-close
  } // main-close