/** Create worker-specific fileinstall.cfg file for the master. */
    public void confCreate(final String worker) {
      ensureConf();
      final Conf conf = conf();

      if (!ResourceUtil.hasProject(worker)) {
        Plugin.logWarn("Project#confCreate: missing project: " + worker);
        return;
      }

      final Map<String, String> variables = //
          Conf.variables(ResourceUtil.workspacePath(), worker, ResourceUtil.projectPath(worker));

      final String path =
          ConfUtil.replace( //
              conf.fileinstallPath(), variables);

      final String tempalte =
          ConfUtil.replace( //
              conf.fileinstallTemplate(), variables);

      final File file = ProjectUtil.file(project(), path);

      try {
        FileUtil.writeTextFile(file, tempalte);
        Plugin.logOK("Project#confCreate file: " + file);
      } catch (final Throwable e) {
        Plugin.logErrr("Project#confCreate failure", e);
      }
    }
 /** Delete all fileinstall.cfg file for the master. */
 public void confDelete() {
   ensureConf();
   final Conf conf = conf();
   final List<String> list = conf.eclipseList();
   for (final String worker : list) {
     confDelete(worker);
   }
 }
 @Test
 public void testMovieFileIsProperlyCreated() throws Exception {
   Conf conf = new Conf("/test-conf.properties");
   String movieName = "Luther.S02E01.720p.HDTV.x264-3.mp3";
   MoviePartMetadata moviePartMetadata =
       new MoviePartMetadata(conf.getCachosDir(), movieName, 0, 1024);
   File actualMovieFile = moviePartMetadata.getCacho().getMovieFile();
   File expectedMovieFile = new File(conf.getCachosDir(), movieName + "-0-1024.part");
   Assert.assertEquals(expectedMovieFile, actualMovieFile);
 }
 /** Create all live workers for a master. */
 public void confCreate() {
   ensureConf();
   final Conf conf = conf();
   final List<String> list = conf.eclipseList();
   for (final String worker : list) {
     if (ResourceUtil.hasProjectOpen(worker)) {
       confCreate(worker);
     }
   }
 }
  public double getPerformanceMetric() {
    int numberOfQueries = ThroughputTest.getNumberOfQueries();
    int S = conf.getNumberOfStreams();

    int Q = 3 * S * numberOfQueries;
    long Ttt = getThroughputTest1Time() + getThroughputTest2Time();
    double Tld = 0.01 * S * getLoadTestTime();
    long Tpt = getPowerTestTime() * S;
    double performanceMetric = conf.getScale() * Q * 1.0 / ((Tpt + Ttt + Tld) / (3600 * 1000));

    return performanceMetric;
  }
  public void testUrlEncode() {
    Conf conf = new Conf();
    OutboundRule rule1 = new OutboundRule();
    rule1.setFrom("/aaa");
    rule1.setTo("/bbb");
    conf.addOutboundRule(rule1);
    conf.initialise();
    UrlRewriter urlRewriter = new UrlRewriter(conf);

    MockRequest request = new MockRequest("doesn't matter");
    UrlRewriteWrappedResponse urlRewriteWrappedResponse =
        new UrlRewriteWrappedResponse(response, request, urlRewriter);

    assertEquals("/bbb;mockencoded=test", urlRewriteWrappedResponse.encodeURL("/aaa"));
  }
  private long[][] getQueryRunStatistics(TestPhase testPhase) {
    int numberOfQueries = ThroughputTest.getNumberOfQueries();
    int numberOfStreams = conf.getNumberOfStreams();
    int[] sortedQueryIds = ThroughputTest.getSortedQueryIds();

    // for each query, an array of long type consists of min, median, max time
    long[][] queryRunStatistics = new long[numberOfQueries][3];

    for (int i = 0; i < numberOfQueries; i++) {
      long[] queryTime = new long[numberOfStreams];
      for (int j = 0; j < numberOfStreams; j++) {
        queryTime[j] = getQueryTime(testPhase, j, sortedQueryIds[i]);
      }
      Arrays.sort(queryTime);
      // min query time
      queryRunStatistics[i][0] = queryTime[0];
      // max query time
      queryRunStatistics[i][2] = queryTime[numberOfStreams - 1];
      // median query time
      // note that numberOfStreams is an even number
      queryRunStatistics[i][1] =
          (queryTime[numberOfStreams / 2] + queryTime[numberOfStreams / 2 - 1]) / 2;
    }

    return queryRunStatistics;
  }
Example #8
0
  public static void main(String[] args) {
    try {
      Runtime.getRuntime()
          .addShutdownHook(
              new Thread() {
                @Override
                public void run() {
                  Cleanup();
                }
              });

      Conf.ParseArgs(args);
      StdoutWriter.Start();
      DB.Init();

      // It seems that this needs to go through the end of the main function.
      // strange thread model.
      if (Conf.stream_seed_users) StreamSeedUsers.Start();

      CrawlTweets.Run();
    } catch (Exception e) {
      e.printStackTrace();
      System.out.println("Exception caught: " + e);
      System.exit(1);
    }
  }
Example #9
0
 public static void main(String[] args) throws InterruptedException, ExecutionException {
   GPathBSP.runBsp(
           "BiconnectBranches",
           Conf.getDefaultConfiguration("local-mapreduce"),
           BiconnectBranchesBSP.class)
       .get();
 }
Example #10
0
 public static String print(Object o) {
   try {
     return print(new StringBuffer(), o, Conf.getDefaults()).toString();
   } catch (Throwable t) {
     return PrettyPrinter.class + " " + t.getClass().getName() + ":" + t.getMessage() + "\n" + o;
   }
 }
  private long getRefreshRunTime(TestPhase testPhase) {
    long time = 0;

    for (int i = 1; i <= conf.getNumberOfStreams() / 2; i++) {
      time += getRefreshDataSetTime(testPhase, i);
    }

    return time;
  }
    /** Delete worker-specific fileinstall.cfg file for the master. */
    public void confDelete(final String worker) {

      if (!ResourceUtil.hasProject(worker)) {
        Plugin.logWarn("Project#confDelete: missing project: " + worker);
        return;
      }

      ensureConf();
      final Conf conf = conf();

      final Map<String, String> variables = //
          Conf.variables(ResourceUtil.workspacePath(), worker, ResourceUtil.projectPath(worker));

      final String path =
          ConfUtil.replace( //
              conf.fileinstallPath(), variables);

      final File file = ProjectUtil.file(project(), path);

      file.delete();

      Plugin.logOK("Project#confDelete file: " + file);
    }
Example #13
0
 private ConfFactory() {
   globalConf = new Conf();
   globalConf.loadResource(new File(CONF_HOME + "/conf_cms.xml"));
   // globalConf.loadResource(new File(CONF_HOME + "/conf-spider.xml"));
   // globalConf.loadResource(new File(CONF_HOME + "/conf-site.xml"));
 }
  @SuppressWarnings("deprecation")
  @Override
  public Boolean crawl(RssEntityBean site) throws CrawlerException {
    log.info("start YouTube crawl channel id = " + site.url);
    Boolean updated = false;
    YouTube youtube =
        new YouTube.Builder(
                HTTP_TRANSPORT,
                JSON_FACTORY,
                new HttpRequestInitializer() {
                  public void initialize(HttpRequest request) throws IOException {}
                })
            .setApplicationName("youtube-cmdline-search-sample")
            .build();

    try {
      YouTube.Search.List search = youtube.search().list("id,snippet");
      String apiKey = Conf.getValue("youtube.appkey");
      search.setKey(apiKey);
      search.setType("video");
      search.setOrder("date");
      search.setMaxResults(maxResult);
      search.setChannelId(site.url);

      SearchListResponse searchResponse = search.execute();
      List<SearchResult> searchResultList = searchResponse.getItems();
      log.info("search response video count = " + searchResultList.size());
      for (SearchResult result : searchResultList) {

        SearchResultSnippet snippet = result.getSnippet();
        DateTime publishDate = snippet.getPublishedAt();
        DateTime lastCrawlDate = null;
        if (site.lastCrawl != null) {
          lastCrawlDate = new DateTime(site.lastCrawl);
        }
        if ((lastCrawlDate == null) || (publishDate.getValue() > lastCrawlDate.getValue())) {
          YouTube.Videos.List videolist = youtube.videos().list("id,snippet");
          videolist.setId(result.getId().getVideoId());
          videolist.setKey(apiKey);
          VideoListResponse videoListResponse = videolist.execute();
          List<Video> videos = videoListResponse.getItems();
          if (videos.size() != 1) {
            log.warn("video not found : " + videolist.getId());
            continue;
          }
          Video video = videos.get(0);

          ArticleEntityBean article = new ArticleEntityBean();
          article.link = video.getId();
          article.title = snippet.getTitle();
          article.description = snippet.getDescription();
          article.imageUrl = snippet.getThumbnails().getDefault().getUrl();
          article.url = site.url;
          article.createdAt = snippet.getPublishedAt();
          String fixHour = Conf.getValue("fix_time");
          Date now = new Date();
          Date fixDate =
              new Date(now.getYear(), now.getMonth(), now.getDate(), Integer.parseInt(fixHour), 0);
          if (now.getTime() > fixDate.getTime()) {
            article.publishedAt = tomorrow;
          } else {
            article.publishedAt = today;
          }
          article.site = site.site;
          article.type = site.type;
          article.tags = site.defaultTag;
          log.info("save article video id = " + article.link + " / video title = " + article.title);
          article.save();
          updated = true;
        }
      }
    } catch (IOException e) {
      log.warn("youtube api error : " + e.getMessage());
      throw new CrawlerException(e);
    }
    log.info("end YouTube crawl");
    return updated;
  }
// Class for Metrics collection.
public class Metrics {
  private static final Logger logger = Logger.getLogger(Metrics.class.getName());

  private HashMap<String, Date> dataMetricsMap = new HashMap<String, Date>();
  private static Metrics metrics = new Metrics();
  private Conf conf = Conf.getConf();

  private Metrics() {}

  public static Metrics getMetrics() {
    return metrics;
  }

  private String getPhaseMetricName(TestPhase testPhase, boolean isStartTime) {
    String metricName = testPhase.toString() + "_";
    if (isStartTime) {
      metricName += "start";
    } else {
      metricName += "end";
    }

    return metricName;
  }

  public void addPhaseMetric(TestPhase testPhase, boolean isStartTime, Date timestamp) {
    dataMetricsMap.put(getPhaseMetricName(testPhase, isStartTime), timestamp);
  }

  private String getQueryMetricName(
      TestPhase testPhase, int streamId, int queryId, boolean isStartTime) {
    String metricName = testPhase.toString() + "_query_" + streamId + "_" + queryId + "_";
    if (isStartTime) {
      metricName += "start";
    } else {
      metricName += "end";
    }

    return metricName;
  }

  // synchronized method for parallel query running
  public synchronized void addQueryMetric(
      TestPhase testPhase, int streamId, int queryId, boolean isStartTime, Date timestamp) {
    dataMetricsMap.put(getQueryMetricName(testPhase, streamId, queryId, isStartTime), timestamp);
  }

  private String getRefreshMetricName(
      TestPhase testPhase, int refreshDataSetIndex, boolean isStartTime) {
    String metricName = testPhase.toString() + "_refresh_" + refreshDataSetIndex + "_";
    if (isStartTime) {
      metricName += "start";
    } else {
      metricName += "end";
    }

    return metricName;
  }

  // Refresh Run is executed alone without running query together
  public void addRefreshMetric(
      TestPhase testPhase, int refreshDataSetIndex, boolean isStartTime, Date timestamp) {
    dataMetricsMap.put(
        getRefreshMetricName(testPhase, refreshDataSetIndex, isStartTime), timestamp);
  }

  private String getDMMetricName(
      TestPhase testPhase, int refreshDataSetIndex, int dmId, boolean isStartTime) {
    String metricName = testPhase.toString() + "_dm_" + refreshDataSetIndex + "_" + dmId + "_";
    if (isStartTime) {
      metricName += "start";
    } else {
      metricName += "end";
    }

    return metricName;
  }

  public void addDMMetric(
      TestPhase testPhase, int refreshDataSetIndex, int dmId, boolean isStartTime, Date timestamp) {
    dataMetricsMap.put(
        getDMMetricName(testPhase, refreshDataSetIndex, dmId, isStartTime), timestamp);
  }

  private long getPhaseTime(TestPhase testPhase) {
    long startTime = dataMetricsMap.get(getPhaseMetricName(testPhase, true)).getTime();
    long endTime = dataMetricsMap.get(getPhaseMetricName(testPhase, false)).getTime();
    return endTime - startTime;
  }

  public long getLoadTestTime() {
    return getPhaseTime(TestPhase.LOAD_TEST);
  }

  public long getPowerTestTime() {
    return getPhaseTime(TestPhase.POWER_TEST);
  }

  public long getThroughputTest1Time() {
    return getPhaseTime(TestPhase.THROUGHPUT_TEST_1);
  }

  public long getThroughputTest2Time() {
    return getPhaseTime(TestPhase.THROUGHPUT_TEST_2);
  }

  public long getQueryTime(TestPhase testPhase, int streamId, int queryId) {
    long startTime =
        dataMetricsMap.get(getQueryMetricName(testPhase, streamId, queryId, true)).getTime();
    long endTime =
        dataMetricsMap.get(getQueryMetricName(testPhase, streamId, queryId, false)).getTime();
    return endTime - startTime;
  }

  public long getPowerTestQueryTime(int queryId) {
    return getQueryTime(TestPhase.POWER_TEST, 0, queryId);
  }

  private long getRefreshDataSetTime(TestPhase testPhase, int refreshDataSetIndex) {
    long startTime =
        dataMetricsMap.get(getRefreshMetricName(testPhase, refreshDataSetIndex, true)).getTime();
    long endTime =
        dataMetricsMap.get(getRefreshMetricName(testPhase, refreshDataSetIndex, false)).getTime();
    return endTime - startTime;
  }

  private long getRefreshRunTime(TestPhase testPhase) {
    long time = 0;

    for (int i = 1; i <= conf.getNumberOfStreams() / 2; i++) {
      time += getRefreshDataSetTime(testPhase, i);
    }

    return time;
  }

  public long getRefreshRun1Time() {
    return getRefreshRunTime(TestPhase.THROUGHPUT_TEST_1);
  }

  public long getRefreshRun2Time() {
    return getRefreshRunTime(TestPhase.THROUGHPUT_TEST_2);
  }

  public long getDMTime(TestPhase testPhase, int refreshDataSetIndex, int dmId) {
    long startTime =
        dataMetricsMap.get(getDMMetricName(testPhase, refreshDataSetIndex, dmId, true)).getTime();
    long endTime =
        dataMetricsMap.get(getDMMetricName(testPhase, refreshDataSetIndex, dmId, false)).getTime();
    return endTime - startTime;
  }

  public long getQueryRun1Time() {
    return getThroughputTest1Time() - getRefreshRun1Time();
  }

  public long getQueryRun2Time() {
    return getThroughputTest2Time() - getRefreshRun2Time();
  }

  private long[][] getQueryRunStatistics(TestPhase testPhase) {
    int numberOfQueries = ThroughputTest.getNumberOfQueries();
    int numberOfStreams = conf.getNumberOfStreams();
    int[] sortedQueryIds = ThroughputTest.getSortedQueryIds();

    // for each query, an array of long type consists of min, median, max time
    long[][] queryRunStatistics = new long[numberOfQueries][3];

    for (int i = 0; i < numberOfQueries; i++) {
      long[] queryTime = new long[numberOfStreams];
      for (int j = 0; j < numberOfStreams; j++) {
        queryTime[j] = getQueryTime(testPhase, j, sortedQueryIds[i]);
      }
      Arrays.sort(queryTime);
      // min query time
      queryRunStatistics[i][0] = queryTime[0];
      // max query time
      queryRunStatistics[i][2] = queryTime[numberOfStreams - 1];
      // median query time
      // note that numberOfStreams is an even number
      queryRunStatistics[i][1] =
          (queryTime[numberOfStreams / 2] + queryTime[numberOfStreams / 2 - 1]) / 2;
    }

    return queryRunStatistics;
  }

  public long[][] getQueryRun1Statistics() {
    return getQueryRunStatistics(TestPhase.THROUGHPUT_TEST_1);
  }

  public long[][] getQueryRun2Statistics() {
    return getQueryRunStatistics(TestPhase.THROUGHPUT_TEST_2);
  }

  public double getPerformanceMetric() {
    int numberOfQueries = ThroughputTest.getNumberOfQueries();
    int S = conf.getNumberOfStreams();

    int Q = 3 * S * numberOfQueries;
    long Ttt = getThroughputTest1Time() + getThroughputTest2Time();
    double Tld = 0.01 * S * getLoadTestTime();
    long Tpt = getPowerTestTime() * S;
    double performanceMetric = conf.getScale() * Q * 1.0 / ((Tpt + Ttt + Tld) / (3600 * 1000));

    return performanceMetric;
  }
}
 @Before
 public void reset() {
   Env.reset();
   Conf.reset();
 }
 public static void init(Conf conf) {
   zkClient = openClient(conf.getZookeeperServer());
   ZkPathUtils.init(conf.getZookeeperRootPath(), zkClient);
 }