/** Create worker-specific fileinstall.cfg file for the master. */
public void confCreate(final String worker) {
ensureConf();
final Conf conf = conf();
if (!ResourceUtil.hasProject(worker)) {
Plugin.logWarn("Project#confCreate: missing project: " + worker);
return;
}
final Map<String, String> variables = //
Conf.variables(ResourceUtil.workspacePath(), worker, ResourceUtil.projectPath(worker));
final String path =
ConfUtil.replace( //
conf.fileinstallPath(), variables);
final String tempalte =
ConfUtil.replace( //
conf.fileinstallTemplate(), variables);
final File file = ProjectUtil.file(project(), path);
try {
FileUtil.writeTextFile(file, tempalte);
Plugin.logOK("Project#confCreate file: " + file);
} catch (final Throwable e) {
Plugin.logErrr("Project#confCreate failure", e);
}
}
/** Delete all fileinstall.cfg file for the master. */
public void confDelete() {
ensureConf();
final Conf conf = conf();
final List<String> list = conf.eclipseList();
for (final String worker : list) {
confDelete(worker);
}
}
@Test
public void testMovieFileIsProperlyCreated() throws Exception {
Conf conf = new Conf("/test-conf.properties");
String movieName = "Luther.S02E01.720p.HDTV.x264-3.mp3";
MoviePartMetadata moviePartMetadata =
new MoviePartMetadata(conf.getCachosDir(), movieName, 0, 1024);
File actualMovieFile = moviePartMetadata.getCacho().getMovieFile();
File expectedMovieFile = new File(conf.getCachosDir(), movieName + "-0-1024.part");
Assert.assertEquals(expectedMovieFile, actualMovieFile);
}
/** Create all live workers for a master. */
public void confCreate() {
ensureConf();
final Conf conf = conf();
final List<String> list = conf.eclipseList();
for (final String worker : list) {
if (ResourceUtil.hasProjectOpen(worker)) {
confCreate(worker);
}
}
}
public double getPerformanceMetric() {
int numberOfQueries = ThroughputTest.getNumberOfQueries();
int S = conf.getNumberOfStreams();
int Q = 3 * S * numberOfQueries;
long Ttt = getThroughputTest1Time() + getThroughputTest2Time();
double Tld = 0.01 * S * getLoadTestTime();
long Tpt = getPowerTestTime() * S;
double performanceMetric = conf.getScale() * Q * 1.0 / ((Tpt + Ttt + Tld) / (3600 * 1000));
return performanceMetric;
}
public void testUrlEncode() {
Conf conf = new Conf();
OutboundRule rule1 = new OutboundRule();
rule1.setFrom("/aaa");
rule1.setTo("/bbb");
conf.addOutboundRule(rule1);
conf.initialise();
UrlRewriter urlRewriter = new UrlRewriter(conf);
MockRequest request = new MockRequest("doesn't matter");
UrlRewriteWrappedResponse urlRewriteWrappedResponse =
new UrlRewriteWrappedResponse(response, request, urlRewriter);
assertEquals("/bbb;mockencoded=test", urlRewriteWrappedResponse.encodeURL("/aaa"));
}
private long[][] getQueryRunStatistics(TestPhase testPhase) {
int numberOfQueries = ThroughputTest.getNumberOfQueries();
int numberOfStreams = conf.getNumberOfStreams();
int[] sortedQueryIds = ThroughputTest.getSortedQueryIds();
// for each query, an array of long type consists of min, median, max time
long[][] queryRunStatistics = new long[numberOfQueries][3];
for (int i = 0; i < numberOfQueries; i++) {
long[] queryTime = new long[numberOfStreams];
for (int j = 0; j < numberOfStreams; j++) {
queryTime[j] = getQueryTime(testPhase, j, sortedQueryIds[i]);
}
Arrays.sort(queryTime);
// min query time
queryRunStatistics[i][0] = queryTime[0];
// max query time
queryRunStatistics[i][2] = queryTime[numberOfStreams - 1];
// median query time
// note that numberOfStreams is an even number
queryRunStatistics[i][1] =
(queryTime[numberOfStreams / 2] + queryTime[numberOfStreams / 2 - 1]) / 2;
}
return queryRunStatistics;
}
public static void main(String[] args) {
try {
Runtime.getRuntime()
.addShutdownHook(
new Thread() {
@Override
public void run() {
Cleanup();
}
});
Conf.ParseArgs(args);
StdoutWriter.Start();
DB.Init();
// It seems that this needs to go through the end of the main function.
// strange thread model.
if (Conf.stream_seed_users) StreamSeedUsers.Start();
CrawlTweets.Run();
} catch (Exception e) {
e.printStackTrace();
System.out.println("Exception caught: " + e);
System.exit(1);
}
}
public static void main(String[] args) throws InterruptedException, ExecutionException {
GPathBSP.runBsp(
"BiconnectBranches",
Conf.getDefaultConfiguration("local-mapreduce"),
BiconnectBranchesBSP.class)
.get();
}
public static String print(Object o) {
try {
return print(new StringBuffer(), o, Conf.getDefaults()).toString();
} catch (Throwable t) {
return PrettyPrinter.class + " " + t.getClass().getName() + ":" + t.getMessage() + "\n" + o;
}
}
private long getRefreshRunTime(TestPhase testPhase) {
long time = 0;
for (int i = 1; i <= conf.getNumberOfStreams() / 2; i++) {
time += getRefreshDataSetTime(testPhase, i);
}
return time;
}
/** Delete worker-specific fileinstall.cfg file for the master. */
public void confDelete(final String worker) {
if (!ResourceUtil.hasProject(worker)) {
Plugin.logWarn("Project#confDelete: missing project: " + worker);
return;
}
ensureConf();
final Conf conf = conf();
final Map<String, String> variables = //
Conf.variables(ResourceUtil.workspacePath(), worker, ResourceUtil.projectPath(worker));
final String path =
ConfUtil.replace( //
conf.fileinstallPath(), variables);
final File file = ProjectUtil.file(project(), path);
file.delete();
Plugin.logOK("Project#confDelete file: " + file);
}
private ConfFactory() {
globalConf = new Conf();
globalConf.loadResource(new File(CONF_HOME + "/conf_cms.xml"));
// globalConf.loadResource(new File(CONF_HOME + "/conf-spider.xml"));
// globalConf.loadResource(new File(CONF_HOME + "/conf-site.xml"));
}
@SuppressWarnings("deprecation")
@Override
public Boolean crawl(RssEntityBean site) throws CrawlerException {
log.info("start YouTube crawl channel id = " + site.url);
Boolean updated = false;
YouTube youtube =
new YouTube.Builder(
HTTP_TRANSPORT,
JSON_FACTORY,
new HttpRequestInitializer() {
public void initialize(HttpRequest request) throws IOException {}
})
.setApplicationName("youtube-cmdline-search-sample")
.build();
try {
YouTube.Search.List search = youtube.search().list("id,snippet");
String apiKey = Conf.getValue("youtube.appkey");
search.setKey(apiKey);
search.setType("video");
search.setOrder("date");
search.setMaxResults(maxResult);
search.setChannelId(site.url);
SearchListResponse searchResponse = search.execute();
List<SearchResult> searchResultList = searchResponse.getItems();
log.info("search response video count = " + searchResultList.size());
for (SearchResult result : searchResultList) {
SearchResultSnippet snippet = result.getSnippet();
DateTime publishDate = snippet.getPublishedAt();
DateTime lastCrawlDate = null;
if (site.lastCrawl != null) {
lastCrawlDate = new DateTime(site.lastCrawl);
}
if ((lastCrawlDate == null) || (publishDate.getValue() > lastCrawlDate.getValue())) {
YouTube.Videos.List videolist = youtube.videos().list("id,snippet");
videolist.setId(result.getId().getVideoId());
videolist.setKey(apiKey);
VideoListResponse videoListResponse = videolist.execute();
List<Video> videos = videoListResponse.getItems();
if (videos.size() != 1) {
log.warn("video not found : " + videolist.getId());
continue;
}
Video video = videos.get(0);
ArticleEntityBean article = new ArticleEntityBean();
article.link = video.getId();
article.title = snippet.getTitle();
article.description = snippet.getDescription();
article.imageUrl = snippet.getThumbnails().getDefault().getUrl();
article.url = site.url;
article.createdAt = snippet.getPublishedAt();
String fixHour = Conf.getValue("fix_time");
Date now = new Date();
Date fixDate =
new Date(now.getYear(), now.getMonth(), now.getDate(), Integer.parseInt(fixHour), 0);
if (now.getTime() > fixDate.getTime()) {
article.publishedAt = tomorrow;
} else {
article.publishedAt = today;
}
article.site = site.site;
article.type = site.type;
article.tags = site.defaultTag;
log.info("save article video id = " + article.link + " / video title = " + article.title);
article.save();
updated = true;
}
}
} catch (IOException e) {
log.warn("youtube api error : " + e.getMessage());
throw new CrawlerException(e);
}
log.info("end YouTube crawl");
return updated;
}
// Class for Metrics collection.
public class Metrics {
private static final Logger logger = Logger.getLogger(Metrics.class.getName());
private HashMap<String, Date> dataMetricsMap = new HashMap<String, Date>();
private static Metrics metrics = new Metrics();
private Conf conf = Conf.getConf();
private Metrics() {}
public static Metrics getMetrics() {
return metrics;
}
private String getPhaseMetricName(TestPhase testPhase, boolean isStartTime) {
String metricName = testPhase.toString() + "_";
if (isStartTime) {
metricName += "start";
} else {
metricName += "end";
}
return metricName;
}
public void addPhaseMetric(TestPhase testPhase, boolean isStartTime, Date timestamp) {
dataMetricsMap.put(getPhaseMetricName(testPhase, isStartTime), timestamp);
}
private String getQueryMetricName(
TestPhase testPhase, int streamId, int queryId, boolean isStartTime) {
String metricName = testPhase.toString() + "_query_" + streamId + "_" + queryId + "_";
if (isStartTime) {
metricName += "start";
} else {
metricName += "end";
}
return metricName;
}
// synchronized method for parallel query running
public synchronized void addQueryMetric(
TestPhase testPhase, int streamId, int queryId, boolean isStartTime, Date timestamp) {
dataMetricsMap.put(getQueryMetricName(testPhase, streamId, queryId, isStartTime), timestamp);
}
private String getRefreshMetricName(
TestPhase testPhase, int refreshDataSetIndex, boolean isStartTime) {
String metricName = testPhase.toString() + "_refresh_" + refreshDataSetIndex + "_";
if (isStartTime) {
metricName += "start";
} else {
metricName += "end";
}
return metricName;
}
// Refresh Run is executed alone without running query together
public void addRefreshMetric(
TestPhase testPhase, int refreshDataSetIndex, boolean isStartTime, Date timestamp) {
dataMetricsMap.put(
getRefreshMetricName(testPhase, refreshDataSetIndex, isStartTime), timestamp);
}
private String getDMMetricName(
TestPhase testPhase, int refreshDataSetIndex, int dmId, boolean isStartTime) {
String metricName = testPhase.toString() + "_dm_" + refreshDataSetIndex + "_" + dmId + "_";
if (isStartTime) {
metricName += "start";
} else {
metricName += "end";
}
return metricName;
}
public void addDMMetric(
TestPhase testPhase, int refreshDataSetIndex, int dmId, boolean isStartTime, Date timestamp) {
dataMetricsMap.put(
getDMMetricName(testPhase, refreshDataSetIndex, dmId, isStartTime), timestamp);
}
private long getPhaseTime(TestPhase testPhase) {
long startTime = dataMetricsMap.get(getPhaseMetricName(testPhase, true)).getTime();
long endTime = dataMetricsMap.get(getPhaseMetricName(testPhase, false)).getTime();
return endTime - startTime;
}
public long getLoadTestTime() {
return getPhaseTime(TestPhase.LOAD_TEST);
}
public long getPowerTestTime() {
return getPhaseTime(TestPhase.POWER_TEST);
}
public long getThroughputTest1Time() {
return getPhaseTime(TestPhase.THROUGHPUT_TEST_1);
}
public long getThroughputTest2Time() {
return getPhaseTime(TestPhase.THROUGHPUT_TEST_2);
}
public long getQueryTime(TestPhase testPhase, int streamId, int queryId) {
long startTime =
dataMetricsMap.get(getQueryMetricName(testPhase, streamId, queryId, true)).getTime();
long endTime =
dataMetricsMap.get(getQueryMetricName(testPhase, streamId, queryId, false)).getTime();
return endTime - startTime;
}
public long getPowerTestQueryTime(int queryId) {
return getQueryTime(TestPhase.POWER_TEST, 0, queryId);
}
private long getRefreshDataSetTime(TestPhase testPhase, int refreshDataSetIndex) {
long startTime =
dataMetricsMap.get(getRefreshMetricName(testPhase, refreshDataSetIndex, true)).getTime();
long endTime =
dataMetricsMap.get(getRefreshMetricName(testPhase, refreshDataSetIndex, false)).getTime();
return endTime - startTime;
}
private long getRefreshRunTime(TestPhase testPhase) {
long time = 0;
for (int i = 1; i <= conf.getNumberOfStreams() / 2; i++) {
time += getRefreshDataSetTime(testPhase, i);
}
return time;
}
public long getRefreshRun1Time() {
return getRefreshRunTime(TestPhase.THROUGHPUT_TEST_1);
}
public long getRefreshRun2Time() {
return getRefreshRunTime(TestPhase.THROUGHPUT_TEST_2);
}
public long getDMTime(TestPhase testPhase, int refreshDataSetIndex, int dmId) {
long startTime =
dataMetricsMap.get(getDMMetricName(testPhase, refreshDataSetIndex, dmId, true)).getTime();
long endTime =
dataMetricsMap.get(getDMMetricName(testPhase, refreshDataSetIndex, dmId, false)).getTime();
return endTime - startTime;
}
public long getQueryRun1Time() {
return getThroughputTest1Time() - getRefreshRun1Time();
}
public long getQueryRun2Time() {
return getThroughputTest2Time() - getRefreshRun2Time();
}
private long[][] getQueryRunStatistics(TestPhase testPhase) {
int numberOfQueries = ThroughputTest.getNumberOfQueries();
int numberOfStreams = conf.getNumberOfStreams();
int[] sortedQueryIds = ThroughputTest.getSortedQueryIds();
// for each query, an array of long type consists of min, median, max time
long[][] queryRunStatistics = new long[numberOfQueries][3];
for (int i = 0; i < numberOfQueries; i++) {
long[] queryTime = new long[numberOfStreams];
for (int j = 0; j < numberOfStreams; j++) {
queryTime[j] = getQueryTime(testPhase, j, sortedQueryIds[i]);
}
Arrays.sort(queryTime);
// min query time
queryRunStatistics[i][0] = queryTime[0];
// max query time
queryRunStatistics[i][2] = queryTime[numberOfStreams - 1];
// median query time
// note that numberOfStreams is an even number
queryRunStatistics[i][1] =
(queryTime[numberOfStreams / 2] + queryTime[numberOfStreams / 2 - 1]) / 2;
}
return queryRunStatistics;
}
public long[][] getQueryRun1Statistics() {
return getQueryRunStatistics(TestPhase.THROUGHPUT_TEST_1);
}
public long[][] getQueryRun2Statistics() {
return getQueryRunStatistics(TestPhase.THROUGHPUT_TEST_2);
}
public double getPerformanceMetric() {
int numberOfQueries = ThroughputTest.getNumberOfQueries();
int S = conf.getNumberOfStreams();
int Q = 3 * S * numberOfQueries;
long Ttt = getThroughputTest1Time() + getThroughputTest2Time();
double Tld = 0.01 * S * getLoadTestTime();
long Tpt = getPowerTestTime() * S;
double performanceMetric = conf.getScale() * Q * 1.0 / ((Tpt + Ttt + Tld) / (3600 * 1000));
return performanceMetric;
}
}
@Before
public void reset() {
Env.reset();
Conf.reset();
}
public static void init(Conf conf) {
zkClient = openClient(conf.getZookeeperServer());
ZkPathUtils.init(conf.getZookeeperRootPath(), zkClient);
}