/** Upload feeds, processes and clusters with different names. */
@BeforeClass(alwaysRun = true)
public void prepareData()
throws IOException, URISyntaxException, AuthenticationException, InterruptedException,
JAXBException {
uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
removeTestClassEntities();
bundles[0] = BundleUtil.readELBundle();
bundles[0] = new Bundle(bundles[0], servers.get(0));
bundles[0].generateUniqueBundle(this);
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
bundles[0].submitBundle(prism);
// submit different clusters, feeds and processes
FeedMerlin feed = new FeedMerlin(bundles[0].getInputFeedFromBundle());
ProcessMerlin process = bundles[0].getProcessObject();
ClusterMerlin cluster = bundles[0].getClusterElement();
String clusterNamePrefix = bundles[0].getClusterElement().getName() + '-';
String processNamePrefix = bundles[0].getProcessName() + '-';
String feedNamePrefix = bundles[0].getInputFeedNameFromBundle() + '-';
List randomNames = getPatternName();
for (Object randomName : randomNames) {
process.setName(processNamePrefix + randomName);
AssertUtil.assertSucceeded(prism.getProcessHelper().submitAndSchedule(process.toString()));
feed.setName(feedNamePrefix + randomName);
AssertUtil.assertSucceeded(prism.getFeedHelper().submitAndSchedule(feed.toString()));
cluster.setName(clusterNamePrefix + randomName);
AssertUtil.assertSucceeded(prism.getClusterHelper().submitEntity(cluster.toString()));
}
}
/**
* Set feed1 to have cluster1 as source, cluster3 as target. Set feed2 clusters vise versa. Add
* both clusters to process and feed2 as input feed. Run process. Update feed1. TODO test case is
* incomplete
*
* @throws Exception
*/
@Test(enabled = true, timeOut = 1800000)
public void updateFeedDependentProcessTest() throws Exception {
// set cluster colos
bundles[0].setCLusterColo(cluster1Colo);
bundles[1].setCLusterColo(cluster2Colo);
bundles[2].setCLusterColo(cluster3Colo);
// submit 3 clusters
Bundle.submitCluster(bundles[0], bundles[1], bundles[2]);
// get 2 unique feeds
FeedMerlin feed01 = new FeedMerlin(bundles[0].getInputFeedFromBundle());
FeedMerlin feed02 = new FeedMerlin(bundles[1].getInputFeedFromBundle());
FeedMerlin outputFeed = new FeedMerlin(bundles[0].getOutputFeedFromBundle());
// set clusters to null;
feed01.clearFeedClusters();
feed02.clearFeedClusters();
outputFeed.clearFeedClusters();
// set new feed input data
feed01.setFeedPathValue(baseTestDir + "/feed01" + MINUTE_DATE_PATTERN);
feed02.setFeedPathValue(baseTestDir + "/feed02" + MINUTE_DATE_PATTERN);
// generate data in both the colos ua1 and ua3
String prefix = feed01.getFeedPrefix();
HadoopUtil.deleteDirIfExists(prefix.substring(1), cluster1FS);
HadoopUtil.lateDataReplenish(cluster1FS, 25, 1, prefix, null);
prefix = feed02.getFeedPrefix();
HadoopUtil.deleteDirIfExists(prefix.substring(1), cluster3FS);
HadoopUtil.lateDataReplenish(cluster3FS, 25, 1, prefix, null);
String startTime = TimeUtil.getTimeWrtSystemTime(-50);
// set clusters for feed01
feed01.addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0)))
.withRetention("hours(10)", ActionType.DELETE)
.withValidity(startTime, "2099-01-01T00:00Z")
.withClusterType(ClusterType.SOURCE)
.build());
feed01.addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0)))
.withRetention("hours(10)", ActionType.DELETE)
.withValidity(startTime, "2099-01-01T00:00Z")
.withClusterType(ClusterType.TARGET)
.build());
// set clusters for feed02
feed02.addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0)))
.withRetention("hours(10)", ActionType.DELETE)
.withValidity(startTime, "2099-01-01T00:00Z")
.withClusterType(ClusterType.TARGET)
.build());
feed02.addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0)))
.withRetention("hours(10)", ActionType.DELETE)
.withValidity(startTime, "2099-01-01T00:00Z")
.withClusterType(ClusterType.SOURCE)
.build());
// set clusters for output feed
outputFeed.addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0)))
.withRetention("hours(10)", ActionType.DELETE)
.withValidity(startTime, "2099-01-01T00:00Z")
.withClusterType(ClusterType.SOURCE)
.build());
outputFeed.addFeedCluster(
new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0)))
.withRetention("hours(10)", ActionType.DELETE)
.withValidity(startTime, "2099-01-01T00:00Z")
.withClusterType(ClusterType.TARGET)
.build());
// submit and schedule feeds
prism.getFeedHelper().submitAndSchedule(feed01.toString());
prism.getFeedHelper().submitAndSchedule(feed02.toString());
prism.getFeedHelper().submitAndSchedule(outputFeed.toString());
// create a process with 2 clusters
// get a process
ProcessMerlin process = new ProcessMerlin(bundles[0].getProcessData());
// add clusters to process
String processStartTime = TimeUtil.getTimeWrtSystemTime(-6);
String processEndTime = TimeUtil.getTimeWrtSystemTime(70);
process.clearProcessCluster();
process.addProcessCluster(
new ProcessMerlin.ProcessClusterBuilder(
Util.readEntityName(bundles[0].getClusters().get(0)))
.withValidity(processStartTime, processEndTime)
.build());
process.addProcessCluster(
new ProcessMerlin.ProcessClusterBuilder(
Util.readEntityName(bundles[2].getClusters().get(0)))
.withValidity(processStartTime, processEndTime)
.build());
process.addInputFeed(feed02.getName(), feed02.getName());
// submit and schedule process
AssertUtil.assertSucceeded(prism.getProcessHelper().submitAndSchedule(process.toString()));
LOGGER.info("Wait till process goes into running ");
int timeout = OSUtil.IS_WINDOWS ? 50 : 25;
InstanceUtil.waitTillInstanceReachState(
serverOC.get(0), process.getName(), 1, Status.RUNNING, EntityType.PROCESS, timeout);
InstanceUtil.waitTillInstanceReachState(
serverOC.get(2), process.getName(), 1, Status.RUNNING, EntityType.PROCESS, timeout);
feed01.setFilePath(alternativeInputPath);
LOGGER.info("updated feed: " + Util.prettyPrintXml(feed01.toString()));
AssertUtil.assertSucceeded(prism.getFeedHelper().update(feed01.toString(), feed01.toString()));
}