@Test
public void testIndexRebuilds() throws Exception {
IOHelper.deleteFile(schedulerStoreDir);
JobSchedulerStoreImpl schedulerStore = createScheduler();
broker = createBroker(schedulerStore);
broker.start();
ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory("vm://localhost");
Connection connection = cf.createConnection();
connection.start();
for (int i = 0; i < NUM_JOBS; ++i) {
scheduleRepeating(connection);
}
connection.close();
JobScheduler scheduler = schedulerStore.getJobScheduler("JMS");
assertNotNull(scheduler);
assertEquals(NUM_JOBS, scheduler.getAllJobs().size());
broker.stop();
IOHelper.delete(new File(schedulerStoreDir, "scheduleDB.data"));
schedulerStore = createScheduler();
broker = createBroker(schedulerStore);
broker.start();
scheduler = schedulerStore.getJobScheduler("JMS");
assertNotNull(scheduler);
assertEquals(NUM_JOBS, scheduler.getAllJobs().size());
}
public void stop() {
LOGGER.info("Stopping BIMserver");
executorService.shutdown();
if (bimDatabase != null) {
bimDatabase.close();
}
if (bimScheduler != null) {
bimScheduler.close();
}
if (longActionManager != null) {
longActionManager.shutdown();
}
if (notificationsManager != null) {
notificationsManager.shutdown();
}
if (embeddedWebServer != null) {
embeddedWebServer.shutdown();
}
if (protocolBuffersServer != null) {
protocolBuffersServer.shutdown();
}
if (commandLine != null) {
commandLine.shutdown();
}
LOGGER.info("BIMserver stopped");
}
/** @see FlowService#executeJob(String, String, String, long, long, String) */
@Override
public Job executeJob(
Credentials credentials,
String token,
String name,
String description,
long flowInstanceId,
long userId,
String userEmail) {
FlowDao flowDao = daoFactory.getFlowDao();
JobDao jobDao = daoFactory.getJobDao();
Flow flowInstance = flowDao.findById(flowInstanceId, false);
Job job = new Job();
job.setToken(token);
job.setName(name);
job.setDescription(description);
job.setFlow(flowInstance);
job.setOwnerId(userId);
job.setOwnerEmail(userEmail);
SimpleCredentials simpleCreds = (SimpleCredentials) credentials;
String serializedCreds = simpleCreds.getUserID() + ":" + new String(simpleCreds.getPassword());
job.setCredentials(serializedCreds);
jobDao.makePersistent(job);
jobScheduler.scheduleJob(job);
job.setJobStatus(JobStatus.SCHEDULED);
job.setScheduleTimestamp(new Date());
jobDao.makePersistent(job);
jobStatusMonitor.start(job, notificationCreator);
return job;
}
private void assignConnectorPolicy(ActivityCluster ac, Map<ActivityId, ActivityPlan> taskMap) {
Map<ConnectorDescriptorId, IConnectorPolicy> cPolicyMap =
new HashMap<ConnectorDescriptorId, IConnectorPolicy>();
Set<ActivityId> activities = ac.getActivityMap().keySet();
BitSet targetBitmap = new BitSet();
for (ActivityId a1 : activities) {
Task[] ac1TaskStates = taskMap.get(a1).getTasks();
int nProducers = ac1TaskStates.length;
List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(a1);
if (outputConns != null) {
for (IConnectorDescriptor c : outputConns) {
ConnectorDescriptorId cdId = c.getConnectorId();
ActivityId a2 = ac.getConsumerActivity(cdId);
Task[] ac2TaskStates = taskMap.get(a2).getTasks();
int nConsumers = ac2TaskStates.length;
int[] fanouts = new int[nProducers];
if (c.allProducersToAllConsumers()) {
for (int i = 0; i < nProducers; ++i) {
fanouts[i] = nConsumers;
}
} else {
for (int i = 0; i < nProducers; ++i) {
c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
fanouts[i] = targetBitmap.cardinality();
}
}
IConnectorPolicy cp = assignConnectorPolicy(ac, c, nProducers, nConsumers, fanouts);
cPolicyMap.put(cdId, cp);
}
}
}
scheduler.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
}
public org.neo4j.server.database.RrdDbWrapper createRrdDbAndSampler(
final Database db, JobScheduler scheduler) throws IOException {
NodeManager nodeManager =
db.getGraph().getDependencyResolver().resolveDependency(NodeManager.class);
Sampleable[] primitives = {
new NodeIdsInUseSampleable(nodeManager),
new PropertyCountSampleable(nodeManager),
new RelationshipCountSampleable(nodeManager)
};
Sampleable[] usage = {};
final String rrdPath =
config.getString(RRDB_LOCATION_PROPERTY_KEY, getDefaultRrdFile(db.getGraph()));
final RrdDbWrapper rrdb =
createRrdb(rrdPath, isEphemereal(db.getGraph()), join(primitives, usage));
scheduler.scheduleAtFixedRate(
new RrdJob(new RrdSamplerImpl(rrdb.get(), primitives)),
RRD_THREAD_NAME + "[primitives]",
SECONDS.toMillis(0),
SECONDS.toMillis(3));
return rrdb;
}
@Override
public String getActiveProgramsList() throws RemoteException {
String list =
String.format(
"%-20s %-10s %-10s %-20s %-20s%n", "Job", "ID", "Status", "Mappers", "Reducers");
list = list.concat(scheduler.getActiveProgramsList());
return list;
}
@Override
public synchronized void jobFinished(boolean success, NodeJob job) throws RemoteException {
if (!success) {
dispatcher.enqueue(job);
} else {
scheduler.jobFinished(job);
}
}
@Override
public int dispatchJob(Class<?> clazz, String filename) throws RemoteException {
Map<Integer, Set<Integer>> blockLocations = fsTable.get(filename);
if (blockLocations == null) {
return -1;
}
return scheduler.issueJob(clazz, filename, blockLocations.size());
}
@Test
public void testIndexRebuildsAfterSomeJobsExpire() throws Exception {
IOHelper.deleteFile(schedulerStoreDir);
JobSchedulerStoreImpl schedulerStore = createScheduler();
broker = createBroker(schedulerStore);
broker.start();
ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory("vm://localhost");
Connection connection = cf.createConnection();
connection.start();
for (int i = 0; i < NUM_JOBS; ++i) {
scheduleRepeating(connection);
scheduleOneShot(connection);
}
connection.close();
JobScheduler scheduler = schedulerStore.getJobScheduler("JMS");
assertNotNull(scheduler);
assertEquals(NUM_JOBS * 2, scheduler.getAllJobs().size());
final JobScheduler awaitingOneShotTimeout = scheduler;
assertTrue(
"One shot jobs should time out",
Wait.waitFor(
new Wait.Condition() {
@Override
public boolean isSatisified() throws Exception {
return awaitingOneShotTimeout.getAllJobs().size() == NUM_JOBS;
}
},
TimeUnit.MINUTES.toMillis(2)));
broker.stop();
IOHelper.delete(new File(schedulerStoreDir, "scheduleDB.data"));
schedulerStore = createScheduler();
broker = createBroker(schedulerStore);
broker.start();
scheduler = schedulerStore.getJobScheduler("JMS");
assertNotNull(scheduler);
assertEquals(NUM_JOBS, scheduler.getAllJobs().size());
}
private TaskCluster getTaskCluster(TaskId tid) {
JobRun run = scheduler.getJobRun();
ActivityCluster ac = run.getActivityClusterGraph().getActivityMap().get(tid.getActivityId());
ActivityClusterPlan acp = run.getActivityClusterPlanMap().get(ac.getId());
Task[] tasks = acp.getActivityPlanMap().get(tid.getActivityId()).getTasks();
Task task = tasks[tid.getPartition()];
assert task.getTaskId().equals(tid);
return task.getTaskCluster();
}
/** @see FlowService#abortJob(long) */
@Override
public void abortJob(long jobId) throws IllegalStateException {
Job job = daoFactory.getJobDao().findById(jobId, false);
if (!job.isRunning()) {
throw new IllegalStateException(
"Cannot abort job " + jobId + " because it has already completed.");
}
try {
jobScheduler.abortJob(job);
} catch (MeandreServerException e) {
throw new ServiceException("Job " + jobId + " could not be aborted.", e);
}
}
public FacilityManagerMasterImpl(Config config)
throws IOException, AlreadyBoundException, InterruptedException {
super(config);
currentNode = 0;
String[] participants = config.getParticipantIps();
int expectedNumParticipants = participants.length;
managers = new FacilityManager[expectedNumParticipants];
managers[getNodeId()] = this;
clusterName = config.getClusterName();
rmiPort = config.getRmiPort();
/* FILESYSTEM INIT */
fsTable = Collections.synchronizedMap(new HashMap<String, Map<Integer, Set<Integer>>>());
Registry r = LocateRegistry.createRegistry(rmiPort);
r.bind(clusterName + REGISTRY_MASTER_KEY, UnicastRemoteObject.exportObject(this, 0));
connectParticipants();
System.out.println("Waiting for slaves to connect...");
while (managers.length != expectedNumParticipants) {
Thread.sleep(1000);
}
healthChecker = new HealthChecker(this, expectedNumParticipants);
scheduler = new JobScheduler(this, config);
dispatcher = new JobDispatcher(this, config);
dispatcher.setScheduler(scheduler);
scheduler.setDispatcher(dispatcher);
scheduler.setHealthChecker(healthChecker);
healthChecker.start();
dispatcher.start();
System.out.println("All slaves connected.");
}
/** @see FlowService#getWorkerStatus(String, int) */
@Override
public MeandreServerProxyStatus getWorkerStatus(String host, int port) {
if (host == null) {
throw new IllegalArgumentException("Host must not be null.");
}
Map<MeandreServerProxyConfig, MeandreServerProxyStatus> status = jobScheduler.getWorkerStatus();
Iterator<MeandreServerProxyConfig> configIterator = status.keySet().iterator();
while (configIterator.hasNext()) {
MeandreServerProxyConfig config = configIterator.next();
if (host.equals(config.getHost()) && port == config.getPort()) {
return status.get(config);
}
}
return null;
}
public ActivityClusterPlan planActivityCluster(ActivityCluster ac) throws HyracksException {
JobRun jobRun = scheduler.getJobRun();
Map<ActivityId, ActivityPartitionDetails> pcMap = computePartitionCounts(ac);
Map<ActivityId, ActivityPlan> activityPlanMap = buildActivityPlanMap(ac, jobRun, pcMap);
assignConnectorPolicy(ac, activityPlanMap);
TaskCluster[] taskClusters = computeTaskClusters(ac, jobRun, activityPlanMap);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Plan for " + ac);
LOGGER.info("Built " + taskClusters.length + " Task Clusters");
for (TaskCluster tc : taskClusters) {
LOGGER.info("Tasks: " + Arrays.toString(tc.getTasks()));
}
}
return new ActivityClusterPlan(taskClusters, activityPlanMap);
}
@PostConstruct
public void init() {
logger.info("Initializing NEMA Flow Service...");
notificationCreator = new JobStatusNotificationCreator(daoFactory);
try {
headServer =
meandreServerProxyFactory.getServerProxyInstance(flowServiceConfig.getHeadConfig(), true);
} catch (MeandreServerException e) {
throw new RuntimeException("Could not instantiate head server.", e);
}
flowServiceConfig.addChangeListener(this);
// Any jobs marked as scheduled in the database will be put back in the
// queue for execution.
JobDao jobDao = daoFactory.getJobDao();
List<Job> scheduledJobs;
try {
Session session = jobDao.getSessionFactory().openSession();
jobDao.startManagedSession(session);
scheduledJobs = jobDao.getJobsByStatus(Job.JobStatus.SCHEDULED);
jobDao.endManagedSession();
session.close();
} catch (HibernateException e) {
throw new RuntimeException("Problem searching for scheduled jobs" + " in the database.", e);
}
if (scheduledJobs != null && scheduledJobs.size() > 0) {
logger.info(
scheduledJobs.size()
+ " scheduled jobs found in the "
+ "database. Jobs will be rescheduled for execution.");
}
for (Job job : scheduledJobs) {
jobScheduler.scheduleJob(job);
jobStatusMonitor.start(job, notificationCreator);
}
}
/** @see ConfigChangeListener#configChanged() */
@Override
public void configChanged() {
logger.info("Received configuration change notification.");
if (!headServer.getConfig().equals(flowServiceConfig.getHeadConfig())) {
MeandreServerProxy newHead;
try {
newHead =
meandreServerProxyFactory.getServerProxyInstance(
flowServiceConfig.getHeadConfig(), true);
} catch (MeandreServerException e) {
throw new RuntimeException("Could not instantiate head server.", e);
}
meandreServerProxyFactory.release(headServer);
headServer = newHead;
logger.info(
"Head server configuration has changed. New head "
+ "server is "
+ headServer.toString());
} else {
logger.info("Head server has not changed.");
}
jobScheduler.setWorkerConfigs(flowServiceConfig.getWorkerConfigs());
}
private void executePhase(ExecutionContext context, Set<ExecutionScript> executions)
throws InterruptedException, IOException {
assert context != null;
assert executions != null;
YSLOG.info(
"I03000",
context.getBatchId(),
context.getFlowId(),
context.getExecutionId(),
context.getPhase());
long start = System.currentTimeMillis();
try {
if (skipFlows.contains(context.getFlowId())) {
YSLOG.info(
"I03002",
context.getBatchId(),
context.getFlowId(),
context.getExecutionId(),
context.getPhase());
return;
}
List<? extends Job> jobs;
ErrorHandler handler;
switch (context.getPhase()) {
case SETUP:
jobs = buildSetupJobs(context);
handler = JobScheduler.STRICT;
break;
case CLEANUP:
jobs = buildCleanupJobs(context);
handler = JobScheduler.BEST_EFFORT;
break;
case FINALIZE:
jobs = buildExecutionJobs(context, executions);
handler = JobScheduler.BEST_EFFORT;
break;
default:
jobs = buildExecutionJobs(context, executions);
handler = JobScheduler.STRICT;
break;
}
PhaseMonitor monitor = obtainPhaseMonitor(context);
try {
scheduler.execute(monitor, context, jobs, handler);
} finally {
monitor.close();
}
YSLOG.info(
"I03001",
context.getBatchId(),
context.getFlowId(),
context.getExecutionId(),
context.getPhase());
} catch (IOException e) {
YSLOG.error(
e,
"E03001",
context.getBatchId(),
context.getFlowId(),
context.getExecutionId(),
context.getPhase());
throw e;
} catch (InterruptedException e) {
YSLOG.warn(
e,
"W03001",
context.getBatchId(),
context.getFlowId(),
context.getExecutionId(),
context.getPhase());
throw e;
} finally {
long end = System.currentTimeMillis();
YSLOG.info(
"I03999",
context.getBatchId(),
context.getFlowId(),
context.getExecutionId(),
context.getPhase(),
end - start);
}
}
private Map<ActivityId, ActivityPartitionDetails> computePartitionCounts(ActivityCluster ac)
throws HyracksException {
PartitionConstraintSolver solver = scheduler.getSolver();
Set<LValueConstraintExpression> lValues = new HashSet<LValueConstraintExpression>();
for (ActivityId anId : ac.getActivityMap().keySet()) {
lValues.add(new PartitionCountExpression(anId.getOperatorDescriptorId()));
}
solver.solve(lValues);
Map<OperatorDescriptorId, Integer> nPartMap = new HashMap<OperatorDescriptorId, Integer>();
for (LValueConstraintExpression lv : lValues) {
Object value = solver.getValue(lv);
if (value == null) {
throw new HyracksException("No value found for " + lv);
}
if (!(value instanceof Number)) {
throw new HyracksException(
"Unexpected type of value bound to "
+ lv
+ ": "
+ value.getClass()
+ "("
+ value
+ ")");
}
int nParts = ((Number) value).intValue();
if (nParts <= 0) {
throw new HyracksException("Unsatisfiable number of partitions for " + lv + ": " + nParts);
}
nPartMap.put(
((PartitionCountExpression) lv).getOperatorDescriptorId(), Integer.valueOf(nParts));
}
Map<ActivityId, ActivityPartitionDetails> activityPartsMap =
new HashMap<ActivityId, ActivityPartitionDetails>();
for (ActivityId anId : ac.getActivityMap().keySet()) {
int nParts = nPartMap.get(anId.getOperatorDescriptorId());
int[] nInputPartitions = null;
List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(anId);
if (inputs != null) {
nInputPartitions = new int[inputs.size()];
for (int i = 0; i < nInputPartitions.length; ++i) {
ConnectorDescriptorId cdId = inputs.get(i).getConnectorId();
ActivityId aid = ac.getProducerActivity(cdId);
Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId());
nInputPartitions[i] = nPartInt;
}
}
int[] nOutputPartitions = null;
List<IConnectorDescriptor> outputs = ac.getActivityOutputMap().get(anId);
if (outputs != null) {
nOutputPartitions = new int[outputs.size()];
for (int i = 0; i < nOutputPartitions.length; ++i) {
ConnectorDescriptorId cdId = outputs.get(i).getConnectorId();
ActivityId aid = ac.getConsumerActivity(cdId);
Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId());
nOutputPartitions[i] = nPartInt;
}
}
ActivityPartitionDetails apd =
new ActivityPartitionDetails(nParts, nInputPartitions, nOutputPartitions);
activityPartsMap.put(anId, apd);
}
return activityPartsMap;
}
public void start()
throws DatabaseInitException, BimserverDatabaseException, PluginException,
DatabaseRestartRequiredException, ServerException {
try {
LOGGER.debug("Starting BIMserver");
SVersion localVersion = versionChecker.getLocalVersion();
if (localVersion != null) {
LOGGER.info(
"Version: "
+ localVersion.getMajor()
+ "."
+ localVersion.getMinor()
+ "."
+ localVersion.getRevision()
+ " - "
+ localVersion.getDate());
} else {
LOGGER.info("Unknown version");
}
try {
pluginManager.addPluginChangeListener(
new PluginChangeListener() {
@Override
public void pluginStateChanged(PluginContext pluginContext, boolean enabled) {
// Reflect this change also in the database
Condition pluginCondition =
new AttributeCondition(
StorePackage.eINSTANCE.getPluginDescriptor_PluginClassName(),
new StringLiteral(pluginContext.getPlugin().getClass().getName()));
DatabaseSession session = bimDatabase.createSession();
try {
Map<Long, PluginDescriptor> pluginsFound =
session.query(pluginCondition, PluginDescriptor.class, Query.getDefault());
if (pluginsFound.size() == 0) {
LOGGER.error(
"Error changing plugin-state in database, plugin "
+ pluginContext.getPlugin().getClass().getName()
+ " not found");
} else if (pluginsFound.size() == 1) {
PluginDescriptor pluginConfiguration = pluginsFound.values().iterator().next();
pluginConfiguration.setEnabled(pluginContext.isEnabled());
session.store(pluginConfiguration);
} else {
LOGGER.error(
"Error, too many plugin-objects found in database for name "
+ pluginContext.getPlugin().getClass().getName());
}
session.commit();
} catch (BimserverDatabaseException e) {
LOGGER.error("", e);
} catch (ServiceException e) {
LOGGER.error("", e);
} finally {
session.close();
}
}
});
pluginManager.loadPlugin(
ObjectIDMPlugin.class,
new File(".").toURI(),
"Internal",
new SchemaFieldObjectIDMPlugin(),
getClass().getClassLoader(),
PluginSourceType.INTERNAL,
null);
} catch (Exception e) {
LOGGER.error("", e);
}
try {
metaDataManager.init();
pluginManager.initAllLoadedPlugins();
} catch (PluginException e) {
LOGGER.error("", e);
}
serverStartTime = new GregorianCalendar();
longActionManager = new LongActionManager();
Set<EPackage> packages = new LinkedHashSet<>();
packages.add(Ifc2x3tc1Package.eINSTANCE);
packages.add(Ifc4Package.eINSTANCE);
templateEngine = new TemplateEngine();
templateEngine.init(config.getResourceFetcher().getResource("templates/"));
Path databaseDir = config.getHomeDir().resolve("database");
BerkeleyKeyValueStore keyValueStore = new BerkeleyKeyValueStore(databaseDir);
schemaConverterManager.registerConverter(new Ifc2x3tc1ToIfc4SchemaConverterFactory());
schemaConverterManager.registerConverter(new Ifc4ToIfc2x3tc1SchemaConverterFactory());
metricsRegistry = new MetricsRegistry();
Query.setPackageMetaDataForDefaultQuery(metaDataManager.getPackageMetaData("store"));
bimDatabase = new Database(this, packages, keyValueStore, metaDataManager);
try {
bimDatabase.init();
} catch (DatabaseRestartRequiredException e) {
bimDatabase.close();
keyValueStore = new BerkeleyKeyValueStore(databaseDir);
bimDatabase = new Database(this, packages, keyValueStore, metaDataManager);
try {
bimDatabase.init();
} catch (InconsistentModelsException e1) {
LOGGER.error("", e);
serverInfoManager.setServerState(ServerState.FATAL_ERROR);
serverInfoManager.setErrorMessage("Inconsistent models");
}
} catch (InconsistentModelsException e) {
LOGGER.error("", e);
serverInfoManager.setServerState(ServerState.FATAL_ERROR);
serverInfoManager.setErrorMessage("Inconsistent models");
}
DatabaseSession encsession = bimDatabase.createSession();
try {
byte[] encryptionkeyBytes = null;
if (!bimDatabase.getRegistry().has(ENCRYPTIONKEY, encsession)) {
encryptionkeyBytes = new byte[16];
new SecureRandom().nextBytes(encryptionkeyBytes);
bimDatabase.getRegistry().save(ENCRYPTIONKEY, encryptionkeyBytes, encsession);
encsession.commit();
} else {
encryptionkeyBytes = bimDatabase.getRegistry().readByteArray(ENCRYPTIONKEY, encsession);
}
encryptionkey = new SecretKeySpec(encryptionkeyBytes, "AES");
} finally {
encsession.close();
}
protocolBuffersMetaData = new ProtocolBuffersMetaData();
protocolBuffersMetaData.load(servicesMap, ProtocolBuffersBimServerClientFactory.class);
serverInfoManager.init(this);
webModuleManager = new WebModuleManager(this);
jsonHandler = new JsonHandler(this);
serializerFactory = new SerializerFactory();
deserializerFactory = new DeserializerFactory();
serverSettingsCache = new ServerSettingsCache(bimDatabase);
serverInfoManager.update();
if (serverInfoManager.getServerState() == ServerState.MIGRATION_REQUIRED) {
serverInfoManager.registerStateChangeListener(
new StateChangeListener() {
@Override
public void stateChanged(ServerState oldState, ServerState newState) {
if (oldState == ServerState.MIGRATION_REQUIRED && newState == ServerState.RUNNING) {
try {
initDatabaseDependantItems();
} catch (BimserverDatabaseException e) {
LOGGER.error("", e);
}
}
}
});
} else {
initDatabaseDependantItems();
}
mailSystem = new MailSystem(this);
diskCacheManager = new DiskCacheManager(this, config.getHomeDir().resolve("cache"));
mergerFactory = new MergerFactory(this);
FileBasedReflectorFactoryBuilder factoryBuilder = new FileBasedReflectorFactoryBuilder();
reflectorFactory = factoryBuilder.newReflectorFactory();
if (reflectorFactory == null) {
throw new RuntimeException("No reflector factory!");
}
servicesMap.setReflectorFactory(reflectorFactory);
bimScheduler = new JobScheduler(this);
bimScheduler.start();
if (config.isStartEmbeddedWebServer()) {
embeddedWebServer.start();
}
if (config.isStartCommandLine()) {
commandLine = new CommandLine(this);
commandLine.start();
}
} catch (Throwable e) {
LOGGER.error("", e);
serverInfoManager.setErrorMessage(e.getMessage());
}
}
public void slaveDied(int id) {
managers[id] = null;
scheduler.nodeDied(id);
}
private TaskCluster[] buildConnectorPolicyAwareTaskClusters(
ActivityCluster ac,
Map<ActivityId, ActivityPlan> activityPlanMap,
Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity) {
Map<TaskId, Set<TaskId>> taskClusterMap = new HashMap<TaskId, Set<TaskId>>();
for (ActivityId anId : ac.getActivityMap().keySet()) {
ActivityPlan ap = activityPlanMap.get(anId);
Task[] tasks = ap.getTasks();
for (Task t : tasks) {
Set<TaskId> cluster = new HashSet<TaskId>();
TaskId tid = t.getTaskId();
cluster.add(tid);
taskClusterMap.put(tid, cluster);
}
}
JobRun jobRun = scheduler.getJobRun();
Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
for (Map.Entry<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> e :
taskConnectivity.entrySet()) {
Set<TaskId> cluster = taskClusterMap.get(e.getKey());
for (Pair<TaskId, ConnectorDescriptorId> p : e.getValue()) {
IConnectorPolicy cPolicy = connectorPolicies.get(p.getRight());
if (cPolicy.requiresProducerConsumerCoscheduling()) {
cluster.add(p.getLeft());
}
}
}
/*
* taskClusterMap contains for every TID x, x -> { coscheduled consumer TIDs U x }
* We compute the transitive closure of this relation to find the largest set of
* tasks that need to be co-scheduled
*/
int counter = 0;
TaskId[] ordinalList = new TaskId[taskClusterMap.size()];
Map<TaskId, Integer> ordinalMap = new HashMap<TaskId, Integer>();
for (TaskId tid : taskClusterMap.keySet()) {
ordinalList[counter] = tid;
ordinalMap.put(tid, counter);
++counter;
}
int n = ordinalList.length;
BitSet[] paths = new BitSet[n];
for (Map.Entry<TaskId, Set<TaskId>> e : taskClusterMap.entrySet()) {
int i = ordinalMap.get(e.getKey());
BitSet bsi = paths[i];
if (bsi == null) {
bsi = new BitSet(n);
paths[i] = bsi;
}
for (TaskId ttid : e.getValue()) {
int j = ordinalMap.get(ttid);
paths[i].set(j);
BitSet bsj = paths[j];
if (bsj == null) {
bsj = new BitSet(n);
paths[j] = bsj;
}
bsj.set(i);
}
}
for (int k = 0; k < n; ++k) {
for (int i = paths[k].nextSetBit(0); i >= 0; i = paths[k].nextSetBit(i + 1)) {
for (int j = paths[i].nextClearBit(0); j < n && j >= 0; j = paths[i].nextClearBit(j + 1)) {
paths[i].set(j, paths[k].get(j));
paths[j].set(i, paths[i].get(j));
}
}
}
BitSet pending = new BitSet(n);
pending.set(0, n);
List<List<TaskId>> clusters = new ArrayList<List<TaskId>>();
for (int i = pending.nextSetBit(0); i >= 0; i = pending.nextSetBit(i)) {
List<TaskId> cluster = new ArrayList<TaskId>();
for (int j = paths[i].nextSetBit(0); j >= 0; j = paths[i].nextSetBit(j + 1)) {
cluster.add(ordinalList[j]);
pending.clear(j);
}
clusters.add(cluster);
}
List<TaskCluster> tcSet = new ArrayList<TaskCluster>();
counter = 0;
for (List<TaskId> cluster : clusters) {
List<Task> taskStates = new ArrayList<Task>();
for (TaskId tid : cluster) {
taskStates.add(activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()]);
}
TaskCluster tc =
new TaskCluster(
new TaskClusterId(ac.getId(), counter++),
ac,
taskStates.toArray(new Task[taskStates.size()]));
tcSet.add(tc);
for (TaskId tid : cluster) {
activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()].setTaskCluster(tc);
}
}
TaskCluster[] taskClusters = tcSet.toArray(new TaskCluster[tcSet.size()]);
return taskClusters;
}
@Override
public String getCompletedProgramsList() throws RemoteException {
return scheduler.getCompletedProgramsList();
}
/** @see FlowService#getWorkerStatus() */
@Override
public Map<MeandreServerProxyConfig, MeandreServerProxyStatus> getWorkerStatus() {
return jobScheduler.getWorkerStatus();
}
/** @see FlowService#getScheduledJobs() */
@Override
public List<Job> getScheduledJobs() {
return jobScheduler.getScheduledJobs();
}
@Override
public void stopProgram(String classname, String filename) throws RemoteException {
scheduler.stopProgram(classname, filename);
}
