private synchronized void handleIntakeJobStartMessage(FeedIntakeInfo intakeJobInfo)
throws Exception {
List<OperatorDescriptorId> intakeOperatorIds = new ArrayList<OperatorDescriptorId>();
Map<OperatorDescriptorId, IOperatorDescriptor> operators =
intakeJobInfo.getSpec().getOperatorMap();
for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) {
IOperatorDescriptor opDesc = entry.getValue();
if (opDesc instanceof FeedIntakeOperatorDescriptor) {
intakeOperatorIds.add(opDesc.getOperatorId());
}
}
IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
JobInfo info = hcc.getJobInfo(intakeJobInfo.getJobId());
List<String> intakeLocations = new ArrayList<String>();
for (OperatorDescriptorId intakeOperatorId : intakeOperatorIds) {
Map<Integer, String> operatorLocations = info.getOperatorLocations().get(intakeOperatorId);
int nOperatorInstances = operatorLocations.size();
for (int i = 0; i < nOperatorInstances; i++) {
intakeLocations.add(operatorLocations.get(i));
}
}
// intakeLocations is an ordered list; element at position i corresponds to location of i'th
// instance of operator
intakeJobInfo.setIntakeLocation(intakeLocations);
intakeJobInfo.getIntakeFeedJoint().setState(State.ACTIVE);
intakeJobInfo.setState(FeedJobState.ACTIVE);
// notify event listeners
notifyFeedEventSubscribers(intakeJobInfo, FeedLifecycleEvent.FEED_INTAKE_STARTED);
}
private void handleFeedIntakeJobFinishMessage(FeedIntakeInfo intakeInfo, Message message)
throws Exception {
IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
JobInfo info = hcc.getJobInfo(message.jobId);
JobStatus status = info.getStatus();
FeedLifecycleEvent event;
event =
status.equals(JobStatus.FAILURE)
? FeedLifecycleEvent.FEED_INTAKE_FAILURE
: FeedLifecycleEvent.FEED_ENDED;
// remove feed joints
deregisterFeedIntakeJob(message.jobId);
// notify event listeners
notifyFeedEventSubscribers(intakeInfo, event);
}
private void handleFeedCollectJobFinishMessage(FeedConnectJobInfo cInfo) throws Exception {
FeedConnectionId connectionId = cInfo.getConnectionId();
IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
JobInfo info = hcc.getJobInfo(cInfo.getJobId());
JobStatus status = info.getStatus();
boolean failure = status != null && status.equals(JobStatus.FAILURE);
FeedPolicyAccessor fpa = new FeedPolicyAccessor(cInfo.getFeedPolicy());
boolean removeJobHistory = !failure;
boolean retainSubsription =
cInfo.getState().equals(FeedJobState.UNDER_RECOVERY)
|| (failure && fpa.continueOnHardwareFailure());
if (!retainSubsription) {
IFeedJoint feedJoint = cInfo.getSourceFeedJoint();
feedJoint.removeReceiver(connectionId);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(
"Subscription "
+ cInfo.getConnectionId()
+ " completed successfully. Removed subscription");
}
removeFeedJointsPostPipelineTermination(cInfo.getConnectionId());
}
if (removeJobHistory) {
connectJobInfos.remove(connectionId);
jobInfos.remove(cInfo.getJobId());
feedIntakeProgressTrackers.remove(cInfo.getConnectionId());
}
deregisterFeedActivity(cInfo);
// notify event listeners
FeedLifecycleEvent event =
failure ? FeedLifecycleEvent.FEED_COLLECT_FAILURE : FeedLifecycleEvent.FEED_ENDED;
notifyFeedEventSubscribers(cInfo, event);
}
private void setLocations(FeedConnectJobInfo cInfo) {
JobSpecification jobSpec = cInfo.getSpec();
List<OperatorDescriptorId> collectOperatorIds = new ArrayList<OperatorDescriptorId>();
List<OperatorDescriptorId> computeOperatorIds = new ArrayList<OperatorDescriptorId>();
List<OperatorDescriptorId> storageOperatorIds = new ArrayList<OperatorDescriptorId>();
Map<OperatorDescriptorId, IOperatorDescriptor> operators = jobSpec.getOperatorMap();
for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) {
IOperatorDescriptor opDesc = entry.getValue();
IOperatorDescriptor actualOp = null;
if (opDesc instanceof FeedMetaOperatorDescriptor) {
actualOp = ((FeedMetaOperatorDescriptor) opDesc).getCoreOperator();
} else {
actualOp = opDesc;
}
if (actualOp instanceof AlgebricksMetaOperatorDescriptor) {
AlgebricksMetaOperatorDescriptor op = ((AlgebricksMetaOperatorDescriptor) actualOp);
IPushRuntimeFactory[] runtimeFactories = op.getPipeline().getRuntimeFactories();
boolean computeOp = false;
for (IPushRuntimeFactory rf : runtimeFactories) {
if (rf instanceof AssignRuntimeFactory) {
IConnectorDescriptor connDesc =
jobSpec.getOperatorInputMap().get(op.getOperatorId()).get(0);
IOperatorDescriptor sourceOp =
jobSpec
.getConnectorOperatorMap()
.get(connDesc.getConnectorId())
.getLeft()
.getLeft();
if (sourceOp instanceof FeedCollectOperatorDescriptor) {
computeOp = true;
break;
}
}
}
if (computeOp) {
computeOperatorIds.add(entry.getKey());
}
} else if (actualOp instanceof LSMTreeIndexInsertUpdateDeleteOperatorDescriptor) {
storageOperatorIds.add(entry.getKey());
} else if (actualOp instanceof FeedCollectOperatorDescriptor) {
collectOperatorIds.add(entry.getKey());
}
}
try {
IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
JobInfo info = hcc.getJobInfo(cInfo.getJobId());
List<String> collectLocations = new ArrayList<String>();
for (OperatorDescriptorId collectOpId : collectOperatorIds) {
Map<Integer, String> operatorLocations = info.getOperatorLocations().get(collectOpId);
int nOperatorInstances = operatorLocations.size();
for (int i = 0; i < nOperatorInstances; i++) {
collectLocations.add(operatorLocations.get(i));
}
}
List<String> computeLocations = new ArrayList<String>();
for (OperatorDescriptorId computeOpId : computeOperatorIds) {
Map<Integer, String> operatorLocations = info.getOperatorLocations().get(computeOpId);
if (operatorLocations != null) {
int nOperatorInstances = operatorLocations.size();
for (int i = 0; i < nOperatorInstances; i++) {
computeLocations.add(operatorLocations.get(i));
}
} else {
computeLocations.clear();
computeLocations.addAll(collectLocations);
}
}
List<String> storageLocations = new ArrayList<String>();
for (OperatorDescriptorId storageOpId : storageOperatorIds) {
Map<Integer, String> operatorLocations = info.getOperatorLocations().get(storageOpId);
if (operatorLocations == null) {
continue;
}
int nOperatorInstances = operatorLocations.size();
for (int i = 0; i < nOperatorInstances; i++) {
storageLocations.add(operatorLocations.get(i));
}
}
cInfo.setCollectLocations(collectLocations);
cInfo.setComputeLocations(computeLocations);
cInfo.setStorageLocations(storageLocations);
} catch (Exception e) {
e.printStackTrace();
}
}