public void registerFeedIntakeJob(FeedId feedId, JobId jobId, JobSpecification jobSpec)
throws HyracksDataException {
if (jobInfos.get(jobId) != null) {
throw new IllegalStateException("Feed job already registered");
}
List<IFeedJoint> joints = feedPipeline.get(feedId);
IFeedJoint intakeJoint = null;
for (IFeedJoint joint : joints) {
if (joint.getType().equals(IFeedJoint.FeedJointType.INTAKE)) {
intakeJoint = joint;
break;
}
}
if (intakeJoint != null) {
FeedIntakeInfo intakeJobInfo =
new FeedIntakeInfo(
jobId,
FeedJobState.CREATED,
FeedJobInfo.JobType.INTAKE,
feedId,
intakeJoint,
jobSpec);
intakeJobInfos.put(feedId, intakeJobInfo);
jobInfos.put(jobId, intakeJobInfo);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Registered feed intake [" + jobId + "]" + " for feed " + feedId);
}
} else {
throw new HyracksDataException(
"Could not register feed intake job [" + jobId + "]" + " for feed " + feedId);
}
}
public IFeedJoint getAvailableFeedJoint(FeedJointKey feedJointKey) {
IFeedJoint feedJoint = getFeedJoint(feedJointKey);
if (feedJoint != null) {
return feedJoint;
} else {
String jointKeyString = feedJointKey.getStringRep();
List<IFeedJoint> jointsOnPipeline = feedPipeline.get(feedJointKey.getFeedId());
IFeedJoint candidateJoint = null;
if (jointsOnPipeline != null) {
for (IFeedJoint joint : jointsOnPipeline) {
if (jointKeyString.contains(joint.getFeedJointKey().getStringRep())) {
if (candidateJoint == null) {
candidateJoint = joint;
} else if (joint
.getFeedJointKey()
.getStringRep()
.contains(
candidateJoint
.getFeedJointKey()
.getStringRep())) { // found feed point is a super set of the earlier find
candidateJoint = joint;
}
}
}
}
return candidateJoint;
}
}
public IFeedJoint getFeedPoint(FeedId sourceFeedId, IFeedJoint.FeedJointType type) {
List<IFeedJoint> joints = feedPipeline.get(sourceFeedId);
for (IFeedJoint joint : joints) {
if (joint.getType().equals(type)) {
return joint;
}
}
return null;
}
public List<String> getFeedComputeLocations(FeedId feedId) {
List<IFeedJoint> feedJoints = feedPipeline.get(feedId);
for (IFeedJoint joint : feedJoints) {
if (joint.getFeedJointKey().getFeedId().equals(feedId)) {
return connectJobInfos.get(joint.getProvider()).getComputeLocations();
}
}
return null;
}
public IFeedJoint getFeedJoint(FeedJointKey feedPointKey) {
List<IFeedJoint> joints = feedPipeline.get(feedPointKey.getFeedId());
if (joints != null && !joints.isEmpty()) {
for (IFeedJoint joint : joints) {
if (joint.getFeedJointKey().equals(feedPointKey)) {
return joint;
}
}
}
return null;
}
public boolean isFeedPointAvailable(FeedJointKey feedJointKey) {
List<IFeedJoint> joints = feedPipeline.get(feedJointKey.getFeedId());
if (joints != null && !joints.isEmpty()) {
for (IFeedJoint joint : joints) {
if (joint.getFeedJointKey().equals(feedJointKey)) {
return true;
}
}
}
return false;
}
public void registerFeedJoint(IFeedJoint feedJoint) {
List<IFeedJoint> feedJointsOnPipeline = feedPipeline.get(feedJoint.getOwnerFeedId());
if (feedJointsOnPipeline == null) {
feedJointsOnPipeline = new ArrayList<IFeedJoint>();
feedPipeline.put(feedJoint.getOwnerFeedId(), feedJointsOnPipeline);
feedJointsOnPipeline.add(feedJoint);
} else {
if (!feedJointsOnPipeline.contains(feedJoint)) {
feedJointsOnPipeline.add(feedJoint);
} else {
throw new IllegalArgumentException("Feed joint " + feedJoint + " already registered");
}
}
}
public void registerFeedCollectionJob(
FeedId sourceFeedId,
FeedConnectionId connectionId,
JobId jobId,
JobSpecification jobSpec,
Map<String, String> feedPolicy) {
if (jobInfos.get(jobId) != null) {
throw new IllegalStateException("Feed job already registered");
}
List<IFeedJoint> feedJoints = feedPipeline.get(sourceFeedId);
FeedConnectionId cid = null;
IFeedJoint sourceFeedJoint = null;
for (IFeedJoint joint : feedJoints) {
cid = joint.getReceiver(connectionId);
if (cid != null) {
sourceFeedJoint = joint;
break;
}
}
if (cid != null) {
FeedConnectJobInfo cInfo =
new FeedConnectJobInfo(
jobId,
FeedJobState.CREATED,
connectionId,
sourceFeedJoint,
null,
jobSpec,
feedPolicy);
jobInfos.put(jobId, cInfo);
connectJobInfos.put(connectionId, cInfo);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Registered feed connection [" + jobId + "]" + " for feed " + connectionId);
}
} else {
if (LOGGER.isLoggable(Level.WARNING)) {
LOGGER.warning(
"Could not register feed collection job ["
+ jobId
+ "]"
+ " for feed connection "
+ connectionId);
}
}
}
public void removeFeedJointsPostPipelineTermination(FeedConnectionId connectionId) {
FeedConnectJobInfo cInfo = connectJobInfos.get(connectionId);
List<IFeedJoint> feedJoints = feedPipeline.get(connectionId.getFeedId());
IFeedJoint sourceJoint = cInfo.getSourceFeedJoint();
List<FeedConnectionId> all = sourceJoint.getReceivers();
boolean removeSourceJoint = all.size() < 2;
if (removeSourceJoint) {
feedJoints.remove(sourceJoint);
}
IFeedJoint computeJoint = cInfo.getComputeFeedJoint();
if (computeJoint != null && computeJoint.getReceivers().size() < 2) {
feedJoints.remove(computeJoint);
}
}
public synchronized void submitFeedConnectionRequest(
IFeedJoint feedJoint, final FeedConnectionRequest request) throws Exception {
List<String> locations = null;
switch (feedJoint.getType()) {
case INTAKE:
FeedIntakeInfo intakeInfo = intakeJobInfos.get(feedJoint.getOwnerFeedId());
locations = intakeInfo.getIntakeLocation();
break;
case COMPUTE:
FeedConnectionId connectionId = feedJoint.getProvider();
FeedConnectJobInfo cInfo = connectJobInfos.get(connectionId);
locations = cInfo.getComputeLocations();
break;
}
SubscribeFeedWork work = new SubscribeFeedWork(locations.toArray(new String[] {}), request);
FeedWorkManager.INSTANCE.submitWork(
work, new SubscribeFeedWork.FeedSubscribeWorkEventListener());
}
private void handleCollectJobStartMessage(FeedConnectJobInfo cInfo)
throws RemoteException, ACIDException {
// set locations of feed sub-operations (intake, compute, store)
setLocations(cInfo);
// activate joints
List<IFeedJoint> joints = feedPipeline.get(cInfo.getConnectionId().getFeedId());
for (IFeedJoint joint : joints) {
if (joint.getProvider().equals(cInfo.getConnectionId())) {
joint.setState(State.ACTIVE);
if (joint.getType().equals(IFeedJoint.FeedJointType.COMPUTE)) {
cInfo.setComputeFeedJoint(joint);
}
}
}
cInfo.setState(FeedJobState.ACTIVE);
// register activity in metadata
registerFeedActivity(cInfo);
// notify event listeners
notifyFeedEventSubscribers(cInfo, FeedLifecycleEvent.FEED_COLLECT_STARTED);
}
private void handleFeedCollectJobFinishMessage(FeedConnectJobInfo cInfo) throws Exception {
FeedConnectionId connectionId = cInfo.getConnectionId();
IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
JobInfo info = hcc.getJobInfo(cInfo.getJobId());
JobStatus status = info.getStatus();
boolean failure = status != null && status.equals(JobStatus.FAILURE);
FeedPolicyAccessor fpa = new FeedPolicyAccessor(cInfo.getFeedPolicy());
boolean removeJobHistory = !failure;
boolean retainSubsription =
cInfo.getState().equals(FeedJobState.UNDER_RECOVERY)
|| (failure && fpa.continueOnHardwareFailure());
if (!retainSubsription) {
IFeedJoint feedJoint = cInfo.getSourceFeedJoint();
feedJoint.removeReceiver(connectionId);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(
"Subscription "
+ cInfo.getConnectionId()
+ " completed successfully. Removed subscription");
}
removeFeedJointsPostPipelineTermination(cInfo.getConnectionId());
}
if (removeJobHistory) {
connectJobInfos.remove(connectionId);
jobInfos.remove(cInfo.getJobId());
feedIntakeProgressTrackers.remove(cInfo.getConnectionId());
}
deregisterFeedActivity(cInfo);
// notify event listeners
FeedLifecycleEvent event =
failure ? FeedLifecycleEvent.FEED_COLLECT_FAILURE : FeedLifecycleEvent.FEED_ENDED;
notifyFeedEventSubscribers(cInfo, event);
}
