private synchronized void handleIntakeJobStartMessage(FeedIntakeInfo intakeJobInfo) throws Exception { List<OperatorDescriptorId> intakeOperatorIds = new ArrayList<OperatorDescriptorId>(); Map<OperatorDescriptorId, IOperatorDescriptor> operators = intakeJobInfo.getSpec().getOperatorMap(); for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) { IOperatorDescriptor opDesc = entry.getValue(); if (opDesc instanceof FeedIntakeOperatorDescriptor) { intakeOperatorIds.add(opDesc.getOperatorId()); } } IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc(); JobInfo info = hcc.getJobInfo(intakeJobInfo.getJobId()); List<String> intakeLocations = new ArrayList<String>(); for (OperatorDescriptorId intakeOperatorId : intakeOperatorIds) { Map<Integer, String> operatorLocations = info.getOperatorLocations().get(intakeOperatorId); int nOperatorInstances = operatorLocations.size(); for (int i = 0; i < nOperatorInstances; i++) { intakeLocations.add(operatorLocations.get(i)); } } // intakeLocations is an ordered list; element at position i corresponds to location of i'th // instance of operator intakeJobInfo.setIntakeLocation(intakeLocations); intakeJobInfo.getIntakeFeedJoint().setState(State.ACTIVE); intakeJobInfo.setState(FeedJobState.ACTIVE); // notify event listeners notifyFeedEventSubscribers(intakeJobInfo, FeedLifecycleEvent.FEED_INTAKE_STARTED); }
public static JobId runJob(JobSpecification spec, boolean waitForCompletion) throws Exception { IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc(); JobId jobId = hcc.startJob(spec); if (waitForCompletion) { hcc.waitForCompletion(jobId); } return jobId; }
public static void executeAQL(String aql) throws Exception { AQLParser parser = new AQLParser(new StringReader(aql)); List<Statement> statements; statements = parser.Statement(); SessionConfig pc = new SessionConfig(out, OutputFormat.ADM); AqlTranslator translator = new AqlTranslator(statements, pc); translator.compileAndExecute( AsterixAppContextInfo.getInstance().getHcc(), null, AqlTranslator.ResultDelivery.SYNC); }
private void handleFeedIntakeJobFinishMessage(FeedIntakeInfo intakeInfo, Message message) throws Exception { IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc(); JobInfo info = hcc.getJobInfo(message.jobId); JobStatus status = info.getStatus(); FeedLifecycleEvent event; event = status.equals(JobStatus.FAILURE) ? FeedLifecycleEvent.FEED_INTAKE_FAILURE : FeedLifecycleEvent.FEED_ENDED; // remove feed joints deregisterFeedIntakeJob(message.jobId); // notify event listeners notifyFeedEventSubscribers(intakeInfo, event); }
private void handleFeedCollectJobFinishMessage(FeedConnectJobInfo cInfo) throws Exception { FeedConnectionId connectionId = cInfo.getConnectionId(); IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc(); JobInfo info = hcc.getJobInfo(cInfo.getJobId()); JobStatus status = info.getStatus(); boolean failure = status != null && status.equals(JobStatus.FAILURE); FeedPolicyAccessor fpa = new FeedPolicyAccessor(cInfo.getFeedPolicy()); boolean removeJobHistory = !failure; boolean retainSubsription = cInfo.getState().equals(FeedJobState.UNDER_RECOVERY) || (failure && fpa.continueOnHardwareFailure()); if (!retainSubsription) { IFeedJoint feedJoint = cInfo.getSourceFeedJoint(); feedJoint.removeReceiver(connectionId); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info( "Subscription " + cInfo.getConnectionId() + " completed successfully. Removed subscription"); } removeFeedJointsPostPipelineTermination(cInfo.getConnectionId()); } if (removeJobHistory) { connectJobInfos.remove(connectionId); jobInfos.remove(cInfo.getJobId()); feedIntakeProgressTrackers.remove(cInfo.getConnectionId()); } deregisterFeedActivity(cInfo); // notify event listeners FeedLifecycleEvent event = failure ? FeedLifecycleEvent.FEED_COLLECT_FAILURE : FeedLifecycleEvent.FEED_ENDED; notifyFeedEventSubscribers(cInfo, event); }
private CentralFeedManager() { this.port = AsterixAppContextInfo.getInstance().getFeedProperties().getFeedCentralManagerPort(); this.feedLoadManager = new FeedLoadManager(); this.feedTrackingManager = new FeedTrackingManager(); this.messageListener = new SocketMessageListener(port, new FeedMessageReceiver(this)); }
private void setLocations(FeedConnectJobInfo cInfo) { JobSpecification jobSpec = cInfo.getSpec(); List<OperatorDescriptorId> collectOperatorIds = new ArrayList<OperatorDescriptorId>(); List<OperatorDescriptorId> computeOperatorIds = new ArrayList<OperatorDescriptorId>(); List<OperatorDescriptorId> storageOperatorIds = new ArrayList<OperatorDescriptorId>(); Map<OperatorDescriptorId, IOperatorDescriptor> operators = jobSpec.getOperatorMap(); for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) { IOperatorDescriptor opDesc = entry.getValue(); IOperatorDescriptor actualOp = null; if (opDesc instanceof FeedMetaOperatorDescriptor) { actualOp = ((FeedMetaOperatorDescriptor) opDesc).getCoreOperator(); } else { actualOp = opDesc; } if (actualOp instanceof AlgebricksMetaOperatorDescriptor) { AlgebricksMetaOperatorDescriptor op = ((AlgebricksMetaOperatorDescriptor) actualOp); IPushRuntimeFactory[] runtimeFactories = op.getPipeline().getRuntimeFactories(); boolean computeOp = false; for (IPushRuntimeFactory rf : runtimeFactories) { if (rf instanceof AssignRuntimeFactory) { IConnectorDescriptor connDesc = jobSpec.getOperatorInputMap().get(op.getOperatorId()).get(0); IOperatorDescriptor sourceOp = jobSpec .getConnectorOperatorMap() .get(connDesc.getConnectorId()) .getLeft() .getLeft(); if (sourceOp instanceof FeedCollectOperatorDescriptor) { computeOp = true; break; } } } if (computeOp) { computeOperatorIds.add(entry.getKey()); } } else if (actualOp instanceof LSMTreeIndexInsertUpdateDeleteOperatorDescriptor) { storageOperatorIds.add(entry.getKey()); } else if (actualOp instanceof FeedCollectOperatorDescriptor) { collectOperatorIds.add(entry.getKey()); } } try { IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc(); JobInfo info = hcc.getJobInfo(cInfo.getJobId()); List<String> collectLocations = new ArrayList<String>(); for (OperatorDescriptorId collectOpId : collectOperatorIds) { Map<Integer, String> operatorLocations = info.getOperatorLocations().get(collectOpId); int nOperatorInstances = operatorLocations.size(); for (int i = 0; i < nOperatorInstances; i++) { collectLocations.add(operatorLocations.get(i)); } } List<String> computeLocations = new ArrayList<String>(); for (OperatorDescriptorId computeOpId : computeOperatorIds) { Map<Integer, String> operatorLocations = info.getOperatorLocations().get(computeOpId); if (operatorLocations != null) { int nOperatorInstances = operatorLocations.size(); for (int i = 0; i < nOperatorInstances; i++) { computeLocations.add(operatorLocations.get(i)); } } else { computeLocations.clear(); computeLocations.addAll(collectLocations); } } List<String> storageLocations = new ArrayList<String>(); for (OperatorDescriptorId storageOpId : storageOperatorIds) { Map<Integer, String> operatorLocations = info.getOperatorLocations().get(storageOpId); if (operatorLocations == null) { continue; } int nOperatorInstances = operatorLocations.size(); for (int i = 0; i < nOperatorInstances; i++) { storageLocations.add(operatorLocations.get(i)); } } cInfo.setCollectLocations(collectLocations); cInfo.setComputeLocations(computeLocations); cInfo.setStorageLocations(storageLocations); } catch (Exception e) { e.printStackTrace(); } }