// Populates the InstancesResult.Instance instance using ExecutionInstance private void populateInstanceInfo( InstancesResult.Instance instanceInfo, ExecutionInstance instance) throws StateStoreException { instanceInfo.cluster = instance.getCluster(); InstanceState.STATE state = STATE_STORE.getExecutionInstance(instance.getId()).getCurrentState(); switch (state) { case SUCCEEDED: instanceInfo.status = InstancesResult.WorkflowStatus.SUCCEEDED; break; case FAILED: instanceInfo.status = InstancesResult.WorkflowStatus.FAILED; break; case KILLED: instanceInfo.status = InstancesResult.WorkflowStatus.KILLED; break; case READY: instanceInfo.status = InstancesResult.WorkflowStatus.READY; break; case WAITING: instanceInfo.status = InstancesResult.WorkflowStatus.WAITING; break; case SUSPENDED: instanceInfo.status = InstancesResult.WorkflowStatus.SUSPENDED; break; case RUNNING: instanceInfo.status = InstancesResult.WorkflowStatus.RUNNING; break; default: instanceInfo.status = InstancesResult.WorkflowStatus.UNDEFINED; break; } // Mask wfParams by default instanceInfo.wfParams = null; }
@Override public InstancesResult getRunningInstances(Entity entity, List<LifeCycle> lifeCycles) throws FalconException { Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity); List<InstancesResult.Instance> runInstances = new ArrayList<>(); for (String cluster : clusters) { Collection<InstanceState> instances = STATE_STORE.getExecutionInstances(entity, cluster, InstanceState.getRunningStates()); for (InstanceState state : instances) { String instanceTimeStr = state.getInstance().getInstanceTime().toString(); InstancesResult.Instance instance = new InstancesResult.Instance( cluster, instanceTimeStr, InstancesResult.WorkflowStatus.RUNNING); instance.startTime = state.getInstance().getActualStart().toDate(); runInstances.add(instance); } } InstancesResult result = new InstancesResult(APIResult.Status.SUCCEEDED, "Running Instances"); result.setInstances(runInstances.toArray(new InstancesResult.Instance[runInstances.size()])); return result; }
/** Test the list feed instances api using an orderBy parameter. Check the order. */ @Test public void testFeedOrderBy() throws URISyntaxException, OozieClientException, JAXBException, AuthenticationException, IOException, InterruptedException { SoftAssert softAssert = new SoftAssert(); // orderBy start time InstancesResult r = prism.getFeedHelper().listInstances(feedName, "orderBy=startTime&sortOrder=desc", null); InstancesResult.Instance[] instances = r.getInstances(); Date previousDate = new Date(); for (InstancesResult.Instance instance : instances) { Date current = instance.getStartTime(); if (current != null) { // e.g if instance is WAITING it doesn't have start time softAssert.assertTrue( current.before(previousDate) || current.equals(previousDate), "Wrong order. Current startTime :" + current + " Previous: " + previousDate); previousDate = (Date) current.clone(); } } // orderBy status r = prism .getFeedHelper() .listInstances( feedName, "start=" + startTime + "&numResults=12&orderBy=status&sortOrder=desc", null); InstanceUtil.validateResponse(r, 12, 1, 1, 4, 6); instances = r.getInstances(); InstancesResult.WorkflowStatus previousStatus = InstancesResult.WorkflowStatus.WAITING; for (InstancesResult.Instance instance : instances) { InstancesResult.WorkflowStatus current = instance.getStatus(); softAssert.assertTrue( current.toString().compareTo(previousStatus.toString()) <= 0, "Wrong order. Compared " + current + " and " + previousStatus + " statuses."); previousStatus = current; } // sort by endTime r = prism .getFeedHelper() .listInstances( feedName, "start=" + startTime + "&numResults=12&orderBy=endTime&sortOrder=desc", null); instances = r.getInstances(); previousDate = new Date(); for (InstancesResult.Instance instance : instances) { Date current = instance.getEndTime(); if (current != null) { // e.g if instance is WAITING it doesn't have end time softAssert.assertTrue( current.before(previousDate) || current.equals(previousDate), "Wrong order. Current startTime :" + current + " Previous: " + previousDate); previousDate = (Date) current.clone(); } } softAssert.assertAll(); }
private InstancesResult.Instance performAction( String cluster, Entity entity, JobAction action, ExecutionInstance instance, Properties userProps, boolean isForced) throws FalconException { EntityExecutor executor = EXECUTION_SERVICE.getEntityExecutor(entity, cluster); InstancesResult.Instance instanceInfo = null; LOG.debug("Retrieving information for {} for action {}", instance.getId(), action); if (StringUtils.isNotEmpty(instance.getExternalID())) { instanceInfo = DAGEngineFactory.getDAGEngine(cluster).info(instance.getExternalID()); } else { instanceInfo = new InstancesResult.Instance(); } switch (action) { case KILL: executor.kill(instance); populateInstanceInfo(instanceInfo, instance); break; case SUSPEND: executor.suspend(instance); populateInstanceInfo(instanceInfo, instance); break; case RESUME: executor.resume(instance); populateInstanceInfo(instanceInfo, instance); break; case RERUN: executor.rerun(instance, userProps, isForced); populateInstanceInfo(instanceInfo, instance); break; case STATUS: // Mask wfParams instanceInfo.wfParams = null; if (StringUtils.isNotEmpty(instance.getExternalID())) { List<InstancesResult.InstanceAction> instanceActions = DAGEngineFactory.getDAGEngine(cluster).getJobDetails(instance.getExternalID()); instanceInfo.actions = instanceActions.toArray(new InstancesResult.InstanceAction[instanceActions.size()]); // If not scheduled externally yet, get details from state } else { populateInstanceInfo(instanceInfo, instance); } break; case PARAMS: // Mask details, log instanceInfo.details = null; instanceInfo.logFile = null; Properties props = DAGEngineFactory.getDAGEngine(cluster).getConfiguration(instance.getExternalID()); InstancesResult.KeyValuePair[] keyValuePairs = new InstancesResult.KeyValuePair[props.size()]; int i = 0; for (String name : props.stringPropertyNames()) { keyValuePairs[i++] = new InstancesResult.KeyValuePair(name, props.getProperty(name)); } instanceInfo.wfParams = keyValuePairs; break; default: throw new IllegalArgumentException("Unhandled action " + action); } return instanceInfo; }
private InstancesResult doJobAction( JobAction action, Entity entity, Date start, Date end, Properties props, List<LifeCycle> lifeCycles, boolean isForced) throws FalconException { Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity); List<String> clusterList = getIncludedClusters(props, FALCON_INSTANCE_ACTION_CLUSTERS); APIResult.Status overallStatus = APIResult.Status.SUCCEEDED; int instanceCount = 0; Collection<InstanceState.STATE> states; switch (action) { case KILL: case SUSPEND: states = InstanceState.getActiveStates(); break; case RESUME: states = new ArrayList<>(); states.add(InstanceState.STATE.SUSPENDED); break; case PARAMS: // Applicable only for running and finished jobs. states = InstanceState.getRunningStates(); states.addAll(InstanceState.getTerminalStates()); states.add(InstanceState.STATE.SUSPENDED); break; case STATUS: states = InstanceState.getActiveStates(); states.addAll(InstanceState.getTerminalStates()); states.add(InstanceState.STATE.SUSPENDED); break; case RERUN: // Applicable only for terminated States states = InstanceState.getTerminalStates(); break; default: throw new IllegalArgumentException("Unhandled action " + action); } List<ExecutionInstance> instancesToActOn = new ArrayList<>(); for (String cluster : clusters) { if (clusterList.size() != 0 && !clusterList.contains(cluster)) { continue; } LOG.debug("Retrieving instances for cluster : {} for action {}", cluster, action); Collection<InstanceState> instances = STATE_STORE.getExecutionInstances( entity, cluster, states, new DateTime(start), new DateTime(end)); for (InstanceState state : instances) { instancesToActOn.add(state.getInstance()); } } // To ensure compatibility with OozieWorkflowEngine. // Also because users would like to see the most recent instances first. sortInstancesDescBySequence(instancesToActOn); List<InstancesResult.Instance> instances = new ArrayList<>(); for (ExecutionInstance ins : instancesToActOn) { instanceCount++; String instanceTimeStr = SchemaHelper.formatDateUTC(ins.getInstanceTime().toDate()); InstancesResult.Instance instance = null; try { instance = performAction(ins.getCluster(), entity, action, ins, props, isForced); instance.instance = instanceTimeStr; } catch (FalconException e) { LOG.warn("Unable to perform action {} on cluster", action, e); instance = new InstancesResult.Instance(ins.getCluster(), instanceTimeStr, null); instance.status = InstancesResult.WorkflowStatus.ERROR; instance.details = e.getMessage(); overallStatus = APIResult.Status.PARTIAL; } instances.add(instance); } if (instanceCount < 2 && overallStatus == APIResult.Status.PARTIAL) { overallStatus = APIResult.Status.FAILED; } InstancesResult instancesResult = new InstancesResult(overallStatus, action.name()); instancesResult.setInstances(instances.toArray(new InstancesResult.Instance[instances.size()])); return instancesResult; }
/** * Test list feed instances using custom filter. Expecting list of feed instances which satisfy * custom filters. */ @Test public void testFeedCustomFilter() throws URISyntaxException, IOException, AuthenticationException, InterruptedException { String params = "start=" + startTime + "&filterBy=status:RUNNING"; InstancesResult r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateResponse(r, 1, 1, 0, 0, 0); params = "start=" + startTime + "&end=" + endTime + "&filterBy=status:RUNNING&offset=2"; r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateSuccessWOInstances(r); params = "start=" + startTime + "&end=" + endTime + "&filterBy=status:WAITING"; r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateResponse(r, 4, 0, 0, 4, 0); params = "start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 41) + "&filterBy=status:WAITING"; r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateResponse(r, 3, 0, 0, 3, 0); params = "start=" + startTime + "&offset=1&numResults=1&filterBy=status:WAITING"; r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateResponse(r, 1, 0, 0, 1, 0); params = "start=" + TimeUtil.addMinsToTime(startTime, 16) + "&offset=2&numResults=12"; r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateResponse(r, 6, 0, 1, 3, 2); String sourceCluster = bundles[0].getClusterNames().get(0); String clusterName = bundles[1].getClusterNames().get(0); params = "start=" + startTime + "&filterBy=STATUS:KILLED,CLUSTER:" + clusterName + "&numResults=5&orderBy=startTime&sortOrder=desc"; r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateResponse(r, 5, 0, 0, 0, 5); // should be ordered by a start time SoftAssert softAssert = new SoftAssert(); InstancesResult.Instance[] instances = r.getInstances(); Date previousDate = new Date(); for (InstancesResult.Instance instance : instances) { Date current = instance.getStartTime(); softAssert.assertNotNull(current, "Start time shouldn't be null for KILLED instance."); softAssert.assertTrue( current.before(previousDate) || current.equals(previousDate), "Wrong order. Current startTime :" + current + " Previous: " + previousDate); previousDate = (Date) current.clone(); } softAssert.assertAll(); // missing 1st, 11th, 12th instances, all other instances should be retrieved. params = "start=" + TimeUtil.addMinsToTime(startTime, 2) + "&offset=2"; r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateResponse(r, 9, 0, 1, 3, 5); // missing the 1st, 11th, 12th instance, all instances which have progressed should be present: // 5 killed + 1 suspended, but numResults=5, so expecting 1 suspended and 4 killed instances. params = "start=" + TimeUtil.addMinsToTime(startTime, 2) + "&filterBy=SOURCECLUSTER:" + sourceCluster + "&offset=1&numResults=5"; r = prism.getFeedHelper().listInstances(feedName, params, null); InstanceUtil.validateResponse(r, 5, 0, 1, 0, 4); }