@Test
  public void testProcessEndtimeUpdate() throws Exception {
    scheduleProcess();
    waitForBundleStart(Job.Status.RUNNING);

    ClientResponse response =
        this.service
            .path("api/entities/definition/process/" + processName)
            .header("Remote-User", REMOTE_USER)
            .accept(MediaType.TEXT_XML)
            .get(ClientResponse.class);
    Process process =
        (Process)
            EntityType.PROCESS
                .getUnmarshaller()
                .unmarshal(new StringReader(response.getEntity(String.class)));

    Validity processValidity = process.getClusters().getClusters().get(0).getValidity();
    processValidity.setEnd(new Date(new Date().getTime() + 60 * 60 * 1000));
    File tmpFile = getTempFile();
    EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
    response =
        this.service
            .path("api/entities/update/process/" + processName)
            .header("Remote-User", REMOTE_USER)
            .accept(MediaType.TEXT_XML)
            .post(ClientResponse.class, getServletInputStream(tmpFile.getAbsolutePath()));
    assertSuccessful(response);

    // Assert that update does not create new bundle
    List<BundleJob> bundles = getBundles();
    Assert.assertEquals(bundles.size(), 1);
  }
  /**
   * Tests should be enabled only in local environments as they need running instance of webserver
   */
  @Test
  public void testUpdateCheckUser() throws Exception {
    Map<String, String> overlay = getUniqueOverlay();
    String tmpFileName = overlayParametersOverTemplate(PROCESS_TEMPLATE, overlay);
    Process process =
        (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(new File(tmpFileName));
    Validity processValidity = process.getClusters().getClusters().get(0).getValidity();
    processValidity.setEnd(new Date(new Date().getTime() + 2 * 24 * 60 * 60 * 1000));
    File tmpFile = getTempFile();
    EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
    scheduleProcess(tmpFile.getAbsolutePath(), overlay);
    waitForBundleStart(Status.RUNNING);

    List<BundleJob> bundles = getBundles();
    Assert.assertEquals(bundles.size(), 1);
    Assert.assertEquals(bundles.get(0).getUser(), REMOTE_USER);

    ClientResponse response =
        this.service
            .path("api/entities/definition/feed/" + outputFeedName)
            .header("Remote-User", REMOTE_USER)
            .accept(MediaType.TEXT_XML)
            .get(ClientResponse.class);
    Feed feed =
        (Feed)
            EntityType.FEED
                .getUnmarshaller()
                .unmarshal(new StringReader(response.getEntity(String.class)));

    // change output feed path and update feed as another user
    feed.getLocations()
        .getLocations()
        .get(0)
        .setPath("/falcon/test/output2/${YEAR}/${MONTH}/${DAY}");
    tmpFile = getTempFile();
    EntityType.FEED.getMarshaller().marshal(feed, tmpFile);
    response =
        this.service
            .path("api/entities/update/feed/" + outputFeedName)
            .header("Remote-User", "testuser")
            .accept(MediaType.TEXT_XML)
            .post(ClientResponse.class, getServletInputStream(tmpFile.getAbsolutePath()));
    assertSuccessful(response);

    bundles = getBundles();
    Assert.assertEquals(bundles.size(), 2);
    Assert.assertEquals(bundles.get(0).getUser(), REMOTE_USER);
    Assert.assertEquals(bundles.get(1).getUser(), REMOTE_USER);
  }
  @Test
  public void testProcessInputUpdate() throws Exception {
    scheduleProcess();
    waitForBundleStart(Job.Status.RUNNING);

    ClientResponse response =
        this.service
            .path("api/entities/definition/process/" + processName)
            .header("Remote-User", REMOTE_USER)
            .accept(MediaType.TEXT_XML)
            .get(ClientResponse.class);
    Process process =
        (Process)
            EntityType.PROCESS
                .getUnmarshaller()
                .unmarshal(new StringReader(response.getEntity(String.class)));

    String feed3 = "f3" + System.currentTimeMillis();
    Map<String, String> overlay = new HashMap<String, String>();
    overlay.put("inputFeedName", feed3);
    overlay.put("cluster", clusterName);
    response = submitToFalcon(FEED_TEMPLATE1, overlay, EntityType.FEED);
    assertSuccessful(response);

    Input input = new Input();
    input.setFeed(feed3);
    input.setName("inputData2");
    input.setStart("today(20,0)");
    input.setEnd("today(20,20)");
    process.getInputs().getInputs().add(input);

    Validity processValidity = process.getClusters().getClusters().get(0).getValidity();
    processValidity.setEnd(new Date(new Date().getTime() + 2 * 24 * 60 * 60 * 1000));
    File tmpFile = getTempFile();
    EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
    response =
        this.service
            .path("api/entities/update/process/" + processName)
            .header("Remote-User", REMOTE_USER)
            .accept(MediaType.TEXT_XML)
            .post(ClientResponse.class, getServletInputStream(tmpFile.getAbsolutePath()));
    assertSuccessful(response);

    // Assert that update creates new bundle
    List<BundleJob> bundles = getBundles();
    Assert.assertEquals(bundles.size(), 2);
  }
  @Test
  public void testProcessDeleteAndSchedule() throws Exception {
    // Submit process with invalid property so that coord submit fails and bundle goes to failed
    // state
    Map<String, String> overlay = getUniqueOverlay();
    String tmpFileName = overlayParametersOverTemplate(PROCESS_TEMPLATE, overlay);
    Process process =
        (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(new File(tmpFileName));
    Property prop = new Property();
    prop.setName("newProp");
    prop.setValue("${formatTim()}");
    process.getProperties().getProperties().add(prop);
    File tmpFile = getTempFile();
    EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
    scheduleProcess(tmpFile.getAbsolutePath(), overlay);
    waitForBundleStart(Status.FAILED);

    // Delete and re-submit the process with correct workflow
    ClientResponse clientRepsonse =
        this.service
            .path("api/entities/delete/process/" + processName)
            .header("Remote-User", REMOTE_USER)
            .accept(MediaType.TEXT_XML)
            .delete(ClientResponse.class);
    assertSuccessful(clientRepsonse);
    process.getWorkflow().setPath("/falcon/test/workflow");
    tmpFile = getTempFile();
    EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
    clientRepsonse =
        this.service
            .path("api/entities/submitAndSchedule/process")
            .header("Remote-User", REMOTE_USER)
            .accept(MediaType.TEXT_XML)
            .type(MediaType.TEXT_XML)
            .post(ClientResponse.class, getServletInputStream(tmpFile.getAbsolutePath()));
    assertSuccessful(clientRepsonse);

    // Assert that new schedule creates new bundle
    List<BundleJob> bundles = getBundles();
    Assert.assertEquals(bundles.size(), 2);
  }
Exemplo n.º 5
0
  @Test
  public void testValidateForInvalidEntityXML() throws IOException {
    ServletInputStream invalidProcessXML = getServletInputStream(SAMPLE_INVALID_PROCESS_XML);
    when(mockHttpServletRequest.getInputStream()).thenReturn(invalidProcessXML);

    try {
      validate(mockHttpServletRequest, EntityType.PROCESS.name(), false);
      Assert.fail("Invalid entity type was accepted by the system");
    } catch (FalconWebException ignore) {
      // ignore
    }
  }
  @Test(enabled = false)
  public void testOptionalInput() throws Exception {
    Map<String, String> overlay = getUniqueOverlay();
    String tmpFileName = overlayParametersOverTemplate(PROCESS_TEMPLATE, overlay);
    Process process =
        (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(new File(tmpFileName));

    Input in1 = process.getInputs().getInputs().get(0);
    Input in2 = new Input();
    in2.setFeed(in1.getFeed());
    in2.setName("input2");
    in2.setOptional(true);
    in2.setPartition(in1.getPartition());
    in2.setStart("now(-1,0)");
    in2.setEnd("now(0,0)");
    process.getInputs().getInputs().add(in2);

    File tmpFile = getTempFile();
    EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
    scheduleProcess(tmpFile.getAbsolutePath(), overlay);
    waitForWorkflowStart(processName);
  }
  private static org.apache.falcon.entity.v0.process.Process bindAttributesInTemplate(
      final String processTemplate,
      final Properties extensionProperties,
      final String extensionName,
      final String wfPath,
      final String wfLibPath)
      throws FalconException {
    if (StringUtils.isBlank(processTemplate) || extensionProperties == null) {
      throw new FalconException("Process template or properties cannot be null");
    }

    org.apache.falcon.entity.v0.process.Process process;
    try {
      Unmarshaller unmarshaller = EntityType.PROCESS.getUnmarshaller();
      // Validation can be skipped for unmarshalling as we want to bind template with the
      // properties.
      // Vaildation is handled as part of marshalling
      unmarshaller.setSchema(null);
      unmarshaller.setEventHandler(
          new ValidationEventHandler() {
            public boolean handleEvent(ValidationEvent validationEvent) {
              return true;
            }
          });
      process =
          (org.apache.falcon.entity.v0.process.Process)
              unmarshaller.unmarshal(new StringReader(processTemplate));
    } catch (Exception e) {
      throw new FalconException(e);
    }

    /* For optional properties user might directly set them in the process xml and might not set it in properties
       file. Before doing the submission validation is done to confirm process xml doesn't have
       EXTENSION_VAR_PATTERN
    */

    String processName = extensionProperties.getProperty(ExtensionProperties.JOB_NAME.getName());
    if (StringUtils.isNotEmpty(processName)) {
      process.setName(processName);
    }

    // DR process template has only one cluster
    bindClusterProperties(process.getClusters().getClusters().get(0), extensionProperties);

    // bind scheduling properties
    String processFrequency =
        extensionProperties.getProperty(ExtensionProperties.FREQUENCY.getName());
    if (StringUtils.isNotEmpty(processFrequency)) {
      process.setFrequency(Frequency.fromString(processFrequency));
    }

    String zone = extensionProperties.getProperty(ExtensionProperties.TIMEZONE.getName());
    if (StringUtils.isNotBlank(zone)) {
      process.setTimezone(TimeZone.getTimeZone(zone));
    } else {
      process.setTimezone(TimeZone.getTimeZone("UTC"));
    }

    bindWorkflowProperties(process.getWorkflow(), extensionName, wfPath, wfLibPath);
    bindRetryProperties(process.getRetry(), extensionProperties);
    bindNotificationProperties(process.getNotification(), extensionProperties);
    bindACLProperties(process.getACL(), extensionProperties);
    bindTagsProperties(process, extensionProperties);
    bindCustomProperties(process.getProperties(), extensionProperties);

    return process;
  }