@Test
  public void testLoadXml() throws Exception {

    OozieJobExecutorJobEntry jobEntry = new OozieJobExecutorJobEntry();
    OozieJobExecutorConfig jobConfig = new OozieJobExecutorConfig();

    jobConfig.setOozieWorkflow("hdfs://localhost:9000/user/test-user/oozie/workflow.xml");
    jobConfig.setOozieWorkflowConfig("file:///User/test-user/oozie/job.properties");
    jobConfig.setOozieUrl("http://localhost:11000/oozie");

    jobEntry.setJobConfig(jobConfig);

    JobEntryCopy jec = new JobEntryCopy(jobEntry);
    jec.setLocation(0, 0);
    String xml = jec.getXML();

    Document d = XMLHandler.loadXMLString(xml);

    OozieJobExecutorJobEntry jobEntry2 = new OozieJobExecutorJobEntry();
    jobEntry2.loadXML(d.getDocumentElement(), null, null, null);

    OozieJobExecutorConfig jobConfig2 = jobEntry2.getJobConfig();
    assertEquals(jobConfig.getOozieWorkflow(), jobConfig2.getOozieWorkflow());
    assertEquals(jobConfig.getOozieWorkflowConfig(), jobConfig2.getOozieWorkflowConfig());
    assertEquals(jobConfig.getOozieUrl(), jobConfig2.getOozieUrl());
  }
  @Override
  public boolean jobElementRead(String xml, RepositoryImportFeedbackInterface feedback) {
    try {
      Document doc = XMLHandler.loadXMLString(getOrCreateDb(), xml);
      Node jobNode = XMLHandler.getSubNode(doc, RepositoryExportSaxParser.STRING_JOB);
      if (!importJob(jobNode, feedback)) {
        return false;
      }
      jobNumber++;
    } catch (Exception e) {
      // Some unexpected error occurred during job import
      // This is usually a problem with a missing plugin or something
      // like that...
      //
      showError(
          BaseMessages.getString(PKG, "RepositoryImporter.UnexpectedErrorDuringJobImport.Title"),
          BaseMessages.getString(PKG, "RepositoryImporter.UnexpectedErrorDuringJobImport.Message"),
          e);

      if (!feedback.askContinueOnErrorQuestion(
          BaseMessages.getString(PKG, "RepositoryImporter.DoYouWantToContinue.Title"),
          BaseMessages.getString(PKG, "RepositoryImporter.DoYouWantToContinue.Message"))) {
        return false;
      }
    }
    return true;
  }
  public void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    if (!request.getRequestURI().equals(CONTEXT_PATH + "/")) return;

    if (log.isDebug()) log.logDebug(toString(), "Slave Server registration requested");

    PrintWriter out = response.getWriter();
    BufferedReader in = request.getReader();
    if (log.isDetailed())
      log.logDetailed(toString(), "Encoding: " + request.getCharacterEncoding());

    // We always use XML to reply here...
    //
    response.setContentType("text/xml");
    out.print(XMLHandler.getXMLHeader());
    response.setStatus(HttpServletResponse.SC_OK);

    try {
      // First read the slave server information in memory from the request
      //
      StringBuilder xml = new StringBuilder(request.getContentLength());
      int c;
      while ((c = in.read()) != -1) {
        xml.append((char) c);
      }

      // Parse the XML, create a transformation configuration
      //
      Document document = XMLHandler.loadXMLString(xml.toString());
      Node node = XMLHandler.getSubNode(document, SlaveServerDetection.XML_TAG);
      SlaveServerDetection slaveServerDetection = new SlaveServerDetection(node);

      // See if this slave server is already in our list...
      //
      String message;
      int index = detections.indexOf(slaveServerDetection);
      if (index < 0) {
        detections.add(slaveServerDetection);
        message =
            "Slave server detection '"
                + slaveServerDetection.getSlaveServer().getName()
                + "' was replaced in the list.";
      } else {
        // replace the data in the old one...
        //
        SlaveServerDetection old = detections.get(index);
        old.setSlaveServer(slaveServerDetection.getSlaveServer());
        old.setActive(slaveServerDetection.isActive());

        // Note: in case it's not the slave server itself doing the sending, it might be possible
        // for it to be inactive...
        //
        if (old.isActive()) {
          old.setLastActiveDate(slaveServerDetection.getLastActiveDate());
        } else {
          old.setLastInactiveDate(slaveServerDetection.getLastInactiveDate());
        }
        message =
            "Slave server detection '"
                + slaveServerDetection.getSlaveServer().getName()
                + "' was added to the list.";
      }

      out.println(new WebResult(WebResult.STRING_OK, message));
    } catch (Exception ex) {
      out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
    }
  }
Esempio n. 4
0
  public void dataNodeToElement(final DataNode rootNode, final RepositoryElementInterface element)
      throws KettleException {
    TransMeta transMeta = (TransMeta) element;

    List<String> privateTransformationDatabases = null;
    // read the private databases
    DataNode privateDatabases = rootNode.getNode(NODE_TRANS_PRIVATE_DATABASES);
    // if we have node than we use new format we could remove unexpected node
    if (privateDatabases != null) {
      privateTransformationDatabases = new ArrayList<String>();
      for (DataNode privateDatabase : privateDatabases.getNodes()) {
        privateTransformationDatabases.add(privateDatabase.getName());
      }
    }
    transMeta.setPrivateTransformationDatabases(privateTransformationDatabases);

    // read the steps...
    //
    DataNode stepsNode = rootNode.getNode(NODE_STEPS);
    for (DataNode stepNode : stepsNode.getNodes()) {

      StepMeta stepMeta = new StepMeta(new StringObjectId(stepNode.getId().toString()));
      stepMeta.setParentTransMeta(transMeta); // for tracing, retain hierarchy

      // Read the basics
      //
      stepMeta.setName(getString(stepNode, PROP_NAME));
      if (stepNode.hasProperty(PROP_DESCRIPTION)) {
        stepMeta.setDescription(getString(stepNode, PROP_DESCRIPTION));
      }
      stepMeta.setDistributes(stepNode.getProperty(PROP_STEP_DISTRIBUTE).getBoolean());
      DataProperty rowDistributionProperty = stepNode.getProperty(PROP_STEP_ROW_DISTRIBUTION);
      String rowDistributionCode =
          rowDistributionProperty == null ? null : rowDistributionProperty.getString();
      RowDistributionInterface rowDistribution =
          PluginRegistry.getInstance()
              .loadClass(
                  RowDistributionPluginType.class,
                  rowDistributionCode,
                  RowDistributionInterface.class);
      stepMeta.setRowDistribution(rowDistribution);
      stepMeta.setDraw(stepNode.getProperty(PROP_STEP_GUI_DRAW).getBoolean());
      int copies = (int) stepNode.getProperty(PROP_STEP_COPIES).getLong();
      String copiesString =
          stepNode.getProperty(PROP_STEP_COPIES_STRING) != null
              ? stepNode.getProperty(PROP_STEP_COPIES_STRING).getString()
              : StringUtils.EMPTY;
      if (!Const.isEmpty(copiesString)) {
        stepMeta.setCopiesString(copiesString);
      } else {
        stepMeta.setCopies(copies); // for backward compatibility
      }

      int x = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_X).getLong();
      int y = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_Y).getLong();
      stepMeta.setLocation(x, y);

      // Load the group attributes map
      //
      AttributesMapUtil.loadAttributesMap(stepNode, stepMeta);

      String stepType = getString(stepNode, PROP_STEP_TYPE);

      // Create a new StepMetaInterface object...
      //
      PluginRegistry registry = PluginRegistry.getInstance();
      PluginInterface stepPlugin = registry.findPluginWithId(StepPluginType.class, stepType);

      StepMetaInterface stepMetaInterface = null;
      if (stepPlugin != null) {
        stepMetaInterface = (StepMetaInterface) registry.loadClass(stepPlugin);
        stepType =
            stepPlugin.getIds()[0]; // revert to the default in case we loaded an alternate version
      } else {
        stepMeta.setStepMetaInterface(
            (StepMetaInterface) new MissingTrans(stepMeta.getName(), stepType));
        transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
      }

      stepMeta.setStepID(stepType);

      // Read the metadata from the repository too...
      //
      RepositoryProxy proxy = new RepositoryProxy(stepNode.getNode(NODE_STEP_CUSTOM));
      if (!stepMeta.isMissing()) {
        readRepCompatibleStepMeta(stepMetaInterface, proxy, null, transMeta.getDatabases());
        stepMetaInterface.readRep(proxy, transMeta.getMetaStore(), null, transMeta.getDatabases());
        stepMeta.setStepMetaInterface(stepMetaInterface);
      }

      // Get the partitioning as well...
      StepPartitioningMeta stepPartitioningMeta = new StepPartitioningMeta();
      if (stepNode.hasProperty(PROP_PARTITIONING_SCHEMA)) {
        String partSchemaId =
            stepNode.getProperty(PROP_PARTITIONING_SCHEMA).getRef().getId().toString();
        String schemaName =
            repo.loadPartitionSchema(new StringObjectId(partSchemaId), null).getName();

        stepPartitioningMeta.setPartitionSchemaName(schemaName);
        String methodCode = getString(stepNode, PROP_PARTITIONING_METHOD);
        stepPartitioningMeta.setMethod(StepPartitioningMeta.getMethod(methodCode));
        if (stepPartitioningMeta.getPartitioner() != null) {
          proxy = new RepositoryProxy(stepNode.getNode(NODE_PARTITIONER_CUSTOM));
          stepPartitioningMeta.getPartitioner().loadRep(proxy, null);
        }
        stepPartitioningMeta.hasChanged(true);
      }
      stepMeta.setStepPartitioningMeta(stepPartitioningMeta);

      stepMeta
          .getStepPartitioningMeta()
          .setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas());
      // Get the cluster schema name
      String clusterSchemaName = getString(stepNode, PROP_CLUSTER_SCHEMA);
      stepMeta.setClusterSchemaName(clusterSchemaName);
      if (clusterSchemaName != null && transMeta.getClusterSchemas() != null) {
        // Get the cluster schema from the given name
        for (ClusterSchema clusterSchema : transMeta.getClusterSchemas()) {
          if (clusterSchema.getName().equals(clusterSchemaName)) {
            stepMeta.setClusterSchema(clusterSchema);
            break;
          }
        }
      }

      transMeta.addStep(stepMeta);
    }

    for (DataNode stepNode : stepsNode.getNodes()) {

      ObjectId stepObjectId = new StringObjectId(stepNode.getId().toString());
      StepMeta stepMeta = StepMeta.findStep(transMeta.getSteps(), stepObjectId);

      // Also load the step error handling metadata
      //
      if (stepNode.hasProperty(PROP_STEP_ERROR_HANDLING_SOURCE_STEP)) {
        StepErrorMeta meta = new StepErrorMeta(transMeta, stepMeta);
        meta.setTargetStep(
            StepMeta.findStep(
                transMeta.getSteps(),
                stepNode.getProperty(PROP_STEP_ERROR_HANDLING_TARGET_STEP).getString()));
        meta.setEnabled(stepNode.getProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED).getBoolean());
        meta.setNrErrorsValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_NR_VALUENAME));
        meta.setErrorDescriptionsValuename(
            getString(stepNode, PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME));
        meta.setErrorFieldsValuename(
            getString(stepNode, PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME));
        meta.setErrorCodesValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_CODES_VALUENAME));
        meta.setMaxErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_ERRORS));
        meta.setMaxPercentErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS));
        meta.setMinPercentRows(getString(stepNode, PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS));
        meta.getSourceStep().setStepErrorMeta(meta); // a bit of a trick, I know.
      }
    }

    // Have all StreamValueLookups, etc. reference the correct source steps...
    //
    for (int i = 0; i < transMeta.nrSteps(); i++) {
      StepMeta stepMeta = transMeta.getStep(i);
      StepMetaInterface sii = stepMeta.getStepMetaInterface();
      if (sii != null) {
        sii.searchInfoAndTargetSteps(transMeta.getSteps());
      }
    }

    // Read the notes...
    //
    DataNode notesNode = rootNode.getNode(NODE_NOTES);
    int nrNotes = (int) notesNode.getProperty(PROP_NR_NOTES).getLong();
    for (DataNode noteNode : notesNode.getNodes()) {
      String xml = getString(noteNode, PROP_XML);
      transMeta.addNote(
          new NotePadMeta(
              XMLHandler.getSubNode(XMLHandler.loadXMLString(xml), NotePadMeta.XML_TAG)));
    }
    if (transMeta.nrNotes() != nrNotes) {
      throw new KettleException(
          "The number of notes read ["
              + transMeta.nrNotes()
              + "] was not the number we expected ["
              + nrNotes
              + "]");
    }

    // Read the hops...
    //
    DataNode hopsNode = rootNode.getNode(NODE_HOPS);
    int nrHops = (int) hopsNode.getProperty(PROP_NR_HOPS).getLong();
    for (DataNode hopNode : hopsNode.getNodes()) {
      String stepFromName = getString(hopNode, TRANS_HOP_FROM);
      String stepToName = getString(hopNode, TRANS_HOP_TO);
      boolean enabled = true;
      if (hopNode.hasProperty(TRANS_HOP_ENABLED)) {
        enabled = hopNode.getProperty(TRANS_HOP_ENABLED).getBoolean();
      }

      StepMeta stepFrom = StepMeta.findStep(transMeta.getSteps(), stepFromName);
      StepMeta stepTo = StepMeta.findStep(transMeta.getSteps(), stepToName);

      // Make sure to only accept valid hops PDI-5519
      //
      if (stepFrom != null && stepTo != null) {
        transMeta.addTransHop(new TransHopMeta(stepFrom, stepTo, enabled));
      }
    }
    if (transMeta.nrTransHops() != nrHops) {
      throw new KettleException(
          "The number of hops read ["
              + transMeta.nrTransHops()
              + "] was not the number we expected ["
              + nrHops
              + "]");
    }

    // Load the details at the end, to make sure we reference the databases correctly, etc.
    //
    loadTransformationDetails(rootNode, transMeta);

    transMeta.eraseParameters();

    DataNode paramsNode = rootNode.getNode(NODE_PARAMETERS);

    int count = (int) paramsNode.getProperty(PROP_NR_PARAMETERS).getLong();
    for (int idx = 0; idx < count; idx++) {
      DataNode paramNode = paramsNode.getNode(TRANS_PARAM_PREFIX + idx);
      String key = getString(paramNode, PARAM_KEY);
      String def = getString(paramNode, PARAM_DEFAULT);
      String desc = getString(paramNode, PARAM_DESC);
      transMeta.addParameterDefinition(key, def, desc);
    }

    transMeta.activateParameters();
  }