@Test
public void testLoadXml() throws Exception {
OozieJobExecutorJobEntry jobEntry = new OozieJobExecutorJobEntry();
OozieJobExecutorConfig jobConfig = new OozieJobExecutorConfig();
jobConfig.setOozieWorkflow("hdfs://localhost:9000/user/test-user/oozie/workflow.xml");
jobConfig.setOozieWorkflowConfig("file:///User/test-user/oozie/job.properties");
jobConfig.setOozieUrl("http://localhost:11000/oozie");
jobEntry.setJobConfig(jobConfig);
JobEntryCopy jec = new JobEntryCopy(jobEntry);
jec.setLocation(0, 0);
String xml = jec.getXML();
Document d = XMLHandler.loadXMLString(xml);
OozieJobExecutorJobEntry jobEntry2 = new OozieJobExecutorJobEntry();
jobEntry2.loadXML(d.getDocumentElement(), null, null, null);
OozieJobExecutorConfig jobConfig2 = jobEntry2.getJobConfig();
assertEquals(jobConfig.getOozieWorkflow(), jobConfig2.getOozieWorkflow());
assertEquals(jobConfig.getOozieWorkflowConfig(), jobConfig2.getOozieWorkflowConfig());
assertEquals(jobConfig.getOozieUrl(), jobConfig2.getOozieUrl());
}
@Override
public boolean jobElementRead(String xml, RepositoryImportFeedbackInterface feedback) {
try {
Document doc = XMLHandler.loadXMLString(getOrCreateDb(), xml);
Node jobNode = XMLHandler.getSubNode(doc, RepositoryExportSaxParser.STRING_JOB);
if (!importJob(jobNode, feedback)) {
return false;
}
jobNumber++;
} catch (Exception e) {
// Some unexpected error occurred during job import
// This is usually a problem with a missing plugin or something
// like that...
//
showError(
BaseMessages.getString(PKG, "RepositoryImporter.UnexpectedErrorDuringJobImport.Title"),
BaseMessages.getString(PKG, "RepositoryImporter.UnexpectedErrorDuringJobImport.Message"),
e);
if (!feedback.askContinueOnErrorQuestion(
BaseMessages.getString(PKG, "RepositoryImporter.DoYouWantToContinue.Title"),
BaseMessages.getString(PKG, "RepositoryImporter.DoYouWantToContinue.Message"))) {
return false;
}
}
return true;
}
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
if (!request.getRequestURI().equals(CONTEXT_PATH + "/")) return;
if (log.isDebug()) log.logDebug(toString(), "Slave Server registration requested");
PrintWriter out = response.getWriter();
BufferedReader in = request.getReader();
if (log.isDetailed())
log.logDetailed(toString(), "Encoding: " + request.getCharacterEncoding());
// We always use XML to reply here...
//
response.setContentType("text/xml");
out.print(XMLHandler.getXMLHeader());
response.setStatus(HttpServletResponse.SC_OK);
try {
// First read the slave server information in memory from the request
//
StringBuilder xml = new StringBuilder(request.getContentLength());
int c;
while ((c = in.read()) != -1) {
xml.append((char) c);
}
// Parse the XML, create a transformation configuration
//
Document document = XMLHandler.loadXMLString(xml.toString());
Node node = XMLHandler.getSubNode(document, SlaveServerDetection.XML_TAG);
SlaveServerDetection slaveServerDetection = new SlaveServerDetection(node);
// See if this slave server is already in our list...
//
String message;
int index = detections.indexOf(slaveServerDetection);
if (index < 0) {
detections.add(slaveServerDetection);
message =
"Slave server detection '"
+ slaveServerDetection.getSlaveServer().getName()
+ "' was replaced in the list.";
} else {
// replace the data in the old one...
//
SlaveServerDetection old = detections.get(index);
old.setSlaveServer(slaveServerDetection.getSlaveServer());
old.setActive(slaveServerDetection.isActive());
// Note: in case it's not the slave server itself doing the sending, it might be possible
// for it to be inactive...
//
if (old.isActive()) {
old.setLastActiveDate(slaveServerDetection.getLastActiveDate());
} else {
old.setLastInactiveDate(slaveServerDetection.getLastInactiveDate());
}
message =
"Slave server detection '"
+ slaveServerDetection.getSlaveServer().getName()
+ "' was added to the list.";
}
out.println(new WebResult(WebResult.STRING_OK, message));
} catch (Exception ex) {
out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
}
}
public void dataNodeToElement(final DataNode rootNode, final RepositoryElementInterface element)
throws KettleException {
TransMeta transMeta = (TransMeta) element;
List<String> privateTransformationDatabases = null;
// read the private databases
DataNode privateDatabases = rootNode.getNode(NODE_TRANS_PRIVATE_DATABASES);
// if we have node than we use new format we could remove unexpected node
if (privateDatabases != null) {
privateTransformationDatabases = new ArrayList<String>();
for (DataNode privateDatabase : privateDatabases.getNodes()) {
privateTransformationDatabases.add(privateDatabase.getName());
}
}
transMeta.setPrivateTransformationDatabases(privateTransformationDatabases);
// read the steps...
//
DataNode stepsNode = rootNode.getNode(NODE_STEPS);
for (DataNode stepNode : stepsNode.getNodes()) {
StepMeta stepMeta = new StepMeta(new StringObjectId(stepNode.getId().toString()));
stepMeta.setParentTransMeta(transMeta); // for tracing, retain hierarchy
// Read the basics
//
stepMeta.setName(getString(stepNode, PROP_NAME));
if (stepNode.hasProperty(PROP_DESCRIPTION)) {
stepMeta.setDescription(getString(stepNode, PROP_DESCRIPTION));
}
stepMeta.setDistributes(stepNode.getProperty(PROP_STEP_DISTRIBUTE).getBoolean());
DataProperty rowDistributionProperty = stepNode.getProperty(PROP_STEP_ROW_DISTRIBUTION);
String rowDistributionCode =
rowDistributionProperty == null ? null : rowDistributionProperty.getString();
RowDistributionInterface rowDistribution =
PluginRegistry.getInstance()
.loadClass(
RowDistributionPluginType.class,
rowDistributionCode,
RowDistributionInterface.class);
stepMeta.setRowDistribution(rowDistribution);
stepMeta.setDraw(stepNode.getProperty(PROP_STEP_GUI_DRAW).getBoolean());
int copies = (int) stepNode.getProperty(PROP_STEP_COPIES).getLong();
String copiesString =
stepNode.getProperty(PROP_STEP_COPIES_STRING) != null
? stepNode.getProperty(PROP_STEP_COPIES_STRING).getString()
: StringUtils.EMPTY;
if (!Const.isEmpty(copiesString)) {
stepMeta.setCopiesString(copiesString);
} else {
stepMeta.setCopies(copies); // for backward compatibility
}
int x = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_X).getLong();
int y = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_Y).getLong();
stepMeta.setLocation(x, y);
// Load the group attributes map
//
AttributesMapUtil.loadAttributesMap(stepNode, stepMeta);
String stepType = getString(stepNode, PROP_STEP_TYPE);
// Create a new StepMetaInterface object...
//
PluginRegistry registry = PluginRegistry.getInstance();
PluginInterface stepPlugin = registry.findPluginWithId(StepPluginType.class, stepType);
StepMetaInterface stepMetaInterface = null;
if (stepPlugin != null) {
stepMetaInterface = (StepMetaInterface) registry.loadClass(stepPlugin);
stepType =
stepPlugin.getIds()[0]; // revert to the default in case we loaded an alternate version
} else {
stepMeta.setStepMetaInterface(
(StepMetaInterface) new MissingTrans(stepMeta.getName(), stepType));
transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
}
stepMeta.setStepID(stepType);
// Read the metadata from the repository too...
//
RepositoryProxy proxy = new RepositoryProxy(stepNode.getNode(NODE_STEP_CUSTOM));
if (!stepMeta.isMissing()) {
readRepCompatibleStepMeta(stepMetaInterface, proxy, null, transMeta.getDatabases());
stepMetaInterface.readRep(proxy, transMeta.getMetaStore(), null, transMeta.getDatabases());
stepMeta.setStepMetaInterface(stepMetaInterface);
}
// Get the partitioning as well...
StepPartitioningMeta stepPartitioningMeta = new StepPartitioningMeta();
if (stepNode.hasProperty(PROP_PARTITIONING_SCHEMA)) {
String partSchemaId =
stepNode.getProperty(PROP_PARTITIONING_SCHEMA).getRef().getId().toString();
String schemaName =
repo.loadPartitionSchema(new StringObjectId(partSchemaId), null).getName();
stepPartitioningMeta.setPartitionSchemaName(schemaName);
String methodCode = getString(stepNode, PROP_PARTITIONING_METHOD);
stepPartitioningMeta.setMethod(StepPartitioningMeta.getMethod(methodCode));
if (stepPartitioningMeta.getPartitioner() != null) {
proxy = new RepositoryProxy(stepNode.getNode(NODE_PARTITIONER_CUSTOM));
stepPartitioningMeta.getPartitioner().loadRep(proxy, null);
}
stepPartitioningMeta.hasChanged(true);
}
stepMeta.setStepPartitioningMeta(stepPartitioningMeta);
stepMeta
.getStepPartitioningMeta()
.setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas());
// Get the cluster schema name
String clusterSchemaName = getString(stepNode, PROP_CLUSTER_SCHEMA);
stepMeta.setClusterSchemaName(clusterSchemaName);
if (clusterSchemaName != null && transMeta.getClusterSchemas() != null) {
// Get the cluster schema from the given name
for (ClusterSchema clusterSchema : transMeta.getClusterSchemas()) {
if (clusterSchema.getName().equals(clusterSchemaName)) {
stepMeta.setClusterSchema(clusterSchema);
break;
}
}
}
transMeta.addStep(stepMeta);
}
for (DataNode stepNode : stepsNode.getNodes()) {
ObjectId stepObjectId = new StringObjectId(stepNode.getId().toString());
StepMeta stepMeta = StepMeta.findStep(transMeta.getSteps(), stepObjectId);
// Also load the step error handling metadata
//
if (stepNode.hasProperty(PROP_STEP_ERROR_HANDLING_SOURCE_STEP)) {
StepErrorMeta meta = new StepErrorMeta(transMeta, stepMeta);
meta.setTargetStep(
StepMeta.findStep(
transMeta.getSteps(),
stepNode.getProperty(PROP_STEP_ERROR_HANDLING_TARGET_STEP).getString()));
meta.setEnabled(stepNode.getProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED).getBoolean());
meta.setNrErrorsValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_NR_VALUENAME));
meta.setErrorDescriptionsValuename(
getString(stepNode, PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME));
meta.setErrorFieldsValuename(
getString(stepNode, PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME));
meta.setErrorCodesValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_CODES_VALUENAME));
meta.setMaxErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_ERRORS));
meta.setMaxPercentErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS));
meta.setMinPercentRows(getString(stepNode, PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS));
meta.getSourceStep().setStepErrorMeta(meta); // a bit of a trick, I know.
}
}
// Have all StreamValueLookups, etc. reference the correct source steps...
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepMeta stepMeta = transMeta.getStep(i);
StepMetaInterface sii = stepMeta.getStepMetaInterface();
if (sii != null) {
sii.searchInfoAndTargetSteps(transMeta.getSteps());
}
}
// Read the notes...
//
DataNode notesNode = rootNode.getNode(NODE_NOTES);
int nrNotes = (int) notesNode.getProperty(PROP_NR_NOTES).getLong();
for (DataNode noteNode : notesNode.getNodes()) {
String xml = getString(noteNode, PROP_XML);
transMeta.addNote(
new NotePadMeta(
XMLHandler.getSubNode(XMLHandler.loadXMLString(xml), NotePadMeta.XML_TAG)));
}
if (transMeta.nrNotes() != nrNotes) {
throw new KettleException(
"The number of notes read ["
+ transMeta.nrNotes()
+ "] was not the number we expected ["
+ nrNotes
+ "]");
}
// Read the hops...
//
DataNode hopsNode = rootNode.getNode(NODE_HOPS);
int nrHops = (int) hopsNode.getProperty(PROP_NR_HOPS).getLong();
for (DataNode hopNode : hopsNode.getNodes()) {
String stepFromName = getString(hopNode, TRANS_HOP_FROM);
String stepToName = getString(hopNode, TRANS_HOP_TO);
boolean enabled = true;
if (hopNode.hasProperty(TRANS_HOP_ENABLED)) {
enabled = hopNode.getProperty(TRANS_HOP_ENABLED).getBoolean();
}
StepMeta stepFrom = StepMeta.findStep(transMeta.getSteps(), stepFromName);
StepMeta stepTo = StepMeta.findStep(transMeta.getSteps(), stepToName);
// Make sure to only accept valid hops PDI-5519
//
if (stepFrom != null && stepTo != null) {
transMeta.addTransHop(new TransHopMeta(stepFrom, stepTo, enabled));
}
}
if (transMeta.nrTransHops() != nrHops) {
throw new KettleException(
"The number of hops read ["
+ transMeta.nrTransHops()
+ "] was not the number we expected ["
+ nrHops
+ "]");
}
// Load the details at the end, to make sure we reference the databases correctly, etc.
//
loadTransformationDetails(rootNode, transMeta);
transMeta.eraseParameters();
DataNode paramsNode = rootNode.getNode(NODE_PARAMETERS);
int count = (int) paramsNode.getProperty(PROP_NR_PARAMETERS).getLong();
for (int idx = 0; idx < count; idx++) {
DataNode paramNode = paramsNode.getNode(TRANS_PARAM_PREFIX + idx);
String key = getString(paramNode, PARAM_KEY);
String def = getString(paramNode, PARAM_DEFAULT);
String desc = getString(paramNode, PARAM_DESC);
transMeta.addParameterDefinition(key, def, desc);
}
transMeta.activateParameters();
}