public static Trans getTrans(
final Configuration conf, final String transXml, boolean singleThreaded)
throws KettleException {
initKettleEnvironment(conf);
TransConfiguration transConfiguration = TransConfiguration.fromXML(transXml);
TransMeta transMeta = transConfiguration.getTransMeta();
String carteObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject =
new SimpleLoggingObject("HADOOP_MAPPER", LoggingObjectType.CARTE, null); // $NON-NLS-1$
servletLoggingObject.setContainerObjectId(carteObjectId);
TransExecutionConfiguration executionConfiguration =
transConfiguration.getTransExecutionConfiguration();
servletLoggingObject.setLogLevel(executionConfiguration.getLogLevel());
if (singleThreaded) {
// Set the type to single threaded in case the user forgot...
//
transMeta.setTransformationType(TransformationType.SingleThreaded);
// Disable thread priority management as it will slow things down needlessly.
// The single threaded engine doesn't use threads and doesn't need row locking.
//
transMeta.setUsingThreadPriorityManagment(false);
} else {
transMeta.setTransformationType(TransformationType.Normal);
}
return new Trans(transMeta, servletLoggingObject);
}
protected void splitTrans(
final TransMeta transMeta, final TransExecutionConfiguration executionConfiguration)
throws KettleException {
try {
final TransSplitter transSplitter = new TransSplitter(transMeta);
transSplitter.splitOriginalTransformation();
TransMeta master = transSplitter.getMaster();
SlaveServer masterServer = null;
List<StepMeta> masterSteps = master.getTransHopSteps(false);
// add transgraph of transmetas if showing is true
SlaveServer slaves[] = transSplitter.getSlaveTargets();
if (executionConfiguration.isClusterShowingTransformation()) {
if (masterSteps.size() > 0) // If there is something that needs to
// be done on the master...
{
masterServer = transSplitter.getMasterServer();
addTransGraph(master);
}
// Then the slaves...
//
for (int i = 0; i < slaves.length; i++) {
TransMeta slaveTrans = (TransMeta) transSplitter.getSlaveTransMap().get(slaves[i]);
addTransGraph(slaveTrans);
}
}
// Inject certain internal variables to make it more intuitive.
//
for (String var : Const.INTERNAL_TRANS_VARIABLES)
executionConfiguration.getVariables().put(var, transMeta.getVariable(var));
for (String var : Const.INTERNAL_JOB_VARIABLES)
executionConfiguration.getVariables().put(var, transMeta.getVariable(var));
// Parameters override the variables.
// For the time being we're passing the parameters over the wire as variables...
//
TransMeta ot = transSplitter.getOriginalTransformation();
for (String param : ot.listParameters()) {
String value =
Const.NVL(
ot.getParameterValue(param),
Const.NVL(ot.getParameterDefault(param), ot.getVariable(param)));
if (!Const.isEmpty(value)) {
executionConfiguration.getVariables().put(param, value);
}
}
Trans.executeClustered(transSplitter, executionConfiguration);
if (executionConfiguration.isClusterPosting()) {
// Now add monitors for the master and all the slave servers
//
if (masterServer != null) {
spoon.addSpoonSlave(masterServer);
for (int i = 0; i < slaves.length; i++) {
spoon.addSpoonSlave(slaves[i]);
}
}
}
// OK, we should also start monitoring of the cluster in the background.
// Stop them all if one goes bad.
// Also clean up afterwards, close sockets, etc.
//
// Launch in a separate thread to prevent GUI blocking...
//
new Thread(
new Runnable() {
public void run() {
Trans.monitorClusteredTransformation(transMeta.toString(), transSplitter, null);
Result result =
Trans.getClusteredTransformationResult(
transMeta.toString(), transSplitter, null);
LogWriter log = LogWriter.getInstance();
log.logBasic(
transMeta.toString(),
"-----------------------------------------------------");
log.logBasic(
transMeta.toString(), "Got result back from clustered transformation:");
log.logBasic(
transMeta.toString(),
"-----------------------------------------------------");
log.logBasic(transMeta.toString(), "Errors : " + result.getNrErrors());
log.logBasic(transMeta.toString(), "Input : " + result.getNrLinesInput());
log.logBasic(transMeta.toString(), "Output : " + result.getNrLinesOutput());
log.logBasic(transMeta.toString(), "Updated : " + result.getNrLinesUpdated());
log.logBasic(transMeta.toString(), "Read : " + result.getNrLinesRead());
log.logBasic(transMeta.toString(), "Written : " + result.getNrLinesWritten());
log.logBasic(transMeta.toString(), "Rejected : " + result.getNrLinesRejected());
log.logBasic(
transMeta.toString(),
"-----------------------------------------------------");
}
})
.start();
} catch (Exception e) {
throw new KettleException(e);
}
}
public void executeTransformation(
final TransMeta transMeta,
final boolean local,
final boolean remote,
final boolean cluster,
final boolean preview,
final boolean debug,
final Date replayDate,
final boolean safe)
throws KettleException {
if (transMeta == null) {
return;
}
// See if we need to ask for debugging information...
//
TransDebugMeta transDebugMeta = null;
TransExecutionConfiguration executionConfiguration = null;
if (preview) {
executionConfiguration = spoon.getTransPreviewExecutionConfiguration();
} else if (debug) {
executionConfiguration = spoon.getTransDebugExecutionConfiguration();
} else {
executionConfiguration = spoon.getTransExecutionConfiguration();
}
if (debug) {
// See if we have debugging information stored somewhere?
//
transDebugMeta = transDebugMetaMap.get(transMeta);
if (transDebugMeta == null) {
transDebugMeta = new TransDebugMeta(transMeta);
transDebugMetaMap.put(transMeta, transDebugMeta);
}
// Set the default number of rows to retrieve on all selected steps...
//
StepMeta[] selectedSteps = transMeta.getSelectedSteps();
if (selectedSteps != null && selectedSteps.length > 0) {
transDebugMeta.getStepDebugMetaMap().clear();
for (StepMeta stepMeta : transMeta.getSelectedSteps()) {
StepDebugMeta stepDebugMeta = new StepDebugMeta(stepMeta);
stepDebugMeta.setRowCount(PropsUI.getInstance().getDefaultPreviewSize());
stepDebugMeta.setPausingOnBreakPoint(true);
stepDebugMeta.setReadingFirstRows(false);
transDebugMeta.getStepDebugMetaMap().put(stepMeta, stepDebugMeta);
}
}
} else if (preview) {
// See if we have preview information stored somewhere?
//
transDebugMeta = transPreviewMetaMap.get(transMeta);
if (transDebugMeta == null) {
transDebugMeta = new TransDebugMeta(transMeta);
transPreviewMetaMap.put(transMeta, transDebugMeta);
}
// Set the default number of preview rows on all selected steps...
//
StepMeta[] selectedSteps = transMeta.getSelectedSteps();
if (selectedSteps != null && selectedSteps.length > 0) {
transDebugMeta.getStepDebugMetaMap().clear();
for (StepMeta stepMeta : transMeta.getSelectedSteps()) {
StepDebugMeta stepDebugMeta = new StepDebugMeta(stepMeta);
stepDebugMeta.setRowCount(PropsUI.getInstance().getDefaultPreviewSize());
stepDebugMeta.setPausingOnBreakPoint(false);
stepDebugMeta.setReadingFirstRows(true);
transDebugMeta.getStepDebugMetaMap().put(stepMeta, stepDebugMeta);
}
}
}
executionConfiguration.setRepository(spoon.rep);
executionConfiguration.setSafeModeEnabled(safe);
int debugAnswer = TransDebugDialog.DEBUG_CONFIG;
if (debug || preview) {
transDebugMeta.getTransMeta().setRepository(spoon.rep); // pass repository for mappings
TransDebugDialog transDebugDialog = new TransDebugDialog(spoon.getShell(), transDebugMeta);
debugAnswer = transDebugDialog.open();
if (debugAnswer != TransDebugDialog.DEBUG_CANCEL) {
executionConfiguration.setExecutingLocally(true);
executionConfiguration.setExecutingRemotely(false);
executionConfiguration.setExecutingClustered(false);
} else {
// If we cancel the debug dialog, we don't go further with the execution either.
//
return;
}
} else {
if (transMeta.findFirstUsedClusterSchema() != null) {
executionConfiguration.setExecutingLocally(false);
executionConfiguration.setExecutingRemotely(false);
executionConfiguration.setExecutingClustered(true);
} else {
executionConfiguration.setExecutingLocally(true);
executionConfiguration.setExecutingRemotely(false);
executionConfiguration.setExecutingClustered(false);
}
}
Object data[] = spoon.variables.getData();
String fields[] = spoon.variables.getRowMeta().getFieldNames();
Map<String, String> variableMap = new HashMap<String, String>();
variableMap.putAll(executionConfiguration.getVariables()); // the default
for (int idx = 0; idx < fields.length; idx++) {
String value = executionConfiguration.getVariables().get(fields[idx]);
if (Const.isEmpty(value)) value = data[idx].toString();
variableMap.put(fields[idx], value);
}
executionConfiguration.setVariables(variableMap);
executionConfiguration.getUsedVariables(transMeta);
executionConfiguration.getUsedArguments(transMeta, spoon.getArguments());
executionConfiguration.setReplayDate(replayDate);
executionConfiguration.setLogLevel(spoon.getLog().getLogLevel());
boolean execConfigAnswer = true;
if (debugAnswer == TransDebugDialog.DEBUG_CONFIG && replayDate == null) {
TransExecutionConfigurationDialog dialog =
new TransExecutionConfigurationDialog(
spoon.getShell(), executionConfiguration, transMeta);
execConfigAnswer = dialog.open();
}
if (execConfigAnswer) {
// Verify if there is at least one step specified to debug or preview...
//
if (debug || preview) {
if (transDebugMeta.getNrOfUsedSteps() == 0) {
MessageBox box = new MessageBox(spoon.getShell(), SWT.ICON_WARNING | SWT.YES | SWT.NO);
box.setText(Messages.getString("Spoon.Dialog.Warning.NoPreviewOrDebugSteps.Title"));
box.setMessage(Messages.getString("Spoon.Dialog.Warning.NoPreviewOrDebugSteps.Message"));
int answer = box.open();
if (answer != SWT.YES) {
return;
}
}
}
// addTransLog(transMeta, executionConfiguration.isExecutingLocally());
// TransLog transLog = spoon.getActiveTransLog();
TransGraph activeTransGraph = spoon.getActiveTransGraph();
// Is this a local execution?
//
if (executionConfiguration.isExecutingLocally()) {
if (debug || preview) {
activeTransGraph.debug(executionConfiguration, transDebugMeta);
} else {
activeTransGraph.start(executionConfiguration);
}
// Are we executing remotely?
//
} else if (executionConfiguration.isExecutingRemotely()) {
if (executionConfiguration.getRemoteServer() != null) {
Trans.sendToSlaveServer(transMeta, executionConfiguration, spoon.rep);
monitorRemoteTrans(transMeta, executionConfiguration.getRemoteServer());
spoon.delegates.slaves.addSpoonSlave(executionConfiguration.getRemoteServer());
} else {
MessageBox mb = new MessageBox(spoon.getShell(), SWT.OK | SWT.ICON_INFORMATION);
mb.setMessage(Messages.getString("Spoon.Dialog.NoRemoteServerSpecified.Message"));
mb.setText(Messages.getString("Spoon.Dialog.NoRemoteServerSpecified.Title"));
mb.open();
}
// Are we executing clustered?
//
} else if (executionConfiguration.isExecutingClustered()) {
splitTrans(transMeta, executionConfiguration);
}
}
}
/**
* <div id="mindtouch">
*
* <h1>/kettle/prepareExec</h1>
*
* <a name="GET"></a>
*
* <h2>GET</h2>
*
* <p>Prepares previously loaded transformation for execution. Method is used for preparing
* previously uploaded transformation for execution by its name.
*
* <p><b>Example Request:</b><br>
*
* <pre function="syntax.xml">
* GET /kettle/prepareExec/?name=dummy-trans2&xml=Y
* </pre>
*
* <h3>Parameters</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>name</th>
* <th>description</th>
* <th>type</th>
* </tr>
* <tr>
* <td>name</td>
* <td>Name of the transformation to be prepared for execution.</td>
* <td>query</td>
* </tr>
* <tr>
* <td>xml</td>
* <td>Boolean flag which sets the output format required. Use <code>Y</code> to receive XML response.</td>
* <td>boolean</td>
* </tr>
* <tr>
* <td>id</td>
* <td>Carte transformation ID of the transformation to be prepared for execution.</td>
* <td>query, optional</td>
* </tr>
* </tbody>
* </table>
*
* <h3>Response Body</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <td align="right">text:</td>
* <td>HTML</td>
* </tr>
* <tr>
* <td align="right">media types:</td>
* <td>text/xml, text/html</td>
* </tr>
* </tbody>
* </table>
*
* <p>Response XML or HTML containing operation result. When using xml=Y <code>result</code> field
* indicates whether operation was successful (<code>OK</code>) or not (<code>ERROR</code>).
*
* <p><b>Example Response:</b>
*
* <pre function="syntax.xml">
* <?xml version="1.0" encoding="UTF-8"?>
* <webresult>
* <result>OK</result>
* <message/>
* <id/>
* </webresult>
* </pre>
*
* <h3>Status Codes</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>code</th>
* <th>description</th>
* </tr>
* <tr>
* <td>200</td>
* <td>Request was processed.</td>
* </tr>
* <tr>
* <td>500</td>
* <td>Internal server error occurs during request processing.</td>
* </tr>
* </tbody>
* </table>
*
* </div>
*/
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "PrepareExecutionTransServlet.TransPrepareExecutionRequested"));
}
String transName = request.getParameter("name");
String id = request.getParameter("id");
boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
response.setStatus(HttpServletResponse.SC_OK);
PrintWriter out = response.getWriter();
if (useXML) {
response.setContentType("text/xml");
out.print(XMLHandler.getXMLHeader(Const.XML_ENCODING));
} else {
response.setCharacterEncoding("UTF-8");
response.setContentType("text/html;charset=UTF-8");
out.println("<HTML>");
out.println("<HEAD>");
out.println(
"<TITLE>"
+ BaseMessages.getString(PKG, "PrepareExecutionTransServlet.TransPrepareExecution")
+ "</TITLE>");
out.println(
"<META http-equiv=\"Refresh\" content=\"2;url="
+ convertContextPath(GetTransStatusServlet.CONTEXT_PATH)
+ "?name="
+ URLEncoder.encode(transName, "UTF-8")
+ "\">");
out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
out.println("</HEAD>");
out.println("<BODY>");
}
try {
// ID is optional...
//
Trans trans;
CarteObjectEntry entry;
if (Utils.isEmpty(id)) {
// get the first transformation that matches...
//
entry = getTransformationMap().getFirstCarteObjectEntry(transName);
if (entry == null) {
trans = null;
} else {
id = entry.getId();
trans = getTransformationMap().getTransformation(entry);
}
} else {
// Take the ID into account!
//
entry = new CarteObjectEntry(transName, id);
trans = getTransformationMap().getTransformation(entry);
}
TransConfiguration transConfiguration = getTransformationMap().getConfiguration(entry);
if (trans != null && transConfiguration != null) {
TransExecutionConfiguration executionConfiguration =
transConfiguration.getTransExecutionConfiguration();
// Set the appropriate logging, variables, arguments, replay date, ...
// etc.
trans.setArguments(executionConfiguration.getArgumentStrings());
trans.setReplayDate(executionConfiguration.getReplayDate());
trans.setSafeModeEnabled(executionConfiguration.isSafeModeEnabled());
trans.setGatheringMetrics(executionConfiguration.isGatheringMetrics());
trans.injectVariables(executionConfiguration.getVariables());
trans.setPreviousResult(executionConfiguration.getPreviousResult());
try {
trans.prepareExecution(null);
if (useXML) {
out.println(WebResult.OK.getXML());
} else {
out.println(
"<H1>"
+ Encode.forHtml(
BaseMessages.getString(
PKG, "PrepareExecutionTransServlet.TransPrepared", transName))
+ "</H1>");
out.println(
"<a href=\""
+ convertContextPath(GetTransStatusServlet.CONTEXT_PATH)
+ "?name="
+ URLEncoder.encode(transName, "UTF-8")
+ "&id="
+ URLEncoder.encode(id, "UTF-8")
+ "\">"
+ BaseMessages.getString(PKG, "TransStatusServlet.BackToTransStatusPage")
+ "</a><p>");
}
} catch (Exception e) {
String logText =
KettleLogStore.getAppender()
.getBuffer(trans.getLogChannel().getLogChannelId(), true)
.toString();
if (useXML) {
out.println(
new WebResult(
WebResult.STRING_ERROR,
BaseMessages.getString(
PKG,
"PrepareExecutionTransServlet.Error.TransInitFailed",
Const.CR + logText + Const.CR + Const.getStackTracker(e))));
} else {
out.println(
"<H1>"
+ Encode.forHtml(
BaseMessages.getString(
PKG, "PrepareExecutionTransServlet.Log.TransNotInit", transName))
+ "</H1>");
out.println("<pre>");
out.println(Encode.forHtml(logText));
out.println(Encode.forHtml(Const.getStackTracker(e)));
out.println("</pre>");
out.println(
"<a href=\""
+ convertContextPath(GetTransStatusServlet.CONTEXT_PATH)
+ "?name="
+ URLEncoder.encode(transName, "UTF-8")
+ "&id="
+ id
+ "\">"
+ BaseMessages.getString(PKG, "TransStatusServlet.BackToTransStatusPage")
+ "</a><p>");
}
}
} else {
if (useXML) {
out.println(
new WebResult(
WebResult.STRING_ERROR,
BaseMessages.getString(
PKG, "TransStatusServlet.Log.CoundNotFindSpecTrans", transName)));
} else {
out.println(
"<H1>"
+ Encode.forHtml(
BaseMessages.getString(
PKG, "TransStatusServlet.Log.CoundNotFindTrans", transName))
+ "</H1>");
out.println(
"<a href=\""
+ convertContextPath(GetStatusServlet.CONTEXT_PATH)
+ "\">"
+ BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage")
+ "</a><p>");
}
}
} catch (Exception ex) {
if (useXML) {
out.println(
new WebResult(
WebResult.STRING_ERROR,
BaseMessages.getString(
PKG,
"PrepareExecutionTransServlet.Error.UnexpectedError",
Const.CR + Const.getStackTracker(ex))));
} else {
out.println("<p>");
out.println("<pre>");
out.println(Encode.forHtml(Const.getStackTracker(ex)));
out.println("</pre>");
}
}
if (!useXML) {
out.println("<p>");
out.println("</BODY>");
out.println("</HTML>");
}
}