// Grab the first x lines from the given file... // private List<String> getFirst(FixedInputMeta meta, int limit) throws IOException, KettleValueException { List<String> lines = new ArrayList<String>(); FixedInputMeta oneMeta = new FixedInputMeta(); getInfo(oneMeta); // Add a single field with the width of the line... // int lineWidth = Integer.parseInt(oneMeta.getLineWidth()); if (lineWidth <= 0) { throw new IOException("The width of a line can not be 0 or less."); } oneMeta.allocate(1); FixedFileInputField field = new FixedFileInputField(); field.setName("Field1"); field.setType(ValueMetaInterface.TYPE_STRING); field.setWidth(lineWidth); // CHECKSTYLE:Indentation:OFF oneMeta.getFieldDefinition()[0] = field; TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation(transMeta, oneMeta, wStepname.getText()); TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog( shell, previewMeta, new String[] {wStepname.getText()}, new int[] {limit}); progressDialog.open(); Trans trans = progressDialog.getTrans(); String loggingText = progressDialog.getLoggingText(); if (!progressDialog.isCancelled()) { if (trans.getResult() != null && trans.getResult().getNrErrors() > 0) { EnterTextDialog etd = new EnterTextDialog( shell, BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title"), BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message"), loggingText, true); etd.setReadOnly(); etd.open(); } } // The rows are in the transformation... // RowMetaInterface previewRowsMeta = progressDialog.getPreviewRowsMeta(wStepname.getText()); List<Object[]> previewRowsData = progressDialog.getPreviewRows(wStepname.getText()); for (int i = 0; i < previewRowsData.size(); i++) { String line = previewRowsMeta.getString(previewRowsData.get(i), 0); lines.add(line); } return lines; }
@Test @PrepareForTest({Encode.class}) public void testStartExecutionTransServletEscapesHtmlWhenTransFound() throws ServletException, IOException { KettleLogStore.init(); HttpServletRequest mockHttpServletRequest = mock(HttpServletRequest.class); HttpServletResponse mockHttpServletResponse = mock(HttpServletResponse.class); Trans mockTrans = mock(Trans.class); TransMeta mockTransMeta = mock(TransMeta.class); LogChannelInterface mockChannelInterface = mock(LogChannelInterface.class); StringWriter out = new StringWriter(); PrintWriter printWriter = new PrintWriter(out); PowerMockito.spy(Encode.class); when(mockHttpServletRequest.getContextPath()) .thenReturn(StartExecutionTransServlet.CONTEXT_PATH); when(mockHttpServletRequest.getParameter(anyString())) .thenReturn(ServletTestUtils.BAD_STRING_TO_TEST); when(mockHttpServletResponse.getWriter()).thenReturn(printWriter); when(mockTransformationMap.getTransformation(any(CarteObjectEntry.class))) .thenReturn(mockTrans); when(mockTrans.getLogChannel()).thenReturn(mockChannelInterface); when(mockTrans.isReadyToStart()).thenReturn(true); when(mockTrans.getLogChannelId()).thenReturn("test"); when(mockTrans.getTransMeta()).thenReturn(mockTransMeta); when(mockTransMeta.getMaximum()).thenReturn(new Point(10, 10)); startExecutionTransServlet.doGet(mockHttpServletRequest, mockHttpServletResponse); assertFalse(ServletTestUtils.hasBadText(ServletTestUtils.getInsideOfTag("H1", out.toString()))); PowerMockito.verifyStatic(atLeastOnce()); Encode.forHtml(anyString()); }
private void preview() { PaloDimInputMeta oneMeta = new PaloDimInputMeta(); try { getInfo(oneMeta); } catch (KettleException e) { new ErrorDialog( shell, BaseMessages.getString(PKG, "PaloInputDialog.Illegal.Dialog.Settings.Title"), BaseMessages.getString(PKG, "PaloInputDialog.Illegal.Dialog.Settings.Message"), e); return; } TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation( transMeta, oneMeta, textStepName.getText()); EnterNumberDialog numberDialog = new EnterNumberDialog( shell, 500, BaseMessages.getString("System.Dialog.EnterPreviewSize.Title"), BaseMessages.getString("System.Dialog.EnterPreviewSize.Message")); int previewSize = numberDialog.open(); if (previewSize > 0) { TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog( shell, previewMeta, new String[] {textStepName.getText()}, new int[] {previewSize}); progressDialog.open(); Trans trans = progressDialog.getTrans(); String loggingText = progressDialog.getLoggingText(); if (!progressDialog.isCancelled()) { if (trans.getResult() != null && trans.getResult().getNrErrors() > 0) { EnterTextDialog etd = new EnterTextDialog( shell, BaseMessages.getString("System.Dialog.PreviewError.Title"), BaseMessages.getString("System.Dialog.PreviewError.Message"), loggingText, true); etd.setReadOnly(); etd.open(); } } PreviewRowsDialog prd = new PreviewRowsDialog( shell, transMeta, SWT.NONE, textStepName.getText(), progressDialog.getPreviewRowsMeta(textStepName.getText()), progressDialog.getPreviewRows(textStepName.getText()), loggingText); prd.open(); } }
// Preview the data private void preview() { try { GetFilesRowsCountMeta oneMeta = new GetFilesRowsCountMeta(); getInfo(oneMeta); TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation( transMeta, oneMeta, wStepname.getText()); EnterNumberDialog numberDialog = new EnterNumberDialog( shell, props.getDefaultPreviewSize(), BaseMessages.getString(PKG, "GetFilesRowsCountDialog.NumberRows.DialogTitle"), BaseMessages.getString(PKG, "GetFilesRowsCountDialog.NumberRows.DialogMessage")); int previewSize = numberDialog.open(); if (previewSize > 0) { TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog( shell, previewMeta, new String[] {wStepname.getText()}, new int[] {previewSize}); progressDialog.open(); if (!progressDialog.isCancelled()) { Trans trans = progressDialog.getTrans(); String loggingText = progressDialog.getLoggingText(); if (trans.getResult() != null && trans.getResult().getNrErrors() > 0) { EnterTextDialog etd = new EnterTextDialog( shell, BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title"), BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message"), loggingText, true); etd.setReadOnly(); etd.open(); } PreviewRowsDialog prd = new PreviewRowsDialog( shell, transMeta, SWT.NONE, wStepname.getText(), progressDialog.getPreviewRowsMeta(wStepname.getText()), progressDialog.getPreviewRows(wStepname.getText()), loggingText); prd.open(); } } } catch (KettleException e) { new ErrorDialog( shell, BaseMessages.getString(PKG, "GetFilesRowsCountDialog.ErrorPreviewingData.DialogTitle"), BaseMessages.getString(PKG, "GetFilesRowsCountDialog.ErrorPreviewingData.DialogMessage"), e); } }
// Preview the data private void preview() { GetTableNamesMeta oneMeta = new GetTableNamesMeta(); getInfo(oneMeta); if (oneMeta.getDatabase() == null) { MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); mb.setMessage( BaseMessages.getString(PKG, "GetTableNamesDialog.InvalidConnection.DialogMessage")); mb.setText(BaseMessages.getString(PKG, "GetTableNamesDialog.InvalidConnection.DialogTitle")); mb.open(); return; } if (!checkUserInput(oneMeta)) return; TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation(transMeta, oneMeta, wStepname.getText()); EnterNumberDialog numberDialog = new EnterNumberDialog( shell, props.getDefaultPreviewSize(), BaseMessages.getString(PKG, "GetTableNamesDialog.PreviewSize.DialogTitle"), BaseMessages.getString(PKG, "GetTableNamesDialog.PreviewSize.DialogMessage")); int previewSize = numberDialog.open(); if (previewSize > 0) { TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog( shell, previewMeta, new String[] {wStepname.getText()}, new int[] {previewSize}); progressDialog.open(); if (!progressDialog.isCancelled()) { Trans trans = progressDialog.getTrans(); String loggingText = progressDialog.getLoggingText(); if (trans.getResult() != null && trans.getResult().getNrErrors() > 0) { EnterTextDialog etd = new EnterTextDialog( shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "GetTableNamesDialog.ErrorInPreview.DialogMessage"), loggingText, true); etd.setReadOnly(); etd.open(); } PreviewRowsDialog prd = new PreviewRowsDialog( shell, transMeta, SWT.NONE, wStepname.getText(), progressDialog.getPreviewRowsMeta(wStepname.getText()), progressDialog.getPreviewRows(wStepname.getText()), loggingText); prd.open(); } } }
public boolean init(StepMetaInterface smi, StepDataInterface sdi) { meta = (OraBulkLoaderMeta) smi; data = (OraBulkLoaderData) sdi; Trans trans = getTrans(); preview = trans.isPreview(); return super.init(smi, sdi); }
public boolean init(StepMetaInterface smi, StepDataInterface sdi) { meta = (GPLoadMeta) smi; data = (GPLoadData) sdi; Trans trans = getTrans(); preview = trans.isPreview(); if (super.init(smi, sdi)) { return true; } return false; }
// Preview the data private void preview() { // execute a complete preview transformation in the background. // This is how we do it... // FixedInputMeta oneMeta = new FixedInputMeta(); getInfo(oneMeta); TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation(transMeta, oneMeta, wStepname.getText()); EnterNumberDialog numberDialog = new EnterNumberDialog( shell, props.getDefaultPreviewSize(), BaseMessages.getString(PKG, "FixedInputDialog.PreviewSize.DialogTitle"), BaseMessages.getString(PKG, "FixedInputDialog.PreviewSize.DialogMessage")); int previewSize = numberDialog.open(); if (previewSize > 0) { TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog( shell, previewMeta, new String[] {wStepname.getText()}, new int[] {previewSize}); progressDialog.open(); Trans trans = progressDialog.getTrans(); String loggingText = progressDialog.getLoggingText(); if (!progressDialog.isCancelled()) { if (trans.getResult() != null && trans.getResult().getNrErrors() > 0) { EnterTextDialog etd = new EnterTextDialog( shell, BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title"), BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message"), loggingText, true); etd.setReadOnly(); etd.open(); } } PreviewRowsDialog prd = new PreviewRowsDialog( shell, transMeta, SWT.NONE, wStepname.getText(), progressDialog.getPreviewRowsMeta(wStepname.getText()), progressDialog.getPreviewRows(wStepname.getText()), loggingText); prd.open(); } }
// Preview the data // unused // preserve for later private void preview() { // Create the XML input step SapInputMeta oneMeta = new SapInputMeta(); getInfo(oneMeta); TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation(transMeta, oneMeta, wStepname.getText()); transMeta.getVariable("Internal.Transformation.Filename.Directory"); previewMeta.getVariable("Internal.Transformation.Filename.Directory"); EnterNumberDialog numberDialog = new EnterNumberDialog( shell, props.getDefaultPreviewSize(), BaseMessages.getString(PKG, "CsvInputDialog.PreviewSize.DialogTitle"), BaseMessages.getString(PKG, "CsvInputDialog.PreviewSize.DialogMessage")); int previewSize = numberDialog.open(); if (previewSize > 0) { TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog( shell, previewMeta, new String[] {wStepname.getText()}, new int[] {previewSize}); progressDialog.open(); Trans trans = progressDialog.getTrans(); String loggingText = progressDialog.getLoggingText(); if (!progressDialog.isCancelled()) { if (trans.getResult() != null && trans.getResult().getNrErrors() > 0) { EnterTextDialog etd = new EnterTextDialog( shell, BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title"), BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message"), loggingText, true); etd.setReadOnly(); etd.open(); } } PreviewRowsDialog prd = new PreviewRowsDialog( shell, transMeta, SWT.NONE, wStepname.getText(), progressDialog.getPreviewRowsMeta(wStepname.getText()), progressDialog.getPreviewRows(wStepname.getText()), loggingText); prd.open(); } }
@Test public void testExecuteConcurrentModification() throws Exception { String sql = "SELECT * FROM " + DATA_SERVICE_NAME; DataServiceExecutor executor = new DataServiceExecutor.Builder(new SQL(sql), dataService, context) .prepareExecution(false) .sqlTransGenerator(sqlTransGenerator) .serviceTrans(serviceTrans) .genTrans(genTrans) .build(); final DataServiceExecutor.ExecutionPoint stage = DataServiceExecutor.ExecutionPoint.OPTIMIZE; final ListMultimap<DataServiceExecutor.ExecutionPoint, Runnable> listenerMap = executor.getListenerMap(); final Runnable task = new Runnable() { @Override public void run() { // Remove itself on run assertTrue(listenerMap.remove(stage, this)); } }; listenerMap.put(stage, task); executor.executeQuery(); // Note the error reported to logs verify(genTrans.getLogChannel()) .logError(anyString(), eq(stage), eq(ImmutableList.of(task)), eq(ImmutableList.of())); }
@Override public void run() { try { trans.prepareExecution(null); } catch (KettleException e) { Throwables.propagate(e); } }
@Before public void setUp() throws Exception { doAnswer(RETURNS_SELF).when(transMeta).realClone(anyBoolean()); doAnswer(RETURNS_SELF).when(transMeta).clone(); when(serviceTrans.getContainerObjectId()).thenReturn(CONTAINER_ID); when(sqlTransGenerator.getInjectorStepName()).thenReturn(INJECTOR_STEP_NAME); when(sqlTransGenerator.getResultStepName()).thenReturn(RESULT_STEP_NAME); }
@Test public void testLogging() throws Exception { when(serviceTrans.getTransMeta()).thenReturn(transMeta); TransMeta genTransMeta = mock(TransMeta.class); when(genTrans.getTransMeta()).thenReturn(genTransMeta); new DataServiceExecutor.Builder( new SQL("SELECT * FROM " + DATA_SERVICE_NAME), dataService, context) .serviceTrans(serviceTrans) .genTrans(genTrans) .prepareExecution(false) .logLevel(LogLevel.DETAILED) .build(); verify(serviceTrans).setLogLevel(LogLevel.DETAILED); verify(transMeta).setLogLevel(LogLevel.DETAILED); verify(genTrans).setLogLevel(LogLevel.DETAILED); verify(genTransMeta).setLogLevel(LogLevel.DETAILED); }
@Test public void testStop() throws KettleException { String sql = "SELECT * FROM " + DATA_SERVICE_NAME; when(serviceTrans.isRunning()).thenReturn(true); when(genTrans.isRunning()).thenReturn(true); DataServiceExecutor executor = new DataServiceExecutor.Builder(new SQL(sql), dataService, context) .serviceTrans(serviceTrans) .sqlTransGenerator(sqlTransGenerator) .genTrans(genTrans) .build(); executor.stop(); verify(serviceTrans).stopAll(); verify(genTrans).stopAll(); }
@Test public void testStepShouldStopProcessingInput_IfUnderlyingTransitionIsStopped() throws Exception { MappingInput mappingInput = mock(MappingInput.class); when(mappingInput.getStepname()).thenReturn(MAPPING_INPUT_STEP_NAME); stepMockHelper.processRowsStepDataInterface.mappingInput = mappingInput; RowProducer rowProducer = mock(RowProducer.class); when(rowProducer.putRow(any(RowMetaInterface.class), any(Object[].class), anyBoolean())) .thenReturn(true); StepInterface stepInterface = mock(StepInterface.class); Trans mappingTrans = mock(Trans.class); when(mappingTrans.addRowProducer(anyString(), anyInt())).thenReturn(rowProducer); when(mappingTrans.findStepInterface(anyString(), anyInt())).thenReturn(stepInterface); when(mappingTrans.isFinishedOrStopped()).thenReturn(Boolean.FALSE).thenReturn(Boolean.TRUE); stepMockHelper.processRowsStepDataInterface.mappingTrans = mappingTrans; MappingOutput mappingOutput = mock(MappingOutput.class); when(mappingOutput.getStepname()).thenReturn(MAPPING_OUTPUT_STEP_NAME); stepMockHelper.processRowsStepDataInterface.mappingOutput = mappingOutput; smp = new SimpleMapping( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans); smp.init(stepMockHelper.initStepMetaInterface, simpleMpData); smp.getInputRowSets().add(stepMockHelper.getMockInputRowSet(new Object[] {})); smp.getInputRowSets().add(stepMockHelper.getMockInputRowSet(new Object[] {})); assertTrue( smp.processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface)); assertFalse( smp.processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface)); }
/** Test case for injector step... also a show case on how to use injector. */ public void testInjector() throws Exception { KettleEnvironment.init(); // // Create a new transformation... // TransMeta transMeta = new TransMeta(); transMeta.setName("injectortest"); PluginRegistry registry = PluginRegistry.getInstance(); // // create an injector step... // String injectorStepname = "injector step"; InjectorMeta im = new InjectorMeta(); // Set the information of the injector. String injectorPid = registry.getPluginId(StepPluginType.class, im); StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, (StepMetaInterface) im); transMeta.addStep(injectorStep); // // Create a dummy step // String dummyStepname = "dummy step"; DummyTransMeta dm = new DummyTransMeta(); String dummyPid = registry.getPluginId(StepPluginType.class, dm); StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, (StepMetaInterface) dm); transMeta.addStep(dummyStep); TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep); transMeta.addTransHop(hi); // Now execute the transformation... Trans trans = new Trans(transMeta); trans.prepareExecution(null); StepInterface si = trans.getStepInterface(dummyStepname, 0); RowStepCollector rc = new RowStepCollector(); si.addRowListener(rc); RowProducer rp = trans.addRowProducer(injectorStepname, 0); trans.startThreads(); // add rows List<RowMetaAndData> inputList = createData(); for (RowMetaAndData rm : inputList) { rp.putRow(rm.getRowMeta(), rm.getData()); } rp.finished(); trans.waitUntilFinished(); List<RowMetaAndData> resultRows = rc.getRowsWritten(); checkRows(resultRows, inputList); }
@Test public void testIsComplete() throws KettleException { String sql = "SELECT * FROM " + DATA_SERVICE_NAME; when(genTrans.isStopped()).thenReturn(true); DataServiceExecutor executor = new DataServiceExecutor.Builder(new SQL(sql), dataService, context) .serviceTrans(serviceTrans) .sqlTransGenerator(sqlTransGenerator) .genTrans(genTrans) .build(); assertTrue(executor.isStopped()); }
@Test public void testStopTransServletEscapesHtmlWhenTransFound() throws ServletException, IOException { KettleLogStore.init(); HttpServletRequest mockHttpServletRequest = mock(HttpServletRequest.class); HttpServletResponse mockHttpServletResponse = mock(HttpServletResponse.class); Trans mockTrans = mock(Trans.class); TransMeta mockTransMeta = mock(TransMeta.class); LogChannelInterface mockChannelInterface = mock(LogChannelInterface.class); StringWriter out = new StringWriter(); PrintWriter printWriter = new PrintWriter(out); when(mockHttpServletRequest.getContextPath()).thenReturn(StopTransServlet.CONTEXT_PATH); when(mockHttpServletRequest.getParameter(anyString())).thenReturn(ServletTestUtils.BAD_STRING); when(mockHttpServletResponse.getWriter()).thenReturn(printWriter); when(mockTransformationMap.getTransformation(any(CarteObjectEntry.class))) .thenReturn(mockTrans); when(mockTrans.getLogChannel()).thenReturn(mockChannelInterface); when(mockTrans.getLogChannelId()).thenReturn("test"); when(mockTrans.getTransMeta()).thenReturn(mockTransMeta); when(mockTransMeta.getMaximum()).thenReturn(new Point(10, 10)); stopTransServlet.doGet(mockHttpServletRequest, mockHttpServletResponse); assertFalse(ServletTestUtils.hasBadText(ServletTestUtils.getInsideOfTag("H1", out.toString()))); }
public static Trans loadAndRunTransformation(String path, Object... parameters) throws Exception { TransMeta transMeta = new TransMeta(path); transMeta.setTransformationType(TransMeta.TransformationType.Normal); Trans trans = new Trans(transMeta); if (parameters != null) { if (parameters.length % 2 == 1) { throw new IllegalArgumentException( "Parameters should be an array of pairs 'parameter'-'value'-..."); } for (int i = 0; i < parameters.length; i += 2) { Object parameter = parameters[i]; Object value = parameters[i + 1]; trans.setParameterValue(parameter.toString(), value.toString()); } } trans.prepareExecution(null); trans.startThreads(); trans.waitUntilFinished(); return trans; }
/** Test case for Constant step. Row generator attached to a constant step. */ public void testConstant1() throws Exception { KettleEnvironment.init(); // // Create a new transformation... // TransMeta transMeta = new TransMeta(); transMeta.setName("constanttest1"); PluginRegistry registry = PluginRegistry.getInstance(); // // create a row generator step... // String rowGeneratorStepname = "row generator step"; RowGeneratorMeta rm = new RowGeneratorMeta(); // Set the information of the row generator. String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rm); StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, (StepMetaInterface) rm); transMeta.addStep(rowGeneratorStep); // // Generate 1 empty row // String fieldName[] = {}; String type[] = {}; String value[] = {}; String fieldFormat[] = {}; String group[] = {}; String decimal[] = {}; int intDummies[] = {}; rm.setDefault(); rm.setFieldName(fieldName); rm.setFieldType(type); rm.setValue(value); rm.setFieldLength(intDummies); rm.setFieldPrecision(intDummies); rm.setRowLimit("1"); rm.setFieldFormat(fieldFormat); rm.setGroup(group); rm.setDecimal(decimal); // // Add constant step. // String constStepname1 = "constant 1"; ConstantMeta cnst1 = new ConstantMeta(); String fieldName1[] = { "boolean1", "boolean2", "boolean3", "boolean4", "boolean5", "boolean6", "boolean7", "string1", "string2", "string3", "integer1", "integer2", "integer3", "integer4", "number1", "number2", "number3", "number4", }; String type1[] = { "boolean", "Boolean", "bOOLEAN", "BOOLEAN", "boolean", "boolean", "boolean", "string", "string", "String", "integer", "integer", "integer", "integer", "number", "number", "number", "number" }; String value1[] = { "Y", "T", "a", "TRUE", "0", "9", "", "AAAAAAAAAAAAAA", " ", "", "-100", "0", "212", "", "-100.2", "0.0", "212.23", "" }; String fieldFormat1[] = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" }; String group1[] = {"", "", "", "", "", "", "", "", "", "", "", "", "", "", ",", ",", ",", ","}; String decimal1[] = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", ".", ".", ".", "." }; String currency[] = {"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""}; int intDummies1[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; cnst1.setFieldName(fieldName1); cnst1.setFieldType(type1); cnst1.setValue(value1); cnst1.setFieldLength(intDummies1); cnst1.setFieldPrecision(intDummies1); cnst1.setFieldFormat(fieldFormat1); cnst1.setGroup(group1); cnst1.setDecimal(decimal1); cnst1.setCurrency(currency); String addSeqPid1 = registry.getPluginId(StepPluginType.class, cnst1); StepMeta addSeqStep1 = new StepMeta(addSeqPid1, constStepname1, (StepMetaInterface) cnst1); transMeta.addStep(addSeqStep1); TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, addSeqStep1); transMeta.addTransHop(hi1); // Now execute the transformation... Trans trans = new Trans(transMeta); trans.prepareExecution(null); StepInterface si = trans.getStepInterface(constStepname1, 0); RowStepCollector endRc = new RowStepCollector(); si.addRowListener(endRc); trans.startThreads(); trans.waitUntilFinished(); // Now check whether the output is still as we expect. List<RowMetaAndData> goldenImageRows = createResultData1(); List<RowMetaAndData> resultRows1 = endRc.getRowsWritten(); checkRows(resultRows1, goldenImageRows); }
@Test public void testExecuteQuery() throws Exception { SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME); StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP); StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME); when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]); PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class); when(optimization.isEnabled()).thenReturn(true); dataService.getPushDownOptimizationMeta().add(optimization); IMetaStore metastore = mock(IMetaStore.class); DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context) .serviceTrans(serviceTrans) .sqlTransGenerator(sqlTransGenerator) .genTrans(genTrans) .metastore(metastore) .build(); ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class); verify(serviceTrans).setContainerObjectId(objectIds.capture()); when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue()); verify(genTrans).setContainerObjectId(objectIds.capture()); when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue()); verify(serviceTrans).setMetaStore(metastore); verify(genTrans).setMetaStore(metastore); RowProducer sqlTransRowProducer = mock(RowProducer.class); when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); // Start Execution executor.executeQuery(new DataOutputStream(outputStream)); // Check header was written assertThat(outputStream.size(), greaterThan(0)); outputStream.reset(); InOrder genTransStartup = inOrder(genTrans, resultStep); InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep); ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class); ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class); ArgumentCaptor<TransListener> transListenerCaptor = ArgumentCaptor.forClass(TransListener.class); genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture()); genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0); genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture()); genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture()); RowListener clientRowListener = listenerArgumentCaptor.getValue(); genTransStartup.verify(genTrans).startThreads(); serviceTransStartup.verify(optimization).activate(executor); serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture()); serviceTransStartup.verify(serviceTrans).startThreads(); // Verify linkage RowListener serviceRowListener = listenerArgumentCaptor.getValue(); assertNotNull(serviceRowListener); // Push row from service to sql Trans RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME); Object[] data; for (int i = 0; i < 50; i++) { data = new Object[] {i}; Object[] dataClone = {i}; when(rowMeta.cloneRow(data)).thenReturn(dataClone); serviceRowListener.rowWrittenEvent(rowMeta, data); verify(sqlTransRowProducer) .putRowWait( same(rowMeta), and(eq(dataClone), not(same(data))), any(Long.class), any(TimeUnit.class)); verify(rowMeta).cloneRow(data); } doReturn(true).when(serviceTrans).isRunning(); resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep); verify(serviceTrans).stopAll(); // Verify Service Trans finished ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class); verify(serviceStep).addStepListener(serviceStepListener.capture()); serviceStepListener .getValue() .stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep); verify(sqlTransRowProducer).finished(); // Push row from service to sql Trans for (int i = 0; i < 50; i++) { Object[] row = {i}; clientRowListener.rowWrittenEvent(rowMeta, row); } transListenerCaptor.getValue().transFinished(genTrans); InOrder writeRows = inOrder(rowMeta); ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class); writeRows.verify(rowMeta).writeMeta(streamCaptor.capture()); DataOutputStream dataOutputStream = streamCaptor.getValue(); writeRows .verify(rowMeta, times(50)) .writeData(same(dataOutputStream), argThat(arrayWithSize(1))); writeRows.verifyNoMoreInteractions(); executor.waitUntilFinished(); verify(serviceTrans).waitUntilFinished(); verify(genTrans).waitUntilFinished(); }
protected void splitTrans( final TransMeta transMeta, final TransExecutionConfiguration executionConfiguration) throws KettleException { try { final TransSplitter transSplitter = new TransSplitter(transMeta); transSplitter.splitOriginalTransformation(); TransMeta master = transSplitter.getMaster(); SlaveServer masterServer = null; List<StepMeta> masterSteps = master.getTransHopSteps(false); // add transgraph of transmetas if showing is true SlaveServer slaves[] = transSplitter.getSlaveTargets(); if (executionConfiguration.isClusterShowingTransformation()) { if (masterSteps.size() > 0) // If there is something that needs to // be done on the master... { masterServer = transSplitter.getMasterServer(); addTransGraph(master); } // Then the slaves... // for (int i = 0; i < slaves.length; i++) { TransMeta slaveTrans = (TransMeta) transSplitter.getSlaveTransMap().get(slaves[i]); addTransGraph(slaveTrans); } } // Inject certain internal variables to make it more intuitive. // for (String var : Const.INTERNAL_TRANS_VARIABLES) executionConfiguration.getVariables().put(var, transMeta.getVariable(var)); for (String var : Const.INTERNAL_JOB_VARIABLES) executionConfiguration.getVariables().put(var, transMeta.getVariable(var)); // Parameters override the variables. // For the time being we're passing the parameters over the wire as variables... // TransMeta ot = transSplitter.getOriginalTransformation(); for (String param : ot.listParameters()) { String value = Const.NVL( ot.getParameterValue(param), Const.NVL(ot.getParameterDefault(param), ot.getVariable(param))); if (!Const.isEmpty(value)) { executionConfiguration.getVariables().put(param, value); } } Trans.executeClustered(transSplitter, executionConfiguration); if (executionConfiguration.isClusterPosting()) { // Now add monitors for the master and all the slave servers // if (masterServer != null) { spoon.addSpoonSlave(masterServer); for (int i = 0; i < slaves.length; i++) { spoon.addSpoonSlave(slaves[i]); } } } // OK, we should also start monitoring of the cluster in the background. // Stop them all if one goes bad. // Also clean up afterwards, close sockets, etc. // // Launch in a separate thread to prevent GUI blocking... // new Thread( new Runnable() { public void run() { Trans.monitorClusteredTransformation(transMeta.toString(), transSplitter, null); Result result = Trans.getClusteredTransformationResult( transMeta.toString(), transSplitter, null); LogWriter log = LogWriter.getInstance(); log.logBasic( transMeta.toString(), "-----------------------------------------------------"); log.logBasic( transMeta.toString(), "Got result back from clustered transformation:"); log.logBasic( transMeta.toString(), "-----------------------------------------------------"); log.logBasic(transMeta.toString(), "Errors : " + result.getNrErrors()); log.logBasic(transMeta.toString(), "Input : " + result.getNrLinesInput()); log.logBasic(transMeta.toString(), "Output : " + result.getNrLinesOutput()); log.logBasic(transMeta.toString(), "Updated : " + result.getNrLinesUpdated()); log.logBasic(transMeta.toString(), "Read : " + result.getNrLinesRead()); log.logBasic(transMeta.toString(), "Written : " + result.getNrLinesWritten()); log.logBasic(transMeta.toString(), "Rejected : " + result.getNrLinesRejected()); log.logBasic( transMeta.toString(), "-----------------------------------------------------"); } }) .start(); } catch (Exception e) { throw new KettleException(e); } }
public void executeTransformation( final TransMeta transMeta, final boolean local, final boolean remote, final boolean cluster, final boolean preview, final boolean debug, final Date replayDate, final boolean safe) throws KettleException { if (transMeta == null) { return; } // See if we need to ask for debugging information... // TransDebugMeta transDebugMeta = null; TransExecutionConfiguration executionConfiguration = null; if (preview) { executionConfiguration = spoon.getTransPreviewExecutionConfiguration(); } else if (debug) { executionConfiguration = spoon.getTransDebugExecutionConfiguration(); } else { executionConfiguration = spoon.getTransExecutionConfiguration(); } if (debug) { // See if we have debugging information stored somewhere? // transDebugMeta = transDebugMetaMap.get(transMeta); if (transDebugMeta == null) { transDebugMeta = new TransDebugMeta(transMeta); transDebugMetaMap.put(transMeta, transDebugMeta); } // Set the default number of rows to retrieve on all selected steps... // StepMeta[] selectedSteps = transMeta.getSelectedSteps(); if (selectedSteps != null && selectedSteps.length > 0) { transDebugMeta.getStepDebugMetaMap().clear(); for (StepMeta stepMeta : transMeta.getSelectedSteps()) { StepDebugMeta stepDebugMeta = new StepDebugMeta(stepMeta); stepDebugMeta.setRowCount(PropsUI.getInstance().getDefaultPreviewSize()); stepDebugMeta.setPausingOnBreakPoint(true); stepDebugMeta.setReadingFirstRows(false); transDebugMeta.getStepDebugMetaMap().put(stepMeta, stepDebugMeta); } } } else if (preview) { // See if we have preview information stored somewhere? // transDebugMeta = transPreviewMetaMap.get(transMeta); if (transDebugMeta == null) { transDebugMeta = new TransDebugMeta(transMeta); transPreviewMetaMap.put(transMeta, transDebugMeta); } // Set the default number of preview rows on all selected steps... // StepMeta[] selectedSteps = transMeta.getSelectedSteps(); if (selectedSteps != null && selectedSteps.length > 0) { transDebugMeta.getStepDebugMetaMap().clear(); for (StepMeta stepMeta : transMeta.getSelectedSteps()) { StepDebugMeta stepDebugMeta = new StepDebugMeta(stepMeta); stepDebugMeta.setRowCount(PropsUI.getInstance().getDefaultPreviewSize()); stepDebugMeta.setPausingOnBreakPoint(false); stepDebugMeta.setReadingFirstRows(true); transDebugMeta.getStepDebugMetaMap().put(stepMeta, stepDebugMeta); } } } executionConfiguration.setRepository(spoon.rep); executionConfiguration.setSafeModeEnabled(safe); int debugAnswer = TransDebugDialog.DEBUG_CONFIG; if (debug || preview) { transDebugMeta.getTransMeta().setRepository(spoon.rep); // pass repository for mappings TransDebugDialog transDebugDialog = new TransDebugDialog(spoon.getShell(), transDebugMeta); debugAnswer = transDebugDialog.open(); if (debugAnswer != TransDebugDialog.DEBUG_CANCEL) { executionConfiguration.setExecutingLocally(true); executionConfiguration.setExecutingRemotely(false); executionConfiguration.setExecutingClustered(false); } else { // If we cancel the debug dialog, we don't go further with the execution either. // return; } } else { if (transMeta.findFirstUsedClusterSchema() != null) { executionConfiguration.setExecutingLocally(false); executionConfiguration.setExecutingRemotely(false); executionConfiguration.setExecutingClustered(true); } else { executionConfiguration.setExecutingLocally(true); executionConfiguration.setExecutingRemotely(false); executionConfiguration.setExecutingClustered(false); } } Object data[] = spoon.variables.getData(); String fields[] = spoon.variables.getRowMeta().getFieldNames(); Map<String, String> variableMap = new HashMap<String, String>(); variableMap.putAll(executionConfiguration.getVariables()); // the default for (int idx = 0; idx < fields.length; idx++) { String value = executionConfiguration.getVariables().get(fields[idx]); if (Const.isEmpty(value)) value = data[idx].toString(); variableMap.put(fields[idx], value); } executionConfiguration.setVariables(variableMap); executionConfiguration.getUsedVariables(transMeta); executionConfiguration.getUsedArguments(transMeta, spoon.getArguments()); executionConfiguration.setReplayDate(replayDate); executionConfiguration.setLogLevel(spoon.getLog().getLogLevel()); boolean execConfigAnswer = true; if (debugAnswer == TransDebugDialog.DEBUG_CONFIG && replayDate == null) { TransExecutionConfigurationDialog dialog = new TransExecutionConfigurationDialog( spoon.getShell(), executionConfiguration, transMeta); execConfigAnswer = dialog.open(); } if (execConfigAnswer) { // Verify if there is at least one step specified to debug or preview... // if (debug || preview) { if (transDebugMeta.getNrOfUsedSteps() == 0) { MessageBox box = new MessageBox(spoon.getShell(), SWT.ICON_WARNING | SWT.YES | SWT.NO); box.setText(Messages.getString("Spoon.Dialog.Warning.NoPreviewOrDebugSteps.Title")); box.setMessage(Messages.getString("Spoon.Dialog.Warning.NoPreviewOrDebugSteps.Message")); int answer = box.open(); if (answer != SWT.YES) { return; } } } // addTransLog(transMeta, executionConfiguration.isExecutingLocally()); // TransLog transLog = spoon.getActiveTransLog(); TransGraph activeTransGraph = spoon.getActiveTransGraph(); // Is this a local execution? // if (executionConfiguration.isExecutingLocally()) { if (debug || preview) { activeTransGraph.debug(executionConfiguration, transDebugMeta); } else { activeTransGraph.start(executionConfiguration); } // Are we executing remotely? // } else if (executionConfiguration.isExecutingRemotely()) { if (executionConfiguration.getRemoteServer() != null) { Trans.sendToSlaveServer(transMeta, executionConfiguration, spoon.rep); monitorRemoteTrans(transMeta, executionConfiguration.getRemoteServer()); spoon.delegates.slaves.addSpoonSlave(executionConfiguration.getRemoteServer()); } else { MessageBox mb = new MessageBox(spoon.getShell(), SWT.OK | SWT.ICON_INFORMATION); mb.setMessage(Messages.getString("Spoon.Dialog.NoRemoteServerSpecified.Message")); mb.setText(Messages.getString("Spoon.Dialog.NoRemoteServerSpecified.Title")); mb.open(); } // Are we executing clustered? // } else if (executionConfiguration.isExecutingClustered()) { splitTrans(transMeta, executionConfiguration); } } }
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) { return; } if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "TransStatusServlet.Log.SniffStepRequested")); } String transName = request.getParameter("trans"); String id = request.getParameter("id"); String stepName = request.getParameter("step"); int copyNr = Const.toInt(request.getParameter("copynr"), 0); final int nrLines = Const.toInt(request.getParameter("lines"), 0); String type = Const.NVL(request.getParameter("type"), TYPE_OUTPUT); boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml")); response.setStatus(HttpServletResponse.SC_OK); if (useXML) { response.setContentType("text/xml"); response.setCharacterEncoding(Const.XML_ENCODING); } else { response.setContentType("text/html;charset=UTF-8"); } PrintWriter out = response.getWriter(); // ID is optional... // Trans trans; CarteObjectEntry entry; if (Const.isEmpty(id)) { // get the first transformation that matches... // entry = getTransformationMap().getFirstCarteObjectEntry(transName); if (entry == null) { trans = null; } else { id = entry.getId(); trans = getTransformationMap().getTransformation(entry); } } else { // Take the ID into account! // entry = new CarteObjectEntry(transName, id); trans = getTransformationMap().getTransformation(entry); } Encoder encoder = ESAPI.encoder(); if (trans != null) { // Find the step to look at... // StepInterface step = null; List<StepInterface> stepInterfaces = trans.findBaseSteps(stepName); for (int i = 0; i < stepInterfaces.size(); i++) { StepInterface look = stepInterfaces.get(i); if (look.getCopy() == copyNr) { step = look; } } if (step != null) { // Add a listener to the transformation step... // final boolean read = type.equalsIgnoreCase(TYPE_INPUT); final boolean written = type.equalsIgnoreCase(TYPE_OUTPUT) || !read; final MetaAndData metaData = new MetaAndData(); metaData.bufferRowMeta = null; metaData.bufferRowData = new ArrayList<Object[]>(); RowListener rowListener = new RowListener() { public void rowReadEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException { if (read && metaData.bufferRowData.size() < nrLines) { metaData.bufferRowMeta = rowMeta; metaData.bufferRowData.add(row); } } public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException { if (written && metaData.bufferRowData.size() < nrLines) { metaData.bufferRowMeta = rowMeta; metaData.bufferRowData.add(row); } } public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {} }; step.addRowListener(rowListener); // Wait until we have enough rows... // while (metaData.bufferRowData.size() < nrLines && step.isRunning() && !trans.isFinished() && !trans.isStopped()) { try { Thread.sleep(100); } catch (InterruptedException e) { // Ignore // break; } } // Remove the row listener // step.removeRowListener(rowListener); // Pass along the rows of data... // if (useXML) { // Send the result back as XML // response.setContentType("text/xml"); response.setCharacterEncoding(Const.XML_ENCODING); out.print(XMLHandler.getXMLHeader(Const.XML_ENCODING)); out.println(XMLHandler.openTag(XML_TAG)); if (metaData.bufferRowMeta != null) { // Row Meta data // out.println(metaData.bufferRowMeta.getMetaXML()); // Nr of lines // out.println(XMLHandler.addTagValue("nr_rows", metaData.bufferRowData.size())); // Rows of data // for (int i = 0; i < metaData.bufferRowData.size(); i++) { Object[] rowData = metaData.bufferRowData.get(i); out.println(metaData.bufferRowMeta.getDataXML(rowData)); } } out.println(XMLHandler.closeTag(XML_TAG)); } else { response.setContentType("text/html;charset=UTF-8"); out.println("<HTML>"); out.println("<HEAD>"); out.println( "<TITLE>" + BaseMessages.getString(PKG, "SniffStepServlet.SniffResults") + "</TITLE>"); out.println( "<META http-equiv=\"Refresh\" content=\"10;url=" + convertContextPath(CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">"); out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">"); out.println("</HEAD>"); out.println("<BODY>"); out.println( "<H1>" + encoder.encodeForHTML( BaseMessages.getString(PKG, "SniffStepServlet.SniffResultsForStep", stepName)) + "</H1>"); try { out.println("<table border=\"1\">"); if (metaData.bufferRowMeta != null) { // Print a header row containing all the field names... // out.print("<tr><th>#</th>"); for (ValueMetaInterface valueMeta : metaData.bufferRowMeta.getValueMetaList()) { out.print("<th>" + valueMeta.getName() + "</th>"); } out.println("</tr>"); // Now output the data rows... // for (int r = 0; r < metaData.bufferRowData.size(); r++) { Object[] rowData = metaData.bufferRowData.get(r); out.print("<tr>"); out.println("<td>" + (r + 1) + "</td>"); for (int v = 0; v < metaData.bufferRowMeta.size(); v++) { ValueMetaInterface valueMeta = metaData.bufferRowMeta.getValueMeta(v); Object valueData = rowData[v]; out.println("<td>" + valueMeta.getString(valueData) + "</td>"); } out.println("</tr>"); } } out.println("</table>"); out.println("<p>"); } catch (Exception ex) { out.println("<p>"); out.println("<pre>"); out.println(encoder.encodeForHTML(Const.getStackTracker(ex))); out.println("</pre>"); } out.println("<p>"); out.println("</BODY>"); out.println("</HTML>"); } } else { if (useXML) { out.println( new WebResult( WebResult.STRING_ERROR, BaseMessages.getString( PKG, "SniffStepServlet.Log.CoundNotFindSpecStep", stepName)) .getXML()); } else { out.println( "<H1>" + encoder.encodeForHTML( BaseMessages.getString( PKG, "SniffStepServlet.Log.CoundNotFindSpecStep", stepName)) + "</H1>"); out.println( "<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>"); } } } else { if (useXML) { out.println( new WebResult( WebResult.STRING_ERROR, BaseMessages.getString( PKG, "SniffStepServlet.Log.CoundNotFindSpecTrans", transName)) .getXML()); } else { out.println( "<H1>" + encoder.encodeForHTML( BaseMessages.getString( PKG, "SniffStepServlet.Log.CoundNotFindTrans", transName)) + "</H1>"); out.println( "<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>"); } } }
/** Create a copy of {@code trans} */ public static Trans recreateTrans(Trans trans) { return new Trans(trans.getTransMeta(), trans.getParent()); }
public void testCalculator1() throws Exception { KettleEnvironment.init(); PluginRegistry registry = PluginRegistry.getInstance(); // // Create a new transformation... // TransMeta transMeta = new TransMeta(); transMeta.setName("calculatortest1"); // // create a row generator step... // String rowGeneratorStepname = "row generator step"; RowGeneratorMeta rm = new RowGeneratorMeta(); // Set the information of the row generator. String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rm); StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm); transMeta.addStep(rowGeneratorStep); // // Generate 1 empty row // String[] strDummies = {}; int[] intDummies = {}; rm.setDefault(); rm.setFieldName(strDummies); rm.setFieldType(strDummies); rm.setValue(strDummies); rm.setFieldLength(intDummies); rm.setFieldPrecision(intDummies); rm.setRowLimit("1"); rm.setFieldFormat(strDummies); rm.setGroup(strDummies); rm.setDecimal(strDummies); // // Add calculator step. // String calculatorStepname1 = "calculator 1"; CalculatorMeta calc1 = new CalculatorMeta(); CalculatorMetaFunction[] calculations = new CalculatorMetaFunction[] { new CalculatorMetaFunction( "timestamp1", // fieldName CalculatorMetaFunction.CALC_CONSTANT, // calctype "1970-01-01 00:00:00.100100", // fieldA "", // String fieldB "", // String fieldC ValueMetaInterface.TYPE_TIMESTAMP, // valueType, 0, // int valueLength, 0, // int valuePrecision, false, // boolean removedFromResult, "", // String conversionMask, "", // String decimalSymbol, "", // String groupingSymbol, "" // String currencySymbol ), new CalculatorMetaFunction( "int1", // fieldName CalculatorMetaFunction.CALC_CONSTANT, // calctype "1", // fieldA "", // String fieldB "", // String fieldC ValueMetaInterface.TYPE_INTEGER, // valueType, 0, // int valueLength, 0, // int valuePrecision, false, // boolean removedFromResult, "", // String conversionMask, "", // String decimalSymbol, "", // String groupingSymbol, "" // String currencySymbol ), new CalculatorMetaFunction( "timestamp plus 1 day", // fieldName CalculatorMetaFunction.CALC_ADD_DAYS, // calctype "timestamp1", // fieldA "int1", // String fieldB "", // String fieldC ValueMetaInterface.TYPE_DATE, // valueType, 0, // int valueLength, 0, // int valuePrecision, false, // boolean removedFromResult, "", // String conversionMask, "", // String decimalSymbol, "", // String groupingSymbol, "" // String currencySymbol ) }; calc1.setCalculation(calculations); // String calculatorPid1 = registry.getPluginId(StepPluginType.class, calc1); StepMeta calcualtorStep1 = new StepMeta(calculatorPid1, calculatorStepname1, calc1); transMeta.addStep(calcualtorStep1); // TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, calcualtorStep1); transMeta.addTransHop(hi1); // Now execute the transformation... Trans trans = new Trans(transMeta); trans.prepareExecution(null); StepInterface si = trans.getStepInterface(calculatorStepname1, 0); RowStepCollector endRc = new RowStepCollector(); si.addRowListener(endRc); trans.startThreads(); trans.waitUntilFinished(); // Now check whether the output is still as we expect. List<RowMetaAndData> goldenImageRows = createResultData1(); List<RowMetaAndData> resultRows1 = endRc.getRowsWritten(); checkRows(resultRows1, goldenImageRows); }
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) { return; } if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "TransStatusServlet.Log.TransStatusRequested")); } String transName = request.getParameter("name"); String id = request.getParameter("id"); boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml")); int startLineNr = Const.toInt(request.getParameter("from"), 0); response.setStatus(HttpServletResponse.SC_OK); if (useXML) { response.setContentType("text/xml"); response.setCharacterEncoding(Const.XML_ENCODING); } else { response.setCharacterEncoding("UTF-8"); response.setContentType("text/html;charset=UTF-8"); } PrintWriter out = response.getWriter(); // ID is optional... // Trans trans; CarteObjectEntry entry; if (Const.isEmpty(id)) { // get the first transformation that matches... // entry = getTransformationMap().getFirstCarteObjectEntry(transName); if (entry == null) { trans = null; } else { id = entry.getId(); trans = getTransformationMap().getTransformation(entry); } } else { // Take the ID into account! // entry = new CarteObjectEntry(transName, id); trans = getTransformationMap().getTransformation(entry); } Encoder encoder = ESAPI.encoder(); if (trans != null) { String status = trans.getStatus(); int lastLineNr = KettleLogStore.getLastBufferLineNr(); String logText = KettleLogStore.getAppender() .getBuffer(trans.getLogChannel().getLogChannelId(), false, startLineNr, lastLineNr) .toString(); if (useXML) { response.setContentType("text/xml"); response.setCharacterEncoding(Const.XML_ENCODING); out.print(XMLHandler.getXMLHeader(Const.XML_ENCODING)); SlaveServerTransStatus transStatus = new SlaveServerTransStatus(transName, entry.getId(), status); transStatus.setFirstLoggingLineNr(startLineNr); transStatus.setLastLoggingLineNr(lastLineNr); for (int i = 0; i < trans.nrSteps(); i++) { StepInterface baseStep = trans.getRunThread(i); if ((baseStep.isRunning()) || baseStep.getStatus() != StepExecutionStatus.STATUS_EMPTY) { StepStatus stepStatus = new StepStatus(baseStep); transStatus.getStepStatusList().add(stepStatus); } } // The log can be quite large at times, we are going to put a base64 encoding around a // compressed stream // of bytes to handle this one. String loggingString = HttpUtil.encodeBase64ZippedString(logText); transStatus.setLoggingString(loggingString); // Also set the result object... // transStatus.setResult(trans.getResult()); // Is the transformation paused? // transStatus.setPaused(trans.isPaused()); // Send the result back as XML // try { out.println(transStatus.getXML()); } catch (KettleException e) { throw new ServletException("Unable to get the transformation status in XML format", e); } } else { response.setContentType("text/html;charset=UTF-8"); out.println("<HTML>"); out.println("<HEAD>"); out.println( "<TITLE>" + BaseMessages.getString(PKG, "TransStatusServlet.KettleTransStatus") + "</TITLE>"); out.println( "<META http-equiv=\"Refresh\" content=\"10;url=" + convertContextPath(CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">"); out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">"); out.println("</HEAD>"); out.println("<BODY>"); out.println( "<H1>" + encoder.encodeForHTML( BaseMessages.getString(PKG, "TransStatusServlet.TopTransStatus", transName)) + "</H1>"); try { out.println("<table border=\"1\">"); out.print( "<tr> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.TransName") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.CarteObjectId") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.TransStatus") + "</th> </tr>"); out.print("<tr>"); out.print("<td>" + encoder.encodeForHTML(transName) + "</td>"); out.print("<td>" + encoder.encodeForHTML(id) + "</td>"); out.print("<td>" + encoder.encodeForHTML(status) + "</td>"); out.print("</tr>"); out.print("</table>"); out.print("<p>"); // Get the transformation image // // out.print("<a href=\"" + convertContextPath(GetTransImageServlet.CONTEXT_PATH) + // "?name=" + // URLEncoder.encode(transName, "UTF-8") + "&id="+id+"\">" // + BaseMessages.getString(PKG, "TransStatusServlet.GetTransImage") + "</a>"); Point max = trans.getTransMeta().getMaximum(); max.x += 20; max.y += 20; out.print( "<iframe height=\"" + max.y + "\" width=\"" + max.x + "\" seamless src=\"" + convertContextPath(GetTransImageServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\"></iframe>"); out.print("<p>"); if ((trans.isFinished() && trans.isRunning()) || (!trans.isRunning() && !trans.isPreparing() && !trans.isInitializing())) { out.print( "<a href=\"" + convertContextPath(StartTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.StartTrans") + "</a>"); out.print("<p>"); out.print( "<a href=\"" + convertContextPath(PrepareExecutionTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.PrepareTrans") + "</a><br>"); } else if (trans.isRunning()) { out.print( "<a href=\"" + convertContextPath(PauseTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "PauseStatusServlet.PauseResumeTrans") + "</a><br>"); out.print( "<a href=\"" + convertContextPath(StopTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.StopTrans") + "</a>"); out.print("<p>"); } out.print( "<a href=\"" + convertContextPath(CleanupTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.CleanupTrans") + "</a>"); out.print("<p>"); out.println("<table border=\"1\">"); out.print( "<tr> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Stepname") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.CopyNr") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Read") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Written") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Input") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Output") + "</th> " + "<th>" + BaseMessages.getString(PKG, "TransStatusServlet.Updated") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Rejected") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Errors") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Active") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Time") + "</th> " + "<th>" + BaseMessages.getString(PKG, "TransStatusServlet.Speed") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.prinout") + "</th> </tr>"); for (int i = 0; i < trans.nrSteps(); i++) { StepInterface step = trans.getRunThread(i); if ((step.isRunning()) || step.getStatus() != StepExecutionStatus.STATUS_EMPTY) { StepStatus stepStatus = new StepStatus(step); boolean snif = false; if (step.isRunning() && !step.isStopped() && !step.isPaused()) { snif = true; String sniffLink = " <a href=\"" + convertContextPath(SniffStepServlet.CONTEXT_PATH) + "?trans=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "&lines=50" + "©nr=" + step.getCopy() + "&type=" + SniffStepServlet.TYPE_OUTPUT + "&step=" + URLEncoder.encode(step.getStepname(), "UTF-8") + "\">" + encoder.encodeForHTML(stepStatus.getStepname()) + "</a>"; stepStatus.setStepname(sniffLink); } out.print(stepStatus.getHTMLTableRow(snif)); } } out.println("</table>"); out.println("<p>"); out.print( "<a href=\"" + convertContextPath(GetTransStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "&xml=y\">" + BaseMessages.getString(PKG, "TransStatusServlet.ShowAsXml") + "</a><br>"); out.print( "<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><br>"); out.print( "<p><a href=\"" + convertContextPath(GetTransStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.Refresh") + "</a>"); // Put the logging below that. out.println("<p>"); out.println( "<textarea id=\"translog\" cols=\"120\" rows=\"20\" " + "wrap=\"off\" name=\"Transformation log\" readonly=\"readonly\">" + encoder.encodeForHTML(logText) + "</textarea>"); out.println("<script type=\"text/javascript\"> "); out.println(" translog.scrollTop=translog.scrollHeight; "); out.println("</script> "); out.println("<p>"); } catch (Exception ex) { out.println("<p>"); out.println("<pre>"); out.println(encoder.encodeForHTML(Const.getStackTracker(ex))); out.println("</pre>"); } out.println("<p>"); out.println("</BODY>"); out.println("</HTML>"); } } else { if (useXML) { out.println( new WebResult( WebResult.STRING_ERROR, BaseMessages.getString( PKG, "TransStatusServlet.Log.CoundNotFindSpecTrans", transName))); } else { out.println( "<H1>" + encoder.encodeForHTML( BaseMessages.getString( PKG, "TransStatusServlet.Log.CoundNotFindTrans", transName)) + "</H1>"); out.println( "<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>"); } } }
/** * <div id="mindtouch"> * * <h1>/kettle/prepareExec</h1> * * <a name="GET"></a> * * <h2>GET</h2> * * <p>Prepares previously loaded transformation for execution. Method is used for preparing * previously uploaded transformation for execution by its name. * * <p><b>Example Request:</b><br> * * <pre function="syntax.xml"> * GET /kettle/prepareExec/?name=dummy-trans2&xml=Y * </pre> * * <h3>Parameters</h3> * * <table class="pentaho-table"> * <tbody> * <tr> * <th>name</th> * <th>description</th> * <th>type</th> * </tr> * <tr> * <td>name</td> * <td>Name of the transformation to be prepared for execution.</td> * <td>query</td> * </tr> * <tr> * <td>xml</td> * <td>Boolean flag which sets the output format required. Use <code>Y</code> to receive XML response.</td> * <td>boolean</td> * </tr> * <tr> * <td>id</td> * <td>Carte transformation ID of the transformation to be prepared for execution.</td> * <td>query, optional</td> * </tr> * </tbody> * </table> * * <h3>Response Body</h3> * * <table class="pentaho-table"> * <tbody> * <tr> * <td align="right">text:</td> * <td>HTML</td> * </tr> * <tr> * <td align="right">media types:</td> * <td>text/xml, text/html</td> * </tr> * </tbody> * </table> * * <p>Response XML or HTML containing operation result. When using xml=Y <code>result</code> field * indicates whether operation was successful (<code>OK</code>) or not (<code>ERROR</code>). * * <p><b>Example Response:</b> * * <pre function="syntax.xml"> * <?xml version="1.0" encoding="UTF-8"?> * <webresult> * <result>OK</result> * <message/> * <id/> * </webresult> * </pre> * * <h3>Status Codes</h3> * * <table class="pentaho-table"> * <tbody> * <tr> * <th>code</th> * <th>description</th> * </tr> * <tr> * <td>200</td> * <td>Request was processed.</td> * </tr> * <tr> * <td>500</td> * <td>Internal server error occurs during request processing.</td> * </tr> * </tbody> * </table> * * </div> */ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) { return; } if (log.isDebug()) { logDebug( BaseMessages.getString( PKG, "PrepareExecutionTransServlet.TransPrepareExecutionRequested")); } String transName = request.getParameter("name"); String id = request.getParameter("id"); boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml")); response.setStatus(HttpServletResponse.SC_OK); PrintWriter out = response.getWriter(); if (useXML) { response.setContentType("text/xml"); out.print(XMLHandler.getXMLHeader(Const.XML_ENCODING)); } else { response.setCharacterEncoding("UTF-8"); response.setContentType("text/html;charset=UTF-8"); out.println("<HTML>"); out.println("<HEAD>"); out.println( "<TITLE>" + BaseMessages.getString(PKG, "PrepareExecutionTransServlet.TransPrepareExecution") + "</TITLE>"); out.println( "<META http-equiv=\"Refresh\" content=\"2;url=" + convertContextPath(GetTransStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "\">"); out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">"); out.println("</HEAD>"); out.println("<BODY>"); } try { // ID is optional... // Trans trans; CarteObjectEntry entry; if (Utils.isEmpty(id)) { // get the first transformation that matches... // entry = getTransformationMap().getFirstCarteObjectEntry(transName); if (entry == null) { trans = null; } else { id = entry.getId(); trans = getTransformationMap().getTransformation(entry); } } else { // Take the ID into account! // entry = new CarteObjectEntry(transName, id); trans = getTransformationMap().getTransformation(entry); } TransConfiguration transConfiguration = getTransformationMap().getConfiguration(entry); if (trans != null && transConfiguration != null) { TransExecutionConfiguration executionConfiguration = transConfiguration.getTransExecutionConfiguration(); // Set the appropriate logging, variables, arguments, replay date, ... // etc. trans.setArguments(executionConfiguration.getArgumentStrings()); trans.setReplayDate(executionConfiguration.getReplayDate()); trans.setSafeModeEnabled(executionConfiguration.isSafeModeEnabled()); trans.setGatheringMetrics(executionConfiguration.isGatheringMetrics()); trans.injectVariables(executionConfiguration.getVariables()); trans.setPreviousResult(executionConfiguration.getPreviousResult()); try { trans.prepareExecution(null); if (useXML) { out.println(WebResult.OK.getXML()); } else { out.println( "<H1>" + Encode.forHtml( BaseMessages.getString( PKG, "PrepareExecutionTransServlet.TransPrepared", transName)) + "</H1>"); out.println( "<a href=\"" + convertContextPath(GetTransStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToTransStatusPage") + "</a><p>"); } } catch (Exception e) { String logText = KettleLogStore.getAppender() .getBuffer(trans.getLogChannel().getLogChannelId(), true) .toString(); if (useXML) { out.println( new WebResult( WebResult.STRING_ERROR, BaseMessages.getString( PKG, "PrepareExecutionTransServlet.Error.TransInitFailed", Const.CR + logText + Const.CR + Const.getStackTracker(e)))); } else { out.println( "<H1>" + Encode.forHtml( BaseMessages.getString( PKG, "PrepareExecutionTransServlet.Log.TransNotInit", transName)) + "</H1>"); out.println("<pre>"); out.println(Encode.forHtml(logText)); out.println(Encode.forHtml(Const.getStackTracker(e))); out.println("</pre>"); out.println( "<a href=\"" + convertContextPath(GetTransStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + id + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToTransStatusPage") + "</a><p>"); } } } else { if (useXML) { out.println( new WebResult( WebResult.STRING_ERROR, BaseMessages.getString( PKG, "TransStatusServlet.Log.CoundNotFindSpecTrans", transName))); } else { out.println( "<H1>" + Encode.forHtml( BaseMessages.getString( PKG, "TransStatusServlet.Log.CoundNotFindTrans", transName)) + "</H1>"); out.println( "<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>"); } } } catch (Exception ex) { if (useXML) { out.println( new WebResult( WebResult.STRING_ERROR, BaseMessages.getString( PKG, "PrepareExecutionTransServlet.Error.UnexpectedError", Const.CR + Const.getStackTracker(ex)))); } else { out.println("<p>"); out.println("<pre>"); out.println(Encode.forHtml(Const.getStackTracker(ex))); out.println("</pre>"); } } if (!useXML) { out.println("<p>"); out.println("</BODY>"); out.println("</HTML>"); } }
/** Test case for janino step. */ public void testJaninoStep() throws Exception { KettleEnvironment.init(); // // Create a new transformation... // TransMeta transMeta = new TransMeta(); transMeta.setName("janino test"); PluginRegistry registry = PluginRegistry.getInstance(); // create an injector step... String injectorStepName = "injector step"; InjectorMeta im = new InjectorMeta(); // Set the information of the injector. String injectorPid = registry.getPluginId(StepPluginType.class, im); StepMeta injectorStep = new StepMeta(injectorPid, injectorStepName, im); transMeta.addStep(injectorStep); // // create a janino step... // String stepname = "janino"; JaninoMeta jm = new JaninoMeta(); // Set the information of the step String janinoPid = registry.getPluginId(StepPluginType.class, jm); StepMeta janinoStep = new StepMeta(janinoPid, stepname, jm); transMeta.addStep(janinoStep); jm.setDefault(); JaninoMetaFunction[] formulas = { new JaninoMetaFunction( "string", "(string==null)?null:\"string-value\"", ValueMeta.TYPE_STRING, -1, -1, "string"), new JaninoMetaFunction( "integer", "(integer==null)?null:new Long(42L)", ValueMeta.TYPE_INTEGER, -1, -1, "integer"), new JaninoMetaFunction( "number", "(number==null)?null:new Double(23.0)", ValueMeta.TYPE_NUMBER, -1, -1, "number"), new JaninoMetaFunction( "bigdecimal", "(bigdecimal==null)?null:new java.math.BigDecimal(11.0)", ValueMeta.TYPE_BIGNUMBER, -1, -1, "bigdecimal"), new JaninoMetaFunction( "date", "(date==null)?null:new java.util.Date(10000000)", ValueMeta.TYPE_DATE, -1, -1, "date"), new JaninoMetaFunction( "binary", "(binary==null)?null:new byte[]{1,2,3,4,5}", ValueMeta.TYPE_BINARY, -1, -1, "binary"), new JaninoMetaFunction( "bool", "(bool==null)?null:Boolean.TRUE", ValueMeta.TYPE_BOOLEAN, -1, -1, "bool"), }; jm.setFormula(formulas); transMeta.addTransHop(new TransHopMeta(injectorStep, janinoStep)); // // Create a dummy step // String dummyStepname = "dummy step"; DummyTransMeta dm = new DummyTransMeta(); String dummyPid = registry.getPluginId(StepPluginType.class, dm); StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm); transMeta.addStep(dummyStep); TransHopMeta hi = new TransHopMeta(janinoStep, dummyStep); transMeta.addTransHop(hi); // Now execute the transformation... Trans trans = new Trans(transMeta); trans.prepareExecution(null); StepInterface si = trans.getStepInterface(dummyStepname, 0); RowStepCollector rc = new RowStepCollector(); si.addRowListener(rc); RowProducer rp = trans.addRowProducer(injectorStepName, 0); trans.startThreads(); for (RowMetaAndData rm : createInputList()) { rp.putRow(rm.getRowMeta(), rm.getData()); } rp.finished(); trans.waitUntilFinished(); List<RowMetaAndData> checkList = createExpectedList(); List<RowMetaAndData> resultRows = rc.getRowsWritten(); checkRows(resultRows, checkList); }