public synchronized void openServerSocket(BaseStep baseStep) throws IOException { this.baseStep = baseStep; int portNumber = Integer.parseInt(baseStep.environmentSubstitute(port)); SocketRepository socketRepository = baseStep.getSocketRepository(); serverSocket = socketRepository.openServerSocket( portNumber, baseStep.getTransMeta().getName() + " - " + baseStep.toString()); // Add this socket to the steps server socket list // That way, the socket can be closed during transformation cleanup // That is called when the cluster has finished processing. // baseStep.getServerSockets().add(serverSocket); }
public void _init() { super._init(); myProjectName.setText(myOptions.getProjectName()); myProjectPath.setPath(myOptions.getProjectPath()); updateProjectPath(); }
private Object[] getRowOfData(RowMetaInterface rowMeta) throws KettleFileException { Object[] rowData = null; while (!baseStep.isStopped() && rowData == null) { try { rowData = rowMeta.readData(inputStream); } catch (SocketTimeoutException e) { rowData = null; // try again. } } return rowData; }
/** Close left-over sockets, streams and so on. */ public void cleanup() { if (socket != null && socket.isConnected() && !socket.isClosed()) { try { if (socket != null && !socket.isOutputShutdown()) { socket.shutdownOutput(); } if (socket != null && !socket.isInputShutdown()) { socket.shutdownInput(); } if (socket != null && !socket.isClosed()) { socket.close(); } if (bufferedInputStream != null) { bufferedInputStream.close(); bufferedInputStream = null; } if (gzipInputStream != null) { gzipInputStream.close(); gzipInputStream = null; } if (inputStream != null) { inputStream.close(); inputStream = null; } if (gzipOutputStream != null) { gzipOutputStream.close(); gzipOutputStream = null; } if (bufferedOutputStream != null) { bufferedOutputStream.close(); bufferedOutputStream = null; } if (outputStream != null) { outputStream.close(); outputStream = null; } } catch (Exception e) { baseStep.logError("Error closing socket", e); } } }
public HashMap<String, Object> Do(BaseStep step) throws PageObjectNotFoundException, UnexpectedValueException, InterruptedException, FileNotFoundException { step.Driver = Driver; step.SmallWait = new WebDriverWait(Driver, 3); step.Wait = new WebDriverWait(Driver, 10); step.LongWait = new WebDriverWait(Driver, 30); step.SiteUrls = SiteUrl; step.Browser = Browser; IsFailed = true; HashMap<String, Object> stepReturns = new HashMap<String, Object>(); step.Execute(stepReturns); IsFailed = false; return stepReturns; }
public void _commit(boolean finishChosen) throws CommitStepException { super._commit(finishChosen); myOptions.setProjectName(myProjectName.getText()); myOptions.setProjectPath(myProjectPath.getPath()); }
public synchronized BlockingRowSet openReaderSocket(final BaseStep baseStep) throws IOException, KettleException { this.baseStep = baseStep; final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset()); // Make sure we handle the case with multiple step copies running on a // slave... // rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr); rowSet.setRemoteSlaveServerName(targetSlaveServerName); final int portNumber = Integer.parseInt(baseStep.environmentSubstitute(port)); final String realHostname = baseStep.environmentSubstitute(hostname); // Connect to the server socket (started during BaseStep.init()) // Because the accept() call on the server socket can be called after we // reached this code // it is best to build in a retry loop with a time-out here. // long startTime = System.currentTimeMillis(); boolean connected = false; KettleException lastException = null; // // timeout with retry until connected while (!connected && (TIMEOUT_IN_SECONDS > (System.currentTimeMillis() - startTime) / 1000) && !baseStep.isStopped()) { try { socket = new Socket(); socket.setReuseAddress(true); baseStep.logDetailed( "Step variable MASTER_HOST : [" + baseStep.getVariable("MASTER_HOST") + "]"); baseStep.logDetailed( "Opening client (reader) socket to server [" + Const.NVL(realHostname, "") + ":" + port + "]"); socket.connect(new InetSocketAddress(realHostname, portNumber), 5000); connected = true; if (compressingStreams) { gzipInputStream = new GZIPInputStream(socket.getInputStream()); bufferedInputStream = new BufferedInputStream(gzipInputStream, bufferSize); } else { bufferedInputStream = new BufferedInputStream(socket.getInputStream(), bufferSize); } inputStream = new DataInputStream(bufferedInputStream); lastException = null; } catch (Exception e) { lastException = new KettleException( "Unable to open socket to server " + realHostname + " port " + portNumber, e); } if (lastException != null) { // Sleep for a while try { Thread.sleep(250); } catch (InterruptedException e) { if (socket != null) { socket.shutdownInput(); socket.shutdownOutput(); socket.close(); baseStep.logDetailed( "Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort()); } throw new KettleException( "Interrupted while trying to connect to server socket: " + e.toString()); } } } // See if all was OK... if (lastException != null) { baseStep.logError("Error initialising step: " + lastException.toString()); if (socket != null) { socket.shutdownInput(); socket.shutdownOutput(); socket.close(); baseStep.logDetailed( "Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort()); } throw lastException; } else { if (inputStream == null) throw new KettleException( "Unable to connect to the SocketWriter in the " + TIMEOUT_IN_SECONDS + "s timeout period."); } baseStep.logDetailed( "Opened connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort()); // Create a thread to take care of the reading from the client socket. // The rows read will be put in a RowSet buffer. // That buffer will hand over the rows to the step that has this RemoteStep // object defined // as a remote input step. // Runnable runnable = new Runnable() { public void run() { try { // First read the row meta data from the socket... // RowMetaInterface rowMeta = null; while (!baseStep.isStopped() && rowMeta == null) { try { rowMeta = new RowMeta(inputStream); } catch (SocketTimeoutException e) { rowMeta = null; } } if (rowMeta == null) { throw new KettleEOFException(); // leave now. } // And a first row of data... // Object[] rowData = getRowOfData(rowMeta); // Now get the data itself, row by row... // while (rowData != null && !baseStep.isStopped()) { baseStep.incrementLinesInput(); baseStep.decrementLinesRead(); if (baseStep.log.isDebug()) baseStep.logDebug("Received row from remote step: " + rowMeta.getString(rowData)); baseStep.putRowTo(rowMeta, rowData, rowSet); baseStep.decrementLinesWritten(); rowData = getRowOfData(rowMeta); } } catch (KettleEOFException e) { // Nothing, we're simply done reading... // if (baseStep.log.isDebug()) baseStep.logDebug( "Finished reading from remote step on server " + hostname + " port " + portNumber); } catch (Exception e) { baseStep.logError("Error reading from client socket to remote step", e); baseStep.setErrors(1); baseStep.stopAll(); } finally { // Close the input socket if (socket != null && !socket.isClosed() && !socket.isInputShutdown()) { try { socket.shutdownInput(); } catch (Exception e) { baseStep.logError( "Error shutting down input channel on client socket connection to remote step", e); } } if (socket != null && !socket.isClosed() && !socket.isOutputShutdown()) { try { socket.shutdownOutput(); } catch (Exception e) { baseStep.logError( "Error shutting down output channel on client socket connection to remote step", e); } } if (socket != null && !socket.isClosed()) { try { socket.close(); } catch (Exception e) { baseStep.logError( "Error shutting down client socket connection to remote step", e); } } if (inputStream != null) { try { inputStream.close(); } catch (Exception e) { baseStep.logError( "Error closing input stream on socket connection to remote step", e); } inputStream = null; } if (bufferedInputStream != null) { try { bufferedInputStream.close(); } catch (Exception e) { baseStep.logError( "Error closing input stream on socket connection to remote step", e); } } bufferedInputStream = null; if (gzipInputStream != null) { try { gzipInputStream.close(); } catch (Exception e) { baseStep.logError( "Error closing input stream on socket connection to remote step", e); } } gzipInputStream = null; baseStep.logDetailed( "Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort()); } // signal baseStep that nothing else comes from this step. // rowSet.setDone(); } }; new Thread(runnable).start(); return rowSet; }
/** * Open a socket for writing. * * @return the RowSet created that will accept the rows for the remote step * @throws IOException */ public synchronized BlockingRowSet openWriterSocket() throws IOException { // Create an output row set: to be added to BaseStep.outputRowSets // final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset()); // Set the details for the source and target step as well as the target slave server. // This will help us determine the pre-calculated partition nr later in the game. (putRow()) // rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr); rowSet.setRemoteSlaveServerName(targetSlaveServerName); // Start a thread that will read out the output row set and send the data over the wire... // This will make everything else transparent, copying, distributing, including partitioning, // etc. // Runnable runnable = new Runnable() { public void run() { try { // Accept the socket, create a connection // This blocks until something comes through... // socket = serverSocket.accept(); // Create the output stream... if (compressingStreams) { gzipOutputStream = new GZIPOutputStream(socket.getOutputStream(), 50000); bufferedOutputStream = new BufferedOutputStream(gzipOutputStream, bufferSize); } else { bufferedOutputStream = new BufferedOutputStream(socket.getOutputStream(), bufferSize); } outputStream = new DataOutputStream(bufferedOutputStream); baseStep.logBasic( "Server socket accepted for port [" + port + "], reading from server " + targetSlaveServerName); // get a row of data... // Object[] rowData = baseStep.getRowFrom(rowSet); if (rowData != null) { rowSet.getRowMeta().writeMeta(outputStream); } // Send that row to the remote step // while (rowData != null && !baseStep.isStopped()) { // It's too confusing to count these twice, so decrement baseStep.decrementLinesRead(); baseStep.decrementLinesWritten(); // Write the row to the remote step via the output stream.... // rowSet.getRowMeta().writeData(outputStream, rowData); baseStep.incrementLinesOutput(); if (baseStep.log.isDebug()) baseStep.logDebug( "Sent row to port " + port + " : " + rowSet.getRowMeta().getString(rowData)); rowData = baseStep.getRowFrom(rowSet); } if (compressingStreams) { outputStream.flush(); gzipOutputStream.finish(); } else { outputStream.flush(); } } catch (Exception e) { baseStep.logError("Error writing to remote step", e); baseStep.setErrors(1); baseStep.stopAll(); } finally { try { socket.shutdownOutput(); } catch (Exception e) { baseStep.logError( "Error shutting down output channel on the server socket of remote step", e); baseStep.setErrors(1L); baseStep.stopAll(); } try { if (outputStream != null) { outputStream.flush(); outputStream.close(); bufferedOutputStream.close(); if (gzipOutputStream != null) { gzipOutputStream.close(); } } } catch (Exception e) { baseStep.logError( "Error shutting down output streams on the server socket of remote step", e); baseStep.setErrors(1L); baseStep.stopAll(); } outputStream = null; bufferedOutputStream = null; gzipOutputStream = null; // // Now we can't close the server socket. // This would immediately kill all the remaining data on the client side. // The close of the server socket will happen when all the transformation in the // cluster have finished. // Then Trans.cleanup() will be called. } } }; // Fire this off in the in a separate thread... // new Thread(runnable).start(); // Return the rowSet to be added to the output row set of baseStep // return rowSet; }