public synchronized void openServerSocket(BaseStep baseStep) throws IOException {
this.baseStep = baseStep;
int portNumber = Integer.parseInt(baseStep.environmentSubstitute(port));
SocketRepository socketRepository = baseStep.getSocketRepository();
serverSocket =
socketRepository.openServerSocket(
portNumber, baseStep.getTransMeta().getName() + " - " + baseStep.toString());
// Add this socket to the steps server socket list
// That way, the socket can be closed during transformation cleanup
// That is called when the cluster has finished processing.
//
baseStep.getServerSockets().add(serverSocket);
}
public void _init() {
super._init();
myProjectName.setText(myOptions.getProjectName());
myProjectPath.setPath(myOptions.getProjectPath());
updateProjectPath();
}
private Object[] getRowOfData(RowMetaInterface rowMeta) throws KettleFileException {
Object[] rowData = null;
while (!baseStep.isStopped() && rowData == null) {
try {
rowData = rowMeta.readData(inputStream);
} catch (SocketTimeoutException e) {
rowData = null; // try again.
}
}
return rowData;
}
/** Close left-over sockets, streams and so on. */
public void cleanup() {
if (socket != null && socket.isConnected() && !socket.isClosed()) {
try {
if (socket != null && !socket.isOutputShutdown()) {
socket.shutdownOutput();
}
if (socket != null && !socket.isInputShutdown()) {
socket.shutdownInput();
}
if (socket != null && !socket.isClosed()) {
socket.close();
}
if (bufferedInputStream != null) {
bufferedInputStream.close();
bufferedInputStream = null;
}
if (gzipInputStream != null) {
gzipInputStream.close();
gzipInputStream = null;
}
if (inputStream != null) {
inputStream.close();
inputStream = null;
}
if (gzipOutputStream != null) {
gzipOutputStream.close();
gzipOutputStream = null;
}
if (bufferedOutputStream != null) {
bufferedOutputStream.close();
bufferedOutputStream = null;
}
if (outputStream != null) {
outputStream.close();
outputStream = null;
}
} catch (Exception e) {
baseStep.logError("Error closing socket", e);
}
}
}
public HashMap<String, Object> Do(BaseStep step)
throws PageObjectNotFoundException, UnexpectedValueException, InterruptedException,
FileNotFoundException {
step.Driver = Driver;
step.SmallWait = new WebDriverWait(Driver, 3);
step.Wait = new WebDriverWait(Driver, 10);
step.LongWait = new WebDriverWait(Driver, 30);
step.SiteUrls = SiteUrl;
step.Browser = Browser;
IsFailed = true;
HashMap<String, Object> stepReturns = new HashMap<String, Object>();
step.Execute(stepReturns);
IsFailed = false;
return stepReturns;
}
public void _commit(boolean finishChosen) throws CommitStepException {
super._commit(finishChosen);
myOptions.setProjectName(myProjectName.getText());
myOptions.setProjectPath(myProjectPath.getPath());
}
public synchronized BlockingRowSet openReaderSocket(final BaseStep baseStep)
throws IOException, KettleException {
this.baseStep = baseStep;
final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset());
// Make sure we handle the case with multiple step copies running on a
// slave...
//
rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr);
rowSet.setRemoteSlaveServerName(targetSlaveServerName);
final int portNumber = Integer.parseInt(baseStep.environmentSubstitute(port));
final String realHostname = baseStep.environmentSubstitute(hostname);
// Connect to the server socket (started during BaseStep.init())
// Because the accept() call on the server socket can be called after we
// reached this code
// it is best to build in a retry loop with a time-out here.
//
long startTime = System.currentTimeMillis();
boolean connected = false;
KettleException lastException = null;
// // timeout with retry until connected
while (!connected
&& (TIMEOUT_IN_SECONDS > (System.currentTimeMillis() - startTime) / 1000)
&& !baseStep.isStopped()) {
try {
socket = new Socket();
socket.setReuseAddress(true);
baseStep.logDetailed(
"Step variable MASTER_HOST : [" + baseStep.getVariable("MASTER_HOST") + "]");
baseStep.logDetailed(
"Opening client (reader) socket to server ["
+ Const.NVL(realHostname, "")
+ ":"
+ port
+ "]");
socket.connect(new InetSocketAddress(realHostname, portNumber), 5000);
connected = true;
if (compressingStreams) {
gzipInputStream = new GZIPInputStream(socket.getInputStream());
bufferedInputStream = new BufferedInputStream(gzipInputStream, bufferSize);
} else {
bufferedInputStream = new BufferedInputStream(socket.getInputStream(), bufferSize);
}
inputStream = new DataInputStream(bufferedInputStream);
lastException = null;
} catch (Exception e) {
lastException =
new KettleException(
"Unable to open socket to server " + realHostname + " port " + portNumber, e);
}
if (lastException != null) {
// Sleep for a while
try {
Thread.sleep(250);
} catch (InterruptedException e) {
if (socket != null) {
socket.shutdownInput();
socket.shutdownOutput();
socket.close();
baseStep.logDetailed(
"Closed connection to server socket to read rows from remote step on server "
+ realHostname
+ " port "
+ portNumber
+ " - Local port="
+ socket.getLocalPort());
}
throw new KettleException(
"Interrupted while trying to connect to server socket: " + e.toString());
}
}
}
// See if all was OK...
if (lastException != null) {
baseStep.logError("Error initialising step: " + lastException.toString());
if (socket != null) {
socket.shutdownInput();
socket.shutdownOutput();
socket.close();
baseStep.logDetailed(
"Closed connection to server socket to read rows from remote step on server "
+ realHostname
+ " port "
+ portNumber
+ " - Local port="
+ socket.getLocalPort());
}
throw lastException;
} else {
if (inputStream == null)
throw new KettleException(
"Unable to connect to the SocketWriter in the "
+ TIMEOUT_IN_SECONDS
+ "s timeout period.");
}
baseStep.logDetailed(
"Opened connection to server socket to read rows from remote step on server "
+ realHostname
+ " port "
+ portNumber
+ " - Local port="
+ socket.getLocalPort());
// Create a thread to take care of the reading from the client socket.
// The rows read will be put in a RowSet buffer.
// That buffer will hand over the rows to the step that has this RemoteStep
// object defined
// as a remote input step.
//
Runnable runnable =
new Runnable() {
public void run() {
try {
// First read the row meta data from the socket...
//
RowMetaInterface rowMeta = null;
while (!baseStep.isStopped() && rowMeta == null) {
try {
rowMeta = new RowMeta(inputStream);
} catch (SocketTimeoutException e) {
rowMeta = null;
}
}
if (rowMeta == null) {
throw new KettleEOFException(); // leave now.
}
// And a first row of data...
//
Object[] rowData = getRowOfData(rowMeta);
// Now get the data itself, row by row...
//
while (rowData != null && !baseStep.isStopped()) {
baseStep.incrementLinesInput();
baseStep.decrementLinesRead();
if (baseStep.log.isDebug())
baseStep.logDebug("Received row from remote step: " + rowMeta.getString(rowData));
baseStep.putRowTo(rowMeta, rowData, rowSet);
baseStep.decrementLinesWritten();
rowData = getRowOfData(rowMeta);
}
} catch (KettleEOFException e) {
// Nothing, we're simply done reading...
//
if (baseStep.log.isDebug())
baseStep.logDebug(
"Finished reading from remote step on server "
+ hostname
+ " port "
+ portNumber);
} catch (Exception e) {
baseStep.logError("Error reading from client socket to remote step", e);
baseStep.setErrors(1);
baseStep.stopAll();
} finally {
// Close the input socket
if (socket != null && !socket.isClosed() && !socket.isInputShutdown()) {
try {
socket.shutdownInput();
} catch (Exception e) {
baseStep.logError(
"Error shutting down input channel on client socket connection to remote step",
e);
}
}
if (socket != null && !socket.isClosed() && !socket.isOutputShutdown()) {
try {
socket.shutdownOutput();
} catch (Exception e) {
baseStep.logError(
"Error shutting down output channel on client socket connection to remote step",
e);
}
}
if (socket != null && !socket.isClosed()) {
try {
socket.close();
} catch (Exception e) {
baseStep.logError(
"Error shutting down client socket connection to remote step", e);
}
}
if (inputStream != null) {
try {
inputStream.close();
} catch (Exception e) {
baseStep.logError(
"Error closing input stream on socket connection to remote step", e);
}
inputStream = null;
}
if (bufferedInputStream != null) {
try {
bufferedInputStream.close();
} catch (Exception e) {
baseStep.logError(
"Error closing input stream on socket connection to remote step", e);
}
}
bufferedInputStream = null;
if (gzipInputStream != null) {
try {
gzipInputStream.close();
} catch (Exception e) {
baseStep.logError(
"Error closing input stream on socket connection to remote step", e);
}
}
gzipInputStream = null;
baseStep.logDetailed(
"Closed connection to server socket to read rows from remote step on server "
+ realHostname
+ " port "
+ portNumber
+ " - Local port="
+ socket.getLocalPort());
}
// signal baseStep that nothing else comes from this step.
//
rowSet.setDone();
}
};
new Thread(runnable).start();
return rowSet;
}
/**
* Open a socket for writing.
*
* @return the RowSet created that will accept the rows for the remote step
* @throws IOException
*/
public synchronized BlockingRowSet openWriterSocket() throws IOException {
// Create an output row set: to be added to BaseStep.outputRowSets
//
final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset());
// Set the details for the source and target step as well as the target slave server.
// This will help us determine the pre-calculated partition nr later in the game. (putRow())
//
rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr);
rowSet.setRemoteSlaveServerName(targetSlaveServerName);
// Start a thread that will read out the output row set and send the data over the wire...
// This will make everything else transparent, copying, distributing, including partitioning,
// etc.
//
Runnable runnable =
new Runnable() {
public void run() {
try {
// Accept the socket, create a connection
// This blocks until something comes through...
//
socket = serverSocket.accept();
// Create the output stream...
if (compressingStreams) {
gzipOutputStream = new GZIPOutputStream(socket.getOutputStream(), 50000);
bufferedOutputStream = new BufferedOutputStream(gzipOutputStream, bufferSize);
} else {
bufferedOutputStream =
new BufferedOutputStream(socket.getOutputStream(), bufferSize);
}
outputStream = new DataOutputStream(bufferedOutputStream);
baseStep.logBasic(
"Server socket accepted for port ["
+ port
+ "], reading from server "
+ targetSlaveServerName);
// get a row of data...
//
Object[] rowData = baseStep.getRowFrom(rowSet);
if (rowData != null) {
rowSet.getRowMeta().writeMeta(outputStream);
}
// Send that row to the remote step
//
while (rowData != null && !baseStep.isStopped()) {
// It's too confusing to count these twice, so decrement
baseStep.decrementLinesRead();
baseStep.decrementLinesWritten();
// Write the row to the remote step via the output stream....
//
rowSet.getRowMeta().writeData(outputStream, rowData);
baseStep.incrementLinesOutput();
if (baseStep.log.isDebug())
baseStep.logDebug(
"Sent row to port " + port + " : " + rowSet.getRowMeta().getString(rowData));
rowData = baseStep.getRowFrom(rowSet);
}
if (compressingStreams) {
outputStream.flush();
gzipOutputStream.finish();
} else {
outputStream.flush();
}
} catch (Exception e) {
baseStep.logError("Error writing to remote step", e);
baseStep.setErrors(1);
baseStep.stopAll();
} finally {
try {
socket.shutdownOutput();
} catch (Exception e) {
baseStep.logError(
"Error shutting down output channel on the server socket of remote step", e);
baseStep.setErrors(1L);
baseStep.stopAll();
}
try {
if (outputStream != null) {
outputStream.flush();
outputStream.close();
bufferedOutputStream.close();
if (gzipOutputStream != null) {
gzipOutputStream.close();
}
}
} catch (Exception e) {
baseStep.logError(
"Error shutting down output streams on the server socket of remote step", e);
baseStep.setErrors(1L);
baseStep.stopAll();
}
outputStream = null;
bufferedOutputStream = null;
gzipOutputStream = null;
//
// Now we can't close the server socket.
// This would immediately kill all the remaining data on the client side.
// The close of the server socket will happen when all the transformation in the
// cluster have finished.
// Then Trans.cleanup() will be called.
}
}
};
// Fire this off in the in a separate thread...
//
new Thread(runnable).start();
// Return the rowSet to be added to the output row set of baseStep
//
return rowSet;
}
