/**
* Create the command line for a psql process depending on the meta information supplied.
*
* @param meta The meta data to create the command line from
* @param password Use the real password or not
* @return The string to execute.
* @throws KettleException Upon any exception
*/
public String createCommandLine(GPLoadMeta meta, boolean password) throws KettleException {
StringBuffer sb = new StringBuffer(300);
if (meta.getGploadPath() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getGploadPath()), getTransMeta());
String psqlexec = KettleVFS.getFilename(fileObject);
// sb.append('\'').append(psqlexec).append('\'');
sb.append(psqlexec);
} catch (Exception ex) {
throw new KettleException("Error retrieving sqlldr string", ex);
}
} else {
throw new KettleException("No psql application specified");
}
if (meta.getControlFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
sb.append(" -f ");
// sb.append('\'').append(KettleVFS.getFilename(fileObject)).append('\'');
sb.append(KettleVFS.getFilename(fileObject));
} catch (Exception ex) {
throw new KettleException("Error retrieving controlfile string", ex);
}
} else {
throw new KettleException("No control file specified");
}
if (meta.getLogFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getLogFile()), getTransMeta());
sb.append(" -l ");
sb.append('\'').append(KettleVFS.getFilename(fileObject)).append('\'');
} catch (Exception ex) {
throw new KettleException("Error retrieving logfile string", ex);
}
}
// hostname, port and so on are passed through the control file
//
return sb.toString();
}
protected void registerXmlPlugins() throws KettlePluginException {
for (PluginFolderInterface folder : pluginFolders) {
if (folder.isPluginXmlFolder()) {
List<FileObject> pluginXmlFiles = findPluginXmlFiles(folder.getFolder());
for (FileObject file : pluginXmlFiles) {
try {
Document document = XMLHandler.loadXMLFile(file);
Node pluginNode = XMLHandler.getSubNode(document, "plugin");
if (pluginNode != null) {
registerPluginFromXmlResource(
pluginNode,
KettleVFS.getFilename(file.getParent()),
this.getClass(),
false,
file.getParent().getURL());
}
} catch (Exception e) {
// We want to report this plugin.xml error, perhaps an XML typo or something like
// that...
//
log.logError(
"Error found while reading step plugin.xml file: " + file.getName().toString(), e);
}
}
}
}
}
private void readFileOrString() throws Exception {
if (data.file != null) {
data.filename = KettleVFS.getFilename(data.file);
// Add additional fields?
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
data.shortFilename = data.file.getName().getBaseName();
}
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
data.path = KettleVFS.getFilename(data.file.getParent());
}
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
data.hidden = data.file.isHidden();
}
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
data.extension = data.file.getName().getExtension();
}
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
}
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
data.uriName = data.file.getName().getURI();
}
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
data.rootUriName = data.file.getName().getRootURI();
}
// Check if file is empty
long fileSize = data.file.getContent().getSize();
if (meta.getSizeField() != null && meta.getSizeField().length() > 0) {
data.size = fileSize;
}
// Move file pointer ahead!
data.filenr++;
if (log.isDetailed())
logDetailed(BaseMessages.getString(PKG, "JsonInput.Log.OpeningFile", data.file.toString()));
addFileToResultFilesname(data.file);
}
parseJson();
}
public void dispose(StepMetaInterface smi, StepDataInterface sdi) {
meta = (GPLoadMeta) smi;
data = (GPLoadData) sdi;
super.dispose(smi, sdi);
if (!preview && meta.isEraseFiles()) {
// Erase the created cfg/dat files if requested. We don't erase
// the rest of the files because it would be "stupid" to erase them
// right after creation. If you don't want them, don't fill them in.
FileObject fileObject = null;
String method = meta.getLoadMethod();
if ( // GPLoadMeta.METHOD_AUTO_CONCURRENT.equals(method) ||
GPLoadMeta.METHOD_AUTO_END.equals(method)) {
/*
if ( meta.getControlFile() != null )
{
try
{
fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
fileObject.delete();
fileObject.close();
}
catch ( Exception ex )
{
logError("Error deleting control file \'" + KettleVFS.getFilename(fileObject) + "\': " + ex.getMessage());
}
}*/
}
if (GPLoadMeta.METHOD_AUTO_END.equals(method)) {
// In concurrent mode the data is written to the control file.
if (meta.getDataFile() != null) {
try {
fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getDataFile()), getTransMeta());
fileObject.delete();
fileObject.close();
} catch (Exception ex) {
logError(
"Error deleting data file \'"
+ KettleVFS.getFilename(fileObject)
+ "\': "
+ ex.getMessage(),
ex);
}
}
}
if (GPLoadMeta.METHOD_MANUAL.equals(method)) {
logBasic("Deletion of files is not compatible with \'manual load method\'");
}
}
}
private void checkFilesLocked(FileObject[] files) throws KettleException {
for (int i = 0; i < files.length && !oneFileLocked; i++) {
FileObject file = files[i];
String filename = KettleVFS.getFilename(file);
LockFile locked = new LockFile(filename);
if (locked.isLocked()) {
oneFileLocked = true;
logError(BaseMessages.getString(PKG, "JobCheckFilesLocked.Log.FileLocked", filename));
} else {
if (isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "JobCheckFilesLocked.Log.FileNotLocked", filename));
}
}
}
}
private FileObject createTemporaryShellFile(FileObject tempFile, String fileContent)
throws Exception {
// Create a unique new temporary filename in the working directory, put the script in there
// Set the permissions to execute and then run it...
//
if (tempFile != null && fileContent != null) {
try {
// flag indicates if current OS is Windows or not
boolean isWindows = Const.isWindows();
if (!isWindows) {
fileContent = replaceWinEOL(fileContent);
}
tempFile.createFile();
OutputStream outputStream = tempFile.getContent().getOutputStream();
outputStream.write(fileContent.getBytes());
outputStream.close();
if (!isWindows) {
String tempFilename = KettleVFS.getFilename(tempFile);
// Now we have to make this file executable...
// On Unix-like systems this is done using the command "/bin/chmod +x filename"
//
ProcessBuilder procBuilder = new ProcessBuilder("chmod", "+x", tempFilename);
Process proc = procBuilder.start();
// Eat/log stderr/stdout all messages in a different thread...
StreamLogger errorLogger =
new StreamLogger(log, proc.getErrorStream(), toString() + " (stderr)");
StreamLogger outputLogger =
new StreamLogger(log, proc.getInputStream(), toString() + " (stdout)");
new Thread(errorLogger).start();
new Thread(outputLogger).start();
proc.waitFor();
}
} catch (Exception e) {
throw new Exception("Unable to create temporary file to execute script", e);
}
}
return tempFile;
}
private boolean openNextFile() {
try {
if (meta.getIsInFields()) {
data.readrow = getRow(); // Grab another row ...
if (data.readrow == null) // finished processing!
{
if (isDetailed())
logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
return false;
}
if (first) {
first = false;
data.inputRowMeta = getInputRowMeta();
data.outputRowMeta = data.inputRowMeta.clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
// Create convert meta-data objects that will contain Date & Number formatters
//
data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);
if (meta.getIsInFields()) {
// Check is filename field is provided
if (Const.isEmpty(meta.getDynamicFilenameField())) {
logError(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
throw new KettleException(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
}
// cache the position of the field
if (data.indexOfFilenameField < 0) {
data.indexOfFilenameField =
data.inputRowMeta.indexOfValue(meta.getDynamicFilenameField());
if (data.indexOfFilenameField < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(PKG, "LoadFileInput.Log.ErrorFindingField")
+ "["
+ meta.getDynamicFilenameField()
+ "]"); //$NON-NLS-1$ //$NON-NLS-2$
throw new KettleException(
BaseMessages.getString(
PKG,
"LoadFileInput.Exception.CouldnotFindField",
meta.getDynamicFilenameField())); // $NON-NLS-1$ //$NON-NLS-2$
}
}
// Get the number of previous fields
data.totalpreviousfields = data.inputRowMeta.size();
}
} // end if first
// get field value
String Fieldvalue = data.inputRowMeta.getString(data.readrow, data.indexOfFilenameField);
if (isDetailed())
logDetailed(
BaseMessages.getString(
PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue));
FileObject file = null;
try {
// Source is a file.
data.file = KettleVFS.getFileObject(Fieldvalue);
} catch (Exception e) {
throw new KettleException(e);
} finally {
try {
if (file != null) file.close();
} catch (Exception e) {
}
}
} else {
if (data.filenr >= data.files.nrOfFiles()) // finished processing!
{
if (isDetailed())
logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
return false;
}
// Is this the last file?
data.last_file = (data.filenr == data.files.nrOfFiles() - 1);
data.file = (FileObject) data.files.getFile(data.filenr);
}
// Check if file is empty
data.fileSize = data.file.getContent().getSize();
// Move file pointer ahead!
data.filenr++;
if (meta.isIgnoreEmptyFile() && data.fileSize == 0) {
logError(
BaseMessages.getString(
PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName()));
openNextFile();
} else {
if (isDetailed())
logDetailed(
BaseMessages.getString(PKG, "LoadFileInput.Log.OpeningFile", data.file.toString()));
data.filename = KettleVFS.getFilename(data.file);
// Add additional fields?
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
data.shortFilename = data.file.getName().getBaseName();
}
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
data.path = KettleVFS.getFilename(data.file.getParent());
}
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
data.hidden = data.file.isHidden();
}
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
data.extension = data.file.getName().getExtension();
}
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
}
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
data.uriName = data.file.getName().getURI();
}
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
data.rootUriName = data.file.getName().getRootURI();
}
// get File content
getFileContent();
addFileToResultFilesname(data.file);
if (isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "LoadFileInput.Log.FileOpened", data.file.toString()));
}
}
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG,
"LoadFileInput.Log.UnableToOpenFile",
"" + data.filenr,
data.file.toString(),
e.toString()));
stopAll();
setErrors(1);
return false;
}
return true;
}
protected void pickFileVFS() {
FileDialog dialog = new FileDialog(shell, SWT.OPEN);
dialog.setFilterExtensions(Const.STRING_TRANS_FILTER_EXT);
dialog.setFilterNames(Const.getTransformationFilterNames());
String prevName = jobMeta.environmentSubstitute(wPath.getText());
String parentFolder = null;
try {
parentFolder =
KettleVFS.getFilename(
KettleVFS.getFileObject(jobMeta.environmentSubstitute(jobMeta.getFilename()))
.getParent());
} catch (Exception e) {
// not that important
}
if (!Utils.isEmpty(prevName)) {
try {
if (KettleVFS.fileExists(prevName)) {
dialog.setFilterPath(
KettleVFS.getFilename(KettleVFS.getFileObject(prevName).getParent()));
} else {
if (!prevName.endsWith(".ktr")) {
prevName = getEntryName(Const.trim(wPath.getText()) + ".ktr");
}
if (KettleVFS.fileExists(prevName)) {
specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
wPath.setText(prevName);
return;
} else {
// File specified doesn't exist. Ask if we should create the file...
//
MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION);
mb.setMessage(
BaseMessages.getString(
PKG, "JobTrans.Dialog.CreateTransformationQuestion.Message"));
mb.setText(
BaseMessages.getString(
PKG, "JobTrans.Dialog.CreateTransformationQuestion.Title")); // Sorry!
int answer = mb.open();
if (answer == SWT.YES) {
Spoon spoon = Spoon.getInstance();
spoon.newTransFile();
TransMeta transMeta = spoon.getActiveTransformation();
transMeta.initializeVariablesFrom(jobEntry);
transMeta.setFilename(jobMeta.environmentSubstitute(prevName));
wPath.setText(prevName);
specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
spoon.saveFile();
return;
}
}
}
} catch (Exception e) {
dialog.setFilterPath(parentFolder);
}
} else if (!Utils.isEmpty(parentFolder)) {
dialog.setFilterPath(parentFolder);
}
String fname = dialog.open();
if (fname != null) {
File file = new File(fname);
String name = file.getName();
String parentFolderSelection = file.getParentFile().toString();
if (!Utils.isEmpty(parentFolder) && parentFolder.equals(parentFolderSelection)) {
wPath.setText(getEntryName(name));
} else {
wPath.setText(fname);
}
}
}
/**
* Resolve file name.
*
* @param fileName the filename to resolve. may contain Kettle Environment variables.
* @return the data file name.
* @throws KettleException the kettle exception
*/
@SuppressWarnings("unused")
private String resolveFileName(final String fileName) throws KettleException {
final FileObject fileObject = KettleVFS.getFileObject(parent.environmentSubstitute(fileName));
return KettleVFS.getFilename(fileObject);
}
public boolean validate() {
boolean retval = false;
FileObject xmlfile = null;
FileObject DTDfile = null;
ByteArrayInputStream ba = null;
try {
if (xmlfilename != null
&& ((getDTDFilename() != null && !isInternDTD()) || (isInternDTD()))) {
xmlfile = KettleVFS.getFileObject(getXMLFilename());
if (xmlfile.exists()) {
URL xmlFile = new File(KettleVFS.getFilename(xmlfile)).toURI().toURL();
StringBuffer xmlStringbuffer = new StringBuffer("");
BufferedReader xmlBufferedReader = null;
InputStreamReader is = null;
try {
// open XML File
is = new InputStreamReader(xmlFile.openStream());
xmlBufferedReader = new BufferedReader(is);
char[] buffertXML = new char[1024];
int LenXML = -1;
while ((LenXML = xmlBufferedReader.read(buffertXML)) != -1) {
xmlStringbuffer.append(buffertXML, 0, LenXML);
}
} finally {
if (is != null) {
is.close();
}
if (xmlBufferedReader != null) {
xmlBufferedReader.close();
}
}
// Prepare parsing ...
DocumentBuilderFactory DocBuilderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder DocBuilder = DocBuilderFactory.newDocumentBuilder();
// Let's try to get XML document encoding
DocBuilderFactory.setValidating(false);
ba = new ByteArrayInputStream(xmlStringbuffer.toString().getBytes("UTF-8"));
Document xmlDocDTD = DocBuilder.parse(ba);
if (ba != null) {
ba.close();
}
String encoding = null;
if (xmlDocDTD.getXmlEncoding() == null) {
encoding = "UTF-8";
} else {
encoding = xmlDocDTD.getXmlEncoding();
}
int xmlStartDTD = xmlStringbuffer.indexOf("<!DOCTYPE");
if (isInternDTD()) {
// DTD find in the XML document
if (xmlStartDTD != -1) {
log.logBasic(
BaseMessages.getString(
PKG, "JobEntryDTDValidator.ERRORDTDFound.Label", getXMLFilename()));
} else {
setErrorMessage(
BaseMessages.getString(
PKG, "JobEntryDTDValidator.ERRORDTDNotFound.Label", getXMLFilename()));
}
} else {
// DTD in external document
// If we find an intern declaration, we remove it
DTDfile = KettleVFS.getFileObject(getDTDFilename());
if (DTDfile.exists()) {
if (xmlStartDTD != -1) {
int EndDTD = xmlStringbuffer.indexOf(">", xmlStartDTD);
// String DocTypeDTD = xmlStringbuffer.substring(xmlStartDTD, EndDTD + 1);
xmlStringbuffer.replace(xmlStartDTD, EndDTD + 1, "");
}
String xmlRootnodeDTD = xmlDocDTD.getDocumentElement().getNodeName();
String RefDTD =
"<?xml version='"
+ xmlDocDTD.getXmlVersion()
+ "' encoding='"
+ encoding
+ "'?>\n<!DOCTYPE "
+ xmlRootnodeDTD
+ " SYSTEM '"
+ KettleVFS.getFilename(DTDfile)
+ "'>\n";
int xmloffsetDTD = xmlStringbuffer.indexOf("<" + xmlRootnodeDTD);
xmlStringbuffer.replace(0, xmloffsetDTD, RefDTD);
} else {
log.logError(
BaseMessages.getString(PKG, "JobEntryDTDValidator.ERRORDTDFileNotExists.Subject"),
BaseMessages.getString(
PKG, "JobEntryDTDValidator.ERRORDTDFileNotExists.Msg", getDTDFilename()));
}
}
if (!(isInternDTD() && xmlStartDTD == -1 || (!isInternDTD() && !DTDfile.exists()))) {
// Let's parse now ...
MyErrorHandler error = new MyErrorHandler();
DocBuilderFactory.setValidating(true);
DocBuilder = DocBuilderFactory.newDocumentBuilder();
DocBuilder.setErrorHandler(error);
ba = new ByteArrayInputStream(xmlStringbuffer.toString().getBytes(encoding));
xmlDocDTD = DocBuilder.parse(ba);
if (error.errorMessage == null) {
log.logBasic(
BaseMessages.getString(PKG, "JobEntryDTDValidator.DTDValidatorOK.Subject"),
BaseMessages.getString(
PKG, "JobEntryDTDValidator.DTDValidatorOK.Label", getXMLFilename()));
// Everything is OK
retval = true;
} else {
// Invalid DTD
setNrErrors(error.nrErrors);
setErrorMessage(
BaseMessages.getString(
PKG,
"JobEntryDTDValidator.DTDValidatorKO",
getXMLFilename(),
error.nrErrors,
error.errorMessage));
}
}
} else {
if (!xmlfile.exists()) {
setErrorMessage(
BaseMessages.getString(
PKG, "JobEntryDTDValidator.FileDoesNotExist.Label", getXMLFilename()));
}
}
} else {
setErrorMessage(BaseMessages.getString(PKG, "JobEntryDTDValidator.AllFilesNotNull.Label"));
}
} catch (Exception e) {
setErrorMessage(
BaseMessages.getString(
PKG,
"JobEntryDTDValidator.ErrorDTDValidator.Label",
getXMLFilename(),
getDTDFilename(),
e.getMessage()));
} finally {
try {
if (xmlfile != null) {
xmlfile.close();
}
if (DTDfile != null) {
DTDfile.close();
}
if (ba != null) {
ba.close();
}
} catch (IOException e) {
// Ignore close errors
}
}
return retval;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (AutoDocMeta) smi;
data = (AutoDocData) sdi;
Object[] row = getRow();
if (row == null) {
if (data.filenames.isEmpty()) {
// Nothing to see here, move along!
//
setOutputDone();
return false;
}
// End of the line, create the documentation...
//
FileObject targetFile =
KettleVFS.getFileObject(environmentSubstitute(meta.getTargetFilename()));
String targetFilename = KettleVFS.getFilename(targetFile);
// Create the report builder
//
KettleReportBuilder kettleReportBuilder =
new KettleReportBuilder(this, data.filenames, KettleVFS.getFilename(targetFile), meta);
try {
// Try to get the Classic Reporting Engine to boot inside of the plugin class loader...
//
if (ClassicEngineBoot.getInstance().isBootDone() == false) {
ObjectUtilities.setClassLoader(getClass().getClassLoader());
ObjectUtilities.setClassLoaderSource(ObjectUtilities.CLASS_CONTEXT);
LibLoaderBoot.getInstance().start();
LibFontBoot.getInstance().start();
ClassicEngineBoot.getInstance().start();
}
// Do the reporting thing...
//
kettleReportBuilder.createReport();
kettleReportBuilder.render();
Object[] outputRowData = RowDataUtil.allocateRowData(data.outputRowMeta.size());
int outputIndex = 0;
outputRowData[outputIndex++] = targetFilename;
// Pass along the data to the next steps...
//
putRow(data.outputRowMeta, outputRowData);
// Add the target file to the result file list
//
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, targetFile, getTransMeta().getName(), toString());
resultFile.setComment("This file was generated by the 'Auto Documentation Output' step");
addResultFile(resultFile);
} catch (Exception e) {
throw new KettleException(
BaseMessages.getString(PKG, "AutoDoc.Exception.UnableToRenderReport"), e);
}
setOutputDone();
return false;
}
if (first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
// Get the filename field index...
//
String filenameField = environmentSubstitute(meta.getFilenameField());
data.fileNameFieldIndex = getInputRowMeta().indexOfValue(filenameField);
if (data.fileNameFieldIndex < 0) {
throw new KettleException(
BaseMessages.getString(PKG, "AutoDoc.Exception.FilenameFieldNotFound", filenameField));
}
// Get the file type field index...
//
String fileTypeField = environmentSubstitute(meta.getFileTypeField());
data.fileTypeFieldIndex = getInputRowMeta().indexOfValue(fileTypeField);
if (data.fileTypeFieldIndex < 0) {
throw new KettleException(
BaseMessages.getString(PKG, "AutoDoc.Exception.FileTypeFieldNotFound", fileTypeField));
}
data.repository = getTrans().getRepository();
if (data.repository != null) {
data.tree = data.repository.loadRepositoryDirectoryTree();
}
// Initialize the repository information handlers (images, metadata, loading, etc)
//
TransformationInformation.init(getTrans().getRepository());
JobInformation.init(getTrans().getRepository());
}
// One more transformation or job to place in the documentation.
//
String fileName = getInputRowMeta().getString(row, data.fileNameFieldIndex);
String fileType = getInputRowMeta().getString(row, data.fileTypeFieldIndex);
RepositoryObjectType objectType;
if ("Transformation".equalsIgnoreCase(fileType)) {
objectType = RepositoryObjectType.TRANSFORMATION;
} else if ("Job".equalsIgnoreCase(fileType)) {
objectType = RepositoryObjectType.JOB;
} else {
throw new KettleException(
BaseMessages.getString(PKG, "AutoDoc.Exception.UnknownFileTypeValue", fileType));
}
ReportSubjectLocation location = null;
if (getTrans().getRepository() == null) {
switch (objectType) {
case TRANSFORMATION:
location = new ReportSubjectLocation(fileName, null, null, objectType);
break;
case JOB:
location = new ReportSubjectLocation(fileName, null, null, objectType);
break;
default:
break;
}
} else {
int lastSlashIndex = fileName.lastIndexOf(RepositoryDirectory.DIRECTORY_SEPARATOR);
if (lastSlashIndex < 0) {
fileName = RepositoryDirectory.DIRECTORY_SEPARATOR + fileName;
lastSlashIndex = 0;
}
String directoryName = fileName.substring(0, lastSlashIndex + 1);
String objectName = fileName.substring(lastSlashIndex + 1);
RepositoryDirectoryInterface directory = data.tree.findDirectory(directoryName);
if (directory == null) {
throw new KettleException(
BaseMessages.getString(
PKG, "AutoDoc.Exception.RepositoryDirectoryNotFound", directoryName));
}
location = new ReportSubjectLocation(null, directory, objectName, objectType);
}
if (location == null) {
throw new KettleException(
BaseMessages.getString(
PKG, "AutoDoc.Exception.UnableToDetermineLocation", fileName, fileType));
}
if (meta.getOutputType() != OutputType.METADATA) {
// Add the file location to the list for later processing in one output report
//
data.filenames.add(location);
} else {
// Load the metadata from the transformation / job...
// Output it in one row for each input row
//
Object[] outputRow = RowDataUtil.resizeArray(row, data.outputRowMeta.size());
int outputIndex = getInputRowMeta().size();
List<AreaOwner> imageAreaList = null;
switch (location.getObjectType()) {
case TRANSFORMATION:
TransformationInformation ti = TransformationInformation.getInstance();
TransMeta transMeta = ti.getTransMeta(location);
imageAreaList = ti.getImageAreaList(location);
// TransMeta
outputRow[outputIndex++] = transMeta;
break;
case JOB:
JobInformation ji = JobInformation.getInstance();
JobMeta jobMeta = ji.getJobMeta(location);
imageAreaList = ji.getImageAreaList(location);
// TransMeta
outputRow[outputIndex++] = jobMeta;
break;
default:
break;
}
// Name
if (meta.isIncludingName()) {
outputRow[outputIndex++] = KettleFileTableModel.getName(location);
}
// Description
if (meta.isIncludingDescription()) {
outputRow[outputIndex++] = KettleFileTableModel.getDescription(location);
}
// Extended Description
if (meta.isIncludingExtendedDescription()) {
outputRow[outputIndex++] = KettleFileTableModel.getExtendedDescription(location);
}
// created
if (meta.isIncludingCreated()) {
outputRow[outputIndex++] = KettleFileTableModel.getCreation(location);
}
// modified
if (meta.isIncludingModified()) {
outputRow[outputIndex++] = KettleFileTableModel.getModification(location);
}
// image
if (meta.isIncludingImage()) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try {
BufferedImage image = KettleFileTableModel.getImage(location);
ImageIO.write(image, "png", outputStream);
outputRow[outputIndex++] = outputStream.toByteArray();
} catch (Exception e) {
throw new KettleException("Unable to serialize image to PNG", e);
} finally {
try {
outputStream.close();
} catch (IOException e) {
throw new KettleException("Unable to serialize image to PNG", e);
}
}
}
if (meta.isIncludingLoggingConfiguration()) {
outputRow[outputIndex++] = KettleFileTableModel.getLogging(location);
}
if (meta.isIncludingLastExecutionResult()) {
outputRow[outputIndex++] = KettleFileTableModel.getLogging(location);
}
if (meta.isIncludingImageAreaList()) {
outputRow[outputIndex++] = imageAreaList;
}
putRow(data.outputRowMeta, outputRow);
}
return true;
}
public void dispose(StepMetaInterface smi, StepDataInterface sdi) {
meta = (OraBulkLoaderMeta) smi;
data = (OraBulkLoaderData) sdi;
super.dispose(smi, sdi);
// close output stream (may terminate running sqlldr)
if (output != null) {
// Close the output
try {
output.close();
} catch (IOException e) {
logError("Error while closing output", e);
}
output = null;
}
// running sqlldr process must be terminated
if (sqlldrProcess != null) {
try {
int exitVal = sqlldrProcess.waitFor();
sqlldrProcess = null;
logBasic(BaseMessages.getString(PKG, "OraBulkLoader.Log.ExitValueSqlldr", "" + exitVal));
} catch (InterruptedException e) {
/* process should be destroyed */
e.printStackTrace();
if (sqlldrProcess != null) {
sqlldrProcess.destroy();
}
}
}
if (!preview && meta.isEraseFiles()) {
// Erase the created cfg/dat files if requested. We don't erase
// the rest of the files because it would be "stupid" to erase them
// right after creation. If you don't want them, don't fill them in.
FileObject fileObject = null;
String method = meta.getLoadMethod();
// OraBulkLoaderMeta.METHOD_AUTO_CONCURRENT.equals(method) ||
if (OraBulkLoaderMeta.METHOD_AUTO_END.equals(method)) {
if (meta.getControlFile() != null) {
try {
fileObject =
KettleVFS.getFileObject(
environmentSubstitute(meta.getControlFile()), getTransMeta());
fileObject.delete();
fileObject.close();
} catch (Exception ex) {
logError(
"Error deleting control file \'"
+ KettleVFS.getFilename(fileObject)
+ "\': "
+ ex.getMessage(),
ex);
}
}
}
if (OraBulkLoaderMeta.METHOD_AUTO_END.equals(method)) {
// In concurrent mode the data is written to the control file.
if (meta.getDataFile() != null) {
try {
fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getDataFile()), getTransMeta());
fileObject.delete();
fileObject.close();
} catch (Exception ex) {
logError(
"Error deleting data file \'"
+ KettleVFS.getFilename(fileObject)
+ "\': "
+ ex.getMessage(),
ex);
}
}
}
if (OraBulkLoaderMeta.METHOD_MANUAL.equals(method)) {
logBasic("Deletion of files is not compatible with \'manual load method\'");
}
}
}
/**
* Create the command line for an sqlldr process depending on the meta information supplied.
*
* @param meta The meta data to create the command line from
* @param password Use the real password or not
* @return The string to execute.
* @throws KettleException Upon any exception
*/
public String createCommandLine(OraBulkLoaderMeta meta, boolean password) throws KettleException {
StringBuilder sb = new StringBuilder(300);
if (meta.getSqlldr() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getSqlldr()), getTransMeta());
String sqlldr = KettleVFS.getFilename(fileObject);
sb.append(sqlldr);
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving sqlldr string", ex);
}
} else {
throw new KettleException("No sqlldr application specified");
}
if (meta.getControlFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
sb.append(" control=\'");
sb.append(KettleVFS.getFilename(fileObject));
sb.append("\'");
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving controlfile string", ex);
}
} else {
throw new KettleException("No control file specified");
}
if (OraBulkLoaderMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod())) {
sb.append(" data=\'-\'");
}
if (meta.getLogFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getLogFile()), getTransMeta());
sb.append(" log=\'");
sb.append(KettleVFS.getFilename(fileObject));
sb.append("\'");
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving logfile string", ex);
}
}
if (meta.getBadFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getBadFile()), getTransMeta());
sb.append(" bad=\'");
sb.append(KettleVFS.getFilename(fileObject));
sb.append("\'");
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving badfile string", ex);
}
}
if (meta.getDiscardFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getDiscardFile()), getTransMeta());
sb.append(" discard=\'");
sb.append(KettleVFS.getFilename(fileObject));
sb.append("\'");
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving discardfile string", ex);
}
}
DatabaseMeta dm = meta.getDatabaseMeta();
if (dm != null) {
String user = Const.NVL(dm.getUsername(), "");
String pass =
Const.NVL(
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(dm.getPassword())), "");
if (!password) {
pass = "******";
}
String dns = Const.NVL(dm.getDatabaseName(), "");
sb.append(" userid=")
.append(environmentSubstitute(user))
.append("/")
.append(environmentSubstitute(pass))
.append("@");
String overrideName = meta.getDbNameOverride();
if (Utils.isEmpty(Const.rtrim(overrideName))) {
sb.append(environmentSubstitute(dns));
} else {
// if the database name override is filled in, do that one.
sb.append(environmentSubstitute(overrideName));
}
} else {
throw new KettleException("No connection specified");
}
if (meta.isDirectPath()) {
sb.append(" DIRECT=TRUE");
if (getStepMeta().getCopies() > 1 || meta.isParallel()) {
sb.append(" PARALLEL=TRUE");
}
}
return sb.toString();
}
/**
* Create the command line for a sql process depending on the meta information supplied.
*
* @param meta The meta data to create the command line from
* @return The string to execute.
* @throws KettleException Upon any exception
*/
public String createCommandLine(IngresVectorwiseLoaderMeta meta) throws KettleException {
StringBuffer sb = new StringBuffer(300);
if (!Const.isEmpty(meta.getSqlPath())) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getSqlPath()), getTransMeta());
String sqlexec = Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject));
sb.append(sqlexec);
// sql @tc-dwh-test.timocom.net,tcp_ip,VW[ingres,pwd]::dwh
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving command string", ex);
}
} else {
if (meta.isUsingVwload()) {
if (isDetailed()) logDetailed("vwload defaults to system path");
sb.append("vwload");
} else {
if (isDetailed()) logDetailed("sql defaults to system path");
sb.append("sql");
}
}
DatabaseMeta dm = meta.getDatabaseMeta();
if (dm != null) {
String databaseName = environmentSubstitute(Const.NVL(dm.getDatabaseName(), ""));
String password =
Encr.decryptPasswordOptionallyEncrypted(
environmentSubstitute(Const.NVL(dm.getDatabaseInterface().getPassword(), "")));
String port =
environmentSubstitute(Const.NVL(dm.getDatabasePortNumberString(), "")).replace("7", "");
String username =
environmentSubstitute(Const.NVL(dm.getDatabaseInterface().getUsername(), ""));
String hostname =
environmentSubstitute(Const.NVL(dm.getDatabaseInterface().getHostname(), ""));
String schemaTable =
dm.getQuotedSchemaTableCombination(null, environmentSubstitute(meta.getTablename()));
String encoding = environmentSubstitute(Const.NVL(meta.getEncoding(), ""));
String fifoFile =
Const.optionallyQuoteStringByOS(
environmentSubstitute(Const.NVL(meta.getFifoFileName(), "")));
String errorFile =
Const.optionallyQuoteStringByOS(
environmentSubstitute(Const.NVL(meta.getErrorFileName(), "")));
int maxNrErrors =
Const.toInt(environmentSubstitute(Const.NVL(meta.getMaxNrErrors(), "0")), 0);
if (meta.isUsingVwload()) {
sb.append(" -u ").append(username);
sb.append(" -P ").append(password);
sb.append(" -f ").append(meta.getDelimiter()).append("");
sb.append(" -t ").append(schemaTable);
if (!Const.isEmpty(encoding)) {
sb.append(" -C ").append(encoding);
}
if (!Const.isEmpty(errorFile)) {
sb.append(" -l ").append(errorFile);
}
if (maxNrErrors > 0) {
sb.append(" -x ").append(maxNrErrors);
}
sb.append(" ").append(databaseName);
sb.append(" ").append(fifoFile);
} else if (meta.isUseDynamicVNode()) {
// logical portname in JDBC use a 7
sb.append(" @")
.append(hostname)
.append(",")
.append(port)
.append("[")
.append(username)
.append(",")
.append(password)
.append("]::")
.append(databaseName);
} else {
// Database Name
//
sb.append(" ").append(databaseName);
if (meta.isUseAuthentication()) {
sb.append("-P").append(password);
}
}
} else {
throw new KettleException("No connection specified");
}
return sb.toString();
}
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
FileObject fileObject = null;
String realScript = null;
FileObject tempFile = null;
try {
// What's the exact command?
String[] base = null;
List<String> cmds = new ArrayList<String>();
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
}
if (insertScript) {
realScript = environmentSubstitute(script);
} else {
String realFilename = environmentSubstitute(getFilename());
fileObject = KettleVFS.getFileObject(realFilename, this);
}
if (Const.getOS().equals("Windows 95")) {
base = new String[] {"command.com", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else if (Const.getOS().startsWith("Windows")) {
base = new String[] {"cmd.exe", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else {
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
base = new String[] {KettleVFS.getFilename(fileObject)};
}
// Construct the arguments...
if (argFromPrevious && cmdRows != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
}
} else if (args != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
for (int i = 0; i < args.length; i++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
for (int i = 0; i < args.length; i++) {
cmds.add(args[i]);
}
}
}
StringBuffer command = new StringBuffer();
Iterator<String> it = cmds.iterator();
boolean first = true;
while (it.hasNext()) {
if (!first) {
command.append(' ');
} else {
first = false;
}
command.append(it.next());
}
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
}
// Build the environment variable list...
ProcessBuilder procBuilder = new ProcessBuilder(cmds);
Map<String, String> env = procBuilder.environment();
String[] variables = listVariables();
for (int i = 0; i < variables.length; i++) {
env.put(variables[i], getVariable(variables[i]));
}
if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
String vfsFilename = environmentSubstitute(getWorkDirectory());
File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
procBuilder.directory(file);
}
Process proc = procBuilder.start();
// any error message?
StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);
// any output?
StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");
// kick them off
Thread errorLoggerThread = new Thread(errorLogger);
errorLoggerThread.start();
Thread outputLoggerThread = new Thread(outputLogger);
outputLoggerThread.start();
proc.waitFor();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
}
// What's the exit status?
result.setExitStatus(proc.exitValue());
if (result.getExitStatus() != 0) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobShell.ExitStatus",
environmentSubstitute(getFilename()),
"" + result.getExitStatus()));
}
result.setNrErrors(1);
}
// wait until loggers read all data from stdout and stderr
errorLoggerThread.join();
outputLoggerThread.join();
// close the streams
// otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
proc.getErrorStream().close();
proc.getOutputStream().close();
} catch (IOException ioe) {
logError(
BaseMessages.getString(
PKG,
"JobShell.ErrorRunningShell",
environmentSubstitute(getFilename()),
ioe.toString()),
ioe);
result.setNrErrors(1);
} catch (InterruptedException ie) {
logError(
BaseMessages.getString(
PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
ie);
result.setNrErrors(1);
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
e);
result.setNrErrors(1);
} finally {
// If we created a temporary file, remove it...
//
if (tempFile != null) {
try {
tempFile.delete();
} catch (Exception e) {
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
}
}
}
if (result.getNrErrors() > 0) {
result.setResult(false);
} else {
result.setResult(true);
}
}
