/** Changes directly the filesystem attribute. */
protected void doSetPreferredLoader(FileObject fo, DataLoader loader) throws IOException {
if (loader == null) {
fo.setAttribute("NetBeansAttrAssignedLoader", null);
} else {
Class c = loader.getClass();
fo.setAttribute("NetBeansAttrAssignedLoader", c.getName());
}
}
private synchronized void reTryFailedMove(PollTableEntry entry, FileObject fileObject)
throws AxisFault {
try {
String moveToDirectoryURI = entry.getMoveAfterMoveFailure();
FileObject moveToDirectory = fsManager.resolveFile(moveToDirectoryURI);
if (!moveToDirectory.exists()) {
moveToDirectory.createFolder();
}
String prefix;
if (entry.getMoveTimestampFormat() != null) {
prefix = entry.getMoveTimestampFormat().format(new Date());
} else {
prefix = "";
}
FileObject dest = moveToDirectory.resolveFile(prefix + fileObject.getName().getBaseName());
if (log.isDebugEnabled()) {
log.debug("The failed file is moving to :" + dest.getName().getURI());
}
try {
fileObject.moveTo(
dest); // FIXME - when an exception occurs here it causes the in folder to vanish
} catch (FileSystemException e) {
handleException(
"Error moving the failed file : " + fileObject + " to " + moveToDirectoryURI, e);
}
} catch (FileSystemException e) {
handleException("Cloud not move the failed file object '" + fileObject + "'", e);
} catch (IOException e) {
handleException("Cloud not create the folder", e);
}
}
/**
* Take specified action to either move or delete the processed file, depending on the outcome
*
* @param entry the PollTableEntry for the file that has been processed
* @param fileObject the FileObject representing the file to be moved or deleted
*/
private void moveOrDeleteAfterProcessing(final PollTableEntry entry, FileObject fileObject)
throws AxisFault {
String moveToDirectoryURI = null;
try {
switch (entry.getLastPollState()) {
case PollTableEntry.SUCCSESSFUL:
if (entry.getActionAfterProcess() == PollTableEntry.MOVE) {
moveToDirectoryURI = entry.getMoveAfterProcess();
}
break;
case PollTableEntry.FAILED:
if (entry.getActionAfterFailure() == PollTableEntry.MOVE) {
moveToDirectoryURI = entry.getMoveAfterFailure();
}
break;
default:
return;
}
if (moveToDirectoryURI != null) {
FileObject moveToDirectory = fsManager.resolveFile(moveToDirectoryURI);
String prefix;
if (entry.getMoveTimestampFormat() != null) {
prefix = entry.getMoveTimestampFormat().format(new Date());
} else {
prefix = "";
}
FileObject dest = moveToDirectory.resolveFile(prefix + fileObject.getName().getBaseName());
if (log.isDebugEnabled()) {
log.debug("Moving to file :" + dest.getName().getURI());
}
try {
fileObject.moveTo(dest);
} catch (FileSystemException e) {
handleException("Error moving file : " + fileObject + " to " + moveToDirectoryURI, e);
}
} else {
try {
if (log.isDebugEnabled()) {
log.debug("Deleting file :" + fileObject);
}
fileObject.close();
if (!fileObject.delete()) {
String msg = "Cannot delete file : " + fileObject;
log.error(msg);
throw new AxisFault(msg);
}
} catch (FileSystemException e) {
log.error("Error deleting file : " + fileObject, e);
}
}
} catch (FileSystemException e) {
handleException(
"Error resolving directory to move after processing : " + moveToDirectoryURI, e);
}
}
/**
* The DicomEventListener implementation. Listen for an event from the SCP, log the file, and move
* it to the dicom-import directory for anonymization and export.
*
* @param event the event identifying the file that was received.
*/
public void dicomEventOccurred(DicomEvent event) {
if ((event.getStatus() == 0)
&& event.serviceAsString(event.getService()).equals("C_STORE_RQ")) {
File inFile = new File(event.getFilename());
Log.message(dicomImportServiceName + ": Image received: " + inFile.getName());
// Make the output directory in case it doesn't exist.
File outDir = new File(TrialConfig.basepath + TrialConfig.dicomImportDir);
outDir.mkdirs();
// Put the new file in it, using the overwrite attribute of the trial to determine
// whether duplicate SOPInstanceUIDs are to be renamed so as not to lose them.
FileObject fileObject = new FileObject(inFile);
fileObject.moveToDirectory(outDir, TrialConfig.allowOverwrite());
} else if (event.getStatus() != 0xff00)
Log.message(dicomImportServiceName + ": unexpected status: " + event.toStringNoPath());
}
private synchronized void addFailedRecord(
PollTableEntry pollTableEntry, FileObject failedObject, String timeString) {
try {
String record =
failedObject.getName().getBaseName() + VFSConstants.FAILED_RECORD_DELIMITER + timeString;
String recordFile =
pollTableEntry.getFailedRecordFileDestination()
+ pollTableEntry.getFailedRecordFileName();
File failedRecordFile = new File(recordFile);
if (!failedRecordFile.exists()) {
FileUtils.writeStringToFile(failedRecordFile, record);
if (log.isDebugEnabled()) {
log.debug("Added fail record '" + record + "' into the record file '" + recordFile + "'");
}
} else {
List<String> content = FileUtils.readLines(failedRecordFile);
if (!content.contains(record)) {
content.add(record);
}
FileUtils.writeLines(failedRecordFile, content);
}
} catch (IOException e) {
log.fatal("Failure while writing the failed records!", e);
}
}
// Look through the http-import directory and process all
// the files there, oldest first. Note that these are actual
// files, not queue elements.
private void processHttpImportFiles() {
File importDirFile = new File(TrialConfig.basepath + TrialConfig.httpImportDir);
if (!importDirFile.exists()) return;
File[] files = importDirFile.listFiles();
for (int k = 0; k < files.length; k++) {
File next = files[k];
if (next.canRead() && next.canWrite()) {
FileObject fileObject = FileObject.getObject(next);
if (preprocess(fileObject)) {
process(fileObject);
if (!queueForDatabase(fileObject))
Log.message(Quarantine.file(next, processorServiceName));
else Log.message(processorServiceName + ": Processing complete: " + next.getName());
}
// If the file still exists, then there must be a bug
// somewhere; log it and quarantine the file so we don't
// fall into an infinite loop.
if (next.exists()) {
logger.warn(
"File still in queue after processing:\n" + next + "The file will be quarantined.");
Log.message(Quarantine.file(next, processorServiceName));
}
Thread.currentThread().yield();
}
}
}
// Queue a file for the DatabaseExportService.
private boolean queueForDatabase(FileObject fileObject) {
if (TrialConfig.getDatabaseExportMode().equals("auto")) {
File databaseExportDirectoryFile = TrialConfig.getDatabaseExportDirectoryFile();
try {
ExportQueueElement eqe = ExportQueueElement.createEQE(fileObject);
eqe.queue(databaseExportDirectoryFile);
} catch (Exception e) {
Log.message(
processorServiceName
+ ": Unable to queue "
+ fileObject.getFile().getName()
+ " for database export");
logger.warn("Unable to queue " + fileObject.getFile().getName() + " for database export");
}
}
return true;
}
// Process a FileObject.
private void process(FileObject fileObject) {
if (fileObject instanceof DicomObject) process((DicomObject) fileObject);
else if (fileObject instanceof XmlObject) process((XmlObject) fileObject);
else if (fileObject instanceof ZipObject) process((ZipObject) fileObject);
else {
try {
// This is a file that can't be put into a MIRCdocument directory.
// Put the file in the bullpen to get it out of the input queue
// and to place it somewhere for the DatabaseExportService to find.
fileObject.moveToDirectory(StorageConfig.getBullpenFile());
} catch (Exception ex) {
}
}
}
public void run() {
if (log.isDebugEnabled()) {
log.debug(
"New file remove task is starting..thread id : " + Thread.currentThread().getName());
}
// there has been a failure, basically it should be a move operation
// failure. we'll re-try to move in a loop suspending with the
// configured amount of time
// we'll check if the lock is still there and if the lock is there
// then we assume that the respective file object is also there
// try to remove the folder waiting on a busy loop, if the remove operation success
// we just exit from the busy loop and mark end of the file move task.
boolean isDeletionSucceed = false;
int nextRetryDuration = pollTableEntry.getNextRetryDuration();
int count = 0;
while (!isDeletionSucceed) {
try {
reTryFailedMove(pollTableEntry, failedFileObject);
isDeletionSucceed = true;
removeTaskState = STATE_STOPPED;
} catch (AxisFault axisFault) {
removeTaskState = STATE_RUNNING;
try {
log.error(
"Remove attempt '"
+ (count++)
+ "' failed for the file '"
+ failedFileObject.getURL().toString()
+ "', next re-try will be "
+ "after '"
+ nextRetryDuration
+ "' milliseconds");
} catch (FileSystemException e) {
log.error(
"Error while retrying the file url of the file object '" + failedFileObject + "'");
}
try {
Thread.sleep(nextRetryDuration);
} catch (InterruptedException ignore) {
// ignore
}
}
}
}
private boolean isFailedRecord(FileObject fileObject, PollTableEntry entry) {
String failedFile = entry.getFailedRecordFileDestination() + entry.getFailedRecordFileName();
File file = new File(failedFile);
if (file.exists()) {
try {
List list = FileUtils.readLines(file);
for (Object aList : list) {
String str = (String) aList;
StringTokenizer st = new StringTokenizer(str, VFSConstants.FAILED_RECORD_DELIMITER);
String fileName = st.nextToken();
if (fileName != null && fileName.equals(fileObject.getName().getBaseName())) {
return true;
}
}
} catch (IOException e) {
log.fatal("Error while reading the file '" + failedFile + "'", e);
}
}
return false;
}
/**
* Process a single file through Axis2
*
* @param entry the PollTableEntry for the file (or its parent directory or archive)
* @param file the file that contains the actual message pumped into Axis2
* @throws AxisFault on error
*/
private void processFile(PollTableEntry entry, FileObject file) throws AxisFault {
try {
FileContent content = file.getContent();
String fileName = file.getName().getBaseName();
String filePath = file.getName().getPath();
metrics.incrementBytesReceived(content.getSize());
Map<String, Object> transportHeaders = new HashMap<String, Object>();
transportHeaders.put(VFSConstants.FILE_PATH, filePath);
transportHeaders.put(VFSConstants.FILE_NAME, fileName);
try {
transportHeaders.put(VFSConstants.FILE_LENGTH, content.getSize());
transportHeaders.put(VFSConstants.LAST_MODIFIED, content.getLastModifiedTime());
} catch (FileSystemException ignore) {
}
MessageContext msgContext = entry.createMessageContext();
String contentType = entry.getContentType();
if (BaseUtils.isBlank(contentType)) {
if (file.getName().getExtension().toLowerCase().endsWith(".xml")) {
contentType = "text/xml";
} else if (file.getName().getExtension().toLowerCase().endsWith(".txt")) {
contentType = "text/plain";
}
} else {
// Extract the charset encoding from the configured content type and
// set the CHARACTER_SET_ENCODING property as e.g. SOAPBuilder relies on this.
String charSetEnc = null;
try {
if (contentType != null) {
charSetEnc = new ContentType(contentType).getParameter("charset");
}
} catch (ParseException ex) {
// ignore
}
msgContext.setProperty(Constants.Configuration.CHARACTER_SET_ENCODING, charSetEnc);
}
// if the content type was not found, but the service defined it.. use it
if (contentType == null) {
if (entry.getContentType() != null) {
contentType = entry.getContentType();
} else if (VFSUtils.getProperty(content, BaseConstants.CONTENT_TYPE) != null) {
contentType = VFSUtils.getProperty(content, BaseConstants.CONTENT_TYPE);
}
}
// does the service specify a default reply file URI ?
String replyFileURI = entry.getReplyFileURI();
if (replyFileURI != null) {
msgContext.setProperty(
Constants.OUT_TRANSPORT_INFO,
new VFSOutTransportInfo(replyFileURI, entry.isFileLockingEnabled()));
}
// Determine the message builder to use
Builder builder;
if (contentType == null) {
log.debug("No content type specified. Using SOAP builder.");
builder = new SOAPBuilder();
} else {
int index = contentType.indexOf(';');
String type = index > 0 ? contentType.substring(0, index) : contentType;
builder = BuilderUtil.getBuilderFromSelector(type, msgContext);
if (builder == null) {
if (log.isDebugEnabled()) {
log.debug("No message builder found for type '" + type + "'. Falling back to SOAP.");
}
builder = new SOAPBuilder();
}
}
// set the message payload to the message context
InputStream in;
ManagedDataSource dataSource;
if (builder instanceof DataSourceMessageBuilder && entry.isStreaming()) {
in = null;
dataSource = ManagedDataSourceFactory.create(new FileObjectDataSource(file, contentType));
} else {
in = new AutoCloseInputStream(content.getInputStream());
dataSource = null;
}
try {
OMElement documentElement;
if (in != null) {
documentElement = builder.processDocument(in, contentType, msgContext);
} else {
documentElement =
((DataSourceMessageBuilder) builder)
.processDocument(dataSource, contentType, msgContext);
}
msgContext.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement));
handleIncomingMessage(
msgContext,
transportHeaders,
null, // * SOAP Action - not applicable *//
contentType);
} finally {
if (dataSource != null) {
dataSource.destroy();
}
}
if (log.isDebugEnabled()) {
log.debug("Processed file : " + file + " of Content-type : " + contentType);
}
} catch (FileSystemException e) {
handleException("Error reading file content or attributes : " + file, e);
} finally {
try {
file.close();
} catch (FileSystemException warn) {
// log.warn("Cannot close file after processing : " + file.getName().getPath(), warn);
// ignore the warning, since we handed over the stream close job to AutocloseInputstream..
}
}
}
/**
* Search for files that match the given regex pattern and create a list Then process each of
* these files and update the status of the scan on the poll table
*
* @param entry the poll table entry for the scan
* @param fileURI the file or directory to be scanned
*/
private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) {
FileObject fileObject = null;
if (log.isDebugEnabled()) {
log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
}
boolean wasError = true;
int retryCount = 0;
int maxRetryCount = entry.getMaxRetryCount();
long reconnectionTimeout = entry.getReconnectTimeout();
while (wasError) {
try {
retryCount++;
fileObject = fsManager.resolveFile(fileURI);
if (fileObject == null) {
log.error("fileObject is null");
throw new FileSystemException("fileObject is null");
}
wasError = false;
} catch (FileSystemException e) {
if (retryCount >= maxRetryCount) {
processFailure(
"Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI),
e,
entry);
return;
} else {
log.warn(
"Failed to resolve the file URI: "
+ VFSUtils.maskURLPassword(fileURI)
+ ", in attempt "
+ retryCount
+ ", "
+ e.getMessage()
+ " Retrying in "
+ reconnectionTimeout
+ " milliseconds.");
}
}
if (wasError) {
try {
Thread.sleep(reconnectionTimeout);
} catch (InterruptedException e2) {
log.error("Thread was interrupted while waiting to reconnect.", e2);
}
}
}
try {
if (fileObject.exists() && fileObject.isReadable()) {
entry.setLastPollState(PollTableEntry.NONE);
FileObject[] children = null;
try {
children = fileObject.getChildren();
} catch (FileSystemException ignore) {
}
// if this is a file that would translate to a single message
if (children == null || children.length == 0) {
boolean isFailedRecord = false;
if (entry.getMoveAfterMoveFailure() != null) {
isFailedRecord = isFailedRecord(fileObject, entry);
}
if (fileObject.getType() == FileType.FILE && !isFailedRecord) {
if (!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) {
try {
processFile(entry, fileObject);
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
metrics.incrementMessagesReceived();
} catch (AxisFault e) {
logException("Error processing File URI : " + fileObject.getName(), e);
entry.setLastPollState(PollTableEntry.FAILED);
metrics.incrementFaultsReceiving();
}
try {
moveOrDeleteAfterProcessing(entry, fileObject);
} catch (AxisFault axisFault) {
logException(
"File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved",
axisFault);
entry.setLastPollState(PollTableEntry.FAILED);
String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
addFailedRecord(entry, fileObject, timeStamp);
}
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, fileObject);
if (log.isDebugEnabled()) {
log.debug(
"Removed the lock file '"
+ fileObject.toString()
+ ".lock' of the file '"
+ fileObject.toString());
}
}
} else if (log.isDebugEnabled()) {
log.debug("Couldn't get the lock for processing the file : " + fileObject.getName());
} else if (isFailedRecord) {
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, fileObject);
}
// schedule a cleanup task if the file is there
if (fsManager.resolveFile(fileObject.getURL().toString()) != null
&& removeTaskState == STATE_STOPPED
&& entry.getMoveAfterMoveFailure() != null) {
workerPool.execute(new FileRemoveTask(entry, fileObject));
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed"
+ " record, it will not process");
}
}
}
} else {
int failCount = 0;
int successCount = 0;
if (log.isDebugEnabled()) {
log.debug("File name pattern : " + entry.getFileNamePattern());
}
for (FileObject child : children) {
boolean isFailedRecord = false;
if (entry.getMoveAfterMoveFailure() != null) {
isFailedRecord = isFailedRecord(child, entry);
}
if (log.isDebugEnabled()) {
log.debug("Matching file : " + child.getName().getBaseName());
}
if ((entry.getFileNamePattern() != null)
&& (child.getName().getBaseName().matches(entry.getFileNamePattern()))
&& (!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child)))
&& !isFailedRecord) {
try {
if (log.isDebugEnabled()) {
log.debug("Processing file :" + child);
}
processFile(entry, child);
successCount++;
// tell moveOrDeleteAfterProcessing() file was success
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
metrics.incrementMessagesReceived();
} catch (Exception e) {
logException("Error processing File URI : " + child.getName(), e);
failCount++;
// tell moveOrDeleteAfterProcessing() file failed
entry.setLastPollState(PollTableEntry.FAILED);
metrics.incrementFaultsReceiving();
}
try {
moveOrDeleteAfterProcessing(entry, child);
} catch (AxisFault axisFault) {
logException(
"File object '" + child.getURL().toString() + "'cloud not be moved", axisFault);
failCount++;
entry.setLastPollState(PollTableEntry.FAILED);
String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
addFailedRecord(entry, child, timeStamp);
}
// if there is a failure or not we'll try to release the lock
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, child);
}
} else if (!(!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled()
&& VFSUtils.acquireLock(fsManager, fileObject)))
&& log.isDebugEnabled()) {
log.debug("Couldn't get the lock for processing the file : " + child.getName());
} else if (isFailedRecord) {
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, child);
VFSUtils.releaseLock(fsManager, fileObject);
}
if (fsManager.resolveFile(child.getURL().toString()) != null
&& removeTaskState == STATE_STOPPED
&& entry.getMoveAfterMoveFailure() != null) {
workerPool.execute(new FileRemoveTask(entry, child));
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed record, it will not "
+ "process");
}
}
}
if (failCount == 0 && successCount > 0) {
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
} else if (successCount == 0 && failCount > 0) {
entry.setLastPollState(PollTableEntry.FAILED);
} else {
entry.setLastPollState(PollTableEntry.WITH_ERRORS);
}
}
// processing of this poll table entry is complete
long now = System.currentTimeMillis();
entry.setLastPollTime(now);
entry.setNextPollTime(now + entry.getPollInterval());
} else if (log.isDebugEnabled()) {
log.debug(
"Unable to access or read file or directory : "
+ VFSUtils.maskURLPassword(fileURI)
+ "."
+ " Reason: "
+ (fileObject.exists()
? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!")
: "The file does not exists!"));
}
onPollCompletion(entry);
} catch (FileSystemException e) {
processFailure(
"Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI),
e,
entry);
}
}
private void processPath(FileObject fileObject, Project project)
throws CacheCorruptedException {
File file = fileObject.getFile();
final String path = file.getPath();
try {
if (CompilerManager.MAKE_ENABLED) {
byte[] fileContent = fileObject.getContent();
// the file is assumed to exist!
final JavaDependencyCache dependencyCache =
myCompileContext.getDependencyCache().findChild(JavaDependencyCache.class);
final int newClassQName = dependencyCache.reparseClassFile(file, fileContent);
final Cache newClassesCache = dependencyCache.getNewClassesCache();
final String sourceFileName = newClassesCache.getSourceFileName(newClassQName);
final String qName = dependencyCache.resolve(newClassQName);
String relativePathToSource =
"/" + JavaMakeUtil.createRelativePathToSource(qName, sourceFileName);
putName(sourceFileName, newClassQName, relativePathToSource, path);
boolean haveToInstrument =
myAddNotNullAssertions
&& hasNotNullAnnotations(
newClassesCache, dependencyCache.getSymbolTable(), newClassQName, project);
if (haveToInstrument) {
try {
ClassReader reader = new ClassReader(fileContent, 0, fileContent.length);
ClassWriter writer = new PsiClassWriter(myProject, myIsJdk16);
if (NotNullVerifyingInstrumenter.processClassFile(reader, writer)) {
fileObject = new FileObject(file, writer.toByteArray());
}
} catch (Exception ignored) {
LOG.info(ignored);
}
}
fileObject.save();
} else {
final String _path = FileUtil.toSystemIndependentName(path);
final int dollarIndex = _path.indexOf('$');
final int tailIndex = dollarIndex >= 0 ? dollarIndex : _path.length() - ".class".length();
final int slashIndex = _path.lastIndexOf('/');
final String sourceFileName = _path.substring(slashIndex + 1, tailIndex) + ".java";
String relativePathToSource = _path.substring(myOutputDir.length(), tailIndex) + ".java";
putName(
sourceFileName,
0 /*doesn't matter here*/,
relativePathToSource.startsWith("/")
? relativePathToSource
: "/" + relativePathToSource,
path);
}
} catch (ClsFormatException e) {
final String m = e.getMessage();
String message =
CompilerBundle.message(
"error.bad.class.file.format", StringUtil.isEmpty(m) ? path : m + "\n" + path);
myCompileContext.addMessage(CompilerMessageCategory.ERROR, message, null, -1, -1);
LOG.info(e);
} catch (IOException e) {
myCompileContext.addMessage(CompilerMessageCategory.ERROR, e.getMessage(), null, -1, -1);
LOG.info(e);
} finally {
myStatistics.incClassesCount();
updateStatistics();
}
}