/**
* @param uri The URI of the file to identify
* @param request The Identification Request
* @throws CommandExecutionException When an exception happens during execution
* @throws CommandExecutionException When an exception happens during archive file input/output
*/
public final void identify(final URI uri, final IdentificationRequest request)
throws CommandExecutionException {
final String newPath = makeContainerURI("gzip", request.getFileName());
setSlash1("");
final URI newUri = URI.create(GzipUtils.getUncompressedFilename(uri.toString()));
final RequestIdentifier identifier = new RequestIdentifier(newUri);
final RequestMetaData metaData = new RequestMetaData(SIZE, TIME, uri.getPath());
final GZipIdentificationRequest gzRequest =
new GZipIdentificationRequest(metaData, identifier, getTmpDir());
GzipCompressorInputStream gzin = null;
try {
gzin = new GzipCompressorInputStream(new FileInputStream(request.getSourceFile()), true);
expandContainer(gzRequest, gzin, newPath);
} catch (IOException ioe) {
System.err.println(ioe + " (" + newPath + ")"); // continue after corrupt archive
} finally {
if (gzin != null) {
try {
gzin.close();
} catch (IOException ioe) {
throw new CommandExecutionException(ioe.getMessage(), ioe);
}
}
}
}
private IdentificationResultCollection handleContainer(
IdentificationRequest request, IdentificationResultCollection results) throws IOException {
// process a container format (ole2, odf, ooxml etc)
String containerFormat = getContainerFormat(results);
try {
if (containerFormatResolver != null && containerFormat != null) {
ContainerIdentifier containerIdentifier =
containerIdentifierFactory.getIdentifier(containerFormat);
containerIdentifier.setMaxBytesToScan(maxBytesToScan);
IdentificationResultCollection containerResults = containerIdentifier.submit(request);
droidCore.removeLowerPriorityHits(containerResults);
droidCore.checkForExtensionsMismatches(containerResults, request.getExtension());
containerResults.setFileLength(request.size());
containerResults.setRequestMetaData(request.getRequestMetaData());
return containerResults.getResults().isEmpty() ? null : containerResults;
}
// CHECKSTYLE:OFF - rules say don't catch this, but other code keeps on throwing them.
// Don't prejudice any results so far because other code isn't following 'the rules'.
} catch (Exception e) {
// CHECKSTYLE:ON
String causeMessage = "";
if (e.getCause() != null) {
causeMessage = e.getCause().getMessage();
}
final String message =
String.format(
CONTAINER_ERROR,
containerFormat,
request.getIdentifier().getUri().toString(),
e.getMessage(),
causeMessage);
log.warn(message);
}
return null;
}
private void closeRequest() {
requests.remove(request);
try {
request.close();
} catch (IOException e) {
log.error(String.format("Error closing request [%s]", request.getIdentifier().getUri()), e);
}
}
/**
* @param request The archive request to handle.
* @param results The previous identification results for the archive format.
* @return
*/
private boolean handleArchive(
IdentificationRequest request, IdentificationResultCollection results) {
boolean jobCountDecremented = false;
String archiveFormat = getArchiveFormat(results);
if (archiveFormat != null) {
results.setArchive(true);
ResourceId id = resultHandler.handle(results);
jobCounter.incrementPostProcess();
RequestIdentifier identifier = request.getIdentifier();
identifier.setResourceId(id);
if (identifier.getAncestorId() == null) {
identifier.setAncestorId(id.getId());
}
submissionQueue.add(request.getIdentifier());
jobCounter.decrement();
jobCountDecremented = true;
try {
// BNO: Does this always return the same archive handler for any given container format?
// And will it end up using the same submission gateway, or a new one with a different
// thread pool?
ArchiveHandler handler = archiveHandlerFactory.getHandler(archiveFormat);
handler.handle(request);
// CHECKSTYLE:OFF
} catch (Exception e) {
// CHECKSTYLE:ON
String causeMessage = "";
if (e.getCause() != null) {
causeMessage = e.getCause().getMessage();
}
final String message =
String.format(
ARCHIVE_ERROR,
archiveFormat,
request.getIdentifier().getUri().toString(),
e.getMessage(),
causeMessage);
log.warn(message);
resultHandler.handleError(
new IdentificationException(request, IdentificationErrorType.OTHER, e));
} finally {
submissionQueue.remove(request.getIdentifier());
jobCounter.decrementPostProcess();
}
} else {
ResourceId id = resultHandler.handle(results);
request.getIdentifier().setNodeId(id.getId());
}
return jobCountDecremented;
}
private IdentificationResultCollection handleExtensions(
IdentificationRequest request, IdentificationResultCollection results) {
IdentificationResultCollection extensionResults = results;
try {
List<IdentificationResult> resultList = results.getResults();
if (resultList != null && resultList.isEmpty()) {
// If we call matchExtensions with "true", it will match
// ALL files formats which have a given extension.
// If "false", it will only match file formats for which
// there is no other signature defined.
IdentificationResultCollection checkExtensionResults =
droidCore.matchExtensions(request, matchAllExtensions);
if (checkExtensionResults != null) {
extensionResults = checkExtensionResults;
}
} else {
droidCore.checkForExtensionsMismatches(extensionResults, request.getExtension());
}
// CHECKSTYLE:OFF - do not allow any errors in other code to
// prevent results so far from being recorded.
} catch (Exception e) {
log.error(e);
}
// CHECKSTYLE:ON
return extensionResults;
}
private void generateHash(IdentificationRequest request) throws IOException {
if (generateHash) {
try {
InputStream in = request.getSourceInputStream();
try {
String hash = hashGenerator.hash(in);
request.getRequestMetaData().setHash(hash);
} finally {
if (in != null) {
in.close();
}
}
// CHECKSTYLE:OFF - generating a hash can't prejudice any other results
} catch (Exception e) {
log.error(e);
}
// CHECKSTYLE:ON
}
}
@Override
protected void done() {
boolean jobCountDecremented = false;
try {
generateHash(request);
IdentificationResultCollection results = get();
IdentificationResultCollection containerResults = handleContainer(request, results);
if (containerResults == null) {
// no container results - process the normal results.
droidCore.removeLowerPriorityHits(results);
results = handleExtensions(request, results);
// Are we processing archive formats?
if ((processArchives || processWebArchives) && archiveFormatResolver != null) {
jobCountDecremented = handleArchive(request, results);
} else { // just process the results so far:
results.setArchive(getArchiveFormat(results) != null);
ResourceId id = resultHandler.handle(results);
request.getIdentifier().setResourceId(id);
}
} else { // we have possible container formats:
droidCore.removeLowerPriorityHits(containerResults);
containerResults = handleExtensions(request, containerResults);
ResourceId id = resultHandler.handle(containerResults);
request.getIdentifier().setResourceId(id);
}
} catch (ExecutionException e) {
final Throwable cause = e.getCause();
log.error(cause.getStackTrace(), cause);
resultHandler.handleError(
new IdentificationException(request, IdentificationErrorType.OTHER, cause));
} catch (InterruptedException e) {
log.debug(e);
} catch (IOException e) {
resultHandler.handleError(
new IdentificationException(request, IdentificationErrorType.OTHER, e));
} finally {
closeRequest();
if (!jobCountDecremented) {
jobCounter.decrement();
}
}
}
/*
** Similar to above - except that there is no second occurrence of the final left fragment - and
** the sole occurrence of this fragment is beyond the maximum offset for the sequence as a whole
** from BOF. N.B. This test passes in the previous version DROID 6.2.1
*/
@Test
public void testFinalLeftFragmentBeyondSeqMaxOffset() throws Exception {
final String sigFile = "left-frag-1200.xml";
final String fileToScan = "left1200a.ext";
final String expectedPuid = "dev/1200";
BinarySignatureIdentifier droid = new BinarySignatureIdentifier();
droid.setSignatureFile(TESTAREA + sigFile);
try {
droid.init();
} catch (SignatureParseException x) {
assertEquals("Can't parse signature file", x.getMessage());
}
File file = new File(TESTAREA + fileToScan);
assertTrue(file.exists());
URI resourceUri = file.toURI();
RequestMetaData metaData = new RequestMetaData(file.length(), file.lastModified(), fileToScan);
RequestIdentifier identifier = new RequestIdentifier(resourceUri);
identifier.setParentId(1L);
IdentificationRequest<File> request = new FileSystemIdentificationRequest(metaData, identifier);
request.open(file);
IdentificationResultCollection resultsCollection = droid.matchBinarySignatures(request);
List<IdentificationResult> results = resultsCollection.getResults();
assertEquals(0, results.size());
Iterator<IdentificationResult> iter = results.iterator();
while (iter.hasNext()) {
IdentificationResult result = iter.next();
assertEquals(expectedPuid, result.getPuid());
}
}
/**
* Shuts down the executor service and closes any in-flight requests.
*
* @throws IOException if temp files could not be deleted.
*/
public void close() throws IOException {
executorService.shutdownNow();
for (IdentificationRequest request : requests) {
request.close();
}
}
/**
* Run droid identification on file
*
* @param filePath Absolute file path
* @return Result list
* @throws FileNotFoundException
* @throws IOException
*/
@Override
public HashMap<String, String> identify(File file) throws FileNotFoundException {
HashMap<String, String> droidIdRes = new HashMap<String, String>();
InputStream in = null;
IdentificationRequest request = null;
try {
URI resourceUri = file.toURI();
in = new FileInputStream(file);
LOG.debug("Identification of resource: " + resourceUri.toString());
RequestMetaData metaData = new RequestMetaData(file.length(), file.lastModified(), file.getName());
LOG.debug("File length: " + file.length());
LOG.debug("File modified: " + file.lastModified());
LOG.debug("File name: " + file.getName());
RequestIdentifier identifier = new RequestIdentifier(resourceUri);
request = new FileSystemIdentificationRequest(metaData, identifier);
request.open(in);
IdentificationResultCollection results = bsi.matchBinarySignatures(request);
bsi.removeLowerPriorityHits(results);
if (results == null || results.getResults() == null || results.getResults().isEmpty()) {
LOG.debug("No identification result");
} else {
List<IdentificationResult> result = results.getResults();
if (result != null && !result.isEmpty()) {
for (IdentificationResult ir : result) {
String mime = ir.getMimeType();
if (mime != null && !mime.isEmpty()) {
// take first mime, ignore others
if (!droidIdRes.containsKey("mime")) {
droidIdRes.put("mime", mime);
}
}
String puid = ir.getPuid();
if (puid != null && !puid.isEmpty()) {
// take first puid, ignore others
if (!droidIdRes.containsKey("puid")) {
droidIdRes.put("puid", puid);
}
}
}
}
}
in.close();
request.close();
} catch (IOException ex) {
LOG.error("I/O Exception", ex);
} finally {
try {
in.close();
request.close();
} catch (IOException _) {
}
}
if (!droidIdRes.containsKey("mime")) {
droidIdRes.put("mime", MIME_UNKNOWN);
}
if (!droidIdRes.containsKey("puid")) {
droidIdRes.put("puid", "fmt/0");
}
return droidIdRes;
}