/** * @param uri The URI of the file to identify * @param request The Identification Request * @throws CommandExecutionException When an exception happens during execution * @throws CommandExecutionException When an exception happens during archive file input/output */ public final void identify(final URI uri, final IdentificationRequest request) throws CommandExecutionException { final String newPath = makeContainerURI("gzip", request.getFileName()); setSlash1(""); final URI newUri = URI.create(GzipUtils.getUncompressedFilename(uri.toString())); final RequestIdentifier identifier = new RequestIdentifier(newUri); final RequestMetaData metaData = new RequestMetaData(SIZE, TIME, uri.getPath()); final GZipIdentificationRequest gzRequest = new GZipIdentificationRequest(metaData, identifier, getTmpDir()); GzipCompressorInputStream gzin = null; try { gzin = new GzipCompressorInputStream(new FileInputStream(request.getSourceFile()), true); expandContainer(gzRequest, gzin, newPath); } catch (IOException ioe) { System.err.println(ioe + " (" + newPath + ")"); // continue after corrupt archive } finally { if (gzin != null) { try { gzin.close(); } catch (IOException ioe) { throw new CommandExecutionException(ioe.getMessage(), ioe); } } } }
private IdentificationResultCollection handleContainer( IdentificationRequest request, IdentificationResultCollection results) throws IOException { // process a container format (ole2, odf, ooxml etc) String containerFormat = getContainerFormat(results); try { if (containerFormatResolver != null && containerFormat != null) { ContainerIdentifier containerIdentifier = containerIdentifierFactory.getIdentifier(containerFormat); containerIdentifier.setMaxBytesToScan(maxBytesToScan); IdentificationResultCollection containerResults = containerIdentifier.submit(request); droidCore.removeLowerPriorityHits(containerResults); droidCore.checkForExtensionsMismatches(containerResults, request.getExtension()); containerResults.setFileLength(request.size()); containerResults.setRequestMetaData(request.getRequestMetaData()); return containerResults.getResults().isEmpty() ? null : containerResults; } // CHECKSTYLE:OFF - rules say don't catch this, but other code keeps on throwing them. // Don't prejudice any results so far because other code isn't following 'the rules'. } catch (Exception e) { // CHECKSTYLE:ON String causeMessage = ""; if (e.getCause() != null) { causeMessage = e.getCause().getMessage(); } final String message = String.format( CONTAINER_ERROR, containerFormat, request.getIdentifier().getUri().toString(), e.getMessage(), causeMessage); log.warn(message); } return null; }
private void closeRequest() { requests.remove(request); try { request.close(); } catch (IOException e) { log.error(String.format("Error closing request [%s]", request.getIdentifier().getUri()), e); } }
/** * @param request The archive request to handle. * @param results The previous identification results for the archive format. * @return */ private boolean handleArchive( IdentificationRequest request, IdentificationResultCollection results) { boolean jobCountDecremented = false; String archiveFormat = getArchiveFormat(results); if (archiveFormat != null) { results.setArchive(true); ResourceId id = resultHandler.handle(results); jobCounter.incrementPostProcess(); RequestIdentifier identifier = request.getIdentifier(); identifier.setResourceId(id); if (identifier.getAncestorId() == null) { identifier.setAncestorId(id.getId()); } submissionQueue.add(request.getIdentifier()); jobCounter.decrement(); jobCountDecremented = true; try { // BNO: Does this always return the same archive handler for any given container format? // And will it end up using the same submission gateway, or a new one with a different // thread pool? ArchiveHandler handler = archiveHandlerFactory.getHandler(archiveFormat); handler.handle(request); // CHECKSTYLE:OFF } catch (Exception e) { // CHECKSTYLE:ON String causeMessage = ""; if (e.getCause() != null) { causeMessage = e.getCause().getMessage(); } final String message = String.format( ARCHIVE_ERROR, archiveFormat, request.getIdentifier().getUri().toString(), e.getMessage(), causeMessage); log.warn(message); resultHandler.handleError( new IdentificationException(request, IdentificationErrorType.OTHER, e)); } finally { submissionQueue.remove(request.getIdentifier()); jobCounter.decrementPostProcess(); } } else { ResourceId id = resultHandler.handle(results); request.getIdentifier().setNodeId(id.getId()); } return jobCountDecremented; }
private IdentificationResultCollection handleExtensions( IdentificationRequest request, IdentificationResultCollection results) { IdentificationResultCollection extensionResults = results; try { List<IdentificationResult> resultList = results.getResults(); if (resultList != null && resultList.isEmpty()) { // If we call matchExtensions with "true", it will match // ALL files formats which have a given extension. // If "false", it will only match file formats for which // there is no other signature defined. IdentificationResultCollection checkExtensionResults = droidCore.matchExtensions(request, matchAllExtensions); if (checkExtensionResults != null) { extensionResults = checkExtensionResults; } } else { droidCore.checkForExtensionsMismatches(extensionResults, request.getExtension()); } // CHECKSTYLE:OFF - do not allow any errors in other code to // prevent results so far from being recorded. } catch (Exception e) { log.error(e); } // CHECKSTYLE:ON return extensionResults; }
private void generateHash(IdentificationRequest request) throws IOException { if (generateHash) { try { InputStream in = request.getSourceInputStream(); try { String hash = hashGenerator.hash(in); request.getRequestMetaData().setHash(hash); } finally { if (in != null) { in.close(); } } // CHECKSTYLE:OFF - generating a hash can't prejudice any other results } catch (Exception e) { log.error(e); } // CHECKSTYLE:ON } }
@Override protected void done() { boolean jobCountDecremented = false; try { generateHash(request); IdentificationResultCollection results = get(); IdentificationResultCollection containerResults = handleContainer(request, results); if (containerResults == null) { // no container results - process the normal results. droidCore.removeLowerPriorityHits(results); results = handleExtensions(request, results); // Are we processing archive formats? if ((processArchives || processWebArchives) && archiveFormatResolver != null) { jobCountDecremented = handleArchive(request, results); } else { // just process the results so far: results.setArchive(getArchiveFormat(results) != null); ResourceId id = resultHandler.handle(results); request.getIdentifier().setResourceId(id); } } else { // we have possible container formats: droidCore.removeLowerPriorityHits(containerResults); containerResults = handleExtensions(request, containerResults); ResourceId id = resultHandler.handle(containerResults); request.getIdentifier().setResourceId(id); } } catch (ExecutionException e) { final Throwable cause = e.getCause(); log.error(cause.getStackTrace(), cause); resultHandler.handleError( new IdentificationException(request, IdentificationErrorType.OTHER, cause)); } catch (InterruptedException e) { log.debug(e); } catch (IOException e) { resultHandler.handleError( new IdentificationException(request, IdentificationErrorType.OTHER, e)); } finally { closeRequest(); if (!jobCountDecremented) { jobCounter.decrement(); } } }
/* ** Similar to above - except that there is no second occurrence of the final left fragment - and ** the sole occurrence of this fragment is beyond the maximum offset for the sequence as a whole ** from BOF. N.B. This test passes in the previous version DROID 6.2.1 */ @Test public void testFinalLeftFragmentBeyondSeqMaxOffset() throws Exception { final String sigFile = "left-frag-1200.xml"; final String fileToScan = "left1200a.ext"; final String expectedPuid = "dev/1200"; BinarySignatureIdentifier droid = new BinarySignatureIdentifier(); droid.setSignatureFile(TESTAREA + sigFile); try { droid.init(); } catch (SignatureParseException x) { assertEquals("Can't parse signature file", x.getMessage()); } File file = new File(TESTAREA + fileToScan); assertTrue(file.exists()); URI resourceUri = file.toURI(); RequestMetaData metaData = new RequestMetaData(file.length(), file.lastModified(), fileToScan); RequestIdentifier identifier = new RequestIdentifier(resourceUri); identifier.setParentId(1L); IdentificationRequest<File> request = new FileSystemIdentificationRequest(metaData, identifier); request.open(file); IdentificationResultCollection resultsCollection = droid.matchBinarySignatures(request); List<IdentificationResult> results = resultsCollection.getResults(); assertEquals(0, results.size()); Iterator<IdentificationResult> iter = results.iterator(); while (iter.hasNext()) { IdentificationResult result = iter.next(); assertEquals(expectedPuid, result.getPuid()); } }
/** * Shuts down the executor service and closes any in-flight requests. * * @throws IOException if temp files could not be deleted. */ public void close() throws IOException { executorService.shutdownNow(); for (IdentificationRequest request : requests) { request.close(); } }
/** * Run droid identification on file * * @param filePath Absolute file path * @return Result list * @throws FileNotFoundException * @throws IOException */ @Override public HashMap<String, String> identify(File file) throws FileNotFoundException { HashMap<String, String> droidIdRes = new HashMap<String, String>(); InputStream in = null; IdentificationRequest request = null; try { URI resourceUri = file.toURI(); in = new FileInputStream(file); LOG.debug("Identification of resource: " + resourceUri.toString()); RequestMetaData metaData = new RequestMetaData(file.length(), file.lastModified(), file.getName()); LOG.debug("File length: " + file.length()); LOG.debug("File modified: " + file.lastModified()); LOG.debug("File name: " + file.getName()); RequestIdentifier identifier = new RequestIdentifier(resourceUri); request = new FileSystemIdentificationRequest(metaData, identifier); request.open(in); IdentificationResultCollection results = bsi.matchBinarySignatures(request); bsi.removeLowerPriorityHits(results); if (results == null || results.getResults() == null || results.getResults().isEmpty()) { LOG.debug("No identification result"); } else { List<IdentificationResult> result = results.getResults(); if (result != null && !result.isEmpty()) { for (IdentificationResult ir : result) { String mime = ir.getMimeType(); if (mime != null && !mime.isEmpty()) { // take first mime, ignore others if (!droidIdRes.containsKey("mime")) { droidIdRes.put("mime", mime); } } String puid = ir.getPuid(); if (puid != null && !puid.isEmpty()) { // take first puid, ignore others if (!droidIdRes.containsKey("puid")) { droidIdRes.put("puid", puid); } } } } } in.close(); request.close(); } catch (IOException ex) { LOG.error("I/O Exception", ex); } finally { try { in.close(); request.close(); } catch (IOException _) { } } if (!droidIdRes.containsKey("mime")) { droidIdRes.put("mime", MIME_UNKNOWN); } if (!droidIdRes.containsKey("puid")) { droidIdRes.put("puid", "fmt/0"); } return droidIdRes; }