@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context;
refCounter.incrementAndGet(context.getJobId());
synchronized (SampleFileIngestModule.class) {
if (attrId == -1) {
// For this sample, make a new attribute type to use to post
// results to the blackboard. There are many standard blackboard
// artifact and attribute types and you should use them instead
// creating new ones to facilitate use of your results by other
// modules.
Case autopsyCase = Case.getCurrentCase();
SleuthkitCase sleuthkitCase = autopsyCase.getSleuthkitCase();
try {
// See if the attribute type has already been defined.
attrId = sleuthkitCase.getAttrTypeID("ATTR_SAMPLE");
if (attrId == -1) {
attrId = sleuthkitCase.addAttrType("ATTR_SAMPLE", "Sample Attribute");
}
} catch (TskCoreException ex) {
IngestServices ingestServices = IngestServices.getInstance();
Logger logger = ingestServices.getLogger(SampleIngestModuleFactory.getModuleName());
logger.log(Level.SEVERE, "Failed to create blackboard attribute", ex);
attrId = -1;
throw new IngestModuleException(ex.getLocalizedMessage());
}
}
}
}
static synchronized void reportBlackboardPostCount(long ingestJobId) {
Long refCount = refCounter.decrementAndGet(ingestJobId);
if (refCount == 0) {
Long filesCount = artifactCountsForIngestJobs.remove(ingestJobId);
String msgText = String.format("Posted %d times to the blackboard", filesCount);
IngestMessage message =
IngestMessage.createMessage(
IngestMessage.MessageType.INFO, SampleIngestModuleFactory.getModuleName(), msgText);
IngestServices.getInstance().postMessage(message);
}
}
@SuppressWarnings("deprecation")
protected void initialize() {
followUpToggle.setOnAction(
(ActionEvent t) -> {
if (followUpToggle.isSelected() == true) {
globalSelectionModel.clearAndSelect(fileID);
try {
AddDrawableTagAction.getInstance().addTag(TagUtils.getFollowUpTagName(), "");
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Failed to add follow up tag. Could not load TagName.", ex);
}
} else {
// TODO: convert this to an action!
final ImageGalleryController controller = ImageGalleryController.getDefault();
try {
// remove file from old category group
controller
.getGroupManager()
.removeFromGroup(
new GroupKey<TagName>(DrawableAttribute.TAGS, TagUtils.getFollowUpTagName()),
fileID);
List<ContentTag> contentTagsByContent =
Case.getCurrentCase()
.getServices()
.getTagsManager()
.getContentTagsByContent(file);
for (ContentTag ct : contentTagsByContent) {
if (ct.getName()
.getDisplayName()
.equals(TagUtils.getFollowUpTagName().getDisplayName())) {
Case.getCurrentCase().getServices().getTagsManager().deleteContentTag(ct);
}
}
IngestServices.getInstance()
.fireModuleDataEvent(
new ModuleDataEvent(
"TagAction", BlackboardArtifact.ARTIFACT_TYPE.TSK_TAG_FILE)); // NON-NLS
controller
.getGroupManager()
.handleFileUpdate(
FileUpdateEvent.newUpdateEvent(
Collections.singleton(fileID), DrawableAttribute.TAGS));
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Failed to delete follow up tag.", ex);
}
}
});
}
SevenZipExtractor(
IngestJobContext context,
FileTypeDetector fileTypeDetector,
String moduleDirRelative,
String moduleDirAbsolute)
throws IngestModuleException {
if (!SevenZip.isInitializedSuccessfully()
&& (SevenZip.getLastInitializationException() == null)) {
try {
SevenZip.initSevenZipFromPlatformJAR();
String platform = SevenZip.getUsedPlatform();
logger.log(
Level.INFO,
"7-Zip-JBinding library was initialized on supported platform: {0}",
platform); // NON-NLS
} catch (SevenZipNativeInitializationException e) {
logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); // NON-NLS
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errInitModule.msg",
EmbeddedFileExtractorModuleFactory.getModuleName());
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errCantInitLib",
e.getMessage());
services.postMessage(
IngestMessage.createErrorMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
throw new IngestModuleException(e.getMessage());
}
}
this.context = context;
this.fileTypeDetector = fileTypeDetector;
this.moduleDirRelative = moduleDirRelative;
this.moduleDirAbsolute = moduleDirAbsolute;
this.archiveDepthCountTree = new ArchiveDepthCountTree();
}
/** Query for history databases and add artifacts */
private void getHistory() {
FileManager fileManager = currentCase.getServices().getFileManager();
List<AbstractFile> historyFiles;
try {
historyFiles = fileManager.findFiles(dataSource, "History", "Chrome"); // NON-NLS
} catch (TskCoreException ex) {
String msg = NbBundle.getMessage(this.getClass(), "Chrome.getHistory.errMsg.errGettingFiles");
logger.log(Level.SEVERE, msg, ex);
this.addErrorMessage(this.getName() + ": " + msg);
return;
}
// get only the allocated ones, for now
List<AbstractFile> allocatedHistoryFiles = new ArrayList<>();
for (AbstractFile historyFile : historyFiles) {
if (historyFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
allocatedHistoryFiles.add(historyFile);
}
}
// log a message if we don't have any allocated history files
if (allocatedHistoryFiles.isEmpty()) {
String msg =
NbBundle.getMessage(this.getClass(), "Chrome.getHistory.errMsg.couldntFindAnyFiles");
logger.log(Level.INFO, msg);
return;
}
dataFound = true;
int j = 0;
while (j < historyFiles.size()) {
String temps =
RAImageIngestModule.getRATempPath(currentCase, "chrome")
+ File.separator
+ historyFiles.get(j).getName().toString()
+ j
+ ".db"; // NON-NLS
final AbstractFile historyFile = historyFiles.get(j++);
if (historyFile.getSize() == 0) {
continue;
}
try {
ContentUtils.writeToFile(historyFile, new File(temps));
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error writing temp sqlite db for Chrome web history artifacts.{0}",
ex); // NON-NLS
this.addErrorMessage(
NbBundle.getMessage(
this.getClass(),
"Chrome.getHistory.errMsg.errAnalyzingFile",
this.getName(),
historyFile.getName()));
continue;
}
File dbFile = new File(temps);
if (context.dataSourceIngestIsCancelled()) {
dbFile.delete();
break;
}
List<HashMap<String, Object>> tempList;
tempList = this.dbConnect(temps, historyQuery);
logger.log(
Level.INFO,
"{0}- Now getting history from {1} with {2}artifacts identified.",
new Object[] {moduleName, temps, tempList.size()}); // NON-NLS
for (HashMap<String, Object> result : tempList) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_URL.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("url").toString() != null)
? result.get("url").toString()
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
(Long.valueOf(result.get("last_visit_time").toString()) / 1000000)
- Long.valueOf("11644473600"))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_REFERRER.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("from_visit").toString() != null)
? result.get("from_visit").toString()
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_TITLE.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("title").toString() != null)
? result.get("title").toString()
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
(Util.extractDomain(
(result.get("url").toString() != null)
? result.get("url").toString()
: "")))); // NON-NLS
this.addArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY, historyFile, bbattributes);
}
dbFile.delete();
}
IngestServices.getInstance()
.fireModuleDataEvent(
new ModuleDataEvent(
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY));
}
/** Queries for login files and adds artifacts */
private void getLogin() {
FileManager fileManager = currentCase.getServices().getFileManager();
List<AbstractFile> signonFiles;
try {
signonFiles = fileManager.findFiles(dataSource, "signons.sqlite", "Chrome"); // NON-NLS
} catch (TskCoreException ex) {
String msg = NbBundle.getMessage(this.getClass(), "Chrome.getLogin.errMsg.errGettingFiles");
logger.log(Level.SEVERE, msg, ex);
this.addErrorMessage(this.getName() + ": " + msg);
return;
}
if (signonFiles.isEmpty()) {
logger.log(Level.INFO, "Didn't find any Chrome signon files."); // NON-NLS
return;
}
dataFound = true;
int j = 0;
while (j < signonFiles.size()) {
AbstractFile signonFile = signonFiles.get(j++);
if (signonFile.getSize() == 0) {
continue;
}
String temps =
RAImageIngestModule.getRATempPath(currentCase, "chrome")
+ File.separator
+ signonFile.getName().toString()
+ j
+ ".db"; // NON-NLS
try {
ContentUtils.writeToFile(signonFile, new File(temps));
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error writing temp sqlite db for Chrome login artifacts.{0}",
ex); // NON-NLS
this.addErrorMessage(
NbBundle.getMessage(
this.getClass(),
"Chrome.getLogin.errMsg.errAnalyzingFiles",
this.getName(),
signonFile.getName()));
continue;
}
File dbFile = new File(temps);
if (context.dataSourceIngestIsCancelled()) {
dbFile.delete();
break;
}
List<HashMap<String, Object>> tempList = this.dbConnect(temps, loginQuery);
logger.log(
Level.INFO,
"{0}- Now getting login information from {1} with {2}artifacts identified.",
new Object[] {moduleName, temps, tempList.size()}); // NON-NLS
for (HashMap<String, Object> result : tempList) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_URL.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("origin_url").toString() != null)
? result.get("origin_url").toString()
: ""))); // NON-NLS
// bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(),
// "Recent Activity", ((result.get("origin_url").toString() != null) ?
// EscapeUtil.decodeURL(result.get("origin_url").toString()) : "")));
// TODO Revisit usage of deprecated constructor as per TSK-583
// bbattributes.add(new
// BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(), "Recent Activity",
// "Last Visited", ((Long.valueOf(result.get("last_visit_time").toString())) / 1000000)));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
(Long.valueOf(result.get("last_visit_time").toString()) / 1000000)
- Long.valueOf("11644473600"))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_REFERRER.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("from_visit").toString() != null)
? result.get("from_visit").toString()
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("title").toString() != null)
? result.get("title").toString()
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
(Util.extractDomain(
(result.get("origin_url").toString() != null)
? result.get("url").toString()
: "")))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_USER_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("username_value").toString() != null)
? result.get("username_value").toString().replaceAll("'", "''")
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
result.get("signon_realm").toString())); // NON-NLS
this.addArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY, signonFile, bbattributes);
Collection<BlackboardAttribute> osAcctAttributes = new ArrayList<>();
osAcctAttributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_USER_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("username_value").toString() != null)
? result.get("username_value").toString().replaceAll("'", "''")
: ""))); // NON-NLS
this.addArtifact(ARTIFACT_TYPE.TSK_OS_ACCOUNT, signonFile, osAcctAttributes);
}
dbFile.delete();
}
IngestServices.getInstance()
.fireModuleDataEvent(
new ModuleDataEvent(
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY));
}
/** Queries for download files and adds artifacts */
private void getDownload() {
FileManager fileManager = currentCase.getServices().getFileManager();
List<AbstractFile> downloadFiles = null;
try {
downloadFiles = fileManager.findFiles(dataSource, "History", "Chrome"); // NON-NLS
} catch (TskCoreException ex) {
String msg =
NbBundle.getMessage(this.getClass(), "Chrome.getDownload.errMsg.errGettingFiles");
logger.log(Level.SEVERE, msg, ex);
this.addErrorMessage(this.getName() + ": " + msg);
return;
}
if (downloadFiles.isEmpty()) {
logger.log(Level.INFO, "Didn't find any Chrome download files."); // NON-NLS
return;
}
dataFound = true;
int j = 0;
while (j < downloadFiles.size()) {
AbstractFile downloadFile = downloadFiles.get(j++);
if (downloadFile.getSize() == 0) {
continue;
}
String temps =
RAImageIngestModule.getRATempPath(currentCase, "chrome")
+ File.separator
+ downloadFile.getName().toString()
+ j
+ ".db"; // NON-NLS
try {
ContentUtils.writeToFile(downloadFile, new File(temps));
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error writing temp sqlite db for Chrome download artifacts.{0}",
ex); // NON-NLS
this.addErrorMessage(
NbBundle.getMessage(
this.getClass(),
"Chrome.getDownload.errMsg.errAnalyzeFiles1",
this.getName(),
downloadFile.getName()));
continue;
}
File dbFile = new File(temps);
if (context.dataSourceIngestIsCancelled()) {
dbFile.delete();
break;
}
List<HashMap<String, Object>> tempList;
if (isChromePreVersion30(temps)) {
tempList = this.dbConnect(temps, downloadQuery);
} else {
tempList = this.dbConnect(temps, downloadQueryVersion30);
}
logger.log(
Level.INFO,
"{0}- Now getting downloads from {1} with {2}artifacts identified.",
new Object[] {moduleName, temps, tempList.size()}); // NON-NLS
for (HashMap<String, Object> result : tempList) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_PATH.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
(result.get("full_path").toString()))); // NON-NLS
long pathID = Util.findID(dataSource, (result.get("full_path").toString())); // NON-NLS
if (pathID != -1) {
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_PATH_ID.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
pathID));
}
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_URL.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("url").toString() != null)
? result.get("url").toString()
: ""))); // NON-NLS
// bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(),
// "Recent Activity", ((result.get("url").toString() != null) ?
// EscapeUtil.decodeURL(result.get("url").toString()) : "")));
Long time =
(Long.valueOf(result.get("start_time").toString()) / 1000000)
- Long.valueOf("11644473600"); // NON-NLS
// TODO Revisit usage of deprecated constructor as per TSK-583
// bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LAST_ACCESSED.getTypeID(),
// "Recent Activity", "Last Visited", time));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
time));
String domain =
Util.extractDomain(
(result.get("url").toString() != null)
? result.get("url").toString()
: ""); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
domain));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
this.addArtifact(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD, downloadFile, bbattributes);
}
dbFile.delete();
}
IngestServices.getInstance()
.fireModuleDataEvent(
new ModuleDataEvent(
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD));
}
/** Queries for cookie files and adds artifacts */
private void getCookie() {
FileManager fileManager = currentCase.getServices().getFileManager();
List<AbstractFile> cookiesFiles;
try {
cookiesFiles = fileManager.findFiles(dataSource, "Cookies", "Chrome"); // NON-NLS
} catch (TskCoreException ex) {
String msg = NbBundle.getMessage(this.getClass(), "Chrome.getCookie.errMsg.errGettingFiles");
logger.log(Level.SEVERE, msg, ex);
this.addErrorMessage(this.getName() + ": " + msg);
return;
}
if (cookiesFiles.isEmpty()) {
logger.log(Level.INFO, "Didn't find any Chrome cookies files."); // NON-NLS
return;
}
dataFound = true;
int j = 0;
while (j < cookiesFiles.size()) {
AbstractFile cookiesFile = cookiesFiles.get(j++);
if (cookiesFile.getSize() == 0) {
continue;
}
String temps =
RAImageIngestModule.getRATempPath(currentCase, "chrome")
+ File.separator
+ cookiesFile.getName().toString()
+ j
+ ".db"; // NON-NLS
try {
ContentUtils.writeToFile(cookiesFile, new File(temps));
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error writing temp sqlite db for Chrome cookie artifacts.{0}",
ex); // NON-NLS
this.addErrorMessage(
NbBundle.getMessage(
this.getClass(),
"Chrome.getCookie.errMsg.errAnalyzeFile",
this.getName(),
cookiesFile.getName()));
continue;
}
File dbFile = new File(temps);
if (context.dataSourceIngestIsCancelled()) {
dbFile.delete();
break;
}
List<HashMap<String, Object>> tempList = this.dbConnect(temps, cookieQuery);
logger.log(
Level.INFO,
"{0}- Now getting cookies from {1} with {2}artifacts identified.",
new Object[] {moduleName, temps, tempList.size()}); // NON-NLS
for (HashMap<String, Object> result : tempList) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_URL.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("host_key").toString() != null)
? result.get("host_key").toString()
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
(Long.valueOf(result.get("last_access_utc").toString()) / 1000000)
- Long.valueOf("11644473600"))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("name").toString() != null)
? result.get("name").toString()
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_VALUE.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
((result.get("value").toString() != null)
? result.get("value").toString()
: ""))); // NON-NLS
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
String domain = result.get("host_key").toString(); // NON-NLS
domain = domain.replaceFirst("^\\.+(?!$)", "");
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
domain));
this.addArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes);
}
dbFile.delete();
}
IngestServices.getInstance()
.fireModuleDataEvent(
new ModuleDataEvent(
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE));
}
/** Search for bookmark files and make artifacts. */
private void getBookmark() {
FileManager fileManager = currentCase.getServices().getFileManager();
List<AbstractFile> bookmarkFiles = null;
try {
bookmarkFiles = fileManager.findFiles(dataSource, "Bookmarks", "Chrome"); // NON-NLS
} catch (TskCoreException ex) {
String msg =
NbBundle.getMessage(this.getClass(), "Chrome.getBookmark.errMsg.errGettingFiles");
logger.log(Level.SEVERE, msg, ex);
this.addErrorMessage(this.getName() + ": " + msg);
return;
}
if (bookmarkFiles.isEmpty()) {
logger.log(Level.INFO, "Didn't find any Chrome bookmark files."); // NON-NLS
return;
}
dataFound = true;
int j = 0;
while (j < bookmarkFiles.size()) {
AbstractFile bookmarkFile = bookmarkFiles.get(j++);
if (bookmarkFile.getSize() == 0) {
continue;
}
String temps =
RAImageIngestModule.getRATempPath(currentCase, "chrome")
+ File.separator
+ bookmarkFile.getName().toString()
+ j
+ ".db"; // NON-NLS
try {
ContentUtils.writeToFile(bookmarkFile, new File(temps));
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error writing temp sqlite db for Chrome bookmark artifacts.{0}",
ex); // NON-NLS
this.addErrorMessage(
NbBundle.getMessage(
this.getClass(),
"Chrome.getBookmark.errMsg.errAnalyzingFile",
this.getName(),
bookmarkFile.getName()));
continue;
}
logger.log(
Level.INFO,
"{0}- Now getting Bookmarks from {1}",
new Object[] {moduleName, temps}); // NON-NLS
File dbFile = new File(temps);
if (context.dataSourceIngestIsCancelled()) {
dbFile.delete();
break;
}
FileReader tempReader;
try {
tempReader = new FileReader(temps);
} catch (FileNotFoundException ex) {
logger.log(
Level.SEVERE,
"Error while trying to read into the Bookmarks for Chrome.",
ex); // NON-NLS
this.addErrorMessage(
NbBundle.getMessage(
this.getClass(),
"Chrome.getBookmark.errMsg.errAnalyzeFile",
this.getName(),
bookmarkFile.getName()));
continue;
}
final JsonParser parser = new JsonParser();
JsonElement jsonElement;
JsonObject jElement, jRoot, jBookmark;
JsonArray jBookmarkArray;
try {
jsonElement = parser.parse(tempReader);
jElement = jsonElement.getAsJsonObject();
jRoot = jElement.get("roots").getAsJsonObject(); // NON-NLS
jBookmark = jRoot.get("bookmark_bar").getAsJsonObject(); // NON-NLS
jBookmarkArray = jBookmark.getAsJsonArray("children"); // NON-NLS
} catch (JsonIOException | JsonSyntaxException | IllegalStateException ex) {
logger.log(Level.WARNING, "Error parsing Json from Chrome Bookmark.", ex); // NON-NLS
this.addErrorMessage(
NbBundle.getMessage(
this.getClass(),
"Chrome.getBookmark.errMsg.errAnalyzingFile3",
this.getName(),
bookmarkFile.getName()));
continue;
}
for (JsonElement result : jBookmarkArray) {
JsonObject address = result.getAsJsonObject();
if (address == null) {
continue;
}
JsonElement urlEl = address.get("url"); // NON-NLS
String url;
if (urlEl != null) {
url = urlEl.getAsString();
} else {
url = "";
}
String name;
JsonElement nameEl = address.get("name"); // NON-NLS
if (nameEl != null) {
name = nameEl.getAsString();
} else {
name = "";
}
Long date;
JsonElement dateEl = address.get("date_added"); // NON-NLS
if (dateEl != null) {
date = dateEl.getAsLong();
} else {
date = Long.valueOf(0);
}
String domain = Util.extractDomain(url);
try {
BlackboardArtifact bbart = bookmarkFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
// TODO Revisit usage of deprecated constructor as per TSK-583
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_URL.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
url));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_TITLE.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
name));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DATETIME_CREATED.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
(date / 1000000) - Long.valueOf("11644473600")));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
bbattributes.add(
new BlackboardAttribute(
ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(),
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
domain));
bbart.addAttributes(bbattributes);
} catch (TskCoreException ex) {
logger.log(
Level.SEVERE,
"Error while trying to insert Chrome bookmark artifact{0}",
ex); // NON-NLS
this.addErrorMessage(
NbBundle.getMessage(
this.getClass(),
"Chrome.getBookmark.errMsg.errAnalyzingFile4",
this.getName(),
bookmarkFile.getName()));
}
}
dbFile.delete();
}
IngestServices.getInstance()
.fireModuleDataEvent(
new ModuleDataEvent(
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK));
}
@Override
public IngestModule.ProcessResult process(AbstractFile file) {
if (attrId == -1) {
return IngestModule.ProcessResult.ERROR;
}
// Skip anything other than actual file system files.
if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
|| (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
|| (file.isFile() == false)) {
return IngestModule.ProcessResult.OK;
}
// Skip NSRL / known files.
if (skipKnownFiles && file.getKnown() == TskData.FileKnown.KNOWN) {
return IngestModule.ProcessResult.OK;
}
// Do a nonsensical calculation of the number of 0x00 bytes
// in the first 1024-bytes of the file. This is for demo
// purposes only.
try {
byte buffer[] = new byte[1024];
int len = file.read(buffer, 0, 1024);
int count = 0;
for (int i = 0; i < len; i++) {
if (buffer[i] == 0x00) {
count++;
}
}
// Make an attribute using the ID for the attribute type that
// was previously created.
BlackboardAttribute attr =
new BlackboardAttribute(attrId, SampleIngestModuleFactory.getModuleName(), count);
// Add the to the general info artifact for the file. In a
// real module, you would likely have more complex data types
// and be making more specific artifacts.
BlackboardArtifact art = file.getGenInfoArtifact();
art.addAttribute(attr);
// This method is thread-safe with per ingest job reference counted
// management of shared data.
addToBlackboardPostCount(context.getJobId(), 1L);
// Fire an event to notify any listeners for blackboard postings.
ModuleDataEvent event =
new ModuleDataEvent(
SampleIngestModuleFactory.getModuleName(), ARTIFACT_TYPE.TSK_GEN_INFO);
IngestServices.getInstance().fireModuleDataEvent(event);
return IngestModule.ProcessResult.OK;
} catch (TskCoreException ex) {
IngestServices ingestServices = IngestServices.getInstance();
Logger logger = ingestServices.getLogger(SampleIngestModuleFactory.getModuleName());
logger.log(Level.SEVERE, "Error processing file (id = " + file.getId() + ")", ex);
return IngestModule.ProcessResult.ERROR;
}
}
/** @inheritDoc */
@Override
@Messages({
"FilesIdentifierIngestModule.indexError.message=Failed to index interesting file hit artifact for keyword search."
})
public ProcessResult process(AbstractFile file) {
blackboard = Case.getCurrentCase().getServices().getBlackboard();
// See if the file belongs to any defined interesting files set.
List<FilesSet> filesSets =
FilesIdentifierIngestModule.interestingFileSetsByJob.get(this.context.getJobId());
for (FilesSet filesSet : filesSets) {
String ruleSatisfied = filesSet.fileIsMemberOf(file);
if (ruleSatisfied != null) {
try {
// Post an interesting files set hit artifact to the
// blackboard.
String moduleName = InterestingItemsIngestModuleFactory.getModuleName();
BlackboardArtifact artifact =
file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT);
// Add a set name attribute to the artifact. This adds a
// fair amount of redundant data to the attributes table
// (i.e., rows that differ only in artifact id), but doing
// otherwise would requires reworking the interesting files
// set hit artifact.
BlackboardAttribute setNameAttribute =
new BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, moduleName, filesSet.getName());
artifact.addAttribute(setNameAttribute);
// Add a category attribute to the artifact to record the
// interesting files set membership rule that was satisfied.
BlackboardAttribute ruleNameAttribute =
new BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY, moduleName, ruleSatisfied);
artifact.addAttribute(ruleNameAttribute);
try {
// index the artifact for keyword search
blackboard.indexArtifact(artifact);
} catch (Blackboard.BlackboardException ex) {
logger.log(
Level.SEVERE,
"Unable to index blackboard artifact " + artifact.getArtifactID(),
ex); // NON-NLS
MessageNotifyUtil.Notify.error(
Bundle.FilesIdentifierIngestModule_indexError_message(), artifact.getDisplayName());
}
IngestServices.getInstance()
.fireModuleDataEvent(
new ModuleDataEvent(
moduleName,
BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT,
Collections.singletonList(artifact)));
} catch (TskCoreException ex) {
FilesIdentifierIngestModule.logger.log(
Level.SEVERE, "Error posting to the blackboard", ex); // NOI18N NON-NLS
}
}
}
return ProcessResult.OK;
}
class SevenZipExtractor {
private static final Logger logger = Logger.getLogger(SevenZipExtractor.class.getName());
private IngestServices services = IngestServices.getInstance();
private final IngestJobContext context;
private final FileTypeDetector fileTypeDetector;
static final String[] SUPPORTED_EXTENSIONS = {
"zip", "rar", "arj", "7z", "7zip", "gzip", "gz", "bzip2", "tar", "tgz",
}; // NON-NLS
// encryption type strings
private static final String ENCRYPTION_FILE_LEVEL =
NbBundle.getMessage(
EmbeddedFileExtractorIngestModule.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel");
private static final String ENCRYPTION_FULL =
NbBundle.getMessage(
EmbeddedFileExtractorIngestModule.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFull");
// zip bomb detection
private static final int MAX_DEPTH = 4;
private static final int MAX_COMPRESSION_RATIO = 600;
private static final long MIN_COMPRESSION_RATIO_SIZE = 500 * 1000000L;
private static final long MIN_FREE_DISK_SPACE = 1 * 1000 * 1000000L; // 1GB
// counts archive depth
private ArchiveDepthCountTree archiveDepthCountTree;
private String moduleDirRelative;
private String moduleDirAbsolute;
private String getLocalRootAbsPath(String uniqueArchiveFileName) {
return moduleDirAbsolute + File.separator + uniqueArchiveFileName;
}
/** Enum of mimetypes which support archive extraction */
private enum SupportedArchiveExtractionFormats {
ZIP("application/zip"),
SEVENZ("application/x-7z-compressed"),
GZIP("application/gzip"),
XGZIP("application/x-gzip"),
XBZIP2("application/x-bzip2"),
XTAR("application/x-tar");
private final String mimeType;
SupportedArchiveExtractionFormats(final String mimeType) {
this.mimeType = mimeType;
}
@Override
public String toString() {
return this.mimeType;
}
// TODO Expand to support more formats after upgrading Tika
}
SevenZipExtractor(
IngestJobContext context,
FileTypeDetector fileTypeDetector,
String moduleDirRelative,
String moduleDirAbsolute)
throws IngestModuleException {
if (!SevenZip.isInitializedSuccessfully()
&& (SevenZip.getLastInitializationException() == null)) {
try {
SevenZip.initSevenZipFromPlatformJAR();
String platform = SevenZip.getUsedPlatform();
logger.log(
Level.INFO,
"7-Zip-JBinding library was initialized on supported platform: {0}",
platform); // NON-NLS
} catch (SevenZipNativeInitializationException e) {
logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); // NON-NLS
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errInitModule.msg",
EmbeddedFileExtractorModuleFactory.getModuleName());
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errCantInitLib",
e.getMessage());
services.postMessage(
IngestMessage.createErrorMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
throw new IngestModuleException(e.getMessage());
}
}
this.context = context;
this.fileTypeDetector = fileTypeDetector;
this.moduleDirRelative = moduleDirRelative;
this.moduleDirAbsolute = moduleDirAbsolute;
this.archiveDepthCountTree = new ArchiveDepthCountTree();
}
/**
* This method returns true if the file format is currently supported. Else it returns false.
* Attempt extension based detection in case Apache Tika based detection fails.
*
* @param abstractFile The AbstractFilw whose mimetype is to be determined.
* @return This method returns true if the file format is currently supported. Else it returns
* false.
*/
boolean isSevenZipExtractionSupported(AbstractFile abstractFile) {
try {
String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile);
for (SupportedArchiveExtractionFormats s : SupportedArchiveExtractionFormats.values()) {
if (s.toString().equals(abstractFileMimeType)) {
return true;
}
}
return false;
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Error executing FileTypeDetector.getFileType()", ex); // NON-NLS
}
// attempt extension matching
final String extension = abstractFile.getNameExtension();
for (String supportedExtension : SUPPORTED_EXTENSIONS) {
if (extension.equals(supportedExtension)) {
return true;
}
}
return false;
}
/**
* Check if the item inside archive is a potential zipbomb
*
* <p>Currently checks compression ratio.
*
* <p>More heuristics to be added here
*
* @param archiveName the parent archive
* @param archiveFileItem the archive item
* @return true if potential zip bomb, false otherwise
*/
private boolean isZipBombArchiveItemCheck(
AbstractFile archiveFile, ISimpleInArchiveItem archiveFileItem) {
try {
final Long archiveItemSize = archiveFileItem.getSize();
// skip the check for small files
if (archiveItemSize == null || archiveItemSize < MIN_COMPRESSION_RATIO_SIZE) {
return false;
}
final Long archiveItemPackedSize = archiveFileItem.getPackedSize();
if (archiveItemPackedSize == null || archiveItemPackedSize <= 0) {
logger.log(
Level.WARNING,
"Cannot getting compression ratio, cannot detect if zipbomb: {0}, item: {1}",
new Object[] {archiveFile.getName(), archiveFileItem.getPath()}); // NON-NLS
return false;
}
int cRatio = (int) (archiveItemSize / archiveItemPackedSize);
if (cRatio >= MAX_COMPRESSION_RATIO) {
String itemName = archiveFileItem.getPath();
logger.log(
Level.INFO,
"Possible zip bomb detected, compression ration: {0} for in archive item: {1}",
new Object[] {cRatio, itemName}); // NON-NLS
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg",
archiveFile.getName(),
itemName);
String path;
try {
path = archiveFile.getUniquePath();
} catch (TskCoreException ex) {
path = archiveFile.getParentPath() + archiveFile.getName();
}
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails",
cRatio,
path);
// MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(
IngestMessage.createWarningMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
return true;
} else {
return false;
}
} catch (SevenZipException ex) {
logger.log(
Level.SEVERE,
"Error getting archive item size and cannot detect if zipbomb. ",
ex); // NON-NLS
return false;
}
}
/**
* Check file extension and return appropriate input options for SevenZip.openInArchive()
*
* @param archiveFile file to check file extension
* @return input parameter for SevenZip.openInArchive()
*/
private ArchiveFormat get7ZipOptions(AbstractFile archiveFile) {
// try to get the file type from the BB
String detectedFormat = null;
try {
ArrayList<BlackboardAttribute> attributes =
archiveFile.getGenInfoAttributes(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG);
for (BlackboardAttribute attribute : attributes) {
detectedFormat = attribute.getValueString();
break;
}
} catch (TskCoreException ex) {
logger.log(
Level.WARNING,
"Couldn't obtain file attributes for file: " + archiveFile.toString(),
ex); // NON-NLS
}
if (detectedFormat == null) {
logger.log(Level.WARNING, "Could not detect format for file: " + archiveFile); // NON-NLS
// if we don't have attribute info then use file extension
String extension = archiveFile.getNameExtension();
if ("rar".equals(extension)) // NON-NLS
{
// for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP
// archive inside RAR archive
// it will be opened incorrectly when using 7zip's built-in auto-detect functionality
return RAR;
}
// Otherwise open the archive using 7zip's built-in auto-detect functionality
return null;
} else if (detectedFormat.contains("application/x-rar-compressed")) // NON-NLS
{
// for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive
// inside RAR archive
// it will be opened incorrectly when using 7zip's built-in auto-detect functionality
return RAR;
}
// Otherwise open the archive using 7zip's built-in auto-detect functionality
return null;
}
/**
* Unpack the file to local folder and return a list of derived files
*
* @param pipelineContext current ingest context
* @param archiveFile file to unpack
* @return list of unpacked derived files
*/
void unpack(AbstractFile archiveFile) {
String archiveFilePath;
try {
archiveFilePath = archiveFile.getUniquePath();
} catch (TskCoreException ex) {
archiveFilePath = archiveFile.getParentPath() + archiveFile.getName();
}
// check if already has derived files, skip
try {
if (archiveFile.hasChildren()) {
// check if local unpacked dir exists
if (new File(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)).exists()) {
logger.log(
Level.INFO,
"File already has been processed as it has children and local unpacked file, skipping: {0}",
archiveFilePath); // NON-NLS
return;
}
}
} catch (TskCoreException e) {
logger.log(
Level.INFO,
"Error checking if file already has been processed, skipping: {0}",
archiveFilePath); // NON-NLS
return;
}
List<AbstractFile> unpackedFiles = Collections.<AbstractFile>emptyList();
// recursion depth check for zip bomb
final long archiveId = archiveFile.getId();
SevenZipExtractor.ArchiveDepthCountTree.Archive parentAr =
archiveDepthCountTree.findArchive(archiveId);
if (parentAr == null) {
parentAr = archiveDepthCountTree.addArchive(null, archiveId);
} else if (parentAr.getDepth() == MAX_DEPTH) {
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnMsg.zipBomb",
archiveFile.getName());
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
parentAr.getDepth(),
archiveFilePath);
// MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(
IngestMessage.createWarningMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
return;
}
boolean hasEncrypted = false;
boolean fullEncryption = true;
ISevenZipInArchive inArchive = null;
SevenZipContentReadStream stream = null;
final ProgressHandle progress =
ProgressHandleFactory.createHandle(
NbBundle.getMessage(
this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.moduleName"));
int processedItems = 0;
boolean progressStarted = false;
try {
stream = new SevenZipContentReadStream(new ReadContentInputStream(archiveFile));
// for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive
// inside RAR archive
// it will be opened incorrectly when using 7zip's built-in auto-detect functionality.
// All other archive formats are still opened using 7zip built-in auto-detect functionality.
ArchiveFormat options = get7ZipOptions(archiveFile);
inArchive = SevenZip.openInArchive(options, stream);
int numItems = inArchive.getNumberOfItems();
logger.log(
Level.INFO,
"Count of items in archive: {0}: {1}",
new Object[] {archiveFilePath, numItems}); // NON-NLS
progress.start(numItems);
progressStarted = true;
final ISimpleInArchive simpleInArchive = inArchive.getSimpleInterface();
// setup the archive local root folder
final String uniqueArchiveFileName =
EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile);
final String localRootAbsPath = getLocalRootAbsPath(uniqueArchiveFileName);
final File localRoot = new File(localRootAbsPath);
if (!localRoot.exists()) {
try {
localRoot.mkdirs();
} catch (SecurityException e) {
logger.log(
Level.SEVERE,
"Error setting up output path for archive root: {0}",
localRootAbsPath); // NON-NLS
// bail
return;
}
}
// initialize tree hierarchy to keep track of unpacked file structure
SevenZipExtractor.UnpackedTree unpackedTree =
new SevenZipExtractor.UnpackedTree(
moduleDirRelative + "/" + uniqueArchiveFileName, archiveFile);
long freeDiskSpace = services.getFreeDiskSpace();
// unpack and process every item in archive
int itemNumber = 0;
for (ISimpleInArchiveItem item : simpleInArchive.getArchiveItems()) {
String pathInArchive = item.getPath();
if (pathInArchive == null || pathInArchive.isEmpty()) {
// some formats (.tar.gz) may not be handled correctly -- file in archive has no name/path
// handle this for .tar.gz and tgz but assuming the child is tar,
// otherwise, unpack using itemNumber as name
// TODO this should really be signature based, not extension based
String archName = archiveFile.getName();
int dotI = archName.lastIndexOf(".");
String useName = null;
if (dotI != -1) {
String base = archName.substring(0, dotI);
String ext = archName.substring(dotI);
switch (ext) {
case ".gz": // NON-NLS
useName = base;
break;
case ".tgz": // NON-NLS
useName = base + ".tar"; // NON-NLS
break;
}
}
if (useName == null) {
pathInArchive = "/" + archName + "/" + Integer.toString(itemNumber);
} else {
pathInArchive = "/" + useName;
}
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.unknownPath.msg",
archiveFilePath,
pathInArchive);
logger.log(Level.WARNING, msg);
}
++itemNumber;
logger.log(Level.INFO, "Extracted item path: {0}", pathInArchive); // NON-NLS
// check if possible zip bomb
if (isZipBombArchiveItemCheck(archiveFile, item)) {
continue; // skip the item
}
// find this node in the hierarchy, create if needed
SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode =
unpackedTree.addNode(pathInArchive);
String fileName = unpackedNode.getFileName();
// update progress bar
progress.progress(archiveFile.getName() + ": " + fileName, processedItems);
final boolean isEncrypted = item.isEncrypted();
final boolean isDir = item.isFolder();
if (isEncrypted) {
logger.log(
Level.WARNING, "Skipping encrypted file in archive: {0}", pathInArchive); // NON-NLS
hasEncrypted = true;
continue;
} else {
fullEncryption = false;
}
final Long size = item.getSize();
if (size == null) {
// If the size property cannot be determined, out-of-disk-space
// situations cannot be ascertained.
// Hence skip this file.
logger.log(
Level.WARNING,
"Size cannot be determined. Skipping file in archive: {0}",
pathInArchive); // NON-NLS
continue;
}
// check if unpacking this file will result in out of disk space
// this is additional to zip bomb prevention mechanism
if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN
&& size > 0) { // if known free space and file not empty
long newDiskSpace = freeDiskSpace - size;
if (newDiskSpace < MIN_FREE_DISK_SPACE) {
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
archiveFilePath,
fileName);
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
// MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(
IngestMessage.createErrorMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
logger.log(
Level.INFO,
"Skipping archive item due to insufficient disk space: {0}, {1}",
new Object[] {archiveFilePath, fileName}); // NON-NLS
logger.log(
Level.INFO, "Available disk space: {0}", new Object[] {freeDiskSpace}); // NON-NLS
continue; // skip this file
} else {
// update est. disk space during this archive, so we don't need to poll for every file
// extracted
freeDiskSpace = newDiskSpace;
}
}
final String uniqueExtractedName =
uniqueArchiveFileName
+ File.separator
+ (item.getItemIndex() / 1000)
+ File.separator
+ item.getItemIndex()
+ new File(pathInArchive).getName();
// final String localRelPath = unpackDir + File.separator + localFileRelPath;
final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
final String localAbsPath = moduleDirAbsolute + File.separator + uniqueExtractedName;
// create local dirs and empty files before extracted
File localFile = new java.io.File(localAbsPath);
// cannot rely on files in top-bottom order
if (!localFile.exists()) {
try {
if (isDir) {
localFile.mkdirs();
} else {
localFile.getParentFile().mkdirs();
try {
localFile.createNewFile();
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error creating extracted file: " + localFile.getAbsolutePath(),
ex); // NON-NLS
}
}
} catch (SecurityException e) {
logger.log(
Level.SEVERE,
"Error setting up output path for unpacked file: {0}",
pathInArchive); // NON-NLS
// TODO consider bail out / msg to the user
}
}
// skip the rest of this loop if we couldn't create the file
if (localFile.exists() == false) {
continue;
}
final Date createTime = item.getCreationTime();
final Date accessTime = item.getLastAccessTime();
final Date writeTime = item.getLastWriteTime();
final long createtime = createTime == null ? 0L : createTime.getTime() / 1000;
final long modtime = writeTime == null ? 0L : writeTime.getTime() / 1000;
final long accesstime = accessTime == null ? 0L : accessTime.getTime() / 1000;
// record derived data in unode, to be traversed later after unpacking the archive
unpackedNode.addDerivedInfo(
size, !isDir, 0L, createtime, accesstime, modtime, localRelPath);
// unpack locally if a file
if (!isDir) {
SevenZipExtractor.UnpackStream unpackStream = null;
try {
unpackStream = new SevenZipExtractor.UnpackStream(localAbsPath);
item.extractSlow(unpackStream);
} catch (Exception e) {
// could be something unexpected with this file, move on
logger.log(
Level.WARNING,
"Could not extract file from archive: " + localAbsPath,
e); // NON-NLS
} finally {
if (unpackStream != null) {
unpackStream.close();
}
}
}
// update units for progress bar
++processedItems;
}
// add them to the DB. We wait until the end so that we have the metadata on all of the
// intermediate nodes since the order is not guaranteed
try {
unpackedTree.addDerivedFilesToCase();
unpackedFiles = unpackedTree.getAllFileObjects();
// check if children are archives, update archive depth tracking
for (AbstractFile unpackedFile : unpackedFiles) {
if (isSevenZipExtractionSupported(unpackedFile)) {
archiveDepthCountTree.addArchive(parentAr, unpackedFile.getId());
}
}
} catch (TskCoreException e) {
logger.log(
Level.SEVERE,
"Error populating complete derived file hierarchy from the unpacked dir structure"); // NON-NLS
// TODO decide if anything to cleanup, for now bailing
}
} catch (SevenZipException ex) {
logger.log(Level.SEVERE, "Error unpacking file: " + archiveFile, ex); // NON-NLS
// inbox message
// print a message if the file is allocated
if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
archiveFile.getName());
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
archiveFilePath,
ex.getMessage());
services.postMessage(
IngestMessage.createErrorMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
}
} finally {
if (inArchive != null) {
try {
inArchive.close();
} catch (SevenZipException e) {
logger.log(Level.SEVERE, "Error closing archive: " + archiveFile, e); // NON-NLS
}
}
if (stream != null) {
try {
stream.close();
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error closing stream after unpacking archive: " + archiveFile,
ex); // NON-NLS
}
}
// close progress bar
if (progressStarted) {
progress.finish();
}
}
// create artifact and send user message
if (hasEncrypted) {
String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
try {
BlackboardArtifact artifact =
archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
artifact.addAttribute(
new BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(),
EmbeddedFileExtractorModuleFactory.getModuleName(),
encryptionType));
services.fireModuleDataEvent(
new ModuleDataEvent(
EmbeddedFileExtractorModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
} catch (TskCoreException ex) {
logger.log(
Level.SEVERE,
"Error creating blackboard artifact for encryption detected for file: "
+ archiveFilePath,
ex); // NON-NLS
}
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
archiveFile.getName(),
EmbeddedFileExtractorModuleFactory.getModuleName());
services.postMessage(
IngestMessage.createWarningMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
}
// adding unpacked extracted derived files to the job after closing relevant resources.
if (!unpackedFiles.isEmpty()) {
// currently sending a single event for all new files
services.fireModuleContentEvent(new ModuleContentEvent(archiveFile));
context.addFilesToJob(unpackedFiles);
}
}
/** Stream used to unpack the archive to local file */
private static class UnpackStream implements ISequentialOutStream {
private OutputStream output;
private String localAbsPath;
UnpackStream(String localAbsPath) {
try {
output = new BufferedOutputStream(new FileOutputStream(localAbsPath));
} catch (FileNotFoundException ex) {
logger.log(Level.SEVERE, "Error writing extracted file: " + localAbsPath, ex); // NON-NLS
}
}
@Override
public int write(byte[] bytes) throws SevenZipException {
try {
output.write(bytes);
} catch (IOException ex) {
throw new SevenZipException(
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
localAbsPath),
ex);
}
return bytes.length;
}
public void close() {
if (output != null) {
try {
output.flush();
output.close();
} catch (IOException e) {
logger.log(
Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); // NON-NLS
}
}
}
}
/**
* Representation of the files in the archive. Used to track of local tree file hierarchy, archive
* depth, and files created to easily and reliably get parent AbstractFile for unpacked file. So
* that we don't have to depend on type of traversal of unpacked files handed to us by 7zip
* unpacker.
*/
private class UnpackedTree {
final UnpackedNode rootNode;
/**
* @param localPathRoot Path in module output folder that files will be saved to
* @param archiveFile Archive file being extracted
* @param fileManager
*/
UnpackedTree(String localPathRoot, AbstractFile archiveFile) {
this.rootNode = new UnpackedNode();
this.rootNode.setFile(archiveFile);
this.rootNode.setFileName(archiveFile.getName());
this.rootNode.localRelPath = localPathRoot;
}
/**
* Creates a node in the tree at the given path. Makes intermediate nodes if needed. If a node
* already exists at that path, it is returned.
*
* @param filePath file path with 1 or more tokens separated by /
* @return child node for the last file token in the filePath
*/
UnpackedNode addNode(String filePath) {
String[] toks = filePath.split("[\\/\\\\]");
List<String> tokens = new ArrayList<>();
for (int i = 0; i < toks.length; ++i) {
if (!toks[i].isEmpty()) {
tokens.add(toks[i]);
}
}
return addNode(rootNode, tokens);
}
/**
* recursive method that traverses the path
*
* @param tokenPath
* @return
*/
private UnpackedNode addNode(UnpackedNode parent, List<String> tokenPath) {
// we found all of the tokens
if (tokenPath.isEmpty()) {
return parent;
}
// get the next name in the path and look it up
String childName = tokenPath.remove(0);
UnpackedNode child = parent.getChild(childName);
// create new node
if (child == null) {
child = new UnpackedNode(childName, parent);
}
// go down one more level
return addNode(child, tokenPath);
}
/**
* Get the root file objects (after createDerivedFiles() ) of this tree, so that they can be
* rescheduled.
*
* @return root objects of this unpacked tree
*/
List<AbstractFile> getRootFileObjects() {
List<AbstractFile> ret = new ArrayList<>();
for (UnpackedNode child : rootNode.children) {
ret.add(child.getFile());
}
return ret;
}
/**
* Get the all file objects (after createDerivedFiles() ) of this tree, so that they can be
* rescheduled.
*
* @return all file objects of this unpacked tree
*/
List<AbstractFile> getAllFileObjects() {
List<AbstractFile> ret = new ArrayList<>();
for (UnpackedNode child : rootNode.children) {
getAllFileObjectsRec(ret, child);
}
return ret;
}
private void getAllFileObjectsRec(List<AbstractFile> list, UnpackedNode parent) {
list.add(parent.getFile());
for (UnpackedNode child : parent.children) {
getAllFileObjectsRec(list, child);
}
}
/**
* Traverse the tree top-down after unzipping is done and create derived files for the entire
* hierarchy
*/
void addDerivedFilesToCase() throws TskCoreException {
final FileManager fileManager = Case.getCurrentCase().getServices().getFileManager();
for (UnpackedNode child : rootNode.children) {
addDerivedFilesToCaseRec(child, fileManager);
}
}
private void addDerivedFilesToCaseRec(UnpackedNode node, FileManager fileManager)
throws TskCoreException {
final String fileName = node.getFileName();
try {
DerivedFile df =
fileManager.addDerivedFile(
fileName,
node.getLocalRelPath(),
node.getSize(),
node.getCtime(),
node.getCrtime(),
node.getAtime(),
node.getMtime(),
node.isIsFile(),
node.getParent().getFile(),
"",
EmbeddedFileExtractorModuleFactory.getModuleName(),
"",
"");
node.setFile(df);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error adding a derived file to db:" + fileName, ex); // NON-NLS
throw new TskCoreException(
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
fileName),
ex);
}
// recurse
for (UnpackedNode child : node.children) {
addDerivedFilesToCaseRec(child, fileManager);
}
}
/** A node in the unpacked tree that represents a file or folder. */
private class UnpackedNode {
private String fileName;
private AbstractFile file;
private List<UnpackedNode> children = new ArrayList<>();
private String localRelPath = "";
private long size;
private long ctime, crtime, atime, mtime;
private boolean isFile;
private UnpackedNode parent;
// root constructor
UnpackedNode() {}
// child node constructor
UnpackedNode(String fileName, UnpackedNode parent) {
this.fileName = fileName;
this.parent = parent;
// this.localRelPath = parent.localRelPath + File.separator + fileName;
// new child derived file will be set by unpack() method
parent.children.add(this);
}
public long getCtime() {
return ctime;
}
public long getCrtime() {
return crtime;
}
public long getAtime() {
return atime;
}
public long getMtime() {
return mtime;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
UnpackedNode getParent() {
return parent;
}
void addDerivedInfo(
long size,
boolean isFile,
long ctime,
long crtime,
long atime,
long mtime,
String relLocalPath) {
this.size = size;
this.isFile = isFile;
this.ctime = ctime;
this.crtime = crtime;
this.atime = atime;
this.mtime = mtime;
this.localRelPath = relLocalPath;
}
void setFile(AbstractFile file) {
this.file = file;
}
/**
* get child by name or null if it doesn't exist
*
* @param childFileName
* @return
*/
UnpackedNode getChild(String childFileName) {
UnpackedNode ret = null;
for (UnpackedNode child : children) {
if (child.fileName.equals(childFileName)) {
ret = child;
break;
}
}
return ret;
}
public String getFileName() {
return fileName;
}
public AbstractFile getFile() {
return file;
}
public String getLocalRelPath() {
return localRelPath;
}
public long getSize() {
return size;
}
public boolean isIsFile() {
return isFile;
}
}
}
/** Tracks archive hierarchy and archive depth */
private static class ArchiveDepthCountTree {
// keeps all nodes refs for easy search
private final List<Archive> archives = new ArrayList<>();
/**
* Search for previously added parent archive by id
*
* @param objectId parent archive object id
* @return the archive node or null if not found
*/
Archive findArchive(long objectId) {
for (Archive ar : archives) {
if (ar.objectId == objectId) {
return ar;
}
}
return null;
}
/**
* Add a new archive to track of depth
*
* @param parent parent archive or null
* @param objectId object id of the new archive
* @return the archive added
*/
Archive addArchive(Archive parent, long objectId) {
Archive child = new Archive(parent, objectId);
archives.add(child);
return child;
}
private static class Archive {
int depth;
long objectId;
Archive parent;
List<Archive> children;
Archive(Archive parent, long objectId) {
this.parent = parent;
this.objectId = objectId;
children = new ArrayList<>();
if (parent != null) {
parent.children.add(this);
this.depth = parent.depth + 1;
} else {
this.depth = 0;
}
}
/**
* get archive depth of this archive
*
* @return
*/
int getDepth() {
return depth;
}
}
}
}
/**
* Unpack the file to local folder and return a list of derived files
*
* @param pipelineContext current ingest context
* @param archiveFile file to unpack
* @return list of unpacked derived files
*/
void unpack(AbstractFile archiveFile) {
String archiveFilePath;
try {
archiveFilePath = archiveFile.getUniquePath();
} catch (TskCoreException ex) {
archiveFilePath = archiveFile.getParentPath() + archiveFile.getName();
}
// check if already has derived files, skip
try {
if (archiveFile.hasChildren()) {
// check if local unpacked dir exists
if (new File(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)).exists()) {
logger.log(
Level.INFO,
"File already has been processed as it has children and local unpacked file, skipping: {0}",
archiveFilePath); // NON-NLS
return;
}
}
} catch (TskCoreException e) {
logger.log(
Level.INFO,
"Error checking if file already has been processed, skipping: {0}",
archiveFilePath); // NON-NLS
return;
}
List<AbstractFile> unpackedFiles = Collections.<AbstractFile>emptyList();
// recursion depth check for zip bomb
final long archiveId = archiveFile.getId();
SevenZipExtractor.ArchiveDepthCountTree.Archive parentAr =
archiveDepthCountTree.findArchive(archiveId);
if (parentAr == null) {
parentAr = archiveDepthCountTree.addArchive(null, archiveId);
} else if (parentAr.getDepth() == MAX_DEPTH) {
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnMsg.zipBomb",
archiveFile.getName());
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
parentAr.getDepth(),
archiveFilePath);
// MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(
IngestMessage.createWarningMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
return;
}
boolean hasEncrypted = false;
boolean fullEncryption = true;
ISevenZipInArchive inArchive = null;
SevenZipContentReadStream stream = null;
final ProgressHandle progress =
ProgressHandleFactory.createHandle(
NbBundle.getMessage(
this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.moduleName"));
int processedItems = 0;
boolean progressStarted = false;
try {
stream = new SevenZipContentReadStream(new ReadContentInputStream(archiveFile));
// for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive
// inside RAR archive
// it will be opened incorrectly when using 7zip's built-in auto-detect functionality.
// All other archive formats are still opened using 7zip built-in auto-detect functionality.
ArchiveFormat options = get7ZipOptions(archiveFile);
inArchive = SevenZip.openInArchive(options, stream);
int numItems = inArchive.getNumberOfItems();
logger.log(
Level.INFO,
"Count of items in archive: {0}: {1}",
new Object[] {archiveFilePath, numItems}); // NON-NLS
progress.start(numItems);
progressStarted = true;
final ISimpleInArchive simpleInArchive = inArchive.getSimpleInterface();
// setup the archive local root folder
final String uniqueArchiveFileName =
EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile);
final String localRootAbsPath = getLocalRootAbsPath(uniqueArchiveFileName);
final File localRoot = new File(localRootAbsPath);
if (!localRoot.exists()) {
try {
localRoot.mkdirs();
} catch (SecurityException e) {
logger.log(
Level.SEVERE,
"Error setting up output path for archive root: {0}",
localRootAbsPath); // NON-NLS
// bail
return;
}
}
// initialize tree hierarchy to keep track of unpacked file structure
SevenZipExtractor.UnpackedTree unpackedTree =
new SevenZipExtractor.UnpackedTree(
moduleDirRelative + "/" + uniqueArchiveFileName, archiveFile);
long freeDiskSpace = services.getFreeDiskSpace();
// unpack and process every item in archive
int itemNumber = 0;
for (ISimpleInArchiveItem item : simpleInArchive.getArchiveItems()) {
String pathInArchive = item.getPath();
if (pathInArchive == null || pathInArchive.isEmpty()) {
// some formats (.tar.gz) may not be handled correctly -- file in archive has no name/path
// handle this for .tar.gz and tgz but assuming the child is tar,
// otherwise, unpack using itemNumber as name
// TODO this should really be signature based, not extension based
String archName = archiveFile.getName();
int dotI = archName.lastIndexOf(".");
String useName = null;
if (dotI != -1) {
String base = archName.substring(0, dotI);
String ext = archName.substring(dotI);
switch (ext) {
case ".gz": // NON-NLS
useName = base;
break;
case ".tgz": // NON-NLS
useName = base + ".tar"; // NON-NLS
break;
}
}
if (useName == null) {
pathInArchive = "/" + archName + "/" + Integer.toString(itemNumber);
} else {
pathInArchive = "/" + useName;
}
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.unknownPath.msg",
archiveFilePath,
pathInArchive);
logger.log(Level.WARNING, msg);
}
++itemNumber;
logger.log(Level.INFO, "Extracted item path: {0}", pathInArchive); // NON-NLS
// check if possible zip bomb
if (isZipBombArchiveItemCheck(archiveFile, item)) {
continue; // skip the item
}
// find this node in the hierarchy, create if needed
SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode =
unpackedTree.addNode(pathInArchive);
String fileName = unpackedNode.getFileName();
// update progress bar
progress.progress(archiveFile.getName() + ": " + fileName, processedItems);
final boolean isEncrypted = item.isEncrypted();
final boolean isDir = item.isFolder();
if (isEncrypted) {
logger.log(
Level.WARNING, "Skipping encrypted file in archive: {0}", pathInArchive); // NON-NLS
hasEncrypted = true;
continue;
} else {
fullEncryption = false;
}
final Long size = item.getSize();
if (size == null) {
// If the size property cannot be determined, out-of-disk-space
// situations cannot be ascertained.
// Hence skip this file.
logger.log(
Level.WARNING,
"Size cannot be determined. Skipping file in archive: {0}",
pathInArchive); // NON-NLS
continue;
}
// check if unpacking this file will result in out of disk space
// this is additional to zip bomb prevention mechanism
if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN
&& size > 0) { // if known free space and file not empty
long newDiskSpace = freeDiskSpace - size;
if (newDiskSpace < MIN_FREE_DISK_SPACE) {
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
archiveFilePath,
fileName);
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
// MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(
IngestMessage.createErrorMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
logger.log(
Level.INFO,
"Skipping archive item due to insufficient disk space: {0}, {1}",
new Object[] {archiveFilePath, fileName}); // NON-NLS
logger.log(
Level.INFO, "Available disk space: {0}", new Object[] {freeDiskSpace}); // NON-NLS
continue; // skip this file
} else {
// update est. disk space during this archive, so we don't need to poll for every file
// extracted
freeDiskSpace = newDiskSpace;
}
}
final String uniqueExtractedName =
uniqueArchiveFileName
+ File.separator
+ (item.getItemIndex() / 1000)
+ File.separator
+ item.getItemIndex()
+ new File(pathInArchive).getName();
// final String localRelPath = unpackDir + File.separator + localFileRelPath;
final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
final String localAbsPath = moduleDirAbsolute + File.separator + uniqueExtractedName;
// create local dirs and empty files before extracted
File localFile = new java.io.File(localAbsPath);
// cannot rely on files in top-bottom order
if (!localFile.exists()) {
try {
if (isDir) {
localFile.mkdirs();
} else {
localFile.getParentFile().mkdirs();
try {
localFile.createNewFile();
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error creating extracted file: " + localFile.getAbsolutePath(),
ex); // NON-NLS
}
}
} catch (SecurityException e) {
logger.log(
Level.SEVERE,
"Error setting up output path for unpacked file: {0}",
pathInArchive); // NON-NLS
// TODO consider bail out / msg to the user
}
}
// skip the rest of this loop if we couldn't create the file
if (localFile.exists() == false) {
continue;
}
final Date createTime = item.getCreationTime();
final Date accessTime = item.getLastAccessTime();
final Date writeTime = item.getLastWriteTime();
final long createtime = createTime == null ? 0L : createTime.getTime() / 1000;
final long modtime = writeTime == null ? 0L : writeTime.getTime() / 1000;
final long accesstime = accessTime == null ? 0L : accessTime.getTime() / 1000;
// record derived data in unode, to be traversed later after unpacking the archive
unpackedNode.addDerivedInfo(
size, !isDir, 0L, createtime, accesstime, modtime, localRelPath);
// unpack locally if a file
if (!isDir) {
SevenZipExtractor.UnpackStream unpackStream = null;
try {
unpackStream = new SevenZipExtractor.UnpackStream(localAbsPath);
item.extractSlow(unpackStream);
} catch (Exception e) {
// could be something unexpected with this file, move on
logger.log(
Level.WARNING,
"Could not extract file from archive: " + localAbsPath,
e); // NON-NLS
} finally {
if (unpackStream != null) {
unpackStream.close();
}
}
}
// update units for progress bar
++processedItems;
}
// add them to the DB. We wait until the end so that we have the metadata on all of the
// intermediate nodes since the order is not guaranteed
try {
unpackedTree.addDerivedFilesToCase();
unpackedFiles = unpackedTree.getAllFileObjects();
// check if children are archives, update archive depth tracking
for (AbstractFile unpackedFile : unpackedFiles) {
if (isSevenZipExtractionSupported(unpackedFile)) {
archiveDepthCountTree.addArchive(parentAr, unpackedFile.getId());
}
}
} catch (TskCoreException e) {
logger.log(
Level.SEVERE,
"Error populating complete derived file hierarchy from the unpacked dir structure"); // NON-NLS
// TODO decide if anything to cleanup, for now bailing
}
} catch (SevenZipException ex) {
logger.log(Level.SEVERE, "Error unpacking file: " + archiveFile, ex); // NON-NLS
// inbox message
// print a message if the file is allocated
if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
archiveFile.getName());
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
archiveFilePath,
ex.getMessage());
services.postMessage(
IngestMessage.createErrorMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
}
} finally {
if (inArchive != null) {
try {
inArchive.close();
} catch (SevenZipException e) {
logger.log(Level.SEVERE, "Error closing archive: " + archiveFile, e); // NON-NLS
}
}
if (stream != null) {
try {
stream.close();
} catch (IOException ex) {
logger.log(
Level.SEVERE,
"Error closing stream after unpacking archive: " + archiveFile,
ex); // NON-NLS
}
}
// close progress bar
if (progressStarted) {
progress.finish();
}
}
// create artifact and send user message
if (hasEncrypted) {
String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
try {
BlackboardArtifact artifact =
archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
artifact.addAttribute(
new BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(),
EmbeddedFileExtractorModuleFactory.getModuleName(),
encryptionType));
services.fireModuleDataEvent(
new ModuleDataEvent(
EmbeddedFileExtractorModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
} catch (TskCoreException ex) {
logger.log(
Level.SEVERE,
"Error creating blackboard artifact for encryption detected for file: "
+ archiveFilePath,
ex); // NON-NLS
}
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
archiveFile.getName(),
EmbeddedFileExtractorModuleFactory.getModuleName());
services.postMessage(
IngestMessage.createWarningMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
}
// adding unpacked extracted derived files to the job after closing relevant resources.
if (!unpackedFiles.isEmpty()) {
// currently sending a single event for all new files
services.fireModuleContentEvent(new ModuleContentEvent(archiveFile));
context.addFilesToJob(unpackedFiles);
}
}
/**
* Check if the item inside archive is a potential zipbomb
*
* <p>Currently checks compression ratio.
*
* <p>More heuristics to be added here
*
* @param archiveName the parent archive
* @param archiveFileItem the archive item
* @return true if potential zip bomb, false otherwise
*/
private boolean isZipBombArchiveItemCheck(
AbstractFile archiveFile, ISimpleInArchiveItem archiveFileItem) {
try {
final Long archiveItemSize = archiveFileItem.getSize();
// skip the check for small files
if (archiveItemSize == null || archiveItemSize < MIN_COMPRESSION_RATIO_SIZE) {
return false;
}
final Long archiveItemPackedSize = archiveFileItem.getPackedSize();
if (archiveItemPackedSize == null || archiveItemPackedSize <= 0) {
logger.log(
Level.WARNING,
"Cannot getting compression ratio, cannot detect if zipbomb: {0}, item: {1}",
new Object[] {archiveFile.getName(), archiveFileItem.getPath()}); // NON-NLS
return false;
}
int cRatio = (int) (archiveItemSize / archiveItemPackedSize);
if (cRatio >= MAX_COMPRESSION_RATIO) {
String itemName = archiveFileItem.getPath();
logger.log(
Level.INFO,
"Possible zip bomb detected, compression ration: {0} for in archive item: {1}",
new Object[] {cRatio, itemName}); // NON-NLS
String msg =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg",
archiveFile.getName(),
itemName);
String path;
try {
path = archiveFile.getUniquePath();
} catch (TskCoreException ex) {
path = archiveFile.getParentPath() + archiveFile.getName();
}
String details =
NbBundle.getMessage(
this.getClass(),
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails",
cRatio,
path);
// MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(
IngestMessage.createWarningMessage(
EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
return true;
} else {
return false;
}
} catch (SevenZipException ex) {
logger.log(
Level.SEVERE,
"Error getting archive item size and cannot detect if zipbomb. ",
ex); // NON-NLS
return false;
}
}