@Test
public void testZipAndUnzipDirectory() throws IOException {
File basedir = File.createTempFile(ZipUtils.class.getSimpleName(), ".tmp");
basedir.delete();
assertTrue("Could not create temporary base directory.", basedir.mkdir());
for (int i = 0; i < 5; i++) {
createSubDirWithFiles(basedir, i);
}
File zipFile = File.createTempFile(ZipUtils.class.getSimpleName(), ".zip");
ZipArchiveOutputStream zipOutputStream =
new ZipArchiveOutputStream(new FileOutputStream(zipFile));
ZipUtils.addFilesToZipRecursively("mybase", basedir, null, zipOutputStream);
zipOutputStream.close();
final File unzipBase = File.createTempFile(ZipUtils.class.getSimpleName(), ".unzipped");
ZipUtils.unzip(zipFile, unzipBase);
IOUtils.processFiles(
unzipBase,
new FileProcessor() {
public void process(File file) {
try {
assertEquals(
IOUtils.getFileIdentifier(unzipBase, file), FileUtils.readFileToString(file));
} catch (IOException e) {
throw new RuntimeException("Could not process file.", e);
}
}
},
null);
}
@Override
public void onConfigurationFailure(ITestResult iTestResult) {
super.onConfigurationFailure(iTestResult);
String testName = iTestResult.getTestClass().getName();
String configurationName = iTestResult.getMethod().toString().split("\\(|\\)")[0];
if (!enableLogstash && isAfter(iTestResult)) {
DumpUtils.copyBeforeConfigurationsLogToTestDir(testName, suiteName);
testName = testMethodName;
}
if (suiteName
== null) { // this is in case the suite has a @BeforeSuite method. which is invoked before
// the onStart is.
suiteName = System.getProperty("iTests.suiteName", "sgtest");
}
LogUtils.log("Configuration Failed: " + configurationName, iTestResult.getThrowable());
if (enableLogstash && iTestResult.getMethod().isBeforeClassConfiguration()) {
initLogstash2(iTestResult);
}
String newmanTestFolder = System.getProperty("newman.test.path");
if (newmanTestFolder != null) {
File testFolder = new File(newmanTestFolder);
ZipUtils.unzipArchive(testFolder);
try {
copyAllFilesToLogDir(testFolder, testFolder);
} catch (IOException e) {
LogUtils.log("Failed to copy all log files");
}
} else {
ZipUtils.unzipArchive(testMethodName, suiteName);
}
if (enableLogstash
&& isAfter(iTestResult)
&& !iTestResult.getMethod().isAfterClassConfiguration()
&& !iTestResult.getMethod().isAfterSuiteConfiguration()) {
testName = testMethodName;
}
File testFolder = DumpUtils.createTestFolder(testName, suiteName);
write2LogFile(iTestResult, testFolder);
write2ErrorTxtFile(iTestResult, testFolder);
if (isAfter(iTestResult)) {
if (enableLogstash) {
if (process != null) {
killLogstashAgent(1, logstashLogPath);
}
if (process2 != null && iTestResult.getMethod().isAfterClassConfiguration()) {
killLogstashAgent(2, logstashLogPath2);
}
}
}
}
/**
* ************* Packs an application folder into a zip file.
*
* @param application the application object as read from the application file.
* @param applicationDir the directory where the application was read from.
* @param additionalServiceFiles additional files that should be packaged into each service
* directory.
* @return the packaged zip file.
* @throws IOException .
* @throws PackagingException .
*/
public static File packApplication(
final Application application, final File applicationDir, final File[] additionalServiceFiles)
throws IOException, PackagingException {
boolean hasExtendedServices = false;
for (final Service service : application.getServices()) {
if (!service.getExtendedServicesPaths().isEmpty()) {
hasExtendedServices = true;
break;
}
}
File applicationFolderToPack = applicationDir;
// If there are no extended service we don't need to prepare an application folder to pack with
// all the
// extended services content.
if (hasExtendedServices) {
final File destApplicationFolder = createCopyDirectory(applicationFolderToPack);
for (final Service service : application.getServices()) {
final File extFolder = new File(destApplicationFolder + "/" + service.getName());
final File recipeFile =
DSLReader.findDefaultDSLFile(
DSLReader.SERVICE_DSL_FILE_NAME_SUFFIX,
new File(applicationDir + "/" + service.getName()));
copyExtendedServiceFiles(service, recipeFile, extFolder);
}
// Pack the prepared folder instead of the original application folder.
applicationFolderToPack = destApplicationFolder;
}
if ((additionalServiceFiles != null) && (additionalServiceFiles.length > 0)) {
// if a copy directory was already created, use the existing one, otherwise
// create a new one.
if (applicationFolderToPack == applicationDir) {
applicationFolderToPack = createCopyDirectory(applicationFolderToPack);
}
List<Service> services = application.getServices();
for (Service service : services) {
File serviceDir = new File(applicationFolderToPack, service.getName());
if (!serviceDir.exists()) {
throw new PackagingException("Could not find service folder at: " + serviceDir);
}
if (!serviceDir.isDirectory()) {
throw new PackagingException("Was expecting a directory at: " + serviceDir);
}
for (File fileToCopy : additionalServiceFiles) {
FileUtils.copyFileToDirectory(fileToCopy, serviceDir);
}
}
}
// zip the application folder.
final File zipFile = File.createTempFile("application", ".zip");
zipFile.deleteOnExit();
ZipUtils.zip(applicationFolderToPack, zipFile);
return zipFile;
}
@Test
public void getDirectorySize() throws Throwable {
File root = temp.getRoot();
File file = new File(root, "file1.txt");
FileUtils.write(file, "a");
file = new File(root, "file2.txt");
FileUtils.write(file, "ab");
File dir = new File(root, "dir");
dir.mkdir();
file = new File(dir, "file3.txt");
FileUtils.write(file, "abc");
assertEquals(6, ZipUtils.getDirectorySize(root));
}
private static File createZippedPu(
final Service service, final File puFolderToZip, final File recipeFile)
throws IOException, PackagingException {
logger.finer("trying to zip " + puFolderToZip.getAbsolutePath());
final String serviceName =
service.getName() != null ? service.getName() : recipeFile.getParentFile().getName();
// create a temp dir under the system temp dir
final File tmpFile = File.createTempFile("ServicePackage", null);
tmpFile.delete();
tmpFile.mkdir();
final File zipFile = new File(tmpFile, serviceName + ".zip");
// files will be deleted in reverse order
tmpFile.deleteOnExit();
zipFile.deleteOnExit();
ServiceReader.validateFolderSize(puFolderToZip, service.getMaxJarSize());
ZipUtils.zip(puFolderToZip, zipFile);
logger.finer("zipped folder successfully to " + zipFile.getAbsolutePath());
return zipFile;
}
/** {@inheritDoc} */
@SuppressWarnings("null")
@Override
public RawDataFile execute() throws MSDKException {
String osName = System.getProperty("os.name").toUpperCase();
if (!osName.contains("WINDOWS")) {
throw new MSDKException("Native data format import only works on MS Windows");
}
logger.info("Started parsing file " + sourceFile);
try {
// Decompress the thermo raw dump executable to a temporary folder
File tempFolder = Files.createTempDirectory("msdk").toFile();
tempFolder.deleteOnExit();
InputStream dumpArchive =
this.getClass().getClassLoader().getResourceAsStream("thermorawdump.zip");
if (dumpArchive == null)
throw new MSDKException("Failed to load the thermorawdump.zip archive from the MSDK jar");
ZipUtils.extractStreamToFolder(dumpArchive, tempFolder);
// Path to the rawdump executable
File rawDumpPath = new File(tempFolder, "ThermoRawDump.exe");
if (!rawDumpPath.canExecute())
throw new MSDKException("Cannot execute program " + rawDumpPath);
// Create a separate process and execute RAWdump.exe
final String cmdLine[] = new String[] {rawDumpPath.getPath(), sourceFile.getPath()};
dumperProcess = Runtime.getRuntime().exec(cmdLine);
// Get the stdout of RAWdump process as InputStream
InputStream dumpStream = dumperProcess.getInputStream();
// Create the new RawDataFile
String fileName = sourceFile.getName();
newRawFile = MSDKObjectBuilder.getRawDataFile(fileName, sourceFile, fileType, dataStore);
// Read the dump data
parser = new RawDumpParser(newRawFile, dataStore);
parser.readRAWDump(dumpStream);
// Cleanup
dumpStream.close();
dumperProcess.destroy();
try {
FileUtils.deleteDirectory(tempFolder);
} catch (IOException e) {
// Ignore errors while deleting the tmp folder
}
if (canceled) return null;
} catch (Throwable e) {
if (dumperProcess != null) dumperProcess.destroy();
throw new MSDKException(e);
}
logger.info("Finished parsing " + sourceFile);
return newRawFile;
}
public static void generateMainPage(File mainFile, File sourceProjDir)
throws IOException, ProjectFileParsingException, NeuroMLException {
SimpleXMLElement root = new SimpleXMLElement("document");
SimpleXMLElement header = new SimpleXMLElement("header");
root.addChildElement(header);
SimpleXMLElement title = new SimpleXMLElement("title");
header.addChildElement(title);
SimpleXMLElement body = new SimpleXMLElement("body");
root.addChildElement(body);
SimpleXMLElement intro = new SimpleXMLElement("p");
body.addChildElement(intro);
if (!mainFile.getParentFile().exists()) mainFile.getParentFile().mkdir();
File targetDownloadDir = new File(mainFile.getParentFile(), "downloads");
if (!targetDownloadDir.exists()) targetDownloadDir.mkdir();
if (sourceProjDir.getName().indexOf("examples") >= 0) {
title.addContent("neuroConstruct example projects");
intro.addContent(
"Downloadable neuroConstruct example projects. These <strong>illustrate the core "
+ "functionality of neuroConstruct</strong>, as opposed to providing electrophysiologically accurate "
+ "models. Projects based on published conductance based models can be found <a href=\"../models/index.html\">here</a>");
}
if (sourceProjDir.getName().indexOf("models") >= 0) {
title.addContent("neuroConstruct projects based on published neuronal and network models");
intro.addContent(
"Downloadable neuroConstruct projects <strong>based on published conductance based models</strong>. "
+ "Some examples to illustrate the core functionality of neuroConstruct, as opposed to "
+ "providing electrophysiologically accurate models can be found <a href=\"../samples/index.html\">here</a>."
+ "<p>Note: These models are currently being moved to a repository to allow open source, collaborative development of NeuroML models.</p>"
+ "<p>See the <a href=\"http://www.opensourcebrain.org\">Open Source Brain</a> website for full details. "
+ "<img alt=\"Open Source Brain\" src=\"http://www.opensourcebrain.org/images/logo.png\"/></p>");
}
File[] fileArray = sourceProjDir.listFiles();
fileArray = GeneralUtils.reorderAlphabetically(fileArray, true);
ArrayList<File> files = GeneralUtils.toArrayList(fileArray);
// if (files.contains(""))
ArrayList<String> toIgnore = new ArrayList<String>();
// toIgnore.add("Thalamocortical"); // temporarily
// toIgnore.add("CA1PyramidalCell"); // temporarily
// toIgnore.add("SolinasEtAl-GolgiCell"); // temporarily
for (File exProjDir : files) {
File morphDir = new File(exProjDir, "cellMechanisms");
if (morphDir.isDirectory() && !toIgnore.contains(exProjDir.getName())) {
String projName = exProjDir.getName();
SimpleXMLElement section = new SimpleXMLElement("section");
body.addChildElement(section);
SimpleXMLElement secTitle = new SimpleXMLElement("title");
section.addChildElement(secTitle);
secTitle.addContent(projName);
SimpleXMLElement anchor = new SimpleXMLElement("anchor");
section.addChildElement(anchor);
anchor.addAttribute("id", projName);
SimpleXMLElement table = new SimpleXMLElement("table");
section.addChildElement(table);
SimpleXMLElement row = new SimpleXMLElement("tr");
table.addChildElement(row);
String largeImg = "large.png";
String smallImg = "small.png";
File targetImageDir = new File(mainFile.getParentFile(), "images");
if (!targetImageDir.exists()) targetImageDir.mkdir();
File targetProjImageDir = new File(targetImageDir, projName);
if (!targetProjImageDir.exists()) targetProjImageDir.mkdir();
File smallImgFile = new File(exProjDir, "images/" + smallImg);
File largeImgFile = new File(exProjDir, "images/" + largeImg);
if (smallImgFile.exists()) {
GeneralUtils.copyFileIntoDir(smallImgFile, targetProjImageDir);
SimpleXMLElement col2 = new SimpleXMLElement("td");
row.addChildElement(col2);
col2.addAttribute("width", "120");
SimpleXMLElement secImg = new SimpleXMLElement("p");
col2.addChildElement(secImg);
SimpleXMLElement img = new SimpleXMLElement("img");
img.addAttribute("src", "images/" + projName + "/small.png");
img.addAttribute("alt", "Screenshot of " + projName);
if (largeImgFile.exists()) {
GeneralUtils.copyFileIntoDir(largeImgFile, targetProjImageDir);
SimpleXMLElement imgRef = new SimpleXMLElement("a");
img.addAttribute("title", "Click to enlarge");
imgRef.addAttribute("href", "images/" + projName + "/" + largeImg);
imgRef.addChildElement(img);
secImg.addChildElement(imgRef);
} else {
secImg.addChildElement(img);
}
}
SimpleXMLElement secIntro = new SimpleXMLElement("p");
SimpleXMLElement colMid = new SimpleXMLElement("td");
SimpleXMLElement colRight = new SimpleXMLElement("td");
row.addChildElement(colMid);
row.addChildElement(colRight);
colRight.addAttribute("width", "150");
colMid.addChildElement(secIntro);
secIntro.addContent("Project name: <strong>" + projName + "</strong>");
File projFile = ProjectStructure.findProjectFile(exProjDir);
Project project = Project.loadProject(projFile, null);
String descFull = project.getProjectDescription();
String breakpoint = "\n\n";
String descShort = new String(descFull);
if (descFull.indexOf(breakpoint) > 0) {
descShort = descFull.substring(0, descFull.indexOf(breakpoint));
}
SimpleXMLElement desc = new SimpleXMLElement("p");
colMid.addChildElement(desc);
desc.addContent(GeneralUtils.parseForHyperlinks(descShort));
SimpleXMLElement modified = new SimpleXMLElement("p");
colMid.addChildElement(modified);
SimpleDateFormat formatter = new SimpleDateFormat("EEEE MMMM d, yyyy");
java.util.Date date = new java.util.Date(projFile.lastModified());
modified.addContent("Project last modified: " + formatter.format(date));
File zipFile = null;
String zipFileName =
targetDownloadDir.getAbsolutePath()
+ "/"
+ projName
+ ProjectStructure.getNewProjectZipFileExtension();
ArrayList<String> ignore = new ArrayList<String>();
ArrayList<String> ignoreNone = new ArrayList<String>();
ArrayList<String> ignoreExtns = new ArrayList<String>();
ignore.add("i686");
ignore.add("x86_64");
ignore.add(".svn");
ignore.add("simulations");
ignore.add("generatedNEURON");
ignore.add("generatedNeuroML");
ignore.add("generatedGENESIS");
ignore.add("generatedMOOSE");
ignore.add("generatedPyNN");
ignore.add("generatedPSICS");
ignore.add("dataSets");
ignoreExtns.add("bak");
zipFile = ZipUtils.zipUp(exProjDir, zipFileName, ignore, ignoreExtns);
logger.logComment(
"The zip file: "
+ zipFile.getAbsolutePath()
+ " ("
+ zipFile.length()
+ " bytes) contains all of the project files");
SimpleXMLElement downloads = new SimpleXMLElement("p");
colRight.addChildElement(downloads);
downloads.addContent("Downloads<a href=\"#downloadInfo\">*</a>:");
SimpleXMLElement downloadProj = new SimpleXMLElement("p");
colRight.addChildElement(downloadProj);
SimpleXMLElement link = new SimpleXMLElement("a");
link.addAttribute("href", "downloads/" + zipFile.getName());
link.addContent("neuroConstruct project");
link.addAttribute("title", "Download full project for loading into neuroConstruct");
downloadProj.addChildElement(link);
ArrayList<String> noNeuroML = new ArrayList<String>();
noNeuroML.add("Ex3_Morphology");
noNeuroML.add("DentateGyrus");
noNeuroML.add("RothmanEtAl_KoleEtAl_PyrCell");
if (!noNeuroML.contains(projName)) {
project.neuromlFileManager.generateNeuroMLFiles(
null, new OriginalCompartmentalisation(), 1234, false);
File neuroMLDir = ProjectStructure.getNeuroML1Dir(project.getProjectMainDirectory());
String nmlZipFileName =
targetDownloadDir.getAbsolutePath() + "/" + projName + "_NeuroML.zip";
zipFile = ZipUtils.zipUp(neuroMLDir, nmlZipFileName, ignoreNone, ignoreNone);
SimpleXMLElement downloadNml = new SimpleXMLElement("p");
colRight.addChildElement(downloadNml);
// downloadNml.addContent("Download project as pure NeuroML: ");
SimpleXMLElement img = new SimpleXMLElement("img");
img.addAttribute("src", "../images/NeuroMLSmall.png");
String info = "Download core project elements in NeuroML format";
img.addAttribute("alt", info);
SimpleXMLElement imgRef = new SimpleXMLElement("a");
img.addAttribute("title", info);
imgRef.addAttribute("href", "downloads/" + zipFile.getName());
imgRef.addChildElement(img);
downloadNml.addChildElement(imgRef);
}
}
}
SimpleXMLElement end = new SimpleXMLElement("p");
body.addChildElement(end);
end.addContent(" ");
SimpleXMLElement infoDlanchor = new SimpleXMLElement("anchor");
body.addChildElement(infoDlanchor);
end.addAttribute("id", "downloadInfo");
SimpleXMLElement infoDl = new SimpleXMLElement("p");
body.addChildElement(infoDl);
end.addContent(
"* Note: neuroConstruct project downloads (most of which are included with the standard software distribution) "
+ "can be loaded directly into neuroConstruct to generate cell and network scripts for NEURON, GENESIS, etc.,"
+ " but NeuroML downloads just consist of the core elements of the project"
+ " (morphologies, channels, etc.) which have been exported in NeuroML format. The latter can be useful for testing NeuroML compliant applications. "
+ "If no NeuroML download link is present, this usually indicates that the model is mainly implemented using channel/synapse mechanisms in a simulator's "
+ "native language (e.g. mod files) which have not fully been converted to ChannelML yet.");
SimpleXMLElement end2 = new SimpleXMLElement("p");
body.addChildElement(end2);
end2.addContent(" ");
FileWriter fw = null;
try {
fw = new FileWriter(mainFile);
fw.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); // quick hack, todo: add to
// SimpleXMLDoc...
fw.write(
"<!DOCTYPE document PUBLIC \"-//APACHE//DTD Documentation V2.0//EN\" \"http://forrest.apache.org/dtd/document-v20.dtd\">\n\n");
fw.write(root.getXMLString("", false));
fw.flush();
fw.close();
} catch (IOException ex) {
logger.logError("Problem: ", ex);
fw.close();
}
/*
<header>
<title>Examples of neuroConstruct in use</title>
</header>
<body>
<p>Some screenshots of neuroConstruct in action are given below.
Click on the thumbnails to see a full size version of the screenshots</p>
<section>
<title>Examples included with distribution</title>*/
}
@Test
public void zipAndUnzipDirectory() throws Throwable {
File root = temp.getRoot();
File zip = new File(root, "zip.zip");
{
File dir = new File(root, "zipMe");
dir.mkdir();
File file1 = new File(dir, "file1.txt");
FileUtils.write(file1, "data1");
File file2 = new File(dir, "file2.txt");
FileUtils.write(file2, "data2");
File dir2 = new File(dir, "folder2");
dir2.mkdir();
File file3 = new File(dir2, "file3.txt");
FileUtils.write(file3, "data3");
File dir3 = new File(dir, "folder3");
dir3.mkdir();
ZipListener listener = mock(ZipListener.class);
ZipUtils.zipDirectory(dir, zip, listener);
verify(listener).onZippedFile(file1);
verify(listener).onZippedFile(file2);
verify(listener).onZippedFile(file3);
Set<String> actualPaths = new HashSet<String>();
ZipFile zipFile = new ZipFile(zip);
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
actualPaths.add(entry.getName());
}
zipFile.close();
Set<String> expectedPaths = new HashSet<String>();
expectedPaths.add("/" + dir.getName() + "/" + file1.getName());
expectedPaths.add("/" + dir.getName() + "/" + file2.getName());
expectedPaths.add("/" + dir.getName() + "/" + dir2.getName() + "/" + file3.getName());
expectedPaths.add("/" + dir.getName() + "/" + dir3.getName() + "/empty");
assertEquals(expectedPaths, actualPaths);
}
{
File destinationDir = new File(root, "destination");
ZipUtils.unzip(destinationDir, zip);
assertEquals(
"data1", FileUtils.readFileToString(new File(destinationDir, "zipMe/file1.txt")));
assertEquals(
"data2", FileUtils.readFileToString(new File(destinationDir, "zipMe/file2.txt")));
assertEquals(
"data3", FileUtils.readFileToString(new File(destinationDir, "zipMe/folder2/file3.txt")));
assertTrue(new File(destinationDir, "zipMe/folder3").isDirectory());
assertFalse(new File(destinationDir, "zipMe/folder3/empty").exists());
}
}
