@Override
public void finalStatistics(EvolutionState state, int result) {
super.finalStatistics(state, result);
if (compress) {
try {
taos.close();
} catch (IOException ex) {
Logger.getLogger(SampleSolutionsStat.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
private void run(File f) throws IOException, ParserException {
writeLogFile(logFile, "Start HDT on file " + f.getAbsolutePath(), false);
long start = System.currentTimeMillis();
OutputStream os = new FileOutputStream(f.getAbsolutePath() + ".hdt.tar.bz2", false);
OutputStream bzos = new BZip2CompressorOutputStream(os);
TarArchiveOutputStream aos = new TarArchiveOutputStream(bzos);
HDT hdt =
HDTManager.generateHDT(
f.getAbsolutePath(),
"urn:rdfcomp",
RDFNotation.parse("ntriples"),
new HDTSpecification(),
null);
hdt.saveToHDT(tmpDir + "/" + name + "_data.hdt", null);
long saveToHDT = System.currentTimeMillis() - start;
File filePrefix = new File(tmpDir + "/" + name + "_data.hdt");
TarArchiveEntry entry = new TarArchiveEntry(filePrefix, "mappings.hdt");
entry.setSize(filePrefix.length());
aos.putArchiveEntry(entry);
IOUtils.copy(new FileInputStream(filePrefix), aos);
aos.closeArchiveEntry();
aos.finish();
aos.close();
bzos.close();
os.close();
long saveToTarBzip2 = System.currentTimeMillis() - saveToHDT;
long overall = System.currentTimeMillis() - start;
String log =
"Original size: "
+ f.length()
+ "B = "
+ f.length() / 1024
+ " KB"
+ " ="
+ f.length() / (1024 * 1024)
+ " MB";
writeLogFile(logFile, log, true);
Model model = ModelLoader.getModel(file.getAbsolutePath());
long ntBzip2 = computeOrginalNTriple(model, file);
log =
"NT+BZIP size: "
+ ntBzip2
+ " B= "
+ ntBzip2 / 1024
+ " KB= "
+ ntBzip2 / (1024 * 1024)
+ " MB";
writeLogFile(logFile, log, true);
log =
"HDT size: "
+ filePrefix.length()
+ " B= "
+ filePrefix.length() / 1024
+ " KB= "
+ filePrefix.length() / (1024 * 1024)
+ " MB";
writeLogFile(logFile, log, true);
long hdtBzip2Size = new File(f.getAbsolutePath() + ".hdt.tar.bz2").length();
log =
"HDT+BZIP2 size: "
+ hdtBzip2Size
+ " B= "
+ hdtBzip2Size / 1024
+ " KB= "
+ hdtBzip2Size / (1024 * 1024)
+ " MB";
writeLogFile(logFile, log, true);
double ratio = new Double(filePrefix.length()) / new Double(ntBzip2);
log = "HDT / NTBZip2 ratio= " + ratio;
writeLogFile(logFile, log, true);
ratio = new Double(hdtBzip2Size) / new Double(ntBzip2);
log = "HDT+BZIP2 / NTBZip2 ratio= " + ratio + " \n\n";
writeLogFile(logFile, log, true);
log = "Time HDT: " + saveToHDT + "ms = " + saveToHDT / 1000 + " s ";
writeLogFile(logFile, log, true);
log += "Time HDT+BZIP: " + saveToTarBzip2 + "ms =" + saveToTarBzip2 / 1000 + "s";
writeLogFile(logFile, log, true);
log += "Time overall: " + overall + "ms = " + overall / 1000 + " s ";
writeLogFile(logFile, log, true);
}
/**
* Build the data archive of the deb from the provided DataProducers
*
* @param pData
* @param pOutput
* @param pChecksums
* @param compression the compression method used for the data file
* @return
* @throws NoSuchAlgorithmException
* @throws IOException
* @throws CompressorException
*/
BigInteger buildData(
final DataProducer[] pData,
final File pOutput,
final StringBuilder pChecksums,
Compression compression)
throws NoSuchAlgorithmException, IOException, CompressorException {
final File dir = pOutput.getParentFile();
if (dir != null && (!dir.exists() || !dir.isDirectory())) {
throw new IOException("Cannot write data file at '" + pOutput.getAbsolutePath() + "'");
}
final TarArchiveOutputStream tarOutputStream =
new TarArchiveOutputStream(
compression.toCompressedOutputStream(new FileOutputStream(pOutput)));
tarOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
final MessageDigest digest = MessageDigest.getInstance("MD5");
final Total dataSize = new Total();
final List<String> addedDirectories = new ArrayList<String>();
final DataConsumer receiver =
new DataConsumer() {
public void onEachDir(
String dirname,
String linkname,
String user,
int uid,
String group,
int gid,
int mode,
long size)
throws IOException {
dirname = fixPath(dirname);
createParentDirectories((new File(dirname)).getParent(), user, uid, group, gid);
// The directory passed in explicitly by the caller also gets the passed-in mode.
// (Unlike
// the parent directories for now. See related comments at "int mode =" in
// createParentDirectories, including about a possible bug.)
createDirectory(dirname, user, uid, group, gid, mode, 0);
console.info("dir: " + dirname);
}
public void onEachFile(
InputStream inputStream,
String filename,
String linkname,
String user,
int uid,
String group,
int gid,
int mode,
long size)
throws IOException {
filename = fixPath(filename);
createParentDirectories((new File(filename)).getParent(), user, uid, group, gid);
final TarArchiveEntry entry = new TarArchiveEntry(filename, true);
entry.setUserName(user);
entry.setUserId(uid);
entry.setGroupName(group);
entry.setGroupId(gid);
entry.setMode(mode);
entry.setSize(size);
tarOutputStream.putArchiveEntry(entry);
dataSize.add(size);
digest.reset();
Utils.copy(inputStream, new DigestOutputStream(tarOutputStream, digest));
final String md5 = Utils.toHex(digest.digest());
tarOutputStream.closeArchiveEntry();
console.info(
"file:"
+ entry.getName()
+ " size:"
+ entry.getSize()
+ " mode:"
+ entry.getMode()
+ " linkname:"
+ entry.getLinkName()
+ " username:"******" userid:"
+ entry.getUserId()
+ " groupname:"
+ entry.getGroupName()
+ " groupid:"
+ entry.getGroupId()
+ " modtime:"
+ entry.getModTime()
+ " md5: "
+ md5);
// append to file md5 list
pChecksums.append(md5).append(" ").append(entry.getName()).append('\n');
}
public void onEachLink(
String path,
String linkName,
boolean symlink,
String user,
int uid,
String group,
int gid,
int mode)
throws IOException {
path = fixPath(path);
createParentDirectories((new File(path)).getParent(), user, uid, group, gid);
final TarArchiveEntry entry =
new TarArchiveEntry(
path, symlink ? TarArchiveEntry.LF_SYMLINK : TarArchiveEntry.LF_LINK);
entry.setLinkName(linkName);
entry.setUserName(user);
entry.setUserId(uid);
entry.setGroupName(group);
entry.setGroupId(gid);
entry.setMode(mode);
tarOutputStream.putArchiveEntry(entry);
tarOutputStream.closeArchiveEntry();
console.info(
"link:"
+ entry.getName()
+ " mode:"
+ entry.getMode()
+ " linkname:"
+ entry.getLinkName()
+ " username:"******" userid:"
+ entry.getUserId()
+ " groupname:"
+ entry.getGroupName()
+ " groupid:"
+ entry.getGroupId());
}
private String fixPath(String path) {
// If we're receiving directory names from Windows, then we'll convert to use slash
// This does eliminate the ability to use of a backslash in a directory name on *NIX,
// but in practice, this is a non-issue
if (path.indexOf('\\') > -1) {
path = path.replace('\\', '/');
}
// ensure the path is like : ./foo/bar
if (path.startsWith("/")) {
path = "." + path;
} else if (!path.startsWith("./")) {
path = "./" + path;
}
return path;
}
private void createDirectory(
String directory, String user, int uid, String group, int gid, int mode, long size)
throws IOException {
// All dirs should end with "/" when created, or the test
// DebAndTaskTestCase.testTarFileSet() thinks its a file
// and so thinks it has the wrong permission.
// This consistency also helps when checking if a directory already exists in
// addedDirectories.
if (!directory.endsWith("/")) {
directory += "/";
}
if (!addedDirectories.contains(directory)) {
TarArchiveEntry entry = new TarArchiveEntry(directory, true);
entry.setUserName(user);
entry.setUserId(uid);
entry.setGroupName(group);
entry.setGroupId(gid);
entry.setMode(mode);
entry.setSize(size);
tarOutputStream.putArchiveEntry(entry);
tarOutputStream.closeArchiveEntry();
addedDirectories.add(
directory); // so addedDirectories consistently have "/" for finding duplicates.
}
}
private void createParentDirectories(
String dirname, String user, int uid, String group, int gid) throws IOException {
// Debian packages must have parent directories created
// before sub-directories or files can be installed.
// For example, if an entry of ./usr/lib/foo/bar existed
// in a .deb package, but the ./usr/lib/foo directory didn't
// exist, the package installation would fail. The .deb must
// then have an entry for ./usr/lib/foo and then ./usr/lib/foo/bar
if (dirname == null) {
return;
}
// The loop below will create entries for all parent directories
// to ensure that .deb packages will install correctly.
String[] pathParts = dirname.split("\\/");
String parentDir = "./";
for (int i = 1; i < pathParts.length; i++) {
parentDir += pathParts[i] + "/";
// Make it so the dirs can be traversed by users.
// We could instead try something more granular, like setting the directory
// permission to 'rx' for each of the 3 user/group/other read permissions
// found on the file being added (ie, only if "other" has read
// permission on the main node, then add o+rx permission on all the containing
// directories, same w/ user & group), and then also we'd have to
// check the parentDirs collection of those already added to
// see if those permissions need to be similarly updated. (Note, it hasn't
// been demonstrated, but there might be a bug if a user specifically
// requests a directory with certain permissions,
// that has already been auto-created because it was a parent, and if so, go set
// the user-requested mode on that directory instead of this automatic one.)
// But for now, keeping it simple by making every dir a+rx. Examples are:
// drw-r----- fs/fs # what you get with setMode(mode)
// drwxr-xr-x fs/fs # Usable. Too loose?
int mode = TarArchiveEntry.DEFAULT_DIR_MODE;
createDirectory(parentDir, user, uid, group, gid, mode, 0);
}
}
};
try {
for (DataProducer data : pData) {
data.produce(receiver);
}
} finally {
tarOutputStream.close();
}
console.info("Total size: " + dataSize);
return dataSize.count;
}
protected void cleanUp() throws IOException {
super.cleanUp();
tOut.close();
}
/**
* Build control archive of the deb
*
* @param pControlFiles
* @param pDataSize
* @param pChecksums
* @param pOutput
* @return
* @throws FileNotFoundException
* @throws IOException
* @throws ParseException
*/
private PackageDescriptor buildControl(
final File[] pControlFiles,
final BigInteger pDataSize,
final StringBuilder pChecksums,
final File pOutput)
throws IOException, ParseException {
final File dir = pOutput.getParentFile();
if (dir != null && (!dir.exists() || !dir.isDirectory())) {
throw new IOException("Cannot write control file at '" + pOutput.getAbsolutePath() + "'");
}
final TarArchiveOutputStream outputStream =
new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(pOutput)));
outputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
List<FilteredConfigurationFile> configurationFiles = new ArrayList<FilteredConfigurationFile>();
// create a descriptor out of the "control" file, copy all other files, ignore directories
PackageDescriptor packageDescriptor = null;
for (File file : pControlFiles) {
if (file.isDirectory()) {
// warn about the misplaced directory, except for directories ignored by default (.svn, cvs,
// etc)
if (!isDefaultExcludes(file)) {
console.info(
"Found directory '"
+ file
+ "' in the control directory. Maybe you are pointing to wrong dir?");
}
continue;
}
if (CONFIGURATION_FILENAMES.contains(file.getName())
|| MAINTAINER_SCRIPTS.contains(file.getName())) {
FilteredConfigurationFile configurationFile =
new FilteredConfigurationFile(file.getName(), new FileInputStream(file), resolver);
configurationFiles.add(configurationFile);
} else if ("control".equals(file.getName())) {
packageDescriptor = createPackageDescriptor(file, pDataSize);
} else {
// initialize the information stream to guess the type of the file
InformationInputStream infoStream = new InformationInputStream(new FileInputStream(file));
Utils.copy(infoStream, NullOutputStream.NULL_OUTPUT_STREAM);
infoStream.close();
// fix line endings for shell scripts
InputStream in = new FileInputStream(file);
if (infoStream.isShell() && !infoStream.hasUnixLineEndings()) {
byte[] buf = Utils.toUnixLineEndings(in);
in = new ByteArrayInputStream(buf);
}
addControlEntry(file.getName(), IOUtils.toString(in), outputStream);
in.close();
}
}
if (packageDescriptor == null) {
throw new FileNotFoundException(
"No 'control' file found in " + Arrays.toString(pControlFiles));
}
for (FilteredConfigurationFile configurationFile : configurationFiles) {
addControlEntry(configurationFile.getName(), configurationFile.toString(), outputStream);
}
addControlEntry("control", packageDescriptor.toString(), outputStream);
addControlEntry("md5sums", pChecksums.toString(), outputStream);
outputStream.close();
return packageDescriptor;
}