void createArchive(Path projectPath, Path output, Config overrideParams) throws IOException {
out.println("Creating " + output + "...");
ProjectArchive project =
projectLoader.load(projectPath, WorkflowResourceMatcher.defaultMatcher(), overrideParams);
ArchiveMetadata meta = project.getArchiveMetadata();
try (TarArchiveOutputStream tar =
new TarArchiveOutputStream(new GzipCompressorOutputStream(Files.newOutputStream(output)))) {
// default mode for file names longer than 100 bytes is throwing an exception (LONGFILE_ERROR)
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
project.listFiles(
(resourceName, absPath) -> {
if (!Files.isDirectory(absPath)) {
out.println(" Archiving " + resourceName);
TarArchiveEntry e = buildTarArchiveEntry(project, absPath, resourceName);
tar.putArchiveEntry(e);
if (e.isSymbolicLink()) {
out.println(" symlink -> " + e.getLinkName());
} else {
try (InputStream in = Files.newInputStream(absPath)) {
ByteStreams.copy(in, tar);
}
tar.closeArchiveEntry();
}
}
});
// create .digdag.dig
// TODO set default time zone if not set?
byte[] metaBody = yamlMapper.toYaml(meta).getBytes(StandardCharsets.UTF_8);
TarArchiveEntry metaEntry = new TarArchiveEntry(ArchiveMetadata.FILE_NAME);
metaEntry.setSize(metaBody.length);
metaEntry.setModTime(new Date());
tar.putArchiveEntry(metaEntry);
tar.write(metaBody);
tar.closeArchiveEntry();
}
out.println("Workflows:");
for (WorkflowDefinition workflow : meta.getWorkflowList().get()) {
out.println(" " + workflow.getName());
}
out.println("");
}
public TarArchiveEntry map(final TarArchiveEntry entry) {
final String name = entry.getName();
final TarArchiveEntry newEntry =
new TarArchiveEntry(prefix + '/' + Utils.stripPath(strip, name), true);
// Set ownership
if (uid > -1) {
newEntry.setUserId(uid);
} else {
newEntry.setUserId(entry.getUserId());
}
if (gid > -1) {
newEntry.setGroupId(gid);
} else {
newEntry.setGroupId(entry.getGroupId());
}
if (user != null) {
newEntry.setUserName(user);
} else {
newEntry.setUserName(entry.getUserName());
}
if (group != null) {
newEntry.setGroupName(group);
} else {
newEntry.setGroupName(entry.getGroupName());
}
// Set permissions
if (newEntry.isDirectory()) {
if (dirMode > -1) {
newEntry.setMode(dirMode);
} else {
newEntry.setMode(entry.getMode());
}
} else {
if (fileMode > -1) {
newEntry.setMode(fileMode);
} else {
newEntry.setMode(entry.getMode());
}
}
newEntry.setSize(entry.getSize());
return newEntry;
}
private static void addControlEntry(
final String pName, final String pContent, final TarArchiveOutputStream pOutput)
throws IOException {
final byte[] data = pContent.getBytes("UTF-8");
final TarArchiveEntry entry = new TarArchiveEntry("./" + pName, true);
entry.setSize(data.length);
entry.setNames("root", "root");
if (MAINTAINER_SCRIPTS.contains(pName)) {
entry.setMode(PermMapper.toMode("755"));
} else {
entry.setMode(PermMapper.toMode("644"));
}
pOutput.putArchiveEntry(entry);
pOutput.write(data);
pOutput.closeArchiveEntry();
}
private void run(File f) throws IOException, ParserException {
writeLogFile(logFile, "Start HDT on file " + f.getAbsolutePath(), false);
long start = System.currentTimeMillis();
OutputStream os = new FileOutputStream(f.getAbsolutePath() + ".hdt.tar.bz2", false);
OutputStream bzos = new BZip2CompressorOutputStream(os);
TarArchiveOutputStream aos = new TarArchiveOutputStream(bzos);
HDT hdt =
HDTManager.generateHDT(
f.getAbsolutePath(),
"urn:rdfcomp",
RDFNotation.parse("ntriples"),
new HDTSpecification(),
null);
hdt.saveToHDT(tmpDir + "/" + name + "_data.hdt", null);
long saveToHDT = System.currentTimeMillis() - start;
File filePrefix = new File(tmpDir + "/" + name + "_data.hdt");
TarArchiveEntry entry = new TarArchiveEntry(filePrefix, "mappings.hdt");
entry.setSize(filePrefix.length());
aos.putArchiveEntry(entry);
IOUtils.copy(new FileInputStream(filePrefix), aos);
aos.closeArchiveEntry();
aos.finish();
aos.close();
bzos.close();
os.close();
long saveToTarBzip2 = System.currentTimeMillis() - saveToHDT;
long overall = System.currentTimeMillis() - start;
String log =
"Original size: "
+ f.length()
+ "B = "
+ f.length() / 1024
+ " KB"
+ " ="
+ f.length() / (1024 * 1024)
+ " MB";
writeLogFile(logFile, log, true);
Model model = ModelLoader.getModel(file.getAbsolutePath());
long ntBzip2 = computeOrginalNTriple(model, file);
log =
"NT+BZIP size: "
+ ntBzip2
+ " B= "
+ ntBzip2 / 1024
+ " KB= "
+ ntBzip2 / (1024 * 1024)
+ " MB";
writeLogFile(logFile, log, true);
log =
"HDT size: "
+ filePrefix.length()
+ " B= "
+ filePrefix.length() / 1024
+ " KB= "
+ filePrefix.length() / (1024 * 1024)
+ " MB";
writeLogFile(logFile, log, true);
long hdtBzip2Size = new File(f.getAbsolutePath() + ".hdt.tar.bz2").length();
log =
"HDT+BZIP2 size: "
+ hdtBzip2Size
+ " B= "
+ hdtBzip2Size / 1024
+ " KB= "
+ hdtBzip2Size / (1024 * 1024)
+ " MB";
writeLogFile(logFile, log, true);
double ratio = new Double(filePrefix.length()) / new Double(ntBzip2);
log = "HDT / NTBZip2 ratio= " + ratio;
writeLogFile(logFile, log, true);
ratio = new Double(hdtBzip2Size) / new Double(ntBzip2);
log = "HDT+BZIP2 / NTBZip2 ratio= " + ratio + " \n\n";
writeLogFile(logFile, log, true);
log = "Time HDT: " + saveToHDT + "ms = " + saveToHDT / 1000 + " s ";
writeLogFile(logFile, log, true);
log += "Time HDT+BZIP: " + saveToTarBzip2 + "ms =" + saveToTarBzip2 / 1000 + "s";
writeLogFile(logFile, log, true);
log += "Time overall: " + overall + "ms = " + overall / 1000 + " s ";
writeLogFile(logFile, log, true);
}
public void produce(final DataConsumer pReceiver) throws IOException {
String user = Producers.ROOT_NAME;
int uid = Producers.ROOT_UID;
String group = Producers.ROOT_NAME;
int gid = Producers.ROOT_UID;
int filemode = TarEntry.DEFAULT_FILE_MODE;
int dirmode = TarEntry.DEFAULT_DIR_MODE;
String prefix = "";
if (fileset instanceof Tar.TarFileSet) {
Tar.TarFileSet tarfileset = (Tar.TarFileSet) fileset;
user = tarfileset.getUserName();
uid = tarfileset.getUid();
group = tarfileset.getGroup();
gid = tarfileset.getGid();
filemode = tarfileset.getMode();
dirmode = tarfileset.getDirMode(tarfileset.getProject());
prefix = tarfileset.getPrefix(tarfileset.getProject());
}
final DirectoryScanner scanner = fileset.getDirectoryScanner(fileset.getProject());
scanner.scan();
final File basedir = scanner.getBasedir();
for (String directory : scanner.getIncludedDirectories()) {
String name = directory.replace('\\', '/');
final TarArchiveEntry entry = new TarArchiveEntry(prefix + "/" + name);
entry.setUserName(user);
entry.setUserId(uid);
entry.setGroupName(group);
entry.setGroupId(gid);
entry.setMode(dirmode);
pReceiver.onEachDir(entry);
}
for (String filename : scanner.getIncludedFiles()) {
final String name = filename.replace('\\', '/');
final File file = new File(basedir, name);
final InputStream inputStream = new FileInputStream(file);
try {
final String entryName = prefix + "/" + name;
final File entryPath = new File(entryName);
final boolean symbolicLink = SymlinkUtils.isSymbolicLink(entryPath);
final TarArchiveEntry e;
if (symbolicLink) {
e = new TarArchiveEntry(entryName, TarConstants.LF_SYMLINK);
e.setLinkName(SymlinkUtils.readSymbolicLink(entryPath));
} else {
e = new TarArchiveEntry(entryName, true);
}
e.setUserId(uid);
e.setGroupId(gid);
e.setUserName(user);
e.setGroupName(group);
e.setMode(filemode);
e.setSize(file.length());
pReceiver.onEachFile(inputStream, e);
} finally {
inputStream.close();
}
}
}
/**
* tar a file
*
* @param entry the file to tar
* @param tOut the output stream
* @param vPath the path name of the file to tar
* @throws IOException on error
*/
protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String vPath)
throws ArchiverException, IOException {
// don't add "" to the archive
if (vPath.length() <= 0) {
return;
}
if (entry.getResource().isDirectory() && !vPath.endsWith("/")) {
vPath += "/";
}
if (vPath.startsWith("/") && !options.getPreserveLeadingSlashes()) {
int l = vPath.length();
if (l <= 1) {
// we would end up adding "" to the archive
return;
}
vPath = vPath.substring(1, l);
}
int pathLength = vPath.length();
InputStream fIn = null;
try {
TarArchiveEntry te;
if (!longFileMode.isGnuMode()
&& pathLength >= org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN) {
int maxPosixPathLen =
org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN
+ org.apache.commons.compress.archivers.tar.TarConstants.PREFIXLEN;
if (longFileMode.isPosixMode()) {
} else if (longFileMode.isPosixWarnMode()) {
if (pathLength > maxPosixPathLen) {
getLogger()
.warn("Entry: " + vPath + " longer than " + maxPosixPathLen + " characters.");
if (!longWarningGiven) {
getLogger()
.warn(
"Resulting tar file can only be processed "
+ "successfully by GNU compatible tar commands");
longWarningGiven = true;
}
}
} else if (longFileMode.isOmitMode()) {
getLogger().info("Omitting: " + vPath);
return;
} else if (longFileMode.isWarnMode()) {
getLogger()
.warn(
"Entry: "
+ vPath
+ " longer than "
+ org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN
+ " characters.");
if (!longWarningGiven) {
getLogger()
.warn(
"Resulting tar file can only be processed "
+ "successfully by GNU compatible tar commands");
longWarningGiven = true;
}
} else if (longFileMode.isFailMode()) {
throw new ArchiverException(
"Entry: "
+ vPath
+ " longer than "
+ org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN
+ " characters.");
} else {
throw new IllegalStateException("Non gnu mode should never get here?");
}
}
if (entry.getType() == ArchiveEntry.SYMLINK) {
final SymlinkDestinationSupplier plexusIoSymlinkResource =
(SymlinkDestinationSupplier) entry.getResource();
te = new TarArchiveEntry(vPath, TarArchiveEntry.LF_SYMLINK);
te.setLinkName(plexusIoSymlinkResource.getSymlinkDestination());
} else {
te = new TarArchiveEntry(vPath);
}
long teLastModified = entry.getResource().getLastModified();
te.setModTime(
teLastModified == PlexusIoResource.UNKNOWN_MODIFICATION_DATE
? System.currentTimeMillis()
: teLastModified);
if (entry.getType() == ArchiveEntry.SYMLINK) {
te.setSize(0);
} else if (!entry.getResource().isDirectory()) {
final long size = entry.getResource().getSize();
te.setSize(size == PlexusIoResource.UNKNOWN_RESOURCE_SIZE ? 0 : size);
}
te.setMode(entry.getMode());
PlexusIoResourceAttributes attributes = entry.getResourceAttributes();
te.setUserName(
(attributes != null && attributes.getUserName() != null)
? attributes.getUserName()
: options.getUserName());
te.setGroupName(
(attributes != null && attributes.getGroupName() != null)
? attributes.getGroupName()
: options.getGroup());
final int userId =
(attributes != null && attributes.getUserId() != null)
? attributes.getUserId()
: options.getUid();
if (userId >= 0) {
te.setUserId(userId);
}
final int groupId =
(attributes != null && attributes.getGroupId() != null)
? attributes.getGroupId()
: options.getGid();
if (groupId >= 0) {
te.setGroupId(groupId);
}
tOut.putArchiveEntry(te);
try {
if (entry.getResource().isFile() && !(entry.getType() == ArchiveEntry.SYMLINK)) {
fIn = entry.getInputStream();
Streams.copyFullyDontCloseOutput(fIn, tOut, "xAR");
}
} catch (Throwable e) {
getLogger().warn("When creating tar entry", e);
} finally {
tOut.closeArchiveEntry();
}
} finally {
IOUtil.close(fIn);
}
}