protected void execute() throws ArchiverException, IOException {
if (!checkForced()) {
return;
}
ResourceIterator iter = getResources();
if (!iter.hasNext()) {
throw new ArchiverException("You must set at least one file.");
}
File tarFile = getDestFile();
if (tarFile == null) {
throw new ArchiverException("You must set the destination tar file.");
}
if (tarFile.exists() && !tarFile.isFile()) {
throw new ArchiverException(tarFile + " isn't a file.");
}
if (tarFile.exists() && !tarFile.canWrite()) {
throw new ArchiverException(tarFile + " is read-only.");
}
getLogger().info("Building tar: " + tarFile.getAbsolutePath());
final OutputStream bufferedOutputStream = bufferedOutputStream(new FileOutputStream(tarFile));
tOut = new TarArchiveOutputStream(compress(compression, bufferedOutputStream), "UTF8");
if (longFileMode.isTruncateMode()) {
tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_TRUNCATE);
} else if (longFileMode.isPosixMode() || longFileMode.isPosixWarnMode()) {
tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
// Todo: Patch 2.5.1 for this fix. Also make closeable fix on 2.5.1
tOut.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
} else if (longFileMode.isFailMode() || longFileMode.isOmitMode()) {
tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_ERROR);
} else {
// warn or GNU
tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
}
longWarningGiven = false;
try {
while (iter.hasNext()) {
ArchiveEntry entry = iter.next();
// Check if we don't add tar file in itself
if (ResourceUtils.isSame(entry.getResource(), tarFile)) {
throw new ArchiverException("A tar file cannot include itself.");
}
String fileName = entry.getName();
String name = StringUtils.replace(fileName, File.separatorChar, '/');
tarFile(entry, tOut, name);
}
} finally {
IOUtil.close(tOut);
}
}
void createArchive(Path projectPath, Path output, Config overrideParams) throws IOException {
out.println("Creating " + output + "...");
ProjectArchive project =
projectLoader.load(projectPath, WorkflowResourceMatcher.defaultMatcher(), overrideParams);
ArchiveMetadata meta = project.getArchiveMetadata();
try (TarArchiveOutputStream tar =
new TarArchiveOutputStream(new GzipCompressorOutputStream(Files.newOutputStream(output)))) {
// default mode for file names longer than 100 bytes is throwing an exception (LONGFILE_ERROR)
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
project.listFiles(
(resourceName, absPath) -> {
if (!Files.isDirectory(absPath)) {
out.println(" Archiving " + resourceName);
TarArchiveEntry e = buildTarArchiveEntry(project, absPath, resourceName);
tar.putArchiveEntry(e);
if (e.isSymbolicLink()) {
out.println(" symlink -> " + e.getLinkName());
} else {
try (InputStream in = Files.newInputStream(absPath)) {
ByteStreams.copy(in, tar);
}
tar.closeArchiveEntry();
}
}
});
// create .digdag.dig
// TODO set default time zone if not set?
byte[] metaBody = yamlMapper.toYaml(meta).getBytes(StandardCharsets.UTF_8);
TarArchiveEntry metaEntry = new TarArchiveEntry(ArchiveMetadata.FILE_NAME);
metaEntry.setSize(metaBody.length);
metaEntry.setModTime(new Date());
tar.putArchiveEntry(metaEntry);
tar.write(metaBody);
tar.closeArchiveEntry();
}
out.println("Workflows:");
for (WorkflowDefinition workflow : meta.getWorkflowList().get()) {
out.println(" " + workflow.getName());
}
out.println("");
}
public static void pack(
File sourceFolder,
OutputStream output,
FileFilter filter,
boolean includeTopFolder,
String addedTopFolder)
throws IOException {
TarArchiveOutputStream tarOut = new TarArchiveOutputStream(output);
tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
String absName = sourceFolder.getAbsolutePath();
int baseNameLen = absName.length() + 1;
if (includeTopFolder) baseNameLen -= (sourceFolder.getName().length() + 1);
try {
append(sourceFolder, filter, baseNameLen, addedTopFolder, tarOut);
} finally {
StreamUtil.close(tarOut);
}
}
/**
* Build the data archive of the deb from the provided DataProducers
*
* @param pData
* @param pOutput
* @param pChecksums
* @param compression the compression method used for the data file
* @return
* @throws NoSuchAlgorithmException
* @throws IOException
* @throws CompressorException
*/
BigInteger buildData(
final DataProducer[] pData,
final File pOutput,
final StringBuilder pChecksums,
Compression compression)
throws NoSuchAlgorithmException, IOException, CompressorException {
final File dir = pOutput.getParentFile();
if (dir != null && (!dir.exists() || !dir.isDirectory())) {
throw new IOException("Cannot write data file at '" + pOutput.getAbsolutePath() + "'");
}
final TarArchiveOutputStream tarOutputStream =
new TarArchiveOutputStream(
compression.toCompressedOutputStream(new FileOutputStream(pOutput)));
tarOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
final MessageDigest digest = MessageDigest.getInstance("MD5");
final Total dataSize = new Total();
final List<String> addedDirectories = new ArrayList<String>();
final DataConsumer receiver =
new DataConsumer() {
public void onEachDir(
String dirname,
String linkname,
String user,
int uid,
String group,
int gid,
int mode,
long size)
throws IOException {
dirname = fixPath(dirname);
createParentDirectories((new File(dirname)).getParent(), user, uid, group, gid);
// The directory passed in explicitly by the caller also gets the passed-in mode.
// (Unlike
// the parent directories for now. See related comments at "int mode =" in
// createParentDirectories, including about a possible bug.)
createDirectory(dirname, user, uid, group, gid, mode, 0);
console.info("dir: " + dirname);
}
public void onEachFile(
InputStream inputStream,
String filename,
String linkname,
String user,
int uid,
String group,
int gid,
int mode,
long size)
throws IOException {
filename = fixPath(filename);
createParentDirectories((new File(filename)).getParent(), user, uid, group, gid);
final TarArchiveEntry entry = new TarArchiveEntry(filename, true);
entry.setUserName(user);
entry.setUserId(uid);
entry.setGroupName(group);
entry.setGroupId(gid);
entry.setMode(mode);
entry.setSize(size);
tarOutputStream.putArchiveEntry(entry);
dataSize.add(size);
digest.reset();
Utils.copy(inputStream, new DigestOutputStream(tarOutputStream, digest));
final String md5 = Utils.toHex(digest.digest());
tarOutputStream.closeArchiveEntry();
console.info(
"file:"
+ entry.getName()
+ " size:"
+ entry.getSize()
+ " mode:"
+ entry.getMode()
+ " linkname:"
+ entry.getLinkName()
+ " username:"******" userid:"
+ entry.getUserId()
+ " groupname:"
+ entry.getGroupName()
+ " groupid:"
+ entry.getGroupId()
+ " modtime:"
+ entry.getModTime()
+ " md5: "
+ md5);
// append to file md5 list
pChecksums.append(md5).append(" ").append(entry.getName()).append('\n');
}
public void onEachLink(
String path,
String linkName,
boolean symlink,
String user,
int uid,
String group,
int gid,
int mode)
throws IOException {
path = fixPath(path);
createParentDirectories((new File(path)).getParent(), user, uid, group, gid);
final TarArchiveEntry entry =
new TarArchiveEntry(
path, symlink ? TarArchiveEntry.LF_SYMLINK : TarArchiveEntry.LF_LINK);
entry.setLinkName(linkName);
entry.setUserName(user);
entry.setUserId(uid);
entry.setGroupName(group);
entry.setGroupId(gid);
entry.setMode(mode);
tarOutputStream.putArchiveEntry(entry);
tarOutputStream.closeArchiveEntry();
console.info(
"link:"
+ entry.getName()
+ " mode:"
+ entry.getMode()
+ " linkname:"
+ entry.getLinkName()
+ " username:"******" userid:"
+ entry.getUserId()
+ " groupname:"
+ entry.getGroupName()
+ " groupid:"
+ entry.getGroupId());
}
private String fixPath(String path) {
// If we're receiving directory names from Windows, then we'll convert to use slash
// This does eliminate the ability to use of a backslash in a directory name on *NIX,
// but in practice, this is a non-issue
if (path.indexOf('\\') > -1) {
path = path.replace('\\', '/');
}
// ensure the path is like : ./foo/bar
if (path.startsWith("/")) {
path = "." + path;
} else if (!path.startsWith("./")) {
path = "./" + path;
}
return path;
}
private void createDirectory(
String directory, String user, int uid, String group, int gid, int mode, long size)
throws IOException {
// All dirs should end with "/" when created, or the test
// DebAndTaskTestCase.testTarFileSet() thinks its a file
// and so thinks it has the wrong permission.
// This consistency also helps when checking if a directory already exists in
// addedDirectories.
if (!directory.endsWith("/")) {
directory += "/";
}
if (!addedDirectories.contains(directory)) {
TarArchiveEntry entry = new TarArchiveEntry(directory, true);
entry.setUserName(user);
entry.setUserId(uid);
entry.setGroupName(group);
entry.setGroupId(gid);
entry.setMode(mode);
entry.setSize(size);
tarOutputStream.putArchiveEntry(entry);
tarOutputStream.closeArchiveEntry();
addedDirectories.add(
directory); // so addedDirectories consistently have "/" for finding duplicates.
}
}
private void createParentDirectories(
String dirname, String user, int uid, String group, int gid) throws IOException {
// Debian packages must have parent directories created
// before sub-directories or files can be installed.
// For example, if an entry of ./usr/lib/foo/bar existed
// in a .deb package, but the ./usr/lib/foo directory didn't
// exist, the package installation would fail. The .deb must
// then have an entry for ./usr/lib/foo and then ./usr/lib/foo/bar
if (dirname == null) {
return;
}
// The loop below will create entries for all parent directories
// to ensure that .deb packages will install correctly.
String[] pathParts = dirname.split("\\/");
String parentDir = "./";
for (int i = 1; i < pathParts.length; i++) {
parentDir += pathParts[i] + "/";
// Make it so the dirs can be traversed by users.
// We could instead try something more granular, like setting the directory
// permission to 'rx' for each of the 3 user/group/other read permissions
// found on the file being added (ie, only if "other" has read
// permission on the main node, then add o+rx permission on all the containing
// directories, same w/ user & group), and then also we'd have to
// check the parentDirs collection of those already added to
// see if those permissions need to be similarly updated. (Note, it hasn't
// been demonstrated, but there might be a bug if a user specifically
// requests a directory with certain permissions,
// that has already been auto-created because it was a parent, and if so, go set
// the user-requested mode on that directory instead of this automatic one.)
// But for now, keeping it simple by making every dir a+rx. Examples are:
// drw-r----- fs/fs # what you get with setMode(mode)
// drwxr-xr-x fs/fs # Usable. Too loose?
int mode = TarArchiveEntry.DEFAULT_DIR_MODE;
createDirectory(parentDir, user, uid, group, gid, mode, 0);
}
}
};
try {
for (DataProducer data : pData) {
data.produce(receiver);
}
} finally {
tarOutputStream.close();
}
console.info("Total size: " + dataSize);
return dataSize.count;
}
/**
* Build control archive of the deb
*
* @param pControlFiles
* @param pDataSize
* @param pChecksums
* @param pOutput
* @return
* @throws FileNotFoundException
* @throws IOException
* @throws ParseException
*/
private PackageDescriptor buildControl(
final File[] pControlFiles,
final BigInteger pDataSize,
final StringBuilder pChecksums,
final File pOutput)
throws IOException, ParseException {
final File dir = pOutput.getParentFile();
if (dir != null && (!dir.exists() || !dir.isDirectory())) {
throw new IOException("Cannot write control file at '" + pOutput.getAbsolutePath() + "'");
}
final TarArchiveOutputStream outputStream =
new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(pOutput)));
outputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
List<FilteredConfigurationFile> configurationFiles = new ArrayList<FilteredConfigurationFile>();
// create a descriptor out of the "control" file, copy all other files, ignore directories
PackageDescriptor packageDescriptor = null;
for (File file : pControlFiles) {
if (file.isDirectory()) {
// warn about the misplaced directory, except for directories ignored by default (.svn, cvs,
// etc)
if (!isDefaultExcludes(file)) {
console.info(
"Found directory '"
+ file
+ "' in the control directory. Maybe you are pointing to wrong dir?");
}
continue;
}
if (CONFIGURATION_FILENAMES.contains(file.getName())
|| MAINTAINER_SCRIPTS.contains(file.getName())) {
FilteredConfigurationFile configurationFile =
new FilteredConfigurationFile(file.getName(), new FileInputStream(file), resolver);
configurationFiles.add(configurationFile);
} else if ("control".equals(file.getName())) {
packageDescriptor = createPackageDescriptor(file, pDataSize);
} else {
// initialize the information stream to guess the type of the file
InformationInputStream infoStream = new InformationInputStream(new FileInputStream(file));
Utils.copy(infoStream, NullOutputStream.NULL_OUTPUT_STREAM);
infoStream.close();
// fix line endings for shell scripts
InputStream in = new FileInputStream(file);
if (infoStream.isShell() && !infoStream.hasUnixLineEndings()) {
byte[] buf = Utils.toUnixLineEndings(in);
in = new ByteArrayInputStream(buf);
}
addControlEntry(file.getName(), IOUtils.toString(in), outputStream);
in.close();
}
}
if (packageDescriptor == null) {
throw new FileNotFoundException(
"No 'control' file found in " + Arrays.toString(pControlFiles));
}
for (FilteredConfigurationFile configurationFile : configurationFiles) {
addControlEntry(configurationFile.getName(), configurationFile.toString(), outputStream);
}
addControlEntry("control", packageDescriptor.toString(), outputStream);
addControlEntry("md5sums", pChecksums.toString(), outputStream);
outputStream.close();
return packageDescriptor;
}