protected void execute() throws ArchiverException, IOException { if (!checkForced()) { return; } ResourceIterator iter = getResources(); if (!iter.hasNext()) { throw new ArchiverException("You must set at least one file."); } File tarFile = getDestFile(); if (tarFile == null) { throw new ArchiverException("You must set the destination tar file."); } if (tarFile.exists() && !tarFile.isFile()) { throw new ArchiverException(tarFile + " isn't a file."); } if (tarFile.exists() && !tarFile.canWrite()) { throw new ArchiverException(tarFile + " is read-only."); } getLogger().info("Building tar: " + tarFile.getAbsolutePath()); final OutputStream bufferedOutputStream = bufferedOutputStream(new FileOutputStream(tarFile)); tOut = new TarArchiveOutputStream(compress(compression, bufferedOutputStream), "UTF8"); if (longFileMode.isTruncateMode()) { tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_TRUNCATE); } else if (longFileMode.isPosixMode() || longFileMode.isPosixWarnMode()) { tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); // Todo: Patch 2.5.1 for this fix. Also make closeable fix on 2.5.1 tOut.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); } else if (longFileMode.isFailMode() || longFileMode.isOmitMode()) { tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_ERROR); } else { // warn or GNU tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); } longWarningGiven = false; try { while (iter.hasNext()) { ArchiveEntry entry = iter.next(); // Check if we don't add tar file in itself if (ResourceUtils.isSame(entry.getResource(), tarFile)) { throw new ArchiverException("A tar file cannot include itself."); } String fileName = entry.getName(); String name = StringUtils.replace(fileName, File.separatorChar, '/'); tarFile(entry, tOut, name); } } finally { IOUtil.close(tOut); } }
private static void append( File file, FileFilter filter, int baseNameLen, String addedTopFolder, TarArchiveOutputStream tarOut) throws IOException { String name = file.getAbsolutePath(); if (name.length() <= baseNameLen) name = ""; else name = name.substring(baseNameLen); if (File.separatorChar == '\\') name = name.replace('\\', '/'); if (addedTopFolder != null) name = addedTopFolder + '/' + name; if (FileUtils.isSymlink(file)) { String linkTarget = FileUtils.readSymbolicLink(file); if (linkTarget != null) { TarArchiveEntry entry = new TarArchiveEntry(name, TarConstants.LF_SYMLINK); entry.setName(name); entry.setLinkName(linkTarget); tarOut.putArchiveEntry(entry); } return; } ArchiveEntry entry = tarOut.createArchiveEntry(file, name); tarOut.putArchiveEntry(entry); File[] children = file.listFiles(filter); if (children != null) { tarOut.closeArchiveEntry(); // This is a directory. Append its children for (File child : children) append(child, filter, baseNameLen, addedTopFolder, tarOut); return; } // Append the content of the file InputStream input = new FileInputStream(file); try { StreamUtil.copy(input, tarOut); tarOut.closeArchiveEntry(); } finally { StreamUtil.close(input); } }
public static void pack( File sourceFolder, OutputStream output, FileFilter filter, boolean includeTopFolder, String addedTopFolder) throws IOException { TarArchiveOutputStream tarOut = new TarArchiveOutputStream(output); tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); String absName = sourceFolder.getAbsolutePath(); int baseNameLen = absName.length() + 1; if (includeTopFolder) baseNameLen -= (sourceFolder.getName().length() + 1); try { append(sourceFolder, filter, baseNameLen, addedTopFolder, tarOut); } finally { StreamUtil.close(tarOut); } }
private static void addControlEntry( final String pName, final String pContent, final TarArchiveOutputStream pOutput) throws IOException { final byte[] data = pContent.getBytes("UTF-8"); final TarArchiveEntry entry = new TarArchiveEntry("./" + pName, true); entry.setSize(data.length); entry.setNames("root", "root"); if (MAINTAINER_SCRIPTS.contains(pName)) { entry.setMode(PermMapper.toMode("755")); } else { entry.setMode(PermMapper.toMode("644")); } pOutput.putArchiveEntry(entry); pOutput.write(data); pOutput.closeArchiveEntry(); }
@Override public void finalStatistics(EvolutionState state, int result) { super.finalStatistics(state, result); if (compress) { try { taos.close(); } catch (IOException ex) { Logger.getLogger(SampleSolutionsStat.class.getName()).log(Level.SEVERE, null, ex); } } }
private void addFileToTar(TarArchiveOutputStream tOut, File f, String base) throws IOException { String entryName = base + f.getName(); TarArchiveEntry tarEntry = new TarArchiveEntry(f, entryName); tOut.putArchiveEntry(tarEntry); if (f.isFile()) { try (InputStream is = new FileInputStream(f)) { IOUtils.copy(is, tOut); tOut.closeArchiveEntry(); } } else { tOut.closeArchiveEntry(); File[] children = f.listFiles(); if (children != null) { for (File child : children) { addFileToTar(tOut, child, entryName + "/"); } } } }
void createArchive(Path projectPath, Path output, Config overrideParams) throws IOException { out.println("Creating " + output + "..."); ProjectArchive project = projectLoader.load(projectPath, WorkflowResourceMatcher.defaultMatcher(), overrideParams); ArchiveMetadata meta = project.getArchiveMetadata(); try (TarArchiveOutputStream tar = new TarArchiveOutputStream(new GzipCompressorOutputStream(Files.newOutputStream(output)))) { // default mode for file names longer than 100 bytes is throwing an exception (LONGFILE_ERROR) tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); project.listFiles( (resourceName, absPath) -> { if (!Files.isDirectory(absPath)) { out.println(" Archiving " + resourceName); TarArchiveEntry e = buildTarArchiveEntry(project, absPath, resourceName); tar.putArchiveEntry(e); if (e.isSymbolicLink()) { out.println(" symlink -> " + e.getLinkName()); } else { try (InputStream in = Files.newInputStream(absPath)) { ByteStreams.copy(in, tar); } tar.closeArchiveEntry(); } } }); // create .digdag.dig // TODO set default time zone if not set? byte[] metaBody = yamlMapper.toYaml(meta).getBytes(StandardCharsets.UTF_8); TarArchiveEntry metaEntry = new TarArchiveEntry(ArchiveMetadata.FILE_NAME); metaEntry.setSize(metaBody.length); metaEntry.setModTime(new Date()); tar.putArchiveEntry(metaEntry); tar.write(metaBody); tar.closeArchiveEntry(); } out.println("Workflows:"); for (WorkflowDefinition workflow : meta.getWorkflowList().get()) { out.println(" " + workflow.getName()); } out.println(""); }
private void run(File f) throws IOException, ParserException { writeLogFile(logFile, "Start HDT on file " + f.getAbsolutePath(), false); long start = System.currentTimeMillis(); OutputStream os = new FileOutputStream(f.getAbsolutePath() + ".hdt.tar.bz2", false); OutputStream bzos = new BZip2CompressorOutputStream(os); TarArchiveOutputStream aos = new TarArchiveOutputStream(bzos); HDT hdt = HDTManager.generateHDT( f.getAbsolutePath(), "urn:rdfcomp", RDFNotation.parse("ntriples"), new HDTSpecification(), null); hdt.saveToHDT(tmpDir + "/" + name + "_data.hdt", null); long saveToHDT = System.currentTimeMillis() - start; File filePrefix = new File(tmpDir + "/" + name + "_data.hdt"); TarArchiveEntry entry = new TarArchiveEntry(filePrefix, "mappings.hdt"); entry.setSize(filePrefix.length()); aos.putArchiveEntry(entry); IOUtils.copy(new FileInputStream(filePrefix), aos); aos.closeArchiveEntry(); aos.finish(); aos.close(); bzos.close(); os.close(); long saveToTarBzip2 = System.currentTimeMillis() - saveToHDT; long overall = System.currentTimeMillis() - start; String log = "Original size: " + f.length() + "B = " + f.length() / 1024 + " KB" + " =" + f.length() / (1024 * 1024) + " MB"; writeLogFile(logFile, log, true); Model model = ModelLoader.getModel(file.getAbsolutePath()); long ntBzip2 = computeOrginalNTriple(model, file); log = "NT+BZIP size: " + ntBzip2 + " B= " + ntBzip2 / 1024 + " KB= " + ntBzip2 / (1024 * 1024) + " MB"; writeLogFile(logFile, log, true); log = "HDT size: " + filePrefix.length() + " B= " + filePrefix.length() / 1024 + " KB= " + filePrefix.length() / (1024 * 1024) + " MB"; writeLogFile(logFile, log, true); long hdtBzip2Size = new File(f.getAbsolutePath() + ".hdt.tar.bz2").length(); log = "HDT+BZIP2 size: " + hdtBzip2Size + " B= " + hdtBzip2Size / 1024 + " KB= " + hdtBzip2Size / (1024 * 1024) + " MB"; writeLogFile(logFile, log, true); double ratio = new Double(filePrefix.length()) / new Double(ntBzip2); log = "HDT / NTBZip2 ratio= " + ratio; writeLogFile(logFile, log, true); ratio = new Double(hdtBzip2Size) / new Double(ntBzip2); log = "HDT+BZIP2 / NTBZip2 ratio= " + ratio + " \n\n"; writeLogFile(logFile, log, true); log = "Time HDT: " + saveToHDT + "ms = " + saveToHDT / 1000 + " s "; writeLogFile(logFile, log, true); log += "Time HDT+BZIP: " + saveToTarBzip2 + "ms =" + saveToTarBzip2 / 1000 + "s"; writeLogFile(logFile, log, true); log += "Time overall: " + overall + "ms = " + overall / 1000 + " s "; writeLogFile(logFile, log, true); }
protected void cleanUp() throws IOException { super.cleanUp(); tOut.close(); }
/** * tar a file * * @param entry the file to tar * @param tOut the output stream * @param vPath the path name of the file to tar * @throws IOException on error */ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String vPath) throws ArchiverException, IOException { // don't add "" to the archive if (vPath.length() <= 0) { return; } if (entry.getResource().isDirectory() && !vPath.endsWith("/")) { vPath += "/"; } if (vPath.startsWith("/") && !options.getPreserveLeadingSlashes()) { int l = vPath.length(); if (l <= 1) { // we would end up adding "" to the archive return; } vPath = vPath.substring(1, l); } int pathLength = vPath.length(); InputStream fIn = null; try { TarArchiveEntry te; if (!longFileMode.isGnuMode() && pathLength >= org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN) { int maxPosixPathLen = org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN + org.apache.commons.compress.archivers.tar.TarConstants.PREFIXLEN; if (longFileMode.isPosixMode()) { } else if (longFileMode.isPosixWarnMode()) { if (pathLength > maxPosixPathLen) { getLogger() .warn("Entry: " + vPath + " longer than " + maxPosixPathLen + " characters."); if (!longWarningGiven) { getLogger() .warn( "Resulting tar file can only be processed " + "successfully by GNU compatible tar commands"); longWarningGiven = true; } } } else if (longFileMode.isOmitMode()) { getLogger().info("Omitting: " + vPath); return; } else if (longFileMode.isWarnMode()) { getLogger() .warn( "Entry: " + vPath + " longer than " + org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN + " characters."); if (!longWarningGiven) { getLogger() .warn( "Resulting tar file can only be processed " + "successfully by GNU compatible tar commands"); longWarningGiven = true; } } else if (longFileMode.isFailMode()) { throw new ArchiverException( "Entry: " + vPath + " longer than " + org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN + " characters."); } else { throw new IllegalStateException("Non gnu mode should never get here?"); } } if (entry.getType() == ArchiveEntry.SYMLINK) { final SymlinkDestinationSupplier plexusIoSymlinkResource = (SymlinkDestinationSupplier) entry.getResource(); te = new TarArchiveEntry(vPath, TarArchiveEntry.LF_SYMLINK); te.setLinkName(plexusIoSymlinkResource.getSymlinkDestination()); } else { te = new TarArchiveEntry(vPath); } long teLastModified = entry.getResource().getLastModified(); te.setModTime( teLastModified == PlexusIoResource.UNKNOWN_MODIFICATION_DATE ? System.currentTimeMillis() : teLastModified); if (entry.getType() == ArchiveEntry.SYMLINK) { te.setSize(0); } else if (!entry.getResource().isDirectory()) { final long size = entry.getResource().getSize(); te.setSize(size == PlexusIoResource.UNKNOWN_RESOURCE_SIZE ? 0 : size); } te.setMode(entry.getMode()); PlexusIoResourceAttributes attributes = entry.getResourceAttributes(); te.setUserName( (attributes != null && attributes.getUserName() != null) ? attributes.getUserName() : options.getUserName()); te.setGroupName( (attributes != null && attributes.getGroupName() != null) ? attributes.getGroupName() : options.getGroup()); final int userId = (attributes != null && attributes.getUserId() != null) ? attributes.getUserId() : options.getUid(); if (userId >= 0) { te.setUserId(userId); } final int groupId = (attributes != null && attributes.getGroupId() != null) ? attributes.getGroupId() : options.getGid(); if (groupId >= 0) { te.setGroupId(groupId); } tOut.putArchiveEntry(te); try { if (entry.getResource().isFile() && !(entry.getType() == ArchiveEntry.SYMLINK)) { fIn = entry.getInputStream(); Streams.copyFullyDontCloseOutput(fIn, tOut, "xAR"); } } catch (Throwable e) { getLogger().warn("When creating tar entry", e); } finally { tOut.closeArchiveEntry(); } } finally { IOUtil.close(fIn); } }
/** * Build the data archive of the deb from the provided DataProducers * * @param pData * @param pOutput * @param pChecksums * @param compression the compression method used for the data file * @return * @throws NoSuchAlgorithmException * @throws IOException * @throws CompressorException */ BigInteger buildData( final DataProducer[] pData, final File pOutput, final StringBuilder pChecksums, Compression compression) throws NoSuchAlgorithmException, IOException, CompressorException { final File dir = pOutput.getParentFile(); if (dir != null && (!dir.exists() || !dir.isDirectory())) { throw new IOException("Cannot write data file at '" + pOutput.getAbsolutePath() + "'"); } final TarArchiveOutputStream tarOutputStream = new TarArchiveOutputStream( compression.toCompressedOutputStream(new FileOutputStream(pOutput))); tarOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); final MessageDigest digest = MessageDigest.getInstance("MD5"); final Total dataSize = new Total(); final List<String> addedDirectories = new ArrayList<String>(); final DataConsumer receiver = new DataConsumer() { public void onEachDir( String dirname, String linkname, String user, int uid, String group, int gid, int mode, long size) throws IOException { dirname = fixPath(dirname); createParentDirectories((new File(dirname)).getParent(), user, uid, group, gid); // The directory passed in explicitly by the caller also gets the passed-in mode. // (Unlike // the parent directories for now. See related comments at "int mode =" in // createParentDirectories, including about a possible bug.) createDirectory(dirname, user, uid, group, gid, mode, 0); console.info("dir: " + dirname); } public void onEachFile( InputStream inputStream, String filename, String linkname, String user, int uid, String group, int gid, int mode, long size) throws IOException { filename = fixPath(filename); createParentDirectories((new File(filename)).getParent(), user, uid, group, gid); final TarArchiveEntry entry = new TarArchiveEntry(filename, true); entry.setUserName(user); entry.setUserId(uid); entry.setGroupName(group); entry.setGroupId(gid); entry.setMode(mode); entry.setSize(size); tarOutputStream.putArchiveEntry(entry); dataSize.add(size); digest.reset(); Utils.copy(inputStream, new DigestOutputStream(tarOutputStream, digest)); final String md5 = Utils.toHex(digest.digest()); tarOutputStream.closeArchiveEntry(); console.info( "file:" + entry.getName() + " size:" + entry.getSize() + " mode:" + entry.getMode() + " linkname:" + entry.getLinkName() + " username:"******" userid:" + entry.getUserId() + " groupname:" + entry.getGroupName() + " groupid:" + entry.getGroupId() + " modtime:" + entry.getModTime() + " md5: " + md5); // append to file md5 list pChecksums.append(md5).append(" ").append(entry.getName()).append('\n'); } public void onEachLink( String path, String linkName, boolean symlink, String user, int uid, String group, int gid, int mode) throws IOException { path = fixPath(path); createParentDirectories((new File(path)).getParent(), user, uid, group, gid); final TarArchiveEntry entry = new TarArchiveEntry( path, symlink ? TarArchiveEntry.LF_SYMLINK : TarArchiveEntry.LF_LINK); entry.setLinkName(linkName); entry.setUserName(user); entry.setUserId(uid); entry.setGroupName(group); entry.setGroupId(gid); entry.setMode(mode); tarOutputStream.putArchiveEntry(entry); tarOutputStream.closeArchiveEntry(); console.info( "link:" + entry.getName() + " mode:" + entry.getMode() + " linkname:" + entry.getLinkName() + " username:"******" userid:" + entry.getUserId() + " groupname:" + entry.getGroupName() + " groupid:" + entry.getGroupId()); } private String fixPath(String path) { // If we're receiving directory names from Windows, then we'll convert to use slash // This does eliminate the ability to use of a backslash in a directory name on *NIX, // but in practice, this is a non-issue if (path.indexOf('\\') > -1) { path = path.replace('\\', '/'); } // ensure the path is like : ./foo/bar if (path.startsWith("/")) { path = "." + path; } else if (!path.startsWith("./")) { path = "./" + path; } return path; } private void createDirectory( String directory, String user, int uid, String group, int gid, int mode, long size) throws IOException { // All dirs should end with "/" when created, or the test // DebAndTaskTestCase.testTarFileSet() thinks its a file // and so thinks it has the wrong permission. // This consistency also helps when checking if a directory already exists in // addedDirectories. if (!directory.endsWith("/")) { directory += "/"; } if (!addedDirectories.contains(directory)) { TarArchiveEntry entry = new TarArchiveEntry(directory, true); entry.setUserName(user); entry.setUserId(uid); entry.setGroupName(group); entry.setGroupId(gid); entry.setMode(mode); entry.setSize(size); tarOutputStream.putArchiveEntry(entry); tarOutputStream.closeArchiveEntry(); addedDirectories.add( directory); // so addedDirectories consistently have "/" for finding duplicates. } } private void createParentDirectories( String dirname, String user, int uid, String group, int gid) throws IOException { // Debian packages must have parent directories created // before sub-directories or files can be installed. // For example, if an entry of ./usr/lib/foo/bar existed // in a .deb package, but the ./usr/lib/foo directory didn't // exist, the package installation would fail. The .deb must // then have an entry for ./usr/lib/foo and then ./usr/lib/foo/bar if (dirname == null) { return; } // The loop below will create entries for all parent directories // to ensure that .deb packages will install correctly. String[] pathParts = dirname.split("\\/"); String parentDir = "./"; for (int i = 1; i < pathParts.length; i++) { parentDir += pathParts[i] + "/"; // Make it so the dirs can be traversed by users. // We could instead try something more granular, like setting the directory // permission to 'rx' for each of the 3 user/group/other read permissions // found on the file being added (ie, only if "other" has read // permission on the main node, then add o+rx permission on all the containing // directories, same w/ user & group), and then also we'd have to // check the parentDirs collection of those already added to // see if those permissions need to be similarly updated. (Note, it hasn't // been demonstrated, but there might be a bug if a user specifically // requests a directory with certain permissions, // that has already been auto-created because it was a parent, and if so, go set // the user-requested mode on that directory instead of this automatic one.) // But for now, keeping it simple by making every dir a+rx. Examples are: // drw-r----- fs/fs # what you get with setMode(mode) // drwxr-xr-x fs/fs # Usable. Too loose? int mode = TarArchiveEntry.DEFAULT_DIR_MODE; createDirectory(parentDir, user, uid, group, gid, mode, 0); } } }; try { for (DataProducer data : pData) { data.produce(receiver); } } finally { tarOutputStream.close(); } console.info("Total size: " + dataSize); return dataSize.count; }
/** * Build control archive of the deb * * @param pControlFiles * @param pDataSize * @param pChecksums * @param pOutput * @return * @throws FileNotFoundException * @throws IOException * @throws ParseException */ private PackageDescriptor buildControl( final File[] pControlFiles, final BigInteger pDataSize, final StringBuilder pChecksums, final File pOutput) throws IOException, ParseException { final File dir = pOutput.getParentFile(); if (dir != null && (!dir.exists() || !dir.isDirectory())) { throw new IOException("Cannot write control file at '" + pOutput.getAbsolutePath() + "'"); } final TarArchiveOutputStream outputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(pOutput))); outputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); List<FilteredConfigurationFile> configurationFiles = new ArrayList<FilteredConfigurationFile>(); // create a descriptor out of the "control" file, copy all other files, ignore directories PackageDescriptor packageDescriptor = null; for (File file : pControlFiles) { if (file.isDirectory()) { // warn about the misplaced directory, except for directories ignored by default (.svn, cvs, // etc) if (!isDefaultExcludes(file)) { console.info( "Found directory '" + file + "' in the control directory. Maybe you are pointing to wrong dir?"); } continue; } if (CONFIGURATION_FILENAMES.contains(file.getName()) || MAINTAINER_SCRIPTS.contains(file.getName())) { FilteredConfigurationFile configurationFile = new FilteredConfigurationFile(file.getName(), new FileInputStream(file), resolver); configurationFiles.add(configurationFile); } else if ("control".equals(file.getName())) { packageDescriptor = createPackageDescriptor(file, pDataSize); } else { // initialize the information stream to guess the type of the file InformationInputStream infoStream = new InformationInputStream(new FileInputStream(file)); Utils.copy(infoStream, NullOutputStream.NULL_OUTPUT_STREAM); infoStream.close(); // fix line endings for shell scripts InputStream in = new FileInputStream(file); if (infoStream.isShell() && !infoStream.hasUnixLineEndings()) { byte[] buf = Utils.toUnixLineEndings(in); in = new ByteArrayInputStream(buf); } addControlEntry(file.getName(), IOUtils.toString(in), outputStream); in.close(); } } if (packageDescriptor == null) { throw new FileNotFoundException( "No 'control' file found in " + Arrays.toString(pControlFiles)); } for (FilteredConfigurationFile configurationFile : configurationFiles) { addControlEntry(configurationFile.getName(), configurationFile.toString(), outputStream); } addControlEntry("control", packageDescriptor.toString(), outputStream); addControlEntry("md5sums", pChecksums.toString(), outputStream); outputStream.close(); return packageDescriptor; }