Ejemplo n.º 1
0
 @Override
 public void finalStatistics(EvolutionState state, int result) {
   super.finalStatistics(state, result);
   if (compress) {
     try {
       taos.close();
     } catch (IOException ex) {
       Logger.getLogger(SampleSolutionsStat.class.getName()).log(Level.SEVERE, null, ex);
     }
   }
 }
Ejemplo n.º 2
0
  private void run(File f) throws IOException, ParserException {
    writeLogFile(logFile, "Start HDT on file " + f.getAbsolutePath(), false);
    long start = System.currentTimeMillis();
    OutputStream os = new FileOutputStream(f.getAbsolutePath() + ".hdt.tar.bz2", false);
    OutputStream bzos = new BZip2CompressorOutputStream(os);
    TarArchiveOutputStream aos = new TarArchiveOutputStream(bzos);

    HDT hdt =
        HDTManager.generateHDT(
            f.getAbsolutePath(),
            "urn:rdfcomp",
            RDFNotation.parse("ntriples"),
            new HDTSpecification(),
            null);

    hdt.saveToHDT(tmpDir + "/" + name + "_data.hdt", null);
    long saveToHDT = System.currentTimeMillis() - start;
    File filePrefix = new File(tmpDir + "/" + name + "_data.hdt");
    TarArchiveEntry entry = new TarArchiveEntry(filePrefix, "mappings.hdt");
    entry.setSize(filePrefix.length());
    aos.putArchiveEntry(entry);
    IOUtils.copy(new FileInputStream(filePrefix), aos);
    aos.closeArchiveEntry();
    aos.finish();
    aos.close();
    bzos.close();
    os.close();
    long saveToTarBzip2 = System.currentTimeMillis() - saveToHDT;
    long overall = System.currentTimeMillis() - start;
    String log =
        "Original size: "
            + f.length()
            + "B = "
            + f.length() / 1024
            + " KB"
            + " ="
            + f.length() / (1024 * 1024)
            + " MB";
    writeLogFile(logFile, log, true);

    Model model = ModelLoader.getModel(file.getAbsolutePath());
    long ntBzip2 = computeOrginalNTriple(model, file);

    log =
        "NT+BZIP size: "
            + ntBzip2
            + " B= "
            + ntBzip2 / 1024
            + " KB= "
            + ntBzip2 / (1024 * 1024)
            + " MB";
    writeLogFile(logFile, log, true);
    log =
        "HDT size: "
            + filePrefix.length()
            + " B= "
            + filePrefix.length() / 1024
            + " KB= "
            + filePrefix.length() / (1024 * 1024)
            + " MB";
    writeLogFile(logFile, log, true);
    long hdtBzip2Size = new File(f.getAbsolutePath() + ".hdt.tar.bz2").length();
    log =
        "HDT+BZIP2 size: "
            + hdtBzip2Size
            + " B= "
            + hdtBzip2Size / 1024
            + " KB= "
            + hdtBzip2Size / (1024 * 1024)
            + " MB";
    writeLogFile(logFile, log, true);

    double ratio = new Double(filePrefix.length()) / new Double(ntBzip2);
    log = "HDT / NTBZip2 ratio= " + ratio;
    writeLogFile(logFile, log, true);
    ratio = new Double(hdtBzip2Size) / new Double(ntBzip2);
    log = "HDT+BZIP2 / NTBZip2 ratio= " + ratio + " \n\n";
    writeLogFile(logFile, log, true);

    log = "Time HDT: " + saveToHDT + "ms = " + saveToHDT / 1000 + " s ";
    writeLogFile(logFile, log, true);
    log += "Time HDT+BZIP: " + saveToTarBzip2 + "ms =" + saveToTarBzip2 / 1000 + "s";
    writeLogFile(logFile, log, true);
    log += "Time overall: " + overall + "ms = " + overall / 1000 + " s ";
    writeLogFile(logFile, log, true);
  }
Ejemplo n.º 3
0
  /**
   * Build the data archive of the deb from the provided DataProducers
   *
   * @param pData
   * @param pOutput
   * @param pChecksums
   * @param compression the compression method used for the data file
   * @return
   * @throws NoSuchAlgorithmException
   * @throws IOException
   * @throws CompressorException
   */
  BigInteger buildData(
      final DataProducer[] pData,
      final File pOutput,
      final StringBuilder pChecksums,
      Compression compression)
      throws NoSuchAlgorithmException, IOException, CompressorException {

    final File dir = pOutput.getParentFile();
    if (dir != null && (!dir.exists() || !dir.isDirectory())) {
      throw new IOException("Cannot write data file at '" + pOutput.getAbsolutePath() + "'");
    }

    final TarArchiveOutputStream tarOutputStream =
        new TarArchiveOutputStream(
            compression.toCompressedOutputStream(new FileOutputStream(pOutput)));
    tarOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);

    final MessageDigest digest = MessageDigest.getInstance("MD5");

    final Total dataSize = new Total();

    final List<String> addedDirectories = new ArrayList<String>();
    final DataConsumer receiver =
        new DataConsumer() {
          public void onEachDir(
              String dirname,
              String linkname,
              String user,
              int uid,
              String group,
              int gid,
              int mode,
              long size)
              throws IOException {
            dirname = fixPath(dirname);

            createParentDirectories((new File(dirname)).getParent(), user, uid, group, gid);

            // The directory passed in explicitly by the caller also gets the passed-in mode.
            // (Unlike
            // the parent directories for now.  See related comments at "int mode =" in
            // createParentDirectories, including about a possible bug.)
            createDirectory(dirname, user, uid, group, gid, mode, 0);

            console.info("dir: " + dirname);
          }

          public void onEachFile(
              InputStream inputStream,
              String filename,
              String linkname,
              String user,
              int uid,
              String group,
              int gid,
              int mode,
              long size)
              throws IOException {
            filename = fixPath(filename);

            createParentDirectories((new File(filename)).getParent(), user, uid, group, gid);

            final TarArchiveEntry entry = new TarArchiveEntry(filename, true);

            entry.setUserName(user);
            entry.setUserId(uid);
            entry.setGroupName(group);
            entry.setGroupId(gid);
            entry.setMode(mode);
            entry.setSize(size);

            tarOutputStream.putArchiveEntry(entry);

            dataSize.add(size);
            digest.reset();

            Utils.copy(inputStream, new DigestOutputStream(tarOutputStream, digest));

            final String md5 = Utils.toHex(digest.digest());

            tarOutputStream.closeArchiveEntry();

            console.info(
                "file:"
                    + entry.getName()
                    + " size:"
                    + entry.getSize()
                    + " mode:"
                    + entry.getMode()
                    + " linkname:"
                    + entry.getLinkName()
                    + " username:"******" userid:"
                    + entry.getUserId()
                    + " groupname:"
                    + entry.getGroupName()
                    + " groupid:"
                    + entry.getGroupId()
                    + " modtime:"
                    + entry.getModTime()
                    + " md5: "
                    + md5);

            // append to file md5 list
            pChecksums.append(md5).append(" ").append(entry.getName()).append('\n');
          }

          public void onEachLink(
              String path,
              String linkName,
              boolean symlink,
              String user,
              int uid,
              String group,
              int gid,
              int mode)
              throws IOException {
            path = fixPath(path);

            createParentDirectories((new File(path)).getParent(), user, uid, group, gid);

            final TarArchiveEntry entry =
                new TarArchiveEntry(
                    path, symlink ? TarArchiveEntry.LF_SYMLINK : TarArchiveEntry.LF_LINK);
            entry.setLinkName(linkName);

            entry.setUserName(user);
            entry.setUserId(uid);
            entry.setGroupName(group);
            entry.setGroupId(gid);
            entry.setMode(mode);

            tarOutputStream.putArchiveEntry(entry);
            tarOutputStream.closeArchiveEntry();

            console.info(
                "link:"
                    + entry.getName()
                    + " mode:"
                    + entry.getMode()
                    + " linkname:"
                    + entry.getLinkName()
                    + " username:"******" userid:"
                    + entry.getUserId()
                    + " groupname:"
                    + entry.getGroupName()
                    + " groupid:"
                    + entry.getGroupId());
          }

          private String fixPath(String path) {
            // If we're receiving directory names from Windows, then we'll convert to use slash
            // This does eliminate the ability to use of a backslash in a directory name on *NIX,
            // but in practice, this is a non-issue
            if (path.indexOf('\\') > -1) {
              path = path.replace('\\', '/');
            }
            // ensure the path is like : ./foo/bar
            if (path.startsWith("/")) {
              path = "." + path;
            } else if (!path.startsWith("./")) {
              path = "./" + path;
            }
            return path;
          }

          private void createDirectory(
              String directory, String user, int uid, String group, int gid, int mode, long size)
              throws IOException {
            // All dirs should end with "/" when created, or the test
            // DebAndTaskTestCase.testTarFileSet() thinks its a file
            // and so thinks it has the wrong permission.
            // This consistency also helps when checking if a directory already exists in
            // addedDirectories.

            if (!directory.endsWith("/")) {
              directory += "/";
            }

            if (!addedDirectories.contains(directory)) {
              TarArchiveEntry entry = new TarArchiveEntry(directory, true);
              entry.setUserName(user);
              entry.setUserId(uid);
              entry.setGroupName(group);
              entry.setGroupId(gid);
              entry.setMode(mode);
              entry.setSize(size);

              tarOutputStream.putArchiveEntry(entry);
              tarOutputStream.closeArchiveEntry();
              addedDirectories.add(
                  directory); // so addedDirectories consistently have "/" for finding duplicates.
            }
          }

          private void createParentDirectories(
              String dirname, String user, int uid, String group, int gid) throws IOException {
            // Debian packages must have parent directories created
            // before sub-directories or files can be installed.
            // For example, if an entry of ./usr/lib/foo/bar existed
            // in a .deb package, but the ./usr/lib/foo directory didn't
            // exist, the package installation would fail.  The .deb must
            // then have an entry for ./usr/lib/foo and then ./usr/lib/foo/bar

            if (dirname == null) {
              return;
            }

            // The loop below will create entries for all parent directories
            // to ensure that .deb packages will install correctly.
            String[] pathParts = dirname.split("\\/");
            String parentDir = "./";
            for (int i = 1; i < pathParts.length; i++) {
              parentDir += pathParts[i] + "/";
              // Make it so the dirs can be traversed by users.
              // We could instead try something more granular, like setting the directory
              // permission to 'rx' for each of the 3 user/group/other read permissions
              // found on the file being added (ie, only if "other" has read
              // permission on the main node, then add o+rx permission on all the containing
              // directories, same w/ user & group), and then also we'd have to
              // check the parentDirs collection of those already added to
              // see if those permissions need to be similarly updated.  (Note, it hasn't
              // been demonstrated, but there might be a bug if a user specifically
              // requests a directory with certain permissions,
              // that has already been auto-created because it was a parent, and if so, go set
              // the user-requested mode on that directory instead of this automatic one.)
              // But for now, keeping it simple by making every dir a+rx.   Examples are:
              // drw-r----- fs/fs   # what you get with setMode(mode)
              // drwxr-xr-x fs/fs   # Usable. Too loose?
              int mode = TarArchiveEntry.DEFAULT_DIR_MODE;

              createDirectory(parentDir, user, uid, group, gid, mode, 0);
            }
          }
        };

    try {
      for (DataProducer data : pData) {
        data.produce(receiver);
      }
    } finally {
      tarOutputStream.close();
    }

    console.info("Total size: " + dataSize);

    return dataSize.count;
  }
Ejemplo n.º 4
0
 protected void cleanUp() throws IOException {
   super.cleanUp();
   tOut.close();
 }
Ejemplo n.º 5
0
  /**
   * Build control archive of the deb
   *
   * @param pControlFiles
   * @param pDataSize
   * @param pChecksums
   * @param pOutput
   * @return
   * @throws FileNotFoundException
   * @throws IOException
   * @throws ParseException
   */
  private PackageDescriptor buildControl(
      final File[] pControlFiles,
      final BigInteger pDataSize,
      final StringBuilder pChecksums,
      final File pOutput)
      throws IOException, ParseException {

    final File dir = pOutput.getParentFile();
    if (dir != null && (!dir.exists() || !dir.isDirectory())) {
      throw new IOException("Cannot write control file at '" + pOutput.getAbsolutePath() + "'");
    }

    final TarArchiveOutputStream outputStream =
        new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(pOutput)));
    outputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);

    List<FilteredConfigurationFile> configurationFiles = new ArrayList<FilteredConfigurationFile>();

    // create a descriptor out of the "control" file, copy all other files, ignore directories
    PackageDescriptor packageDescriptor = null;
    for (File file : pControlFiles) {
      if (file.isDirectory()) {
        // warn about the misplaced directory, except for directories ignored by default (.svn, cvs,
        // etc)
        if (!isDefaultExcludes(file)) {
          console.info(
              "Found directory '"
                  + file
                  + "' in the control directory. Maybe you are pointing to wrong dir?");
        }
        continue;
      }

      if (CONFIGURATION_FILENAMES.contains(file.getName())
          || MAINTAINER_SCRIPTS.contains(file.getName())) {
        FilteredConfigurationFile configurationFile =
            new FilteredConfigurationFile(file.getName(), new FileInputStream(file), resolver);
        configurationFiles.add(configurationFile);

      } else if ("control".equals(file.getName())) {
        packageDescriptor = createPackageDescriptor(file, pDataSize);

      } else {
        // initialize the information stream to guess the type of the file
        InformationInputStream infoStream = new InformationInputStream(new FileInputStream(file));
        Utils.copy(infoStream, NullOutputStream.NULL_OUTPUT_STREAM);
        infoStream.close();

        // fix line endings for shell scripts
        InputStream in = new FileInputStream(file);
        if (infoStream.isShell() && !infoStream.hasUnixLineEndings()) {
          byte[] buf = Utils.toUnixLineEndings(in);
          in = new ByteArrayInputStream(buf);
        }

        addControlEntry(file.getName(), IOUtils.toString(in), outputStream);

        in.close();
      }
    }

    if (packageDescriptor == null) {
      throw new FileNotFoundException(
          "No 'control' file found in " + Arrays.toString(pControlFiles));
    }

    for (FilteredConfigurationFile configurationFile : configurationFiles) {
      addControlEntry(configurationFile.getName(), configurationFile.toString(), outputStream);
    }
    addControlEntry("control", packageDescriptor.toString(), outputStream);
    addControlEntry("md5sums", pChecksums.toString(), outputStream);

    outputStream.close();

    return packageDescriptor;
  }