コード例 #1
0
  private void buildXmlPropertiesFile() {
    System.out.println(
        "\nBuilding and checking your pipeline properties file -> " + truncPipePropFile);
    StringBuilder toPrint = new StringBuilder();

    // walk through the prop file
    String[] prop = IO.loadFileIntoStringArray(truncPipePropFile);
    Pattern val = Pattern.compile("(<entry key.+>)([D|A|B].*)</entry>");
    boolean missingFile = false;
    for (String s : prop) {
      // does it match a file needing prepending? Data/, Apps/, Bed/
      Matcher mat = val.matcher(s);
      if (mat.matches()) {
        File test = new File(referenceDir, mat.group(2));
        if (test.exists()) {
          System.out.println("Found\t" + test);
          toPrint.append(mat.group(1));
          toPrint.append(test.toString());
          toPrint.append("</entry>");
        } else {
          System.out.println("Misssing\t" + test);
          missingFile = true;
        }
      }
      // threads?
      else if (s.contains("threads"))
        toPrint.append("<entry key=\"threads\">" + threads + "</entry>");

      // nope just save it and add a line return
      else toPrint.append(s);
      toPrint.append("\n");
    }
    // anything missing? if so exit
    if (missingFile)
      Misc.printErrAndExit(
          "\nFailed to find all of the files in your properties file, see above.\n");

    // OK, write it out
    completePipelinePropFile = new File(outputDirectory, "pipelineProperties.xml");
    if (IO.writeString(toPrint.toString(), completePipelinePropFile) == false)
      Misc.printErrAndExit("Problem writing -> " + truncPipePropFile);
  }
コード例 #2
0
 public static void processFile(File file) {
   System.out.println(file.toString() + " <<<<<< FILE");
 }
コード例 #3
0
  /**
   * Method to move files from HDFS to local filesystem
   *
   * <p>localPath: Path on the machines filesystem fs:FileSystem object from HDFS pathList:List of
   * paths for files that might need to be backed up size:max size in bytes to be backed up
   *
   * <p>ReturnsDate of the last files backed up if reached size limit, else, zero
   */
  public long backupFiles(
      String localPath, String preservePath, FileSystem fs, ArrayList<Path> pathList, long size) {
    Path fsPath;
    long tmpSize = 0;
    long tmpDate = 0;

    // Start iterating over all paths
    for (Path hdfsPath : pathList) {
      try {
        long nFileSize = fs.getContentSummary(hdfsPath).getLength();
        tmpSize = tmpSize + nFileSize;

        if ((tmpSize <= size) || (size == 0)) {
          FileStatus stat = fs.getFileStatus(hdfsPath);

          System.err.println(
              "File "
                  + hdfsPath.toUri().getPath()
                  + " "
                  + nFileSize
                  + " bytes, "
                  + "perms: "
                  + stat.getOwner()
                  + "/"
                  + stat.getGroup()
                  + ", "
                  + stat.getPermission().toString());

          tmpDate = stat.getModificationTime() / 1000;

          String sFsPath = localPath + hdfsPath.toUri().getPath();
          fsPath = new Path(sFsPath);

          File f = new File(sFsPath);

          // COMMENTED OUT: until a few backup cycles run
          // and the mtime gets in fact set on all copied
          // files.
          //
          // ignore it if the file exists and has the same mtime
          // if (f.exists() && f.isFile() && f.lastModified() == stat.getModificationTime())
          // {
          // System.out.println("no need to backup " + f.toString() + ", mtime matches hdfs");
          // continue;
          // }

          if (false == m_bDryRun) {
            // check if we need to back up the local file
            // (not directory), if it already exists.
            if (f.exists() && f.isFile()) {
              // ignore files with substrings in the
              // no-preserve file
              if (true == doPreserveFile(sFsPath)) {
                // move it to the backup path
                String sNewPath = preservePath + hdfsPath.toUri().getPath();
                File newFile = new File(sNewPath);

                // create directory structure for new file?
                if (false == newFile.getParentFile().exists()) {
                  if (false == newFile.getParentFile().mkdirs()) {
                    System.err.println("Failed to mkdirs " + newFile.getParentFile().toString());
                    System.exit(1);
                  }
                }

                // rename existing file to new location
                if (false == f.renameTo(newFile)) {
                  System.err.println(
                      "Failed to renameTo " + f.toString() + " to " + newFile.toString());
                  System.exit(1);
                }

                System.out.println("preserved " + f.toString() + " into " + newFile.toString());
              } else {
                System.out.println("skipped preservation of " + f.toString());
              }
            }

            // copy from hdfs to local filesystem
            fs.copyToLocalFile(hdfsPath, fsPath);

            // set the mtime to match hdfs file
            f.setLastModified(stat.getModificationTime());

            // compare checksums on both files
            compareChecksums(fs, hdfsPath, sFsPath);
          }

          // don't print the progress after every file -- go
          // by at least 1% increments
          long nPercentDone = (long) (100 * tmpSize / m_nTotalBytes);
          if (nPercentDone > m_nLastPercentBytesDone) {
            System.out.println(
                "progress: copied "
                    + prettyPrintBytes(tmpSize)
                    + ", "
                    + nPercentDone
                    + "% done"
                    + ", tstamp="
                    + tmpDate);

            m_nLastPercentBytesDone = nPercentDone;
          }

          if (m_nSleepSeconds > 0) {
            try {
              Thread.sleep(1000 * m_nSleepSeconds);
            } catch (Exception e2) {
              // ignore
            }
          }
        } else {
          return tmpDate;
        }
      } catch (IOException e) {
        System.err.println("FATAL ERROR: Something wrong with the file");
        System.err.println(e);
        System.out.println(tmpDate);
        System.exit(1);

        return 0;
      }
    }

    return 0;
  }