private void buildXmlPropertiesFile() {
System.out.println(
"\nBuilding and checking your pipeline properties file -> " + truncPipePropFile);
StringBuilder toPrint = new StringBuilder();
// walk through the prop file
String[] prop = IO.loadFileIntoStringArray(truncPipePropFile);
Pattern val = Pattern.compile("(<entry key.+>)([D|A|B].*)</entry>");
boolean missingFile = false;
for (String s : prop) {
// does it match a file needing prepending? Data/, Apps/, Bed/
Matcher mat = val.matcher(s);
if (mat.matches()) {
File test = new File(referenceDir, mat.group(2));
if (test.exists()) {
System.out.println("Found\t" + test);
toPrint.append(mat.group(1));
toPrint.append(test.toString());
toPrint.append("</entry>");
} else {
System.out.println("Misssing\t" + test);
missingFile = true;
}
}
// threads?
else if (s.contains("threads"))
toPrint.append("<entry key=\"threads\">" + threads + "</entry>");
// nope just save it and add a line return
else toPrint.append(s);
toPrint.append("\n");
}
// anything missing? if so exit
if (missingFile)
Misc.printErrAndExit(
"\nFailed to find all of the files in your properties file, see above.\n");
// OK, write it out
completePipelinePropFile = new File(outputDirectory, "pipelineProperties.xml");
if (IO.writeString(toPrint.toString(), completePipelinePropFile) == false)
Misc.printErrAndExit("Problem writing -> " + truncPipePropFile);
}
public static void processFile(File file) {
System.out.println(file.toString() + " <<<<<< FILE");
}
/**
* Method to move files from HDFS to local filesystem
*
* <p>localPath: Path on the machines filesystem fs:FileSystem object from HDFS pathList:List of
* paths for files that might need to be backed up size:max size in bytes to be backed up
*
* <p>ReturnsDate of the last files backed up if reached size limit, else, zero
*/
public long backupFiles(
String localPath, String preservePath, FileSystem fs, ArrayList<Path> pathList, long size) {
Path fsPath;
long tmpSize = 0;
long tmpDate = 0;
// Start iterating over all paths
for (Path hdfsPath : pathList) {
try {
long nFileSize = fs.getContentSummary(hdfsPath).getLength();
tmpSize = tmpSize + nFileSize;
if ((tmpSize <= size) || (size == 0)) {
FileStatus stat = fs.getFileStatus(hdfsPath);
System.err.println(
"File "
+ hdfsPath.toUri().getPath()
+ " "
+ nFileSize
+ " bytes, "
+ "perms: "
+ stat.getOwner()
+ "/"
+ stat.getGroup()
+ ", "
+ stat.getPermission().toString());
tmpDate = stat.getModificationTime() / 1000;
String sFsPath = localPath + hdfsPath.toUri().getPath();
fsPath = new Path(sFsPath);
File f = new File(sFsPath);
// COMMENTED OUT: until a few backup cycles run
// and the mtime gets in fact set on all copied
// files.
//
// ignore it if the file exists and has the same mtime
// if (f.exists() && f.isFile() && f.lastModified() == stat.getModificationTime())
// {
// System.out.println("no need to backup " + f.toString() + ", mtime matches hdfs");
// continue;
// }
if (false == m_bDryRun) {
// check if we need to back up the local file
// (not directory), if it already exists.
if (f.exists() && f.isFile()) {
// ignore files with substrings in the
// no-preserve file
if (true == doPreserveFile(sFsPath)) {
// move it to the backup path
String sNewPath = preservePath + hdfsPath.toUri().getPath();
File newFile = new File(sNewPath);
// create directory structure for new file?
if (false == newFile.getParentFile().exists()) {
if (false == newFile.getParentFile().mkdirs()) {
System.err.println("Failed to mkdirs " + newFile.getParentFile().toString());
System.exit(1);
}
}
// rename existing file to new location
if (false == f.renameTo(newFile)) {
System.err.println(
"Failed to renameTo " + f.toString() + " to " + newFile.toString());
System.exit(1);
}
System.out.println("preserved " + f.toString() + " into " + newFile.toString());
} else {
System.out.println("skipped preservation of " + f.toString());
}
}
// copy from hdfs to local filesystem
fs.copyToLocalFile(hdfsPath, fsPath);
// set the mtime to match hdfs file
f.setLastModified(stat.getModificationTime());
// compare checksums on both files
compareChecksums(fs, hdfsPath, sFsPath);
}
// don't print the progress after every file -- go
// by at least 1% increments
long nPercentDone = (long) (100 * tmpSize / m_nTotalBytes);
if (nPercentDone > m_nLastPercentBytesDone) {
System.out.println(
"progress: copied "
+ prettyPrintBytes(tmpSize)
+ ", "
+ nPercentDone
+ "% done"
+ ", tstamp="
+ tmpDate);
m_nLastPercentBytesDone = nPercentDone;
}
if (m_nSleepSeconds > 0) {
try {
Thread.sleep(1000 * m_nSleepSeconds);
} catch (Exception e2) {
// ignore
}
}
} else {
return tmpDate;
}
} catch (IOException e) {
System.err.println("FATAL ERROR: Something wrong with the file");
System.err.println(e);
System.out.println(tmpDate);
System.exit(1);
return 0;
}
}
return 0;
}