// ==============================================================================
public final int[] renderGlyph(
char glyph, Paint paint, android.graphics.Matrix matrix, Rect bounds) {
Path p = new Path();
paint.getTextPath(String.valueOf(glyph), 0, 1, 0.0f, 0.0f, p);
RectF boundsF = new RectF();
p.computeBounds(boundsF, true);
matrix.mapRect(boundsF);
boundsF.roundOut(bounds);
bounds.left--;
bounds.right++;
final int w = bounds.width();
final int h = Math.max(1, bounds.height());
Bitmap bm = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888);
Canvas c = new Canvas(bm);
matrix.postTranslate(-bounds.left, -bounds.top);
c.setMatrix(matrix);
c.drawPath(p, paint);
final int sizeNeeded = w * h;
if (cachedRenderArray.length < sizeNeeded) cachedRenderArray = new int[sizeNeeded];
bm.getPixels(cachedRenderArray, 0, w, 0, 0, w, h);
bm.recycle();
return cachedRenderArray;
}
/**
* Interate through all the paths and find the longest one
*
* @return longest path distance
*/
public double getLongestPath() {
double max = 0;
for (Path p : getPaths()) {
if (p.calculateDistance() > max) max = p.calculateDistance();
}
return max;
}
/** Check that a cancelled key will never be queued */
static void testCancel(Path dir) throws IOException {
System.out.println("-- Cancel --");
try (WatchService watcher = FileSystems.getDefault().newWatchService()) {
System.out.format("register %s for events\n", dir);
WatchKey myKey = dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE});
checkKey(myKey, dir);
System.out.println("cancel key");
myKey.cancel();
// create a file in the directory
Path file = dir.resolve("mars");
System.out.format("create: %s\n", file);
Files.createFile(file);
// poll for keys - there will be none
System.out.println("poll...");
try {
WatchKey key = watcher.poll(3000, TimeUnit.MILLISECONDS);
if (key != null) throw new RuntimeException("key should not be queued");
} catch (InterruptedException x) {
throw new RuntimeException(x);
}
// done
Files.delete(file);
System.out.println("OKAY");
}
}
public void testAbort() throws IOException {
JobConf job = new JobConf();
setConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.setWorkOutputPath(job, committer.getTempTaskOutputPath(tContext));
// do setup
committer.setupJob(jContext);
committer.setupTask(tContext);
String file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
// write output
FileSystem localFs = FileSystem.getLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter);
writeOutput(theRecordWriter, reporter);
// do abort
committer.abortTask(tContext);
File expectedFile =
new File(new Path(committer.getTempTaskOutputPath(tContext), file).toString());
assertFalse("task temp dir still exists", expectedFile.exists());
committer.abortJob(jContext, JobStatus.State.FAILED);
expectedFile = new File(new Path(outDir, FileOutputCommitter.TEMP_DIR_NAME).toString());
assertFalse("job temp dir still exists", expectedFile.exists());
assertEquals("Output directory not empty", 0, new File(outDir.toString()).listFiles().length);
FileUtil.fullyDelete(new File(outDir.toString()));
}
private static IntWritable deduceInputFile(JobConf job) {
Path[] inputPaths = FileInputFormat.getInputPaths(job);
Path inputFile = new Path(job.get("map.input.file"));
// value == one for sort-input; value == two for sort-output
return (inputFile.getParent().equals(inputPaths[0])) ? sortInput : sortOutput;
}
private static void finalize(
Configuration conf, JobConf jobconf, final Path destPath, String presevedAttributes)
throws IOException {
if (presevedAttributes == null) {
return;
}
EnumSet<FileAttribute> preseved = FileAttribute.parse(presevedAttributes);
if (!preseved.contains(FileAttribute.USER)
&& !preseved.contains(FileAttribute.GROUP)
&& !preseved.contains(FileAttribute.PERMISSION)) {
return;
}
FileSystem dstfs = destPath.getFileSystem(conf);
Path dstdirlist = new Path(jobconf.get(DST_DIR_LIST_LABEL));
SequenceFile.Reader in = null;
try {
in = new SequenceFile.Reader(dstdirlist.getFileSystem(jobconf), dstdirlist, jobconf);
Text dsttext = new Text();
FilePair pair = new FilePair();
for (; in.next(dsttext, pair); ) {
Path absdst = new Path(destPath, pair.output);
updatePermissions(pair.input, dstfs.getFileStatus(absdst), preseved, dstfs);
}
} finally {
checkAndClose(in);
}
}
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
FileSplit split = (FileSplit) genericSplit;
Configuration job = context.getConfiguration();
m_Sb.setLength(0);
m_Start = split.getStart();
m_End = m_Start + split.getLength();
final Path file = split.getPath();
compressionCodecs = new CompressionCodecFactory(job);
final CompressionCodec codec = compressionCodecs.getCodec(file);
// open the file and seek to the m_Start of the split
FileSystem fs = file.getFileSystem(job);
// getFileStatus fileStatus = fs.getFileStatus(split.getPath());
//noinspection deprecation
@SuppressWarnings(value = "deprecated")
long length = fs.getLength(file);
FSDataInputStream fileIn = fs.open(split.getPath());
if (m_Start > 0) fileIn.seek(m_Start);
if (codec != null) {
CompressionInputStream inputStream = codec.createInputStream(fileIn);
m_Input = new BufferedReader(new InputStreamReader(inputStream));
m_End = length;
} else {
m_Input = new BufferedReader(new InputStreamReader(fileIn));
}
m_Current = m_Start;
m_Key = split.getPath().getName();
}
public static FileDesc loadFile(Path root, Path file, int blocSize)
throws NoSuchAlgorithmException, FileNotFoundException, IOException {
MessageDigest md = MessageDigest.getInstance("SHA-512");
MessageDigest fileMd = MessageDigest.getInstance("SHA-512");
FileDesc desc = new FileDesc(file.toString(), null, null);
List<Bloc> list = new ArrayList<Bloc>();
try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) {
byte[] buf = new byte[blocSize];
byte[] h;
int s;
while ((s = fis.read(buf)) != -1) {
int c;
while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c;
fileMd.update(buf, 0, s);
// padding
byte p = 0;
while (s < buf.length) buf[s++] = ++p;
h = md.digest(buf);
Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h));
list.add(bloc);
}
h = fileMd.digest();
desc.fileHash = new Hash(h);
desc.blocs = list.toArray(new Bloc[0]);
}
return desc;
}
public int run(String[] args) throws Exception {
Configuration conf = getConf();
conf.setLong("mapreduce.task.timeout", 10000 * 60 * 60);
Path train_file = new Path(args[0]);
Path test_file = new Path(args[1]);
conf.set("train_file", train_file.getParent().getName());
Job job = new Job(conf, "MapTestID");
job.setJarByClass(MapTestID.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setMapperClass(MapTestID.MapTestIDMap.class);
job.setReducerClass(MapTestID.MapTestIDReduce.class);
job.setNumReduceTasks(300);
FileInputFormat.addInputPath(job, train_file);
FileInputFormat.addInputPath(job, test_file);
FileOutputFormat.setOutputPath(job, new Path(args[2]));
return job.waitForCompletion(true) ? 0 : 1;
}
/**
* Recursively rebuild the tree nodes.
*
* @param root The full abstract path to the root saved layout directory.
* @param local The current directory relative to root (null if none).
* @param tnode The current parent tree node.
*/
private void rebuildTreeModel(Path root, Path local, DefaultMutableTreeNode tnode) {
TreeSet<Path> subdirs = new TreeSet<Path>();
TreeSet<String> layouts = new TreeSet<String>();
{
Path current = new Path(root, local);
File files[] = current.toFile().listFiles();
if (files != null) {
int wk;
for (wk = 0; wk < files.length; wk++) {
String name = files[wk].getName();
if (files[wk].isDirectory()) subdirs.add(new Path(local, name));
else if (files[wk].isFile()) layouts.add(name);
}
}
}
for (Path subdir : subdirs) {
TreeData data = new TreeData(subdir);
DefaultMutableTreeNode child = new DefaultMutableTreeNode(data, true);
tnode.add(child);
rebuildTreeModel(root, subdir, child);
}
for (String lname : layouts) {
TreeData data = new TreeData(new Path(local, lname), lname);
DefaultMutableTreeNode child = new DefaultMutableTreeNode(data, false);
tnode.add(child);
}
}
public static void main(String[] args) throws Exception {
Path dir1 = TestUtil.createTemporaryDirectory();
try {
// Same directory
testCopyFileToFile(dir1, dir1, TestUtil.supportsLinks(dir1));
testMove(dir1, dir1, TestUtil.supportsLinks(dir1));
// Different directories. Use test.dir if possible as it might be
// a different volume/file system and so improve test coverage.
String testDir = System.getProperty("test.dir", ".");
Path dir2 = TestUtil.createTemporaryDirectory(testDir);
try {
boolean testSymbolicLinks = TestUtil.supportsLinks(dir1) && TestUtil.supportsLinks(dir2);
testCopyFileToFile(dir1, dir2, testSymbolicLinks);
testMove(dir1, dir2, testSymbolicLinks);
} finally {
TestUtil.removeAll(dir2);
}
// Target is location associated with custom provider
Path dir3 = PassThroughFileSystem.create().getPath(dir1.toString());
testCopyFileToFile(dir1, dir3, false);
testMove(dir1, dir3, false);
// Test copy(InputStream,Path) and copy(Path,OutputStream)
testCopyInputStreamToFile();
testCopyFileToOuputStream();
} finally {
TestUtil.removeAll(dir1);
}
}
private void uploadFile(String relPath, Path fullPath) throws DropboxException, IOException {
if (!fullPath.toFile().exists()) return;
/*if(api.metadata(relPath, 1, null, false, null).rev == MetaHandler.getRev(relPath)){
logger.debug("File "+relPath+" not changed");
return;
}*/
VOSync.debug("Uploading " + relPath);
String rev = MetaHandler.getRev(relPath);
InputStream inp = new FileInputStream(fullPath.toFile());
Entry fileEntry = api.putFile(relPath, inp, fullPath.toFile().length(), rev, null);
inp.close();
Path destFilePath =
FileSystems.getDefault()
.getPath(fullPath.toFile().getParentFile().getPath(), fileEntry.fileName());
MetaHandler.setFile(relPath, destFilePath.toFile(), fileEntry.rev);
logger.debug(relPath + " put to db");
// if the file was renamed, move the file on disk and download the current from server
if (!fileEntry.fileName().equals(fullPath.toFile().getName())) {
logger.error(fileEntry.fileName() + " != " + fullPath.toFile().getName());
fullPath.toFile().renameTo(destFilePath.toFile());
}
}
@Override
public Item item(final QueryContext qc, final InputInfo ii) throws QueryException {
checkCreate(qc);
final Path path = toPath(0, qc);
final B64 archive = toB64(exprs[1], qc, false);
final TokenSet hs = entries(2, qc);
try (ArchiveIn in = ArchiveIn.get(archive.input(info), info)) {
while (in.more()) {
final ZipEntry ze = in.entry();
final String name = ze.getName();
if (hs == null || hs.delete(token(name)) != 0) {
final Path file = path.resolve(name);
if (ze.isDirectory()) {
Files.createDirectories(file);
} else {
Files.createDirectories(file.getParent());
Files.write(file, in.read());
}
}
}
} catch (final IOException ex) {
throw ARCH_FAIL_X.get(info, ex);
}
return null;
}
// Mostly for setting up the symlinks. Note that when we setup the distributed
// cache, we didn't create the symlinks. This is done on a per task basis
// by the currently executing task.
public static void setupWorkDir(JobConf conf) throws IOException {
File workDir = new File(".").getAbsoluteFile();
FileUtil.fullyDelete(workDir);
if (DistributedCache.getSymlink(conf)) {
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
if (archives != null) {
for (int i = 0; i < archives.length; i++) {
String link = archives[i].getFragment();
if (link != null) {
link = workDir.toString() + Path.SEPARATOR + link;
File flink = new File(link);
if (!flink.exists()) {
FileUtil.symLink(localArchives[i].toString(), link);
}
}
}
}
if (files != null) {
for (int i = 0; i < files.length; i++) {
String link = files[i].getFragment();
if (link != null) {
link = workDir.toString() + Path.SEPARATOR + link;
File flink = new File(link);
if (!flink.exists()) {
FileUtil.symLink(localFiles[i].toString(), link);
}
}
}
}
}
File jobCacheDir = null;
if (conf.getJar() != null) {
jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
}
// create symlinks for all the files in job cache dir in current
// workingdir for streaming
try {
DistributedCache.createAllSymlink(conf, jobCacheDir, workDir);
} catch (IOException ie) {
// Do not exit even if symlinks have not been created.
LOG.warn(StringUtils.stringifyException(ie));
}
// add java.io.tmpdir given by mapred.child.tmp
String tmp = conf.get("mapred.child.tmp", "./tmp");
Path tmpDir = new Path(tmp);
// if temp directory path is not absolute
// prepend it with workDir.
if (!tmpDir.isAbsolute()) {
tmpDir = new Path(workDir.toString(), tmp);
FileSystem localFs = FileSystem.getLocal(conf);
if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
throw new IOException("Mkdirs failed to create " + tmpDir.toString());
}
}
}
private void directory_to_directory(
String source_base_name, String destination_base_name, String directory_name)
throws IOException, InterruptedException {
// E.g.
// source_base_name: /foo/bar
// destination_base_name: /x/y
// directory_name: abc
//
// - Create /x/y/abc
// - Copy /foo/bar/abc/* to /x/y/abc/*
// Get source directory.
String source_directory_name = Path.concatenate(source_base_name, directory_name);
File source_directory = File.create(source_directory_name);
Assertion.check(source_directory.isDirectory());
// Get destination directory, creating it if
// necessary.
String destination_directory_name = Path.concatenate(destination_base_name, directory_name);
File destination_directory = File.create(destination_directory_name);
destination_directory.mkdirs();
// Get files in source directory
String[] source_file_names = source_directory.list();
Vector source_files = new Vector();
for (int i = 0; i < source_file_names.length; i++)
source_files.addElement(File.create(source_directory_name, source_file_names[i]));
// Copy
to_directory(source_files, destination_directory);
}
private static void addFolder2(
FileSystem fs, Path p, ArrayList<String> keys, ArrayList<String> failed) {
try {
if (fs == null) return;
Futures futures = new Futures();
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder2(fs, pfs, keys, failed);
} else {
long size = file.getLen();
Key res;
if (pfs.getName().endsWith(Extensions.JSON)) {
throw H2O.unimpl();
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
throw H2O.unimpl();
} else {
Key k = null;
keys.add((k = HdfsFileVec.make(file, futures)).toString());
Log.info("PersistHdfs: DKV.put(" + k + ")");
}
}
}
} catch (Exception e) {
Log.err(e);
failed.add(p.toString());
}
}
private InputStream OpenMultiplePartsWithOffset(FileSystem fs, Path pt, long offset)
throws IOException {
RemoteIterator<LocatedFileStatus> rit = fs.listFiles(pt, false);
Vector<FSDataInputStream> fileHandleList = new Vector<FSDataInputStream>();
while (rit.hasNext()) {
Path path = rit.next().getPath();
String filename =
path.toString().substring(path.getParent().toString().length(), path.toString().length());
if (filename.startsWith("/part-")) {
long filesize = fs.getFileStatus(path).getLen();
if (offset < filesize) {
FSDataInputStream handle = fs.open(path);
if (offset > 0) {
handle.seek(offset);
}
fileHandleList.add(handle);
}
offset -= filesize;
}
}
if (fileHandleList.size() == 1) return fileHandleList.get(0);
else if (fileHandleList.size() > 1) {
Enumeration<FSDataInputStream> enu = fileHandleList.elements();
return new SequenceInputStream(enu);
} else {
System.err.println("Error, no source file loaded. run genSeedDataset.sh fisrt!");
return null;
}
}
/*
* Fetch a file that is in a Hadoop file system. Return a local File.
* Interruptible.
*/
private File hdfsFetch(Path fromPath, Reporter reporter)
throws IOException, InterruptedException {
UUID uniqueId = UUID.randomUUID();
File toFile = new File(tempDir, uniqueId.toString() + "/" + fromPath.getName());
File toDir = new File(toFile.getParent());
if (toDir.exists()) {
FileUtils.deleteDirectory(toDir);
}
toDir.mkdirs();
Path toPath = new Path(toFile.getCanonicalPath());
FileSystem fS = fromPath.getFileSystem(hadoopConf);
FileSystem tofS = FileSystem.getLocal(hadoopConf);
Throttler throttler = new Throttler((double) bytesPerSecThrottle);
try {
for (FileStatus fStatus : fS.globStatus(fromPath)) {
log.info("Copying " + fStatus.getPath() + " to " + toPath);
long bytesSoFar = 0;
FSDataInputStream iS = fS.open(fStatus.getPath());
FSDataOutputStream oS = tofS.create(toPath);
byte[] buffer = new byte[downloadBufferSize];
int nRead;
while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
// Needed to being able to be interrupted at any moment.
if (Thread.interrupted()) {
iS.close();
oS.close();
cleanDirNoExceptions(toDir);
throw new InterruptedException();
}
bytesSoFar += nRead;
oS.write(buffer, 0, nRead);
throttler.incrementAndThrottle(nRead);
if (bytesSoFar >= bytesToReportProgress) {
reporter.progress(bytesSoFar);
bytesSoFar = 0l;
}
}
if (reporter != null) {
reporter.progress(bytesSoFar);
}
oS.close();
iS.close();
}
return toDir;
} catch (ClosedByInterruptException e) {
// This can be thrown by the method read.
cleanDirNoExceptions(toDir);
throw new InterruptedIOException();
}
}
// copy source to target with verification
static void copyAndVerify(Path source, Path target, CopyOption... options) throws IOException {
Path result = copy(source, target, options);
assertTrue(result == target);
// get attributes of source and target file to verify copy
boolean followLinks = true;
LinkOption[] linkOptions = new LinkOption[0];
boolean copyAttributes = false;
for (CopyOption opt : options) {
if (opt == NOFOLLOW_LINKS) {
followLinks = false;
linkOptions = new LinkOption[] {NOFOLLOW_LINKS};
}
if (opt == COPY_ATTRIBUTES) copyAttributes = true;
}
BasicFileAttributes basicAttributes =
readAttributes(source, BasicFileAttributes.class, linkOptions);
// check hash if regular file
if (basicAttributes.isRegularFile()) assertTrue(computeHash(source) == computeHash(target));
// check link target if symbolic link
if (basicAttributes.isSymbolicLink())
assert (readSymbolicLink(source).equals(readSymbolicLink(target)));
// check that attributes are copied
if (copyAttributes && followLinks) {
checkBasicAttributes(
basicAttributes, readAttributes(source, BasicFileAttributes.class, linkOptions));
// verify other attributes when same provider
if (source.getFileSystem().provider() == target.getFileSystem().provider()) {
// check POSIX attributes are copied
String os = System.getProperty("os.name");
if ((os.equals("SunOS") || os.equals("Linux")) && testPosixAttributes) {
checkPosixAttributes(
readAttributes(source, PosixFileAttributes.class, linkOptions),
readAttributes(target, PosixFileAttributes.class, linkOptions));
}
// check DOS attributes are copied
if (os.startsWith("Windows")) {
checkDosAttributes(
readAttributes(source, DosFileAttributes.class, linkOptions),
readAttributes(target, DosFileAttributes.class, linkOptions));
}
// check named attributes are copied
if (followLinks
&& getFileStore(source).supportsFileAttributeView("xattr")
&& getFileStore(target).supportsFileAttributeView("xattr")) {
checkUserDefinedFileAttributes(
readUserDefinedFileAttributes(source), readUserDefinedFileAttributes(target));
}
}
}
}
public static void addFolder2(Path p, ArrayList<String> keys, ArrayList<String> failed)
throws IOException {
FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
if (!fs.exists(p)) {
failed.add("Path does not exist: '" + p.toString() + "'");
return;
}
addFolder2(fs, p, keys, failed);
}
private ArrayList<Path> holeLaufwerkeWindows() {
ArrayList<Path> laufwerksRoot = new ArrayList<>();
Iterable<Path> rootDirectories = FileSystems.getDefault().getRootDirectories();
for (Path path : rootDirectories) {
laufwerksRoot.add(path.getRoot());
}
return laufwerksRoot;
}
public static String absolutePath(Path p) {
if (p == null) return "";
StringBuilder sb = new StringBuilder();
Path parentPath = p.getParent();
if (parentPath == null) return "/";
sb.append(absolutePath(parentPath));
if (sb.length() > 1) sb.append("/");
sb.append(p.getName());
return sb.toString();
}
public void testKryoSerializationRoundtrip() throws IOException {
Path testDir = Files.createTempDirectory("test");
try {
SliceManager sliceManager =
new SliceManager(new KryoSliceSerializer(), new FileStorageManager(testDir));
doSerializationRoundtrip("kryo", sliceManager);
} finally {
FileHelper.delete(testDir.toFile());
}
}
/**
* Generate the list of files and make them into FileSplits. This needs to be copied to insert a
* filter on acceptable data
*/
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
long desiredMappers =
job.getConfiguration().getLong("org.systemsbiology.jxtandem.DesiredXMLInputMappers", 0);
// generate splits
List<InputSplit> splits = new ArrayList<InputSplit>();
List<FileStatus> fileStatuses = listStatus(job);
boolean forceNumberMappers = fileStatuses.size() == 1;
for (FileStatus file : fileStatuses) {
Path path = file.getPath();
if (!isPathAcceptable(path)) // filter acceptable data
continue;
FileSystem fs = path.getFileSystem(job.getConfiguration());
long length = file.getLen();
BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
if ((length != 0) && isSplitable(job, path)) {
long blockSize = file.getBlockSize();
// use desired mappers to force more splits
if (forceNumberMappers && desiredMappers > 0)
maxSize = Math.min(maxSize, (length / desiredMappers));
long splitSize = computeSplitSize(blockSize, minSize, maxSize);
long bytesRemaining = length;
while (withinSlop(splitSize, bytesRemaining)) {
int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
splits.add(
new FileSplit(
path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts()));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0) {
splits.add(
new FileSplit(
path,
length - bytesRemaining,
bytesRemaining,
blkLocations[blkLocations.length - 1].getHosts()));
}
} else if (length != 0) {
splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
} else {
// Create empty hosts array for zero length files
splits.add(new FileSplit(path, 0, length, new String[0]));
}
}
System.out.println("Total # of splits: " + splits.size());
// LOG.debug("Total # of splits: " + splits.size());
return splits;
}
/**
* Test the layout of a directory against the sieve.
*
* @param dir the directory to test
* @return true if the directory structure matches the sieve's configuration.
*/
public boolean matches(File dir) {
for (String str : includes) {
File f = new File(dir, Path.translateFile(str));
if (!f.exists()) return false;
}
for (String str : excludes) {
File f = new File(dir, Path.translateFile(str));
if (f.exists()) return false;
}
return true;
}
/**
* Add an file path to the current set of classpath entries It adds the file to cache as well.
*
* @param file Path of the file to be added
* @param conf Configuration that contains the classpath setting
*/
public static void addFileToClassPath(Path file, Configuration conf) throws IOException {
String classpath = conf.get("mapred.job.classpath.files");
conf.set(
"mapred.job.classpath.files",
classpath == null
? file.toString()
: classpath + System.getProperty("path.separator") + file.toString());
URI uri = file.makeQualified(file.getFileSystem(conf)).toUri();
addCacheFile(uri, conf);
}
public static void addFolder(Path p, JsonArray succeeded, JsonArray failed) throws IOException {
FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
if (!fs.exists(p)) {
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, "Path does not exist!");
failed.add(o);
return;
}
addFolder(fs, p, succeeded, failed);
}
/** Sanity check for srcPath */
private static void checkSrcPath(Configuration conf, List<Path> srcPaths) throws IOException {
List<IOException> rslt = new ArrayList<IOException>();
for (Path p : srcPaths) {
FileSystem fs = p.getFileSystem(conf);
if (!fs.exists(p)) {
rslt.add(new IOException("Input source " + p + " does not exist."));
}
}
if (!rslt.isEmpty()) {
throw new InvalidInputException(rslt);
}
}
/**
* Construct a {@link SubProcessHeavy SubProcessHeavy} instance which when executed will fulfill
* the given action agenda.
*
* <p>
*
* @param agenda The agenda to be accomplished by the action.
* @param outFile The file to which all STDOUT output is redirected.
* @param errFile The file to which all STDERR output is redirected.
* @return The SubProcess which will fulfill the agenda.
* @throws PipelineException If unable to prepare a SubProcess due to illegal, missing or
* imcompatable information in the action agenda or a general failure of the prep method code.
*/
public SubProcessHeavy prep(ActionAgenda agenda, File outFile, File errFile)
throws PipelineException {
NodeID nodeID = agenda.getNodeID();
/* sanity checks */
Path fromPath = null;
FileSeq fromSeq = null;
FileSeq toSeq = null;
{
ArrayList<String> formats = new ArrayList<String>();
formats.add("xml");
String sname = (String) getSingleParamValue("XmlSource");
if (sname == null) throw new PipelineException("The Image Source was not set!");
String xmlpath = null;
if (sname != null) {
FileSeq fxml = agenda.getPrimarySource(sname);
xmlpath = fxml.toString();
}
NodeID snodeID = new NodeID(nodeID, sname);
ArrayList<String> suffixes = new ArrayList<String>();
suffixes.add("html");
suffixes.add("htm");
Path targetPath = getPrimaryTargetPath(agenda, suffixes, "Html fil");
// ********************************************************************
// Build the mencoder command
// ********************************************************************
ArrayList<String> args = new ArrayList<String>();
args.add("/intranet/jpt/phpCliPrograms/Edit2pipe.php");
args.add("none");
args.add(xmlpath);
args.add(xmlpath);
args.add(targetPath.toOsString());
// Create sub process
return createSubProcess(
agenda,
"php5",
args,
agenda.getEnvironment(),
agenda.getTargetPath().toFile(),
outFile,
errFile);
}
}
public void setClassPath(final Path classpath) {
this.pathComponents.removeAllElements();
if (classpath != null) {
final Path actualClasspath = classpath.concatSystemClasspath("ignore");
final String[] pathElements = actualClasspath.list();
for (int i = 0; i < pathElements.length; ++i) {
try {
this.addPathElement(pathElements[i]);
} catch (BuildException ex) {
}
}
}
}
