private static int[] parseJobSetup(Path jobFile) {
int[] result = new int[2];
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
try {
DocumentBuilder db = dbf.newDocumentBuilder();
Document doc = db.parse(jobFile.toString());
Element configElement = doc.getDocumentElement();
NodeList nodes = configElement.getElementsByTagName("property");
if (nodes != null && nodes.getLength() > 0) {
for (int i = 0; i < nodes.getLength(); i++) {
Element property = (Element) nodes.item(i);
String elName = xmlGetSingleValue(property, "name");
if (elName == "example.count") {
result[0] = Integer.parseInt(xmlGetSingleValue(property, "value"));
} else if (elName == "batch.size") {
result[1] = Integer.parseInt(xmlGetSingleValue(property, "value"));
}
}
}
} catch (ParserConfigurationException pce) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(pce));
return null;
} catch (SAXException se) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(se));
return null;
} catch (IOException ioe) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(ioe));
return null;
}
return result;
}
/**
* This method gets called everytime before any read/write to make sure that any change to
* localDirs is reflected immediately.
*/
private synchronized void confChanged(Configuration conf) throws IOException {
String newLocalDirs = conf.get(contextCfgItemName);
if (!newLocalDirs.equals(savedLocalDirs)) {
String[] localDirs = conf.getStrings(contextCfgItemName);
localFS = FileSystem.getLocal(conf);
int numDirs = localDirs.length;
ArrayList<String> dirs = new ArrayList<String>(numDirs);
ArrayList<DF> dfList = new ArrayList<DF>(numDirs);
for (int i = 0; i < numDirs; i++) {
try {
// filter problematic directories
Path tmpDir = new Path(localDirs[i]);
if (localFS.mkdirs(tmpDir) || localFS.exists(tmpDir)) {
try {
DiskChecker.checkDir(new File(localDirs[i]));
dirs.add(localDirs[i]);
dfList.add(new DF(new File(localDirs[i]), 30000));
} catch (DiskErrorException de) {
LOG.warn(localDirs[i] + "is not writable\n" + StringUtils.stringifyException(de));
}
} else {
LOG.warn("Failed to create " + localDirs[i]);
}
} catch (IOException ie) {
LOG.warn(
"Failed to create "
+ localDirs[i]
+ ": "
+ ie.getMessage()
+ "\n"
+ StringUtils.stringifyException(ie));
} // ignore
}
localDirsPath = new Path[dirs.size()];
for (int i = 0; i < localDirsPath.length; i++) {
localDirsPath[i] = new Path(dirs.get(i));
}
dirDF = dfList.toArray(new DF[dirs.size()]);
savedLocalDirs = newLocalDirs;
// randomize the first disk picked in the round-robin selection
dirNumLastAccessed = dirIndexRandomizer.nextInt(dirs.size());
}
}
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Usage: Injector <crawldb> <url_dir>");
return -1;
}
try {
inject(new Path(args[0]), new Path(args[1]));
return 0;
} catch (Exception e) {
LOG.error("Injector: " + StringUtils.stringifyException(e));
return -1;
}
}
public void configure(Configuration conf) {
if (conf.getBoolean("minibatch.job.setup", false)) {
Path[] jobSetupFiles = new Path[0];
try {
jobSetupFiles = DistributedCache.getLocalCacheFiles(conf);
} catch (IOException ioe) {
System.err.println(
"Caught exception while getting cached files: "
+ StringUtils.stringifyException(ioe));
}
for (Path jobSetup : jobSetupFiles) {
parseJobSetup(jobSetup);
}
}
}
private Path createPath(Path path, boolean checkWrite) throws IOException {
Path file = new Path(localDirsPath[dirNumLastAccessed], path);
if (checkWrite) {
// check whether we are able to create a directory here. If the disk
// happens to be RDONLY we will fail
try {
DiskChecker.checkDir(new File(file.getParent().toUri().getPath()));
} catch (DiskErrorException d) {
LOG.warn(StringUtils.stringifyException(d));
return null;
}
}
return file;
}
private String runResolveCommand(List<String> args) {
int loopCount = 0;
if (args.size() == 0) {
return null;
}
StringBuffer allOutput = new StringBuffer();
int numProcessed = 0;
if (maxArgs < MIN_ALLOWABLE_ARGS) {
LOG.warn(
"Invalid value "
+ Integer.toString(maxArgs)
+ " for "
+ SCRIPT_ARG_COUNT_KEY
+ "; must be >= "
+ Integer.toString(MIN_ALLOWABLE_ARGS));
return null;
}
while (numProcessed != args.size()) {
int start = maxArgs * loopCount;
List<String> cmdList = new ArrayList<String>();
cmdList.add(scriptName);
for (numProcessed = start;
numProcessed < (start + maxArgs) && numProcessed < args.size();
numProcessed++) {
cmdList.add(args.get(numProcessed));
}
File dir = null;
String userDir;
if ((userDir = System.getProperty("user.dir")) != null) {
dir = new File(userDir);
}
ShellCommandExecutor s = new ShellCommandExecutor(cmdList.toArray(new String[0]), dir);
try {
s.execute();
allOutput.append(s.getOutput() + " ");
} catch (Exception e) {
LOG.warn(StringUtils.stringifyException(e));
return null;
}
loopCount++;
}
return allOutput.toString();
}