private static int[] parseJobSetup(Path jobFile) { int[] result = new int[2]; DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); try { DocumentBuilder db = dbf.newDocumentBuilder(); Document doc = db.parse(jobFile.toString()); Element configElement = doc.getDocumentElement(); NodeList nodes = configElement.getElementsByTagName("property"); if (nodes != null && nodes.getLength() > 0) { for (int i = 0; i < nodes.getLength(); i++) { Element property = (Element) nodes.item(i); String elName = xmlGetSingleValue(property, "name"); if (elName == "example.count") { result[0] = Integer.parseInt(xmlGetSingleValue(property, "value")); } else if (elName == "batch.size") { result[1] = Integer.parseInt(xmlGetSingleValue(property, "value")); } } } } catch (ParserConfigurationException pce) { System.err.println( "Caught exception while parsing the cached file '" + jobFile + "' : " + StringUtils.stringifyException(pce)); return null; } catch (SAXException se) { System.err.println( "Caught exception while parsing the cached file '" + jobFile + "' : " + StringUtils.stringifyException(se)); return null; } catch (IOException ioe) { System.err.println( "Caught exception while parsing the cached file '" + jobFile + "' : " + StringUtils.stringifyException(ioe)); return null; } return result; }
/** * This method gets called everytime before any read/write to make sure that any change to * localDirs is reflected immediately. */ private synchronized void confChanged(Configuration conf) throws IOException { String newLocalDirs = conf.get(contextCfgItemName); if (!newLocalDirs.equals(savedLocalDirs)) { String[] localDirs = conf.getStrings(contextCfgItemName); localFS = FileSystem.getLocal(conf); int numDirs = localDirs.length; ArrayList<String> dirs = new ArrayList<String>(numDirs); ArrayList<DF> dfList = new ArrayList<DF>(numDirs); for (int i = 0; i < numDirs; i++) { try { // filter problematic directories Path tmpDir = new Path(localDirs[i]); if (localFS.mkdirs(tmpDir) || localFS.exists(tmpDir)) { try { DiskChecker.checkDir(new File(localDirs[i])); dirs.add(localDirs[i]); dfList.add(new DF(new File(localDirs[i]), 30000)); } catch (DiskErrorException de) { LOG.warn(localDirs[i] + "is not writable\n" + StringUtils.stringifyException(de)); } } else { LOG.warn("Failed to create " + localDirs[i]); } } catch (IOException ie) { LOG.warn( "Failed to create " + localDirs[i] + ": " + ie.getMessage() + "\n" + StringUtils.stringifyException(ie)); } // ignore } localDirsPath = new Path[dirs.size()]; for (int i = 0; i < localDirsPath.length; i++) { localDirsPath[i] = new Path(dirs.get(i)); } dirDF = dfList.toArray(new DF[dirs.size()]); savedLocalDirs = newLocalDirs; // randomize the first disk picked in the round-robin selection dirNumLastAccessed = dirIndexRandomizer.nextInt(dirs.size()); } }
public int run(String[] args) throws Exception { if (args.length < 2) { System.err.println("Usage: Injector <crawldb> <url_dir>"); return -1; } try { inject(new Path(args[0]), new Path(args[1])); return 0; } catch (Exception e) { LOG.error("Injector: " + StringUtils.stringifyException(e)); return -1; } }
public void configure(Configuration conf) { if (conf.getBoolean("minibatch.job.setup", false)) { Path[] jobSetupFiles = new Path[0]; try { jobSetupFiles = DistributedCache.getLocalCacheFiles(conf); } catch (IOException ioe) { System.err.println( "Caught exception while getting cached files: " + StringUtils.stringifyException(ioe)); } for (Path jobSetup : jobSetupFiles) { parseJobSetup(jobSetup); } } }
private Path createPath(Path path, boolean checkWrite) throws IOException { Path file = new Path(localDirsPath[dirNumLastAccessed], path); if (checkWrite) { // check whether we are able to create a directory here. If the disk // happens to be RDONLY we will fail try { DiskChecker.checkDir(new File(file.getParent().toUri().getPath())); } catch (DiskErrorException d) { LOG.warn(StringUtils.stringifyException(d)); return null; } } return file; }
private String runResolveCommand(List<String> args) { int loopCount = 0; if (args.size() == 0) { return null; } StringBuffer allOutput = new StringBuffer(); int numProcessed = 0; if (maxArgs < MIN_ALLOWABLE_ARGS) { LOG.warn( "Invalid value " + Integer.toString(maxArgs) + " for " + SCRIPT_ARG_COUNT_KEY + "; must be >= " + Integer.toString(MIN_ALLOWABLE_ARGS)); return null; } while (numProcessed != args.size()) { int start = maxArgs * loopCount; List<String> cmdList = new ArrayList<String>(); cmdList.add(scriptName); for (numProcessed = start; numProcessed < (start + maxArgs) && numProcessed < args.size(); numProcessed++) { cmdList.add(args.get(numProcessed)); } File dir = null; String userDir; if ((userDir = System.getProperty("user.dir")) != null) { dir = new File(userDir); } ShellCommandExecutor s = new ShellCommandExecutor(cmdList.toArray(new String[0]), dir); try { s.execute(); allOutput.append(s.getOutput() + " "); } catch (Exception e) { LOG.warn(StringUtils.stringifyException(e)); return null; } loopCount++; } return allOutput.toString(); }