Beispiel #1
0
 public int run(String[] args) throws Exception {
   if (args.length < 2) {
     System.err.println("Usage: Injector <crawldb> <url_dir>");
     return -1;
   }
   try {
     inject(new Path(args[0]), new Path(args[1]));
     return 0;
   } catch (Exception e) {
     LOG.error("Injector: " + StringUtils.stringifyException(e));
     return -1;
   }
 }
    private String runResolveCommand(List<String> args) {
      int loopCount = 0;
      if (args.size() == 0) {
        return null;
      }
      StringBuffer allOutput = new StringBuffer();
      int numProcessed = 0;
      if (maxArgs < MIN_ALLOWABLE_ARGS) {
        LOG.warn(
            "Invalid value "
                + Integer.toString(maxArgs)
                + " for "
                + SCRIPT_ARG_COUNT_KEY
                + "; must be >= "
                + Integer.toString(MIN_ALLOWABLE_ARGS));
        return null;
      }

      while (numProcessed != args.size()) {
        int start = maxArgs * loopCount;
        List<String> cmdList = new ArrayList<String>();
        cmdList.add(scriptName);
        for (numProcessed = start;
            numProcessed < (start + maxArgs) && numProcessed < args.size();
            numProcessed++) {
          cmdList.add(args.get(numProcessed));
        }
        File dir = null;
        String userDir;
        if ((userDir = System.getProperty("user.dir")) != null) {
          dir = new File(userDir);
        }
        ShellCommandExecutor s = new ShellCommandExecutor(cmdList.toArray(new String[0]), dir);
        try {
          s.execute();
          allOutput.append(s.getOutput() + " ");
        } catch (Exception e) {
          LOG.warn(StringUtils.stringifyException(e));
          return null;
        }
        loopCount++;
      }
      return allOutput.toString();
    }
 // TODO: Can also collect #procs, time taken by each thread to measure fairness.
 @Override
 public Integer call() throws IOException {
   while (true) {
     if (workersFailed.get()) {
       return EXIT_FAILURE;
     }
     long procId = procIds.getAndIncrement();
     if (procId >= numProcs) {
       break;
     }
     if (procId != 0 && procId % 10000 == 0) {
       long ms = System.currentTimeMillis() - start;
       System.out.println("Wrote " + procId + " procedures in " + StringUtils.humanTimeDiff(ms));
     }
     try {
       if (procId > 0 && procId % numProcsPerWal == 0) {
         store.rollWriterForTesting();
         System.out.println(
             "Starting new log : "
                 + store.getActiveLogs().get(store.getActiveLogs().size() - 1));
       }
     } catch (IOException ioe) {
       // Ask other threads to quit too.
       workersFailed.set(true);
       System.err.println("Exception when rolling log file. Current procId = " + procId);
       ioe.printStackTrace();
       return EXIT_FAILURE;
     }
     ProcedureTestingUtility.TestProcedure proc =
         new ProcedureTestingUtility.TestProcedure(procId);
     proc.setData(serializedState);
     store.insert(proc, null);
     store.update(proc);
   }
   return EXIT_SUCCESS;
 }
 /**
  * Return the path array of the localized files
  *
  * @param conf Configuration that contains the localized files
  * @return A path array of localized files
  * @throws IOException
  */
 public static Path[] getLocalSharedCacheFiles(Configuration conf) throws IOException {
   return StringUtils.stringToPath(conf.getStrings("mapred.cache.shared.localFiles"));
 }
 /**
  * Return the path array of the localized caches
  *
  * @param conf Configuration that contains the localized archives
  * @return A path array of localized caches
  * @throws IOException
  */
 public static Path[] getLocalCacheArchives(Configuration conf) throws IOException {
   return StringUtils.stringToPath(conf.getStrings("mapred.cache.localArchives"));
 }
 /**
  * Get cache files set in the Configuration
  *
  * @param conf The configuration which contains the files
  * @return A URI array of the files set in the Configuration
  * @throws IOException
  */
 public static URI[] getSharedCacheFiles(Configuration conf) throws IOException {
   return StringUtils.stringToURI(conf.getStrings("mapred.cache.shared.files"));
 }
 /**
  * Get cache archives set in the Configuration
  *
  * @param conf The configuration which contains the archives
  * @return A URI array of the caches set in the Configuration
  * @throws IOException
  */
 public static URI[] getCacheArchives(Configuration conf) throws IOException {
   return StringUtils.stringToURI(conf.getStrings("mapred.cache.archives"));
 }
 /**
  * Set the configuration with the given set of files
  *
  * @param files The list of files that need to be localized
  * @param conf Configuration which will be changed
  */
 public static void setCacheFiles(URI[] files, Configuration conf) {
   String sfiles = StringUtils.uriToString(files);
   conf.set("mapred.cache.files", sfiles);
 }
 /**
  * Set the configuration with the given set of archives
  *
  * @param archives The list of archives that need to be localized
  * @param conf Configuration which will be changed
  */
 public static void setCacheArchives(URI[] archives, Configuration conf) {
   String sarchives = StringUtils.uriToString(archives);
   conf.set("mapred.cache.archives", sarchives);
 }
 /**
  * Return the path array of the localized files. Intended to be used by user code.
  *
  * @param conf Configuration that contains the localized files
  * @return A path array of localized files
  * @throws IOException
  */
 public static Path[] getLocalCacheFiles(Configuration conf) throws IOException {
   return StringUtils.stringToPath(conf.getStrings(CACHE_LOCALFILES));
 }
 /**
  * Get cache files set in the Configuration. Used by internal DistributedCache and MapReduce code.
  *
  * @param conf The configuration which contains the files
  * @return Am array of the files set in the Configuration
  * @throws IOException
  */
 public static URI[] getCacheFiles(Configuration conf) throws IOException {
   return StringUtils.stringToURI(conf.getStrings(CACHE_FILES));
 }
 /**
  * Get cache archives set in the Configuration. Used by internal DistributedCache and MapReduce
  * code.
  *
  * @param conf The configuration which contains the archives
  * @return An array of the caches set in the Configuration
  * @throws IOException
  */
 public static URI[] getCacheArchives(Configuration conf) throws IOException {
   return StringUtils.stringToURI(conf.getStrings(CACHE_ARCHIVES));
 }
 /**
  * Set the configuration with the given set of files. Intended to be used by user code.
  *
  * @param files The list of files that need to be localized
  * @param conf Configuration which will be changed
  */
 public static void setCacheFiles(URI[] files, Configuration conf) {
   String sfiles = StringUtils.uriToString(files);
   conf.set(CACHE_FILES, sfiles);
 }
 /**
  * Set the configuration with the given set of archives. Intended to be used by user code.
  *
  * @param archives The list of archives that need to be localized
  * @param conf Configuration which will be changed
  */
 public static void setCacheArchives(URI[] archives, Configuration conf) {
   String sarchives = StringUtils.uriToString(archives);
   conf.set(CACHE_ARCHIVES, sarchives);
 }