コード例 #1
0
ファイル: Utils.java プロジェクト: JananiC/stratosphere
 private static void addPathToConfig(Configuration conf, File path) {
   // chain-in a new classloader
   URL fileUrl = null;
   try {
     fileUrl = path.toURL();
   } catch (MalformedURLException e) {
     throw new RuntimeException("Erroneous config file path", e);
   }
   URL[] urls = {fileUrl};
   ClassLoader cl = new URLClassLoader(urls, conf.getClassLoader());
   conf.setClassLoader(cl);
 }
コード例 #2
0
 private static int launch(Configuration conf, Tool tool, String[] args) throws Exception {
   if (LOG.isDebugEnabled()) {
     LOG.debug(
         MessageFormat.format(
             "Launching application: {0}{1}", //$NON-NLS-1$
             tool.getClass().getName(), Arrays.toString(args)));
   }
   ClassLoader contextClassLoader = switchContextClassLoader(conf.getClassLoader());
   try {
     return ToolRunner.run(conf, tool, args);
   } finally {
     switchContextClassLoader(contextClassLoader);
   }
 }
コード例 #3
0
ファイル: DatasetAccessor.java プロジェクト: ngopina/cdap
  private static Dataset instantiate(@Nullable Configuration conf, Id.DatasetInstance dsInstanceId)
      throws IOException {
    ContextManager.Context context = ContextManager.getContext(conf);
    if (context == null) {
      throw new NullJobConfException();
    }

    Id.DatasetInstance datasetInstanceId =
        dsInstanceId != null ? dsInstanceId : getDatasetInstanceId(conf);

    if (datasetInstanceId == null) {
      throw new IOException("Dataset name property could not be found.");
    }

    try {
      DatasetFramework framework = context.getDatasetFramework();

      if (conf == null) {
        return framework.getDataset(datasetInstanceId, DatasetDefinition.NO_ARGUMENTS, null);
      }

      String queryId = conf.get(Constants.Explore.QUERY_ID);
      if (queryId == null) {
        throw new IOException("QueryId property could not be found");
      }
      QueryHandle queryHandle = QueryHandle.fromId(queryId);
      ClassLoader classLoader =
          DATASET_CLASSLOADER_MAP.getUnchecked(queryHandle).get(datasetInstanceId);
      Dataset dataset;
      if (classLoader == null) {
        classLoader = conf.getClassLoader();
        dataset = firstLoad(framework, queryHandle, datasetInstanceId, classLoader);
      } else {
        dataset =
            framework.getDataset(datasetInstanceId, DatasetDefinition.NO_ARGUMENTS, classLoader);
      }
      return dataset;
    } catch (DatasetManagementException e) {
      throw new IOException(e);
    } finally {
      context.close();
    }
  }
コード例 #4
0
  /**
   * Modify configuration according user-specified generic options
   *
   * @param conf Configuration to be modified
   * @param line User-specified generic options
   */
  private void processGeneralOptions(Configuration conf, CommandLine line) throws IOException {
    if (line.hasOption("fs")) {
      FileSystem.setDefaultUri(conf, line.getOptionValue("fs"));
    }

    if (line.hasOption("jt")) {
      conf.set("mapred.job.tracker", line.getOptionValue("jt"), "from -jt command line option");
    }
    if (line.hasOption("conf")) {
      String[] values = line.getOptionValues("conf");
      for (String value : values) {
        conf.addResource(new Path(value));
      }
    }
    if (line.hasOption("libjars")) {
      conf.set(
          "tmpjars",
          validateFiles(line.getOptionValue("libjars"), conf),
          "from -libjars command line option");
      // setting libjars in client classpath
      URL[] libjars = getLibJars(conf);
      if (libjars != null && libjars.length > 0) {
        conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
        Thread.currentThread()
            .setContextClassLoader(
                new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
      }
    }
    if (line.hasOption("files")) {
      conf.set(
          "tmpfiles",
          validateFiles(line.getOptionValue("files"), conf),
          "from -files command line option");
    }
    if (line.hasOption("archives")) {
      conf.set(
          "tmparchives",
          validateFiles(line.getOptionValue("archives"), conf),
          "from -archives command line option");
    }
    if (line.hasOption('D')) {
      String[] property = line.getOptionValues('D');
      for (String prop : property) {
        String[] keyval = prop.split("=", 2);
        if (keyval.length == 2) {
          conf.set(keyval[0], keyval[1], "from command line");
        }
      }
    }
    conf.setBoolean("mapreduce.client.genericoptionsparser.used", true);

    // tokensFile
    if (line.hasOption("tokenCacheFile")) {
      String fileName = line.getOptionValue("tokenCacheFile");
      // check if the local file exists
      FileSystem localFs = FileSystem.getLocal(conf);
      Path p = new Path(fileName);
      if (!localFs.exists(p)) {
        throw new FileNotFoundException("File " + fileName + " does not exist.");
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("setting conf tokensFile: " + fileName);
      }
      conf.set(
          "mapreduce.job.credentials.json",
          localFs.makeQualified(p).toString(),
          "from -tokenCacheFile command line option");
    }
  }