Пример #1
0
  @Parameters
  public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {

    LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();

    for (int i = 1; i <= NUM_PROGRAMS; i++) {
      Configuration config = new Configuration();
      config.setInteger("ProgramId", i);
      tConfigs.add(config);
    }

    return toParameterList(tConfigs);
  }
Пример #2
0
  /**
   * Configures this JDBCOutputFormat.
   *
   * @param parameters Configuration containing all parameters.
   */
  @Override
  public void configure(Configuration parameters) {
    this.driverName = parameters.getString(DRIVER_KEY, null);
    this.username = parameters.getString(USERNAME_KEY, null);
    this.password = parameters.getString(PASSWORD_KEY, null);
    this.dbURL = parameters.getString(URL_KEY, null);
    this.query = parameters.getString(QUERY_KEY, null);
    this.fieldCount = parameters.getInteger(FIELD_COUNT_KEY, 0);
    this.batchInterval = parameters.getInteger(BATCH_INTERVAL, DEFAULT_BATCH_INTERVERAL);

    @SuppressWarnings("unchecked")
    Class<Value>[] classes = new Class[this.fieldCount];
    this.fieldClasses = classes;

    for (int i = 0; i < this.fieldCount; i++) {
      @SuppressWarnings("unchecked")
      Class<? extends Value> clazz =
          (Class<? extends Value>) parameters.getClass(FIELD_TYPE_KEY + i, null);
      if (clazz == null) {
        throw new IllegalArgumentException(
            "Invalid configuration for JDBCOutputFormat: " + "No type class for parameter " + i);
      }
      this.fieldClasses[i] = clazz;
    }
  }
Пример #3
0
  /**
   * Registers an newly incoming runtime task with the task manager.
   *
   * @param id the ID of the task to register
   * @param jobConfiguration the job configuration that has been attached to the original job graph
   * @param environment the environment of the task to be registered
   * @return the task to be started or <code>null</code> if a task with the same ID was already
   *     running
   */
  private Task createAndRegisterTask(
      final ExecutionVertexID id,
      final Configuration jobConfiguration,
      final RuntimeEnvironment environment)
      throws InsufficientResourcesException, IOException {

    if (id == null) {
      throw new IllegalArgumentException("Argument id is null");
    }

    if (environment == null) {
      throw new IllegalArgumentException("Argument environment is null");
    }

    // Task creation and registration must be atomic
    Task task;

    synchronized (this) {
      final Task runningTask = this.runningTasks.get(id);
      boolean registerTask = true;
      if (runningTask == null) {
        task = new Task(id, environment, this);
      } else {

        if (runningTask instanceof Task) {
          // Task is already running
          return null;
        } else {
          // There is already a replay task running, we will simply restart it
          task = runningTask;
          registerTask = false;
        }
      }

      if (registerTask) {
        // Register the task with the byte buffered channel manager
        this.channelManager.register(task);

        boolean enableProfiling = false;
        if (this.profiler != null
            && jobConfiguration.getBoolean(ProfilingUtils.PROFILE_JOB_KEY, true)) {
          enableProfiling = true;
        }

        // Register environment, input, and output gates for profiling
        if (enableProfiling) {
          task.registerProfiler(this.profiler, jobConfiguration);
        }

        this.runningTasks.put(id, task);
      }
    }
    return task;
  }
Пример #4
0
  @Override
  public void configure(Configuration parameters) {
    super.configure(parameters);

    // read own parameters
    this.blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE);
    if (this.blockSize < 1 && this.blockSize != NATIVE_BLOCK_SIZE)
      throw new IllegalArgumentException("The block size parameter must be set and larger than 0.");
    if (this.blockSize > Integer.MAX_VALUE)
      throw new UnsupportedOperationException(
          "Currently only block size up to Integer.MAX_VALUE are supported");
  }
Пример #5
0
  @Override
  public void configure(Configuration parameters) {
    super.configure(parameters);

    // get the charset for the decoding
    String charsetName = parameters.getString(CHARSET_NAME, DEFAULT_CHARSET_NAME);
    if (charsetName == null || !Charset.isSupported(charsetName)) {
      throw new RuntimeException("Unsupported charset: " + charsetName);
    }

    if (charsetName.equals("ISO-8859-1") || charsetName.equalsIgnoreCase("ASCII")) {
      this.ascii = true;
    } else {
      this.decoder = Charset.forName(charsetName).newDecoder();
      this.byteWrapper = ByteBuffer.allocate(1);
    }

    // get the field position to write in the record
    this.pos = parameters.getInteger(FIELD_POS, 0);
    if (this.pos < 0) {
      throw new RuntimeException(
          "Illegal configuration value for the target position: " + this.pos);
    }
  }
Пример #6
0
  /**
   * Entry point for the program.
   *
   * @param args arguments from the command line
   * @throws IOException
   */
  @SuppressWarnings("static-access")
  public static void main(String[] args) throws IOException {
    Option configDirOpt =
        OptionBuilder.withArgName("config directory")
            .hasArg()
            .withDescription("Specify configuration directory.")
            .create("configDir");
    // tempDir option is used by the YARN client.
    Option tempDir =
        OptionBuilder.withArgName("temporary directory (overwrites configured option)")
            .hasArg()
            .withDescription("Specify temporary directory.")
            .create(ARG_CONF_DIR);
    configDirOpt.setRequired(true);
    tempDir.setRequired(false);
    Options options = new Options();
    options.addOption(configDirOpt);
    options.addOption(tempDir);

    CommandLineParser parser = new GnuParser();
    CommandLine line = null;
    try {
      line = parser.parse(options, args);
    } catch (ParseException e) {
      System.err.println("CLI Parsing failed. Reason: " + e.getMessage());
      System.exit(FAILURE_RETURN_CODE);
    }

    String configDir = line.getOptionValue(configDirOpt.getOpt(), null);
    String tempDirVal = line.getOptionValue(tempDir.getOpt(), null);

    // First, try to load global configuration
    GlobalConfiguration.loadConfiguration(configDir);
    if (tempDirVal != null // the YARN TM runner has set a value for the temp dir
        // the configuration does not contain a temp direcory
        && GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null) == null) {
      Configuration c = GlobalConfiguration.getConfiguration();
      c.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tempDirVal);
      LOG.info("Setting temporary directory to " + tempDirVal);
      GlobalConfiguration.includeConfiguration(c);
    }
    System.err.println("Configuration " + GlobalConfiguration.getConfiguration());
    LOG.info("Current user " + UserGroupInformation.getCurrentUser().getShortUserName());

    {
      // log the available JVM memory
      long maxMemoryMiBytes = Runtime.getRuntime().maxMemory() >>> 20;
      LOG.info(
          "Starting TaskManager in a JVM with " + maxMemoryMiBytes + " MiBytes maximum heap size.");
    }

    // Create a new task manager object
    try {
      new TaskManager(ExecutionMode.CLUSTER);
    } catch (Exception e) {
      LOG.fatal("Taskmanager startup failed: " + e.getMessage(), e);
      System.exit(FAILURE_RETURN_CODE);
    }

    // park the main thread to keep the JVM alive (all other threads may be daemon threads)
    Object mon = new Object();
    synchronized (mon) {
      try {
        mon.wait();
      } catch (InterruptedException ex) {
      }
    }
  }
Пример #7
0
 public boolean usesConvergenceCriterion() {
   return config.getString(ITERATION_CONVERGENCE_CRITERION, null) != null;
 }