コード例 #1
0
ファイル: SharedTree.java プロジェクト: herberteuler/h2o-3
  @Override
  protected void checkMemoryFootPrint() {
    if (_model._output._ntrees == 0) return;
    int trees_so_far = _model._output._ntrees; // existing trees
    long model_mem_size =
        new ComputeModelSize(trees_so_far, _model._output._treeKeys).doAllNodes()._model_mem_size;
    _model._output._treeStats._byte_size = model_mem_size;
    double avg_tree_mem_size = (double) model_mem_size / trees_so_far;
    Log.debug(
        "Average tree size (for all classes): " + PrettyPrint.bytes((long) avg_tree_mem_size));

    // all the compressed trees are stored on the driver node
    long max_mem = H2O.SELF.get_max_mem();
    if (_parms._ntrees * avg_tree_mem_size > max_mem) {
      String msg =
          "The tree model will not fit in the driver node's memory ("
              + PrettyPrint.bytes((long) avg_tree_mem_size)
              + " per tree x "
              + _parms._ntrees
              + " > "
              + PrettyPrint.bytes(max_mem)
              + ") - try decreasing ntrees and/or max_depth or increasing min_rows!";
      error("_ntrees", msg);
      cancel(msg);
    }
  }
コード例 #2
0
ファイル: KMeans.java プロジェクト: huamichaelchen/h2o-3
 protected void checkMemoryFootPrint() {
   long mem_usage = 8 /*doubles*/ * _parms._k * _train.numCols() * (_parms._standardize ? 2 : 1);
   long max_mem = H2O.SELF._heartbeat.get_free_mem();
   if (mem_usage > max_mem) {
     String msg =
         "Centroids won't fit in the driver node's memory ("
             + PrettyPrint.bytes(mem_usage)
             + " > "
             + PrettyPrint.bytes(max_mem)
             + ") - try reducing the number of columns and/or the number of categorical factors.";
     error("_train", msg);
     cancel(msg);
   }
 }
コード例 #3
0
ファイル: PCA.java プロジェクト: liangexiang/h2o-3
 @Override
 protected void checkMemoryFootPrint() {
   HeartBeat hb = H2O.SELF._heartbeat;
   double p = _train.degreesOfFreedom();
   long mem_usage =
       (long)
           (hb._cpus_allowed
               * p
               * p
               * 8 /*doubles*/
               * Math.log((double) _train.lastVec().nChunks())
               / Math.log(2.)); // one gram per core
   long max_mem = hb.get_max_mem();
   if (mem_usage > max_mem) {
     String msg =
         "Gram matrices (one per thread) won't fit in the driver node's memory ("
             + PrettyPrint.bytes(mem_usage)
             + " > "
             + PrettyPrint.bytes(max_mem)
             + ") - try reducing the number of columns and/or the number of categorical factors.";
     error("_train", msg);
     cancel(msg);
   }
 }
コード例 #4
0
  public Job<Frame> execImpl() {
    if (integer_fraction + binary_fraction + categorical_fraction > 1)
      throw new IllegalArgumentException(
          "Integer, binary and categorical fractions must add up to <= 1.");
    if (Math.abs(missing_fraction) > 1)
      throw new IllegalArgumentException("Missing fraction must be between 0 and 1.");
    if (Math.abs(integer_fraction) > 1)
      throw new IllegalArgumentException("Integer fraction must be between 0 and 1.");
    if (Math.abs(binary_fraction) > 1)
      throw new IllegalArgumentException("Binary fraction must be between 0 and 1.");
    if (Math.abs(binary_ones_fraction) > 1)
      throw new IllegalArgumentException("Binary ones fraction must be between 0 and 1.");
    if (Math.abs(categorical_fraction) > 1)
      throw new IllegalArgumentException("Categorical fraction must be between 0 and 1.");
    if (categorical_fraction > 0 && factors <= 1)
      throw new IllegalArgumentException("Factors must be larger than 2 for categorical data.");
    if (response_factors < 1)
      throw new IllegalArgumentException(
          "Response factors must be either 1 (real-valued response), or >=2 (factor levels).");
    if (response_factors > 1024)
      throw new IllegalArgumentException("Response factors must be <= 1024.");
    if (factors > 1000000)
      throw new IllegalArgumentException("Number of factors must be <= 1,000,000).");
    if (cols <= 0 || rows <= 0)
      throw new IllegalArgumentException("Must have number of rows > 0 and columns > 1.");

    // estimate byte size of the frame
    double byte_estimate =
        randomize
            ? rows
                    * cols
                    * (binary_fraction * 1. / 8 // bits
                        + categorical_fraction * (factors < 128 ? 1 : factors < 32768 ? 2 : 4)
                        + integer_fraction
                            * (integer_range < 128
                                ? 1
                                : integer_range < 32768 ? 2 : integer_range < (1 << 31) ? 4 : 8)
                        + (1 - integer_fraction - binary_fraction - categorical_fraction)
                            * 8) // reals
                + rows * 1 // response is
            : 0; // all constants - should be small

    if (byte_estimate > H2O.CLOUD._memary[0].get_max_mem() * H2O.CLOUD.size())
      throw new IllegalArgumentException(
          "Frame is expected to require "
              + PrettyPrint.bytes((long) byte_estimate)
              + ", won't fit into H2O's memory.");

    if (!randomize) {
      if (integer_fraction != 0 || categorical_fraction != 0)
        throw new IllegalArgumentException(
            "Cannot have integer or categorical fractions > 0 unless randomize=true.");
    } else {
      if (value != 0)
        throw new IllegalArgumentException(
            "Cannot set data to a constant value if randomize=true.");
    }
    if (_dest == null) throw new IllegalArgumentException("Destination key cannot be null.");

    FrameCreator fc = new FrameCreator(this, this._key);
    start(fc, fc.nChunks() * 5);
    return this;
  }
コード例 #5
0
  /**
   * Create a summary table
   *
   * @return
   */
  TwoDimTable createSummaryTable() {
    Neurons[] neurons = DeepLearningTask.makeNeuronsForTesting(this);
    long byte_size = new AutoBuffer().put(this).buf().length;
    TwoDimTable table =
        new TwoDimTable(
            "Status of Neuron Layers",
            (get_params()._diagnostics ? "" : "diagnostics disabled, ")
                + (!get_params()._autoencoder ? ("predicting " + _train.lastVecName() + ", ") : "")
                + (get_params()._autoencoder
                    ? "auto-encoder"
                    : _classification
                        ? (units[units.length - 1] + "-class classification")
                        : "regression")
                + ", "
                + get_params()._distribution
                + " distribution, "
                + get_params()._loss
                + " loss, "
                + String.format("%,d", size())
                + " weights/biases, "
                + PrettyPrint.bytes(byte_size)
                + ", "
                + String.format("%,d", get_processed_global())
                + " training samples, "
                + "mini-batch size "
                + String.format("%,d", get_params()._mini_batch_size),
            new String[neurons.length],
            new String[] {
              "Layer",
              "Units",
              "Type",
              "Dropout",
              "L1",
              "L2",
              "Mean Rate",
              "Rate RMS",
              "Momentum",
              "Mean Weight",
              "Weight RMS",
              "Mean Bias",
              "Bias RMS"
            },
            new String[] {
              "int", "int", "string", "double", "double", "double", "double", "double", "double",
              "double", "double", "double", "double"
            },
            new String[] {
              "%d",
              "%d",
              "%s",
              "%2.2f %%",
              "%5f",
              "%5f",
              "%5f",
              "%5f",
              "%5f",
              "%5f",
              "%5f",
              "%5f",
              "%5f"
            },
            "");

    for (int i = 0; i < neurons.length; ++i) {
      table.set(i, 0, i + 1);
      table.set(i, 1, neurons[i].units);
      table.set(i, 2, neurons[i].getClass().getSimpleName());

      if (i == 0) {
        table.set(i, 3, neurons[i].params._input_dropout_ratio * 100);
        continue;
      } else if (i < neurons.length - 1) {
        if (neurons[i].params._hidden_dropout_ratios == null) {
          table.set(i, 3, 0);
        } else {
          table.set(i, 3, neurons[i].params._hidden_dropout_ratios[i - 1] * 100);
        }
      }
      table.set(i, 4, neurons[i].params._l1);
      table.set(i, 5, neurons[i].params._l2);
      table.set(
          i,
          6,
          (get_params()._adaptive_rate ? mean_rate[i] : neurons[i].rate(get_processed_total())));
      table.set(i, 7, (get_params()._adaptive_rate ? rms_rate[i] : 0));
      table.set(i, 8, get_params()._adaptive_rate ? 0 : neurons[i].momentum(get_processed_total()));
      table.set(i, 9, mean_weight[i]);
      table.set(i, 10, rms_weight[i]);
      table.set(i, 11, mean_bias[i]);
      table.set(i, 12, rms_bias[i]);
    }
    summaryTable = table;
    return summaryTable;
  }