Esempio n. 1
0
    private TwoDimTable createScoringHistoryTable(KMeansModel.KMeansOutput output) {
      List<String> colHeaders = new ArrayList<>();
      List<String> colTypes = new ArrayList<>();
      List<String> colFormat = new ArrayList<>();
      colHeaders.add("Timestamp");
      colTypes.add("string");
      colFormat.add("%s");
      colHeaders.add("Duration");
      colTypes.add("string");
      colFormat.add("%s");
      colHeaders.add("Iteration");
      colTypes.add("long");
      colFormat.add("%d");
      colHeaders.add("Avg. Change of Std. Centroids");
      colTypes.add("double");
      colFormat.add("%.5f");
      colHeaders.add("Within Cluster Sum Of Squares");
      colTypes.add("double");
      colFormat.add("%.5f");

      final int rows = output._avg_centroids_chg.length;
      TwoDimTable table =
          new TwoDimTable(
              "Scoring History",
              null,
              new String[rows],
              colHeaders.toArray(new String[0]),
              colTypes.toArray(new String[0]),
              colFormat.toArray(new String[0]),
              "");
      int row = 0;
      for (int i = 0; i < rows; i++) {
        int col = 0;
        assert (row < table.getRowDim());
        assert (col < table.getColDim());
        DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
        table.set(row, col++, fmt.print(output._training_time_ms[i]));
        table.set(row, col++, PrettyPrint.msecs(output._training_time_ms[i] - _start_time, true));
        table.set(row, col++, i);
        table.set(row, col++, output._avg_centroids_chg[i]);
        table.set(row, col++, output._history_withinss[i]);
        row++;
      }
      return table;
    }
Esempio n. 2
0
  @Ignore
  @Test
  public void matrixVecTest() {
    int rows = 2048;
    int cols = 8192;
    int loops = 5;
    int warmup_loops = 5;
    long seed = 0x533D;
    float nnz_ratio_vec = 0.01f; // fraction of non-zeroes for vector
    float nnz_ratio_mat = 0.1f; // fraction of non-zeroes for matrix

    float[] a = new float[rows * cols];
    float[] x = new float[cols];
    float[] y = new float[rows];
    float[] res = new float[rows];
    byte[] bits = new byte[rows];

    for (int row = 0; row < rows; ++row) {
      y[row] = 0;
      res[row] = 0;
      bits[row] = (byte) ("abcdefghijklmnopqrstuvwxyz".toCharArray()[row % 26]);
    }
    Random rng = new Random(seed);
    for (int col = 0; col < cols; ++col)
      if (rng.nextFloat() < nnz_ratio_vec) x[col] = ((float) col) / cols;

    for (int row = 0; row < rows; ++row) {
      int off = row * cols;
      for (int col = 0; col < cols; ++col) {
        if (rng.nextFloat() < nnz_ratio_mat) a[off + col] = ((float) (row + col)) / cols;
      }
    }
    Storage.DenseRowMatrix dra = new Storage.DenseRowMatrix(a, rows, cols);
    Storage.DenseColMatrix dca = new Storage.DenseColMatrix(dra, rows, cols);
    Storage.SparseRowMatrix sra = new Storage.SparseRowMatrix(dra, rows, cols);
    Storage.SparseColMatrix sca = new Storage.SparseColMatrix(dca, rows, cols);
    Storage.DenseVector dx = new Storage.DenseVector(x);
    Storage.DenseVector dy = new Storage.DenseVector(y);
    Storage.DenseVector dres = new Storage.DenseVector(res);
    Storage.SparseVector sx = new Storage.SparseVector(x);

    /** warmup */
    System.out.println("warming up.");
    float sum = 0;
    for (int l = 0; l < warmup_loops; ++l) {
      gemv_naive(res, a, x, y, bits);
      sum += res[rows / 2];
    }
    for (int l = 0; l < warmup_loops; ++l) {
      gemv_naive(dres, dra, dx, dy, bits);
      sum += res[rows / 2];
    }
    for (int l = 0; l < warmup_loops; ++l) {
      gemv_row_optimized(res, a, x, y, bits);
      sum += res[rows / 2];
    }
    for (int l = 0; l < warmup_loops; ++l) {
      gemv(dres, dca, dx, dy, bits);
      sum += res[rows / 2];
    }
    for (int l = 0; l < warmup_loops; ++l) {
      gemv(dres, dra, sx, dy, bits);
      sum += res[rows / 2];
    }
    for (int l = 0; l < warmup_loops; ++l) {
      gemv(dres, dca, sx, dy, bits);
      sum += res[rows / 2];
    }
    for (int l = 0; l < warmup_loops; ++l) {
      gemv(dres, sra, sx, dy, bits);
      sum += res[rows / 2];
    }
    for (int l = 0; l < warmup_loops; ++l) {
      gemv(dres, sca, sx, dy, bits);
      sum += res[rows / 2];
    }

    /** naive version */
    System.out.println("\nstarting naive.");
    sum = 0;
    long start = System.currentTimeMillis();
    for (int l = 0; l < loops; ++l) {
      gemv_naive(res, a, x, y, bits);
      sum += res[rows / 2]; // do something useful
    }
    System.out.println("result: " + sum + " and " + ArrayUtils.sum(res));
    System.out.println(
        "naive time: " + PrettyPrint.msecs(System.currentTimeMillis() - start, true));

    System.out.println("\nstarting dense row * dense.");
    sum = 0;
    start = System.currentTimeMillis();
    for (int l = 0; l < loops; ++l) {
      gemv_naive(dres, dra, dx, dy, bits);
      sum += res[rows / 2]; // do something useful
    }
    System.out.println("result: " + sum + " and " + ArrayUtils.sum(res));
    System.out.println(
        "dense row * dense time: " + PrettyPrint.msecs(System.currentTimeMillis() - start, true));

    System.out.println("\nstarting optimized dense row * dense.");
    sum = 0;
    start = System.currentTimeMillis();
    for (int l = 0; l < loops; ++l) {
      gemv_row_optimized(res, a, x, y, bits);
      sum += res[rows / 2]; // do something useful
    }
    System.out.println("result: " + sum + " and " + ArrayUtils.sum(res));
    System.out.println(
        "optimized dense row * dense time: "
            + PrettyPrint.msecs(System.currentTimeMillis() - start, true));

    System.out.println("\nstarting dense col * dense.");
    sum = 0;
    start = System.currentTimeMillis();
    for (int l = 0; l < loops; ++l) {
      gemv(dres, dca, dx, dy, bits);
      sum += res[rows / 2]; // do something useful
    }
    System.out.println("result: " + sum + " and " + ArrayUtils.sum(res));
    System.out.println(
        "dense col * dense time: " + PrettyPrint.msecs(System.currentTimeMillis() - start, true));

    System.out.println("\nstarting dense row * sparse.");
    sum = 0;
    start = System.currentTimeMillis();
    for (int l = 0; l < loops; ++l) {
      gemv(dres, dra, sx, dy, bits);
      sum += res[rows / 2]; // do something useful
    }
    System.out.println("result: " + sum + " and " + ArrayUtils.sum(res));
    System.out.println(
        "dense row * sparse time: " + PrettyPrint.msecs(System.currentTimeMillis() - start, true));

    System.out.println("\nstarting dense col * sparse.");
    sum = 0;
    start = System.currentTimeMillis();
    for (int l = 0; l < loops; ++l) {
      gemv(dres, dca, sx, dy, bits);
      sum += res[rows / 2]; // do something useful
    }
    System.out.println("result: " + sum + " and " + ArrayUtils.sum(res));
    System.out.println(
        "dense col * sparse time: " + PrettyPrint.msecs(System.currentTimeMillis() - start, true));

    System.out.println("\nstarting sparse row * sparse.");
    sum = 0;
    start = System.currentTimeMillis();
    for (int l = 0; l < loops; ++l) {
      gemv(dres, sra, sx, dy, bits);
      sum += res[rows / 2]; // do something useful
    }
    System.out.println("result: " + sum + " and " + ArrayUtils.sum(res));
    System.out.println(
        "sparse row * sparse time: " + PrettyPrint.msecs(System.currentTimeMillis() - start, true));

    System.out.println("\nstarting sparse col * sparse.");
    sum = 0;
    start = System.currentTimeMillis();
    for (int l = 0; l < loops; ++l) {
      gemv(dres, sca, sx, dy, bits);
      sum += res[rows / 2]; // do something useful
    }
    System.out.println("result: " + sum + " and " + ArrayUtils.sum(res));
    System.out.println(
        "sparse col * sparse time: " + PrettyPrint.msecs(System.currentTimeMillis() - start, true));
  }
Esempio n. 3
0
    /**
     * Train a Deep Learning neural net model
     *
     * @param model Input model (e.g., from initModel(), or from a previous training run)
     * @return Trained model
     */
    public final DeepLearningModel trainModel(DeepLearningModel model) {
      Frame validScoreFrame = null;
      Frame train, trainScoreFrame;
      try {
        //      if (checkpoint == null && !quiet_mode) logStart(); //if checkpoint is given, some
        // Job's params might be uninitialized (but the restarted model's parameters are correct)
        if (model == null) {
          model = DKV.get(dest()).get();
        }
        Log.info(
            "Model category: "
                + (_parms._autoencoder
                    ? "Auto-Encoder"
                    : isClassifier() ? "Classification" : "Regression"));
        final long model_size = model.model_info().size();
        Log.info(
            "Number of model parameters (weights/biases): " + String.format("%,d", model_size));
        model.write_lock(_job);
        _job.update(0, "Setting up training data...");
        final DeepLearningParameters mp = model.model_info().get_params();

        // temporary frames of the same "name" as the orig _train/_valid (asking the parameter's
        // Key, not the actual frame)
        // Note: don't put into DKV or they would overwrite the _train/_valid frames!
        Frame tra_fr = new Frame(mp._train, _train.names(), _train.vecs());
        Frame val_fr = _valid != null ? new Frame(mp._valid, _valid.names(), _valid.vecs()) : null;

        train = tra_fr;
        if (model._output.isClassifier() && mp._balance_classes) {
          _job.update(0, "Balancing class distribution of training data...");
          float[] trainSamplingFactors =
              new float
                  [train
                      .lastVec()
                      .domain()
                      .length]; // leave initialized to 0 -> will be filled up below
          if (mp._class_sampling_factors != null) {
            if (mp._class_sampling_factors.length != train.lastVec().domain().length)
              throw new IllegalArgumentException(
                  "class_sampling_factors must have "
                      + train.lastVec().domain().length
                      + " elements");
            trainSamplingFactors =
                mp._class_sampling_factors.clone(); // clone: don't modify the original
          }
          train =
              sampleFrameStratified(
                  train,
                  train.lastVec(),
                  train.vec(model._output.weightsName()),
                  trainSamplingFactors,
                  (long) (mp._max_after_balance_size * train.numRows()),
                  mp._seed,
                  true,
                  false);
          Vec l = train.lastVec();
          Vec w = train.vec(model._output.weightsName());
          MRUtils.ClassDist cd = new MRUtils.ClassDist(l);
          model._output._modelClassDist =
              _weights != null ? cd.doAll(l, w).rel_dist() : cd.doAll(l).rel_dist();
        }
        model.training_rows = train.numRows();
        if (_weights != null && _weights.min() == 0 && _weights.max() == 1 && _weights.isInt()) {
          model.training_rows = Math.round(train.numRows() * _weights.mean());
          Log.warn(
              "Not counting "
                  + (train.numRows() - model.training_rows)
                  + " rows with weight=0 towards an epoch.");
        }
        Log.info("One epoch corresponds to " + model.training_rows + " training data rows.");
        trainScoreFrame =
            sampleFrame(
                train,
                mp._score_training_samples,
                mp._seed); // training scoring dataset is always sampled uniformly from the training
                           // dataset
        if (trainScoreFrame != train) Scope.track(trainScoreFrame);

        if (!_parms._quiet_mode)
          Log.info("Number of chunks of the training data: " + train.anyVec().nChunks());
        if (val_fr != null) {
          model.validation_rows = val_fr.numRows();
          // validation scoring dataset can be sampled in multiple ways from the given validation
          // dataset
          if (model._output.isClassifier()
              && mp._balance_classes
              && mp._score_validation_sampling
                  == DeepLearningParameters.ClassSamplingMethod.Stratified) {
            _job.update(0, "Sampling validation data (stratified)...");
            validScoreFrame =
                sampleFrameStratified(
                    val_fr,
                    val_fr.lastVec(),
                    val_fr.vec(model._output.weightsName()),
                    null,
                    mp._score_validation_samples > 0
                        ? mp._score_validation_samples
                        : val_fr.numRows(),
                    mp._seed + 1,
                    false /* no oversampling */,
                    false);
          } else {
            _job.update(0, "Sampling validation data...");
            validScoreFrame = sampleFrame(val_fr, mp._score_validation_samples, mp._seed + 1);
            if (validScoreFrame != val_fr) Scope.track(validScoreFrame);
          }
          if (!_parms._quiet_mode)
            Log.info(
                "Number of chunks of the validation data: " + validScoreFrame.anyVec().nChunks());
        }

        // Set train_samples_per_iteration size (cannot be done earlier since this depends on
        // whether stratified sampling is done)
        model.actual_train_samples_per_iteration =
            computeTrainSamplesPerIteration(mp, model.training_rows, model);
        // Determine whether shuffling is enforced
        if (mp._replicate_training_data
            && (model.actual_train_samples_per_iteration
                == model.training_rows * (mp._single_node_mode ? 1 : H2O.CLOUD.size()))
            && !mp._shuffle_training_data
            && H2O.CLOUD.size() > 1
            && !mp._reproducible) {
          if (!mp._quiet_mode)
            Log.info(
                "Enabling training data shuffling, because all nodes train on the full dataset (replicated training data).");
          mp._shuffle_training_data = true;
        }
        if (!mp._shuffle_training_data
            && model.actual_train_samples_per_iteration == model.training_rows
            && train.anyVec().nChunks() == 1) {
          if (!mp._quiet_mode)
            Log.info(
                "Enabling training data shuffling to avoid training rows in the same order over and over (no Hogwild since there's only 1 chunk).");
          mp._shuffle_training_data = true;
        }

        //        if (!mp._quiet_mode) Log.info("Initial model:\n" + model.model_info());
        long now = System.currentTimeMillis();
        model._timeLastIterationEnter = now;
        if (_parms._autoencoder) {
          _job.update(0, "Scoring null model of autoencoder...");
          if (!mp._quiet_mode) Log.info("Scoring the null model of the autoencoder.");
          model.doScoring(
              trainScoreFrame,
              validScoreFrame,
              _job._key,
              0,
              false); // get the null model reconstruction error
        }
        // put the initial version of the model into DKV
        model.update(_job);
        model.total_setup_time_ms += now - _job.start_time();
        Log.info("Total setup time: " + PrettyPrint.msecs(model.total_setup_time_ms, true));
        Log.info("Starting to train the Deep Learning model.");
        _job.update(0, "Training...");

        // main loop
        for (; ; ) {
          model.iterations++;
          model.set_model_info(
              mp._epochs == 0
                  ? model.model_info()
                  : H2O.CLOUD.size() > 1 && mp._replicate_training_data
                      ? (mp._single_node_mode
                          ? new DeepLearningTask2(
                                  _job._key,
                                  train,
                                  model.model_info(),
                                  rowFraction(train, mp, model),
                                  model.iterations)
                              .doAll(Key.make(H2O.SELF))
                              .model_info()
                          : // replicated data + single node mode
                          new DeepLearningTask2(
                                  _job._key,
                                  train,
                                  model.model_info(),
                                  rowFraction(train, mp, model),
                                  model.iterations)
                              .doAllNodes()
                              .model_info())
                      : // replicated data + multi-node mode
                      new DeepLearningTask(
                              _job._key,
                              model.model_info(),
                              rowFraction(train, mp, model),
                              model.iterations)
                          .doAll(train)
                          .model_info()); // distributed data (always in multi-node mode)
          if (stop_requested() && !timeout()) break; // cancellation
          if (!model.doScoring(
              trainScoreFrame, validScoreFrame, _job._key, model.iterations, false))
            break; // finished training (or early stopping or convergence)
          if (timeout()) break; // stop after scoring
        }

        // replace the model with the best model so far (if it's better)
        if (!stop_requested()
            && _parms._overwrite_with_best_model
            && model.actual_best_model_key != null
            && _parms._nfolds == 0) {
          DeepLearningModel best_model = DKV.getGet(model.actual_best_model_key);
          if (best_model != null
              && best_model.loss() < model.loss()
              && Arrays.equals(best_model.model_info().units, model.model_info().units)) {
            if (!_parms._quiet_mode)
              Log.info("Setting the model to be the best model so far (based on scoring history).");
            DeepLearningModelInfo mi = best_model.model_info().deep_clone();
            // Don't cheat - count full amount of training samples, since that's the amount of
            // training it took to train (without finding anything better)
            mi.set_processed_global(model.model_info().get_processed_global());
            mi.set_processed_local(model.model_info().get_processed_local());
            model.set_model_info(mi);
            model.update(_job);
            model.doScoring(trainScoreFrame, validScoreFrame, _job._key, model.iterations, true);
            assert (best_model.loss() == model.loss());
          }
        }
        // store coefficient names for future use
        // possibly change
        model.model_info().data_info().coefNames();
        if (!_parms._quiet_mode) {
          Log.info(
              "==============================================================================================================================================================================");
          if (stop_requested()) {
            Log.info("Deep Learning model training was interrupted.");
          } else {
            Log.info("Finished training the Deep Learning model.");
            Log.info(model);
          }
          Log.info(
              "==============================================================================================================================================================================");
        }
      } finally {
        if (model != null) {
          model.deleteElasticAverageModels();
          model.unlock(_job);
          if (model.actual_best_model_key != null) {
            assert (model.actual_best_model_key != model._key);
            DKV.remove(model.actual_best_model_key);
          }
        }
      }
      return model;
    }
Esempio n. 4
0
  private TwoDimTable createScoringHistoryTable(SharedTreeModel.SharedTreeOutput _output) {
    List<String> colHeaders = new ArrayList<>();
    List<String> colTypes = new ArrayList<>();
    List<String> colFormat = new ArrayList<>();
    colHeaders.add("Timestamp");
    colTypes.add("string");
    colFormat.add("%s");
    colHeaders.add("Duration");
    colTypes.add("string");
    colFormat.add("%s");
    colHeaders.add("Number of Trees");
    colTypes.add("long");
    colFormat.add("%d");
    colHeaders.add("Training MSE");
    colTypes.add("double");
    colFormat.add("%.5f");
    if (_output.isClassifier()) {
      colHeaders.add("Training LogLoss");
      colTypes.add("double");
      colFormat.add("%.5f");
    }
    if (_output.getModelCategory() == ModelCategory.Binomial) {
      colHeaders.add("Training AUC");
      colTypes.add("double");
      colFormat.add("%.5f");
    }
    if (_output.getModelCategory() == ModelCategory.Binomial
        || _output.getModelCategory() == ModelCategory.Multinomial) {
      colHeaders.add("Training Classification Error");
      colTypes.add("double");
      colFormat.add("%.5f");
    }

    if (valid() != null) {
      colHeaders.add("Validation MSE");
      colTypes.add("double");
      colFormat.add("%.5f");
      if (_output.isClassifier()) {
        colHeaders.add("Validation LogLoss");
        colTypes.add("double");
        colFormat.add("%.5f");
      }
      if (_output.getModelCategory() == ModelCategory.Binomial) {
        colHeaders.add("Validation AUC");
        colTypes.add("double");
        colFormat.add("%.5f");
      }
      if (_output.isClassifier()) {
        colHeaders.add("Validation Classification Error");
        colTypes.add("double");
        colFormat.add("%.5f");
      }
    }

    int rows = 0;
    for (int i = 1; i < _output._scored_train.length; i++) {
      if (!Double.isNaN(_output._scored_train[i]._mse)) ++rows;
    }
    TwoDimTable table =
        new TwoDimTable(
            "Scoring History",
            null,
            new String[rows],
            colHeaders.toArray(new String[0]),
            colTypes.toArray(new String[0]),
            colFormat.toArray(new String[0]),
            "");
    int row = 0;
    for (int i = 1; i < _output._scored_train.length; i++) {
      if (Double.isNaN(_output._scored_train[i]._mse)) continue;
      int col = 0;
      assert (row < table.getRowDim());
      assert (col < table.getColDim());
      DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
      table.set(row, col++, fmt.print(_output._training_time_ms[i]));
      table.set(row, col++, PrettyPrint.msecs(_output._training_time_ms[i] - _start_time, true));
      table.set(row, col++, i);
      ScoreKeeper st = _output._scored_train[i];
      table.set(row, col++, st._mse);
      if (_output.isClassifier()) table.set(row, col++, st._logloss);
      if (_output.getModelCategory() == ModelCategory.Binomial) table.set(row, col++, st._AUC);
      if (_output.isClassifier()) table.set(row, col++, st._classError);

      if (_valid != null) {
        st = _output._scored_valid[i];
        table.set(row, col++, st._mse);
        if (_output.isClassifier()) table.set(row, col++, st._logloss);
        if (_output.getModelCategory() == ModelCategory.Binomial) table.set(row, col++, st._AUC);
        if (_output.isClassifier()) table.set(row, col++, st._classError);
      }
      row++;
    }
    return table;
  }