private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
TotSS(double[] means, double[] mults, int[] modes, String[][] isCats, int[] card) {
_means = means;
_mults = mults;
_modes = modes;
_tss = 0;
_isCats = isCats;
_card = card;
// Mean of numeric col is zero when standardized
_gc = mults != null ? new double[means.length] : Arrays.copyOf(means, means.length);
for (int i = 0; i < means.length; i++) {
if (isCats[i] != null)
_gc[i] =
Math.min(
Math.round(means[i]),
_card[i] - 1); // TODO: Should set to majority class of categorical column
}
}
/**
* Main constructor
*
* @param params Model parameters
* @param dinfo Data Info
* @param nClasses number of classes (1 for regression, 0 for autoencoder)
* @param train User-given training data frame, prepared by AdaptTestTrain
* @param valid User-specified validation data frame, prepared by AdaptTestTrain
*/
public DeepLearningModelInfo(
final DeepLearningParameters params,
final DataInfo dinfo,
int nClasses,
Frame train,
Frame valid) {
_classification = nClasses > 1;
_train = train;
_valid = valid;
data_info = dinfo;
parameters =
(DeepLearningParameters) params.clone(); // make a copy, don't change model's parameters
DeepLearningParameters.Sanity.modifyParms(
parameters, parameters, nClasses); // sanitize the model_info's parameters
final int num_input = dinfo.fullN();
final int num_output =
get_params()._autoencoder
? num_input
: (_classification ? train.lastVec().cardinality() : 1);
if (!get_params()._autoencoder) assert (num_output == nClasses);
_saw_missing_cats = dinfo._cats > 0 ? new boolean[data_info._cats] : null;
assert (num_input > 0);
assert (num_output > 0);
if (has_momenta() && adaDelta())
throw new IllegalArgumentException(
"Cannot have non-zero momentum and adaptive rate at the same time.");
final int layers = get_params()._hidden.length;
// units (# neurons for each layer)
units = new int[layers + 2];
if (get_params()._max_categorical_features <= Integer.MAX_VALUE - dinfo._nums)
units[0] = Math.min(dinfo._nums + get_params()._max_categorical_features, num_input);
else units[0] = num_input;
System.arraycopy(get_params()._hidden, 0, units, 1, layers);
units[layers + 1] = num_output;
boolean printLevels = units[0] > 1000L;
boolean warn = units[0] > 100000L;
if (printLevels) {
final String[][] domains = dinfo._adaptedFrame.domains();
int[] levels = new int[domains.length];
for (int i = 0; i < levels.length; ++i) {
levels[i] = domains[i] != null ? domains[i].length : 0;
}
Arrays.sort(levels);
if (warn) {
Log.warn(
"===================================================================================================================================");
Log.warn(
num_input
+ " input features"
+ (dinfo._cats > 0 ? " (after categorical one-hot encoding)" : "")
+ ". Can be slow and require a lot of memory.");
}
if (levels[levels.length - 1] > 0) {
int levelcutoff = levels[levels.length - 1 - Math.min(10, levels.length - 1)];
int count = 0;
for (int i = 0;
i < dinfo._adaptedFrame.numCols() - (get_params()._autoencoder ? 0 : 1) && count < 10;
++i) {
if (dinfo._adaptedFrame.domains()[i] != null
&& dinfo._adaptedFrame.domains()[i].length >= levelcutoff) {
if (warn) {
Log.warn(
"Categorical feature '"
+ dinfo._adaptedFrame._names[i]
+ "' has cardinality "
+ dinfo._adaptedFrame.domains()[i].length
+ ".");
} else {
Log.info(
"Categorical feature '"
+ dinfo._adaptedFrame._names[i]
+ "' has cardinality "
+ dinfo._adaptedFrame.domains()[i].length
+ ".");
}
}
count++;
}
}
if (warn) {
Log.warn("Suggestions:");
Log.warn(" *) Limit the size of the first hidden layer");
if (dinfo._cats > 0) {
Log.warn(
" *) Limit the total number of one-hot encoded features with the parameter 'max_categorical_features'");
Log.warn(
" *) Run h2o.interaction(...,pairwise=F) on high-cardinality categorical columns to limit the factor count, see http://learn.h2o.ai");
}
Log.warn(
"===================================================================================================================================");
}
}
// weights (to connect layers)
dense_row_weights = new Storage.DenseRowMatrix[layers + 1];
dense_col_weights = new Storage.DenseColMatrix[layers + 1];
// decide format of weight matrices row-major or col-major
if (get_params()._col_major)
dense_col_weights[0] = new Storage.DenseColMatrix(units[1], units[0]);
else dense_row_weights[0] = new Storage.DenseRowMatrix(units[1], units[0]);
for (int i = 1; i <= layers; ++i)
dense_row_weights[i] = new Storage.DenseRowMatrix(units[i + 1] /*rows*/, units[i] /*cols*/);
// biases (only for hidden layers and output layer)
biases = new Storage.DenseVector[layers + 1];
for (int i = 0; i <= layers; ++i) biases[i] = new Storage.DenseVector(units[i + 1]);
// average activation (only for hidden layers)
if (get_params()._autoencoder && get_params()._sparsity_beta > 0) {
avg_activations = new Storage.DenseVector[layers];
mean_a = new float[layers];
for (int i = 0; i < layers; ++i) avg_activations[i] = new Storage.DenseVector(units[i + 1]);
}
allocateHelperArrays();
// for diagnostics
mean_rate = new float[units.length];
rms_rate = new float[units.length];
mean_bias = new float[units.length];
rms_bias = new float[units.length];
mean_weight = new float[units.length];
rms_weight = new float[units.length];
}
// Progress reporting for the job/progress page
@Override
public float progress() {
return Math.min(1f, _iteration / (float) _maxIter);
}
// public final double[] rproxgrad_x(double[] u, double alpha) { return rproxgrad(u, alpha,
// _gamma_x, _regularization_x, RandomUtils.getRNG(_seed)); }
// public final double[] rproxgrad_y(double[] u, double alpha) { return rproxgrad(u, alpha,
// _gamma_y, _regularization_y, RandomUtils.getRNG(_seed)); }
static double[] rproxgrad(
double[] u, double alpha, double gamma, Regularizer regularization, Random rand) {
if (u == null || alpha == 0 || gamma == 0) return u;
double[] v = new double[u.length];
switch (regularization) {
case None:
return u;
case Quadratic:
for (int i = 0; i < u.length; i++) v[i] = u[i] / (1 + 2 * alpha * gamma);
return v;
case L2:
// Proof uses Moreau decomposition; see section 6.5.1 of Parikh and Boyd
// https://web.stanford.edu/~boyd/papers/pdf/prox_algs.pdf
double weight = 1 - alpha * gamma / ArrayUtils.l2norm(u);
if (weight < 0) return v; // Zero vector
for (int i = 0; i < u.length; i++) v[i] = weight * u[i];
return v;
case L1:
for (int i = 0; i < u.length; i++)
v[i] = Math.max(u[i] - alpha * gamma, 0) + Math.min(u[i] + alpha * gamma, 0);
return v;
case NonNegative:
for (int i = 0; i < u.length; i++) v[i] = Math.max(u[i], 0);
return v;
case OneSparse:
int idx = ArrayUtils.maxIndex(u, rand);
v[idx] = u[idx] > 0 ? u[idx] : 1e-6;
return v;
case UnitOneSparse:
idx = ArrayUtils.maxIndex(u, rand);
v[idx] = 1;
return v;
case Simplex:
// Proximal gradient algorithm by Chen and Ye in http://arxiv.org/pdf/1101.6081v2.pdf
// 1) Sort input vector u in ascending order: u[1] <= ... <= u[n]
int n = u.length;
int[] idxs = new int[n];
for (int i = 0; i < n; i++) idxs[i] = i;
ArrayUtils.sort(idxs, u);
// 2) Calculate cumulative sum of u in descending order
// cumsum(u) = (..., u[n-2]+u[n-1]+u[n], u[n-1]+u[n], u[n])
double[] ucsum = new double[n];
ucsum[n - 1] = u[idxs[n - 1]];
for (int i = n - 2; i >= 0; i--) ucsum[i] = ucsum[i + 1] + u[idxs[i]];
// 3) Let t_i = (\sum_{j=i+1}^n u[j] - 1)/(n - i)
// For i = n-1,...,1, set optimal t* to first t_i >= u[i]
double t = (ucsum[0] - 1) / n; // Default t* = (\sum_{j=1}^n u[j] - 1)/n
for (int i = n - 1; i >= 1; i--) {
double tmp = (ucsum[i] - 1) / (n - i);
if (tmp >= u[idxs[i - 1]]) {
t = tmp;
break;
}
}
// 4) Return max(u - t*, 0) as projection of u onto simplex
double[] x = new double[u.length];
for (int i = 0; i < u.length; i++) x[i] = Math.max(u[i] - t, 0);
return x;
default:
throw new RuntimeException("Unknown regularization function " + regularization);
}
}
@Override
public float progress() {
if (_status == Status.Done) return 1.0f;
return Math.min(0.99f, (float) ((double) _count / (double) _nchunks));
}