private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) { try { if (fs == null) return; for (FileStatus file : fs.listStatus(p)) { Path pfs = file.getPath(); if (file.isDir()) { addFolder(fs, pfs, succeeded, failed); } else { Key k = Key.make(pfs.toString()); long size = file.getLen(); Value val = null; if (pfs.getName().endsWith(Extensions.JSON)) { JsonParser parser = new JsonParser(); JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject(); JsonElement v = json.get(Constants.VERSION); if (v == null) throw new RuntimeException("Missing version"); JsonElement type = json.get(Constants.TYPE); if (type == null) throw new RuntimeException("Missing type"); Class c = Class.forName(type.getAsString()); OldModel model = (OldModel) c.newInstance(); model.fromJson(json); } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file? FSDataInputStream s = fs.open(pfs); int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg byte[] mem = MemoryManager.malloc1(sz); s.readFully(mem); // Convert to a ValueArray (hope it fits in 1Meg!) ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem)); val = new Value(k, ary, Value.HDFS); } else if (size >= 2 * ValueArray.CHUNK_SZ) { val = new Value( k, new ValueArray(k, size), Value.HDFS); // ValueArray byte wrapper over a large file } else { val = new Value(k, (int) size, Value.HDFS); // Plain Value val.setdsk(); } DKV.put(k, val); Log.info("PersistHdfs: DKV.put(" + k + ")"); JsonObject o = new JsonObject(); o.addProperty(Constants.KEY, k.toString()); o.addProperty(Constants.FILE, pfs.toString()); o.addProperty(Constants.VALUE_SIZE, file.getLen()); succeeded.add(o); } } } catch (Exception e) { Log.err(e); JsonObject o = new JsonObject(); o.addProperty(Constants.FILE, p.toString()); o.addProperty(Constants.ERROR, e.getMessage()); failed.add(o); } }
@Override public final double score(double[] data) { int j = 0; for (int i = 0; i < _xCols.length; ++i) _row[j++] = (_catMap == null || _catMap[i] == null) ? data[_xCols[i]] : translateCat(i, data[_xCols[i]]); return M.score0(_row); }
@Override public double score(ValueArray data, AutoBuffer ab, int row) { int j = 0; for (int i = 0; i < _xCols.length; ++i) _row[j++] = data.isNA(ab, row, _xCols[i]) ? Double.NaN : (_catMap == null || _catMap[i] == null) ? data.datad(ab, row, _xCols[i]) : translateCat(i, (int) data.data(ab, row, _xCols[i])); return M.score0(_row); }
@Override public final OldModel adapt(String[] cols) { return M.adapt(cols); }
// keep only one adaptor layer! (just in case there would be multiple adapt calls...) @Override public final OldModel adapt(ValueArray ary) { return M.adapt(ary); }
@Override public JsonObject toJson() { return M.toJson(); }