Esempio n. 1
0
 private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
   try {
     if (fs == null) return;
     for (FileStatus file : fs.listStatus(p)) {
       Path pfs = file.getPath();
       if (file.isDir()) {
         addFolder(fs, pfs, succeeded, failed);
       } else {
         Key k = Key.make(pfs.toString());
         long size = file.getLen();
         Value val = null;
         if (pfs.getName().endsWith(Extensions.JSON)) {
           JsonParser parser = new JsonParser();
           JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
           JsonElement v = json.get(Constants.VERSION);
           if (v == null) throw new RuntimeException("Missing version");
           JsonElement type = json.get(Constants.TYPE);
           if (type == null) throw new RuntimeException("Missing type");
           Class c = Class.forName(type.getAsString());
           OldModel model = (OldModel) c.newInstance();
           model.fromJson(json);
         } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
           FSDataInputStream s = fs.open(pfs);
           int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
           byte[] mem = MemoryManager.malloc1(sz);
           s.readFully(mem);
           // Convert to a ValueArray (hope it fits in 1Meg!)
           ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
           val = new Value(k, ary, Value.HDFS);
         } else if (size >= 2 * ValueArray.CHUNK_SZ) {
           val =
               new Value(
                   k,
                   new ValueArray(k, size),
                   Value.HDFS); // ValueArray byte wrapper over a large file
         } else {
           val = new Value(k, (int) size, Value.HDFS); // Plain Value
           val.setdsk();
         }
         DKV.put(k, val);
         Log.info("PersistHdfs: DKV.put(" + k + ")");
         JsonObject o = new JsonObject();
         o.addProperty(Constants.KEY, k.toString());
         o.addProperty(Constants.FILE, pfs.toString());
         o.addProperty(Constants.VALUE_SIZE, file.getLen());
         succeeded.add(o);
       }
     }
   } catch (Exception e) {
     Log.err(e);
     JsonObject o = new JsonObject();
     o.addProperty(Constants.FILE, p.toString());
     o.addProperty(Constants.ERROR, e.getMessage());
     failed.add(o);
   }
 }
Esempio n. 2
0
 @Override
 public final double score(double[] data) {
   int j = 0;
   for (int i = 0; i < _xCols.length; ++i)
     _row[j++] =
         (_catMap == null || _catMap[i] == null)
             ? data[_xCols[i]]
             : translateCat(i, data[_xCols[i]]);
   return M.score0(_row);
 }
Esempio n. 3
0
 @Override
 public double score(ValueArray data, AutoBuffer ab, int row) {
   int j = 0;
   for (int i = 0; i < _xCols.length; ++i)
     _row[j++] =
         data.isNA(ab, row, _xCols[i])
             ? Double.NaN
             : (_catMap == null || _catMap[i] == null)
                 ? data.datad(ab, row, _xCols[i])
                 : translateCat(i, (int) data.data(ab, row, _xCols[i]));
   return M.score0(_row);
 }
Esempio n. 4
0
 @Override
 public final OldModel adapt(String[] cols) {
   return M.adapt(cols);
 }
Esempio n. 5
0
 // keep only one adaptor layer! (just in case there would be multiple adapt calls...)
 @Override
 public final OldModel adapt(ValueArray ary) {
   return M.adapt(ary);
 }
Esempio n. 6
0
 @Override
 public JsonObject toJson() {
   return M.toJson();
 }