Example #1
0
 /**
  * Annotate the number of columns and rows of the training data set in the job parameter JSON
  *
  * @return JsonObject annotated with num_cols and num_rows of the training data set
  */
 @Override
 public JsonObject toJSON() {
   JsonObject jo = super.toJSON();
   if (source != null) {
     jo.getAsJsonObject("source").addProperty("num_cols", source.numCols());
     jo.getAsJsonObject("source").addProperty("num_rows", source.numRows());
   }
   return jo;
 }
Example #2
0
 /**
  * Annotate the number of columns and rows of the validation data set in the job parameter JSON
  *
  * @return JsonObject annotated with num_cols and num_rows of the validation data set
  */
 @Override
 public JsonObject toJSON() {
   JsonObject jo = super.toJSON();
   if (validation != null) {
     jo.getAsJsonObject("validation").addProperty("num_cols", validation.numCols());
     jo.getAsJsonObject("validation").addProperty("num_rows", validation.numRows());
   }
   return jo;
 }
Example #3
0
 private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
   try {
     if (fs == null) return;
     for (FileStatus file : fs.listStatus(p)) {
       Path pfs = file.getPath();
       if (file.isDir()) {
         addFolder(fs, pfs, succeeded, failed);
       } else {
         Key k = Key.make(pfs.toString());
         long size = file.getLen();
         Value val = null;
         if (pfs.getName().endsWith(Extensions.JSON)) {
           JsonParser parser = new JsonParser();
           JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
           JsonElement v = json.get(Constants.VERSION);
           if (v == null) throw new RuntimeException("Missing version");
           JsonElement type = json.get(Constants.TYPE);
           if (type == null) throw new RuntimeException("Missing type");
           Class c = Class.forName(type.getAsString());
           OldModel model = (OldModel) c.newInstance();
           model.fromJson(json);
         } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
           FSDataInputStream s = fs.open(pfs);
           int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
           byte[] mem = MemoryManager.malloc1(sz);
           s.readFully(mem);
           // Convert to a ValueArray (hope it fits in 1Meg!)
           ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
           val = new Value(k, ary, Value.HDFS);
         } else if (size >= 2 * ValueArray.CHUNK_SZ) {
           val =
               new Value(
                   k,
                   new ValueArray(k, size),
                   Value.HDFS); // ValueArray byte wrapper over a large file
         } else {
           val = new Value(k, (int) size, Value.HDFS); // Plain Value
           val.setdsk();
         }
         DKV.put(k, val);
         Log.info("PersistHdfs: DKV.put(" + k + ")");
         JsonObject o = new JsonObject();
         o.addProperty(Constants.KEY, k.toString());
         o.addProperty(Constants.FILE, pfs.toString());
         o.addProperty(Constants.VALUE_SIZE, file.getLen());
         succeeded.add(o);
       }
     }
   } catch (Exception e) {
     Log.err(e);
     JsonObject o = new JsonObject();
     o.addProperty(Constants.FILE, p.toString());
     o.addProperty(Constants.ERROR, e.getMessage());
     failed.add(o);
   }
 }
Example #4
0
 public JsonObject toJson() {
   JsonObject res = new JsonObject();
   JsonArray rows = new JsonArray();
   for (int i = 0; i < _rows.length; ++i) rows.add(new JsonPrimitive(_rows[i]));
   JsonArray dist = new JsonArray();
   for (int i = 0; i < _dist.length; ++i) dist.add(new JsonPrimitive(_dist[i]));
   res.add("rows_per_cluster", rows);
   res.add("sqr_error_per_cluster", dist);
   return res;
 }
Example #5
0
 public static void addFolder(Path p, JsonArray succeeded, JsonArray failed) throws IOException {
   FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
   if (!fs.exists(p)) {
     JsonObject o = new JsonObject();
     o.addProperty(Constants.FILE, p.toString());
     o.addProperty(Constants.ERROR, "Path does not exist!");
     failed.add(o);
     return;
   }
   addFolder(fs, p, succeeded, failed);
 }
Example #6
0
 /**
  * Annotate the name of the response column in the job parameter JSON
  *
  * @return JsonObject annotated with the name of the response column
  */
 @Override
 public JsonObject toJSON() {
   JsonObject jo = super.toJSON();
   if (source != null) {
     int idx = source.find(response);
     if (idx == -1) {
       Vec vm = response.masterVec();
       if (vm != null) idx = source.find(vm);
     }
     jo.getAsJsonObject("response")
         .add("name", new JsonPrimitive(idx == -1 ? "null" : source._names[idx]));
   }
   return jo;
 }
Example #7
0
 @Override
 public JsonObject toJson() {
   JsonObject res = new JsonObject();
   res.addProperty(Constants.VERSION, H2O.VERSION);
   res.addProperty(Constants.TYPE, KMeansModel.class.getName());
   res.addProperty(Constants.ERROR, _error);
   JsonArray ary = new JsonArray();
   for (double[] dd : clusters()) {
     JsonArray ary2 = new JsonArray();
     for (double d : dd) ary2.add(new JsonPrimitive(d));
     ary.add(ary2);
   }
   res.add(Constants.CLUSTERS, ary);
   return res;
 }
Example #8
0
 /**
  * Annotate the used and ignored columns in the job parameter JSON For both the used and the
  * ignored columns, the following rules apply: If the number of columns is less or equal than
  * 100, a dense list of used columns is reported. If the number of columns is greater than 100,
  * the number of columns is reported. If the number of columns is 0, a "N/A" is reported.
  *
  * @return JsonObject annotated with used/ignored columns
  */
 @Override
 public JsonObject toJSON() {
   JsonObject jo = super.toJSON();
   if (!jo.has("source") || source == null) return jo;
   HashMap<String, int[]> map = new HashMap<String, int[]>();
   map.put("used_cols", cols);
   map.put("ignored_cols", ignored_cols);
   for (String key : map.keySet()) {
     int[] val = map.get(key);
     if (val != null) {
       if (val.length > 100) jo.getAsJsonObject("source").addProperty("num_" + key, val.length);
       else if (val.length > 0) {
         StringBuilder sb = new StringBuilder();
         for (int c : val) sb.append(c + ",");
         jo.getAsJsonObject("source")
             .addProperty(key, sb.toString().substring(0, sb.length() - 1));
       } else {
         jo.getAsJsonObject("source").add(key, JsonNull.INSTANCE);
       }
     }
   }
   return jo;
 }