private static void addFolder2( FileSystem fs, Path p, ArrayList<String> keys, ArrayList<String> failed) { try { if (fs == null) return; Futures futures = new Futures(); for (FileStatus file : fs.listStatus(p)) { Path pfs = file.getPath(); if (file.isDir()) { addFolder2(fs, pfs, keys, failed); } else { long size = file.getLen(); Key res; if (pfs.getName().endsWith(Extensions.JSON)) { throw H2O.unimpl(); } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file? throw H2O.unimpl(); } else { Key k = null; keys.add((k = HdfsFileVec.make(file, futures)).toString()); Log.info("PersistHdfs: DKV.put(" + k + ")"); } } } } catch (Exception e) { Log.err(e); failed.add(p.toString()); } }
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) { try { if (fs == null) return; for (FileStatus file : fs.listStatus(p)) { Path pfs = file.getPath(); if (file.isDir()) { addFolder(fs, pfs, succeeded, failed); } else { Key k = Key.make(pfs.toString()); long size = file.getLen(); Value val = null; if (pfs.getName().endsWith(Extensions.JSON)) { JsonParser parser = new JsonParser(); JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject(); JsonElement v = json.get(Constants.VERSION); if (v == null) throw new RuntimeException("Missing version"); JsonElement type = json.get(Constants.TYPE); if (type == null) throw new RuntimeException("Missing type"); Class c = Class.forName(type.getAsString()); OldModel model = (OldModel) c.newInstance(); model.fromJson(json); } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file? FSDataInputStream s = fs.open(pfs); int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg byte[] mem = MemoryManager.malloc1(sz); s.readFully(mem); // Convert to a ValueArray (hope it fits in 1Meg!) ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem)); val = new Value(k, ary, Value.HDFS); } else if (size >= 2 * ValueArray.CHUNK_SZ) { val = new Value( k, new ValueArray(k, size), Value.HDFS); // ValueArray byte wrapper over a large file } else { val = new Value(k, (int) size, Value.HDFS); // Plain Value val.setdsk(); } DKV.put(k, val); Log.info("PersistHdfs: DKV.put(" + k + ")"); JsonObject o = new JsonObject(); o.addProperty(Constants.KEY, k.toString()); o.addProperty(Constants.FILE, pfs.toString()); o.addProperty(Constants.VALUE_SIZE, file.getLen()); succeeded.add(o); } } } catch (Exception e) { Log.err(e); JsonObject o = new JsonObject(); o.addProperty(Constants.FILE, p.toString()); o.addProperty(Constants.ERROR, e.getMessage()); failed.add(o); } }
private static void run(Callable c, boolean read, int size) { // Count all i/o time from here, including all retry overheads long start_io_ms = System.currentTimeMillis(); while (true) { try { long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats c.call(); TimeLine.record_IOclose(start_ns, start_io_ms, read ? 1 : 0, size, Value.HDFS); break; // Explicitly ignore the following exceptions but // fail on the rest IOExceptions } catch (EOFException e) { ignoreAndWait(e, false); } catch (SocketTimeoutException e) { ignoreAndWait(e, false); } catch (S3Exception e) { // Preserve S3Exception before IOException // Since this is tricky code - we are supporting different HDFS version // New version declares S3Exception as IOException // But old versions (0.20.xxx) declares it as RuntimeException // So we have to catch it before IOException !!! ignoreAndWait(e, false); } catch (IOException e) { ignoreAndWait(e, true); } catch (Exception e) { throw Log.errRTExcept(e); } } }
@Override protected void init() { super.init(); // Reject request if classification is required and response column is float // Argument a4class = find("classification"); // get UI control // String p4class = input("classification"); // get value from HTTP requests // if there is UI control and classification field was passed final boolean classificationFieldSpecified = true; // ROLLBACK: a4class!=null ? p4class!=null : /* we are not in UI so expect that // parameter is specified correctly */ true; if (!classificationFieldSpecified) { // can happen if a client sends a request which does not // specify classification parameter classification = response.isEnum(); Log.warn( "Classification field is not specified - deriving according to response! The classification field set to " + classification); } else { if (classification && response.isFloat()) throw new H2OIllegalArgumentException( find("classification"), "Requested classification on float column!"); if (!classification && response.isEnum()) throw new H2OIllegalArgumentException( find("classification"), "Requested regression on enum column!"); } }
public final void print() { StringBuilder sb = new StringBuilder(); sb.append("I: ").append(_iteration).append("["); double[][] c = clusters(); for (int i = 0; i < c.length; i++) sb.append(c[i][2]).append(","); sb.append("]"); Log.debug(Sys.KMEAN, sb); }
// Loading/Writing ice to HDFS PersistHdfs(URI uri) { try { _iceRoot = new Path(uri + "/ice" + H2O.SELF_ADDRESS.getHostAddress() + "-" + H2O.API_PORT); // Make the directory as-needed FileSystem fs = FileSystem.get(_iceRoot.toUri(), CONF); fs.mkdirs(_iceRoot); } catch (Exception e) { throw Log.errRTExcept(e); } }
static { Configuration conf = null; if (H2O.OPT_ARGS.hdfs_config != null) { conf = new Configuration(); File p = new File(H2O.OPT_ARGS.hdfs_config); if (!p.exists()) Log.die("Unable to open hdfs configuration file " + p.getAbsolutePath()); conf.addResource(new Path(p.getAbsolutePath())); Log.debug(Sys.HDFS_, "resource ", p.getAbsolutePath(), " added to the hadoop configuration"); } else { conf = new Configuration(); if (!Strings.isNullOrEmpty(H2O.OPT_ARGS.hdfs)) { // setup default remote Filesystem - for version 0.21 and higher conf.set("fs.defaultFS", H2O.OPT_ARGS.hdfs); // To provide compatibility with version 0.20.0 it is necessary to setup the property // fs.default.name which was in newer version renamed to 'fs.defaultFS' conf.set("fs.default.name", H2O.OPT_ARGS.hdfs); } } CONF = conf; }
private void cancel(final String msg, JobState resultingState) { if (resultingState == JobState.CANCELLED) { Log.info("Job " + self() + "(" + description + ") was cancelled."); } else { Log.err("Job " + self() + "(" + description + ") failed."); Log.err(msg); } exception = msg; state = resultingState; // replace finished job by a job handle replaceByJobHandle(); DKV.write_barrier(); final Job job = this; H2O.submitTask( new H2OCountedCompleter() { @Override public void compute2() { job.onCancelled(); } }); }
public static InputStream openStream(Key k, ProgressMonitor pmon) throws IOException { H2OHdfsInputStream res = null; Path p = new Path(k.toString()); try { res = new H2OHdfsInputStream(p, 0, pmon); } catch (IOException e) { try { Thread.sleep(1000); } catch (Exception ex) { } Log.warn("Error while opening HDFS key " + k.toString() + ", will wait and retry."); res = new H2OHdfsInputStream(p, 0, pmon); } return res; }
private static void run(Callable c, boolean read, int size) { // Count all i/o time from here, including all retry overheads long start_io_ms = System.currentTimeMillis(); while (true) { try { long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats c.call(); TimeLine.record_IOclose(start_ns, start_io_ms, read ? 1 : 0, size, Value.HDFS); break; // Explicitly ignore the following exceptions but // fail on the rest IOExceptions } catch (EOFException e) { ignoreAndWait(e, false); } catch (SocketTimeoutException e) { ignoreAndWait(e, false); } catch (IOException e) { ignoreAndWait(e, true); } catch (Exception e) { throw Log.errRTExcept(e); } } }