private static void addFolder2(
FileSystem fs, Path p, ArrayList<String> keys, ArrayList<String> failed) {
try {
if (fs == null) return;
Futures futures = new Futures();
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder2(fs, pfs, keys, failed);
} else {
long size = file.getLen();
Key res;
if (pfs.getName().endsWith(Extensions.JSON)) {
throw H2O.unimpl();
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
throw H2O.unimpl();
} else {
Key k = null;
keys.add((k = HdfsFileVec.make(file, futures)).toString());
Log.info("PersistHdfs: DKV.put(" + k + ")");
}
}
}
} catch (Exception e) {
Log.err(e);
failed.add(p.toString());
}
}
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
private static void run(Callable c, boolean read, int size) {
// Count all i/o time from here, including all retry overheads
long start_io_ms = System.currentTimeMillis();
while (true) {
try {
long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats
c.call();
TimeLine.record_IOclose(start_ns, start_io_ms, read ? 1 : 0, size, Value.HDFS);
break;
// Explicitly ignore the following exceptions but
// fail on the rest IOExceptions
} catch (EOFException e) {
ignoreAndWait(e, false);
} catch (SocketTimeoutException e) {
ignoreAndWait(e, false);
} catch (S3Exception e) {
// Preserve S3Exception before IOException
// Since this is tricky code - we are supporting different HDFS version
// New version declares S3Exception as IOException
// But old versions (0.20.xxx) declares it as RuntimeException
// So we have to catch it before IOException !!!
ignoreAndWait(e, false);
} catch (IOException e) {
ignoreAndWait(e, true);
} catch (Exception e) {
throw Log.errRTExcept(e);
}
}
}
@Override
protected void init() {
super.init();
// Reject request if classification is required and response column is float
// Argument a4class = find("classification"); // get UI control
// String p4class = input("classification"); // get value from HTTP requests
// if there is UI control and classification field was passed
final boolean classificationFieldSpecified =
true; // ROLLBACK: a4class!=null ? p4class!=null : /* we are not in UI so expect that
// parameter is specified correctly */ true;
if (!classificationFieldSpecified) { // can happen if a client sends a request which does not
// specify classification parameter
classification = response.isEnum();
Log.warn(
"Classification field is not specified - deriving according to response! The classification field set to "
+ classification);
} else {
if (classification && response.isFloat())
throw new H2OIllegalArgumentException(
find("classification"), "Requested classification on float column!");
if (!classification && response.isEnum())
throw new H2OIllegalArgumentException(
find("classification"), "Requested regression on enum column!");
}
}
public final void print() {
StringBuilder sb = new StringBuilder();
sb.append("I: ").append(_iteration).append("[");
double[][] c = clusters();
for (int i = 0; i < c.length; i++) sb.append(c[i][2]).append(",");
sb.append("]");
Log.debug(Sys.KMEAN, sb);
}
// Loading/Writing ice to HDFS
PersistHdfs(URI uri) {
try {
_iceRoot = new Path(uri + "/ice" + H2O.SELF_ADDRESS.getHostAddress() + "-" + H2O.API_PORT);
// Make the directory as-needed
FileSystem fs = FileSystem.get(_iceRoot.toUri(), CONF);
fs.mkdirs(_iceRoot);
} catch (Exception e) {
throw Log.errRTExcept(e);
}
}
static {
Configuration conf = null;
if (H2O.OPT_ARGS.hdfs_config != null) {
conf = new Configuration();
File p = new File(H2O.OPT_ARGS.hdfs_config);
if (!p.exists()) Log.die("Unable to open hdfs configuration file " + p.getAbsolutePath());
conf.addResource(new Path(p.getAbsolutePath()));
Log.debug(Sys.HDFS_, "resource ", p.getAbsolutePath(), " added to the hadoop configuration");
} else {
conf = new Configuration();
if (!Strings.isNullOrEmpty(H2O.OPT_ARGS.hdfs)) {
// setup default remote Filesystem - for version 0.21 and higher
conf.set("fs.defaultFS", H2O.OPT_ARGS.hdfs);
// To provide compatibility with version 0.20.0 it is necessary to setup the property
// fs.default.name which was in newer version renamed to 'fs.defaultFS'
conf.set("fs.default.name", H2O.OPT_ARGS.hdfs);
}
}
CONF = conf;
}
private void cancel(final String msg, JobState resultingState) {
if (resultingState == JobState.CANCELLED) {
Log.info("Job " + self() + "(" + description + ") was cancelled.");
} else {
Log.err("Job " + self() + "(" + description + ") failed.");
Log.err(msg);
}
exception = msg;
state = resultingState;
// replace finished job by a job handle
replaceByJobHandle();
DKV.write_barrier();
final Job job = this;
H2O.submitTask(
new H2OCountedCompleter() {
@Override
public void compute2() {
job.onCancelled();
}
});
}
public static InputStream openStream(Key k, ProgressMonitor pmon) throws IOException {
H2OHdfsInputStream res = null;
Path p = new Path(k.toString());
try {
res = new H2OHdfsInputStream(p, 0, pmon);
} catch (IOException e) {
try {
Thread.sleep(1000);
} catch (Exception ex) {
}
Log.warn("Error while opening HDFS key " + k.toString() + ", will wait and retry.");
res = new H2OHdfsInputStream(p, 0, pmon);
}
return res;
}
private static void run(Callable c, boolean read, int size) {
// Count all i/o time from here, including all retry overheads
long start_io_ms = System.currentTimeMillis();
while (true) {
try {
long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats
c.call();
TimeLine.record_IOclose(start_ns, start_io_ms, read ? 1 : 0, size, Value.HDFS);
break;
// Explicitly ignore the following exceptions but
// fail on the rest IOExceptions
} catch (EOFException e) {
ignoreAndWait(e, false);
} catch (SocketTimeoutException e) {
ignoreAndWait(e, false);
} catch (IOException e) {
ignoreAndWait(e, true);
} catch (Exception e) {
throw Log.errRTExcept(e);
}
}
}