public static String store2Hdfs(Key srcKey) {
assert srcKey._kb[0] != Key.ARRAYLET_CHUNK;
assert PersistHdfs.getPathForKey(srcKey) != null; // Validate key name
Value v = DKV.get(srcKey);
if (v == null) return "Key " + srcKey + " not found";
if (!v.isArray()) { // Simple chunk?
v.setHdfs(); // Set to HDFS and be done
return null; // Success
}
// For ValueArrays, make the .hex header
ValueArray ary = v.get();
String err = PersistHdfs.freeze(srcKey, ary);
if (err != null) return err;
// The task managing which chunks to write next,
// store in a known key
TaskStore2HDFS ts = new TaskStore2HDFS(srcKey);
Key selfKey = ts.selfKey();
UKV.put(selfKey, ts);
// Then start writing chunks in-order with the zero chunk
H2ONode chk0_home = ValueArray.getChunkKey(0, srcKey).home_node();
RPC.call(ts.chunkHome(), ts);
// Watch the progress key until it gets removed or an error appears
long idx = 0;
while ((ts = UKV.get(selfKey, TaskStore2HDFS.class)) != null) {
if (ts._indexFrom != idx) {
System.out.print(" " + idx + "/" + ary.chunks());
idx = ts._indexFrom;
}
if (ts._err != null) { // Found an error?
UKV.remove(selfKey); // Cleanup & report
return ts._err;
}
try {
Thread.sleep(100);
} catch (InterruptedException e) {
}
}
System.out.println(" " + ary.chunks() + "/" + ary.chunks());
// PersistHdfs.refreshHDFSKeys();
return null;
}
// Store Value v to disk.
@Override
public void store(Value v) {
// Only the home node does persistence on NFS
if (!v._key.home()) return;
// A perhaps useless cutout: the upper layers should test this first.
if (v.isPersisted()) return;
// Never store arraylets on NFS, instead we'll store the entire array.
assert !v.isArray();
try {
File f = getFileForKey(v._key);
f.mkdirs();
FileOutputStream s = new FileOutputStream(f);
try {
byte[] m = v.memOrLoad();
assert (m == null || m.length == v._max); // Assert not saving partial files
if (m != null) new AutoBuffer(s.getChannel(), false, Value.NFS).putA1(m, m.length).close();
v.setdsk(); // Set as write-complete to disk
} finally {
s.close();
}
} catch (IOException e) {
H2O.ignore(e);
}
}