private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
@Override
public byte[] load(final Value v) {
final byte[] b = MemoryManager.malloc1(v._max);
long skip = 0;
Key k = v._key;
final Path p;
if (_iceRoot != null) {
p = new Path(_iceRoot, getIceName(v));
} else {
// Convert an arraylet chunk into a long-offset from the base file.
if (k._kb[0] == Key.ARRAYLET_CHUNK) {
skip = ValueArray.getChunkOffset(k); // The offset
k = ValueArray.getArrayKey(k); // From the base file key
if (k.toString().endsWith(Extensions.HEX)) { // Hex file?
int value_len = DKV.get(k).memOrLoad().length; // How long is the ValueArray header?
skip += value_len;
}
}
p = new Path(k.toString());
}
final long skip_ = skip;
run(
new Callable() {
@Override
public Object call() throws Exception {
FileSystem fs = FileSystem.get(p.toUri(), CONF);
FSDataInputStream s = null;
try {
s = fs.open(p);
// NOTE:
// The following line degrades performance of HDFS load from S3 API:
// s.readFully(skip,b,0,b.length);
// Google API's simple seek has better performance
// Load of 300MB file via Google API ~ 14sec, via s.readFully ~ 5min (under the same
// condition)
ByteStreams.skipFully(s, skip_);
ByteStreams.readFully(s, b);
assert v.isPersisted();
} finally {
Utils.close(s);
}
return null;
}
},
true,
v._max);
return b;
}
@Override
public byte[] load(final Value v) {
final byte[] b = MemoryManager.malloc1(v._max);
long skip = 0;
Key k = v._key;
if (k._kb[0] == Key.ARRAYLET_CHUNK) {
skip = ValueArray.getChunkOffset(k); // The offset
k = ValueArray.getArrayKey(k); // From the base file key
} else if (k._kb[0] == Key.DVEC) {
skip = water.fvec.NFSFileVec.chunkOffset(k); // The offset
}
final Path p =
_iceRoot == null ? new Path(getPathForKey(k)) : new Path(_iceRoot, getIceName(v));
final long skip_ = skip;
run(
new Callable() {
@Override
public Object call() throws Exception {
FileSystem fs = FileSystem.get(p.toUri(), CONF);
FSDataInputStream s = null;
try {
s = fs.open(p);
// NOTE:
// The following line degrades performance of HDFS load from S3 API:
// s.readFully(skip,b,0,b.length);
// Google API's simple seek has better performance
// Load of 300MB file via Google API ~ 14sec, via s.readFully ~ 5min (under the same
// condition)
ByteStreams.skipFully(s, skip_);
ByteStreams.readFully(s, b);
assert v.isPersisted();
} finally {
Utils.close(s);
}
return null;
}
},
true,
v._max);
return b;
}