/** * Stores the stream to its chunk using the atomic union. After the data from the stream is * stored, its memory is freed up. */ public void store() { assert _ab.eof(); Key k = ValueArray.getChunkKey(_chunkIndex, _resultKey); AtomicUnion u = new AtomicUnion(_ab.bufClose(), _chunkOffset); alsoBlockFor(u.fork(k)); _ab = null; // free mem }
// User-Weak-Get a Key from the distributed cloud. // Right now, just gets chunk#0 from a ValueArray, or a normal Value otherwise. public static Value get(Key key) { Value val = DKV.get(key); if (val != null && val._isArray != 0) { Key k2 = ValueArray.getChunkKey(0, key); Value vchunk0 = DKV.get(k2); assert vchunk0 != null : "missed looking for key " + k2 + " from " + key; return vchunk0; // Else just get the prefix asked for } return val; }
// Recursively remove, gathering all the pending remote key-deletes private static void remove(Key key, Futures fs) { Value val = DKV.get(key, 32); // Get the existing Value, if any if (val == null) return; // Trivial delete if (val._isArray != 0) { // See if this is an Array ValueArray ary = ValueArray.value(val); for (long i = 0; i < ary.chunks(); i++) // Delete all the chunks remove(ary.getChunkKey(i), fs); } if (key._kb[0] == Key.KEY_OF_KEYS) // Key-of-keys? for (Key k : val.flatten()) // Then recursively delete remove(k, fs); DKV.remove(key, fs); }
public static void put(Key key, Value val, Futures fs) { Value res = DKV.put(key, val, fs); // If the old Value was a large array, we need to delete the leftover // chunks - they are unrelated to the new Value which might be either // bigger or smaller than the old Value. if (res != null && res._isArray != 0) { ValueArray ary = ValueArray.value(res); for (long i = 0; i < ary.chunks(); i++) // Delete all the chunks DKV.remove(ary.getChunkKey(i), fs); } if (key._kb[0] == Key.KEY_OF_KEYS) // Key-of-keys? for (Key k : res.flatten()) // Then recursively delete remove(k, fs); if (res != null) res.freeMem(); }
public static String store2Hdfs(Key srcKey) { assert srcKey._kb[0] != Key.ARRAYLET_CHUNK; assert PersistHdfs.getPathForKey(srcKey) != null; // Validate key name Value v = DKV.get(srcKey); if (v == null) return "Key " + srcKey + " not found"; if (v._isArray == 0) { // Simple chunk? v.setHdfs(); // Set to HDFS and be done return null; // Success } // For ValueArrays, make the .hex header ValueArray ary = ValueArray.value(v); String err = PersistHdfs.freeze(srcKey, ary); if (err != null) return err; // The task managing which chunks to write next, // store in a known key TaskStore2HDFS ts = new TaskStore2HDFS(srcKey); Key selfKey = ts.selfKey(); UKV.put(selfKey, ts); // Then start writing chunks in-order with the zero chunk H2ONode chk0_home = ValueArray.getChunkKey(0, srcKey).home_node(); RPC.call(ts.chunkHome(), ts); // Watch the progress key until it gets removed or an error appears long idx = 0; while (UKV.get(selfKey, ts) != null) { if (ts._indexFrom != idx) { System.out.print(" " + idx + "/" + ary.chunks()); idx = ts._indexFrom; } if (ts._err != null) { // Found an error? UKV.remove(selfKey); // Cleanup & report return ts._err; } try { Thread.sleep(100); } catch (InterruptedException e) { } } System.out.println(" " + ary.chunks() + "/" + ary.chunks()); // PersistHdfs.refreshHDFSKeys(); return null; }
@Override public void compute() { String path = null; // getPathFromValue(val); ValueArray ary = ValueArray.value(_arykey); Key self = selfKey(); while (_indexFrom < ary.chunks()) { Key ckey = ary.getChunkKey(_indexFrom++); if (!ckey.home()) { // Next chunk not At Home? RPC.call(chunkHome(), this); // Hand the baton off to the next node/chunk return; } Value val = DKV.get(ckey); // It IS home, so get the data _err = PersistHdfs.appendChunk(_arykey, val); if (_err != null) return; UKV.put(self, this); // Update the progress/self key } // We did the last chunk. Removing the selfKey is the signal to the web // thread that All Done. UKV.remove(self); }
private void updateClusters( int[] clusters, int count, long chunk, long numrows, int rpc, long updatedRow) { final int offset = (int) (updatedRow - (rpc * chunk)); final Key chunkKey = ValueArray.getChunkKey(chunk, _job.dest()); final int[] message; if (count == clusters.length) message = clusters; else { message = new int[count]; System.arraycopy(clusters, 0, message, 0, message.length); } final int rows = ValueArray.rpc(chunk, rpc, numrows); new Atomic() { @Override public Value atomic(Value val) { assert val == null || val._key.equals(chunkKey); AutoBuffer b = new AutoBuffer(rows * ROW_SIZE); if (val != null) b._bb.put(val.memOrLoad()); for (int i = 0; i < message.length; i++) b.put4((offset + i) * 4, message[i]); b.position(b.limit()); return new Value(chunkKey, b.buf()); } }.invoke(chunkKey); }
private H2ONode chunkHome() { return ValueArray.getChunkKey(_indexFrom, _arykey).home_node(); }