Example #1
0
 // Read up to 'len' bytes of Value. Value should already be persisted to
 // disk.  A racing delete can trigger a failure where we get a null return,
 // but no crash (although one could argue that a racing load&delete is a bug
 // no matter what).
 @Override
 public byte[] load(Value v) {
   long skip = 0;
   Key k = v._key;
   // Convert an arraylet chunk into a long-offset from the base file.
   if (k._kb[0] == Key.ARRAYLET_CHUNK) {
     skip = ValueArray.getChunkOffset(k); // The offset
     k = ValueArray.getArrayKey(k); // From the base file key
   }
   if (k._kb[0] == Key.DVEC) {
     skip = water.fvec.NFSFileVec.chunkOffset(k); // The offset
   }
   try {
     FileInputStream s = null;
     try {
       s = new FileInputStream(getFileForKey(k));
       FileChannel fc = s.getChannel();
       fc.position(skip);
       AutoBuffer ab = new AutoBuffer(fc, true, Value.NFS);
       byte[] b = ab.getA1(v._max);
       ab.close();
       assert v.isPersisted();
       return b;
     } finally {
       if (s != null) s.close();
     }
   } catch (IOException e) { // Broken disk / short-file???
     H2O.ignore(e);
     return null;
   }
 }
Example #2
0
 /**
  * Stores the stream to its chunk using the atomic union. After the data from the stream is
  * stored, its memory is freed up.
  */
 public void store() {
   assert _ab.eof();
   Key k = ValueArray.getChunkKey(_chunkIndex, _resultKey);
   AtomicUnion u = new AtomicUnion(_ab.bufClose(), _chunkOffset);
   alsoBlockFor(u.fork(k));
   _ab = null; // free mem
 }
Example #3
0
 public static Compression guessCompressionMethod(byte [] bits){
   AutoBuffer ab = new AutoBuffer(bits);
   // Look for ZIP magic
   if( bits.length > ZipFile.LOCHDR && ab.get4(0) == ZipFile.LOCSIG )
     return Compression.ZIP;
   if( bits.length > 2 && ab.get2(0) == GZIPInputStream.GZIP_MAGIC )
     return Compression.GZIP;
   return Compression.NONE;
 }
Example #4
0
  /** Adds string (enum) value to the column. */
  public void addStrCol(int colIdx, ValueString str) {
    if (colIdx >= _ncolumns) return;
    switch (_phase) {
      case ONE:
        ++_colIdx;
        // If this is a yet unspecified but non-numeric column, attempt a time-parse
        if (_colTypes[colIdx] == UCOL) {
          long time = attemptTimeParse(str);
          if (time != Long.MIN_VALUE) _colTypes[colIdx] = TCOL;
        } else if (_colTypes[colIdx] == TCOL) {
          return;
        }

        // Now attempt to make this an Enum col
        Enum e = _enums[colIdx];
        if (e == null || e.isKilled()) return;
        if (_colTypes[colIdx] == UCOL) _colTypes[colIdx] = ECOL;
        e.addKey(str);
        ++_invalidValues[
            colIdx]; // invalid count in phase0 is in fact number of non-numbers (it is used for
        // mean computation, is recomputed in 2nd pass)
        break;
      case TWO:
        if (_enums[colIdx] != null) {
          ++_colIdx;
          int id = _enums[colIdx].getTokenId(str);
          // we do not expect any misses here
          assert 0 <= id && id < _enums[colIdx].size();
          switch (_colTypes[colIdx]) {
            case BYTE:
              _ab.put1(id);
              break;
            case SHORT:
              _ab.put2((char) id);
              break;
            case INT:
              _ab.put4(id);
              break;
            default:
              assert false : "illegal case: " + _colTypes[colIdx];
          }
        } else if (_colTypes[colIdx] == LONG) {
          ++_colIdx;
          // Times are strings with a numeric column type of LONG
          _ab.put8(attemptTimeParse(str));
        } else {
          addInvalidCol(colIdx);
        }
        break;
      default:
        assert (false);
    }
  }
Example #5
0
 /** Adds invalid value to the column. */
 public void addInvalidCol(int colIdx) {
   ++_colIdx;
   if (colIdx >= _ncolumns) return;
   ++_invalidValues[colIdx];
   if (_phase == Pass.ONE) return;
   switch (_colTypes[colIdx]) {
     case BYTE:
     case DBYTE:
       _ab.put1(-1);
       break;
     case SHORT:
     case DSHORT:
       _ab.put2((short) -1);
       break;
     case INT:
       _ab.put4(Integer.MIN_VALUE);
       break;
     case LONG:
       _ab.put8(Long.MIN_VALUE);
       break;
     case FLOAT:
       _ab.put4f(Float.NaN);
       break;
     case DOUBLE:
       _ab.put8d(Double.NaN);
       break;
     case STRINGCOL:
       // TODO, replace with empty space!
       _ab.put1(-1);
       break;
     default:
       assert false : "illegal case: " + _colTypes[colIdx];
   }
 }
Example #6
0
 @Override
 public C2SChunk read(AutoBuffer bb) {
   _mem = bb.bufClose();
   _start = -1;
   _len = (_mem.length - OFF) >> 1;
   _scale = UDP.get8d(_mem, 0);
   _bias = UDP.get4(_mem, 8);
   return this;
 }
Example #7
0
 /** Advances to new line. In phase two it also must make sure that the */
 public void newLine() {
   ++_myrows;
   if (_phase == Pass.TWO) {
     while (_colIdx < _ncolumns) addInvalidCol(_colIdx);
     _colIdx = 0;
     // if we are at the end of current stream, move to the next one
     if (_ab.eof()) {
       _outputStreams2[_outputIdx].store();
       ++_outputIdx;
       if (_outputIdx < _outputStreams2.length) {
         _ab = _outputStreams2[_outputIdx].initialize();
       } else {
         _ab = null; // just to be sure we throw a NPE if there is a problem
       }
     }
   }
 }
Example #8
0
 @Override public AutoBuffer write(AutoBuffer bb) {
   bb.put4(size());
   for(Map.Entry<K, V> e:entrySet())bb.put(e.getKey()).put(e.getValue());
   return bb;
 }
Example #9
0
 @Override public IcedArrayList<T> read(AutoBuffer bb) {
   int n = bb.get4();
   for(int i = 0; i < n; ++i)
     add(bb.<T>get());
   return this;
 }
Example #10
0
 @Override public AutoBuffer write(AutoBuffer bb) {
   bb.put4(size());
   for(T t:this)
     bb.put(t);
   return bb;
 }
Example #11
0
 @Override
 protected Keyed readAll_impl(AutoBuffer ab, Futures fs) {
   ab.getKey(_output._u_key, fs);
   ab.getKey(_output._v_key, fs);
   return super.readAll_impl(ab, fs);
 }
Example #12
0
 @Override
 protected Keyed readAll_impl(AutoBuffer ab, Futures fs) {
   ab.getKey(_output._init_key, fs);
   ab.getKey(_output._representation_key, fs);
   return super.readAll_impl(ab, fs);
 }
Example #13
0
 /** Write out K/V pairs */
 @Override
 protected AutoBuffer writeAll_impl(AutoBuffer ab) {
   ab.putKey(_output._init_key);
   ab.putKey(_output._representation_key);
   return super.writeAll_impl(ab);
 }
Example #14
0
 @Override public IcedHashMap<K,V> read(AutoBuffer bb) {
   int n = bb.get4();
   for(int i = 0; i < n; ++i)
     put(bb.<K>get(),bb.<V>get());
   return this;
 }
Example #15
0
 @SuppressWarnings("fallthrough")
 public void addNumCol(int colIdx, long number, int exp) {
   ++_colIdx;
   if (colIdx >= _ncolumns) return;
   switch (_phase) {
     case ONE:
       double d = number * pow10(exp);
       if (d < _min[colIdx]) _min[colIdx] = d;
       if (d > _max[colIdx]) _max[colIdx] = d;
       _mean[colIdx] += d;
       if (exp != 0) {
         if (exp < _scale[colIdx]) _scale[colIdx] = exp;
         if (_colTypes[colIdx] != DCOL) {
           if (Math.abs(number) > MAX_FLOAT_MANTISSA || exp < -35 || exp > 35)
             _colTypes[colIdx] = DCOL;
           else _colTypes[colIdx] = FCOL;
         }
       } else if (_colTypes[colIdx] < ICOL) {
         _colTypes[colIdx] = ICOL;
       }
       break;
     case TWO:
       switch (_colTypes[colIdx]) {
         case BYTE:
           _ab.put1((byte) (number * pow10i(exp - _scale[colIdx]) - _bases[colIdx]));
           break;
         case SHORT:
           _ab.put2((short) (number * pow10i(exp - _scale[colIdx]) - _bases[colIdx]));
           break;
         case INT:
           _ab.put4((int) (number * pow10i(exp - _scale[colIdx]) - _bases[colIdx]));
           break;
         case LONG:
           _ab.put8(number * pow10i(exp - _scale[colIdx]));
           break;
         case FLOAT:
           _ab.put4f((float) (number * pow10(exp)));
           break;
         case DOUBLE:
           _ab.put8d(number * pow10(exp));
           break;
         case DBYTE:
           _ab.put1((short) (number * pow10i(exp - _scale[colIdx]) - _bases[colIdx]));
           break;
         case DSHORT:
           // scale is computed as negative in the first pass,
           // therefore to compute the positive exponent after scale, we add scale and the original
           // exponent
           _ab.put2((short) (number * pow10i(exp - _scale[colIdx]) - _bases[colIdx]));
           break;
         case STRINGCOL:
           break;
       }
       // update sigma
       if (!Double.isNaN(_mean[colIdx])) {
         d = number * pow10(exp) - _mean[colIdx];
         _sigma[colIdx] += d * d;
       }
       break;
     default:
       assert (false);
   }
 }
Example #16
0
 @Override
 public AutoBuffer write(AutoBuffer bb) {
   return bb.putA1(_mem, _mem.length);
 }
Example #17
0
  // TODO: Drop this writeJSON_impl and use the default one.
  // TODO: Pull out the help text & metadata into the ParameterSchema for the front-end to display.
  public final AutoBuffer writeJSON_impl(AutoBuffer ab) {
    ab.putJSON("job", job);
    ab.put1(',');
    ab.putJSONStr("algo", algo);
    ab.put1(',');
    ab.putJSONStr("algo_full_name", algo_full_name);
    ab.put1(',');
    ab.putJSONAEnum("can_build", can_build);
    ab.put1(',');
    ab.putJSONEnum("visibility", visibility);
    ab.put1(',');
    ab.putJSONA("messages", messages);
    ab.put1(',');
    ab.putJSON4("error_count", error_count);
    ab.put1(',');

    // Builds ModelParameterSchemaV2 objects for each field, and then calls writeJSON on the array
    ModelParametersSchemaV3.writeParametersJSON(
        ab,
        parameters,
        createParametersSchema().fillFromImpl((Model.Parameters) parameters.createImpl()));
    return ab;
  }
 // deep clone all weights/biases
 DeepLearningModelInfo deep_clone() {
   AutoBuffer ab = new AutoBuffer();
   this.write(ab);
   ab.flipForReading();
   return (DeepLearningModelInfo) new DeepLearningModelInfo().read(ab);
 }
Example #19
0
 public DataInfo deep_clone() {
   AutoBuffer ab = new AutoBuffer();
   this.write(ab);
   ab.flipForReading();
   return new DataInfo().read(ab);
 }
Example #20
0
 // Insert just the predictions: a single byte/short if we are predicting a
 // single class, or else the full distribution.
 @Override
 protected AutoBuffer compress(AutoBuffer ab) {
   assert !Double.isNaN(_pred);
   return ab.put4f((float) _pred);
 }
Example #21
0
 /** Write out K/V pairs */
 @Override
 protected AutoBuffer writeAll_impl(AutoBuffer ab) {
   ab.putKey(_output._u_key);
   ab.putKey(_output._v_key);
   return super.writeAll_impl(ab);
 }