Beispiel #1
0
  private static void addFolder2(
      FileSystem fs, Path p, ArrayList<String> keys, ArrayList<String> failed) {
    try {
      if (fs == null) return;

      Futures futures = new Futures();
      for (FileStatus file : fs.listStatus(p)) {
        Path pfs = file.getPath();
        if (file.isDir()) {
          addFolder2(fs, pfs, keys, failed);
        } else {
          long size = file.getLen();
          Key res;
          if (pfs.getName().endsWith(Extensions.JSON)) {
            throw H2O.unimpl();
          } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
            throw H2O.unimpl();
          } else {
            Key k = null;
            keys.add((k = HdfsFileVec.make(file, futures)).toString());
            Log.info("PersistHdfs: DKV.put(" + k + ")");
          }
        }
      }
    } catch (Exception e) {
      Log.err(e);
      failed.add(p.toString());
    }
  }
Beispiel #2
0
 private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
   try {
     if (fs == null) return;
     for (FileStatus file : fs.listStatus(p)) {
       Path pfs = file.getPath();
       if (file.isDir()) {
         addFolder(fs, pfs, succeeded, failed);
       } else {
         Key k = Key.make(pfs.toString());
         long size = file.getLen();
         Value val = null;
         if (pfs.getName().endsWith(Extensions.JSON)) {
           JsonParser parser = new JsonParser();
           JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
           JsonElement v = json.get(Constants.VERSION);
           if (v == null) throw new RuntimeException("Missing version");
           JsonElement type = json.get(Constants.TYPE);
           if (type == null) throw new RuntimeException("Missing type");
           Class c = Class.forName(type.getAsString());
           OldModel model = (OldModel) c.newInstance();
           model.fromJson(json);
         } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
           FSDataInputStream s = fs.open(pfs);
           int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
           byte[] mem = MemoryManager.malloc1(sz);
           s.readFully(mem);
           // Convert to a ValueArray (hope it fits in 1Meg!)
           ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
           val = new Value(k, ary, Value.HDFS);
         } else if (size >= 2 * ValueArray.CHUNK_SZ) {
           val =
               new Value(
                   k,
                   new ValueArray(k, size),
                   Value.HDFS); // ValueArray byte wrapper over a large file
         } else {
           val = new Value(k, (int) size, Value.HDFS); // Plain Value
           val.setdsk();
         }
         DKV.put(k, val);
         Log.info("PersistHdfs: DKV.put(" + k + ")");
         JsonObject o = new JsonObject();
         o.addProperty(Constants.KEY, k.toString());
         o.addProperty(Constants.FILE, pfs.toString());
         o.addProperty(Constants.VALUE_SIZE, file.getLen());
         succeeded.add(o);
       }
     }
   } catch (Exception e) {
     Log.err(e);
     JsonObject o = new JsonObject();
     o.addProperty(Constants.FILE, p.toString());
     o.addProperty(Constants.ERROR, e.getMessage());
     failed.add(o);
   }
 }
Beispiel #3
0
 public static byte [] unzipBytes(byte [] bs, Compression cmp) {
   InputStream is = null;
   int off = 0;
   try {
     switch(cmp) {
     case NONE: // No compression
       return bs;
     case ZIP: {
       ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(bs));
       ZipEntry ze = zis.getNextEntry(); // Get the *FIRST* entry
       // There is at least one entry in zip file and it is not a directory.
       if( ze != null && !ze.isDirectory() ) {
         is = zis;
         break;
       }
       zis.close();
       return bs; // Don't crash, ignore file if cannot unzip
     }
     case GZIP:
       is = new GZIPInputStream(new ByteArrayInputStream(bs));
       break;
     default:
       assert false:"cmp = " + cmp;
     }
     // If reading from a compressed stream, estimate we can read 2x uncompressed
     assert( is != null ):"is is NULL, cmp = " + cmp;
     bs = new byte[bs.length * 2];
     // Now read from the (possibly compressed) stream
     while( off < bs.length ) {
       int len = is.read(bs, off, bs.length - off);
       if( len < 0 )
         break;
       off += len;
       if( off == bs.length ) { // Dataset is uncompressing alot! Need more space...
         if( bs.length >= ValueArray.CHUNK_SZ )
           break; // Already got enough
         bs = Arrays.copyOf(bs, bs.length * 2);
       }
     }
   } catch( IOException ioe ) { // Stop at any io error
     Log.err(ioe);
   } finally {
     Utils.close(is);
   }
   return bs;
 }