Beispiel #1
0
  /**
   * Load in the filesystem image. It's a big list of filenames and blocks. Return whether we should
   * "re-save" and consolidate the edit-logs
   */
  boolean loadFSImage(File fsdir, File edits) throws IOException {
    //
    // Atomic move sequence, to recover from interrupted save
    //
    File curFile = new File(fsdir, FS_IMAGE);
    File newFile = new File(fsdir, NEW_FS_IMAGE);
    File oldFile = new File(fsdir, OLD_FS_IMAGE);

    // Maybe we were interrupted between 2 and 4
    if (oldFile.exists() && curFile.exists()) {
      oldFile.delete();
      if (edits.exists()) {
        edits.delete();
      }
    } else if (oldFile.exists() && newFile.exists()) {
      // Or maybe between 1 and 2
      newFile.renameTo(curFile);
      oldFile.delete();
    } else if (curFile.exists() && newFile.exists()) {
      // Or else before stage 1, in which case we lose the edits
      newFile.delete();
    }

    //
    // Load in bits
    //
    if (curFile.exists()) {
      DataInputStream in =
          new DataInputStream(new BufferedInputStream(new FileInputStream(curFile)));
      try {
        int numFiles = in.readInt();
        for (int i = 0; i < numFiles; i++) {
          UTF8 name = new UTF8();
          name.readFields(in);
          int numBlocks = in.readInt();
          if (numBlocks == 0) {
            unprotectedAddFile(name, null);
          } else {
            Block blocks[] = new Block[numBlocks];
            for (int j = 0; j < numBlocks; j++) {
              blocks[j] = new Block();
              blocks[j].readFields(in);
            }
            unprotectedAddFile(name, blocks);
          }
        }
      } finally {
        in.close();
      }
    }

    if (edits.exists() && loadFSEdits(edits) > 0) {
      return true;
    } else {
      return false;
    }
  }
Beispiel #2
0
  /** Write a {@link Writable}, {@link String}, primitive type, or an array of the preceding. */
  public static void writeObject(
      DataOutput out, Object instance, Class declaredClass, Configuration conf) throws IOException {

    if (instance == null) { // null
      instance = new NullInstance(declaredClass, conf);
      declaredClass = Writable.class;
    }

    UTF8.writeString(out, declaredClass.getName()); // always write declared

    if (declaredClass.isArray()) { // array
      int length = Array.getLength(instance);
      out.writeInt(length);
      for (int i = 0; i < length; i++) {
        writeObject(out, Array.get(instance, i), declaredClass.getComponentType(), conf);
      }

    } else if (declaredClass == String.class) { // String
      UTF8.writeString(out, (String) instance);

    } else if (declaredClass.isPrimitive()) { // primitive type

      if (declaredClass == Boolean.TYPE) { // boolean
        out.writeBoolean(((Boolean) instance).booleanValue());
      } else if (declaredClass == Character.TYPE) { // char
        out.writeChar(((Character) instance).charValue());
      } else if (declaredClass == Byte.TYPE) { // byte
        out.writeByte(((Byte) instance).byteValue());
      } else if (declaredClass == Short.TYPE) { // short
        out.writeShort(((Short) instance).shortValue());
      } else if (declaredClass == Integer.TYPE) { // int
        out.writeInt(((Integer) instance).intValue());
      } else if (declaredClass == Long.TYPE) { // long
        out.writeLong(((Long) instance).longValue());
      } else if (declaredClass == Float.TYPE) { // float
        out.writeFloat(((Float) instance).floatValue());
      } else if (declaredClass == Double.TYPE) { // double
        out.writeDouble(((Double) instance).doubleValue());
      } else if (declaredClass == Void.TYPE) { // void
      } else {
        throw new IllegalArgumentException("Not a primitive: " + declaredClass);
      }
    } else if (declaredClass.isEnum()) { // enum
      UTF8.writeString(out, ((Enum) instance).name());
    } else if (Writable.class.isAssignableFrom(declaredClass)) { // Writable
      UTF8.writeString(out, instance.getClass().getName());
      ((Writable) instance).write(out);

    } else {
      throw new IOException("Can't write: " + instance + " as " + declaredClass);
    }
  }
 public void write(DataOutput out) throws IOException {
   out.write(BASE_TYPE);
   out.writeUTF(name);
   out.writeInt(sampleStrs.size());
   for (int i = 0; i < sampleStrs.size(); i++) {
     UTF8.writeString(out, sampleStrs.get(i));
   }
   out.writeInt(tokenClassIdentifier);
   out.writeBoolean(tokenParameter != null);
   if (tokenParameter != null) {
     UTF8.writeString(out, tokenParameter);
   }
 }
 public void readFields(DataInput in) throws IOException {
   // instance-specific
   this.sampleStrs = new ArrayList<String>();
   int numSamples = in.readInt();
   for (int i = 0; i < numSamples; i++) {
     sampleStrs.add(UTF8.readString(in).toString());
   }
   this.tokenClassIdentifier = in.readInt();
   if (in.readBoolean()) {
     this.tokenParameter = UTF8.readString(in);
   } else {
     this.tokenParameter = null;
   }
   this.schema = computeAvroSchema();
 }
Beispiel #5
0
 String normalizePath(UTF8 src) {
   String srcs = src.toString();
   if (srcs.length() > 1 && srcs.endsWith("/")) {
     srcs = srcs.substring(0, srcs.length() - 1);
   }
   return srcs;
 }
Beispiel #6
0
 /** Get the blocks associated with the file */
 public Block[] getFile(UTF8 src) {
   waitForReady();
   synchronized (rootDir) {
     INode targetNode = rootDir.getNode(src.toString());
     if (targetNode == null) {
       return null;
     } else {
       return targetNode.blocks;
     }
   }
 }
Beispiel #7
0
 boolean unprotectedAddFile(UTF8 name, Block blocks[]) {
   synchronized (rootDir) {
     if (blocks != null) {
       // Add file->block mapping
       for (int i = 0; i < blocks.length; i++) {
         activeBlocks.add(blocks[i]);
       }
     }
     return (rootDir.addNode(name.toString(), blocks) != null);
   }
 }
Beispiel #8
0
 public void readFields(DataInput in) throws IOException {
   String className = UTF8.readString(in);
   declaredClass = PRIMITIVE_NAMES.get(className);
   if (declaredClass == null) {
     try {
       declaredClass = getConf().getClassByName(className);
     } catch (ClassNotFoundException e) {
       throw new RuntimeException(e.toString());
     }
   }
 }
Beispiel #9
0
  /** Add the given filename to the fs. */
  public boolean addFile(UTF8 src, Block blocks[]) {
    waitForReady();

    // Always do an implicit mkdirs for parent directory tree
    mkdirs(DFSFile.getDFSParent(src.toString()));
    if (unprotectedAddFile(src, blocks)) {
      logEdit(OP_ADD, src, new ArrayWritable(Block.class, blocks));
      return true;
    } else {
      return false;
    }
  }
Beispiel #10
0
 boolean unprotectedRenameTo(UTF8 src, UTF8 dst) {
   synchronized (rootDir) {
     INode removedNode = rootDir.getNode(src.toString());
     if (removedNode == null) {
       return false;
     }
     removedNode.removeNode();
     if (isDir(dst)) {
       dst = new UTF8(dst.toString() + "/" + new File(src.toString()).getName());
     }
     INode newNode = rootDir.addNode(dst.toString(), removedNode.blocks);
     if (newNode != null) {
       newNode.children = removedNode.children;
       for (Iterator it = newNode.children.values().iterator(); it.hasNext(); ) {
         INode child = (INode) it.next();
         child.parent = newNode;
       }
       return true;
     } else {
       rootDir.addNode(src.toString(), removedNode.blocks);
       return false;
     }
   }
 }
Beispiel #11
0
 Block[] unprotectedDelete(UTF8 src) {
   synchronized (rootDir) {
     INode targetNode = rootDir.getNode(src.toString());
     if (targetNode == null) {
       return null;
     } else {
       //
       // Remove the node from the namespace and GC all
       // the blocks underneath the node.
       //
       if (!targetNode.removeNode()) {
         return null;
       } else {
         Vector v = new Vector();
         targetNode.collectSubtreeBlocks(v);
         for (Iterator it = v.iterator(); it.hasNext(); ) {
           Block b = (Block) it.next();
           activeBlocks.remove(b);
         }
         return (Block[]) v.toArray(new Block[v.size()]);
       }
     }
   }
 }
Beispiel #12
0
 /** Create the given directory and all its parent dirs. */
 public boolean mkdirs(UTF8 src) {
   return mkdirs(src.toString());
 }
Beispiel #13
0
  /**
   * Load an edit log, and apply the changes to the in-memory structure
   *
   * <p>This is where we apply edits that we've been writing to disk all along.
   */
  int loadFSEdits(File edits) throws IOException {
    int numEdits = 0;

    if (edits.exists()) {
      DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(edits)));
      try {
        while (in.available() > 0) {
          byte opcode = in.readByte();
          numEdits++;
          switch (opcode) {
            case OP_ADD:
              {
                UTF8 name = new UTF8();
                name.readFields(in);
                ArrayWritable aw = new ArrayWritable(Block.class);
                aw.readFields(in);
                Writable writables[] = (Writable[]) aw.get();
                Block blocks[] = new Block[writables.length];
                System.arraycopy(writables, 0, blocks, 0, blocks.length);
                unprotectedAddFile(name, blocks);
                break;
              }
            case OP_RENAME:
              {
                UTF8 src = new UTF8();
                UTF8 dst = new UTF8();
                src.readFields(in);
                dst.readFields(in);
                unprotectedRenameTo(src, dst);
                break;
              }
            case OP_DELETE:
              {
                UTF8 src = new UTF8();
                src.readFields(in);
                unprotectedDelete(src);
                break;
              }
            case OP_MKDIR:
              {
                UTF8 src = new UTF8();
                src.readFields(in);
                unprotectedMkdir(src.toString());
                break;
              }
            default:
              {
                throw new IOException("Never seen opcode " + opcode);
              }
          }
        }
      } finally {
        in.close();
      }
    }
    return numEdits;
  }
Beispiel #14
0
  /** Read a {@link Writable}, {@link String}, primitive type, or an array of the preceding. */
  @SuppressWarnings("unchecked")
  public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
      throws IOException {
    String className = UTF8.readString(in);
    Class<?> declaredClass = PRIMITIVE_NAMES.get(className);
    if (declaredClass == null) {
      declaredClass = loadClass(conf, className);
    }

    Object instance;

    if (declaredClass.isPrimitive()) { // primitive types

      if (declaredClass == Boolean.TYPE) { // boolean
        instance = Boolean.valueOf(in.readBoolean());
      } else if (declaredClass == Character.TYPE) { // char
        instance = Character.valueOf(in.readChar());
      } else if (declaredClass == Byte.TYPE) { // byte
        instance = Byte.valueOf(in.readByte());
      } else if (declaredClass == Short.TYPE) { // short
        instance = Short.valueOf(in.readShort());
      } else if (declaredClass == Integer.TYPE) { // int
        instance = Integer.valueOf(in.readInt());
      } else if (declaredClass == Long.TYPE) { // long
        instance = Long.valueOf(in.readLong());
      } else if (declaredClass == Float.TYPE) { // float
        instance = Float.valueOf(in.readFloat());
      } else if (declaredClass == Double.TYPE) { // double
        instance = Double.valueOf(in.readDouble());
      } else if (declaredClass == Void.TYPE) { // void
        instance = null;
      } else {
        throw new IllegalArgumentException("Not a primitive: " + declaredClass);
      }

    } else if (declaredClass.isArray()) { // array
      int length = in.readInt();
      instance = Array.newInstance(declaredClass.getComponentType(), length);
      for (int i = 0; i < length; i++) {
        Array.set(instance, i, readObject(in, conf));
      }

    } else if (declaredClass == String.class) { // String
      instance = UTF8.readString(in);
    } else if (declaredClass.isEnum()) { // enum
      instance = Enum.valueOf((Class<? extends Enum>) declaredClass, UTF8.readString(in));
    } else { // Writable
      Class instanceClass = null;
      String str = UTF8.readString(in);
      instanceClass = loadClass(conf, str);

      Writable writable = WritableFactories.newInstance(instanceClass, conf);
      writable.readFields(in);
      instance = writable;

      if (instanceClass == NullInstance.class) { // null
        declaredClass = ((NullInstance) instance).declaredClass;
        instance = null;
      }
    }

    if (objectWritable != null) { // store values
      objectWritable.declaredClass = declaredClass;
      objectWritable.instance = instance;
    }

    return instance;
  }
Beispiel #15
0
 public void write(DataOutput out) throws IOException {
   UTF8.writeString(out, declaredClass.getName());
 }