/**
* Load in the filesystem image. It's a big list of filenames and blocks. Return whether we should
* "re-save" and consolidate the edit-logs
*/
boolean loadFSImage(File fsdir, File edits) throws IOException {
//
// Atomic move sequence, to recover from interrupted save
//
File curFile = new File(fsdir, FS_IMAGE);
File newFile = new File(fsdir, NEW_FS_IMAGE);
File oldFile = new File(fsdir, OLD_FS_IMAGE);
// Maybe we were interrupted between 2 and 4
if (oldFile.exists() && curFile.exists()) {
oldFile.delete();
if (edits.exists()) {
edits.delete();
}
} else if (oldFile.exists() && newFile.exists()) {
// Or maybe between 1 and 2
newFile.renameTo(curFile);
oldFile.delete();
} else if (curFile.exists() && newFile.exists()) {
// Or else before stage 1, in which case we lose the edits
newFile.delete();
}
//
// Load in bits
//
if (curFile.exists()) {
DataInputStream in =
new DataInputStream(new BufferedInputStream(new FileInputStream(curFile)));
try {
int numFiles = in.readInt();
for (int i = 0; i < numFiles; i++) {
UTF8 name = new UTF8();
name.readFields(in);
int numBlocks = in.readInt();
if (numBlocks == 0) {
unprotectedAddFile(name, null);
} else {
Block blocks[] = new Block[numBlocks];
for (int j = 0; j < numBlocks; j++) {
blocks[j] = new Block();
blocks[j].readFields(in);
}
unprotectedAddFile(name, blocks);
}
}
} finally {
in.close();
}
}
if (edits.exists() && loadFSEdits(edits) > 0) {
return true;
} else {
return false;
}
}
/** Write a {@link Writable}, {@link String}, primitive type, or an array of the preceding. */
public static void writeObject(
DataOutput out, Object instance, Class declaredClass, Configuration conf) throws IOException {
if (instance == null) { // null
instance = new NullInstance(declaredClass, conf);
declaredClass = Writable.class;
}
UTF8.writeString(out, declaredClass.getName()); // always write declared
if (declaredClass.isArray()) { // array
int length = Array.getLength(instance);
out.writeInt(length);
for (int i = 0; i < length; i++) {
writeObject(out, Array.get(instance, i), declaredClass.getComponentType(), conf);
}
} else if (declaredClass == String.class) { // String
UTF8.writeString(out, (String) instance);
} else if (declaredClass.isPrimitive()) { // primitive type
if (declaredClass == Boolean.TYPE) { // boolean
out.writeBoolean(((Boolean) instance).booleanValue());
} else if (declaredClass == Character.TYPE) { // char
out.writeChar(((Character) instance).charValue());
} else if (declaredClass == Byte.TYPE) { // byte
out.writeByte(((Byte) instance).byteValue());
} else if (declaredClass == Short.TYPE) { // short
out.writeShort(((Short) instance).shortValue());
} else if (declaredClass == Integer.TYPE) { // int
out.writeInt(((Integer) instance).intValue());
} else if (declaredClass == Long.TYPE) { // long
out.writeLong(((Long) instance).longValue());
} else if (declaredClass == Float.TYPE) { // float
out.writeFloat(((Float) instance).floatValue());
} else if (declaredClass == Double.TYPE) { // double
out.writeDouble(((Double) instance).doubleValue());
} else if (declaredClass == Void.TYPE) { // void
} else {
throw new IllegalArgumentException("Not a primitive: " + declaredClass);
}
} else if (declaredClass.isEnum()) { // enum
UTF8.writeString(out, ((Enum) instance).name());
} else if (Writable.class.isAssignableFrom(declaredClass)) { // Writable
UTF8.writeString(out, instance.getClass().getName());
((Writable) instance).write(out);
} else {
throw new IOException("Can't write: " + instance + " as " + declaredClass);
}
}
public void write(DataOutput out) throws IOException {
out.write(BASE_TYPE);
out.writeUTF(name);
out.writeInt(sampleStrs.size());
for (int i = 0; i < sampleStrs.size(); i++) {
UTF8.writeString(out, sampleStrs.get(i));
}
out.writeInt(tokenClassIdentifier);
out.writeBoolean(tokenParameter != null);
if (tokenParameter != null) {
UTF8.writeString(out, tokenParameter);
}
}
public void readFields(DataInput in) throws IOException {
// instance-specific
this.sampleStrs = new ArrayList<String>();
int numSamples = in.readInt();
for (int i = 0; i < numSamples; i++) {
sampleStrs.add(UTF8.readString(in).toString());
}
this.tokenClassIdentifier = in.readInt();
if (in.readBoolean()) {
this.tokenParameter = UTF8.readString(in);
} else {
this.tokenParameter = null;
}
this.schema = computeAvroSchema();
}
String normalizePath(UTF8 src) {
String srcs = src.toString();
if (srcs.length() > 1 && srcs.endsWith("/")) {
srcs = srcs.substring(0, srcs.length() - 1);
}
return srcs;
}
/** Get the blocks associated with the file */
public Block[] getFile(UTF8 src) {
waitForReady();
synchronized (rootDir) {
INode targetNode = rootDir.getNode(src.toString());
if (targetNode == null) {
return null;
} else {
return targetNode.blocks;
}
}
}
boolean unprotectedAddFile(UTF8 name, Block blocks[]) {
synchronized (rootDir) {
if (blocks != null) {
// Add file->block mapping
for (int i = 0; i < blocks.length; i++) {
activeBlocks.add(blocks[i]);
}
}
return (rootDir.addNode(name.toString(), blocks) != null);
}
}
public void readFields(DataInput in) throws IOException {
String className = UTF8.readString(in);
declaredClass = PRIMITIVE_NAMES.get(className);
if (declaredClass == null) {
try {
declaredClass = getConf().getClassByName(className);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e.toString());
}
}
}
/** Add the given filename to the fs. */
public boolean addFile(UTF8 src, Block blocks[]) {
waitForReady();
// Always do an implicit mkdirs for parent directory tree
mkdirs(DFSFile.getDFSParent(src.toString()));
if (unprotectedAddFile(src, blocks)) {
logEdit(OP_ADD, src, new ArrayWritable(Block.class, blocks));
return true;
} else {
return false;
}
}
boolean unprotectedRenameTo(UTF8 src, UTF8 dst) {
synchronized (rootDir) {
INode removedNode = rootDir.getNode(src.toString());
if (removedNode == null) {
return false;
}
removedNode.removeNode();
if (isDir(dst)) {
dst = new UTF8(dst.toString() + "/" + new File(src.toString()).getName());
}
INode newNode = rootDir.addNode(dst.toString(), removedNode.blocks);
if (newNode != null) {
newNode.children = removedNode.children;
for (Iterator it = newNode.children.values().iterator(); it.hasNext(); ) {
INode child = (INode) it.next();
child.parent = newNode;
}
return true;
} else {
rootDir.addNode(src.toString(), removedNode.blocks);
return false;
}
}
}
Block[] unprotectedDelete(UTF8 src) {
synchronized (rootDir) {
INode targetNode = rootDir.getNode(src.toString());
if (targetNode == null) {
return null;
} else {
//
// Remove the node from the namespace and GC all
// the blocks underneath the node.
//
if (!targetNode.removeNode()) {
return null;
} else {
Vector v = new Vector();
targetNode.collectSubtreeBlocks(v);
for (Iterator it = v.iterator(); it.hasNext(); ) {
Block b = (Block) it.next();
activeBlocks.remove(b);
}
return (Block[]) v.toArray(new Block[v.size()]);
}
}
}
}
/** Create the given directory and all its parent dirs. */
public boolean mkdirs(UTF8 src) {
return mkdirs(src.toString());
}
/**
* Load an edit log, and apply the changes to the in-memory structure
*
* <p>This is where we apply edits that we've been writing to disk all along.
*/
int loadFSEdits(File edits) throws IOException {
int numEdits = 0;
if (edits.exists()) {
DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(edits)));
try {
while (in.available() > 0) {
byte opcode = in.readByte();
numEdits++;
switch (opcode) {
case OP_ADD:
{
UTF8 name = new UTF8();
name.readFields(in);
ArrayWritable aw = new ArrayWritable(Block.class);
aw.readFields(in);
Writable writables[] = (Writable[]) aw.get();
Block blocks[] = new Block[writables.length];
System.arraycopy(writables, 0, blocks, 0, blocks.length);
unprotectedAddFile(name, blocks);
break;
}
case OP_RENAME:
{
UTF8 src = new UTF8();
UTF8 dst = new UTF8();
src.readFields(in);
dst.readFields(in);
unprotectedRenameTo(src, dst);
break;
}
case OP_DELETE:
{
UTF8 src = new UTF8();
src.readFields(in);
unprotectedDelete(src);
break;
}
case OP_MKDIR:
{
UTF8 src = new UTF8();
src.readFields(in);
unprotectedMkdir(src.toString());
break;
}
default:
{
throw new IOException("Never seen opcode " + opcode);
}
}
}
} finally {
in.close();
}
}
return numEdits;
}
/** Read a {@link Writable}, {@link String}, primitive type, or an array of the preceding. */
@SuppressWarnings("unchecked")
public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
throws IOException {
String className = UTF8.readString(in);
Class<?> declaredClass = PRIMITIVE_NAMES.get(className);
if (declaredClass == null) {
declaredClass = loadClass(conf, className);
}
Object instance;
if (declaredClass.isPrimitive()) { // primitive types
if (declaredClass == Boolean.TYPE) { // boolean
instance = Boolean.valueOf(in.readBoolean());
} else if (declaredClass == Character.TYPE) { // char
instance = Character.valueOf(in.readChar());
} else if (declaredClass == Byte.TYPE) { // byte
instance = Byte.valueOf(in.readByte());
} else if (declaredClass == Short.TYPE) { // short
instance = Short.valueOf(in.readShort());
} else if (declaredClass == Integer.TYPE) { // int
instance = Integer.valueOf(in.readInt());
} else if (declaredClass == Long.TYPE) { // long
instance = Long.valueOf(in.readLong());
} else if (declaredClass == Float.TYPE) { // float
instance = Float.valueOf(in.readFloat());
} else if (declaredClass == Double.TYPE) { // double
instance = Double.valueOf(in.readDouble());
} else if (declaredClass == Void.TYPE) { // void
instance = null;
} else {
throw new IllegalArgumentException("Not a primitive: " + declaredClass);
}
} else if (declaredClass.isArray()) { // array
int length = in.readInt();
instance = Array.newInstance(declaredClass.getComponentType(), length);
for (int i = 0; i < length; i++) {
Array.set(instance, i, readObject(in, conf));
}
} else if (declaredClass == String.class) { // String
instance = UTF8.readString(in);
} else if (declaredClass.isEnum()) { // enum
instance = Enum.valueOf((Class<? extends Enum>) declaredClass, UTF8.readString(in));
} else { // Writable
Class instanceClass = null;
String str = UTF8.readString(in);
instanceClass = loadClass(conf, str);
Writable writable = WritableFactories.newInstance(instanceClass, conf);
writable.readFields(in);
instance = writable;
if (instanceClass == NullInstance.class) { // null
declaredClass = ((NullInstance) instance).declaredClass;
instance = null;
}
}
if (objectWritable != null) { // store values
objectWritable.declaredClass = declaredClass;
objectWritable.instance = instance;
}
return instance;
}
public void write(DataOutput out) throws IOException {
UTF8.writeString(out, declaredClass.getName());
}