public static void ensureExists(File thing, String resource) {
System.err.println("configfile = " + thing);
if (!thing.exists()) {
try {
System.err.println("Creating: " + thing + " from " + resource);
if (resource.indexOf("/") != 0) {
resource = "/" + resource;
}
InputStream is = Config.class.getResourceAsStream(resource);
if (is == null) {
throw new NullPointerException("Can't find resource: " + resource);
}
getParentFile(thing).mkdirs();
OutputStream os = new FileOutputStream(thing);
for (int next = is.read(); next != -1; next = is.read()) {
os.write(next);
}
os.flush();
os.close();
} catch (FileNotFoundException fnfe) {
throw new Error("Can't create resource: " + fnfe.getMessage());
} catch (IOException ioe) {
throw new Error("Can't create resource: " + ioe.getMessage());
}
}
}
public void loadPropfile(File file) {
propFile = file;
try {
FileInputStream in = new FileInputStream(propFile);
prop.load(in);
in.close();
System.out.println("Util(): Using properties file '" + propFile + "'.");
} catch (FileNotFoundException exc) {
propFile = null;
System.out.println("Util(): FileNotFoundException " + exc.getMessage());
} catch (IOException exc) {
propFile = null;
System.out.println("Util(): IOException " + exc.getMessage());
}
}
/** Load index data from .fbi file to {@link #indexData}. */
private void loadIndex() {
// Loading .fbi makes sense only if both .fbi and .fbk files are available.
if (fbiURL != null && fbkURL != null) {
FbsEntryPoint[] newIndex;
int numRecordsRead = 0;
byte[] newInitData;
try {
// Connect.
URLConnection connection = fbiURL.openConnection();
connection.connect();
DataInputStream is = new DataInputStream(connection.getInputStream());
// Check file signature.
byte[] b = new byte[12];
is.readFully(b);
if (b[0] != 'F'
|| b[1] != 'B'
|| b[2] != 'I'
|| b[3] != ' '
|| b[4] != '0'
|| b[5] != '0'
|| b[6] != '1'
|| b[7] != '.'
|| b[8] < '0'
|| b[8] > '9'
|| b[9] < '0'
|| b[9] > '9'
|| b[10] < '0'
|| b[10] > '9'
|| b[11] != '\n') {
System.err.println("Could not load index: bad .fbi file signature");
return;
}
// Read the record counter and allocate index array.
int numRecords = is.readInt();
if (numRecords <= 0) {
System.err.println("Could not load index: bad .fbi record counter");
return;
}
newIndex = new FbsEntryPoint[numRecords];
// Read byte counter and allocate byte array for RFB initialization.
int initSize = is.readInt();
if (initSize <= 0) {
System.err.println("Could not load index: bad RFB init data size");
return;
}
newInitData = new byte[initSize];
// Load index from the .fbi file.
try {
for (int i = 0; i < numRecords; i++) {
FbsEntryPoint record = new FbsEntryPoint();
record.timestamp = (long) is.readInt() & 0xFFFFFFFFL;
record.key_fpos = (long) is.readInt() & 0xFFFFFFFFL;
record.key_size = (long) is.readInt() & 0xFFFFFFFFL;
record.fbs_fpos = (long) is.readInt() & 0xFFFFFFFFL;
record.fbs_skip = (long) is.readInt() & 0xFFFFFFFFL;
newIndex[i] = record;
numRecordsRead++;
}
} catch (EOFException e) {
System.err.println("Preliminary end of .fbi file");
} catch (IOException e) {
System.err.println("Ignored exception: " + e);
}
if (numRecordsRead == 0) {
System.err.println("Could not load index: failed to read .fbi data");
return;
} else if (numRecordsRead != numRecords) {
System.err.println("Warning: read not as much .fbi data as expected");
}
is.readFully(newInitData);
} catch (FileNotFoundException e) {
System.err.println("Could not load index: .fbi file not found: " + e.getMessage());
return;
} catch (IOException e) {
System.err.println(e);
System.err.println("Could not load index: failed to load .fbi file");
return;
}
// Check correctness of the data read.
for (int i = 1; i < numRecordsRead; i++) {
if (newIndex[i].timestamp <= newIndex[i - 1].timestamp) {
System.err.println("Could not load index: wrong .fbi file contents");
return;
}
}
// Loaded successfully.
indexData = newIndex;
numIndexRecords = numRecordsRead;
rfbInitData = newInitData;
System.err.println("Loaded index data, " + numRecordsRead + " records");
}
}
/** Tests mod & access time in DFS. */
public void testTimes() throws IOException {
Configuration conf = new HdfsConfiguration();
final int MAX_IDLE_TIME = 2000; // 2s
conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
cluster.waitActive();
final int nnport = cluster.getNameNodePort();
InetSocketAddress addr = new InetSocketAddress("localhost", cluster.getNameNodePort());
DFSClient client = new DFSClient(addr, conf);
DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE);
assertEquals("Number of Datanodes ", numDatanodes, info.length);
FileSystem fileSys = cluster.getFileSystem();
int replicas = 1;
assertTrue(fileSys instanceof DistributedFileSystem);
try {
//
// create file and record atime/mtime
//
System.out.println("Creating testdir1 and testdir1/test1.dat.");
Path dir1 = new Path("testdir1");
Path file1 = new Path(dir1, "test1.dat");
FSDataOutputStream stm = writeFile(fileSys, file1, replicas);
FileStatus stat = fileSys.getFileStatus(file1);
long atimeBeforeClose = stat.getAccessTime();
String adate = dateForm.format(new Date(atimeBeforeClose));
System.out.println(
"atime on " + file1 + " before close is " + adate + " (" + atimeBeforeClose + ")");
assertTrue(atimeBeforeClose != 0);
stm.close();
stat = fileSys.getFileStatus(file1);
long atime1 = stat.getAccessTime();
long mtime1 = stat.getModificationTime();
adate = dateForm.format(new Date(atime1));
String mdate = dateForm.format(new Date(mtime1));
System.out.println("atime on " + file1 + " is " + adate + " (" + atime1 + ")");
System.out.println("mtime on " + file1 + " is " + mdate + " (" + mtime1 + ")");
assertTrue(atime1 != 0);
//
// record dir times
//
stat = fileSys.getFileStatus(dir1);
long mdir1 = stat.getAccessTime();
assertTrue(mdir1 == 0);
// set the access time to be one day in the past
long atime2 = atime1 - (24L * 3600L * 1000L);
fileSys.setTimes(file1, -1, atime2);
// check new access time on file
stat = fileSys.getFileStatus(file1);
long atime3 = stat.getAccessTime();
String adate3 = dateForm.format(new Date(atime3));
System.out.println("new atime on " + file1 + " is " + adate3 + " (" + atime3 + ")");
assertTrue(atime2 == atime3);
assertTrue(mtime1 == stat.getModificationTime());
// set the modification time to be 1 hour in the past
long mtime2 = mtime1 - (3600L * 1000L);
fileSys.setTimes(file1, mtime2, -1);
// check new modification time on file
stat = fileSys.getFileStatus(file1);
long mtime3 = stat.getModificationTime();
String mdate3 = dateForm.format(new Date(mtime3));
System.out.println("new mtime on " + file1 + " is " + mdate3 + " (" + mtime3 + ")");
assertTrue(atime2 == stat.getAccessTime());
assertTrue(mtime2 == mtime3);
long mtime4 = System.currentTimeMillis() - (3600L * 1000L);
long atime4 = System.currentTimeMillis();
fileSys.setTimes(dir1, mtime4, atime4);
// check new modification time on file
stat = fileSys.getFileStatus(dir1);
assertTrue("Not matching the modification times", mtime4 == stat.getModificationTime());
assertTrue("Not matching the access times", atime4 == stat.getAccessTime());
Path nonExistingDir = new Path(dir1, "/nonExistingDir/");
try {
fileSys.setTimes(nonExistingDir, mtime4, atime4);
fail("Expecting FileNotFoundException");
} catch (FileNotFoundException e) {
assertTrue(
e.getMessage()
.contains("File/Directory " + nonExistingDir.toString() + " does not exist."));
}
// shutdown cluster and restart
cluster.shutdown();
try {
Thread.sleep(2 * MAX_IDLE_TIME);
} catch (InterruptedException e) {
}
cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport).format(false).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
// verify that access times and modification times persist after a
// cluster restart.
System.out.println("Verifying times after cluster restart");
stat = fileSys.getFileStatus(file1);
assertTrue(atime2 == stat.getAccessTime());
assertTrue(mtime3 == stat.getModificationTime());
cleanupFile(fileSys, file1);
cleanupFile(fileSys, dir1);
} catch (IOException e) {
info = client.datanodeReport(DatanodeReportType.ALL);
printDatanodeReport(info);
throw e;
} finally {
fileSys.close();
cluster.shutdown();
}
}
