public static void ensureExists(File thing, String resource) { System.err.println("configfile = " + thing); if (!thing.exists()) { try { System.err.println("Creating: " + thing + " from " + resource); if (resource.indexOf("/") != 0) { resource = "/" + resource; } InputStream is = Config.class.getResourceAsStream(resource); if (is == null) { throw new NullPointerException("Can't find resource: " + resource); } getParentFile(thing).mkdirs(); OutputStream os = new FileOutputStream(thing); for (int next = is.read(); next != -1; next = is.read()) { os.write(next); } os.flush(); os.close(); } catch (FileNotFoundException fnfe) { throw new Error("Can't create resource: " + fnfe.getMessage()); } catch (IOException ioe) { throw new Error("Can't create resource: " + ioe.getMessage()); } } }
public void loadPropfile(File file) { propFile = file; try { FileInputStream in = new FileInputStream(propFile); prop.load(in); in.close(); System.out.println("Util(): Using properties file '" + propFile + "'."); } catch (FileNotFoundException exc) { propFile = null; System.out.println("Util(): FileNotFoundException " + exc.getMessage()); } catch (IOException exc) { propFile = null; System.out.println("Util(): IOException " + exc.getMessage()); } }
/** Load index data from .fbi file to {@link #indexData}. */ private void loadIndex() { // Loading .fbi makes sense only if both .fbi and .fbk files are available. if (fbiURL != null && fbkURL != null) { FbsEntryPoint[] newIndex; int numRecordsRead = 0; byte[] newInitData; try { // Connect. URLConnection connection = fbiURL.openConnection(); connection.connect(); DataInputStream is = new DataInputStream(connection.getInputStream()); // Check file signature. byte[] b = new byte[12]; is.readFully(b); if (b[0] != 'F' || b[1] != 'B' || b[2] != 'I' || b[3] != ' ' || b[4] != '0' || b[5] != '0' || b[6] != '1' || b[7] != '.' || b[8] < '0' || b[8] > '9' || b[9] < '0' || b[9] > '9' || b[10] < '0' || b[10] > '9' || b[11] != '\n') { System.err.println("Could not load index: bad .fbi file signature"); return; } // Read the record counter and allocate index array. int numRecords = is.readInt(); if (numRecords <= 0) { System.err.println("Could not load index: bad .fbi record counter"); return; } newIndex = new FbsEntryPoint[numRecords]; // Read byte counter and allocate byte array for RFB initialization. int initSize = is.readInt(); if (initSize <= 0) { System.err.println("Could not load index: bad RFB init data size"); return; } newInitData = new byte[initSize]; // Load index from the .fbi file. try { for (int i = 0; i < numRecords; i++) { FbsEntryPoint record = new FbsEntryPoint(); record.timestamp = (long) is.readInt() & 0xFFFFFFFFL; record.key_fpos = (long) is.readInt() & 0xFFFFFFFFL; record.key_size = (long) is.readInt() & 0xFFFFFFFFL; record.fbs_fpos = (long) is.readInt() & 0xFFFFFFFFL; record.fbs_skip = (long) is.readInt() & 0xFFFFFFFFL; newIndex[i] = record; numRecordsRead++; } } catch (EOFException e) { System.err.println("Preliminary end of .fbi file"); } catch (IOException e) { System.err.println("Ignored exception: " + e); } if (numRecordsRead == 0) { System.err.println("Could not load index: failed to read .fbi data"); return; } else if (numRecordsRead != numRecords) { System.err.println("Warning: read not as much .fbi data as expected"); } is.readFully(newInitData); } catch (FileNotFoundException e) { System.err.println("Could not load index: .fbi file not found: " + e.getMessage()); return; } catch (IOException e) { System.err.println(e); System.err.println("Could not load index: failed to load .fbi file"); return; } // Check correctness of the data read. for (int i = 1; i < numRecordsRead; i++) { if (newIndex[i].timestamp <= newIndex[i - 1].timestamp) { System.err.println("Could not load index: wrong .fbi file contents"); return; } } // Loaded successfully. indexData = newIndex; numIndexRecords = numRecordsRead; rfbInitData = newInitData; System.err.println("Loaded index data, " + numRecordsRead + " records"); } }
/** Tests mod & access time in DFS. */ public void testTimes() throws IOException { Configuration conf = new HdfsConfiguration(); final int MAX_IDLE_TIME = 2000; // 2s conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build(); cluster.waitActive(); final int nnport = cluster.getNameNodePort(); InetSocketAddress addr = new InetSocketAddress("localhost", cluster.getNameNodePort()); DFSClient client = new DFSClient(addr, conf); DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); assertEquals("Number of Datanodes ", numDatanodes, info.length); FileSystem fileSys = cluster.getFileSystem(); int replicas = 1; assertTrue(fileSys instanceof DistributedFileSystem); try { // // create file and record atime/mtime // System.out.println("Creating testdir1 and testdir1/test1.dat."); Path dir1 = new Path("testdir1"); Path file1 = new Path(dir1, "test1.dat"); FSDataOutputStream stm = writeFile(fileSys, file1, replicas); FileStatus stat = fileSys.getFileStatus(file1); long atimeBeforeClose = stat.getAccessTime(); String adate = dateForm.format(new Date(atimeBeforeClose)); System.out.println( "atime on " + file1 + " before close is " + adate + " (" + atimeBeforeClose + ")"); assertTrue(atimeBeforeClose != 0); stm.close(); stat = fileSys.getFileStatus(file1); long atime1 = stat.getAccessTime(); long mtime1 = stat.getModificationTime(); adate = dateForm.format(new Date(atime1)); String mdate = dateForm.format(new Date(mtime1)); System.out.println("atime on " + file1 + " is " + adate + " (" + atime1 + ")"); System.out.println("mtime on " + file1 + " is " + mdate + " (" + mtime1 + ")"); assertTrue(atime1 != 0); // // record dir times // stat = fileSys.getFileStatus(dir1); long mdir1 = stat.getAccessTime(); assertTrue(mdir1 == 0); // set the access time to be one day in the past long atime2 = atime1 - (24L * 3600L * 1000L); fileSys.setTimes(file1, -1, atime2); // check new access time on file stat = fileSys.getFileStatus(file1); long atime3 = stat.getAccessTime(); String adate3 = dateForm.format(new Date(atime3)); System.out.println("new atime on " + file1 + " is " + adate3 + " (" + atime3 + ")"); assertTrue(atime2 == atime3); assertTrue(mtime1 == stat.getModificationTime()); // set the modification time to be 1 hour in the past long mtime2 = mtime1 - (3600L * 1000L); fileSys.setTimes(file1, mtime2, -1); // check new modification time on file stat = fileSys.getFileStatus(file1); long mtime3 = stat.getModificationTime(); String mdate3 = dateForm.format(new Date(mtime3)); System.out.println("new mtime on " + file1 + " is " + mdate3 + " (" + mtime3 + ")"); assertTrue(atime2 == stat.getAccessTime()); assertTrue(mtime2 == mtime3); long mtime4 = System.currentTimeMillis() - (3600L * 1000L); long atime4 = System.currentTimeMillis(); fileSys.setTimes(dir1, mtime4, atime4); // check new modification time on file stat = fileSys.getFileStatus(dir1); assertTrue("Not matching the modification times", mtime4 == stat.getModificationTime()); assertTrue("Not matching the access times", atime4 == stat.getAccessTime()); Path nonExistingDir = new Path(dir1, "/nonExistingDir/"); try { fileSys.setTimes(nonExistingDir, mtime4, atime4); fail("Expecting FileNotFoundException"); } catch (FileNotFoundException e) { assertTrue( e.getMessage() .contains("File/Directory " + nonExistingDir.toString() + " does not exist.")); } // shutdown cluster and restart cluster.shutdown(); try { Thread.sleep(2 * MAX_IDLE_TIME); } catch (InterruptedException e) { } cluster = new MiniDFSCluster.Builder(conf).nameNodePort(nnport).format(false).build(); cluster.waitActive(); fileSys = cluster.getFileSystem(); // verify that access times and modification times persist after a // cluster restart. System.out.println("Verifying times after cluster restart"); stat = fileSys.getFileStatus(file1); assertTrue(atime2 == stat.getAccessTime()); assertTrue(mtime3 == stat.getModificationTime()); cleanupFile(fileSys, file1); cleanupFile(fileSys, dir1); } catch (IOException e) { info = client.datanodeReport(DatanodeReportType.ALL); printDatanodeReport(info); throw e; } finally { fileSys.close(); cluster.shutdown(); } }