/** * If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially capped. If * ramDiskStorageLimit < 0 then it is ignored. */ protected final void startUpCluster( final int numDataNodes, final StorageType[] storageTypes, final long ramDiskStorageLimit, final boolean useSCR) throws IOException { Configuration conf = new Configuration(); conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); conf.setInt( DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC); conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL_SEC); conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, HEARTBEAT_RECHECK_INTERVAL_MSEC); conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, LAZY_WRITER_INTERVAL_SEC); if (useSCR) { conf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, useSCR); conf.set(DFS_CLIENT_CONTEXT, UUID.randomUUID().toString()); sockDir = new TemporarySocketDirectory(); conf.set( DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), this.getClass().getSimpleName() + "._PORT.sock") .getAbsolutePath()); conf.set( DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, UserGroupInformation.getCurrentUser().getShortUserName()); } cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDataNodes) .storageTypes( storageTypes != null ? storageTypes : new StorageType[] {DEFAULT, DEFAULT}) .build(); fs = cluster.getFileSystem(); client = fs.getClient(); // Artificially cap the storage capacity of the RAM_DISK volume. if (ramDiskStorageLimit >= 0) { List<? extends FsVolumeSpi> volumes = cluster.getDataNodes().get(0).getFSDataset().getVolumes(); for (FsVolumeSpi volume : volumes) { if (volume.getStorageType() == RAM_DISK) { ((FsVolumeImpl) volume).setCapacityForTesting(ramDiskStorageLimit); } } } LOG.info("Cluster startup complete"); }