/**
* The list operation randomly picks a directory in the test space and list the directory
* content.
*/
private void list() throws IOException {
String dirName = dirs.get(r.nextInt(dirs.size()));
long startTime = Time.now();
fc.listStatus(new Path(dirName));
executionTime[LIST] += (Time.now() - startTime);
totalNumOfOps[LIST]++;
}
/** Create a file with a length of <code>fileSize</code>. The file is filled with 'a'. */
private void genFile(Path file, long fileSize) throws IOException {
long startTime = Time.now();
FSDataOutputStream out = null;
try {
out =
fc.create(
file,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
CreateOpts.createParent(),
CreateOpts.bufferSize(4096),
CreateOpts.repFac((short) 3));
executionTime[CREATE] += (Time.now() - startTime);
numOfOps[CREATE]++;
long i = fileSize;
while (i > 0) {
long s = Math.min(fileSize, WRITE_CONTENTS.length);
out.write(WRITE_CONTENTS, 0, (int) s);
i -= s;
}
startTime = Time.now();
executionTime[WRITE_CLOSE] += (Time.now() - startTime);
numOfOps[WRITE_CLOSE]++;
} finally {
IOUtils.cleanup(LOG, out);
}
}
@Test
public void sleepRatio2() {
setWaitForRatio(1);
long start = Time.now();
sleep(100);
long end = Time.now();
assertEquals(end - start, 100 * getWaitForRatio(), 50 * getWaitForRatio());
}
/** Read operation randomly picks a file in the test space and reads the entire file */
private void read() throws IOException {
String fileName = files.get(r.nextInt(files.size()));
long startTime = Time.now();
InputStream in = fc.open(new Path(fileName));
executionTime[OPEN] += (Time.now() - startTime);
totalNumOfOps[OPEN]++;
while (in.read(buffer) != -1) {}
in.close();
}
/**
* The write operation randomly picks a directory in the test space and creates a file whose
* name consists of the current machine's host name and the thread id. The length of the file
* follows Gaussian distribution with an average size of 2 blocks and the standard deviation of
* 1 block. The new file is filled with 'a'. Immediately after the file creation completes, the
* file is deleted from the test space.
*/
private void write() throws IOException {
String dirName = dirs.get(r.nextInt(dirs.size()));
Path file = new Path(dirName, hostname + id);
double fileSize = 0;
while ((fileSize = r.nextGaussian() + 2) <= 0) {}
genFile(file, (long) (fileSize * BLOCK_SIZE));
long startTime = Time.now();
fc.delete(file, true);
executionTime[DELETE] += (Time.now() - startTime);
totalNumOfOps[DELETE]++;
}
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
long startTime = 0;
if (LOG.isDebugEnabled()) {
startTime = Time.now();
}
ObjectWritable value =
(ObjectWritable)
client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId);
if (LOG.isDebugEnabled()) {
long callTime = Time.now() - startTime;
LOG.debug("Call: " + method.getName() + " " + callTime);
}
return value.get();
}
/**
* Read in the cached DU value and return it if it is less than 600 seconds old (DU update
* interval). Slight imprecision of dfsUsed is not critical and skipping DU can significantly
* shorten the startup time. If the cached value is not available or too old, -1 is returned.
*/
long loadDfsUsed() {
long cachedDfsUsed;
long mtime;
Scanner sc;
try {
sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8");
} catch (FileNotFoundException fnfe) {
return -1;
}
try {
// Get the recorded dfsUsed from the file.
if (sc.hasNextLong()) {
cachedDfsUsed = sc.nextLong();
} else {
return -1;
}
// Get the recorded mtime from the file.
if (sc.hasNextLong()) {
mtime = sc.nextLong();
} else {
return -1;
}
// Return the cached value if mtime is okay.
if (mtime > 0 && (Time.now() - mtime < 600000L)) {
FsDatasetImpl.LOG.info("Cached dfsUsed found for " + currentDir + ": " + cachedDfsUsed);
return cachedDfsUsed;
}
return -1;
} finally {
sc.close();
}
}
private void requestNewHdfsDelegationToken(
ApplicationId applicationId, String user, boolean shouldCancelAtEnd)
throws IOException, InterruptedException {
// Get new hdfs tokens for this user
Credentials credentials = new Credentials();
Token<?>[] newTokens = obtainSystemTokensForUser(user, credentials);
// Add new tokens to the toRenew list.
LOG.info(
"Received new tokens for " + applicationId + ". Received " + newTokens.length + " tokens.");
if (newTokens.length > 0) {
for (Token<?> token : newTokens) {
if (token.isManaged()) {
DelegationTokenToRenew tokenToRenew =
new DelegationTokenToRenew(
applicationId, token, getConfig(), Time.now(), shouldCancelAtEnd, user);
// renew the token to get the next expiration date.
renewToken(tokenToRenew);
setTimerForTokenRenewal(tokenToRenew);
appTokens.get(applicationId).add(tokenToRenew);
LOG.info("Received new token " + token);
}
}
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer byteBuffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
rmContext.getSystemCredentialsForApps().put(applicationId, byteBuffer);
}
/** Add a snapshot. */
Snapshot addSnapshot(int id, String name) throws SnapshotException, QuotaExceededException {
// check snapshot quota
final int n = getNumSnapshots();
if (n + 1 > snapshotQuota) {
throw new SnapshotException(
"Failed to add snapshot: there are already "
+ n
+ " snapshot(s) and the snapshot quota is "
+ snapshotQuota);
}
final Snapshot s = new Snapshot(id, name, this);
final byte[] nameBytes = s.getRoot().getLocalNameBytes();
final int i = searchSnapshot(nameBytes);
if (i >= 0) {
throw new SnapshotException(
"Failed to add snapshot: there is already a "
+ "snapshot with the same name \""
+ Snapshot.getSnapshotName(s)
+ "\".");
}
final DirectoryDiff d = getDiffs().addDiff(s, this);
d.snapshotINode = s.getRoot();
snapshotsByNames.add(-i - 1, s);
// set modification time
updateModificationTime(Time.now(), null, null);
s.getRoot().setModificationTime(getModificationTime(), null, null);
return s;
}
@Test
public void waitFor() {
long start = Time.now();
long waited =
waitFor(
1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return true;
}
});
long end = Time.now();
assertEquals(waited, 0, 50);
assertEquals(end - start - waited, 0, 50);
}
@Test
public void waitForTimeOutRatio2() {
setWaitForRatio(2);
long start = Time.now();
long waited =
waitFor(
200,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return false;
}
});
long end = Time.now();
assertEquals(waited, -1);
assertEquals(end - start, 200 * getWaitForRatio(), 50 * getWaitForRatio());
}
/**
* Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
* that are supported by the server
*
* @param protocol protocol class
* @param clientVersion client version
* @param addr remote address
* @param conf configuration to use
* @param rpcTimeout timeout for each RPC
* @param timeout time in milliseconds before giving up
* @return the proxy
* @throws IOException if the far end through a RemoteException
*/
public static <T> ProtocolProxy<T> waitForProtocolProxy(
Class<T> protocol,
long clientVersion,
InetSocketAddress addr,
Configuration conf,
int rpcTimeout,
RetryPolicy connectionRetryPolicy,
long timeout)
throws IOException {
long startTime = Time.now();
IOException ioe;
while (true) {
try {
return getProtocolProxy(
protocol,
clientVersion,
addr,
UserGroupInformation.getCurrentUser(),
conf,
NetUtils.getDefaultSocketFactory(conf),
rpcTimeout,
connectionRetryPolicy);
} catch (ConnectException se) { // namenode has not been started
LOG.info("Server at " + addr + " not available yet, Zzzzz...");
ioe = se;
} catch (SocketTimeoutException te) { // namenode is busy
LOG.info("Problem connecting to server: " + addr);
ioe = te;
} catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over
LOG.info("No route to host for server: " + addr);
ioe = nrthe;
}
// check if timed out
if (Time.now() - timeout >= startTime) {
throw ioe;
}
// wait for retry
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
// IGNORE
}
}
}
/**
* Returns when the current number of seconds from the epoch equals the command line argument
* given by <code>-startTime</code>. This allows multiple instances of this program, running on
* clock synchronized nodes, to start at roughly the same time.
*/
private static void barrier() {
long sleepTime;
while ((sleepTime = startTime - Time.now()) > 0) {
try {
Thread.sleep(sleepTime);
} catch (InterruptedException ex) {
}
}
}
static String newBlockPoolID() throws UnknownHostException {
String ip = "unknownIP";
try {
ip = DNS.getDefaultIP("default");
} catch (UnknownHostException e) {
System.out.println("Could not find ip address of \"default\" inteface.");
throw e;
}
int rand = DFSUtil.getSecureRandom().nextInt(Integer.MAX_VALUE);
String bpid = "BP-" + rand + "-" + ip + "-" + Time.now();
return bpid;
}
/**
* Checks whether {@link DataNode#checkDiskErrorAsync()} is being called or not. Before
* refactoring the code the above function was not getting called
*
* @throws IOException, InterruptedException
*/
@Test
public void testcheckDiskError() throws IOException, InterruptedException {
if (cluster.getDataNodes().size() <= 0) {
cluster.startDataNodes(conf, 1, true, null, null);
cluster.waitActive();
}
DataNode dataNode = cluster.getDataNodes().get(0);
long slackTime = dataNode.checkDiskErrorInterval / 2;
// checking for disk error
dataNode.checkDiskErrorAsync();
Thread.sleep(dataNode.checkDiskErrorInterval);
long lastDiskErrorCheck = dataNode.getLastDiskErrorCheck();
assertTrue(
"Disk Error check is not performed within " + dataNode.checkDiskErrorInterval + " ms",
((Time.monotonicNow() - lastDiskErrorCheck)
< (dataNode.checkDiskErrorInterval + slackTime)));
}
/** Write the current dfsUsed to the cache file. */
void saveDfsUsed() {
File outFile = new File(currentDir, DU_CACHE_FILE);
if (outFile.exists() && !outFile.delete()) {
FsDatasetImpl.LOG.warn("Failed to delete old dfsUsed file in " + outFile.getParent());
}
try {
long used = getDfsUsed();
try (Writer out = new OutputStreamWriter(new FileOutputStream(outFile), "UTF-8")) {
// mtime is written last, so that truncated writes won't be valid.
out.write(Long.toString(used) + " " + Long.toString(Time.now()));
out.flush();
}
} catch (IOException ioe) {
// If write failed, the volume might be bad. Since the cache file is
// not critical, log the error and continue.
FsDatasetImpl.LOG.warn("Failed to write dfsUsed to " + outFile, ioe);
}
}
/**
* Keeps a Collection for every named machine containing blocks that have recently been invalidated
* and are thought to live on the machine in question.
*/
@InterfaceAudience.Private
class InvalidateBlocks {
/** Mapping: DatanodeInfo -> Collection of Blocks */
private final Map<DatanodeInfo, LightWeightHashSet<Block>> node2blocks =
new TreeMap<DatanodeInfo, LightWeightHashSet<Block>>();
/** The total number of blocks in the map. */
private long numBlocks = 0L;
private final int blockInvalidateLimit;
/** The period of pending time for block invalidation since the NameNode startup */
private final long pendingPeriodInMs;
/** the startup time */
private final long startupTime = Time.monotonicNow();
InvalidateBlocks(final int blockInvalidateLimit, long pendingPeriodInMs) {
this.blockInvalidateLimit = blockInvalidateLimit;
this.pendingPeriodInMs = pendingPeriodInMs;
printBlockDeletionTime(BlockManager.LOG);
}
private void printBlockDeletionTime(final Log log) {
log.info(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY
+ " is set to "
+ DFSUtil.durationToString(pendingPeriodInMs));
SimpleDateFormat sdf = new SimpleDateFormat("yyyy MMM dd HH:mm:ss");
Calendar calendar = new GregorianCalendar();
calendar.add(Calendar.SECOND, (int) (this.pendingPeriodInMs / 1000));
log.info("The block deletion will start around " + sdf.format(calendar.getTime()));
}
/** @return the number of blocks to be invalidated . */
synchronized long numBlocks() {
return numBlocks;
}
/**
* @return true if the given storage has the given block listed for invalidation. Blocks are
* compared including their generation stamps: if a block is pending invalidation but with a
* different generation stamp, returns false.
*/
synchronized boolean contains(final DatanodeInfo dn, final Block block) {
final LightWeightHashSet<Block> s = node2blocks.get(dn);
if (s == null) {
return false; // no invalidate blocks for this storage ID
}
Block blockInSet = s.getElement(block);
return blockInSet != null && block.getGenerationStamp() == blockInSet.getGenerationStamp();
}
/** Add a block to the block collection which will be invalidated on the specified datanode. */
synchronized void add(final Block block, final DatanodeInfo datanode, final boolean log) {
LightWeightHashSet<Block> set = node2blocks.get(datanode);
if (set == null) {
set = new LightWeightHashSet<Block>();
node2blocks.put(datanode, set);
}
if (set.add(block)) {
numBlocks++;
if (log) {
NameNode.blockStateChangeLog.info(
"BLOCK* " + getClass().getSimpleName() + ": add " + block + " to " + datanode);
}
}
}
/** Remove a storage from the invalidatesSet */
synchronized void remove(final DatanodeInfo dn) {
final LightWeightHashSet<Block> blocks = node2blocks.remove(dn);
if (blocks != null) {
numBlocks -= blocks.size();
}
}
/** Remove the block from the specified storage. */
synchronized void remove(final DatanodeInfo dn, final Block block) {
final LightWeightHashSet<Block> v = node2blocks.get(dn);
if (v != null && v.remove(block)) {
numBlocks--;
if (v.isEmpty()) {
node2blocks.remove(dn);
}
}
}
/** Print the contents to out. */
synchronized void dump(final PrintWriter out) {
final int size = node2blocks.values().size();
out.println("Metasave: Blocks " + numBlocks + " waiting deletion from " + size + " datanodes.");
if (size == 0) {
return;
}
for (Map.Entry<DatanodeInfo, LightWeightHashSet<Block>> entry : node2blocks.entrySet()) {
final LightWeightHashSet<Block> blocks = entry.getValue();
if (blocks.size() > 0) {
out.println(entry.getKey());
out.println(blocks);
}
}
}
/** @return a list of the storage IDs. */
synchronized List<DatanodeInfo> getDatanodes() {
return new ArrayList<DatanodeInfo>(node2blocks.keySet());
}
/** @return the remianing pending time */
@VisibleForTesting
long getInvalidationDelay() {
return pendingPeriodInMs - (Time.monotonicNow() - startupTime);
}
synchronized List<Block> invalidateWork(final DatanodeDescriptor dn) {
final long delay = getInvalidationDelay();
if (delay > 0) {
if (BlockManager.LOG.isDebugEnabled()) {
BlockManager.LOG.debug(
"Block deletion is delayed during NameNode startup. "
+ "The deletion will start after "
+ delay
+ " ms.");
}
return null;
}
final LightWeightHashSet<Block> set = node2blocks.get(dn);
if (set == null) {
return null;
}
// # blocks that can be sent in one message is limited
final int limit = blockInvalidateLimit;
final List<Block> toInvalidate = set.pollN(limit);
// If we send everything in this message, remove this node entry
if (set.isEmpty()) {
remove(dn);
}
dn.addBlocksToBeInvalidated(toInvalidate);
numBlocks -= toInvalidate.size();
return toInvalidate;
}
synchronized void clear() {
node2blocks.clear();
numBlocks = 0;
}
}
/** @return the remianing pending time */
@VisibleForTesting
long getInvalidationDelay() {
return pendingPeriodInMs - (Time.monotonicNow() - startupTime);
}
/** process datanode heartbeat or stats initialization. */
public void updateHeartbeatState(
StorageReport[] reports,
long cacheCapacity,
long cacheUsed,
int xceiverCount,
int volFailures,
VolumeFailureSummary volumeFailureSummary) {
long totalCapacity = 0;
long totalRemaining = 0;
long totalBlockPoolUsed = 0;
long totalDfsUsed = 0;
Set<DatanodeStorageInfo> failedStorageInfos = null;
// Decide if we should check for any missing StorageReport and mark it as
// failed. There are different scenarios.
// 1. When DN is running, a storage failed. Given the current DN
// implementation doesn't add recovered storage back to its storage list
// until DN restart, we can assume volFailures won't decrease
// during the current DN registration session.
// When volumeFailures == this.volumeFailures, it implies there is no
// state change. No need to check for failed storage. This is an
// optimization. Recent versions of the DataNode report a
// VolumeFailureSummary containing the date/time of the last volume
// failure. If that's available, then we check that instead for greater
// accuracy.
// 2. After DN restarts, volFailures might not increase and it is possible
// we still have new failed storage. For example, admins reduce
// available storages in configuration. Another corner case
// is the failed volumes might change after restart; a) there
// is one good storage A, one restored good storage B, so there is
// one element in storageReports and that is A. b) A failed. c) Before
// DN sends HB to NN to indicate A has failed, DN restarts. d) After DN
// restarts, storageReports has one element which is B.
final boolean checkFailedStorages;
if (volumeFailureSummary != null && this.volumeFailureSummary != null) {
checkFailedStorages =
volumeFailureSummary.getLastVolumeFailureDate()
> this.volumeFailureSummary.getLastVolumeFailureDate();
} else {
checkFailedStorages = (volFailures > this.volumeFailures) || !heartbeatedSinceRegistration;
}
if (checkFailedStorages) {
LOG.info(
"Number of failed storage changes from " + this.volumeFailures + " to " + volFailures);
synchronized (storageMap) {
failedStorageInfos = new HashSet<>(storageMap.values());
}
}
setCacheCapacity(cacheCapacity);
setCacheUsed(cacheUsed);
setXceiverCount(xceiverCount);
setLastUpdate(Time.now());
setLastUpdateMonotonic(Time.monotonicNow());
this.volumeFailures = volFailures;
this.volumeFailureSummary = volumeFailureSummary;
for (StorageReport report : reports) {
DatanodeStorageInfo storage = updateStorage(report.getStorage());
if (checkFailedStorages) {
failedStorageInfos.remove(storage);
}
storage.receivedHeartbeat(report);
totalCapacity += report.getCapacity();
totalRemaining += report.getRemaining();
totalBlockPoolUsed += report.getBlockPoolUsed();
totalDfsUsed += report.getDfsUsed();
}
rollBlocksScheduled(getLastUpdateMonotonic());
// Update total metrics for the node.
setCapacity(totalCapacity);
setRemaining(totalRemaining);
setBlockPoolUsed(totalBlockPoolUsed);
setDfsUsed(totalDfsUsed);
if (checkFailedStorages) {
updateFailedStorage(failedStorageInfos);
}
long storageMapSize;
synchronized (storageMap) {
storageMapSize = storageMap.size();
}
if (storageMapSize != reports.length) {
pruneStorageMap(reports);
}
}
@Override
public Writable call(
org.apache.hadoop.ipc.RPC.Server server,
String protocolName,
Writable rpcRequest,
long receivedTime)
throws IOException {
try {
Invocation call = (Invocation) rpcRequest;
if (server.verbose) log("Call: " + call);
// Verify rpc version
if (call.getRpcVersion() != writableRpcVersion) {
// Client is using a different version of WritableRpc
throw new IOException(
"WritableRpc version mismatch, client side version="
+ call.getRpcVersion()
+ ", server side version="
+ writableRpcVersion);
}
long clientVersion = call.getProtocolVersion();
final String protoName;
ProtoClassProtoImpl protocolImpl;
if (call.declaringClassProtocolName.equals(VersionedProtocol.class.getName())) {
// VersionProtocol methods are often used by client to figure out
// which version of protocol to use.
//
// Versioned protocol methods should go the protocolName protocol
// rather than the declaring class of the method since the
// the declaring class is VersionedProtocol which is not
// registered directly.
// Send the call to the highest protocol version
VerProtocolImpl highest =
server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protocolName);
if (highest == null) {
throw new IOException("Unknown protocol: " + protocolName);
}
protocolImpl = highest.protocolTarget;
} else {
protoName = call.declaringClassProtocolName;
// Find the right impl for the protocol based on client version.
ProtoNameVer pv = new ProtoNameVer(call.declaringClassProtocolName, clientVersion);
protocolImpl = server.getProtocolImplMap(RPC.RpcKind.RPC_WRITABLE).get(pv);
if (protocolImpl == null) { // no match for Protocol AND Version
VerProtocolImpl highest =
server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protoName);
if (highest == null) {
throw new IOException("Unknown protocol: " + protoName);
} else { // protocol supported but not the version that client wants
throw new RPC.VersionMismatch(protoName, clientVersion, highest.version);
}
}
}
// Invoke the protocol method
long startTime = Time.now();
Method method =
protocolImpl.protocolClass.getMethod(
call.getMethodName(), call.getParameterClasses());
method.setAccessible(true);
server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
Object value = method.invoke(protocolImpl.protocolImpl, call.getParameters());
int processingTime = (int) (Time.now() - startTime);
int qTime = (int) (startTime - receivedTime);
if (LOG.isDebugEnabled()) {
LOG.debug(
"Served: "
+ call.getMethodName()
+ " queueTime= "
+ qTime
+ " procesingTime= "
+ processingTime);
}
server.rpcMetrics.addRpcQueueTime(qTime);
server.rpcMetrics.addRpcProcessingTime(processingTime);
server.rpcDetailedMetrics.addProcessingTime(call.getMethodName(), processingTime);
if (server.verbose) log("Return: " + value);
return new ObjectWritable(method.getReturnType(), value);
} catch (InvocationTargetException e) {
Throwable target = e.getTargetException();
if (target instanceof IOException) {
throw (IOException) target;
} else {
IOException ioe = new IOException(target.toString());
ioe.setStackTrace(target.getStackTrace());
throw ioe;
}
} catch (Throwable e) {
if (!(e instanceof IOException)) {
LOG.error("Unexpected throwable object ", e);
}
IOException ioe = new IOException(e.toString());
ioe.setStackTrace(e.getStackTrace());
throw ioe;
}
}
/** Check files on DFS, starting from the indicated path. */
public void fsck() {
final long startTime = Time.now();
try {
String msg =
"FSCK started by "
+ UserGroupInformation.getCurrentUser()
+ " from "
+ remoteAddress
+ " for path "
+ path
+ " at "
+ new Date();
LOG.info(msg);
out.println(msg);
namenode.getNamesystem().logFsckEvent(path, remoteAddress);
if (snapshottableDirs != null) {
SnapshottableDirectoryStatus[] snapshotDirs =
namenode.getRpcServer().getSnapshottableDirListing();
if (snapshotDirs != null) {
for (SnapshottableDirectoryStatus dir : snapshotDirs) {
snapshottableDirs.add(dir.getFullPath().toString());
}
}
}
final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(path);
if (file != null) {
if (showCorruptFileBlocks) {
listCorruptFileBlocks();
return;
}
Result res = new Result(conf);
check(path, file, res);
out.println(res);
out.println(" Number of data-nodes:\t\t" + totalDatanodes);
out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks());
out.println(
"FSCK ended at " + new Date() + " in " + (Time.now() - startTime + " milliseconds"));
// If there were internal errors during the fsck operation, we want to
// return FAILURE_STATUS, even if those errors were not immediately
// fatal. Otherwise many unit tests will pass even when there are bugs.
if (internalError) {
throw new IOException("fsck encountered internal errors!");
}
// DFSck client scans for the string HEALTHY/CORRUPT to check the status
// of file system and return appropriate code. Changing the output
// string might break testcases. Also note this must be the last line
// of the report.
if (res.isHealthy()) {
out.print("\n\nThe filesystem under path '" + path + "' " + HEALTHY_STATUS);
} else {
out.print("\n\nThe filesystem under path '" + path + "' " + CORRUPT_STATUS);
}
} else {
out.print("\n\nPath '" + path + "' " + NONEXISTENT_STATUS);
}
} catch (Exception e) {
String errMsg = "Fsck on path '" + path + "' " + FAILURE_STATUS;
LOG.warn(errMsg, e);
out.println(
"FSCK ended at " + new Date() + " in " + (Time.now() - startTime + " milliseconds"));
out.println(e.getMessage());
out.print("\n\n" + errMsg);
} finally {
out.close();
}
}
/**
* This is the constructor with the signature needed by {@link FileSystem#createFileSystem(URI,
* Configuration)}
*
* <p>After this constructor is called initialize() is called.
*
* @throws IOException
*/
public ViewFileSystem() throws IOException {
ugi = UserGroupInformation.getCurrentUser();
creationTime = Time.now();
}
/**
* Add replicas under the given directory to the volume map
*
* @param volumeMap the replicas map
* @param dir an input directory
* @param lazyWriteReplicaMap Map of replicas on transient storage.
* @param isFinalized true if the directory has finalized replicas; false if the directory has rbw
* replicas
*/
void addToReplicasMap(
ReplicaMap volumeMap,
File dir,
final RamDiskReplicaTracker lazyWriteReplicaMap,
boolean isFinalized)
throws IOException {
File files[] = FileUtil.listFiles(dir);
for (File file : files) {
if (file.isDirectory()) {
addToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized);
}
if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) {
file = recoverTempUnlinkedBlock(file);
if (file == null) { // the original block still exists, so we cover it
// in another iteration and can continue here
continue;
}
}
if (!Block.isBlockFilename(file)) continue;
long genStamp = FsDatasetUtil.getGenerationStampFromFile(files, file);
long blockId = Block.filename2id(file.getName());
ReplicaInfo newReplica = null;
if (isFinalized) {
newReplica =
new FinalizedReplica(blockId, file.length(), genStamp, volume, file.getParentFile());
} else {
boolean loadRwr = true;
File restartMeta =
new File(file.getParent() + File.pathSeparator + "." + file.getName() + ".restart");
Scanner sc = null;
try {
sc = new Scanner(restartMeta, "UTF-8");
// The restart meta file exists
if (sc.hasNextLong() && (sc.nextLong() > Time.now())) {
// It didn't expire. Load the replica as a RBW.
// We don't know the expected block length, so just use 0
// and don't reserve any more space for writes.
newReplica =
new ReplicaBeingWritten(
blockId,
validateIntegrityAndSetLength(file, genStamp),
genStamp,
volume,
file.getParentFile(),
null,
0);
loadRwr = false;
}
sc.close();
if (!restartMeta.delete()) {
FsDatasetImpl.LOG.warn("Failed to delete restart meta file: " + restartMeta.getPath());
}
} catch (FileNotFoundException fnfe) {
// nothing to do hereFile dir =
} finally {
if (sc != null) {
sc.close();
}
}
// Restart meta doesn't exist or expired.
if (loadRwr) {
newReplica =
new ReplicaWaitingToBeRecovered(
blockId,
validateIntegrityAndSetLength(file, genStamp),
genStamp,
volume,
file.getParentFile());
}
}
ReplicaInfo oldReplica = volumeMap.get(bpid, newReplica.getBlockId());
if (oldReplica == null) {
volumeMap.add(bpid, newReplica);
} else {
// We have multiple replicas of the same block so decide which one
// to keep.
newReplica = resolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
}
// If we are retaining a replica on transient storage make sure
// it is in the lazyWriteReplicaMap so it can be persisted
// eventually.
if (newReplica.getVolume().isTransientStorage()) {
lazyWriteReplicaMap.addReplica(bpid, blockId, (FsVolumeImpl) newReplica.getVolume());
} else {
lazyWriteReplicaMap.discardReplica(bpid, blockId, false);
}
}
}
/**
* The load generator is a tool for testing NameNode behavior under different client loads. Note
* there is a subclass of this clas that lets you run a the load generator as a MapReduce job (see
* LoadGeneratorMR in the MapReduce project.
*
* <p>The loadGenerator allows the user to generate different mixes of read, write, and list
* requests by specifying the probabilities of read and write. The user controls the intensity of
* the load by adjusting parameters for the number of worker threads and the delay between
* operations. While load generators are running, the user can profile and monitor the running of
* the NameNode. When a load generator exits, it print some NameNode statistics like the average
* execution time of each kind of operations and the NameNode throughput.
*
* <p>The program can run in one of two forms. As a regular single process command that runs
* multiple threads to generate load on the NN or as a Map Reduce program that runs multiple
* (multi-threaded) map tasks that generate load on the NN; the results summary is generated by a
* single reduce task.
*
* <p>The user may either specify constant duration, read and write probabilities via the command
* line, or may specify a text file that acts as a script of which read and write probabilities to
* use for specified durations. If no duration is specified the program runs till killed (duration
* required if run as MapReduce).
*
* <p>The script takes the form of lines of duration in seconds, read probability and write
* probability, each separated by white space. Blank lines and lines starting with # (comments) are
* ignored. If load generator is run as a MapReduce program then the script file needs to be
* accessible on the the Map task as a HDFS file.
*
* <p>After command line argument parsing and data initialization, the load generator spawns the
* number of worker threads as specified by the user. Each thread sends a stream of requests to the
* NameNode. For each iteration, it first decides if it is going to read a file, create a file, or
* listing a directory following the read and write probabilities specified by the user. When
* reading, it randomly picks a file in the test space and reads the entire file. When writing, it
* randomly picks a directory in the test space and creates a file whose name consists of the
* current machine's host name and the thread id. The length of the file follows Gaussian
* distribution with an average size of 2 blocks and the standard deviation of 1 block. The new file
* is filled with 'a'. Immediately after the file creation completes, the file is deleted from the
* test space. While listing, it randomly picks a directory in the test space and list the directory
* content. Between two consecutive operations, the thread pauses for a random amount of time in the
* range of [0, maxDelayBetweenOps] if the specified max delay is not zero. All threads are stopped
* when the specified elapsed time has passed in command-line execution, or all the lines of script
* have been executed, if using a script. Before exiting, the program prints the average execution
* for each kind of NameNode operations, and the number of requests served by the NameNode.
*
* <p>The synopsis of the command is java LoadGenerator -readProbability <read probability>: read
* probability [0, 1] with a default value of 0.3333. -writeProbability <write probability>: write
* probability [0, 1] with a default value of 0.3333. -root <root>: test space with a default value
* of /testLoadSpace -maxDelayBetweenOps <maxDelayBetweenOpsInMillis>: Max delay in the unit of
* milliseconds between two operations with a default value of 0 indicating no delay. -numOfThreads
* <numOfThreads>: number of threads to spawn with a default value of 200. -elapsedTime
* <elapsedTimeInSecs>: the elapsed time of program with a default value of 0 indicating running
* forever -startTime <startTimeInMillis> : when the threads start to run. -scriptFile <file name>:
* text file to parse for scripted operation
*/
public class LoadGenerator extends Configured implements Tool {
public static final Log LOG = LogFactory.getLog(LoadGenerator.class);
private static volatile boolean shouldRun = true;
protected static Path root = DataGenerator.DEFAULT_ROOT;
private static FileContext fc;
protected static int maxDelayBetweenOps = 0;
protected static int numOfThreads = 200;
protected static long[] durations = {0};
protected static double[] readProbs = {0.3333};
protected static double[] writeProbs = {0.3333};
private static volatile int currentIndex = 0;
protected static long totalTime = 0;
protected static long startTime = Time.now() + 10000;
private static final int BLOCK_SIZE = 10;
private static ArrayList<String> files = new ArrayList<String>(); // a table of file names
private static ArrayList<String> dirs = new ArrayList<String>(); // a table of directory names
protected static Random r = null;
protected static long seed = 0;
protected static String scriptFile = null;
protected static final String FLAGFILE_DEFAULT = "/tmp/flagFile";
protected static Path flagFile = new Path(FLAGFILE_DEFAULT);
protected String hostname;
private static final String USAGE_CMD = "java LoadGenerator\n";
protected static final String USAGE_ARGS =
"-readProbability <read probability>\n"
+ "-writeProbability <write probability>\n"
+ "-root <root>\n"
+ "-maxDelayBetweenOps <maxDelayBetweenOpsInMillis>\n"
+ "-numOfThreads <numOfThreads>\n"
+ "-elapsedTime <elapsedTimeInSecs>\n"
+ "-startTime <startTimeInMillis>\n"
+ "-scriptFile <filename>\n"
+ "-flagFile <filename>";
private static final String USAGE = USAGE_CMD + USAGE_ARGS;
private final byte[] WRITE_CONTENTS = new byte[4096];
private static final int ERR_TEST_FAILED = 2;
/** Constructor */
public LoadGenerator() throws IOException, UnknownHostException {
InetAddress addr = InetAddress.getLocalHost();
hostname = addr.getHostName();
Arrays.fill(WRITE_CONTENTS, (byte) 'a');
}
public LoadGenerator(Configuration conf) throws IOException, UnknownHostException {
this();
setConf(conf);
}
protected static final int OPEN = 0;
protected static final int LIST = 1;
protected static final int CREATE = 2;
protected static final int WRITE_CLOSE = 3;
protected static final int DELETE = 4;
protected static final int TOTAL_OP_TYPES = 5;
protected static long[] executionTime = new long[TOTAL_OP_TYPES];
protected static long[] numOfOps = new long[TOTAL_OP_TYPES];
protected static long totalOps = 0; // across all of types
/**
* A thread sends a stream of requests to the NameNode. At each iteration, it first decides if it
* is going to read a file, create a file, or listing a directory following the read and write
* probabilities. When reading, it randomly picks a file in the test space and reads the entire
* file. When writing, it randomly picks a directory in the test space and creates a file whose
* name consists of the current machine's host name and the thread id. The length of the file
* follows Gaussian distribution with an average size of 2 blocks and the standard deviation of 1
* block. The new file is filled with 'a'. Immediately after the file creation completes, the file
* is deleted from the test space. While listing, it randomly picks a directory in the test space
* and list the directory content. Between two consecutive operations, the thread pauses for a
* random amount of time in the range of [0, maxDelayBetweenOps] if the specified max delay is not
* zero. A thread runs for the specified elapsed time if the time isn't zero. Otherwise, it runs
* forever.
*/
private class DFSClientThread extends Thread {
private int id;
private long[] executionTime = new long[TOTAL_OP_TYPES];
private long[] totalNumOfOps = new long[TOTAL_OP_TYPES];
private byte[] buffer = new byte[1024];
private boolean failed;
private DFSClientThread(int id) {
this.id = id;
}
/**
* Main loop for each thread Each iteration decides what's the next operation and then pauses.
*/
@Override
public void run() {
try {
while (shouldRun) {
nextOp();
delay();
}
} catch (Exception ioe) {
System.err.println(ioe.getLocalizedMessage());
ioe.printStackTrace();
failed = true;
}
}
/**
* Let the thread pause for a random amount of time in the range of [0, maxDelayBetweenOps] if
* the delay is not zero. Otherwise, no pause.
*/
private void delay() throws InterruptedException {
if (maxDelayBetweenOps > 0) {
int delay = r.nextInt(maxDelayBetweenOps);
Thread.sleep(delay);
}
}
/**
* Perform the next operation.
*
* <p>Depending on the read and write probabilities, the next operation could be either read,
* write, or list.
*/
private void nextOp() throws IOException {
double rn = r.nextDouble();
int i = currentIndex;
if (LOG.isDebugEnabled()) LOG.debug("Thread " + this.id + " moving to index " + i);
if (rn < readProbs[i]) {
read();
} else if (rn < readProbs[i] + writeProbs[i]) {
write();
} else {
list();
}
}
/** Read operation randomly picks a file in the test space and reads the entire file */
private void read() throws IOException {
String fileName = files.get(r.nextInt(files.size()));
long startTime = Time.now();
InputStream in = fc.open(new Path(fileName));
executionTime[OPEN] += (Time.now() - startTime);
totalNumOfOps[OPEN]++;
while (in.read(buffer) != -1) {}
in.close();
}
/**
* The write operation randomly picks a directory in the test space and creates a file whose
* name consists of the current machine's host name and the thread id. The length of the file
* follows Gaussian distribution with an average size of 2 blocks and the standard deviation of
* 1 block. The new file is filled with 'a'. Immediately after the file creation completes, the
* file is deleted from the test space.
*/
private void write() throws IOException {
String dirName = dirs.get(r.nextInt(dirs.size()));
Path file = new Path(dirName, hostname + id);
double fileSize = 0;
while ((fileSize = r.nextGaussian() + 2) <= 0) {}
genFile(file, (long) (fileSize * BLOCK_SIZE));
long startTime = Time.now();
fc.delete(file, true);
executionTime[DELETE] += (Time.now() - startTime);
totalNumOfOps[DELETE]++;
}
/**
* The list operation randomly picks a directory in the test space and list the directory
* content.
*/
private void list() throws IOException {
String dirName = dirs.get(r.nextInt(dirs.size()));
long startTime = Time.now();
fc.listStatus(new Path(dirName));
executionTime[LIST] += (Time.now() - startTime);
totalNumOfOps[LIST]++;
}
/** Create a file with a length of <code>fileSize</code>. The file is filled with 'a'. */
private void genFile(Path file, long fileSize) throws IOException {
long startTime = Time.now();
FSDataOutputStream out = null;
try {
out =
fc.create(
file,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
CreateOpts.createParent(),
CreateOpts.bufferSize(4096),
CreateOpts.repFac((short) 3));
executionTime[CREATE] += (Time.now() - startTime);
numOfOps[CREATE]++;
long i = fileSize;
while (i > 0) {
long s = Math.min(fileSize, WRITE_CONTENTS.length);
out.write(WRITE_CONTENTS, 0, (int) s);
i -= s;
}
startTime = Time.now();
executionTime[WRITE_CLOSE] += (Time.now() - startTime);
numOfOps[WRITE_CLOSE]++;
} finally {
IOUtils.cleanup(LOG, out);
}
}
}
/**
* Main function called by tool runner. It first initializes data by parsing the command line
* arguments. It then calls the loadGenerator
*/
@Override
public int run(String[] args) throws Exception {
int exitCode = parseArgs(false, args);
if (exitCode != 0) {
return exitCode;
}
System.out.println(
"Running LoadGenerator against fileSystem: "
+ FileContext.getFileContext().getDefaultFileSystem().getUri());
exitCode = generateLoadOnNN();
printResults(System.out);
return exitCode;
}
boolean stopFileCreated() {
try {
fc.getFileStatus(flagFile);
} catch (FileNotFoundException e) {
return false;
} catch (IOException e) {
LOG.error("Got error when checking if file exists:" + flagFile, e);
}
LOG.info("Flag file was created. Stopping the test.");
return true;
}
/**
* This is the main function - run threads to generate load on NN It starts the number of
* DFSClient threads as specified by the user. It stops all the threads when the specified elapsed
* time is passed.
*/
protected int generateLoadOnNN() throws InterruptedException {
int hostHashCode = hostname.hashCode();
if (seed == 0) {
r = new Random(System.currentTimeMillis() + hostHashCode);
} else {
r = new Random(seed + hostHashCode);
}
try {
fc = FileContext.getFileContext(getConf());
} catch (IOException ioe) {
System.err.println("Can not initialize the file system: " + ioe.getLocalizedMessage());
return -1;
}
int status = initFileDirTables();
if (status != 0) {
return status;
}
barrier();
DFSClientThread[] threads = new DFSClientThread[numOfThreads];
for (int i = 0; i < numOfThreads; i++) {
threads[i] = new DFSClientThread(i);
threads[i].start();
}
if (durations[0] > 0) {
if (durations.length == 1) { // There is a fixed run time
while (shouldRun) {
Thread.sleep(2000);
totalTime += 2;
if (totalTime >= durations[0] || stopFileCreated()) {
shouldRun = false;
}
}
} else {
// script run
while (shouldRun) {
Thread.sleep(durations[currentIndex] * 1000);
totalTime += durations[currentIndex];
// Are we on the final line of the script?
if ((currentIndex + 1) == durations.length || stopFileCreated()) {
shouldRun = false;
} else {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Moving to index "
+ currentIndex
+ ": r = "
+ readProbs[currentIndex]
+ ", w = "
+ writeProbs
+ " for duration "
+ durations[currentIndex]);
}
currentIndex++;
}
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Done with testing. Waiting for threads to finish.");
}
boolean failed = false;
for (DFSClientThread thread : threads) {
thread.join();
for (int i = 0; i < TOTAL_OP_TYPES; i++) {
executionTime[i] += thread.executionTime[i];
numOfOps[i] += thread.totalNumOfOps[i];
}
failed = failed || thread.failed;
}
int exitCode = 0;
if (failed) {
exitCode = -ERR_TEST_FAILED;
}
totalOps = 0;
for (int i = 0; i < TOTAL_OP_TYPES; i++) {
totalOps += numOfOps[i];
}
return exitCode;
}
protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
out.println(
"Result of running LoadGenerator against fileSystem: "
+ FileContext.getFileContext().getDefaultFileSystem().getUri());
if (numOfOps[OPEN] != 0) {
out.println(
"Average open execution time: " + (double) executionTime[OPEN] / numOfOps[OPEN] + "ms");
}
if (numOfOps[LIST] != 0) {
out.println(
"Average list execution time: " + (double) executionTime[LIST] / numOfOps[LIST] + "ms");
}
if (numOfOps[DELETE] != 0) {
out.println(
"Average deletion execution time: "
+ (double) executionTime[DELETE] / numOfOps[DELETE]
+ "ms");
out.println(
"Average create execution time: "
+ (double) executionTime[CREATE] / numOfOps[CREATE]
+ "ms");
out.println(
"Average write_close execution time: "
+ (double) executionTime[WRITE_CLOSE] / numOfOps[WRITE_CLOSE]
+ "ms");
}
if (totalTime != 0) {
out.println("Average operations per second: " + (double) totalOps / totalTime + "ops/s");
}
out.println();
}
/** Parse the command line arguments and initialize the data */
protected int parseArgs(boolean runAsMapReduce, String[] args) throws IOException {
try {
for (int i = 0; i < args.length; i++) { // parse command line
if (args[i].equals("-scriptFile")) {
scriptFile = args[++i];
if (durations[0] > 0) {
System.err.println("Can't specify elapsedTime and use script.");
return -1;
}
} else if (args[i].equals("-readProbability")) {
if (scriptFile != null) {
System.err.println("Can't specify probabilities and use script.");
return -1;
}
readProbs[0] = Double.parseDouble(args[++i]);
if (readProbs[0] < 0 || readProbs[0] > 1) {
System.err.println("The read probability must be [0, 1]: " + readProbs[0]);
return -1;
}
} else if (args[i].equals("-writeProbability")) {
if (scriptFile != null) {
System.err.println("Can't specify probabilities and use script.");
return -1;
}
writeProbs[0] = Double.parseDouble(args[++i]);
if (writeProbs[0] < 0 || writeProbs[0] > 1) {
System.err.println("The write probability must be [0, 1]: " + writeProbs[0]);
return -1;
}
} else if (args[i].equals("-root")) {
root = new Path(args[++i]);
} else if (args[i].equals("-maxDelayBetweenOps")) {
maxDelayBetweenOps = Integer.parseInt(args[++i]); // in milliseconds
} else if (args[i].equals("-numOfThreads")) {
numOfThreads = Integer.parseInt(args[++i]);
if (numOfThreads <= 0) {
System.err.println("Number of threads must be positive: " + numOfThreads);
return -1;
}
} else if (args[i].equals("-startTime")) {
startTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-elapsedTime")) {
if (scriptFile != null) {
System.err.println("Can't specify elapsedTime and use script.");
return -1;
}
durations[0] = Long.parseLong(args[++i]);
} else if (args[i].equals("-seed")) {
seed = Long.parseLong(args[++i]);
r = new Random(seed);
} else if (args[i].equals("-flagFile")) {
LOG.info("got flagFile:" + flagFile);
flagFile = new Path(args[++i]);
} else {
System.err.println(USAGE);
ToolRunner.printGenericCommandUsage(System.err);
return -1;
}
}
} catch (NumberFormatException e) {
System.err.println("Illegal parameter: " + e.getLocalizedMessage());
System.err.println(USAGE);
return -1;
}
// Load Script File if not MR; for MR scriptFile is loaded by Mapper
if (!runAsMapReduce && scriptFile != null) {
if (loadScriptFile(scriptFile, true) == -1) return -1;
}
for (int i = 0; i < readProbs.length; i++) {
if (readProbs[i] + writeProbs[i] < 0 || readProbs[i] + writeProbs[i] > 1) {
System.err.println(
"The sum of read probability and write probability must be [0, 1]: "
+ readProbs[i]
+ " "
+ writeProbs[i]);
return -1;
}
}
return 0;
}
private static void parseScriptLine(
String line,
ArrayList<Long> duration,
ArrayList<Double> readProb,
ArrayList<Double> writeProb) {
String[] a = line.split("\\s");
if (a.length != 3) {
throw new IllegalArgumentException("Incorrect number of parameters: " + line);
}
try {
long d = Long.parseLong(a[0]);
double r = Double.parseDouble(a[1]);
double w = Double.parseDouble(a[2]);
Preconditions.checkArgument(d >= 0, "Invalid duration: " + d);
Preconditions.checkArgument(0 <= r && r <= 1.0, "The read probability must be [0, 1]: " + r);
Preconditions.checkArgument(0 <= w && w <= 1.0, "The read probability must be [0, 1]: " + w);
readProb.add(r);
duration.add(d);
writeProb.add(w);
} catch (NumberFormatException nfe) {
throw new IllegalArgumentException("Cannot parse: " + line);
}
}
/**
* Read a script file of the form: lines of text with duration in seconds, read probability and
* write probability, separated by white space.
*
* @param filename Script file
* @return 0 if successful, -1 if not
* @throws IOException if errors with file IO
*/
protected static int loadScriptFile(String filename, boolean readLocally) throws IOException {
FileContext fc;
if (readLocally) { // read locally - program is run without MR
fc = FileContext.getLocalFSFileContext();
} else {
fc = FileContext.getFileContext(); // use default file system
}
DataInputStream in = null;
try {
in = fc.open(new Path(filename));
} catch (IOException e) {
System.err.println("Unable to open scriptFile: " + filename);
System.exit(-1);
}
InputStreamReader inr = new InputStreamReader(in);
BufferedReader br = new BufferedReader(inr);
ArrayList<Long> duration = new ArrayList<Long>();
ArrayList<Double> readProb = new ArrayList<Double>();
ArrayList<Double> writeProb = new ArrayList<Double>();
int lineNum = 0;
String line;
// Read script, parse values, build array of duration, read and write probs
try {
while ((line = br.readLine()) != null) {
lineNum++;
if (line.startsWith("#") || line.isEmpty()) // skip comments and blanks
continue;
parseScriptLine(line, duration, readProb, writeProb);
}
} catch (IllegalArgumentException e) {
System.err.println("Line: " + lineNum + ", " + e.getMessage());
return -1;
} finally {
IOUtils.cleanup(LOG, br);
}
// Copy vectors to arrays of values, to avoid autoboxing overhead later
durations = new long[duration.size()];
readProbs = new double[readProb.size()];
writeProbs = new double[writeProb.size()];
for (int i = 0; i < durations.length; i++) {
durations[i] = duration.get(i);
readProbs[i] = readProb.get(i);
writeProbs[i] = writeProb.get(i);
}
if (durations[0] == 0)
System.err.println("Initial duration set to 0. " + "Will loop until stopped manually.");
return 0;
}
/**
* Create a table that contains all directories under root and another table that contains all
* files under root.
*/
private int initFileDirTables() {
try {
initFileDirTables(root);
} catch (IOException e) {
System.err.println(e.getLocalizedMessage());
e.printStackTrace();
return -1;
}
if (dirs.isEmpty()) {
System.err.println("The test space " + root + " is empty");
return -1;
}
if (files.isEmpty()) {
System.err.println("The test space " + root + " does not have any file");
return -1;
}
return 0;
}
/**
* Create a table that contains all directories under the specified path and another table that
* contains all files under the specified path and whose name starts with "_file_".
*/
private void initFileDirTables(Path path) throws IOException {
FileStatus[] stats = fc.util().listStatus(path);
for (FileStatus stat : stats) {
if (stat.isDirectory()) {
dirs.add(stat.getPath().toString());
initFileDirTables(stat.getPath());
} else {
Path filePath = stat.getPath();
if (filePath.getName().startsWith(StructureGenerator.FILE_NAME_PREFIX)) {
files.add(filePath.toString());
}
}
}
}
/**
* Returns when the current number of seconds from the epoch equals the command line argument
* given by <code>-startTime</code>. This allows multiple instances of this program, running on
* clock synchronized nodes, to start at roughly the same time.
*/
private static void barrier() {
long sleepTime;
while ((sleepTime = startTime - Time.now()) > 0) {
try {
Thread.sleep(sleepTime);
} catch (InterruptedException ex) {
}
}
}
/**
* Main program
*
* @param args command line arguments
* @throws Exception
*/
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new LoadGenerator(), args);
System.exit(res);
}
}
