コード例 #1
0
ファイル: LoadGenerator.java プロジェクト: FloodDragon/hadoop
 /**
  * The list operation randomly picks a directory in the test space and list the directory
  * content.
  */
 private void list() throws IOException {
   String dirName = dirs.get(r.nextInt(dirs.size()));
   long startTime = Time.now();
   fc.listStatus(new Path(dirName));
   executionTime[LIST] += (Time.now() - startTime);
   totalNumOfOps[LIST]++;
 }
コード例 #2
0
ファイル: LoadGenerator.java プロジェクト: FloodDragon/hadoop
    /** Create a file with a length of <code>fileSize</code>. The file is filled with 'a'. */
    private void genFile(Path file, long fileSize) throws IOException {
      long startTime = Time.now();
      FSDataOutputStream out = null;
      try {
        out =
            fc.create(
                file,
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                CreateOpts.createParent(),
                CreateOpts.bufferSize(4096),
                CreateOpts.repFac((short) 3));
        executionTime[CREATE] += (Time.now() - startTime);
        numOfOps[CREATE]++;

        long i = fileSize;
        while (i > 0) {
          long s = Math.min(fileSize, WRITE_CONTENTS.length);
          out.write(WRITE_CONTENTS, 0, (int) s);
          i -= s;
        }

        startTime = Time.now();
        executionTime[WRITE_CLOSE] += (Time.now() - startTime);
        numOfOps[WRITE_CLOSE]++;
      } finally {
        IOUtils.cleanup(LOG, out);
      }
    }
コード例 #3
0
ファイル: TestHTestCase.java プロジェクト: FloodDragon/hadoop
 @Test
 public void sleepRatio2() {
   setWaitForRatio(1);
   long start = Time.now();
   sleep(100);
   long end = Time.now();
   assertEquals(end - start, 100 * getWaitForRatio(), 50 * getWaitForRatio());
 }
コード例 #4
0
ファイル: LoadGenerator.java プロジェクト: FloodDragon/hadoop
 /** Read operation randomly picks a file in the test space and reads the entire file */
 private void read() throws IOException {
   String fileName = files.get(r.nextInt(files.size()));
   long startTime = Time.now();
   InputStream in = fc.open(new Path(fileName));
   executionTime[OPEN] += (Time.now() - startTime);
   totalNumOfOps[OPEN]++;
   while (in.read(buffer) != -1) {}
   in.close();
 }
コード例 #5
0
ファイル: LoadGenerator.java プロジェクト: FloodDragon/hadoop
 /**
  * The write operation randomly picks a directory in the test space and creates a file whose
  * name consists of the current machine's host name and the thread id. The length of the file
  * follows Gaussian distribution with an average size of 2 blocks and the standard deviation of
  * 1 block. The new file is filled with 'a'. Immediately after the file creation completes, the
  * file is deleted from the test space.
  */
 private void write() throws IOException {
   String dirName = dirs.get(r.nextInt(dirs.size()));
   Path file = new Path(dirName, hostname + id);
   double fileSize = 0;
   while ((fileSize = r.nextGaussian() + 2) <= 0) {}
   genFile(file, (long) (fileSize * BLOCK_SIZE));
   long startTime = Time.now();
   fc.delete(file, true);
   executionTime[DELETE] += (Time.now() - startTime);
   totalNumOfOps[DELETE]++;
 }
コード例 #6
0
    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
      long startTime = 0;
      if (LOG.isDebugEnabled()) {
        startTime = Time.now();
      }

      ObjectWritable value =
          (ObjectWritable)
              client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId);
      if (LOG.isDebugEnabled()) {
        long callTime = Time.now() - startTime;
        LOG.debug("Call: " + method.getName() + " " + callTime);
      }
      return value.get();
    }
コード例 #7
0
  /**
   * Read in the cached DU value and return it if it is less than 600 seconds old (DU update
   * interval). Slight imprecision of dfsUsed is not critical and skipping DU can significantly
   * shorten the startup time. If the cached value is not available or too old, -1 is returned.
   */
  long loadDfsUsed() {
    long cachedDfsUsed;
    long mtime;
    Scanner sc;

    try {
      sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8");
    } catch (FileNotFoundException fnfe) {
      return -1;
    }

    try {
      // Get the recorded dfsUsed from the file.
      if (sc.hasNextLong()) {
        cachedDfsUsed = sc.nextLong();
      } else {
        return -1;
      }
      // Get the recorded mtime from the file.
      if (sc.hasNextLong()) {
        mtime = sc.nextLong();
      } else {
        return -1;
      }

      // Return the cached value if mtime is okay.
      if (mtime > 0 && (Time.now() - mtime < 600000L)) {
        FsDatasetImpl.LOG.info("Cached dfsUsed found for " + currentDir + ": " + cachedDfsUsed);
        return cachedDfsUsed;
      }
      return -1;
    } finally {
      sc.close();
    }
  }
コード例 #8
0
  private void requestNewHdfsDelegationToken(
      ApplicationId applicationId, String user, boolean shouldCancelAtEnd)
      throws IOException, InterruptedException {
    // Get new hdfs tokens for this user
    Credentials credentials = new Credentials();
    Token<?>[] newTokens = obtainSystemTokensForUser(user, credentials);

    // Add new tokens to the toRenew list.
    LOG.info(
        "Received new tokens for " + applicationId + ". Received " + newTokens.length + " tokens.");
    if (newTokens.length > 0) {
      for (Token<?> token : newTokens) {
        if (token.isManaged()) {
          DelegationTokenToRenew tokenToRenew =
              new DelegationTokenToRenew(
                  applicationId, token, getConfig(), Time.now(), shouldCancelAtEnd, user);
          // renew the token to get the next expiration date.
          renewToken(tokenToRenew);
          setTimerForTokenRenewal(tokenToRenew);
          appTokens.get(applicationId).add(tokenToRenew);
          LOG.info("Received new token " + token);
        }
      }
    }
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer byteBuffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    rmContext.getSystemCredentialsForApps().put(applicationId, byteBuffer);
  }
コード例 #9
0
  /** Add a snapshot. */
  Snapshot addSnapshot(int id, String name) throws SnapshotException, QuotaExceededException {
    // check snapshot quota
    final int n = getNumSnapshots();
    if (n + 1 > snapshotQuota) {
      throw new SnapshotException(
          "Failed to add snapshot: there are already "
              + n
              + " snapshot(s) and the snapshot quota is "
              + snapshotQuota);
    }
    final Snapshot s = new Snapshot(id, name, this);
    final byte[] nameBytes = s.getRoot().getLocalNameBytes();
    final int i = searchSnapshot(nameBytes);
    if (i >= 0) {
      throw new SnapshotException(
          "Failed to add snapshot: there is already a "
              + "snapshot with the same name \""
              + Snapshot.getSnapshotName(s)
              + "\".");
    }

    final DirectoryDiff d = getDiffs().addDiff(s, this);
    d.snapshotINode = s.getRoot();
    snapshotsByNames.add(-i - 1, s);

    // set modification time
    updateModificationTime(Time.now(), null, null);
    s.getRoot().setModificationTime(getModificationTime(), null, null);
    return s;
  }
コード例 #10
0
ファイル: TestHTestCase.java プロジェクト: FloodDragon/hadoop
 @Test
 public void waitFor() {
   long start = Time.now();
   long waited =
       waitFor(
           1000,
           new Predicate() {
             @Override
             public boolean evaluate() throws Exception {
               return true;
             }
           });
   long end = Time.now();
   assertEquals(waited, 0, 50);
   assertEquals(end - start - waited, 0, 50);
 }
コード例 #11
0
ファイル: TestHTestCase.java プロジェクト: FloodDragon/hadoop
 @Test
 public void waitForTimeOutRatio2() {
   setWaitForRatio(2);
   long start = Time.now();
   long waited =
       waitFor(
           200,
           new Predicate() {
             @Override
             public boolean evaluate() throws Exception {
               return false;
             }
           });
   long end = Time.now();
   assertEquals(waited, -1);
   assertEquals(end - start, 200 * getWaitForRatio(), 50 * getWaitForRatio());
 }
コード例 #12
0
ファイル: RPC.java プロジェクト: jonathangizmo/HadoopDistJ
  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param rpcTimeout timeout for each RPC
   * @param timeout time in milliseconds before giving up
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      int rpcTimeout,
      RetryPolicy connectionRetryPolicy,
      long timeout)
      throws IOException {
    long startTime = Time.now();
    IOException ioe;
    while (true) {
      try {
        return getProtocolProxy(
            protocol,
            clientVersion,
            addr,
            UserGroupInformation.getCurrentUser(),
            conf,
            NetUtils.getDefaultSocketFactory(conf),
            rpcTimeout,
            connectionRetryPolicy);
      } catch (ConnectException se) { // namenode has not been started
        LOG.info("Server at " + addr + " not available yet, Zzzzz...");
        ioe = se;
      } catch (SocketTimeoutException te) { // namenode is busy
        LOG.info("Problem connecting to server: " + addr);
        ioe = te;
      } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over
        LOG.info("No route to host for server: " + addr);
        ioe = nrthe;
      }
      // check if timed out
      if (Time.now() - timeout >= startTime) {
        throw ioe;
      }

      // wait for retry
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        // IGNORE
      }
    }
  }
コード例 #13
0
ファイル: LoadGenerator.java プロジェクト: FloodDragon/hadoop
 /**
  * Returns when the current number of seconds from the epoch equals the command line argument
  * given by <code>-startTime</code>. This allows multiple instances of this program, running on
  * clock synchronized nodes, to start at roughly the same time.
  */
 private static void barrier() {
   long sleepTime;
   while ((sleepTime = startTime - Time.now()) > 0) {
     try {
       Thread.sleep(sleepTime);
     } catch (InterruptedException ex) {
     }
   }
 }
コード例 #14
0
ファイル: StorageInfo.java プロジェクト: bcleenders/hops
  static String newBlockPoolID() throws UnknownHostException {
    String ip = "unknownIP";
    try {
      ip = DNS.getDefaultIP("default");
    } catch (UnknownHostException e) {
      System.out.println("Could not find ip address of \"default\" inteface.");
      throw e;
    }

    int rand = DFSUtil.getSecureRandom().nextInt(Integer.MAX_VALUE);
    String bpid = "BP-" + rand + "-" + ip + "-" + Time.now();
    return bpid;
  }
コード例 #15
0
ファイル: TestDiskError.java プロジェクト: aixuebo/had2.6.0
 /**
  * Checks whether {@link DataNode#checkDiskErrorAsync()} is being called or not. Before
  * refactoring the code the above function was not getting called
  *
  * @throws IOException, InterruptedException
  */
 @Test
 public void testcheckDiskError() throws IOException, InterruptedException {
   if (cluster.getDataNodes().size() <= 0) {
     cluster.startDataNodes(conf, 1, true, null, null);
     cluster.waitActive();
   }
   DataNode dataNode = cluster.getDataNodes().get(0);
   long slackTime = dataNode.checkDiskErrorInterval / 2;
   // checking for disk error
   dataNode.checkDiskErrorAsync();
   Thread.sleep(dataNode.checkDiskErrorInterval);
   long lastDiskErrorCheck = dataNode.getLastDiskErrorCheck();
   assertTrue(
       "Disk Error check is not performed within  " + dataNode.checkDiskErrorInterval + "  ms",
       ((Time.monotonicNow() - lastDiskErrorCheck)
           < (dataNode.checkDiskErrorInterval + slackTime)));
 }
コード例 #16
0
  /** Write the current dfsUsed to the cache file. */
  void saveDfsUsed() {
    File outFile = new File(currentDir, DU_CACHE_FILE);
    if (outFile.exists() && !outFile.delete()) {
      FsDatasetImpl.LOG.warn("Failed to delete old dfsUsed file in " + outFile.getParent());
    }

    try {
      long used = getDfsUsed();
      try (Writer out = new OutputStreamWriter(new FileOutputStream(outFile), "UTF-8")) {
        // mtime is written last, so that truncated writes won't be valid.
        out.write(Long.toString(used) + " " + Long.toString(Time.now()));
        out.flush();
      }
    } catch (IOException ioe) {
      // If write failed, the volume might be bad. Since the cache file is
      // not critical, log the error and continue.
      FsDatasetImpl.LOG.warn("Failed to write dfsUsed to " + outFile, ioe);
    }
  }
コード例 #17
0
/**
 * Keeps a Collection for every named machine containing blocks that have recently been invalidated
 * and are thought to live on the machine in question.
 */
@InterfaceAudience.Private
class InvalidateBlocks {
  /** Mapping: DatanodeInfo -> Collection of Blocks */
  private final Map<DatanodeInfo, LightWeightHashSet<Block>> node2blocks =
      new TreeMap<DatanodeInfo, LightWeightHashSet<Block>>();
  /** The total number of blocks in the map. */
  private long numBlocks = 0L;

  private final int blockInvalidateLimit;

  /** The period of pending time for block invalidation since the NameNode startup */
  private final long pendingPeriodInMs;
  /** the startup time */
  private final long startupTime = Time.monotonicNow();

  InvalidateBlocks(final int blockInvalidateLimit, long pendingPeriodInMs) {
    this.blockInvalidateLimit = blockInvalidateLimit;
    this.pendingPeriodInMs = pendingPeriodInMs;
    printBlockDeletionTime(BlockManager.LOG);
  }

  private void printBlockDeletionTime(final Log log) {
    log.info(
        DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY
            + " is set to "
            + DFSUtil.durationToString(pendingPeriodInMs));
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy MMM dd HH:mm:ss");
    Calendar calendar = new GregorianCalendar();
    calendar.add(Calendar.SECOND, (int) (this.pendingPeriodInMs / 1000));
    log.info("The block deletion will start around " + sdf.format(calendar.getTime()));
  }

  /** @return the number of blocks to be invalidated . */
  synchronized long numBlocks() {
    return numBlocks;
  }

  /**
   * @return true if the given storage has the given block listed for invalidation. Blocks are
   *     compared including their generation stamps: if a block is pending invalidation but with a
   *     different generation stamp, returns false.
   */
  synchronized boolean contains(final DatanodeInfo dn, final Block block) {
    final LightWeightHashSet<Block> s = node2blocks.get(dn);
    if (s == null) {
      return false; // no invalidate blocks for this storage ID
    }
    Block blockInSet = s.getElement(block);
    return blockInSet != null && block.getGenerationStamp() == blockInSet.getGenerationStamp();
  }

  /** Add a block to the block collection which will be invalidated on the specified datanode. */
  synchronized void add(final Block block, final DatanodeInfo datanode, final boolean log) {
    LightWeightHashSet<Block> set = node2blocks.get(datanode);
    if (set == null) {
      set = new LightWeightHashSet<Block>();
      node2blocks.put(datanode, set);
    }
    if (set.add(block)) {
      numBlocks++;
      if (log) {
        NameNode.blockStateChangeLog.info(
            "BLOCK* " + getClass().getSimpleName() + ": add " + block + " to " + datanode);
      }
    }
  }

  /** Remove a storage from the invalidatesSet */
  synchronized void remove(final DatanodeInfo dn) {
    final LightWeightHashSet<Block> blocks = node2blocks.remove(dn);
    if (blocks != null) {
      numBlocks -= blocks.size();
    }
  }

  /** Remove the block from the specified storage. */
  synchronized void remove(final DatanodeInfo dn, final Block block) {
    final LightWeightHashSet<Block> v = node2blocks.get(dn);
    if (v != null && v.remove(block)) {
      numBlocks--;
      if (v.isEmpty()) {
        node2blocks.remove(dn);
      }
    }
  }

  /** Print the contents to out. */
  synchronized void dump(final PrintWriter out) {
    final int size = node2blocks.values().size();
    out.println("Metasave: Blocks " + numBlocks + " waiting deletion from " + size + " datanodes.");
    if (size == 0) {
      return;
    }

    for (Map.Entry<DatanodeInfo, LightWeightHashSet<Block>> entry : node2blocks.entrySet()) {
      final LightWeightHashSet<Block> blocks = entry.getValue();
      if (blocks.size() > 0) {
        out.println(entry.getKey());
        out.println(blocks);
      }
    }
  }

  /** @return a list of the storage IDs. */
  synchronized List<DatanodeInfo> getDatanodes() {
    return new ArrayList<DatanodeInfo>(node2blocks.keySet());
  }

  /** @return the remianing pending time */
  @VisibleForTesting
  long getInvalidationDelay() {
    return pendingPeriodInMs - (Time.monotonicNow() - startupTime);
  }

  synchronized List<Block> invalidateWork(final DatanodeDescriptor dn) {
    final long delay = getInvalidationDelay();
    if (delay > 0) {
      if (BlockManager.LOG.isDebugEnabled()) {
        BlockManager.LOG.debug(
            "Block deletion is delayed during NameNode startup. "
                + "The deletion will start after "
                + delay
                + " ms.");
      }
      return null;
    }
    final LightWeightHashSet<Block> set = node2blocks.get(dn);
    if (set == null) {
      return null;
    }

    // # blocks that can be sent in one message is limited
    final int limit = blockInvalidateLimit;
    final List<Block> toInvalidate = set.pollN(limit);

    // If we send everything in this message, remove this node entry
    if (set.isEmpty()) {
      remove(dn);
    }

    dn.addBlocksToBeInvalidated(toInvalidate);
    numBlocks -= toInvalidate.size();
    return toInvalidate;
  }

  synchronized void clear() {
    node2blocks.clear();
    numBlocks = 0;
  }
}
コード例 #18
0
 /** @return the remianing pending time */
 @VisibleForTesting
 long getInvalidationDelay() {
   return pendingPeriodInMs - (Time.monotonicNow() - startupTime);
 }
コード例 #19
0
  /** process datanode heartbeat or stats initialization. */
  public void updateHeartbeatState(
      StorageReport[] reports,
      long cacheCapacity,
      long cacheUsed,
      int xceiverCount,
      int volFailures,
      VolumeFailureSummary volumeFailureSummary) {
    long totalCapacity = 0;
    long totalRemaining = 0;
    long totalBlockPoolUsed = 0;
    long totalDfsUsed = 0;
    Set<DatanodeStorageInfo> failedStorageInfos = null;

    // Decide if we should check for any missing StorageReport and mark it as
    // failed. There are different scenarios.
    // 1. When DN is running, a storage failed. Given the current DN
    //    implementation doesn't add recovered storage back to its storage list
    //    until DN restart, we can assume volFailures won't decrease
    //    during the current DN registration session.
    //    When volumeFailures == this.volumeFailures, it implies there is no
    //    state change. No need to check for failed storage. This is an
    //    optimization.  Recent versions of the DataNode report a
    //    VolumeFailureSummary containing the date/time of the last volume
    //    failure.  If that's available, then we check that instead for greater
    //    accuracy.
    // 2. After DN restarts, volFailures might not increase and it is possible
    //    we still have new failed storage. For example, admins reduce
    //    available storages in configuration. Another corner case
    //    is the failed volumes might change after restart; a) there
    //    is one good storage A, one restored good storage B, so there is
    //    one element in storageReports and that is A. b) A failed. c) Before
    //    DN sends HB to NN to indicate A has failed, DN restarts. d) After DN
    //    restarts, storageReports has one element which is B.
    final boolean checkFailedStorages;
    if (volumeFailureSummary != null && this.volumeFailureSummary != null) {
      checkFailedStorages =
          volumeFailureSummary.getLastVolumeFailureDate()
              > this.volumeFailureSummary.getLastVolumeFailureDate();
    } else {
      checkFailedStorages = (volFailures > this.volumeFailures) || !heartbeatedSinceRegistration;
    }

    if (checkFailedStorages) {
      LOG.info(
          "Number of failed storage changes from " + this.volumeFailures + " to " + volFailures);
      synchronized (storageMap) {
        failedStorageInfos = new HashSet<>(storageMap.values());
      }
    }

    setCacheCapacity(cacheCapacity);
    setCacheUsed(cacheUsed);
    setXceiverCount(xceiverCount);
    setLastUpdate(Time.now());
    setLastUpdateMonotonic(Time.monotonicNow());
    this.volumeFailures = volFailures;
    this.volumeFailureSummary = volumeFailureSummary;
    for (StorageReport report : reports) {
      DatanodeStorageInfo storage = updateStorage(report.getStorage());
      if (checkFailedStorages) {
        failedStorageInfos.remove(storage);
      }

      storage.receivedHeartbeat(report);
      totalCapacity += report.getCapacity();
      totalRemaining += report.getRemaining();
      totalBlockPoolUsed += report.getBlockPoolUsed();
      totalDfsUsed += report.getDfsUsed();
    }
    rollBlocksScheduled(getLastUpdateMonotonic());

    // Update total metrics for the node.
    setCapacity(totalCapacity);
    setRemaining(totalRemaining);
    setBlockPoolUsed(totalBlockPoolUsed);
    setDfsUsed(totalDfsUsed);
    if (checkFailedStorages) {
      updateFailedStorage(failedStorageInfos);
    }
    long storageMapSize;
    synchronized (storageMap) {
      storageMapSize = storageMap.size();
    }
    if (storageMapSize != reports.length) {
      pruneStorageMap(reports);
    }
  }
コード例 #20
0
      @Override
      public Writable call(
          org.apache.hadoop.ipc.RPC.Server server,
          String protocolName,
          Writable rpcRequest,
          long receivedTime)
          throws IOException {
        try {
          Invocation call = (Invocation) rpcRequest;
          if (server.verbose) log("Call: " + call);

          // Verify rpc version
          if (call.getRpcVersion() != writableRpcVersion) {
            // Client is using a different version of WritableRpc
            throw new IOException(
                "WritableRpc version mismatch, client side version="
                    + call.getRpcVersion()
                    + ", server side version="
                    + writableRpcVersion);
          }

          long clientVersion = call.getProtocolVersion();
          final String protoName;
          ProtoClassProtoImpl protocolImpl;
          if (call.declaringClassProtocolName.equals(VersionedProtocol.class.getName())) {
            // VersionProtocol methods are often used by client to figure out
            // which version of protocol to use.
            //
            // Versioned protocol methods should go the protocolName protocol
            // rather than the declaring class of the method since the
            // the declaring class is VersionedProtocol which is not
            // registered directly.
            // Send the call to the highest  protocol version
            VerProtocolImpl highest =
                server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protocolName);
            if (highest == null) {
              throw new IOException("Unknown protocol: " + protocolName);
            }
            protocolImpl = highest.protocolTarget;
          } else {
            protoName = call.declaringClassProtocolName;

            // Find the right impl for the protocol based on client version.
            ProtoNameVer pv = new ProtoNameVer(call.declaringClassProtocolName, clientVersion);
            protocolImpl = server.getProtocolImplMap(RPC.RpcKind.RPC_WRITABLE).get(pv);
            if (protocolImpl == null) { // no match for Protocol AND Version
              VerProtocolImpl highest =
                  server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protoName);
              if (highest == null) {
                throw new IOException("Unknown protocol: " + protoName);
              } else { // protocol supported but not the version that client wants
                throw new RPC.VersionMismatch(protoName, clientVersion, highest.version);
              }
            }
          }

          // Invoke the protocol method

          long startTime = Time.now();
          Method method =
              protocolImpl.protocolClass.getMethod(
                  call.getMethodName(), call.getParameterClasses());
          method.setAccessible(true);
          server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
          Object value = method.invoke(protocolImpl.protocolImpl, call.getParameters());
          int processingTime = (int) (Time.now() - startTime);
          int qTime = (int) (startTime - receivedTime);
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                "Served: "
                    + call.getMethodName()
                    + " queueTime= "
                    + qTime
                    + " procesingTime= "
                    + processingTime);
          }
          server.rpcMetrics.addRpcQueueTime(qTime);
          server.rpcMetrics.addRpcProcessingTime(processingTime);
          server.rpcDetailedMetrics.addProcessingTime(call.getMethodName(), processingTime);
          if (server.verbose) log("Return: " + value);

          return new ObjectWritable(method.getReturnType(), value);

        } catch (InvocationTargetException e) {
          Throwable target = e.getTargetException();
          if (target instanceof IOException) {
            throw (IOException) target;
          } else {
            IOException ioe = new IOException(target.toString());
            ioe.setStackTrace(target.getStackTrace());
            throw ioe;
          }
        } catch (Throwable e) {
          if (!(e instanceof IOException)) {
            LOG.error("Unexpected throwable object ", e);
          }
          IOException ioe = new IOException(e.toString());
          ioe.setStackTrace(e.getStackTrace());
          throw ioe;
        }
      }
コード例 #21
0
  /** Check files on DFS, starting from the indicated path. */
  public void fsck() {
    final long startTime = Time.now();
    try {
      String msg =
          "FSCK started by "
              + UserGroupInformation.getCurrentUser()
              + " from "
              + remoteAddress
              + " for path "
              + path
              + " at "
              + new Date();
      LOG.info(msg);
      out.println(msg);
      namenode.getNamesystem().logFsckEvent(path, remoteAddress);

      if (snapshottableDirs != null) {
        SnapshottableDirectoryStatus[] snapshotDirs =
            namenode.getRpcServer().getSnapshottableDirListing();
        if (snapshotDirs != null) {
          for (SnapshottableDirectoryStatus dir : snapshotDirs) {
            snapshottableDirs.add(dir.getFullPath().toString());
          }
        }
      }

      final HdfsFileStatus file = namenode.getRpcServer().getFileInfo(path);
      if (file != null) {

        if (showCorruptFileBlocks) {
          listCorruptFileBlocks();
          return;
        }

        Result res = new Result(conf);

        check(path, file, res);

        out.println(res);
        out.println(" Number of data-nodes:\t\t" + totalDatanodes);
        out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks());

        out.println(
            "FSCK ended at " + new Date() + " in " + (Time.now() - startTime + " milliseconds"));

        // If there were internal errors during the fsck operation, we want to
        // return FAILURE_STATUS, even if those errors were not immediately
        // fatal.  Otherwise many unit tests will pass even when there are bugs.
        if (internalError) {
          throw new IOException("fsck encountered internal errors!");
        }

        // DFSck client scans for the string HEALTHY/CORRUPT to check the status
        // of file system and return appropriate code. Changing the output
        // string might break testcases. Also note this must be the last line
        // of the report.
        if (res.isHealthy()) {
          out.print("\n\nThe filesystem under path '" + path + "' " + HEALTHY_STATUS);
        } else {
          out.print("\n\nThe filesystem under path '" + path + "' " + CORRUPT_STATUS);
        }

      } else {
        out.print("\n\nPath '" + path + "' " + NONEXISTENT_STATUS);
      }
    } catch (Exception e) {
      String errMsg = "Fsck on path '" + path + "' " + FAILURE_STATUS;
      LOG.warn(errMsg, e);
      out.println(
          "FSCK ended at " + new Date() + " in " + (Time.now() - startTime + " milliseconds"));
      out.println(e.getMessage());
      out.print("\n\n" + errMsg);
    } finally {
      out.close();
    }
  }
コード例 #22
0
ファイル: ViewFileSystem.java プロジェクト: wangxuemin/coding
 /**
  * This is the constructor with the signature needed by {@link FileSystem#createFileSystem(URI,
  * Configuration)}
  *
  * <p>After this constructor is called initialize() is called.
  *
  * @throws IOException
  */
 public ViewFileSystem() throws IOException {
   ugi = UserGroupInformation.getCurrentUser();
   creationTime = Time.now();
 }
コード例 #23
0
  /**
   * Add replicas under the given directory to the volume map
   *
   * @param volumeMap the replicas map
   * @param dir an input directory
   * @param lazyWriteReplicaMap Map of replicas on transient storage.
   * @param isFinalized true if the directory has finalized replicas; false if the directory has rbw
   *     replicas
   */
  void addToReplicasMap(
      ReplicaMap volumeMap,
      File dir,
      final RamDiskReplicaTracker lazyWriteReplicaMap,
      boolean isFinalized)
      throws IOException {
    File files[] = FileUtil.listFiles(dir);
    for (File file : files) {
      if (file.isDirectory()) {
        addToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized);
      }

      if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) {
        file = recoverTempUnlinkedBlock(file);
        if (file == null) { // the original block still exists, so we cover it
          // in another iteration and can continue here
          continue;
        }
      }
      if (!Block.isBlockFilename(file)) continue;

      long genStamp = FsDatasetUtil.getGenerationStampFromFile(files, file);
      long blockId = Block.filename2id(file.getName());
      ReplicaInfo newReplica = null;
      if (isFinalized) {
        newReplica =
            new FinalizedReplica(blockId, file.length(), genStamp, volume, file.getParentFile());
      } else {

        boolean loadRwr = true;
        File restartMeta =
            new File(file.getParent() + File.pathSeparator + "." + file.getName() + ".restart");
        Scanner sc = null;
        try {
          sc = new Scanner(restartMeta, "UTF-8");
          // The restart meta file exists
          if (sc.hasNextLong() && (sc.nextLong() > Time.now())) {
            // It didn't expire. Load the replica as a RBW.
            // We don't know the expected block length, so just use 0
            // and don't reserve any more space for writes.
            newReplica =
                new ReplicaBeingWritten(
                    blockId,
                    validateIntegrityAndSetLength(file, genStamp),
                    genStamp,
                    volume,
                    file.getParentFile(),
                    null,
                    0);
            loadRwr = false;
          }
          sc.close();
          if (!restartMeta.delete()) {
            FsDatasetImpl.LOG.warn("Failed to delete restart meta file: " + restartMeta.getPath());
          }
        } catch (FileNotFoundException fnfe) {
          // nothing to do hereFile dir =
        } finally {
          if (sc != null) {
            sc.close();
          }
        }
        // Restart meta doesn't exist or expired.
        if (loadRwr) {
          newReplica =
              new ReplicaWaitingToBeRecovered(
                  blockId,
                  validateIntegrityAndSetLength(file, genStamp),
                  genStamp,
                  volume,
                  file.getParentFile());
        }
      }

      ReplicaInfo oldReplica = volumeMap.get(bpid, newReplica.getBlockId());
      if (oldReplica == null) {
        volumeMap.add(bpid, newReplica);
      } else {
        // We have multiple replicas of the same block so decide which one
        // to keep.
        newReplica = resolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
      }

      // If we are retaining a replica on transient storage make sure
      // it is in the lazyWriteReplicaMap so it can be persisted
      // eventually.
      if (newReplica.getVolume().isTransientStorage()) {
        lazyWriteReplicaMap.addReplica(bpid, blockId, (FsVolumeImpl) newReplica.getVolume());
      } else {
        lazyWriteReplicaMap.discardReplica(bpid, blockId, false);
      }
    }
  }
コード例 #24
0
ファイル: LoadGenerator.java プロジェクト: FloodDragon/hadoop
/**
 * The load generator is a tool for testing NameNode behavior under different client loads. Note
 * there is a subclass of this clas that lets you run a the load generator as a MapReduce job (see
 * LoadGeneratorMR in the MapReduce project.
 *
 * <p>The loadGenerator allows the user to generate different mixes of read, write, and list
 * requests by specifying the probabilities of read and write. The user controls the intensity of
 * the load by adjusting parameters for the number of worker threads and the delay between
 * operations. While load generators are running, the user can profile and monitor the running of
 * the NameNode. When a load generator exits, it print some NameNode statistics like the average
 * execution time of each kind of operations and the NameNode throughput.
 *
 * <p>The program can run in one of two forms. As a regular single process command that runs
 * multiple threads to generate load on the NN or as a Map Reduce program that runs multiple
 * (multi-threaded) map tasks that generate load on the NN; the results summary is generated by a
 * single reduce task.
 *
 * <p>The user may either specify constant duration, read and write probabilities via the command
 * line, or may specify a text file that acts as a script of which read and write probabilities to
 * use for specified durations. If no duration is specified the program runs till killed (duration
 * required if run as MapReduce).
 *
 * <p>The script takes the form of lines of duration in seconds, read probability and write
 * probability, each separated by white space. Blank lines and lines starting with # (comments) are
 * ignored. If load generator is run as a MapReduce program then the script file needs to be
 * accessible on the the Map task as a HDFS file.
 *
 * <p>After command line argument parsing and data initialization, the load generator spawns the
 * number of worker threads as specified by the user. Each thread sends a stream of requests to the
 * NameNode. For each iteration, it first decides if it is going to read a file, create a file, or
 * listing a directory following the read and write probabilities specified by the user. When
 * reading, it randomly picks a file in the test space and reads the entire file. When writing, it
 * randomly picks a directory in the test space and creates a file whose name consists of the
 * current machine's host name and the thread id. The length of the file follows Gaussian
 * distribution with an average size of 2 blocks and the standard deviation of 1 block. The new file
 * is filled with 'a'. Immediately after the file creation completes, the file is deleted from the
 * test space. While listing, it randomly picks a directory in the test space and list the directory
 * content. Between two consecutive operations, the thread pauses for a random amount of time in the
 * range of [0, maxDelayBetweenOps] if the specified max delay is not zero. All threads are stopped
 * when the specified elapsed time has passed in command-line execution, or all the lines of script
 * have been executed, if using a script. Before exiting, the program prints the average execution
 * for each kind of NameNode operations, and the number of requests served by the NameNode.
 *
 * <p>The synopsis of the command is java LoadGenerator -readProbability <read probability>: read
 * probability [0, 1] with a default value of 0.3333. -writeProbability <write probability>: write
 * probability [0, 1] with a default value of 0.3333. -root <root>: test space with a default value
 * of /testLoadSpace -maxDelayBetweenOps <maxDelayBetweenOpsInMillis>: Max delay in the unit of
 * milliseconds between two operations with a default value of 0 indicating no delay. -numOfThreads
 * <numOfThreads>: number of threads to spawn with a default value of 200. -elapsedTime
 * <elapsedTimeInSecs>: the elapsed time of program with a default value of 0 indicating running
 * forever -startTime <startTimeInMillis> : when the threads start to run. -scriptFile <file name>:
 * text file to parse for scripted operation
 */
public class LoadGenerator extends Configured implements Tool {
  public static final Log LOG = LogFactory.getLog(LoadGenerator.class);

  private static volatile boolean shouldRun = true;
  protected static Path root = DataGenerator.DEFAULT_ROOT;
  private static FileContext fc;
  protected static int maxDelayBetweenOps = 0;
  protected static int numOfThreads = 200;
  protected static long[] durations = {0};
  protected static double[] readProbs = {0.3333};
  protected static double[] writeProbs = {0.3333};
  private static volatile int currentIndex = 0;
  protected static long totalTime = 0;
  protected static long startTime = Time.now() + 10000;
  private static final int BLOCK_SIZE = 10;
  private static ArrayList<String> files = new ArrayList<String>(); // a table of file names
  private static ArrayList<String> dirs = new ArrayList<String>(); // a table of directory names
  protected static Random r = null;
  protected static long seed = 0;
  protected static String scriptFile = null;
  protected static final String FLAGFILE_DEFAULT = "/tmp/flagFile";
  protected static Path flagFile = new Path(FLAGFILE_DEFAULT);
  protected String hostname;
  private static final String USAGE_CMD = "java LoadGenerator\n";
  protected static final String USAGE_ARGS =
      "-readProbability <read probability>\n"
          + "-writeProbability <write probability>\n"
          + "-root <root>\n"
          + "-maxDelayBetweenOps <maxDelayBetweenOpsInMillis>\n"
          + "-numOfThreads <numOfThreads>\n"
          + "-elapsedTime <elapsedTimeInSecs>\n"
          + "-startTime <startTimeInMillis>\n"
          + "-scriptFile <filename>\n"
          + "-flagFile <filename>";
  private static final String USAGE = USAGE_CMD + USAGE_ARGS;

  private final byte[] WRITE_CONTENTS = new byte[4096];

  private static final int ERR_TEST_FAILED = 2;

  /** Constructor */
  public LoadGenerator() throws IOException, UnknownHostException {
    InetAddress addr = InetAddress.getLocalHost();
    hostname = addr.getHostName();
    Arrays.fill(WRITE_CONTENTS, (byte) 'a');
  }

  public LoadGenerator(Configuration conf) throws IOException, UnknownHostException {
    this();
    setConf(conf);
  }

  protected static final int OPEN = 0;
  protected static final int LIST = 1;
  protected static final int CREATE = 2;
  protected static final int WRITE_CLOSE = 3;
  protected static final int DELETE = 4;
  protected static final int TOTAL_OP_TYPES = 5;
  protected static long[] executionTime = new long[TOTAL_OP_TYPES];
  protected static long[] numOfOps = new long[TOTAL_OP_TYPES];
  protected static long totalOps = 0; // across all of types

  /**
   * A thread sends a stream of requests to the NameNode. At each iteration, it first decides if it
   * is going to read a file, create a file, or listing a directory following the read and write
   * probabilities. When reading, it randomly picks a file in the test space and reads the entire
   * file. When writing, it randomly picks a directory in the test space and creates a file whose
   * name consists of the current machine's host name and the thread id. The length of the file
   * follows Gaussian distribution with an average size of 2 blocks and the standard deviation of 1
   * block. The new file is filled with 'a'. Immediately after the file creation completes, the file
   * is deleted from the test space. While listing, it randomly picks a directory in the test space
   * and list the directory content. Between two consecutive operations, the thread pauses for a
   * random amount of time in the range of [0, maxDelayBetweenOps] if the specified max delay is not
   * zero. A thread runs for the specified elapsed time if the time isn't zero. Otherwise, it runs
   * forever.
   */
  private class DFSClientThread extends Thread {
    private int id;
    private long[] executionTime = new long[TOTAL_OP_TYPES];
    private long[] totalNumOfOps = new long[TOTAL_OP_TYPES];
    private byte[] buffer = new byte[1024];
    private boolean failed;

    private DFSClientThread(int id) {
      this.id = id;
    }

    /**
     * Main loop for each thread Each iteration decides what's the next operation and then pauses.
     */
    @Override
    public void run() {
      try {
        while (shouldRun) {
          nextOp();
          delay();
        }
      } catch (Exception ioe) {
        System.err.println(ioe.getLocalizedMessage());
        ioe.printStackTrace();
        failed = true;
      }
    }

    /**
     * Let the thread pause for a random amount of time in the range of [0, maxDelayBetweenOps] if
     * the delay is not zero. Otherwise, no pause.
     */
    private void delay() throws InterruptedException {
      if (maxDelayBetweenOps > 0) {
        int delay = r.nextInt(maxDelayBetweenOps);
        Thread.sleep(delay);
      }
    }

    /**
     * Perform the next operation.
     *
     * <p>Depending on the read and write probabilities, the next operation could be either read,
     * write, or list.
     */
    private void nextOp() throws IOException {
      double rn = r.nextDouble();
      int i = currentIndex;

      if (LOG.isDebugEnabled()) LOG.debug("Thread " + this.id + " moving to index " + i);

      if (rn < readProbs[i]) {
        read();
      } else if (rn < readProbs[i] + writeProbs[i]) {
        write();
      } else {
        list();
      }
    }

    /** Read operation randomly picks a file in the test space and reads the entire file */
    private void read() throws IOException {
      String fileName = files.get(r.nextInt(files.size()));
      long startTime = Time.now();
      InputStream in = fc.open(new Path(fileName));
      executionTime[OPEN] += (Time.now() - startTime);
      totalNumOfOps[OPEN]++;
      while (in.read(buffer) != -1) {}
      in.close();
    }

    /**
     * The write operation randomly picks a directory in the test space and creates a file whose
     * name consists of the current machine's host name and the thread id. The length of the file
     * follows Gaussian distribution with an average size of 2 blocks and the standard deviation of
     * 1 block. The new file is filled with 'a'. Immediately after the file creation completes, the
     * file is deleted from the test space.
     */
    private void write() throws IOException {
      String dirName = dirs.get(r.nextInt(dirs.size()));
      Path file = new Path(dirName, hostname + id);
      double fileSize = 0;
      while ((fileSize = r.nextGaussian() + 2) <= 0) {}
      genFile(file, (long) (fileSize * BLOCK_SIZE));
      long startTime = Time.now();
      fc.delete(file, true);
      executionTime[DELETE] += (Time.now() - startTime);
      totalNumOfOps[DELETE]++;
    }

    /**
     * The list operation randomly picks a directory in the test space and list the directory
     * content.
     */
    private void list() throws IOException {
      String dirName = dirs.get(r.nextInt(dirs.size()));
      long startTime = Time.now();
      fc.listStatus(new Path(dirName));
      executionTime[LIST] += (Time.now() - startTime);
      totalNumOfOps[LIST]++;
    }

    /** Create a file with a length of <code>fileSize</code>. The file is filled with 'a'. */
    private void genFile(Path file, long fileSize) throws IOException {
      long startTime = Time.now();
      FSDataOutputStream out = null;
      try {
        out =
            fc.create(
                file,
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                CreateOpts.createParent(),
                CreateOpts.bufferSize(4096),
                CreateOpts.repFac((short) 3));
        executionTime[CREATE] += (Time.now() - startTime);
        numOfOps[CREATE]++;

        long i = fileSize;
        while (i > 0) {
          long s = Math.min(fileSize, WRITE_CONTENTS.length);
          out.write(WRITE_CONTENTS, 0, (int) s);
          i -= s;
        }

        startTime = Time.now();
        executionTime[WRITE_CLOSE] += (Time.now() - startTime);
        numOfOps[WRITE_CLOSE]++;
      } finally {
        IOUtils.cleanup(LOG, out);
      }
    }
  }

  /**
   * Main function called by tool runner. It first initializes data by parsing the command line
   * arguments. It then calls the loadGenerator
   */
  @Override
  public int run(String[] args) throws Exception {
    int exitCode = parseArgs(false, args);
    if (exitCode != 0) {
      return exitCode;
    }
    System.out.println(
        "Running LoadGenerator against fileSystem: "
            + FileContext.getFileContext().getDefaultFileSystem().getUri());
    exitCode = generateLoadOnNN();
    printResults(System.out);
    return exitCode;
  }

  boolean stopFileCreated() {
    try {
      fc.getFileStatus(flagFile);
    } catch (FileNotFoundException e) {
      return false;
    } catch (IOException e) {
      LOG.error("Got error when checking if file exists:" + flagFile, e);
    }
    LOG.info("Flag file was created. Stopping the test.");
    return true;
  }

  /**
   * This is the main function - run threads to generate load on NN It starts the number of
   * DFSClient threads as specified by the user. It stops all the threads when the specified elapsed
   * time is passed.
   */
  protected int generateLoadOnNN() throws InterruptedException {
    int hostHashCode = hostname.hashCode();
    if (seed == 0) {
      r = new Random(System.currentTimeMillis() + hostHashCode);
    } else {
      r = new Random(seed + hostHashCode);
    }
    try {
      fc = FileContext.getFileContext(getConf());
    } catch (IOException ioe) {
      System.err.println("Can not initialize the file system: " + ioe.getLocalizedMessage());
      return -1;
    }

    int status = initFileDirTables();
    if (status != 0) {
      return status;
    }
    barrier();

    DFSClientThread[] threads = new DFSClientThread[numOfThreads];
    for (int i = 0; i < numOfThreads; i++) {
      threads[i] = new DFSClientThread(i);
      threads[i].start();
    }

    if (durations[0] > 0) {
      if (durations.length == 1) { // There is a fixed run time
        while (shouldRun) {
          Thread.sleep(2000);
          totalTime += 2;
          if (totalTime >= durations[0] || stopFileCreated()) {
            shouldRun = false;
          }
        }
      } else {
        // script run

        while (shouldRun) {
          Thread.sleep(durations[currentIndex] * 1000);
          totalTime += durations[currentIndex];
          // Are we on the final line of the script?
          if ((currentIndex + 1) == durations.length || stopFileCreated()) {
            shouldRun = false;
          } else {
            if (LOG.isDebugEnabled()) {
              LOG.debug(
                  "Moving to index "
                      + currentIndex
                      + ": r = "
                      + readProbs[currentIndex]
                      + ", w = "
                      + writeProbs
                      + " for duration "
                      + durations[currentIndex]);
            }
            currentIndex++;
          }
        }
      }
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Done with testing.  Waiting for threads to finish.");
    }

    boolean failed = false;
    for (DFSClientThread thread : threads) {
      thread.join();
      for (int i = 0; i < TOTAL_OP_TYPES; i++) {
        executionTime[i] += thread.executionTime[i];
        numOfOps[i] += thread.totalNumOfOps[i];
      }
      failed = failed || thread.failed;
    }
    int exitCode = 0;
    if (failed) {
      exitCode = -ERR_TEST_FAILED;
    }

    totalOps = 0;
    for (int i = 0; i < TOTAL_OP_TYPES; i++) {
      totalOps += numOfOps[i];
    }
    return exitCode;
  }

  protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
    out.println(
        "Result of running LoadGenerator against fileSystem: "
            + FileContext.getFileContext().getDefaultFileSystem().getUri());
    if (numOfOps[OPEN] != 0) {
      out.println(
          "Average open execution time: " + (double) executionTime[OPEN] / numOfOps[OPEN] + "ms");
    }
    if (numOfOps[LIST] != 0) {
      out.println(
          "Average list execution time: " + (double) executionTime[LIST] / numOfOps[LIST] + "ms");
    }
    if (numOfOps[DELETE] != 0) {
      out.println(
          "Average deletion execution time: "
              + (double) executionTime[DELETE] / numOfOps[DELETE]
              + "ms");
      out.println(
          "Average create execution time: "
              + (double) executionTime[CREATE] / numOfOps[CREATE]
              + "ms");
      out.println(
          "Average write_close execution time: "
              + (double) executionTime[WRITE_CLOSE] / numOfOps[WRITE_CLOSE]
              + "ms");
    }
    if (totalTime != 0) {
      out.println("Average operations per second: " + (double) totalOps / totalTime + "ops/s");
    }
    out.println();
  }

  /** Parse the command line arguments and initialize the data */
  protected int parseArgs(boolean runAsMapReduce, String[] args) throws IOException {
    try {
      for (int i = 0; i < args.length; i++) { // parse command line
        if (args[i].equals("-scriptFile")) {
          scriptFile = args[++i];
          if (durations[0] > 0) {
            System.err.println("Can't specify elapsedTime and use script.");
            return -1;
          }
        } else if (args[i].equals("-readProbability")) {
          if (scriptFile != null) {
            System.err.println("Can't specify probabilities and use script.");
            return -1;
          }
          readProbs[0] = Double.parseDouble(args[++i]);
          if (readProbs[0] < 0 || readProbs[0] > 1) {
            System.err.println("The read probability must be [0, 1]: " + readProbs[0]);
            return -1;
          }
        } else if (args[i].equals("-writeProbability")) {
          if (scriptFile != null) {
            System.err.println("Can't specify probabilities and use script.");
            return -1;
          }
          writeProbs[0] = Double.parseDouble(args[++i]);
          if (writeProbs[0] < 0 || writeProbs[0] > 1) {
            System.err.println("The write probability must be [0, 1]: " + writeProbs[0]);
            return -1;
          }
        } else if (args[i].equals("-root")) {
          root = new Path(args[++i]);
        } else if (args[i].equals("-maxDelayBetweenOps")) {
          maxDelayBetweenOps = Integer.parseInt(args[++i]); // in milliseconds
        } else if (args[i].equals("-numOfThreads")) {
          numOfThreads = Integer.parseInt(args[++i]);
          if (numOfThreads <= 0) {
            System.err.println("Number of threads must be positive: " + numOfThreads);
            return -1;
          }
        } else if (args[i].equals("-startTime")) {
          startTime = Long.parseLong(args[++i]);
        } else if (args[i].equals("-elapsedTime")) {
          if (scriptFile != null) {
            System.err.println("Can't specify elapsedTime and use script.");
            return -1;
          }
          durations[0] = Long.parseLong(args[++i]);
        } else if (args[i].equals("-seed")) {
          seed = Long.parseLong(args[++i]);
          r = new Random(seed);
        } else if (args[i].equals("-flagFile")) {
          LOG.info("got flagFile:" + flagFile);
          flagFile = new Path(args[++i]);
        } else {
          System.err.println(USAGE);
          ToolRunner.printGenericCommandUsage(System.err);
          return -1;
        }
      }
    } catch (NumberFormatException e) {
      System.err.println("Illegal parameter: " + e.getLocalizedMessage());
      System.err.println(USAGE);
      return -1;
    }

    // Load Script File if not MR; for MR scriptFile is loaded by Mapper
    if (!runAsMapReduce && scriptFile != null) {
      if (loadScriptFile(scriptFile, true) == -1) return -1;
    }

    for (int i = 0; i < readProbs.length; i++) {
      if (readProbs[i] + writeProbs[i] < 0 || readProbs[i] + writeProbs[i] > 1) {
        System.err.println(
            "The sum of read probability and write probability must be [0, 1]: "
                + readProbs[i]
                + " "
                + writeProbs[i]);
        return -1;
      }
    }
    return 0;
  }

  private static void parseScriptLine(
      String line,
      ArrayList<Long> duration,
      ArrayList<Double> readProb,
      ArrayList<Double> writeProb) {
    String[] a = line.split("\\s");

    if (a.length != 3) {
      throw new IllegalArgumentException("Incorrect number of parameters: " + line);
    }

    try {
      long d = Long.parseLong(a[0]);
      double r = Double.parseDouble(a[1]);
      double w = Double.parseDouble(a[2]);

      Preconditions.checkArgument(d >= 0, "Invalid duration: " + d);
      Preconditions.checkArgument(0 <= r && r <= 1.0, "The read probability must be [0, 1]: " + r);
      Preconditions.checkArgument(0 <= w && w <= 1.0, "The read probability must be [0, 1]: " + w);

      readProb.add(r);
      duration.add(d);
      writeProb.add(w);
    } catch (NumberFormatException nfe) {
      throw new IllegalArgumentException("Cannot parse: " + line);
    }
  }

  /**
   * Read a script file of the form: lines of text with duration in seconds, read probability and
   * write probability, separated by white space.
   *
   * @param filename Script file
   * @return 0 if successful, -1 if not
   * @throws IOException if errors with file IO
   */
  protected static int loadScriptFile(String filename, boolean readLocally) throws IOException {

    FileContext fc;
    if (readLocally) { // read locally - program is run without MR
      fc = FileContext.getLocalFSFileContext();
    } else {
      fc = FileContext.getFileContext(); // use default file system
    }
    DataInputStream in = null;
    try {
      in = fc.open(new Path(filename));
    } catch (IOException e) {
      System.err.println("Unable to open scriptFile: " + filename);

      System.exit(-1);
    }
    InputStreamReader inr = new InputStreamReader(in);

    BufferedReader br = new BufferedReader(inr);
    ArrayList<Long> duration = new ArrayList<Long>();
    ArrayList<Double> readProb = new ArrayList<Double>();
    ArrayList<Double> writeProb = new ArrayList<Double>();
    int lineNum = 0;

    String line;
    // Read script, parse values, build array of duration, read and write probs

    try {
      while ((line = br.readLine()) != null) {
        lineNum++;
        if (line.startsWith("#") || line.isEmpty()) // skip comments and blanks
        continue;

        parseScriptLine(line, duration, readProb, writeProb);
      }
    } catch (IllegalArgumentException e) {
      System.err.println("Line: " + lineNum + ", " + e.getMessage());
      return -1;
    } finally {
      IOUtils.cleanup(LOG, br);
    }

    // Copy vectors to arrays of values, to avoid autoboxing overhead later
    durations = new long[duration.size()];
    readProbs = new double[readProb.size()];
    writeProbs = new double[writeProb.size()];

    for (int i = 0; i < durations.length; i++) {
      durations[i] = duration.get(i);
      readProbs[i] = readProb.get(i);
      writeProbs[i] = writeProb.get(i);
    }

    if (durations[0] == 0)
      System.err.println("Initial duration set to 0.  " + "Will loop until stopped manually.");

    return 0;
  }

  /**
   * Create a table that contains all directories under root and another table that contains all
   * files under root.
   */
  private int initFileDirTables() {
    try {
      initFileDirTables(root);
    } catch (IOException e) {
      System.err.println(e.getLocalizedMessage());
      e.printStackTrace();
      return -1;
    }
    if (dirs.isEmpty()) {
      System.err.println("The test space " + root + " is empty");
      return -1;
    }
    if (files.isEmpty()) {
      System.err.println("The test space " + root + " does not have any file");
      return -1;
    }
    return 0;
  }

  /**
   * Create a table that contains all directories under the specified path and another table that
   * contains all files under the specified path and whose name starts with "_file_".
   */
  private void initFileDirTables(Path path) throws IOException {
    FileStatus[] stats = fc.util().listStatus(path);

    for (FileStatus stat : stats) {
      if (stat.isDirectory()) {
        dirs.add(stat.getPath().toString());
        initFileDirTables(stat.getPath());
      } else {
        Path filePath = stat.getPath();
        if (filePath.getName().startsWith(StructureGenerator.FILE_NAME_PREFIX)) {
          files.add(filePath.toString());
        }
      }
    }
  }

  /**
   * Returns when the current number of seconds from the epoch equals the command line argument
   * given by <code>-startTime</code>. This allows multiple instances of this program, running on
   * clock synchronized nodes, to start at roughly the same time.
   */
  private static void barrier() {
    long sleepTime;
    while ((sleepTime = startTime - Time.now()) > 0) {
      try {
        Thread.sleep(sleepTime);
      } catch (InterruptedException ex) {
      }
    }
  }

  /**
   * Main program
   *
   * @param args command line arguments
   * @throws Exception
   */
  public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new Configuration(), new LoadGenerator(), args);
    System.exit(res);
  }
}