Пример #1
0
  /* This test start a one-node cluster, fill the node to be 30% full;
   * It then adds an empty node and start balancing.
   * @param newCapacity new node's capacity
   * @param new
   */
  private void test(long[] capacities, String[] racks, long newCapacity, String newRack)
      throws Exception {
    int numOfDatanodes = capacities.length;
    assertEquals(numOfDatanodes, racks.length);
    cluster = new MiniDFSCluster(0, CONF, capacities.length, true, true, null, racks, capacities);
    try {
      cluster.waitActive();
      client = DFSClient.createNamenode(CONF);

      long totalCapacity = 0L;
      for (long capacity : capacities) {
        totalCapacity += capacity;
      }
      // fill up the cluster to be 30% full
      long totalUsedSpace = totalCapacity * 3 / 10;
      createFile(totalUsedSpace / numOfDatanodes, (short) numOfDatanodes);
      // start up an empty node with the same capacity and on the same rack
      cluster.startDataNodes(CONF, 1, true, null, new String[] {newRack}, new long[] {newCapacity});

      totalCapacity += newCapacity;

      // run balancer and validate results
      runBalancer(totalUsedSpace, totalCapacity);
    } finally {
      cluster.shutdown();
    }
  }
Пример #2
0
  /* fill up a cluster with <code>numNodes</code> datanodes
   * whose used space to be <code>size</code>
   */
  private Block[] generateBlocks(long size, short numNodes) throws IOException {
    cluster = new MiniDFSCluster(CONF, numNodes, true, null);
    try {
      cluster.waitActive();
      client = DFSClient.createNamenode(CONF);

      short replicationFactor = (short) (numNodes - 1);
      long fileLen = size / replicationFactor;
      createFile(fileLen, replicationFactor);

      List<LocatedBlock> locatedBlocks =
          client.getBlockLocations(fileName, 0, fileLen).getLocatedBlocks();

      int numOfBlocks = locatedBlocks.size();
      Block[] blocks = new Block[numOfBlocks];
      for (int i = 0; i < numOfBlocks; i++) {
        Block b = locatedBlocks.get(i).getBlock();
        blocks[i] = new Block(b.getBlockId(), b.getNumBytes(), b.getGenerationStamp());
      }

      return blocks;
    } finally {
      cluster.shutdown();
    }
  }
Пример #3
0
  /* we first start a cluster and fill the cluster up to a certain size.
   * then redistribute blocks according the required distribution.
   * Afterwards a balancer is running to balance the cluster.
   */
  private void testUnevenDistribution(long distribution[], long capacities[], String[] racks)
      throws Exception {
    int numDatanodes = distribution.length;
    if (capacities.length != numDatanodes || racks.length != numDatanodes) {
      throw new IllegalArgumentException("Array length is not the same");
    }

    // calculate total space that need to be filled
    long totalUsedSpace = 0L;
    for (int i = 0; i < distribution.length; i++) {
      totalUsedSpace += distribution[i];
    }

    // fill the cluster
    Block[] blocks = generateBlocks(totalUsedSpace, (short) numDatanodes);

    // redistribute blocks
    Block[][] blocksDN = distributeBlocks(blocks, (short) (numDatanodes - 1), distribution);

    // restart the cluster: do NOT format the cluster
    CONF.set("dfs.safemode.threshold.pct", "0.0f");
    cluster = new MiniDFSCluster(0, CONF, numDatanodes, false, true, null, racks, capacities);
    cluster.waitActive();
    client = DFSClient.createNamenode(CONF);

    cluster.injectBlocks(blocksDN);

    long totalCapacity = 0L;
    for (long capacity : capacities) {
      totalCapacity += capacity;
    }
    runBalancer(totalUsedSpace, totalCapacity);
  }
Пример #4
0
 /** Create a {@link NameNode} proxy from the current {@link ServletContext}. */
 protected ClientProtocol createNameNodeProxy() throws IOException {
   ServletContext context = getServletContext();
   // if we are running in the Name Node, use it directly rather than via rpc
   NameNode nn = (NameNode) context.getAttribute("name.node");
   if (nn != null) {
     return nn;
   }
   InetSocketAddress nnAddr = (InetSocketAddress) context.getAttribute("name.node.address");
   Configuration conf =
       new Configuration((Configuration) context.getAttribute(JspHelper.CURRENT_CONF));
   return DFSClient.createNamenode(nnAddr, conf);
 }
Пример #5
0
  @Test
  public void testDfsClient() throws IOException {
    Configuration conf = MyConf.getConfiguration();
    DFSClient client = new DFSClient(conf);
    if (logger.isInfoEnabled()) {
      logger.info("testDfsClient() - DFSClient client=" + client.toString()); // $NON-NLS-1$
    }

    ClientProtocol protocol = DFSClient.createNamenode(conf);
    System.out.println(protocol.getClass().toString());
    ContentSummary summary = protocol.getContentSummary("/penglin");
    System.out.println(summary.getFileCount());
    System.out.println(summary.toString());
    System.out.println(summary.getSpaceConsumed());
  }