/* This test start a one-node cluster, fill the node to be 30% full;
* It then adds an empty node and start balancing.
* @param newCapacity new node's capacity
* @param new
*/
private void test(long[] capacities, String[] racks, long newCapacity, String newRack)
throws Exception {
int numOfDatanodes = capacities.length;
assertEquals(numOfDatanodes, racks.length);
cluster = new MiniDFSCluster(0, CONF, capacities.length, true, true, null, racks, capacities);
try {
cluster.waitActive();
client = DFSClient.createNamenode(CONF);
long totalCapacity = 0L;
for (long capacity : capacities) {
totalCapacity += capacity;
}
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity * 3 / 10;
createFile(totalUsedSpace / numOfDatanodes, (short) numOfDatanodes);
// start up an empty node with the same capacity and on the same rack
cluster.startDataNodes(CONF, 1, true, null, new String[] {newRack}, new long[] {newCapacity});
totalCapacity += newCapacity;
// run balancer and validate results
runBalancer(totalUsedSpace, totalCapacity);
} finally {
cluster.shutdown();
}
}
/* fill up a cluster with <code>numNodes</code> datanodes
* whose used space to be <code>size</code>
*/
private Block[] generateBlocks(long size, short numNodes) throws IOException {
cluster = new MiniDFSCluster(CONF, numNodes, true, null);
try {
cluster.waitActive();
client = DFSClient.createNamenode(CONF);
short replicationFactor = (short) (numNodes - 1);
long fileLen = size / replicationFactor;
createFile(fileLen, replicationFactor);
List<LocatedBlock> locatedBlocks =
client.getBlockLocations(fileName, 0, fileLen).getLocatedBlocks();
int numOfBlocks = locatedBlocks.size();
Block[] blocks = new Block[numOfBlocks];
for (int i = 0; i < numOfBlocks; i++) {
Block b = locatedBlocks.get(i).getBlock();
blocks[i] = new Block(b.getBlockId(), b.getNumBytes(), b.getGenerationStamp());
}
return blocks;
} finally {
cluster.shutdown();
}
}
/* we first start a cluster and fill the cluster up to a certain size.
* then redistribute blocks according the required distribution.
* Afterwards a balancer is running to balance the cluster.
*/
private void testUnevenDistribution(long distribution[], long capacities[], String[] racks)
throws Exception {
int numDatanodes = distribution.length;
if (capacities.length != numDatanodes || racks.length != numDatanodes) {
throw new IllegalArgumentException("Array length is not the same");
}
// calculate total space that need to be filled
long totalUsedSpace = 0L;
for (int i = 0; i < distribution.length; i++) {
totalUsedSpace += distribution[i];
}
// fill the cluster
Block[] blocks = generateBlocks(totalUsedSpace, (short) numDatanodes);
// redistribute blocks
Block[][] blocksDN = distributeBlocks(blocks, (short) (numDatanodes - 1), distribution);
// restart the cluster: do NOT format the cluster
CONF.set("dfs.safemode.threshold.pct", "0.0f");
cluster = new MiniDFSCluster(0, CONF, numDatanodes, false, true, null, racks, capacities);
cluster.waitActive();
client = DFSClient.createNamenode(CONF);
cluster.injectBlocks(blocksDN);
long totalCapacity = 0L;
for (long capacity : capacities) {
totalCapacity += capacity;
}
runBalancer(totalUsedSpace, totalCapacity);
}
/** Create a {@link NameNode} proxy from the current {@link ServletContext}. */
protected ClientProtocol createNameNodeProxy() throws IOException {
ServletContext context = getServletContext();
// if we are running in the Name Node, use it directly rather than via rpc
NameNode nn = (NameNode) context.getAttribute("name.node");
if (nn != null) {
return nn;
}
InetSocketAddress nnAddr = (InetSocketAddress) context.getAttribute("name.node.address");
Configuration conf =
new Configuration((Configuration) context.getAttribute(JspHelper.CURRENT_CONF));
return DFSClient.createNamenode(nnAddr, conf);
}
@Test
public void testDfsClient() throws IOException {
Configuration conf = MyConf.getConfiguration();
DFSClient client = new DFSClient(conf);
if (logger.isInfoEnabled()) {
logger.info("testDfsClient() - DFSClient client=" + client.toString()); // $NON-NLS-1$
}
ClientProtocol protocol = DFSClient.createNamenode(conf);
System.out.println(protocol.getClass().toString());
ContentSummary summary = protocol.getContentSummary("/penglin");
System.out.println(summary.getFileCount());
System.out.println(summary.toString());
System.out.println(summary.getSpaceConsumed());
}