Ejemplo n.º 1
0
 /** Verify the first block of the file is corrupted (for all its replica). */
 private void verifyFirstBlockCorrupted(Path filePath, boolean isCorrupted)
     throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException {
   final LocatedBlocks locatedBlocks =
       dfs.dfs.getNamenode().getBlockLocations(filePath.toUri().getPath(), 0, Long.MAX_VALUE);
   final LocatedBlock firstLocatedBlock = locatedBlocks.get(0);
   Assert.assertEquals(isCorrupted, firstLocatedBlock.isCorrupt());
 }
Ejemplo n.º 2
0
    public void reorderBlocks(Configuration conf, LocatedBlocks lbs, String src)
        throws IOException {

      ServerName sn = AbstractFSWALProvider.getServerNameFromWALDirectoryName(conf, src);
      if (sn == null) {
        // It's not an WAL
        return;
      }

      // Ok, so it's an WAL
      String hostName = sn.getHostname();
      if (LOG.isTraceEnabled()) {
        LOG.trace(src + " is an WAL file, so reordering blocks, last hostname will be:" + hostName);
      }

      // Just check for all blocks
      for (LocatedBlock lb : lbs.getLocatedBlocks()) {
        DatanodeInfo[] dnis = lb.getLocations();
        if (dnis != null && dnis.length > 1) {
          boolean found = false;
          for (int i = 0; i < dnis.length - 1 && !found; i++) {
            if (hostName.equals(dnis[i].getHostName())) {
              // advance the other locations by one and put this one at the last place.
              DatanodeInfo toLast = dnis[i];
              System.arraycopy(dnis, i + 1, dnis, i, dnis.length - i - 1);
              dnis[dnis.length - 1] = toLast;
              found = true;
            }
          }
        }
      }
    }
Ejemplo n.º 3
0
  private void checkForCorruptOpenFiles(FileStatus file, List<FileStatus> corruptFiles)
      throws IOException {
    String filePath = file.getPath().toUri().getPath();

    if (file.isDir()) {
      for (FileStatus fileStatus : nn.namesystem.dir.getListing(filePath)) {
        checkForCorruptOpenFiles(fileStatus, corruptFiles);
      }

    } else {
      LeaseManager.Lease lease = nn.getNamesystem().leaseManager.getLeaseByPath(filePath);
      // Condition:
      //  1. lease has expired hard limit
      //  2. the file is open for write
      //  3. the last block has 0 locations
      if (lease != null && lease.expiredHardLimit()) {
        LocatedBlocks blocks = nn.getNamesystem().getBlockLocations(filePath, 0, file.getLen());
        List<LocatedBlock> locatedBlockList = blocks.getLocatedBlocks();
        LocatedBlock lastBlock = locatedBlockList.get(locatedBlockList.size() - 1);

        if (blocks.isUnderConstruction() && lastBlock.getLocations().length == 0) {
          corruptFiles.add(file);
        }
      }
    }
  }
Ejemplo n.º 4
0
  /**
   * File partially fit in RamDisk after eviction. RamDisk can fit 2 blocks. Write a file with 5
   * blocks. Expect 2 or less blocks are on RamDisk and 3 or more on disk.
   *
   * @throws IOException
   */
  @Test
  public void testFallbackToDiskPartial() throws IOException, InterruptedException {
    startUpCluster(true, 2);
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path path = new Path("/" + METHOD_NAME + ".dat");

    makeTestFile(path, BLOCK_SIZE * 5, true);

    // Sleep for a short time to allow the lazy writer thread to do its job
    Thread.sleep(6 * LAZY_WRITER_INTERVAL_SEC * 1000);

    triggerBlockReport();

    int numBlocksOnRamDisk = 0;
    int numBlocksOnDisk = 0;

    long fileLength = client.getFileInfo(path.toString()).getLen();
    LocatedBlocks locatedBlocks = client.getLocatedBlocks(path.toString(), 0, fileLength);
    for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
      if (locatedBlock.getStorageTypes()[0] == RAM_DISK) {
        numBlocksOnRamDisk++;
      } else if (locatedBlock.getStorageTypes()[0] == DEFAULT) {
        numBlocksOnDisk++;
      }
    }

    // Since eviction is asynchronous, depending on the timing of eviction
    // wrt writes, we may get 2 or less blocks on RAM disk.
    assert (numBlocksOnRamDisk <= 2);
    assert (numBlocksOnDisk >= 3);
  }
Ejemplo n.º 5
0
  /**
   * Make sure at least one non-transient volume has a saved copy of the replica. An infinite loop
   * is used to ensure the async lazy persist tasks are completely done before verification. Caller
   * of ensureLazyPersistBlocksAreSaved expects either a successful pass or timeout failure.
   */
  protected final void ensureLazyPersistBlocksAreSaved(LocatedBlocks locatedBlocks)
      throws IOException, InterruptedException {
    final String bpid = cluster.getNamesystem().getBlockPoolId();
    List<? extends FsVolumeSpi> volumes = cluster.getDataNodes().get(0).getFSDataset().getVolumes();
    final Set<Long> persistedBlockIds = new HashSet<Long>();

    while (persistedBlockIds.size() < locatedBlocks.getLocatedBlocks().size()) {
      // Take 1 second sleep before each verification iteration
      Thread.sleep(1000);

      for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
        for (FsVolumeSpi v : volumes) {
          if (v.isTransientStorage()) {
            continue;
          }

          FsVolumeImpl volume = (FsVolumeImpl) v;
          File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();

          long blockId = lb.getBlock().getBlockId();
          File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, blockId);
          File blockFile = new File(targetDir, lb.getBlock().getBlockName());
          if (blockFile.exists()) {
            // Found a persisted copy for this block and added to the Set
            persistedBlockIds.add(blockId);
          }
        }
      }
    }

    // We should have found a persisted copy for each located block.
    assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
  }
 /**
  * Get blocks in the specified range. Includes only the complete blocks. Fetch them from the
  * namenode if not cached.
  */
 private synchronized List<LocatedBlock> getFinalizedBlockRange(long offset, long length)
     throws IOException {
   assert (locatedBlocks != null) : "locatedBlocks is null";
   List<LocatedBlock> blockRange = new ArrayList<LocatedBlock>();
   // search cached blocks first
   int blockIdx = locatedBlocks.findBlock(offset);
   if (blockIdx < 0) { // block is not cached
     blockIdx = LocatedBlocks.getInsertIndex(blockIdx);
   }
   long remaining = length;
   long curOff = offset;
   while (remaining > 0) {
     LocatedBlock blk = null;
     if (blockIdx < locatedBlocks.locatedBlockCount()) blk = locatedBlocks.get(blockIdx);
     if (blk == null || curOff < blk.getStartOffset()) {
       LocatedBlocks newBlocks;
       newBlocks = DFSClient.callGetBlockLocations(dfsClient.namenode, src, curOff, remaining);
       locatedBlocks.insertRange(blockIdx, newBlocks.getLocatedBlocks());
       continue;
     }
     assert curOff >= blk.getStartOffset() : "Block not found";
     blockRange.add(blk);
     long bytesRead = blk.getStartOffset() + blk.getBlockSize() - curOff;
     remaining -= bytesRead;
     curOff += bytesRead;
     blockIdx++;
   }
   return blockRange;
 }
  static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy(
      DatanodeID datanodeid,
      Configuration conf,
      int socketTimeout,
      boolean connectToDnViaHostname,
      LocatedBlock locatedBlock)
      throws IOException {
    final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
    InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
    }

    // Since we're creating a new UserGroupInformation here, we know that no
    // future RPC proxies will be able to re-use the same connection. And
    // usages of this proxy tend to be one-off calls.
    //
    // This is a temporary fix: callers should really achieve this by using
    // RPC.stopProxy() on the resulting object, but this is currently not
    // working in trunk. See the discussion on HDFS-1965.
    Configuration confWithNoIpcIdle = new Configuration(conf);
    confWithNoIpcIdle.setInt(
        CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);

    UserGroupInformation ticket =
        UserGroupInformation.createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
    ticket.addToken(locatedBlock.getBlockToken());
    return createClientDatanodeProtocolProxy(
        addr, ticket, confWithNoIpcIdle, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
  }
Ejemplo n.º 8
0
 /**
  * Verify the number of corrupted block replicas by fetching the block location from name node.
  */
 private void verifyCorruptedBlockCount(Path filePath, int expectedReplicas)
     throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException {
   final LocatedBlocks lBlocks =
       dfs.dfs.getNamenode().getBlockLocations(filePath.toUri().getPath(), 0, Long.MAX_VALUE);
   // we expect only the first block of the file is used for this test
   LocatedBlock firstLocatedBlock = lBlocks.get(0);
   Assert.assertEquals(expectedReplicas, firstLocatedBlock.getLocations().length);
 }
  /**
   * Setup a {@link MiniDFSCluster}. Create a block with both {@link State#NORMAL} and {@link
   * State#READ_ONLY_SHARED} replicas.
   */
  @Before
  public void setup() throws IOException, InterruptedException {
    conf = new HdfsConfiguration();
    SimulatedFSDataset.setFactory(conf);

    Configuration[] overlays = new Configuration[NUM_DATANODES];
    for (int i = 0; i < overlays.length; i++) {
      overlays[i] = new Configuration();
      if (i == RO_NODE_INDEX) {
        overlays[i].setEnum(
            SimulatedFSDataset.CONFIG_PROPERTY_STATE,
            i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL);
      }
    }

    cluster =
        new MiniDFSCluster.Builder(conf)
            .numDataNodes(NUM_DATANODES)
            .dataNodeConfOverlays(overlays)
            .build();
    fs = cluster.getFileSystem();
    blockManager = cluster.getNameNode().getNamesystem().getBlockManager();
    datanodeManager = blockManager.getDatanodeManager();
    client =
        new DFSClient(
            new InetSocketAddress("localhost", cluster.getNameNodePort()),
            cluster.getConfiguration(0));

    for (int i = 0; i < NUM_DATANODES; i++) {
      DataNode dataNode = cluster.getDataNodes().get(i);
      validateStorageState(
          BlockManagerTestUtil.getStorageReportsForDatanode(
              datanodeManager.getDatanode(dataNode.getDatanodeId())),
          i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL);
    }

    // Create a 1 block file
    DFSTestUtil.createFile(fs, PATH, BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE, (short) 1, seed);

    LocatedBlock locatedBlock = getLocatedBlock();
    extendedBlock = locatedBlock.getBlock();
    block = extendedBlock.getLocalBlock();

    assertThat(locatedBlock.getLocations().length, is(1));
    normalDataNode = locatedBlock.getLocations()[0];
    readOnlyDataNode =
        datanodeManager.getDatanode(cluster.getDataNodes().get(RO_NODE_INDEX).getDatanodeId());
    assertThat(normalDataNode, is(not(readOnlyDataNode)));

    validateNumberReplicas(1);

    // Inject the block into the datanode with READ_ONLY_SHARED storage
    cluster.injectBlocks(0, RO_NODE_INDEX, Collections.singleton(block));

    // There should now be 2 *locations* for the block
    // Must wait until the NameNode has processed the block report for the injected blocks
    waitForLocations(2);
  }
  private DNAddrPair chooseDataNode(LocatedBlock block) throws IOException {
    while (true) {
      DatanodeInfo[] nodes = block.getLocations();
      try {
        DatanodeInfo chosenNode = bestNode(nodes, deadNodes);
        InetSocketAddress targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
        return new DNAddrPair(chosenNode, targetAddr);
      } catch (IOException ie) {
        String blockInfo = block.getBlock() + " file=" + src;
        if (failures >= dfsClient.getMaxBlockAcquireFailures()) {
          throw new BlockMissingException(
              src, "Could not obtain block: " + blockInfo, block.getStartOffset());
        }

        if (nodes == null || nodes.length == 0) {
          DFSClient.LOG.info("No node available for block: " + blockInfo);
        }
        DFSClient.LOG.info(
            "Could not obtain block "
                + block.getBlock()
                + " from any node: "
                + ie
                + ". Will get new block locations from namenode and retry...");
        try {
          // Introducing a random factor to the wait time before another retry.
          // The wait time is dependent on # of failures and a random factor.
          // At the first time of getting a BlockMissingException, the wait time
          // is a random number between 0..3000 ms. If the first retry
          // still fails, we will wait 3000 ms grace period before the 2nd retry.
          // Also at the second retry, the waiting window is expanded to 6000 ms
          // alleviating the request rate from the server. Similarly the 3rd retry
          // will wait 6000ms grace period before retry and the waiting window is
          // expanded to 9000ms.
          double waitTime =
              timeWindow * failures
                  + // grace period for the last round of attempt
                  timeWindow
                      * (failures + 1)
                      * DFSUtil.getRandom().nextDouble(); // expanding time window for each failure
          DFSClient.LOG.warn(
              "DFS chooseDataNode: got # "
                  + (failures + 1)
                  + " IOException, will wait for "
                  + waitTime
                  + " msec.");
          Thread.sleep((long) waitTime);
        } catch (InterruptedException iex) {
        }
        deadNodes.clear(); // 2nd option is to remove only nodes[blockId]
        openInfo();
        block = getBlockAt(block.getStartOffset(), false);
        failures++;
        continue;
      }
    }
  }
  /**
   * The following test first creates a file. It verifies the block information from a datanode.
   * Then, it updates the block with new information and verifies again.
   */
  @Test
  public void testBlockMetaDataInfo() throws Exception {
    MiniDFSCluster cluster = null;

    try {
      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
      cluster.waitActive();

      // create a file
      DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
      String filestr = "/foo";
      Path filepath = new Path(filestr);
      DFSTestUtil.createFile(dfs, filepath, 1024L, (short) 3, 0L);
      assertTrue(dfs.exists(filepath));

      // get block info
      LocatedBlock locatedblock =
          getLastLocatedBlock(DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
      DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
      assertTrue(datanodeinfo.length > 0);

      // connect to a data node
      DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
      InterDatanodeProtocol idp =
          DataNodeTestUtils.createInterDatanodeProtocolProxy(datanode, datanodeinfo[0], conf);

      // stop block scanner, so we could compare lastScanTime
      DataNodeTestUtils.shutdownBlockScanner(datanode);

      // verify BlockMetaDataInfo
      ExtendedBlock b = locatedblock.getBlock();
      InterDatanodeProtocol.LOG.info("b=" + b + ", " + b.getClass());
      checkMetaInfo(b, datanode);
      long recoveryId = b.getGenerationStamp() + 1;
      idp.initReplicaRecovery(new RecoveringBlock(b, locatedblock.getLocations(), recoveryId));

      // verify updateBlock
      ExtendedBlock newblock =
          new ExtendedBlock(
              b.getBlockPoolId(), b.getBlockId(), b.getNumBytes() / 2, b.getGenerationStamp() + 1);
      idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes());
      checkMetaInfo(newblock, datanode);

      // Verify correct null response trying to init recovery for a missing block
      ExtendedBlock badBlock = new ExtendedBlock("fake-pool", b.getBlockId(), 0, 0);
      assertNull(
          idp.initReplicaRecovery(
              new RecoveringBlock(badBlock, locatedblock.getLocations(), recoveryId)));
    } finally {
      if (cluster != null) {
        cluster.shutdown();
      }
    }
  }
 private void addBlocks(String fileName, String clientName) throws IOException {
   for (int jdx = 0; jdx < blocksPerFile; jdx++) {
     LocatedBlock loc = nameNode.addBlock(fileName, clientName);
     for (DatanodeInfo dnInfo : loc.getLocations()) {
       int dnIdx = Arrays.binarySearch(datanodes, dnInfo.getName());
       datanodes[dnIdx].addBlock(loc.getBlock());
       nameNode.blockReceived(
           datanodes[dnIdx].dnRegistration, new Block[] {loc.getBlock()}, new String[] {""});
     }
   }
 }
Ejemplo n.º 13
0
  /** Test constructing LocatedBlock with null cachedLocs */
  @Test
  public void testLocatedBlockConstructorWithNullCachedLocs() {
    DatanodeInfo d = DFSTestUtil.getLocalDatanodeInfo();
    DatanodeInfo[] ds = new DatanodeInfo[1];
    ds[0] = d;

    ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1);
    LocatedBlock l1 = new LocatedBlock(b1, ds, null, null, 0, false, null);
    final DatanodeInfo[] cachedLocs = l1.getCachedLocations();
    assertTrue(cachedLocs.length == 0);
  }
  /**
   * TC11: Racing rename
   *
   * @throws IOException an exception might be thrown
   */
  public void testTC11() throws Exception {
    final Path p = new Path("/TC11/foo");
    System.out.println("p=" + p);

    // a. Create file and write one block of data. Close file.
    final int len1 = (int) BLOCK_SIZE;
    {
      FSDataOutputStream out = fs.create(p, false, buffersize, REPLICATION, BLOCK_SIZE);
      AppendTestUtil.write(out, 0, len1);
      out.close();
    }

    // b. Reopen file in "append" mode. Append half block of data.
    FSDataOutputStream out = fs.append(p);
    final int len2 = (int) BLOCK_SIZE / 2;
    AppendTestUtil.write(out, len1, len2);
    out.hflush();

    // c. Rename file to file.new.
    final Path pnew = new Path(p + ".new");
    assertTrue(fs.rename(p, pnew));

    // d. Close file handle that was opened in (b).
    try {
      out.close();
      fail("close() should throw an exception");
    } catch (Exception e) {
      AppendTestUtil.LOG.info("GOOD!", e);
    }

    // wait for the lease recovery
    cluster.setLeasePeriod(1000, 1000);
    AppendTestUtil.sleep(5000);

    // check block sizes
    final long len = fs.getFileStatus(pnew).getLen();
    final LocatedBlocks locatedblocks =
        fs.dfs.getNamenode().getBlockLocations(pnew.toString(), 0L, len);
    final int numblock = locatedblocks.locatedBlockCount();
    for (int i = 0; i < numblock; i++) {
      final LocatedBlock lb = locatedblocks.get(i);
      final Block blk = lb.getBlock();
      final long size = lb.getBlockSize();
      if (i < numblock - 1) {
        assertEquals(BLOCK_SIZE, size);
      }
      for (DatanodeInfo datanodeinfo : lb.getLocations()) {
        final DataNode dn = cluster.getDataNode(datanodeinfo.getIpcPort());
        final Block metainfo = dn.data.getStoredBlock(blk.getBlockId());
        assertEquals(size, metainfo.getNumBytes());
      }
    }
  }
Ejemplo n.º 15
0
 protected final LocatedBlocks ensureFileReplicasOnStorageType(Path path, StorageType storageType)
     throws IOException {
   // Ensure that returned block locations returned are correct!
   LOG.info("Ensure path: " + path + " is on StorageType: " + storageType);
   assertThat(fs.exists(path), is(true));
   long fileLength = client.getFileInfo(path.toString()).getLen();
   LocatedBlocks locatedBlocks = client.getLocatedBlocks(path.toString(), 0, fileLength);
   for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
     assertThat(locatedBlock.getStorageTypes()[0], is(storageType));
   }
   return locatedBlocks;
 }
Ejemplo n.º 16
0
  private BlockInfo createBlockInfo(Path file, LocatedBlock b) {
    DatanodeInfo[] locations = b.getLocations();
    String[] hosts = new String[locations.length];
    String[] names = new String[locations.length];
    for (int i = 0; i < locations.length; ++i) {
      DatanodeInfo d = locations[i];
      hosts[i] = d.getHost();
      names[i] = d.getName();
    }

    BlockLocation loc = new BlockLocation(names, hosts, b.getStartOffset(), b.getBlockSize());
    return new BlockInfo(loc, file);
  }
  /** Read the block length from one of the datanodes. */
  private long readBlockLength(LocatedBlock locatedblock) throws IOException {
    assert locatedblock != null : "LocatedBlock cannot be null";
    int replicaNotFoundCount = locatedblock.getLocations().length;

    for (DatanodeInfo datanode : locatedblock.getLocations()) {
      ClientDatanodeProtocol cdp = null;

      try {
        cdp =
            DFSUtil.createClientDatanodeProtocolProxy(
                datanode, dfsClient.conf, dfsClient.getConf().socketTimeout, locatedblock);

        final long n = cdp.getReplicaVisibleLength(locatedblock.getBlock());

        if (n >= 0) {
          return n;
        }
      } catch (IOException ioe) {
        if (ioe instanceof RemoteException
            && (((RemoteException) ioe).unwrapRemoteException()
                instanceof ReplicaNotFoundException)) {
          // special case : replica might not be on the DN, treat as 0 length
          replicaNotFoundCount--;
        }

        if (DFSClient.LOG.isDebugEnabled()) {
          DFSClient.LOG.debug(
              "Failed to getReplicaVisibleLength from datanode "
                  + datanode
                  + " for block "
                  + locatedblock.getBlock(),
              ioe);
        }
      } finally {
        if (cdp != null) {
          RPC.stopProxy(cdp);
        }
      }
    }

    // Namenode told us about these locations, but none know about the replica
    // means that we hit the race between pipeline creation start and end.
    // we require all 3 because some other exception could have happened
    // on a DN that has it.  we want to report that error
    if (replicaNotFoundCount == 0) {
      return 0;
    }

    throw new IOException("Cannot obtain block length for " + locatedblock);
  }
Ejemplo n.º 18
0
  /** Convert a Json map to LocatedBlock. */
  private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
    if (m == null) {
      return null;
    }

    final Block b = toBlock((Map<?, ?>) m.get("block"));
    final DatanodeInfo[] locations = toDatanodeInfoArray((Object[]) m.get("locations"));
    final long startOffset = (Long) m.get("startOffset");
    final boolean isCorrupt = (Boolean) m.get("isCorrupt");

    final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
    locatedblock.setBlockToken(toBlockToken((Map<?, ?>) m.get("blockToken")));
    return locatedblock;
  }
Ejemplo n.º 19
0
  /** Convert a LocatedBlock to a Json map. */
  private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock) throws IOException {
    if (locatedblock == null) {
      return null;
    }

    final Map<String, Object> m = new TreeMap<String, Object>();
    m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
    m.put("isCorrupt", locatedblock.isCorrupt());
    m.put("startOffset", locatedblock.getStartOffset());
    m.put("block", toJsonMap(locatedblock.getBlock()));

    m.put("locations", toJsonArray(locatedblock.getLocations()));
    return m;
  }
 private void waitForLocations(int locations) throws IOException, InterruptedException {
   for (int tries = 0; tries < RETRIES; )
     try {
       LocatedBlock locatedBlock = getLocatedBlock();
       assertThat(locatedBlock.getLocations().length, is(locations));
       break;
     } catch (AssertionError e) {
       if (++tries < RETRIES) {
         Thread.sleep(1000);
       } else {
         throw e;
       }
     }
 }
Ejemplo n.º 21
0
  /** Test that the {@link PlacementMonitor.BlockAndDatanodeResolver} works correctly. */
  @Test
  public void testBlockAndDatanodeResolver() throws Exception {
    setupCluster();
    try {
      Path src = new Path("/dir/file");
      Path parity = new Path("/raid/dir/file");
      DFSTestUtil.createFile(fs, src, 20, (short) 2, 0L);
      DFSTestUtil.createFile(fs, parity, 11, (short) 2, 0L);
      DFSTestUtil.waitReplication(fs, src, (short) 2);
      DFSTestUtil.waitReplication(fs, parity, (short) 2);

      LocatedBlocks srcLbs, parityLbs;
      List<BlockInfo> srcInfos, parityInfos;
      srcLbs = namenode.getBlockLocations(src.toString(), 4, 10);
      srcInfos = placementMonitor.getBlockInfos(fs, src, 4, 10);
      parityLbs = namenode.getBlockLocations(parity.toString(), 3, 7);
      parityInfos = placementMonitor.getBlockInfos(fs, parity, 3, 7);

      Assert.assertEquals(10, srcLbs.getLocatedBlocks().size());
      Assert.assertEquals(7, parityLbs.getLocatedBlocks().size());
      Assert.assertEquals(10, srcInfos.size());
      Assert.assertEquals(7, parityInfos.size());

      BlockAndDatanodeResolver resolver = new BlockAndDatanodeResolver(src, fs, parity, fs);
      for (int i = 0; i < srcInfos.size(); ++i) {
        LocatedBlock lb = resolver.getLocatedBlock(srcInfos.get(i));
        Assert.assertEquals(srcLbs.get(i).getBlock(), lb.getBlock());
        for (String nodeName : srcInfos.get(i).getNames()) {
          DatanodeInfo node = resolver.getDatanodeInfo(nodeName);
          Assert.assertEquals(node.getName(), nodeName);
        }
      }
      for (int i = 0; i < parityInfos.size(); ++i) {
        LocatedBlock lb = resolver.getLocatedBlock(parityInfos.get(i));
        Assert.assertEquals(parityLbs.get(i).getBlock(), lb.getBlock());
        for (String nodeName : parityInfos.get(i).getNames()) {
          DatanodeInfo node = resolver.getDatanodeInfo(nodeName);
          Assert.assertEquals(node.getName(), nodeName);
        }
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown();
      }
      if (placementMonitor != null) {
        placementMonitor.stop();
      }
    }
  }
  /**
   * try to access a block on a data node. If fails - throws exception
   *
   * @param datanode
   * @param lblock
   * @throws IOException
   */
  private void accessBlock(DatanodeInfo datanode, LocatedBlock lblock) throws IOException {
    InetSocketAddress targetAddr = null;
    ExtendedBlock block = lblock.getBlock();

    targetAddr = NetUtils.createSocketAddr(datanode.getXferAddr());

    BlockReader blockReader =
        new BlockReaderFactory(new DfsClientConf(conf))
            .setInetSocketAddress(targetAddr)
            .setBlock(block)
            .setFileName(
                BlockReaderFactory.getFileName(targetAddr, "test-blockpoolid", block.getBlockId()))
            .setBlockToken(lblock.getBlockToken())
            .setStartOffset(0)
            .setLength(-1)
            .setVerifyChecksum(true)
            .setClientName("TestDataNodeVolumeFailure")
            .setDatanodeInfo(datanode)
            .setCachingStrategy(CachingStrategy.newDefaultStrategy())
            .setClientCacheContext(ClientContext.getFromConf(conf))
            .setConfiguration(conf)
            .setTracer(FsTracer.get(conf))
            .setRemotePeerFactory(
                new RemotePeerFactory() {
                  @Override
                  public Peer newConnectedPeer(
                      InetSocketAddress addr,
                      Token<BlockTokenIdentifier> blockToken,
                      DatanodeID datanodeId)
                      throws IOException {
                    Peer peer = null;
                    Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
                    try {
                      sock.connect(addr, HdfsConstants.READ_TIMEOUT);
                      sock.setSoTimeout(HdfsConstants.READ_TIMEOUT);
                      peer = DFSUtilClient.peerFromSocket(sock);
                    } finally {
                      if (peer == null) {
                        IOUtils.closeSocket(sock);
                      }
                    }
                    return peer;
                  }
                })
            .build();
    blockReader.close();
  }
Ejemplo n.º 23
0
 /**
  * Test that {@link PlacementMonitor} moves block correctly
  *
  * @throws Exception
  */
 @Test
 public void testMoveBlock() throws Exception {
   setupCluster();
   try {
     Path path = new Path("/dir/file");
     DFSTestUtil.createFile(fs, path, 1, (short) 1, 0L);
     DFSTestUtil.waitReplication(fs, path, (short) 1);
     FileStatus status = fs.getFileStatus(path);
     LocatedBlocks blocks = namenode.getBlockLocations(path.toString(), 0, status.getLen());
     Assert.assertEquals(1, blocks.getLocatedBlocks().size());
     LocatedBlock block = blocks.getLocatedBlocks().get(0);
     Assert.assertEquals(1, block.getLocations().length);
     DatanodeInfo source = block.getLocations()[0];
     Set<DatanodeInfo> excluded = new HashSet<DatanodeInfo>();
     for (DatanodeInfo d : datanodes) {
       excluded.add(d);
     }
     excluded.remove(source);
     DatanodeInfo target = excluded.iterator().next();
     excluded.add(source);
     excluded.remove(target);
     BlockMover.BlockMoveAction action =
         blockMover.new BlockMoveAction(block, source, excluded, 1);
     LOG.info("Start moving block from " + source + " to " + target);
     action.run();
     LOG.info("Done moving block");
     boolean blockMoved = false;
     for (int i = 0; i < 100; ++i) {
       blocks = namenode.getBlockLocations(path.toString(), 0, status.getLen());
       block = blocks.getLocatedBlocks().get(0);
       if (block.getLocations().length == 1 && block.getLocations()[0].equals((target))) {
         blockMoved = true;
         break;
       }
       Thread.sleep(100L);
     }
     Assert.assertTrue(blockMoved);
   } finally {
     if (cluster != null) {
       cluster.shutdown();
     }
     if (placementMonitor != null) {
       placementMonitor.stop();
     }
   }
 }
  /** removes a specified block from MiniDFS storage and reports it as corrupt */
  private void removeAndReportBlock(
      DistributedFileSystem blockDfs, Path filePath, LocatedBlock block) throws IOException {
    TestRaidDfs.corruptBlock(filePath, block.getBlock(), NUM_DATANODES, true);

    // report deleted block to the name node
    LocatedBlock[] toReport = {block};
    blockDfs.getClient().namenode.reportBadBlocks(toReport);
  }
  /**
   * Read bytes starting from the specified position.
   *
   * @param position start read from this position
   * @param buffer read buffer
   * @param offset offset into buffer
   * @param length number of bytes to read
   * @return actual number of bytes read
   */
  @Override
  public int read(long position, byte[] buffer, int offset, int length) throws IOException {
    // sanity checks
    dfsClient.checkOpen();
    if (closed) {
      throw new IOException("Stream closed");
    }
    failures = 0;
    long filelen = getFileLength();
    if ((position < 0) || (position >= filelen)) {
      return -1;
    }
    int realLen = length;
    if ((position + length) > filelen) {
      realLen = (int) (filelen - position);
    }

    // determine the block and byte range within the block
    // corresponding to position and realLen
    List<LocatedBlock> blockRange = getBlockRange(position, realLen);
    int remaining = realLen;
    Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap =
        new HashMap<ExtendedBlock, Set<DatanodeInfo>>();
    for (LocatedBlock blk : blockRange) {
      long targetStart = position - blk.getStartOffset();
      long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
      try {
        fetchBlockByteRange(
            blk, targetStart, targetStart + bytesToRead - 1, buffer, offset, corruptedBlockMap);
      } finally {
        // Check and report if any block replicas are corrupted.
        // BlockMissingException may be caught if all block replicas are
        // corrupted.
        reportCheckSumFailure(corruptedBlockMap, blk.getLocations().length);
      }

      remaining -= bytesToRead;
      position += bytesToRead;
      offset += bytesToRead;
    }
    assert remaining == 0 : "Wrong number of bytes read.";
    if (dfsClient.stats != null) {
      dfsClient.stats.incrementBytesRead(realLen);
    }
    return realLen;
  }
  /**
   * go to each block on the 2nd DataNode until it fails...
   *
   * @param path
   * @param size
   * @throws IOException
   */
  private void triggerFailure(String path, long size) throws IOException {
    NamenodeProtocols nn = cluster.getNameNodeRpc();
    List<LocatedBlock> locatedBlocks = nn.getBlockLocations(path, 0, size).getLocatedBlocks();

    for (LocatedBlock lb : locatedBlocks) {
      DatanodeInfo dinfo = lb.getLocations()[1];
      ExtendedBlock b = lb.getBlock();
      try {
        accessBlock(dinfo, lb);
      } catch (IOException e) {
        System.out.println(
            "Failure triggered, on block: "
                + b.getBlockId()
                + "; corresponding volume should be removed by now");
        break;
      }
    }
  }
  /**
   * TC7: Corrupted replicas are present.
   *
   * @throws IOException an exception might be thrown
   */
  public void testTC7() throws Exception {
    final short repl = 2;
    final Path p = new Path("/TC7/foo");
    System.out.println("p=" + p);

    // a. Create file with replication factor of 2. Write half block of data. Close file.
    final int len1 = (int) (BLOCK_SIZE / 2);
    {
      FSDataOutputStream out = fs.create(p, false, buffersize, repl, BLOCK_SIZE);
      AppendTestUtil.write(out, 0, len1);
      out.close();
    }
    DFSTestUtil.waitReplication(fs, p, repl);

    // b. Log into one datanode that has one replica of this block.
    //   Find the block file on this datanode and truncate it to zero size.
    final LocatedBlocks locatedblocks =
        fs.dfs.getNamenode().getBlockLocations(p.toString(), 0L, len1);
    assertEquals(1, locatedblocks.locatedBlockCount());
    final LocatedBlock lb = locatedblocks.get(0);
    final Block blk = lb.getBlock();
    assertEquals(len1, lb.getBlockSize());

    DatanodeInfo[] datanodeinfos = lb.getLocations();
    assertEquals(repl, datanodeinfos.length);
    final DataNode dn = cluster.getDataNode(datanodeinfos[0].getIpcPort());
    final FSDataset data = (FSDataset) dn.getFSDataset();
    final RandomAccessFile raf = new RandomAccessFile(data.getBlockFile(blk), "rw");
    AppendTestUtil.LOG.info("dn=" + dn + ", blk=" + blk + " (length=" + blk.getNumBytes() + ")");
    assertEquals(len1, raf.length());
    raf.setLength(0);
    raf.close();

    // c. Open file in "append mode".  Append a new block worth of data. Close file.
    final int len2 = (int) BLOCK_SIZE;
    {
      FSDataOutputStream out = fs.append(p);
      AppendTestUtil.write(out, len1, len2);
      out.close();
    }

    // d. Reopen file and read two blocks worth of data.
    AppendTestUtil.check(fs, p, len1 + len2);
  }
 private int printLocatedBlocks(Path filePath) throws Exception {
   LocatedBlocks lbs = dfs.getLocatedBlocks(filePath, 0L, Integer.MAX_VALUE);
   StringBuilder sb = new StringBuilder();
   sb.append("Path " + filePath + ":");
   int maxRepl = 0;
   for (LocatedBlock lb : lbs.getLocatedBlocks()) {
     sb.append(lb.getBlock());
     sb.append(":");
     for (DatanodeInfo loc : lb.getLocations()) {
       sb.append(loc.getHostName());
       sb.append(" ");
     }
     if (lb.getLocations().length > maxRepl) {
       maxRepl = lb.getLocations().length;
     }
   }
   LOG.info(sb.toString());
   return maxRepl;
 }
  /**
   * Get block at the specified position. Fetch it from the namenode if not cached.
   *
   * @param offset
   * @param updatePosition whether to update current position
   * @return located block
   * @throws IOException
   */
  private synchronized LocatedBlock getBlockAt(long offset, boolean updatePosition)
      throws IOException {
    assert (locatedBlocks != null) : "locatedBlocks is null";

    final LocatedBlock blk;

    // check offset
    if (offset < 0 || offset >= getFileLength()) {
      throw new IOException(
          "offset < 0 || offset > getFileLength(), offset="
              + offset
              + ", updatePosition="
              + updatePosition
              + ", locatedBlocks="
              + locatedBlocks);
    } else if (offset >= locatedBlocks.getFileLength()) {
      // offset to the portion of the last block,
      // which is not known to the name-node yet;
      // getting the last block
      blk = locatedBlocks.getLastLocatedBlock();
    } else {
      // search cached blocks first
      int targetBlockIdx = locatedBlocks.findBlock(offset);
      if (targetBlockIdx < 0) { // block is not cached
        targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
        // fetch more blocks
        LocatedBlocks newBlocks;
        newBlocks = DFSClient.callGetBlockLocations(dfsClient.namenode, src, offset, prefetchSize);
        assert (newBlocks != null) : "Could not find target position " + offset;
        locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
      }
      blk = locatedBlocks.get(targetBlockIdx);
    }

    // update current position
    if (updatePosition) {
      pos = offset;
      blockEnd = blk.getStartOffset() + blk.getBlockSize() - 1;
      currentLocatedBlock = blk;
    }
    return blk;
  }
Ejemplo n.º 30
0
  /*
   * Since NameNode will not persist any locations of the block, addBlock()
   * retry call after restart NN should re-select the locations and return to
   * client. refer HDFS-5257
   */
  @Test
  public void testAddBlockRetryShouldReturnBlockWithLocations() throws Exception {
    final String src = "/testAddBlockRetryShouldReturnBlockWithLocations";
    NamenodeProtocols nameNodeRpc = cluster.getNameNodeRpc();
    // create file
    nameNodeRpc.create(
        src,
        FsPermission.getFileDefault(),
        "clientName",
        new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE)),
        true,
        (short) 3,
        1024,
        null);
    // start first addBlock()
    LOG.info("Starting first addBlock for " + src);
    LocatedBlock lb1 =
        nameNodeRpc.addBlock(src, "clientName", null, null, INodeId.GRANDFATHER_INODE_ID, null);
    assertTrue("Block locations should be present", lb1.getLocations().length > 0);

    cluster.restartNameNode();
    nameNodeRpc = cluster.getNameNodeRpc();
    LocatedBlock lb2 =
        nameNodeRpc.addBlock(src, "clientName", null, null, INodeId.GRANDFATHER_INODE_ID, null);
    assertEquals("Blocks are not equal", lb1.getBlock(), lb2.getBlock());
    assertTrue("Wrong locations with retry", lb2.getLocations().length > 0);
  }