예제 #1
0
 public void updateRecord(MetaRecord rec) {
   try {
     long ts = System.currentTimeMillis();
     putRecord(rec, ts);
     String node = ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(rec.getId()));
     // setData会异步触发所有机器(包括本机)上的H2MetaTableTracker.nodeDataChanged
     // 然后触发下列调用:
     // =>org.h2.engine.Database.updateDatabaseObject(int)
     //  =>org.h2.engine.Database.update(Session, DbObject)
     //      =>org.h2.engine.Database.addMeta0(Session, DbObject, boolean)
     //          =>又到此方法
     // 所以会造成循环
     synchronized (this) { // 避免setData后立刻触发nodeDataChanged,此时IdVersion还未更新
       ZKUtil.setData(watcher, node, Bytes.toBytes(ts));
       // setData后watch不见了,所以要继续watch,监听其他人对此node的修改
       // ZKUtil.watchAndCheckExists(watcher, node);
       Stat stat = new Stat();
       ZKUtil.getDataAndWatch(watcher, node, stat);
       // 这里记录下id的最新版本,触发nodeDataChanged时再检查一下是否版本一样,
       // 如果不大于这里的版本那么就不再执行updateDatabaseObject操作
       tracker.updateIdVersion(rec.getId(), stat.getVersion());
     }
   } catch (Exception e) {
     throw new RuntimeException(e);
   }
 }
예제 #2
0
 private void waitForNewProcedures() {
   // watch for new procedues that we need to start subprocedures for
   LOG.debug("Looking for new procedures under znode:'" + zkController.getAcquiredBarrier() + "'");
   List<String> runningProcedures = null;
   try {
     runningProcedures =
         ZKUtil.listChildrenAndWatchForNewChildren(
             zkController.getWatcher(), zkController.getAcquiredBarrier());
     if (runningProcedures == null) {
       LOG.debug("No running procedures.");
       return;
     }
   } catch (KeeperException e) {
     member.controllerConnectionFailure(
         "General failure when watching for new procedures", e, null);
   }
   if (runningProcedures == null) {
     LOG.debug("No running procedures.");
     return;
   }
   for (String procName : runningProcedures) {
     // then read in the procedure information
     String path = ZKUtil.joinZNode(zkController.getAcquiredBarrier(), procName);
     startNewSubprocedure(path);
   }
 }
  @Override
  public Configuration getPeerConf(String peerId) throws ReplicationException {
    String znode = ZKUtil.joinZNode(this.peersZNode, peerId);
    byte[] data = null;
    try {
      data = ZKUtil.getData(this.zookeeper, znode);
    } catch (KeeperException e) {
      throw new ReplicationException("Error getting configuration for peer with id=" + peerId, e);
    }
    if (data == null) {
      LOG.error("Could not get configuration for peer because it doesn't exist. peerId=" + peerId);
      return null;
    }
    String otherClusterKey = "";
    try {
      otherClusterKey = parsePeerFrom(data);
    } catch (DeserializationException e) {
      LOG.warn(
          "Failed to parse cluster key from peerId="
              + peerId
              + ", specifically the content from the following znode: "
              + znode);
      return null;
    }

    Configuration otherConf = new Configuration(this.conf);
    try {
      ZKUtil.applyClusterKeyToConf(otherConf, otherClusterKey);
    } catch (IOException e) {
      LOG.error("Can't get peer configuration for peerId=" + peerId + " because:", e);
      return null;
    }
    return otherConf;
  }
  @Override
  public void addPeer(String id, String clusterKey, String tableCFs) throws ReplicationException {
    try {
      if (peerExists(id)) {
        throw new IllegalArgumentException(
            "Cannot add a peer with id=" + id + " because that id already exists.");
      }
      ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
      ZKUtil.createAndWatch(
          this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id), toByteArray(clusterKey));
      // There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the
      // peer-state znode. This happens while adding a peer.
      // The peer state data is set as "ENABLED" by default.
      ZKUtil.createNodeIfNotExistsAndWatch(
          this.zookeeper, getPeerStateNode(id), ENABLED_ZNODE_BYTES);
      // A peer is enabled by default

      String tableCFsStr = (tableCFs == null) ? "" : tableCFs;
      ZKUtil.createNodeIfNotExistsAndWatch(
          this.zookeeper, getTableCFsNode(id), Bytes.toBytes(tableCFsStr));
    } catch (KeeperException e) {
      throw new ReplicationException(
          "Could not add peer with id=" + id + ", clusterKey=" + clusterKey, e);
    }
  }
예제 #5
0
  /**
   * This attempts to create an acquired state znode for the procedure (snapshot name).
   *
   * <p>It then looks for the reached znode to trigger in-barrier execution. If not present we have
   * a watcher, if present then trigger the in-barrier action.
   */
  @Override
  public void sendMemberAcquired(Subprocedure sub) throws IOException {
    String procName = sub.getName();
    try {
      LOG.debug(
          "Member: '"
              + memberName
              + "' joining acquired barrier for procedure ("
              + procName
              + ") in zk");
      String acquiredZNode =
          ZKUtil.joinZNode(
              ZKProcedureUtil.getAcquireBarrierNode(zkController, procName), memberName);
      ZKUtil.createAndFailSilent(zkController.getWatcher(), acquiredZNode);

      // watch for the complete node for this snapshot
      String reachedBarrier = zkController.getReachedBarrierNode(procName);
      LOG.debug("Watch for global barrier reached:" + reachedBarrier);
      if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), reachedBarrier)) {
        receivedReachedGlobalBarrier(reachedBarrier);
      }
    } catch (KeeperException e) {
      member.controllerConnectionFailure(
          "Failed to acquire barrier for procedure: " + procName + " and member: " + memberName,
          e,
          procName);
    }
  }
예제 #6
0
  @Test
  public void TestMap() throws Exception {
    String prefix = "0000";
    final String fileName = "19691231f2cd014ea28f42788214560a21a44cef";
    final String mobFilePath = prefix + fileName;

    ImmutableBytesWritable r = new ImmutableBytesWritable(Bytes.toBytes("r"));
    final KeyValue[] kvList = new KeyValue[1];
    kvList[0] =
        new KeyValue(
            Bytes.toBytes("row"),
            Bytes.toBytes("family"),
            Bytes.toBytes("column"),
            Bytes.toBytes(mobFilePath));

    Result columns = mock(Result.class);
    when(columns.rawCells()).thenReturn(kvList);

    Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
    ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable());
    TableName tn = TableName.valueOf("testSweepMapper");
    TableName lockName = MobUtils.getTableLockName(tn);
    String znode = ZKUtil.joinZNode(zkw.tableLockZNode, lockName.getNameAsString());
    configuration.set(SweepJob.SWEEP_JOB_ID, "1");
    configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode);
    ServerName serverName = SweepJob.getCurrentServerName(configuration);
    configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString());

    TableLockManager tableLockManager =
        TableLockManager.createTableLockManager(configuration, zkw, serverName);
    TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool");
    lock.acquire();
    try {
      Mapper<ImmutableBytesWritable, Result, Text, KeyValue>.Context ctx =
          mock(Mapper.Context.class);
      when(ctx.getConfiguration()).thenReturn(configuration);
      SweepMapper map = new SweepMapper();
      doAnswer(
              new Answer<Void>() {

                @Override
                public Void answer(InvocationOnMock invocation) throws Throwable {
                  Text text = (Text) invocation.getArguments()[0];
                  KeyValue kv = (KeyValue) invocation.getArguments()[1];

                  assertEquals(Bytes.toString(text.getBytes(), 0, text.getLength()), fileName);
                  assertEquals(0, Bytes.compareTo(kv.getKey(), kvList[0].getKey()));

                  return null;
                }
              })
          .when(ctx)
          .write(any(Text.class), any(KeyValue.class));

      map.map(r, columns, ctx);
    } finally {
      lock.release();
    }
  }
예제 #7
0
 private static String getPgPortEphemeralNodePath(ServerName sn, int port, boolean isMaster) {
   String znode =
       (isMaster ? "M" : "S")
           + ":"
           + sn.getHostAndPort()
           + Addressing.HOSTNAME_PORT_SEPARATOR
           + port;
   return ZKUtil.joinZNode(ZooKeeperAdmin.PG_SERVER_NODE, znode);
 }
예제 #8
0
 public void addRecord(MetaRecord rec) {
   try {
     putRecord(rec, System.currentTimeMillis());
     String node = ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(rec.getId()));
     ZKUtil.createAndWatch(watcher, node, EMPTY_BYTE_ARRAY);
     tracker.updateIdVersion(rec.getId(), 0);
   } catch (Exception e) {
     throw new RuntimeException(e);
   }
 }
 public LeaderElector(ZooKeeperWatcher watcher, String serverName) {
   setDaemon(true);
   setName("ZKSecretWatcher-leaderElector");
   zkLeader =
       new ZKLeaderManager(
           watcher,
           ZKUtil.joinZNode(zkWatcher.getRootKeyZNode(), "keymaster"),
           Bytes.toBytes(serverName),
           this);
 }
예제 #10
0
  @Test
  public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception {
    // this test is for when distributed log replay is enabled
    if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return;

    LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization");
    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
    MasterFileSystem fs = master.getMasterFileSystem();

    String failedRegion = "failedRegoin1";
    String staleRegion = "staleRegion";
    ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1");
    ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1");
    String walPath =
        "/hbase/data/.logs/" + inRecoveryServerName.getServerName() + "-splitting/test";
    // Create a ZKW to use in the test
    ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
    zkw.getRecoverableZooKeeper()
        .create(
            ZKSplitLog.getEncodedNodeName(zkw, walPath),
            new SplitLogTask.Owned(inRecoveryServerName, fs.getLogRecoveryMode()).toByteArray(),
            Ids.OPEN_ACL_UNSAFE,
            CreateMode.PERSISTENT);
    String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion);
    ZKUtil.createWithParents(zkw, staleRegionPath);
    String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion);
    inRecoveringRegionPath =
        ZKUtil.joinZNode(inRecoveringRegionPath, inRecoveryServerName.getServerName());
    ZKUtil.createWithParents(zkw, inRecoveringRegionPath);
    Set<ServerName> servers = new HashSet<ServerName>();
    servers.add(previouselyFaildServerName);
    fs.removeStaleRecoveringRegionsFromZK(servers);

    // verification
    assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1);
    assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1);

    ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode);
    ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode);
    zkw.close();
  }
 @Override
 public void removePeer(String id) throws ReplicationException {
   try {
     if (!peerExists(id)) {
       throw new IllegalArgumentException(
           "Cannot remove peer with id=" + id + " because that id does not exist.");
     }
     ZKUtil.deleteNodeRecursively(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id));
   } catch (KeeperException e) {
     throw new ReplicationException("Could not remove peer with id=" + id, e);
   }
 }
 private static HRegionServer setDrainingServer(final HRegionServer hrs) throws KeeperException {
   LOG.info(
       "Making "
           + hrs.getServerName()
           + " the draining server; "
           + "it has "
           + hrs.getNumberOfOnlineRegions()
           + " online regions");
   ZooKeeperWatcher zkw = hrs.getZooKeeper();
   String hrsDrainingZnode = ZKUtil.joinZNode(zkw.drainingZNode, hrs.getServerName().toString());
   ZKUtil.createWithParents(zkw, hrsDrainingZnode);
   return hrs;
 }
예제 #13
0
 public void removeRecord(int id) {
   try {
     // System.out.println("removeRecord id: " + id);
     // new Error().printStackTrace();
     Delete delete = new Delete(Bytes.toBytes(id));
     table.delete(delete);
     ZKUtil.deleteNodeFailSilent(
         watcher, ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(id)));
     tracker.removeId(id);
   } catch (Exception e) {
     throw new RuntimeException(e);
   }
 }
예제 #14
0
  public void loadMetaRecords(List<MetaRecord> records) throws Exception {
    MetaRecord rec;
    for (Result r : table.getScanner(new Scan())) {
      if (r.isEmpty()) continue;
      rec = getMetaRecord(r);
      records.add(rec);

      if (!tracker.contains(rec.getId()))
        ZKUtil.createNodeIfNotExistsAndWatch(
            watcher,
            ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(rec.getId())),
            EMPTY_BYTE_ARRAY);
    }
  }
예제 #15
0
 private void watchForAbortedProcedures() {
   LOG.debug("Checking for aborted procedures on node: '" + zkController.getAbortZnode() + "'");
   try {
     // this is the list of the currently aborted procedues
     for (String node :
         ZKUtil.listChildrenAndWatchForNewChildren(
             zkController.getWatcher(), zkController.getAbortZnode())) {
       String abortNode = ZKUtil.joinZNode(zkController.getAbortZnode(), node);
       abort(abortNode);
     }
   } catch (KeeperException e) {
     member.controllerConnectionFailure(
         "Failed to list children for abort node:" + zkController.getAbortZnode(), e, null);
   }
 }
예제 #16
0
 /** This acts as the ack for a completed procedure */
 @Override
 public void sendMemberCompleted(Subprocedure sub, byte[] data) throws IOException {
   String procName = sub.getName();
   LOG.debug(
       "Marking procedure  '" + procName + "' completed for member '" + memberName + "' in zk");
   String joinPath = ZKUtil.joinZNode(zkController.getReachedBarrierNode(procName), memberName);
   // ProtobufUtil.prependPBMagic does not take care of null
   if (data == null) {
     data = new byte[0];
   }
   try {
     ZKUtil.createAndFailSilent(
         zkController.getWatcher(), joinPath, ProtobufUtil.prependPBMagic(data));
   } catch (KeeperException e) {
     member.controllerConnectionFailure(
         "Failed to post zk node:" + joinPath + " to join procedure barrier.", e, procName);
   }
 }
 @Override
 public Map<String, String> getAllPeerClusterKeys() {
   Map<String, String> peers = new TreeMap<String, String>();
   List<String> ids = null;
   try {
     ids = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
     for (String id : ids) {
       byte[] bytes = ZKUtil.getData(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id));
       String clusterKey = null;
       try {
         clusterKey = parsePeerFrom(bytes);
       } catch (DeserializationException de) {
         LOG.warn("Failed parse of clusterid=" + id + " znode content, continuing.");
         continue;
       }
       peers.put(id, clusterKey);
     }
   } catch (KeeperException e) {
     this.abortable.abort("Cannot get the list of peers ", e);
   }
   return peers;
 }
  public boolean shouldAddCheckerMaster() {

    ZooKeeperWatcher zk = super.getZooKeeper();
    String numberN = ZKUtil.joinZNode(zk.baseZNode, CCIndexConstants.CheckNumNode);
    try {
      if (ZKUtil.checkExists(zk, numberN) != -1) {
        ZKUtil.createSetData(zk, numberN, Bytes.toBytes(1));
      } else {
        int num = Bytes.toInt(ZKUtil.getData(zk, numberN));
        if (num < this.checkMasterN) {
          ZKUtil.setData(zk, numberN, Bytes.toBytes(num + 1));
          return true;
        } else {
          return false;
        }
      }
    } catch (KeeperException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }

    return false;
  }
 /**
  * Wait for tasks to become available at /hbase/splitlog zknode. Grab a task one at a time. This
  * policy puts an upper-limit on the number of simultaneous log splitting that could be happening
  * in a cluster.
  *
  * <p>Synchronization using {@link #taskReadyLock} ensures that it will try to grab every task
  * that has been put up
  *
  * @throws InterruptedException
  */
 @Override
 public void taskLoop() throws InterruptedException {
   while (!shouldStop) {
     int seq_start = taskReadySeq;
     List<String> paths = null;
     paths = getTaskList();
     if (paths == null) {
       LOG.warn(
           "Could not get tasks, did someone remove "
               + watcher.splitLogZNode
               + " ... worker thread exiting.");
       return;
     }
     // pick meta wal firstly
     int offset = (int) (Math.random() * paths.size());
     for (int i = 0; i < paths.size(); i++) {
       if (DefaultWALProvider.isMetaFile(paths.get(i))) {
         offset = i;
         break;
       }
     }
     int numTasks = paths.size();
     for (int i = 0; i < numTasks; i++) {
       int idx = (i + offset) % paths.size();
       // don't call ZKSplitLog.getNodeName() because that will lead to
       // double encoding of the path name
       if (this.calculateAvailableSplitters(numTasks) > 0) {
         grabTask(ZKUtil.joinZNode(watcher.splitLogZNode, paths.get(idx)));
       } else {
         LOG.debug(
             "Current region server "
                 + server.getServerName()
                 + " has "
                 + this.tasksInProgress.get()
                 + " tasks in progress and can't take more.");
         break;
       }
       if (shouldStop) {
         return;
       }
     }
     SplitLogCounters.tot_wkr_task_grabing.incrementAndGet();
     synchronized (taskReadyLock) {
       while (seq_start == taskReadySeq) {
         taskReadyLock.wait(checkInterval);
         if (server != null) {
           // check to see if we have stale recovering regions in our internal memory state
           Map<String, HRegion> recoveringRegions = server.getRecoveringRegions();
           if (!recoveringRegions.isEmpty()) {
             // Make a local copy to prevent ConcurrentModificationException when other threads
             // modify recoveringRegions
             List<String> tmpCopy = new ArrayList<String>(recoveringRegions.keySet());
             int listSize = tmpCopy.size();
             for (int i = 0; i < listSize; i++) {
               String region = tmpCopy.get(i);
               String nodePath = ZKUtil.joinZNode(watcher.recoveringRegionsZNode, region);
               try {
                 if (ZKUtil.checkExists(watcher, nodePath) == -1) {
                   HRegion r = recoveringRegions.remove(region);
                   if (r != null) {
                     r.setRecovering(false);
                   }
                   LOG.debug("Mark recovering region:" + region + " up.");
                 } else {
                   // current check is a defensive(or redundant) mechanism to prevent us from
                   // having stale recovering regions in our internal RS memory state while
                   // zookeeper(source of truth) says differently. We stop at the first good one
                   // because we should not have a single instance such as this in normal case so
                   // check the first one is good enough.
                   break;
                 }
               } catch (KeeperException e) {
                 // ignore zookeeper error
                 LOG.debug("Got a zookeeper when trying to open a recovering region", e);
                 break;
               }
             }
           }
         }
       }
     }
   }
 }
 private String getTableCFsNode(String id) {
   return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.tableCFsNodeName));
 }
예제 #21
0
  @Test
  public void testRun() throws Exception {

    TableName tn = TableName.valueOf(tableName);
    byte[] mobValueBytes = new byte[100];

    // get the path where mob files lie in
    Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, family);

    Put put = new Put(Bytes.toBytes(row));
    put.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes);
    Put put2 = new Put(Bytes.toBytes(row + "ignore"));
    put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes);
    table.mutate(put);
    table.mutate(put2);
    table.flush();
    admin.flush(tn);

    FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
    // check the generation of a mob file
    assertEquals(1, fileStatuses.length);

    String mobFile1 = fileStatuses[0].getPath().getName();

    Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
    configuration.setFloat(MobConstants.MOB_SWEEP_TOOL_COMPACTION_RATIO, 0.6f);
    configuration.setStrings(TableInputFormat.INPUT_TABLE, tableName);
    configuration.setStrings(TableInputFormat.SCAN_COLUMN_FAMILY, family);
    configuration.setStrings(SweepJob.WORKING_VISITED_DIR_KEY, "jobWorkingNamesDir");
    configuration.setStrings(SweepJob.WORKING_FILES_DIR_KEY, "compactionFileDir");
    configuration.setStrings(
        CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, JavaSerialization.class.getName());
    configuration.set(SweepJob.WORKING_VISITED_DIR_KEY, "compactionVisitedDir");
    configuration.setLong(
        MobConstants.MOB_SWEEP_TOOL_COMPACTION_START_DATE,
        System.currentTimeMillis() + 24 * 3600 * 1000);

    ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable());
    TableName lockName = MobUtils.getTableLockName(tn);
    String znode = ZKUtil.joinZNode(zkw.tableLockZNode, lockName.getNameAsString());
    configuration.set(SweepJob.SWEEP_JOB_ID, "1");
    configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode);
    ServerName serverName = SweepJob.getCurrentServerName(configuration);
    configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString());

    TableLockManager tableLockManager =
        TableLockManager.createTableLockManager(configuration, zkw, serverName);
    TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool");
    lock.acquire();
    try {
      // use the same counter when mocking
      Counter counter = new GenericCounter();
      Reducer<Text, KeyValue, Writable, Writable>.Context ctx = mock(Reducer.Context.class);
      when(ctx.getConfiguration()).thenReturn(configuration);
      when(ctx.getCounter(Matchers.any(SweepCounter.class))).thenReturn(counter);
      when(ctx.nextKey()).thenReturn(true).thenReturn(false);
      when(ctx.getCurrentKey()).thenReturn(new Text(mobFile1));

      byte[] refBytes = Bytes.toBytes(mobFile1);
      long valueLength = refBytes.length;
      byte[] newValue = Bytes.add(Bytes.toBytes(valueLength), refBytes);
      KeyValue kv2 =
          new KeyValue(
              Bytes.toBytes(row),
              Bytes.toBytes(family),
              Bytes.toBytes(qf),
              1,
              KeyValue.Type.Put,
              newValue);
      List<KeyValue> list = new ArrayList<KeyValue>();
      list.add(kv2);

      when(ctx.getValues()).thenReturn(list);

      SweepReducer reducer = new SweepReducer();
      reducer.run(ctx);
    } finally {
      lock.release();
    }
    FileStatus[] filsStatuses2 = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
    String mobFile2 = filsStatuses2[0].getPath().getName();
    // new mob file is generated, old one has been archived
    assertEquals(1, filsStatuses2.length);
    assertEquals(false, mobFile2.equalsIgnoreCase(mobFile1));

    // test sequence file
    String workingPath = configuration.get(SweepJob.WORKING_VISITED_DIR_KEY);
    FileStatus[] statuses = TEST_UTIL.getTestFileSystem().listStatus(new Path(workingPath));
    Set<String> files = new TreeSet<String>();
    for (FileStatus st : statuses) {
      files.addAll(
          getKeyFromSequenceFile(TEST_UTIL.getTestFileSystem(), st.getPath(), configuration));
    }
    assertEquals(1, files.size());
    assertEquals(true, files.contains(mobFile1));
  }
 private String getPeerStateNode(String id) {
   return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.peerStateNodeName));
 }
 private static HRegionServer unsetDrainingServer(final HRegionServer hrs) throws KeeperException {
   ZooKeeperWatcher zkw = hrs.getZooKeeper();
   String hrsDrainingZnode = ZKUtil.joinZNode(zkw.drainingZNode, hrs.getServerName().toString());
   ZKUtil.deleteNode(zkw, hrsDrainingZnode);
   return hrs;
 }