@Test
  public void testCheckVersion() throws Exception {
    CuratorFramework client =
        CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
    client.start();
    try {
      client.create().forPath("/foo");
      Stat stat = client.setData().forPath("/foo", "new".getBytes()); // up the version

      try {
        client
            .inTransaction()
            .check()
            .withVersion(stat.getVersion() + 1)
            .forPath("/foo") // force a bad version
            .and()
            .create()
            .forPath("/bar")
            .and()
            .commit();

        Assert.fail();
      } catch (KeeperException.BadVersionException correct) {
        // correct
      }

      Assert.assertNull(client.checkExists().forPath("/bar"));
    } finally {
      client.close();
    }
  }
Ejemplo n.º 2
0
  @Test
  public void testNamespace() throws Exception {
    Timing timing = new Timing();
    ChildReaper reaper = null;
    CuratorFramework client =
        CuratorFrameworkFactory.builder()
            .connectString(server.getConnectString())
            .sessionTimeoutMs(timing.session())
            .connectionTimeoutMs(timing.connection())
            .retryPolicy(new RetryOneTime(1))
            .namespace("foo")
            .build();
    try {
      client.start();

      for (int i = 0; i < 10; ++i) {
        client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
      }

      reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
      reaper.start();

      timing.forWaiting().sleepABit();

      Stat stat = client.checkExists().forPath("/test");
      Assert.assertEquals(stat.getNumChildren(), 0);

      stat = client.usingNamespace(null).checkExists().forPath("/foo/test");
      Assert.assertNotNull(stat);
      Assert.assertEquals(stat.getNumChildren(), 0);
    } finally {
      Closeables.closeQuietly(reaper);
      Closeables.closeQuietly(client);
    }
  }
  @Test
  public void testBasic() throws Exception {
    CuratorFramework client =
        CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
    client.start();
    try {
      Collection<CuratorTransactionResult> results =
          client
              .inTransaction()
              .create()
              .forPath("/foo")
              .and()
              .create()
              .forPath("/foo/bar", "snafu".getBytes())
              .and()
              .commit();

      Assert.assertTrue(client.checkExists().forPath("/foo/bar") != null);
      Assert.assertEquals(client.getData().forPath("/foo/bar"), "snafu".getBytes());

      CuratorTransactionResult fooResult =
          Iterables.find(
              results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/foo"));
      CuratorTransactionResult fooBarResult =
          Iterables.find(
              results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/foo/bar"));
      Assert.assertNotNull(fooResult);
      Assert.assertNotNull(fooBarResult);
      Assert.assertNotSame(fooResult, fooBarResult);
      Assert.assertEquals(fooResult.getResultPath(), "/foo");
      Assert.assertEquals(fooBarResult.getResultPath(), "/foo/bar");
    } finally {
      client.close();
    }
  }
Ejemplo n.º 4
0
  @Test
  public void testUpdateShuffleNode() throws Exception {
    JobId jobId = JobId.newJobId(createTestAppId(), 123456);

    DragonZooKeeper.NodeData node = new DragonZooKeeper.NodeData();
    node.nodeId = createTestNodeId();
    node.taskId = TaskId.newTaskId(jobId, 34565, TaskType.MAP);

    dragonZK.createShufflePath(jobId);
    dragonZK.createShuffleNode(jobId, newArrayList(node));

    node.nodeId.setHost("NewServerHost");
    dragonZK.updateShuffleNode(jobId, newArrayList(node));

    Stat stat =
        zkClient
            .checkExists()
            .forPath(
                "/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565");
    assertNotNull(stat);

    byte[] data =
        zkClient
            .getData()
            .forPath(
                "/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565");
    assertEquals("NewServerHost:2345", new String(data));
  }
Ejemplo n.º 5
0
  @Test
  public void testSomeNodes() throws Exception {

    Timing timing = new Timing();
    ChildReaper reaper = null;
    CuratorFramework client =
        CuratorFrameworkFactory.newClient(
            server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
    try {
      client.start();

      Random r = new Random();
      int nonEmptyNodes = 0;
      for (int i = 0; i < 10; ++i) {
        client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
        if (r.nextBoolean()) {
          client.create().forPath("/test/" + Integer.toString(i) + "/foo");
          ++nonEmptyNodes;
        }
      }

      reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
      reaper.start();

      timing.forWaiting().sleepABit();

      Stat stat = client.checkExists().forPath("/test");
      Assert.assertEquals(stat.getNumChildren(), nonEmptyNodes);
    } finally {
      Closeables.closeQuietly(reaper);
      Closeables.closeQuietly(client);
    }
  }
  /**
   * Try to atomically update a node in ZooKeeper, creating it if it doesn't exist. This is meant to
   * be used within an optimistic concurrency model.
   *
   * @param pathSuffix suffix to use to build path in ZooKeeper.
   * @param f function used to initialize the node, or transform the data already there.
   * @return true if node was created/updated, false if a concurrent modification occurred and
   *     succeeded while trying to update/create the node.
   * @throws Exception
   */
  private boolean tryAtomicUpdate(final String pathSuffix, final NodeFunction f) throws Exception {
    final String path = buildZookeeperPath(pathSuffix);
    final Stat stat = zk.checkExists().forPath(path);

    if (stat == null) {
      try {
        zk.create()
            .creatingParentsIfNeeded()
            .withMode(CreateMode.PERSISTENT)
            .forPath(path, f.initialize());
      } catch (KeeperException.NodeExistsException e) {
        LOG.debug("Concurrent creation of " + path + ", retrying", e);
        return false;
      }
    } else {
      Mod<byte[]> newVal = f.apply(zk.getData().forPath(path));

      if (newVal.hasModification()) {
        try {
          zk.setData().withVersion(stat.getVersion()).forPath(path, newVal.get());
        } catch (KeeperException.BadVersionException e) {
          LOG.debug("Concurrent update to " + path + ", retrying.", e);
          return false;
        }
      }
    }

    return true;
  }
  @Test
  public void testWithNamespace() throws Exception {
    CuratorFramework client =
        CuratorFrameworkFactory.builder()
            .connectString(server.getConnectString())
            .retryPolicy(new RetryOneTime(1))
            .namespace("galt")
            .build();
    client.start();
    try {
      Collection<CuratorTransactionResult> results =
          client
              .inTransaction()
              .create()
              .forPath("/foo", "one".getBytes())
              .and()
              .create()
              .withMode(CreateMode.PERSISTENT_SEQUENTIAL)
              .forPath("/test-", "one".getBytes())
              .and()
              .setData()
              .forPath("/foo", "two".getBytes())
              .and()
              .create()
              .forPath("/foo/bar")
              .and()
              .delete()
              .forPath("/foo/bar")
              .and()
              .commit();

      Assert.assertTrue(client.checkExists().forPath("/foo") != null);
      Assert.assertTrue(client.nonNamespaceView().checkExists().forPath("/galt/foo") != null);
      Assert.assertEquals(client.getData().forPath("/foo"), "two".getBytes());
      Assert.assertTrue(client.checkExists().forPath("/foo/bar") == null);

      CuratorTransactionResult ephemeralResult =
          Iterables.find(
              results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/test-"));
      Assert.assertNotNull(ephemeralResult);
      Assert.assertNotEquals(ephemeralResult.getResultPath(), "/test-");
      Assert.assertTrue(ephemeralResult.getResultPath().startsWith("/test-"));
    } finally {
      client.close();
    }
  }
Ejemplo n.º 8
0
  @Test
  public void testRegisterNodeManager() throws Exception {
    NodeId nodeId = createTestNodeId();

    dragonZK.registerNodeManager(nodeId);

    Stat stat = zkClient.checkExists().forPath("/dragon/nodemanagers/ServerHost:2345");
    assertNotNull(stat);
  }
  // 检查节点是否已经存在
  private synchronized boolean checkNodeExisted(String path, CuratorFramework client)
      throws Exception {

    Stat stat = client.checkExists().forPath(path);

    if (stat != null) {
      return true;
    }

    return false;
  }
Ejemplo n.º 10
0
  @Test
  public void testCreateShufflePath() throws Exception {
    ApplicationId appId = createTestAppId();

    JobId jobId = JobId.newJobId(appId, 123456);

    dragonZK.createShufflePath(jobId);

    Stat stat = zkClient.checkExists().forPath("/dragon/job_458495849584_89894985_123456/shuffle");
    assertNotNull(stat);
  }
Ejemplo n.º 11
0
  private boolean internalLockLoop(long startMillis, Long millisToWait, String ourPath)
      throws Exception {
    if (revocable.get() != null) {
      client.getData().usingWatcher(revocableWatcher).forPath(ourPath);
    }

    boolean haveTheLock = false;
    boolean doDelete = false;
    try {
      while (client.isStarted() && !haveTheLock) {
        List<String> children = getSortedChildren();
        String sequenceNodeName =
            ourPath.substring(basePath.length() + 1); // +1 to include the slash

        PredicateResults predicateResults =
            driver.getsTheLock(client, children, sequenceNodeName, maxLeases);
        if (predicateResults.getsTheLock()) {
          haveTheLock = true;
        } else {
          String previousSequencePath = basePath + "/" + predicateResults.getPathToWatch();

          synchronized (this) {
            Stat stat = client.checkExists().usingWatcher(watcher).forPath(previousSequencePath);
            if (stat != null) {
              if (millisToWait != null) {
                millisToWait -= (System.currentTimeMillis() - startMillis);
                startMillis = System.currentTimeMillis();
                if (millisToWait <= 0) {
                  doDelete = true; // timed out - delete our node
                  break;
                }

                wait(millisToWait);
              } else {
                wait();
              }
            }
          }
          // else it may have been deleted (i.e. lock released). Try to acquire again
        }
      }
    } catch (KeeperException e) {
      // ignore this and let the retry policy handle it
      throw e;
    } catch (Exception e) {
      doDelete = true;
      throw e;
    } finally {
      if (doDelete) {
        client.delete().guaranteed().forPath(ourPath);
      }
    }
    return haveTheLock;
  }
Ejemplo n.º 12
0
  private void checkLeadership(List<String> children) throws Exception {
    final String localOurPath = ourPath.get();
    List<String> sortedChildren = LockInternals.getSortedChildren(LOCK_NAME, sorter, children);
    int ourIndex =
        (localOurPath != null) ? sortedChildren.indexOf(ZKPaths.getNodeFromPath(localOurPath)) : -1;
    if (ourIndex < 0) {
      log.error("Can't find our node. Resetting. Index: " + ourIndex);
      reset();
    } else if (ourIndex == 0) {
      setLeadership(true);
    } else {
      String watchPath = sortedChildren.get(ourIndex - 1);
      Watcher watcher =
          new Watcher() {
            @Override
            public void process(WatchedEvent event) {
              if ((state.get() == State.STARTED)
                  && (event.getType() == Event.EventType.NodeDeleted)
                  && (localOurPath != null)) {
                try {
                  getChildren();
                } catch (Exception ex) {
                  log.error("An error occurred checking the leadership.", ex);
                }
              }
            }
          };

      BackgroundCallback callback =
          new BackgroundCallback() {
            @Override
            public void processResult(CuratorFramework client, CuratorEvent event)
                throws Exception {
              if (event.getResultCode() == KeeperException.Code.NONODE.intValue()) {
                // previous node is gone - reset
                reset();
              }
            }
          };
      client
          .checkExists()
          .usingWatcher(watcher)
          .inBackground(callback)
          .forPath(ZKPaths.makePath(latchPath, watchPath));
    }
  }
  /** Amoeba启动的时候,初始化zookeeper客户端,另外,给zookeeper预热 */
  public void init() throws Exception {

    Properties props = Utils.readGlobalSeqConfigProps();
    String zkHosts = props.getProperty("zkHosts", "127.0.0.1:2181");
    Integer connTimeOut = Integer.valueOf(props.getProperty("zkConnTimeOut", "60"));

    log.info(String.format("zookeeper config: connect string= %s", zkHosts));
    log.info(String.format("zookeeper config: connect timeout= %d seconds", connTimeOut));

    client =
        CuratorFrameworkFactory.builder()
            .connectString(zkHosts)
            .retryPolicy(new ExponentialBackoffRetry(BASE_SLEEP_TIME, RETRY_TIMES))
            .connectionTimeoutMs(connTimeOut * 1000)
            .build(); // 时间要由秒转成毫秒

    client.start();

    // 只是为client热身而调用
    client.checkExists().forPath(ROOT_PATH);
  }
Ejemplo n.º 14
0
  private void checkForLeadership() throws Exception {
    List<String> sortedChildren =
        LockInternals.getSortedChildren(client, latchPath, LOCK_NAME, sorter);
    if (sortedChildren.size() == 0) {
      throw new Exception("no children - unexpected state");
    }

    int ourIndex = sortedChildren.indexOf(ZKPaths.getNodeFromPath(ourPath));
    if (ourIndex == 0) {
      setLeadership(true);
    } else {
      final String ourPathWhenWatched =
          ourPath; // protected against a lost/suspended connection and an old watcher - I'm not
                   // sure if this is possible but it can't hurt
      String watchPath = sortedChildren.get(ourIndex - 1);
      Watcher watcher =
          new Watcher() {
            @Override
            public void process(WatchedEvent event) {
              if ((event.getType() == Event.EventType.NodeDeleted)
                  && (ourPath != null)
                  && ourPath.equals(ourPathWhenWatched)) {
                try {
                  checkForLeadership();
                } catch (Exception ex) {
                  log.error("An error ocurred checking the leadership.", ex);
                }
              }
            }
          };
      if (client.checkExists().usingWatcher(watcher).forPath(ZKPaths.makePath(latchPath, watchPath))
          == null) {
        // the previous Participant may be down, so we need to reevaluate the list
        // to get the actual previous Participant or get the leadership
        checkForLeadership();
      }
    }
  }
Ejemplo n.º 15
0
  @Test
  public void testGetShuffleNodeByTaskId() throws Exception {
    JobId jobId = JobId.newJobId(createTestAppId(), 123456);
    TaskId taskId = TaskId.newTaskId(jobId, 34565, TaskType.MAP);
    DragonZooKeeper.NodeData node = new DragonZooKeeper.NodeData();
    node.nodeId = createTestNodeId();
    node.taskId = taskId;

    if (zkClient.checkExists().forPath("/dragon/job_458495849584_89894985_123456/shuffle")
        == null) {
      dragonZK.createShufflePath(jobId);
    }

    PathChildrenCache shuffleNodeCache =
        new PathChildrenCache(zkClient, dragonZK.getShufflePath(jobId), true);
    dragonZK.setShuffleNodeCache(shuffleNodeCache);

    dragonZK.createShuffleNode(jobId, newArrayList(node));
    Thread.sleep(1000);

    NodeId nodeId = dragonZK.getShuffleNodeByTaskId(jobId, taskId);
    assertNotNull(nodeId);
    assertEquals(node.nodeId, nodeId);
  }