@Override
    public void eventReceived(CuratorFramework client, CuratorEvent event) throws Exception {
      if (event.getType() == CuratorEventType.CHILDREN) {
        if (event.getChildren().isEmpty()) {
          client.getChildren().inBackground(event.getContext()).forPath(event.getPath());
        } else {
          String path = event.getPath() + "/" + event.getChildren().get(0);
          LOG.info("Operations Node registered in ZK. Waiting for transports configration");
          client.getData().inBackground(event.getContext()).forPath(path);
        }
      } else if (event.getType() == CuratorEventType.GET_DATA) {
        if (event.getData() == null) {
          client.getData().inBackground(event.getContext()).forPath(event.getPath());
        } else {
          OperationsNodeInfo nodeInfo =
              OPERATIONS_NODE_INFO_CONVERTER.fromByteArray(event.getData());
          boolean isTransportInitialized = !nodeInfo.getTransports().isEmpty();

          if (isTransportInitialized) {
            LOG.info("Operations Node updated tarnsports configuration in ZK");
            ((CountDownLatch) event.getContext()).countDown();
          } else {
            client.getData().inBackground(event.getContext()).forPath(event.getPath());
          }
        }
      }
    }
  public Map<String, Map<String, ClientLeaseInfo>> loadAndWatchTopicExistingLeases(String topic)
      throws Exception {
    Map<String, Map<String, ClientLeaseInfo>> topicExistingLeases = new HashMap<>();

    CuratorFramework client = m_zkClient.get();
    String topicPath = ZKPaths.makePath(ZKPathUtils.getBrokerLeaseRootZkPath(), topic);

    client.getData().usingWatcher(m_brokerLeaseChangedWatcher).forPath(topicPath);
    m_brokerLeaseChangedWatcher.addWatchedPath(topicPath);
    addWatchedTopic(topic);

    List<String> partitions = client.getChildren().forPath(topicPath);

    if (partitions != null && !partitions.isEmpty()) {
      for (String partition : partitions) {
        String partitionPath = ZKPaths.makePath(topicPath, partition);
        byte[] data = client.getData().forPath(partitionPath);
        Map<String, ClientLeaseInfo> existingLeases = deserializeExistingLeases(data);
        if (existingLeases != null) {
          topicExistingLeases.put(partitionPath, existingLeases);
        }
      }
    }

    return topicExistingLeases;
  }
  public static long getOffsetFromZooKeeper(
      CuratorFramework curatorClient, String groupId, String topic, int partition)
      throws Exception {
    ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
    String path = topicDirs.consumerOffsetDir() + "/" + partition;
    curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());

    byte[] data = curatorClient.getData().forPath(path);

    if (data == null) {
      return OFFSET_NOT_SET;
    } else {
      String asString = new String(data);
      if (asString.length() == 0) {
        return OFFSET_NOT_SET;
      } else {
        try {
          return Long.parseLong(asString);
        } catch (NumberFormatException e) {
          throw new Exception(
              String.format(
                  "The offset in ZooKeeper for group '%s', topic '%s', partition %d is a malformed string: %s",
                  groupId, topic, partition, asString));
        }
      }
    }
  }
  private boolean verifyUpdate(ObjectMapper jsonMapper, String path, CuratorFramework remoteCf)
      throws Exception {
    DruidServerMetadata announced =
        jsonMapper.readValue(remoteCf.getData().forPath(path), DruidServerMetadata.class);

    return (118 == announced.getMaxSize());
  }
Exemple #5
0
  public byte[] getData(CuratorFramework zk, String path, boolean watch) throws Exception {
    String npath = PathUtils.normalize_path(path);
    try {
      if (existsNode(zk, npath, watch)) {
        if (watch) {
          return zk.getData().watched().forPath(npath);
        } else {
          return zk.getData().forPath(npath);
        }
      }
    } catch (KeeperException e) {
      LOG.error("zookeeper getdata for path" + path, e);
    }

    return null;
  }
  public static void main(String[] args) {
    TestingServer server = null;
    CuratorFramework client = null;
    CuratorFramework optionsClient = null;

    try {
      server = new TestingServer();
      client = createSimple(server.getConnectString());
      client.start();
      client.create().creatingParentsIfNeeded().forPath(PATH, "test".getBytes());

      optionsClient =
          createWithOptions(
              server.getConnectString(), new ExponentialBackoffRetry(1000, 3), 1000, 1000);
      optionsClient.start();
      log.info("[{}]", new String(optionsClient.getData().forPath(PATH)));
    } catch (Exception e) {
      log.info("exception throws cause {}", Throwables.getStackTraceAsString(e));
    } finally {
      if (client != null) {
        CloseableUtils.closeQuietly(client);
      }
      if (optionsClient != null) {
        CloseableUtils.closeQuietly(optionsClient);
      }
    }
  }
Exemple #7
0
 protected String getFirstService(String containerPath) throws Exception {
   CuratorFramework curatorFramework = curator.get();
   if (curatorFramework != null) {
     byte[] data = curatorFramework.getData().forPath(containerPath);
     if (data != null && data.length > 0) {
       String text = new String(data).trim();
       if (!text.isEmpty()) {
         ObjectMapper mapper = new ObjectMapper();
         Map<String, Object> map = mapper.readValue(data, HashMap.class);
         Object serviceValue = map.get("services");
         if (serviceValue instanceof List) {
           List services = (List) serviceValue;
           if (services != null) {
             if (!services.isEmpty()) {
               List<String> serviceTexts = new ArrayList<String>();
               for (Object service : services) {
                 String serviceText = getSubstitutedData(curatorFramework, service.toString());
                 if (io.fabric8.common.util.Strings.isNotBlank(serviceText)) {
                   return serviceText;
                 }
               }
             }
           }
         }
       }
     }
   }
   return null;
 }
  /**
   * Load the {@link org.springframework.xd.dirt.core.Job} instance for a given job name<i>if the
   * job is deployed</i>.
   *
   * @param client curator client
   * @param jobName the name of the job to load
   * @param jobFactory job factory used to create instance of job
   * @return the job instance, or {@code null} if the job does not exist or is not deployed
   * @throws Exception
   */
  public static Job loadJob(CuratorFramework client, String jobName, JobFactory jobFactory)
      throws Exception {
    try {
      byte[] definition = client.getData().forPath(Paths.build(Paths.JOBS, jobName));
      Map<String, String> definitionMap = ZooKeeperUtils.bytesToMap(definition);

      byte[] deploymentPropertiesData =
          client.getData().forPath(Paths.build(Paths.JOB_DEPLOYMENTS, jobName));
      if (deploymentPropertiesData != null && deploymentPropertiesData.length > 0) {
        definitionMap.put("deploymentProperties", new String(deploymentPropertiesData, "UTF-8"));
      }
      return jobFactory.createJob(jobName, definitionMap);
    } catch (KeeperException.NoNodeException e) {
      // job is not deployed
    }
    return null;
  }
  /**
   * Load the {@link org.springframework.xd.dirt.core.Stream} instance for a given stream name <i>if
   * the stream is deployed</i>. It will include the stream definition as well as any deployment
   * properties data for the stream deployment.
   *
   * @param client curator client
   * @param streamName the name of the stream to load
   * @param streamFactory stream factory used to create instance of stream
   * @return the stream instance, or {@code null} if the stream does not exist or is not deployed
   * @throws Exception if ZooKeeper access fails for any reason
   */
  public static Stream loadStream(
      CuratorFramework client, String streamName, StreamFactory streamFactory) throws Exception {
    try {
      byte[] definition = client.getData().forPath(Paths.build(Paths.STREAMS, streamName));
      Map<String, String> definitionMap = ZooKeeperUtils.bytesToMap(definition);

      byte[] deploymentPropertiesData =
          client.getData().forPath(Paths.build(Paths.STREAM_DEPLOYMENTS, streamName));
      if (deploymentPropertiesData != null && deploymentPropertiesData.length > 0) {
        definitionMap.put("deploymentProperties", new String(deploymentPropertiesData, "UTF-8"));
      }
      return streamFactory.createStream(streamName, definitionMap);
    } catch (KeeperException.NoNodeException e) {
      // stream is not deployed or does not exist
    }
    return null;
  }
 public int queryRoomChildRed5Node(String roomname) throws Throwable {
   logger.debug(String.format("查询房间%s的子节点red5数据", roomname));
   byte[] data = client.getData().forPath(ROOMNODE + "/" + roomname + RED5);
   String json = new String(data);
   @SuppressWarnings("unchecked")
   Map<String, Object> map = (Map<String, Object>) JsonUtil.deserialize(json, Map.class);
   int red5UserCount = (Integer) map.get("red5UserCount");
   return red5UserCount;
 }
  public Long recoverLastFinishedTime(int partitionId) {
    String path = this.zkRoot + "/" + partitionId + "/" + ZNODE_LAST_FINISH_TIME;
    try {
      return Long.valueOf(new String(curator.getData().forPath(path)));
    } catch (Exception e) {
      LOG.error("failed to recover last finish time {}", e);
    }

    return 0L;
  }
 /**
  * Return a service instance POJO
  *
  * @param name name of the service
  * @param id ID of the instance
  * @return the instance or <code>null</code> if not found
  * @throws Exception errors
  */
 @Override
 public ServiceInstance<T> queryForInstance(String name, String id) throws Exception {
   String path = pathForInstance(name, id);
   try {
     byte[] bytes = client.getData().forPath(path);
     return serializer.deserialize(bytes);
   } catch (KeeperException.NoNodeException ignore) {
     // ignore
   }
   return null;
 }
Exemple #13
0
 public byte[] readBytes(String path) {
   try {
     if (_curator.checkExists().forPath(path) != null) {
       return _curator.getData().forPath(path);
     } else {
       return null;
     }
   } catch (Exception e) {
     throw new RuntimeException(e);
   }
 }
 private InetSocketAddress toInetAdrress(
     String nodePath, ZookeeperServiceReader reader, CuratorFramework zkClient) {
   try {
     byte[] data = zkClient.getData().forPath(nodePath);
     if (data == null) {
       // clustered service member went away, but path still exists
       return null;
     }
     return reader.deserialize(data);
   } catch (Exception e) {
     logger.error("Rethrowing exception ", e);
     throw new RuntimeException(e);
   }
 }
  @Override
  public String readData(String statePath) {
    try {
      if (curatorFramework.checkExists().forPath(statePath) == null) {
        // do we want to throw an exception if path doesn't exist??
        return null;
      } else {
        byte[] bytes = curatorFramework.getData().forPath(statePath);
        String data = new String(bytes);

        logger.info(String.format("data was retrieved. path: %s, data: %s.", statePath, data));

        return data;
      }
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  @Test
  public void testRunTask() throws Exception {
    cf.create()
        .creatingParentsIfNeeded()
        .forPath(joiner.join(tasksPath, task.getId()), jsonMapper.writeValueAsBytes(task));

    Assert.assertTrue(
        TestUtils.conditionValid(
            new IndexingServiceCondition() {
              @Override
              public boolean isValid() {
                try {
                  return cf.checkExists().forPath(joiner.join(tasksPath, task.getId())) == null;
                } catch (Exception e) {
                  return false;
                }
              }
            }));

    Assert.assertTrue(
        TestUtils.conditionValid(
            new IndexingServiceCondition() {
              @Override
              public boolean isValid() {
                try {
                  return cf.checkExists().forPath(joiner.join(statusPath, task.getId())) != null;
                } catch (Exception e) {
                  return false;
                }
              }
            }));

    TaskAnnouncement taskAnnouncement =
        jsonMapper.readValue(
            cf.getData().forPath(joiner.join(statusPath, task.getId())), TaskAnnouncement.class);

    Assert.assertEquals(task.getId(), taskAnnouncement.getTaskStatus().getId());
    Assert.assertEquals(
        TaskStatus.Status.RUNNING, taskAnnouncement.getTaskStatus().getStatusCode());
  }
  @Test
  public void testBasicFunctionality() throws Exception {

    // Check ZNode for timestamp exists (storage instantiation should create it)
    Stat zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
    assertEquals(zNodeStats.getVersion(), 0);

    // Initial checks
    assertEquals(storage.getMaxTimestamp(), INITIAL_MAX_TS_VALUE);
    byte[] data = zkClient.getData().forPath(TIMESTAMP_ZNODE);
    assertEquals(data.length, BYTES_IN_LONG);

    // Check new timestamp does not allow negative values...
    try {
      storage.updateMaxTimestamp(INITIAL_MAX_TS_VALUE, NEGATIVE_TS);
      fail();
    } catch (IllegalArgumentException e) {
      LOG.info("Expected exception", e);
    }

    // ...nor is less than previous timestamp
    try {
      storage.updateMaxTimestamp(1, 0);
      fail();
    } catch (IllegalArgumentException e) {
      LOG.info("Expected exception", e);
    }

    // Check that the original version is still there
    zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
    assertEquals(zNodeStats.getVersion(), 0);

    // Iterate updating the timestamp and check the final value
    long previousMaxTimestamp = INITIAL_MAX_TS_VALUE;
    for (int i = 0; i < ITERATION_COUNT; i++) {
      long newMaxTimestamp = previousMaxTimestamp + 1_000_000;
      storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
      previousMaxTimestamp = newMaxTimestamp;
    }
    assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
    // Check the znode version has changed accordingly
    zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
    assertEquals(zNodeStats.getVersion(), ITERATION_COUNT);

    // Check exceptions
    doThrow(new RuntimeException()).when(storageInternalZKClient).getData();
    try {
      storage.getMaxTimestamp();
      fail();
    } catch (IOException e) {
      LOG.info("Expected exception", e);
    }

    doThrow(new RuntimeException()).when(storageInternalZKClient).setData();
    try {
      storage.updateMaxTimestamp(INITIAL_MAX_TS_VALUE, INITIAL_MAX_TS_VALUE + 1_000_000);
      fail();
    } catch (IOException e) {
      LOG.info("Expected exception", e);
    }

    // Reset the mock and double-check last result
    Mockito.reset(storageInternalZKClient);
    assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);

    // Finally check the znode version is still the same
    zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
    assertEquals(zNodeStats.getVersion(), ITERATION_COUNT);
  }
  public Map<String, Pair<Map<String, String>, AppInfo>> recoverYarnApp(String yarnAppId)
      throws Exception {
    Map<String, Pair<Map<String, String>, AppInfo>> result = new HashMap<>();
    String path = this.zkRoot + "/" + yarnAppId;
    try {
      lock.acquire();
      if (curator.checkExists().forPath(path) == null) {
        return result;
      }
      List<String> jobIds = curator.getChildren().forPath(path);
      if (jobIds.size() == 0) {
        LOG.info("delete empty path {}", path);
        delete(yarnAppId);
      }

      for (String jobId : jobIds) {
        String jobPath = path + "/" + jobId;
        LOG.info("recover path {}", jobPath);
        String fields = new String(curator.getData().forPath(jobPath), "UTF-8");
        if (fields.length() == 0) {
          // LOG.info("delete empty path {}", jobPath);
          // delete(yarnAppId, jobId);
          continue;
        }
        JSONObject object = new JSONObject(fields);
        Map<String, Map<String, String>> parseResult = parse(object);

        Map<String, String> appInfoMap = parseResult.get(APP_INFO_KEY);
        AppInfo appInfo = new AppInfo();
        appInfo.setId(appInfoMap.get("id"));
        appInfo.setUser(appInfoMap.get("user"));
        appInfo.setName(appInfoMap.get("name"));
        appInfo.setQueue(appInfoMap.get("queue"));
        appInfo.setState(appInfoMap.get("state"));
        appInfo.setFinalStatus(appInfoMap.get("finalStatus"));
        appInfo.setProgress(Double.parseDouble(appInfoMap.get("progress")));
        appInfo.setTrackingUI(appInfoMap.get("trackingUI"));
        appInfo.setDiagnostics(appInfoMap.get("diagnostics"));
        appInfo.setTrackingUrl(appInfoMap.get("trackingUrl"));
        appInfo.setClusterId(appInfoMap.get("clusterId"));
        appInfo.setApplicationType(appInfoMap.get("applicationType"));
        appInfo.setStartedTime(Long.parseLong(appInfoMap.get("startedTime")));
        appInfo.setFinishedTime(Long.parseLong(appInfoMap.get("finishedTime")));
        appInfo.setElapsedTime(Long.parseLong(appInfoMap.get("elapsedTime")));
        appInfo.setAmContainerLogs(
            appInfoMap.get("amContainerLogs") == null ? "" : appInfoMap.get("amContainerLogs"));
        appInfo.setAmHostHttpAddress(
            appInfoMap.get("amHostHttpAddress") == null ? "" : appInfoMap.get("amHostHttpAddress"));
        appInfo.setAllocatedMB(Long.parseLong(appInfoMap.get("allocatedMB")));
        appInfo.setAllocatedVCores(Integer.parseInt(appInfoMap.get("allocatedVCores")));
        appInfo.setRunningContainers(Integer.parseInt(appInfoMap.get("runningContainers")));

        Map<String, String> tags = parseResult.get(ENTITY_TAGS_KEY);
        result.put(jobId, Pair.of(tags, appInfo));
      }
    } catch (Exception e) {
      LOG.error("fail to recoverYarnApp", e);
      throw new RuntimeException(e);
    } finally {
      try {
        lock.release();
      } catch (Exception e) {
        LOG.error("fail releasing lock", e);
      }
    }

    return result;
  }