@Override public void eventReceived(CuratorFramework client, CuratorEvent event) throws Exception { if (event.getType() == CuratorEventType.CHILDREN) { if (event.getChildren().isEmpty()) { client.getChildren().inBackground(event.getContext()).forPath(event.getPath()); } else { String path = event.getPath() + "/" + event.getChildren().get(0); LOG.info("Operations Node registered in ZK. Waiting for transports configration"); client.getData().inBackground(event.getContext()).forPath(path); } } else if (event.getType() == CuratorEventType.GET_DATA) { if (event.getData() == null) { client.getData().inBackground(event.getContext()).forPath(event.getPath()); } else { OperationsNodeInfo nodeInfo = OPERATIONS_NODE_INFO_CONVERTER.fromByteArray(event.getData()); boolean isTransportInitialized = !nodeInfo.getTransports().isEmpty(); if (isTransportInitialized) { LOG.info("Operations Node updated tarnsports configuration in ZK"); ((CountDownLatch) event.getContext()).countDown(); } else { client.getData().inBackground(event.getContext()).forPath(event.getPath()); } } } }
public Map<String, Map<String, ClientLeaseInfo>> loadAndWatchTopicExistingLeases(String topic) throws Exception { Map<String, Map<String, ClientLeaseInfo>> topicExistingLeases = new HashMap<>(); CuratorFramework client = m_zkClient.get(); String topicPath = ZKPaths.makePath(ZKPathUtils.getBrokerLeaseRootZkPath(), topic); client.getData().usingWatcher(m_brokerLeaseChangedWatcher).forPath(topicPath); m_brokerLeaseChangedWatcher.addWatchedPath(topicPath); addWatchedTopic(topic); List<String> partitions = client.getChildren().forPath(topicPath); if (partitions != null && !partitions.isEmpty()) { for (String partition : partitions) { String partitionPath = ZKPaths.makePath(topicPath, partition); byte[] data = client.getData().forPath(partitionPath); Map<String, ClientLeaseInfo> existingLeases = deserializeExistingLeases(data); if (existingLeases != null) { topicExistingLeases.put(partitionPath, existingLeases); } } } return topicExistingLeases; }
public static long getOffsetFromZooKeeper( CuratorFramework curatorClient, String groupId, String topic, int partition) throws Exception { ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic); String path = topicDirs.consumerOffsetDir() + "/" + partition; curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient()); byte[] data = curatorClient.getData().forPath(path); if (data == null) { return OFFSET_NOT_SET; } else { String asString = new String(data); if (asString.length() == 0) { return OFFSET_NOT_SET; } else { try { return Long.parseLong(asString); } catch (NumberFormatException e) { throw new Exception( String.format( "The offset in ZooKeeper for group '%s', topic '%s', partition %d is a malformed string: %s", groupId, topic, partition, asString)); } } } }
private boolean verifyUpdate(ObjectMapper jsonMapper, String path, CuratorFramework remoteCf) throws Exception { DruidServerMetadata announced = jsonMapper.readValue(remoteCf.getData().forPath(path), DruidServerMetadata.class); return (118 == announced.getMaxSize()); }
public byte[] getData(CuratorFramework zk, String path, boolean watch) throws Exception { String npath = PathUtils.normalize_path(path); try { if (existsNode(zk, npath, watch)) { if (watch) { return zk.getData().watched().forPath(npath); } else { return zk.getData().forPath(npath); } } } catch (KeeperException e) { LOG.error("zookeeper getdata for path" + path, e); } return null; }
public static void main(String[] args) { TestingServer server = null; CuratorFramework client = null; CuratorFramework optionsClient = null; try { server = new TestingServer(); client = createSimple(server.getConnectString()); client.start(); client.create().creatingParentsIfNeeded().forPath(PATH, "test".getBytes()); optionsClient = createWithOptions( server.getConnectString(), new ExponentialBackoffRetry(1000, 3), 1000, 1000); optionsClient.start(); log.info("[{}]", new String(optionsClient.getData().forPath(PATH))); } catch (Exception e) { log.info("exception throws cause {}", Throwables.getStackTraceAsString(e)); } finally { if (client != null) { CloseableUtils.closeQuietly(client); } if (optionsClient != null) { CloseableUtils.closeQuietly(optionsClient); } } }
protected String getFirstService(String containerPath) throws Exception { CuratorFramework curatorFramework = curator.get(); if (curatorFramework != null) { byte[] data = curatorFramework.getData().forPath(containerPath); if (data != null && data.length > 0) { String text = new String(data).trim(); if (!text.isEmpty()) { ObjectMapper mapper = new ObjectMapper(); Map<String, Object> map = mapper.readValue(data, HashMap.class); Object serviceValue = map.get("services"); if (serviceValue instanceof List) { List services = (List) serviceValue; if (services != null) { if (!services.isEmpty()) { List<String> serviceTexts = new ArrayList<String>(); for (Object service : services) { String serviceText = getSubstitutedData(curatorFramework, service.toString()); if (io.fabric8.common.util.Strings.isNotBlank(serviceText)) { return serviceText; } } } } } } } } return null; }
/** * Load the {@link org.springframework.xd.dirt.core.Job} instance for a given job name<i>if the * job is deployed</i>. * * @param client curator client * @param jobName the name of the job to load * @param jobFactory job factory used to create instance of job * @return the job instance, or {@code null} if the job does not exist or is not deployed * @throws Exception */ public static Job loadJob(CuratorFramework client, String jobName, JobFactory jobFactory) throws Exception { try { byte[] definition = client.getData().forPath(Paths.build(Paths.JOBS, jobName)); Map<String, String> definitionMap = ZooKeeperUtils.bytesToMap(definition); byte[] deploymentPropertiesData = client.getData().forPath(Paths.build(Paths.JOB_DEPLOYMENTS, jobName)); if (deploymentPropertiesData != null && deploymentPropertiesData.length > 0) { definitionMap.put("deploymentProperties", new String(deploymentPropertiesData, "UTF-8")); } return jobFactory.createJob(jobName, definitionMap); } catch (KeeperException.NoNodeException e) { // job is not deployed } return null; }
/** * Load the {@link org.springframework.xd.dirt.core.Stream} instance for a given stream name <i>if * the stream is deployed</i>. It will include the stream definition as well as any deployment * properties data for the stream deployment. * * @param client curator client * @param streamName the name of the stream to load * @param streamFactory stream factory used to create instance of stream * @return the stream instance, or {@code null} if the stream does not exist or is not deployed * @throws Exception if ZooKeeper access fails for any reason */ public static Stream loadStream( CuratorFramework client, String streamName, StreamFactory streamFactory) throws Exception { try { byte[] definition = client.getData().forPath(Paths.build(Paths.STREAMS, streamName)); Map<String, String> definitionMap = ZooKeeperUtils.bytesToMap(definition); byte[] deploymentPropertiesData = client.getData().forPath(Paths.build(Paths.STREAM_DEPLOYMENTS, streamName)); if (deploymentPropertiesData != null && deploymentPropertiesData.length > 0) { definitionMap.put("deploymentProperties", new String(deploymentPropertiesData, "UTF-8")); } return streamFactory.createStream(streamName, definitionMap); } catch (KeeperException.NoNodeException e) { // stream is not deployed or does not exist } return null; }
public int queryRoomChildRed5Node(String roomname) throws Throwable { logger.debug(String.format("查询房间%s的子节点red5数据", roomname)); byte[] data = client.getData().forPath(ROOMNODE + "/" + roomname + RED5); String json = new String(data); @SuppressWarnings("unchecked") Map<String, Object> map = (Map<String, Object>) JsonUtil.deserialize(json, Map.class); int red5UserCount = (Integer) map.get("red5UserCount"); return red5UserCount; }
public Long recoverLastFinishedTime(int partitionId) { String path = this.zkRoot + "/" + partitionId + "/" + ZNODE_LAST_FINISH_TIME; try { return Long.valueOf(new String(curator.getData().forPath(path))); } catch (Exception e) { LOG.error("failed to recover last finish time {}", e); } return 0L; }
/** * Return a service instance POJO * * @param name name of the service * @param id ID of the instance * @return the instance or <code>null</code> if not found * @throws Exception errors */ @Override public ServiceInstance<T> queryForInstance(String name, String id) throws Exception { String path = pathForInstance(name, id); try { byte[] bytes = client.getData().forPath(path); return serializer.deserialize(bytes); } catch (KeeperException.NoNodeException ignore) { // ignore } return null; }
public byte[] readBytes(String path) { try { if (_curator.checkExists().forPath(path) != null) { return _curator.getData().forPath(path); } else { return null; } } catch (Exception e) { throw new RuntimeException(e); } }
private InetSocketAddress toInetAdrress( String nodePath, ZookeeperServiceReader reader, CuratorFramework zkClient) { try { byte[] data = zkClient.getData().forPath(nodePath); if (data == null) { // clustered service member went away, but path still exists return null; } return reader.deserialize(data); } catch (Exception e) { logger.error("Rethrowing exception ", e); throw new RuntimeException(e); } }
@Override public String readData(String statePath) { try { if (curatorFramework.checkExists().forPath(statePath) == null) { // do we want to throw an exception if path doesn't exist?? return null; } else { byte[] bytes = curatorFramework.getData().forPath(statePath); String data = new String(bytes); logger.info(String.format("data was retrieved. path: %s, data: %s.", statePath, data)); return data; } } catch (Exception e) { throw new RuntimeException(e); } }
@Test public void testRunTask() throws Exception { cf.create() .creatingParentsIfNeeded() .forPath(joiner.join(tasksPath, task.getId()), jsonMapper.writeValueAsBytes(task)); Assert.assertTrue( TestUtils.conditionValid( new IndexingServiceCondition() { @Override public boolean isValid() { try { return cf.checkExists().forPath(joiner.join(tasksPath, task.getId())) == null; } catch (Exception e) { return false; } } })); Assert.assertTrue( TestUtils.conditionValid( new IndexingServiceCondition() { @Override public boolean isValid() { try { return cf.checkExists().forPath(joiner.join(statusPath, task.getId())) != null; } catch (Exception e) { return false; } } })); TaskAnnouncement taskAnnouncement = jsonMapper.readValue( cf.getData().forPath(joiner.join(statusPath, task.getId())), TaskAnnouncement.class); Assert.assertEquals(task.getId(), taskAnnouncement.getTaskStatus().getId()); Assert.assertEquals( TaskStatus.Status.RUNNING, taskAnnouncement.getTaskStatus().getStatusCode()); }
@Test public void testBasicFunctionality() throws Exception { // Check ZNode for timestamp exists (storage instantiation should create it) Stat zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE); assertEquals(zNodeStats.getVersion(), 0); // Initial checks assertEquals(storage.getMaxTimestamp(), INITIAL_MAX_TS_VALUE); byte[] data = zkClient.getData().forPath(TIMESTAMP_ZNODE); assertEquals(data.length, BYTES_IN_LONG); // Check new timestamp does not allow negative values... try { storage.updateMaxTimestamp(INITIAL_MAX_TS_VALUE, NEGATIVE_TS); fail(); } catch (IllegalArgumentException e) { LOG.info("Expected exception", e); } // ...nor is less than previous timestamp try { storage.updateMaxTimestamp(1, 0); fail(); } catch (IllegalArgumentException e) { LOG.info("Expected exception", e); } // Check that the original version is still there zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE); assertEquals(zNodeStats.getVersion(), 0); // Iterate updating the timestamp and check the final value long previousMaxTimestamp = INITIAL_MAX_TS_VALUE; for (int i = 0; i < ITERATION_COUNT; i++) { long newMaxTimestamp = previousMaxTimestamp + 1_000_000; storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp); previousMaxTimestamp = newMaxTimestamp; } assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT); // Check the znode version has changed accordingly zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE); assertEquals(zNodeStats.getVersion(), ITERATION_COUNT); // Check exceptions doThrow(new RuntimeException()).when(storageInternalZKClient).getData(); try { storage.getMaxTimestamp(); fail(); } catch (IOException e) { LOG.info("Expected exception", e); } doThrow(new RuntimeException()).when(storageInternalZKClient).setData(); try { storage.updateMaxTimestamp(INITIAL_MAX_TS_VALUE, INITIAL_MAX_TS_VALUE + 1_000_000); fail(); } catch (IOException e) { LOG.info("Expected exception", e); } // Reset the mock and double-check last result Mockito.reset(storageInternalZKClient); assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT); // Finally check the znode version is still the same zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE); assertEquals(zNodeStats.getVersion(), ITERATION_COUNT); }
public Map<String, Pair<Map<String, String>, AppInfo>> recoverYarnApp(String yarnAppId) throws Exception { Map<String, Pair<Map<String, String>, AppInfo>> result = new HashMap<>(); String path = this.zkRoot + "/" + yarnAppId; try { lock.acquire(); if (curator.checkExists().forPath(path) == null) { return result; } List<String> jobIds = curator.getChildren().forPath(path); if (jobIds.size() == 0) { LOG.info("delete empty path {}", path); delete(yarnAppId); } for (String jobId : jobIds) { String jobPath = path + "/" + jobId; LOG.info("recover path {}", jobPath); String fields = new String(curator.getData().forPath(jobPath), "UTF-8"); if (fields.length() == 0) { // LOG.info("delete empty path {}", jobPath); // delete(yarnAppId, jobId); continue; } JSONObject object = new JSONObject(fields); Map<String, Map<String, String>> parseResult = parse(object); Map<String, String> appInfoMap = parseResult.get(APP_INFO_KEY); AppInfo appInfo = new AppInfo(); appInfo.setId(appInfoMap.get("id")); appInfo.setUser(appInfoMap.get("user")); appInfo.setName(appInfoMap.get("name")); appInfo.setQueue(appInfoMap.get("queue")); appInfo.setState(appInfoMap.get("state")); appInfo.setFinalStatus(appInfoMap.get("finalStatus")); appInfo.setProgress(Double.parseDouble(appInfoMap.get("progress"))); appInfo.setTrackingUI(appInfoMap.get("trackingUI")); appInfo.setDiagnostics(appInfoMap.get("diagnostics")); appInfo.setTrackingUrl(appInfoMap.get("trackingUrl")); appInfo.setClusterId(appInfoMap.get("clusterId")); appInfo.setApplicationType(appInfoMap.get("applicationType")); appInfo.setStartedTime(Long.parseLong(appInfoMap.get("startedTime"))); appInfo.setFinishedTime(Long.parseLong(appInfoMap.get("finishedTime"))); appInfo.setElapsedTime(Long.parseLong(appInfoMap.get("elapsedTime"))); appInfo.setAmContainerLogs( appInfoMap.get("amContainerLogs") == null ? "" : appInfoMap.get("amContainerLogs")); appInfo.setAmHostHttpAddress( appInfoMap.get("amHostHttpAddress") == null ? "" : appInfoMap.get("amHostHttpAddress")); appInfo.setAllocatedMB(Long.parseLong(appInfoMap.get("allocatedMB"))); appInfo.setAllocatedVCores(Integer.parseInt(appInfoMap.get("allocatedVCores"))); appInfo.setRunningContainers(Integer.parseInt(appInfoMap.get("runningContainers"))); Map<String, String> tags = parseResult.get(ENTITY_TAGS_KEY); result.put(jobId, Pair.of(tags, appInfo)); } } catch (Exception e) { LOG.error("fail to recoverYarnApp", e); throw new RuntimeException(e); } finally { try { lock.release(); } catch (Exception e) { LOG.error("fail releasing lock", e); } } return result; }