public static long getOffsetFromZooKeeper(
CuratorFramework curatorClient, String groupId, String topic, int partition)
throws Exception {
ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
String path = topicDirs.consumerOffsetDir() + "/" + partition;
curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());
byte[] data = curatorClient.getData().forPath(path);
if (data == null) {
return OFFSET_NOT_SET;
} else {
String asString = new String(data);
if (asString.length() == 0) {
return OFFSET_NOT_SET;
} else {
try {
return Long.parseLong(asString);
} catch (NumberFormatException e) {
throw new Exception(
String.format(
"The offset in ZooKeeper for group '%s', topic '%s', partition %d is a malformed string: %s",
groupId, topic, partition, asString));
}
}
}
}
public static void setOffsetInZooKeeper(
CuratorFramework curatorClient, String groupId, String topic, int partition, long offset)
throws Exception {
ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
String path = topicDirs.consumerOffsetDir() + "/" + partition;
curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());
byte[] data = Long.toString(offset).getBytes();
curatorClient.setData().forPath(path, data);
}
/**
* Ensure the existence of the given path.
*
* @param client curator client
* @param path path to create, if needed
*/
public static void ensurePath(CuratorFramework client, String path) {
EnsurePath ensurePath = ensurePaths.get(path);
if (ensurePath == null) {
ensurePaths.putIfAbsent(path, client.newNamespaceAwareEnsurePath(path));
ensurePath = ensurePaths.get(path);
}
try {
ensurePath.ensure(client.getZookeeperClient());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
protected List<String> setupDefaultMaster(final int offset, String... args) throws Exception {
// TODO (dano): Move this bootstrapping to something reusable
final CuratorFramework curator = zk.curatorWithSuperAuth();
curator.newNamespaceAwareEnsurePath(Paths.configHosts()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.configJobs()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.configJobRefs()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.statusHosts()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.statusMasters()).ensure(curator.getZookeeperClient());
curator.newNamespaceAwareEnsurePath(Paths.historyJobs()).ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(Paths.configId(zkClusterId))
.ensure(curator.getZookeeperClient());
final List<String> argsList =
Lists.newArrayList(
"-vvvv",
"--no-log-setup",
"--http",
"http://0.0.0.0:" + (masterPort() + offset),
"--admin",
"http://0.0.0.0:" + (masterAdminPort() + offset),
"--domain",
"",
"--zk",
zk.connectString(),
"--zk-enable-acls",
"--zk-acl-agent-user",
AGENT_USER,
"--zk-acl-agent-digest",
AGENT_DIGEST,
"--zk-acl-master-user",
MASTER_USER,
"--zk-acl-master-password",
MASTER_PASSWORD);
final String name;
if (asList(args).contains("--name")) {
name = args[asList(args).indexOf("--name") + 1];
} else {
name = TEST_MASTER + offset;
argsList.addAll(asList("--name", TEST_MASTER));
}
final String stateDir = masterStateDirs.resolve(name).toString();
argsList.addAll(asList("--state-dir", stateDir));
argsList.addAll(asList(args));
return argsList;
}
@LifecycleStart
public void start() throws IOException {
synchronized (lock) {
if (started) {
return;
}
log.info("Starting zkCoordinator for server[%s]", me.getName());
final String loadQueueLocation = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName());
final String servedSegmentsLocation =
ZKPaths.makePath(zkPaths.getServedSegmentsPath(), me.getName());
final String liveSegmentsLocation =
ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), me.getName());
loadQueueCache = new PathChildrenCache(curator, loadQueueLocation, true, true, loadingExec);
try {
curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(servedSegmentsLocation)
.ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(liveSegmentsLocation)
.ensure(curator.getZookeeperClient());
loadLocalCache();
loadQueueCache
.getListenable()
.addListener(
new PathChildrenCacheListener() {
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event)
throws Exception {
final ChildData child = event.getData();
switch (event.getType()) {
case CHILD_ADDED:
final String path = child.getPath();
final DataSegmentChangeRequest request =
jsonMapper.readValue(child.getData(), DataSegmentChangeRequest.class);
log.info("New request[%s] with zNode[%s].", request.asString(), path);
try {
request.go(
getDataSegmentChangeHandler(),
new DataSegmentChangeCallback() {
boolean hasRun = false;
@Override
public void execute() {
try {
if (!hasRun) {
curator.delete().guaranteed().forPath(path);
log.info("Completed request [%s]", request.asString());
hasRun = true;
}
} catch (Exception e) {
try {
curator.delete().guaranteed().forPath(path);
} catch (Exception e1) {
log.error(
e1,
"Failed to delete zNode[%s], but ignoring exception.",
path);
}
log.error(e, "Exception while removing zNode[%s]", path);
throw Throwables.propagate(e);
}
}
});
} catch (Exception e) {
try {
curator.delete().guaranteed().forPath(path);
} catch (Exception e1) {
log.error(
e1, "Failed to delete zNode[%s], but ignoring exception.", path);
}
log.makeAlert(e, "Segment load/unload: uncaught exception.")
.addData("node", path)
.addData("nodeProperties", request)
.emit();
}
break;
case CHILD_REMOVED:
log.info("zNode[%s] was removed", event.getData().getPath());
break;
default:
log.info("Ignoring event[%s]", event);
}
}
});
loadQueueCache.start();
} catch (Exception e) {
Throwables.propagateIfPossible(e, IOException.class);
throw Throwables.propagate(e);
}
started = true;
}
}
@LifecycleStart
public void start() throws IOException {
log.info("Starting zkCoordinator for server[%s]", me);
synchronized (lock) {
if (started) {
return;
}
final String loadQueueLocation = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName());
final String servedSegmentsLocation =
ZKPaths.makePath(zkPaths.getServedSegmentsPath(), me.getName());
final String liveSegmentsLocation =
ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), me.getName());
loadQueueCache =
new PathChildrenCache(
curator,
loadQueueLocation,
true,
true,
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ZkCoordinator-%s").build());
try {
config.getInfoDir().mkdirs();
curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(servedSegmentsLocation)
.ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(liveSegmentsLocation)
.ensure(curator.getZookeeperClient());
loadCache();
loadQueueCache
.getListenable()
.addListener(
new PathChildrenCacheListener() {
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event)
throws Exception {
final ChildData child = event.getData();
switch (event.getType()) {
case CHILD_ADDED:
final String path = child.getPath();
final DataSegmentChangeRequest segment =
jsonMapper.readValue(child.getData(), DataSegmentChangeRequest.class);
log.info("New node[%s] with segmentClass[%s]", path, segment.getClass());
try {
segment.go(ZkCoordinator.this);
curator.delete().guaranteed().forPath(path);
log.info("Completed processing for node[%s]", path);
} catch (Exception e) {
try {
curator.delete().guaranteed().forPath(path);
} catch (Exception e1) {
log.info(
e1, "Failed to delete node[%s], but ignoring exception.", path);
}
log.makeAlert(e, "Segment load/unload: uncaught exception.")
.addData("node", path)
.addData("nodeProperties", segment)
.emit();
}
break;
case CHILD_REMOVED:
log.info("%s was removed", event.getData().getPath());
break;
default:
log.info("Ignoring event[%s]", event);
}
}
});
loadQueueCache.start();
} catch (Exception e) {
Throwables.propagateIfPossible(e, IOException.class);
throw Throwables.propagate(e);
}
started = true;
}
}
