public String createEphemeralSequential(final String path, final byte[] data) throws Exception {
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
try {
return zk.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case OPERATIONTIMEOUT:
LOG.warn("Possibly transient ZooKeeper exception: " + e);
if (!retryCounter.shouldRetry()) {
LOG.error(
"ZooKeeper create failed after " + retryCounter.getMaxRetries() + " retries");
throw e;
}
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
/**
* Utility - return true if the given exception is retry-able
*
* @param exception exception to check
* @return true/false
*/
public static boolean isRetryException(Throwable exception) {
if (exception instanceof KeeperException) {
KeeperException keeperException = (KeeperException) exception;
return shouldRetry(keeperException.code().intValue());
}
return false;
}
public byte[] getData(String path, Watcher watcher, Stat stat)
throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
try {
// 120227 by DaeJin Choi - What about Metadata of data ( ex: Magic
// number and so on )
return zk.getData(path, watcher, stat);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case OPERATIONTIMEOUT:
LOG.warn("Possibly transient ZooKeeper exception: " + e);
if (!retryCounter.shouldRetry()) {
LOG.error(
"ZooKeeper getData failed after " + retryCounter.getMaxRetries() + " retries");
throw e;
}
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
private String createSequential(String path, byte[] data, List<ACL> acl, CreateMode createMode)
throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
boolean first = true;
String newPath = path + this.identifier;
while (true) {
try {
if (!first) {
// Check if we succeeded on a previous attempt
String previousResult = findPreviousSequentialNode(newPath);
if (previousResult != null) {
return previousResult;
}
}
first = false;
return zk.create(newPath, data, acl, createMode);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "create");
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
public List<String> getChildren(String path) throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
try {
return zk.getChildren(path, true);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case OPERATIONTIMEOUT:
LOG.warn("Possibly transient ZooKeeper exception: " + e);
if (!retryCounter.shouldRetry()) {
LOG.error(
"ZooKeeper getChildren failed after "
+ retryCounter.getMaxRetries()
+ " retries");
throw e;
}
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
public void createPersistent(final String path, final byte[] data)
throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
try {
zk.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
return;
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case OPERATIONTIMEOUT:
LOG.warn("Possibly transient ZooKeeper exception: " + e);
if (!retryCounter.shouldRetry()) {
LOG.error(
"ZooKeeper create failed after " + retryCounter.getMaxRetries() + " retries");
throw e;
}
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
/**
* Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper
*
* @param conf configuration for the instance
* @param instanceId instanceId for the instance
* @param hostAddress Address that monitor process is bound to
*/
public static void startLogListener(
AccumuloConfiguration conf, String instanceId, String hostAddress) {
try {
SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT));
// getLocalPort will return the actual ephemeral port used when '0' was provided.
String logForwardingAddr = hostAddress + ":" + server.getLocalPort();
log.debug("Setting monitor log4j log-forwarding address to: " + logForwardingAddr);
final String path = ZooUtil.getRoot(instanceId) + Constants.ZMONITOR_LOG4J_ADDR;
final ZooReaderWriter zoo = ZooReaderWriter.getInstance();
// Delete before we try to re-create in case the previous session hasn't yet expired
try {
zoo.delete(path, -1);
} catch (KeeperException e) {
// We don't care if the node is already gone
if (!KeeperException.Code.NONODE.equals(e.code())) {
throw e;
}
}
zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8));
new Daemon(server).start();
} catch (Throwable t) {
log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t);
}
}
/**
* Try to own the task by transitioning the zk node data from UNASSIGNED to OWNED.
*
* <p>This method is also used to periodically heartbeat the task progress by transitioning the
* node from OWNED to OWNED.
*
* <p>
*
* @param isFirstTime shows whther it's the first attempt.
* @param zkw zk wathcer
* @param server name
* @param task to own
* @param taskZKVersion version of the task in zk
* @return non-negative integer value when task can be owned by current region server otherwise -1
*/
protected static int attemptToOwnTask(
boolean isFirstTime,
ZooKeeperWatcher zkw,
ServerName server,
String task,
RecoveryMode mode,
int taskZKVersion) {
int latestZKVersion = FAILED_TO_OWN_TASK;
try {
SplitLogTask slt = new SplitLogTask.Owned(server, mode);
Stat stat = zkw.getRecoverableZooKeeper().setData(task, slt.toByteArray(), taskZKVersion);
if (stat == null) {
LOG.warn("zk.setData() returned null for path " + task);
SplitLogCounters.tot_wkr_task_heartbeat_failed.incrementAndGet();
return FAILED_TO_OWN_TASK;
}
latestZKVersion = stat.getVersion();
SplitLogCounters.tot_wkr_task_heartbeat.incrementAndGet();
return latestZKVersion;
} catch (KeeperException e) {
if (!isFirstTime) {
if (e.code().equals(KeeperException.Code.NONODE)) {
LOG.warn("NONODE failed to assert ownership for " + task, e);
} else if (e.code().equals(KeeperException.Code.BADVERSION)) {
LOG.warn("BADVERSION failed to assert ownership for " + task, e);
} else {
LOG.warn("failed to assert ownership for " + task, e);
}
}
} catch (InterruptedException e1) {
LOG.warn(
"Interrupted while trying to assert ownership of "
+ task
+ " "
+ StringUtils.stringifyException(e1));
Thread.currentThread().interrupt();
}
SplitLogCounters.tot_wkr_task_heartbeat_failed.incrementAndGet();
return FAILED_TO_OWN_TASK;
}
protected KeeperException.Code createFromByte(String path, byte[] data, CreateMode mode) {
try {
zooKeeper.create(path, data, acl, mode);
} catch (KeeperException e) {
return e.code();
} catch (Exception e) {
return KeeperException.Code.SYSTEMERROR;
}
return KeeperException.Code.OK;
}
public static boolean createIfNotExists(
ZooKeeperConnection zooKeeper, String path, byte[] value, CreateMode createMode)
throws KeeperException, InterruptedException {
if (zooKeeper.exists(path, false) == null) {
try {
zooKeeper.create(path, value, ZooDefs.Ids.OPEN_ACL_UNSAFE, createMode);
} catch (KeeperException e) {
if (e.code() != KeeperException.Code.NODEEXISTS) throw e;
return false;
}
return true;
}
return false;
}
private List<ReplicationPeer> listValidReplicationPeers() {
Map<String, ReplicationPeerConfig> peers = listPeerConfigs();
if (peers == null || peers.size() <= 0) {
return null;
}
List<ReplicationPeer> validPeers = new ArrayList<ReplicationPeer>(peers.size());
for (Entry<String, ReplicationPeerConfig> peerEntry : peers.entrySet()) {
String peerId = peerEntry.getKey();
String clusterKey = peerEntry.getValue().getClusterKey();
Configuration peerConf = new Configuration(this.connection.getConfiguration());
Stat s = null;
try {
ZKUtil.applyClusterKeyToConf(peerConf, clusterKey);
Pair<ReplicationPeerConfig, Configuration> pair = this.replicationPeers.getPeerConf(peerId);
ReplicationPeer peer = new ReplicationPeerZKImpl(peerConf, peerId, pair.getFirst());
s =
zkw.getRecoverableZooKeeper()
.exists(peerConf.get(HConstants.ZOOKEEPER_ZNODE_PARENT), null);
if (null == s) {
LOG.info(peerId + ' ' + clusterKey + " is invalid now.");
continue;
}
validPeers.add(peer);
} catch (ReplicationException e) {
LOG.warn(
"Failed to get valid replication peers. "
+ "Error connecting to peer cluster with peerId="
+ peerId);
LOG.debug("Failure details to get valid replication peers.", e);
continue;
} catch (KeeperException e) {
LOG.warn(
"Failed to get valid replication peers. KeeperException code=" + e.code().intValue());
LOG.debug("Failure details to get valid replication peers.", e);
continue;
} catch (InterruptedException e) {
LOG.warn("Failed to get valid replication peers due to InterruptedException.");
LOG.debug("Failure details to get valid replication peers.", e);
Thread.currentThread().interrupt();
continue;
} catch (IOException e) {
LOG.warn("Failed to get valid replication peers due to IOException.");
LOG.debug("Failure details to get valid replication peers.", e);
continue;
}
}
return validPeers;
}
private String createNonSequential(String path, byte[] data, List<ACL> acl, CreateMode createMode)
throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
boolean isRetry = false; // False for first attempt, true for all retries.
while (true) {
try {
return zk.create(path, data, acl, createMode);
} catch (KeeperException e) {
switch (e.code()) {
case NODEEXISTS:
if (isRetry) {
// If the connection was lost, there is still a possibility that
// we have successfully created the node at our previous attempt,
// so we read the node and compare.
byte[] currentData = zk.getData(path, false, null);
if (currentData != null && Bytes.compareTo(currentData, data) == 0) {
// We successfully created a non-sequential node
return path;
}
LOG.error(
"Node "
+ path
+ " already exists with "
+ Bytes.toStringBinary(currentData)
+ ", could not write "
+ Bytes.toStringBinary(data));
throw e;
}
LOG.info("Node " + path + " already exists and this is not a " + "retry");
throw e;
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "create");
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
isRetry = true;
}
}
protected KeeperException.Code create(String path, String data, CreateMode mode) {
try {
byte[] byteData = null;
if (data != null) {
byteData = data.getBytes();
}
zooKeeper.create(path, byteData, acl, mode);
} catch (KeeperException e) {
return e.code();
} catch (Exception e) {
return KeeperException.Code.SYSTEMERROR;
}
return KeeperException.Code.OK;
}
/** Get metrics data for this service node (self) for current interval. */
public MetricsData getMyMetrics(String clusterId, String serviceId) {
String key = clusterId + "/" + serviceId + "/" + getContext().getZkNodeId().getPathToken();
ExportMeta exportMeta = exportPathMap.get(key);
if (exportMeta == null) {
logger.trace(
"MetricsData not found: data has not been exported: clusterId={}; serviceId={}; exportMeta={}",
clusterId,
serviceId,
exportMeta);
return null;
}
if (exportMeta.dataPath == null) {
logger.trace(
"MetricsData not found: waiting for data to be reported in ZK: clusterId={}; serviceId={}; exportMeta.dataPath={}",
clusterId,
serviceId,
exportMeta.dataPath);
synchronized (exportMeta) {
try {
exportMeta.wait();
} catch (InterruptedException e) {
logger.warn("Interrupted while waiting: " + e, e);
}
}
}
try {
logger.debug("Retrieving metrics: path={}", exportMeta.dataPath);
Stat stat = new Stat();
byte[] bytes = getContext().getZkClient().getData(exportMeta.dataPath, true, stat);
MetricsData metricsData = JacksonUtil.getObjectMapper().readValue(bytes, MetricsData.class);
metricsData.setClusterId(clusterId);
metricsData.setServiceId(serviceId);
metricsData.setLastUpdatedTimestamp(stat.getMtime());
return metricsData;
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.NONODE) {
return null;
}
throw new ReignException(e);
} catch (Exception e) {
throw new ReignException(e);
}
}
@Override
public void dropUser(String user) throws AccumuloSecurityException {
try {
synchronized (zooCache) {
zooCache.clear();
ZooReaderWriter.getInstance()
.recursiveDelete(ZKUserPath + "/" + user, NodeMissingPolicy.FAIL);
}
} catch (InterruptedException e) {
log.error("{}", e.getMessage(), e);
throw new RuntimeException(e);
} catch (KeeperException e) {
if (e.code().equals(KeeperException.Code.NONODE)) {
throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
}
log.error("{}", e.getMessage(), e);
throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
}
}
@Override
public void createUser(String principal, AuthenticationToken token)
throws AccumuloSecurityException {
try {
if (!(token instanceof PasswordToken))
throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
PasswordToken pt = (PasswordToken) token;
constructUser(principal, ZKSecurityTool.createPass(pt.getPassword()));
} catch (KeeperException e) {
if (e.code().equals(KeeperException.Code.NODEEXISTS))
throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_EXISTS, e);
throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
} catch (InterruptedException e) {
log.error("{}", e.getMessage(), e);
throw new RuntimeException(e);
} catch (AccumuloException e) {
log.error("{}", e.getMessage(), e);
throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
}
}
/**
* exists is an idempotent operation. Retry before throwing exception
*
* @return A Stat instance
*/
public Stat exists(String path, boolean watch) throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
try {
return zk.exists(path, watch);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "exists");
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
/** Simply queue the request, which will be processed in FIFO order. */
public void processRequest(Request request) {
if (!finished) {
Request upgradeRequest = null;
try {
upgradeRequest = zks.checkUpgradeSession(request);
} catch (KeeperException ke) {
if (request.getHdr() != null) {
request.getHdr().setType(OpCode.error);
request.setTxn(new ErrorTxn(ke.code().intValue()));
}
request.setException(ke);
LOG.info("Error creating upgrade request", ke);
} catch (IOException ie) {
LOG.error("Unexpected error in upgrade", ie);
}
if (upgradeRequest != null) {
queuedRequests.add(upgradeRequest);
}
queuedRequests.add(request);
}
}
/**
* setData is NOT an idempotent operation. Retry may cause BadVersion Exception Adding an
* identifier field into the data to check whether badversion is caused by the result of previous
* correctly setData
*
* @return Stat instance
*/
public Stat setData(String path, byte[] data, int version)
throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
byte[] newData = appendMetaData(data);
boolean isRetry = false;
while (true) {
try {
return zk.setData(path, newData, version);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "setData");
break;
case BADVERSION:
if (isRetry) {
// try to verify whether the previous setData success or not
try {
Stat stat = new Stat();
byte[] revData = zk.getData(path, false, stat);
if (Bytes.compareTo(revData, newData) == 0) {
// the bad version is caused by previous successful setData
return stat;
}
} catch (KeeperException keeperException) {
// the ZK is not reliable at this moment. just throwing exception
throw keeperException;
}
}
// throw other exceptions and verified bad version exceptions
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
isRetry = true;
}
}
/**
* getChildren is an idempotent operation. Retry before throwing exception
*
* @return List of children znodes
*/
public List<String> getChildren(String path, Watcher watcher)
throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
try {
return zk.getChildren(path, watcher);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "getChildren");
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
/** Run multiple operations in a transactional manner. Retry before throwing exception */
public List<OpResult> multi(Iterable<Op> ops) throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
Iterable<Op> multiOps = prepareZKMulti(ops);
while (true) {
try {
return zk.multi(multiOps);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "multi");
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
/**
* getData is an idemnpotent operation. Retry before throwing exception
*
* @return Data
*/
public byte[] getData(String path, boolean watch, Stat stat)
throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
try {
byte[] revData = zk.getData(path, watch, stat);
return this.removeMetaData(revData);
} catch (KeeperException e) {
switch (e.code()) {
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "getData");
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
/**
* delete is an idempotent operation. Retry before throwing exception. This function will not
* throw NoNodeException if the path does not exist.
*/
public void delete(String path, int version) throws InterruptedException, KeeperException {
RetryCounter retryCounter = retryCounterFactory.create();
boolean isRetry = false; // False for first attempt, true for all retries.
while (true) {
try {
zk.delete(path, version);
return;
} catch (KeeperException e) {
switch (e.code()) {
case NONODE:
if (isRetry) {
LOG.info(
"Node "
+ path
+ " already deleted. Assuming that a "
+ "previous attempt succeeded.");
return;
}
LOG.warn("Node " + path + " already deleted, and this is not a " + "retry");
throw e;
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "delete");
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
isRetry = true;
}
}
/**
* This method will be called inside the ProcessRequestThread, which is a singleton, so there will
* be a single thread calling this code.
*
* @param request
*/
@SuppressWarnings("unchecked")
protected void pRequest(Request request) throws RequestProcessorException {
// LOG.info("Prep>>> cxid = " + request.cxid + " type = " +
// request.type + " id = 0x" + Long.toHexString(request.sessionId));
TxnHeader txnHeader = null;
Record txn = null;
try {
switch (request.type) {
case OpCode.create:
txnHeader =
new TxnHeader(
request.sessionId, request.cxid, zks.getNextZxid(), zks.getTime(), OpCode.create);
zks.sessionTracker.checkSession(request.sessionId, request.getOwner());
CreateRequest createRequest = new CreateRequest();
ZooKeeperServer.byteBuffer2Record(request.request, createRequest);
String path = createRequest.getPath();
int lastSlash = path.lastIndexOf('/');
if (lastSlash == -1 || path.indexOf('\0') != -1 || failCreate) {
LOG.info(
"Invalid path " + path + " with session 0x" + Long.toHexString(request.sessionId));
throw new KeeperException.BadArgumentsException(path);
}
if (!fixupACL(request.authInfo, createRequest.getAcl())) {
throw new KeeperException.InvalidACLException(path);
}
String parentPath = path.substring(0, lastSlash);
ChangeRecord parentRecord = getRecordForPath(parentPath);
checkACL(zks, parentRecord.acl, ZooDefs.Perms.CREATE, request.authInfo);
int parentCVersion = parentRecord.stat.getCversion();
CreateMode createMode = CreateMode.fromFlag(createRequest.getFlags());
if (createMode.isSequential()) {
path = path + String.format(Locale.ENGLISH, "%010d", parentCVersion);
}
try {
PathUtils.validatePath(path);
} catch (IllegalArgumentException ie) {
LOG.info(
"Invalid path " + path + " with session 0x" + Long.toHexString(request.sessionId));
throw new KeeperException.BadArgumentsException(path);
}
try {
if (getRecordForPath(path) != null) {
throw new KeeperException.NodeExistsException(path);
}
} catch (KeeperException.NoNodeException e) {
// ignore this one
}
boolean ephemeralParent = parentRecord.stat.getEphemeralOwner() != 0;
if (ephemeralParent) {
throw new KeeperException.NoChildrenForEphemeralsException(path);
}
txn =
new CreateTxn(
path, createRequest.getData(), createRequest.getAcl(), createMode.isEphemeral());
StatPersisted s = new StatPersisted();
if (createMode.isEphemeral()) {
s.setEphemeralOwner(request.sessionId);
}
parentRecord = parentRecord.duplicate(txnHeader.getZxid());
parentRecord.childCount++;
parentRecord.stat.setCversion(parentRecord.stat.getCversion() + 1);
addChangeRecord(parentRecord);
addChangeRecord(
new ChangeRecord(txnHeader.getZxid(), path, s, 0, createRequest.getAcl()));
break;
case OpCode.delete:
txnHeader =
new TxnHeader(
request.sessionId, request.cxid, zks.getNextZxid(), zks.getTime(), OpCode.delete);
zks.sessionTracker.checkSession(request.sessionId, request.getOwner());
DeleteRequest deleteRequest = new DeleteRequest();
ZooKeeperServer.byteBuffer2Record(request.request, deleteRequest);
path = deleteRequest.getPath();
lastSlash = path.lastIndexOf('/');
if (lastSlash == -1
|| path.indexOf('\0') != -1
|| zks.getZKDatabase().isSpecialPath(path)) {
throw new KeeperException.BadArgumentsException(path);
}
parentPath = path.substring(0, lastSlash);
parentRecord = getRecordForPath(parentPath);
ChangeRecord nodeRecord = getRecordForPath(path);
checkACL(zks, parentRecord.acl, ZooDefs.Perms.DELETE, request.authInfo);
int version = deleteRequest.getVersion();
if (version != -1 && nodeRecord.stat.getVersion() != version) {
throw new KeeperException.BadVersionException(path);
}
if (nodeRecord.childCount > 0) {
throw new KeeperException.NotEmptyException(path);
}
txn = new DeleteTxn(path);
parentRecord = parentRecord.duplicate(txnHeader.getZxid());
parentRecord.childCount--;
parentRecord.stat.setCversion(parentRecord.stat.getCversion() + 1);
addChangeRecord(parentRecord);
addChangeRecord(new ChangeRecord(txnHeader.getZxid(), path, null, -1, null));
break;
case OpCode.setData:
txnHeader =
new TxnHeader(
request.sessionId,
request.cxid,
zks.getNextZxid(),
zks.getTime(),
OpCode.setData);
zks.sessionTracker.checkSession(request.sessionId, request.getOwner());
SetDataRequest setDataRequest = new SetDataRequest();
ZooKeeperServer.byteBuffer2Record(request.request, setDataRequest);
path = setDataRequest.getPath();
nodeRecord = getRecordForPath(path);
checkACL(zks, nodeRecord.acl, ZooDefs.Perms.WRITE, request.authInfo);
version = setDataRequest.getVersion();
int currentVersion = nodeRecord.stat.getVersion();
if (version != -1 && version != currentVersion) {
throw new KeeperException.BadVersionException(path);
}
version = currentVersion + 1;
txn = new SetDataTxn(path, setDataRequest.getData(), version);
nodeRecord = nodeRecord.duplicate(txnHeader.getZxid());
nodeRecord.stat.setVersion(version);
addChangeRecord(nodeRecord);
break;
case OpCode.setACL:
txnHeader =
new TxnHeader(
request.sessionId, request.cxid, zks.getNextZxid(), zks.getTime(), OpCode.setACL);
zks.sessionTracker.checkSession(request.sessionId, request.getOwner());
SetACLRequest setAclRequest = new SetACLRequest();
ZooKeeperServer.byteBuffer2Record(request.request, setAclRequest);
path = setAclRequest.getPath();
if (!fixupACL(request.authInfo, setAclRequest.getAcl())) {
throw new KeeperException.InvalidACLException(path);
}
nodeRecord = getRecordForPath(path);
checkACL(zks, nodeRecord.acl, ZooDefs.Perms.ADMIN, request.authInfo);
version = setAclRequest.getVersion();
currentVersion = nodeRecord.stat.getAversion();
if (version != -1 && version != currentVersion) {
throw new KeeperException.BadVersionException(path);
}
version = currentVersion + 1;
txn = new SetACLTxn(path, setAclRequest.getAcl(), version);
nodeRecord = nodeRecord.duplicate(txnHeader.getZxid());
nodeRecord.stat.setAversion(version);
addChangeRecord(nodeRecord);
break;
case OpCode.createSession:
txnHeader =
new TxnHeader(
request.sessionId,
request.cxid,
zks.getNextZxid(),
zks.getTime(),
OpCode.createSession);
request.request.rewind();
int to = request.request.getInt();
txn = new CreateSessionTxn(to);
request.request.rewind();
zks.sessionTracker.addSession(request.sessionId, to);
zks.setOwner(request.sessionId, request.getOwner());
break;
case OpCode.closeSession:
txnHeader =
new TxnHeader(
request.sessionId,
request.cxid,
zks.getNextZxid(),
zks.getTime(),
OpCode.closeSession);
// We don't want to do this check since the session expiration thread
// queues up this operation without being the session owner.
// this request is the last of the session so it should be ok
// zks.sessionTracker.checkSession(request.sessionId, request.getOwner());
HashSet<String> es = zks.getZKDatabase().getEphemerals(request.sessionId);
synchronized (zks.outstandingChanges) {
for (ChangeRecord c : zks.outstandingChanges) {
if (c.stat == null) {
// Doing a delete
es.remove(c.path);
} else if (c.stat.getEphemeralOwner() == request.sessionId) {
es.add(c.path);
}
}
for (String path2Delete : es) {
addChangeRecord(new ChangeRecord(txnHeader.getZxid(), path2Delete, null, 0, null));
}
zks.sessionTracker.setSessionClosing(request.sessionId);
}
LOG.info(
"Processed session termination for sessionid: 0x"
+ Long.toHexString(request.sessionId));
break;
case OpCode.sync:
case OpCode.exists:
case OpCode.getData:
case OpCode.getACL:
case OpCode.getChildren:
case OpCode.getChildren2:
case OpCode.ping:
case OpCode.setWatches:
zks.sessionTracker.checkSession(request.sessionId, request.getOwner());
break;
}
} catch (KeeperException e) {
if (txnHeader != null) {
txnHeader.setType(OpCode.error);
txn = new ErrorTxn(e.code().intValue());
}
LOG.info(
"Got user-level KeeperException when processing "
+ request.toString()
+ " Error Path:"
+ e.getPath()
+ " Error:"
+ e.getMessage());
request.setException(e);
} catch (Exception e) {
// log at error level as we are returning a marshalling
// error to the user
LOG.error("Failed to process " + request, e);
StringBuilder sb = new StringBuilder();
ByteBuffer bb = request.request;
if (bb != null) {
bb.rewind();
while (bb.hasRemaining()) {
sb.append(Integer.toHexString(bb.get() & 0xff));
}
} else {
sb.append("request buffer is null");
}
LOG.error("Dumping request buffer: 0x" + sb.toString());
if (txnHeader != null) {
txnHeader.setType(OpCode.error);
txn = new ErrorTxn(Code.MARSHALLINGERROR.intValue());
}
}
request.hdr = txnHeader;
request.txn = txn;
request.zxid = zks.getZxid();
nextProcessor.processRequest(request);
}
MetricsData getMetricsFromDataNode(String clusterId, String serviceId, String dataNode) {
PathScheme pathScheme = getContext().getPathScheme();
String dataPath = null;
if (dataNode != null) {
dataPath =
pathScheme.getAbsolutePath(
PathType.METRICS, pathScheme.joinTokens(clusterId, serviceId, dataNode));
} else {
dataPath =
pathScheme.getAbsolutePath(PathType.METRICS, pathScheme.joinTokens(clusterId, serviceId));
}
byte[] bytes = null;
try {
Stat stat = new Stat();
bytes = getContext().getZkClient().getData(dataPath, true, stat);
MetricsData metricsData = JacksonUtil.getObjectMapper().readValue(bytes, MetricsData.class);
metricsData.setLastUpdatedTimestamp(stat.getMtime());
return metricsData;
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.NONODE) {
return null;
}
logger.warn(
"Error retrieving data node: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ "; dataPath="
+ dataPath
+ "; dataAsString="
+ (new String(bytes, UTF_8))
+ ": "
+ e,
e);
throw new ReignException(
"Error retrieving data node: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ "; dataPath="
+ dataPath
+ "; dataAsString="
+ (new String(bytes, UTF_8)),
e);
} catch (UnrecognizedPropertyException e) {
logger.warn(
"Error retrieving data node: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ "; dataPath="
+ dataPath
+ "; dataAsString="
+ (new String(bytes, UTF_8))
+ ": "
+ e,
e);
return null;
} catch (Exception e) {
logger.warn(
"Error retrieving data node: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ "; dataPath="
+ dataPath
+ "; dataAsString="
+ (new String(bytes, UTF_8))
+ ": "
+ e,
e);
throw new ReignException(
"Error retrieving data node: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ "; dataPath="
+ dataPath
+ "; dataAsString="
+ (new String(bytes, UTF_8)),
e);
}
}
@Override
public ResponseMessage handleMessage(RequestMessage requestMessage) {
ResponseMessage responseMessage = new SimpleResponseMessage();
try {
if (logger.isTraceEnabled()) {
logger.trace(
"Received message: nodeId={}; request='{}:{}'",
requestMessage.getSenderId(),
requestMessage.getTargetService(),
requestMessage.getBody());
}
/** preprocess request * */
ParsedRequestMessage parsedRequestMessage = new ParsedRequestMessage(requestMessage);
String resource = parsedRequestMessage.getResource();
// strip beginning and ending slashes "/"
boolean endsWithSlash = false;
if (resource.startsWith("/")) {
resource = resource.substring(1);
}
if (resource.endsWith("/")) {
endsWithSlash = true;
resource = resource.substring(0, resource.length() - 1);
}
/** get response * */
if ("observe".equals(parsedRequestMessage.getMeta())) {
responseMessage = new SimpleResponseMessage(ResponseStatus.OK);
String[] tokens = getPathScheme().tokenizePath(resource);
if (tokens.length == 2) {
this.observe(
tokens[0],
tokens[1],
this.getClientObserver(
parsedRequestMessage.getSenderId(), tokens[0], tokens[1], null));
} else if (tokens.length == 3) {
this.observe(
tokens[0],
tokens[1],
this.getClientObserver(
parsedRequestMessage.getSenderId(), tokens[0], tokens[1], tokens[2]));
} else {
responseMessage.setComment("Observing not supported: " + resource);
}
} else if ("observe-stop".equals(parsedRequestMessage.getMeta())) {
responseMessage = new SimpleResponseMessage(ResponseStatus.OK);
String absolutePath = getPathScheme().getAbsolutePath(PathType.METRICS, resource);
getContext()
.getObserverManager()
.removeByOwnerId(parsedRequestMessage.getSenderId().toString(), absolutePath);
} else {
if (resource.length() == 0) {
// list available clusters
String path = getContext().getPathScheme().getAbsolutePath(PathType.METRICS);
List<String> clusterList = getContext().getZkClient().getChildren(path, false);
responseMessage.setBody(clusterList);
} else {
String[] tokens = getPathScheme().tokenizePath(resource);
// logger.debug("tokens.length={}", tokens.length);
if (tokens.length == 1) {
// list available services
String path = getContext().getPathScheme().getAbsolutePath(PathType.METRICS, tokens[0]);
List<String> serviceList = getContext().getZkClient().getChildren(path, false);
responseMessage.setBody(serviceList);
if (serviceList == null) {
responseMessage.setComment("Not found: " + resource);
}
} else if (tokens.length == 2) {
if (endsWithSlash) {
// list available nodes for a given service
String path =
getContext()
.getPathScheme()
.getAbsolutePath(PathType.METRICS, tokens[0], tokens[1]);
List<String> nodeList = getContext().getZkClient().getChildren(path, false);
responseMessage.setBody(nodeList);
if (nodeList == null) {
responseMessage.setComment("Not found: " + resource);
}
} else {
// get metrics data for service
MetricsData metricsData = getMetricsFromDataNode(tokens[0], tokens[1], null);
if (metricsData == null) {
responseMessage.setComment("Not found: " + resource);
} else {
responseMessage.setBody(metricsData);
}
}
} else if (tokens.length == 3) {
// get metrics data for single data node
MetricsData metricsData = getMetricsFromDataNode(tokens[0], tokens[1], tokens[2]);
if (metricsData == null) {
responseMessage.setComment("Not found: " + resource);
} else {
responseMessage.setBody(metricsData);
}
}
}
} // if observe
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.NONODE) {
responseMessage.setBody(Collections.EMPTY_LIST);
} else {
responseMessage.setStatus(ResponseStatus.ERROR_UNEXPECTED, "" + e);
}
} catch (Exception e) {
logger.error("" + e, e);
responseMessage.setStatus(ResponseStatus.ERROR_UNEXPECTED, "" + e);
}
responseMessage.setId(requestMessage.getId());
return responseMessage;
}
@Override
public void run() {
long startTimeNanos = System.nanoTime();
logger.trace("AggregationRunnable starting: hashCode={}", this.hashCode());
// list all services in cluster
PresenceService presenceService = getContext().getService("presence");
CoordinationService coordinationService = getContext().getService("coord");
ZkClient zkClient = getContext().getZkClient();
PathScheme pathScheme = getContext().getPathScheme();
// list all services in cluster
List<String> clusterIds = presenceService.getClusters();
for (String clusterId : clusterIds) {
// only proceed if in cluster
if (!presenceService.isMemberOf(clusterId)
|| clusterId.equals(getContext().getPathScheme().getFrameworkClusterId())) {
continue;
}
List<String> allServiceIds = presenceService.getServices(clusterId);
List<String> memberServiceIds = new ArrayList<String>(allServiceIds.size());
for (String serviceId : allServiceIds) {
// only aggregate if node is in service
if (presenceService.isMemberOf(clusterId, serviceId)) {
memberServiceIds.add(serviceId);
}
}
// go through member service list in deterministic order so
// locks are acquired in the same order across
// nodes
Collections.sort(memberServiceIds);
for (int i = 0; i < memberServiceIds.size(); i++) {
long currentTimestamp = System.currentTimeMillis();
String serviceId = memberServiceIds.get(i);
logger.trace("Finding data nodes: clusterId={}; serviceId={}", clusterId, serviceId);
// get lock for a service
DistributedLock lock =
coordinationService.getLock("reign", "metrics-" + clusterId + "-" + serviceId);
if (!lock.tryLock()) {
continue;
}
try {
// get all data nodes for a service
String dataParentPath =
pathScheme.getAbsolutePath(
PathType.METRICS, pathScheme.joinTokens(clusterId, serviceId));
List<String> dataNodes = zkClient.getChildren(dataParentPath, false);
/** iterate through service data nodes and gather up data to aggregate */
Map<String, List<CounterData>> counterMap =
new HashMap<String, List<CounterData>>(dataNodes.size() + 1, 1.0f);
Map<String, List<GaugeData>> gaugeMap =
new HashMap<String, List<GaugeData>>(dataNodes.size() + 1, 1.0f);
Map<String, List<HistogramData>> histogramMap =
new HashMap<String, List<HistogramData>>(dataNodes.size() + 1, 1.0f);
Map<String, List<MeterData>> meterMap =
new HashMap<String, List<MeterData>>(dataNodes.size() + 1, 1.0f);
Map<String, List<TimerData>> timerMap =
new HashMap<String, List<TimerData>>(dataNodes.size() + 1, 1.0f);
int dataNodeCount = 0;
int dataNodeInWindowCount = 0;
Integer intervalLength = null;
TimeUnit intervalLengthUnit = null;
for (String dataNode : dataNodes) {
dataNodeCount++;
logger.trace(
"Found data node: clusterId={}; serviceId={}; nodeId={}",
clusterId,
serviceId,
dataNode);
String dataPath = null;
MetricsData metricsData = null;
dataPath =
pathScheme.getAbsolutePath(
PathType.METRICS, pathScheme.joinTokens(clusterId, serviceId, dataNode));
try {
metricsData = getMetricsFromDataNode(clusterId, serviceId, dataNode);
if (metricsData == null) {
continue;
}
} catch (Exception e) {
logger.warn(
"Error trying to aggregate data directory for service: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ ": "
+ e,
e);
continue;
}
// skip data node if not within interval
long millisToExpiry = millisToExpiry(metricsData, currentTimestamp);
if (millisToExpiry <= 0) {
continue;
}
intervalLength = metricsData.getIntervalLength();
intervalLengthUnit = metricsData.getIntervalLengthUnit();
// aggregate service stats for data nodes that
// within current rotation interval
logger.trace(
"Aggregating data node: path={}; millisToExpiry={}", dataPath, millisToExpiry);
// increment node count
dataNodeInWindowCount++;
// counters
Map<String, CounterData> counters = metricsData.getCounters();
for (String key : counters.keySet()) {
CounterData counter = counters.get(key);
List<CounterData> counterList = counterMap.get(key);
if (counterList == null) {
counterList = new ArrayList<CounterData>(dataNodes.size());
counterMap.put(key, counterList);
}
counterList.add(counter);
}
// gauges
Map<String, GaugeData> gauges = metricsData.getGauges();
for (String key : gauges.keySet()) {
GaugeData gauge = gauges.get(key);
List<GaugeData> gaugeList = gaugeMap.get(key);
if (gaugeList == null) {
gaugeList = new ArrayList<GaugeData>(dataNodes.size());
gaugeMap.put(key, gaugeList);
}
gaugeList.add(gauge);
}
// histogram
Map<String, HistogramData> histograms = metricsData.getHistograms();
for (String key : histograms.keySet()) {
HistogramData histogram = histograms.get(key);
List<HistogramData> histogramList = histogramMap.get(key);
if (histogramList == null) {
histogramList = new ArrayList<HistogramData>(dataNodes.size());
histogramMap.put(key, histogramList);
}
histogramList.add(histogram);
}
// meters
Map<String, MeterData> meters = metricsData.getMeters();
for (String key : meters.keySet()) {
MeterData meter = meters.get(key);
List<MeterData> meterList = meterMap.get(key);
if (meterList == null) {
meterList = new ArrayList<MeterData>(dataNodes.size());
meterMap.put(key, meterList);
}
meterList.add(meter);
}
// timers
Map<String, TimerData> timers = metricsData.getTimers();
for (String key : timers.keySet()) {
TimerData timer = timers.get(key);
List<TimerData> meterList = timerMap.get(key);
if (meterList == null) {
meterList = new ArrayList<TimerData>(dataNodes.size());
timerMap.put(key, meterList);
}
meterList.add(timer);
}
} // for dataNodes
/** aggregate data and write to ZK * */
MetricsData serviceMetricsData = new MetricsData();
// counters
Map<String, CounterData> counters =
new HashMap<String, CounterData>(counterMap.size() + 1, 1.0f);
for (String key : counterMap.keySet()) {
List<CounterData> counterList = counterMap.get(key);
// if (counterList.size() != dataNodeCount) {
// logger.warn(
// "counterList size does not match nodeCount: counterList.size={}; nodeCount={}",
// counterList.size(), dataNodeCount);
// }
CounterData counterData = CounterData.merge(counterList);
counters.put(key, counterData);
}
serviceMetricsData.setCounters(counters);
// gauges
Map<String, GaugeData> gauges =
new HashMap<String, GaugeData>(gaugeMap.size() + 1, 1.0f);
for (String key : gaugeMap.keySet()) {
List<GaugeData> gaugeList = gaugeMap.get(key);
// if (gaugeList.size() != dataNodeCount) {
// logger.warn(
// "gaugeList size does not match nodeCount: gaugeList.size={}; nodeCount={}",
// gaugeList.size(), dataNodeCount);
// }
GaugeData gaugeData = GaugeData.merge(gaugeList);
gauges.put(key, gaugeData);
}
serviceMetricsData.setGauges(gauges);
// histograms
Map<String, HistogramData> histograms =
new HashMap<String, HistogramData>(histogramMap.size() + 1, 1.0f);
for (String key : histogramMap.keySet()) {
List<HistogramData> histogramList = histogramMap.get(key);
// if (histogramList.size() != dataNodeCount) {
// logger.warn(
// "histogramList size does not match nodeCount: histogramList.size={};
// nodeCount={}",
// histogramList.size(), dataNodeCount);
// }
HistogramData histogramData = HistogramData.merge(histogramList);
histograms.put(key, histogramData);
}
serviceMetricsData.setHistograms(histograms);
// meters
Map<String, MeterData> meters =
new HashMap<String, MeterData>(meterMap.size() + 1, 1.0f);
for (String key : meterMap.keySet()) {
List<MeterData> meterList = meterMap.get(key);
// if (meterList.size() != dataNodeCount) {
// logger.warn(
// "meterList size does not match nodeCount: meterList.size={}; nodeCount={}",
// meterList.size(), dataNodeCount);
// }
MeterData meterData = MeterData.merge(meterList);
meters.put(key, meterData);
}
serviceMetricsData.setMeters(meters);
// timers
Map<String, TimerData> timers =
new HashMap<String, TimerData>(timerMap.size() + 1, 1.0f);
for (String key : timerMap.keySet()) {
List<TimerData> timerList = timerMap.get(key);
// if (timerList.size() != dataNodeCount) {
// logger.warn(
// "timerList size does not match nodeCount: timerList.size={}; nodeCount={}",
// timerList.size(), dataNodeCount);
// }
TimerData timerData = TimerData.merge(timerList);
timers.put(key, timerData);
}
serviceMetricsData.setTimers(timers);
serviceMetricsData.setDataNodeCount(dataNodeCount);
serviceMetricsData.setDataNodeInWindowCount(dataNodeInWindowCount);
serviceMetricsData.setClusterId(clusterId);
serviceMetricsData.setServiceId(serviceId);
serviceMetricsData.setIntervalLength(intervalLength);
serviceMetricsData.setIntervalLengthUnit(intervalLengthUnit);
serviceMetricsData.setLastUpdatedTimestamp(System.currentTimeMillis());
// write to ZK
String dataPath =
pathScheme.getAbsolutePath(
PathType.METRICS, pathScheme.joinTokens(clusterId, serviceId));
String serviceMetricsDataString =
JacksonUtil.getObjectMapper().writeValueAsString(serviceMetricsData);
zkClientUtil.updatePath(
getContext().getZkClient(),
getContext().getPathScheme(),
dataPath,
serviceMetricsDataString.getBytes(UTF_8),
getContext().getDefaultZkAclList(),
CreateMode.PERSISTENT,
-1);
// sleep to hold lock before next interval so that
// updates don't happen too frequently with
// more nodes in service
if (i == memberServiceIds.size() - 1) {
try {
long elapsedMillis = (System.nanoTime() - startTimeNanos) / 1000000;
long sleepIntervalMillis = (updateIntervalMillis - elapsedMillis) / 2;
if (sleepIntervalMillis < 0) {
sleepIntervalMillis = updateIntervalMillis;
}
logger.debug(
"AggregationRunnable SLEEPING btw. services: sleepIntervalMillis={}; memberServiceIds.size={}",
sleepIntervalMillis,
memberServiceIds.size());
Thread.sleep(sleepIntervalMillis);
} catch (InterruptedException e) {
logger.warn("Interrupted while sleeping at end of aggregation: " + e, e);
}
}
} catch (KeeperException e) {
if (e.code() != KeeperException.Code.NONODE) {
logger.warn(
"Error trying to aggregate data directory for service: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ ": "
+ e,
e);
}
} catch (Exception e) {
logger.warn(
"Error trying to aggregate data directory for service: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ ": "
+ e,
e);
} finally {
logger.trace("Releasing lock: metrics-aggregation-{}-{}", clusterId, serviceId);
lock.unlock();
lock.destroy();
logger.trace(
"Released and destroyed lock: metrics-aggregation-{}-{}", clusterId, serviceId);
} // try
} // for service
// store aggregated results in ZK at service level
} // for cluster
} // run
public void processRequest(Request request) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing request:: " + request);
}
// request.addRQRec(">final");
long traceMask = ZooTrace.CLIENT_REQUEST_TRACE_MASK;
if (request.type == OpCode.ping) {
traceMask = ZooTrace.SERVER_PING_TRACE_MASK;
}
if (LOG.isTraceEnabled()) {
ZooTrace.logRequest(LOG, traceMask, 'E', request, "");
}
ProcessTxnResult rc = null;
synchronized (zks.outstandingChanges) {
while (!zks.outstandingChanges.isEmpty()
&& zks.outstandingChanges.get(0).zxid <= request.zxid) {
ChangeRecord cr = zks.outstandingChanges.remove(0);
if (cr.zxid < request.zxid) {
LOG.warn("Zxid outstanding " + cr.zxid + " is less than current " + request.zxid);
}
if (zks.outstandingChangesForPath.get(cr.path) == cr) {
zks.outstandingChangesForPath.remove(cr.path);
}
}
if (request.hdr != null) {
TxnHeader hdr = request.hdr;
Record txn = request.txn;
rc = zks.processTxn(hdr, txn);
}
// do not add non quorum packets to the queue.
if (Request.isQuorum(request.type)) {
zks.getZKDatabase().addCommittedProposal(request);
}
}
if (request.hdr != null && request.hdr.getType() == OpCode.closeSession) {
ServerCnxnFactory scxn = zks.getServerCnxnFactory();
// this might be possible since
// we might just be playing diffs from the leader
if (scxn != null && request.cnxn == null) {
// calling this if we have the cnxn results in the client's
// close session response being lost - we've already closed
// the session/socket here before we can send the closeSession
// in the switch block below
scxn.closeSession(request.sessionId);
return;
}
}
if (request.cnxn == null) {
return;
}
ServerCnxn cnxn = request.cnxn;
String lastOp = "NA";
zks.decInProcess();
Code err = Code.OK;
Record rsp = null;
boolean closeSession = false;
try {
if (request.hdr != null && request.hdr.getType() == OpCode.error) {
throw KeeperException.create(KeeperException.Code.get(((ErrorTxn) request.txn).getErr()));
}
KeeperException ke = request.getException();
if (ke != null && request.type != OpCode.multi) {
throw ke;
}
if (LOG.isDebugEnabled()) {
LOG.debug("{}", request);
}
switch (request.type) {
case OpCode.ping:
{
zks.serverStats().updateLatency(request.createTime);
lastOp = "PING";
cnxn.updateStatsForResponse(
request.cxid, request.zxid, lastOp, request.createTime, System.currentTimeMillis());
cnxn.sendResponse(
new ReplyHeader(-2, zks.getZKDatabase().getDataTreeLastProcessedZxid(), 0),
null,
"response");
return;
}
case OpCode.createSession:
{
zks.serverStats().updateLatency(request.createTime);
lastOp = "SESS";
cnxn.updateStatsForResponse(
request.cxid, request.zxid, lastOp, request.createTime, System.currentTimeMillis());
zks.finishSessionInit(request.cnxn, true);
return;
}
case OpCode.multi:
{
lastOp = "MULT";
rsp = new MultiResponse();
for (ProcessTxnResult subTxnResult : rc.multiResult) {
OpResult subResult;
switch (subTxnResult.type) {
case OpCode.check:
subResult = new CheckResult();
break;
case OpCode.create:
subResult = new CreateResult(subTxnResult.path);
break;
case OpCode.delete:
subResult = new DeleteResult();
break;
case OpCode.setData:
subResult = new SetDataResult(subTxnResult.stat);
break;
case OpCode.error:
subResult = new ErrorResult(subTxnResult.err);
break;
default:
throw new IOException("Invalid type of op");
}
((MultiResponse) rsp).add(subResult);
}
break;
}
case OpCode.create:
{
lastOp = "CREA";
rsp = new CreateResponse(rc.path);
err = Code.get(rc.err);
break;
}
case OpCode.delete:
{
lastOp = "DELE";
err = Code.get(rc.err);
break;
}
case OpCode.setData:
{
lastOp = "SETD";
rsp = new SetDataResponse(rc.stat);
err = Code.get(rc.err);
break;
}
case OpCode.setACL:
{
lastOp = "SETA";
rsp = new SetACLResponse(rc.stat);
err = Code.get(rc.err);
break;
}
case OpCode.closeSession:
{
lastOp = "CLOS";
closeSession = true;
err = Code.get(rc.err);
break;
}
case OpCode.sync:
{
lastOp = "SYNC";
SyncRequest syncRequest = new SyncRequest();
ByteBufferInputStream.byteBuffer2Record(request.request, syncRequest);
rsp = new SyncResponse(syncRequest.getPath());
break;
}
case OpCode.check:
{
lastOp = "CHEC";
rsp = new SetDataResponse(rc.stat);
err = Code.get(rc.err);
break;
}
case OpCode.exists:
{
lastOp = "EXIS";
// TODO we need to figure out the security requirement for this!
ExistsRequest existsRequest = new ExistsRequest();
ByteBufferInputStream.byteBuffer2Record(request.request, existsRequest);
String path = existsRequest.getPath();
if (path.indexOf('\0') != -1) {
throw new KeeperException.BadArgumentsException();
}
Stat stat = zks.getZKDatabase().statNode(path, existsRequest.getWatch() ? cnxn : null);
rsp = new ExistsResponse(stat);
break;
}
case OpCode.getData:
{
lastOp = "GETD";
GetDataRequest getDataRequest = new GetDataRequest();
ByteBufferInputStream.byteBuffer2Record(request.request, getDataRequest);
DataNode n = zks.getZKDatabase().getNode(getDataRequest.getPath());
if (n == null) {
throw new KeeperException.NoNodeException();
}
Long aclL;
synchronized (n) {
aclL = n.acl;
}
PrepRequestProcessor.checkACL(
zks, zks.getZKDatabase().convertLong(aclL), ZooDefs.Perms.READ, request.authInfo);
Stat stat = new Stat();
byte b[] =
zks.getZKDatabase()
.getData(
getDataRequest.getPath(), stat, getDataRequest.getWatch() ? cnxn : null);
rsp = new GetDataResponse(b, stat);
break;
}
case OpCode.setWatches:
{
lastOp = "SETW";
SetWatches setWatches = new SetWatches();
// XXX We really should NOT need this!!!!
request.request.rewind();
ByteBufferInputStream.byteBuffer2Record(request.request, setWatches);
long relativeZxid = setWatches.getRelativeZxid();
zks.getZKDatabase()
.setWatches(
relativeZxid,
setWatches.getDataWatches(),
setWatches.getExistWatches(),
setWatches.getChildWatches(),
cnxn);
break;
}
case OpCode.getACL:
{
lastOp = "GETA";
GetACLRequest getACLRequest = new GetACLRequest();
ByteBufferInputStream.byteBuffer2Record(request.request, getACLRequest);
Stat stat = new Stat();
List<ACL> acl = zks.getZKDatabase().getACL(getACLRequest.getPath(), stat);
rsp = new GetACLResponse(acl, stat);
break;
}
case OpCode.getChildren:
{
lastOp = "GETC";
GetChildrenRequest getChildrenRequest = new GetChildrenRequest();
ByteBufferInputStream.byteBuffer2Record(request.request, getChildrenRequest);
DataNode n = zks.getZKDatabase().getNode(getChildrenRequest.getPath());
if (n == null) {
throw new KeeperException.NoNodeException();
}
Long aclG;
synchronized (n) {
aclG = n.acl;
}
PrepRequestProcessor.checkACL(
zks, zks.getZKDatabase().convertLong(aclG), ZooDefs.Perms.READ, request.authInfo);
List<String> children =
zks.getZKDatabase()
.getChildren(
getChildrenRequest.getPath(),
null,
getChildrenRequest.getWatch() ? cnxn : null);
rsp = new GetChildrenResponse(children);
break;
}
case OpCode.getChildren2:
{
lastOp = "GETC";
GetChildren2Request getChildren2Request = new GetChildren2Request();
ByteBufferInputStream.byteBuffer2Record(request.request, getChildren2Request);
Stat stat = new Stat();
DataNode n = zks.getZKDatabase().getNode(getChildren2Request.getPath());
if (n == null) {
throw new KeeperException.NoNodeException();
}
Long aclG;
synchronized (n) {
aclG = n.acl;
}
PrepRequestProcessor.checkACL(
zks, zks.getZKDatabase().convertLong(aclG), ZooDefs.Perms.READ, request.authInfo);
List<String> children =
zks.getZKDatabase()
.getChildren(
getChildren2Request.getPath(),
stat,
getChildren2Request.getWatch() ? cnxn : null);
rsp = new GetChildren2Response(children, stat);
break;
}
}
} catch (SessionMovedException e) {
// session moved is a connection level error, we need to tear
// down the connection otw ZOOKEEPER-710 might happen
// ie client on slow follower starts to renew session, fails
// before this completes, then tries the fast follower (leader)
// and is successful, however the initial renew is then
// successfully fwd/processed by the leader and as a result
// the client and leader disagree on where the client is most
// recently attached (and therefore invalid SESSION MOVED generated)
cnxn.sendCloseSession();
return;
} catch (KeeperException e) {
err = e.code();
} catch (Exception e) {
// log at error level as we are returning a marshalling
// error to the user
LOG.error("Failed to process " + request, e);
StringBuilder sb = new StringBuilder();
ByteBuffer bb = request.request;
bb.rewind();
while (bb.hasRemaining()) {
sb.append(Integer.toHexString(bb.get() & 0xff));
}
LOG.error("Dumping request buffer: 0x" + sb.toString());
err = Code.MARSHALLINGERROR;
}
long lastZxid = zks.getZKDatabase().getDataTreeLastProcessedZxid();
ReplyHeader hdr = new ReplyHeader(request.cxid, lastZxid, err.intValue());
zks.serverStats().updateLatency(request.createTime);
cnxn.updateStatsForResponse(
request.cxid, lastZxid, lastOp, request.createTime, System.currentTimeMillis());
try {
cnxn.sendResponse(hdr, rsp, "response");
if (closeSession) {
cnxn.sendCloseSession();
}
} catch (IOException e) {
LOG.error("FIXMSG", e);
}
}
public ProcessTxnResult processTxn(TxnHeader header, Record txn) {
ProcessTxnResult rc = new ProcessTxnResult();
String debug = "";
try {
rc.clientId = header.getClientId();
rc.cxid = header.getCxid();
rc.zxid = header.getZxid();
rc.type = header.getType();
rc.err = 0;
if (rc.zxid > lastProcessedZxid) {
lastProcessedZxid = rc.zxid;
}
switch (header.getType()) {
case OpCode.create:
CreateTxn createTxn = (CreateTxn) txn;
debug = "Create transaction for " + createTxn.getPath();
createNode(
createTxn.getPath(),
createTxn.getData(),
createTxn.getAcl(),
createTxn.getEphemeral() ? header.getClientId() : 0,
header.getZxid(),
header.getTime());
rc.path = createTxn.getPath();
break;
case OpCode.delete:
DeleteTxn deleteTxn = (DeleteTxn) txn;
debug = "Delete transaction for " + deleteTxn.getPath();
deleteNode(deleteTxn.getPath(), header.getZxid());
break;
case OpCode.setData:
SetDataTxn setDataTxn = (SetDataTxn) txn;
debug = "Set data for transaction for " + setDataTxn.getPath();
rc.stat =
setData(
setDataTxn.getPath(),
setDataTxn.getData(),
setDataTxn.getVersion(),
header.getZxid(),
header.getTime());
break;
case OpCode.setACL:
SetACLTxn setACLTxn = (SetACLTxn) txn;
debug = "Set ACL for transaction for " + setACLTxn.getPath();
rc.stat = setACL(setACLTxn.getPath(), setACLTxn.getAcl(), setACLTxn.getVersion());
break;
case OpCode.closeSession:
killSession(header.getClientId(), header.getZxid());
break;
case OpCode.error:
ErrorTxn errTxn = (ErrorTxn) txn;
rc.err = errTxn.getErr();
break;
}
} catch (KeeperException e) {
// These are expected errors since we take a lazy snapshot
if (initialized || (e.code() != Code.NONODE && e.code() != Code.NODEEXISTS)) {
LOG.warn("Failed:" + debug, e);
}
}
return rc;
}
@Override
public void run() {
logger.trace("CleanerRunnable starting: hashCode={}", this.hashCode());
PresenceService presenceService = getContext().getService("presence");
CoordinationService coordinationService = getContext().getService("coord");
ZkClient zkClient = getContext().getZkClient();
PathScheme pathScheme = getContext().getPathScheme();
// list all services in cluster
List<String> clusterIds = presenceService.getClusters();
for (String clusterId : clusterIds) {
// only proceed if in cluster
if (!presenceService.isMemberOf(clusterId)
|| clusterId.equals(getContext().getPathScheme().getFrameworkClusterId())) {
continue;
}
List<String> serviceIds = presenceService.getServices(clusterId);
for (String serviceId : serviceIds) {
logger.trace(
"Checking data nodes expiry: clusterId={}; serviceId={}", clusterId, serviceId);
// only proceed if in service
if (!presenceService.isMemberOf(clusterId, serviceId)) {
continue;
}
long currentTimestamp = System.currentTimeMillis();
// get lock for a service
DistributedLock lock =
coordinationService.getLock("reign", "metrics-" + clusterId + "-" + serviceId);
if (!lock.tryLock()) {
continue;
}
String dataPath = null;
try {
// get all data nodes for a service
String dataParentPath =
pathScheme.getAbsolutePath(
PathType.METRICS, pathScheme.joinTokens(clusterId, serviceId));
List<String> dataNodes = zkClient.getChildren(dataParentPath, false);
// remove all nodes that are older than rotation
// interval
for (String dataNode : dataNodes) {
try {
logger.trace(
"Checking data node expiry: clusterId={}; serviceId={}; nodeId={}",
clusterId,
serviceId,
dataNode);
dataPath =
pathScheme.getAbsolutePath(
PathType.METRICS, pathScheme.joinTokens(clusterId, serviceId, dataNode));
MetricsData metricsData = getMetricsFromDataNode(clusterId, serviceId, dataNode);
if (metricsData == null) {
logger.warn(
"Removing unrecognized/corrupted/deprecated data node: path={}", dataPath);
zkClient.delete(dataPath, -1);
continue;
}
// keep last few hours worth of data
long millisToExpiry =
millisToExpiry(metricsData, currentTimestamp - (86400000 / 6));
// delete data that is older than some threshold
boolean dataTooOld =
currentTimestamp - metricsData.getIntervalStartTimestamp() > 86400000;
// delete old and expired data
if (millisToExpiry <= 0 || dataTooOld) {
logger.info(
"Removing expired data node: path={}; millisToExpiry={}",
dataPath,
millisToExpiry);
zkClient.delete(dataPath, -1);
} else {
logger.trace(
"Data node is not yet expired: path={}; millisToExpiry={}",
dataPath,
millisToExpiry);
}
} catch (Exception e) {
logger.warn(
"Error trying to clean up data directory for service: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ "; dataPath="
+ dataPath
+ ": "
+ e,
e);
} // try
} // for
} catch (KeeperException e) {
if (e.code() != KeeperException.Code.NONODE) {
logger.warn(
"Error trying to clean up data directory for service: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ "; dataPath="
+ dataPath
+ ": "
+ e,
e);
}
} catch (Exception e) {
logger.warn(
"Error trying to clean up data directory for service: clusterId="
+ clusterId
+ "; serviceId="
+ serviceId
+ "; dataPath="
+ dataPath
+ ": "
+ e,
e);
} finally {
lock.unlock();
lock.destroy();
} // try
} // for service
} // for cluster
} // run()
