/**
* Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper
*
* @param conf configuration for the instance
* @param instanceId instanceId for the instance
* @param hostAddress Address that monitor process is bound to
*/
public static void startLogListener(
AccumuloConfiguration conf, String instanceId, String hostAddress) {
try {
SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT));
// getLocalPort will return the actual ephemeral port used when '0' was provided.
String logForwardingAddr = hostAddress + ":" + server.getLocalPort();
log.debug("Setting monitor log4j log-forwarding address to: " + logForwardingAddr);
final String path = ZooUtil.getRoot(instanceId) + Constants.ZMONITOR_LOG4J_ADDR;
final ZooReaderWriter zoo = ZooReaderWriter.getInstance();
// Delete before we try to re-create in case the previous session hasn't yet expired
try {
zoo.delete(path, -1);
} catch (KeeperException e) {
// We don't care if the node is already gone
if (!KeeperException.Code.NONODE.equals(e.code())) {
throw e;
}
}
zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8));
new Daemon(server).start();
} catch (Throwable t) {
log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t);
}
}
private static GCStatus fetchGcStatus() {
GCStatus result = null;
HostAndPort address = null;
try {
// Read the gc location from its lock
ZooReaderWriter zk = ZooReaderWriter.getInstance();
String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
List<String> locks = zk.getChildren(path, null);
if (locks != null && locks.size() > 0) {
Collections.sort(locks);
address =
new ServerServices(
new String(zk.getData(path + "/" + locks.get(0), null), StandardCharsets.UTF_8))
.getAddress(Service.GC_CLIENT);
GCMonitorService.Client client =
ThriftUtil.getClient(
new GCMonitorService.Client.Factory(), address, config.getConfiguration());
try {
result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
} finally {
ThriftUtil.returnClient(client);
}
}
} catch (Exception ex) {
log.warn("Unable to contact the garbage collector at " + address, ex);
}
return result;
}
/** @param args : the name or UUID of the instance to be deleted */
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
opts.parseArgs(DeleteZooInstance.class.getName(), args);
IZooReaderWriter zk = ZooReaderWriter.getInstance();
// try instance name:
Set<String> instances =
new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
Set<String> uuids = new HashSet<String>(zk.getChildren(Constants.ZROOT));
uuids.remove("instances");
if (instances.contains(opts.instance)) {
String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + opts.instance;
byte[] data = zk.getData(path, null);
deleteRetry(zk, path);
deleteRetry(zk, Constants.ZROOT + "/" + new String(data, UTF_8));
} else if (uuids.contains(opts.instance)) {
// look for the real instance name
for (String instance : instances) {
String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
byte[] data = zk.getData(path, null);
if (opts.instance.equals(new String(data, UTF_8))) deleteRetry(zk, path);
}
deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
}
}
@Override
public void initializeSecurity(TCredentials credentials, String principal, byte[] token)
throws AccumuloSecurityException {
try {
// remove old settings from zookeeper first, if any
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
synchronized (zooCache) {
zooCache.clear();
if (zoo.exists(ZKUserPath)) {
zoo.recursiveDelete(ZKUserPath, NodeMissingPolicy.SKIP);
log.info("Removed " + ZKUserPath + "/" + " from zookeeper");
}
// prep parent node of users with root username
zoo.putPersistentData(ZKUserPath, principal.getBytes(UTF_8), NodeExistsPolicy.FAIL);
constructUser(principal, ZKSecurityTool.createPass(token));
}
} catch (KeeperException e) {
log.error("{}", e.getMessage(), e);
throw new RuntimeException(e);
} catch (InterruptedException e) {
log.error("{}", e.getMessage(), e);
throw new RuntimeException(e);
} catch (AccumuloException e) {
log.error("{}", e.getMessage(), e);
throw new RuntimeException(e);
}
}
@Override
public void changePassword(String principal, AuthenticationToken token)
throws AccumuloSecurityException {
if (!(token instanceof PasswordToken))
throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
PasswordToken pt = (PasswordToken) token;
if (userExists(principal)) {
try {
synchronized (zooCache) {
zooCache.clear(ZKUserPath + "/" + principal);
ZooReaderWriter.getInstance()
.putPrivatePersistentData(
ZKUserPath + "/" + principal,
ZKSecurityTool.createPass(pt.getPassword()),
NodeExistsPolicy.OVERWRITE);
}
} catch (KeeperException e) {
log.error("{}", e.getMessage(), e);
throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
} catch (InterruptedException e) {
log.error("{}", e.getMessage(), e);
throw new RuntimeException(e);
} catch (AccumuloException e) {
log.error("{}", e.getMessage(), e);
throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
}
} else
throw new AccumuloSecurityException(
principal, SecurityErrorCode.USER_DOESNT_EXIST); // user doesn't exist
}
public static void removeUnusedWALEntries(
KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
if (extent.isRootTablet()) {
for (LogEntry entry : logEntries) {
String root = getZookeeperLogLocation();
while (true) {
try {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
if (zoo.isLockHeld(zooLock.getLockID()))
zoo.recursiveDelete(root + "/" + entry.filename, NodeMissingPolicy.SKIP);
break;
} catch (Exception e) {
log.error(e, e);
}
UtilWaitThread.sleep(1000);
}
}
} else {
Mutation m = new Mutation(extent.getMetadataEntry());
for (LogEntry entry : logEntries) {
m.putDelete(LogColumnFamily.NAME, new Text(entry.toString()));
}
update(SystemCredentials.get(), zooLock, m, extent);
}
}
public synchronized String getNextName() {
while (next >= maxAllocated) {
final int allocate = 100 + rand.nextInt(100);
try {
byte[] max =
ZooReaderWriter.getRetryingInstance()
.mutate(
nextNamePath,
null,
ZooUtil.PRIVATE,
new ZooReaderWriter.Mutator() {
public byte[] mutate(byte[] currentValue) throws Exception {
long l = Long.parseLong(new String(currentValue), Character.MAX_RADIX);
l += allocate;
return Long.toString(l, Character.MAX_RADIX).getBytes();
}
});
maxAllocated = Long.parseLong(new String(max), Character.MAX_RADIX);
next = maxAllocated - allocate;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]));
}
public static void addLogEntry(Credentials credentials, LogEntry entry, ZooLock zooLock) {
if (entry.extent.isRootTablet()) {
String root = getZookeeperLogLocation();
while (true) {
try {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
if (zoo.isLockHeld(zooLock.getLockID())) {
String[] parts = entry.filename.split("/");
String uniqueId = parts[parts.length - 1];
zoo.putPersistentData(
root + "/" + uniqueId, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
}
break;
} catch (KeeperException e) {
log.error(e, e);
} catch (InterruptedException e) {
log.error(e, e);
} catch (IOException e) {
log.error(e, e);
}
UtilWaitThread.sleep(1000);
}
} else {
Mutation m = new Mutation(entry.getRow());
m.put(entry.getColumnFamily(), entry.getColumnQualifier(), entry.getValue());
update(credentials, zooLock, m, entry.extent);
}
}
/**
* Sets up the user in ZK for the provided user. No checking for existence is done here, it should
* be done before calling.
*/
private void constructUser(String user, byte[] pass)
throws KeeperException, InterruptedException {
synchronized (zooCache) {
zooCache.clear();
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
zoo.putPrivatePersistentData(ZKUserPath + "/" + user, pass, NodeExistsPolicy.FAIL);
}
}
public static String getRootTabletDir() throws IOException {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
try {
return new String(zoo.getData(zpath, null), Constants.UTF8);
} catch (KeeperException e) {
throw new IOException(e);
} catch (InterruptedException e) {
throw new IOException(e);
}
}
@Override
public void put(String path, byte[] bs) throws DistributedStoreException {
try {
path = relative(path);
ZooReaderWriter.getInstance().putPersistentData(path, bs, NodeExistsPolicy.OVERWRITE);
cache.clear();
log.debug("Wrote " + new String(bs) + " to " + path);
} catch (Exception ex) {
throw new DistributedStoreException(ex);
}
}
@Override
public void remove(String path) throws DistributedStoreException {
try {
log.debug("Removing " + path);
path = relative(path);
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
if (zoo.exists(path)) zoo.recursiveDelete(path, NodeMissingPolicy.SKIP);
cache.clear();
} catch (Exception ex) {
throw new DistributedStoreException(ex);
}
}
@Override
public Repo<Master> call(long id, Master master) throws Exception {
Instance instance = master.getInstance();
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
Utils.tableNameLock.lock();
try {
Utils.checkNamespaceDoesNotExist(instance, newName, namespaceId, TableOperation.RENAME);
final String tap =
ZooUtil.getRoot(instance)
+ Constants.ZNAMESPACES
+ "/"
+ namespaceId
+ Constants.ZNAMESPACE_NAME;
zoo.mutate(
tap,
null,
null,
new Mutator() {
@Override
public byte[] mutate(byte[] current) throws Exception {
final String currentName = new String(current);
if (currentName.equals(newName))
return null; // assume in this case the operation is running again, so we are done
if (!currentName.equals(oldName)) {
throw new AcceptableThriftTableOperationException(
null,
oldName,
TableOperation.RENAME,
TableOperationExceptionType.NAMESPACE_NOTFOUND,
"Name changed while processing");
}
return newName.getBytes();
}
});
Tables.clearCache(instance);
} finally {
Utils.tableNameLock.unlock();
Utils.unreserveNamespace(namespaceId, id, true);
}
LoggerFactory.getLogger(RenameNamespace.class)
.debug("Renamed namespace " + namespaceId + " " + oldName + " " + newName);
return null;
}
@Override
public void dropUser(String user) throws AccumuloSecurityException {
try {
synchronized (zooCache) {
zooCache.clear();
ZooReaderWriter.getInstance()
.recursiveDelete(ZKUserPath + "/" + user, NodeMissingPolicy.FAIL);
}
} catch (InterruptedException e) {
log.error("{}", e.getMessage(), e);
throw new RuntimeException(e);
} catch (KeeperException e) {
if (e.code().equals(KeeperException.Code.NONODE)) {
throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
}
log.error("{}", e.getMessage(), e);
throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
}
}
@Override
public void process(WatchedEvent event) {
log.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
if (event.getState() == KeeperState.Expired) {
log.warn("Trace server lost zookeeper registration at " + event.getPath());
server.stop();
} else if (event.getType() == EventType.NodeDeleted) {
log.warn("Trace server zookeeper entry lost " + event.getPath());
server.stop();
}
if (event.getPath() != null) {
try {
if (ZooReaderWriter.getInstance().exists(event.getPath(), this)) return;
} catch (Exception ex) {
log.error(ex, ex);
}
log.warn("Trace server unable to reset watch on zookeeper registration");
server.stop();
}
}
private static void getRootLogEntries(ArrayList<LogEntry> result)
throws KeeperException, InterruptedException, IOException {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
String root = getZookeeperLogLocation();
// there's a little race between getting the children and fetching
// the data. The log can be removed in between.
while (true) {
result.clear();
for (String child : zoo.getChildren(root)) {
LogEntry e = new LogEntry();
try {
e.fromBytes(zoo.getData(root + "/" + child, null));
// upgrade from !0;!0<< -> +r<<
e.extent = RootTable.EXTENT;
result.add(e);
} catch (KeeperException.NoNodeException ex) {
continue;
}
}
break;
}
}
private void registerInZooKeeper(String name) throws Exception {
String root = ZooUtil.getRoot(serverConfiguration.getInstance()) + Constants.ZTRACERS;
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes());
zoo.exists(path, this);
}
/**
* Get the monitor lock in ZooKeeper
*
* @throws KeeperException
* @throws InterruptedException
*/
private void getMonitorLock() throws KeeperException, InterruptedException {
final String zRoot = ZooUtil.getRoot(instance);
final String monitorPath = zRoot + Constants.ZMONITOR;
final String monitorLockPath = zRoot + Constants.ZMONITOR_LOCK;
// Ensure that everything is kosher with ZK as this has changed.
ZooReaderWriter zoo = ZooReaderWriter.getInstance();
if (zoo.exists(monitorPath)) {
byte[] data = zoo.getData(monitorPath, null);
// If the node isn't empty, it's from a previous install (has hostname:port for HTTP server)
if (0 != data.length) {
// Recursively delete from that parent node
zoo.recursiveDelete(monitorPath, NodeMissingPolicy.SKIP);
// And then make the nodes that we expect for the incoming ephemeral nodes
zoo.putPersistentData(monitorPath, new byte[0], NodeExistsPolicy.FAIL);
zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
} else if (!zoo.exists(monitorLockPath)) {
// monitor node in ZK exists and is empty as we expect
// but the monitor/lock node does not
zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
}
} else {
// 1.5.0 and earlier
zoo.putPersistentData(zRoot + Constants.ZMONITOR, new byte[0], NodeExistsPolicy.FAIL);
if (!zoo.exists(monitorLockPath)) {
// Somehow the monitor node exists but not monitor/lock
zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
}
}
// Get a ZooLock for the monitor
while (true) {
MoniterLockWatcher monitorLockWatcher = new MoniterLockWatcher();
monitorLock = new ZooLock(monitorLockPath);
monitorLock.lockAsync(monitorLockWatcher, new byte[0]);
monitorLockWatcher.waitForChange();
if (monitorLockWatcher.acquiredLock) {
break;
}
if (!monitorLockWatcher.failedToAcquireLock) {
throw new IllegalStateException("monitor lock in unknown state");
}
monitorLock.tryToCancelAsyncLockOrUnlock();
UtilWaitThread.sleep(
getSystemConfiguration().getTimeInMillis(Property.MONITOR_LOCK_CHECK_INTERVAL));
}
log.info("Got Monitor lock.");
}
public void run(String hostname) {
try {
getMonitorLock();
} catch (Exception e) {
log.error("Failed to get Monitor ZooKeeper lock");
throw new RuntimeException(e);
}
Monitor.START_TIME = System.currentTimeMillis();
int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
try {
log.debug("Creating monitor on port " + port);
server = new EmbeddedWebServer(hostname, port);
} catch (Throwable ex) {
log.error("Unable to start embedded web server", ex);
throw new RuntimeException(ex);
}
server.addServlet(DefaultServlet.class, "/");
server.addServlet(OperationServlet.class, "/op");
server.addServlet(MasterServlet.class, "/master");
server.addServlet(TablesServlet.class, "/tables");
server.addServlet(TServersServlet.class, "/tservers");
server.addServlet(ProblemServlet.class, "/problems");
server.addServlet(GcStatusServlet.class, "/gc");
server.addServlet(LogServlet.class, "/log");
server.addServlet(XMLServlet.class, "/xml");
server.addServlet(JSONServlet.class, "/json");
server.addServlet(VisServlet.class, "/vis");
server.addServlet(Summary.class, "/trace/summary");
server.addServlet(ListType.class, "/trace/listType");
server.addServlet(ShowTrace.class, "/trace/show");
if (server.isUsingSsl()) server.addServlet(ShellServlet.class, "/shell");
server.start();
try {
hostname = InetAddress.getLocalHost().getHostName();
log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");
String monitorAddress = HostAndPort.fromParts(hostname, server.getPort()).toString();
ZooReaderWriter.getInstance()
.putPersistentData(
ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR,
monitorAddress.getBytes(StandardCharsets.UTF_8),
NodeExistsPolicy.OVERWRITE);
log.info("Set monitor address in zookeeper to " + monitorAddress);
} catch (Exception ex) {
log.error("Unable to set monitor HTTP address in zookeeper", ex);
}
if (null != hostname) {
LogService.startLogListener(
Monitor.getSystemConfiguration(), instance.getInstanceID(), hostname);
} else {
log.warn("Not starting log4j listener as we could not determine address to use");
}
new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();
// need to regularly fetch data so plot data is updated
new Daemon(
new LoggingRunnable(
log,
new Runnable() {
@Override
public void run() {
while (true) {
try {
Monitor.fetchData();
} catch (Exception e) {
log.warn(e.getMessage(), e);
}
UtilWaitThread.sleep(333);
}
}
}),
"Data fetcher")
.start();
}