コード例 #1
0
  /**
   * Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper
   *
   * @param conf configuration for the instance
   * @param instanceId instanceId for the instance
   * @param hostAddress Address that monitor process is bound to
   */
  public static void startLogListener(
      AccumuloConfiguration conf, String instanceId, String hostAddress) {
    try {
      SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT));

      // getLocalPort will return the actual ephemeral port used when '0' was provided.
      String logForwardingAddr = hostAddress + ":" + server.getLocalPort();

      log.debug("Setting monitor log4j log-forwarding address to: " + logForwardingAddr);

      final String path = ZooUtil.getRoot(instanceId) + Constants.ZMONITOR_LOG4J_ADDR;
      final ZooReaderWriter zoo = ZooReaderWriter.getInstance();

      // Delete before we try to re-create in case the previous session hasn't yet expired
      try {
        zoo.delete(path, -1);
      } catch (KeeperException e) {
        // We don't care if the node is already gone
        if (!KeeperException.Code.NONODE.equals(e.code())) {
          throw e;
        }
      }

      zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8));

      new Daemon(server).start();
    } catch (Throwable t) {
      log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t);
    }
  }
コード例 #2
0
ファイル: Monitor.java プロジェクト: joshelser/accumulo
 private static GCStatus fetchGcStatus() {
   GCStatus result = null;
   HostAndPort address = null;
   try {
     // Read the gc location from its lock
     ZooReaderWriter zk = ZooReaderWriter.getInstance();
     String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
     List<String> locks = zk.getChildren(path, null);
     if (locks != null && locks.size() > 0) {
       Collections.sort(locks);
       address =
           new ServerServices(
                   new String(zk.getData(path + "/" + locks.get(0), null), StandardCharsets.UTF_8))
               .getAddress(Service.GC_CLIENT);
       GCMonitorService.Client client =
           ThriftUtil.getClient(
               new GCMonitorService.Client.Factory(), address, config.getConfiguration());
       try {
         result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
       } finally {
         ThriftUtil.returnClient(client);
       }
     }
   } catch (Exception ex) {
     log.warn("Unable to contact the garbage collector at " + address, ex);
   }
   return result;
 }
コード例 #3
0
  /** @param args : the name or UUID of the instance to be deleted */
  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(DeleteZooInstance.class.getName(), args);

    IZooReaderWriter zk = ZooReaderWriter.getInstance();
    // try instance name:
    Set<String> instances =
        new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
    Set<String> uuids = new HashSet<String>(zk.getChildren(Constants.ZROOT));
    uuids.remove("instances");
    if (instances.contains(opts.instance)) {
      String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + opts.instance;
      byte[] data = zk.getData(path, null);
      deleteRetry(zk, path);
      deleteRetry(zk, Constants.ZROOT + "/" + new String(data, UTF_8));
    } else if (uuids.contains(opts.instance)) {
      // look for the real instance name
      for (String instance : instances) {
        String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
        byte[] data = zk.getData(path, null);
        if (opts.instance.equals(new String(data, UTF_8))) deleteRetry(zk, path);
      }
      deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
    }
  }
コード例 #4
0
  @Override
  public void initializeSecurity(TCredentials credentials, String principal, byte[] token)
      throws AccumuloSecurityException {
    try {
      // remove old settings from zookeeper first, if any
      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
      synchronized (zooCache) {
        zooCache.clear();
        if (zoo.exists(ZKUserPath)) {
          zoo.recursiveDelete(ZKUserPath, NodeMissingPolicy.SKIP);
          log.info("Removed " + ZKUserPath + "/" + " from zookeeper");
        }

        // prep parent node of users with root username
        zoo.putPersistentData(ZKUserPath, principal.getBytes(UTF_8), NodeExistsPolicy.FAIL);

        constructUser(principal, ZKSecurityTool.createPass(token));
      }
    } catch (KeeperException e) {
      log.error("{}", e.getMessage(), e);
      throw new RuntimeException(e);
    } catch (InterruptedException e) {
      log.error("{}", e.getMessage(), e);
      throw new RuntimeException(e);
    } catch (AccumuloException e) {
      log.error("{}", e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }
コード例 #5
0
 @Override
 public void changePassword(String principal, AuthenticationToken token)
     throws AccumuloSecurityException {
   if (!(token instanceof PasswordToken))
     throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
   PasswordToken pt = (PasswordToken) token;
   if (userExists(principal)) {
     try {
       synchronized (zooCache) {
         zooCache.clear(ZKUserPath + "/" + principal);
         ZooReaderWriter.getInstance()
             .putPrivatePersistentData(
                 ZKUserPath + "/" + principal,
                 ZKSecurityTool.createPass(pt.getPassword()),
                 NodeExistsPolicy.OVERWRITE);
       }
     } catch (KeeperException e) {
       log.error("{}", e.getMessage(), e);
       throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
     } catch (InterruptedException e) {
       log.error("{}", e.getMessage(), e);
       throw new RuntimeException(e);
     } catch (AccumuloException e) {
       log.error("{}", e.getMessage(), e);
       throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
     }
   } else
     throw new AccumuloSecurityException(
         principal, SecurityErrorCode.USER_DOESNT_EXIST); // user doesn't exist
 }
コード例 #6
0
 public static void removeUnusedWALEntries(
     KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
   if (extent.isRootTablet()) {
     for (LogEntry entry : logEntries) {
       String root = getZookeeperLogLocation();
       while (true) {
         try {
           IZooReaderWriter zoo = ZooReaderWriter.getInstance();
           if (zoo.isLockHeld(zooLock.getLockID()))
             zoo.recursiveDelete(root + "/" + entry.filename, NodeMissingPolicy.SKIP);
           break;
         } catch (Exception e) {
           log.error(e, e);
         }
         UtilWaitThread.sleep(1000);
       }
     }
   } else {
     Mutation m = new Mutation(extent.getMetadataEntry());
     for (LogEntry entry : logEntries) {
       m.putDelete(LogColumnFamily.NAME, new Text(entry.toString()));
     }
     update(SystemCredentials.get(), zooLock, m, extent);
   }
 }
コード例 #7
0
  public synchronized String getNextName() {

    while (next >= maxAllocated) {
      final int allocate = 100 + rand.nextInt(100);

      try {
        byte[] max =
            ZooReaderWriter.getRetryingInstance()
                .mutate(
                    nextNamePath,
                    null,
                    ZooUtil.PRIVATE,
                    new ZooReaderWriter.Mutator() {
                      public byte[] mutate(byte[] currentValue) throws Exception {
                        long l = Long.parseLong(new String(currentValue), Character.MAX_RADIX);
                        l += allocate;
                        return Long.toString(l, Character.MAX_RADIX).getBytes();
                      }
                    });

        maxAllocated = Long.parseLong(new String(max), Character.MAX_RADIX);
        next = maxAllocated - allocate;

      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    }

    return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]));
  }
コード例 #8
0
 public static void addLogEntry(Credentials credentials, LogEntry entry, ZooLock zooLock) {
   if (entry.extent.isRootTablet()) {
     String root = getZookeeperLogLocation();
     while (true) {
       try {
         IZooReaderWriter zoo = ZooReaderWriter.getInstance();
         if (zoo.isLockHeld(zooLock.getLockID())) {
           String[] parts = entry.filename.split("/");
           String uniqueId = parts[parts.length - 1];
           zoo.putPersistentData(
               root + "/" + uniqueId, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
         }
         break;
       } catch (KeeperException e) {
         log.error(e, e);
       } catch (InterruptedException e) {
         log.error(e, e);
       } catch (IOException e) {
         log.error(e, e);
       }
       UtilWaitThread.sleep(1000);
     }
   } else {
     Mutation m = new Mutation(entry.getRow());
     m.put(entry.getColumnFamily(), entry.getColumnQualifier(), entry.getValue());
     update(credentials, zooLock, m, entry.extent);
   }
 }
コード例 #9
0
 /**
  * Sets up the user in ZK for the provided user. No checking for existence is done here, it should
  * be done before calling.
  */
 private void constructUser(String user, byte[] pass)
     throws KeeperException, InterruptedException {
   synchronized (zooCache) {
     zooCache.clear();
     IZooReaderWriter zoo = ZooReaderWriter.getInstance();
     zoo.putPrivatePersistentData(ZKUserPath + "/" + user, pass, NodeExistsPolicy.FAIL);
   }
 }
コード例 #10
0
 public static String getRootTabletDir() throws IOException {
   IZooReaderWriter zoo = ZooReaderWriter.getInstance();
   String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
   try {
     return new String(zoo.getData(zpath, null), Constants.UTF8);
   } catch (KeeperException e) {
     throw new IOException(e);
   } catch (InterruptedException e) {
     throw new IOException(e);
   }
 }
コード例 #11
0
ファイル: ZooStore.java プロジェクト: ekohlwey/accumulo
 @Override
 public void put(String path, byte[] bs) throws DistributedStoreException {
   try {
     path = relative(path);
     ZooReaderWriter.getInstance().putPersistentData(path, bs, NodeExistsPolicy.OVERWRITE);
     cache.clear();
     log.debug("Wrote " + new String(bs) + " to " + path);
   } catch (Exception ex) {
     throw new DistributedStoreException(ex);
   }
 }
コード例 #12
0
ファイル: ZooStore.java プロジェクト: ekohlwey/accumulo
 @Override
 public void remove(String path) throws DistributedStoreException {
   try {
     log.debug("Removing " + path);
     path = relative(path);
     IZooReaderWriter zoo = ZooReaderWriter.getInstance();
     if (zoo.exists(path)) zoo.recursiveDelete(path, NodeMissingPolicy.SKIP);
     cache.clear();
   } catch (Exception ex) {
     throw new DistributedStoreException(ex);
   }
 }
コード例 #13
0
  @Override
  public Repo<Master> call(long id, Master master) throws Exception {

    Instance instance = master.getInstance();

    IZooReaderWriter zoo = ZooReaderWriter.getInstance();

    Utils.tableNameLock.lock();
    try {
      Utils.checkNamespaceDoesNotExist(instance, newName, namespaceId, TableOperation.RENAME);

      final String tap =
          ZooUtil.getRoot(instance)
              + Constants.ZNAMESPACES
              + "/"
              + namespaceId
              + Constants.ZNAMESPACE_NAME;

      zoo.mutate(
          tap,
          null,
          null,
          new Mutator() {
            @Override
            public byte[] mutate(byte[] current) throws Exception {
              final String currentName = new String(current);
              if (currentName.equals(newName))
                return null; // assume in this case the operation is running again, so we are done
              if (!currentName.equals(oldName)) {
                throw new AcceptableThriftTableOperationException(
                    null,
                    oldName,
                    TableOperation.RENAME,
                    TableOperationExceptionType.NAMESPACE_NOTFOUND,
                    "Name changed while processing");
              }
              return newName.getBytes();
            }
          });
      Tables.clearCache(instance);
    } finally {
      Utils.tableNameLock.unlock();
      Utils.unreserveNamespace(namespaceId, id, true);
    }

    LoggerFactory.getLogger(RenameNamespace.class)
        .debug("Renamed namespace " + namespaceId + " " + oldName + " " + newName);

    return null;
  }
コード例 #14
0
 @Override
 public void dropUser(String user) throws AccumuloSecurityException {
   try {
     synchronized (zooCache) {
       zooCache.clear();
       ZooReaderWriter.getInstance()
           .recursiveDelete(ZKUserPath + "/" + user, NodeMissingPolicy.FAIL);
     }
   } catch (InterruptedException e) {
     log.error("{}", e.getMessage(), e);
     throw new RuntimeException(e);
   } catch (KeeperException e) {
     if (e.code().equals(KeeperException.Code.NONODE)) {
       throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
     }
     log.error("{}", e.getMessage(), e);
     throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
   }
 }
コード例 #15
0
ファイル: TraceServer.java プロジェクト: harayz/accumulo
 @Override
 public void process(WatchedEvent event) {
   log.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
   if (event.getState() == KeeperState.Expired) {
     log.warn("Trace server lost zookeeper registration at " + event.getPath());
     server.stop();
   } else if (event.getType() == EventType.NodeDeleted) {
     log.warn("Trace server zookeeper entry lost " + event.getPath());
     server.stop();
   }
   if (event.getPath() != null) {
     try {
       if (ZooReaderWriter.getInstance().exists(event.getPath(), this)) return;
     } catch (Exception ex) {
       log.error(ex, ex);
     }
     log.warn("Trace server unable to reset watch on zookeeper registration");
     server.stop();
   }
 }
コード例 #16
0
 private static void getRootLogEntries(ArrayList<LogEntry> result)
     throws KeeperException, InterruptedException, IOException {
   IZooReaderWriter zoo = ZooReaderWriter.getInstance();
   String root = getZookeeperLogLocation();
   // there's a little race between getting the children and fetching
   // the data. The log can be removed in between.
   while (true) {
     result.clear();
     for (String child : zoo.getChildren(root)) {
       LogEntry e = new LogEntry();
       try {
         e.fromBytes(zoo.getData(root + "/" + child, null));
         // upgrade from !0;!0<< -> +r<<
         e.extent = RootTable.EXTENT;
         result.add(e);
       } catch (KeeperException.NoNodeException ex) {
         continue;
       }
     }
     break;
   }
 }
コード例 #17
0
ファイル: TraceServer.java プロジェクト: harayz/accumulo
 private void registerInZooKeeper(String name) throws Exception {
   String root = ZooUtil.getRoot(serverConfiguration.getInstance()) + Constants.ZTRACERS;
   IZooReaderWriter zoo = ZooReaderWriter.getInstance();
   String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes());
   zoo.exists(path, this);
 }
コード例 #18
0
ファイル: Monitor.java プロジェクト: joshelser/accumulo
  /**
   * Get the monitor lock in ZooKeeper
   *
   * @throws KeeperException
   * @throws InterruptedException
   */
  private void getMonitorLock() throws KeeperException, InterruptedException {
    final String zRoot = ZooUtil.getRoot(instance);
    final String monitorPath = zRoot + Constants.ZMONITOR;
    final String monitorLockPath = zRoot + Constants.ZMONITOR_LOCK;

    // Ensure that everything is kosher with ZK as this has changed.
    ZooReaderWriter zoo = ZooReaderWriter.getInstance();
    if (zoo.exists(monitorPath)) {
      byte[] data = zoo.getData(monitorPath, null);
      // If the node isn't empty, it's from a previous install (has hostname:port for HTTP server)
      if (0 != data.length) {
        // Recursively delete from that parent node
        zoo.recursiveDelete(monitorPath, NodeMissingPolicy.SKIP);

        // And then make the nodes that we expect for the incoming ephemeral nodes
        zoo.putPersistentData(monitorPath, new byte[0], NodeExistsPolicy.FAIL);
        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
      } else if (!zoo.exists(monitorLockPath)) {
        // monitor node in ZK exists and is empty as we expect
        // but the monitor/lock node does not
        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
      }
    } else {
      // 1.5.0 and earlier
      zoo.putPersistentData(zRoot + Constants.ZMONITOR, new byte[0], NodeExistsPolicy.FAIL);
      if (!zoo.exists(monitorLockPath)) {
        // Somehow the monitor node exists but not monitor/lock
        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
      }
    }

    // Get a ZooLock for the monitor
    while (true) {
      MoniterLockWatcher monitorLockWatcher = new MoniterLockWatcher();
      monitorLock = new ZooLock(monitorLockPath);
      monitorLock.lockAsync(monitorLockWatcher, new byte[0]);

      monitorLockWatcher.waitForChange();

      if (monitorLockWatcher.acquiredLock) {
        break;
      }

      if (!monitorLockWatcher.failedToAcquireLock) {
        throw new IllegalStateException("monitor lock in unknown state");
      }

      monitorLock.tryToCancelAsyncLockOrUnlock();

      UtilWaitThread.sleep(
          getSystemConfiguration().getTimeInMillis(Property.MONITOR_LOCK_CHECK_INTERVAL));
    }

    log.info("Got Monitor lock.");
  }
コード例 #19
0
ファイル: Monitor.java プロジェクト: joshelser/accumulo
  public void run(String hostname) {
    try {
      getMonitorLock();
    } catch (Exception e) {
      log.error("Failed to get Monitor ZooKeeper lock");
      throw new RuntimeException(e);
    }

    Monitor.START_TIME = System.currentTimeMillis();
    int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
    try {
      log.debug("Creating monitor on port " + port);
      server = new EmbeddedWebServer(hostname, port);
    } catch (Throwable ex) {
      log.error("Unable to start embedded web server", ex);
      throw new RuntimeException(ex);
    }

    server.addServlet(DefaultServlet.class, "/");
    server.addServlet(OperationServlet.class, "/op");
    server.addServlet(MasterServlet.class, "/master");
    server.addServlet(TablesServlet.class, "/tables");
    server.addServlet(TServersServlet.class, "/tservers");
    server.addServlet(ProblemServlet.class, "/problems");
    server.addServlet(GcStatusServlet.class, "/gc");
    server.addServlet(LogServlet.class, "/log");
    server.addServlet(XMLServlet.class, "/xml");
    server.addServlet(JSONServlet.class, "/json");
    server.addServlet(VisServlet.class, "/vis");
    server.addServlet(Summary.class, "/trace/summary");
    server.addServlet(ListType.class, "/trace/listType");
    server.addServlet(ShowTrace.class, "/trace/show");
    if (server.isUsingSsl()) server.addServlet(ShellServlet.class, "/shell");
    server.start();

    try {
      hostname = InetAddress.getLocalHost().getHostName();

      log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");

      String monitorAddress = HostAndPort.fromParts(hostname, server.getPort()).toString();

      ZooReaderWriter.getInstance()
          .putPersistentData(
              ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR,
              monitorAddress.getBytes(StandardCharsets.UTF_8),
              NodeExistsPolicy.OVERWRITE);
      log.info("Set monitor address in zookeeper to " + monitorAddress);
    } catch (Exception ex) {
      log.error("Unable to set monitor HTTP address in zookeeper", ex);
    }

    if (null != hostname) {
      LogService.startLogListener(
          Monitor.getSystemConfiguration(), instance.getInstanceID(), hostname);
    } else {
      log.warn("Not starting log4j listener as we could not determine address to use");
    }

    new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();

    // need to regularly fetch data so plot data is updated
    new Daemon(
            new LoggingRunnable(
                log,
                new Runnable() {

                  @Override
                  public void run() {
                    while (true) {
                      try {
                        Monitor.fetchData();
                      } catch (Exception e) {
                        log.warn(e.getMessage(), e);
                      }

                      UtilWaitThread.sleep(333);
                    }
                  }
                }),
            "Data fetcher")
        .start();
  }