Ejemplo n.º 1
0
 TxRecord(final String persistenceContext, final String uuid, final StackTraceElement ste) {
   this.persistenceContext = persistenceContext;
   this.uuid = uuid;
   this.ste = ste;
   this.stack = Threads.currentStackString();
   this.startTime = System.currentTimeMillis();
 }
Ejemplo n.º 2
0
 private static CascadingTx createTransaction(final Object obj)
     throws RecoverablePersistenceException, RuntimeException {
   final String ctx = lookatPersistenceContext(obj);
   final CascadingTx ret = new CascadingTx(ctx);
   try {
     ret.begin();
     if (txRootThreadLocal.get() == null) {
       final String txId = makeTxRootName(ret);
       LOG.trace(
           "Creating root entry for transaction tree: "
               + txId
               + " at: \n"
               + Threads.currentStackString());
       txRootThreadLocal.set(txId);
     }
     txStateThreadLocal.get().put(ctx, ret);
     return ret;
   } catch (RuntimeException ex) {
     try {
       ret.rollback();
     } catch (RuntimeException ex1) {
       throw ex1;
     }
     throw ex;
   }
 }
Ejemplo n.º 3
0
 TransactionState(final String ctx) {
   this.startTime = System.currentTimeMillis();
   this.txUuid = String.format("%s:%s", ctx, UUID.randomUUID().toString());
   this.stopWatch = new StopWatch();
   this.stopWatch.start();
   this.owner = Logs.isExtrrreeeme() ? Threads.currentStackString() : "n/a";
   try {
     this.eventLog(TxStep.BEGIN, TxEvent.CREATE);
     final EntityManagerFactory anemf =
         (EntityManagerFactoryImpl) PersistenceContexts.getEntityManagerFactory(ctx);
     checkParam(anemf, notNullValue());
     this.em = anemf.createEntityManager();
     checkParam(this.em, notNullValue());
     this.transaction = this.em.getTransaction();
     this.transaction.begin();
     this.session = new WeakReference<Session>((Session) this.em.getDelegate());
     this.eventLog(TxStep.END, TxEvent.CREATE);
   } catch (final Throwable ex) {
     Logs.exhaust().error(ex, ex);
     this.eventLog(TxStep.FAIL, TxEvent.CREATE);
     this.rollback();
     throw new RuntimeException(PersistenceExceptions.throwFiltered(ex));
   } finally {
     outstanding.put(this.txUuid, this);
   }
 }
Ejemplo n.º 4
0
 public String getMessage() {
   if (Logs.isExtrrreeeme()) {
     return Threads.currentStackString();
   } else {
     return "n.a";
   }
 }
Ejemplo n.º 5
0
 @Override
 public void fireEvent(final ClockTick event) {
   if (Topology.isEnabledLocally(Eucalyptus.class) && ready.compareAndSet(true, false)) {
     try {
       Threads.enqueue(Eucalyptus.class, Volumes.class, this);
     } catch (final Exception ex) {
       ready.set(true);
     }
   }
 }
Ejemplo n.º 6
0
 public static <E, T> Predicate<T> asTransaction(final Predicate<T> predicate) {
   final List<Class> generics = Classes.genericsToClasses(predicate);
   for (final Class<?> type : generics) {
     if (PersistenceContexts.isPersistentClass(type)) {
       return asTransaction(type, predicate);
     }
   }
   throw new IllegalArgumentException(
       "Failed to find generics for provided predicate, cannot make into transaction: "
           + Threads.currentStackString());
 }
Ejemplo n.º 7
0
 /**
  * @delegate Do not change semantics here.
  * @see javax.persistence.EntityTransaction#commit()
  */
 @Override
 public void commit() throws RecoverablePersistenceException {
   removeTransaction(this);
   if ((this.txState != null) && this.txState.isActive()) {
     try {
       this.txState.commit();
     } catch (final RuntimeException ex) {
       throw PersistenceExceptions.throwFiltered(ex);
     }
   } else {
     Logs.extreme().error("Duplicate call to commit( ): " + Threads.currentStackString());
   }
 }
Ejemplo n.º 8
0
 /**
  * Private for a reason.
  *
  * @see {@link CascadingTx#get(Class)}
  * @param persistenceContext
  * @throws RecoverablePersistenceException
  */
 @SuppressWarnings("unchecked")
 CascadingTx(final String ctx) throws RecoverablePersistenceException {
   final StackTraceElement ste = Threads.currentStackFrame(4);
   final String uuid = UUID.randomUUID().toString();
   this.record = new TxRecord(ctx, uuid, ste);
   try {
     this.txState = new TxState(ctx);
   } catch (final RuntimeException ex) {
     Logs.extreme().error(ex, ex);
     this.rollback();
     throw PersistenceExceptions.throwFiltered(ex);
   }
 }
Ejemplo n.º 9
0
 public static <T, R> Function<T, R> asTransaction(final Function<T, R> function) {
   if (function instanceof TransactionalFunction) {
     return function;
   } else {
     final List<Class> generics = Classes.genericsToClasses(function);
     for (final Class<?> type : generics) {
       if (PersistenceContexts.isPersistentClass(type)) {
         return asTransaction(type, function);
       }
     }
     throw new IllegalArgumentException(
         "Failed to find generics for provided function, cannot make into transaction: "
             + Threads.currentStackString());
   }
 }
Ejemplo n.º 10
0
 public static void awaitSynchronized() {
   if (!isVolatile()) {
     return;
   } else {
     Collection<StackTraceElement> stack = Threads.filteredStack(stackFilter);
     String caller = (stack.isEmpty() ? "" : stack.iterator().next().toString());
     for (int i = 0; i < MAX_TX_START_SYNC_RETRIES && isVolatile(); i++) {
       try {
         TimeUnit.MILLISECONDS.sleep(1000);
         LOG.debug("Transaction blocked on sync: " + caller);
       } catch (InterruptedException ex) {
         Exceptions.maybeInterrupted(ex);
         return;
       }
     }
     if (isVolatile()) {
       throw new DatabaseStateException(
           "Transaction begin failed due to concurrent database synchronization: "
               + Hosts.listDatabases()
               + " for caller:\n"
               + Joiner.on("\n\tat ").join(stack));
     }
   }
 }
Ejemplo n.º 11
0
public class Databases {

  private static final Logger LOG = Logger.getLogger(Databases.class);
  private static final int MAX_DEACTIVATION_RETRIES = 10;
  private static final int MAX_TX_START_SYNC_RETRIES = 120;
  private static final AtomicInteger counter = new AtomicInteger(500);
  private static final Predicate<Host> FILTER_SYNCING_DBS =
      Predicates.and(DbFilter.INSTANCE, Predicates.not(SyncedDbFilter.INSTANCE));
  private static final ScriptedDbBootstrapper singleton = new ScriptedDbBootstrapper();
  private static final String jdbcJmxDomain = "net.sf.hajdbc";
  private static final ExecutorService dbSyncExecutors =
      Executors.newCachedThreadPool(); // NOTE:GRZE:special case thread handling.
  private static final ReentrantReadWriteLock canHas = new ReentrantReadWriteLock();

  private static Supplier<Set<String>> activeHosts =
      Suppliers.memoizeWithExpiration(ActiveHostSet.ACTIVATED, 2, TimeUnit.SECONDS);
  private static Supplier<Set<String>> hostDatabases =
      Suppliers.memoizeWithExpiration(ActiveHostSet.DBHOSTS, 1, TimeUnit.SECONDS);

  private static Predicate<StackTraceElement> notStackFilterYouAreLookingFor =
      Predicates.or(
          Threads.filterStackByQualifiedName("com\\.eucalyptus\\.entities\\..*"),
          Threads.filterStackByQualifiedName("java\\.lang\\.Thread.*"),
          Threads.filterStackByQualifiedName("com\\.eucalyptus\\.system\\.Threads.*"),
          Threads.filterStackByQualifiedName("com\\.eucalyptus\\.bootstrap\\.Databases.*"));
  private static Predicate<StackTraceElement> stackFilter =
      Predicates.not(notStackFilterYouAreLookingFor);

  private static final int DATABASE_WEIGHT_PRIMARY = 100;
  private static final int DATABASE_WEIGHT_SECONDARY = 1;

  public static class DatabaseStateException extends IllegalStateException {
    private static final long serialVersionUID = 1L;

    public DatabaseStateException(String string) {
      super(string);
    }
  }

  public enum Locks {
    DISABLED {
      @Override
      void isLocked() {
        File dbLockFile = this.getLockFile();
        if (dbLockFile.exists() && Hosts.isCoordinator()) {
          this.failStop();
        }
      }

      @Override
      public void failStop() {
        Faults.forComponent(Eucalyptus.class)
            .havingId(1010)
            .withVar(DB_LOCK_FILE, this.getLockFile().getAbsolutePath())
            .log();
        LOG.error(
            "WARNING : DISABLED CLC STARTED OUT OF ORDER, REMOVE THE "
                + this.getLockName()
                + "FILE TO PROCEED WITH RISK");
        System.exit(1);
      }
    },
    PARTITIONED {
      @Override
      void isLocked() {
        if (this.getLockFile().exists()) {
          failStop();
        }
      }

      @Override
      public void failStop() {
        Faults.forComponent(Eucalyptus.class)
            .havingId(1011)
            .withVar(DB_LOCK_FILE, this.getLockFile().getAbsolutePath())
            .log();
        LOG.error("PARTITION DETECTED -- FAIL-STOP TO AVOID POSSIBLE INCONSISTENCY.");
        LOG.error(
            "PARTITION DETECTED -- Shutting down CLC after experiencing a possible split-brain partition.");
        LOG.error("PARTITION DETECTED -- See cloud-fault.log for guidance.");
        System.exit(1);
      }

      @Override
      public void create() {
        super.create();
        Faults.forComponent(Eucalyptus.class)
            .havingId(1011)
            .withVar(DB_LOCK_FILE, this.getLockFile().getAbsolutePath())
            .log();
      }
    };
    public static final String DB_LOCK_FILE = "DB_LOCK_FILE";

    public void delete() {
      this.getLockFile().delete();
      LOG.debug("The " + this.getLockFile().getAbsolutePath() + " file was deleted");
    }

    protected String getLockName() {
      return this.name().toLowerCase() + ".lock";
    }

    abstract void isLocked();

    public abstract void failStop();

    protected File getLockFile() {
      return SubDirectory.DB.getChildFile("data", this.getLockName());
    }

    public void create(String reason) {
      LOG.error(this.getLockName() + ": Caused by: " + reason);
      this.create();
    }

    public void create() {
      try {
        if (getLockFile().createNewFile()) {
          LOG.debug(
              this.getLockName()
                  + ": The "
                  + this.getLockFile().getAbsolutePath()
                  + " file was created.");
        }
      } catch (IOException e) {
        LOG.debug(
            "Unable to create the "
                + this.getLockFile().getAbsolutePath()
                + " file: "
                + e.getMessage());
      }
    }
  }

  public enum Events {
    INSTANCE;

    public static Sql getConnection() throws Exception {
      return Databases.getBootstrapper().getConnection(INSTANCE.getName());
    }

    public String getName() {
      return "database_events";
    }

    public static void create() {
      if (!getBootstrapper().listDatabases().contains(INSTANCE.getName())) {
        try {
          getBootstrapper().createDatabase(INSTANCE.getName());
        } catch (Exception ex) {
          LOG.error(ex, ex);
        }
      }
    }
  }

  enum SyncState {
    IRRELEVANT,
    NOTSYNCED,
    SYNCING {

      @Override
      public boolean set() {
        return syncState.compareAndSet(NOTSYNCED, SYNCING);
      }
    },
    DESYNCING,
    SYNCED {

      @Override
      public boolean isCurrent() {
        if (Hosts.isCoordinator()) {
          syncState.set(this);
        }
        return super.isCurrent();
      }
    };
    private static final AtomicReference<SyncState> syncState =
        new AtomicReference<>(SyncState.NOTSYNCED);

    public static SyncState get() {
      return syncState.get();
    }

    public boolean set() {
      syncState.set(this);
      return true;
    }

    public boolean isCurrent() {
      return this.equals(syncState.get());
    }
  }

  enum ExecuteRunnable implements Function<Runnable, Future<Runnable>> {
    INSTANCE;

    @Override
    public Future<Runnable> apply(Runnable input) {
      Logs.extreme().debug("SUBMIT: " + input);
      return dbSyncExecutors.submit(input, input);
    }
  }

  @Provides(Empyrean.class)
  @RunDuring(Bootstrap.Stage.PoolInit)
  public static class DatabasePoolBootstrapper extends Bootstrapper.Simple {
    private static final int INITIAL_DB_SYNC_RETRY_WAIT = 5;

    @Override
    public boolean load() throws Exception {
      Hosts.awaitDatabases();
      Locks.DISABLED.isLocked();
      Locks.PARTITIONED.isLocked();

      Groovyness.run("setup_dbpool.groovy");
      OrderedShutdown.registerShutdownHook(
          Empyrean.class,
          new Runnable() {

            @Override
            public void run() {
              try {
                for (String ctx : PersistenceContexts.list()) {
                  try {
                    DatabaseClusterMBean db = Databases.lookup(ctx, TimeUnit.SECONDS.toMillis(5));
                    for (String host : db.getinactiveDatabases()) {
                      Databases.disable(host);
                    }
                    for (String host : db.getactiveDatabases()) {
                      Databases.disable(host);
                    }
                  } catch (Exception ex) {
                    LOG.error(ex);
                  }
                }
              } catch (NoSuchElementException ex) {
                LOG.error(ex);
              }
            }
          });
      TimeUnit.SECONDS.sleep(INITIAL_DB_SYNC_RETRY_WAIT);
      if (!Hosts.isCoordinator() && Hosts.localHost().hasDatabase()) {
        while (!Databases.enable(Hosts.localHost())) {
          LOG.warn(
              LogUtil.subheader("Synchronization of the database failed: " + Hosts.localHost()));
          if (counter.decrementAndGet() == 0) {
            LOG.fatal("Restarting process to force re-synchronization.");
            System.exit(123);
          } else {
            LOG.warn(
                "Sleeping for " + INITIAL_DB_SYNC_RETRY_WAIT + " seconds before trying again.");
            TimeUnit.SECONDS.sleep(INITIAL_DB_SYNC_RETRY_WAIT);
          }
        }

        Locks.DISABLED.create();

        Hosts.UpdateEntry.INSTANCE.apply(Hosts.localHost());
        LOG.info(LogUtil.subheader("Database synchronization complete: " + Hosts.localHost()));
      }
      return true;
    }

    @Override
    public boolean check() throws Exception {
      if (Bootstrap.isOperational()
          && !Hosts.localHost().hasDatabase()) { // This check is redundant on hosts with DBs
        final List<Host.DBStatus> statusList = Hosts.localHost().getDatabaseStatus();
        for (final Host.DBStatus status : statusList) {
          for (String error : status.getError().asSet()) {
            LOG.error(error);
          }
        }
      }
      return super.check();
    }
  }

  private static void runDbStateChange(Function<String, Runnable> runnableFunction) {
    Logs.extreme().info("DB STATE CHANGE: " + runnableFunction);
    try {
      Logs.extreme().info("Attempting to acquire db state lock: " + runnableFunction);
      if (canHas.writeLock().tryLock(5, TimeUnit.MINUTES)) {

        try {
          Logs.extreme().info("Acquired db state lock: " + runnableFunction);
          Map<Runnable, Future<Runnable>> runnables = Maps.newHashMap();
          for (final String ctx : listDatabases()) {
            Runnable run = runnableFunction.apply(ctx);
            runnables.put(run, ExecuteRunnable.INSTANCE.apply(run));
          }
          Map<Runnable, Future<Runnable>> succeeded = Futures.waitAll(runnables);
          MapDifference<Runnable, Future<Runnable>> failed = Maps.difference(runnables, succeeded);
          StringBuilder builder = new StringBuilder();
          builder.append(Joiner.on("\nSUCCESS: ").join(succeeded.keySet()));
          builder.append(Joiner.on("\nFAILED:  ").join(failed.entriesOnlyOnLeft().keySet()));
          Logs.extreme().debug(builder.toString());
          if (!failed.entriesOnlyOnLeft().isEmpty()) {
            throw Exceptions.toUndeclared(builder.toString());
          }
        } finally {
          canHas.writeLock().unlock();
        }
      } else {
        throw new LockTimeoutException(
            "DB STATE CHANGE ABORTED (failed to get lock): " + runnableFunction);
      }
    } catch (RuntimeException ex) {
      LOG.error(ex);
      Logs.extreme().error(ex, ex);
      throw ex;
    } catch (InterruptedException ex) {
      Exceptions.maybeInterrupted(ex);
      throw Exceptions.toUndeclared(ex);
    }
  }

  enum DeactivateHostFunction implements Function<String, Function<String, Runnable>> {
    INSTANCE;
    /** @see Function#apply(Object) */
    @Override
    public Function<String, Runnable> apply(final String hostName) {
      return new Function<String, Runnable>() {
        @Override
        public Runnable apply(final String contextName) {
          return new Runnable() {
            @Override
            public void run() {
              if (Internets.testLocal(hostName)) {
                return;
              }
              try {
                final boolean wasPrimary;
                try {
                  wasPrimary =
                      lookupDatabase(contextName, hostName).getweight() == DATABASE_WEIGHT_PRIMARY;
                } catch (Exception ex1) {
                  return;
                }

                LOG.info(
                    "Tearing down database connections for: "
                        + hostName
                        + " to context: "
                        + contextName);
                final DatabaseClusterMBean cluster =
                    lookup(contextName, TimeUnit.SECONDS.toMillis(5));

                // deactivate database
                for (int i = 0;
                    i < MAX_DEACTIVATION_RETRIES && cluster.getactiveDatabases().contains(hostName);
                    i++)
                  try {
                    LOG.debug(
                        "Deactivating database connections for: "
                            + hostName
                            + " to context: "
                            + contextName);
                    cluster.deactivate(hostName);
                    LOG.debug(
                        "Deactived database connections for: "
                            + hostName
                            + " to context: "
                            + contextName);
                    break;
                  } catch (Exception ex) {
                    LOG.error(ex);
                    Logs.extreme().error(ex, ex);
                  }

                // remove database
                for (int i = 0;
                    i < MAX_DEACTIVATION_RETRIES
                        && cluster.getinactiveDatabases().contains(hostName)
                        && !Hosts.contains(hostName);
                    i++)
                  try {
                    LOG.debug(
                        "Removing database registration for: "
                            + hostName
                            + " to context: "
                            + contextName);
                    cluster.remove(hostName);
                    LOG.debug(
                        "Removed database registration for: "
                            + hostName
                            + " to context: "
                            + contextName);
                    break;
                  } catch (Exception ex) {
                    LOG.error(ex);
                    Logs.extreme().error(ex, ex);
                  }

                // promote other
                final Host coordinator = Hosts.getCoordinator();
                if (wasPrimary && coordinator != null) {
                  try {
                    lookupDatabase(contextName, coordinator.getDisplayName())
                        .setweight(DATABASE_WEIGHT_PRIMARY);
                  } catch (NoSuchElementException e) {
                    // Weight will be set on activation
                  } catch (Exception e) {
                    LOG.error(
                        "Error setting primary weight for host "
                            + coordinator.getDisplayName()
                            + " context "
                            + contextName,
                        e);
                  }
                }

                // refresh pooled connections to ensure closed if removed
                for (int i = 0;
                    i < MAX_DEACTIVATION_RETRIES
                        && !cluster.getactiveDatabases().contains(hostName)
                        && !cluster.getinactiveDatabases().contains(hostName);
                    i++)
                  try {
                    // Release any open connections
                    LOG.debug("Refreshing idle pooled connections for context: " + contextName);
                    ProxoolFacade.killAllConnections(contextName, "Database deregistered", true);
                    LOG.debug("Refreshed idle pooled connections for context: " + contextName);
                    break;
                  } catch (Exception ex) {
                    LOG.error(ex);
                    Logs.extreme().error(ex, ex);
                  }
              } catch (final Exception ex1) {
                LOG.error(ex1);
                Logs.extreme().error(ex1, ex1);
              }
            }

            @Override
            public String toString() {
              return "Databases.disable(): " + hostName + " " + contextName;
            }
          };
        }

        @Override
        public String toString() {
          return "Databases.disable(): " + hostName;
        }
      };
    }

    @Override
    public String toString() {
      return "Databases.disable()";
    }
  }

  enum ActivateHostFunction implements Function<Host, Function<String, Runnable>> {
    INSTANCE;

    private static void prepareConnections(final Host host, final String contextName)
        throws NoSuchElementException {
      final String dbUrl =
          "jdbc:" + ServiceUris.remote(Database.class, host.getBindAddress(), contextName);
      final String hostName = host.getDisplayName();
      final DriverDatabaseMBean database = Databases.lookupDatabase(contextName, hostName);
      database.setuser(getUserName());
      database.setpassword(getPassword());
      database.setweight(
          Hosts.isCoordinator(host) ? DATABASE_WEIGHT_PRIMARY : DATABASE_WEIGHT_SECONDARY);
      database.setlocal(host.isLocalHost());
      database.setlocation(dbUrl);
    }

    @Override
    public Function<String, Runnable> apply(final Host host) {
      return new Function<String, Runnable>() {
        @Override
        public Runnable apply(final String ctx) {
          final String hostName = host.getBindAddress().getHostAddress();
          final String contextName = ctx;
          Runnable removeRunner =
              new Runnable() {
                @Override
                public void run() {
                  try {
                    final boolean fullSync =
                        !Hosts.isCoordinator()
                            && host.isLocalHost()
                            && BootstrapArgs.isCloudController()
                            && !Databases.isSynchronized();
                    final boolean passiveSync = !fullSync && host.hasSynced();
                    if (!fullSync && !passiveSync) {
                      throw Exceptions.toUndeclared("Host is not ready to be activated: " + host);
                    } else {
                      final DatabaseClusterMBean cluster =
                          lookup(contextName, TimeUnit.SECONDS.toMillis(30));
                      final boolean activated = cluster.getactiveDatabases().contains(hostName);
                      final boolean deactivated = cluster.getinactiveDatabases().contains(hostName);
                      final String passiveStrategy = cluster.getdefaultSynchronizationStrategy();
                      final String fullStrategy =
                          Iterables.getFirst(
                              Sets.difference(
                                  cluster.getsynchronizationStrategies(),
                                  Collections.singleton(passiveStrategy)),
                              "full");
                      final String syncStrategy = fullSync ? fullStrategy : passiveStrategy;
                      if (activated) {
                        resetDatabaseWeights(contextName);
                        return;
                      } else if (deactivated) {
                        ActivateHostFunction.prepareConnections(host, contextName);
                      } else {
                        LOG.info("Creating database " + ctx + " connections for: " + host);
                        try {
                          lookupDatabase(contextName, hostName);
                        } catch (NoSuchElementException e) {
                          try {
                            cluster.add(hostName);
                            Logs.extreme()
                                .debug(
                                    "Added database " + ctx + " connections for host: " + hostName);
                          } catch (IllegalArgumentException ex) {
                            Logs.extreme()
                                .debug(
                                    "Skipping addition of database "
                                        + ctx
                                        + " connections for host which already exists: "
                                        + hostName);
                          } catch (IllegalStateException ex) {
                            if (Exceptions.isCausedBy(ex, InstanceAlreadyExistsException.class)) {
                              ManagementFactory.getPlatformMBeanServer()
                                  .unregisterMBean(
                                      new ObjectName(
                                          jdbcJmxDomain,
                                          new Hashtable<>(
                                              ImmutableMap.of(
                                                  "cluster",
                                                  ctx,
                                                  "type",
                                                  "Database",
                                                  "database",
                                                  hostName))));
                              cluster.add(hostName);
                            } else {
                              throw ex;
                            }
                          }
                        }
                        ActivateHostFunction.prepareConnections(host, contextName);
                      }

                      // demote others
                      if (Hosts.isCoordinator(host)) {
                        for (Host secondaryHost : Hosts.listActiveDatabases())
                          if (!secondaryHost.equals(host))
                            try {
                              lookupDatabase(contextName, secondaryHost.getDisplayName())
                                  .setweight(DATABASE_WEIGHT_SECONDARY);
                            } catch (NoSuchElementException e) {
                              // Weight will be set on activation
                            } catch (Exception e) {
                              LOG.error(
                                  "Error setting primary weight for host "
                                      + secondaryHost.getDisplayName()
                                      + " context "
                                      + contextName,
                                  e);
                            }
                      }

                      try {
                        if (fullSync) {
                          LOG.info(
                              "Full sync of database "
                                  + ctx
                                  + " on: "
                                  + host
                                  + " using: "
                                  + fullStrategy);
                        } else {
                          LOG.info(
                              "Passive activation of database " + ctx + " connections to: " + host);
                        }
                        cluster.activate(hostName, syncStrategy);
                        if (fullSync) {
                          LOG.info(
                              "Full sync of database "
                                  + ctx
                                  + " on: "
                                  + host
                                  + " using "
                                  + cluster.getactiveDatabases());
                        } else {
                          LOG.info(
                              "Passive activation of database "
                                  + ctx
                                  + " on: "
                                  + host
                                  + " using "
                                  + cluster.getactiveDatabases());
                        }
                      } catch (Exception ex) {
                        throw Exceptions.toUndeclared(ex);
                      }

                      // refresh pooled connections
                      try {
                        // Release any open connections
                        LOG.debug("Refreshing idle pooled connections for context: " + contextName);
                        ProxoolFacade.killAllConnections(contextName, "Database registered", true);
                        LOG.debug("Refreshed idle pooled connections for context: " + contextName);
                      } catch (Exception ex) {
                        LOG.error(
                            "Error refreshing connections on activation of context: " + contextName,
                            ex);
                      }
                    }
                  } catch (final NoSuchElementException ex1) {
                    LOG.error(ex1);
                    Logs.extreme().error(ex1, ex1);
                    return;
                  } catch (final IllegalStateException ex1) {
                    LOG.error(ex1);
                    Logs.extreme().error(ex1, ex1);
                    return;
                  } catch (final Exception ex1) {
                    LOG.error(ex1);
                    Logs.extreme().error(ex1, ex1);
                    throw Exceptions.toUndeclared(
                        "Failed to activate database "
                            + ctx
                            + " host "
                            + host
                            + " because of: "
                            + ex1.getMessage(),
                        ex1);
                  }
                }

                @Override
                public String toString() {
                  return "Databases.enable(): " + host.getDisplayName() + " " + contextName;
                }
              };
          return removeRunner;
        }

        @Override
        public String toString() {
          return "Databases.enable(): " + host;
        }
      };
    }

    @Override
    public String toString() {
      return "Databases.enable()";
    }

    private static void rollback(final Host host, Exception ex) {
      try {
        Databases.runDbStateChange(
            Databases.DeactivateHostFunction.INSTANCE.apply(host.getDisplayName()));
      } catch (LockTimeoutException ex1) {
        Databases.LOG.error("Databases.enable(): failed because of: " + ex.getMessage());
      } catch (Exception ex1) {
        Databases.LOG.error("Databases.enable(): failed because of: " + ex.getMessage());
        Logs.extreme().error(ex, ex);
      }
    }
  }

  private static void resetDatabaseWeights(final String contextName) {
    for (final Host host : Hosts.listActiveDatabases())
      try {
        lookupDatabase(contextName, host.getDisplayName())
            .setweight(
                Hosts.isCoordinator(host) ? DATABASE_WEIGHT_PRIMARY : DATABASE_WEIGHT_SECONDARY);
      } catch (NoSuchElementException e) {
        // Weight will be set on activation
      } catch (Exception e) {
        LOG.error(
            "Error resetting weight for host " + host.getDisplayName() + " context " + contextName,
            e);
      }
  }

  private static DatabaseClusterMBean lookup(final String ctx, final long timeout)
      throws NoSuchElementException {
    return lookupMBean(
        ImmutableMap.of("cluster", ctx, "type", "DatabaseCluster"),
        DatabaseClusterMBean.class,
        new Predicate<DatabaseClusterMBean>() {
          @Override
          public boolean apply(final DatabaseClusterMBean cluster) {
            cluster.getinactiveDatabases();
            return true;
          }
        },
        timeout);
  }

  private static DriverDatabaseMBean lookupDatabase(final String contextName, final String hostName)
      throws NoSuchElementException {
    return lookupMBean(
        ImmutableMap.of("cluster", contextName, "type", "Database", "database", hostName),
        DriverDatabaseMBean.class,
        new Predicate<DriverDatabaseMBean>() {
          @Override
          public boolean apply(final DriverDatabaseMBean database) {
            database.getid();
            return true;
          }
        },
        0);
  }

  private static <T> T lookupMBean(
      final Map<String, String> props,
      final Class<T> type,
      final Predicate<T> tester,
      final long timeout) {
    long until = System.currentTimeMillis() + timeout;
    do
      try {
        final T bean = Mbeans.lookup(jdbcJmxDomain, props, type);
        tester.apply(bean);
        return bean;
      } catch (UndeclaredThrowableException e) {
        if (Exceptions.isCausedBy(e, InstanceNotFoundException.class)) {
          if (System.currentTimeMillis() < until) {
            try {
              TimeUnit.SECONDS.sleep(5);
            } catch (InterruptedException e1) {
              Thread.interrupted();
              break;
            }
            LOG.debug("Waiting for MBean " + type.getSimpleName() + "/" + props);
            continue;
          }
          throw new NoSuchElementException(type.getSimpleName() + " " + props.toString());
        } else {
          throw Exceptions.toUndeclared(e);
        }
      }
    while (System.currentTimeMillis() < until);

    throw new NoSuchElementException(type.getSimpleName() + " " + props.toString());
  }

  static boolean disable(final String hostName) {
    if (!Bootstrap.isFinished()) {
      return false;
    } else if (Internets.testLocal(hostName)) {
      return true;
    } else {
      try {
        runDbStateChange(DeactivateHostFunction.INSTANCE.apply(hostName));
        return true;
      } catch (Exception ex) {
        Logs.extreme().debug(ex);
        return false;
      }
    }
  }

  static boolean enable(final Host host) {
    if (!host.hasDatabase() || Bootstrap.isShuttingDown()) {
      return false;
    } else if (!Hosts.contains(host.getGroupsId())) {
      Hosts.remove(host.getGroupsId());
      return false;
    } else {
      if (host.isLocalHost()) {
        if (SyncState.SYNCING.set()) {
          try {
            runDbStateChange(ActivateHostFunction.INSTANCE.apply(host));
            SyncState.SYNCED.set();
            return true;
          } catch (LockTimeoutException ex) {
            SyncState.NOTSYNCED.set();
            return false;
          } catch (Exception ex) {
            SyncState.NOTSYNCED.set();
            LOG.error(ex);
            Logs.extreme().error(ex, ex);
            return false;
          }
        } else if (!SyncState.SYNCING.isCurrent()) {
          try {
            runDbStateChange(ActivateHostFunction.INSTANCE.apply(host));
            return true;
          } catch (LockTimeoutException ex) {
            return false;
          } catch (Exception ex) {
            LOG.error(ex);
            Logs.extreme().error(ex, ex);
            return false;
          }
        } else {
          try {
            runDbStateChange(ActivateHostFunction.INSTANCE.apply(host));
            SyncState.SYNCED.set();
            return true;
          } catch (LockTimeoutException ex) {
            SyncState.NOTSYNCED.set();
            return false;
          } catch (Exception ex) {
            SyncState.NOTSYNCED.set();
            LOG.error(ex, ex);
            return false;
          }
        }
      } else if (!ActiveHostSet.ACTIVATED.get().contains(host.getDisplayName())) {
        try {
          runDbStateChange(ActivateHostFunction.INSTANCE.apply(host));
          return true;
        } catch (LockTimeoutException ex) {
          return false;
        } catch (Exception ex) {
          Logs.extreme().debug(ex);
          ActivateHostFunction.rollback(host, ex);
          return false;
        }
      } else {
        return ActiveHostSet.ACTIVATED.get().contains(host.getDisplayName());
      }
    }
  }

  static boolean shouldInitialize() { // GRZE:WARNING:HACKHACKHACK do not duplicate pls thanks.
    for (final Host h : Hosts.listActiveDatabases()) {
      final String url =
          String.format(
              "jdbc:%s",
              ServiceUris.remote(Database.class, h.getBindAddress(), "eucalyptus_config"));
      try {
        final Connection conn =
            DriverManager.getConnection(url, Databases.getUserName(), Databases.getPassword());
        try {
          final PreparedStatement statement =
              conn.prepareStatement(
                  "select config_component_hostname from config_component_base where config_component_partition='eucalyptus';");
          final ResultSet result = statement.executeQuery();
          while (result.next()) {
            final Object columnValue = result.getObject(1);
            if (Internets.testLocal(columnValue.toString())) {
              return true;
            }
          }
        } finally {
          conn.close();
        }
      } catch (final Exception ex) {
        LOG.error(ex, ex);
      }
    }
    return false;
  }

  static Set<String> listPrimaryActiveDatabases(final String cluster) {
    return listDatabases(cluster, true, Optional.of(DATABASE_WEIGHT_PRIMARY));
  }

  static Set<String> listSecondaryActiveDatabases(final String cluster) {
    return listDatabases(cluster, true, Optional.of(DATABASE_WEIGHT_SECONDARY));
  }

  static Set<String> listInactiveDatabases(final String cluster) {
    return listDatabases(cluster, false, Optional.<Integer>absent());
  }

  private static Set<String> listDatabases(
      final String clusterName, final boolean active, final Optional<Integer> weight) {
    final Set<String> databases = Sets.newLinkedHashSet();
    try {
      final DatabaseClusterMBean cluster = lookup(clusterName, 0);
      final Iterable<String> databaseIds =
          active ? cluster.getactiveDatabases() : cluster.getinactiveDatabases();
      for (final String databaseId : databaseIds)
        try {
          final DriverDatabaseMBean database = lookupDatabase(clusterName, databaseId);
          if (!weight.isPresent() || database.getweight() == weight.get()) {
            databases.add(databaseId);
          }
        } catch (NoSuchElementException e) {
          // ignore database
        }
    } catch (NoSuchElementException e) {
      // no databases
    } catch (Exception e) {
      LOG.error(e, e);
    }
    return databases;
  }

  static Set<String> listRegisteredDatabases() {
    return Mbeans.listPropertyValues(
        jdbcJmxDomain, "cluster", ImmutableMap.of("type", "DatabaseCluster"));
  }

  /** List all known databases. */
  static Set<String> listDatabases() {
    final Set<String> dbNames = Sets.newHashSet();
    final Predicate<String> dbNamePredicate =
        Predicates.or(Strings.startsWith("eucalyptus_"), Predicates.equalTo("database_events"));

    for (final Host h : Hosts.listActiveDatabases()) {
      Iterables.addAll(
          dbNames,
          Iterables.filter(
              Databases.getBootstrapper().listDatabases(h.getBindAddress()), dbNamePredicate));
    }

    return dbNames;
  }

  /** @return LockTimeoutException */
  public static Boolean isSynchronized() {
    return SyncState.SYNCED.isCurrent();
  }

  public static Boolean isVolatile() {
    if (!Bootstrap.isFinished() || BootstrapArgs.isInitializeSystem()) {
      return false;
    } else if (!Hosts.isCoordinator() && BootstrapArgs.isCloudController()) {
      return !isSynchronized() || !activeHosts.get().containsAll(hostDatabases.get());
    } else if (!activeHosts.get().equals(hostDatabases.get())) {
      return true;
    } else {
      return !Hosts.list(FILTER_SYNCING_DBS).isEmpty();
    }
  }

  enum ActiveHostSet implements Supplier<Set<String>> {
    ACTIVATED {
      @Override
      public Set<String> get() {
        Set<String> hosts = DBHOSTS.get();
        Set<String> union = Sets.newHashSet();
        Set<String> intersection = Sets.newHashSet(hosts);
        Logs.extreme().debug("ActiveHostSet: universe of db hosts: " + hosts);
        for (String ctx : PersistenceContexts.list()) {
          try {
            Set<String> activeDatabases = Databases.lookup(ctx, 0).getactiveDatabases();
            if (BootstrapArgs.isCloudController()) {
              activeDatabases.add(
                  Internets
                      .localHostIdentifier()); // GRZE: use Internets.localHostIdentifier() which is
                                               // static, rather than the Hosts reference as it is
                                               // stateful
            }
            union.addAll(activeDatabases);
            intersection.retainAll(activeDatabases);
          } catch (Exception ex) {
          }
        }
        Logs.extreme().debug("ActiveHostSet: union of activated db connections: " + union);
        Logs.extreme()
            .debug(
                "ActiveHostSet: intersection of db hosts and activated db connections: "
                    + intersection);
        boolean dbVolatile = !hosts.equals(intersection);
        String msg =
            String.format(
                "ActiveHostSet: %-14.14s %s%s%s",
                dbVolatile ? "volatile" : "synchronized",
                hosts,
                dbVolatile ? "!=" : "=",
                intersection);
        if (dbVolatile) {
          if (last.compareAndSet(false, dbVolatile)) {
            LOG.warn(msg);
          } else {
            LOG.debug(msg);
          }
        } else {
          if (last.compareAndSet(true, dbVolatile)) {
            LOG.warn(msg);
          } else {
            Logs.extreme().info(msg);
          }
        }
        return intersection;
      }
    },
    DBHOSTS {
      @Override
      public Set<String> get() {
        return Sets.newHashSet(
            Collections2.transform(Hosts.listDatabases(), Hosts.NameTransform.INSTANCE));
      }
    };
    private static final AtomicBoolean last = new AtomicBoolean(false);

    @Override
    public abstract Set<String> get();
  }

  public static void awaitSynchronized() {
    if (!isVolatile()) {
      return;
    } else {
      Collection<StackTraceElement> stack = Threads.filteredStack(stackFilter);
      String caller = (stack.isEmpty() ? "" : stack.iterator().next().toString());
      for (int i = 0; i < MAX_TX_START_SYNC_RETRIES && isVolatile(); i++) {
        try {
          TimeUnit.MILLISECONDS.sleep(1000);
          LOG.debug("Transaction blocked on sync: " + caller);
        } catch (InterruptedException ex) {
          Exceptions.maybeInterrupted(ex);
          return;
        }
      }
      if (isVolatile()) {
        throw new DatabaseStateException(
            "Transaction begin failed due to concurrent database synchronization: "
                + Hosts.listDatabases()
                + " for caller:\n"
                + Joiner.on("\n\tat ").join(stack));
      }
    }
  }

  public static String getUserName() {
    return singleton.getUserName();
  }

  public static String getPassword() {
    return singleton.getPassword();
  }

  public static String getDriverName() {
    return singleton.getDriverName();
  }

  public static String getJdbcDialect() {
    return singleton.getJdbcDialect();
  }

  public static String getHibernateDialect() {
    return singleton.getHibernateDialect();
  }

  public static DatabaseBootstrapper getBootstrapper() {
    return singleton;
  }

  public static void initialize() {
    singleton.init();
  }

  @RunDuring(Bootstrap.Stage.DatabaseInit)
  @Provides(Empyrean.class)
  @DependsLocal(Eucalyptus.class)
  public static class ScriptedDbBootstrapper extends Bootstrapper.Simple
      implements DatabaseBootstrapper {
    DatabaseBootstrapper db;

    public ScriptedDbBootstrapper() {
      try {
        this.db = Groovyness.newInstance("setup_db");
      } catch (ScriptExecutionFailedException ex) {
        LOG.error(ex, ex);
      }
    }

    @Override
    public boolean load() throws Exception {
      boolean result = this.db.load();
      Databases.Events.create();
      return result;
    }

    @Override
    public boolean start() throws Exception {
      return this.db.start();
    }

    @Override
    public boolean stop() throws Exception {
      return this.db.stop();
    }

    @Override
    public void destroy() throws Exception {
      this.db.destroy();
    }

    @Override
    public boolean isRunning() {
      return this.db.isRunning();
    }

    @Override
    public void hup() {
      this.db.hup();
    }

    @Override
    public String getUserName() {
      return db.getUserName();
    }

    @Override
    public String getPassword() {
      return db.getPassword();
    }

    @Override
    public String getDriverName() {
      return this.db.getDriverName();
    }

    @Override
    public String getJdbcDialect() {
      return this.db.getJdbcDialect();
    }

    @Override
    public String getHibernateDialect() {
      return this.db.getHibernateDialect();
    }

    @Override
    public void init() {
      try {
        this.db.init();
      } catch (Exception ex) {
        LOG.error(ex, ex);
      }
    }

    public static DatabaseBootstrapper getInstance() {
      return singleton;
    }

    @Override
    public String getServicePath(String... pathParts) {
      return this.db.getServicePath(pathParts);
    }

    @Override
    public Map<String, String> getJdbcUrlQueryParameters() {
      return this.db.getJdbcUrlQueryParameters();
    }

    @Override
    public boolean check() throws Exception {
      return this.db.isRunning();
    }

    /** @see DatabaseBootstrapper#getJdbcScheme() */
    @Override
    public String getJdbcScheme() {
      return this.db.getJdbcScheme();
    }

    /** @see DatabaseBootstrapper#listDatabases() */
    @Override
    public List<String> listDatabases() {
      return this.db.listDatabases();
    }

    /** @see DatabaseBootstrapper#listDatabases(InetAddress) */
    @Override
    public List<String> listDatabases(InetAddress host) {
      return this.db.listDatabases(host);
    }

    /** @see DatabaseBootstrapper#listTables(String) */
    @Override
    public List<String> listTables(String database) {
      return this.db.listTables(database);
    }

    /** @see DatabaseBootstrapper#backupDatabase(String,String) */
    @Override
    public File backupDatabase(String name, String backupIdentifier) {
      return this.db.backupDatabase(name, backupIdentifier);
    }

    /** @see DatabaseBootstrapper#deleteDatabase(String) */
    @Override
    public void deleteDatabase(String name) {
      this.db.deleteDatabase(name);
    }

    /** @see DatabaseBootstrapper#copyDatabase(String, String) */
    @Override
    public void copyDatabase(String from, String to) {
      this.db.copyDatabase(from, to);
    }

    /** @see DatabaseBootstrapper#renameDatabase(String, String) */
    @Override
    public void renameDatabase(String from, String to) {
      this.db.renameDatabase(from, to);
    }

    /** @see DatabaseBootstrapper#getConnection(String) */
    @Override
    public Sql getConnection(String database) throws Exception {
      return this.db.getConnection(database);
    }

    /** @see DatabaseBootstrapper#createDatabase(String) */
    @Override
    public void createDatabase(String name) {
      this.db.createDatabase(name);
    }
  }

  public static boolean isRunning() {
    try {
      return singleton.check();
    } catch (Exception ex) {
      LOG.error(ex, ex);
      return false;
    }
  }

  public static String getServicePath(String... pathParts) {
    return singleton.getServicePath(pathParts);
  }

  public static Map<String, String> getJdbcUrlQueryParameters() {
    return singleton.getJdbcUrlQueryParameters();
  }

  public static String getJdbcScheme() {
    return singleton.getJdbcScheme();
  }

  public static void check() {
    for (String ctx : PersistenceContexts.list()) {
      try {
        DatabaseClusterMBean db = lookup(ctx, TimeUnit.SECONDS.toMillis(5));
        for (String host : db.getactiveDatabases()) {
          Host hostEntry = Hosts.lookup(host);
          if (hostEntry == null) {
            disable(host);
          } else if (!Hosts.contains(hostEntry.getGroupsId())) {
            Hosts.remove(host); // GRZE: this will clean up group state and de-activate db.
          }
        }
      } catch (NoSuchElementException ex) {
        LOG.error(ex, ex);
      }
      return;
    }
  }

  /** HA-JDBC Dynamic MBean proxy interface */
  private interface DatabaseClusterMBean {

    Set<String> getinactiveDatabases();

    Set<String> getactiveDatabases();

    void add(String hostName);

    void remove(String hostName);

    void activate(String hostName, String syncStrategy);

    void deactivate(String hostName);

    Set<String> getsynchronizationStrategies();

    String getdefaultSynchronizationStrategy();
  }

  /** HA-JDBC Dynamic MBean proxy interface */
  private interface DriverDatabaseMBean {

    boolean isActive();

    String getid();

    void setuser(String userName);

    void setpassword(String password);

    int getweight();

    void setweight(int i);

    void setlocal(boolean localHost);

    void setlocation(String url);
  }
}
Ejemplo n.º 12
0
 private static void broadcastTask(Callable<Void> task) {
   Threads.enqueue(Eucalyptus.class, NetworkInfoBroadcaster.class, 5, task);
 }
  @Override
  public void fireEvent(final Hertz event) {
    final long defaultPollIntervalSeconds = TimeUnit.MINUTES.toSeconds(DEFAULT_POLL_INTERVAL_MINS);
    if (!Bootstrap.isOperational()
        || !BootstrapArgs.isCloudController()
        || !event.isAsserted(defaultPollIntervalSeconds)) {
      return;
    } else {
      if (DEFAULT_POLL_INTERVAL_MINS >= 1) {
        COLLECTION_INTERVAL_TIME_MS =
            ((int) TimeUnit.MINUTES.toMillis(DEFAULT_POLL_INTERVAL_MINS) / 2);
      } else {
        COLLECTION_INTERVAL_TIME_MS = 0;
      }

      if (COLLECTION_INTERVAL_TIME_MS == 0 || HISTORY_SIZE > 15 || HISTORY_SIZE < 1) {
        LOG.debug("The instance usage report is disabled");
      } else if (COLLECTION_INTERVAL_TIME_MS <= MAX_WRITE_INTERVAL_MS) {

        try {
          if (event.isAsserted(defaultPollIntervalSeconds)) {
            if (Bootstrap.isFinished() && Hosts.isCoordinator()) {
              CloudWatchHelper.DefaultInstanceInfoProvider.refresh();
              for (final ServiceConfiguration ccConfig :
                  Topology.enabledServices(ClusterController.class)) {
                final String ccHost = ccConfig.getHostName();
                if (busyHosts.replace(ccHost, false, true)
                    || busyHosts.putIfAbsent(ccHost, true) == null) {
                  Threads.lookup(Reporting.class, DescribeSensorsListener.class)
                      .submit(
                          new Callable<Object>() {

                            @Override
                            public Object call() throws Exception {
                              final ExecutorService executorService =
                                  Threads.lookup(
                                          Reporting.class,
                                          DescribeSensorsListener.class,
                                          "response-processing")
                                      .limitTo(4);
                              final long startTime = System.currentTimeMillis();
                              try {
                                final List<String> allInstanceIds =
                                    VmInstances.listWithProjection(
                                        VmInstances.instanceIdProjection(),
                                        VmInstance.criterion(VmState.RUNNING),
                                        VmInstance.zoneCriterion(ccConfig.getPartition()),
                                        VmInstance.nonNullNodeCriterion());
                                final Iterable<List<String>> processInts =
                                    Iterables.partition(allInstanceIds, SENSOR_QUERY_BATCH_SIZE);
                                for (final List<String> instIds : processInts) {
                                  final ArrayList<String> instanceIds = Lists.newArrayList(instIds);
                                  /**
                                   * Here this is hijacking the sensor callback in order to control
                                   * the thread of execution used when firing
                                   */
                                  final DescribeSensorCallback msgCallback =
                                      new DescribeSensorCallback(
                                          HISTORY_SIZE, COLLECTION_INTERVAL_TIME_MS, instanceIds) {
                                        @Override
                                        public void fireException(Throwable e) {}

                                        @Override
                                        public void fire(DescribeSensorsResponse msg) {}
                                      };
                                  /**
                                   * Here we actually get the future reference to the result and on
                                   * a response processing thread, invoke .fire().
                                   */
                                  final DescribeSensorsResponse response =
                                      AsyncRequests.newRequest(msgCallback)
                                          .dispatch(ccConfig)
                                          .get();
                                  executorService.submit(
                                      new Runnable() {
                                        @Override
                                        public void run() {
                                          try {
                                            new DescribeSensorCallback(
                                                    HISTORY_SIZE,
                                                    COLLECTION_INTERVAL_TIME_MS,
                                                    instanceIds)
                                                .fire(response);
                                          } catch (Exception e) {
                                            Exceptions.maybeInterrupted(e);
                                          }
                                        }
                                      });
                                }
                              } finally {
                                /** Only and finally set the busy bit back to false. */
                                busyHosts.put(ccHost, false);
                                LOG.debug(
                                    "Sensor polling for "
                                        + ccHost
                                        + " took "
                                        + (System.currentTimeMillis() - startTime)
                                        + "ms");
                              }
                              return null;
                            }
                          });
                } else {
                  LOG.warn(
                      "Skipping sensors polling for " + ccHost + ", previous poll not complete.");
                }
              }
            }
          }
        } catch (Exception ex) {
          LOG.error("Unable to listen for describe sensors events", ex);
        }

      } else {
        LOG.error(
            "DEFAULT_POLL_INTERVAL_MINS : "
                + DEFAULT_POLL_INTERVAL_MINS
                + " must be less than 1440 minutes");
      }
    }
  }
Ejemplo n.º 14
0
 /**
  * Private for a reason.
  *
  * @see {@link EntityWrapper#get(Class)}
  * @param persistenceContext
  */
 @SuppressWarnings("unchecked")
 private EntityWrapper(final String persistenceContext) {
   this.tx = new TransactionState(persistenceContext);
   this.txStart = Threads.currentStackString();
 }
Ejemplo n.º 15
0
public class ListMetricQueue {

  private static final Logger LOG = Logger.getLogger(ListMetricQueue.class);

  private static class NoDupQueue<T> {
    private LinkedHashSet<T> items = Sets.newLinkedHashSet();

    public synchronized void drainTo(List<T> list) {
      list.addAll(items);
      items.clear();
    }

    public synchronized void drainTo(List<T> list, int maxItems) {
      List<T> intermediateList = Lists.newArrayList();
      int ctr = 0;
      for (T item : items) {
        intermediateList.add(item);
        ctr++;
        if (ctr == maxItems) break;
      }
      list.addAll(intermediateList);
      items.removeAll(intermediateList);
    }

    public synchronized void putAll(List<T> list) {
      items.addAll(list);
    }

    public synchronized void put(T item) {
      items.add(item);
    }
  }

  static final NoDupQueue<ListMetricQueueItem> dataQueue = new NoDupQueue<ListMetricQueueItem>();

  private static final ScheduledExecutorService dataFlushTimer =
      Executors.newSingleThreadScheduledExecutor(
          Threads.threadFactory("cloudwatch-list-metrics-flush-%d"));

  private static ListMetricQueue singleton = getInstance();

  public static ListMetricQueue getInstance() {
    synchronized (ListMetricQueue.class) {
      if (singleton == null) singleton = new ListMetricQueue();
    }
    return singleton;
  }

  private void queue(ListMetricQueueItem metricData) {
    dataQueue.put(metricData);
  }

  private static Runnable safeRunner =
      new Runnable() {
        @Override
        public void run() {
          long before = System.currentTimeMillis();
          try {
            List<ListMetricQueueItem> dataBatch = Lists.newArrayList();
            dataQueue.drainTo(dataBatch);
            ThruputMetrics.addDataPoint(MonitoredAction.LIST_METRIC_SIZE, dataBatch.size());
            long t2 = System.currentTimeMillis();
            dataBatch = prune(dataBatch);
            long t3 = System.currentTimeMillis();
            ThruputMetrics.addDataPoint(MonitoredAction.LIST_METRIC_PRUNE, t3 - t2);
            List<ListMetric> listMetrics = convertToListMetrics(dataBatch);
            long t4 = System.currentTimeMillis();
            ThruputMetrics.addDataPoint(MonitoredAction.LIST_METRIC_CONVERT, t4 - t3);
            ListMetricManager.addMetricBatch(listMetrics);
            long t5 = System.currentTimeMillis();
            ThruputMetrics.addDataPoint(MonitoredAction.LIST_METRIC_MERTIC_ADD_BATCH, t5 - t4);
          } catch (Throwable ex) {
            LOG.debug("ListMetricQueue:error");
            ex.printStackTrace();
            LOG.error(ex, ex);
          } finally {
            ThruputMetrics.addDataPoint(
                MonitoredAction.LIST_METRIC_TIMING, System.currentTimeMillis() - before);
          }
        }
      };

  private static List<ListMetric> convertToListMetrics(List<ListMetricQueueItem> dataBatch) {
    if (dataBatch == null) return null;
    List<ListMetric> listMetrics = Lists.newArrayList();
    for (ListMetricQueueItem item : dataBatch) {
      listMetrics.add(
          ListMetricManager.createListMetric(
              item.getAccountId(),
              item.getMetricName(),
              item.getMetricType(),
              item.getNamespace(),
              item.getDimensionMap()));
    }
    return listMetrics;
  }

  private static List<ListMetricQueueItem> prune(List<ListMetricQueueItem> dataBatch) {
    Set<ListMetricQueueItem> intermediateSet = Sets.newLinkedHashSet(dataBatch);
    List<ListMetricQueueItem> prunedList = Lists.newArrayList(intermediateSet);
    return prunedList;
  }

  static {
    dataFlushTimer.scheduleAtFixedRate(safeRunner, 0, 5, TimeUnit.MINUTES);
  }

  public void addAll(List<SimpleMetricEntity> dataBatch) {
    for (SimpleMetricEntity item : dataBatch) {
      ListMetricQueueItem metricMetadata = new ListMetricQueueItem();
      metricMetadata.setAccountId(item.getAccountId());
      metricMetadata.setNamespace(item.getNamespace());
      metricMetadata.setMetricName(item.getMetricName());
      metricMetadata.setDimensionMap(item.getDimensionMap());
      metricMetadata.setMetricType(item.getMetricType());
      queue(metricMetadata);
    }
  }
}