TxRecord(final String persistenceContext, final String uuid, final StackTraceElement ste) {
this.persistenceContext = persistenceContext;
this.uuid = uuid;
this.ste = ste;
this.stack = Threads.currentStackString();
this.startTime = System.currentTimeMillis();
}
private static CascadingTx createTransaction(final Object obj)
throws RecoverablePersistenceException, RuntimeException {
final String ctx = lookatPersistenceContext(obj);
final CascadingTx ret = new CascadingTx(ctx);
try {
ret.begin();
if (txRootThreadLocal.get() == null) {
final String txId = makeTxRootName(ret);
LOG.trace(
"Creating root entry for transaction tree: "
+ txId
+ " at: \n"
+ Threads.currentStackString());
txRootThreadLocal.set(txId);
}
txStateThreadLocal.get().put(ctx, ret);
return ret;
} catch (RuntimeException ex) {
try {
ret.rollback();
} catch (RuntimeException ex1) {
throw ex1;
}
throw ex;
}
}
TransactionState(final String ctx) {
this.startTime = System.currentTimeMillis();
this.txUuid = String.format("%s:%s", ctx, UUID.randomUUID().toString());
this.stopWatch = new StopWatch();
this.stopWatch.start();
this.owner = Logs.isExtrrreeeme() ? Threads.currentStackString() : "n/a";
try {
this.eventLog(TxStep.BEGIN, TxEvent.CREATE);
final EntityManagerFactory anemf =
(EntityManagerFactoryImpl) PersistenceContexts.getEntityManagerFactory(ctx);
checkParam(anemf, notNullValue());
this.em = anemf.createEntityManager();
checkParam(this.em, notNullValue());
this.transaction = this.em.getTransaction();
this.transaction.begin();
this.session = new WeakReference<Session>((Session) this.em.getDelegate());
this.eventLog(TxStep.END, TxEvent.CREATE);
} catch (final Throwable ex) {
Logs.exhaust().error(ex, ex);
this.eventLog(TxStep.FAIL, TxEvent.CREATE);
this.rollback();
throw new RuntimeException(PersistenceExceptions.throwFiltered(ex));
} finally {
outstanding.put(this.txUuid, this);
}
}
public String getMessage() {
if (Logs.isExtrrreeeme()) {
return Threads.currentStackString();
} else {
return "n.a";
}
}
@Override
public void fireEvent(final ClockTick event) {
if (Topology.isEnabledLocally(Eucalyptus.class) && ready.compareAndSet(true, false)) {
try {
Threads.enqueue(Eucalyptus.class, Volumes.class, this);
} catch (final Exception ex) {
ready.set(true);
}
}
}
public static <E, T> Predicate<T> asTransaction(final Predicate<T> predicate) {
final List<Class> generics = Classes.genericsToClasses(predicate);
for (final Class<?> type : generics) {
if (PersistenceContexts.isPersistentClass(type)) {
return asTransaction(type, predicate);
}
}
throw new IllegalArgumentException(
"Failed to find generics for provided predicate, cannot make into transaction: "
+ Threads.currentStackString());
}
/**
* @delegate Do not change semantics here.
* @see javax.persistence.EntityTransaction#commit()
*/
@Override
public void commit() throws RecoverablePersistenceException {
removeTransaction(this);
if ((this.txState != null) && this.txState.isActive()) {
try {
this.txState.commit();
} catch (final RuntimeException ex) {
throw PersistenceExceptions.throwFiltered(ex);
}
} else {
Logs.extreme().error("Duplicate call to commit( ): " + Threads.currentStackString());
}
}
/**
* Private for a reason.
*
* @see {@link CascadingTx#get(Class)}
* @param persistenceContext
* @throws RecoverablePersistenceException
*/
@SuppressWarnings("unchecked")
CascadingTx(final String ctx) throws RecoverablePersistenceException {
final StackTraceElement ste = Threads.currentStackFrame(4);
final String uuid = UUID.randomUUID().toString();
this.record = new TxRecord(ctx, uuid, ste);
try {
this.txState = new TxState(ctx);
} catch (final RuntimeException ex) {
Logs.extreme().error(ex, ex);
this.rollback();
throw PersistenceExceptions.throwFiltered(ex);
}
}
public static <T, R> Function<T, R> asTransaction(final Function<T, R> function) {
if (function instanceof TransactionalFunction) {
return function;
} else {
final List<Class> generics = Classes.genericsToClasses(function);
for (final Class<?> type : generics) {
if (PersistenceContexts.isPersistentClass(type)) {
return asTransaction(type, function);
}
}
throw new IllegalArgumentException(
"Failed to find generics for provided function, cannot make into transaction: "
+ Threads.currentStackString());
}
}
public static void awaitSynchronized() {
if (!isVolatile()) {
return;
} else {
Collection<StackTraceElement> stack = Threads.filteredStack(stackFilter);
String caller = (stack.isEmpty() ? "" : stack.iterator().next().toString());
for (int i = 0; i < MAX_TX_START_SYNC_RETRIES && isVolatile(); i++) {
try {
TimeUnit.MILLISECONDS.sleep(1000);
LOG.debug("Transaction blocked on sync: " + caller);
} catch (InterruptedException ex) {
Exceptions.maybeInterrupted(ex);
return;
}
}
if (isVolatile()) {
throw new DatabaseStateException(
"Transaction begin failed due to concurrent database synchronization: "
+ Hosts.listDatabases()
+ " for caller:\n"
+ Joiner.on("\n\tat ").join(stack));
}
}
}
public class Databases {
private static final Logger LOG = Logger.getLogger(Databases.class);
private static final int MAX_DEACTIVATION_RETRIES = 10;
private static final int MAX_TX_START_SYNC_RETRIES = 120;
private static final AtomicInteger counter = new AtomicInteger(500);
private static final Predicate<Host> FILTER_SYNCING_DBS =
Predicates.and(DbFilter.INSTANCE, Predicates.not(SyncedDbFilter.INSTANCE));
private static final ScriptedDbBootstrapper singleton = new ScriptedDbBootstrapper();
private static final String jdbcJmxDomain = "net.sf.hajdbc";
private static final ExecutorService dbSyncExecutors =
Executors.newCachedThreadPool(); // NOTE:GRZE:special case thread handling.
private static final ReentrantReadWriteLock canHas = new ReentrantReadWriteLock();
private static Supplier<Set<String>> activeHosts =
Suppliers.memoizeWithExpiration(ActiveHostSet.ACTIVATED, 2, TimeUnit.SECONDS);
private static Supplier<Set<String>> hostDatabases =
Suppliers.memoizeWithExpiration(ActiveHostSet.DBHOSTS, 1, TimeUnit.SECONDS);
private static Predicate<StackTraceElement> notStackFilterYouAreLookingFor =
Predicates.or(
Threads.filterStackByQualifiedName("com\\.eucalyptus\\.entities\\..*"),
Threads.filterStackByQualifiedName("java\\.lang\\.Thread.*"),
Threads.filterStackByQualifiedName("com\\.eucalyptus\\.system\\.Threads.*"),
Threads.filterStackByQualifiedName("com\\.eucalyptus\\.bootstrap\\.Databases.*"));
private static Predicate<StackTraceElement> stackFilter =
Predicates.not(notStackFilterYouAreLookingFor);
private static final int DATABASE_WEIGHT_PRIMARY = 100;
private static final int DATABASE_WEIGHT_SECONDARY = 1;
public static class DatabaseStateException extends IllegalStateException {
private static final long serialVersionUID = 1L;
public DatabaseStateException(String string) {
super(string);
}
}
public enum Locks {
DISABLED {
@Override
void isLocked() {
File dbLockFile = this.getLockFile();
if (dbLockFile.exists() && Hosts.isCoordinator()) {
this.failStop();
}
}
@Override
public void failStop() {
Faults.forComponent(Eucalyptus.class)
.havingId(1010)
.withVar(DB_LOCK_FILE, this.getLockFile().getAbsolutePath())
.log();
LOG.error(
"WARNING : DISABLED CLC STARTED OUT OF ORDER, REMOVE THE "
+ this.getLockName()
+ "FILE TO PROCEED WITH RISK");
System.exit(1);
}
},
PARTITIONED {
@Override
void isLocked() {
if (this.getLockFile().exists()) {
failStop();
}
}
@Override
public void failStop() {
Faults.forComponent(Eucalyptus.class)
.havingId(1011)
.withVar(DB_LOCK_FILE, this.getLockFile().getAbsolutePath())
.log();
LOG.error("PARTITION DETECTED -- FAIL-STOP TO AVOID POSSIBLE INCONSISTENCY.");
LOG.error(
"PARTITION DETECTED -- Shutting down CLC after experiencing a possible split-brain partition.");
LOG.error("PARTITION DETECTED -- See cloud-fault.log for guidance.");
System.exit(1);
}
@Override
public void create() {
super.create();
Faults.forComponent(Eucalyptus.class)
.havingId(1011)
.withVar(DB_LOCK_FILE, this.getLockFile().getAbsolutePath())
.log();
}
};
public static final String DB_LOCK_FILE = "DB_LOCK_FILE";
public void delete() {
this.getLockFile().delete();
LOG.debug("The " + this.getLockFile().getAbsolutePath() + " file was deleted");
}
protected String getLockName() {
return this.name().toLowerCase() + ".lock";
}
abstract void isLocked();
public abstract void failStop();
protected File getLockFile() {
return SubDirectory.DB.getChildFile("data", this.getLockName());
}
public void create(String reason) {
LOG.error(this.getLockName() + ": Caused by: " + reason);
this.create();
}
public void create() {
try {
if (getLockFile().createNewFile()) {
LOG.debug(
this.getLockName()
+ ": The "
+ this.getLockFile().getAbsolutePath()
+ " file was created.");
}
} catch (IOException e) {
LOG.debug(
"Unable to create the "
+ this.getLockFile().getAbsolutePath()
+ " file: "
+ e.getMessage());
}
}
}
public enum Events {
INSTANCE;
public static Sql getConnection() throws Exception {
return Databases.getBootstrapper().getConnection(INSTANCE.getName());
}
public String getName() {
return "database_events";
}
public static void create() {
if (!getBootstrapper().listDatabases().contains(INSTANCE.getName())) {
try {
getBootstrapper().createDatabase(INSTANCE.getName());
} catch (Exception ex) {
LOG.error(ex, ex);
}
}
}
}
enum SyncState {
IRRELEVANT,
NOTSYNCED,
SYNCING {
@Override
public boolean set() {
return syncState.compareAndSet(NOTSYNCED, SYNCING);
}
},
DESYNCING,
SYNCED {
@Override
public boolean isCurrent() {
if (Hosts.isCoordinator()) {
syncState.set(this);
}
return super.isCurrent();
}
};
private static final AtomicReference<SyncState> syncState =
new AtomicReference<>(SyncState.NOTSYNCED);
public static SyncState get() {
return syncState.get();
}
public boolean set() {
syncState.set(this);
return true;
}
public boolean isCurrent() {
return this.equals(syncState.get());
}
}
enum ExecuteRunnable implements Function<Runnable, Future<Runnable>> {
INSTANCE;
@Override
public Future<Runnable> apply(Runnable input) {
Logs.extreme().debug("SUBMIT: " + input);
return dbSyncExecutors.submit(input, input);
}
}
@Provides(Empyrean.class)
@RunDuring(Bootstrap.Stage.PoolInit)
public static class DatabasePoolBootstrapper extends Bootstrapper.Simple {
private static final int INITIAL_DB_SYNC_RETRY_WAIT = 5;
@Override
public boolean load() throws Exception {
Hosts.awaitDatabases();
Locks.DISABLED.isLocked();
Locks.PARTITIONED.isLocked();
Groovyness.run("setup_dbpool.groovy");
OrderedShutdown.registerShutdownHook(
Empyrean.class,
new Runnable() {
@Override
public void run() {
try {
for (String ctx : PersistenceContexts.list()) {
try {
DatabaseClusterMBean db = Databases.lookup(ctx, TimeUnit.SECONDS.toMillis(5));
for (String host : db.getinactiveDatabases()) {
Databases.disable(host);
}
for (String host : db.getactiveDatabases()) {
Databases.disable(host);
}
} catch (Exception ex) {
LOG.error(ex);
}
}
} catch (NoSuchElementException ex) {
LOG.error(ex);
}
}
});
TimeUnit.SECONDS.sleep(INITIAL_DB_SYNC_RETRY_WAIT);
if (!Hosts.isCoordinator() && Hosts.localHost().hasDatabase()) {
while (!Databases.enable(Hosts.localHost())) {
LOG.warn(
LogUtil.subheader("Synchronization of the database failed: " + Hosts.localHost()));
if (counter.decrementAndGet() == 0) {
LOG.fatal("Restarting process to force re-synchronization.");
System.exit(123);
} else {
LOG.warn(
"Sleeping for " + INITIAL_DB_SYNC_RETRY_WAIT + " seconds before trying again.");
TimeUnit.SECONDS.sleep(INITIAL_DB_SYNC_RETRY_WAIT);
}
}
Locks.DISABLED.create();
Hosts.UpdateEntry.INSTANCE.apply(Hosts.localHost());
LOG.info(LogUtil.subheader("Database synchronization complete: " + Hosts.localHost()));
}
return true;
}
@Override
public boolean check() throws Exception {
if (Bootstrap.isOperational()
&& !Hosts.localHost().hasDatabase()) { // This check is redundant on hosts with DBs
final List<Host.DBStatus> statusList = Hosts.localHost().getDatabaseStatus();
for (final Host.DBStatus status : statusList) {
for (String error : status.getError().asSet()) {
LOG.error(error);
}
}
}
return super.check();
}
}
private static void runDbStateChange(Function<String, Runnable> runnableFunction) {
Logs.extreme().info("DB STATE CHANGE: " + runnableFunction);
try {
Logs.extreme().info("Attempting to acquire db state lock: " + runnableFunction);
if (canHas.writeLock().tryLock(5, TimeUnit.MINUTES)) {
try {
Logs.extreme().info("Acquired db state lock: " + runnableFunction);
Map<Runnable, Future<Runnable>> runnables = Maps.newHashMap();
for (final String ctx : listDatabases()) {
Runnable run = runnableFunction.apply(ctx);
runnables.put(run, ExecuteRunnable.INSTANCE.apply(run));
}
Map<Runnable, Future<Runnable>> succeeded = Futures.waitAll(runnables);
MapDifference<Runnable, Future<Runnable>> failed = Maps.difference(runnables, succeeded);
StringBuilder builder = new StringBuilder();
builder.append(Joiner.on("\nSUCCESS: ").join(succeeded.keySet()));
builder.append(Joiner.on("\nFAILED: ").join(failed.entriesOnlyOnLeft().keySet()));
Logs.extreme().debug(builder.toString());
if (!failed.entriesOnlyOnLeft().isEmpty()) {
throw Exceptions.toUndeclared(builder.toString());
}
} finally {
canHas.writeLock().unlock();
}
} else {
throw new LockTimeoutException(
"DB STATE CHANGE ABORTED (failed to get lock): " + runnableFunction);
}
} catch (RuntimeException ex) {
LOG.error(ex);
Logs.extreme().error(ex, ex);
throw ex;
} catch (InterruptedException ex) {
Exceptions.maybeInterrupted(ex);
throw Exceptions.toUndeclared(ex);
}
}
enum DeactivateHostFunction implements Function<String, Function<String, Runnable>> {
INSTANCE;
/** @see Function#apply(Object) */
@Override
public Function<String, Runnable> apply(final String hostName) {
return new Function<String, Runnable>() {
@Override
public Runnable apply(final String contextName) {
return new Runnable() {
@Override
public void run() {
if (Internets.testLocal(hostName)) {
return;
}
try {
final boolean wasPrimary;
try {
wasPrimary =
lookupDatabase(contextName, hostName).getweight() == DATABASE_WEIGHT_PRIMARY;
} catch (Exception ex1) {
return;
}
LOG.info(
"Tearing down database connections for: "
+ hostName
+ " to context: "
+ contextName);
final DatabaseClusterMBean cluster =
lookup(contextName, TimeUnit.SECONDS.toMillis(5));
// deactivate database
for (int i = 0;
i < MAX_DEACTIVATION_RETRIES && cluster.getactiveDatabases().contains(hostName);
i++)
try {
LOG.debug(
"Deactivating database connections for: "
+ hostName
+ " to context: "
+ contextName);
cluster.deactivate(hostName);
LOG.debug(
"Deactived database connections for: "
+ hostName
+ " to context: "
+ contextName);
break;
} catch (Exception ex) {
LOG.error(ex);
Logs.extreme().error(ex, ex);
}
// remove database
for (int i = 0;
i < MAX_DEACTIVATION_RETRIES
&& cluster.getinactiveDatabases().contains(hostName)
&& !Hosts.contains(hostName);
i++)
try {
LOG.debug(
"Removing database registration for: "
+ hostName
+ " to context: "
+ contextName);
cluster.remove(hostName);
LOG.debug(
"Removed database registration for: "
+ hostName
+ " to context: "
+ contextName);
break;
} catch (Exception ex) {
LOG.error(ex);
Logs.extreme().error(ex, ex);
}
// promote other
final Host coordinator = Hosts.getCoordinator();
if (wasPrimary && coordinator != null) {
try {
lookupDatabase(contextName, coordinator.getDisplayName())
.setweight(DATABASE_WEIGHT_PRIMARY);
} catch (NoSuchElementException e) {
// Weight will be set on activation
} catch (Exception e) {
LOG.error(
"Error setting primary weight for host "
+ coordinator.getDisplayName()
+ " context "
+ contextName,
e);
}
}
// refresh pooled connections to ensure closed if removed
for (int i = 0;
i < MAX_DEACTIVATION_RETRIES
&& !cluster.getactiveDatabases().contains(hostName)
&& !cluster.getinactiveDatabases().contains(hostName);
i++)
try {
// Release any open connections
LOG.debug("Refreshing idle pooled connections for context: " + contextName);
ProxoolFacade.killAllConnections(contextName, "Database deregistered", true);
LOG.debug("Refreshed idle pooled connections for context: " + contextName);
break;
} catch (Exception ex) {
LOG.error(ex);
Logs.extreme().error(ex, ex);
}
} catch (final Exception ex1) {
LOG.error(ex1);
Logs.extreme().error(ex1, ex1);
}
}
@Override
public String toString() {
return "Databases.disable(): " + hostName + " " + contextName;
}
};
}
@Override
public String toString() {
return "Databases.disable(): " + hostName;
}
};
}
@Override
public String toString() {
return "Databases.disable()";
}
}
enum ActivateHostFunction implements Function<Host, Function<String, Runnable>> {
INSTANCE;
private static void prepareConnections(final Host host, final String contextName)
throws NoSuchElementException {
final String dbUrl =
"jdbc:" + ServiceUris.remote(Database.class, host.getBindAddress(), contextName);
final String hostName = host.getDisplayName();
final DriverDatabaseMBean database = Databases.lookupDatabase(contextName, hostName);
database.setuser(getUserName());
database.setpassword(getPassword());
database.setweight(
Hosts.isCoordinator(host) ? DATABASE_WEIGHT_PRIMARY : DATABASE_WEIGHT_SECONDARY);
database.setlocal(host.isLocalHost());
database.setlocation(dbUrl);
}
@Override
public Function<String, Runnable> apply(final Host host) {
return new Function<String, Runnable>() {
@Override
public Runnable apply(final String ctx) {
final String hostName = host.getBindAddress().getHostAddress();
final String contextName = ctx;
Runnable removeRunner =
new Runnable() {
@Override
public void run() {
try {
final boolean fullSync =
!Hosts.isCoordinator()
&& host.isLocalHost()
&& BootstrapArgs.isCloudController()
&& !Databases.isSynchronized();
final boolean passiveSync = !fullSync && host.hasSynced();
if (!fullSync && !passiveSync) {
throw Exceptions.toUndeclared("Host is not ready to be activated: " + host);
} else {
final DatabaseClusterMBean cluster =
lookup(contextName, TimeUnit.SECONDS.toMillis(30));
final boolean activated = cluster.getactiveDatabases().contains(hostName);
final boolean deactivated = cluster.getinactiveDatabases().contains(hostName);
final String passiveStrategy = cluster.getdefaultSynchronizationStrategy();
final String fullStrategy =
Iterables.getFirst(
Sets.difference(
cluster.getsynchronizationStrategies(),
Collections.singleton(passiveStrategy)),
"full");
final String syncStrategy = fullSync ? fullStrategy : passiveStrategy;
if (activated) {
resetDatabaseWeights(contextName);
return;
} else if (deactivated) {
ActivateHostFunction.prepareConnections(host, contextName);
} else {
LOG.info("Creating database " + ctx + " connections for: " + host);
try {
lookupDatabase(contextName, hostName);
} catch (NoSuchElementException e) {
try {
cluster.add(hostName);
Logs.extreme()
.debug(
"Added database " + ctx + " connections for host: " + hostName);
} catch (IllegalArgumentException ex) {
Logs.extreme()
.debug(
"Skipping addition of database "
+ ctx
+ " connections for host which already exists: "
+ hostName);
} catch (IllegalStateException ex) {
if (Exceptions.isCausedBy(ex, InstanceAlreadyExistsException.class)) {
ManagementFactory.getPlatformMBeanServer()
.unregisterMBean(
new ObjectName(
jdbcJmxDomain,
new Hashtable<>(
ImmutableMap.of(
"cluster",
ctx,
"type",
"Database",
"database",
hostName))));
cluster.add(hostName);
} else {
throw ex;
}
}
}
ActivateHostFunction.prepareConnections(host, contextName);
}
// demote others
if (Hosts.isCoordinator(host)) {
for (Host secondaryHost : Hosts.listActiveDatabases())
if (!secondaryHost.equals(host))
try {
lookupDatabase(contextName, secondaryHost.getDisplayName())
.setweight(DATABASE_WEIGHT_SECONDARY);
} catch (NoSuchElementException e) {
// Weight will be set on activation
} catch (Exception e) {
LOG.error(
"Error setting primary weight for host "
+ secondaryHost.getDisplayName()
+ " context "
+ contextName,
e);
}
}
try {
if (fullSync) {
LOG.info(
"Full sync of database "
+ ctx
+ " on: "
+ host
+ " using: "
+ fullStrategy);
} else {
LOG.info(
"Passive activation of database " + ctx + " connections to: " + host);
}
cluster.activate(hostName, syncStrategy);
if (fullSync) {
LOG.info(
"Full sync of database "
+ ctx
+ " on: "
+ host
+ " using "
+ cluster.getactiveDatabases());
} else {
LOG.info(
"Passive activation of database "
+ ctx
+ " on: "
+ host
+ " using "
+ cluster.getactiveDatabases());
}
} catch (Exception ex) {
throw Exceptions.toUndeclared(ex);
}
// refresh pooled connections
try {
// Release any open connections
LOG.debug("Refreshing idle pooled connections for context: " + contextName);
ProxoolFacade.killAllConnections(contextName, "Database registered", true);
LOG.debug("Refreshed idle pooled connections for context: " + contextName);
} catch (Exception ex) {
LOG.error(
"Error refreshing connections on activation of context: " + contextName,
ex);
}
}
} catch (final NoSuchElementException ex1) {
LOG.error(ex1);
Logs.extreme().error(ex1, ex1);
return;
} catch (final IllegalStateException ex1) {
LOG.error(ex1);
Logs.extreme().error(ex1, ex1);
return;
} catch (final Exception ex1) {
LOG.error(ex1);
Logs.extreme().error(ex1, ex1);
throw Exceptions.toUndeclared(
"Failed to activate database "
+ ctx
+ " host "
+ host
+ " because of: "
+ ex1.getMessage(),
ex1);
}
}
@Override
public String toString() {
return "Databases.enable(): " + host.getDisplayName() + " " + contextName;
}
};
return removeRunner;
}
@Override
public String toString() {
return "Databases.enable(): " + host;
}
};
}
@Override
public String toString() {
return "Databases.enable()";
}
private static void rollback(final Host host, Exception ex) {
try {
Databases.runDbStateChange(
Databases.DeactivateHostFunction.INSTANCE.apply(host.getDisplayName()));
} catch (LockTimeoutException ex1) {
Databases.LOG.error("Databases.enable(): failed because of: " + ex.getMessage());
} catch (Exception ex1) {
Databases.LOG.error("Databases.enable(): failed because of: " + ex.getMessage());
Logs.extreme().error(ex, ex);
}
}
}
private static void resetDatabaseWeights(final String contextName) {
for (final Host host : Hosts.listActiveDatabases())
try {
lookupDatabase(contextName, host.getDisplayName())
.setweight(
Hosts.isCoordinator(host) ? DATABASE_WEIGHT_PRIMARY : DATABASE_WEIGHT_SECONDARY);
} catch (NoSuchElementException e) {
// Weight will be set on activation
} catch (Exception e) {
LOG.error(
"Error resetting weight for host " + host.getDisplayName() + " context " + contextName,
e);
}
}
private static DatabaseClusterMBean lookup(final String ctx, final long timeout)
throws NoSuchElementException {
return lookupMBean(
ImmutableMap.of("cluster", ctx, "type", "DatabaseCluster"),
DatabaseClusterMBean.class,
new Predicate<DatabaseClusterMBean>() {
@Override
public boolean apply(final DatabaseClusterMBean cluster) {
cluster.getinactiveDatabases();
return true;
}
},
timeout);
}
private static DriverDatabaseMBean lookupDatabase(final String contextName, final String hostName)
throws NoSuchElementException {
return lookupMBean(
ImmutableMap.of("cluster", contextName, "type", "Database", "database", hostName),
DriverDatabaseMBean.class,
new Predicate<DriverDatabaseMBean>() {
@Override
public boolean apply(final DriverDatabaseMBean database) {
database.getid();
return true;
}
},
0);
}
private static <T> T lookupMBean(
final Map<String, String> props,
final Class<T> type,
final Predicate<T> tester,
final long timeout) {
long until = System.currentTimeMillis() + timeout;
do
try {
final T bean = Mbeans.lookup(jdbcJmxDomain, props, type);
tester.apply(bean);
return bean;
} catch (UndeclaredThrowableException e) {
if (Exceptions.isCausedBy(e, InstanceNotFoundException.class)) {
if (System.currentTimeMillis() < until) {
try {
TimeUnit.SECONDS.sleep(5);
} catch (InterruptedException e1) {
Thread.interrupted();
break;
}
LOG.debug("Waiting for MBean " + type.getSimpleName() + "/" + props);
continue;
}
throw new NoSuchElementException(type.getSimpleName() + " " + props.toString());
} else {
throw Exceptions.toUndeclared(e);
}
}
while (System.currentTimeMillis() < until);
throw new NoSuchElementException(type.getSimpleName() + " " + props.toString());
}
static boolean disable(final String hostName) {
if (!Bootstrap.isFinished()) {
return false;
} else if (Internets.testLocal(hostName)) {
return true;
} else {
try {
runDbStateChange(DeactivateHostFunction.INSTANCE.apply(hostName));
return true;
} catch (Exception ex) {
Logs.extreme().debug(ex);
return false;
}
}
}
static boolean enable(final Host host) {
if (!host.hasDatabase() || Bootstrap.isShuttingDown()) {
return false;
} else if (!Hosts.contains(host.getGroupsId())) {
Hosts.remove(host.getGroupsId());
return false;
} else {
if (host.isLocalHost()) {
if (SyncState.SYNCING.set()) {
try {
runDbStateChange(ActivateHostFunction.INSTANCE.apply(host));
SyncState.SYNCED.set();
return true;
} catch (LockTimeoutException ex) {
SyncState.NOTSYNCED.set();
return false;
} catch (Exception ex) {
SyncState.NOTSYNCED.set();
LOG.error(ex);
Logs.extreme().error(ex, ex);
return false;
}
} else if (!SyncState.SYNCING.isCurrent()) {
try {
runDbStateChange(ActivateHostFunction.INSTANCE.apply(host));
return true;
} catch (LockTimeoutException ex) {
return false;
} catch (Exception ex) {
LOG.error(ex);
Logs.extreme().error(ex, ex);
return false;
}
} else {
try {
runDbStateChange(ActivateHostFunction.INSTANCE.apply(host));
SyncState.SYNCED.set();
return true;
} catch (LockTimeoutException ex) {
SyncState.NOTSYNCED.set();
return false;
} catch (Exception ex) {
SyncState.NOTSYNCED.set();
LOG.error(ex, ex);
return false;
}
}
} else if (!ActiveHostSet.ACTIVATED.get().contains(host.getDisplayName())) {
try {
runDbStateChange(ActivateHostFunction.INSTANCE.apply(host));
return true;
} catch (LockTimeoutException ex) {
return false;
} catch (Exception ex) {
Logs.extreme().debug(ex);
ActivateHostFunction.rollback(host, ex);
return false;
}
} else {
return ActiveHostSet.ACTIVATED.get().contains(host.getDisplayName());
}
}
}
static boolean shouldInitialize() { // GRZE:WARNING:HACKHACKHACK do not duplicate pls thanks.
for (final Host h : Hosts.listActiveDatabases()) {
final String url =
String.format(
"jdbc:%s",
ServiceUris.remote(Database.class, h.getBindAddress(), "eucalyptus_config"));
try {
final Connection conn =
DriverManager.getConnection(url, Databases.getUserName(), Databases.getPassword());
try {
final PreparedStatement statement =
conn.prepareStatement(
"select config_component_hostname from config_component_base where config_component_partition='eucalyptus';");
final ResultSet result = statement.executeQuery();
while (result.next()) {
final Object columnValue = result.getObject(1);
if (Internets.testLocal(columnValue.toString())) {
return true;
}
}
} finally {
conn.close();
}
} catch (final Exception ex) {
LOG.error(ex, ex);
}
}
return false;
}
static Set<String> listPrimaryActiveDatabases(final String cluster) {
return listDatabases(cluster, true, Optional.of(DATABASE_WEIGHT_PRIMARY));
}
static Set<String> listSecondaryActiveDatabases(final String cluster) {
return listDatabases(cluster, true, Optional.of(DATABASE_WEIGHT_SECONDARY));
}
static Set<String> listInactiveDatabases(final String cluster) {
return listDatabases(cluster, false, Optional.<Integer>absent());
}
private static Set<String> listDatabases(
final String clusterName, final boolean active, final Optional<Integer> weight) {
final Set<String> databases = Sets.newLinkedHashSet();
try {
final DatabaseClusterMBean cluster = lookup(clusterName, 0);
final Iterable<String> databaseIds =
active ? cluster.getactiveDatabases() : cluster.getinactiveDatabases();
for (final String databaseId : databaseIds)
try {
final DriverDatabaseMBean database = lookupDatabase(clusterName, databaseId);
if (!weight.isPresent() || database.getweight() == weight.get()) {
databases.add(databaseId);
}
} catch (NoSuchElementException e) {
// ignore database
}
} catch (NoSuchElementException e) {
// no databases
} catch (Exception e) {
LOG.error(e, e);
}
return databases;
}
static Set<String> listRegisteredDatabases() {
return Mbeans.listPropertyValues(
jdbcJmxDomain, "cluster", ImmutableMap.of("type", "DatabaseCluster"));
}
/** List all known databases. */
static Set<String> listDatabases() {
final Set<String> dbNames = Sets.newHashSet();
final Predicate<String> dbNamePredicate =
Predicates.or(Strings.startsWith("eucalyptus_"), Predicates.equalTo("database_events"));
for (final Host h : Hosts.listActiveDatabases()) {
Iterables.addAll(
dbNames,
Iterables.filter(
Databases.getBootstrapper().listDatabases(h.getBindAddress()), dbNamePredicate));
}
return dbNames;
}
/** @return LockTimeoutException */
public static Boolean isSynchronized() {
return SyncState.SYNCED.isCurrent();
}
public static Boolean isVolatile() {
if (!Bootstrap.isFinished() || BootstrapArgs.isInitializeSystem()) {
return false;
} else if (!Hosts.isCoordinator() && BootstrapArgs.isCloudController()) {
return !isSynchronized() || !activeHosts.get().containsAll(hostDatabases.get());
} else if (!activeHosts.get().equals(hostDatabases.get())) {
return true;
} else {
return !Hosts.list(FILTER_SYNCING_DBS).isEmpty();
}
}
enum ActiveHostSet implements Supplier<Set<String>> {
ACTIVATED {
@Override
public Set<String> get() {
Set<String> hosts = DBHOSTS.get();
Set<String> union = Sets.newHashSet();
Set<String> intersection = Sets.newHashSet(hosts);
Logs.extreme().debug("ActiveHostSet: universe of db hosts: " + hosts);
for (String ctx : PersistenceContexts.list()) {
try {
Set<String> activeDatabases = Databases.lookup(ctx, 0).getactiveDatabases();
if (BootstrapArgs.isCloudController()) {
activeDatabases.add(
Internets
.localHostIdentifier()); // GRZE: use Internets.localHostIdentifier() which is
// static, rather than the Hosts reference as it is
// stateful
}
union.addAll(activeDatabases);
intersection.retainAll(activeDatabases);
} catch (Exception ex) {
}
}
Logs.extreme().debug("ActiveHostSet: union of activated db connections: " + union);
Logs.extreme()
.debug(
"ActiveHostSet: intersection of db hosts and activated db connections: "
+ intersection);
boolean dbVolatile = !hosts.equals(intersection);
String msg =
String.format(
"ActiveHostSet: %-14.14s %s%s%s",
dbVolatile ? "volatile" : "synchronized",
hosts,
dbVolatile ? "!=" : "=",
intersection);
if (dbVolatile) {
if (last.compareAndSet(false, dbVolatile)) {
LOG.warn(msg);
} else {
LOG.debug(msg);
}
} else {
if (last.compareAndSet(true, dbVolatile)) {
LOG.warn(msg);
} else {
Logs.extreme().info(msg);
}
}
return intersection;
}
},
DBHOSTS {
@Override
public Set<String> get() {
return Sets.newHashSet(
Collections2.transform(Hosts.listDatabases(), Hosts.NameTransform.INSTANCE));
}
};
private static final AtomicBoolean last = new AtomicBoolean(false);
@Override
public abstract Set<String> get();
}
public static void awaitSynchronized() {
if (!isVolatile()) {
return;
} else {
Collection<StackTraceElement> stack = Threads.filteredStack(stackFilter);
String caller = (stack.isEmpty() ? "" : stack.iterator().next().toString());
for (int i = 0; i < MAX_TX_START_SYNC_RETRIES && isVolatile(); i++) {
try {
TimeUnit.MILLISECONDS.sleep(1000);
LOG.debug("Transaction blocked on sync: " + caller);
} catch (InterruptedException ex) {
Exceptions.maybeInterrupted(ex);
return;
}
}
if (isVolatile()) {
throw new DatabaseStateException(
"Transaction begin failed due to concurrent database synchronization: "
+ Hosts.listDatabases()
+ " for caller:\n"
+ Joiner.on("\n\tat ").join(stack));
}
}
}
public static String getUserName() {
return singleton.getUserName();
}
public static String getPassword() {
return singleton.getPassword();
}
public static String getDriverName() {
return singleton.getDriverName();
}
public static String getJdbcDialect() {
return singleton.getJdbcDialect();
}
public static String getHibernateDialect() {
return singleton.getHibernateDialect();
}
public static DatabaseBootstrapper getBootstrapper() {
return singleton;
}
public static void initialize() {
singleton.init();
}
@RunDuring(Bootstrap.Stage.DatabaseInit)
@Provides(Empyrean.class)
@DependsLocal(Eucalyptus.class)
public static class ScriptedDbBootstrapper extends Bootstrapper.Simple
implements DatabaseBootstrapper {
DatabaseBootstrapper db;
public ScriptedDbBootstrapper() {
try {
this.db = Groovyness.newInstance("setup_db");
} catch (ScriptExecutionFailedException ex) {
LOG.error(ex, ex);
}
}
@Override
public boolean load() throws Exception {
boolean result = this.db.load();
Databases.Events.create();
return result;
}
@Override
public boolean start() throws Exception {
return this.db.start();
}
@Override
public boolean stop() throws Exception {
return this.db.stop();
}
@Override
public void destroy() throws Exception {
this.db.destroy();
}
@Override
public boolean isRunning() {
return this.db.isRunning();
}
@Override
public void hup() {
this.db.hup();
}
@Override
public String getUserName() {
return db.getUserName();
}
@Override
public String getPassword() {
return db.getPassword();
}
@Override
public String getDriverName() {
return this.db.getDriverName();
}
@Override
public String getJdbcDialect() {
return this.db.getJdbcDialect();
}
@Override
public String getHibernateDialect() {
return this.db.getHibernateDialect();
}
@Override
public void init() {
try {
this.db.init();
} catch (Exception ex) {
LOG.error(ex, ex);
}
}
public static DatabaseBootstrapper getInstance() {
return singleton;
}
@Override
public String getServicePath(String... pathParts) {
return this.db.getServicePath(pathParts);
}
@Override
public Map<String, String> getJdbcUrlQueryParameters() {
return this.db.getJdbcUrlQueryParameters();
}
@Override
public boolean check() throws Exception {
return this.db.isRunning();
}
/** @see DatabaseBootstrapper#getJdbcScheme() */
@Override
public String getJdbcScheme() {
return this.db.getJdbcScheme();
}
/** @see DatabaseBootstrapper#listDatabases() */
@Override
public List<String> listDatabases() {
return this.db.listDatabases();
}
/** @see DatabaseBootstrapper#listDatabases(InetAddress) */
@Override
public List<String> listDatabases(InetAddress host) {
return this.db.listDatabases(host);
}
/** @see DatabaseBootstrapper#listTables(String) */
@Override
public List<String> listTables(String database) {
return this.db.listTables(database);
}
/** @see DatabaseBootstrapper#backupDatabase(String,String) */
@Override
public File backupDatabase(String name, String backupIdentifier) {
return this.db.backupDatabase(name, backupIdentifier);
}
/** @see DatabaseBootstrapper#deleteDatabase(String) */
@Override
public void deleteDatabase(String name) {
this.db.deleteDatabase(name);
}
/** @see DatabaseBootstrapper#copyDatabase(String, String) */
@Override
public void copyDatabase(String from, String to) {
this.db.copyDatabase(from, to);
}
/** @see DatabaseBootstrapper#renameDatabase(String, String) */
@Override
public void renameDatabase(String from, String to) {
this.db.renameDatabase(from, to);
}
/** @see DatabaseBootstrapper#getConnection(String) */
@Override
public Sql getConnection(String database) throws Exception {
return this.db.getConnection(database);
}
/** @see DatabaseBootstrapper#createDatabase(String) */
@Override
public void createDatabase(String name) {
this.db.createDatabase(name);
}
}
public static boolean isRunning() {
try {
return singleton.check();
} catch (Exception ex) {
LOG.error(ex, ex);
return false;
}
}
public static String getServicePath(String... pathParts) {
return singleton.getServicePath(pathParts);
}
public static Map<String, String> getJdbcUrlQueryParameters() {
return singleton.getJdbcUrlQueryParameters();
}
public static String getJdbcScheme() {
return singleton.getJdbcScheme();
}
public static void check() {
for (String ctx : PersistenceContexts.list()) {
try {
DatabaseClusterMBean db = lookup(ctx, TimeUnit.SECONDS.toMillis(5));
for (String host : db.getactiveDatabases()) {
Host hostEntry = Hosts.lookup(host);
if (hostEntry == null) {
disable(host);
} else if (!Hosts.contains(hostEntry.getGroupsId())) {
Hosts.remove(host); // GRZE: this will clean up group state and de-activate db.
}
}
} catch (NoSuchElementException ex) {
LOG.error(ex, ex);
}
return;
}
}
/** HA-JDBC Dynamic MBean proxy interface */
private interface DatabaseClusterMBean {
Set<String> getinactiveDatabases();
Set<String> getactiveDatabases();
void add(String hostName);
void remove(String hostName);
void activate(String hostName, String syncStrategy);
void deactivate(String hostName);
Set<String> getsynchronizationStrategies();
String getdefaultSynchronizationStrategy();
}
/** HA-JDBC Dynamic MBean proxy interface */
private interface DriverDatabaseMBean {
boolean isActive();
String getid();
void setuser(String userName);
void setpassword(String password);
int getweight();
void setweight(int i);
void setlocal(boolean localHost);
void setlocation(String url);
}
}
private static void broadcastTask(Callable<Void> task) {
Threads.enqueue(Eucalyptus.class, NetworkInfoBroadcaster.class, 5, task);
}
@Override
public void fireEvent(final Hertz event) {
final long defaultPollIntervalSeconds = TimeUnit.MINUTES.toSeconds(DEFAULT_POLL_INTERVAL_MINS);
if (!Bootstrap.isOperational()
|| !BootstrapArgs.isCloudController()
|| !event.isAsserted(defaultPollIntervalSeconds)) {
return;
} else {
if (DEFAULT_POLL_INTERVAL_MINS >= 1) {
COLLECTION_INTERVAL_TIME_MS =
((int) TimeUnit.MINUTES.toMillis(DEFAULT_POLL_INTERVAL_MINS) / 2);
} else {
COLLECTION_INTERVAL_TIME_MS = 0;
}
if (COLLECTION_INTERVAL_TIME_MS == 0 || HISTORY_SIZE > 15 || HISTORY_SIZE < 1) {
LOG.debug("The instance usage report is disabled");
} else if (COLLECTION_INTERVAL_TIME_MS <= MAX_WRITE_INTERVAL_MS) {
try {
if (event.isAsserted(defaultPollIntervalSeconds)) {
if (Bootstrap.isFinished() && Hosts.isCoordinator()) {
CloudWatchHelper.DefaultInstanceInfoProvider.refresh();
for (final ServiceConfiguration ccConfig :
Topology.enabledServices(ClusterController.class)) {
final String ccHost = ccConfig.getHostName();
if (busyHosts.replace(ccHost, false, true)
|| busyHosts.putIfAbsent(ccHost, true) == null) {
Threads.lookup(Reporting.class, DescribeSensorsListener.class)
.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
final ExecutorService executorService =
Threads.lookup(
Reporting.class,
DescribeSensorsListener.class,
"response-processing")
.limitTo(4);
final long startTime = System.currentTimeMillis();
try {
final List<String> allInstanceIds =
VmInstances.listWithProjection(
VmInstances.instanceIdProjection(),
VmInstance.criterion(VmState.RUNNING),
VmInstance.zoneCriterion(ccConfig.getPartition()),
VmInstance.nonNullNodeCriterion());
final Iterable<List<String>> processInts =
Iterables.partition(allInstanceIds, SENSOR_QUERY_BATCH_SIZE);
for (final List<String> instIds : processInts) {
final ArrayList<String> instanceIds = Lists.newArrayList(instIds);
/**
* Here this is hijacking the sensor callback in order to control
* the thread of execution used when firing
*/
final DescribeSensorCallback msgCallback =
new DescribeSensorCallback(
HISTORY_SIZE, COLLECTION_INTERVAL_TIME_MS, instanceIds) {
@Override
public void fireException(Throwable e) {}
@Override
public void fire(DescribeSensorsResponse msg) {}
};
/**
* Here we actually get the future reference to the result and on
* a response processing thread, invoke .fire().
*/
final DescribeSensorsResponse response =
AsyncRequests.newRequest(msgCallback)
.dispatch(ccConfig)
.get();
executorService.submit(
new Runnable() {
@Override
public void run() {
try {
new DescribeSensorCallback(
HISTORY_SIZE,
COLLECTION_INTERVAL_TIME_MS,
instanceIds)
.fire(response);
} catch (Exception e) {
Exceptions.maybeInterrupted(e);
}
}
});
}
} finally {
/** Only and finally set the busy bit back to false. */
busyHosts.put(ccHost, false);
LOG.debug(
"Sensor polling for "
+ ccHost
+ " took "
+ (System.currentTimeMillis() - startTime)
+ "ms");
}
return null;
}
});
} else {
LOG.warn(
"Skipping sensors polling for " + ccHost + ", previous poll not complete.");
}
}
}
}
} catch (Exception ex) {
LOG.error("Unable to listen for describe sensors events", ex);
}
} else {
LOG.error(
"DEFAULT_POLL_INTERVAL_MINS : "
+ DEFAULT_POLL_INTERVAL_MINS
+ " must be less than 1440 minutes");
}
}
}
/**
* Private for a reason.
*
* @see {@link EntityWrapper#get(Class)}
* @param persistenceContext
*/
@SuppressWarnings("unchecked")
private EntityWrapper(final String persistenceContext) {
this.tx = new TransactionState(persistenceContext);
this.txStart = Threads.currentStackString();
}
public class ListMetricQueue {
private static final Logger LOG = Logger.getLogger(ListMetricQueue.class);
private static class NoDupQueue<T> {
private LinkedHashSet<T> items = Sets.newLinkedHashSet();
public synchronized void drainTo(List<T> list) {
list.addAll(items);
items.clear();
}
public synchronized void drainTo(List<T> list, int maxItems) {
List<T> intermediateList = Lists.newArrayList();
int ctr = 0;
for (T item : items) {
intermediateList.add(item);
ctr++;
if (ctr == maxItems) break;
}
list.addAll(intermediateList);
items.removeAll(intermediateList);
}
public synchronized void putAll(List<T> list) {
items.addAll(list);
}
public synchronized void put(T item) {
items.add(item);
}
}
static final NoDupQueue<ListMetricQueueItem> dataQueue = new NoDupQueue<ListMetricQueueItem>();
private static final ScheduledExecutorService dataFlushTimer =
Executors.newSingleThreadScheduledExecutor(
Threads.threadFactory("cloudwatch-list-metrics-flush-%d"));
private static ListMetricQueue singleton = getInstance();
public static ListMetricQueue getInstance() {
synchronized (ListMetricQueue.class) {
if (singleton == null) singleton = new ListMetricQueue();
}
return singleton;
}
private void queue(ListMetricQueueItem metricData) {
dataQueue.put(metricData);
}
private static Runnable safeRunner =
new Runnable() {
@Override
public void run() {
long before = System.currentTimeMillis();
try {
List<ListMetricQueueItem> dataBatch = Lists.newArrayList();
dataQueue.drainTo(dataBatch);
ThruputMetrics.addDataPoint(MonitoredAction.LIST_METRIC_SIZE, dataBatch.size());
long t2 = System.currentTimeMillis();
dataBatch = prune(dataBatch);
long t3 = System.currentTimeMillis();
ThruputMetrics.addDataPoint(MonitoredAction.LIST_METRIC_PRUNE, t3 - t2);
List<ListMetric> listMetrics = convertToListMetrics(dataBatch);
long t4 = System.currentTimeMillis();
ThruputMetrics.addDataPoint(MonitoredAction.LIST_METRIC_CONVERT, t4 - t3);
ListMetricManager.addMetricBatch(listMetrics);
long t5 = System.currentTimeMillis();
ThruputMetrics.addDataPoint(MonitoredAction.LIST_METRIC_MERTIC_ADD_BATCH, t5 - t4);
} catch (Throwable ex) {
LOG.debug("ListMetricQueue:error");
ex.printStackTrace();
LOG.error(ex, ex);
} finally {
ThruputMetrics.addDataPoint(
MonitoredAction.LIST_METRIC_TIMING, System.currentTimeMillis() - before);
}
}
};
private static List<ListMetric> convertToListMetrics(List<ListMetricQueueItem> dataBatch) {
if (dataBatch == null) return null;
List<ListMetric> listMetrics = Lists.newArrayList();
for (ListMetricQueueItem item : dataBatch) {
listMetrics.add(
ListMetricManager.createListMetric(
item.getAccountId(),
item.getMetricName(),
item.getMetricType(),
item.getNamespace(),
item.getDimensionMap()));
}
return listMetrics;
}
private static List<ListMetricQueueItem> prune(List<ListMetricQueueItem> dataBatch) {
Set<ListMetricQueueItem> intermediateSet = Sets.newLinkedHashSet(dataBatch);
List<ListMetricQueueItem> prunedList = Lists.newArrayList(intermediateSet);
return prunedList;
}
static {
dataFlushTimer.scheduleAtFixedRate(safeRunner, 0, 5, TimeUnit.MINUTES);
}
public void addAll(List<SimpleMetricEntity> dataBatch) {
for (SimpleMetricEntity item : dataBatch) {
ListMetricQueueItem metricMetadata = new ListMetricQueueItem();
metricMetadata.setAccountId(item.getAccountId());
metricMetadata.setNamespace(item.getNamespace());
metricMetadata.setMetricName(item.getMetricName());
metricMetadata.setDimensionMap(item.getDimensionMap());
metricMetadata.setMetricType(item.getMetricType());
queue(metricMetadata);
}
}
}