private void startThreadedServer(
final InetSocketAddress serverAddress, final ReadStrategy readStrategy) throws Exception {
final AtomicBoolean readyToAccept = new AtomicBoolean(false);
final Runnable runner =
new Runnable() {
@Override
public void run() {
// startServer();
try {
runTestServer(serverAddress, readStrategy, readyToAccept);
} catch (final Exception e) {
e.printStackTrace();
}
}
};
final Thread t = new Thread(runner, "### starter");
t.setDaemon(true);
t.start();
synchronized (readyToAccept) {
if (!readyToAccept.get()) {
readyToAccept.wait(4000);
}
}
assertTrue("Not ready to accept?", readyToAccept.get());
Thread.yield();
Thread.yield();
Thread.yield();
Thread.yield();
}
public void destinationReady() throws InterruptedException {
if (refreshWanted.get()) {
synchronized (refreshWanted) {
log.debug(
"Waiting for new Temporary ReplyTo queue to be assigned before we can continue");
refreshWanted.wait();
}
}
}
void awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
LOGGER.debugf("awaitTermination: %dms", unit.toMillis(timeout));
synchronized (shutdownInitiated) {
if (shutdownInitiated.get() == false) {
LOGGER.debugf("shutdownInitiated.wait");
shutdownInitiated.wait(2000);
}
}
serviceContainer.awaitTermination(timeout == 0 ? Long.MAX_VALUE : timeout, unit);
}
void syncWithMockAppLauncher(
boolean allowScheduling, AtomicBoolean mockAppLauncherGoFlag, MockTezClient tezClient)
throws Exception {
synchronized (mockAppLauncherGoFlag) {
while (!mockAppLauncherGoFlag.get()) {
mockAppLauncherGoFlag.wait();
}
mockApp = tezClient.getLocalClient().getMockApp();
mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(allowScheduling);
mockAppLauncherGoFlag.notify();
}
}
public static void main(String[] args) throws Exception {
final Config config = Config.loadFromDisk("nexus-importer");
final HTTPCache http = HttpClient.createHttpCache(config);
final XmlParser xmlParser = new XmlParser();
final BoneCPDataSource boneCp = config.createBoneCp();
XmlParser.debugXml = false;
ObjectManager<NexusServerDto, ActorRef<NexusServer>> serverManager =
new ObjectManager<>(
"Nexus server",
Collections.<NexusServerDto>emptySet(),
new ObjectFactory<NexusServerDto, ActorRef<NexusServer>>() {
public ActorRef<NexusServer> create(NexusServerDto server) {
final NexusClient client = new NexusClient(http, server.url);
String name = server.name;
return ObjectUtil.threadedActor(
name,
config.nexusUpdateInterval,
boneCp,
"Nexus Server: " + name,
new NexusServer(client, server, xmlParser));
}
});
final AtomicBoolean shouldRun = new AtomicBoolean(true);
config.addShutdownHook(currentThread(), shouldRun);
while (shouldRun.get()) {
try {
List<NexusServerDto> newKeys;
try (Connection c = boneCp.getConnection()) {
newKeys = new NexusDao(c).selectServer();
}
serverManager.update(newKeys);
} catch (SQLException e) {
e.printStackTrace(System.out);
}
synchronized (shouldRun) {
shouldRun.wait(60 * 1000);
}
}
serverManager.close();
}
public void syncRefresh() throws IOException {
synchronized (isRefreshing) {
if (!isRefreshing.get()) {
doRefresh();
} else {
while (isRefreshing.get() && getCacheState(this.headerCache.get()) != CacheState.Good) {
try {
isRefreshing.wait(250);
} catch (InterruptedException e) {
throw new IOException(e);
}
}
}
}
}
public boolean matches(CharSequence input) {
while (!lock.compareAndSet(false, true)) {
try {
lock.wait();
} catch (InterruptedException e) {
}
}
matcher = pattern.matcher(input);
if (matcher.matches()) {
return true;
} else {
release();
return false;
}
}
protected HashResult getHashResult(Downloadable downloadable, File file)
throws InterruptedException {
if (JsonConfig.create(GeneralSettings.class).isHashCheckEnabled()
&& downloadable.isHashCheckEnabled()) {
AtomicBoolean hashCheckLock = new AtomicBoolean(false);
synchronized (HASHCHECK_QEUEU) {
HASHCHECK_QEUEU.add(hashCheckLock);
hashCheckLock.set(HASHCHECK_QEUEU.indexOf(hashCheckLock) != 0);
}
try {
if (hashCheckLock.get()) {
synchronized (hashCheckLock) {
if (hashCheckLock.get()) {
final PluginProgress hashProgress =
new HashCheckPluginProgress(null, Color.YELLOW.darker().darker(), null);
try {
downloadable.addPluginProgress(hashProgress);
hashCheckLock.wait();
} finally {
downloadable.removePluginProgress(hashProgress);
}
}
}
}
final HashInfo hashInfo = downloadable.getHashInfo();
final HashResult hashResult = downloadable.getHashResult(hashInfo, file);
return hashResult;
} finally {
synchronized (HASHCHECK_QEUEU) {
boolean callNext = HASHCHECK_QEUEU.indexOf(hashCheckLock) == 0;
HASHCHECK_QEUEU.remove(hashCheckLock);
if (HASHCHECK_QEUEU.size() > 0 && callNext) {
hashCheckLock = HASHCHECK_QEUEU.get(0);
} else {
hashCheckLock = null;
}
}
if (hashCheckLock != null) {
synchronized (hashCheckLock) {
hashCheckLock.set(false);
hashCheckLock.notifyAll();
}
}
}
}
return null;
}
/**
* Modify table is async so wait on completion of the table operation in master.
*
* @param tableName
* @param htd
* @throws IOException
*/
private void modifyTable(final byte[] tableName, final HTableDescriptor htd) throws IOException {
MasterServices services = TEST_UTIL.getMiniHBaseCluster().getMaster();
ExecutorService executor = services.getExecutorService();
AtomicBoolean done = new AtomicBoolean(false);
executor.registerListener(EventType.C_M_MODIFY_TABLE, new DoneListener(done));
this.admin.modifyTable(tableName, htd);
while (!done.get()) {
synchronized (done) {
try {
done.wait(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
executor.unregisterListener(EventType.C_M_MODIFY_TABLE);
}
private void waitForNotification(AtomicBoolean notifier, long timeout)
throws TimeoutException, RetryException {
// Wait for the RepRap to respond
try {
notifier.wait(timeout);
} catch (InterruptedException e) {
// Presumably we're shutting down
Thread.currentThread().interrupt();
return;
}
if (notifier.get() == true) {
return;
} else {
throw new RetryException();
}
}
/**
* Gets the current location for <code>.META.</code> if available and waits for up to the
* specified timeout if not immediately available. Throws an exception if timed out waiting. This
* method differs from {@link #waitForMeta()} in that it will go ahead and verify the location
* gotten from ZooKeeper and -ROOT- region by trying to use returned connection.
*
* @param timeout maximum time to wait for meta availability, in milliseconds
* @return {@link ServerName} for server hosting <code>.META.</code> or null if none available
* @throws InterruptedException if interrupted while waiting
* @throws IOException unexpected exception connecting to meta server
* @throws NotAllMetaRegionsOnlineException if meta not available before timeout
*/
public ServerName waitForMeta(long timeout)
throws InterruptedException, IOException, NotAllMetaRegionsOnlineException {
long stop = System.currentTimeMillis() + timeout;
long waitTime = Math.min(50, timeout);
synchronized (metaAvailable) {
while (!stopped && (timeout == 0 || System.currentTimeMillis() < stop)) {
if (getMetaServerConnection() != null) {
return metaLocation;
}
// perhaps -ROOT- region isn't available, let us wait a bit and retry.
metaAvailable.wait(waitTime);
}
if (getMetaServerConnection() == null) {
throw new NotAllMetaRegionsOnlineException("Timed out (" + timeout + "ms)");
}
return metaLocation;
}
}
public void terminate() {
// make sure the LogFlusher thread started before terminating it.
synchronized (isStarted) {
while (!isStarted.get()) {
try {
isStarted.wait();
} catch (InterruptedException e) {
// ignore
}
}
}
terminateFlag.set(true);
if (flushPage != null) {
synchronized (flushPage) {
flushPage.isStop(true);
flushPage.notify();
}
}
// [Notice]
// The return value doesn't need to be checked
// since terminateFlag will trigger termination if the flushQ is full.
flushQ.offer(POISON_PILL);
}
public void terminate() throws InterruptedException {
shouldBeActive.set(false);
synchronized (isActive) {
while (isActive.get()) isActive.wait();
}
}
void getSystemAndStartQuery() {
try {
Authentication groupId = DOFObjectID.Authentication.create(providerId.getBase());
DOFObjectID.Attribute attr = providerId.getAttribute(DOFObjectID.Attribute.GROUP);
Domain domainId = DOFObjectID.Domain.create(attr.getValueObjectID());
DOFCredentials domainCredentials =
DOFCredentials.create(hubProvideFactory.coreCredentials, domainId);
//
// @formatter:off
DOFSystem.Config domainSystemConfig =
new DOFSystem.Config.Builder()
.setName(groupId.getDataString() + "." + domainId.getDataString() + "-DOFSystem")
.setCredentials(domainCredentials)
.setPermissionsExtendAllowed(true)
.setTunnelDomains(true)
.build();
// @formatter:on
DOFDomain.Config domainConfig = new DOFDomain.Config.Builder(domainCredentials).build();
DOFDomain serviceDomain = hubProvideFactory.dof.createDomain(domainConfig);
serviceDomain.addStateListener(this);
long t0 = System.currentTimeMillis();
long to = hubProvideFactory.commTimeout;
synchronized (domainFound) {
while (!domainFound.get()) {
try {
domainFound.wait(to);
if (domainFound.get()) break;
long delta = System.currentTimeMillis() - t0;
if (delta >= hubProvideFactory.commTimeout)
throw new TimeoutException(
"timed out: " + to + " waiting for Domain listener to report completed");
to =
hubProvideFactory.commTimeout
- delta; // spurious wakeup, or wait(to) slightly off System.currentTimeMillis
} finally {
serviceDomain.removeStateListener(this);
}
}
}
DOFSystem system =
hubProvideFactory.dof.createSystem(domainSystemConfig, hubProvideFactory.commTimeout);
systemData = new CreateSystemTask.SystemData(groupId, domainId, domainCredentials, system);
} catch (InterruptedException e) {
hubProvideFactory.removeHubRequestMonitor(providerId);
log.debug("Create system task interrupted.");
return;
} catch (Exception e) {
hubProvideFactory.removeHubRequestMonitor(providerId);
log.debug("Create system failed {} - " + e.getCause(), providerId);
return;
}
synchronized (this) {
if (activateInterestOperation == null || activateInterestOperation.isComplete())
activateInterestOperation =
systemData.system.beginInterest(
systemData.groupId,
HubRequestInterface.IID,
DOFInterestLevel.ACTIVATE,
DOF.TIMEOUT_NEVER,
null,
null);
if (queryOperation == null || queryOperation.isComplete()) {
DOFQuery query =
new DOFQuery.Builder()
.addFilter(providerId)
.addRestriction(HubRequestInterface.IID)
.build();
queryOperation = systemData.system.beginQuery(query, DOF.TIMEOUT_NEVER, this, null);
}
createSystemFuture = null;
log.debug(
"HubManager dof: "
+ hubProvideFactory.dof.getState().getName()
+ " beginQuery for: "
+ providerId.toStandardString()
+ ":"
+ HubRequestInterface.IID);
}
}
@Override
public void run() {
AtomicBoolean pause = engine.getPause();
if (pause.get()) {
synchronized (pause) {
log(LOG_WAITING_FOR_RESUME, memoryCacheKey);
try {
pause.wait();
} catch (InterruptedException e) {
L.e(LOG_TASK_INTERRUPTED, memoryCacheKey);
return;
}
log(LOG_RESUME_AFTER_PAUSE, memoryCacheKey);
}
}
if (checkTaskIsNotActual()) return;
if (options.shouldDelayBeforeLoading()) {
log(LOG_DELAY_BEFORE_LOADING, options.getDelayBeforeLoading(), memoryCacheKey);
try {
Thread.sleep(options.getDelayBeforeLoading());
} catch (InterruptedException e) {
L.e(LOG_TASK_INTERRUPTED, memoryCacheKey);
return;
}
if (checkTaskIsNotActual()) return;
}
ReentrantLock loadFromUriLock = imageLoadingInfo.loadFromUriLock;
log(LOG_START_DISPLAY_IMAGE_TASK, memoryCacheKey);
if (loadFromUriLock.isLocked()) {
log(LOG_WAITING_FOR_IMAGE_LOADED, memoryCacheKey);
}
loadFromUriLock.lock();
Bitmap bmp;
try {
if (checkTaskIsNotActual()) return;
bmp = configuration.memoryCache.get(memoryCacheKey);
if (bmp == null) {
bmp = tryLoadBitmap();
if (bmp == null) return;
if (checkTaskIsNotActual() || checkTaskIsInterrupted()) return;
if (options.isCacheInMemory()) {
if (options.shouldPreProcess()) {
log(LOG_PREPROCESS_IMAGE, memoryCacheKey);
bmp = options.getPreProcessor().process(bmp, imageView);
}
log(LOG_CACHE_IMAGE_IN_MEMORY, memoryCacheKey);
configuration.memoryCache.put(memoryCacheKey, bmp);
}
} else {
log(LOG_GET_IMAGE_FROM_MEMORY_CACHE_AFTER_WAITING, memoryCacheKey);
}
if (options.shouldPostProcess()) {
log(LOG_POSTPROCESS_IMAGE, memoryCacheKey);
bmp = options.getPostProcessor().process(bmp, imageView);
}
} finally {
loadFromUriLock.unlock();
}
if (checkTaskIsNotActual() || checkTaskIsInterrupted()) return;
DisplayBitmapTask displayBitmapTask = new DisplayBitmapTask(bmp, imageLoadingInfo, engine);
displayBitmapTask.setLoggingEnabled(loggingEnabled);
handler.post(displayBitmapTask);
}
@Test
public void testFeedServer() throws Exception {
final InetSocketAddress serverAddress = startThreadedServer();
synchronized (m_dataServerStarted) {
if (!m_dataServerStarted.get()) {
m_dataServerStarted.wait(3000);
}
}
final ServerDataFeeder feeder = new ServerDataFeeder(serverAddress);
final Map<InetSocketAddress, Collection<ByteBuffer>> addressesToMessages =
new HashMap<InetSocketAddress, Collection<ByteBuffer>>();
final Collection<SendIndication> messages = new LinkedList<SendIndication>();
final AtomicInteger totalDataSent = new AtomicInteger(0);
final AtomicInteger totalDataReceived = new AtomicInteger(0);
final IoSession session =
new IoSessionStub() {
@Override
public WriteFuture write(final Object message) {
final SendIndication di = (SendIndication) message;
final InetSocketAddress address = di.getRemoteAddress();
if (addressesToMessages.containsKey(address)) {
final Collection<ByteBuffer> bufs = addressesToMessages.get(address);
addRaw(bufs, di);
} else {
final Collection<ByteBuffer> bufs = new LinkedList<ByteBuffer>();
addressesToMessages.put(address, bufs);
addRaw(bufs, di);
}
messages.add(di);
if (messages.size() == NUM_REMOTE_HOSTS) {
synchronized (messages) {
messages.notify();
}
}
return null;
}
// This method is required to extract the data from the TCP frames
// to verify it matches the original data sent (since the server
// just echoes everything it sees).
private void addRaw(final Collection<ByteBuffer> bufs, final SendIndication si) {
final byte[] rawData = si.getData();
final byte[] noFrame = ArrayUtils.subarray(rawData, 2, rawData.length);
totalDataReceived.addAndGet(noFrame.length);
bufs.add(ByteBuffer.wrap(noFrame));
synchronized (totalDataReceived) {
if (totalDataReceived.get() == totalDataSent.get()) {
totalDataReceived.notify();
}
}
}
};
for (int j = 0; j < NUM_MESSAGES_PER_HOST; j++) {
for (int i = 0; i < NUM_REMOTE_HOSTS; i++) {
final InetSocketAddress remoteAddress =
new InetSocketAddress("44.52.67." + (1 + i), 4728 + i);
feeder.onRemoteAddressOpened(remoteAddress, session);
feeder.onData(remoteAddress, session, DATA);
totalDataSent.addAndGet(DATA.length);
}
}
synchronized (m_dataServerSockets) {
if (m_dataServerSockets.get() < NUM_REMOTE_HOSTS) {
m_dataServerSockets.wait(6000);
}
}
assertEquals(NUM_REMOTE_HOSTS, m_dataServerSockets.get());
synchronized (totalDataReceived) {
if (totalDataReceived.get() < totalDataSent.get()) {
totalDataReceived.wait(6000);
}
}
assertEquals(totalDataSent.get(), totalDataReceived.get());
// The messages received should be TURN Send Indications that wrap
// TCP Frames. They're already placed in the appropriate buckets for
// each remote host. Now we just need to verify the data.
for (final Map.Entry<InetSocketAddress, Collection<ByteBuffer>> entry :
addressesToMessages.entrySet()) {
verifyData(entry.getValue());
}
}
/**
* Update ZK, ROOT or META. This can take a while if for example the .META. is not available -- if
* server hosting .META. crashed and we are waiting on it to come back -- so run in a thread and
* keep updating znode state meantime so master doesn't timeout our region-in-transition. Caller
* must cleanup region if this fails.
*/
boolean updateMeta(final HRegion r) {
if (this.server.isStopped() || this.rsServices.isStopping()) {
return false;
}
// Object we do wait/notify on. Make it boolean. If set, we're done.
// Else, wait.
final AtomicBoolean signaller = new AtomicBoolean(false);
PostOpenDeployTasksThread t =
new PostOpenDeployTasksThread(r, this.server, this.rsServices, signaller);
t.start();
int assignmentTimeout =
this.server
.getConfiguration()
.getInt("hbase.master.assignment.timeoutmonitor.period", 10000);
// Total timeout for meta edit. If we fail adding the edit then close out
// the region and let it be assigned elsewhere.
long timeout = assignmentTimeout * 10;
long now = System.currentTimeMillis();
long endTime = now + timeout;
// Let our period at which we update OPENING state to be be 1/3rd of the
// regions-in-transition timeout period.
long period = Math.max(1, assignmentTimeout / 3);
long lastUpdate = now;
boolean tickleOpening = true;
while (!signaller.get()
&& t.isAlive()
&& !this.server.isStopped()
&& !this.rsServices.isStopping()
&& (endTime > now)) {
long elapsed = now - lastUpdate;
if (elapsed > period) {
// Only tickle OPENING if postOpenDeployTasks is taking some time.
lastUpdate = now;
tickleOpening = tickleOpening("post_open_deploy");
}
synchronized (signaller) {
try {
signaller.wait(period);
} catch (InterruptedException e) {
// Go to the loop check.
}
}
now = System.currentTimeMillis();
}
// Is thread still alive? We may have left above loop because server is
// stopping or we timed out the edit. Is so, interrupt it.
if (t.isAlive()) {
if (!signaller.get()) {
// Thread still running; interrupt
LOG.debug("Interrupting thread " + t);
t.interrupt();
}
try {
t.join();
} catch (InterruptedException ie) {
LOG.warn("Interrupted joining " + r.getRegionInfo().getRegionNameAsString(), ie);
Thread.currentThread().interrupt();
}
}
// Was there an exception opening the region? This should trigger on
// InterruptedException too. If so, we failed. Even if tickle opening fails
// then it is a failure.
return ((!Thread.interrupted() && t.getException() == null) && tickleOpening);
}