public static boolean testCommand(
final Context context,
final String command,
final String contains,
final List<String> arguments) {
try {
final List<String> commandAndArgs = new ArrayList<String>();
commandAndArgs.add(command);
commandAndArgs.addAll(arguments);
final ProcessBuilder pb = new ProcessBuilder(commandAndArgs);
logCommand(context, pb);
final Process compilation = pb.start();
final ConsumeStream result = ConsumeStream.start(compilation.getInputStream(), null);
final ConsumeStream error = ConsumeStream.start(compilation.getErrorStream(), null);
compilation.waitFor();
result.join();
error.join();
return error.output.toString().contains(contains)
|| result.output.toString().contains(contains);
} catch (IOException ex) {
context.log(ex.getMessage());
return false;
} catch (InterruptedException ex) {
context.log(ex.getMessage());
return false;
}
}
@Override
public void run() {
try {
if (Fecs.getApplicationContext() == null) return;
Long currentTime = System.currentTimeMillis();
double deltaTime = (currentTime - lastUpdateTime) * 0.001;
if (getEngineState() == STATE_START) { // last bit is 1 = started
int s = getCircumstanceState() - 1; // 0 is null state(error)
if (s >= CircumstanceType.values().length || s < 0)
throw new Exception("unstable state value with " + String.valueOf(s));
Circumstance.get(CircumstanceType.values()[s])
.setParameter("currentTime", currentTime)
.setParameter("deltaTime", deltaTime)
.trigger();
for (Cabin cabin : cabins.values()) updateCabin(cabin, deltaTime);
}
lastUpdateTime = currentTime;
draw();
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
try {
Thread.sleep(1);
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
}
SwingUtilities.invokeLater(this);
}
}
private void setExecutablePermissions(File gradleHome) {
if (isWindows()) {
return;
}
File gradleCommand = new File(gradleHome, "bin/gradle");
String errorMessage = null;
try {
ProcessBuilder pb = new ProcessBuilder("chmod", "755", gradleCommand.getCanonicalPath());
Process p = pb.start();
if (p.waitFor() == 0) {
System.out.println("Set executable permissions for: " + gradleCommand.getAbsolutePath());
} else {
BufferedReader is = new BufferedReader(new InputStreamReader(p.getInputStream()));
Formatter stdout = new Formatter();
String line;
while ((line = is.readLine()) != null) {
stdout.format("%s%n", line);
}
errorMessage = stdout.toString();
}
} catch (IOException e) {
errorMessage = e.getMessage();
} catch (InterruptedException e) {
errorMessage = e.getMessage();
}
if (errorMessage != null) {
System.out.println(
"Could not set executable permissions for: " + gradleCommand.getAbsolutePath());
System.out.println("Please do this manually if you want to use the Gradle UI.");
}
}
private void decompressTile(final File outputFile, int jp2TileX, int jp2TileY)
throws IOException {
final int tileIndex = bandInfo.imageLayout.numXTiles * jp2TileY + jp2TileX;
final Process process =
new ProcessBuilder(
EXE,
"-i",
imageFile.getPath(),
"-o",
outputFile.getPath(),
"-r",
getLevel() + "",
"-t",
tileIndex + "")
.directory(cacheDir)
.start();
try {
final int exitCode = process.waitFor();
if (exitCode != 0) {
System.err.println("Failed to uncompress tile: exitCode = " + exitCode);
}
} catch (InterruptedException e) {
System.err.println("InterruptedException: " + e.getMessage());
}
}
protected void runSootDirectly(String mainClass) {
int length = getSootCommandList().getList().size();
String temp[] = new String[length];
getSootCommandList().getList().toArray(temp);
sendSootOutputEvent(mainClass);
sendSootOutputEvent(" ");
final String[] cmdAsArray = temp;
for (int i = 0; i < temp.length; i++) {
sendSootOutputEvent(temp[i]);
sendSootOutputEvent(" ");
}
sendSootOutputEvent("\n");
IRunnableWithProgress op;
try {
newProcessStarting();
op = new SootRunner(temp, Display.getCurrent(), mainClass);
((SootRunner) op).setParent(this);
ModalContext.run(op, true, new NullProgressMonitor(), Display.getCurrent());
} catch (InvocationTargetException e1) {
// handle exception
System.out.println("InvocationTargetException: " + e1.getMessage());
System.out.println("InvocationTargetException: " + e1.getTargetException());
System.out.println(e1.getStackTrace());
} catch (InterruptedException e2) {
// handle cancelation
System.out.println("InterruptedException: " + e2.getMessage());
}
}
/**
* Return {@code true} if the specified dn is contained in the parent set, or in the specified DN
* cache. This would indicate that the parent has already been processed. It returns {@code false}
* otherwise.
*
* <p>It will optionally check the dn2id database for the dn if the specified cleared backend
* boolean is {@code true}.
*
* @param dn The DN to check for.
* @param dnCache The importer DN cache.
* @param clearedBackend Set to {@code true} if the import process cleared the backend before
* processing.
* @return {@code true} if the dn is contained in the parent ID, or {@code false} otherwise.
* @throws DatabaseException If an error occurred searching the DN cache, or dn2id database.
* @throws InterruptedException If an error occurred processing the pending map.
*/
public boolean isParentProcessed(DN dn, DNCache dnCache, boolean clearedBackend)
throws DatabaseException, InterruptedException {
synchronized (synchObject) {
if (parentSet.contains(dn)) {
return true;
}
}
// The DN was not in the parent set. Make sure it isn't pending.
try {
assureNotPending(dn);
} catch (InterruptedException e) {
logger.error(ERR_IMPORT_LDIF_PENDING_ERR, e.getMessage());
throw e;
}
// Either parent is in the DN cache,
// or else check the dn2id database for the DN (only if backend wasn't cleared)
final boolean parentThere =
dnCache.contains(dn)
|| (!clearedBackend && getDN2ID().get(null, dn, LockMode.DEFAULT) != null);
// Add the DN to the parent set if needed.
if (parentThere) {
synchronized (synchObject) {
if (parentSet.size() >= PARENT_ID_SET_SIZE) {
Iterator<DN> iterator = parentSet.iterator();
iterator.next();
iterator.remove();
}
parentSet.add(dn);
}
}
return parentThere;
}
@Override
public Node.PropertySet[] getPropertySets() {
final Node.PropertySet[][] props = new Node.PropertySet[1][];
Runnable runnable =
new Runnable() {
@Override
public void run() {
FormLAF.executeWithLAFLocks(
new Runnable() {
@Override
public void run() {
props[0] = component.getProperties();
}
});
}
};
if (EventQueue.isDispatchThread()) {
runnable.run();
} else {
try {
// We have made some attempts to keep initialization
// of properties outside AWT thread, but it always
// deadlocked with AWT thread for various reasons.
EventQueue.invokeAndWait(runnable);
} catch (InterruptedException iex) {
FormUtils.LOGGER.log(Level.INFO, iex.getMessage(), iex);
} catch (InvocationTargetException itex) {
FormUtils.LOGGER.log(Level.INFO, itex.getMessage(), itex);
}
}
return props[0];
}
@Async
public Future<Boolean> stop(String nsr_id, String vnfr_id) {
log.debug("Stopping ExecutionTask/CooldownTask for VNFR with id: " + vnfr_id);
int i = 60;
while (!actionMonitor.isTerminated(vnfr_id)
&& actionMonitor.getAction(vnfr_id) != Action.INACTIVE
&& i >= 0) {
actionMonitor.terminate(vnfr_id);
log.debug(
"Waiting for finishing ExecutionTask/Cooldown for VNFR with id: "
+ vnfr_id
+ " ("
+ i
+ "s)");
log.debug(actionMonitor.toString());
try {
Thread.sleep(1_000);
} catch (InterruptedException e) {
log.error(e.getMessage(), e);
}
i--;
if (i <= 0) {
actionMonitor.removeId(vnfr_id);
log.error(
"Were not able to wait until ExecutionTask finished for VNFR with id: " + vnfr_id);
return new AsyncResult<>(false);
}
}
actionMonitor.removeId(vnfr_id);
log.info("Stopped ExecutionTask for VNFR with id: " + vnfr_id);
return new AsyncResult<>(true);
}
public static void main(String[] args) {
long head = 0;
num_thread = Integer.parseInt(args[0]);
long total = Long.parseLong(args[1]);
section = total / ((long) num_thread);
sectionHead = new long[num_thread];
long start = System.currentTimeMillis();
Thread[] threads = new Thread[num_thread];
for (int i = 0; i < num_thread; i++) {
threads[i] = new Thread(new CoinFlip(i));
threads[i].start();
}
for (int j = 0; j < num_thread; j++) {
try {
threads[j].join();
head = head + sectionHead[j];
} catch (InterruptedException e) {
System.out.println(
"Thread interrupted. Exception: " + e.toString() + " Message: " + e.getMessage());
return;
}
}
long end = System.currentTimeMillis();
System.out.println(head + " heads in " + args[1] + " coin tosses");
System.out.println("Elapsed time: " + ((end - start)) + "ms");
}
// This may be aborted if there are stagnant requests sitting in queue.
// This needs fixed to discard outstanding save requests.
public synchronized void forceSave() {
try {
Future<?> future = delayedSave(configFile);
if (future != null) {
future.get();
}
} catch (InterruptedException ex) {
LOGGER.log(Level.SEVERE, ex.getMessage(), ex);
} catch (ExecutionException ex) {
LOGGER.log(Level.SEVERE, ex.getMessage(), ex);
}
}
@Override
public List<SecretAuthenticationKey> authenticate(
@WebParam(name = "authenticationData", targetNamespace = "")
List<AuthenticationTriple> authenticationData)
throws AuthenticationExceptionException, SNAAExceptionException {
Map<String, Set<AuthenticationTriple>> intersectionPrefixSet =
getIntersectionPrefixSetAT(authenticationData);
Set<Future<List<SecretAuthenticationKey>>> futures =
new HashSet<Future<List<SecretAuthenticationKey>>>();
for (String wsEndpointUrl : intersectionPrefixSet.keySet()) {
AuthenticationCallable authenticationCallable =
new AuthenticationCallable(
wsEndpointUrl,
new ArrayList<AuthenticationTriple>(intersectionPrefixSet.get(wsEndpointUrl)));
Future<List<SecretAuthenticationKey>> future = executorService.submit(authenticationCallable);
futures.add(future);
}
List<SecretAuthenticationKey> resultSet = new LinkedList<SecretAuthenticationKey>();
for (Future<List<SecretAuthenticationKey>> future : futures) {
try {
resultSet.addAll(future.get());
} catch (InterruptedException e) {
SNAAException exception = new SNAAException();
exception.setMessage(e.getMessage());
throw new SNAAExceptionException(e.getMessage(), exception, e);
} catch (ExecutionException e) {
SNAAException exception = new SNAAException();
exception.setMessage(e.getMessage());
throw new SNAAExceptionException(e.getMessage(), exception, e);
}
}
return resultSet;
}
boolean awaitLogin() {
try {
if (!loginSuccess.await(10L, TimeUnit.SECONDS)) {
disconnectAbnormally("Timed out waiting for login response");
throw new RuntimeException("Timed out waiting for login response");
}
return isLoggedIn();
} catch (InterruptedException e) {
errorStack.push(new Error(null, e.getMessage(), wireTrace.list()));
throw new RuntimeException("Interruption while awaiting server login", e);
}
}
@Override
protected void done() {
super.done();
try {
this.get();
listener.done();
} catch (InterruptedException e) {
listener.setStatus("Interrupted Exception: " + e.getMessage());
e.printStackTrace(new PrintStream(callbacks.getStderr()));
} catch (ExecutionException e) {
listener.setStatus("Execution Exception: " + e.getMessage());
e.printStackTrace(new PrintStream(callbacks.getStderr()));
} catch (Throwable e) {
listener.setStatus(e.getMessage());
e.printStackTrace(new PrintStream(callbacks.getStderr()));
}
}
/** Обработчик всех событий помещенных в очередь */
void processEvents() {
for (; ; ) {
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
LOG.log(Level.SEVERE, x.getMessage());
return;
}
Path dir = keys.get(key);
if (dir == null) {
LOG.log(Level.SEVERE, "Входной каталог не найден!");
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind kind = event.kind();
// TODO - подумать над обработчиком события OVERFLOW
if (kind == OVERFLOW) {
continue;
}
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
// логируем событие
if (kind == ENTRY_CREATE) {
LOG.log(Level.FINEST, "{0}: {1}", new Object[] {event.kind().name(), child});
Runnable worker = new WorkerThread(child);
executor.execute(worker);
}
}
boolean valid = key.reset();
if (!valid) {
keys.remove(key);
if (keys.isEmpty()) {
break;
}
}
}
}
/**
* Returns control when task is complete.
*
* @param json
* @param logger
*/
private String waitForDeploymentCompletion(JSON json, OctopusApi api, Log logger) {
final long WAIT_TIME = 5000;
final double WAIT_RANDOM_SCALER = 100.0;
JSONObject jsonObj = (JSONObject) json;
String id = jsonObj.getString("TaskId");
Task task = null;
String lastState = "Unknown";
try {
task = api.getTask(id);
} catch (IOException ex) {
logger.error("Error getting task: " + ex.getMessage());
return null;
}
logger.info("Task info:");
logger.info("\tId: " + task.getId());
logger.info("\tName: " + task.getName());
logger.info("\tDesc: " + task.getDescription());
logger.info("\tState: " + task.getState());
logger.info("\n\nStarting wait...");
boolean completed = task.getIsCompleted();
while (!completed) {
try {
task = api.getTask(id);
} catch (IOException ex) {
logger.error("Error getting task: " + ex.getMessage());
return null;
}
completed = task.getIsCompleted();
lastState = task.getState();
logger.info("Task state: " + lastState);
if (completed) {
break;
}
try {
Thread.sleep(WAIT_TIME + (long) (Math.random() * WAIT_RANDOM_SCALER));
} catch (InterruptedException ex) {
logger.info("Wait interrupted!");
logger.info(ex.getMessage());
completed = true; // bail out of wait loop
}
}
logger.info("Wait complete!");
return lastState;
}
/**
* 获得给定url页面的源码
*
* @param crawlerPage
* @param url
* @return
*/
private HtmlBean getNewSourceCode(CrawlerPage crawlerPage, String url) {
boolean useProxy = (boolean) CrawlerConfigurableProperties.getProperty("tb.use.proxy");
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
logger.warn(e.getMessage());
}
crawlerPage.getUriData().setEscapeSetCookie(true);
crawlerPage.getUriData().setId(crawlerPage.getUriData().getId());
crawlerPage.getUriData().setUri(URIUtils.getHttpURL(url));
crawlerPage.getUriData().setSave(true);
crawlerPage.getProxy().setUseProxy(useProxy);
crawlerPage.getProxy().setUseSameProxy(true);
Fetcher fetcher = new Fetcher();
fetcher.process(crawlerPage);
return new HtmlBean(
crawlerPage.getSourceData().getSourceCode(),
crawlerPage.getUriData().getHbaseParam().getKey());
}
protected void runSootAsProcess(String cmd) {
SootProcessRunner op;
try {
newProcessStarting();
op = new SootProcessRunner(Display.getCurrent(), cmd, sootClasspath);
if (window == null) {
window = SootPlugin.getDefault().getWorkbench().getActiveWorkbenchWindow();
}
new ProgressMonitorDialog(window.getShell()).run(true, true, op);
} catch (InvocationTargetException e1) {
// handle exception
} catch (InterruptedException e2) {
// handle cancelation
System.out.println(e2.getMessage());
}
}
private static BufferedImage convertToBufferedImage(Image image) throws IOException {
if (image instanceof BufferedImage) {
return (BufferedImage) image;
} else {
MediaTracker tracker = new MediaTracker(null /*not sure how this is used*/);
tracker.addImage(image, 0);
try {
tracker.waitForAll();
} catch (InterruptedException e) {
throw new IOException(e.getMessage());
}
BufferedImage bufImage =
new BufferedImage(
image.getWidth(null), image.getHeight(null), BufferedImage.TYPE_INT_RGB);
Graphics g = bufImage.createGraphics();
g.drawImage(image, 0, 0, null);
return bufImage;
}
}
@Override
public boolean isAuthorized(
@WebParam(name = "authenticationData", targetNamespace = "")
List<SecretAuthenticationKey> authenticationData,
@WebParam(name = "action", targetNamespace = "") Action action)
throws SNAAExceptionException {
if (authenticationData == null || action == null) {
throw createSNAAException("Arguments must not be null!");
}
Map<String, Set<SecretAuthenticationKey>> intersectionPrefixSet =
getIntersectionPrefixSetSAK(authenticationData);
Set<Future<Boolean>> futures = new HashSet<Future<Boolean>>();
for (String urnPrefix : intersectionPrefixSet.keySet()) {
IsAuthorizedCallable authenticationCallable =
new IsAuthorizedCallable(
getWsnUrlFromUrnPrefix(urnPrefix),
new ArrayList<SecretAuthenticationKey>(intersectionPrefixSet.get(urnPrefix)),
action);
Future<Boolean> future = executorService.submit(authenticationCallable);
futures.add(future);
}
for (Future<Boolean> future : futures) {
try {
if (!future.get()) {
return false;
}
} catch (InterruptedException e) {
throw createSNAAException(e.getMessage());
} catch (ExecutionException e) {
throw createSNAAException(e.getMessage());
}
}
return true;
}
@SuppressWarnings({"unchecked", "boxing", "rawtypes"})
private DiscoveryManager getRemoteProxyToDiscoveryManager(final String ip, final int port) {
final String address = ip + ":" + port;
final int timeout = 500; // 500 ms RemoteAPIImpl if need detailed version...
this.logger.status("remoteproxytodiscovery/start", "ip", ip, "port", port);
final DiscoveryManager newClient =
Proxies.newClient(
EXPORT_NAME, address, getClass().getClassLoader(), DiscoveryManager.class);
// Execute collection asynchronously (TODO: cache pool usage could be improved)
final ExecutorService cachePool =
Executors.newCachedThreadPool(RemoteDiscoveryImpl.threadFactory);
final ExecutorCompletionService<String> ecs = new ExecutorCompletionService(cachePool);
final Future<String> future =
ecs.submit(
new Callable<String>() {
public String call() throws Exception {
return AccessController.doPrivileged(
new PrivilegedAction<String>() {
public String run() {
return newClient.ping(667) == 667 ? "OK" : null;
}
});
}
});
// Wait at most half a second (TODO: Make this configurable)
try {
final String string = future.get(timeout, TimeUnit.MILLISECONDS);
if (string == null) return null;
return newClient;
} catch (final InterruptedException e) {
this.logger.status("remoteproxytodiscovery/exception/interrupted", "message", e.getMessage());
e.printStackTrace();
} catch (final ExecutionException e) {
this.logger.status(
"remoteproxytodiscovery/exception/executionexception", "message", e.getMessage());
} catch (final TimeoutException e) {
this.logger.status(
"remoteproxytodiscovery/exception/timeoutexception", "message", e.getMessage());
} catch (final SecurityException e) {
this.logger.status(
"remoteproxytodiscovery/exception/securityexception", "message", e.getMessage());
e.printStackTrace();
} finally {
AccessController.doPrivileged(
new PrivilegedAction<Object>() {
public Object run() {
future.cancel(true);
cachePool.shutdownNow();
return null;
}
});
this.logger.status("remoteproxytodiscovery/end");
}
this.logger.status("remoteproxytodiscovery/end");
return null;
}
@Override
public void run() {
FeedMedia media = DBReader.getFeedMedia(DownloadService.this, request.getFeedfileId());
if (media == null) {
throw new IllegalStateException("Could not find downloaded media object in database");
}
boolean chaptersRead = false;
media.setDownloaded(true);
media.setFile_url(request.getDestination());
// Get duration
MediaMetadataRetriever mmr = null;
try {
mmr = new MediaMetadataRetriever();
mmr.setDataSource(media.getFile_url());
String durationStr = mmr.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION);
media.setDuration(Integer.parseInt(durationStr));
if (BuildConfig.DEBUG) Log.d(TAG, "Duration of file is " + media.getDuration());
} catch (NumberFormatException e) {
e.printStackTrace();
} catch (RuntimeException e) {
e.printStackTrace();
} finally {
if (mmr != null) {
mmr.release();
}
}
if (media.getItem().getChapters() == null) {
ChapterUtils.loadChaptersFromFileUrl(media);
if (media.getItem().getChapters() != null) {
chaptersRead = true;
}
}
try {
if (chaptersRead) {
DBWriter.setFeedItem(DownloadService.this, media.getItem()).get();
}
DBWriter.setFeedMedia(DownloadService.this, media).get();
if (!DBTasks.isInQueue(DownloadService.this, media.getItem().getId())) {
DBWriter.addQueueItem(DownloadService.this, media.getItem().getId()).get();
}
} catch (ExecutionException e) {
e.printStackTrace();
status =
new DownloadStatus(
media,
media.getEpisodeTitle(),
DownloadError.ERROR_DB_ACCESS_ERROR,
false,
e.getMessage());
} catch (InterruptedException e) {
e.printStackTrace();
status =
new DownloadStatus(
media,
media.getEpisodeTitle(),
DownloadError.ERROR_DB_ACCESS_ERROR,
false,
e.getMessage());
}
saveDownloadStatus(status);
sendDownloadHandledIntent();
numberOfDownloads.decrementAndGet();
queryDownloadsAsync();
}
@Override
public void run() {
try {
UUID exchangeId = UUID.randomUUID();
logger.info("Starting non blocking background syncer (Exchange: " + exchangeId + ")");
this.eventAggregator.stop();
FetchObjectStoreExchangeHandler fetchObjectStoreExchangeHandler =
new FetchObjectStoreExchangeHandler(this.node, this.nodeManager, exchangeId);
int nrOfRunningResponseCallbackExchanges =
this.node.getObjectDataReplyHandler().getResponseCallbackHandlers().size();
if (0 < nrOfRunningResponseCallbackExchanges
|| this.node.getObjectDataReplyHandler().areRequestCallbacksRunning()) {
// other callbacks are currently in progress
// -> postpone this run to next interval
logger.info(
"Skipping execution of background synchronisation since other exchanges are currently running");
return;
}
this.node
.getObjectDataReplyHandler()
.addResponseCallbackHandler(exchangeId, fetchObjectStoreExchangeHandler);
Thread fetchObjectStoreExchangeHandlerThread = new Thread(fetchObjectStoreExchangeHandler);
fetchObjectStoreExchangeHandlerThread.setName(
"FetchObjectStoreExchangeHandler-" + exchangeId);
fetchObjectStoreExchangeHandlerThread.start();
logger.info("Waiting for fetching of object stores to complete");
try {
fetchObjectStoreExchangeHandler.await();
} catch (InterruptedException e) {
logger.error("Got interrupted while waiting for fetching all object stores");
}
this.node.getObjectDataReplyHandler().removeResponseCallbackHandler(exchangeId);
if (!fetchObjectStoreExchangeHandler.isCompleted()) {
logger.error(
"FetchObjectStoreExchangeHandler should be completed after awaiting. Since we do not know about the other clients object store, we abort background sync for exchange "
+ exchangeId);
return;
}
FetchObjectStoreExchangeHandlerResult result = fetchObjectStoreExchangeHandler.getResult();
Map<ClientDevice, IObjectStore> objectStores = Zip.unzipObjectStore(this.objectStore, result);
// use a tree map for guaranteed ordering
Map<String, ClientDevice> deletedPaths = new TreeMap<>(new StringLengthComparator());
Map<String, ClientDevice> updatedPaths = new TreeMap<>(new StringLengthComparator());
Map<String, ClientDevice> conflictPaths = new TreeMap<>(new StringLengthComparator());
for (Map.Entry<ClientDevice, IObjectStore> entry : objectStores.entrySet()) {
HashMap<ObjectStore.MergedObjectType, Set<String>> outdatedOrDeletedPaths =
this.objectStore.mergeObjectStore(entry.getValue());
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.CHANGED)
.stream()
.filter(outDatedPath -> !this.isIgnored(outDatedPath))
.forEach(outDatedPath -> updatedPaths.put(outDatedPath, entry.getKey()));
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.DELETED)
.stream()
.filter(deletedPath -> !this.isIgnored(deletedPath))
.forEach(deletedPath -> deletedPaths.put(deletedPath, entry.getKey()));
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.CONFLICT)
.stream()
.filter(conflictPath -> !this.isIgnored(conflictPath))
.forEach(conflictPath -> conflictPaths.put(conflictPath, entry.getKey()));
entry.getValue().getObjectManager().getStorageAdapater().delete(new TreePathElement("./"));
this.objectStore
.getObjectManager()
.getStorageAdapater()
.delete(new TreePathElement(entry.getKey().getClientDeviceId().toString()));
}
// delete all removed files
logger.info("Removing all (" + deletedPaths.size() + ") deleted files");
for (Map.Entry<String, ClientDevice> entry : deletedPaths.entrySet()) {
logger.debug("Removing deleted path " + entry.getKey());
TreePathElement elementToDelete = new TreePathElement(entry.getKey());
// only delete the file on disk if it actually exists
if (this.storageAdapter.exists(StorageType.DIRECTORY, elementToDelete)
|| this.storageAdapter.exists(StorageType.FILE, elementToDelete)) {
this.storageAdapter.delete(elementToDelete);
}
}
logger.info("Creating all (" + conflictPaths.size() + ") conflict files");
for (Map.Entry<String, ClientDevice> entry : conflictPaths.entrySet()) {
logger.debug("Creating conflict file " + entry.getKey());
Path conflictFilePath =
ConflictHandler.createConflictFile(
this.globalEventBus,
this.node.getClientDeviceId().toString(),
this.objectStore,
this.storageAdapter,
new TreePathElement(entry.getKey()));
// we have to emit an ignore event here for the file syncer
// otherwise the final difference calculation will get an updated
// path (i.e. actually an create event) and will try to sync it to the other clients
if (null != conflictFilePath) {
this.globalEventBus.publish(
new IgnoreBusEvent(
new ModifyEvent(
conflictFilePath,
conflictFilePath.getFileName().toString(),
"weIgnoreTheHash",
System.currentTimeMillis())));
}
// now add this to the updated paths, so that we can fetch the original again
updatedPaths.put(entry.getKey(), entry.getValue());
}
// fetch all missing files
logger.info("Fetching all (" + updatedPaths.size() + ") missing files");
for (Map.Entry<String, ClientDevice> entry : updatedPaths.entrySet()) {
UUID subExchangeId = UUID.randomUUID();
logger.debug(
"Starting to fetch file "
+ entry.getKey()
+ " with subExchangeId "
+ subExchangeId
+ " (non blocking background sync "
+ exchangeId
+ ")");
// before updating, check the actual content hash on disk
// to prevent data loss during sync
PathObject mergedPathObject =
this.objectStore.getObjectManager().getObjectForPath(entry.getKey());
Version lastVersion =
mergedPathObject
.getVersions()
.get(Math.max(0, mergedPathObject.getVersions().size() - 1));
PathType pathType = mergedPathObject.getPathType();
StorageType storageType =
pathType.equals(PathType.DIRECTORY) ? StorageType.DIRECTORY : StorageType.FILE;
// only check version, if the file does exist on our disk,
// if not, we have to fetch it anyway
TreePathElement mergedTreeElement = new TreePathElement(mergedPathObject.getAbsolutePath());
if (this.storageAdapter.exists(storageType, mergedTreeElement)) {
this.objectStore.syncFile(mergedTreeElement);
PathObject modifiedPathObject =
this.objectStore.getObjectManager().getObjectForPath(entry.getKey());
Version modifiedLastVersion =
modifiedPathObject
.getVersions()
.get(Math.max(0, modifiedPathObject.getVersions().size() - 1));
if (!modifiedLastVersion.equals(lastVersion)) {
// we just changed the file on this client while syncing...
// therefore we use this state and do not request an outdated state from another client
logger.info(
"Detected file change while merging object store (from other client or end user)... Using our state");
continue;
}
}
// add owner, access type and sharers for object store to prevent overwriting when a file
// is fetched which does not exist yet
this.globalEventBus.publish(
new AddOwnerAndAccessTypeToObjectStoreBusEvent(
mergedPathObject.getOwner(), mergedPathObject.getAccessType(), entry.getKey()));
this.globalEventBus.publish(
new AddSharerToObjectStoreBusEvent(entry.getKey(), mergedPathObject.getSharers()));
FileDemandExchangeHandler fileDemandExchangeHandler =
new FileDemandExchangeHandler(
this.storageAdapter,
this.node,
this.nodeManager,
this.globalEventBus,
new NodeLocation(
entry.getValue().getUserName(),
entry.getValue().getClientDeviceId(),
entry.getValue().getPeerAddress()),
entry.getKey(),
subExchangeId);
this.node
.getObjectDataReplyHandler()
.addResponseCallbackHandler(subExchangeId, fileDemandExchangeHandler);
Thread fileDemandExchangeHandlerThread = new Thread(fileDemandExchangeHandler);
fileDemandExchangeHandlerThread.setName("FileDemandExchangeHandlerThread-" + subExchangeId);
fileDemandExchangeHandlerThread.start();
try {
fileDemandExchangeHandler.await();
} catch (InterruptedException e) {
logger.error(
"Got interrupted while waiting for fileDemandExchangeHandler "
+ subExchangeId
+ " to complete. Message: "
+ e.getMessage());
}
this.node.getObjectDataReplyHandler().removeResponseCallbackHandler(subExchangeId);
if (!fileDemandExchangeHandler.isCompleted()) {
logger.error(
"FileDemandExchangeHandler " + subExchangeId + " should be completed after wait.");
}
}
// start event aggregator
logger.info(
"Starting event aggregator on client ("
+ this.node.getPeerAddress().inetAddress().getHostName()
+ ":"
+ this.node.getPeerAddress().tcpPort()
+ "): Non-blocking background sync "
+ exchangeId);
this.eventAggregator.start();
logger.info(
"Reconciling local disk changes with merged object store (non-blocking background sync "
+ exchangeId
+ ")");
// create a temporary second object store to get changes made in the mean time of syncing
ITreeStorageAdapter objectStoreStorageManager =
this.objectStore.getObjectManager().getStorageAdapater();
TreePathElement pathElement = new TreePathElement("nonBlockingBackgroundSyncObjectStore");
if (objectStoreStorageManager.exists(StorageType.DIRECTORY, pathElement)) {
objectStoreStorageManager.delete(pathElement);
}
objectStoreStorageManager.persist(StorageType.DIRECTORY, pathElement, null);
// create the temporary object store in the .sync folder
ITreeStorageAdapter changeObjectStoreStorageManager =
new LocalStorageAdapter(
Paths.get(objectStoreStorageManager.getRootDir().getPath())
.resolve(pathElement.getPath()));
IObjectStore changeObjectStore =
new ObjectStore(
this.storageAdapter, "index.json", "object", changeObjectStoreStorageManager);
// build object store for differences in the mean time
List<String> ignoredPaths = new ArrayList<>();
Path origSyncFolder =
Paths.get(this.objectStore.getObjectManager().getStorageAdapater().getRootDir().getPath())
.getFileName();
ignoredPaths.add(origSyncFolder.toString());
changeObjectStore.sync(ignoredPaths);
// get differences between disk and merged object store
HashMap<ObjectStore.MergedObjectType, Set<String>> updatedOrDeletedPaths =
this.objectStore.mergeObjectStore(changeObjectStore);
// remove change object store again
changeObjectStoreStorageManager.delete(new TreePathElement("./"));
Set<String> deletedPathsInTheMeanTime =
updatedOrDeletedPaths.get(ObjectStore.MergedObjectType.DELETED);
Set<String> updatedPathsInTheMeanTime =
updatedOrDeletedPaths.get(ObjectStore.MergedObjectType.CHANGED);
logger.info(
"Found "
+ deletedPathsInTheMeanTime.size()
+ " paths which have been deleted in the mean time of syncing");
for (String deletedPath : deletedPathsInTheMeanTime) {
if (this.isIgnored(deletedPath)) {
logger.info("Ignore deletion of " + deletedPath + " since it matches an ignore pattern");
}
// publish a delete event to the SyncFileChangeListener
logger.trace("Creating delete event for " + deletedPath);
this.globalEventBus.publish(
new CreateBusEvent(
new DeleteEvent(
Paths.get(deletedPath),
Paths.get(deletedPath).getFileName().toString(),
null,
System.currentTimeMillis())));
}
logger.info(
"Found "
+ updatedPathsInTheMeanTime.size()
+ " paths which have changed in the mean time of syncing");
for (String updatedPath : updatedPathsInTheMeanTime) {
if (this.isIgnored(updatedPath)) {
logger.info("Ignore updating of " + updatedPath + " since it matches an ignore pattern");
}
// publish modify events to SyncFileChangeListener
logger.trace("Creating modify event for " + updatedPath);
PathObject updatedPathObject =
this.objectStore.getObjectManager().getObjectForPath(updatedPath);
this.globalEventBus.publish(
new CreateBusEvent(
new ModifyEvent(
Paths.get(updatedPath),
Paths.get(updatedPath).getFileName().toString(),
updatedPathObject
.getVersions()
.get(Math.max(0, updatedPathObject.getVersions().size() - 1))
.getHash(),
System.currentTimeMillis())));
}
logger.info("Completed non-blocking background sync " + exchangeId);
} catch (Exception e) {
logger.error("Got exception in NonBlockingBackgroundSyncer. Message: " + e.getMessage(), e);
}
}
/** {@inheritDoc} */
@Override()
public LDIFImportResult importLDIF(LDIFImportConfig importConfig) throws DirectoryException {
RuntimeInformation.logInfo();
// If the backend already has the root container open, we must use the same
// underlying root container
boolean openRootContainer = rootContainer == null;
// If the rootContainer is open, the backend is initialized by something
// else.
// We can't do import while the backend is online.
if (!openRootContainer) {
Message message = ERR_JEB_IMPORT_BACKEND_ONLINE.get();
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
}
try {
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true);
envConfig.setTransactional(false);
envConfig.setDurability(Durability.COMMIT_NO_SYNC);
envConfig.setLockTimeout(0, TimeUnit.SECONDS);
envConfig.setTxnTimeout(0, TimeUnit.SECONDS);
envConfig.setConfigParam(
EnvironmentConfig.CLEANER_MIN_FILE_UTILIZATION,
String.valueOf(cfg.getDBCleanerMinUtilization()));
envConfig.setConfigParam(
EnvironmentConfig.LOG_FILE_MAX, String.valueOf(cfg.getDBLogFileMax()));
if (!importConfig.appendToExistingData()) {
if (importConfig.clearBackend() || cfg.getBaseDN().size() <= 1) {
// We have the writer lock on the environment, now delete the
// environment and re-open it. Only do this when we are
// importing to all the base DNs in the backend or if the backend only
// have one base DN.
File parentDirectory = getFileForPath(cfg.getDBDirectory());
File backendDirectory = new File(parentDirectory, cfg.getBackendId());
// If the backend does not exist the import will create it.
if (backendDirectory.exists()) {
EnvManager.removeFiles(backendDirectory.getPath());
}
}
}
Importer importer = new Importer(importConfig, cfg, envConfig);
rootContainer = initializeRootContainer(envConfig);
return importer.processImport(rootContainer);
} catch (ExecutionException execEx) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, execEx);
}
if (execEx.getCause() instanceof DirectoryException) {
throw ((DirectoryException) execEx.getCause());
} else {
Message message = ERR_EXECUTION_ERROR.get(execEx.getMessage());
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
}
} catch (InterruptedException intEx) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, intEx);
}
Message message = ERR_INTERRUPTED_ERROR.get(intEx.getMessage());
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
} catch (JebException je) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, je);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), je.getMessageObject());
} catch (InitializationException ie) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, ie);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), ie.getMessageObject());
} catch (ConfigException ce) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, ce);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), ce.getMessageObject());
} finally {
// leave the backend in the same state.
try {
if (rootContainer != null) {
long startTime = System.currentTimeMillis();
rootContainer.close();
long finishTime = System.currentTimeMillis();
long closeTime = (finishTime - startTime) / 1000;
Message msg = NOTE_JEB_IMPORT_LDIF_ROOTCONTAINER_CLOSE.get(closeTime);
logError(msg);
rootContainer = null;
}
// Sync the environment to disk.
if (debugEnabled()) {
Message message = NOTE_JEB_IMPORT_CLOSING_DATABASE.get();
TRACER.debugInfo(message.toString());
}
} catch (DatabaseException de) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, de);
}
}
}
}
/**
* Rebuild index(es) in the backend instance. Note that the server will not explicitly initialize
* this backend before calling this method.
*
* @param rebuildConfig The rebuild configuration.
* @throws ConfigException If an unrecoverable problem arises during initialization.
* @throws InitializationException If a problem occurs during initialization that is not related
* to the server configuration.
* @throws DirectoryException If a Directory Server error occurs.
*/
public void rebuildBackend(RebuildConfig rebuildConfig)
throws InitializationException, ConfigException, DirectoryException {
// If the backend already has the root container open, we must use the same
// underlying root container
boolean openRootContainer = rootContainer == null;
/*
* If the rootContainer is open, the backend is initialized by something
* else. We can't do any rebuild of system indexes while others are using
* this backend.
*/
if (!openRootContainer && rebuildConfig.includesSystemIndex()) {
Message message = ERR_JEB_REBUILD_BACKEND_ONLINE.get();
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
}
try {
EnvironmentConfig envConfig;
if (openRootContainer) {
envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true);
envConfig.setTransactional(false);
envConfig.setDurability(Durability.COMMIT_NO_SYNC);
envConfig.setLockTimeout(0, TimeUnit.SECONDS);
envConfig.setTxnTimeout(0, TimeUnit.SECONDS);
envConfig.setConfigParam(
EnvironmentConfig.CLEANER_MIN_FILE_UTILIZATION,
String.valueOf(cfg.getDBCleanerMinUtilization()));
envConfig.setConfigParam(
EnvironmentConfig.LOG_FILE_MAX, String.valueOf(cfg.getDBLogFileMax()));
Importer importer = new Importer(rebuildConfig, cfg, envConfig);
rootContainer = initializeRootContainer(envConfig);
importer.rebuildIndexes(rootContainer);
} else {
envConfig = ConfigurableEnvironment.parseConfigEntry(cfg);
Importer importer = new Importer(rebuildConfig, cfg, envConfig);
importer.rebuildIndexes(rootContainer);
}
} catch (ExecutionException execEx) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, execEx);
}
Message message = ERR_EXECUTION_ERROR.get(execEx.getMessage());
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
} catch (InterruptedException intEx) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, intEx);
}
Message message = ERR_INTERRUPTED_ERROR.get(intEx.getMessage());
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
} catch (ConfigException ce) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, ce);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), ce.getMessageObject());
} catch (JebException e) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
throw new DirectoryException(
DirectoryServer.getServerErrorResultCode(), e.getMessageObject());
} catch (InitializationException e) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
throw new InitializationException(e.getMessageObject());
} finally {
// If a root container was opened in this method as read only, close it
// to leave the backend in the same state.
if (openRootContainer && rootContainer != null) {
try {
rootContainer.close();
rootContainer = null;
} catch (DatabaseException e) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
}
}
}
}
/**
* Handles event from ZooKeeper for this coordinate.
*
* @param event
*/
@Override
public void process(WatchedEvent event) {
LOG.info("Got an event from ZooKeeper " + event.toString());
synchronized (lastStatusVersionMonitor) {
switch (event.getType()) {
case None:
switch (event.getState()) {
case SyncConnected:
break;
case Disconnected:
case AuthFailed:
case Expired:
default:
// If we lost connection, we don't attempt to register another watcher as
// this might be blocking forever. Parent will try to reconnect (reclaim)
// later.
isSynchronizedWithZooKeeper.set(false);
sendEventToCoordinateListener(
CoordinateListener.Event.NO_CONNECTION_TO_STORAGE, event.toString());
return;
}
return;
case NodeDeleted:
// If node is deleted, we have no node to place a new watcher so we stop watching.
isSynchronizedWithZooKeeper.set(false);
sendEventToCoordinateListener(CoordinateListener.Event.NOT_OWNER, event.toString());
return;
case NodeDataChanged:
LOG.fine("Node data changed, check versions.");
boolean verifiedSynchronized = false;
try {
final Stat stat = zkClient.getZookeeper().exists(path, this);
if (stat == null) {
LOG.info("Could not stat path, setting out of synch, will retry claim");
} else {
LOG.fine("Previous version is " + lastStatusVersion + " now is " + stat.getVersion());
if (stat.getVersion() != lastStatusVersion) {
LOG.fine("Version mismatch, sending out of sync.");
} else {
verifiedSynchronized = true;
}
}
} catch (KeeperException e) {
LOG.fine(
"Problems with zookeeper, sending consistencyState out of sync: " + e.getMessage());
} catch (InterruptedException e) {
LOG.fine("Got interrupted: " + e.getMessage());
return;
} finally {
isSynchronizedWithZooKeeper.set(verifiedSynchronized);
}
if (verifiedSynchronized) {
sendEventToCoordinateListener(
CoordinateListener.Event.COORDINATE_OUT_OF_SYNC, event.toString());
}
return;
case NodeChildrenChanged:
case NodeCreated:
// This should not happen..
isSynchronizedWithZooKeeper.set(false);
sendEventToCoordinateListener(
CoordinateListener.Event.COORDINATE_OUT_OF_SYNC, event.toString());
return;
}
}
}
// current eviction check period is 1 second.
// about 30.000 records can be put in one second
// so the size should be adapted
@Test
public void testEvictionSpeedTest() {
final int k = 3;
final int size = 10000;
final CountDownLatch latch = new CountDownLatch(k);
final String mapName = "testEvictionSpeedTest";
Config cfg = new Config();
final MapConfig mc = cfg.getMapConfig(mapName);
mc.setEvictionPolicy(MapConfig.EvictionPolicy.LRU);
mc.setEvictionPercentage(25);
final MaxSizeConfig msc = new MaxSizeConfig();
msc.setMaxSizePolicy(MaxSizeConfig.MaxSizePolicy.PER_NODE);
msc.setSize(size);
mc.setMaxSizeConfig(msc);
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
final HazelcastInstance[] instances = factory.newInstances(cfg);
final AtomicBoolean success = new AtomicBoolean(true);
new Thread() {
final IMap map = instances[0].getMap(mapName);
public void run() {
try {
Thread.sleep(1000);
while (latch.getCount() != 0) {
try {
int msize = map.size();
if (msize > (size * k + size * k * 10 / 100)) {
success.set(false);
break;
}
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}.start();
for (int i = 0; i < k; i++) {
final IMap map = instances[i].getMap(mapName);
new Thread() {
public void run() {
for (int j = 0; j < size; j++) {
map.put(k + "-" + j, j);
}
latch.countDown();
}
}.start();
}
try {
Assert.assertTrue(latch.await(10, TimeUnit.MINUTES));
Assert.assertTrue(success.get());
} catch (InterruptedException e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testEvictionSpeedTestPerPartition() {
final int k = 2;
final int size = 100;
final CountDownLatch latch = new CountDownLatch(k);
final String mapName = "testEvictionSpeedTestPerPartition";
Config cfg = new Config();
final MapConfig mc = cfg.getMapConfig(mapName);
mc.setEvictionPolicy(MapConfig.EvictionPolicy.LRU);
mc.setEvictionPercentage(25);
final MaxSizeConfig msc = new MaxSizeConfig();
msc.setMaxSizePolicy(MaxSizeConfig.MaxSizePolicy.PER_PARTITION);
msc.setSize(size);
mc.setMaxSizeConfig(msc);
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
final HazelcastInstance[] instances = factory.newInstances(cfg);
final int pnum = instances[0].getPartitionService().getPartitions().size();
final AtomicInteger failCount = new AtomicInteger(0);
new Thread() {
final IMap map = instances[0].getMap(mapName);
public void run() {
try {
Thread.sleep(1000);
while (latch.getCount() != 0) {
try {
int msize = map.size();
if (msize > (size * pnum * 1.2)) {
failCount.incrementAndGet();
}
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}.start();
for (int i = 0; i < k; i++) {
final IMap map = instances[i].getMap(mapName);
new Thread() {
public void run() {
for (int j = 0; j < 100000; j++) {
map.put(k + "-" + j, j);
}
latch.countDown();
}
}.start();
}
try {
Assert.assertEquals(latch.await(10, TimeUnit.MINUTES), true);
Assert.assertEquals(0, failCount.get());
} catch (InterruptedException e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
