private byte[] getFromCache(int site) {
int blockNumber = site / myNumLinesPerInterval;
byte[][] result = myGenoCache.getIfPresent(blockNumber);
if (result == null) {
CompletableFuture<byte[]> future = new CompletableFuture<>();
CompletableFuture<byte[]> temp = myFutureQueue.putIfAbsent(site, future);
if (temp != null) {
future = temp;
}
if (myCurrentlyProcessingBlocks.add(blockNumber)) {
myThreadPool.submit(new ProcessLines(site));
}
try {
result = myGenoCache.getIfPresent(blockNumber);
if (result != null) {
myFutureQueue.remove(site);
future.complete(result[site % myNumLinesPerInterval]);
return result[site % myNumLinesPerInterval];
} else {
return future.get();
}
} catch (Exception e) {
myLogger.error(e.getMessage(), e);
}
}
return result[site % myNumLinesPerInterval];
}
public static Map<Long, Integer> getTermFrequencyMapForDocument(
GraphDatabaseService db, Long classId) {
Map<Long, Integer> termDocumentMatrix;
String cacheKey = "TERM_DOCUMENT_FREQUENCY_" + classId;
if (vectorSpaceModelCache.getIfPresent(cacheKey) == null) {
Node classNode = db.getNodeById(classId);
termDocumentMatrix = new HashMap<>();
IteratorUtil.asCollection(
db.traversalDescription()
.depthFirst()
.relationships(withName("HAS_CLASS"), Direction.INCOMING)
.evaluator(Evaluators.fromDepth(1))
.evaluator(Evaluators.toDepth(1))
.traverse(classNode))
.stream()
.forEach(
p ->
termDocumentMatrix.put(
p.endNode().getId(), (Integer) p.lastRelationship().getProperty("matches")));
vectorSpaceModelCache.put(cacheKey, termDocumentMatrix);
} else {
termDocumentMatrix = (Map<Long, Integer>) vectorSpaceModelCache.getIfPresent(cacheKey);
}
return termDocumentMatrix;
}
/**
* Get a dependency entry for a given uid.
*
* <p>This is a convenience routine to check both the local and global cache for a value.
*
* <p>Please look at {@link #localDependencies} if you are mucking in here.
*
* <p>Side Effects:
*
* <ul>
* <li>If a dependency is found in the global cache, it is populated into the local cache.
* </ul>
*
* @param uid the uid may be null, if so, it only checks the local cache.
* @param descriptor the descriptor, used for both global and local cache lookups.
* @return the DependencyEntry or null if none present.
*/
private DependencyEntry getDE(String uid, DefDescriptor<?> descriptor) {
// See localDependencies comment
String key = makeLocalKey(descriptor);
DependencyEntry de;
if (uid != null) {
de = localDependencies.get(uid);
if (de != null) {
return de;
}
de = depsCache.getIfPresent(makeGlobalKey(uid, descriptor));
} else {
// See localDependencies comment
de = localDependencies.get(key);
if (de != null) {
return de;
}
de = depsCache.getIfPresent(key);
}
if (de != null) {
// See localDependencies comment
localDependencies.put(de.uid, de);
localDependencies.put(key, de);
}
return de;
}
public static int getDocumentSizeForFeature(GraphDatabaseService db, Long id) {
int documentSize;
String cacheKey = "DOCUMENT_SIZE_FEATURE_" + id;
if (vectorSpaceModelCache.getIfPresent(cacheKey) == null) {
Node startNode = db.getNodeById(id);
Iterator<Node> classes =
db.traversalDescription()
.depthFirst()
.relationships(withName("HAS_CLASS"), Direction.OUTGOING)
.evaluator(Evaluators.fromDepth(1))
.evaluator(Evaluators.toDepth(1))
.traverse(startNode)
.nodes()
.iterator();
documentSize = IteratorUtil.count(classes);
vectorSpaceModelCache.put(cacheKey, documentSize);
} else {
documentSize = (Integer) vectorSpaceModelCache.getIfPresent(cacheKey);
}
return documentSize;
}
/**
* Decide whether a task should be migrated based on the time of last migration and the size of
* the task
*
* @param task The task to be migrated
* @param targetHostId The host ID being considered for migration
* @return True if the task should be migrated there
*/
public boolean shouldMigrateTaskToHost(JobTask task, String targetHostId) {
String taskHost;
if (task == null
|| targetHostId == null
|| task.getByteCount() == 0
|| (taskHost = task.getHostUUID()) == null) {
return false; // Suspicious tasks should not be migrated
}
return shouldKickTaskOnHost(targetHostId)
&& migrateHosts.getIfPresent(taskHost) == null
&& migrateHosts.getIfPresent(targetHostId) == null;
}
public static int getDocumentSize(GraphDatabaseService db) {
int documentSize;
String cacheKey = "GLOBAL_DOCUMENT_SIZE";
if (vectorSpaceModelCache.getIfPresent(cacheKey) == null) {
documentSize =
IteratorUtil.count(
GlobalGraphOperations.at(db).getAllNodesWithLabel(DynamicLabel.label("Class")));
vectorSpaceModelCache.put(cacheKey, documentSize);
} else {
documentSize = (Integer) vectorSpaceModelCache.getIfPresent(cacheKey);
}
return documentSize;
}
/**
* returns true if the specified entry must be replicated. We should always replicate meta
* operations (e.g. flush) and use the user HTD flag to decide whether or not replicate the
* memstore.
*/
private boolean requiresReplication(final TableName tableName, final List<Entry> entries)
throws IOException {
// unit-tests may not the TableDescriptors, bypass the check and always replicate
if (tableDescriptors == null) return true;
Boolean requiresReplication = memstoreReplicationEnabled.getIfPresent(tableName);
if (requiresReplication == null) {
// check if the table requires memstore replication
// some unit-test drop the table, so we should do a bypass check and always replicate.
HTableDescriptor htd = tableDescriptors.get(tableName);
requiresReplication = htd == null || htd.hasRegionMemstoreReplication();
memstoreReplicationEnabled.put(tableName, requiresReplication);
}
// if memstore replication is not required, check the entries.
// meta edits (e.g. flush) must be always replicated.
if (!requiresReplication) {
int skipEdits = 0;
java.util.Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
if (entry.getEdit().isMetaEdit()) {
requiresReplication = true;
} else {
it.remove();
skipEdits++;
}
}
skippedEdits.addAndGet(skipEdits);
}
return requiresReplication;
}
@Override
@Traced
public org.vertexium.Authorizations getAuthorizations(
User user, String... additionalAuthorizations) {
Set<String> userAuthorizations;
if (user instanceof SystemUser) {
userAuthorizations = new HashSet<>();
userAuthorizations.add(VisalloVisibility.SUPER_USER_VISIBILITY_STRING);
} else {
Vertex userVertex = userVertexCache.getIfPresent(user.getUserId());
if (userVertex != null) {
userAuthorizations = getAuthorizations(userVertex);
} else {
userAuthorizations = null;
}
}
if (userAuthorizations == null) {
LOGGER.debug("BEGIN getAuthorizations query");
Vertex userVertex = findByIdUserVertex(user.getUserId());
userAuthorizations = getAuthorizations(userVertex);
LOGGER.debug("END getAuthorizations query");
}
Set<String> authorizationsSet = new HashSet<>(userAuthorizations);
Collections.addAll(authorizationsSet, additionalAuthorizations);
return graph.createAuthorizations(authorizationsSet);
}
/**
* Getting the data related to the given data key.
*
* @param pDataKey searched for
* @return the related data
* @throws TTIOException if the read to the persistent storage fails
*/
public IData getData(final long pDataKey) throws TTIOException {
checkArgument(pDataKey >= 0);
checkState(!mClose, "Transaction already closed");
// Calculate bucket and data part for given datakey.
final long seqBucketKey = pDataKey >> IConstants.INDIRECT_BUCKET_COUNT[3];
final int dataBucketOffset = dataBucketOffset(pDataKey);
DataBucket bucket = mCache.getIfPresent(seqBucketKey);
if (bucket == null) {
final List<DataBucket> listRevs = getSnapshotBuckets(seqBucketKey);
final DataBucket[] revs = listRevs.toArray(new DataBucket[listRevs.size()]);
checkState(revs.length > 0, "Number of Buckets to reconstruct must be larger than 0");
// Build up the complete bucket.
final IRevisioning revision = mSession.getConfig().mRevision;
bucket = revision.combineBuckets(revs);
mCache.put(seqBucketKey, bucket);
}
final IData returnVal = bucket.getData(dataBucketOffset);
// root-fsys is excluded from the checkagainst deletion based on the necesssity of the
// data-layer to
// reference against this data while creation of the transaction
if (pDataKey == 0) {
return returnVal;
} else {
return checkItemIfDeleted(returnVal);
}
}
/**
* Returns a cached copy of the given file. The cached copy is guaranteed to not be modified or
* removed.
*
* @param original The source file.
* @param baseDirFactory A factory that can provide a base directory for the file cache.
* @return The cached file.
*/
public File getCachedJar(File original, Factory<File> baseDirFactory) {
File source = FileUtils.canonicalize(original);
FileInfo fileInfo;
// TODO - do some of this work concurrently
lock.lock();
try {
fileInfo = cachedFiles.getIfPresent(source);
if (fileInfo == null || !fileInfo.cachedFile.exists()) {
// TODO - use the hashing service for this
long lastModified = source.lastModified();
long length = source.length();
HashValue hashValue = HashUtil.createHash(source, "sha1");
fileInfo = copyIntoCache(baseDirFactory, source, lastModified, length, hashValue);
} else {
// TODO - use the change detection service for this
long lastModified = source.lastModified();
long length = source.length();
if (lastModified != fileInfo.lastModified || length != fileInfo.length) {
HashValue hashValue = HashUtil.createHash(source, "sha1");
if (!hashValue.equals(fileInfo.hashValue)) {
fileInfo = copyIntoCache(baseDirFactory, source, lastModified, length, hashValue);
}
}
}
} finally {
lock.unlock();
}
return fileInfo.cachedFile;
}
public boolean isValidToken(String token) {
boolean validToken = false;
if (tokenCache.getIfPresent(token) != null) {
validToken = true;
}
return validToken;
}
public static Integer getShard(String s) {
Integer shard = shardCache.getIfPresent(s);
if (shard == null) {
shard = computeShard(s);
shardCache.put(s, shard);
}
return shard;
}
private synchronized void unLoadUser(final String app) throws TddlException {
String dataId = DATAID_PREFIX + this.cluster + "." + app + DATAID_SUFFIX;
ConfigDataHandler cdh = cdhs.getIfPresent(dataId);
if (cdh != null) {
cdh.destroy();
cdhs.invalidate(dataId);
}
String newDataId = DATAID_PREFIX + app + DATAID_SUFFIX;
cdh = cdhs.getIfPresent(newDataId);
if (cdh != null) {
cdh.destroy();
cdhs.invalidate(dataId);
}
users.remove(app);
}
@Nullable
public ClassLoaderDetails maybeGetDetails(ClassLoader classLoader) {
lock.lock();
try {
return classLoaderDetails.getIfPresent(classLoader);
} finally {
lock.unlock();
}
}
@Override
public EconomyData get(UUID uniqueId) {
EconomyData data = cache.getIfPresent(uniqueId);
if (data == null) {
data = new EconomyData(uniqueId, 0.0D);
cache.put(uniqueId, data);
}
return data;
}
public static ZimletUserProperties getProperties(Account account) {
String key = account.getId();
ZimletUserProperties prop = CACHE.getIfPresent(key);
if (prop == null) {
prop = new ZimletUserProperties(account);
CACHE.put(key, prop);
}
return prop;
}
private void prepareTransUnitUpdatedEvent(
int previousVersionNum, ContentState previousState, HTextFlowTarget target) {
LocaleId localeId = target.getLocaleId();
HTextFlow textFlow = target.getTextFlow();
HDocument document = textFlow.getDocument();
HProjectIteration projectIteration = document.getProjectIteration();
String iterationSlug = projectIteration.getSlug();
String projectSlug = projectIteration.getProject().getSlug();
ProjectType projectType = projectIteration.getProjectType();
WorkspaceId workspaceId =
new WorkspaceId(new ProjectIterationId(projectSlug, iterationSlug, projectType), localeId);
Optional<TranslationWorkspace> workspaceOptional =
translationWorkspaceManager.tryGetWorkspace(workspaceId);
if (!workspaceOptional.isPresent()) {
return;
}
TransUnitTransformer transUnitTransformer =
serviceLocator.getInstance(TransUnitTransformer.class);
TransUnit transUnit = transUnitTransformer.transform(textFlow, target, target.getLocale());
DocumentId documentId = new DocumentId(document.getId(), document.getDocId());
int wordCount = textFlow.getWordCount().intValue();
TransUnitUpdateInfo updateInfo =
createTransUnitUpdateInfo(
transUnit, documentId, wordCount, previousVersionNum, previousState);
CacheValue context =
updateContext.getIfPresent(new CacheKey(transUnit.getId(), transUnit.getLocaleId()));
TransUnitUpdated updated;
if (context != null) {
updated = new TransUnitUpdated(updateInfo, context.editorClientId, context.updateType);
log.debug("about to publish trans unit updated event {}", updated);
} else if (ServletContexts.instance().getRequest() != null) {
String sessionId = ServletContexts.instance().getRequest().getSession().getId();
EditorClientId editorClientId = new EditorClientId(sessionId, -1);
updated =
new TransUnitUpdated(
updateInfo, editorClientId, TransUnitUpdated.UpdateType.NonEditorSave);
} else {
updated =
new TransUnitUpdated(
updateInfo,
new EditorClientId("unknown", -1),
TransUnitUpdated.UpdateType.NonEditorSave);
}
if (Events.exists()) {
Events.instance()
.raiseTransactionSuccessEvent(
TextFlowTargetUpdatedEvent.EVENT_NAME,
new TextFlowTargetUpdatedEvent(workspaceOptional.get(), target.getId(), updated));
}
}
public static MemoStorable load(Key key) {
MemoStorable res = deletedCache.getIfPresent(key);
if (res != null) return res;
try {
Entity ent = datastore.get(key);
return load(ent);
} catch (EntityNotFoundException e) {
return null;
}
}
@Override
public Observable<Entity> load(final Id entityId) {
final Entity entity = entityCache.getIfPresent(entityId);
if (entity != null) {
return Observable.just(entity);
}
return targetEntityCollectionManager.load(entityId).doOnNext(cacheAdd);
}
@Override
public <T extends Definition> String getCachedString(
String uid, DefDescriptor<?> descriptor, String key) {
DependencyEntry de = localDependencies.get(uid);
if (de != null) {
return stringsCache.getIfPresent(getKey(de, descriptor, key));
}
return null;
}
@Override
public CheckedFuture<? extends T, SchemaSourceException> getSource(
final SourceIdentifier sourceIdentifier) {
final T present = cache.getIfPresent(sourceIdentifier);
if (present != null) {
return Futures.immediateCheckedFuture(present);
}
return Futures.<T, SchemaSourceException>immediateFailedCheckedFuture(
new MissingSchemaSourceException("Source not found", sourceIdentifier));
}
@Override
public Optional<T> request(final ServerStorageKey<T> key) {
checkOpen();
if (key.get().size() != mapper.getPlan().getNumberOfKeys())
throw new BadKeyException(key.get().size(), mapper.getPlan().getNumberOfKeys());
final T result = cache.getIfPresent(key.hashCode());
if (result != null) return Optional.of(result);
return Optional.ofNullable(newStructureFromResult(createResultSetFromKey(key)));
}
@Traced
public Vertex findByIdUserVertex(String userId) {
Vertex userVertex = userVertexCache.getIfPresent(userId);
if (userVertex != null) {
return userVertex;
}
userVertex = graph.getVertex(userId, authorizations);
if (userVertex != null) {
userVertexCache.put(userId, userVertex);
}
return userVertex;
}
public static synchronized Session getSession(String host, int port) throws UnknownHostException {
Session session = hostConnectionMap.getIfPresent(host);
if (session == null || session.isClosed()) {
Cluster.Builder builder = Cluster.builder().addContactPoint(host).withPort(port);
Cluster cluster = builder.build();
session = cluster.connect();
hostConnectionMap.put(host, session);
logger.debug("Created connection to {}.", host);
logger.debug("Number of sessions opened are {}.", hostConnectionMap.size());
}
return session;
}
@Override
public void storeIncomingStatistics(List<LiveStatistics> liveStatisticsList) {
for (LiveStatistics ls : liveStatisticsList) {
long hoursSince1970 = ls.getTimeperiod() / 240;
int fifteenSecondPeriodsSinceStartOfHour =
LiveStatisticsUtil.getFifteensecondTimeperiodsSinceStartOfHour(ls.getTimeperiod() * 15);
Double calculatedValue =
LiveStatisticsUtil.calculateValueBasedOnUnitType(
ls.getValue(), UnitType.fromValue(ls.getUnitType()));
BasicMetricHour mhToStore =
metricHoursToStoreHash.get(
ls.getAccountName() + ";" + ls.getGuiPath() + ";" + hoursSince1970);
boolean wasInStoreHash = mhToStore != null;
if (mhToStore == null) {
// Not in store-hash chech in metricHourCache
mhToStore =
metricHourCache.getIfPresent(
ls.getAccountName() + ";" + ls.getGuiPath() + ";" + hoursSince1970);
}
if (mhToStore == null) {
// Not in metricHourCache, fetch from Riak
mhToStore = getMetricHour(ls.getAccountName(), ls.getGuiPath(), hoursSince1970);
}
if (mhToStore == null) {
// Not in Riak, create
mhToStore =
new BasicMetricHour(
ls.getGuiPath(),
ls.getAccountName(),
hoursSince1970,
ls.getValueType(),
ls.getUnitType());
}
mhToStore.getMetrics()[fifteenSecondPeriodsSinceStartOfHour] =
LiveStatisticsUtil.calculateValueBasedOnValueType(
ls, calculatedValue, ValueType.fromValue(ls.getValueType()));
if (!wasInStoreHash) {
metricHoursToStoreHash.put(
ls.getAccountName() + ";" + ls.getGuiPath() + ";" + hoursSince1970, mhToStore);
}
}
persistRecentMetricHours();
}
/**
* The simplified converter service - we only return the rate value where an object representing
* various related settings can be returned.
*
* @param fromCurrency
* @param toCurrency
* @return
*/
public String convert(final String fromCurrency, final String toCurrency) {
if (Strings.isNullOrEmpty(fromCurrency) || Strings.isNullOrEmpty(toCurrency)) return null;
if (fromCurrency.length() != 3 || toCurrency.length() != 3) return null;
// Make a cache key and try from cache first
final String key = String.format("%s.%s", fromCurrency, toCurrency);
if (rates.getIfPresent(key) != null) return rates.getIfPresent(key);
try {
// Invoke the service using YQL
JsonElement response =
reader.queryJson(
"http://query.yahooapis.com/v1/public/yql?"
+ "q=select%20*%20from%20xml%20where%20url%3D%27"
+ "http%3A%2F%2Fwww.webservicex.net%2FCurrencyConvertor.asmx%2FConversionRate%3F"
+ "FromCurrency%3D"
+ fromCurrency
+ "%26ToCurrency%3D"
+ toCurrency
+ "%27&format=json&diagnostics=true");
if (!(response instanceof JsonObject)) return null;
JsonObject json = response.getAsJsonObject();
// Parse the results
// (very simplistic as just going after the single exchange rate value)
JsonObject query = json.getAsJsonObject("query");
if (query == null || query.get("results") instanceof JsonNull) return null;
JsonObject results = query.getAsJsonObject("results");
JsonElement rate = results != null ? results.getAsJsonObject("double").get("content") : null;
String value = rate != null ? rate.getAsString() : null;
// Store in the cache
if (value != null) rates.put(key, value);
return value;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* @param structure
* @return The record in cache if exists or the record itself and cache it.
*/
private T initStructure(final T structure, final boolean created) {
final T inCache = cache.getIfPresent(structure.hashCode());
if (inCache != null) return inCache;
cache.put(structure.hashCode(), structure);
final ServerStorageStructurePrivateBase privateBase =
((ServerStorageStructurePrivateBase) structure);
privateBase.setOwnerAndTracker(this, changeTracker);
if (created) privateBase.onCreate();
return structure;
}
@Test
public void shouldReturnNoElementAfterInvalidation() throws Exception {
// given
final Cache<Long, String> cache =
CacheBuilder.newBuilder() //
.expireAfterWrite(EXPIRATION_TIME_IN_MILLISECONDS * 3, TimeUnit.MILLISECONDS) //
.build();
cache.put(KEY, FOO);
// when
cache.invalidate(KEY);
final String value = cache.getIfPresent(KEY);
// then
assertNull(value);
}
private DnsData getDnsData(String hostKey) {
Thread.yield();
DnsData dnsData = dnsCache.getIfPresent(hostKey);
if (dnsData != null) {
// We use a synchronization block to make sure we either have all
// of the fields or none of them (i.e. not a half processed entry).
synchronized (dnsData) {
if (dnsData.getSourceIp() == null) {
return null;
}
}
}
return dnsData;
}
@Test
public void temporizedCacheShouldDeleteEntriesAfterTimeout() throws Exception {
// given
final Cache<Long, String> cache =
CacheBuilder.newBuilder() //
.expireAfterWrite(EXPIRATION_TIME_IN_MILLISECONDS, TimeUnit.MILLISECONDS) //
.build();
cache.put(KEY, FOO);
Thread.sleep(EXPIRATION_TIME_IN_MILLISECONDS + 200);
// when
final String value = cache.getIfPresent(KEY);
// then
assertNull(value);
}
