@Test
public void testFinish() throws Exception {
SourceOperator operator = createExchangeOperator();
operator.addSplit(newRemoteSplit(TASK_1_ID));
operator.addSplit(newRemoteSplit(TASK_2_ID));
operator.addSplit(newRemoteSplit(TASK_3_ID));
operator.noMoreSplits();
// add pages and leave buffers open
taskBuffers.getUnchecked(TASK_1_ID).addPages(1, false);
taskBuffers.getUnchecked(TASK_2_ID).addPages(1, false);
taskBuffers.getUnchecked(TASK_3_ID).addPages(1, false);
// read 3 pages
waitForPages(operator, 3);
// verify state
assertEquals(operator.isFinished(), false);
assertEquals(operator.needsInput(), false);
assertEquals(operator.getOutput(), null);
// finish without closing buffers
operator.finish();
// wait for finished
waitForFinished(operator);
}
@Test
public void test401ShouldRetry() {
HttpCommand command = createMock(HttpCommand.class);
HttpRequest request = createMock(HttpRequest.class);
HttpResponse response = createMock(HttpResponse.class);
@SuppressWarnings("unchecked")
LoadingCache<Credentials, Access> cache = createMock(LoadingCache.class);
BackoffLimitedRetryHandler backoffHandler = createMock(BackoffLimitedRetryHandler.class);
expect(command.getCurrentRequest()).andReturn(request);
cache.invalidateAll();
expectLastCall();
expect(response.getPayload()).andReturn(Payloads.newStringPayload("")).anyTimes();
expect(response.getStatusCode()).andReturn(401).atLeastOnce();
replay(command);
replay(response);
replay(cache);
replay(backoffHandler);
RetryOnRenew retry = new RetryOnRenew(cache, backoffHandler);
assertTrue(retry.shouldRetryRequest(command, response));
verify(command);
verify(response);
verify(cache);
}
@Test
public void test401ShouldRetry4Times() {
HttpCommand command = createMock(HttpCommand.class);
HttpRequest request = createMock(HttpRequest.class);
HttpResponse response = createMock(HttpResponse.class);
@SuppressWarnings("unchecked")
LoadingCache<Credentials, Access> cache = createMock(LoadingCache.class);
BackoffLimitedRetryHandler backoffHandler = createMock(BackoffLimitedRetryHandler.class);
expect(command.getCurrentRequest()).andReturn(request).anyTimes();
expect(request.getHeaders()).andStubReturn(null);
cache.invalidateAll();
expectLastCall().anyTimes();
expect(response.getPayload()).andReturn(Payloads.newStringPayload("")).anyTimes();
expect(response.getStatusCode()).andReturn(401).anyTimes();
replay(command, request, response, cache);
RetryOnRenew retry = new RetryOnRenew(cache, backoffHandler);
for (int i = 0; i < RetryOnRenew.NUM_RETRIES - 1; ++i) {
assertTrue(retry.shouldRetryRequest(command, response), "Expected retry to succeed");
}
assertFalse(
retry.shouldRetryRequest(command, response),
"Expected retry to fail on attempt " + RetryOnRenew.NUM_RETRIES);
verify(command, response, cache);
}
@Override
public void write(E entity) {
Preconditions.checkState(
state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);
accessor.keyFor(entity, provided, reusedKey);
DatasetWriter<E> writer = cachedWriters.getIfPresent(reusedKey);
if (writer == null) {
// avoid checking in every whether the entity belongs in the view by only
// checking when a new writer is created
Preconditions.checkArgument(
view.includes(entity), "View %s does not include entity %s", view, entity);
// get a new key because it is stored in the cache
StorageKey key = StorageKey.copy(reusedKey);
try {
writer = cachedWriters.getUnchecked(key);
} catch (UncheckedExecutionException ex) {
throw new IllegalArgumentException(
"Problem creating view for entity: " + entity, ex.getCause());
}
}
writer.write(entity);
}
@Test
public void testWaitForNoMoreSplits() throws Exception {
SourceOperator operator = createExchangeOperator();
// add a buffer location containing one page and close the buffer
operator.addSplit(newRemoteSplit(TASK_1_ID));
// add pages and leave buffers open
taskBuffers.getUnchecked(TASK_1_ID).addPages(1, true);
// read page
waitForPages(operator, 1);
// verify state
assertEquals(operator.isFinished(), false);
assertEquals(operator.needsInput(), false);
assertEquals(operator.getOutput(), null);
// add a buffer location
operator.addSplit(newRemoteSplit(TASK_2_ID));
// set no more splits (buffer locations)
operator.noMoreSplits();
// add two pages and close the last buffer
taskBuffers.getUnchecked(TASK_2_ID).addPages(2, true);
// read all pages
waitForPages(operator, 2);
// wait for finished
waitForFinished(operator);
}
protected void put(
Map<String, InputStream> map,
LoadingCache<String, Node> store,
String key,
String id,
String name) {
assertEquals(store.size(), 0);
assertEquals(map.size(), 0);
map.put(
key, new ByteArrayInputStream(String.format("id: %s\nname: %s\n", id, name).getBytes()));
store.getUnchecked(key);
}
@Override
public ImmutableMap<K, V> getAll(final Iterable<? extends K> elementKeys)
throws ExecutionException {
final ImmutableMap.Builder<K, V> builder = ImmutableMap.builder();
for (final K key : elementKeys) {
final LoadingCache<K, V> cache = getCache(key);
if (cache != null) {
final V value = cache.get(key);
builder.put(key, value);
}
}
return builder.build();
}
protected void remove(
Map<String, InputStream> map, LoadingCache<String, Node> store, String key) {
store.invalidate(key);
assertEquals(store.size(), 0);
map.remove(key);
assertEquals(map.size(), 0);
try {
assertEquals(store.getUnchecked(key), null);
fail("should not work as null is invalid");
} catch (UncheckedExecutionException e) {
}
assertEquals(map.get(key), null);
}
@Override
public FileObject getFileForOutput(
Location location, String packageName, String relativeName, FileObject sibling)
throws IOException {
URI uri = uriForFileObject(location, packageName, relativeName);
return inMemoryFileObjects.getUnchecked(uri);
}
public void closeLogFile(String taskId) {
try {
logStreamCache.invalidate(taskId);
} catch (Throwable throwable) {
log.warn("Unable to close logfile for {}", taskId, throwable);
}
}
public void uncache(String identifier) {
subjectCache.invalidate(identifier);
PermissionsEx manager = plugin.getManager();
if (manager != null) {
manager.uncache(getIdentifier(), identifier);
}
}
private <T> void evaluateBundle(
final AppliedPTransform<?, ?, ?> transform,
final CommittedBundle<T> bundle,
final CompletionCallback onComplete) {
TransformExecutorService transformExecutor;
if (isKeyed(bundle.getPCollection())) {
final StepAndKey stepAndKey = StepAndKey.of(transform, bundle.getKey());
// This executor will remain reachable until it has executed all scheduled transforms.
// The TransformExecutors keep a strong reference to the Executor, the ExecutorService keeps
// a reference to the scheduled TransformExecutor callable. Follow-up TransformExecutors
// (scheduled due to the completion of another TransformExecutor) are provided to the
// ExecutorService before the Earlier TransformExecutor callable completes.
transformExecutor = executorServices.getUnchecked(stepAndKey);
} else {
transformExecutor = parallelExecutorService;
}
Collection<ModelEnforcementFactory> enforcements =
MoreObjects.firstNonNull(
transformEnforcements.get(transform.getTransform().getClass()),
Collections.<ModelEnforcementFactory>emptyList());
TransformExecutor<T> callable =
TransformExecutor.create(
evaluationContext,
registry,
enforcements,
bundle,
transform,
onComplete,
transformExecutor);
outstandingWork.incrementAndGet();
transformExecutor.schedule(callable);
}
@Override
public void update(final User user) {
if (user == null) {
throw new IllegalArgumentException("User can't null");
}
if (Strings.isNullOrEmpty(user.getUsername())) {
throw new IllegalArgumentException("Username of the user can't be null or empty");
}
User old = find(user.getUsername());
String newEncryptPassword = generateEncryptPassword(user);
if (!old.getPassword().equals(newEncryptPassword)) {
user.setPassword(newEncryptPassword);
}
mongoDBUtil.keepConnect(
userCollectionName,
new CollectionCallback<Void>() {
@Override
public Void process(MongoCollection<Document> collection) {
collection.updateOne(
new Document(usernameKey, user.getUsername()),
new Document("$set", new Document(usernameKey, user.getPassword())));
return null;
}
});
cache.put(user.getUsername(), user);
}
public CodeView getCode(String id) {
try {
return cache.get(id + DELIMITER + CashregisterSession.get().getLocale().getLanguage());
} catch (ExecutionException e) {
throw new WicketRuntimeException(e);
}
}
public void unlock(Object name) {
Monitor lock;
synchronized (stringLocks) {
lock = stringLocks.getUnchecked(name);
}
lock.leave();
}
public static void updateRates(String type, long rate) {
LOGGER.debug("updateRates : type={}, rate={}", type, rate);
Iterator<RateLimiterDescriptor> iterator = rateLimiterMap.asMap().keySet().iterator();
while (iterator.hasNext()) {
RateLimiterDescriptor oldRateDesc = iterator.next();
if (type.equals(oldRateDesc.getType())) {
LOGGER.trace("updateRates : invalidated rate={}", oldRateDesc);
rateLimiterMap.invalidate(oldRateDesc);
RateLimiterDescriptor newRateDesc =
new RateLimiterDescriptor(oldRateDesc.getType(), oldRateDesc.getAppId(), rate);
LOGGER.trace("updateRates : updating rate desc={}, rate={}", newRateDesc, rate);
rateLimiterMap.put(newRateDesc, RateLimiter.create(rate));
rateLimiterMap.refresh(newRateDesc);
}
}
}
private static <K, V> Map<K, V> getAll(LoadingCache<K, V> cache, Iterable<K> keys) {
try {
return cache.getAll(keys);
} catch (ExecutionException | UncheckedExecutionException | ExecutionError e) {
throw Throwables.propagate(e.getCause());
}
}
@Override
protected List<DynamicProperty> create() {
List<DynamicProperty> properties = new ArrayList<DynamicProperty>();
for (ObjectType type : getTypes()) {
if (type.getObjectClass() == null) {
continue;
}
String beanProperty = type.getJavaBeanProperty();
if (ObjectUtils.isBlank(beanProperty)) {
continue;
}
try {
properties.add(
new DynamicProperty(type, beanProperty, beanPropertyIndexes.get(beanProperty)));
} catch (Exception error) {
// Failed introspection so probably not a valid bean
// property.
}
}
return ImmutableList.copyOf(properties);
}
private void blockUntilRunningAndAssignElasticIpsToInstancesOrPutIntoBadMap(
Set<RunningInstance> input, Map<NodeMetadata, Exception> badNodes) {
Map<RegionAndName, RunningInstance> instancesById =
Maps.uniqueIndex(input, instanceToRegionAndName);
for (Map.Entry<RegionAndName, RunningInstance> entry : instancesById.entrySet()) {
RegionAndName id = entry.getKey();
RunningInstance instance = entry.getValue();
try {
logger.debug("<< allocating elastic IP instance(%s)", id);
String ip = client.getElasticIPAddressServices().allocateAddressInRegion(id.getRegion());
// block until instance is running
logger.debug(">> awaiting status running instance(%s)", id);
AtomicReference<NodeMetadata> node =
newReference(runningInstanceToNodeMetadata.apply(instance));
nodeRunning.apply(node);
logger.trace("<< running instance(%s)", id);
logger.debug(">> associating elastic IP %s to instance %s", ip, id);
client
.getElasticIPAddressServices()
.associateAddressInRegion(id.getRegion(), ip, id.getName());
logger.trace("<< associated elastic IP %s to instance %s", ip, id);
// add mapping of instance to ip into the cache
elasticIpCache.put(id, ip);
} catch (RuntimeException e) {
badNodes.put(runningInstanceToNodeMetadata.apply(instancesById.get(id)), e);
}
}
}
private static <K, V> V get(LoadingCache<K, V> cache, K key) {
try {
return cache.get(key);
} catch (ExecutionException | UncheckedExecutionException | ExecutionError e) {
throw Throwables.propagate(e.getCause());
}
}
@Override
public Response<List<Address>> ancestorOfAddresses(Integer anyId) {
Response<List<Address>> result = new Response<List<Address>>();
if (anyId == null) {
log.error("id can not be null");
result.setError("params.not.null");
return result;
}
List<Address> addresses = Lists.newArrayListWithExpectedSize(3);
try {
Integer id = anyId;
while (id > 0) {
Address address = addressCache.getUnchecked(id);
addresses.add(address);
id = address.getParentId();
}
result.setResult(addresses);
return result;
} catch (Exception e) {
log.error(
"failed find ancestors of address(id={}), cause:{}",
anyId,
Throwables.getStackTraceAsString(e));
result.setError("ancestor.query.fail");
return result;
}
}
@VisibleForTesting
public Set<String> getSecurityGroupsForTagAndOptions(
String region, @Nullable String group, TemplateOptions options) {
Builder<String> groups = ImmutableSet.builder();
if (group != null) {
String markerGroup = namingConvention.create().sharedNameForGroup(group);
groups.add(markerGroup);
RegionNameAndIngressRules regionNameAndIngressRulesForMarkerGroup;
if (userSpecifiedTheirOwnGroups(options)) {
regionNameAndIngressRulesForMarkerGroup =
new RegionNameAndIngressRules(region, markerGroup, new int[] {}, false);
groups.addAll(EC2TemplateOptions.class.cast(options).getGroups());
} else {
regionNameAndIngressRulesForMarkerGroup =
new RegionNameAndIngressRules(region, markerGroup, options.getInboundPorts(), true);
}
// this will create if not yet exists.
securityGroupMap.getUnchecked(regionNameAndIngressRulesForMarkerGroup);
}
return groups.build();
}
@VisibleForTesting
public CxxPreprocessAndCompile createPreprocessBuildRule(String name, CxxSource source) {
Preconditions.checkArgument(CxxSourceTypes.isPreprocessableType(source.getType()));
BuildTarget target = createPreprocessBuildTarget(name, source.getType());
PreprocessorDelegate preprocessorDelegate =
preprocessorDelegates.getUnchecked(
PreprocessAndCompilePreprocessorDelegateKey.of(source.getType(), source.getFlags()));
// Build the CxxCompile rule and add it to our sorted set of build rules.
CxxPreprocessAndCompile result =
CxxPreprocessAndCompile.preprocess(
getParams()
.copyWithChanges(
target,
new DepsBuilder()
.addPreprocessDeps()
.add(preprocessorDelegate.getPreprocessor())
.add(source),
Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of())),
getPathResolver(),
preprocessorDelegate,
new CompilerDelegate(
getPathResolver(),
getCxxPlatform().getDebugPathSanitizer(),
getCompiler(source.getType()),
computeCompilerFlags(source.getType(), source.getFlags())),
getPreprocessOutputPath(target, source.getType(), name),
source.getPath(),
source.getType(),
getCxxPlatform().getDebugPathSanitizer());
getResolver().addToIndex(result);
return result;
}
public boolean tryLock(Object name) {
Monitor lock;
synchronized (stringLocks) {
lock = stringLocks.getUnchecked(name);
}
return lock.tryEnter();
}
TopLevelItem get(LazyTopLevelItem.Key key) {
try {
return cache.get(key);
} catch (ExecutionException ex) {
Logger.getLogger(TopLevelItemsCache.class.getName()).log(Level.SEVERE, null, ex);
return null;
}
}
public UserInfo loadUserInfo(final PendingOIDCAuthenticationToken token) {
try {
return cache.get(token);
} catch (UncheckedExecutionException | ExecutionException e) {
logger.warn("Couldn't load User Info from token: " + e.getMessage());
return null;
}
}
@NotNull
private PackageDetails refreshAndGetPackageDetailsFromPyPI(
@NotNull String packageName, boolean alwaysRefresh) throws IOException {
if (alwaysRefresh) {
myPackageToDetails.invalidate(packageName);
}
return getCachedValueOrRethrowIO(myPackageToDetails, packageName);
}
/**
* Implemented as specified in {@link QuantumStrategy}.
*
* @see QuantumStrategy#quantize(double)
*/
@Override
public int quantize(double value) throws QuantizationException {
try {
return values.get(getMiddleRange(value));
} catch (ExecutionException e) {
throw new QuantizationException(e);
}
}
/**
* Makes sure that name is a valid repository name and creates a new RepositoryName using it.
*
* @throws LabelSyntaxException if the name is invalid
*/
public static RepositoryName create(String name) throws LabelSyntaxException {
try {
return repositoryNameCache.get(name);
} catch (ExecutionException e) {
Throwables.propagateIfInstanceOf(e.getCause(), LabelSyntaxException.class);
throw new IllegalStateException("Failed to create RepositoryName from " + name, e);
}
}
private Mustache compileIfChanged(String filename, boolean partial) {
Mustache template = mustacheCache.getIfPresent(filename);
if (template == null) {
String desc = partial ? "partial" : "template";
Log.debug("Compiling Mustache " + desc, "name", filename);
Res res = getResource(filename, partial);
template = customCompile(filename, res);
res.onChange(
"mustache",
new Runnable() {
@Override
public void run() {
invalidateCache();
}
})
.trackChanges();
mustacheCache.put(filename, template);
}
return template;
}
