private void scrubCells(
TransactionManager txManager,
Multimap<String, Cell> tableNameToCells,
long scrubTimestamp,
Transaction.TransactionType transactionType) {
for (Entry<String, Collection<Cell>> entry : tableNameToCells.asMap().entrySet()) {
String tableName = entry.getKey();
if (log.isInfoEnabled()) {
log.info(
"Attempting to immediately scrub "
+ entry.getValue().size()
+ " cells from table "
+ tableName);
}
for (List<Cell> cells : Iterables.partition(entry.getValue(), batchSizeSupplier.get())) {
Multimap<Cell, Long> timestampsToDelete =
HashMultimap.create(
keyValueService.getAllTimestamps(
tableName, ImmutableSet.copyOf(cells), scrubTimestamp));
for (Cell cell : ImmutableList.copyOf(timestampsToDelete.keySet())) {
// Don't scrub garbage collection sentinels
timestampsToDelete.remove(cell, Value.INVALID_VALUE_TIMESTAMP);
}
// If transactionType == TransactionType.AGGRESSIVE_HARD_DELETE this might
// force other transactions to abort or retry
deleteCellsAtTimestamps(txManager, tableName, timestampsToDelete, transactionType);
}
if (log.isInfoEnabled()) {
log.info(
"Immediately scrubbed " + entry.getValue().size() + " cells from table " + tableName);
}
}
}
@Test
public void testApplyWithIllegalStateException() throws UnknownHostException {
final NovaApi api = createMock(NovaApi.class);
KeyPairApi keyApi = createMock(KeyPairApi.class);
@SuppressWarnings("unchecked")
final Supplier<String> uniqueIdSupplier = createMock(Supplier.class);
KeyPair pair = createMock(KeyPair.class);
expect(api.getKeyPairExtensionForZone("zone")).andReturn(Optional.of(keyApi)).atLeastOnce();
expect(uniqueIdSupplier.get()).andReturn("1");
expect(keyApi.createKeyPair("group-1")).andThrow(new IllegalStateException());
expect(uniqueIdSupplier.get()).andReturn("2");
expect(keyApi.createKeyPair("group-2")).andReturn(pair);
replay(api, keyApi, uniqueIdSupplier);
CreateUniqueKeyPair parser =
Guice.createInjector(
new AbstractModule() {
@Override
protected void configure() {
bind(new TypeLiteral<Supplier<String>>() {}).toInstance(uniqueIdSupplier);
bind(NovaApi.class).toInstance(api);
}
})
.getInstance(CreateUniqueKeyPair.class);
assertEquals(parser.load(ZoneAndName.fromZoneAndName("zone", "group")), pair);
verify(api, keyApi, uniqueIdSupplier);
}
@Override
public EzSecurityToken refreshSecurityToken(EzSecurityToken token)
throws EzSecurityTokenException {
EzSecurityToken refreshedToken;
EzSecurity.Client client = null;
try {
TokenRequest request =
new TokenRequest(securityId, System.currentTimeMillis(), token.getType());
request.setTokenPrincipal(token);
client = pool.get().getClient(ezsecurityConstants.SERVICE_NAME, EzSecurity.Client.class);
refreshedToken = client.refreshToken(request, "");
} catch (AppNotRegisteredException e) {
log.error("Application {} is not registered with EzSecurity", securityId, e);
throw new EzSecurityTokenException("Application not registered " + e.getMessage());
} catch (TException e) {
log.error("Unexpected thrift exception getting security token: {}", e.getMessage());
throw new EzSecurityTokenException("TException getting security token: " + e.getMessage());
} finally {
if (client != null) {
pool.get().returnToPool(client);
}
}
return refreshedToken;
}
@BeforeMethod(groups = "unit")
public void init() {
factory = new RawExtractJobFactoryImpl();
stagingDirectoryFactory = EasyMock.createMock(CredentialedStagingDirectoryFactory.class);
stagingDirectory = EasyMock.createMock(StagingDirectory.class);
EasyMock.expect(stagingDirectory.getAbsolutePath()).andStubReturn(PATH);
EasyMock.expect(stagingDirectory.getSep()).andStubReturn("/");
final Supplier<DisposableResourceTracker> trackerSupplier = EasyMockUtils.createMockSupplier();
tracker = createMock(DisposableResourceTracker.class);
expect(trackerSupplier.get()).andReturn(tracker);
replay(trackerSupplier);
rawPopulator = EasyMock.createMock(InputContext.class);
converter = EasyMock.createMock(DTAToMzXMLConverter.class);
options = new DTAToMzXMLOptions();
factory.setDtaToMzXMLConverter(converter);
factory.setDtaToMxXMLOptions(options);
factory.setDisposableResourceTrackerSupplier(trackerSupplier);
factory.setCredentialedStagingDirectoryFactory(stagingDirectoryFactory);
mockObjects =
MockObjectCollection.fromObjects(
stagingDirectoryFactory, stagingDirectory, tracker, rawPopulator, converter);
}
@Override
public void run() {
// gather data from provider
final DataProvider dataProvider = getDataProvider();
checkNotNull(
dataProvider,
"Probable problem: your MASConfiguration/ObjectiveFunction/"
+ "PostProcessor is not fully serializable.");
final Supplier<Scenario> scenario = getDataProvider().getParameter(scenarioId);
final MASConfiguration configuration = getDataProvider().getParameter(configurationId);
final ObjectiveFunction objectiveFunction =
getDataProvider().getParameter(objectiveFunctionId);
final PostProcessor<?> postProcessor = getDataProvider().getParameter(postProcessorId);
final Scenario s = scenario.get();
final SimArgs simArgs =
SimArgs.create(s, configuration, seed, objectiveFunction, false, postProcessor, null);
final Object simResult = Experiment.perform(simArgs);
checkArgument(
simResult instanceof Serializable,
"Your PostProcessor must generate Serializable objects, found %s.",
simResult);
setResult(simResult);
}
public void destroyEngine() {
super.destroyEngine();
cassandraSessionManager.get().shutdown();
scheduledExecutorService.get().shutdown();
}
@Override
public long getSweepTimestamp(SweepStrategy sweepStrategy) {
if (sweepStrategy == SweepStrategy.CONSERVATIVE) {
return Math.min(unreadableTimestampSupplier.get(), immutableTimestampSupplier.get());
} else {
return immutableTimestampSupplier.get();
}
}
public static void spinAssertEquals(Object expected, Supplier<Object> s, int timeoutInSeconds) {
long now = System.currentTimeMillis();
while (System.currentTimeMillis() - now < now + (1000 * timeoutInSeconds)) {
if (s.get().equals(expected)) break;
Thread.yield();
}
assertEquals(expected, s.get());
}
@Override
public void createTaskTables() {
if (config.get().isCreateTables()) {
final MetadataStorageTablesConfig tablesConfig = tablesConfigSupplier.get();
final String entryType = tablesConfig.getTaskEntryType();
createEntryTable(tablesConfig.getEntryTable(entryType));
createLogTable(tablesConfig.getLogTable(entryType), entryType);
createLockTable(tablesConfig.getLockTable(entryType), entryType);
}
}
public List<Rule> getRulesWithDefault(final String dataSource) {
List<Rule> retVal = Lists.newArrayList();
Map<String, List<Rule>> theRules = rules.get();
if (theRules.get(dataSource) != null) {
retVal.addAll(theRules.get(dataSource));
}
if (theRules.get(config.get().getDefaultRule()) != null) {
retVal.addAll(theRules.get(config.get().getDefaultRule()));
}
return retVal;
}
static final <T> Set<T> getApis(final Class<T> superclass) {
Set<T> implementations = Sets.newHashSet();
for (Supplier<?> supplier : suppliers.values()) {
Object implementation = supplier.get();
if (superclass.isInstance(implementation)) {
implementations.add(superclass.cast(implementation));
}
}
return implementations;
}
/**
* Invoked when we have detected that a plugin has been enabled.
*
* @param plugin - the enabled plugin.
*/
private void onPluginEnabled(Plugin plugin) {
String key = plugin.getName();
Supplier<? extends Component> supplier = suppliers.get(key);
// Only register the component once
if (isRegistered() && supplier != null) {
Component component = components.get(key);
// Construct and register the component
if (component == null) components.put(key, component = supplier.get());
if (!component.isRegistered()) component.register(plugin, bus);
}
}
private static FilteredPipeline findAccepting(final HttpRequest request) {
final FilteredPipeline candidate = null;
for (final FilteredPipeline f : pipelines) {
if (f.checkAccepts(request)) {
return f;
}
}
if (request.getHeader(HttpHeaders.Names.HOST).contains("amazonaws.com")
|| request.getHeader(HttpHeaders.Names.HOST).contains(subDomain.get())) {
String hostHeader = request.getHeader(HttpHeaders.Names.HOST);
LOG.debug("Trying to intercept request for " + hostHeader);
for (final FilteredPipeline f : pipelines) {
if (Ats.from(f).has(ComponentPart.class)) {
Class<? extends ComponentId> compIdClass = Ats.from(f).get(ComponentPart.class).value();
ComponentId compId = ComponentIds.lookup(compIdClass);
if (Ats.from(compIdClass).has(PublicService.class)) {
if (request.getHeaderNames().contains("SOAPAction")
&& f.addHandlers(Channels.pipeline()).get(SoapHandler.class) == null) {
continue; // Skip pipeline which doesn't handle SOAP for this SOAP request
} else if (!request.getHeaderNames().contains("SOAPAction")
&& f.addHandlers(Channels.pipeline()).get(SoapHandler.class) != null) {
continue; // Skip pipeline which handles SOAP for this non-SOAP request
}
LOG.debug("Maybe intercepting: " + hostHeader + " using " + f.getClass());
if (Ats.from(compIdClass).has(AwsServiceName.class)
&& request
.getHeader(HttpHeaders.Names.HOST)
.matches(
"[\\w\\.-_]*" + compId.getAwsServiceName() + "\\.\\w+\\.amazonaws.com")) {
return f; // Return pipeline which can handle the request for
// ${service}.${region}.amazonaws.com
} else if (request
.getHeader(HttpHeaders.Names.HOST)
.matches("[\\w\\.-_]*" + compId.name() + "\\." + subDomain.get())) {
return f; // Return pipeline which can handle the request for
// ${service}.${system.dns.dnsdomain}
}
}
}
}
}
if (candidate == null) {
for (final FilteredPipeline f : internalPipelines) {
if (f.checkAccepts(request)) {
return f;
}
}
}
return candidate;
}
public RowUniquenessConstraint(
Keyspace keyspace, ColumnFamily<K, C> columnFamily, K key, Supplier<C> uniqueColumnSupplier) {
this.keyspace = keyspace;
this.columnFamily = columnFamily;
this.uniqueColumn = uniqueColumnSupplier.get();
this.key = key;
}
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public NodeMetadata apply(ServerInfo from) {
NodeMetadataBuilder builder = new NodeMetadataBuilder();
builder.ids(from.getUuid());
builder.name(from.getName());
builder.location(locationSupplier.get());
builder.group(parseGroupFromName(from.getName()));
String imageId = getImageIdFromServer.apply(from);
if (imageId != null) {
Image image = findImageForId.apply(imageId);
if (image != null) {
builder.operatingSystem(image.getOperatingSystem());
builder.adminPassword(image.getAdminPassword());
}
}
builder.hardware(
new HardwareBuilder()
.ids(from.getUuid())
.processors(ImmutableList.of(new Processor(1, from.getCpu())))
.ram(from.getMem())
.volumes(
(List)
ImmutableList.of(
Iterables.transform(from.getDevices().values(), deviceToVolume)))
.build());
builder.state(serverStatusToNodeState.get(from.getStatus()));
builder.publicAddresses(ImmutableSet.<String>of(from.getVnc().getIp()));
builder.privateAddresses(ImmutableSet.<String>of());
builder.credentials(credentialStore.get("node#" + from.getUuid()));
return builder.build();
}
@Override
public HttpRequest filter(HttpRequest request) throws HttpException {
return request
.toBuilder()
.replaceHeader(AuthHeaders.AUTH_TOKEN, authTokenProvider.get())
.build();
}
// adds filters for the lower bound, starting at a given field, with conditions on the higher
// levels supplied
private static List<PartitionFilter> addLowerFilters(
String[] fields,
int position,
PartitionKey keyLower,
List<PartitionFilter> filters,
Supplier<PartitionFilter.Builder> supplier) {
if (position >= fields.length) {
return filters;
}
String fieldName = fields[position];
int lower = (Integer) keyLower.getField(fieldName);
// if this field is at the finest granularity (minutes), we must include its value in the range
// otherwise we exclude it from the range and descend into the next finer granularity with a
// value
// constraints on the current field name. For example:
// - for hour:15/minute:10, we add a filter for hour>=16 (excluding 15) and descent for hour=15
// - now the remaining field is minute:10, we add a filter for minute>=10 (including 10)
int lowerBound = position == fields.length - 1 ? lower : lower + 1;
// only add the filter if this condition is satisfiable. For example, not for hour>=24 or
// month>=13
if (isSatisfiableLowerBound(fieldName, lowerBound)) {
filters.add(supplier.get().addRangeCondition(fieldName, lowerBound, null).build());
}
return addLowerFilters(
fields, position + 1, keyLower, filters, nextSupplier(supplier, fieldName, lower));
}
@Override
public NodeMetadata createNodeWithGroupEncodedIntoNameThenStoreCredentials(
String group, String name, Template template, Map<String, Credentials> credentialStore) {
NodeMetadataBuilder builder = new NodeMetadataBuilder();
String id = idProvider.get() + "";
builder.ids(id);
builder.name(name);
// using a predictable name so tests will pass
builder.hostname(group);
builder.tags(template.getOptions().getTags());
builder.group(group);
builder.location(location.get());
builder.imageId(template.getImage().getId());
builder.operatingSystem(template.getImage().getOperatingSystem());
builder.state(NodeState.PENDING);
builder.publicAddresses(ImmutableSet.<String>of(publicIpPrefix + id));
builder.privateAddresses(ImmutableSet.<String>of(privateIpPrefix + id));
Credentials creds = template.getOptions().getOverridingCredentials();
if (creds == null) creds = new Credentials(null, null);
if (creds.identity == null) creds = creds.toBuilder().identity("root").build();
if (creds.credential == null) creds = creds.toBuilder().credential(passwordPrefix + id).build();
builder.credentials(creds);
NodeMetadata node = builder.build();
credentialStore.put("node#" + node.getId(), node.getCredentials());
nodes.put(node.getId(), node);
StubComputeServiceDependenciesModule.setState(node, NodeState.RUNNING, 100);
return node;
}
@Override
public String toString() {
return Objects.toStringHelper(name)
.add("user", user)
.add("privateKey", String.format("(sha-256:%s)", privateKeyDigest.get()))
.toString();
}
@Transactional(retryOn = {ONeedRetryException.class, ORecordDuplicatedException.class})
protected void doPutContent(
final String path, final Supplier<InputStream> streamSupplier, final Payload payload)
throws IOException {
StorageTx tx = UnitOfWork.currentTransaction();
final Bucket bucket = tx.getBucket();
Component component = getComponent(tx, path, bucket);
Asset asset;
if (component == null) {
// CREATE
component =
tx.createComponent(bucket, getRepository().getFormat())
.group(getGroup(path))
.name(getName(path));
// Set attributes map to contain "raw" format-specific metadata (in this case, path)
component.formatAttributes().set(P_PATH, path);
tx.saveComponent(component);
asset = tx.createAsset(bucket, component);
asset.name(component.name());
} else {
// UPDATE
asset = tx.firstAsset(component);
}
AttributesMap contentAttributes = null;
if (payload instanceof Content) {
contentAttributes = ((Content) payload).getAttributes();
}
Content.applyToAsset(asset, Content.maintainLastModified(asset, contentAttributes));
tx.setBlob(asset, path, streamSupplier.get(), hashAlgorithms, null, payload.getContentType());
tx.saveAsset(asset);
}
public T deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
JsonToken token = jp.getCurrentToken();
if (token == JsonToken.START_OBJECT) {
token = jp.nextToken();
}
R reader = readerSupplier.get();
while (token != null && token != JsonToken.END_OBJECT) {
if (token != JsonToken.FIELD_NAME) {
APIParsingException.raise(
"Parsing of json failed. Expected to be at field name token but was " + token.name(),
jp);
}
String name = jp.getCurrentName();
jp.nextToken();
Optional<FieldParser<R>> fieldParser = registry.getFieldParser(name);
if (fieldParser.isPresent()) {
fieldParser.get().parse(reader, jp, ctxt);
} else {
jp.skipChildren();
}
token = jp.nextToken();
}
return reader.validateAndBuild();
}
// Warning: scrubTimestamp MUST be the start timestamp of the hard delete transaction that
// triggers
// the scrubbing; we need this start timestamp to check whether the hard delete transaction was
// actually committed before we do any scrubbing
/* package */ void queueCellsForScrubbing(
Multimap<String, Cell> tableNameToCell, long scrubTimestamp) {
if (tableNameToCell.isEmpty()) {
return;
}
scrubberStore.queueCellsForScrubbing(tableNameToCell, scrubTimestamp, batchSizeSupplier.get());
}
/** in UltraDNS, zones are scoped to an account. */
@Override
public Iterator<String> list() {
return api.getZoneApi()
.listByAccount(account.get().getId())
.transform(ZoneName.INSTANCE)
.iterator();
}
@SuppressWarnings("unchecked")
public URI apply(Object from) {
Iterable<Object> orgTasksList = (Iterable<Object>) checkNotNull(from, "args");
Object org = Iterables.get(orgTasksList, 0);
Object tasksList = Iterables.get(orgTasksList, 1);
if (org == null && tasksList == null) return defaultTasksList.get().getHref();
else if (org == null) org = defaultOrg.get().getName();
try {
Map<String, ReferenceType> tasksLists = checkNotNull(orgMap.get().get(org)).getTasksLists();
return tasksList == null
? Iterables.getLast(tasksLists.values()).getHref()
: tasksLists.get(tasksList).getHref();
} catch (NullPointerException e) {
throw new NoSuchElementException(org + "/" + tasksList + " not found in " + orgMap.get());
}
}
public static Slice toJson(Type rowType, ConnectorSession session, Slice row) {
Object object = rowType.getObjectValue(session, createBlock(rowType, row), 0);
try {
return Slices.utf8Slice(OBJECT_MAPPER.get().writeValueAsString(object));
} catch (JsonProcessingException e) {
throw Throwables.propagate(e);
}
}
/** {@inheritDoc} */
@Override
public CalculatedValue<Void> calculate(final Map<Statistic, Calculator<?>> dependencies) {
final HistogramStatistic.HistogramAccumulator calculator =
(HistogramStatistic.HistogramAccumulator) dependencies.get(HISTOGRAM_STATISTIC.get());
return new CalculatedValue.Builder<Void>()
.setValue(calculator.calculate(((TPStatistic) getStatistic()).getPercentile()))
.build();
}
@Override
public Multimap<String, String> buildFormParameters() {
if (getAwsAccessKeyId() == null) {
checkState(creds != null, "creds should have been injected");
bucketOwnedBy(creds.get().identity);
}
return super.buildFormParameters();
}
@Override
public Statement apply(String group) {
checkNotNull(group, "group");
String validatorClientName = validatorName.get();
PrivateKey validatorKey = validatorCredential.get();
JsonBall bootstrapConfig = null;
try {
bootstrapConfig = bootstrapConfigForGroup.load(group);
} catch (Exception e) {
throw propagate(e);
}
Map<String, JsonBall> config =
json.fromJson(bootstrapConfig.toString(), BootstrapConfigForGroup.BOOTSTRAP_CONFIG_TYPE);
Optional<JsonBall> environment = Optional.fromNullable(config.get("environment"));
String chefConfigDir = "{root}etc{fs}chef";
Statement createChefConfigDir = exec("{md} " + chefConfigDir);
Statement createClientRb =
appendFile(
chefConfigDir + "{fs}client.rb",
ImmutableList.of(
"require 'rubygems'",
"require 'ohai'",
"o = Ohai::System.new",
"o.all_plugins",
String.format("node_name \"%s-\" + o[:ipaddress]", group),
"log_level :info",
"log_location STDOUT",
String.format("validation_client_name \"%s\"", validatorClientName),
String.format("chef_server_url \"%s\"", endpoint.get())));
Statement createValidationPem =
appendFile(
chefConfigDir + "{fs}validation.pem",
Splitter.on(newLinePattern).split(Pems.pem(validatorKey)));
String chefBootFile = chefConfigDir + "{fs}first-boot.json";
Statement createFirstBoot =
appendFile(chefBootFile, Collections.singleton(json.toJson(bootstrapConfig)));
ImmutableMap.Builder<String, String> options = ImmutableMap.builder();
options.put("-j", chefBootFile);
if (environment.isPresent()) {
options.put("-E", environment.get().toString());
}
String strOptions = Joiner.on(' ').withKeyValueSeparator(" ").join(options.build());
Statement runChef = exec("chef-client " + strOptions);
return newStatementList(
new ExitInsteadOfReturn(installChef),
createChefConfigDir,
createClientRb,
createValidationPem,
createFirstBoot,
runChef);
}
private CompactStriped(int stripes, Supplier<L> supplier) {
super(stripes);
Preconditions.checkArgument(stripes <= Ints.MAX_POWER_OF_TWO, "Stripes must be <= 2^30)");
this.array = new Object[mask + 1];
for (int i = 0; i < array.length; i++) {
array[i] = supplier.get();
}
}
private String getRuleURL() throws URISyntaxException {
Server server = selector.pick();
if (server == null) {
log.error("No instances found for [%s]!", config.get().getCoordinatorServiceName());
return null;
}
return new URI(
server.getScheme(),
null,
server.getAddress(),
server.getPort(),
config.get().getRulesEndpoint(),
null,
null)
.toString();
}
