private Multimap<String, String> computeUserParameters() {
Multimap<String, String> result = LinkedHashMultimap.create();
for (String key : suite.parameterNames()) {
// first check if the user has specified values
Collection<String> userValues = userParameterArguments.get(key);
if (!userValues.isEmpty()) {
result.putAll(key, userValues);
// TODO: type convert 'em to validate?
} else { // otherwise use the default values from the suite
Set<String> values = suite.parameterValues(key);
if (values.isEmpty()) {
throw new ConfigurationException(
key
+ " has no values. "
+ "Did you forget a -D"
+ key
+ "=<value> command line argument?");
}
result.putAll(key, values);
}
}
return result;
}
@Override
public Collection<SourcedDeviceUpnp> getDeviceList() {
// horribly unoptimal impl!
Map<String, SourcedDeviceUpnp> baseXmpp =
convertToSourcedDimmableLightMap(mXmppConnector.getDeviceList(), Source.XMPP);
Map<String, SourcedDeviceUpnp> baseLocal =
convertToSourcedDimmableLightMap(mLocalConnector.getDeviceList(), Source.LOCAL);
Multimap<String, SourcedDeviceUpnp> multiMap = ArrayListMultimap.create();
multiMap.putAll(Multimaps.forMap(baseXmpp));
multiMap.putAll(Multimaps.forMap(baseLocal));
Iterable<SourcedDeviceUpnp> lights =
Iterables.transform(
multiMap.asMap().values(),
new Function<Collection<SourcedDeviceUpnp>, SourcedDeviceUpnp>() {
@Override
public SourcedDeviceUpnp apply(Collection<SourcedDeviceUpnp> lightsCollection) {
EnumSet<Source> sources = EnumSet.noneOf(Source.class);
for (SourcedDeviceUpnp light : lightsCollection) {
sources.addAll(light.getSources());
}
// we make srong assumption here that all sources on list have same state
// (name/switch state/brightness)
SourcedDeviceUpnp first = Iterables.getFirst(lightsCollection, null);
first.setSources(sources);
return first;
}
});
return Lists.newArrayList(lights);
}
private Multimap<Cell, Long> getTimestampsFromRowResults(
List<RowResult<Set<Long>>> cellsToSweep, SweepStrategy sweepStrategy) {
Multimap<Cell, Long> cellTsMappings = HashMultimap.create();
for (RowResult<Set<Long>> rowResult : cellsToSweep) {
for (Map.Entry<Cell, Set<Long>> entry : rowResult.getCells()) {
if (sweepStrategy == SweepStrategy.CONSERVATIVE) {
cellTsMappings.putAll(
entry.getKey(), Sets.difference(entry.getValue(), invalidTimestamps));
} else {
cellTsMappings.putAll(entry.getKey(), entry.getValue());
}
}
}
return cellTsMappings;
}
private Multimap<Cell, Long> getCellTsPairsToSweep(
Multimap<Cell, Long> cellTsMappings,
PeekingIterator<RowResult<Value>> values,
long sweepTimestamp,
SweepStrategy sweepStrategy,
@Output Set<Cell> sentinelsToAdd) {
Multimap<Cell, Long> cellTsMappingsToSweep = HashMultimap.create();
Map<Long, Long> startTsToCommitTs = transactionService.get(cellTsMappings.values());
for (Map.Entry<Cell, Collection<Long>> entry : cellTsMappings.asMap().entrySet()) {
Cell cell = entry.getKey();
Collection<Long> timestamps = entry.getValue();
boolean sweepLastCommitted = isLatestValueEmpty(cell, values);
Iterable<? extends Long> timestampsToSweep =
getTimestampsToSweep(
cell,
timestamps,
startTsToCommitTs,
sentinelsToAdd,
sweepTimestamp,
sweepLastCommitted,
sweepStrategy);
cellTsMappingsToSweep.putAll(entry.getKey(), timestampsToSweep);
}
return cellTsMappingsToSweep;
}
@Test
public void multimap_assertions_examples() {
Multimap<String, String> nbaTeams = ArrayListMultimap.create();
nbaTeams.putAll("Lakers", newArrayList("Kobe Bryant", "Magic Johnson", "Kareem Abdul Jabbar"));
nbaTeams.putAll("Spurs", newArrayList("Tony Parker", "Tim Duncan", "Manu Ginobili"));
assertThat(nbaTeams).hasSize(6);
assertThat(nbaTeams).containsKeys("Lakers", "Spurs");
assertThat(nbaTeams).contains(entry("Lakers", "Kobe Bryant"), entry("Spurs", "Tim Duncan"));
assertThat(nbaTeams).containsValues("Kareem Abdul Jabbar", "Tony Parker");
assertThat(nbaTeams).hasSameEntriesAs(nbaTeams);
assertThat(nbaTeams).containsAllEntriesOf(nbaTeams);
Multimap<String, String> emptyMultimap = ArrayListMultimap.create();
assertThat(emptyMultimap).isEmpty();
}
private List<I_C_RfQResponseLine> getActiveRfqResponseLines_ForBPartnerId(final int bpartnerId) {
final Multimap<Integer, I_C_RfQResponseLine> bpartnerId2activeRfqResponseLines =
getActiveRfqResponseLines_IndexedByBPartnerId();
if (!bpartnerId2activeRfqResponseLines.containsKey(bpartnerId)) {
final List<I_C_RfQResponseLine> rfqResponseLines =
pmmRfQDAO.retrieveActiveResponseLines(getCtx(), bpartnerId);
bpartnerId2activeRfqResponseLines.putAll(bpartnerId, rfqResponseLines);
}
return ImmutableList.copyOf(bpartnerId2activeRfqResponseLines.get(bpartnerId));
}
/**
* Replaces the current payload with one that is a urlencoded payload including the following
* parameters and any formerly set.
*
* @see HttpRequest#getPayload()
*/
public T addFormParams(Multimap<String, String> parameters) {
checkNotNull(endpoint, "endpoint");
Multimap<String, String> map =
payload != null
? queryParser().apply(payload.getRawContent().toString())
: LinkedHashMultimap.<String, String>create();
map.putAll(parameters);
payload = newUrlEncodedFormPayload(map);
return self();
}
private void retrieveRegisteredListeners() {
for (MouseAction action : EnumSet.of(MOUSE_PRESSED, MOUSE_DRAGGED)) {
ArgumentCaptor<MouseActionListener> captor =
ArgumentCaptor.forClass(MouseActionListener.class);
verify(externalMouseDrivenComponent, atLeastOnce())
.addMouseActionListener(eq(action), captor.capture());
registeredListeners.putAll(action, captor.getAllValues());
}
}
public <S, P, T, D extends Pda<S, P>> D expand(
Pda<S, P> pda,
Function<S, Pda<S, P>> expand,
Function<S, T> tokens,
PdaFactory<D, S, P, T> fact) {
D result = fact.create(tokens.apply(pda.getStart()), tokens.apply(pda.getStop()));
Identity<S> identity = new Identity<S>();
Map<S, S> idstates = Maps.newIdentityHashMap();
Multimap<S, S> followers = LinkedHashMultimap.create();
for (S s_old : nfaUtil.collect(pda)) {
S s_new = idstates.get(s_old);
if (s_new == null) {
Pda<S, P> sub = expand.apply(s_old);
if (sub != null) {
S s_start = identity.get(fact.createPush(result, tokens.apply(s_old)));
S s_stop = identity.get(fact.createPop(result, tokens.apply(s_old)));
idstates.put(s_old, s_start);
idstates.put(sub.getStart(), s_start);
idstates.put(sub.getStop(), s_stop);
followers.putAll(s_start, sub.getFollowers(sub.getStart()));
followers.putAll(s_stop, pda.getFollowers(s_old));
for (S f_old : nfaUtil.collect(sub))
if (f_old != sub.getStart() && f_old != sub.getStop()) {
S f_new = idstates.get(f_old);
if (f_new == null) {
idstates.put(f_old, f_new = clone(f_old, sub, result, tokens, fact, identity));
followers.putAll(f_new, pda.getFollowers(f_old));
}
}
} else {
idstates.put(s_old, s_new = clone(s_old, pda, result, tokens, fact, identity));
followers.putAll(s_new, pda.getFollowers(s_old));
}
}
}
for (Map.Entry<S, Collection<S>> entry : followers.asMap().entrySet()) {
Set<S> f = Sets.newLinkedHashSet();
for (S s : entry.getValue()) f.add(idstates.get(s));
fact.setFollowers(result, entry.getKey(), f);
}
return result;
}
public static Multimap<String, String> forMap(Map<String, Collection<String>> map) {
Check.notNull(map, "map");
Multimap<String, String> result = HashMultimap.create();
for (String key : map.keySet()) {
result.putAll(key, map.get(key));
}
return result;
}
/**
* expand super types after scanning, for super types that were not scanned. this is helpful in
* finding the transitive closure without scanning all 3rd party dependencies. it uses {@link
* ReflectionUtils#getSuperTypes(Class)}.
*
* <p>for example, for classes A,B,C where A supertype of B, B supertype of C:
*
* <ul>
* <li>if scanning C resulted in B (B->C in store), but A was not scanned (although A supertype
* of B) - then getSubTypes(A) will not return C
* <li>if expanding supertypes, B will be expanded with A (A->B in store) - then getSubTypes(A)
* will return C
* </ul>
*/
public void expandSuperTypes() {
if (store.keySet().contains(index(SubTypesScanner.class))) {
Multimap<String, String> mmap = store.get(index(SubTypesScanner.class));
Sets.SetView<String> keys = Sets.difference(mmap.keySet(), Sets.newHashSet(mmap.values()));
Multimap<String, String> expand = HashMultimap.create();
for (String key : keys) {
expandSupertypes(expand, key, forName(key));
}
mmap.putAll(expand);
}
}
public void addResponses(final Iterable<Response> responses) {
typeToResponse.putAll(
Multimaps.index(
responses,
new Function<Response, String>() {
@Override
public String apply(final Response input) {
return input.type().asString();
}
}));
}
private List<I_C_Flatrate_Term> getC_Flatrate_Terms_ForBPartnerId(final int bpartnerId) {
final Multimap<Integer, I_C_Flatrate_Term> bpartnerId2contract =
getC_Flatrate_Terms_IndexedByBPartnerId();
if (!bpartnerId2contract.containsKey(bpartnerId)) {
final List<I_C_Flatrate_Term> contracts =
pmmContractsDAO.retrieveRunningContractsOnDateForBPartner(date, bpartnerId);
bpartnerId2contract.putAll(bpartnerId, contracts);
}
return ImmutableList.copyOf(bpartnerId2contract.get(bpartnerId));
}
private ImmutableSetMultimap<String, String> parsePermissions(
@Nullable String database, Ini.Section rolesSection, Ini.Section groupsSection) {
ImmutableSetMultimap.Builder<String, String> resultBuilder = ImmutableSetMultimap.builder();
Multimap<String, String> roleNameToPrivilegeMap = HashMultimap.create();
List<? extends RoleValidator> validators =
Lists.newArrayList(
new ServersAllIsInvalid(),
new DatabaseMustMatch(),
new DatabaseRequiredInRole(),
new ServerNameMustMatch(serverName));
for (Map.Entry<String, String> entry : rolesSection.entrySet()) {
String roleName = Strings.nullToEmpty(entry.getKey()).trim();
String roleValue = Strings.nullToEmpty(entry.getValue()).trim();
boolean invalidConfiguration = false;
if (roleName.isEmpty()) {
LOGGER.warn("Empty role name encountered in {}", resourcePath);
invalidConfiguration = true;
}
if (roleValue.isEmpty()) {
LOGGER.warn("Empty role value encountered in {}", resourcePath);
invalidConfiguration = true;
}
if (roleNameToPrivilegeMap.containsKey(roleName)) {
LOGGER.warn("Role {} defined twice in {}", roleName, resourcePath);
}
Set<String> roles = PermissionUtils.toPermissionStrings(roleValue);
if (!invalidConfiguration && roles != null) {
for (String role : roles) {
for (RoleValidator validator : validators) {
validator.validate(database, role.trim());
}
}
roleNameToPrivilegeMap.putAll(roleName, roles);
}
}
Splitter roleSplitter = ROLE_SPLITTER.omitEmptyStrings().trimResults();
for (Map.Entry<String, String> entry : groupsSection.entrySet()) {
String groupName = Strings.nullToEmpty(entry.getKey()).trim();
String groupPrivileges = Strings.nullToEmpty(entry.getValue()).trim();
Collection<String> resolvedGroupPrivileges = Sets.newHashSet();
for (String roleName : roleSplitter.split(groupPrivileges)) {
if (roleNameToPrivilegeMap.containsKey(roleName)) {
resolvedGroupPrivileges.addAll(roleNameToPrivilegeMap.get(roleName));
} else {
LOGGER.warn(
"Role {} for group {} does not exist in privileges section in {}",
new Object[] {roleName, groupName, resourcePath});
}
}
resultBuilder.putAll(groupName, resolvedGroupPrivileges);
}
return resultBuilder.build();
}
@Override
public Boolean caseCompoundElement(CompoundElement object) {
Multimap<String, AbstractElement> prevAssignedFeatures = newMultimap(assignedFeatures);
for (AbstractElement element : object.getElements()) {
doSwitch(element);
}
if (GrammarUtil.isMultipleCardinality(object)) {
for (AbstractElement element : object.getElements()) {
doSwitch(element);
}
}
if (GrammarUtil.isOptionalCardinality(object)) assignedFeatures.putAll(prevAssignedFeatures);
return Boolean.FALSE;
}
private void handleSuppressWarning(List<AnnotationTree> annotationTrees, int endLine) {
int startLine = 0;
List<String> warnings = Lists.newArrayList();
for (AnnotationTree annotationTree : annotationTrees) {
if (isSuppressWarningsAnnotation(annotationTree)) {
startLine = ((JavaTree) annotationTree).getLine();
warnings.addAll(getSuppressWarningArgs(annotationTree));
break;
}
}
for (int i = startLine; i <= endLine; i++) {
suppressWarningLines.putAll(i, warnings);
}
}
@Override
public Map<String, Collection<String>> revsDiff(Multimap<String, String> revisions) {
Preconditions.checkState(this.isOpen(), "Database is closed");
Preconditions.checkNotNull(revisions, "Input revisions must not be null");
Multimap<String, String> missingRevs = ArrayListMultimap.create();
// Break the potentially big multimap into small ones so for each map,
// a single query can be use to check if the <id, revision> pairs in sqlDb or not
List<Multimap<String, String>> batches =
this.multiMapPartitions(revisions, SQLITE_QUERY_PLACEHOLDERS_LIMIT);
for (Multimap<String, String> batch : batches) {
this.revsDiffBatch(batch);
missingRevs.putAll(batch);
}
return missingRevs.asMap();
}
@Override
public Boolean caseAlternatives(Alternatives object) {
Multimap<String, AbstractElement> prevAssignedFeatures = assignedFeatures;
Multimap<String, AbstractElement> mergedAssignedFeatures = LinkedHashMultimap.create();
for (AbstractElement element : object.getElements()) {
assignedFeatures = newMultimap(prevAssignedFeatures);
doSwitch(element);
mergedAssignedFeatures.putAll(assignedFeatures);
}
if (GrammarUtil.isOptionalCardinality(object)) {
mergedAssignedFeatures.putAll(prevAssignedFeatures);
}
assignedFeatures = mergedAssignedFeatures;
if (GrammarUtil.isMultipleCardinality(object)) {
prevAssignedFeatures = assignedFeatures;
for (AbstractElement element : object.getElements()) {
assignedFeatures = newMultimap(prevAssignedFeatures);
doSwitch(element);
mergedAssignedFeatures.putAll(assignedFeatures);
}
assignedFeatures = mergedAssignedFeatures;
}
return Boolean.FALSE;
}
@BeforeClass
public static void setUp() throws IOException {
InputStream dataFileInputStream =
NaraSoundexTest.class.getResourceAsStream("DaitchMokotoffSoundexTest.data");
List<String> lines = CharStreams.readLines(new InputStreamReader(dataFileInputStream));
for (String currentLine : lines) {
if (!currentLine.startsWith(";")) {
Iterable<String> components = Splitter.on("->").trimResults().split(currentLine);
Iterable<String> expectedValues =
Splitter.on(",").trimResults().split(Iterables.get(components, 1));
testCases.putAll(Iterables.get(components, 0), expectedValues);
}
}
}
@Test
public void bulk_index_active_rules() throws IOException {
List<ActiveRuleDto> activeRules =
newArrayList(
new ActiveRuleDto()
.setId(1)
.setProfileId(10)
.setRuleId(1)
.setSeverity(Severity.MAJOR)
.setParentId(5));
Multimap<Integer, ActiveRuleParamDto> paramsByActiveRule = ArrayListMultimap.create();
paramsByActiveRule.putAll(
1,
newArrayList(
new ActiveRuleParamDto()
.setId(1)
.setActiveRuleId(1)
.setRulesParameterId(1)
.setKey("key")
.setValue("RuleWithParameters")));
esActiveRule.bulkIndexActiveRules(activeRules, paramsByActiveRule);
assertThat(esSetup.exists("rules", "active_rule", "1"));
SearchHit[] parentHit =
esSetup
.client()
.prepareSearch("rules")
.setPostFilter(hasChildFilter("active_rule", termFilter("profileId", 10)))
.execute()
.actionGet()
.getHits()
.getHits();
assertThat(parentHit).hasSize(1);
assertThat(parentHit[0].getId()).isEqualTo("1");
SearchHit[] childHit =
esSetup
.client()
.prepareSearch("rules")
.setPostFilter(hasParentFilter("rule", termFilter("key", "RuleWithParameters")))
.execute()
.actionGet()
.getHits()
.getHits();
assertThat(childHit).hasSize(1);
assertThat(childHit[0].getId()).isEqualTo("1");
}
public Multimap<String, String> openRevisions(int start, int end) {
Preconditions.checkArgument(start >= 0, "Start position must be greater or equal to zero.");
Preconditions.checkArgument(end > start, "End position must be greater than start.");
Preconditions.checkArgument(
end <= this.size(), "End position must be smaller than changes feed size.");
Multimap<String, String> openRevisions = HashMultimap.create();
for (int i = start; i < end; i++) {
ChangesResult.Row row = this.getResults().get(i);
List<ChangesResult.Row.Rev> revisions = row.getChanges();
Set<String> openRevs = new HashSet<String>(revisions.size());
for (ChangesResult.Row.Rev rev : revisions) {
openRevs.add(rev.getRev());
}
openRevisions.putAll(row.getId(), openRevs);
}
return openRevisions;
}
protected void applyValidation(de.jutzig.jabylon.properties.Property property) {
Collection<Review> reviewList =
reviewService.review(descriptor, currentItem.getSourceProperty(), property);
reviews.removeAll(currentItem.getKey());
if (reviewList.isEmpty()) {
if (translated.getComponentError() != null) translated.setComponentError(null);
} else {
reviews.putAll((String) currentItem.getKey(), reviewList);
List<ErrorMessage> errors = new ArrayList<ErrorMessage>(reviewList.size());
for (Review review : reviewList) {
UserError error =
new UserError(review.getMessage(), UserError.CONTENT_TEXT, ErrorMessage.ERROR);
errors.add(error);
}
CompositeErrorMessage message = new CompositeErrorMessage(errors);
translated.setComponentError(message);
}
}
/**
* Prunes a multimap based on a predicate, returning the pruned values. The input map will be
* modified.
*
* @param map The multimap to prune.
* @param filterRule The pruning rule. When the predicate returns {@code false} for an entry, it
* will be pruned, otherwise it will be retained.
* @param <K> The key type in the multimap.
* @param <V> The value type in the multimap.
* @return A new multimap, containing the pruned keys/values.
*/
public static <K, V> Multimap<K, V> prune(
Multimap<K, V> map, Predicate<Collection<V>> filterRule) {
Preconditions.checkNotNull(map);
Preconditions.checkNotNull(filterRule);
Set<K> prunedKeys = Sets.newHashSet();
for (K key : map.keySet()) {
if (!filterRule.apply(map.get(key))) {
prunedKeys.add(key);
}
}
Multimap<K, V> pruned = ArrayListMultimap.create();
for (K key : prunedKeys) {
pruned.putAll(key, map.removeAll(key));
}
return pruned;
}
/**
* Populates the configuration specific mnemonicToExtraActionMap based on all action_listers
* selected by the user (via the blaze option {@code --experimental_action_listener=<target>}).
*/
private Multimap<String, ExtraActionSpec> computeMnemonicsToExtraActionMap() {
// We copy the multimap here every time. This could be expensive.
Multimap<String, ExtraActionSpec> mnemonicToExtraActionMap = HashMultimap.create();
for (TransitiveInfoCollection actionListener :
ruleContext.getPrerequisites(":action_listener", Mode.HOST)) {
ExtraActionMapProvider provider = actionListener.getProvider(ExtraActionMapProvider.class);
if (provider == null) {
ruleContext.ruleError(
String.format(
"Unable to match experimental_action_listeners to this rule. "
+ "Specified target %s is not an action_listener rule",
actionListener.getLabel().toString()));
} else {
mnemonicToExtraActionMap.putAll(provider.getExtraActionMap());
}
}
return mnemonicToExtraActionMap;
}
void inferTypes(NodeTraversal t, Node n, Scope scope) {
TypeInference typeInference =
new TypeInference(
compiler,
computeCfg(n),
reverseInterpreter,
scope,
assertionFunctionsMap,
getUnflowableVars(scope));
try {
typeInference.analyze();
escapedLocalVars.putAll(typeInference.getAssignedOuterLocalVars());
// Resolve any new type names found during the inference.
compiler.getTypeRegistry().resolveTypesInScope(scope);
} catch (DataFlowAnalysis.MaxIterationsExceededException e) {
compiler.report(t.makeError(n, DATAFLOW_ERROR));
}
}
public HttpResponse apply(Response nativeResponse) {
InputStream in = null;
try {
in =
BaseHttpCommandExecutorService.consumeOnClose(nativeResponse.getResponseBodyAsStream());
} catch (IOException e) {
Closeables.closeQuietly(in);
propagate(e);
assert false : "should have propagated exception";
}
Payload payload = in != null ? Payloads.newInputStreamPayload(in) : null;
HttpResponse response =
new HttpResponse(nativeResponse.getStatusCode(), nativeResponse.getStatusText(), payload);
Multimap<String, String> headers = LinkedHashMultimap.create();
for (Entry<String, List<String>> header : nativeResponse.getHeaders()) {
headers.putAll(header.getKey(), header.getValue());
}
utils.setPayloadPropertiesFromHeaders(headers, response);
return response;
}
/**
* Get SkyKeys for the FileValues for the given {@code pathFragments}. To do this, we look for a
* package lookup node for each path fragment, since package lookup nodes contain the "root" of a
* package. The returned SkyKeys correspond to FileValues that may not exist in the graph.
*/
private Collection<SkyKey> getSkyKeysForFileFragments(Iterable<PathFragment> pathFragments) {
Set<SkyKey> result = new HashSet<>();
Multimap<PathFragment, PathFragment> currentToOriginal = ArrayListMultimap.create();
for (PathFragment pathFragment : pathFragments) {
currentToOriginal.put(pathFragment, pathFragment);
}
while (!currentToOriginal.isEmpty()) {
Map<SkyKey, PathFragment> keys = new HashMap<>();
for (PathFragment pathFragment : currentToOriginal.keySet()) {
keys.put(
PackageLookupValue.key(PackageIdentifier.createInDefaultRepo(pathFragment)),
pathFragment);
}
Map<SkyKey, SkyValue> lookupValues = graph.getSuccessfulValues(keys.keySet());
for (Map.Entry<SkyKey, SkyValue> entry : lookupValues.entrySet()) {
PackageLookupValue packageLookupValue = (PackageLookupValue) entry.getValue();
if (packageLookupValue.packageExists()) {
PathFragment dir = keys.get(entry.getKey());
Collection<PathFragment> originalFiles = currentToOriginal.get(dir);
Preconditions.checkState(!originalFiles.isEmpty(), entry);
for (PathFragment fileName : originalFiles) {
result.add(
FileValue.key(RootedPath.toRootedPath(packageLookupValue.getRoot(), fileName)));
}
currentToOriginal.removeAll(dir);
}
}
Multimap<PathFragment, PathFragment> newCurrentToOriginal = ArrayListMultimap.create();
for (PathFragment pathFragment : currentToOriginal.keySet()) {
PathFragment parent = pathFragment.getParentDirectory();
if (parent != null) {
newCurrentToOriginal.putAll(parent, currentToOriginal.get(pathFragment));
}
}
currentToOriginal = newCurrentToOriginal;
}
return result;
}
@Override
public boolean accept(final String lex) {
valuesSeen.putAll(forMap(asMap(parseableAs(lex), x -> lex)));
return true;
}
public static Multimap<Language, LocalLink> mergeMaps(
Multimap<Language, LocalLink> map1, Multimap<Language, LocalLink> map2) {
map1.putAll(map2);
return map1;
}
/** @return number of cells read from _scrub table */
private int scrubSomeCells(
final TransactionManager txManager, int maxCellsToScrub, long maxScrubTimestamp) {
SortedMap<Long, Multimap<String, Cell>> scrubTimestampToTableNameToCell =
scrubberStore.getCellsToScrub(maxCellsToScrub, maxScrubTimestamp);
// Don't call expensive toString() if trace logging is off
if (log.isTraceEnabled()) {
log.trace("Attempting to scrub cells: " + scrubTimestampToTableNameToCell);
}
if (log.isInfoEnabled()) {
Set<String> tables = Sets.newHashSet();
for (Multimap<String, Cell> v : scrubTimestampToTableNameToCell.values()) {
tables.addAll(v.keySet());
}
log.info(
"Attempting to scrub "
+ scrubTimestampToTableNameToCell.size()
+ " cells from tables "
+ tables);
}
if (scrubTimestampToTableNameToCell.size() == 0) {
return 0; // No cells left to scrub
}
Multimap<Long, Cell> toRemoveFromScrubQueue = HashMultimap.create();
int numCellsReadFromScrubTable = 0;
for (Map.Entry<Long, Multimap<String, Cell>> entry :
scrubTimestampToTableNameToCell.entrySet()) {
final long scrubTimestamp = entry.getKey();
final Multimap<String, Cell> tableNameToCell = entry.getValue();
numCellsReadFromScrubTable += tableNameToCell.size();
long commitTimestamp = getCommitTimestampRollBackIfNecessary(scrubTimestamp, tableNameToCell);
if (commitTimestamp >= maxScrubTimestamp) {
// We cannot scrub this yet because not all transactions can read this value.
continue;
} else if (commitTimestamp != TransactionConstants.FAILED_COMMIT_TS) {
// This is CRITICAL; don't scrub if the hard delete transaction didn't actually finish
// (we still remove it from the _scrub table with the call to markCellsAsScrubbed though),
// or else we could cause permanent data loss if the hard delete transaction failed after
// queuing cells to scrub but before successfully committing
List<Future<Void>> scrubFutures = Lists.newArrayList();
for (final List<Entry<String, Cell>> batch :
Iterables.partition(tableNameToCell.entries(), batchSizeSupplier.get())) {
final Multimap<String, Cell> batchMultimap = HashMultimap.create();
for (Entry<String, Cell> e : batch) {
batchMultimap.put(e.getKey(), e.getValue());
}
scrubFutures.add(
exec.submit(
new Callable<Void>() {
@Override
public Void call() throws Exception {
scrubCells(
txManager,
batchMultimap,
scrubTimestamp,
aggressiveScrub
? TransactionType.AGGRESSIVE_HARD_DELETE
: TransactionType.HARD_DELETE);
return null;
}
}));
}
for (Future<Void> future : scrubFutures) {
Futures.getUnchecked(future);
}
}
toRemoveFromScrubQueue.putAll(scrubTimestamp, tableNameToCell.values());
}
Multimap<Cell, Long> cellToScrubTimestamp = HashMultimap.create();
scrubberStore.markCellsAsScrubbed(
Multimaps.invertFrom(toRemoveFromScrubQueue, cellToScrubTimestamp),
batchSizeSupplier.get());
if (log.isTraceEnabled()) {
log.trace("Finished scrubbing cells: " + scrubTimestampToTableNameToCell);
}
if (log.isInfoEnabled()) {
Set<String> tables = Sets.newHashSet();
for (Multimap<String, Cell> v : scrubTimestampToTableNameToCell.values()) {
tables.addAll(v.keySet());
}
log.info(
"Finished scrubbing "
+ scrubTimestampToTableNameToCell.size()
+ " cells from tables "
+ tables);
}
return numCellsReadFromScrubTable;
}
