AttributesRoutings(
ImmutableList<ShardRouting> withSameAttribute,
ImmutableList<ShardRouting> withoutSameAttribute) {
this.withSameAttribute = withSameAttribute;
this.withoutSameAttribute = withoutSameAttribute;
this.totalSize = withoutSameAttribute.size() + withSameAttribute.size();
}
private String formatSubcomponentDeclaration(SubcomponentDeclaration subcomponentDeclaration) {
ImmutableList<TypeMirror> moduleSubcomponents =
getModuleSubcomponents(subcomponentDeclaration.moduleAnnotation());
int index =
Iterables.indexOf(
moduleSubcomponents,
MoreTypes.equivalence()
.equivalentTo(subcomponentDeclaration.subcomponentType().asType()));
StringBuilder annotationValue = new StringBuilder();
if (moduleSubcomponents.size() != 1) {
annotationValue.append("{");
}
annotationValue.append(
formatArgumentInList(
index,
moduleSubcomponents.size(),
subcomponentDeclaration.subcomponentType().getQualifiedName() + ".class"));
if (moduleSubcomponents.size() != 1) {
annotationValue.append("}");
}
return String.format(
"@%s(subcomponents = %s) for %s",
simpleName(subcomponentDeclaration.moduleAnnotation()),
annotationValue,
subcomponentDeclaration.contributingModule().get());
}
private static ImmutableMap<String, String> getQueryMap(final String query)
throws MalformedRequestException {
if (StringUtils.isBlank(query)) {
return ImmutableMap.of();
} else {
final Iterable<String> params = Splitter.on('&').split(query);
final ImmutableMap.Builder<String, String> map = ImmutableMap.builder();
for (final String param : params) {
final ImmutableList<String> strings = ImmutableList.copyOf(Splitter.on('=').split(param));
final String name = strings.get(0);
if (strings.size() > 2) {
throw new MalformedRequestException("Multiple '=' for parameter " + name);
}
final String value;
if (strings.size() > 1) {
value = strings.get(1);
} else {
value = null;
}
if (map.build().keySet().contains(name)) {
throw new MalformedRequestException("Duplicate value for parameter " + name);
}
map.put(name, value);
}
return map.build();
}
}
@Override
public String toString() {
return "A similarity matrix comparing "
+ xSubmissions.size()
+ " submissions to "
+ ySubmissions.size();
}
protected void assertTestCountryExtendeds(
ImmutableList<Integer> is,
List<BadgerEntity<CountryExtended>> countryExtendeds,
List<String> currencyIds,
String baseName,
String repeatedString11one,
String repeatedString11two) {
assertEquals(is.size(), countryExtendeds.size());
ImmutableMap.Builder<Integer, BadgerEntity<CountryExtended>> builder = ImmutableMap.builder();
countryExtendeds.forEach(
countryExtended -> builder.put(countryExtended.getMessage().getI(), countryExtended));
ImmutableMap<Integer, BadgerEntity<CountryExtended>> map = builder.build();
Set<Integer> iSet = new HashSet<>(is);
assertEquals(is.size(), iSet.size());
assertEquals(iSet, map.keySet());
map.entrySet()
.forEach(
entry ->
assertTestCountryExtended(
entry.getKey(),
entry.getValue(),
currencyIds,
baseName,
repeatedString11one,
repeatedString11two));
}
public void testComplexBuilder() {
List<Integer> colorElem = asList(0x00, 0x33, 0x66, 0x99, 0xCC, 0xFF);
ImmutableList.Builder<Integer> webSafeColorsBuilder = ImmutableList.builder();
for (Integer red : colorElem) {
for (Integer green : colorElem) {
for (Integer blue : colorElem) {
webSafeColorsBuilder.add((red << 16) + (green << 8) + blue);
}
}
}
ImmutableList<Integer> webSafeColors = webSafeColorsBuilder.build();
assertEquals(216, webSafeColors.size());
Integer[] webSafeColorArray = webSafeColors.toArray(new Integer[webSafeColors.size()]);
assertEquals(0x000000, (int) webSafeColorArray[0]);
assertEquals(0x000033, (int) webSafeColorArray[1]);
assertEquals(0x000066, (int) webSafeColorArray[2]);
assertEquals(0x003300, (int) webSafeColorArray[6]);
assertEquals(0x330000, (int) webSafeColorArray[36]);
assertEquals(0x000066, (int) webSafeColors.get(2));
assertEquals(0x003300, (int) webSafeColors.get(6));
ImmutableList<Integer> addedColor = webSafeColorsBuilder.add(0x00BFFF).build();
assertEquals(
"Modifying the builder should not have changed any already" + " built sets",
216,
webSafeColors.size());
assertEquals("the new array should be one bigger than webSafeColors", 217, addedColor.size());
Integer[] appendColorArray = addedColor.toArray(new Integer[addedColor.size()]);
assertEquals(0x00BFFF, (int) appendColorArray[216]);
}
// IndexOutOfBoundsException is raised when the instance is equal to ROOT
ResourceId parent() {
if (components.size() == 1) {
return ROOT;
} else {
return new ResourceId(components.subList(0, components.size() - 1));
}
}
/** Normalizes all shard routings to the same version. */
public IndexShardRoutingTable normalizeVersions() {
if (shards.isEmpty()) {
return this;
}
if (shards.size() == 1) {
return this;
}
long highestVersion = shards.get(0).version();
boolean requiresNormalization = false;
for (int i = 1; i < shards.size(); i++) {
if (shards.get(i).version() != highestVersion) {
requiresNormalization = true;
}
if (shards.get(i).version() > highestVersion) {
highestVersion = shards.get(i).version();
}
}
if (!requiresNormalization) {
return this;
}
List<ShardRouting> shardRoutings = new ArrayList<>(shards.size());
for (int i = 0; i < shards.size(); i++) {
if (shards.get(i).version() == highestVersion) {
shardRoutings.add(shards.get(i));
} else {
shardRoutings.add(new ShardRouting(shards.get(i), highestVersion));
}
}
return new IndexShardRoutingTable(
shardId, ImmutableList.copyOf(shardRoutings), primaryAllocatedPostApi);
}
private void createAndroidBinaryRuleAndTestCopyNativeLibraryCommand(
ImmutableSet<String> cpuFilters,
String sourceDir,
String destinationDir,
ImmutableList<String> expectedShellCommands) {
BuildRuleResolver ruleResolver = new BuildRuleResolver();
AndroidBinaryRule.Builder builder =
AndroidBinaryRule.newAndroidBinaryRuleBuilder(new FakeAbstractBuildRuleBuilderParams())
.setBuildTarget(BuildTargetFactory.newInstance("//:fbandroid_with_dash_debug_fbsign"))
.setManifest("AndroidManifest.xml")
.setKeystore(addKeystoreRule(ruleResolver))
.setTarget("Google Inc:Google APIs:16");
for (String filter : cpuFilters) {
builder.addCpuFilter(filter);
}
ImmutableList.Builder<Step> commands = ImmutableList.builder();
AndroidBinaryRule buildRule = ruleResolver.buildAndAddToIndex(builder);
buildRule.copyNativeLibrary(sourceDir, destinationDir, commands);
ImmutableList<Step> steps = commands.build();
assertEquals(steps.size(), expectedShellCommands.size());
ExecutionContext context = createMock(ExecutionContext.class);
replay(context);
for (int i = 0; i < steps.size(); ++i) {
Iterable<String> observedArgs = ((BashStep) steps.get(i)).getShellCommand(context);
String observedCommand = Joiner.on(' ').join(observedArgs);
assertEquals(expectedShellCommands.get(i), observedCommand);
}
verify(context);
}
NominalType instantiateGenerics(List<JSType> types) {
ImmutableList<String> typeParams = this.rawType.getTypeParameters();
Preconditions.checkState(types.size() == typeParams.size());
Map<String, JSType> typeMap = new LinkedHashMap<>();
for (int i = 0; i < typeParams.size(); i++) {
typeMap.put(typeParams.get(i), types.get(i));
}
return instantiateGenerics(typeMap);
}
@Override
protected void masterOperation(
final SnapshotsStatusRequest request,
final ClusterState state,
final ActionListener<SnapshotsStatusResponse> listener)
throws Exception {
ImmutableList<SnapshotMetaData.Entry> currentSnapshots =
snapshotsService.currentSnapshots(request.repository(), request.snapshots());
if (currentSnapshots.isEmpty()) {
listener.onResponse(buildResponse(request, currentSnapshots, null));
return;
}
Set<String> nodesIds = newHashSet();
for (SnapshotMetaData.Entry entry : currentSnapshots) {
for (SnapshotMetaData.ShardSnapshotStatus status : entry.shards().values()) {
if (status.nodeId() != null) {
nodesIds.add(status.nodeId());
}
}
}
if (!nodesIds.isEmpty()) {
// There are still some snapshots running - check their progress
SnapshotId[] snapshotIds = new SnapshotId[currentSnapshots.size()];
for (int i = 0; i < currentSnapshots.size(); i++) {
snapshotIds[i] = currentSnapshots.get(i).snapshotId();
}
transportNodesSnapshotsStatus.status(
nodesIds.toArray(new String[nodesIds.size()]),
snapshotIds,
request.masterNodeTimeout(),
new ActionListener<TransportNodesSnapshotsStatus.NodesSnapshotStatus>() {
@Override
public void onResponse(
TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses) {
try {
ImmutableList<SnapshotMetaData.Entry> currentSnapshots =
snapshotsService.currentSnapshots(request.repository(), request.snapshots());
listener.onResponse(buildResponse(request, currentSnapshots, nodeSnapshotStatuses));
} catch (Throwable e) {
listener.onFailure(e);
}
}
@Override
public void onFailure(Throwable e) {
listener.onFailure(e);
}
});
} else {
// We don't have any in-progress shards, just return current stats
listener.onResponse(buildResponse(request, currentSnapshots, null));
}
}
/**
* @param index Index of submission to retrieve
* @return Submission for the given row in the array
*/
public Submission getXSubmission(int index) {
checkArgument(index >= 0, "Index into X submissions must be greater than 0!");
checkArgument(
index < xSubmissions.size(),
"Index into X submissions must be less than X submissions size ("
+ xSubmissions.size()
+ ")!");
return xSubmissions.get(index);
}
/**
* Returns an iterator over active and initializing shards. Making sure though that its random
* within the active shards, and initializing shards are the last to iterate through.
*/
public ShardIterator activeInitializingShardsIt(int seed) {
if (allInitializingShards.isEmpty()) {
return new PlainShardIterator(shardId, shuffler.shuffle(activeShards, seed));
}
ArrayList<ShardRouting> ordered =
new ArrayList<>(activeShards.size() + allInitializingShards.size());
ordered.addAll(shuffler.shuffle(activeShards, seed));
ordered.addAll(allInitializingShards);
return new PlainShardIterator(shardId, ordered);
}
/**
* @param index Index of submission to retrieve
* @return Submission for the given column in the array
*/
public Submission getYSubmission(int index) {
checkArgument(index >= 0, "Index into Y submissions must be greater than 0!");
checkArgument(
index < ySubmissions.size(),
"Index into Y submissions must be less than Y submissions size ("
+ ySubmissions.size()
+ ")!");
return ySubmissions.get(index);
}
public static String removeParent(final String parentPath, final String childPath) {
checkArgument(isChild(parentPath, childPath));
ImmutableList<String> parent = split(parentPath);
ImmutableList<String> child = split(childPath);
child = child.subList(parent.size(), child.size());
String strippedChildPath = child.get(0);
for (int i = 1; i < child.size(); i++) {
appendChild(strippedChildPath, child.get(i));
}
return strippedChildPath;
}
private String toString(final ImmutableList<String> elements) {
final StringBuilder s = new StringBuilder();
for (int i = 0, size = elements.size(); i < size; i++) {
s.append(elements.get(i));
if (i < elements.size() - 1) {
s.append(ELEMENT_DIVIDER);
}
}
return s.toString();
}
public void run() throws InterruptedException {
Row row;
while ((row = source.receive()) != null) {
context.values = row.getValues();
Object[] values = new Object[projects.size()];
for (int i = 0; i < projects.size(); i++) {
Scalar scalar = projects.get(i);
values[i] = scalar.execute(context);
}
sink.send(new Row(values));
}
}
/**
* Get similarities for one submission compared to another.
*
* @param xIndex Index into similarity matrix on the X axis
* @param yIndex Index into similarity matrix on the Y axis
* @return Matrix Entry for given X and Y index
*/
public AlgorithmResults getEntryFor(int xIndex, int yIndex) {
checkArgument(xIndex >= 0, "X index must be greater than 0!");
checkArgument(
xIndex < xSubmissions.size(),
"X index must be less than X submissions size (" + xSubmissions.size() + ")!");
checkArgument(yIndex >= 0, "Y index must be greater than 0!");
checkArgument(
yIndex < ySubmissions.size(),
"Y index must be less than Y submissions size (" + ySubmissions.size() + ")!");
return entries[xIndex][yIndex];
}
protected void assertTestCurrencies(
ImmutableList<Integer> is, List<BadgerEntity<Currency>> currencies) {
assertEquals(is.size(), currencies.size());
ImmutableMap.Builder<Integer, BadgerEntity<Currency>> builder = ImmutableMap.builder();
currencies.forEach(currency -> builder.put(currency.getMessage().getNumericCode(), currency));
ImmutableMap<Integer, BadgerEntity<Currency>> map = builder.build();
Set<Integer> iSet = new HashSet<>(is);
assertEquals(is.size(), iSet.size());
assertEquals(iSet, map.keySet());
map.entrySet().forEach(entry -> assertTestCurrency(entry.getKey(), entry.getValue()));
}
@Test
public final void testFromRunInfos() {
final SignedResult result1 = makeSignedResult(40, 120);
final SignedResult result2 = makeSignedResult(100, 100);
final HistoricalCalculation calc3 = makeHistoricalCalc(70);
final ImmutableList<HistoricalRunInfo> runInfos =
ImmutableList.of(
HistoricalRunInfo.signed(result1),
HistoricalRunInfo.signed(result2),
HistoricalRunInfo.unsigned(calc3));
final int expectedTotal = runInfos.size();
final int expectedSigned = 2;
final int expectedFirstRunAvg = 70;
final int expectedSignAvg = 110;
final UtilizationSummary actualSummary = UtilizationSummary.fromRunInfos(runInfos);
/* Behavior & Verification */
assertEquals(expectedTotal, actualSummary.getTotalCount());
assertEquals(expectedSigned, actualSummary.getSignedCount());
assertEquals(expectedFirstRunAvg, actualSummary.getSecondsToFirstRunAverage());
assertEquals(expectedSignAvg, actualSummary.getSecondsToSignAverage());
// Also verify toString() contract.
assertThat(
actualSummary.toString(),
allOf(
containsString(Integer.toString(expectedTotal)),
containsString(Integer.toString(expectedSigned)),
containsString(Integer.toString(expectedFirstRunAvg)),
containsString(Integer.toString(expectedSignAvg))));
}
@Override
public void visit(NodeTraversal t, Node n, Node parent) {
for (int i = 0, len = rules.size(); i < len; i++) {
Rule rule = rules.get(i);
rule.check(t, n);
}
}
@Test
public void ensureFilesInSubdirectoriesAreKeptInSubDirectories() throws IOException {
BuildRuleResolver ruleResolver = new BuildRuleResolver();
BuildTarget target = BuildTargetFactory.newInstance("//:example");
Genrule rule =
ruleResolver.buildAndAddToIndex(
Genrule.newGenruleBuilder(new FakeAbstractBuildRuleBuilderParams())
.setRelativeToAbsolutePathFunctionForTesting(relativeToAbsolutePathFunction)
.setBuildTarget(target)
.setBash(Optional.of("ignored"))
.addSrc("in-dir.txt")
.addSrc("foo/bar.html")
.addSrc("other/place.txt")
.setOut("example-file"));
ImmutableList.Builder<Step> builder = ImmutableList.builder();
rule.addSymlinkCommands(builder);
ImmutableList<Step> commands = builder.build();
String baseTmpPath = GEN_DIR + "/example__srcs/";
assertEquals(3, commands.size());
MkdirAndSymlinkFileStep linkCmd = (MkdirAndSymlinkFileStep) commands.get(0);
assertEquals("in-dir.txt", linkCmd.getSource());
assertEquals(baseTmpPath + "in-dir.txt", linkCmd.getTarget());
linkCmd = (MkdirAndSymlinkFileStep) commands.get(1);
assertEquals("foo/bar.html", linkCmd.getSource());
assertEquals(baseTmpPath + "foo/bar.html", linkCmd.getTarget());
linkCmd = (MkdirAndSymlinkFileStep) commands.get(2);
assertEquals("other/place.txt", linkCmd.getSource());
assertEquals(baseTmpPath + "other/place.txt", linkCmd.getTarget());
}
public Map<String, Path> buildMultipleAndReturnOutputs(String... args) throws IOException {
// Add in `--show-output` to the build, so we can parse the output paths after the fact.
ImmutableList<String> buildArgs =
ImmutableList.<String>builder().add("--show-output").add(args).build();
ProjectWorkspace.ProcessResult buildResult =
runBuckBuild(buildArgs.toArray(new String[buildArgs.size()]));
buildResult.assertSuccess();
// Grab the stdout lines, which have the build outputs.
List<String> lines =
Splitter.on(CharMatcher.anyOf(System.lineSeparator()))
.trimResults()
.omitEmptyStrings()
.splitToList(buildResult.getStdout());
// Skip the first line, which is just "The outputs are:".
assertThat(lines.get(0), Matchers.equalTo("The outputs are:"));
lines = lines.subList(1, lines.size());
Splitter lineSplitter = Splitter.on(' ').trimResults();
ImmutableMap.Builder<String, Path> builder = ImmutableMap.builder();
for (String line : lines) {
List<String> fields = lineSplitter.splitToList(line);
assertThat(fields, Matchers.hasSize(2));
builder.put(fields.get(0), getPath(fields.get(1)));
}
return builder.build();
}
@Override
public Collection<StoreFile> getUnneededFiles(long maxTs, List<StoreFile> filesCompacting) {
Collection<StoreFile> expiredStoreFiles = null;
ImmutableList<StoreFile> files = storefiles;
// 1) We can never get rid of the last file which has the maximum seqid.
// 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
for (int i = 0; i < files.size() - 1; ++i) {
StoreFile sf = files.get(i);
long fileTs = sf.getReader().getMaxTimestamp();
if (fileTs < maxTs && !filesCompacting.contains(sf)) {
LOG.info(
"Found an expired store file: "
+ sf.getPath()
+ " whose maxTimeStamp is "
+ fileTs
+ ", which is below "
+ maxTs);
if (expiredStoreFiles == null) {
expiredStoreFiles = new ArrayList<StoreFile>();
}
expiredStoreFiles.add(sf);
}
}
return expiredStoreFiles;
}
@Test(dataProvider = "invalidSlaveCount")
public void testInvalidSlaveCount(int slaveCount, String expectedViolations) {
operation.setNewSlaveCount(slaveCount);
ImmutableList<String> violations = validator.validate(operation);
assertThat(violations.size(), is(1));
assertThat(violations.get(0), is(expectedViolations));
}
public ArtifactCache createArtifactCache(
Optional<String> currentWifiSsid, BuckEventBus buckEventBus) {
ImmutableList<String> modes = getArtifactCacheModes();
if (modes.isEmpty()) {
return new NoopArtifactCache();
}
ImmutableList.Builder<ArtifactCache> builder = ImmutableList.builder();
boolean useDistributedCache = isWifiUsableForDistributedCache(currentWifiSsid);
try {
for (String mode : modes) {
switch (ArtifactCacheNames.valueOf(mode)) {
case dir:
ArtifactCache dirArtifactCache = createDirArtifactCache();
buckEventBus.register(dirArtifactCache);
builder.add(dirArtifactCache);
break;
case http:
if (useDistributedCache) {
ArtifactCache httpArtifactCache = createHttpArtifactCache(buckEventBus);
builder.add(httpArtifactCache);
}
break;
}
}
} catch (IllegalArgumentException e) {
throw new HumanReadableException("Unusable cache.mode: '%s'", modes.toString());
}
ImmutableList<ArtifactCache> artifactCaches = builder.build();
if (artifactCaches.size() == 1) {
// Don't bother wrapping a single artifact cache in MultiArtifactCache.
return artifactCaches.get(0);
} else {
return new MultiArtifactCache(artifactCaches);
}
}
@Override
public <T> Set<T> eval(
QueryEnvironment<T> env, ImmutableList<Argument> args, ListeningExecutorService executor)
throws QueryException, InterruptedException {
QueryExpression argument = args.get(args.size() - 1).getExpression();
String attr = CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, args.get(0).getWord());
final String attrValue = args.get(1).getWord();
final Predicate<Object> predicate =
new Predicate<Object>() {
@Override
public boolean apply(Object input) {
return attrValue.equals(input.toString());
}
};
Set<T> result = new LinkedHashSet<>();
for (T target : argument.eval(env, executor)) {
ImmutableSet<Object> matchingObjects = env.filterAttributeContents(target, attr, predicate);
if (!matchingObjects.isEmpty()) {
result.add(target);
}
}
return result;
}
@Value.Lazy
public List<TypeElement> includedTypes() {
Optional<IncludeMirror> includes = include();
ImmutableList<TypeMirror> typeMirrors =
includes.isPresent()
? ImmutableList.copyOf(includes.get().valueMirror())
: ImmutableList.<TypeMirror>of();
FluentIterable<TypeElement> typeElements =
FluentIterable.from(typeMirrors)
.filter(DeclaredType.class)
.transform(DeclatedTypeToElement.FUNCTION);
ImmutableSet<String> uniqueTypeNames =
typeElements.filter(IsPublic.PREDICATE).transform(ElementToName.FUNCTION).toSet();
if (uniqueTypeNames.size() != typeMirrors.size()) {
report()
.annotationNamed(IncludeMirror.simpleName())
.warning(
"Some types were ignored, non-supported for inclusion: duplicates, non declared reference types, non-public");
}
return typeElements.toList();
}
@Test
public void blobApiBean() throws IOException {
final Ds3SpecParser parser = new Ds3SpecParserImpl();
final Ds3ApiSpec spec =
parser.getSpec(Ds3SpecParserImpl_Test.class.getResourceAsStream("/specs/blobApiBean.xml"));
assertThat(spec.getTypes().size(), is(1));
final Ds3Type blobApiBean =
spec.getTypes().get("com.spectralogic.s3.common.platform.domain.BulkObject");
assertThat(blobApiBean, is(notNullValue()));
final ImmutableList<Ds3Element> elements = blobApiBean.getElements();
assertThat(elements.size(), is(7));
assertThat(elements.get(0).getName(), is("InCache"));
assertThat(elements.get(0).getNullable(), is(true));
assertThat(elements.get(1).getName(), is("Latest"));
assertThat(elements.get(1).getNullable(), is(false));
assertThat(elements.get(2).getName(), is("Length"));
assertThat(elements.get(2).getNullable(), is(false));
assertThat(elements.get(3).getName(), is("Name"));
assertThat(elements.get(3).getNullable(), is(true));
assertThat(elements.get(4).getName(), is("Offset"));
assertThat(elements.get(4).getNullable(), is(false));
assertThat(elements.get(5).getName(), is("PhysicalPlacement"));
assertThat(elements.get(5).getNullable(), is(true));
assertThat(elements.get(6).getName(), is("Version"));
assertThat(elements.get(6).getNullable(), is(false));
}
private ResponseHandler getResponseHandler(ImmutableList<ResponseHandler> list) {
if (list.size() == 1) {
return list.get(0);
}
return new AndResponseHandler(list);
}