public Builder setRuleKeys(Iterable<RuleKey> ruleKeys) {
// Make sure we expand any lazy evaluation Iterators so Json serialization works correctly.
List<String> keysAsStrings =
FluentIterable.from(ruleKeys).transform(Functions.toStringFunction()).toList();
data.put("rule_keys", keysAsStrings);
return this;
}
/**
* Internal ctor, called by {@link MixinConfig}
*
* @param parent
* @param mixinName
* @param runTransformers
* @param plugin
* @param suppressPlugin
* @throws ClassNotFoundException
*/
MixinInfo(
MixinConfig parent,
String mixinName,
boolean runTransformers,
IMixinConfigPlugin plugin,
boolean suppressPlugin)
throws ClassNotFoundException {
this.parent = parent;
this.name = mixinName;
this.className = parent.getMixinPackage() + mixinName;
this.plugin = plugin;
this.phase = MixinEnvironment.getCurrentEnvironment().getPhase();
// Read the class bytes and transform
this.mixinBytes = this.loadMixinClass(this.className, runTransformers);
ClassNode classNode = this.getClassNode(0);
this.priority = this.readPriority(classNode);
this.targetClasses = this.readTargetClasses(classNode, suppressPlugin);
this.targetClassNames =
Collections.unmodifiableList(
Lists.transform(this.targetClasses, Functions.toStringFunction()));
this.validationClassNode = classNode;
this.classInfo = ClassInfo.fromClassNode(classNode);
}
@Override
public String toString() {
return function.getName()
+ "("
+ Joiner.on(", ").join(Iterables.transform(args, Functions.toStringFunction()))
+ ")";
}
@Override
public void addOptionsToAlgorithmParameters(
DropdownMutator mutator, List<String> numericColumnNames) {
List<String> options = Lists.newArrayList(numericColumnNames);
options.add(getColumnNameParameterDisablingToken());
mutator.add(getColumnNameParameterId(), options, getColumnNameParameterDisablingToken());
mutator.add(
getScalingParameterId(),
Collections2.transform(EnumSet.allOf(Scaling.class), Functions.toStringFunction()));
}
public static String getDefaultSharedLibrarySoname(BuildTarget target, CxxPlatform platform) {
String libName =
Joiner.on('_')
.join(
ImmutableList.builder()
.addAll(
FluentIterable.from(target.getBasePath())
.transform(Functions.toStringFunction())
.filter(Predicates.not(Predicates.equalTo(""))))
.add(
target
.withoutFlavors(ImmutableSet.of(platform.getFlavor()))
.getShortNameAndFlavorPostfix())
.build());
String extension = platform.getSharedLibraryExtension();
return String.format("lib%s.%s", libName, extension);
}
@Test
public void testImplicitDepsAreAddedCorrectly() throws NoSuchBuildTargetException {
Description<GenruleDescription.Arg> genruleDescription = new GenruleDescription();
Map<String, Object> instance =
ImmutableMap.<String, Object>of(
"srcs", ImmutableList.of(":baz", "//biz:baz"),
"out", "AndroidManifest.xml",
"cmd", "$(exe //bin:executable) $(location :arg)");
ProjectFilesystem projectFilesystem = new AllExistingProjectFilesystem();
BuildRuleFactoryParams params =
new BuildRuleFactoryParams(projectFilesystem, BuildTargetFactory.newInstance("//foo:bar"));
ConstructorArgMarshaller marshaller =
new ConstructorArgMarshaller(
new DefaultTypeCoercerFactory(ObjectMappers.newDefaultInstance()));
ImmutableSet.Builder<BuildTarget> declaredDeps = ImmutableSet.builder();
ImmutableSet.Builder<VisibilityPattern> visibilityPatterns = ImmutableSet.builder();
GenruleDescription.Arg constructorArg = genruleDescription.createUnpopulatedConstructorArg();
try {
marshaller.populate(
createCellRoots(projectFilesystem),
projectFilesystem,
params,
constructorArg,
declaredDeps,
visibilityPatterns,
instance);
} catch (ConstructorArgMarshalException e) {
fail("Expected constructorArg to be correctly populated.");
}
TargetNode<GenruleDescription.Arg> targetNode =
new TargetNodeFactory(new DefaultTypeCoercerFactory(ObjectMappers.newDefaultInstance()))
.create(
Hashing.sha1().hashString(params.target.getFullyQualifiedName(), UTF_8),
genruleDescription,
constructorArg,
params,
declaredDeps.build(),
visibilityPatterns.build(),
createCellRoots(projectFilesystem));
assertEquals(
"SourcePaths and targets from cmd string should be extracted as extra deps.",
ImmutableSet.of("//foo:baz", "//biz:baz", "//bin:executable", "//foo:arg"),
FluentIterable.from(targetNode.getExtraDeps())
.transform(Functions.toStringFunction())
.toSet());
}
private BuildRule generateBytecodeLinking(ImmutableList<SourcePath> allInputs) {
BuildRuleParams linkParams =
params.copyWithChanges(
addBytecodeFlavor(params.getBuildTarget()),
Suppliers.ofInstance(
ImmutableSortedSet.<BuildRule>naturalOrder()
.addAll(pathResolver.filterBuildRuleInputs(allInputs))
.addAll(ocamlContext.getBytecodeLinkDeps())
.addAll(
FluentIterable.from(ocamlContext.getLinkableInput().getArgs())
.append(ocamlContext.getNativeLinkableInput().getArgs())
.transformAndConcat(Arg.getDepsFunction(pathResolver))
.filter(Predicates.not(Predicates.instanceOf(OCamlBuild.class))))
.build()),
Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of()));
ImmutableList<String> linkerInputs =
FluentIterable.from(allInputs)
.transform(pathResolver.deprecatedPathFunction())
.transform(Functions.toStringFunction())
.toList();
ImmutableList.Builder<String> flags = ImmutableList.builder();
flags.addAll(ocamlContext.getFlags());
flags.addAll(ocamlContext.getCommonCLinkerFlags());
OCamlLink link =
new OCamlLink(
linkParams,
pathResolver,
allInputs,
new OCamlLinkStep.Args(
cxxCompiler.getEnvironment(pathResolver),
cxxCompiler.getCommandPrefix(pathResolver),
ocamlContext.getOcamlBytecodeCompiler().get(),
ocamlContext.getBytecodeOutput(),
ocamlContext.getLinkableInput().getArgs(),
ocamlContext.getNativeLinkableInput().getArgs(),
linkerInputs,
flags.build(),
ocamlContext.isLibrary(),
/* isBytecode */ true));
resolver.addToIndex(link);
return link;
}
/** @return the framework search paths with any embedded macros expanded. */
static ImmutableSet<Path> getFrameworkSearchPaths(
Optional<ImmutableSortedSet<FrameworkPath>> frameworks,
CxxPlatform cxxPlatform,
SourcePathResolver resolver) {
ImmutableSet<Path> searchPaths =
FluentIterable.from(frameworks.get())
.transform(
FrameworkPath.getUnexpandedSearchPathFunction(
resolver.getPathFunction(), Functions.<Path>identity()))
.toSet();
return FluentIterable.from(Optional.of(searchPaths).or(ImmutableSet.<Path>of()))
.transform(Functions.toStringFunction())
.transform(CxxFlags.getTranslateMacrosFn(cxxPlatform))
.transform(MorePaths.TO_PATH)
.toSet();
}
@Override
protected String expand(SourcePathResolver resolver, ProjectFilesystem filesystem, BuildRule rule)
throws MacroException {
return Joiner.on(File.pathSeparator)
.join(
FluentIterable.from(getHasClasspathEntries(rule).getTransitiveClasspathDeps())
.transform(
new Function<JavaLibrary, Path>() {
@Nullable
@Override
public Path apply(JavaLibrary input) {
return input.getPathToOutput();
}
})
.filter(Predicates.notNull())
.transform(filesystem.getAbsolutifier())
.transform(Functions.toStringFunction())
.toSortedSet(Ordering.natural()));
}
@TimedResource
@GET
@Path("/permissions")
@Produces(APPLICATION_JSON)
@ApiOperation(
value = "List user permissions",
response = String.class,
responseContainer = "List")
@ApiResponses(value = {})
public Response getCurrentUserPermissions(
@javax.ws.rs.core.Context final HttpServletRequest request) {
// The getCurrentUserPermissions takes a TenantContext which is not used because permissions are
// cross tenants (at this point)
final TenantContext nullTenantContext = null;
final Set<Permission> permissions = securityApi.getCurrentUserPermissions(nullTenantContext);
final List<String> json =
ImmutableList.<String>copyOf(
Iterables.<Permission, String>transform(permissions, Functions.toStringFunction()));
return Response.status(Status.OK).entity(json).build();
}
public static enum Relationship {
ABOUT("about"),
TWITTER_AUDIENCE("twitter:audience"),
TWITTER_AUDIENCE_RELATED("twitter:audience-related"),
TWITTER_AUDIENCE_REALTIME("twitter:audience:realtime"),
TRANSCRIPTION("transcription"),
TRANSCRIPTION_SUBTITLES("transcription:subtitles"),
TRANSCRIPTION_SUBTITLES_REALTIME("transcription:subtitles:realtime");
private final String name;
private Relationship(String name) {
this.name = name;
}
@Override
public String toString() {
return name;
}
private static ImmutableSet<Relationship> ALL = ImmutableSet.copyOf(values());
public static ImmutableSet<Relationship> all() {
return ALL;
}
private static ImmutableMap<String, Optional<Relationship>> LOOKUP =
ImmutableMap.copyOf(
Maps.transformValues(
Maps.uniqueIndex(all(), Functions.toStringFunction()),
new Function<Relationship, Optional<Relationship>>() {
@Override
public Optional<Relationship> apply(@Nullable Relationship input) {
return Optional.fromNullable(input);
}
}));
public static Optional<Relationship> fromString(String relationship) {
Optional<Relationship> possibleRelationship = LOOKUP.get(relationship);
return possibleRelationship != null ? possibleRelationship : Optional.<Relationship>absent();
}
}
@Test
public void testImplicitDepsAreAddedCorrectly()
throws NoSuchBuildTargetException, TargetNode.InvalidSourcePathInputException {
Description<GenruleDescription.Arg> genruleDescription = new GenruleDescription();
Map<String, Object> instance =
ImmutableMap.<String, Object>of(
"srcs", ImmutableList.of(":baz", "//biz:baz"),
"out", "AndroidManifest.xml",
"cmd", "$(exe //bin:executable) $(location :arg)");
ProjectFilesystem projectFilesystem = new AllExistingProjectFilesystem();
BuildRuleFactoryParams params =
new BuildRuleFactoryParams(
projectFilesystem,
BuildTargetFactory.newInstance("//foo:bar"),
new InMemoryBuildFileTree(ImmutableList.<BuildTarget>of()),
/* enforeBuckBoundaryCheck */ true);
ConstructorArgMarshaller marshaller = new ConstructorArgMarshaller();
ImmutableSet.Builder<BuildTarget> declaredDeps = ImmutableSet.builder();
ImmutableSet.Builder<BuildTargetPattern> visibilityPatterns = ImmutableSet.builder();
GenruleDescription.Arg constructorArg = genruleDescription.createUnpopulatedConstructorArg();
try {
marshaller.populate(
projectFilesystem, params, constructorArg, declaredDeps, visibilityPatterns, instance);
} catch (ConstructorArgMarshalException e) {
fail("Expected constructorArg to be correctly populated.");
}
TargetNode<GenruleDescription.Arg> targetNode =
new TargetNode<>(
genruleDescription,
constructorArg,
params,
declaredDeps.build(),
visibilityPatterns.build());
assertEquals(
"SourcePaths and targets from cmd string should be extracted as extra deps.",
ImmutableSet.of("//foo:baz", "//biz:baz", "//bin:executable", "//foo:arg"),
FluentIterable.from(targetNode.getExtraDeps())
.transform(Functions.toStringFunction())
.toSet());
}
boolean verifyLiveNodes(List<ParticipantId> actualLiveNodes) {
Collections.sort(actualLiveNodes);
List<String> rawActualLiveNodes =
Lists.transform(actualLiveNodes, Functions.toStringFunction());
return _expectSortedLiveNodes.equals(rawActualLiveNodes);
}
@Test
public void testBuildJson() throws IOException {
ProjectFilesystem projectFilesystem = new ProjectFilesystem(tmpDir.getRoot().toPath());
ObjectMapper mapper = new ObjectMapper();
ChromeTraceBuildListener listener =
new ChromeTraceBuildListener(
projectFilesystem,
new FakeClock(1409702151000000000L),
mapper,
Locale.US,
TimeZone.getTimeZone("America/Los_Angeles"),
/* tracesToKeep */ 42,
false);
BuildTarget target = BuildTargetFactory.newInstance("//fake:rule");
FakeBuildRule rule =
new FakeBuildRule(
target,
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSortedSet.<BuildRule>of());
RuleKey ruleKey = new RuleKey("abc123");
rule.setRuleKey(ruleKey);
String stepShortName = "fakeStep";
String stepDescription = "I'm a Fake Step!";
UUID stepUuid = UUID.randomUUID();
ExecutionContext context = createMock(ExecutionContext.class);
replay(context);
ImmutableSet<BuildTarget> buildTargets = ImmutableSet.of(target);
Iterable<String> buildArgs = Iterables.transform(buildTargets, Functions.toStringFunction());
Clock fakeClock = new IncrementingFakeClock(TimeUnit.MILLISECONDS.toNanos(1));
BuckEventBus eventBus =
BuckEventBusFactory.newInstance(
fakeClock, new BuildId("ChromeTraceBuildListenerTestBuildId"));
eventBus.register(listener);
eventBus.post(
CommandEvent.started("party", ImmutableList.of("arg1", "arg2"), /* isDaemon */ true));
eventBus.post(ArtifactCacheConnectEvent.started());
eventBus.post(ArtifactCacheConnectEvent.finished());
eventBus.post(BuildEvent.started(buildArgs));
eventBus.post(
ArtifactCacheEvent.started(ArtifactCacheEvent.Operation.FETCH, ImmutableSet.of(ruleKey)));
eventBus.post(
ArtifactCacheEvent.finished(
ArtifactCacheEvent.Operation.FETCH, ImmutableSet.of(ruleKey), CacheResult.hit("http")));
eventBus.post(BuildRuleEvent.started(rule));
eventBus.post(StepEvent.started(stepShortName, stepDescription, stepUuid));
eventBus.post(StepEvent.finished(stepShortName, stepDescription, stepUuid, 0));
eventBus.post(
BuildRuleEvent.finished(
rule,
BuildRuleStatus.SUCCESS,
CacheResult.miss(),
Optional.of(BuildRuleSuccessType.BUILT_LOCALLY),
Optional.<HashCode>absent(),
Optional.<Long>absent()));
try (TraceEventLogger ignored =
TraceEventLogger.start(eventBus, "planning", ImmutableMap.of("nefarious", "true"))) {
eventBus.post(new TraceEvent("scheming", ChromeTraceEvent.Phase.BEGIN));
eventBus.post(
new TraceEvent(
"scheming", ChromeTraceEvent.Phase.END, ImmutableMap.of("success", "false")));
}
eventBus.post(BuildEvent.finished(buildArgs, 0));
eventBus.post(
CommandEvent.finished(
"party", ImmutableList.of("arg1", "arg2"), /* isDaemon */ true, /* exitCode */ 0));
listener.outputTrace(new BuildId("BUILD_ID"));
File resultFile = new File(tmpDir.getRoot(), BuckConstant.BUCK_TRACE_DIR + "/build.trace");
List<ChromeTraceEvent> resultMap =
mapper.readValue(resultFile, new TypeReference<List<ChromeTraceEvent>>() {});
assertEquals(17, resultMap.size());
assertEquals("process_name", resultMap.get(0).getName());
assertEquals(ChromeTraceEvent.Phase.METADATA, resultMap.get(0).getPhase());
assertEquals(ImmutableMap.of("name", "buck"), resultMap.get(0).getArgs());
assertEquals("party", resultMap.get(1).getName());
assertEquals(ChromeTraceEvent.Phase.BEGIN, resultMap.get(1).getPhase());
assertEquals(ImmutableMap.of("command_args", "arg1 arg2"), resultMap.get(1).getArgs());
assertEquals("artifact_connect", resultMap.get(2).getName());
assertEquals(ChromeTraceEvent.Phase.BEGIN, resultMap.get(2).getPhase());
assertEquals("artifact_connect", resultMap.get(3).getName());
assertEquals(ChromeTraceEvent.Phase.END, resultMap.get(3).getPhase());
assertEquals("build", resultMap.get(4).getName());
assertEquals(ChromeTraceEvent.Phase.BEGIN, resultMap.get(4).getPhase());
assertEquals("artifact_fetch", resultMap.get(5).getName());
assertEquals(ChromeTraceEvent.Phase.BEGIN, resultMap.get(5).getPhase());
assertEquals("artifact_fetch", resultMap.get(6).getName());
assertEquals(ChromeTraceEvent.Phase.END, resultMap.get(6).getPhase());
// BuildRuleEvent.Started
assertEquals("//fake:rule", resultMap.get(7).getName());
assertEquals(ChromeTraceEvent.Phase.BEGIN, resultMap.get(7).getPhase());
assertEquals(ImmutableMap.of("rule_key", "abc123"), resultMap.get(7).getArgs());
assertEquals("fakeStep", resultMap.get(8).getName());
assertEquals(ChromeTraceEvent.Phase.BEGIN, resultMap.get(8).getPhase());
assertEquals("fakeStep", resultMap.get(9).getName());
assertEquals(
ImmutableMap.of(
"description", "I'm a Fake Step!",
"exit_code", "0"),
resultMap.get(9).getArgs());
assertEquals(ChromeTraceEvent.Phase.END, resultMap.get(9).getPhase());
// BuildRuleEvent.Finished
assertEquals("//fake:rule", resultMap.get(10).getName());
assertEquals(ChromeTraceEvent.Phase.END, resultMap.get(10).getPhase());
assertEquals(
ImmutableMap.of(
"cache_result", "miss",
"success_type", "BUILT_LOCALLY"),
resultMap.get(10).getArgs());
assertEquals("planning", resultMap.get(11).getName());
assertEquals(ChromeTraceEvent.Phase.BEGIN, resultMap.get(11).getPhase());
assertEquals(ImmutableMap.of("nefarious", "true"), resultMap.get(11).getArgs());
assertEquals("scheming", resultMap.get(12).getName());
assertEquals(ChromeTraceEvent.Phase.BEGIN, resultMap.get(12).getPhase());
assertEquals(ImmutableMap.of(), resultMap.get(12).getArgs());
assertEquals("scheming", resultMap.get(13).getName());
assertEquals(ChromeTraceEvent.Phase.END, resultMap.get(13).getPhase());
assertEquals(ImmutableMap.of("success", "false"), resultMap.get(13).getArgs());
assertEquals("planning", resultMap.get(14).getName());
assertEquals(ChromeTraceEvent.Phase.END, resultMap.get(14).getPhase());
assertEquals(ImmutableMap.of(), resultMap.get(14).getArgs());
assertEquals("build", resultMap.get(15).getName());
assertEquals(ChromeTraceEvent.Phase.END, resultMap.get(15).getPhase());
assertEquals("party", resultMap.get(16).getName());
assertEquals(ChromeTraceEvent.Phase.END, resultMap.get(16).getPhase());
assertEquals(
ImmutableMap.of(
"command_args", "arg1 arg2",
"daemon", "true"),
resultMap.get(16).getArgs());
verify(context);
}
/**
* If the value is a string, it will be quoted, as that's how json strings are represented.
*
* @return key to a json literal of the value
* @see #getRequirements
* @see Json#fromJson
*/
public Map<String, String> getRequirementsAsJsonLiterals() {
return Maps.transformValues(requirements, Functions.toStringFunction());
}
/**
* Contains a grouping of various minimum requirements for provisioning a machine with this
* dataset. For example 'password' indicates that a password must be provided.
*
* <h4>Note</h4>
*
* requirements can contain arbitrarily complex values. If the value has structure, you should use
* {@link #getRequirementsAsJsonLiterals}
*/
public Map<String, String> getRequirements() {
return Maps.transformValues(
requirements, Functions.compose(Functions.toStringFunction(), unquoteString));
}
private TreeSet<String> sortedToStringsFrom(Iterable<?> iterable) {
return Sets.newTreeSet(FluentIterable.from(iterable).transform(Functions.toStringFunction()));
}
/**
* A ProductPartitionTree is a container for a root {@link ProductPartitionNode} that also handles
* applying changes made to the tree under the root.
*/
public class ProductPartitionTree {
/** A generator for temporary (negative) product partition IDs. */
private final Iterator<Long> idGenerator;
/** The ID of the {@link AdGroup} of this tree. */
private final long adGroupId;
private final BiddingStrategyConfiguration biddingStrategyConfig;
/** The root node of this tree. */
private final ProductPartitionNode root;
/**
* The <em>original</em> root node of this tree. This will be null if this tree's ad group
* originally contained no nodes, e.g., the ad group was created via the API. Otherwise, it will
* be a deep copy of the ad group's original root node.
*
* <p>This root will be used to detect changes made to the tree under {@code root}. See {@link
* #createMutateOperationPairs()}.
*/
private final ProductPartitionNode originalRoot;
private final Comparator<ProductDimension> dimensionComparator;
/** The page size to use when retrieving ad group criteria. */
private static final int PAGE_SIZE = 100;
/**
* Required fields for any {@link Selector} used to fetch {@link AdGroupCriterion} objects used by
* an instance of this class.
*/
public static final List<AdGroupCriterionField> REQUIRED_SELECTOR_FIELD_ENUMS =
ImmutableList.of(
AdGroupCriterionField.AdGroupId,
AdGroupCriterionField.Id,
AdGroupCriterionField.ParentCriterionId,
AdGroupCriterionField.PartitionType,
AdGroupCriterionField.CriteriaType,
AdGroupCriterionField.CaseValue,
AdGroupCriterionField.CpcBid,
AdGroupCriterionField.CpcBidSource,
AdGroupCriterionField.Status);
/**
* Required fields for any {@link Selector} used to fetch {@link AdGroupCriterion} objects used by
* an instance of this class.
*
* @deprecated Use the {@code REQUIRED_SELECTOR_FIELD_ENUMS} instead.
*/
@Deprecated
public static final List<String> REQUIRED_SELECTOR_FIELDS =
ImmutableList.copyOf(
Collections2.transform(REQUIRED_SELECTOR_FIELD_ENUMS, Functions.toStringFunction()));
/**
* Constructor that initializes the temp ID generator based on the ID of the root node.
*
* @param adGroupId the ID of the ad group
* @param biddingStrategyConfig the bidding strategy configuration of the ad group
* @param rootNode the root node of the tree
*/
private ProductPartitionTree(
long adGroupId,
BiddingStrategyConfiguration biddingStrategyConfig,
ProductPartitionNode rootNode) {
this.adGroupId = adGroupId;
this.biddingStrategyConfig =
Preconditions.checkNotNull(biddingStrategyConfig, "Null bidding strategy configuration");
this.root = Preconditions.checkNotNull(rootNode, "Null root node");
long startingTempId;
this.dimensionComparator = new ProductDimensionComparator();
if (this.root.getProductPartitionId() < 0L) {
// The root has a temporary ID, so all changes made to this tree should result in ADD
// operations.
originalRoot = null;
startingTempId = -1L;
} else {
// Set originalRoot to a deep copy of the root node.
originalRoot =
new ProductPartitionNode(
null, root.getDimension(), root.getProductPartitionId(), this.dimensionComparator);
long minimumId =
cloneChildrenToNewParent(
originalRoot, root.getChildren(), originalRoot.getProductPartitionId());
// The starting temp ID should be -1 if all nodes are non-temporary (have positive IDs),
// else start at one less than the lowest ID found in the tree.
startingTempId = minimumId >= 0L ? -1L : minimumId - 1L;
}
this.idGenerator =
new AbstractSequentialIterator<Long>(startingTempId) {
@Override
protected Long computeNext(Long previous) {
return Long.MIN_VALUE == previous.longValue() ? null : previous - 1;
}
};
}
/**
* Deeply clones each child in {@code children} and attaches it to {@code newParent}.
*
* @param newParent the new parent to which the cloned children will be added
* @param children the children to clone
* @param minimumId the minimum ID to compare to - may be null
* @return the minimum product partition ID found within the subtrees under {@code children}
*/
private static long cloneChildrenToNewParent(
ProductPartitionNode newParent,
Iterable<ProductPartitionNode> children,
final Long minimumId) {
long updatedMinimumId = minimumId == null ? Long.MAX_VALUE : minimumId;
for (ProductPartitionNode childNode : children) {
newParent.asSubdivision();
// Clone the child and add it to newParent's collection of children.
ProductPartitionNode newChild = newParent.addChild(childNode.getDimension());
newChild = ProductPartitionNode.copyProperties(childNode, newChild);
updatedMinimumId =
Math.min(
updatedMinimumId,
newChild.getProductPartitionId() == null
? Long.MAX_VALUE
: newChild.getProductPartitionId());
// Recursively clone the child's children.
updatedMinimumId =
cloneChildrenToNewParent(newChild, childNode.getChildren(), updatedMinimumId);
}
return updatedMinimumId;
}
/**
* Returns a new instance of this class by retrieving the product partitions of the specified ad
* group. All parameters are required.
*/
public static ProductPartitionTree createAdGroupTree(
AdWordsServices services, AdWordsSession session, Long adGroupId)
throws ApiException, RemoteException {
// Get the AdGroupCriterionService.
AdGroupCriterionServiceInterface criterionService =
services.get(session, AdGroupCriterionServiceInterface.class);
SelectorBuilder selectorBuilder =
new SelectorBuilder()
.fields(
REQUIRED_SELECTOR_FIELD_ENUMS.toArray(
new AdGroupCriterionField[REQUIRED_SELECTOR_FIELD_ENUMS.size()]))
.equals(AdGroupCriterionField.AdGroupId, adGroupId.toString())
.equals(AdGroupCriterionField.CriteriaType, "PRODUCT_PARTITION")
.in(
AdGroupCriterionField.Status,
UserStatus.ENABLED.getValue(),
UserStatus.PAUSED.getValue())
.limit(PAGE_SIZE);
AdGroupCriterionPage adGroupCriterionPage;
// A multimap from each product partition ID to its direct children.
ListMultimap<Long, AdGroupCriterion> parentIdMap = LinkedListMultimap.create();
int offset = 0;
do {
// Get the next page of results.
adGroupCriterionPage = criterionService.get(selectorBuilder.build());
if (adGroupCriterionPage != null && adGroupCriterionPage.getEntries() != null) {
for (AdGroupCriterion adGroupCriterion : adGroupCriterionPage.getEntries()) {
ProductPartition partition = (ProductPartition) adGroupCriterion.getCriterion();
parentIdMap.put(partition.getParentCriterionId(), adGroupCriterion);
}
offset += adGroupCriterionPage.getEntries().length;
selectorBuilder.increaseOffsetBy(PAGE_SIZE);
}
} while (offset < adGroupCriterionPage.getTotalNumEntries());
// Construct the ProductPartitionTree from the parentIdMap.
if (!parentIdMap.containsKey(null)) {
Preconditions.checkState(
parentIdMap.isEmpty(), "No root criterion found in the tree but the tree is not empty");
return createEmptyAdGroupTree(
adGroupId, getAdGroupBiddingStrategyConfiguration(services, session, adGroupId));
}
return createNonEmptyAdGroupTree(adGroupId, parentIdMap);
}
/**
* Returns a new instance of this class based on the collection of ad group criteria provided.
*
* <p>NOTE: If retrieving existing criteria for use with this method, you must include all of the
* fields in {@link #REQUIRED_SELECTOR_FIELDS} in your {@link Selector}.
*
* @param adGroupId the ID of the ad group
* @param biddingStrategyConfig the {@link BiddingStrategyConfiguration} for the ad group
* @param adGroupCriteria the non-null (but possibly empty) list of ad group criteria
* @throws NullPointerException if any argument is null, any element in {@code adGroupCriteria} is
* null, or any required field from {@link #REQUIRED_SELECTOR_FIELDS} is missing from an
* element in {@code adGroupCriteria}
* @throws IllegalArgumentException if {@code adGroupCriteria} does not include the root criterion
* of the product partition tree
*/
public static ProductPartitionTree createAdGroupTree(
Long adGroupId,
BiddingStrategyConfiguration biddingStrategyConfig,
List<AdGroupCriterion> adGroupCriteria) {
Preconditions.checkNotNull(adGroupId, "Null ad group ID");
Preconditions.checkNotNull(biddingStrategyConfig, "Null bidding strategy configuration");
Preconditions.checkNotNull(adGroupCriteria, "Null criteria list");
if (adGroupCriteria.isEmpty()) {
return createEmptyAdGroupTree(adGroupId, biddingStrategyConfig);
}
ListMultimap<Long, AdGroupCriterion> parentIdMap = LinkedListMultimap.create();
for (AdGroupCriterion adGroupCriterion : adGroupCriteria) {
Preconditions.checkNotNull(
adGroupCriterion.getCriterion(), "AdGroupCriterion has a null criterion");
if (adGroupCriterion instanceof BiddableAdGroupCriterion) {
BiddableAdGroupCriterion biddableCriterion = (BiddableAdGroupCriterion) adGroupCriterion;
Preconditions.checkNotNull(
biddableCriterion.getUserStatus(),
"User status is null for criterion ID %s",
biddableCriterion.getCriterion().getId());
if (UserStatus.REMOVED.equals(biddableCriterion.getUserStatus())) {
// Skip REMOVED criteria.
continue;
}
}
if (adGroupCriterion.getCriterion() instanceof ProductPartition) {
ProductPartition partition = (ProductPartition) adGroupCriterion.getCriterion();
parentIdMap.put(partition.getParentCriterionId(), adGroupCriterion);
}
}
return createNonEmptyAdGroupTree(adGroupId, parentIdMap);
}
/**
* Returns a new tree based on a non-empty collection of ad group criteria. All parameters
* required.
*
* @param adGroupId the ID of the ad group
* @param parentIdMap the multimap from parent product partition ID to child criteria
* @return a new ProductPartitionTree
*/
private static ProductPartitionTree createNonEmptyAdGroupTree(
Long adGroupId, ListMultimap<Long, AdGroupCriterion> parentIdMap) {
Preconditions.checkNotNull(adGroupId, "Null ad group ID");
Preconditions.checkArgument(
!parentIdMap.isEmpty(), "parentIdMap passed for ad group ID %s is empty", adGroupId);
Preconditions.checkArgument(
parentIdMap.containsKey(null),
"No root criterion found in the list of ad group criteria for ad group ID %s",
adGroupId);
AdGroupCriterion rootCriterion = Iterables.getOnlyElement(parentIdMap.get(null));
Preconditions.checkState(
rootCriterion instanceof BiddableAdGroupCriterion,
"Root criterion for ad group ID %s is not a BiddableAdGroupCriterion",
adGroupId);
BiddableAdGroupCriterion biddableRootCriterion = (BiddableAdGroupCriterion) rootCriterion;
BiddingStrategyConfiguration biddingStrategyConfig =
biddableRootCriterion.getBiddingStrategyConfiguration();
Preconditions.checkState(
biddingStrategyConfig != null,
"Null bidding strategy config on the root node of ad group ID %s",
adGroupId);
ProductPartitionNode rootNode =
new ProductPartitionNode(
null,
(ProductDimension) null,
rootCriterion.getCriterion().getId(),
new ProductDimensionComparator());
// Set the root's bid if a bid exists on the BiddableAdGroupCriterion.
Money rootNodeBid = getBid(biddableRootCriterion);
if (rootNodeBid != null) {
rootNode = rootNode.asBiddableUnit().setBid(rootNodeBid.getMicroAmount());
}
addChildNodes(rootNode, parentIdMap);
return new ProductPartitionTree(adGroupId, biddingStrategyConfig, rootNode);
}
/**
* Returns a new empty tree.
*
* @param adGroupId the ID of the ad group
* @param biddingStrategyConfig the bidding strategy configuration of the ad group
*/
private static ProductPartitionTree createEmptyAdGroupTree(
Long adGroupId, BiddingStrategyConfiguration biddingStrategyConfig) {
Preconditions.checkNotNull(adGroupId, "Null ad group ID");
Preconditions.checkNotNull(biddingStrategyConfig, "Null bidding strategy configuration");
ProductPartitionNode rootNode =
new ProductPartitionNode(null, null, -1L, new ProductDimensionComparator());
return new ProductPartitionTree(adGroupId, biddingStrategyConfig, rootNode);
}
/** Returns the ID of the ad group for this tree. */
public Long getAdGroupId() {
return adGroupId;
}
/** Returns the root node of this tree. This <em>will not be null</em>. */
public ProductPartitionNode getRoot() {
return root;
}
@Override
public String toString() {
return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE)
.append("adGroupId", adGroupId)
.append("biddingStrategyConfig", biddingStrategyConfig)
.append("originalRoot is empty", originalRoot == null)
// Add a newline before the root so that the tree levels all line up.
.append("root", String.format("%n%s", root.toDetailedString()))
.toString();
}
/**
* Returns the {@code AdGroupCriterionOperation}s that will apply the changes made to this tree.
*/
public List<AdGroupCriterionOperation> getMutateOperations() {
return Lists.transform(
createMutateOperationPairs(),
new Function<OperationPair, AdGroupCriterionOperation>() {
@Override
public AdGroupCriterionOperation apply(OperationPair input) {
return input.operation;
}
});
}
/**
* Creates and returns the pairs of node/operation that will apply the changes made to this tree.
*/
private List<OperationPair> createMutateOperationPairs() {
List<OperationPair> ops = Lists.newArrayList();
if (originalRoot == null) {
// The original tree was empty, so treat all nodes in this tree as ADDs.
return createAddOperations(root);
}
// Add the mutate operations required to apply changes to the root node.
Set<ProductDimension> dimensionsToProcess = addMutateOperations(originalRoot, root, ops);
// Add additional mutate operations for the remaining child dimensions to process.
addMutateOperationsByParent(originalRoot, root, dimensionsToProcess, ops);
return ops;
}
/**
* Adds to the operations list all operations required to mutate the children of {@code
* originalParentNode} to {@code newParentNode}.
*
* @param originalParentNode required - must not be null
* @param newParentNode required - must not be null
* @param childDimensionsToProcess the child dimensions to process
* @param ops the operations list to add to
*/
private void addMutateOperationsByParent(
ProductPartitionNode originalParentNode,
ProductPartitionNode newParentNode,
Set<ProductDimension> childDimensionsToProcess,
List<OperationPair> ops) {
for (ProductDimension dimensionToProcess : childDimensionsToProcess) {
ProductPartitionNode originalChild =
originalParentNode.hasChild(dimensionToProcess)
? originalParentNode.getChild(dimensionToProcess)
: null;
ProductPartitionNode newChild =
newParentNode.hasChild(dimensionToProcess)
? newParentNode.getChild(dimensionToProcess)
: null;
Set<ProductDimension> grandchildDimensionsToProcess =
addMutateOperations(originalChild, newChild, ops);
if (!grandchildDimensionsToProcess.isEmpty()) {
// Logic check - the only condition where further processing of children is required
// is when the parent exists in both trees. If the parent is null in one tree but
// not the other, then the node for dimensionToProcess was either:
// 1) removed from the original OR
// 2) added to the new tree
// In both cases, the call to addMutateOperations above will have already added all of the
// necessary operations to handle the node and all of its children.
Preconditions.checkState(
originalChild != null,
"Original child should not be null if there are children to process");
Preconditions.checkState(
newChild != null, "New child should not be null if there are children to process");
addMutateOperationsByParent(originalChild, newChild, grandchildDimensionsToProcess, ops);
}
}
}
/**
* Adds to the operations list all operations required to mutate {@code originalNode} to the state
* of {@code newNode}.
*
* <p>The returned set of child product dimensions will only <em>potentially</em> be non-empty if
* both {@code originalNode != null} and {@code newNode != null}.
*
* @param originalNode may be null
* @param newNode may be null
* @param ops the operations list to add to
* @return the set of child product dimensions that require further processing
*/
private Set<ProductDimension> addMutateOperations(
@Nullable ProductPartitionNode originalNode,
@Nullable ProductPartitionNode newNode,
List<OperationPair> ops) {
Set<ProductDimension> childDimensionsToProcess = Sets.newTreeSet(dimensionComparator);
NodeDifference nodeDifference =
ProductPartitionNodeDiffer.diff(originalNode, newNode, dimensionComparator);
boolean isProcessChildren;
switch (nodeDifference) {
case NEW_NODE:
ops.addAll(createAddOperations(newNode));
// No need to further process children. The ADD operations above will include operations
// for all children of newNode.
isProcessChildren = false;
break;
case REMOVED_NODE:
ops.add(createRemoveOperation(originalNode));
// No need to further process children. The REMOVE operation above will perform a
// cascading delete of all children of newNode.
isProcessChildren = false;
break;
case PARTITION_TYPE_CHANGE:
case EXCLUDED_UNIT_CHANGE:
ops.add(createRemoveOperation(originalNode));
ops.addAll(createAddOperations(newNode));
// No need to further process children. The ADD operations above will include operations
// for all children of newNode.
isProcessChildren = false;
break;
case BID_CHANGE:
// Ensure that the new node has the proper ID (this may have been lost if the node
// was removed and then re-added).
newNode = newNode.setProductPartitionId(originalNode.getProductPartitionId());
ops.add(createSetBidOperation(newNode));
// Process the children of newNode. The SET operation above will only handle changes
// made to newNode, not its children.
isProcessChildren = true;
break;
case NONE:
// Ensure that the new node has the proper ID (this may have been lost if the node
// was removed and then re-added).
newNode = newNode.setProductPartitionId(originalNode.getProductPartitionId());
// This node does not have changes, but its children may.
isProcessChildren = true;
break;
default:
throw new IllegalStateException("Unrecognized difference: " + nodeDifference);
}
if (isProcessChildren) {
for (ProductPartitionNode childNode :
Iterables.concat(originalNode.getChildren(), newNode.getChildren())) {
childDimensionsToProcess.add(childNode.getDimension());
}
}
return childDimensionsToProcess;
}
/** Returns a SET operation for the specified node. */
private OperationPair createSetBidOperation(ProductPartitionNode node) {
Preconditions.checkNotNull(
node.getProductPartitionId(), "Node for SET operation has no partition ID: %s", node);
Preconditions.checkArgument(
node.getProductPartitionId().longValue() >= 0L,
"Node for SET operation has a negative partition ID: %s",
node);
AdGroupCriterionOperation setOp = new AdGroupCriterionOperation();
setOp.setOperator(Operator.SET);
setOp.setOperand(
ProductPartitionNodeAdapter.createCriterionForSetBid(
node, adGroupId, getBiddingStrategyConfiguration()));
return new OperationPair(node, setOp);
}
/**
* Creates ADD operations for the node and ALL of its children and adds them to the provided
* operations list.
*/
private List<OperationPair> createAddOperations(ProductPartitionNode node) {
AdGroupCriterionOperation addOp = new AdGroupCriterionOperation();
addOp.setOperator(Operator.ADD);
// Set the node's ID to a new temporary ID.
node.setProductPartitionId(idGenerator.next());
addOp.setOperand(
ProductPartitionNodeAdapter.createCriterionForAdd(
node, adGroupId, getBiddingStrategyConfiguration()));
List<OperationPair> operationsList = Lists.newArrayList();
operationsList.add(new OperationPair(node, addOp));
// Recursively add all of this node's children to the operations list.
for (ProductPartitionNode child : node.getChildren()) {
operationsList.addAll(createAddOperations(child));
}
return operationsList;
}
/** Returns a REMOVE operation for the specified node. */
private OperationPair createRemoveOperation(ProductPartitionNode node) {
Preconditions.checkNotNull(
node.getProductPartitionId(), "Node for REMOVE operation has no partition ID: %s", node);
Preconditions.checkArgument(
node.getProductPartitionId().longValue() >= 0L,
"Node for REMOVE operation has a negative partition ID: %s",
node);
AdGroupCriterionOperation removeOp = new AdGroupCriterionOperation();
removeOp.setOperator(Operator.REMOVE);
removeOp.setOperand(ProductPartitionNodeAdapter.createCriterionForRemove(node, adGroupId));
return new OperationPair(node, removeOp);
}
/** Returns a copy of this tree's bidding strategy configuration. */
private BiddingStrategyConfiguration getBiddingStrategyConfiguration() {
// Create a copy of the config.
BiddingStrategyConfiguration copy = new BiddingStrategyConfiguration();
return copy;
}
/**
* An OperartionPair associates a ProductPartitionNode with an AdGroupCriterionOperation that
* mutates the node.
*/
private static class OperationPair {
final ProductPartitionNode node;
final AdGroupCriterionOperation operation;
OperationPair(ProductPartitionNode node, AdGroupCriterionOperation operation) {
this.node = node;
this.operation = operation;
}
@Override
public String toString() {
return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
.append("node", node)
.append("operation", operation)
.toString();
}
}
/**
* Retrieves the {@link BiddingStrategyConfiguration} of an ad group.
*
* @param services the AdWordsServices
* @param session the session to use for the request
* @param adGroupId the ad group ID
* @return the non-null BiddingStrategyConfiguration of the ad group
*/
private static BiddingStrategyConfiguration getAdGroupBiddingStrategyConfiguration(
AdWordsServices services, AdWordsSession session, Long adGroupId)
throws ApiException, RemoteException {
AdGroupServiceInterface adGroupService = services.get(session, AdGroupServiceInterface.class);
Selector selector =
new SelectorBuilder()
.fields(
AdGroupField.Id,
AdGroupField.BiddingStrategyType,
AdGroupField.BiddingStrategyId,
AdGroupField.BiddingStrategyName)
.equalsId(adGroupId)
.build();
AdGroupPage adGroupPage = adGroupService.get(selector);
if (adGroupPage.getEntries() == null || adGroupPage.getEntries().length == 0) {
throw new IllegalArgumentException("No ad group found with ID " + adGroupId);
}
AdGroup adGroup = adGroupPage.getEntries(0);
Preconditions.checkState(
adGroup.getBiddingStrategyConfiguration() != null,
"Unexpected state - ad group ID %s has a null BiddingStrategyConfiguration",
adGroupId);
return adGroup.getBiddingStrategyConfiguration();
}
/**
* Using the criteria in {@code parentIdMap}, recursively adds all children under the partition ID
* of {@code parentNode} to {@code parentNode}.
*
* @param parentNode required
* @param parentIdMap the multimap from parent partition ID to list of child criteria
*/
private static void addChildNodes(
ProductPartitionNode parentNode, ListMultimap<Long, AdGroupCriterion> parentIdMap) {
if (parentIdMap.containsKey(parentNode.getProductPartitionId())) {
parentNode = parentNode.asSubdivision();
}
for (AdGroupCriterion adGroupCriterion : parentIdMap.get(parentNode.getProductPartitionId())) {
ProductPartition partition = (ProductPartition) adGroupCriterion.getCriterion();
ProductPartitionNode childNode = parentNode.addChild(partition.getCaseValue());
childNode = childNode.setProductPartitionId(partition.getId());
if (ProductPartitionType.SUBDIVISION.equals(partition.getPartitionType())) {
childNode = childNode.asSubdivision();
} else {
if (adGroupCriterion instanceof BiddableAdGroupCriterion) {
childNode = childNode.asBiddableUnit();
Money cpcBidAmount = getBid((BiddableAdGroupCriterion) adGroupCriterion);
if (cpcBidAmount != null) {
childNode = childNode.setBid(cpcBidAmount.getMicroAmount());
}
} else {
childNode = childNode.asExcludedUnit();
}
}
addChildNodes(childNode, parentIdMap);
}
}
/** Returns the criterion-level bid, or null if no such bid exists. */
private static Money getBid(BiddableAdGroupCriterion biddableCriterion) {
BiddingStrategyConfiguration biddingConfig =
biddableCriterion.getBiddingStrategyConfiguration();
Money cpcBidAmount = null;
if (biddingConfig.getBids() != null) {
for (Bids bid : biddingConfig.getBids()) {
if (bid instanceof CpcBid) {
CpcBid cpcBid = (CpcBid) bid;
if (BidSource.CRITERION.equals(cpcBid.getCpcBidSource())) {
cpcBidAmount = cpcBid.getBid();
break;
}
}
}
}
return cpcBidAmount;
}
}
@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(
final CommandRunnerParams params,
Iterable<TestRule> tests,
BuildContext buildContext,
ExecutionContext executionContext,
final TestRunningOptions options,
ListeningExecutorService service,
BuildEngine buildEngine,
final StepRunner stepRunner)
throws IOException, ExecutionException, InterruptedException {
if (options.isUsingOneTimeOutputDirectories()) {
BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString());
}
ImmutableSet<JavaLibrary> rulesUnderTest;
// If needed, we first run instrumentation on the class files.
if (options.isCodeCoverageEnabled()) {
rulesUnderTest = getRulesUnderTest(tests);
if (!rulesUnderTest.isEmpty()) {
try {
stepRunner.runStepForBuildTarget(
new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
} else {
rulesUnderTest = ImmutableSet.of();
}
final ImmutableSet<String> testTargets =
FluentIterable.from(tests)
.transform(HasBuildTarget.TO_TARGET)
.transform(Functions.toStringFunction())
.toSet();
final int totalNumberOfTests = Iterables.size(tests);
params
.getBuckEventBus()
.post(
TestRunEvent.started(
options.isRunAllTests(),
options.getTestSelectorList(),
options.shouldExplainTestSelectorList(),
testTargets));
// Start running all of the tests. The result of each java_test() rule is represented as a
// ListenableFuture.
List<ListenableFuture<TestResults>> results = Lists.newArrayList();
// Unless `--verbose 0` is specified, print out test results as they become available.
// Failures with the ListenableFuture should always be printed, as they indicate an error with
// Buck, not the test being run.
Verbosity verbosity = params.getConsole().getVerbosity();
final boolean printTestResults = (verbosity != Verbosity.SILENT);
// For grouping results!
final TestResultsGrouper grouper;
if (options.isIgnoreFailingDependencies()) {
grouper = new TestResultsGrouper(tests);
} else {
grouper = null;
}
TestRuleKeyFileHelper testRuleKeyFileHelper =
new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine);
final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
final List<TestRun> separateTestRuns = Lists.newArrayList();
List<TestRun> parallelTestRuns = Lists.newArrayList();
for (final TestRule test : tests) {
// Determine whether the test needs to be executed.
boolean isTestRunRequired;
isTestRunRequired =
isTestRunRequiredForTest(
test,
buildEngine,
executionContext,
testRuleKeyFileHelper,
options.isResultsCacheEnabled(),
!options.getTestSelectorList().isEmpty());
List<Step> steps;
if (isTestRunRequired) {
params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
final Map<String, UUID> testUUIDMap = new HashMap<>();
List<Step> testSteps =
test.runTests(
buildContext,
executionContext,
options.isDryRun(),
options.isShufflingTests(),
options.getTestSelectorList(),
new TestRule.TestReportingCallback() {
@Override
public void testsDidBegin() {
LOG.debug("Tests for rule %s began", test.getBuildTarget());
}
@Override
public void testDidBegin(String testCaseName, String testName) {
LOG.debug(
"Test rule %s test case %s test name %s began",
test.getBuildTarget(), testCaseName, testName);
UUID testUUID = UUID.randomUUID();
// UUID is immutable and thread-safe as of Java 7, so it's
// safe to stash in a map and use later:
//
// http://bugs.java.com/view_bug.do?bug_id=6611830
testUUIDMap.put(testCaseName + ":" + testName, testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.started(testUUID, testCaseName, testName));
}
@Override
public void testDidEnd(TestResultSummary testResultSummary) {
LOG.debug(
"Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
UUID testUUID =
testUUIDMap.get(
testResultSummary.getTestCaseName()
+ ":"
+ testResultSummary.getTestName());
Preconditions.checkNotNull(testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.finished(testUUID, testResultSummary));
}
@Override
public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
LOG.debug(
"Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
}
});
if (!testSteps.isEmpty()) {
stepsBuilder.addAll(testSteps);
stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
}
steps = stepsBuilder.build();
} else {
steps = ImmutableList.of();
}
TestRun testRun =
TestRun.of(
test,
steps,
getCachingStatusTransformingCallable(
isTestRunRequired,
test.interpretTestResults(
executionContext,
/*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(),
/*isDryRun*/ options.isDryRun())));
// Always run the commands, even if the list of commands as empty. There may be zero
// commands because the rule is cached, but its results must still be processed.
if (test.runTestSeparately()) {
LOG.debug("Running test %s in serial", test);
separateTestRuns.add(testRun);
} else {
LOG.debug("Running test %s in parallel", test);
parallelTestRuns.add(testRun);
}
}
final StepRunner.StepRunningCallback testStepRunningCallback =
new StepRunner.StepRunningCallback() {
@Override
public void stepsWillRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps will run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get()));
}
@Override
public void stepsDidRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps did run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get()));
}
};
for (TestRun testRun : parallelTestRuns) {
ListenableFuture<TestResults> testResults =
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
service,
testStepRunningCallback);
results.add(
transformTestResults(
params,
testResults,
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
final List<TestResults> completedResults = Lists.newArrayList();
final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
ListenableFuture<Void> uberFuture =
stepRunner.addCallback(
parallelTestStepsFuture,
new FutureCallback<List<TestResults>>() {
@Override
public void onSuccess(List<TestResults> parallelTestResults) {
LOG.debug("Parallel tests completed, running separate tests...");
completedResults.addAll(parallelTestResults);
List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
for (TestRun testRun : separateTestRuns) {
separateResultsList.add(
transformTestResults(
params,
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
directExecutorService,
testStepRunningCallback),
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> serialResults =
Futures.allAsList(separateResultsList);
try {
completedResults.addAll(serialResults.get());
} catch (ExecutionException e) {
LOG.error(e, "Error fetching serial test results");
throw new HumanReadableException(e, "Error fetching serial test results");
} catch (InterruptedException e) {
LOG.error(e, "Interrupted fetching serial test results");
try {
serialResults.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw new HumanReadableException(e, "Test cancelled");
}
LOG.debug("Done running serial tests.");
}
@Override
public void onFailure(Throwable e) {
LOG.error(e, "Parallel tests failed, not running serial tests");
throw new HumanReadableException(e, "Parallel tests failed");
}
},
directExecutorService);
try {
// Block until all the tests have finished running.
uberFuture.get();
} catch (ExecutionException e) {
e.printStackTrace(params.getConsole().getStdErr());
return 1;
} catch (InterruptedException e) {
try {
uberFuture.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
}
params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
// Write out the results as XML, if requested.
Optional<String> path = options.getPathToXmlTestOutput();
if (path.isPresent()) {
try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
writeXmlOutput(completedResults, writer);
}
}
// Generate the code coverage report.
if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) {
try {
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional =
Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder());
stepRunner.runStepForBuildTarget(
getReportCommand(
rulesUnderTest,
defaultJavaPackageFinderOptional,
params.getRepository().getFilesystem(),
JUnitStep.JACOCO_OUTPUT_DIR,
options.getCoverageReportFormat()),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
boolean failures =
Iterables.any(
completedResults,
new Predicate<TestResults>() {
@Override
public boolean apply(TestResults results) {
LOG.debug("Checking result %s for failure", results);
return !results.isSuccess();
}
});
return failures ? TEST_FAILURES_EXIT_CODE : 0;
}
private String timeout(final Integer timeout) {
return Optional.fromNullable(timeout).transform(Functions.toStringFunction()).or("NONE");
}