@Override
public synchronized void addElement(T val) {
if (fStartTimesIndex.put(Long.valueOf(val.getStart()), val)) {
fEndTimesIndex.put(Long.valueOf(val.getEnd()), val);
fSize++;
}
}
/** Constructor */
public TreeMapStore() {
/*
* For the start times index, the "key comparator" will compare the
* start times as longs directly. This is the primary comparator for its
* tree map.
*
* The secondary "value" comparator will check the end times first, and
* in the event of a tie, defer to the ISegment's Comparable
* implementation, a.k.a. its natural ordering.
*
* The same is done for the end times index, but swapping the first two
* comparators instead.
*/
fStartTimesIndex =
checkNotNull(
TreeMultimap.<Long, T>create(
SegmentComparators.LONG_COMPARATOR,
Ordering.from(SegmentComparators.INTERVAL_END_COMPARATOR)
.compound(Ordering.natural())));
fEndTimesIndex =
checkNotNull(
TreeMultimap.<Long, T>create(
SegmentComparators.LONG_COMPARATOR,
Ordering.from(SegmentComparators.INTERVAL_START_COMPARATOR)
.compound(Ordering.natural())));
fSize = 0;
}
@Override
public Iterable<T> getIntersectingElements(long start, long end) {
Iterable<T> matchStarts =
Iterables.concat(fStartTimesIndex.asMap().headMap(end, true).values());
Iterable<T> matchEnds = Iterables.concat(fEndTimesIndex.asMap().tailMap(start, true).values());
return checkNotNull(
Sets.intersection(Sets.newHashSet(matchStarts), Sets.newHashSet(matchEnds)));
}
/**
* Filters SchemaContext for yang modules
*
* @param delegate original SchemaContext
* @param rootModules modules (yang schemas) to be available and all their dependencies (modules
* importing rootModule and whole chain of their imports)
* @param additionalModuleIds (additional) modules (yang schemas) to be available and whole chain
* of their imports
*/
public FilteringSchemaContextProxy(
final SchemaContext delegate,
final Collection<ModuleId> rootModules,
final Set<ModuleId> additionalModuleIds) {
Preconditions.checkArgument(rootModules != null, "Base modules cannot be null.");
Preconditions.checkArgument(additionalModuleIds != null, "Additional modules cannot be null.");
final Builder<Module> filteredModulesBuilder = new Builder<>();
final SetMultimap<URI, Module> nsMap =
Multimaps.newSetMultimap(new TreeMap<URI, Collection<Module>>(), MODULE_SET_SUPPLIER);
final SetMultimap<String, Module> nameMap =
Multimaps.newSetMultimap(new TreeMap<String, Collection<Module>>(), MODULE_SET_SUPPLIER);
ImmutableMap.Builder<ModuleIdentifier, String> identifiersToSourcesBuilder =
ImmutableMap.builder();
// preparing map to get all modules with one name but difference in revision
final TreeMultimap<String, Module> nameToModulesAll = getStringModuleTreeMultimap();
nameToModulesAll.putAll(getStringModuleMap(delegate));
// in case there is a particular dependancy to view filteredModules/yang models
// dependancy is checked for module name and imports
processForRootModules(delegate, rootModules, filteredModulesBuilder);
// adding additional modules
processForAdditionalModules(delegate, additionalModuleIds, filteredModulesBuilder);
filteredModulesBuilder.addAll(
getImportedModules(
Maps.uniqueIndex(delegate.getModules(), ModuleId.MODULE_TO_MODULE_ID),
filteredModulesBuilder.build(),
nameToModulesAll));
/**
* Instead of doing this on each invocation of getModules(), pre-compute it once and keep it
* around -- better than the set we got in.
*/
this.filteredModules = filteredModulesBuilder.build();
for (final Module module : filteredModules) {
nameMap.put(module.getName(), module);
nsMap.put(module.getNamespace(), module);
identifiersToSourcesBuilder.put(module, module.getSource());
}
namespaceToModules = ImmutableSetMultimap.copyOf(nsMap);
nameToModules = ImmutableSetMultimap.copyOf(nameMap);
identifiersToSources = identifiersToSourcesBuilder.build();
}
@Override
public Iterable<T> getIntersectingElements(long position) {
/*
* The intervals intersecting 't' are those whose 1) start time is
* *lower* than 't' AND 2) end time is *higher* than 't'.
*/
Iterable<T> matchStarts =
Iterables.concat(fStartTimesIndex.asMap().headMap(position, true).values());
Iterable<T> matchEnds =
Iterables.concat(fEndTimesIndex.asMap().tailMap(position, true).values());
return checkNotNull(
Sets.intersection(Sets.newHashSet(matchStarts), Sets.newHashSet(matchEnds)));
}
private List<Cluster> doPrivilegedLookup(String partitionName, String vmTypeName)
throws NotEnoughResourcesException {
if (Partition.DEFAULT_NAME.equals(partitionName)) {
Iterable<Cluster> authorizedClusters =
Iterables.filter(
Clusters.getInstance().listValues(),
RestrictedTypes.filterPrivilegedWithoutOwner());
Multimap<VmTypeAvailability, Cluster> sorted = TreeMultimap.create();
for (Cluster c : authorizedClusters) {
sorted.put(c.getNodeState().getAvailability(vmTypeName), c);
}
if (sorted.isEmpty()) {
throw new NotEnoughResourcesException(
"Not enough resources: no availability zone is available in which you have permissions to run instances.");
} else {
return Lists.newArrayList(sorted.values());
}
} else {
ServiceConfiguration ccConfig =
Topology.lookup(ClusterController.class, Partitions.lookupByName(partitionName));
Cluster cluster = Clusters.lookup(ccConfig);
if (cluster == null) {
throw new NotEnoughResourcesException("Can't find cluster " + partitionName);
}
if (!RestrictedTypes.filterPrivilegedWithoutOwner().apply(cluster)) {
throw new NotEnoughResourcesException("Not authorized to use cluster " + partitionName);
}
return Lists.newArrayList(cluster);
}
}
public static Multimap<Date, LogEntry> groupByTime(Collection<LogEntry> entries) {
Multimap<Date, LogEntry> time2Entries = TreeMultimap.create();
for (LogEntry entry : entries) {
time2Entries.put(entry.getDate(), entry);
}
return time2Entries;
}
public static Multimap<String, LogEntry> groupByUserAgent(Collection<LogEntry> entries) {
Multimap<String, LogEntry> userAgent2Entries = TreeMultimap.create();
for (LogEntry entry : entries) {
userAgent2Entries.put(entry.getUserAgent(), entry);
}
return userAgent2Entries;
}
public static Multimap<String, LogEntry> groupByIPAddress(Collection<LogEntry> entries) {
Multimap<String, LogEntry> ip2Entries = TreeMultimap.create();
for (LogEntry entry : entries) {
ip2Entries.put(entry.getIp(), entry);
}
return ip2Entries;
}
// dealing with imported module other than root and directly importing root
private static Collection<Module> getImportedModules(
Map<ModuleId, Module> allModules,
Set<Module> baseModules,
TreeMultimap<String, Module> nameToModulesAll) {
List<Module> relatedModules = Lists.newLinkedList();
for (Module module : baseModules) {
for (ModuleImport moduleImport : module.getImports()) {
Date revisionDate =
moduleImport.getRevision() == null
? nameToModulesAll.get(moduleImport.getModuleName()).first().getRevision()
: moduleImport.getRevision();
ModuleId key = new ModuleId(moduleImport.getModuleName(), revisionDate);
Module importedModule = allModules.get(key);
Preconditions.checkArgument(
importedModule != null,
"Invalid schema, cannot find imported module: %s from module: %s, %s, modules:%s",
key,
module.getQNameModule(),
module.getName());
relatedModules.add(importedModule);
// calling imports recursive
relatedModules.addAll(
getImportedModules(
allModules, Collections.singleton(importedModule), nameToModulesAll));
}
}
return relatedModules;
}
@Override
public Metric deserialize(
JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext)
throws JsonParseException {
JsonObject jsonObject = jsonElement.getAsJsonObject();
String name = null;
if (jsonObject.get("name") != null) name = jsonObject.get("name").getAsString();
boolean exclude_tags = false;
if (jsonObject.get("exclude_tags") != null)
exclude_tags = jsonObject.get("exclude_tags").getAsBoolean();
TreeMultimap<String, String> tags = TreeMultimap.create();
JsonElement jeTags = jsonObject.get("tags");
if (jeTags != null) {
JsonObject joTags = jeTags.getAsJsonObject();
int count = 0;
for (Map.Entry<String, JsonElement> tagEntry : joTags.entrySet()) {
String context = "tags[" + count + "]";
if (tagEntry.getKey().isEmpty())
throw new ContextualJsonSyntaxException(context, "name must not be empty");
if (tagEntry.getValue().isJsonArray()) {
for (JsonElement element : tagEntry.getValue().getAsJsonArray()) {
if (element.isJsonNull() || element.getAsString().isEmpty())
throw new ContextualJsonSyntaxException(
context + "." + tagEntry.getKey(), "value must not be null or empty");
tags.put(tagEntry.getKey(), element.getAsString());
}
} else {
if (tagEntry.getValue().isJsonNull() || tagEntry.getValue().getAsString().isEmpty())
throw new ContextualJsonSyntaxException(
context + "." + tagEntry.getKey(), "value must not be null or empty");
tags.put(tagEntry.getKey(), tagEntry.getValue().getAsString());
}
count++;
}
}
Metric ret = new Metric(name, exclude_tags, tags);
JsonElement limit = jsonObject.get("limit");
if (limit != null) ret.setLimit(limit.getAsInt());
return (ret);
}
private void handleApplicationShutdown(ChannelHandlerContext ctx, MessageEvent me) {
synchronized (monitor) {
this.shutdown = true;
emptyAcknowledgementSchedule.clear();
ongoingMessageExchanges.clear();
}
ctx.sendDownstream(me);
}
@Test
public void buildInt() throws IOException {
final TreeMultimap<Integer, Integer> elements = TreeMultimap.create();
for (int i = 0; i < DOCS; i++) {
elements.put(i / 2, i);
}
final com.yandex.yoctodb.util.mutable.IndexToIndexMultiMap mutable =
new com.yandex.yoctodb.util.mutable.impl.IntIndexToIndexMultiMap(elements.asMap().values());
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
mutable.writeTo(baos);
final Buffer buf = Buffer.from(baos.toByteArray());
final IndexToIndexMultiMap result = IndexToIndexMultiMapReader.from(buf);
assertTrue(result instanceof IntIndexToIndexMultiMap);
}
private static TreeMultimap<String, Module> getStringModuleTreeMultimap() {
return TreeMultimap.create(
new Comparator<String>() {
@Override
public int compare(String o1, String o2) {
return o1.compareTo(o2);
}
},
REVISION_COMPARATOR);
}
public OwnersReport updatedWith(OwnersReport other) {
SetMultimap<TargetNode<?>, Path> updatedOwners = TreeMultimap.create(owners);
updatedOwners.putAll(other.owners);
return new OwnersReport(
updatedOwners,
Sets.intersection(inputsWithNoOwners, other.inputsWithNoOwners),
Sets.union(nonExistentInputs, other.nonExistentInputs),
Sets.union(nonFileInputs, other.nonFileInputs));
}
public static String sortAndConcatHeadersIntoString(Multimap<String, String> headers) {
StringBuffer buffer = new StringBuffer();
SortedSetMultimap<String, String> sortedMap = TreeMultimap.create();
sortedMap.putAll(headers);
for (Entry<String, String> header : sortedMap.entries()) {
if (header.getKey() != null)
buffer.append(String.format("%s: %s%n", header.getKey(), header.getValue()));
}
return buffer.toString();
}
@VisibleForTesting
static OwnersReport generateOwnersReport(
CommandRunnerParams params,
TargetNode<?> targetNode,
Iterable<String> filePaths,
boolean guessForDeletedEnabled) {
// Process arguments assuming they are all relative file paths.
Set<Path> inputs = Sets.newHashSet();
Set<String> nonExistentInputs = Sets.newHashSet();
Set<String> nonFileInputs = Sets.newHashSet();
for (String filePath : filePaths) {
File file = params.getCell().getFilesystem().getFileForRelativePath(filePath);
if (!file.exists()) {
nonExistentInputs.add(filePath);
} else if (!file.isFile()) {
nonFileInputs.add(filePath);
} else {
inputs.add(Paths.get(filePath));
}
}
// Try to find owners for each valid and existing file.
Set<Path> inputsWithNoOwners = Sets.newHashSet(inputs);
SetMultimap<TargetNode<?>, Path> owners = TreeMultimap.create();
for (final Path commandInput : inputs) {
Predicate<Path> startsWith =
new Predicate<Path>() {
@Override
public boolean apply(Path input) {
return !commandInput.equals(input) && commandInput.startsWith(input);
}
};
Set<Path> ruleInputs = targetNode.getInputs();
if (ruleInputs.contains(commandInput)
|| FluentIterable.from(ruleInputs).anyMatch(startsWith)) {
inputsWithNoOwners.remove(commandInput);
owners.put(targetNode, commandInput);
}
}
// Try to guess owners for nonexistent files.
if (guessForDeletedEnabled) {
for (String nonExistentInput : nonExistentInputs) {
owners.put(targetNode, new File(nonExistentInput).toPath());
}
}
return new OwnersReport(owners, inputsWithNoOwners, nonExistentInputs, nonFileInputs);
}
private void handleIncomingConfirmableCoapRequest(ChannelHandlerContext ctx, MessageEvent me) {
InetSocketAddress remoteEndpoint = (InetSocketAddress) me.getRemoteAddress();
CoapMessage coapMessage = (CoapMessage) me.getMessage();
IncomingReliableMessageExchange newMessageExchange =
new IncomingReliableMessageExchange(remoteEndpoint, coapMessage.getMessageID());
IncomingMessageExchange oldMessageExchange =
ongoingMessageExchanges.get(remoteEndpoint, coapMessage.getMessageID());
// Check if there is an ongoing
if (oldMessageExchange != null) {
if (oldMessageExchange instanceof IncomingReliableMessageExchange) {
// if the old message exchange is reliable and the empty ACK was already sent send another
// empty ACK
if (((IncomingReliableMessageExchange) oldMessageExchange).isAcknowledgementSent())
writeEmptyAcknowledgement(remoteEndpoint, coapMessage.getMessageID());
}
// if the old message was unreliable and the duplicate message is confirmable send empty ACK
else writeEmptyAcknowledgement(remoteEndpoint, coapMessage.getMessageID());
// As the message is already being processed there is nothing more to do
return;
}
// try to add new reliable message exchange
boolean added = false;
synchronized (monitor) {
Long time = System.currentTimeMillis() + MIN_EMPTY_ACK_DELAY_MILLIS;
// Add message exchange to set of ongoing exchanges to detect duplicates
if (!ongoingMessageExchanges.contains(remoteEndpoint, coapMessage.getMessageID())) {
ongoingMessageExchanges.put(remoteEndpoint, coapMessage.getMessageID(), newMessageExchange);
added = true;
}
// If the scheduling of the empty ACK does not work then it was already scheduled
if (!emptyAcknowledgementSchedule.put(time, newMessageExchange)) {
log.error("Could not schedule empty ACK for message: {}", coapMessage);
ongoingMessageExchanges.remove(remoteEndpoint, coapMessage.getMessageID());
added = false;
}
}
// everything is fine, so further process message
if (added) ctx.sendUpstream(me);
}
/**
* @param executorService the {@link ScheduledExecutorService} to provide the threads that execute
* the operations for reliability.
*/
public IncomingMessageReliabilityHandler(ScheduledExecutorService executorService) {
this.shutdown = false;
this.emptyAcknowledgementSchedule =
TreeMultimap.create(
Ordering.<Long>natural(), Ordering.<IncomingReliableMessageExchange>arbitrary());
this.ongoingMessageExchanges = HashBasedTable.create();
executorService.scheduleAtFixedRate(
new ReliabilityTask(),
RELIABILITY_TASK_PERIOD_MILLIS,
RELIABILITY_TASK_PERIOD_MILLIS,
TimeUnit.MILLISECONDS);
}
/** Print only targets which were identified as owners in JSON. */
@VisibleForTesting
void printOwnersOnlyJsonReport(CommandRunnerParams params, OwnersReport report)
throws IOException {
final Multimap<String, String> output = TreeMultimap.create();
Set<TargetNode<?>> sortedTargetNodes = report.owners.keySet();
for (TargetNode<?> targetNode : sortedTargetNodes) {
Set<Path> files = report.owners.get(targetNode);
for (Path input : files) {
output.put(input.toString(), targetNode.getBuildTarget().getFullyQualifiedName());
}
}
params.getObjectMapper().writeValue(params.getConsole().getStdOut(), output.asMap());
}
private static Multimap<String, String> buildCanonicalizedHeadersMap(HttpRequest request) {
Multimap<String, String> headers = request.getHeaders();
SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
for (Entry<String, String> header : headers.entries()) {
if (header.getKey() == null) continue;
String key = header.getKey().toString().toLowerCase(Locale.getDefault());
// Ignore any headers that are not particularly interesting.
if (key.equalsIgnoreCase(HttpHeaders.CONTENT_TYPE)
|| key.equalsIgnoreCase(HttpHeaders.CONTENT_MD5)
|| key.equalsIgnoreCase(HttpHeaders.HOST)
|| key.startsWith(HEADER_TAG)) {
canonicalizedHeaders.put(key, header.getValue());
}
}
return canonicalizedHeaders;
}
private Map<String, Collection<String>> scanValueRequirementBySecType(
UniqueId portfolioId, ToolContext toolContext) {
AvailableOutputsProvider availableOutputsProvider = toolContext.getAvaliableOutputsProvider();
if (availableOutputsProvider == null) {
throw new OpenGammaRuntimeException("AvailableOutputsProvider missing from ToolContext");
}
final SetMultimap<String, String> valueNamesBySecurityType = TreeMultimap.create();
AvailableOutputs portfolioOutputs =
availableOutputsProvider.getPortfolioOutputs(portfolioId, null);
Set<String> securityTypes = portfolioOutputs.getSecurityTypes();
for (String securityType : securityTypes) {
Set<AvailableOutput> positionOutputs = portfolioOutputs.getPositionOutputs(securityType);
for (AvailableOutput availableOutput : positionOutputs) {
valueNamesBySecurityType.put(securityType, availableOutput.getValueName());
}
}
return valueNamesBySecurityType.asMap();
}
/**
* getLogMessages Returns a list of up to 20 errors (eg encountered when parsing JavaScrip)t for a
* source, sorted by frequency in ascending order
*
* @return
*/
private StringBuffer getLogMessages(boolean bReset) {
if ((null != _messages) && (_messages.size() > 0)) {
StringBuffer messagesString = new StringBuffer();
// Create multimap to store errors in, reverse the order of key (error message) and
// value (count) to sort on error count
Multimap<Integer, String> mm = TreeMultimap.create();
for (java.util.Map.Entry<String, Integer> entry : _messages.entrySet()) {
StringBuffer msg =
new StringBuffer(entry.getKey())
.append(" (Occurences: ")
.append(entry.getValue())
.append(')');
mm.put(-entry.getValue(), msg.toString());
}
// Write the error messages to a Collection<String>
Collection<String> messages = mm.values();
// Append up to the top 20 messages to our StringBuffer and return
int messageCount = 1;
for (String s : messages) {
if (messageCount > 1) {
messagesString.append('\n');
}
messagesString.append(s);
messageCount++;
if (messageCount > 20) break;
}
if (bReset) {
_messages.clear();
}
return messagesString;
} else {
return null;
}
} // TESTED (cut and paste from old code)
private void identifyDuplicates(List<ModContainer> mods) {
TreeMultimap<ModContainer, File> dupsearch =
TreeMultimap.create(new ModIdComparator(), Ordering.arbitrary());
for (ModContainer mc : mods) {
if (mc.getSource() != null) {
dupsearch.put(mc, mc.getSource());
}
}
ImmutableMultiset<ModContainer> duplist = Multisets.copyHighestCountFirst(dupsearch.keys());
SetMultimap<ModContainer, File> dupes = LinkedHashMultimap.create();
for (Entry<ModContainer> e : duplist.entrySet()) {
if (e.getCount() > 1) {
FMLLog.severe(
"Found a duplicate mod %s at %s",
e.getElement().getModId(), dupsearch.get(e.getElement()));
dupes.putAll(e.getElement(), dupsearch.get(e.getElement()));
}
}
if (!dupes.isEmpty()) {
throw new DuplicateModsFoundException(dupes);
}
}
/**
* This class keeps information on the current locally installed SDK. It tries to lazily load
* information as much as possible.
*
* <p>Packages are accessed by their type and a main query attribute, depending on the package type.
* There are different versions of {@link #getPkgInfo} which depend on the query attribute.
*
* <table border='1' cellpadding='3'>
* <tr>
* <th>Type</th>
* <th>Query parameter</th>
* <th>Getter</th>
* </tr>
*
* <tr>
* <td>Tools</td>
* <td>Unique instance</td>
* <td>{@code getPkgInfo(PkgType.PKG_TOOLS)} => {@link LocalPkgInfo}</td>
* </tr>
*
* <tr>
* <td>Platform-Tools</td>
* <td>Unique instance</td>
* <td>{@code getPkgInfo(PkgType.PKG_PLATFORM_TOOLS)} => {@link LocalPkgInfo}</td>
* </tr>
*
* <tr>
* <td>Docs</td>
* <td>Unique instance</td>
* <td>{@code getPkgInfo(PkgType.PKG_DOCS)} => {@link LocalPkgInfo}</td>
* </tr>
*
* <tr>
* <td>Build-Tools</td>
* <td>{@link FullRevision}</td>
* <td>{@code getLatestBuildTool()} => {@link BuildToolInfo}, <br/>
* or {@code getBuildTool(FullRevision)} => {@link BuildToolInfo}, <br/>
* or {@code getPkgInfo(PkgType.PKG_BUILD_TOOLS, FullRevision)} => {@link LocalPkgInfo}, <br/>
* or {@code getPkgsInfos(PkgType.PKG_BUILD_TOOLS)} => {@link LocalPkgInfo}[]</td>
* </tr>
*
* <tr>
* <td>Extras</td>
* <td>String vendor/path</td>
* <td>{@code getExtra(String)} => {@link LocalExtraPkgInfo}, <br/>
* or {@code getPkgInfo(PkgType.PKG_EXTRAS, String)} => {@link LocalPkgInfo}, <br/>
* or {@code getPkgsInfos(PkgType.PKG_EXTRAS)} => {@link LocalPkgInfo}[]</td>
* </tr>
*
* <tr>
* <td>Sources</td>
* <td>{@link AndroidVersion}</td>
* <td>{@code getPkgInfo(PkgType.PKG_SOURCES, AndroidVersion)} => {@link LocalPkgInfo}, <br/>
* or {@code getPkgsInfos(PkgType.PKG_SOURCES)} => {@link LocalPkgInfo}[]</td>
* </tr>
*
* <tr>
* <td>Samples</td>
* <td>{@link AndroidVersion}</td>
* <td>{@code getPkgInfo(PkgType.PKG_SAMPLES, AndroidVersion)} => {@link LocalPkgInfo}, <br/>
* or {@code getPkgsInfos(PkgType.PKG_SAMPLES)} => {@link LocalPkgInfo}[]</td>
* </tr>
*
* <tr>
* <td>Platforms</td>
* <td>{@link AndroidVersion}</td>
* <td>{@code getPkgInfo(PkgType.PKG_PLATFORMS, AndroidVersion)} => {@link LocalPkgInfo}, <br/>
* or {@code getPkgInfo(PkgType.PKG_ADDONS, String)} => {@link LocalPkgInfo}, <br/>
* or {@code getPkgsInfos(PkgType.PKG_PLATFORMS)} => {@link LocalPkgInfo}[], <br/>
* or {@code getTargetFromHashString(String)} => {@link IAndroidTarget}</td>
* </tr>
*
* <tr>
* <td>Add-ons</td>
* <td>{@link AndroidVersion} x String vendor/path</td>
* <td>{@code getPkgInfo(PkgType.PKG_ADDONS, String)} => {@link LocalPkgInfo}, <br/>
* or {@code getPkgsInfos(PkgType.PKG_ADDONS)} => {@link LocalPkgInfo}[], <br/>
* or {@code getTargetFromHashString(String)} => {@link IAndroidTarget}</td>
* </tr>
*
* <tr>
* <td>System images</td>
* <td>{@link AndroidVersion} x {@link String} ABI</td>
* <td>{@code getPkgsInfos(PkgType.PKG_SYS_IMAGES)} => {@link LocalPkgInfo}[]</td>
* </tr>
*
* </table>
*
* Apps/libraries that use it are encouraged to keep an existing instance around (using a singleton
* or similar mechanism).
*
* <p>Threading: All accessor methods are synchronized on the same internal lock so it's safe to
* call them from any thread, even concurrently. <br>
* A method like {@code getPkgsInfos} returns a copy of its data array, which objects are not
* altered after creation, so its value is not influenced by the internal state after it returns.
*
* <p>Implementation Background:
*
* <ul>
* <li>The sdk manager has a set of "Package" classes that cover both local and remote SDK
* operations.
* <li>Goal was to split it in 2 cleanly separated parts: {@link LocalSdk} parses sdk on disk, and
* a separate class wraps the downloaded manifest (this is now handled within Studio only)
* <li>The local SDK should be a singleton accessible somewhere, so there will be one in ADT (via
* the Sdk instance), one in Studio, and one in the command line tool. <br>
* Right now there's a bit of mess with some classes creating a temp LocalSdkParser, some
* others using an SdkManager instance, and that needs to be sorted out.
* <li>As a transition, the SdkManager instance wraps a LocalSdk and uses this. Eventually the
* SdkManager.java class will go away (its name is totally misleading, for starters.)
* <li>The current LocalSdkParser stays as-is for compatibility purposes and the goal is also to
* totally remove it when the SdkManager class goes away.
* </ul>
*
* @version 2 of the {@code SdkManager} class, essentially.
*/
public class LocalSdk {
/** Location of the SDK. Maybe null. Can be changed. */
private File mSdkRoot;
/** File operation object. (Used for overriding in mock testing.) */
private final IFileOp mFileOp;
/** List of package information loaded so far. Lazily populated. */
@GuardedBy(value = "mLocalPackages")
private final Multimap<PkgType, LocalPkgInfo> mLocalPackages = TreeMultimap.create();
/** Directories already parsed into {@link #mLocalPackages}. */
@GuardedBy(value = "mLocalPackages")
private final Multimap<PkgType, LocalDirInfo> mVisitedDirs = HashMultimap.create();
/** A legacy build-tool for older platform-tools < 17. */
private BuildToolInfo mLegacyBuildTools;
/** Cache of targets from local sdk. See {@link #getTargets()}. */
@GuardedBy(value = "mLocalPackages")
private List<IAndroidTarget> mCachedTargets = null;
private Set<MissingTarget> mCachedMissingTargets = null;
/** Creates an initial LocalSdk instance with an unknown location. */
public LocalSdk() {
mFileOp = new FileOp();
}
/**
* Creates an initial LocalSdk instance for a known SDK location.
*
* @param sdkRoot The location of the SDK root folder.
*/
public LocalSdk(@NonNull File sdkRoot) {
this();
setLocation(sdkRoot);
}
/**
* Creates an initial LocalSdk instance with an unknown location. This is designed for unit tests
* to override the {@link FileOp} being used.
*
* @param fileOp The alternate {@link FileOp} to use for all file-based interactions.
*/
@VisibleForTesting(visibility = Visibility.PRIVATE)
public LocalSdk(@NonNull IFileOp fileOp) {
mFileOp = fileOp;
}
/*
* Returns the current IFileOp being used.
*/
@NonNull
public IFileOp getFileOp() {
return mFileOp;
}
/**
* Sets or changes the SDK root location. This also clears any cached information.
*
* @param sdkRoot The location of the SDK root folder.
*/
public void setLocation(@NonNull File sdkRoot) {
assert sdkRoot != null;
mSdkRoot = sdkRoot;
clearLocalPkg(PkgType.PKG_ALL);
}
/**
* Location of the SDK. Maybe null. Can be changed.
*
* @return The location of the SDK. Null if not initialized yet.
*/
@Nullable
public File getLocation() {
return mSdkRoot;
}
/**
* Location of the SDK. Maybe null. Can be changed. The getLocation() API replaces this function.
*
* @return The location of the SDK. Null if not initialized yet.
*/
@Deprecated
@Nullable
public String getPath() {
return mSdkRoot != null ? mSdkRoot.getPath() : null;
}
/**
* Clear the tracked visited folders & the cached {@link LocalPkgInfo} for the given filter types.
*
* @param filters A set of PkgType constants or {@link PkgType#PKG_ALL} to clear everything.
*/
public void clearLocalPkg(@NonNull EnumSet<PkgType> filters) {
mLegacyBuildTools = null;
synchronized (mLocalPackages) {
for (PkgType filter : filters) {
mVisitedDirs.removeAll(filter);
mLocalPackages.removeAll(filter);
}
// Clear the targets if the platforms or addons are being cleared
if (filters.contains(PkgType.PKG_PLATFORM) || filters.contains(PkgType.PKG_ADDON)) {
mCachedMissingTargets = null;
mCachedTargets = null;
}
}
}
/**
* Check the tracked visited folders to see if anything has changed for the requested filter
* types. This does not refresh or reload any package information.
*
* @param filters A set of PkgType constants or {@link PkgType#PKG_ALL} to clear everything.
*/
public boolean hasChanged(@NonNull EnumSet<PkgType> filters) {
for (PkgType filter : filters) {
Collection<LocalDirInfo> dirInfos;
synchronized (mLocalPackages) {
dirInfos = mVisitedDirs.get(filter);
for (LocalDirInfo dirInfo : dirInfos) {
if (dirInfo.hasChanged()) {
return true;
}
}
}
}
return false;
}
// --------- Generic querying ---------
/**
* Retrieves information on a package identified by an {@link IPkgDesc}.
*
* @param descriptor {@link IPkgDesc} describing a package.
* @return The first package found with the same descriptor or null.
*/
@Nullable
public LocalPkgInfo getPkgInfo(@NonNull IPkgDesc descriptor) {
for (LocalPkgInfo pkg : getPkgsInfos(EnumSet.of(descriptor.getType()))) {
IPkgDesc d = pkg.getDesc();
if (d.equals(descriptor)) {
return pkg;
}
}
return null;
}
/**
* Retrieves information on a package identified by an {@link AndroidVersion}.
*
* <p>Note: don't use this for {@link PkgType#PKG_SYS_IMAGE} since there can be more than one ABI
* and this method only returns a single package per filter type.
*
* @param filter {@link PkgType#PKG_PLATFORM}, {@link PkgType#PKG_SAMPLE} or {@link
* PkgType#PKG_SOURCE}.
* @param version The {@link AndroidVersion} specific for this package type.
* @return An existing package information or null if not found.
*/
@Nullable
public LocalPkgInfo getPkgInfo(@NonNull PkgType filter, @NonNull AndroidVersion version) {
assert filter == PkgType.PKG_PLATFORM
|| filter == PkgType.PKG_SAMPLE
|| filter == PkgType.PKG_SOURCE;
for (LocalPkgInfo pkg : getPkgsInfos(filter)) {
IPkgDesc d = pkg.getDesc();
if (d.hasAndroidVersion() && d.getAndroidVersion().equals(version)) {
return pkg;
}
}
return null;
}
/**
* Retrieves information on a package identified by its {@link FullRevision}.
*
* <p>Note that {@link PkgType#PKG_TOOLS} and {@link PkgType#PKG_PLATFORM_TOOLS} are unique in a
* local SDK so you'll want to use {@link #getPkgInfo(PkgType)} to retrieve them instead.
*
* @param filter {@link PkgType#PKG_BUILD_TOOLS}.
* @param revision The {@link FullRevision} uniquely identifying this package.
* @return An existing package information or null if not found.
*/
@Nullable
public LocalPkgInfo getPkgInfo(@NonNull PkgType filter, @NonNull FullRevision revision) {
assert filter == PkgType.PKG_BUILD_TOOLS;
for (LocalPkgInfo pkg : getPkgsInfos(filter)) {
IPkgDesc d = pkg.getDesc();
if (d.hasFullRevision() && d.getFullRevision().equals(revision)) {
return pkg;
}
}
return null;
}
/**
* Retrieves information on a package identified by its {@link String} path.
*
* <p>For add-ons and platforms, the path is the target hash string (see {@link AndroidTargetHash}
* for helpers methods to generate this string.)
*
* @param filter {@link PkgType#PKG_ADDON}, {@link PkgType#PKG_PLATFORM}.
* @param path The vendor/path uniquely identifying this package.
* @return An existing package information or null if not found.
*/
@Nullable
public LocalPkgInfo getPkgInfo(@NonNull PkgType filter, @NonNull String path) {
assert filter == PkgType.PKG_ADDON || filter == PkgType.PKG_PLATFORM;
for (LocalPkgInfo pkg : getPkgsInfos(filter)) {
IPkgDesc d = pkg.getDesc();
if (d.hasPath() && path.equals(d.getPath())) {
return pkg;
}
}
return null;
}
/**
* Retrieves information on a package identified by both vendor and path strings.
*
* <p>For add-ons the path is target hash string (see {@link AndroidTargetHash} for helpers
* methods to generate this string.)
*
* @param filter {@link PkgType#PKG_EXTRA}, {@link PkgType#PKG_ADDON}.
* @param vendor The vendor id of the extra package.
* @param path The path uniquely identifying this package for its vendor.
* @return An existing package information or null if not found.
*/
@Nullable
public LocalPkgInfo getPkgInfo(
@NonNull PkgType filter, @NonNull String vendor, @NonNull String path) {
assert filter == PkgType.PKG_EXTRA || filter == PkgType.PKG_ADDON;
for (LocalPkgInfo pkg : getPkgsInfos(filter)) {
IPkgDesc d = pkg.getDesc();
if (d.hasVendor() && vendor.equals(d.getVendor().getId())) {
if (d.hasPath() && path.equals(d.getPath())) {
return pkg;
}
}
}
return null;
}
/**
* Retrieves information on an extra package identified by its {@link String} vendor/path.
*
* @param vendor The vendor id of the extra package.
* @param path The path uniquely identifying this package for its vendor.
* @return An existing extra package information or null if not found.
*/
@Nullable
public LocalExtraPkgInfo getExtra(@NonNull String vendor, @NonNull String path) {
return (LocalExtraPkgInfo) getPkgInfo(PkgType.PKG_EXTRA, vendor, path);
}
/**
* For unique local packages. Returns the cached LocalPkgInfo for the requested type. Loads it
* from disk if not cached.
*
* @param filter {@link PkgType#PKG_TOOLS} or {@link PkgType#PKG_PLATFORM_TOOLS} or {@link
* PkgType#PKG_DOC}.
* @return null if the package is not installed.
*/
@Nullable
public LocalPkgInfo getPkgInfo(@NonNull PkgType filter) {
if (filter != PkgType.PKG_TOOLS
&& filter != PkgType.PKG_PLATFORM_TOOLS
&& filter != PkgType.PKG_DOC
&& filter != PkgType.PKG_NDK) {
assert false;
return null;
}
LocalPkgInfo info = null;
synchronized (mLocalPackages) {
Collection<LocalPkgInfo> existing = mLocalPackages.get(filter);
assert existing.size() <= 1;
if (!existing.isEmpty()) {
return existing.iterator().next();
}
File uniqueDir = new File(mSdkRoot, filter.getFolderName());
if (!mVisitedDirs.containsEntry(filter, new LocalDirInfo.MapComparator(uniqueDir))) {
switch (filter) {
case PKG_TOOLS:
info = scanTools(uniqueDir);
break;
case PKG_PLATFORM_TOOLS:
info = scanPlatformTools(uniqueDir);
break;
case PKG_DOC:
info = scanDoc(uniqueDir);
break;
case PKG_NDK:
info = scanNdk(uniqueDir);
default:
break;
}
}
// Whether we have found a valid pkg or not, this directory has been visited.
mVisitedDirs.put(filter, new LocalDirInfo(mFileOp, uniqueDir));
if (info != null) {
mLocalPackages.put(filter, info);
}
}
return info;
}
/**
* Retrieve all the info about the requested package type. This is used for the package types that
* have one or more instances, each with different versions. The resulting array is sorted
* according to the PkgInfo's sort order.
*
* <p>Note: you can use this with {@link PkgType#PKG_TOOLS}, {@link PkgType#PKG_PLATFORM_TOOLS}
* and {@link PkgType#PKG_DOC} but since there can only be one package of these types, it is more
* efficient to use {@link #getPkgInfo(PkgType)} to query them.
*
* @param filter One of {@link PkgType} constants.
* @return A list (possibly empty) of matching installed packages. Never returns null.
*/
@NonNull
public LocalPkgInfo[] getPkgsInfos(@NonNull PkgType filter) {
return getPkgsInfos(EnumSet.of(filter));
}
/**
* Retrieve all the info about the requested package types. This is used for the package types
* that have one or more instances, each with different versions. The resulting array is sorted
* according to the PkgInfo's sort order.
*
* <p>To force the LocalSdk parser to load <b>everything</b>, simply call this method with the
* {@link PkgType#PKG_ALL} argument to load all the known package types.
*
* <p>Note: you can use this with {@link PkgType#PKG_TOOLS}, {@link PkgType#PKG_PLATFORM_TOOLS}
* and {@link PkgType#PKG_DOC} but since there can only be one package of these types, it is more
* efficient to use {@link #getPkgInfo(PkgType)} to query them.
*
* @param filters One or more of {@link PkgType#PKG_ADDON}, {@link PkgType#PKG_PLATFORM}, {@link
* PkgType#PKG_BUILD_TOOLS}, {@link PkgType#PKG_EXTRA}, {@link PkgType#PKG_SOURCE}, {@link
* PkgType#PKG_SYS_IMAGE}
* @return A list (possibly empty) of matching installed packages. Never returns null.
*/
@NonNull
public LocalPkgInfo[] getPkgsInfos(@NonNull EnumSet<PkgType> filters) {
List<LocalPkgInfo> list = Lists.newArrayList();
for (PkgType filter : filters) {
if (filter == PkgType.PKG_TOOLS
|| filter == PkgType.PKG_PLATFORM_TOOLS
|| filter == PkgType.PKG_DOC
|| filter == PkgType.PKG_NDK) {
LocalPkgInfo info = getPkgInfo(filter);
if (info != null) {
list.add(info);
}
} else {
synchronized (mLocalPackages) {
Collection<LocalPkgInfo> existing = mLocalPackages.get(filter);
assert existing != null; // Multimap returns an empty set if not found
if (!existing.isEmpty()) {
list.addAll(existing);
continue;
}
File subDir = new File(mSdkRoot, filter.getFolderName());
if (!mVisitedDirs.containsEntry(filter, new LocalDirInfo.MapComparator(subDir))) {
switch (filter) {
case PKG_BUILD_TOOLS:
scanBuildTools(subDir, existing);
break;
case PKG_PLATFORM:
scanPlatforms(subDir, existing);
break;
case PKG_SYS_IMAGE:
scanSysImages(subDir, existing, false);
break;
case PKG_ADDON_SYS_IMAGE:
scanSysImages(subDir, existing, true);
break;
case PKG_ADDON:
scanAddons(subDir, existing);
break;
case PKG_SAMPLE:
scanSamples(subDir, existing);
break;
case PKG_SOURCE:
scanSources(subDir, existing);
break;
case PKG_EXTRA:
scanExtras(subDir, existing);
break;
case PKG_TOOLS:
case PKG_PLATFORM_TOOLS:
case PKG_DOC:
case PKG_NDK:
break;
default:
throw new IllegalArgumentException("Unsupported pkg type " + filter.toString());
}
mVisitedDirs.put(filter, new LocalDirInfo(mFileOp, subDir));
list.addAll(existing);
}
}
}
}
Collections.sort(list);
return list.toArray(new LocalPkgInfo[list.size()]);
}
// ---------- Package-specific querying --------
/**
* Returns the {@link BuildToolInfo} for the given revision.
*
* @param revision The requested revision.
* @return A {@link BuildToolInfo}. Can be null if {@code revision} is null or is not part of the
* known set returned by {@code getPkgsInfos(PkgType.PKG_BUILD_TOOLS)}.
*/
@Nullable
public BuildToolInfo getBuildTool(@Nullable FullRevision revision) {
LocalPkgInfo pkg = getPkgInfo(PkgType.PKG_BUILD_TOOLS, revision);
if (pkg instanceof LocalBuildToolPkgInfo) {
return ((LocalBuildToolPkgInfo) pkg).getBuildToolInfo();
}
return null;
}
/**
* Returns the highest build-tool revision known, or null if there are are no build-tools.
*
* <p>If no specific build-tool package is installed but the platform-tools is lower than 17, then
* this creates and returns a "legacy" built-tool package using platform-tools. (We only split
* build-tools out of platform-tools starting with revision 17, before they were both the same
* thing.)
*
* @return The highest build-tool revision known, or null.
*/
@Nullable
public BuildToolInfo getLatestBuildTool() {
if (mLegacyBuildTools != null) {
return mLegacyBuildTools;
}
LocalPkgInfo[] pkgs = getPkgsInfos(PkgType.PKG_BUILD_TOOLS);
if (pkgs.length == 0) {
LocalPkgInfo ptPkg = getPkgInfo(PkgType.PKG_PLATFORM_TOOLS);
if (ptPkg instanceof LocalPlatformToolPkgInfo
&& ptPkg.getDesc().getFullRevision().compareTo(new FullRevision(17)) < 0) {
// older SDK, create a compatible build-tools
mLegacyBuildTools = createLegacyBuildTools((LocalPlatformToolPkgInfo) ptPkg);
return mLegacyBuildTools;
}
return null;
}
assert pkgs.length > 0;
// Note: the pkgs come from a TreeMultimap so they should already be sorted.
// Just in case, sort them again.
Arrays.sort(pkgs);
// LocalBuildToolPkgInfo's comparator sorts on its FullRevision so we just
// need to take the latest element.
LocalPkgInfo pkg = pkgs[pkgs.length - 1];
if (pkg instanceof LocalBuildToolPkgInfo) {
return ((LocalBuildToolPkgInfo) pkg).getBuildToolInfo();
}
return null;
}
@NonNull
private BuildToolInfo createLegacyBuildTools(@NonNull LocalPlatformToolPkgInfo ptInfo) {
File platformTools = new File(getLocation(), SdkConstants.FD_PLATFORM_TOOLS);
File platformToolsLib = ptInfo.getLocalDir();
File platformToolsRs = new File(platformTools, SdkConstants.FN_FRAMEWORK_RENDERSCRIPT);
return new BuildToolInfo(
ptInfo.getDesc().getFullRevision(),
platformTools,
new File(platformTools, SdkConstants.FN_AAPT),
new File(platformTools, SdkConstants.FN_AIDL),
new File(platformTools, SdkConstants.FN_DX),
new File(platformToolsLib, SdkConstants.FN_DX_JAR),
new File(platformTools, SdkConstants.FN_RENDERSCRIPT),
new File(platformToolsRs, SdkConstants.FN_FRAMEWORK_INCLUDE),
new File(platformToolsRs, SdkConstants.FN_FRAMEWORK_INCLUDE_CLANG),
null,
null,
null,
null,
new File(platformTools, SdkConstants.FN_ZIPALIGN));
}
/**
* Returns the targets (platforms & addons) that are available in the SDK. The target list is
* created on demand the first time then cached. It will not refreshed unless {@link
* #clearLocalPkg} is called to clear platforms and/or add-ons.
*
* <p>The array can be empty but not null.
*/
@NonNull
public IAndroidTarget[] getTargets() {
synchronized (mLocalPackages) {
if (mCachedTargets == null) {
List<IAndroidTarget> result = Lists.newArrayList();
LocalPkgInfo[] pkgsInfos =
getPkgsInfos(EnumSet.of(PkgType.PKG_PLATFORM, PkgType.PKG_ADDON));
for (LocalPkgInfo info : pkgsInfos) {
assert info instanceof LocalPlatformPkgInfo;
IAndroidTarget target = ((LocalPlatformPkgInfo) info).getAndroidTarget();
if (target != null) {
result.add(target);
}
}
mCachedTargets = result;
}
return mCachedTargets.toArray(new IAndroidTarget[mCachedTargets.size()]);
}
}
public IAndroidTarget[] getTargets(boolean includeMissing) {
IAndroidTarget[] result = getTargets();
if (includeMissing) {
result = ObjectArrays.concat(result, getMissingTargets(), IAndroidTarget.class);
}
return result;
}
@NonNull
public IAndroidTarget[] getMissingTargets() {
synchronized (mLocalPackages) {
if (mCachedMissingTargets == null) {
Map<MissingTarget, MissingTarget> result = Maps.newHashMap();
Set<ISystemImage> seen = Sets.newHashSet();
for (IAndroidTarget target : getTargets()) {
Collections.addAll(seen, target.getSystemImages());
}
for (LocalPkgInfo local : getPkgsInfos(PkgType.PKG_ADDON_SYS_IMAGE)) {
LocalAddonSysImgPkgInfo info = (LocalAddonSysImgPkgInfo) local;
ISystemImage image = info.getSystemImage();
if (!seen.contains(image)) {
addOrphanedSystemImage(image, info.getDesc(), result);
}
}
for (LocalPkgInfo local : getPkgsInfos(PkgType.PKG_SYS_IMAGE)) {
LocalSysImgPkgInfo info = (LocalSysImgPkgInfo) local;
ISystemImage image = info.getSystemImage();
if (!seen.contains(image)) {
addOrphanedSystemImage(image, info.getDesc(), result);
}
}
mCachedMissingTargets = result.keySet();
}
return mCachedMissingTargets.toArray(new IAndroidTarget[mCachedMissingTargets.size()]);
}
}
private static void addOrphanedSystemImage(
ISystemImage image, IPkgDesc desc, Map<MissingTarget, MissingTarget> targets) {
IdDisplay vendor = desc.getVendor();
MissingTarget target =
new MissingTarget(
vendor == null ? null : vendor.getDisplay(),
desc.getTag().getDisplay(),
desc.getAndroidVersion());
MissingTarget existing = targets.get(target);
if (existing == null) {
existing = target;
targets.put(target, target);
}
existing.addSystemImage(image);
}
/**
* Returns a target from a hash that was generated by {@link IAndroidTarget#hashString()}.
*
* @param hash the {@link IAndroidTarget} hash string.
* @return The matching {@link IAndroidTarget} or null.
*/
@Nullable
public IAndroidTarget getTargetFromHashString(@Nullable String hash) {
if (hash != null) {
IAndroidTarget[] targets = getTargets(true);
for (IAndroidTarget target : targets) {
if (target != null && hash.equals(AndroidTargetHash.getTargetHashString(target))) {
return target;
}
}
}
return null;
}
// -------------
/** Try to find a tools package at the given location. Returns null if not found. */
private LocalToolPkgInfo scanTools(File toolFolder) {
// Can we find some properties?
Properties props = parseProperties(new File(toolFolder, SdkConstants.FN_SOURCE_PROP));
FullRevision rev = PackageParserUtils.getPropertyFull(props, PkgProps.PKG_REVISION);
if (rev == null) {
return null;
}
FullRevision minPlatToolsRev =
PackageParserUtils.getPropertyFull(props, PkgProps.MIN_PLATFORM_TOOLS_REV);
if (minPlatToolsRev == null) {
minPlatToolsRev = FullRevision.NOT_SPECIFIED;
}
LocalToolPkgInfo info = new LocalToolPkgInfo(this, toolFolder, props, rev, minPlatToolsRev);
// We're not going to check that all tools are present. At the very least
// we should expect to find android and an emulator adapted to the current OS.
boolean hasEmulator = false;
boolean hasAndroid = false;
String android1 = SdkConstants.androidCmdName().replace(".bat", ".exe");
String android2 = android1.indexOf('.') == -1 ? null : android1.replace(".exe", ".bat");
File[] files = mFileOp.listFiles(toolFolder);
for (File file : files) {
String name = file.getName();
if (SdkConstants.FN_EMULATOR.equals(name)) {
hasEmulator = true;
}
if (android1.equals(name) || (android2 != null && android2.equals(name))) {
hasAndroid = true;
}
}
if (!hasAndroid) {
info.appendLoadError("Missing %1$s", SdkConstants.androidCmdName());
}
if (!hasEmulator) {
info.appendLoadError("Missing %1$s", SdkConstants.FN_EMULATOR);
}
return info;
}
/** Try to find a platform-tools package at the given location. Returns null if not found. */
private LocalPlatformToolPkgInfo scanPlatformTools(File ptFolder) {
// Can we find some properties?
Properties props = parseProperties(new File(ptFolder, SdkConstants.FN_SOURCE_PROP));
FullRevision rev = PackageParserUtils.getPropertyFull(props, PkgProps.PKG_REVISION);
if (rev == null) {
return null;
}
LocalPlatformToolPkgInfo info = new LocalPlatformToolPkgInfo(this, ptFolder, props, rev);
return info;
}
/** Try to find a docs package at the given location. Returns null if not found. */
private LocalDocPkgInfo scanDoc(File docFolder) {
// Can we find some properties?
Properties props = parseProperties(new File(docFolder, SdkConstants.FN_SOURCE_PROP));
MajorRevision rev = PackageParserUtils.getPropertyMajor(props, PkgProps.PKG_REVISION);
if (rev == null) {
return null;
}
try {
AndroidVersion vers = new AndroidVersion(props);
LocalDocPkgInfo info = new LocalDocPkgInfo(this, docFolder, props, vers, rev);
// To start with, a doc folder should have an "index.html" to be acceptable.
// We don't actually check the content of the file.
if (!mFileOp.isFile(new File(docFolder, "index.html"))) {
info.appendLoadError("Missing index.html");
}
return info;
} catch (AndroidVersionException e) {
return null; // skip invalid or missing android version.
}
}
/** Try to find an NDK package at the given location. Returns null if not found. */
@Nullable
private LocalNdkPkgInfo scanNdk(@NonNull File ndkFolder) {
// Can we find some properties?
Properties props = parseProperties(new File(ndkFolder, SdkConstants.FN_SOURCE_PROP));
FullRevision rev = PackageParserUtils.getPropertyFull(props, PkgProps.PKG_REVISION);
if (rev == null) {
return null;
}
return new LocalNdkPkgInfo(this, ndkFolder, props, rev);
}
/**
* Helper used by scanXyz methods below to check whether a directory should be visited. It can be
* skipped if it's not a directory or if it's already marked as visited in mVisitedDirs for the
* given package type -- in which case the directory is added to the visited map.
*
* @param pkgType The package type being scanned.
* @param directory The file or directory to check.
* @return False if directory can/should be skipped. True if directory should be visited, in which
* case it's registered in mVisitedDirs.
*/
private boolean shouldVisitDir(@NonNull PkgType pkgType, @NonNull File directory) {
if (!mFileOp.isDirectory(directory)) {
return false;
}
synchronized (mLocalPackages) {
if (mVisitedDirs.containsEntry(pkgType, new LocalDirInfo.MapComparator(directory))) {
return false;
}
mVisitedDirs.put(pkgType, new LocalDirInfo(mFileOp, directory));
}
return true;
}
private void scanBuildTools(File collectionDir, Collection<LocalPkgInfo> outCollection) {
// The build-tool root folder contains a list of per-revision folders.
for (File buildToolDir : mFileOp.listFiles(collectionDir)) {
if (!shouldVisitDir(PkgType.PKG_BUILD_TOOLS, buildToolDir)) {
continue;
}
Properties props = parseProperties(new File(buildToolDir, SdkConstants.FN_SOURCE_PROP));
FullRevision rev = PackageParserUtils.getPropertyFull(props, PkgProps.PKG_REVISION);
if (rev == null) {
continue; // skip, no revision
}
BuildToolInfo btInfo = new BuildToolInfo(rev, buildToolDir);
LocalBuildToolPkgInfo pkgInfo =
new LocalBuildToolPkgInfo(this, buildToolDir, props, rev, btInfo);
outCollection.add(pkgInfo);
}
}
private void scanPlatforms(File collectionDir, Collection<LocalPkgInfo> outCollection) {
for (File platformDir : mFileOp.listFiles(collectionDir)) {
if (!shouldVisitDir(PkgType.PKG_PLATFORM, platformDir)) {
continue;
}
Properties props = parseProperties(new File(platformDir, SdkConstants.FN_SOURCE_PROP));
MajorRevision rev = PackageParserUtils.getPropertyMajor(props, PkgProps.PKG_REVISION);
if (rev == null) {
continue; // skip, no revision
}
FullRevision minToolsRev = PackageParserUtils.getPropertyFull(props, PkgProps.MIN_TOOLS_REV);
if (minToolsRev == null) {
minToolsRev = FullRevision.NOT_SPECIFIED;
}
try {
AndroidVersion vers = new AndroidVersion(props);
LocalPlatformPkgInfo pkgInfo =
new LocalPlatformPkgInfo(this, platformDir, props, vers, rev, minToolsRev);
outCollection.add(pkgInfo);
} catch (AndroidVersionException e) {
continue; // skip invalid or missing android version.
}
}
}
private void scanAddons(File collectionDir, Collection<LocalPkgInfo> outCollection) {
for (File addonDir : mFileOp.listFiles(collectionDir)) {
if (!shouldVisitDir(PkgType.PKG_ADDON, addonDir)) {
continue;
}
Properties props = parseProperties(new File(addonDir, SdkConstants.FN_SOURCE_PROP));
MajorRevision rev = PackageParserUtils.getPropertyMajor(props, PkgProps.PKG_REVISION);
if (rev == null) {
continue; // skip, no revision
}
try {
AndroidVersion vers = new AndroidVersion(props);
// Starting with addon-4.xsd, we have vendor-id and name-id available
// in the add-on source properties so we'll use that directly.
String nameId = props.getProperty(PkgProps.ADDON_NAME_ID);
String nameDisp = props.getProperty(PkgProps.ADDON_NAME_DISPLAY);
String vendorId = props.getProperty(PkgProps.ADDON_VENDOR_ID);
String vendorDisp = props.getProperty(PkgProps.ADDON_VENDOR_DISPLAY);
if (nameId == null) {
// Support earlier add-ons that only had a name display attribute
nameDisp = props.getProperty(PkgProps.ADDON_NAME, "Unknown");
nameId = LocalAddonPkgInfo.sanitizeDisplayToNameId(nameDisp);
}
if (nameId != null && nameDisp == null) {
nameDisp = LocalExtraPkgInfo.getPrettyName(null, nameId);
}
if (vendorId != null && vendorDisp == null) {
vendorDisp = LocalExtraPkgInfo.getPrettyName(null, nameId);
}
if (vendorId == null) {
// Support earlier add-ons that only had a vendor display attribute
vendorDisp = props.getProperty(PkgProps.ADDON_VENDOR, "Unknown");
vendorId = LocalAddonPkgInfo.sanitizeDisplayToNameId(vendorDisp);
}
LocalAddonPkgInfo pkgInfo =
new LocalAddonPkgInfo(
this,
addonDir,
props,
vers,
rev,
new IdDisplay(vendorId, vendorDisp),
new IdDisplay(nameId, nameDisp));
outCollection.add(pkgInfo);
} catch (AndroidVersionException e) {
continue; // skip invalid or missing android version.
}
}
}
private void scanSysImages(
File collectionDir, Collection<LocalPkgInfo> outCollection, boolean scanAddons) {
List<File> propFiles = Lists.newArrayList();
PkgType type = scanAddons ? PkgType.PKG_ADDON_SYS_IMAGE : PkgType.PKG_SYS_IMAGE;
// Create a list of folders that contains a source.properties file matching these patterns:
// sys-img/target/tag/abi
// sys-img/target/abis
// sys-img/add-on-target/abi
// sys-img/target/add-on/abi
for (File platformDir : mFileOp.listFiles(collectionDir)) {
if (!shouldVisitDir(type, platformDir)) {
continue;
}
for (File dir1 : mFileOp.listFiles(platformDir)) {
// dir1 might be either a tag or an abi folder.
if (!shouldVisitDir(type, dir1)) {
continue;
}
File prop1 = new File(dir1, SdkConstants.FN_SOURCE_PROP);
if (mFileOp.isFile(prop1)) {
// dir1 was a legacy abi folder.
if (!propFiles.contains(prop1)) {
propFiles.add(prop1);
}
} else {
File[] dir1Files = mFileOp.listFiles(dir1);
for (File dir2 : dir1Files) {
// dir2 should be an abi folder in a tag folder.
if (!shouldVisitDir(type, dir2)) {
continue;
}
File prop2 = new File(dir2, SdkConstants.FN_SOURCE_PROP);
if (mFileOp.isFile(prop2)) {
if (!propFiles.contains(prop2)) {
propFiles.add(prop2);
}
}
}
}
}
}
for (File propFile : propFiles) {
Properties props = parseProperties(propFile);
MajorRevision rev = PackageParserUtils.getPropertyMajor(props, PkgProps.PKG_REVISION);
if (rev == null) {
continue; // skip, no revision
}
try {
AndroidVersion vers = new AndroidVersion(props);
IdDisplay tag = LocalSysImgPkgInfo.extractTagFromProps(props);
String vendorId = props.getProperty(PkgProps.ADDON_VENDOR_ID, null);
File abiDir = propFile.getParentFile();
if (vendorId == null && !scanAddons) {
LocalSysImgPkgInfo pkgInfo =
new LocalSysImgPkgInfo(this, abiDir, props, vers, tag, abiDir.getName(), rev);
outCollection.add(pkgInfo);
} else if (vendorId != null && scanAddons) {
String vendorDisp = props.getProperty(PkgProps.ADDON_VENDOR_DISPLAY, vendorId);
IdDisplay vendor = new IdDisplay(vendorId, vendorDisp);
LocalAddonSysImgPkgInfo pkgInfo =
new LocalAddonSysImgPkgInfo(
this, abiDir, props, vers, vendor, tag, abiDir.getName(), rev);
outCollection.add(pkgInfo);
}
} catch (AndroidVersionException e) {
continue; // skip invalid or missing android version.
}
}
}
private void scanSamples(File collectionDir, Collection<LocalPkgInfo> outCollection) {
for (File platformDir : mFileOp.listFiles(collectionDir)) {
if (!shouldVisitDir(PkgType.PKG_SAMPLE, platformDir)) {
continue;
}
Properties props = parseProperties(new File(platformDir, SdkConstants.FN_SOURCE_PROP));
MajorRevision rev = PackageParserUtils.getPropertyMajor(props, PkgProps.PKG_REVISION);
if (rev == null) {
continue; // skip, no revision
}
FullRevision minToolsRev = PackageParserUtils.getPropertyFull(props, PkgProps.MIN_TOOLS_REV);
if (minToolsRev == null) {
minToolsRev = FullRevision.NOT_SPECIFIED;
}
try {
AndroidVersion vers = new AndroidVersion(props);
LocalSamplePkgInfo pkgInfo =
new LocalSamplePkgInfo(this, platformDir, props, vers, rev, minToolsRev);
outCollection.add(pkgInfo);
} catch (AndroidVersionException e) {
continue; // skip invalid or missing android version.
}
}
}
private void scanSources(File collectionDir, Collection<LocalPkgInfo> outCollection) {
// The build-tool root folder contains a list of per-revision folders.
for (File platformDir : mFileOp.listFiles(collectionDir)) {
if (!shouldVisitDir(PkgType.PKG_SOURCE, platformDir)) {
continue;
}
Properties props = parseProperties(new File(platformDir, SdkConstants.FN_SOURCE_PROP));
MajorRevision rev = PackageParserUtils.getPropertyMajor(props, PkgProps.PKG_REVISION);
if (rev == null) {
continue; // skip, no revision
}
try {
AndroidVersion vers = new AndroidVersion(props);
LocalSourcePkgInfo pkgInfo = new LocalSourcePkgInfo(this, platformDir, props, vers, rev);
outCollection.add(pkgInfo);
} catch (AndroidVersionException e) {
continue; // skip invalid or missing android version.
}
}
}
private void scanExtras(File collectionDir, Collection<LocalPkgInfo> outCollection) {
for (File vendorDir : mFileOp.listFiles(collectionDir)) {
if (!shouldVisitDir(PkgType.PKG_EXTRA, vendorDir)) {
continue;
}
for (File extraDir : mFileOp.listFiles(vendorDir)) {
if (!shouldVisitDir(PkgType.PKG_EXTRA, extraDir)) {
continue;
}
Properties props = parseProperties(new File(extraDir, SdkConstants.FN_SOURCE_PROP));
NoPreviewRevision rev =
PackageParserUtils.getPropertyNoPreview(props, PkgProps.PKG_REVISION);
if (rev == null) {
continue; // skip, no revision
}
String oldPaths = PackageParserUtils.getProperty(props, PkgProps.EXTRA_OLD_PATHS, null);
String vendorId = vendorDir.getName();
String vendorDisp = props.getProperty(PkgProps.EXTRA_VENDOR_DISPLAY);
if (vendorDisp == null || vendorDisp.isEmpty()) {
vendorDisp = vendorId;
}
String displayName = props.getProperty(PkgProps.EXTRA_NAME_DISPLAY, null);
LocalExtraPkgInfo pkgInfo =
new LocalExtraPkgInfo(
this,
extraDir,
props,
new IdDisplay(vendorId, vendorDisp),
extraDir.getName(),
displayName,
PkgDescExtra.convertOldPaths(oldPaths),
rev);
outCollection.add(pkgInfo);
}
}
}
/**
* Parses the given file as properties file if it exists. Returns null if the file does not exist,
* cannot be parsed or has no properties.
*/
private Properties parseProperties(File propsFile) {
InputStream fis = null;
try {
if (mFileOp.exists(propsFile)) {
fis = mFileOp.newFileInputStream(propsFile);
Properties props = new Properties();
props.load(fis);
// To be valid, there must be at least one property in it.
if (!props.isEmpty()) {
return props;
}
}
} catch (IOException e) {
// Ignore
} finally {
if (fis != null) {
try {
fis.close();
} catch (IOException e) {
}
}
}
return null;
}
}
public static Document generalResultSegmentation(
Document doc, String labeledResult, List<LayoutToken> documentTokens) {
List<Pair<String, String>> labeledTokens = GenericTaggerUtils.getTokensAndLabels(labeledResult);
SortedSetMultimap<String, DocumentPiece> labeledBlocks = TreeMultimap.create();
doc.setLabeledBlocks(labeledBlocks);
/*try {
FileUtils.writeStringToFile(new File("/tmp/x1.txt"), labeledResult);
FileUtils.writeStringToFile(new File("/tmp/x2.txt"), documentTokens.toString());
}
catch(Exception e) {
e.printStackTrace();
}*/
List<Block> docBlocks = doc.getBlocks();
int indexLine = 0;
int blockIndex = 0;
int p = 0; // position in the labeled result
int currentLineEndPos = 0; // position in the global doc. tokenization of the last
// token of the current line
int currentLineStartPos = 0; // position in the global doc.
// tokenization of the first token of the current line
String line = null;
DocumentPointer pointerA = DocumentPointer.START_DOCUMENT_POINTER;
DocumentPointer currentPointer = null;
DocumentPointer lastPointer = null;
String curLabel;
String curPlainLabel = null;
String lastPlainLabel = null;
int lastTokenInd = -1;
for (int i = docBlocks.size() - 1; i >= 0; i--) {
int endToken = docBlocks.get(i).getEndToken();
if (endToken != -1) {
lastTokenInd = endToken;
break;
}
}
// we do this concatenation trick so that we don't have to process stuff after the main loop
// no copying of lists happens because of this, so it's ok to concatenate
String ignoredLabel = "@IGNORED_LABEL@";
for (Pair<String, String> labeledTokenPair :
Iterables.concat(
labeledTokens,
Collections.singleton(new Pair<String, String>("IgnoredToken", ignoredLabel)))) {
if (labeledTokenPair == null) {
p++;
continue;
}
// as we process the document segmentation line by line, we don't use the usual
// tokenization to rebuild the text flow, but we get each line again from the
// text stored in the document blocks (similarly as when generating the features)
line = null;
while ((line == null) && (blockIndex < docBlocks.size())) {
Block block = docBlocks.get(blockIndex);
List<LayoutToken> tokens = block.getTokens();
String localText = block.getText();
if ((tokens == null) || (localText == null) || (localText.trim().length() == 0)) {
blockIndex++;
indexLine = 0;
if (blockIndex < docBlocks.size()) {
block = docBlocks.get(blockIndex);
currentLineStartPos = block.getStartToken();
}
continue;
}
String[] lines = localText.split("[\\n\\r]");
if ((lines.length == 0) || (indexLine >= lines.length)) {
blockIndex++;
indexLine = 0;
if (blockIndex < docBlocks.size()) {
block = docBlocks.get(blockIndex);
currentLineStartPos = block.getStartToken();
}
continue;
} else {
line = lines[indexLine];
indexLine++;
if ((line.trim().length() == 0) || (TextUtilities.filterLine(line))) {
line = null;
continue;
}
if (currentLineStartPos > lastTokenInd) continue;
// adjust the start token position in documentTokens to this non trivial line
// first skip possible space characters and tabs at the beginning of the line
while ((documentTokens.get(currentLineStartPos).t().equals(" ")
|| documentTokens.get(currentLineStartPos).t().equals("\t"))
&& (currentLineStartPos != lastTokenInd)) {
currentLineStartPos++;
}
if (!labeledTokenPair.a.startsWith(documentTokens.get(currentLineStartPos).getText())) {
while (currentLineStartPos < block.getEndToken()) {
if (documentTokens.get(currentLineStartPos).t().equals("\n")
|| documentTokens.get(currentLineStartPos).t().equals("\r")) {
// move to the start of the next line, but ignore space characters and tabs
currentLineStartPos++;
while ((documentTokens.get(currentLineStartPos).t().equals(" ")
|| documentTokens.get(currentLineStartPos).t().equals("\t"))
&& (currentLineStartPos != lastTokenInd)) {
currentLineStartPos++;
}
if ((currentLineStartPos != lastTokenInd)
&& labeledTokenPair.a.startsWith(
documentTokens.get(currentLineStartPos).getText())) {
break;
}
}
currentLineStartPos++;
}
}
// what is then the position of the last token of this line?
currentLineEndPos = currentLineStartPos;
while (currentLineEndPos < block.getEndToken()) {
if (documentTokens.get(currentLineEndPos).t().equals("\n")
|| documentTokens.get(currentLineEndPos).t().equals("\r")) {
currentLineEndPos--;
break;
}
currentLineEndPos++;
}
}
}
curLabel = labeledTokenPair.b;
curPlainLabel = GenericTaggerUtils.getPlainLabel(curLabel);
/*System.out.println("-------------------------------");
System.out.println("block: " + blockIndex);
System.out.println("line: " + line);
System.out.println("token: " + labeledTokenPair.a);
System.out.println("curPlainLabel: " + curPlainLabel);
System.out.println("lastPlainLabel: " + lastPlainLabel);
if ((currentLineStartPos < lastTokenInd) && (currentLineStartPos != -1))
System.out.println("currentLineStartPos: " + currentLineStartPos +
" (" + documentTokens.get(currentLineStartPos) + ")");
if ((currentLineEndPos < lastTokenInd) && (currentLineEndPos != -1))
System.out.println("currentLineEndPos: " + currentLineEndPos +
" (" + documentTokens.get(currentLineEndPos) + ")");*/
if (blockIndex == docBlocks.size()) {
break;
}
currentPointer = new DocumentPointer(doc, blockIndex, currentLineEndPos);
// either a new entity starts or a new beginning of the same type of entity
if ((!curPlainLabel.equals(lastPlainLabel)) && (lastPlainLabel != null)) {
if ((pointerA.getTokenDocPos() <= lastPointer.getTokenDocPos())
&& (pointerA.getTokenDocPos() != -1)) {
labeledBlocks.put(lastPlainLabel, new DocumentPiece(pointerA, lastPointer));
}
pointerA = new DocumentPointer(doc, blockIndex, currentLineStartPos);
// System.out.println("add segment for: " + lastPlainLabel + ", until " +
// (currentLineStartPos-2));
}
// updating stuff for next iteration
lastPlainLabel = curPlainLabel;
lastPointer = currentPointer;
currentLineStartPos = currentLineEndPos + 2; // one shift for the EOL, one for the next line
p++;
}
if (blockIndex == docBlocks.size()) {
// the last labelled piece has still to be added
if ((!curPlainLabel.equals(lastPlainLabel)) && (lastPlainLabel != null)) {
if ((pointerA.getTokenDocPos() <= lastPointer.getTokenDocPos())
&& (pointerA.getTokenDocPos() != -1)) {
labeledBlocks.put(lastPlainLabel, new DocumentPiece(pointerA, lastPointer));
// System.out.println("add segment for: " + lastPlainLabel + ", until " +
// (currentLineStartPos-2));
}
}
}
return doc;
}
@Override
public Iterator<T> iterator() {
return checkNotNull(fStartTimesIndex.values().iterator());
}
@Override
public synchronized void dispose() {
fStartTimesIndex.clear();
fEndTimesIndex.clear();
fSize = 0;
}
/**
* Supplies an arbitrary "default" instance for a wide range of types, often useful in testing
* utilities.
*
* <p>Covers arrays, enums and common types defined in {@code java.lang}, {@code java.lang.reflect},
* {@code java.io}, {@code java.nio}, {@code java.math}, {@code java.util}, {@code
* java.util.concurrent}, {@code java.util.regex}, {@code com.google.common.base}, {@code
* com.google.common.collect} and {@code com.google.common.primitives}. In addition, if the type
* exposes at least one public static final constant of the same type, one of the constants will be
* used; or if the class exposes a public parameter-less constructor then it will be "new"d and
* returned.
*
* <p>All default instances returned by {@link #get} are generics-safe. Clients won't get type
* errors for using {@code get(Comparator.class)} as a {@code Comparator<Foo>}, for example.
* Immutable empty instances are returned for collection types; {@code ""} for string; {@code 0} for
* number types; reasonable default instance for other stateless types. For mutable types, a fresh
* instance is created each time {@code get()} is called.
*
* @author Kevin Bourrillion
* @author Ben Yu
* @since 12.0
*/
@Beta
public final class ArbitraryInstances {
private static final Ordering<Field> BY_FIELD_NAME =
new Ordering<Field>() {
@Override
public int compare(Field left, Field right) {
return left.getName().compareTo(right.getName());
}
};
/**
* Returns a new {@code MatchResult} that corresponds to a successful match. Apache Harmony (used
* in Android) requires a successful match in order to generate a {@code MatchResult}:
* http://goo.gl/5VQFmC
*/
private static MatchResult newMatchResult() {
Matcher matcher = Pattern.compile(".").matcher("X");
matcher.find();
return matcher.toMatchResult();
}
private static final ClassToInstanceMap<Object> DEFAULTS =
ImmutableClassToInstanceMap.builder()
// primitives
.put(Object.class, "")
.put(Number.class, 0)
.put(UnsignedInteger.class, UnsignedInteger.ZERO)
.put(UnsignedLong.class, UnsignedLong.ZERO)
.put(BigInteger.class, BigInteger.ZERO)
.put(BigDecimal.class, BigDecimal.ZERO)
.put(CharSequence.class, "")
.put(String.class, "")
.put(Pattern.class, Pattern.compile(""))
.put(MatchResult.class, newMatchResult())
.put(TimeUnit.class, TimeUnit.SECONDS)
.put(Charset.class, Charsets.UTF_8)
.put(Currency.class, Currency.getInstance(Locale.US))
.put(Locale.class, Locale.US)
// common.base
.put(CharMatcher.class, CharMatcher.NONE)
.put(Joiner.class, Joiner.on(','))
.put(Splitter.class, Splitter.on(','))
.put(Optional.class, Optional.absent())
.put(Predicate.class, Predicates.alwaysTrue())
.put(Equivalence.class, Equivalence.equals())
.put(Ticker.class, Ticker.systemTicker())
.put(Stopwatch.class, Stopwatch.createUnstarted())
// io types
.put(InputStream.class, new ByteArrayInputStream(new byte[0]))
.put(ByteArrayInputStream.class, new ByteArrayInputStream(new byte[0]))
.put(Readable.class, new StringReader(""))
.put(Reader.class, new StringReader(""))
.put(StringReader.class, new StringReader(""))
.put(Buffer.class, ByteBuffer.allocate(0))
.put(CharBuffer.class, CharBuffer.allocate(0))
.put(ByteBuffer.class, ByteBuffer.allocate(0))
.put(ShortBuffer.class, ShortBuffer.allocate(0))
.put(IntBuffer.class, IntBuffer.allocate(0))
.put(LongBuffer.class, LongBuffer.allocate(0))
.put(FloatBuffer.class, FloatBuffer.allocate(0))
.put(DoubleBuffer.class, DoubleBuffer.allocate(0))
.put(File.class, new File(""))
.put(ByteSource.class, ByteSource.empty())
.put(CharSource.class, CharSource.empty())
.put(ByteSink.class, NullByteSink.INSTANCE)
.put(CharSink.class, NullByteSink.INSTANCE.asCharSink(Charsets.UTF_8))
// All collections are immutable empty. So safe for any type parameter.
.put(Iterator.class, ImmutableSet.of().iterator())
.put(PeekingIterator.class, Iterators.peekingIterator(ImmutableSet.of().iterator()))
.put(ListIterator.class, ImmutableList.of().listIterator())
.put(Iterable.class, ImmutableSet.of())
.put(Collection.class, ImmutableList.of())
.put(ImmutableCollection.class, ImmutableList.of())
.put(List.class, ImmutableList.of())
.put(ImmutableList.class, ImmutableList.of())
.put(Set.class, ImmutableSet.of())
.put(ImmutableSet.class, ImmutableSet.of())
.put(SortedSet.class, ImmutableSortedSet.of())
.put(ImmutableSortedSet.class, ImmutableSortedSet.of())
.put(NavigableSet.class, Sets.unmodifiableNavigableSet(Sets.newTreeSet()))
.put(Map.class, ImmutableMap.of())
.put(ImmutableMap.class, ImmutableMap.of())
.put(SortedMap.class, ImmutableSortedMap.of())
.put(ImmutableSortedMap.class, ImmutableSortedMap.of())
.put(NavigableMap.class, Maps.unmodifiableNavigableMap(Maps.newTreeMap()))
.put(Multimap.class, ImmutableMultimap.of())
.put(ImmutableMultimap.class, ImmutableMultimap.of())
.put(ListMultimap.class, ImmutableListMultimap.of())
.put(ImmutableListMultimap.class, ImmutableListMultimap.of())
.put(SetMultimap.class, ImmutableSetMultimap.of())
.put(ImmutableSetMultimap.class, ImmutableSetMultimap.of())
.put(
SortedSetMultimap.class,
Multimaps.unmodifiableSortedSetMultimap(TreeMultimap.create()))
.put(Multiset.class, ImmutableMultiset.of())
.put(ImmutableMultiset.class, ImmutableMultiset.of())
.put(SortedMultiset.class, ImmutableSortedMultiset.of())
.put(ImmutableSortedMultiset.class, ImmutableSortedMultiset.of())
.put(BiMap.class, ImmutableBiMap.of())
.put(ImmutableBiMap.class, ImmutableBiMap.of())
.put(Table.class, ImmutableTable.of())
.put(ImmutableTable.class, ImmutableTable.of())
.put(RowSortedTable.class, Tables.unmodifiableRowSortedTable(TreeBasedTable.create()))
.put(ClassToInstanceMap.class, ImmutableClassToInstanceMap.builder().build())
.put(ImmutableClassToInstanceMap.class, ImmutableClassToInstanceMap.builder().build())
.put(Comparable.class, ByToString.INSTANCE)
.put(Comparator.class, AlwaysEqual.INSTANCE)
.put(Ordering.class, AlwaysEqual.INSTANCE)
.put(Range.class, Range.all())
.put(MapConstraint.class, MapConstraints.notNull())
.put(MapDifference.class, Maps.difference(ImmutableMap.of(), ImmutableMap.of()))
.put(
SortedMapDifference.class,
Maps.difference(ImmutableSortedMap.of(), ImmutableSortedMap.of()))
// reflect
.put(AnnotatedElement.class, Object.class)
.put(GenericDeclaration.class, Object.class)
.put(Type.class, Object.class)
.build();
/**
* type -> implementation. Inherently mutable interfaces and abstract classes are mapped to their
* default implementations and are "new"d upon get().
*/
private static final ConcurrentMap<Class<?>, Class<?>> implementations = Maps.newConcurrentMap();
private static <T> void setImplementation(Class<T> type, Class<? extends T> implementation) {
checkArgument(type != implementation, "Don't register %s to itself!", type);
checkArgument(
!DEFAULTS.containsKey(type), "A default value was already registered for %s", type);
checkArgument(
implementations.put(type, implementation) == null,
"Implementation for %s was already registered",
type);
}
static {
setImplementation(Appendable.class, StringBuilder.class);
setImplementation(BlockingQueue.class, LinkedBlockingDeque.class);
setImplementation(BlockingDeque.class, LinkedBlockingDeque.class);
setImplementation(ConcurrentMap.class, ConcurrentHashMap.class);
setImplementation(ConcurrentNavigableMap.class, ConcurrentSkipListMap.class);
setImplementation(CountDownLatch.class, Dummies.DummyCountDownLatch.class);
setImplementation(Deque.class, ArrayDeque.class);
setImplementation(OutputStream.class, ByteArrayOutputStream.class);
setImplementation(PrintStream.class, Dummies.InMemoryPrintStream.class);
setImplementation(PrintWriter.class, Dummies.InMemoryPrintWriter.class);
setImplementation(Queue.class, ArrayDeque.class);
setImplementation(Random.class, Dummies.DeterministicRandom.class);
setImplementation(
ScheduledThreadPoolExecutor.class, Dummies.DummyScheduledThreadPoolExecutor.class);
setImplementation(ThreadPoolExecutor.class, Dummies.DummyScheduledThreadPoolExecutor.class);
setImplementation(Writer.class, StringWriter.class);
setImplementation(Runnable.class, Dummies.DummyRunnable.class);
setImplementation(ThreadFactory.class, Dummies.DummyThreadFactory.class);
setImplementation(Executor.class, Dummies.DummyExecutor.class);
}
@SuppressWarnings("unchecked") // it's a subtype map
@Nullable
private static <T> Class<? extends T> getImplementation(Class<T> type) {
return (Class<? extends T>) implementations.get(type);
}
private static final Logger logger = Logger.getLogger(ArbitraryInstances.class.getName());
/**
* Returns an arbitrary instance for {@code type}, or {@code null} if no arbitrary instance can be
* determined.
*/
@Nullable
public static <T> T get(Class<T> type) {
T defaultValue = DEFAULTS.getInstance(type);
if (defaultValue != null) {
return defaultValue;
}
Class<? extends T> implementation = getImplementation(type);
if (implementation != null) {
return get(implementation);
}
if (type.isEnum()) {
T[] enumConstants = type.getEnumConstants();
return (enumConstants.length == 0) ? null : enumConstants[0];
}
if (type.isArray()) {
return createEmptyArray(type);
}
T jvmDefault = Defaults.defaultValue(Primitives.unwrap(type));
if (jvmDefault != null) {
return jvmDefault;
}
if (Modifier.isAbstract(type.getModifiers()) || !Modifier.isPublic(type.getModifiers())) {
return arbitraryConstantInstanceOrNull(type);
}
final Constructor<T> constructor;
try {
constructor = type.getConstructor();
} catch (NoSuchMethodException e) {
return arbitraryConstantInstanceOrNull(type);
}
constructor.setAccessible(true); // accessibility check is too slow
try {
return constructor.newInstance();
} catch (InstantiationException impossible) {
throw new AssertionError(impossible);
} catch (IllegalAccessException impossible) {
throw new AssertionError(impossible);
} catch (InvocationTargetException e) {
logger.log(Level.WARNING, "Exception while invoking default constructor.", e.getCause());
return arbitraryConstantInstanceOrNull(type);
}
}
@Nullable
private static <T> T arbitraryConstantInstanceOrNull(Class<T> type) {
Field[] fields = type.getDeclaredFields();
Arrays.sort(fields, BY_FIELD_NAME);
for (Field field : fields) {
if (Modifier.isPublic(field.getModifiers())
&& Modifier.isStatic(field.getModifiers())
&& Modifier.isFinal(field.getModifiers())) {
if (field.getGenericType() == field.getType() && type.isAssignableFrom(field.getType())) {
field.setAccessible(true);
try {
T constant = type.cast(field.get(null));
if (constant != null) {
return constant;
}
} catch (IllegalAccessException impossible) {
throw new AssertionError(impossible);
}
}
}
}
return null;
}
private static <T> T createEmptyArray(Class<T> arrayType) {
return arrayType.cast(Array.newInstance(arrayType.getComponentType(), 0));
}
// Internal implementations of some classes, with public default constructor that get() needs.
private static final class Dummies {
public static final class InMemoryPrintStream extends PrintStream {
public InMemoryPrintStream() {
super(new ByteArrayOutputStream());
}
}
public static final class InMemoryPrintWriter extends PrintWriter {
public InMemoryPrintWriter() {
super(new StringWriter());
}
}
public static final class DeterministicRandom extends Random {
public DeterministicRandom() {
super(0);
}
}
public static final class DummyScheduledThreadPoolExecutor extends ScheduledThreadPoolExecutor {
public DummyScheduledThreadPoolExecutor() {
super(1);
}
}
public static final class DummyCountDownLatch extends CountDownLatch {
public DummyCountDownLatch() {
super(0);
}
}
public static final class DummyRunnable implements Runnable, Serializable {
@Override
public void run() {}
}
public static final class DummyThreadFactory implements ThreadFactory, Serializable {
@Override
public Thread newThread(Runnable r) {
return new Thread(r);
}
}
public static final class DummyExecutor implements Executor, Serializable {
@Override
public void execute(Runnable command) {}
}
}
private static final class NullByteSink extends ByteSink implements Serializable {
private static final NullByteSink INSTANCE = new NullByteSink();
@Override
public OutputStream openStream() {
return ByteStreams.nullOutputStream();
}
}
// Compare by toString() to satisfy 2 properties:
// 1. compareTo(null) should throw NullPointerException
// 2. the order is deterministic and easy to understand, for debugging purpose.
private static final class ByToString implements Comparable<Object>, Serializable {
private static final ByToString INSTANCE = new ByToString();
@Override
public int compareTo(Object o) {
return toString().compareTo(o.toString());
}
@Override
public String toString() {
return "BY_TO_STRING";
}
private Object readResolve() {
return INSTANCE;
}
}
// Always equal is a valid total ordering. And it works for any Object.
private static final class AlwaysEqual extends Ordering<Object> implements Serializable {
private static final AlwaysEqual INSTANCE = new AlwaysEqual();
@Override
public int compare(Object o1, Object o2) {
return 0;
}
@Override
public String toString() {
return "ALWAYS_EQUAL";
}
private Object readResolve() {
return INSTANCE;
}
}
private ArbitraryInstances() {}
}
/**
* Returns a HashMap containing the scores of the hosts. The slowest *normal* host gets a score of
* 1.0 and the fastest gets 0. If the host was a straggler, its score is set to
* HostMetric.STRAGGLER.
*
* @return a HashMap of the host name and its score
*/
@Override
public HashMap<String, Double> getHostsScores(String queryID) {
HashMap<String, Double> scores = new HashMap<>();
List<String> hostNames = myMetadataTracker.getHostNamesInQuery(queryID);
if (hostNames == null || hostNames.size() == 0) {
// Unknown query ID
return null;
}
// Order the hosts by their slowness and assign a slowness score for each starting from 0 to 1.0
TreeMultimap<Long, String> hostsDeltaTime = TreeMultimap.create();
double sum = 0;
double sumSquared = 0;
double numberOfSamples = 0;
synchronized (hostNames) {
for (String hostName : hostNames) {
Long deltaTime = myMetadataTracker.getHostDeltaTime(queryID, hostName);
if (deltaTime == null && !myMetadataTracker.isRusher(queryID, hostName)) {
// That's not a rusher and didn't finish then it is a straggler
scores.put(hostName, HostMetric.STRAGGLER);
continue;
}
if (deltaTime != null) {
sum += deltaTime;
sumSquared += deltaTime * deltaTime;
numberOfSamples++;
hostsDeltaTime.put(deltaTime, hostName);
}
// Else this host is a rusher
}
}
if (numberOfSamples == 0) {
// We had one host only and it is a straggler
return scores;
}
// n cannot be 0 from here on....
// Now mark the non-stragglers but slow hosts
double mean = sum / numberOfSamples;
double variance =
(sumSquared - 2 * sum * mean + numberOfSamples * mean * mean) / numberOfSamples;
double stdDevsAway = mean + multipleStdDevs * Math.sqrt(variance);
if (log.isTraceEnabled()) {
log.trace(
"mean:"
+ mean
+ " stddev:"
+ Math.sqrt(variance)
+ " multiple:"
+ multipleStdDevs
+ " stdDevsAway:"
+ stdDevsAway);
}
ArrayList<Long> keysToDelete = new ArrayList<>();
for (Map.Entry<Long, String> e : hostsDeltaTime.entries()) {
if (e.getKey() > stdDevsAway) {
keysToDelete.add(e.getKey());
scores.put(e.getValue(), HostMetric.SLOW_HOST);
if (log.isTraceEnabled()) {
log.trace("Marking host:" + e.getValue() + " as slow");
}
}
}
// Remove the hosts that needs to be remove
for (Long key : keysToDelete) {
hostsDeltaTime.removeAll(key);
}
// If we have only one host to determine score for, then put 0.5
if (hostsDeltaTime.size() == 1) {
// Only one host responded
scores.put(hostsDeltaTime.values().iterator().next(), 0.5);
return scores;
}
// The number of segments is 1 less than the number of hosts to score. So if we have 3 hosts to
// occupy
// the score range from 0 -> 1, then we have two segments and their scores should be 0, 0.5 and
// 1.
double segmentSize = 1.0 / (hostsDeltaTime.size() - 1);
// Since the data is sorting ascending in turnaround time, the first we meet are the fastest
// hosts
// and so they should get the lowest slowness score
double score = 0;
for (Map.Entry<Long, String> e : hostsDeltaTime.entries()) {
scores.put(e.getValue(), score);
score += segmentSize;
}
return scores;
}
