private static THashSet<String> collectFoldersToExclude(
final Module module, final VirtualFile pubspecYamlFile) {
final THashSet<String> newExcludedPackagesUrls = new THashSet<String>();
final ProjectFileIndex fileIndex =
ProjectRootManager.getInstance(module.getProject()).getFileIndex();
final VirtualFile root = pubspecYamlFile.getParent();
final VirtualFile binFolder = root.findChild("bin");
if (binFolder != null && binFolder.isDirectory() && fileIndex.isInContent(binFolder)) {
newExcludedPackagesUrls.add(binFolder.getUrl() + "/packages");
}
appendPackagesFolders(newExcludedPackagesUrls, root.findChild("benchmark"), fileIndex);
appendPackagesFolders(newExcludedPackagesUrls, root.findChild("example"), fileIndex);
appendPackagesFolders(newExcludedPackagesUrls, root.findChild("test"), fileIndex);
appendPackagesFolders(newExcludedPackagesUrls, root.findChild("tool"), fileIndex);
appendPackagesFolders(newExcludedPackagesUrls, root.findChild("web"), fileIndex);
// Folder packages/ThisProject (where ThisProject is the name specified in pubspec.yaml) is a
// symlink to local 'lib' folder. Exclude it in order not to have duplicates. Resolve goes to
// local 'lib' folder.
// Empty 'ThisProject (link to 'lib' folder)' node is added to Project Structure by
// DartTreeStructureProvider
final VirtualFile libFolder = root.findChild("lib");
if (libFolder != null && libFolder.isDirectory()) {
final String pubspecName = PubspecYamlUtil.getPubspecName(pubspecYamlFile);
if (pubspecName != null) {
newExcludedPackagesUrls.add(root.getUrl() + "/packages/" + pubspecName);
}
}
return newExcludedPackagesUrls;
}
/**
* Loads the specified stopwords file. Used internally by Stopwords(TermPipeline, String[]).
*
* @param stopwordsFilename The filename of the file to use as the stopwords list.
*/
public void loadStopwordsList(String stopwordsFilename) {
// determine encoding to use when reading the stopwords files
// String stopwordsEncoding = null;
try {
// use sys default encoding if none specified
BufferedReader br = new BufferedReader(new FileReader(stopwordsFilename));
String word;
while ((word = br.readLine()) != null) {
word = word.trim();
if (word.length() > 0) {
stopWords.add(word);
}
}
br.close();
} catch (IOException ioe) {
System.err.println(
"Errror: Input/Output Exception while reading stopword list ("
+ stopwordsFilename
+ ") : Stack trace follows.");
ioe.printStackTrace();
}
if (stopWords.size() == 0)
System.err.println("Error: Empty stopwords file was used (" + stopwordsFilename + ")");
}
public void init(PsiElement element) {
myFile = (XmlFile) element.getContainingFile();
if (element instanceof XmlTag) {
myTag = (XmlTag) element;
} else {
final XmlDocument document = myFile.getDocument();
if (document != null) {
myTag = document.getRootTag();
}
}
if (myTag != null) {
myTargetNamespace = myTag.getAttributeValue("targetNamespace");
}
final THashSet<PsiFile> dependenciesSet = new THashSet<PsiFile>();
final Set<PsiFile> redefineProcessingSet = myRedefinedDescriptorsInProcessing.get();
if (redefineProcessingSet != null) {
dependenciesSet.addAll(redefineProcessingSet);
}
collectDependencies(myTag, myFile, dependenciesSet);
dependencies = ArrayUtil.toObjectArray(dependenciesSet);
}
private void addIntermediateGroovyClasses(
Set<VirtualFile> allToCompile, final Set<VirtualFile> groovyFiles) {
final Set<VirtualFile> initialFiles = new THashSet<VirtualFile>(allToCompile);
final THashSet<VirtualFile> visited = new THashSet<VirtualFile>();
for (VirtualFile aClass : initialFiles) {
if (visited.add(aClass)) {
goForIntermediateFiles(
aClass,
allToCompile,
new FactoryMap<VirtualFile, Set<VirtualFile>>() {
@Override
protected Set<VirtualFile> create(final VirtualFile key) {
AccessToken accessToken =
ApplicationManager.getApplication().acquireReadActionLock();
try {
return calcCodeReferenceDependencies(key, groovyFiles);
} finally {
accessToken.finish();
}
}
},
visited);
}
}
}
private static void checkChoice(BnfChoice choice, ProblemsHolder problemsHolder) {
Set<BnfRule> visited = new THashSet<BnfRule>();
THashSet<BnfExpression> first = new THashSet<BnfExpression>();
BnfFirstNextAnalyzer analyzer = new BnfFirstNextAnalyzer();
List<BnfExpression> list = choice.getExpressionList();
for (int i = 0, listSize = list.size() - 1; i < listSize; i++) {
BnfExpression child = list.get(i);
Set<String> firstSet = analyzer.asStrings(analyzer.calcFirstInner(child, first, visited));
if (firstSet.contains(BnfFirstNextAnalyzer.MATCHES_NOTHING)) {
registerProblem(
choice,
child,
"Branch is unable to match anything due to & or ! conditions",
problemsHolder);
} else if (firstSet.contains(BnfFirstNextAnalyzer.MATCHES_EOF)) {
registerProblem(
choice,
child,
"Branch matches empty input making the rest branches unreachable",
problemsHolder);
break;
}
first.clear();
visited.clear();
}
}
@Override
public Set<String> getAllLookupStrings() {
final Set<String> strings = getDelegate().getAllLookupStrings();
final THashSet<String> result = new THashSet<String>();
result.addAll(strings);
result.add(getLookupString());
return result;
}
@Nullable
@Override
public Set<String> fromString(@NotNull String value) {
final THashSet<String> result = new THashSet<String>();
for (String closingTag : StringUtil.split(value, ",")) {
result.add(closingTag.trim());
}
return result;
}
@NotNull
@Override
public List<Pair<LookupElement, Object>> getSortingWeights(
@NotNull Iterable<LookupElement> items, @NotNull ProcessingContext context) {
final THashSet<LookupElement> lifted = newIdentityTroveSet();
Iterable<LookupElement> iterable =
liftShorterElements(ContainerUtil.newArrayList(items), lifted, context);
return ContainerUtil.map(
iterable, element -> new Pair<LookupElement, Object>(element, lifted.contains(element)));
}
private static boolean defaultFunctionalityWorked(final PsiLanguageInjectionHost host) {
final THashSet<String> languages = new THashSet<String>();
final List<Pair<PsiElement, TextRange>> files = InjectedLanguageUtil.getInjectedPsiFiles(host);
if (files == null) return false;
for (Pair<PsiElement, TextRange> pair : files) {
for (Language lang = pair.first.getLanguage(); lang != null; lang = lang.getBaseLanguage()) {
languages.add(lang.getID());
}
}
// todo there is a problem: host i.e. literal expression is confused with "target" i.e.
// parameter
// todo therefore this part doesn't work for java
return Configuration.getInstance().setHostInjectionEnabled(host, languages, false);
}
private void mergeWithDefaultConfiguration() {
final ArrayList<Configuration> cfgList = new ArrayList<Configuration>();
for (LanguageInjectionSupport support : InjectorUtils.getActiveInjectionSupports()) {
final String config = support.getDefaultConfigUrl();
final URL url = config == null ? null : support.getClass().getResource(config);
if (url != null) {
try {
cfgList.add(load(url.openStream()));
} catch (Exception e) {
LOG.warn(e);
}
}
}
final THashSet<Object> visited = new THashSet<Object>();
for (IdeaPluginDescriptor pluginDescriptor : PluginManager.getPlugins()) {
if (pluginDescriptor instanceof IdeaPluginDescriptorImpl
&& !((IdeaPluginDescriptorImpl) pluginDescriptor).isEnabled()) continue;
final ClassLoader loader = pluginDescriptor.getPluginClassLoader();
if (!visited.add(loader)) continue;
if (loader instanceof PluginClassLoader && ((PluginClassLoader) loader).getUrls().isEmpty())
continue;
try {
final Enumeration<URL> enumeration = loader.getResources("META-INF/languageInjections.xml");
if (enumeration == null) continue;
while (enumeration.hasMoreElements()) {
URL url = enumeration.nextElement();
if (!visited.add(url.getFile())) continue; // for DEBUG mode
try {
cfgList.add(load(url.openStream()));
} catch (Exception e) {
LOG.warn(e);
}
}
} catch (Exception e) {
LOG.warn(e);
}
}
final ArrayList<BaseInjection> originalInjections = new ArrayList<BaseInjection>();
final ArrayList<BaseInjection> newInjections = new ArrayList<BaseInjection>();
myDefaultInjections = new ArrayList<BaseInjection>();
for (String supportId : InjectorUtils.getActiveInjectionSupportIds()) {
for (Configuration cfg : cfgList) {
final List<BaseInjection> imported = cfg.getInjections(supportId);
myDefaultInjections.addAll(imported);
importInjections(getInjections(supportId), imported, originalInjections, newInjections);
}
}
replaceInjections(newInjections, originalInjections);
}
public static void excludePackagesFolders(
final Module module, final VirtualFile pubspecYamlFile) {
final VirtualFile root = pubspecYamlFile.getParent();
final VirtualFile contentRoot =
root == null
? null
: ProjectRootManager.getInstance(module.getProject())
.getFileIndex()
.getContentRootForFile(root);
if (contentRoot == null) return;
// http://pub.dartlang.org/doc/glossary.html#entrypoint-directory
// Entrypoint directory: A directory inside your package that is allowed to contain Dart
// entrypoints.
// Pub will ensure all of these directories get a “packages” directory, which is needed for
// “package:” imports to work.
// Pub has a whitelist of these directories: benchmark, bin, example, test, tool, and web.
// Any subdirectories of those (except bin) may also contain entrypoints.
//
// the same can be seen in the pub tool source code: [repo
// root]/sdk/lib/_internal/pub/lib/src/entrypoint.dart
final Collection<String> oldExcludedPackagesUrls =
ContainerUtil.filter(
ModuleRootManager.getInstance(module).getExcludeRootUrls(),
new Condition<String>() {
final String rootUrlPrefix = root.getUrl() + '/';
public boolean value(final String url) {
if (!url.startsWith(rootUrlPrefix)) return false;
if (url.endsWith("/packages")) return true;
// excluded subfolder of 'packages' folder
final int lastSlashIndex = url.lastIndexOf('/');
if (lastSlashIndex > 0 && url.substring(0, lastSlashIndex).endsWith("/packages"))
return true;
return false;
}
});
final THashSet<String> newExcludedPackagesUrls =
collectFoldersToExclude(module, pubspecYamlFile);
if (oldExcludedPackagesUrls.size() != newExcludedPackagesUrls.size()
|| !newExcludedPackagesUrls.containsAll(oldExcludedPackagesUrls)) {
updateExcludedFolders(module, contentRoot, oldExcludedPackagesUrls, newExcludedPackagesUrls);
}
}
public void processTerm(String t) {
if (t == null) return;
// current term is a delimiter
if (blockDelimiterTerms.contains(t)) {
// delimiters should also be indexed
if (indexDelimiters) {
final int[] fieldIds = new int[numFields];
int i = 0;
for (String fieldName : termFields) {
fieldIds[i] = fieldNames.get(fieldName);
i++;
}
((BlockFieldDocumentPostingList) termsInDocument).insert(t, fieldIds, blockId);
if (countDelimiters) numOfTokensInDocument++;
}
numOfTokensInBlock = 0;
blockId++;
} else {
// index non-delimiter term
final int[] fieldIds = new int[numFields];
int i = 0;
for (String fieldName : termFields) {
fieldIds[i] = fieldNames.get(fieldName);
i++;
}
((BlockFieldDocumentPostingList) termsInDocument).insert(t, fieldIds, blockId);
numOfTokensInDocument++;
}
}
public DelimFieldTermProcessor(
String[] _delims, boolean _indexDelimiters, boolean _countDelimiters) {
blockDelimiterTerms = new THashSet<String>();
for (String t : _delims) blockDelimiterTerms.add(t);
indexDelimiters = _indexDelimiters;
countDelimiters = _countDelimiters;
}
public String getHighRecallSegmentation(String[][] data, THashSet<String> allRelatedWords) {
ArrayList<String> startInds = new ArrayList<>();
for (int i = 0; i < data[0].length; i++) {
startInds.add("" + i);
}
String tokNums = "";
for (int i = MAX_LEN; i >= 1; i--) {
for (int j = 0; j <= (data[0].length - i); j++) {
String ind = "" + j;
if (!startInds.contains(ind)) continue;
String lTok = "";
for (int k = j; k < j + i; k++) {
String pos = data[1][k];
String cPos = pos.substring(0, 1);
String l = data[5][k];
lTok += l + "_" + cPos + " ";
}
lTok = lTok.trim();
if (allRelatedWords.contains(lTok)) {
String tokRep = "";
for (int k = j; k < j + i; k++) {
tokRep += k + " ";
ind = "" + k;
startInds.remove(ind);
}
tokRep = tokRep.trim().replaceAll(" ", "_");
tokNums += tokRep + "\t";
}
}
}
tokNums = tokNums.trim();
return tokNums;
}
public CfgInfo(Configuration cfg, final String title) {
this.cfg = cfg;
this.title = title;
bundledInjections.addAll(cfg.getDefaultInjections());
originalInjections =
new ArrayList<BaseInjection>(
ContainerUtil.concat(
InjectorUtils.getActiveInjectionSupportIds(),
new Function<String, Collection<? extends BaseInjection>>() {
public Collection<? extends BaseInjection> fun(final String s) {
List<BaseInjection> injections =
CfgInfo.this.cfg instanceof Configuration.Prj
? ((Configuration.Prj) CfgInfo.this.cfg).getOwnInjections(s)
: CfgInfo.this.cfg.getInjections(s);
return ContainerUtil.findAll(
injections,
new Condition<BaseInjection>() {
public boolean value(final BaseInjection injection) {
String id = injection.getInjectedLanguageId();
return InjectedLanguage.findLanguageById(id) != null
|| ReferenceInjector.findById(id) != null;
}
});
}
}));
sortInjections(originalInjections);
reset();
}
public void addTypeReference(RefJavaElement from) {
if (from != null) {
if (myInTypeReferences == null) {
myInTypeReferences = new THashSet<RefElement>(1);
}
myInTypeReferences.add(from);
((RefJavaElementImpl) from).addOutTypeRefernce(this);
getRefManager().fireNodeMarkedReferenced(this, from, false, false, false);
}
}
private String createNameForClass(ClassDescriptor descriptor) {
String suggestedName = descriptor.getName().asString();
String name = suggestedName;
DeclarationDescriptor containing = descriptor;
while (!nameClashGuard.add(name)) {
containing = containing.getContainingDeclaration();
assert containing != null;
name = suggestedName + '_' + containing.getName().asString();
}
return name;
}
@NotNull
@Override
public Comparable weigh(@NotNull LookupElement element) {
final Object object = element.getObject();
final String name = getLookupObjectName(object);
if (name != null) {
int max = 0;
final List<String> wordsNoDigits = NameUtil.nameToWordsLowerCase(truncDigits(name));
for (ExpectedTypeInfo myExpectedInfo : myExpectedTypes) {
String expectedName = ((ExpectedTypeInfoImpl) myExpectedInfo).getExpectedName().compute();
if (expectedName != null) {
final THashSet<String> set =
new THashSet<String>(NameUtil.nameToWordsLowerCase(truncDigits(expectedName)));
set.retainAll(wordsNoDigits);
max = Math.max(max, set.size());
}
}
return -max;
}
return 0;
}
private static List<String> getLibraryFiles(@NotNull Module module) {
List<String> result = new ArrayList<String>();
List<String> libLocationAndTarget = ProjectStructureUtil.getLibLocationForProject(module);
THashSet<Module> modules = new THashSet<Module>();
collectModuleDependencies(module, modules);
if (!modules.isEmpty()) {
for (Module dependency : modules) {
result.add("@" + dependency.getName());
for (VirtualFile file : getSourceFiles(dependency)) {
result.add(file.getPath());
}
}
}
for (String file : libLocationAndTarget) {
result.add(file);
}
return result;
}
@Override
@NotNull
public String[] getNames(Project project, boolean includeNonProjectItems) {
if (FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping) {
final THashSet<String> names = new THashSet<String>(1000);
IdFilter filter = IdFilter.getProjectIdFilter(project, includeNonProjectItems);
processNames(
new Processor<String>() {
@Override
public boolean process(String s) {
names.add(s);
return true;
}
},
FindSymbolParameters.searchScopeFor(project, includeNonProjectItems),
filter);
if (IdFilter.LOG.isDebugEnabled()) {
IdFilter.LOG.debug("All names retrieved2:" + names.size());
}
return ArrayUtil.toStringArray(names);
} else {
return FilenameIndex.getAllFilenames(project);
}
}
private static List<BaseInjection> loadDefaultInjections() {
final ArrayList<Configuration> cfgList = new ArrayList<Configuration>();
final THashSet<Object> visited = new THashSet<Object>();
for (LanguageInjectionConfigBean configBean :
Extensions.getExtensions(LanguageInjectionSupport.CONFIG_EP_NAME)) {
PluginDescriptor descriptor = configBean.getPluginDescriptor();
final ClassLoader loader = descriptor.getPluginClassLoader();
try {
final Enumeration<URL> enumeration = loader.getResources(configBean.getConfigUrl());
if (enumeration == null || !enumeration.hasMoreElements()) {
LOG.warn(descriptor.getPluginId() + ": " + configBean.getConfigUrl() + " was not found");
} else {
while (enumeration.hasMoreElements()) {
URL url = enumeration.nextElement();
if (!visited.add(url.getFile())) continue; // for DEBUG mode
try {
cfgList.add(load(url.openStream()));
} catch (Exception e) {
LOG.warn(e);
}
}
}
} catch (Exception e) {
LOG.warn(e);
}
}
final ArrayList<BaseInjection> defaultInjections = new ArrayList<BaseInjection>();
for (String supportId : InjectorUtils.getActiveInjectionSupportIds()) {
for (Configuration cfg : cfgList) {
final List<BaseInjection> imported = cfg.getInjections(supportId);
defaultInjections.addAll(imported);
}
}
return defaultInjections;
}
public void processTerm(String t) {
if (t == null) return;
// current term is a delimiter
if (blockDelimiterTerms.contains(t)) {
// delimiters should also be indexed
if (indexDelimiters) {
((BlockDocumentPostingList) termsInDocument).insert(t, blockId);
if (countDelimiters) numOfTokensInDocument++;
}
numOfTokensInBlock = 0;
blockId++;
} else {
// index non-delimiter term
((BlockDocumentPostingList) termsInDocument).insert(t, blockId);
numOfTokensInDocument++;
}
}
public void run(@NotNull ProgressIndicator indicator) {
try {
mySocket = myServerSocket.accept();
DumbService.getInstance(myProject)
.repeatUntilPassesInSmartMode(
new Runnable() {
@Override
public void run() {
myClasses.clear();
myJunit4[0] = ConfigurationUtil.findAllTestClasses(myClassFilter, myClasses);
}
});
myFoundTests = !myClasses.isEmpty();
} catch (IOException e) {
LOG.info(e);
} catch (Throwable e) {
LOG.error(e);
}
}
public void addInstanceReference(RefElement from) {
if (myInstanceReferences == null) {
myInstanceReferences = new THashSet<RefElement>(1);
}
myInstanceReferences.add(from);
}
static {
final THashSet<String> set = new THashSet<String>(FileUtil.PATH_HASHING_STRATEGY);
set.addAll(Arrays.asList("jpg", "jpeg", "gif", "bmp", "png"));
DEFAULT_FILTERING_EXCLUDED_EXTENSIONS = Collections.unmodifiableSet(set);
}
/**
* Clear all stopwords from this stopword list object.
*
* @since 1.1.0
*/
public void clear() {
stopWords.clear();
}
/** Returns true is term t is a stopword */
public boolean isStopword(final String t) {
return stopWords.contains(t);
}
/** Checks to see if term t is a stopword. If so, return null. */
public final String processTerm(final String t) {
return (stopWords.contains(t)) ? null : t;
}
private List<JavaFileObject> findInside(
Location location, String packageName, Set<JavaFileObject.Kind> kinds, boolean lookForFile)
throws IOException {
Iterable<? extends File> path = getLocation(location);
if (path == null) return Collections.emptyList();
String subdirectory = packageName.replace('.', '/');
List<JavaFileObject> results = null;
for (File directory : path) {
VirtualFile dir = LocalFileSystem.getInstance().findFileByIoFile(directory);
if (dir == null) continue;
if (!dir.isDirectory()) {
dir = JarFileSystem.getInstance().getJarRootForLocalFile(dir);
if (dir == null) continue;
}
VirtualFile virtualFile =
StringUtil.isEmptyOrSpaces(subdirectory) ? dir : dir.findFileByRelativePath(subdirectory);
if (virtualFile == null) continue;
VirtualFile[] children;
if (lookForFile) {
if (!virtualFile.isDirectory()) {
children = new VirtualFile[] {virtualFile};
} else continue;
} else {
children = virtualFile.getChildren();
}
for (VirtualFile child : children) {
JavaFileObject.Kind kind = getKind("." + child.getExtension());
if (kinds == null || kinds.contains(kind)) {
if (results == null) results = new SmartList<JavaFileObject>();
if (kind == JavaFileObject.Kind.SOURCE && child.getFileSystem() instanceof JarFileSystem)
continue; // for some reasdon javac looks for java files inside jar
// use VFS to read content inside .jar
String childPath = child.getPath();
JavaFileObject fileObject =
!childPath.contains("!/")
? new JavaIoFile(new File(childPath), kind, myEncoding)
: new JavaVirtualFile(child, kind);
results.add(fileObject);
}
}
}
List<JavaFileObject> ret = results == null ? Collections.<JavaFileObject>emptyList() : results;
if (LOG.isDebugEnabled()) {
// for testing consistency
Collection c = (Collection) myStandardFileManager.list(location, packageName, kinds, false);
Collection<JavaFileObject> sup = new HashSet(c);
assert sup.size() == c.size();
assert new HashSet(c).equals(sup);
THashSet<JavaFileObject> s =
new THashSet<JavaFileObject>(
new TObjectHashingStrategy<JavaFileObject>() {
public int computeHashCode(JavaFileObject object) {
return object.getName().hashCode();
}
public boolean equals(JavaFileObject o1, JavaFileObject o2) {
return o1.getKind() == o2.getKind() && o1.toUri().equals(o2.toUri());
}
});
s.addAll(ret);
s.removeAll(sup);
if (ret.size() != sup.size()) {
assert false : "our implementation differs from javac'";
}
}
return ret;
}
