/**
* Generates the cached tokens in the needed structure for a 'fast' search given a token
* representation (creates a map with the name of the token --> token).
*/
private Map<String, IToken> internalGenerateCachedTokens(
IPythonNature nature,
ICompletionCache completionCache,
String activationToken,
boolean searchSameLevelMods)
throws CompletionRecursionException {
Map<String, IToken> cachedTokens = new HashMap<String, IToken>();
// if still not found, we have to get all the tokens, including regular and wild imports
ICompletionState state =
CompletionStateFactory.getEmptyCompletionState(nature, completionCache);
ICodeCompletionASTManager astManager = nature.getAstManager();
state.setActivationToken(activationToken);
// we don't want to gather builtins in this case.
state.setBuiltinsGotten(true);
IToken[] globalTokens = astManager.getCompletionsForModule(this, state, searchSameLevelMods);
for (IToken token : globalTokens) {
String rep = token.getRepresentation();
IToken t = cachedTokens.get(rep);
if (t != null) {
// only override tokens if it's a getattr that's not defined in the builtin module
if (rep.equals("__getattribute__") || rep.equals("__getattr__")) {
if (!isTokenFromBuiltins(token)) {
cachedTokens.put(rep, token);
}
}
} else {
cachedTokens.put(rep, token);
}
}
return cachedTokens;
}
@Override
public synchronized List<FunctionDefAdapter> getFunctionsInitFiltered() {
if (cache == null) {
cache = new ArrayList<FunctionDefAdapter>();
for (IToken tok : this.tokens) {
if (tok.getType() == IToken.TYPE_FUNCTION
|| tok.getType() == IToken.TYPE_BUILTIN
|| tok.getType() == IToken.TYPE_UNKNOWN) {
String args = tok.getArgs();
List<exprType> arguments = new ArrayList<exprType>();
boolean useAnyArgs = false;
if (args.length() > 0) {
StringTokenizer strTok = new StringTokenizer(args, "( ,)");
if (!strTok.hasMoreTokens()) {
useAnyArgs = true;
} else {
while (strTok.hasMoreTokens()) {
String nextArg = strTok.nextToken();
arguments.add(new Name(nextArg, Name.Load, false));
}
}
} else {
useAnyArgs = true;
}
argumentsType functionArguments =
new argumentsType(
arguments.toArray(new exprType[0]),
null,
null,
null,
null,
null,
null,
null,
null,
null);
if (useAnyArgs) {
Name name = new Name("self", Name.Store, false);
name.addSpecial(new SpecialStr(",", -1, -1), true);
functionArguments.args = new exprType[] {name};
functionArguments.vararg = new NameTok("args", NameTok.VarArg);
functionArguments.kwarg = new NameTok("kwargs", NameTok.KwArg);
}
// System.out.println(tok.getRepresentation()+tok.getArgs());
FunctionDef functionDef =
new FunctionDef(
new NameTok(tok.getRepresentation(), NameTok.FunctionName),
functionArguments,
null,
null,
null);
cache.add(new FunctionDefAdapter(this.getModule(), null, functionDef, adapterPrefs));
}
}
}
return cache;
}
/**
* Adds tokens to the internal HashMap
*
* @param array The array of tokens to be added (maps representation -> token), so, existing
* tokens with the same representation will be replaced.
*/
public synchronized void addTokens(List<IToken> array) {
if (tokens == null) {
tokens = new HashMap<String, IToken>();
}
for (IToken token : array) {
this.tokens.put(token.getRepresentation(), token);
}
}
@Override
public int isInGlobalTokens(
String tok,
IPythonNature nature,
boolean searchSameLevelMods,
boolean ifHasGetAttributeConsiderInTokens,
ICompletionCache completionCache)
throws CompletionRecursionException {
// it's worth checking it if it is not dotted... (much faster as it's in a map already)
if (tok.indexOf(".") == -1) {
if (isInDirectGlobalTokens(tok, completionCache)) {
return IModule.FOUND_TOKEN;
}
}
String[] headAndTail = FullRepIterable.headAndTail(tok);
String head = headAndTail[1];
String generateTokensFor = headAndTail[0];
Map<String, IToken> cachedTokens =
getCachedCompletions(tok, nature, searchSameLevelMods, completionCache, generateTokensFor);
if (cachedTokens.containsKey(head)) {
return IModule.FOUND_TOKEN;
}
if (ifHasGetAttributeConsiderInTokens) {
IToken token = cachedTokens.get("__getattribute__");
if (token == null || isTokenFromBuiltins(token)) {
token = cachedTokens.get("__getattr__");
}
if (token != null && !isTokenFromBuiltins(token)) {
return IModule.FOUND_BECAUSE_OF_GETATTR;
}
// Try to determine if the user specified it has a @DynamicAttrs.
String[] parentsHeadAndTail = FullRepIterable.headAndTail(generateTokensFor);
cachedTokens =
getCachedCompletions(
tok, nature, searchSameLevelMods, completionCache, parentsHeadAndTail[0]);
IToken parentToken = cachedTokens.get(parentsHeadAndTail[1]);
if (parentToken != null) {
String docString = parentToken.getDocStr();
if (docString != null) {
if (docString.indexOf("@DynamicAttrs") != -1) {
// class that has things dynamically defined.
return IModule.FOUND_BECAUSE_OF_GETATTR;
}
}
}
}
// if not found until now, it is not defined
return IModule.NOT_FOUND;
}
private void checkExpected(int expected) {
FastStringBuffer buf = new FastStringBuffer(40 * comps.length);
for (IToken t : comps) {
buf.append(t.getRepresentation());
buf.append(", ");
}
String msg = "Expected " + expected + ". Found: " + buf.toString();
assertEquals(msg, expected, comps.length);
}
/** @return the names that are already imported in the current document */
private HashSet<String> getImportedNames(ICompletionState state) {
List<IToken> tokenImportedModules = state.getTokenImportedModules();
HashSet<String> importedNames = new HashSet<String>();
if (tokenImportedModules != null) {
for (IToken token : tokenImportedModules) {
importedNames.add(token.getRepresentation());
}
}
return importedNames;
}
private void checkExpected(String... expected) {
for (String s : expected) {
assertIsIn(s, comps);
}
List<String> asList = Arrays.asList(expected);
for (IToken t : comps) {
assertContains(asList, t.getRepresentation());
}
checkExpected(expected.length);
}
private IToken[] getComps() {
try {
IToken[] completionsForToken = getManager().getCompletionsForToken(doc, state);
HashMap<String, IToken> map = new HashMap<String, IToken>();
for (IToken iToken : completionsForToken) {
map.put(iToken.getRepresentation(), iToken);
}
return map.values().toArray(new IToken[map.size()]);
} catch (CompletionRecursionException e) {
throw new RuntimeException(e);
}
}
/**
* @param result
* @param message
*/
private void addToResult(List<IMessage> result, IMessage message) {
if (isUnusedImportMessage(message.getType())) {
IToken generator = message.getGenerator();
if (generator instanceof SourceToken) {
String asAbsoluteImport = generator.getAsAbsoluteImport();
if (asAbsoluteImport.indexOf("__future__.") != -1
|| asAbsoluteImport.indexOf("__metaclass__") != -1) {
// do not add from __future__ import xxx
return;
}
}
}
result.add(message);
}
private boolean isTokenFromBuiltins(IToken token) {
String parentPackage = token.getParentPackage();
return parentPackage.equals("__builtin__")
|| parentPackage.startsWith("__builtin__.")
|| parentPackage.equals("builtins")
|| parentPackage.startsWith("builtins.");
}
/**
* Checks if some token is actually undefined and changes its representation if needed
*
* @return a tuple indicating if it really is undefined and the representation that should be
* used.
*/
protected Tuple<Boolean, String> isActuallyUndefined(IToken token, String rep) {
String tokenRepresentation = token.getRepresentation();
if (tokenRepresentation != null) {
String firstPart = FullRepIterable.getFirstPart(tokenRepresentation);
if (this.prefs.getTokensAlwaysInGlobals().contains(firstPart)) {
return new Tuple<Boolean, String>(
false, firstPart); // ok firstPart in not really undefined...
}
}
boolean isActuallyUndefined = true;
if (rep == null) {
rep = tokenRepresentation;
}
int i;
if ((i = rep.indexOf('.')) != -1) {
rep = rep.substring(0, i);
}
String builtinType = NodeUtils.getBuiltinType(rep);
if (builtinType != null) {
isActuallyUndefined = false; // this is a builtin, so, it is defined after all
}
return new Tuple<Boolean, String>(isActuallyUndefined, rep);
}
@Override
public boolean isInGlobalTokens(
String tok, IPythonNature nature, ICompletionCache completionCache) {
// we have to override because there is no way to check if it is in some import from some other
// place if it has dots on the tok...
if (tok.indexOf('.') == -1) {
return isInDirectGlobalTokens(tok, completionCache);
} else {
ICompletionState state =
CompletionStateFactory.getEmptyCompletionState(nature, completionCache);
String[] headAndTail = FullRepIterable.headAndTail(tok);
state.setActivationToken(headAndTail[0]);
String head = headAndTail[1];
IToken[] globalTokens = getGlobalTokens(state, nature.getAstManager());
for (IToken token : globalTokens) {
if (token.getRepresentation().equals(head)) {
return true;
}
}
}
return false;
}
/* (non-Javadoc)
* @see org.python.pydev.editor.codecompletion.IPyCodeCompletion#getCodeCompletionProposals(org.eclipse.jface.text.ITextViewer, org.python.pydev.editor.codecompletion.CompletionRequest)
*/
@SuppressWarnings("unchecked")
public List getCodeCompletionProposals(ITextViewer viewer, CompletionRequest request)
throws CoreException, BadLocationException, IOException, MisconfigurationException,
PythonNatureWithoutProjectException {
if (request.getPySelection().getCursorLineContents().trim().startsWith("#")) {
// this may happen if the context is still not correctly computed in python
return new PyStringCodeCompletion().getCodeCompletionProposals(viewer, request);
}
if (DebugSettings.DEBUG_CODE_COMPLETION) {
Log.toLogFile(this, "Starting getCodeCompletionProposals");
Log.addLogLevel();
Log.toLogFile(this, "Request:" + request);
}
ArrayList<ICompletionProposal> ret = new ArrayList<ICompletionProposal>();
// let's see if we should do a code-completion in the current scope...
if (!isValidCompletionContext(request)) {
request.showTemplates = false;
return ret;
}
try {
IPythonNature pythonNature = request.nature;
checkPythonNature(pythonNature);
ICodeCompletionASTManager astManager = pythonNature.getAstManager();
if (astManager == null) {
// we're probably still loading it.
return ret;
}
// list of Object[], IToken or ICompletionProposal
List<Object> tokensList = new ArrayList<Object>();
lazyStartShell(request);
String trimmed = request.activationToken.replace('.', ' ').trim();
ImportInfo importsTipper = getImportsTipperStr(request);
int line = request.doc.getLineOfOffset(request.documentOffset);
IRegion region = request.doc.getLineInformation(line);
ICompletionState state =
new CompletionState(
line,
request.documentOffset - region.getOffset(),
null,
request.nature,
request.qualifier);
state.setIsInCalltip(request.isInCalltip);
boolean importsTip = false;
if (importsTipper.importsTipperStr.length() != 0) {
// code completion in imports
request.isInCalltip = false; // if found after (, but in an import, it is not a calltip!
importsTip = doImportCompletion(request, astManager, tokensList, importsTipper);
} else if (trimmed.length() > 0 && request.activationToken.indexOf('.') != -1) {
// code completion for a token
doTokenCompletion(request, astManager, tokensList, trimmed, state);
} else {
// go to globals
doGlobalsCompletion(request, astManager, tokensList, state);
}
Map<String, IToken> alreadyChecked = new HashMap<String, IToken>();
String lowerCaseQual = request.qualifier.toLowerCase();
if (lowerCaseQual.length()
>= PyCodeCompletionPreferencesPage.getArgumentsDeepAnalysisNChars()) {
// this can take some time on the analysis, so, let's let the user choose on how many chars
// does he
// want to do the analysis...
state.pushFindResolveImportMemoryCtx();
try {
for (Iterator<Object> it = tokensList.listIterator(); it.hasNext(); ) {
Object o = it.next();
if (o instanceof IToken) {
it
.remove(); // always remove the tokens from the list (they'll be re-added later
// once they are filtered)
IToken initialToken = (IToken) o;
IToken token = initialToken;
String strRep = token.getRepresentation();
IToken prev = alreadyChecked.get(strRep);
if (prev != null) {
if (prev.getArgs().length() != 0) {
continue; // we already have a version with args... just keep going
}
}
if (!strRep.toLowerCase().startsWith(lowerCaseQual)) {
// just re-add it if we're going to actually use it (depending on the qualifier)
continue;
}
while (token.isImportFrom()) {
// we'll only add it here if it is an import from (so, set the flag to false for the
// outer add)
if (token.getArgs().length() > 0) {
// if we already have the args, there's also no reason to do it (that's what we'll
// do here)
break;
}
ICompletionState s =
state.getCopyForResolveImportWithActTok(token.getRepresentation());
s.checkFindResolveImportMemory(token);
IToken token2 = astManager.resolveImport(s, token);
if (token2 != null && initialToken != token2) {
String args = token2.getArgs();
if (args.length() > 0) {
// put it into the map (may override previous if it didn't have args)
initialToken.setArgs(args);
initialToken.setDocStr(token2.getDocStr());
if (initialToken instanceof SourceToken && token2 instanceof SourceToken) {
SourceToken initialSourceToken = (SourceToken) initialToken;
SourceToken token2SourceToken = (SourceToken) token2;
initialSourceToken.setAst(token2SourceToken.getAst());
}
break;
}
if (token2 == null
|| (token2.equals(token)
&& token2.getArgs().equals(token.getArgs())
&& token2.getParentPackage().equals(token.getParentPackage()))) {
break;
}
token = token2;
} else {
break;
}
}
alreadyChecked.put(strRep, initialToken);
}
}
} finally {
state.popFindResolveImportMemoryCtx();
}
}
tokensList.addAll(alreadyChecked.values());
changeItokenToCompletionPropostal(viewer, request, ret, tokensList, importsTip, state);
} catch (CompletionRecursionException e) {
if (onCompletionRecursionException != null) {
onCompletionRecursionException.call(e);
}
if (DebugSettings.DEBUG_CODE_COMPLETION) {
Log.toLogFile(e);
}
// PydevPlugin.log(e);
// ret.add(new CompletionProposal("",request.documentOffset,0,0,null,e.getMessage(),
// null,null));
}
if (DebugSettings.DEBUG_CODE_COMPLETION) {
Log.remLogLevel();
Log.toLogFile(this, "Finished completion. Returned:" + ret.size() + " completions.\r\n");
}
return ret;
}
/** adds a message of some type for a given token */
public void addMessage(int type, IToken token) {
List<IMessage> msgs = getMsgsList(token);
doAddMessage(msgs, type, token.getRepresentation(), token);
}
/** adds a message of some type for some Found instance */
public void addMessage(int type, IToken generator, IToken tok) {
addMessage(type, generator, tok, tok.getRepresentation());
}
/**
* @see
* org.python.pydev.editor.codecompletion.revisited.modules.AbstractModule#getGlobalTokens(java.lang.String)
*/
public IToken[] getGlobalTokens(ICompletionState state, ICodeCompletionASTManager manager) {
String activationToken = state.getActivationToken();
if (activationToken.length() == 0) {
return getGlobalTokens();
}
Map<String, IToken> v = cache.get(activationToken);
if (v != null) {
Collection<IToken> values = v.values();
return values.toArray(new IToken[values.size()]);
}
IToken[] toks = new IToken[0];
if (COMPILED_MODULES_ENABLED) {
try {
final IPythonNature nature = manager.getNature();
final AbstractShell shell;
try {
shell = AbstractShell.getServerShell(nature, AbstractShell.COMPLETION_SHELL);
} catch (Exception e) {
throw new RuntimeException("Unable to create shell for CompiledModule: " + this.name, e);
}
synchronized (shell) {
String act = name + '.' + activationToken;
String tokenToCompletion = act;
if (isPythonBuiltin) {
String replacement = BUILTIN_REPLACEMENTS.get(activationToken);
if (replacement != null) {
tokenToCompletion = name + '.' + replacement;
}
}
List<String[]> completions =
shell.getImportCompletions(
tokenToCompletion,
manager
.getModulesManager()
.getCompletePythonPath(
nature.getProjectInterpreter(),
nature.getRelatedInterpreterManager()))
.o2;
ArrayList<IToken> array = new ArrayList<IToken>();
for (Iterator<String[]> iter = completions.iterator(); iter.hasNext(); ) {
String[] element = iter.next();
if (element.length >= 4) { // it might be a server error
IToken t =
new CompiledToken(
element[0], element[1], element[2], act, Integer.parseInt(element[3]));
array.add(t);
}
}
toks = (CompiledToken[]) array.toArray(new CompiledToken[0]);
HashMap<String, IToken> map = new HashMap<String, IToken>();
for (IToken token : toks) {
map.put(token.getRepresentation(), token);
}
cache.put(activationToken, map);
}
} catch (Exception e) {
Log.log(
"Error while getting info for module:"
+ this.name
+ ". Project: "
+ manager.getNature().getProject(),
e);
}
}
return toks;
}