Example #1
0
  /**
   * Generates the cached tokens in the needed structure for a 'fast' search given a token
   * representation (creates a map with the name of the token --> token).
   */
  private Map<String, IToken> internalGenerateCachedTokens(
      IPythonNature nature,
      ICompletionCache completionCache,
      String activationToken,
      boolean searchSameLevelMods)
      throws CompletionRecursionException {

    Map<String, IToken> cachedTokens = new HashMap<String, IToken>();

    // if still not found, we have to get all the tokens, including regular and wild imports
    ICompletionState state =
        CompletionStateFactory.getEmptyCompletionState(nature, completionCache);
    ICodeCompletionASTManager astManager = nature.getAstManager();
    state.setActivationToken(activationToken);

    // we don't want to gather builtins in this case.
    state.setBuiltinsGotten(true);
    IToken[] globalTokens = astManager.getCompletionsForModule(this, state, searchSameLevelMods);
    for (IToken token : globalTokens) {
      String rep = token.getRepresentation();
      IToken t = cachedTokens.get(rep);
      if (t != null) {
        // only override tokens if it's a getattr that's not defined in the builtin module
        if (rep.equals("__getattribute__") || rep.equals("__getattr__")) {
          if (!isTokenFromBuiltins(token)) {
            cachedTokens.put(rep, token);
          }
        }
      } else {
        cachedTokens.put(rep, token);
      }
    }
    return cachedTokens;
  }
  @Override
  public synchronized List<FunctionDefAdapter> getFunctionsInitFiltered() {
    if (cache == null) {
      cache = new ArrayList<FunctionDefAdapter>();
      for (IToken tok : this.tokens) {
        if (tok.getType() == IToken.TYPE_FUNCTION
            || tok.getType() == IToken.TYPE_BUILTIN
            || tok.getType() == IToken.TYPE_UNKNOWN) {
          String args = tok.getArgs();

          List<exprType> arguments = new ArrayList<exprType>();
          boolean useAnyArgs = false;
          if (args.length() > 0) {
            StringTokenizer strTok = new StringTokenizer(args, "( ,)");
            if (!strTok.hasMoreTokens()) {
              useAnyArgs = true;
            } else {
              while (strTok.hasMoreTokens()) {
                String nextArg = strTok.nextToken();
                arguments.add(new Name(nextArg, Name.Load, false));
              }
            }
          } else {
            useAnyArgs = true;
          }

          argumentsType functionArguments =
              new argumentsType(
                  arguments.toArray(new exprType[0]),
                  null,
                  null,
                  null,
                  null,
                  null,
                  null,
                  null,
                  null,
                  null);
          if (useAnyArgs) {
            Name name = new Name("self", Name.Store, false);
            name.addSpecial(new SpecialStr(",", -1, -1), true);
            functionArguments.args = new exprType[] {name};
            functionArguments.vararg = new NameTok("args", NameTok.VarArg);
            functionArguments.kwarg = new NameTok("kwargs", NameTok.KwArg);
          }
          //                System.out.println(tok.getRepresentation()+tok.getArgs());
          FunctionDef functionDef =
              new FunctionDef(
                  new NameTok(tok.getRepresentation(), NameTok.FunctionName),
                  functionArguments,
                  null,
                  null,
                  null);
          cache.add(new FunctionDefAdapter(this.getModule(), null, functionDef, adapterPrefs));
        }
      }
    }
    return cache;
  }
Example #3
0
  /**
   * Adds tokens to the internal HashMap
   *
   * @param array The array of tokens to be added (maps representation -> token), so, existing
   *     tokens with the same representation will be replaced.
   */
  public synchronized void addTokens(List<IToken> array) {
    if (tokens == null) {
      tokens = new HashMap<String, IToken>();
    }

    for (IToken token : array) {
      this.tokens.put(token.getRepresentation(), token);
    }
  }
Example #4
0
  @Override
  public int isInGlobalTokens(
      String tok,
      IPythonNature nature,
      boolean searchSameLevelMods,
      boolean ifHasGetAttributeConsiderInTokens,
      ICompletionCache completionCache)
      throws CompletionRecursionException {

    // it's worth checking it if it is not dotted... (much faster as it's in a map already)
    if (tok.indexOf(".") == -1) {
      if (isInDirectGlobalTokens(tok, completionCache)) {
        return IModule.FOUND_TOKEN;
      }
    }

    String[] headAndTail = FullRepIterable.headAndTail(tok);
    String head = headAndTail[1];
    String generateTokensFor = headAndTail[0];

    Map<String, IToken> cachedTokens =
        getCachedCompletions(tok, nature, searchSameLevelMods, completionCache, generateTokensFor);

    if (cachedTokens.containsKey(head)) {
      return IModule.FOUND_TOKEN;
    }

    if (ifHasGetAttributeConsiderInTokens) {

      IToken token = cachedTokens.get("__getattribute__");
      if (token == null || isTokenFromBuiltins(token)) {
        token = cachedTokens.get("__getattr__");
      }
      if (token != null && !isTokenFromBuiltins(token)) {
        return IModule.FOUND_BECAUSE_OF_GETATTR;
      }

      // Try to determine if the user specified it has a @DynamicAttrs.
      String[] parentsHeadAndTail = FullRepIterable.headAndTail(generateTokensFor);
      cachedTokens =
          getCachedCompletions(
              tok, nature, searchSameLevelMods, completionCache, parentsHeadAndTail[0]);
      IToken parentToken = cachedTokens.get(parentsHeadAndTail[1]);
      if (parentToken != null) {
        String docString = parentToken.getDocStr();
        if (docString != null) {
          if (docString.indexOf("@DynamicAttrs") != -1) {
            // class that has things dynamically defined.
            return IModule.FOUND_BECAUSE_OF_GETATTR;
          }
        }
      }
    }

    // if not found until now, it is not defined
    return IModule.NOT_FOUND;
  }
Example #5
0
 private void checkExpected(int expected) {
   FastStringBuffer buf = new FastStringBuffer(40 * comps.length);
   for (IToken t : comps) {
     buf.append(t.getRepresentation());
     buf.append(", ");
   }
   String msg = "Expected " + expected + ". Found: " + buf.toString();
   assertEquals(msg, expected, comps.length);
 }
Example #6
0
 /** @return the names that are already imported in the current document */
 private HashSet<String> getImportedNames(ICompletionState state) {
   List<IToken> tokenImportedModules = state.getTokenImportedModules();
   HashSet<String> importedNames = new HashSet<String>();
   if (tokenImportedModules != null) {
     for (IToken token : tokenImportedModules) {
       importedNames.add(token.getRepresentation());
     }
   }
   return importedNames;
 }
Example #7
0
 private void checkExpected(String... expected) {
   for (String s : expected) {
     assertIsIn(s, comps);
   }
   List<String> asList = Arrays.asList(expected);
   for (IToken t : comps) {
     assertContains(asList, t.getRepresentation());
   }
   checkExpected(expected.length);
 }
Example #8
0
  private IToken[] getComps() {
    try {
      IToken[] completionsForToken = getManager().getCompletionsForToken(doc, state);
      HashMap<String, IToken> map = new HashMap<String, IToken>();
      for (IToken iToken : completionsForToken) {
        map.put(iToken.getRepresentation(), iToken);
      }

      return map.values().toArray(new IToken[map.size()]);
    } catch (CompletionRecursionException e) {
      throw new RuntimeException(e);
    }
  }
Example #9
0
 /**
  * @param result
  * @param message
  */
 private void addToResult(List<IMessage> result, IMessage message) {
   if (isUnusedImportMessage(message.getType())) {
     IToken generator = message.getGenerator();
     if (generator instanceof SourceToken) {
       String asAbsoluteImport = generator.getAsAbsoluteImport();
       if (asAbsoluteImport.indexOf("__future__.") != -1
           || asAbsoluteImport.indexOf("__metaclass__") != -1) {
         // do not add from __future__ import xxx
         return;
       }
     }
   }
   result.add(message);
 }
Example #10
0
 private boolean isTokenFromBuiltins(IToken token) {
   String parentPackage = token.getParentPackage();
   return parentPackage.equals("__builtin__")
       || parentPackage.startsWith("__builtin__.")
       || parentPackage.equals("builtins")
       || parentPackage.startsWith("builtins.");
 }
Example #11
0
  /**
   * Checks if some token is actually undefined and changes its representation if needed
   *
   * @return a tuple indicating if it really is undefined and the representation that should be
   *     used.
   */
  protected Tuple<Boolean, String> isActuallyUndefined(IToken token, String rep) {
    String tokenRepresentation = token.getRepresentation();
    if (tokenRepresentation != null) {
      String firstPart = FullRepIterable.getFirstPart(tokenRepresentation);
      if (this.prefs.getTokensAlwaysInGlobals().contains(firstPart)) {
        return new Tuple<Boolean, String>(
            false, firstPart); // ok firstPart in not really undefined...
      }
    }

    boolean isActuallyUndefined = true;
    if (rep == null) {
      rep = tokenRepresentation;
    }

    int i;
    if ((i = rep.indexOf('.')) != -1) {
      rep = rep.substring(0, i);
    }

    String builtinType = NodeUtils.getBuiltinType(rep);
    if (builtinType != null) {
      isActuallyUndefined = false; // this is a builtin, so, it is defined after all
    }
    return new Tuple<Boolean, String>(isActuallyUndefined, rep);
  }
Example #12
0
  @Override
  public boolean isInGlobalTokens(
      String tok, IPythonNature nature, ICompletionCache completionCache) {
    // we have to override because there is no way to check if it is in some import from some other
    // place if it has dots on the tok...

    if (tok.indexOf('.') == -1) {
      return isInDirectGlobalTokens(tok, completionCache);
    } else {
      ICompletionState state =
          CompletionStateFactory.getEmptyCompletionState(nature, completionCache);
      String[] headAndTail = FullRepIterable.headAndTail(tok);
      state.setActivationToken(headAndTail[0]);
      String head = headAndTail[1];
      IToken[] globalTokens = getGlobalTokens(state, nature.getAstManager());
      for (IToken token : globalTokens) {
        if (token.getRepresentation().equals(head)) {
          return true;
        }
      }
    }
    return false;
  }
Example #13
0
  /* (non-Javadoc)
   * @see org.python.pydev.editor.codecompletion.IPyCodeCompletion#getCodeCompletionProposals(org.eclipse.jface.text.ITextViewer, org.python.pydev.editor.codecompletion.CompletionRequest)
   */
  @SuppressWarnings("unchecked")
  public List getCodeCompletionProposals(ITextViewer viewer, CompletionRequest request)
      throws CoreException, BadLocationException, IOException, MisconfigurationException,
          PythonNatureWithoutProjectException {
    if (request.getPySelection().getCursorLineContents().trim().startsWith("#")) {
      // this may happen if the context is still not correctly computed in python
      return new PyStringCodeCompletion().getCodeCompletionProposals(viewer, request);
    }
    if (DebugSettings.DEBUG_CODE_COMPLETION) {
      Log.toLogFile(this, "Starting getCodeCompletionProposals");
      Log.addLogLevel();
      Log.toLogFile(this, "Request:" + request);
    }

    ArrayList<ICompletionProposal> ret = new ArrayList<ICompletionProposal>();

    // let's see if we should do a code-completion in the current scope...
    if (!isValidCompletionContext(request)) {
      request.showTemplates = false;
      return ret;
    }

    try {
      IPythonNature pythonNature = request.nature;
      checkPythonNature(pythonNature);

      ICodeCompletionASTManager astManager = pythonNature.getAstManager();
      if (astManager == null) {
        // we're probably still loading it.
        return ret;
      }

      // list of Object[], IToken or ICompletionProposal
      List<Object> tokensList = new ArrayList<Object>();
      lazyStartShell(request);
      String trimmed = request.activationToken.replace('.', ' ').trim();

      ImportInfo importsTipper = getImportsTipperStr(request);

      int line = request.doc.getLineOfOffset(request.documentOffset);
      IRegion region = request.doc.getLineInformation(line);

      ICompletionState state =
          new CompletionState(
              line,
              request.documentOffset - region.getOffset(),
              null,
              request.nature,
              request.qualifier);
      state.setIsInCalltip(request.isInCalltip);

      boolean importsTip = false;

      if (importsTipper.importsTipperStr.length() != 0) {
        // code completion in imports
        request.isInCalltip = false; // if found after (, but in an import, it is not a calltip!
        importsTip = doImportCompletion(request, astManager, tokensList, importsTipper);

      } else if (trimmed.length() > 0 && request.activationToken.indexOf('.') != -1) {
        // code completion for a token
        doTokenCompletion(request, astManager, tokensList, trimmed, state);

      } else {
        // go to globals
        doGlobalsCompletion(request, astManager, tokensList, state);
      }

      Map<String, IToken> alreadyChecked = new HashMap<String, IToken>();

      String lowerCaseQual = request.qualifier.toLowerCase();
      if (lowerCaseQual.length()
          >= PyCodeCompletionPreferencesPage.getArgumentsDeepAnalysisNChars()) {
        // this can take some time on the analysis, so, let's let the user choose on how many chars
        // does he
        // want to do the analysis...
        state.pushFindResolveImportMemoryCtx();
        try {
          for (Iterator<Object> it = tokensList.listIterator(); it.hasNext(); ) {
            Object o = it.next();
            if (o instanceof IToken) {
              it
                  .remove(); // always remove the tokens from the list (they'll be re-added later
                             // once they are filtered)

              IToken initialToken = (IToken) o;

              IToken token = initialToken;
              String strRep = token.getRepresentation();
              IToken prev = alreadyChecked.get(strRep);

              if (prev != null) {
                if (prev.getArgs().length() != 0) {
                  continue; // we already have a version with args... just keep going
                }
              }

              if (!strRep.toLowerCase().startsWith(lowerCaseQual)) {
                // just re-add it if we're going to actually use it (depending on the qualifier)
                continue;
              }

              while (token.isImportFrom()) {
                // we'll only add it here if it is an import from (so, set the flag to false for the
                // outer add)

                if (token.getArgs().length() > 0) {
                  // if we already have the args, there's also no reason to do it (that's what we'll
                  // do here)
                  break;
                }
                ICompletionState s =
                    state.getCopyForResolveImportWithActTok(token.getRepresentation());
                s.checkFindResolveImportMemory(token);

                IToken token2 = astManager.resolveImport(s, token);
                if (token2 != null && initialToken != token2) {
                  String args = token2.getArgs();
                  if (args.length() > 0) {
                    // put it into the map (may override previous if it didn't have args)
                    initialToken.setArgs(args);
                    initialToken.setDocStr(token2.getDocStr());
                    if (initialToken instanceof SourceToken && token2 instanceof SourceToken) {
                      SourceToken initialSourceToken = (SourceToken) initialToken;
                      SourceToken token2SourceToken = (SourceToken) token2;
                      initialSourceToken.setAst(token2SourceToken.getAst());
                    }
                    break;
                  }
                  if (token2 == null
                      || (token2.equals(token)
                          && token2.getArgs().equals(token.getArgs())
                          && token2.getParentPackage().equals(token.getParentPackage()))) {
                    break;
                  }
                  token = token2;
                } else {
                  break;
                }
              }

              alreadyChecked.put(strRep, initialToken);
            }
          }

        } finally {
          state.popFindResolveImportMemoryCtx();
        }
      }

      tokensList.addAll(alreadyChecked.values());
      changeItokenToCompletionPropostal(viewer, request, ret, tokensList, importsTip, state);
    } catch (CompletionRecursionException e) {
      if (onCompletionRecursionException != null) {
        onCompletionRecursionException.call(e);
      }
      if (DebugSettings.DEBUG_CODE_COMPLETION) {
        Log.toLogFile(e);
      }
      // PydevPlugin.log(e);
      // ret.add(new CompletionProposal("",request.documentOffset,0,0,null,e.getMessage(),
      // null,null));
    }

    if (DebugSettings.DEBUG_CODE_COMPLETION) {
      Log.remLogLevel();
      Log.toLogFile(this, "Finished completion. Returned:" + ret.size() + " completions.\r\n");
    }

    return ret;
  }
Example #14
0
 /** adds a message of some type for a given token */
 public void addMessage(int type, IToken token) {
   List<IMessage> msgs = getMsgsList(token);
   doAddMessage(msgs, type, token.getRepresentation(), token);
 }
Example #15
0
 /** adds a message of some type for some Found instance */
 public void addMessage(int type, IToken generator, IToken tok) {
   addMessage(type, generator, tok, tok.getRepresentation());
 }
Example #16
0
  /**
   * @see
   *     org.python.pydev.editor.codecompletion.revisited.modules.AbstractModule#getGlobalTokens(java.lang.String)
   */
  public IToken[] getGlobalTokens(ICompletionState state, ICodeCompletionASTManager manager) {
    String activationToken = state.getActivationToken();
    if (activationToken.length() == 0) {
      return getGlobalTokens();
    }

    Map<String, IToken> v = cache.get(activationToken);
    if (v != null) {
      Collection<IToken> values = v.values();
      return values.toArray(new IToken[values.size()]);
    }

    IToken[] toks = new IToken[0];

    if (COMPILED_MODULES_ENABLED) {
      try {
        final IPythonNature nature = manager.getNature();

        final AbstractShell shell;
        try {
          shell = AbstractShell.getServerShell(nature, AbstractShell.COMPLETION_SHELL);
        } catch (Exception e) {
          throw new RuntimeException("Unable to create shell for CompiledModule: " + this.name, e);
        }
        synchronized (shell) {
          String act = name + '.' + activationToken;
          String tokenToCompletion = act;
          if (isPythonBuiltin) {
            String replacement = BUILTIN_REPLACEMENTS.get(activationToken);
            if (replacement != null) {
              tokenToCompletion = name + '.' + replacement;
            }
          }

          List<String[]> completions =
              shell.getImportCompletions(
                      tokenToCompletion,
                      manager
                          .getModulesManager()
                          .getCompletePythonPath(
                              nature.getProjectInterpreter(),
                              nature.getRelatedInterpreterManager()))
                  .o2;

          ArrayList<IToken> array = new ArrayList<IToken>();

          for (Iterator<String[]> iter = completions.iterator(); iter.hasNext(); ) {
            String[] element = iter.next();
            if (element.length >= 4) { // it might be a server error
              IToken t =
                  new CompiledToken(
                      element[0], element[1], element[2], act, Integer.parseInt(element[3]));
              array.add(t);
            }
          }
          toks = (CompiledToken[]) array.toArray(new CompiledToken[0]);
          HashMap<String, IToken> map = new HashMap<String, IToken>();
          for (IToken token : toks) {
            map.put(token.getRepresentation(), token);
          }
          cache.put(activationToken, map);
        }
      } catch (Exception e) {
        Log.log(
            "Error while getting info for module:"
                + this.name
                + ". Project: "
                + manager.getNature().getProject(),
            e);
      }
    }
    return toks;
  }