예제 #1
0
 /**
  * @param workspaceMetadataFile
  * @throws IOException
  */
 public void loadFromFile(File pythonpatHelperFile) throws IOException {
   String fileContents = REF.getFileContents(pythonpatHelperFile);
   if (fileContents == null || fileContents.trim().length() == 0) {
     throw new IOException("No loaded contents from: " + pythonpatHelperFile);
   }
   this.pythonpath.addAll(StringUtils.split(fileContents, '\n'));
 }
예제 #2
0
  @Override
  protected void setUp() throws Exception {
    baseDir =
        new File(REF.getFileAbsolutePath(new File("InterpreterInfoBuilderTest.temporary_dir")));
    try {
      REF.deleteDirectoryTree(baseDir);
    } catch (Exception e) {
      // ignore
    }

    libDir = new File(baseDir, "Lib");
    libDir.mkdirs();

    REF.writeStrToFile("class Module1:pass", new File(libDir, "module1.py"));
    REF.writeStrToFile("class Module2:pass", new File(libDir, "module2.py"));
    REF.writeStrToFile("class Module3:pass", new File(libDir, "module3.py"));

    PydevTestUtils.setTestPlatformStateLocation();
    REF.IN_TESTS = true;
  }
예제 #3
0
 /**
  * Adds to both: the memory and the disk
  */
 public synchronized void add(String key, Serializable n) {
     synchronized(cache){
         super.add(key, n);
         File fileForKey = getFileForKey(key);
         if(DEBUG){
             System.out.println("Disk cache - Adding: "+key+" file: "+fileForKey);
         }
         REF.writeToFile(n, fileForKey);
         keys.add(key);
     }
 }
예제 #4
0
  public void saveAstManager() {
    File astOutputFile = getAstOutputFile();
    if (astOutputFile == null) {
      // The project was removed. Nothing to save here.
      Log.log(
          IStatus.INFO,
          "Not saving ast manager for: " + this.project + ". No write area available.",
          null);
      return;
    }

    if (astManager == null) {
      REF.writeToFile(null, astOutputFile);

    } else {
      synchronized (astManager.getLock()) {
        REF.writeToFile(astManager, astOutputFile);
      }
    }
  }
예제 #5
0
 /** Creates a grinder.jar file in the project. */
 protected void createGrinderJar(NullProgressMonitor monitor, IProject project)
     throws CoreException {
   String grinderJarLocatioon = project.getLocation().toOSString() + "/grinder.jar";
   File grinderJarFile = new File(grinderJarLocatioon);
   if (!grinderJarFile.exists()) {
     REF.copyFile(
         TestDependent.TEST_PYDEV_PLUGIN_LOC
             + "tests_completions/org/python/pydev/editor/codecompletion/revisited/javaintegration/grinder.jar",
         grinderJarLocatioon);
   }
   project.refreshLocal(IResource.DEPTH_INFINITE, monitor);
 }
 private void saveTo(File pathToSave) {
   synchronized (lock) {
     if (DEBUG_ADDITIONAL_INFO) {
       System.out.println(
           "Saving info "
               + this.getClass().getName()
               + " to file (size = "
               + getAllTokens().size()
               + ") "
               + pathToSave);
     }
     REF.writeToFile(getInfoToSave(), pathToSave);
   }
 }
예제 #7
0
 /**
  * @param string this is the string that has the pythonpath (separated by |)
  * @param lPath OUT: this list is filled with the pythonpath.
  */
 private static void parsePythonPathFromStr(String string, List<String> lPath) {
   String[] strings = string.split("\\|");
   for (int i = 0; i < strings.length; i++) {
     String defaultPathStr = getDefaultPathStr(strings[i]);
     if (defaultPathStr != null && defaultPathStr.trim().length() > 0) {
       File file = new File(defaultPathStr);
       if (file.exists()) {
         // we have to get it with the appropriate cases and in a canonical form
         String path = REF.getFileAbsolutePath(file);
         lPath.add(path);
       } else {
         lPath.add(defaultPathStr);
       }
     }
   }
 }
예제 #8
0
  /** @param version IPythonNature.PYTHON_VERSION_XXX */
  protected ModuleAdapter createModuleAdapterFromDataSource(String version) throws Throwable {
    codeCompletionTestsBase.restorePythonPath(
        REF.getFileAbsolutePath(data.file.getParentFile()), true);
    PythonModuleManager pythonModuleManager =
        new PythonModuleManager(CodeCompletionTestsBase.nature);
    if (version != null) {
      // As the files will be found in the system, we need to set the system modules manager info.
      IModulesManager modulesManager = pythonModuleManager.getIModuleManager();
      SystemModulesManager systemModulesManager =
          (SystemModulesManager) modulesManager.getSystemModulesManager();
      systemModulesManager.setInfo(new InterpreterInfo(version, "", new ArrayList<String>()));

      CodeCompletionTestsBase.nature.setVersion(version, null);
    }
    ModuleAdapter module =
        VisitorFactory.createModuleAdapter(
            pythonModuleManager,
            data.file,
            new Document(data.source),
            CodeCompletionTestsBase.nature,
            CodeCompletionTestsBase.nature);
    return module;
  }
예제 #9
0
 public synchronized Serializable getObj(String key) {
     synchronized(cache){
         Serializable v = super.getObj(key);
         if(v == null && keys.contains(key)){
             //miss in memory... get from disk
             File file = getFileForKey(key);
             if(file.exists()){
                 v = (Serializable) REF.readFromFile(file);
             }else{
                 if(DEBUG){
                     System.out.println("File: "+file+" is in the cache but does not exist (so, it will be removed).");
                 }
             }
             if(v == null){
                 this.remove(key);
                 return null;
             }
             //put it back in memory
             super.add(key, v);
         }
         return v;
     }
 }
예제 #10
0
 public static ModuleAdapter createModuleAdapter(
     PythonModuleManager pythonModuleManager, File file, IDocument doc, IPythonNature nature)
     throws Throwable {
   if (file != null && file.exists()) {
     if (pythonModuleManager != null) {
       IModulesManager modulesManager = pythonModuleManager.getIModuleManager();
       if (modulesManager != null) {
         String modName = modulesManager.resolveModule(REF.getFileAbsolutePath(file));
         if (modName != null) {
           IModule module = modulesManager.getModule(modName, nature, true);
           if (module instanceof ISourceModule) {
             SourceModule iSourceModule = (SourceModule) module;
             if (iSourceModule.parseError != null) {
               throw iSourceModule.parseError;
             }
             return new ModuleAdapter(pythonModuleManager, ((ISourceModule) module), nature, doc);
           }
         }
       }
     }
   }
   return new ModuleAdapter(pythonModuleManager, file, doc, getRootNode(doc), nature);
 }
예제 #11
0
 @Override
 protected void tearDown() throws Exception {
   REF.deleteDirectoryTree(baseDir);
 }
예제 #12
0
 /**
  * @see
  *     org.python.pydev.core.IProjectModulesManager#resolveModule(org.eclipse.core.resources.IResource,
  *     org.eclipse.core.resources.IProject)
  */
 public String resolveModule(IResource member, IProject container) {
   File inOs = member.getRawLocation().toFile();
   return resolveModule(REF.getFileAbsolutePath(inOs));
 }
예제 #13
0
 public DiskCache(int maxSize, File folderToPersist, String suffix) {
     super(maxSize);
     this.folderToPersist = REF.getFileAbsolutePath(folderToPersist);
     this.suffix = suffix;
 }
예제 #14
0
  /**
   * @param findInfo
   * @see
   *     org.python.pydev.editor.codecompletion.revisited.modules.AbstractModule#findDefinition(java.lang.String,
   *     int, int)
   */
  public Definition[] findDefinition(
      ICompletionState state, int line, int col, IPythonNature nature) throws Exception {
    String token = state.getActivationToken();

    if (TRACE_COMPILED_MODULES) {
      System.out.println("CompiledModule.findDefinition:" + token);
    }
    Definition[] found = this.definitionsFoundCache.getObj(token);
    if (found != null) {
      if (TRACE_COMPILED_MODULES) {
        System.out.println("CompiledModule.findDefinition: found in cache.");
      }
      return found;
    }

    AbstractShell shell = AbstractShell.getServerShell(nature, AbstractShell.COMPLETION_SHELL);
    synchronized (shell) {
      Tuple<String[], int[]> def =
          shell.getLineCol(
              this.name,
              token,
              nature
                  .getAstManager()
                  .getModulesManager()
                  .getCompletePythonPath(
                      nature.getProjectInterpreter(),
                      nature.getRelatedInterpreterManager())); // default
      if (def == null) {
        if (TRACE_COMPILED_MODULES) {
          System.out.println("CompiledModule.findDefinition:" + token + " = empty");
        }
        this.definitionsFoundCache.add(token, EMPTY_DEFINITION);
        return EMPTY_DEFINITION;
      }
      String fPath = def.o1[0];
      if (fPath.equals("None")) {
        if (TRACE_COMPILED_MODULES) {
          System.out.println("CompiledModule.findDefinition:" + token + " = None");
        }
        Definition[] definition =
            new Definition[] {new Definition(def.o2[0], def.o2[1], token, null, null, this)};
        this.definitionsFoundCache.add(token, definition);
        return definition;
      }
      File f = new File(fPath);
      String foundModName = nature.resolveModule(f);
      String foundAs = def.o1[1];

      IModule mod;
      if (foundModName == null) {
        // this can happen in a case where we have a definition that's found from a compiled file
        // which actually
        // maps to a file that's outside of the pythonpath known by Pydev.
        String n = FullRepIterable.getFirstPart(f.getName());
        mod = AbstractModule.createModule(n, f, nature, true);
      } else {
        mod = nature.getAstManager().getModule(foundModName, nature, true);
      }

      if (TRACE_COMPILED_MODULES) {
        System.out.println("CompiledModule.findDefinition: found at:" + mod.getName());
      }
      int foundLine = def.o2[0];
      if (foundLine == 0
          && foundAs != null
          && foundAs.length() > 0
          && mod != null
          && state.canStillCheckFindSourceFromCompiled(mod, foundAs)) {
        // TODO: The nature (and so the grammar to be used) must be defined by the file we'll parse
        // (so, we need to know the system modules manager that actually created it to know the
        // actual nature)
        IModule sourceMod =
            AbstractModule.createModuleFromDoc(
                mod.getName(), f, new Document(REF.getFileContents(f)), nature, true);
        if (sourceMod instanceof SourceModule) {
          Definition[] definitions =
              (Definition[])
                  sourceMod.findDefinition(state.getCopyWithActTok(foundAs), -1, -1, nature);
          if (definitions.length > 0) {
            this.definitionsFoundCache.add(token, definitions);
            return definitions;
          }
        }
      }
      if (mod == null) {
        mod = this;
      }
      int foundCol = def.o2[1];
      if (foundCol < 0) {
        foundCol = 0;
      }
      if (TRACE_COMPILED_MODULES) {
        System.out.println("CompiledModule.findDefinition: found compiled at:" + mod.getName());
      }
      Definition[] definitions =
          new Definition[] {new Definition(foundLine + 1, foundCol + 1, token, null, null, mod)};
      this.definitionsFoundCache.add(token, definitions);
      return definitions;
    }
  }
예제 #15
0
 /** @param pythonpatHelperFile */
 public void saveToFile(File pythonpatHelperFile) {
   REF.writeStrToFile(StringUtils.join("\n", this.pythonpath), pythonpatHelperFile);
 }
예제 #16
0
  /**
   * This method returns the module that corresponds to the path passed as a parameter.
   *
   * @param name the name of the module we're looking for (e.g.: mod1.mod2)
   * @param dontSearchInit is used in a negative form because initially it was isLookingForRelative,
   *     but it actually defines if we should look in __init__ modules too, so, the name matches the
   *     old signature.
   *     <p>NOTE: isLookingForRelative description was: when looking for relative imports, we don't
   *     check for __init__
   * @return the module represented by this name
   */
  protected IModule getModule(
      boolean acceptCompiledModule, String name, IPythonNature nature, boolean dontSearchInit) {
    if (temporaryModules != null) {
      synchronized (lockTemporaryModules) {
        if (temporaryModules != null) {
          // who knows, maybe it became null at that point.
          FastStack<IModule> stack = temporaryModules.get(name);
          if (stack != null && stack.size() > 0) {
            if (DEBUG_TEMPORARY_MODULES) {
              System.out.println("Returning temporary module: " + name);
            }
            return stack.peek();
          }
        }
      }
    }
    AbstractModule n = null;
    ModulesKey keyForCacheAccess = new ModulesKey(null, null);

    if (!dontSearchInit) {
      if (n == null) {
        keyForCacheAccess.name = new StringBuffer(name).append(".__init__").toString();
        n = cache.getObj(keyForCacheAccess, this);
        if (n != null) {
          name += ".__init__";
        }
      }
    }
    if (n == null) {
      keyForCacheAccess.name = name;
      n = cache.getObj(keyForCacheAccess, this);
    }

    if (n instanceof SourceModule) {
      // ok, module exists, let's check if it is synched with the filesystem version...
      SourceModule s = (SourceModule) n;
      if (!s.isSynched()) {
        // change it for an empty and proceed as usual.
        n = (AbstractModule) addModule(createModulesKey(s.getName(), s.getFile()));
      }
    }

    if (n instanceof EmptyModule) {
      EmptyModule e = (EmptyModule) n;

      boolean found = false;

      if (!found && e.f != null) {

        if (!e.f.exists()) {
          // if the file does not exist anymore, just remove it.
          keyForCacheAccess.name = name;
          keyForCacheAccess.file = e.f;
          doRemoveSingleModule(keyForCacheAccess);
          n = null;

        } else {
          // file exists
          n = checkOverride(name, nature, n);

          if (n instanceof EmptyModule) {
            // ok, handle case where the file is actually from a zip file...
            if (e instanceof EmptyModuleForZip) {
              EmptyModuleForZip emptyModuleForZip = (EmptyModuleForZip) e;

              if (emptyModuleForZip.pathInZip.endsWith(".class") || !emptyModuleForZip.isFile) {
                // handle java class... (if it's a class or a folder in a jar)
                n = JythonModulesManagerUtils.createModuleFromJar(emptyModuleForZip);
                n = decorateModule(n, nature);

              } else if (FileTypesPreferencesPage.isValidDll(emptyModuleForZip.pathInZip)) {
                // .pyd
                n = new CompiledModule(name, IToken.TYPE_BUILTIN, nature.getAstManager());
                n = decorateModule(n, nature);

              } else if (PythonPathHelper.isValidSourceFile(emptyModuleForZip.pathInZip)) {
                // handle python file from zip... we have to create it getting the contents from the
                // zip file
                try {
                  IDocument doc =
                      REF.getDocFromZip(emptyModuleForZip.f, emptyModuleForZip.pathInZip);
                  // NOTE: The nature (and so the grammar to be used) must be defined by this
                  // modules
                  // manager (and not by the initial caller)!!
                  n =
                      AbstractModule.createModuleFromDoc(
                          name, emptyModuleForZip.f, doc, this.getNature(), -1, false);
                  SourceModule zipModule = (SourceModule) n;
                  zipModule.zipFilePath = emptyModuleForZip.pathInZip;
                  n = decorateModule(n, nature);
                } catch (Exception exc1) {
                  PydevPlugin.log(exc1);
                  n = null;
                }
              }

            } else {
              // regular case... just go on and create it.
              try {
                // NOTE: The nature (and so the grammar to be used) must be defined by this modules
                // manager (and not by the initial caller)!!
                n = AbstractModule.createModule(name, e.f, this.getNature(), -1);
                n = decorateModule(n, nature);
              } catch (IOException exc) {
                keyForCacheAccess.name = name;
                keyForCacheAccess.file = e.f;
                doRemoveSingleModule(keyForCacheAccess);
                n = null;
              } catch (MisconfigurationException exc) {
                PydevPlugin.log(exc);
                n = null;
              }
            }
          }
        }

      } else { // ok, it does not have a file associated, so, we treat it as a builtin (this can
               // happen in java jars)
        n = checkOverride(name, nature, n);
        if (n instanceof EmptyModule) {
          if (acceptCompiledModule) {
            n = new CompiledModule(name, IToken.TYPE_BUILTIN, nature.getAstManager());
            n = decorateModule(n, nature);
          } else {
            return null;
          }
        }
      }

      if (n != null) {
        doAddSingleModule(createModulesKey(name, e.f), n);
      } else {
        System.err.println("The module " + name + " could not be found nor created!");
      }
    }

    if (n instanceof EmptyModule) {
      throw new RuntimeException("Should not be an empty module anymore!");
    }
    if (n instanceof SourceModule) {
      SourceModule sourceModule = (SourceModule) n;
      // now, here's a catch... it may be a bootstrap module...
      if (sourceModule.isBootstrapModule()) {
        // if it's a bootstrap module, we must replace it for the related compiled module.
        n = new CompiledModule(name, IToken.TYPE_BUILTIN, nature.getAstManager());
        n = decorateModule(n, nature);
      }
    }

    return n;
  }
예제 #17
0
  /**
   * DAMN... when I started thinking this up, it seemed much better... (and easier)
   *
   * @param module - this is the full path of the module. Only for directories or py,pyd,dll,pyo
   *     files.
   * @return a String with the module that the file or folder should represent. E.g.: compiler.ast
   */
  public String resolveModule(
      String fullPath, final boolean requireFileToExist, List<String> pythonPathCopy) {
    fullPath = REF.getFileAbsolutePath(fullPath);
    fullPath = getDefaultPathStr(fullPath);
    String fullPathWithoutExtension;

    if (isValidSourceFile(fullPath) || FileTypesPreferencesPage.isValidDll(fullPath)) {
      fullPathWithoutExtension = FullRepIterable.headAndTail(fullPath)[0];
    } else {
      fullPathWithoutExtension = fullPath;
    }

    final File moduleFile = new File(fullPath);

    if (requireFileToExist && !moduleFile.exists()) {
      return null;
    }

    boolean isFile = moduleFile.isFile();

    // go through our pythonpath and check the beginning
    for (String pathEntry : pythonPathCopy) {

      String element = getDefaultPathStr(pathEntry);
      if (fullPath.startsWith(element)) {
        int len = element.length();
        String s = fullPath.substring(len);
        String sWithoutExtension = fullPathWithoutExtension.substring(len);

        if (s.startsWith("/")) {
          s = s.substring(1);
        }
        if (sWithoutExtension.startsWith("/")) {
          sWithoutExtension = sWithoutExtension.substring(1);
        }

        if (!isValidModuleLastPart(sWithoutExtension)) {
          continue;
        }

        s = s.replaceAll("/", ".");
        if (s.indexOf(".") != -1) {
          File root = new File(element);
          if (root.exists() == false) {
            continue;
          }

          final List<String> temp = StringUtils.dotSplit(s);
          String[] modulesParts = temp.toArray(new String[temp.size()]);

          // this means that more than 1 module is specified, so, in order to get it,
          // we have to go and see if all the folders to that module have __init__.py in it...
          if (modulesParts.length > 1 && isFile) {
            String[] t = new String[modulesParts.length - 1];

            for (int i = 0; i < modulesParts.length - 1; i++) {
              t[i] = modulesParts[i];
            }
            t[t.length - 1] = t[t.length - 1] + "." + modulesParts[modulesParts.length - 1];
            modulesParts = t;
          }

          // here, in modulesParts, we have something like
          // ["compiler", "ast.py"] - if file
          // ["pywin","debugger"] - if folder
          //
          // root starts with the pythonpath folder that starts with the same
          // chars as the full path passed in.
          boolean isValid = true;
          for (int i = 0; i < modulesParts.length && root != null; i++) {
            root = new File(REF.getFileAbsolutePath(root) + "/" + modulesParts[i]);

            // check if file is in root...
            if (isValidFileMod(modulesParts[i])) {
              if (root.exists() && root.isFile()) {
                break;
              }

            } else {
              // this part is a folder part... check if it is a valid module (has init).
              if (isFileOrFolderWithInit(root) == false) {
                isValid = false;
                break;
              }
              // go on and check the next part.
            }
          }
          if (isValid) {
            if (isFile) {
              s = stripExtension(s);
            } else if (moduleFile.exists() == false) {
              // ok, it does not exist, so isFile will not work, let's just check if it is
              // a valid module (ends with .py or .pyw) and if it is, strip the extension
              if (isValidFileMod(s)) {
                s = stripExtension(s);
              }
            }
            return s;
          }
        } else {
          // simple part, we don't have to go into subfolders to check validity...
          if (!isFile && moduleFile.isDirectory() && isFileOrFolderWithInit(moduleFile) == false) {
            return null;
          }
          return s;
        }
      }
    }
    // ok, it was not found in any existing way, so, if we don't require the file to exist, let's
    // just do some simpler search and get the
    // first match (if any)... this is useful if the file we are looking for has just been deleted
    if (!requireFileToExist) {
      // we have to remove the last part (.py, .pyc, .pyw)
      for (String element : pythonPathCopy) {
        element = getDefaultPathStr(element);
        if (fullPathWithoutExtension.startsWith(element)) {
          String s = fullPathWithoutExtension.substring(element.length());
          if (s.startsWith("/")) {
            s = s.substring(1);
          }
          if (!isValidModuleLastPart(s)) {
            continue;
          }
          s = s.replaceAll("/", ".");
          return s;
        }
      }
    }
    return null;
  }