/**
* @param workspaceMetadataFile
* @throws IOException
*/
public void loadFromFile(File pythonpatHelperFile) throws IOException {
String fileContents = REF.getFileContents(pythonpatHelperFile);
if (fileContents == null || fileContents.trim().length() == 0) {
throw new IOException("No loaded contents from: " + pythonpatHelperFile);
}
this.pythonpath.addAll(StringUtils.split(fileContents, '\n'));
}
@Override
protected void setUp() throws Exception {
baseDir =
new File(REF.getFileAbsolutePath(new File("InterpreterInfoBuilderTest.temporary_dir")));
try {
REF.deleteDirectoryTree(baseDir);
} catch (Exception e) {
// ignore
}
libDir = new File(baseDir, "Lib");
libDir.mkdirs();
REF.writeStrToFile("class Module1:pass", new File(libDir, "module1.py"));
REF.writeStrToFile("class Module2:pass", new File(libDir, "module2.py"));
REF.writeStrToFile("class Module3:pass", new File(libDir, "module3.py"));
PydevTestUtils.setTestPlatformStateLocation();
REF.IN_TESTS = true;
}
/**
* Adds to both: the memory and the disk
*/
public synchronized void add(String key, Serializable n) {
synchronized(cache){
super.add(key, n);
File fileForKey = getFileForKey(key);
if(DEBUG){
System.out.println("Disk cache - Adding: "+key+" file: "+fileForKey);
}
REF.writeToFile(n, fileForKey);
keys.add(key);
}
}
public void saveAstManager() {
File astOutputFile = getAstOutputFile();
if (astOutputFile == null) {
// The project was removed. Nothing to save here.
Log.log(
IStatus.INFO,
"Not saving ast manager for: " + this.project + ". No write area available.",
null);
return;
}
if (astManager == null) {
REF.writeToFile(null, astOutputFile);
} else {
synchronized (astManager.getLock()) {
REF.writeToFile(astManager, astOutputFile);
}
}
}
/** Creates a grinder.jar file in the project. */
protected void createGrinderJar(NullProgressMonitor monitor, IProject project)
throws CoreException {
String grinderJarLocatioon = project.getLocation().toOSString() + "/grinder.jar";
File grinderJarFile = new File(grinderJarLocatioon);
if (!grinderJarFile.exists()) {
REF.copyFile(
TestDependent.TEST_PYDEV_PLUGIN_LOC
+ "tests_completions/org/python/pydev/editor/codecompletion/revisited/javaintegration/grinder.jar",
grinderJarLocatioon);
}
project.refreshLocal(IResource.DEPTH_INFINITE, monitor);
}
private void saveTo(File pathToSave) {
synchronized (lock) {
if (DEBUG_ADDITIONAL_INFO) {
System.out.println(
"Saving info "
+ this.getClass().getName()
+ " to file (size = "
+ getAllTokens().size()
+ ") "
+ pathToSave);
}
REF.writeToFile(getInfoToSave(), pathToSave);
}
}
/**
* @param string this is the string that has the pythonpath (separated by |)
* @param lPath OUT: this list is filled with the pythonpath.
*/
private static void parsePythonPathFromStr(String string, List<String> lPath) {
String[] strings = string.split("\\|");
for (int i = 0; i < strings.length; i++) {
String defaultPathStr = getDefaultPathStr(strings[i]);
if (defaultPathStr != null && defaultPathStr.trim().length() > 0) {
File file = new File(defaultPathStr);
if (file.exists()) {
// we have to get it with the appropriate cases and in a canonical form
String path = REF.getFileAbsolutePath(file);
lPath.add(path);
} else {
lPath.add(defaultPathStr);
}
}
}
}
/** @param version IPythonNature.PYTHON_VERSION_XXX */
protected ModuleAdapter createModuleAdapterFromDataSource(String version) throws Throwable {
codeCompletionTestsBase.restorePythonPath(
REF.getFileAbsolutePath(data.file.getParentFile()), true);
PythonModuleManager pythonModuleManager =
new PythonModuleManager(CodeCompletionTestsBase.nature);
if (version != null) {
// As the files will be found in the system, we need to set the system modules manager info.
IModulesManager modulesManager = pythonModuleManager.getIModuleManager();
SystemModulesManager systemModulesManager =
(SystemModulesManager) modulesManager.getSystemModulesManager();
systemModulesManager.setInfo(new InterpreterInfo(version, "", new ArrayList<String>()));
CodeCompletionTestsBase.nature.setVersion(version, null);
}
ModuleAdapter module =
VisitorFactory.createModuleAdapter(
pythonModuleManager,
data.file,
new Document(data.source),
CodeCompletionTestsBase.nature,
CodeCompletionTestsBase.nature);
return module;
}
public synchronized Serializable getObj(String key) {
synchronized(cache){
Serializable v = super.getObj(key);
if(v == null && keys.contains(key)){
//miss in memory... get from disk
File file = getFileForKey(key);
if(file.exists()){
v = (Serializable) REF.readFromFile(file);
}else{
if(DEBUG){
System.out.println("File: "+file+" is in the cache but does not exist (so, it will be removed).");
}
}
if(v == null){
this.remove(key);
return null;
}
//put it back in memory
super.add(key, v);
}
return v;
}
}
public static ModuleAdapter createModuleAdapter(
PythonModuleManager pythonModuleManager, File file, IDocument doc, IPythonNature nature)
throws Throwable {
if (file != null && file.exists()) {
if (pythonModuleManager != null) {
IModulesManager modulesManager = pythonModuleManager.getIModuleManager();
if (modulesManager != null) {
String modName = modulesManager.resolveModule(REF.getFileAbsolutePath(file));
if (modName != null) {
IModule module = modulesManager.getModule(modName, nature, true);
if (module instanceof ISourceModule) {
SourceModule iSourceModule = (SourceModule) module;
if (iSourceModule.parseError != null) {
throw iSourceModule.parseError;
}
return new ModuleAdapter(pythonModuleManager, ((ISourceModule) module), nature, doc);
}
}
}
}
}
return new ModuleAdapter(pythonModuleManager, file, doc, getRootNode(doc), nature);
}
@Override
protected void tearDown() throws Exception {
REF.deleteDirectoryTree(baseDir);
}
/**
* @see
* org.python.pydev.core.IProjectModulesManager#resolveModule(org.eclipse.core.resources.IResource,
* org.eclipse.core.resources.IProject)
*/
public String resolveModule(IResource member, IProject container) {
File inOs = member.getRawLocation().toFile();
return resolveModule(REF.getFileAbsolutePath(inOs));
}
public DiskCache(int maxSize, File folderToPersist, String suffix) {
super(maxSize);
this.folderToPersist = REF.getFileAbsolutePath(folderToPersist);
this.suffix = suffix;
}
/**
* @param findInfo
* @see
* org.python.pydev.editor.codecompletion.revisited.modules.AbstractModule#findDefinition(java.lang.String,
* int, int)
*/
public Definition[] findDefinition(
ICompletionState state, int line, int col, IPythonNature nature) throws Exception {
String token = state.getActivationToken();
if (TRACE_COMPILED_MODULES) {
System.out.println("CompiledModule.findDefinition:" + token);
}
Definition[] found = this.definitionsFoundCache.getObj(token);
if (found != null) {
if (TRACE_COMPILED_MODULES) {
System.out.println("CompiledModule.findDefinition: found in cache.");
}
return found;
}
AbstractShell shell = AbstractShell.getServerShell(nature, AbstractShell.COMPLETION_SHELL);
synchronized (shell) {
Tuple<String[], int[]> def =
shell.getLineCol(
this.name,
token,
nature
.getAstManager()
.getModulesManager()
.getCompletePythonPath(
nature.getProjectInterpreter(),
nature.getRelatedInterpreterManager())); // default
if (def == null) {
if (TRACE_COMPILED_MODULES) {
System.out.println("CompiledModule.findDefinition:" + token + " = empty");
}
this.definitionsFoundCache.add(token, EMPTY_DEFINITION);
return EMPTY_DEFINITION;
}
String fPath = def.o1[0];
if (fPath.equals("None")) {
if (TRACE_COMPILED_MODULES) {
System.out.println("CompiledModule.findDefinition:" + token + " = None");
}
Definition[] definition =
new Definition[] {new Definition(def.o2[0], def.o2[1], token, null, null, this)};
this.definitionsFoundCache.add(token, definition);
return definition;
}
File f = new File(fPath);
String foundModName = nature.resolveModule(f);
String foundAs = def.o1[1];
IModule mod;
if (foundModName == null) {
// this can happen in a case where we have a definition that's found from a compiled file
// which actually
// maps to a file that's outside of the pythonpath known by Pydev.
String n = FullRepIterable.getFirstPart(f.getName());
mod = AbstractModule.createModule(n, f, nature, true);
} else {
mod = nature.getAstManager().getModule(foundModName, nature, true);
}
if (TRACE_COMPILED_MODULES) {
System.out.println("CompiledModule.findDefinition: found at:" + mod.getName());
}
int foundLine = def.o2[0];
if (foundLine == 0
&& foundAs != null
&& foundAs.length() > 0
&& mod != null
&& state.canStillCheckFindSourceFromCompiled(mod, foundAs)) {
// TODO: The nature (and so the grammar to be used) must be defined by the file we'll parse
// (so, we need to know the system modules manager that actually created it to know the
// actual nature)
IModule sourceMod =
AbstractModule.createModuleFromDoc(
mod.getName(), f, new Document(REF.getFileContents(f)), nature, true);
if (sourceMod instanceof SourceModule) {
Definition[] definitions =
(Definition[])
sourceMod.findDefinition(state.getCopyWithActTok(foundAs), -1, -1, nature);
if (definitions.length > 0) {
this.definitionsFoundCache.add(token, definitions);
return definitions;
}
}
}
if (mod == null) {
mod = this;
}
int foundCol = def.o2[1];
if (foundCol < 0) {
foundCol = 0;
}
if (TRACE_COMPILED_MODULES) {
System.out.println("CompiledModule.findDefinition: found compiled at:" + mod.getName());
}
Definition[] definitions =
new Definition[] {new Definition(foundLine + 1, foundCol + 1, token, null, null, mod)};
this.definitionsFoundCache.add(token, definitions);
return definitions;
}
}
/** @param pythonpatHelperFile */
public void saveToFile(File pythonpatHelperFile) {
REF.writeStrToFile(StringUtils.join("\n", this.pythonpath), pythonpatHelperFile);
}
/**
* This method returns the module that corresponds to the path passed as a parameter.
*
* @param name the name of the module we're looking for (e.g.: mod1.mod2)
* @param dontSearchInit is used in a negative form because initially it was isLookingForRelative,
* but it actually defines if we should look in __init__ modules too, so, the name matches the
* old signature.
* <p>NOTE: isLookingForRelative description was: when looking for relative imports, we don't
* check for __init__
* @return the module represented by this name
*/
protected IModule getModule(
boolean acceptCompiledModule, String name, IPythonNature nature, boolean dontSearchInit) {
if (temporaryModules != null) {
synchronized (lockTemporaryModules) {
if (temporaryModules != null) {
// who knows, maybe it became null at that point.
FastStack<IModule> stack = temporaryModules.get(name);
if (stack != null && stack.size() > 0) {
if (DEBUG_TEMPORARY_MODULES) {
System.out.println("Returning temporary module: " + name);
}
return stack.peek();
}
}
}
}
AbstractModule n = null;
ModulesKey keyForCacheAccess = new ModulesKey(null, null);
if (!dontSearchInit) {
if (n == null) {
keyForCacheAccess.name = new StringBuffer(name).append(".__init__").toString();
n = cache.getObj(keyForCacheAccess, this);
if (n != null) {
name += ".__init__";
}
}
}
if (n == null) {
keyForCacheAccess.name = name;
n = cache.getObj(keyForCacheAccess, this);
}
if (n instanceof SourceModule) {
// ok, module exists, let's check if it is synched with the filesystem version...
SourceModule s = (SourceModule) n;
if (!s.isSynched()) {
// change it for an empty and proceed as usual.
n = (AbstractModule) addModule(createModulesKey(s.getName(), s.getFile()));
}
}
if (n instanceof EmptyModule) {
EmptyModule e = (EmptyModule) n;
boolean found = false;
if (!found && e.f != null) {
if (!e.f.exists()) {
// if the file does not exist anymore, just remove it.
keyForCacheAccess.name = name;
keyForCacheAccess.file = e.f;
doRemoveSingleModule(keyForCacheAccess);
n = null;
} else {
// file exists
n = checkOverride(name, nature, n);
if (n instanceof EmptyModule) {
// ok, handle case where the file is actually from a zip file...
if (e instanceof EmptyModuleForZip) {
EmptyModuleForZip emptyModuleForZip = (EmptyModuleForZip) e;
if (emptyModuleForZip.pathInZip.endsWith(".class") || !emptyModuleForZip.isFile) {
// handle java class... (if it's a class or a folder in a jar)
n = JythonModulesManagerUtils.createModuleFromJar(emptyModuleForZip);
n = decorateModule(n, nature);
} else if (FileTypesPreferencesPage.isValidDll(emptyModuleForZip.pathInZip)) {
// .pyd
n = new CompiledModule(name, IToken.TYPE_BUILTIN, nature.getAstManager());
n = decorateModule(n, nature);
} else if (PythonPathHelper.isValidSourceFile(emptyModuleForZip.pathInZip)) {
// handle python file from zip... we have to create it getting the contents from the
// zip file
try {
IDocument doc =
REF.getDocFromZip(emptyModuleForZip.f, emptyModuleForZip.pathInZip);
// NOTE: The nature (and so the grammar to be used) must be defined by this
// modules
// manager (and not by the initial caller)!!
n =
AbstractModule.createModuleFromDoc(
name, emptyModuleForZip.f, doc, this.getNature(), -1, false);
SourceModule zipModule = (SourceModule) n;
zipModule.zipFilePath = emptyModuleForZip.pathInZip;
n = decorateModule(n, nature);
} catch (Exception exc1) {
PydevPlugin.log(exc1);
n = null;
}
}
} else {
// regular case... just go on and create it.
try {
// NOTE: The nature (and so the grammar to be used) must be defined by this modules
// manager (and not by the initial caller)!!
n = AbstractModule.createModule(name, e.f, this.getNature(), -1);
n = decorateModule(n, nature);
} catch (IOException exc) {
keyForCacheAccess.name = name;
keyForCacheAccess.file = e.f;
doRemoveSingleModule(keyForCacheAccess);
n = null;
} catch (MisconfigurationException exc) {
PydevPlugin.log(exc);
n = null;
}
}
}
}
} else { // ok, it does not have a file associated, so, we treat it as a builtin (this can
// happen in java jars)
n = checkOverride(name, nature, n);
if (n instanceof EmptyModule) {
if (acceptCompiledModule) {
n = new CompiledModule(name, IToken.TYPE_BUILTIN, nature.getAstManager());
n = decorateModule(n, nature);
} else {
return null;
}
}
}
if (n != null) {
doAddSingleModule(createModulesKey(name, e.f), n);
} else {
System.err.println("The module " + name + " could not be found nor created!");
}
}
if (n instanceof EmptyModule) {
throw new RuntimeException("Should not be an empty module anymore!");
}
if (n instanceof SourceModule) {
SourceModule sourceModule = (SourceModule) n;
// now, here's a catch... it may be a bootstrap module...
if (sourceModule.isBootstrapModule()) {
// if it's a bootstrap module, we must replace it for the related compiled module.
n = new CompiledModule(name, IToken.TYPE_BUILTIN, nature.getAstManager());
n = decorateModule(n, nature);
}
}
return n;
}
/**
* DAMN... when I started thinking this up, it seemed much better... (and easier)
*
* @param module - this is the full path of the module. Only for directories or py,pyd,dll,pyo
* files.
* @return a String with the module that the file or folder should represent. E.g.: compiler.ast
*/
public String resolveModule(
String fullPath, final boolean requireFileToExist, List<String> pythonPathCopy) {
fullPath = REF.getFileAbsolutePath(fullPath);
fullPath = getDefaultPathStr(fullPath);
String fullPathWithoutExtension;
if (isValidSourceFile(fullPath) || FileTypesPreferencesPage.isValidDll(fullPath)) {
fullPathWithoutExtension = FullRepIterable.headAndTail(fullPath)[0];
} else {
fullPathWithoutExtension = fullPath;
}
final File moduleFile = new File(fullPath);
if (requireFileToExist && !moduleFile.exists()) {
return null;
}
boolean isFile = moduleFile.isFile();
// go through our pythonpath and check the beginning
for (String pathEntry : pythonPathCopy) {
String element = getDefaultPathStr(pathEntry);
if (fullPath.startsWith(element)) {
int len = element.length();
String s = fullPath.substring(len);
String sWithoutExtension = fullPathWithoutExtension.substring(len);
if (s.startsWith("/")) {
s = s.substring(1);
}
if (sWithoutExtension.startsWith("/")) {
sWithoutExtension = sWithoutExtension.substring(1);
}
if (!isValidModuleLastPart(sWithoutExtension)) {
continue;
}
s = s.replaceAll("/", ".");
if (s.indexOf(".") != -1) {
File root = new File(element);
if (root.exists() == false) {
continue;
}
final List<String> temp = StringUtils.dotSplit(s);
String[] modulesParts = temp.toArray(new String[temp.size()]);
// this means that more than 1 module is specified, so, in order to get it,
// we have to go and see if all the folders to that module have __init__.py in it...
if (modulesParts.length > 1 && isFile) {
String[] t = new String[modulesParts.length - 1];
for (int i = 0; i < modulesParts.length - 1; i++) {
t[i] = modulesParts[i];
}
t[t.length - 1] = t[t.length - 1] + "." + modulesParts[modulesParts.length - 1];
modulesParts = t;
}
// here, in modulesParts, we have something like
// ["compiler", "ast.py"] - if file
// ["pywin","debugger"] - if folder
//
// root starts with the pythonpath folder that starts with the same
// chars as the full path passed in.
boolean isValid = true;
for (int i = 0; i < modulesParts.length && root != null; i++) {
root = new File(REF.getFileAbsolutePath(root) + "/" + modulesParts[i]);
// check if file is in root...
if (isValidFileMod(modulesParts[i])) {
if (root.exists() && root.isFile()) {
break;
}
} else {
// this part is a folder part... check if it is a valid module (has init).
if (isFileOrFolderWithInit(root) == false) {
isValid = false;
break;
}
// go on and check the next part.
}
}
if (isValid) {
if (isFile) {
s = stripExtension(s);
} else if (moduleFile.exists() == false) {
// ok, it does not exist, so isFile will not work, let's just check if it is
// a valid module (ends with .py or .pyw) and if it is, strip the extension
if (isValidFileMod(s)) {
s = stripExtension(s);
}
}
return s;
}
} else {
// simple part, we don't have to go into subfolders to check validity...
if (!isFile && moduleFile.isDirectory() && isFileOrFolderWithInit(moduleFile) == false) {
return null;
}
return s;
}
}
}
// ok, it was not found in any existing way, so, if we don't require the file to exist, let's
// just do some simpler search and get the
// first match (if any)... this is useful if the file we are looking for has just been deleted
if (!requireFileToExist) {
// we have to remove the last part (.py, .pyc, .pyw)
for (String element : pythonPathCopy) {
element = getDefaultPathStr(element);
if (fullPathWithoutExtension.startsWith(element)) {
String s = fullPathWithoutExtension.substring(element.length());
if (s.startsWith("/")) {
s = s.substring(1);
}
if (!isValidModuleLastPart(s)) {
continue;
}
s = s.replaceAll("/", ".");
return s;
}
}
}
return null;
}