/**
* Creates the complete package path given by the user (all filled with __init__) and returns the
* last __init__ module created.
*/
public static IFile createPackage(
IProgressMonitor monitor, IContainer validatedSourceFolder, String packageName)
throws CoreException {
IFile lastFile = null;
if (validatedSourceFolder == null) {
return null;
}
IContainer parent = validatedSourceFolder;
for (String packagePart : StringUtils.dotSplit(packageName)) {
IFolder folder = parent.getFolder(new Path(packagePart));
if (!folder.exists()) {
folder.create(true, true, monitor);
}
parent = folder;
IFile file =
parent.getFile(
new Path("__init__" + FileTypesPreferencesPage.getDefaultDottedPythonExtension()));
if (!file.exists()) {
file.create(new ByteArrayInputStream(new byte[0]), true, monitor);
}
lastFile = file;
}
return lastFile;
}
/** @return if the paths maps to a valid python module (depending on its extension). */
public static boolean isValidFileMod(String path) {
boolean ret = false;
if (isValidSourceFile(path)) {
ret = true;
} else if (FileTypesPreferencesPage.isValidDll(path)) {
ret = true;
}
return ret;
}
/** @return if the path passed belongs to a valid python source file (checks for the extension) */
public static boolean isValidSourceFile(String path) {
path = path.toLowerCase();
for (String end : FileTypesPreferencesPage.getDottedValidSourceFiles()) {
if (path.endsWith(end)) {
return true;
}
}
if (path.endsWith(".pypredef")) {
return true;
}
return false;
}
public static void buildKeysForRegularEntries(
IProgressMonitor monitor,
ModulesFoundStructure modulesFound,
PyPublicTreeMap<ModulesKey, ModulesKey> keys,
boolean includeOnlySourceModules) {
String[] dottedValidSourceFiles = FileTypesPreferencesPage.getDottedValidSourceFiles();
int j = 0;
FastStringBuffer buffer = new FastStringBuffer();
// now, create in memory modules for all the loaded files (empty modules).
for (Iterator<Map.Entry<File, String>> iterator =
modulesFound.regularModules.entrySet().iterator();
iterator.hasNext() && monitor.isCanceled() == false;
j++) {
Map.Entry<File, String> entry = iterator.next();
String m = entry.getValue();
if (m != null) {
if (j % 20 == 0) {
// no need to report all the time (that's pretty fast now)
buffer.clear();
monitor.setTaskName(buffer.append("Module resolved: ").append(m).toString());
monitor.worked(1);
}
// we don't load them at this time.
File f = entry.getKey();
if (includeOnlySourceModules) {
// check if we should include only source modules
if (!PythonPathHelper.isValidSourceFile(f.getName())) {
continue;
}
}
ModulesKey modulesKey = new ModulesKey(m, f);
// no conflict (easy)
if (!keys.containsKey(modulesKey)) {
keys.put(modulesKey, modulesKey);
} else {
// we have a conflict, so, let's resolve which one to keep (the old one or this one)
if (PythonPathHelper.isValidSourceFile(f.getName(), dottedValidSourceFiles)) {
// source files have priority over other modules (dlls) -- if both are source, there is
// no real way to resolve
// this priority, so, let's just add it over.
keys.put(modulesKey, modulesKey);
}
}
}
}
}
protected File getFileStructure(File file, String withoutLastPart) {
File f = file;
List<String> split = StringUtils.dotSplit(withoutLastPart);
int size = split.size();
for (int i = 0; i < size; i++) {
if (i == size - 1) {
f = new File(f, split.get(i) + FileTypesPreferencesPage.getDefaultDottedPythonExtension());
} else {
f = new File(f, split.get(i));
}
}
return f;
}
/** @return whether an IFile is a valid source file given its extension */
public static boolean isValidSourceFile(IFile file) {
String ext = file.getFileExtension();
if (ext == null) { // no extension
return false;
}
ext = ext.toLowerCase();
for (String end : FileTypesPreferencesPage.getValidSourceFiles()) {
if (ext.equals(end)) {
return true;
}
}
if (ext.equals(".pypredef")) {
return true;
}
return false;
}
/**
* This method returns the module that corresponds to the path passed as a parameter.
*
* @param name the name of the module we're looking for (e.g.: mod1.mod2)
* @param dontSearchInit is used in a negative form because initially it was isLookingForRelative,
* but it actually defines if we should look in __init__ modules too, so, the name matches the
* old signature.
* <p>NOTE: isLookingForRelative description was: when looking for relative imports, we don't
* check for __init__
* @return the module represented by this name
*/
protected IModule getModule(
boolean acceptCompiledModule, String name, IPythonNature nature, boolean dontSearchInit) {
synchronized (lockTemporaryModules) {
SortedMap<Integer, IModule> map = temporaryModules.get(name);
if (map != null && map.size() > 0) {
if (DEBUG_TEMPORARY_MODULES) {
System.out.println("Returning temporary module: " + name);
}
return map.get(map.lastKey());
}
}
AbstractModule n = null;
ModulesKey keyForCacheAccess = new ModulesKey(null, null);
if (!dontSearchInit) {
if (n == null) {
keyForCacheAccess.name =
(String) StringUtils.join(".", new String[] {name, "__init__"}, null);
n = cache.getObj(keyForCacheAccess, this);
if (n != null) {
name = keyForCacheAccess.name;
}
}
}
if (n == null) {
keyForCacheAccess.name = name;
n = cache.getObj(keyForCacheAccess, this);
}
if (n instanceof SourceModule) {
// ok, module exists, let's check if it is synched with the filesystem version...
SourceModule s = (SourceModule) n;
if (!s.isSynched()) {
// change it for an empty and proceed as usual.
n = (AbstractModule) addModule(createModulesKey(s.getName(), s.getFile()));
}
}
if (n instanceof EmptyModule) {
EmptyModule e = (EmptyModule) n;
if (e.f != null) {
if (!e.f.exists()) {
// if the file does not exist anymore, just remove it.
keyForCacheAccess.name = name;
keyForCacheAccess.file = e.f;
doRemoveSingleModule(keyForCacheAccess);
n = null;
} else {
// file exists
n = checkOverride(name, nature, n);
if (n instanceof EmptyModule) {
// ok, handle case where the file is actually from a zip file...
if (e instanceof EmptyModuleForZip) {
EmptyModuleForZip emptyModuleForZip = (EmptyModuleForZip) e;
if (emptyModuleForZip.pathInZip.endsWith(".class") || !emptyModuleForZip.isFile) {
// handle java class... (if it's a class or a folder in a jar)
n = JythonModulesManagerUtils.createModuleFromJar(emptyModuleForZip);
n = decorateModule(n, nature);
} else if (FileTypesPreferencesPage.isValidDll(emptyModuleForZip.pathInZip)) {
// .pyd
n = new CompiledModule(name, this);
n = decorateModule(n, nature);
} else if (PythonPathHelper.isValidSourceFile(emptyModuleForZip.pathInZip)) {
// handle python file from zip... we have to create it getting the contents from the
// zip file
try {
IDocument doc =
FileUtilsFileBuffer.getDocFromZip(
emptyModuleForZip.f, emptyModuleForZip.pathInZip);
// NOTE: The nature (and so the grammar to be used) must be defined by this
// modules
// manager (and not by the initial caller)!!
n =
AbstractModule.createModuleFromDoc(
name, emptyModuleForZip.f, doc, this.getNature(), false);
SourceModule zipModule = (SourceModule) n;
zipModule.zipFilePath = emptyModuleForZip.pathInZip;
n = decorateModule(n, nature);
} catch (Exception exc1) {
Log.log(exc1);
n = null;
}
}
} else {
// regular case... just go on and create it.
try {
// NOTE: The nature (and so the grammar to be used) must be defined by this modules
// manager (and not by the initial caller)!!
n = AbstractModule.createModule(name, e.f, this.getNature(), true);
n = decorateModule(n, nature);
} catch (IOException exc) {
keyForCacheAccess.name = name;
keyForCacheAccess.file = e.f;
doRemoveSingleModule(keyForCacheAccess);
n = null;
} catch (MisconfigurationException exc) {
Log.log(exc);
n = null;
}
}
}
}
} else { // ok, it does not have a file associated, so, we treat it as a builtin (this can
// happen in java jars)
n = checkOverride(name, nature, n);
if (n instanceof EmptyModule) {
if (acceptCompiledModule) {
n = new CompiledModule(name, this);
n = decorateModule(n, nature);
} else {
return null;
}
}
}
if (n != null) {
doAddSingleModule(createModulesKey(name, e.f), n);
} else {
Log.log(("The module " + name + " could not be found nor created!"));
}
}
if (n instanceof EmptyModule) {
throw new RuntimeException("Should not be an empty module anymore: " + n);
}
if (n instanceof SourceModule) {
SourceModule sourceModule = (SourceModule) n;
// now, here's a catch... it may be a bootstrap module...
if (sourceModule.isBootstrapModule()) {
// if it's a bootstrap module, we must replace it for the related compiled module.
n = new CompiledModule(name, this);
n = decorateModule(n, nature);
}
}
return n;
}
@Override
public void addProps(
MarkerAnnotationAndPosition markerAnnotation,
IAnalysisPreferences analysisPreferences,
String line,
PySelection ps,
int offset,
IPythonNature nature,
PyEdit edit,
List<ICompletionProposal> props)
throws BadLocationException, CoreException {
if (nature == null) {
return;
}
ICodeCompletionASTManager astManager = nature.getAstManager();
if (astManager == null) {
return;
}
if (markerAnnotation.position == null) {
return;
}
IMarker marker = markerAnnotation.markerAnnotation.getMarker();
Integer id = (Integer) marker.getAttribute(AnalysisRunner.PYDEV_ANALYSIS_TYPE);
int start = markerAnnotation.position.offset;
int end = start + markerAnnotation.position.length;
ps.setSelection(start, end);
String markerContents;
try {
markerContents = ps.getSelectedText();
} catch (Exception e1) {
return; // Selection may be wrong.
}
IDocument doc = ps.getDoc();
List<String> parametersAfterCall = ps.getParametersAfterCall(end);
switch (id) {
case IAnalysisPreferences.TYPE_UNDEFINED_VARIABLE:
addCreateClassOption(ps, edit, props, markerContents, parametersAfterCall);
addCreateMethodOption(ps, edit, props, markerContents, parametersAfterCall);
break;
case IAnalysisPreferences.TYPE_UNDEFINED_IMPORT_VARIABLE:
// Say we had something as:
// import sys
// sys.Bar
// in which case 'Bar' is undefined
// in this situation, the activationTokenAndQual would be "sys." and "Bar"
// and we want to get the definition for "sys"
String[] activationTokenAndQual = ps.getActivationTokenAndQual(true);
if (activationTokenAndQual[0].endsWith(".")) {
ArrayList<IDefinition> selected = findDefinitions(nature, edit, start - 2, doc);
for (IDefinition iDefinition : selected) {
IModule module = iDefinition.getModule();
if (module.getFile() != null) {
Definition definition = (Definition) iDefinition;
File file = module.getFile();
if (definition.ast == null) {
// if we have no ast in the definition, it means the module itself was found (global
// scope)
// Add option to create class at the given module!
addCreateClassOption(ps, edit, props, markerContents, parametersAfterCall, file);
addCreateMethodOption(ps, edit, props, markerContents, parametersAfterCall, file);
} else if (definition.ast instanceof ClassDef) {
ClassDef classDef = (ClassDef) definition.ast;
// Ok, we should create a field or method in this case (accessing a classmethod or
// staticmethod)
PyCreateMethodOrField pyCreateMethod = new PyCreateMethodOrField();
String className = NodeUtils.getNameFromNameTok(classDef.name);
pyCreateMethod.setCreateInClass(className);
pyCreateMethod.setCreateAs(PyCreateMethodOrField.CLASSMETHOD);
addCreateClassmethodOption(
ps,
edit,
props,
markerContents,
parametersAfterCall,
pyCreateMethod,
file,
className);
}
}
}
}
break;
case IAnalysisPreferences.TYPE_UNRESOLVED_IMPORT:
// This case is the following: from other_module4 import Foo
// with 'Foo' being undefined.
// So, we have to suggest creating a Foo class/method in other_module4
PyImportsHandling importsHandling = new PyImportsHandling(ps.getDoc(), false);
int offsetLine = ps.getLineOfOffset(start);
String selectedText = ps.getSelectedText();
Tuple<IModule, String> found = null;
String foundFromImportStr = null;
boolean isImportFrom = false;
OUT:
for (ImportHandle handle : importsHandling) {
if (handle.startFoundLine == offsetLine
|| handle.endFoundLine == offsetLine
|| (handle.startFoundLine < offsetLine && handle.endFoundLine > offsetLine)) {
List<ImportHandleInfo> importInfo = handle.getImportInfo();
for (ImportHandleInfo importHandleInfo : importInfo) {
String fromImportStr = importHandleInfo.getFromImportStr();
List<String> importedStr = importHandleInfo.getImportedStr();
for (String imported : importedStr) {
if (selectedText.equals(imported)) {
if (fromImportStr != null) {
foundFromImportStr = fromImportStr + "." + imported;
isImportFrom = true;
} else {
// if fromImportStr == null, it's not a from xxx import yyy (i.e.: simple
// import)
foundFromImportStr = imported;
}
try {
String currentModule = nature.resolveModule(edit.getEditorFile());
ICompletionState state =
CompletionStateFactory.getEmptyCompletionState(
nature, new CompletionCache());
found =
nature
.getAstManager()
.findModule(
foundFromImportStr,
currentModule,
state,
new SourceModule(
currentModule, edit.getEditorFile(), edit.getAST(), null));
} catch (Exception e) {
Log.log(e);
}
break OUT;
}
}
}
break OUT;
}
}
boolean addOptionToCreateClassOrMethod = isImportFrom;
if (found != null && found.o1 != null) {
// Ok, we found a module, now, it may be that we still have to create some intermediary
// modules
// or just create a class or method at the end.
if (found.o1 instanceof SourceModule) {
// if all was found, there's nothing left to create.
if (found.o2 != null && found.o2.length() > 0) {
SourceModule sourceModule = (SourceModule) found.o1;
File file = sourceModule.getFile();
if (found.o2.indexOf('.') != -1) {
// We have to create some intermediary structure.
if (!addOptionToCreateClassOrMethod) {
// Cannot create class or method from the info (only the module structure).
if (sourceModule.getName().endsWith(".__init__")) {
File f = getFileStructure(file.getParentFile(), found.o2);
addCreateModuleOption(ps, edit, props, markerContents, f);
}
} else {
// Ok, the leaf may be a class or method.
if (sourceModule.getName().endsWith(".__init__")) {
String moduleName = FullRepIterable.getWithoutLastPart(sourceModule.getName());
String withoutLastPart = FullRepIterable.getWithoutLastPart(found.o2);
moduleName += "." + withoutLastPart;
String classOrMethodName = FullRepIterable.getLastPart(found.o2);
File f = getFileStructure(file.getParentFile(), withoutLastPart);
addCreateClassInNewModuleOption(
ps, edit, props, classOrMethodName, moduleName, parametersAfterCall, f);
addCreateMethodInNewModuleOption(
ps, edit, props, classOrMethodName, moduleName, parametersAfterCall, f);
}
}
} else {
// Ok, it's all there, we just have to create the leaf.
if (!addOptionToCreateClassOrMethod
|| sourceModule.getName().endsWith(".__init__")) {
// Cannot create class or method from the info (only the module structure).
if (sourceModule.getName().endsWith(".__init__")) {
File f =
new File(
file.getParent(),
found.o2 + FileTypesPreferencesPage.getDefaultDottedPythonExtension());
addCreateModuleOption(ps, edit, props, markerContents, f);
}
} else {
// Ok, the leaf may be a class or method.
addCreateClassOption(ps, edit, props, markerContents, parametersAfterCall, file);
addCreateMethodOption(ps, edit, props, markerContents, parametersAfterCall, file);
}
}
}
}
} else if (foundFromImportStr != null) {
// We couldn't find anything there, so, we have to create the modules structure as needed
// and
// maybe create a class or module at the end (but only if it's an import from).
// Ok, it's all there, we just have to create the leaf.
// Discover the source folder where we should create the structure.
File editorFile = edit.getEditorFile();
String onlyProjectPythonPathStr =
nature.getPythonPathNature().getOnlyProjectPythonPathStr(false);
List<String> split =
StringUtils.splitAndRemoveEmptyTrimmed(onlyProjectPythonPathStr, '|');
for (int i = 0; i < split.size(); i++) {
String fullPath = FileUtils.getFileAbsolutePath(split.get(i));
fullPath = PythonPathHelper.getDefaultPathStr(fullPath);
split.set(i, fullPath);
}
HashSet<String> projectSourcePath = new HashSet<String>(split);
if (projectSourcePath.size() == 0) {
return; // No source folder for editor... this shouldn't happen (code analysis wouldn't
// even run on it).
}
String fullPath = FileUtils.getFileAbsolutePath(editorFile);
fullPath = PythonPathHelper.getDefaultPathStr(fullPath);
String foundSourceFolderFullPath = null;
if (projectSourcePath.size() == 1) {
foundSourceFolderFullPath = projectSourcePath.iterator().next();
} else {
for (String string : projectSourcePath) {
if (fullPath.startsWith(string)) {
// Use this as the source folder
foundSourceFolderFullPath = string;
break;
}
}
}
if (foundSourceFolderFullPath != null) {
if (!addOptionToCreateClassOrMethod) {
// Cannot create class or method from the info (only the module structure).
File f = getFileStructure(new File(foundSourceFolderFullPath), foundFromImportStr);
addCreateModuleOption(ps, edit, props, foundFromImportStr, f);
} else {
// Ok, the leaf may be a class or method.
String moduleName = FullRepIterable.getWithoutLastPart(foundFromImportStr);
File file = getFileStructure(new File(foundSourceFolderFullPath), moduleName);
String lastPart = FullRepIterable.getLastPart(foundFromImportStr);
addCreateClassInNewModuleOption(
ps, edit, props, lastPart, moduleName, parametersAfterCall, file);
addCreateMethodInNewModuleOption(
ps, edit, props, lastPart, moduleName, parametersAfterCall, file);
}
}
}
break;
}
}
/**
* DAMN... when I started thinking this up, it seemed much better... (and easier)
*
* @param module - this is the full path of the module. Only for directories or py,pyd,dll,pyo
* files.
* @return a String with the module that the file or folder should represent. E.g.: compiler.ast
*/
public String resolveModule(
String fullPath, final boolean requireFileToExist, List<String> pythonPathCopy) {
fullPath = REF.getFileAbsolutePath(fullPath);
fullPath = getDefaultPathStr(fullPath);
String fullPathWithoutExtension;
if (isValidSourceFile(fullPath) || FileTypesPreferencesPage.isValidDll(fullPath)) {
fullPathWithoutExtension = FullRepIterable.headAndTail(fullPath)[0];
} else {
fullPathWithoutExtension = fullPath;
}
final File moduleFile = new File(fullPath);
if (requireFileToExist && !moduleFile.exists()) {
return null;
}
boolean isFile = moduleFile.isFile();
// go through our pythonpath and check the beginning
for (String pathEntry : pythonPathCopy) {
String element = getDefaultPathStr(pathEntry);
if (fullPath.startsWith(element)) {
int len = element.length();
String s = fullPath.substring(len);
String sWithoutExtension = fullPathWithoutExtension.substring(len);
if (s.startsWith("/")) {
s = s.substring(1);
}
if (sWithoutExtension.startsWith("/")) {
sWithoutExtension = sWithoutExtension.substring(1);
}
if (!isValidModuleLastPart(sWithoutExtension)) {
continue;
}
s = s.replaceAll("/", ".");
if (s.indexOf(".") != -1) {
File root = new File(element);
if (root.exists() == false) {
continue;
}
final List<String> temp = StringUtils.dotSplit(s);
String[] modulesParts = temp.toArray(new String[temp.size()]);
// this means that more than 1 module is specified, so, in order to get it,
// we have to go and see if all the folders to that module have __init__.py in it...
if (modulesParts.length > 1 && isFile) {
String[] t = new String[modulesParts.length - 1];
for (int i = 0; i < modulesParts.length - 1; i++) {
t[i] = modulesParts[i];
}
t[t.length - 1] = t[t.length - 1] + "." + modulesParts[modulesParts.length - 1];
modulesParts = t;
}
// here, in modulesParts, we have something like
// ["compiler", "ast.py"] - if file
// ["pywin","debugger"] - if folder
//
// root starts with the pythonpath folder that starts with the same
// chars as the full path passed in.
boolean isValid = true;
for (int i = 0; i < modulesParts.length && root != null; i++) {
root = new File(REF.getFileAbsolutePath(root) + "/" + modulesParts[i]);
// check if file is in root...
if (isValidFileMod(modulesParts[i])) {
if (root.exists() && root.isFile()) {
break;
}
} else {
// this part is a folder part... check if it is a valid module (has init).
if (isFileOrFolderWithInit(root) == false) {
isValid = false;
break;
}
// go on and check the next part.
}
}
if (isValid) {
if (isFile) {
s = stripExtension(s);
} else if (moduleFile.exists() == false) {
// ok, it does not exist, so isFile will not work, let's just check if it is
// a valid module (ends with .py or .pyw) and if it is, strip the extension
if (isValidFileMod(s)) {
s = stripExtension(s);
}
}
return s;
}
} else {
// simple part, we don't have to go into subfolders to check validity...
if (!isFile && moduleFile.isDirectory() && isFileOrFolderWithInit(moduleFile) == false) {
return null;
}
return s;
}
}
}
// ok, it was not found in any existing way, so, if we don't require the file to exist, let's
// just do some simpler search and get the
// first match (if any)... this is useful if the file we are looking for has just been deleted
if (!requireFileToExist) {
// we have to remove the last part (.py, .pyc, .pyw)
for (String element : pythonPathCopy) {
element = getDefaultPathStr(element);
if (fullPathWithoutExtension.startsWith(element)) {
String s = fullPathWithoutExtension.substring(element.length());
if (s.startsWith("/")) {
s = s.substring(1);
}
if (!isValidModuleLastPart(s)) {
continue;
}
s = s.replaceAll("/", ".");
return s;
}
}
}
return null;
}
/**
* @param root the zip file to analyze
* @param monitor the monitor, to keep track of what is happening
* @return a list with the name of the found modules in the jar
*/
protected static ModulesFoundStructure.ZipContents getFromZip(
File root, IProgressMonitor monitor) {
String fileName = root.getName();
if (root.isFile()
&& FileTypesPreferencesPage.isValidZipFile(
fileName)) { // ok, it may be a jar file, so let's get its contents and get the
// available modules
// the major difference from handling jars from regular python files is that we don't have to
// check for __init__.py files
ModulesFoundStructure.ZipContents zipContents = new ModulesFoundStructure.ZipContents(root);
// by default it's a zip (for python) -- may change if a .class is found.
zipContents.zipContentsType = ZipContents.ZIP_CONTENTS_TYPE_PY_ZIP;
try {
String zipFileName = root.getName();
ZipFile zipFile = new ZipFile(root);
try {
Enumeration<? extends ZipEntry> entries = zipFile.entries();
int i = 0;
FastStringBuffer buffer = new FastStringBuffer();
// ok, now that we have the zip entries, let's map them to modules
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
String name = entry.getName();
if (!entry.isDirectory()) {
if (isValidFileMod(name) || name.endsWith(".class")) {
if (name.endsWith(".class")) {
zipContents.zipContentsType = ZipContents.ZIP_CONTENTS_TYPE_JAR;
}
// it is a valid python file
if (i % 15 == 0) {
if (monitor.isCanceled()) {
return null;
}
buffer.clear();
monitor.setTaskName(
buffer
.append("Found in ")
.append(zipFileName)
.append(" module ")
.append(name)
.toString());
monitor.worked(1);
}
if (isValidInitFile(name)) {
zipContents.pyInitFilesLowerWithoutExtension.add(
StringUtils.stripExtension(name).toLowerCase());
}
zipContents.pyFilesLowerToRegular.put(name.toLowerCase(), name);
}
} else { // !isDirectory
zipContents.pyfoldersLower.add(name.toLowerCase());
}
i++;
}
} finally {
zipFile.close();
}
// now, on to actually filling the structure if we have a zip file (just add the ones that
// are actually under
// the pythonpath)
zipContents.consolidatePythonpathInfo(monitor);
return zipContents;
} catch (Exception e) {
// that's ok, it is probably not a zip file after all...
PydevPlugin.log(e);
}
}
return null;
}