root/trunk/mf.py
| Revision 526 (by htgoebel, 08/05/08 12:31:49) |
|---|
# # Copyright (C) 2005, Giovanni Bajo # # Based on previous work under copyright (c) 2002 McMillan Enterprises, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA # import sys, string, os, imp, marshal, dircache try: # zipimport is supported starting with Python 2.3 import zipimport except ImportError: zipimport = None import suffixes try: STRINGTYPE = basestring except NameError: STRINGTYPE = type("") if not os.environ.has_key('PYTHONCASEOK') and sys.version_info >= (2, 1): def caseOk(filename): files = dircache.listdir(os.path.dirname(filename)) return os.path.basename(filename) in files else: def caseOk(filename): return True def pyco(): """ Returns correct extension ending: 'c' or 'o' """ if __debug__: return 'c' else: return 'o' #=======================Owners==========================# # An Owner does imports from a particular piece of turf # That is, there's an Owner for each thing on sys.path # There are owners for directories and .pyz files. # There could be owners for zip files, or even URLs. # Note that they replace the string in sys.path, # but str(sys.path[n]) should yield the original string. class Owner: def __init__(self, path, target_platform=None): self.path = path self.target_platform = target_platform def __str__(self): return self.path def getmod(self, nm): return None class DirOwner(Owner): def __init__(self, path, target_platform=None): if path == '': path = os.getcwd() if not os.path.isdir(path): raise ValueError, "%s is not a directory" % path Owner.__init__(self, path, target_platform) def _getsuffixes(self): return suffixes.get_suffixes(self.target_platform) def getmod(self, nm, getsuffixes=None, loadco=marshal.loads): if getsuffixes is None: getsuffixes = self._getsuffixes pth = os.path.join(self.path, nm) possibles = [(pth, 0, None)] if os.path.isdir(pth): possibles.insert(0, (os.path.join(pth, '__init__'), 1, pth)) py = pyc = None for pth, ispkg, pkgpth in possibles: for ext, mode, typ in getsuffixes(): attempt = pth+ext try: st = os.stat(attempt) except Exception, e: #print "DirOwner", e pass else: # Check case if not caseOk(attempt): continue if typ == imp.C_EXTENSION: #print "DirOwner.getmod -> ExtensionModule(%s, %s)" % (nm, attempt) return ExtensionModule(nm, attempt) elif typ == imp.PY_SOURCE: py = (attempt, st) else: pyc = (attempt, st) if py or pyc: break if py is None and pyc is None: #print "DirOwner.getmod -> (py == pyc == None)" return None while 1: # If we have no pyc or py is newer if pyc is None or py and pyc[1][8] < py[1][8]: try: stuff = open(py[0], 'r').read()+'\n' co = compile(string.replace(stuff, "\r\n", "\n"), py[0], 'exec') pth = py[0] + pyco() break except SyntaxError, e: print "Syntax error in", py[0] print e.args raise elif pyc: stuff = open(pyc[0], 'rb').read() try: co = loadco(stuff[8:]) pth = pyc[0] break except (ValueError, EOFError): print "W: bad .pyc found (%s), will use .py" % pyc[0] pyc = None else: #print "DirOwner.getmod while 1 -> None" return None if not os.path.isabs(pth): pth = os.path.abspath(pth) if ispkg: mod = PkgModule(nm, pth, co) else: mod = PyModule(nm, pth, co) #print "DirOwner.getmod -> %s" % mod return mod class PYZOwner(Owner): def __init__(self, path, target_platform=None): import archive self.pyz = archive.ZlibArchive(path) Owner.__init__(self, path, target_platform) def getmod(self, nm): rslt = self.pyz.extract(nm) if rslt: ispkg, co = rslt if ispkg: return PkgInPYZModule(nm, co, self) return PyModule(nm, self.path, co) ZipOwner = None if zipimport: class ZipOwner(Owner): def __init__(self, path, target_platform=None): self.__zip = zipimport.zipimporter(path) Owner.__init__(self, path, target_platform) def getmod(self, nm): try: co = self.__zip.get_code(nm) return PkgInZipModule(nm, co, self) except zipimport.ZipImportError: return None _globalownertypes = filter(None, [ DirOwner, ZipOwner, PYZOwner, Owner, ]) #===================Import Directors====================================# # ImportDirectors live on the metapath # There's one for builtins, one for frozen modules, and one for sys.path # Windows gets one for modules gotten from the Registry # There should be one for Frozen modules # Mac would have them for PY_RESOURCE modules etc. # A generalization of Owner - their concept of "turf" is broader class ImportDirector(Owner): pass class BuiltinImportDirector(ImportDirector): def __init__(self): self.path = 'Builtins' def getmod(self, nm, isbuiltin=imp.is_builtin): if isbuiltin(nm): return BuiltinModule(nm) return None class FrozenImportDirector(ImportDirector): def __init__(self): self.path = 'FrozenModules' def getmod(self, nm, isfrozen=imp.is_frozen): if isfrozen(nm): return FrozenModule(nm) return None class RegistryImportDirector(ImportDirector): # for Windows only def __init__(self): self.path = "WindowsRegistry" self.map = {} try: import win32api import win32con except ImportError: pass else: subkey = r"Software\Python\PythonCore\%s\Modules" % sys.winver for root in (win32con.HKEY_CURRENT_USER, win32con.HKEY_LOCAL_MACHINE): try: #hkey = win32api.RegOpenKeyEx(root, subkey, 0, win32con.KEY_ALL_ACCESS) hkey = win32api.RegOpenKeyEx(root, subkey, 0, win32con.KEY_READ) except Exception, e: #print "RegistryImportDirector", e pass else: numsubkeys, numvalues, lastmodified = win32api.RegQueryInfoKey(hkey) for i in range(numsubkeys): subkeyname = win32api.RegEnumKey(hkey, i) #hskey = win32api.RegOpenKeyEx(hkey, subkeyname, 0, win32con.KEY_ALL_ACCESS) hskey = win32api.RegOpenKeyEx(hkey, subkeyname, 0, win32con.KEY_READ) val = win32api.RegQueryValueEx(hskey, '') desc = getDescr(val[0]) #print " RegistryImportDirector got %s %s" % (val[0], desc) #XXX self.map[subkeyname] = (val[0], desc) hskey.Close() hkey.Close() break def getmod(self, nm): stuff = self.map.get(nm) if stuff: fnm, (suffix, mode, typ) = stuff if typ == imp.C_EXTENSION: return ExtensionModule(nm, fnm) elif typ == imp.PY_SOURCE: try: stuff = open(fnm, 'r').read()+'\n' co = compile(string.replace(stuff, "\r\n", "\n"), fnm, 'exec') except SyntaxError, e: print "Invalid syntax in %s" % py[0] print e.args raise else: stuff = open(fnm, 'rb').read() co = loadco(stuff[8:]) return PyModule(nm, fnm, co) return None class PathImportDirector(ImportDirector): def __init__(self, pathlist=None, importers=None, ownertypes=None, target_platform=None): if pathlist is None: self.path = sys.path else: self.path = pathlist if ownertypes == None: self.ownertypes = _globalownertypes else: self.ownertypes = ownertypes if importers: self.shadowpath = importers else: self.shadowpath = {} self.inMakeOwner = 0 self.building = {} self.target_platform = target_platform def __str__(self): return str(self.path) def getmod(self, nm): mod = None for thing in self.path: if isinstance(thing, STRINGTYPE): owner = self.shadowpath.get(thing, -1) if owner == -1: owner = self.shadowpath[thing] = self.makeOwner(thing) if owner: mod = owner.getmod(nm) else: mod = thing.getmod(nm) if mod: break return mod def makeOwner(self, path): if self.building.get(path): return None self.building[path] = 1 owner = None for klass in self.ownertypes: try: # this may cause an import, which may cause recursion # hence the protection owner = klass(path, self.target_platform) except Exception, e: #print "PathImportDirector", e pass else: break del self.building[path] return owner def getDescr(fnm): ext = os.path.splitext(fnm)[1] for (suffix, mode, typ) in imp.get_suffixes(): if suffix == ext: return (suffix, mode, typ) #=================Import Tracker============================# # This one doesn't really import, just analyzes # If it *were* importing, it would be the one-and-only ImportManager # ie, the builtin import UNTRIED = -1 imptyps = ['top-level', 'conditional', 'delayed', 'delayed, conditional'] import hooks if __debug__: import sys import UserDict class LogDict(UserDict.UserDict): count = 0 def __init__(self, *args): UserDict.UserDict.__init__(self, *args) LogDict.count += 1 self.logfile = open("logdict%s-%d.log" % (".".join(map(str, sys.version_info)), LogDict.count), "w") def __setitem__(self, key, value): self.logfile.write("%s: %s -> %s\n" % (key, self.data.get(key), value)) UserDict.UserDict.__setitem__(self, key, value) def __delitem__(self, key): self.logfile.write(" DEL %s\n" % key) UserDict.UserDict.__delitem__(self, key) else: LogDict = dict class ImportTracker: # really the equivalent of builtin import def __init__(self, xpath=None, hookspath=None, excludes=None, target_platform=None): self.path = [] self.warnings = {} if xpath: self.path = xpath self.path.extend(sys.path) self.modules = LogDict() self.metapath = [ BuiltinImportDirector(), FrozenImportDirector(), RegistryImportDirector(), PathImportDirector(self.path, target_platform=target_platform) ] if hookspath: hooks.__path__.extend(hookspath) self.excludes = excludes if excludes is None: self.excludes = [] self.target_platform = target_platform def analyze_r(self, nm, importernm=None): importer = importernm if importer is None: importer = '__main__' seen = {} nms = self.analyze_one(nm, importernm) nms = map(None, nms, [importer]*len(nms)) i = 0 while i < len(nms): nm, importer = nms[i] if seen.get(nm,0): del nms[i] mod = self.modules[nm] if mod: mod.xref(importer) else: i = i + 1 seen[nm] = 1 j = i mod = self.modules[nm] if mod: mod.xref(importer) for name, isdelayed, isconditional, level in mod.imports: imptyp = isdelayed * 2 + isconditional newnms = self.analyze_one(name, nm, imptyp, level) newnms = map(None, newnms, [nm]*len(newnms)) nms[j:j] = newnms j = j + len(newnms) return map(lambda a: a[0], nms) def analyze_one(self, nm, importernm=None, imptyp=0, level=-1): #print '## analyze_one', nm, importernm, imptyp, level # break the name being imported up so we get: # a.b.c -> [a, b, c] ; ..z -> ['', '', z] if not nm: nm = importernm importernm = None level = 0 nmparts = string.split(nm, '.') if level < 0: # behaviour up to Python 2.4 (and default in Python 2.5) # first see if we could be importing a relative name contexts = [None] if importernm: if self.ispackage(importernm): contexts.insert(0, importernm) else: pkgnm = string.join(string.split(importernm, '.')[:-1], '.') if pkgnm: contexts.insert(0, pkgnm) elif level == 0: # absolute import, do not try relative importernm = None contexts = [None] elif level > 0: # relative import, do not try absolute contexts = [string.join(string.split(importernm, '.')[:-level], '.')] importernm = None _all = None assert contexts # so contexts is [pkgnm, None] or just [None] if nmparts[-1] == '*': del nmparts[-1] _all = [] nms = [] for context in contexts: ctx = context for i in range(len(nmparts)): nm = nmparts[i] if ctx: fqname = ctx + '.' + nm else: fqname = nm mod = self.modules.get(fqname, UNTRIED) if mod is UNTRIED: mod = self.doimport(nm, ctx, fqname) if mod: nms.append(mod.__name__) ctx = fqname else: break else: # no break, point i beyond end i = i + 1 if i: break # now nms is the list of modules that went into sys.modules # just as result of the structure of the name being imported # however, each mod has been scanned and that list is in mod.imports if i<len(nmparts): if ctx: if hasattr(self.modules[ctx], nmparts[i]): return nms if not self.ispackage(ctx): return nms self.warnings["W: no module named %s (%s import by %s)" % (fqname, imptyps[imptyp], importernm or "__main__")] = 1 if self.modules.has_key(fqname): del self.modules[fqname] return nms if _all is None: return nms bottommod = self.modules[ctx] if bottommod.ispackage(): for nm in bottommod._all: if not hasattr(bottommod, nm): mod = self.doimport(nm, ctx, ctx+'.'+nm) if mod: nms.append(mod.__name__) else: bottommod.warnings.append("W: name %s not found" % nm) return nms def analyze_script(self, fnm): try: stuff = open(fnm, 'r').read()+'\n' co = compile(string.replace(stuff, "\r\n", "\n"), fnm, 'exec') except SyntaxError, e: print "Invalid syntax in %s" % fnm print e.args raise mod = PyScript(fnm, co) self.modules['__main__'] = mod return self.analyze_r('__main__') def ispackage(self, nm): return self.modules[nm].ispackage() def doimport(self, nm, ctx, fqname): # Not that nm is NEVER a dotted name at this point assert ("." not in nm), nm if fqname in self.excludes: return None if ctx: parent = self.modules[ctx] if parent.ispackage(): mod = parent.doimport(nm) if mod: # insert the new module in the parent package # FIXME why? setattr(parent, nm, mod) else: # if parent is not a package, there is nothing more to do return None else: # now we're dealing with an absolute import # try to import nm using available directors for director in self.metapath: mod = director.getmod(nm) if mod: break # here we have `mod` from: # mod = parent.doimport(nm) # or # mod = director.getmod(nm) if mod: mod.__name__ = fqname self.modules[fqname] = mod # now look for hooks # this (and scan_code) are instead of doing "exec co in mod.__dict__" try: hookmodnm = 'hook-'+fqname hooks = __import__('hooks', globals(), locals(), [hookmodnm]) hook = getattr(hooks, hookmodnm) #print `hook` except (ImportError, AttributeError): pass else: # rearranged so that hook() has a chance to mess with hiddenimports & attrs if hasattr(hook, 'hook'): mod = hook.hook(mod) if hasattr(hook, 'hiddenimports'): for impnm in hook.hiddenimports: mod.imports.append((impnm, 0, 0, -1)) if hasattr(hook, 'attrs'): for attr, val in hook.attrs: setattr(mod, attr, val) if fqname != mod.__name__: print "W: %s is changing it's name to %s" % (fqname, mod.__name__) self.modules[mod.__name__] = mod else: assert (mod == None), mod self.modules[fqname] = None # should be equivalent using only one # self.modules[fqname] = mod # here return mod def getwarnings(self): warnings = self.warnings.keys() for nm,mod in self.modules.items(): if mod: for w in mod.warnings: warnings.append(w+' - %s (%s)' % (mod.__name__, mod.__file__)) return warnings def getxref(self): mods = self.modules.items() # (nm, mod) mods.sort() rslt = [] for nm, mod in mods: if mod: importers = mod._xref.keys() importers.sort() rslt.append((nm, importers)) return rslt #====================Modules============================# # All we're doing here is tracking, not importing # If we were importing, these would be hooked to the real module objects class Module: _ispkg = 0 typ = 'UNKNOWN' def __init__(self, nm): self.__name__ = nm self.__file__ = None self._all = [] self.imports = [] self.warnings = [] self._xref = {} def ispackage(self): return self._ispkg def doimport(self, nm): pass def xref(self, nm): self._xref[nm] = 1 def __str__(self): return "<Module %s %s %s>" % (self.__name__, self.__file__, self.imports) class BuiltinModule(Module): typ = 'BUILTIN' def __init__(self, nm): Module.__init__(self, nm) class ExtensionModule(Module): typ = 'EXTENSION' def __init__(self, nm, pth): Module.__init__(self, nm) self.__file__ = pth class PyModule(Module): typ = 'PYMODULE' def __init__(self, nm, pth, co): Module.__init__(self, nm) self.co = co self.__file__ = pth if os.path.splitext(self.__file__)[1] == '.py': self.__file__ = self.__file__ + pyco() self.scancode() def scancode(self): self.imports, self.warnings, allnms = scan_code(self.co) if allnms: self._all = allnms class PyScript(PyModule): typ = 'PYSOURCE' def __init__(self, pth, co): Module.__init__(self, '__main__') self.co = co self.__file__ = pth self.scancode() class PkgModule(PyModule): typ = 'PYMODULE' def __init__(self, nm, pth, co): PyModule.__init__(self, nm, pth, co) self._ispkg = 1 pth = os.path.dirname(pth) self.__path__ = [ pth ] self._update_director(force=True) def _update_director(self, force=False): if force or self.subimporter.path != self.__path__: self.subimporter = PathImportDirector(self.__path__) def doimport(self, nm): self._update_director() mod = self.subimporter.getmod(nm) if mod: mod.__name__ = self.__name__ + '.' + mod.__name__ return mod class PkgInPYZModule(PyModule): def __init__(self, nm, co, pyzowner): PyModule.__init__(self, nm, co.co_filename, co) self._ispkg = 1 self.__path__ = [ str(pyzowner) ] self.owner = pyzowner def doimport(self, nm): mod = self.owner.getmod(self.__name__ + '.' + nm) return mod class PkgInZipModule(PyModule): typ = 'ZIPFILE' def __init__(self, nm, co, pyzowner): PyModule.__init__(self, nm, co.co_filename, co) self._ispkg = 0 self.__path__ = [ str(pyzowner) ] self.owner = pyzowner def doimport(self, nm): mod = self.owner.getmod(self.__name__ + '.' + nm) return mod #======================== Utility ================================# # Scan the code object for imports, __all__ and wierd stuff import dis IMPORT_NAME = dis.opname.index('IMPORT_NAME') IMPORT_FROM = dis.opname.index('IMPORT_FROM') try: IMPORT_STAR = dis.opname.index('IMPORT_STAR') except: IMPORT_STAR = 999 STORE_NAME = dis.opname.index('STORE_NAME') STORE_FAST = dis.opname.index('STORE_FAST') STORE_GLOBAL = dis.opname.index('STORE_GLOBAL') LOAD_GLOBAL = dis.opname.index('LOAD_GLOBAL') EXEC_STMT = dis.opname.index('EXEC_STMT') try: SET_LINENO = dis.opname.index('SET_LINENO') except ValueError: SET_LINENO = 999 BUILD_LIST = dis.opname.index('BUILD_LIST') LOAD_CONST = dis.opname.index('LOAD_CONST') if getattr(sys, 'version_info', (0,0,0)) > (2,5,0): LOAD_CONST_level = LOAD_CONST else: LOAD_CONST_level = 999 JUMP_IF_FALSE = dis.opname.index('JUMP_IF_FALSE') JUMP_IF_TRUE = dis.opname.index('JUMP_IF_TRUE') JUMP_FORWARD = dis.opname.index('JUMP_FORWARD') try: STORE_DEREF = dis.opname.index('STORE_DEREF') except ValueError: STORE_DEREF = 999 COND_OPS = [JUMP_IF_TRUE, JUMP_IF_FALSE] STORE_OPS = [STORE_NAME, STORE_FAST, STORE_GLOBAL, STORE_DEREF] #IMPORT_STAR -> IMPORT_NAME mod ; IMPORT_STAR #JUMP_IF_FALSE / JUMP_IF_TRUE / JUMP_FORWARD def pass1(code): instrs = [] i = 0 n = len(code) curline = 0 incondition = 0 out = 0 while i < n: if i >= out: incondition = 0 c = code[i] i = i+1 op = ord(c) if op >= dis.HAVE_ARGUMENT: oparg = ord(code[i]) + ord(code[i+1])*256 i = i+2 else: oparg = None if not incondition and op in COND_OPS: incondition = 1 out = i + oparg elif incondition and op == JUMP_FORWARD: out = max(out, i + oparg) if op == SET_LINENO: curline = oparg else: instrs.append((op, oparg, incondition, curline)) return instrs def scan_code(co, m=None, w=None, nested=0): instrs = pass1(co.co_code) if m is None: m = [] if w is None: w = [] all = None lastname = None level = -1 # import-level, same behaviour as up to Python 2.4 for i in range(len(instrs)): op, oparg, conditional, curline = instrs[i] if op == IMPORT_NAME: if level <= 0: name = lastname = co.co_names[oparg] else: name = lastname = co.co_names[oparg] #print 'import_name', name, `lastname`, level m.append((name, nested, conditional, level)) elif op == IMPORT_FROM: name = co.co_names[oparg] #print 'import_from', name, `lastname`, level, if level > 0 and (not lastname or lastname[-1:] == '.'): name = lastname + name else: name = lastname + '.' + name #print name m.append((name, nested, conditional, level)) assert lastname is not None elif op == IMPORT_STAR: m.append((lastname+'.*', nested, conditional, level)) elif op == STORE_NAME: if co.co_names[oparg] == "__all__": j = i - 1 pop, poparg, pcondtl, pline = instrs[j] if pop != BUILD_LIST: w.append("W: __all__ is built strangely at line %s" % pline) else: all = [] while j > 0: j = j - 1 pop, poparg, pcondtl, pline = instrs[j] if pop == LOAD_CONST: all.append(co.co_consts[poparg]) else: break elif op in STORE_OPS: pass elif op == LOAD_CONST_level: # starting with Python 2.5, _each_ import is preceeded with a # LOAD_CONST to indicate the relative level. if isinstance(co.co_consts[oparg], (int, long)): level = co.co_consts[oparg] elif op == LOAD_GLOBAL: name = co.co_names[oparg] cndtl = ['', 'conditional'][conditional] lvl = ['top-level', 'delayed'][nested] if name == "__import__": w.append("W: %s %s __import__ hack detected at line %s" % (lvl, cndtl, curline)) elif name == "eval": w.append("W: %s %s eval hack detected at line %s" % (lvl, cndtl, curline)) elif op == EXEC_STMT: cndtl = ['', 'conditional'][conditional] lvl = ['top-level', 'delayed'][nested] w.append("W: %s %s exec statement detected at line %s" % (lvl, cndtl, curline)) else: lastname = None for c in co.co_consts: if isinstance(c, type(co)): scan_code(c, m, w, 1) return m, w, all
Note: See TracBrowser for help on using the browser.
