1
# -*- test-case-name: twisted.test.test_modules -*-
4
This module aims to provide a unified, object-oriented view of Python's
7
Python is a very dynamic language with wide variety of introspection utilities.
8
However, these utilities can be hard to use, because there is no consistent
9
API. The introspection API in python is made up of attributes (__name__,
10
__module__, func_name, etc) on instances, modules, classes and functions which
11
vary between those four types, utility modules such as 'inspect' which provide
12
some functionality, the 'imp' module, the "compiler" module, the semantics of
13
PEP 302 support, and setuptools, among other things.
15
At the top, you have "PythonPath", an abstract representation of sys.path which
16
includes methods to locate top-level modules, with or without loading them.
17
The top-level exposed functions in this module for accessing the system path
18
are "walkModules", "iterModules", and "getModule".
20
From most to least specific, here are the objects provided:
25
PathEntry # one entry on sys.path: an importer
28
PythonModule # a module or package that can be loaded
31
PythonAttribute # an attribute of a module (function or class)
34
PythonAttribute # an attribute of a function or class
39
Here's an example of idiomatic usage: this is what you would do to list all of
40
the modules outside the standard library's python-files directory.
43
stdlibdir = os.path.dirname(os.__file__)
45
from twisted.python.modules import iterModules
47
for modinfo in iterModules():
48
if (modinfo.pathEntry.filePath.path != stdlibdir
49
and not modinfo.isPackage()):
50
print 'unpackaged: %s: %s' % (
51
modinfo.name, modinfo.filePath.path)
56
# let's try to keep path imports to a minimum...
57
from os.path import dirname, split as splitpath
62
from errno import ENOTDIR
64
from zope.interface import Interface, implements
66
from twisted.python.components import registerAdapter
67
from twisted.python.filepath import FilePath
68
from twisted.python.zippath import ZipArchive
69
from twisted.python.reflect import namedAny
70
from twisted.python.win32 import WindowsError
71
from twisted.python.win32 import ERROR_DIRECTORY
75
PYTHON_EXTENSIONS = ['.py']
76
OPTIMIZED_MODE = __doc__ is None
78
PYTHON_EXTENSIONS.append('.pyo')
80
PYTHON_EXTENSIONS.append('.pyc')
82
def _isPythonIdentifier(string):
84
cheezy fake test for proper identifier-ness.
86
@param string: a str which might or might not be a valid python identifier.
88
@return: True or False
90
return (' ' not in string and
96
def _isPackagePath(fpath):
97
# Determine if a FilePath-like object is a Python package. TODO: deal with
98
# __init__module.(so|dll|pyd)?
99
extless = fpath.splitext()[0]
100
basend = splitpath(extless)[1]
101
return basend == "__init__"
105
class _ModuleIteratorHelper:
107
This mixin provides common behavior between python module and path entries,
108
since the mechanism for searching sys.path and __path__ attributes is
112
def iterModules(self):
114
Loop over the modules present below this entry or package on PYTHONPATH.
116
For modules which are not packages, this will yield nothing.
118
For packages and path entries, this will only yield modules one level
119
down; i.e. if there is a package a.b.c, iterModules on a will only
120
return a.b. If you want to descend deeply, use walkModules.
122
@return: a generator which yields PythonModule instances that describe
123
modules which can be, or have been, imported.
126
if not self.filePath.exists():
129
for placeToLook in self._packagePaths():
131
children = placeToLook.children()
132
except WindowsError, e:
133
errno = getattr(e, 'winerror', e.errno)
134
if errno == ERROR_DIRECTORY:
135
# It is a non-directory, skip it.
139
if e.errno in (ENOTDIR,):
140
# It is a non-directory, skip it.
144
for potentialTopLevel in children:
145
name, ext = potentialTopLevel.splitext()
146
if ext in PYTHON_EXTENSIONS:
147
# TODO: this should be a little choosier about which path entry
148
# it selects first, and it should do all the .so checking and
150
potentialBasename = potentialTopLevel.basename()[:-len(ext)]
151
if not _isPythonIdentifier(potentialBasename):
153
modname = self._subModuleName(potentialBasename)
154
if modname.split(".")[-1] == '__init__':
155
# This marks the directory as a package so it can't be
158
if modname not in yielded:
159
yielded[modname] = True
160
pm = PythonModule(modname, potentialTopLevel, self._getEntry())
163
elif potentialTopLevel.isdir():
164
modname = self._subModuleName(potentialTopLevel.basename())
165
for ext in PYTHON_EXTENSIONS:
166
initpy = potentialTopLevel.child("__init__"+ext)
168
yielded[modname] = True
169
pm = PythonModule(modname, initpy, self._getEntry())
174
def walkModules(self, importPackages=False):
176
Similar to L{iterModules}, this yields self, and then every module in my
177
package or entry, and every submodule in each package or entry.
179
In other words, this is deep, and L{iterModules} is shallow.
182
for package in self.iterModules():
183
for module in package.walkModules(importPackages=importPackages):
186
def _subModuleName(self, mn):
188
This is a hook to provide packages with the ability to specify their names
189
as a prefix to submodules here.
193
def _packagePaths(self):
195
Implement in subclasses to specify where to look for modules.
197
@return: iterable of FilePath-like objects.
199
raise NotImplementedError()
203
Implement in subclasses to specify what path entry submodules will come
206
@return: a PathEntry instance.
208
raise NotImplementedError()
211
def __getitem__(self, modname):
213
Retrieve a module from below this path or package.
215
@param modname: a str naming a module to be loaded. For entries, this
216
is a top-level, undotted package name, and for packages it is the name
217
of the module without the package prefix. For example, if you have a
218
PythonModule representing the 'twisted' package, you could use:
220
twistedPackageObj['python']['modules']
222
to retrieve this module.
224
@raise: KeyError if the module is not found.
226
@return: a PythonModule.
228
for module in self.iterModules():
229
if module.name == self._subModuleName(modname):
231
raise KeyError(modname)
235
Implemented to raise NotImplementedError for clarity, so that attempting to
236
loop over this object won't call __getitem__.
238
Note: in the future there might be some sensible default for iteration,
239
like 'walkEverything', so this is deliberately untested and undefined
242
raise NotImplementedError()
244
class PythonAttribute:
246
I represent a function, class, or other object that is present.
248
@ivar name: the fully-qualified python name of this attribute.
250
@ivar onObject: a reference to a PythonModule or other PythonAttribute that
251
is this attribute's logical parent.
253
@ivar name: the fully qualified python name of the attribute represented by
256
def __init__(self, name, onObject, loaded, pythonValue):
258
Create a PythonAttribute. This is a private constructor. Do not construct
259
me directly, use PythonModule.iterAttributes.
261
@param name: the FQPN
262
@param onObject: see ivar
263
@param loaded: always True, for now
264
@param pythonValue: the value of the attribute we're pointing to.
267
self.onObject = onObject
268
self._loaded = loaded
269
self.pythonValue = pythonValue
272
return 'PythonAttribute<%r>'%(self.name,)
276
Return a boolean describing whether the attribute this describes has
277
actually been loaded into memory by importing its module.
279
Note: this currently always returns true; there is no Python parser
280
support in this module yet.
284
def load(self, default=_nothing):
286
Load the value associated with this attribute.
288
@return: an arbitrary Python object, or 'default' if there is an error
291
return self.pythonValue
293
def iterAttributes(self):
294
for name, val in inspect.getmembers(self.load()):
295
yield PythonAttribute(self.name+'.'+name, self, True, val)
297
class PythonModule(_ModuleIteratorHelper):
299
Representation of a module which could be imported from sys.path.
301
@ivar name: the fully qualified python name of this module.
303
@ivar filePath: a FilePath-like object which points to the location of this
306
@ivar pathEntry: a L{PathEntry} instance which this module was located
310
def __init__(self, name, filePath, pathEntry):
312
Create a PythonModule. Do not construct this directly, instead inspect a
313
PythonPath or other PythonModule instances.
315
@param name: see ivar
316
@param filePath: see ivar
317
@param pathEntry: see ivar
319
assert not name.endswith(".__init__")
321
self.filePath = filePath
322
self.pathEntry = pathEntry
325
return self.pathEntry
329
Return a string representation including the module name.
331
return 'PythonModule<%r>' % (self.name,)
335
Determine if the module is loaded into sys.modules.
337
@return: a boolean: true if loaded, false if not.
339
return self.name in self.pathEntry.pythonPath.moduleDict
341
def iterAttributes(self):
343
List all the attributes defined in this module.
345
Note: Future work is planned here to make it possible to list python
346
attributes on a module without loading the module by inspecting ASTs or
347
bytecode, but currently any iteration of PythonModule objects insists
348
they must be loaded, and will use inspect.getmodule.
350
@raise NotImplementedError: if this module is not loaded.
352
@return: a generator yielding PythonAttribute instances describing the
353
attributes of this module.
355
if not self.isLoaded():
356
raise NotImplementedError(
357
"You can't load attributes from non-loaded modules yet.")
358
for name, val in inspect.getmembers(self.load()):
359
yield PythonAttribute(self.name+'.'+name, self, True, val)
363
Returns true if this module is also a package, and might yield something
366
return _isPackagePath(self.filePath)
368
def load(self, default=_nothing):
372
@param default: if specified, the value to return in case of an error.
374
@return: a genuine python module.
376
@raise: any type of exception. Importing modules is a risky business;
377
the erorrs of any code run at module scope may be raised from here, as
378
well as ImportError if something bizarre happened to the system path
379
between the discovery of this PythonModule object and the attempt to
380
import it. If you specify a default, the error will be swallowed
381
entirely, and not logged.
383
@rtype: types.ModuleType.
386
return self.pathEntry.pythonPath.moduleLoader(self.name)
387
except: # this needs more thought...
388
if default is not _nothing:
392
def __eq__(self, other):
394
PythonModules with the same name are equal.
396
if not isinstance(other, PythonModule):
398
return other.name == self.name
400
def __ne__(self, other):
402
PythonModules with different names are not equal.
404
if not isinstance(other, PythonModule):
406
return other.name != self.name
408
def walkModules(self, importPackages=False):
409
if importPackages and self.isPackage():
411
return super(PythonModule, self).walkModules(importPackages=importPackages)
413
def _subModuleName(self, mn):
415
submodules of this module are prefixed with our name.
417
return self.name + '.' + mn
419
def _packagePaths(self):
421
Yield a sequence of FilePath-like objects which represent path segments.
423
if not self.isPackage():
426
for fn in self.load().__path__:
427
sfpp = self.filePath.parent()
429
# this should _really_ exist.
433
smp = self.pathEntry.pythonPath._smartPath(fn)
437
yield self.filePath.parent()
440
class PathEntry(_ModuleIteratorHelper):
442
I am a proxy for a single entry on sys.path.
444
@ivar filePath: a FilePath-like object pointing at the filesystem location
445
or archive file where this path entry is stored.
447
@ivar pythonPath: a PythonPath instance.
449
def __init__(self, filePath, pythonPath):
451
Create a PathEntry. This is a private constructor.
453
self.filePath = filePath
454
self.pythonPath = pythonPath
460
return 'PathEntry<%r>' % (self.filePath,)
462
def _packagePaths(self):
465
class IPathImportMapper(Interface):
467
This is an internal interface, used to map importers to factories for
468
FilePath-like objects.
470
def mapPath(self, pathLikeString):
472
Return a FilePath-like object.
474
@param pathLikeString: a path-like string, like one that might be
475
passed to an import hook.
477
@return: a L{FilePath}, or something like it (currently only a
478
L{ZipPath}, but more might be added later).
481
class _DefaultMapImpl:
482
""" Wrapper for the default importer, i.e. None. """
483
implements(IPathImportMapper)
484
def mapPath(self, fsPathString):
485
return FilePath(fsPathString)
486
_theDefaultMapper = _DefaultMapImpl()
489
""" IPathImportMapper implementation for zipimport.ZipImporter. """
490
implements(IPathImportMapper)
491
def __init__(self, importer):
492
self.importer = importer
494
def mapPath(self, fsPathString):
496
Map the given FS path to a ZipPath, by looking at the ZipImporter's
497
"archive" attribute and using it as our ZipArchive root, then walking
498
down into the archive from there.
500
@return: a L{zippath.ZipPath} or L{zippath.ZipArchive} instance.
502
za = ZipArchive(self.importer.archive)
503
myPath = FilePath(self.importer.archive)
504
itsPath = FilePath(fsPathString)
505
if myPath == itsPath:
507
# This is NOT a general-purpose rule for sys.path or __file__:
508
# zipimport specifically uses regular OS path syntax in its pathnames.
509
segs = itsPath.segmentsFrom(myPath)
515
registerAdapter(_ZipMapImpl, zipimport.zipimporter, IPathImportMapper)
517
def _defaultSysPathFactory():
519
Provide the default behavior of PythonPath's sys.path factory, which is to
520
return the current value of sys.path.
529
I represent the very top of the Python object-space, the module list in
530
sys.path and the modules list in sys.modules.
532
@ivar sysPath: a sequence of strings like sys.path. This attribute is
535
@ivar moduleDict: a dictionary mapping string module names to module
536
objects, like sys.modules.
538
@ivar sysPathHooks: a list of PEP-302 path hooks, like sys.path_hooks.
540
@ivar moduleLoader: a function that takes a fully-qualified python name and
541
returns a module, like twisted.python.reflect.namedAny.
546
moduleDict=sys.modules,
547
sysPathHooks=sys.path_hooks,
548
importerCache=sys.path_importer_cache,
549
moduleLoader=namedAny,
550
sysPathFactory=None):
552
Create a PythonPath. You almost certainly want to use
553
modules.theSystemPath, or its aliased methods, rather than creating a
554
new instance yourself, though.
556
All parameters are optional, and if unspecified, will use 'system'
557
equivalents that makes this PythonPath like the global L{theSystemPath}
560
@param sysPath: a sys.path-like list to use for this PythonPath, to
561
specify where to load modules from.
563
@param moduleDict: a sys.modules-like dictionary to use for keeping
564
track of what modules this PythonPath has loaded.
566
@param sysPathHooks: sys.path_hooks-like list of PEP-302 path hooks to
567
be used for this PythonPath, to determie which importers should be
570
@param importerCache: a sys.path_importer_cache-like list of PEP-302
571
importers. This will be used in conjunction with the given
574
@param moduleLoader: a module loader function which takes a string and
575
returns a module. That is to say, it is like L{namedAny} - *not* like
578
@param sysPathFactory: a 0-argument callable which returns the current
579
value of a sys.path-like list of strings. Specify either this, or
580
sysPath, not both. This alternative interface is provided because the
581
way the Python import mechanism works, you can re-bind the 'sys.path'
582
name and that is what is used for current imports, so it must be a
583
factory rather than a value to deal with modification by rebinding
584
rather than modification by mutation. Note: it is not recommended to
585
rebind sys.path. Although this mechanism can deal with that, it is a
586
subtle point which some tools that it is easy for tools which interact
587
with sys.path to miss.
589
if sysPath is not None:
590
sysPathFactory = lambda : sysPath
591
elif sysPathFactory is None:
592
sysPathFactory = _defaultSysPathFactory
593
self._sysPathFactory = sysPathFactory
594
self._sysPath = sysPath
595
self.moduleDict = moduleDict
596
self.sysPathHooks = sysPathHooks
597
self.importerCache = importerCache
598
self._moduleLoader = moduleLoader
601
def _getSysPath(self):
603
Retrieve the current value of sys.path.
605
return self._sysPathFactory()
607
sysPath = property(_getSysPath)
609
def moduleLoader(self, modname):
611
Replicate python2.4+ sys.modules preservation behavior.
613
@param modname: a str module name.
615
@return: an imported module.
617
@raise: any type of exception that may arise from importing.
619
freezeModules = self.moduleDict.copy()
621
return self._moduleLoader(modname)
623
self.moduleDict.clear()
624
self.moduleDict.update(freezeModules)
627
def _findEntryPathString(self, modobj):
629
Determine where a given Python module object came from by looking at path
632
topPackageObj = modobj
633
while '.' in topPackageObj.__name__:
634
topPackageObj = self.moduleDict['.'.join(
635
topPackageObj.__name__.split('.')[:-1])]
636
if _isPackagePath(FilePath(topPackageObj.__file__)):
637
# if package 'foo' is on sys.path at /a/b/foo, package 'foo's
638
# __file__ will be /a/b/foo/__init__.py, and we are looking for
639
# /a/b here, the path-entry; so go up two steps.
640
rval = dirname(dirname(topPackageObj.__file__))
642
# the module is completely top-level, not within any packages. The
643
# path entry it's on is just its dirname.
644
rval = dirname(topPackageObj.__file__)
645
# There are probably some awful tricks that an importer could pull
646
# which would break this, so let's just make sure... it's a loaded
647
# module after all, which means that its path MUST be in
648
# path_importer_cache according to PEP 302 -glyph
649
from pprint import pformat
650
assert rval in self.importerCache, '%r for %r not in import cache %s' % (
651
rval, modobj, pformat(self.importerCache))
654
def _smartPath(self, pathName):
656
Given a path entry from sys.path which may refer to an importer,
657
return the appropriate FilePath-like instance.
659
@param pathName: a str describing the path.
661
@return: a FilePath-like object.
663
importr = self.importerCache.get(pathName, _nothing)
664
if importr is _nothing:
665
for hook in self.sysPathHooks:
667
importr = hook(pathName)
668
except ImportError, ie:
670
if importr is _nothing: # still
672
return IPathImportMapper(importr, _theDefaultMapper).mapPath(pathName)
674
def iterEntries(self):
676
Iterate the entries on my sysPath.
678
@return: a generator yielding PathEntry objects
680
for pathName in self.sysPath:
681
fp = self._smartPath(pathName)
682
yield PathEntry(fp, self)
684
def __getitem__(self, modname):
686
Get a python module by a given fully-qualified name.
688
@return: a PythonModule object.
690
@raise: KeyError, if the module name is a module name.
692
# See if the module is already somewhere in Python-land.
693
if modname in self.moduleDict:
694
# we need 2 paths; one of the path entry and one for the module.
695
moduleObject = self.moduleDict[modname]
698
self._findEntryPathString(moduleObject)),
700
mp = self._smartPath(moduleObject.__file__)
701
return PythonModule(modname, mp, pe)
703
# Recurse if we're trying to get a submodule.
706
for name in modname.split('.'):
710
# Finally do the slowest possible thing and iterate
711
for module in self.iterModules():
712
if module.name == modname:
714
raise KeyError(modname)
718
Display my sysPath and moduleDict in a string representation.
720
return "PythonPath(%r,%r)" % (self.sysPath, self.moduleDict)
722
def iterModules(self):
724
Yield all top-level modules on my sysPath.
726
for entry in self.iterEntries():
727
for module in entry.iterModules():
730
def walkModules(self, importPackages=False):
732
Similar to L{iterModules}, this yields every module on the path, then every
733
submodule in each package or entry.
735
for package in self.iterModules():
736
for module in package.walkModules(importPackages=False):
739
theSystemPath = PythonPath()
741
def walkModules(importPackages=False):
743
Deeply iterate all modules on the global python path.
745
@param importPackages: Import packages as they are seen.
747
return theSystemPath.walkModules(importPackages=importPackages)
751
Iterate all modules and top-level packages on the global Python path, but
752
do not descend into packages.
754
@param importPackages: Import packages as they are seen.
756
return theSystemPath.iterModules()
758
def getModule(moduleName):
760
Retrieve a module from the system path.
762
return theSystemPath[moduleName]
added
removed
twisted/python/modules.py
