1
# -*- test-case-name: twisted.test.test_modules -*-
2
# Copyright (c) 2006-2009 Twisted Matrix Laboratories.
3
# See LICENSE for details.
6
This module aims to provide a unified, object-oriented view of Python's
9
Python is a very dynamic language with wide variety of introspection utilities.
10
However, these utilities can be hard to use, because there is no consistent
11
API. The introspection API in python is made up of attributes (__name__,
12
__module__, func_name, etc) on instances, modules, classes and functions which
13
vary between those four types, utility modules such as 'inspect' which provide
14
some functionality, the 'imp' module, the "compiler" module, the semantics of
15
PEP 302 support, and setuptools, among other things.
17
At the top, you have "PythonPath", an abstract representation of sys.path which
18
includes methods to locate top-level modules, with or without loading them.
19
The top-level exposed functions in this module for accessing the system path
20
are "walkModules", "iterModules", and "getModule".
22
From most to least specific, here are the objects provided::
27
PathEntry # one entry on sys.path: an importer
30
PythonModule # a module or package that can be loaded
33
PythonAttribute # an attribute of a module (function or class)
36
PythonAttribute # an attribute of a function or class
41
Here's an example of idiomatic usage: this is what you would do to list all of
42
the modules outside the standard library's python-files directory::
45
stdlibdir = os.path.dirname(os.__file__)
47
from twisted.python.modules import iterModules
49
for modinfo in iterModules():
50
if (modinfo.pathEntry.filePath.path != stdlibdir
51
and not modinfo.isPackage()):
52
print 'unpackaged: %s: %s' % (
53
modinfo.name, modinfo.filePath.path)
58
# let's try to keep path imports to a minimum...
59
from os.path import dirname, split as splitpath
65
from zope.interface import Interface, implements
67
from twisted.python.components import registerAdapter
68
from twisted.python.filepath import FilePath, UnlistableError
69
from twisted.python.zippath import ZipArchive
70
from twisted.python.reflect import namedAny
74
PYTHON_EXTENSIONS = ['.py']
75
OPTIMIZED_MODE = __doc__ is None
77
PYTHON_EXTENSIONS.append('.pyo')
79
PYTHON_EXTENSIONS.append('.pyc')
81
def _isPythonIdentifier(string):
83
cheezy fake test for proper identifier-ness.
85
@param string: a str which might or might not be a valid python identifier.
87
@return: True or False
89
return (' ' not in string and
95
def _isPackagePath(fpath):
96
# Determine if a FilePath-like object is a Python package. TODO: deal with
97
# __init__module.(so|dll|pyd)?
98
extless = fpath.splitext()[0]
99
basend = splitpath(extless)[1]
100
return basend == "__init__"
104
class _ModuleIteratorHelper:
106
This mixin provides common behavior between python module and path entries,
107
since the mechanism for searching sys.path and __path__ attributes is
111
def iterModules(self):
113
Loop over the modules present below this entry or package on PYTHONPATH.
115
For modules which are not packages, this will yield nothing.
117
For packages and path entries, this will only yield modules one level
118
down; i.e. if there is a package a.b.c, iterModules on a will only
119
return a.b. If you want to descend deeply, use walkModules.
121
@return: a generator which yields PythonModule instances that describe
122
modules which can be, or have been, imported.
125
if not self.filePath.exists():
128
for placeToLook in self._packagePaths():
130
children = placeToLook.children()
131
except UnlistableError:
135
for potentialTopLevel in children:
136
ext = potentialTopLevel.splitext()[1]
137
potentialBasename = potentialTopLevel.basename()[:-len(ext)]
138
if ext in PYTHON_EXTENSIONS:
139
# TODO: this should be a little choosier about which path entry
140
# it selects first, and it should do all the .so checking and
142
if not _isPythonIdentifier(potentialBasename):
144
modname = self._subModuleName(potentialBasename)
145
if modname.split(".")[-1] == '__init__':
146
# This marks the directory as a package so it can't be
149
if modname not in yielded:
150
yielded[modname] = True
151
pm = PythonModule(modname, potentialTopLevel, self._getEntry())
155
if (ext or not _isPythonIdentifier(potentialBasename)
156
or not potentialTopLevel.isdir()):
158
modname = self._subModuleName(potentialTopLevel.basename())
159
for ext in PYTHON_EXTENSIONS:
160
initpy = potentialTopLevel.child("__init__"+ext)
162
yielded[modname] = True
163
pm = PythonModule(modname, initpy, self._getEntry())
168
def walkModules(self, importPackages=False):
170
Similar to L{iterModules}, this yields self, and then every module in my
171
package or entry, and every submodule in each package or entry.
173
In other words, this is deep, and L{iterModules} is shallow.
176
for package in self.iterModules():
177
for module in package.walkModules(importPackages=importPackages):
180
def _subModuleName(self, mn):
182
This is a hook to provide packages with the ability to specify their names
183
as a prefix to submodules here.
187
def _packagePaths(self):
189
Implement in subclasses to specify where to look for modules.
191
@return: iterable of FilePath-like objects.
193
raise NotImplementedError()
197
Implement in subclasses to specify what path entry submodules will come
200
@return: a PathEntry instance.
202
raise NotImplementedError()
205
def __getitem__(self, modname):
207
Retrieve a module from below this path or package.
209
@param modname: a str naming a module to be loaded. For entries, this
210
is a top-level, undotted package name, and for packages it is the name
211
of the module without the package prefix. For example, if you have a
212
PythonModule representing the 'twisted' package, you could use::
214
twistedPackageObj['python']['modules']
216
to retrieve this module.
218
@raise: KeyError if the module is not found.
220
@return: a PythonModule.
222
for module in self.iterModules():
223
if module.name == self._subModuleName(modname):
225
raise KeyError(modname)
229
Implemented to raise NotImplementedError for clarity, so that attempting to
230
loop over this object won't call __getitem__.
232
Note: in the future there might be some sensible default for iteration,
233
like 'walkEverything', so this is deliberately untested and undefined
236
raise NotImplementedError()
238
class PythonAttribute:
240
I represent a function, class, or other object that is present.
242
@ivar name: the fully-qualified python name of this attribute.
244
@ivar onObject: a reference to a PythonModule or other PythonAttribute that
245
is this attribute's logical parent.
247
@ivar name: the fully qualified python name of the attribute represented by
250
def __init__(self, name, onObject, loaded, pythonValue):
252
Create a PythonAttribute. This is a private constructor. Do not construct
253
me directly, use PythonModule.iterAttributes.
255
@param name: the FQPN
256
@param onObject: see ivar
257
@param loaded: always True, for now
258
@param pythonValue: the value of the attribute we're pointing to.
261
self.onObject = onObject
262
self._loaded = loaded
263
self.pythonValue = pythonValue
266
return 'PythonAttribute<%r>'%(self.name,)
270
Return a boolean describing whether the attribute this describes has
271
actually been loaded into memory by importing its module.
273
Note: this currently always returns true; there is no Python parser
274
support in this module yet.
278
def load(self, default=_nothing):
280
Load the value associated with this attribute.
282
@return: an arbitrary Python object, or 'default' if there is an error
285
return self.pythonValue
287
def iterAttributes(self):
288
for name, val in inspect.getmembers(self.load()):
289
yield PythonAttribute(self.name+'.'+name, self, True, val)
291
class PythonModule(_ModuleIteratorHelper):
293
Representation of a module which could be imported from sys.path.
295
@ivar name: the fully qualified python name of this module.
297
@ivar filePath: a FilePath-like object which points to the location of this
300
@ivar pathEntry: a L{PathEntry} instance which this module was located
304
def __init__(self, name, filePath, pathEntry):
306
Create a PythonModule. Do not construct this directly, instead inspect a
307
PythonPath or other PythonModule instances.
309
@param name: see ivar
310
@param filePath: see ivar
311
@param pathEntry: see ivar
313
assert not name.endswith(".__init__")
315
self.filePath = filePath
316
self.parentPath = filePath.parent()
317
self.pathEntry = pathEntry
320
return self.pathEntry
324
Return a string representation including the module name.
326
return 'PythonModule<%r>' % (self.name,)
330
Determine if the module is loaded into sys.modules.
332
@return: a boolean: true if loaded, false if not.
334
return self.name in self.pathEntry.pythonPath.moduleDict
336
def iterAttributes(self):
338
List all the attributes defined in this module.
340
Note: Future work is planned here to make it possible to list python
341
attributes on a module without loading the module by inspecting ASTs or
342
bytecode, but currently any iteration of PythonModule objects insists
343
they must be loaded, and will use inspect.getmodule.
345
@raise NotImplementedError: if this module is not loaded.
347
@return: a generator yielding PythonAttribute instances describing the
348
attributes of this module.
350
if not self.isLoaded():
351
raise NotImplementedError(
352
"You can't load attributes from non-loaded modules yet.")
353
for name, val in inspect.getmembers(self.load()):
354
yield PythonAttribute(self.name+'.'+name, self, True, val)
358
Returns true if this module is also a package, and might yield something
361
return _isPackagePath(self.filePath)
363
def load(self, default=_nothing):
367
@param default: if specified, the value to return in case of an error.
369
@return: a genuine python module.
371
@raise: any type of exception. Importing modules is a risky business;
372
the erorrs of any code run at module scope may be raised from here, as
373
well as ImportError if something bizarre happened to the system path
374
between the discovery of this PythonModule object and the attempt to
375
import it. If you specify a default, the error will be swallowed
376
entirely, and not logged.
378
@rtype: types.ModuleType.
381
return self.pathEntry.pythonPath.moduleLoader(self.name)
382
except: # this needs more thought...
383
if default is not _nothing:
387
def __eq__(self, other):
389
PythonModules with the same name are equal.
391
if not isinstance(other, PythonModule):
393
return other.name == self.name
395
def __ne__(self, other):
397
PythonModules with different names are not equal.
399
if not isinstance(other, PythonModule):
401
return other.name != self.name
403
def walkModules(self, importPackages=False):
404
if importPackages and self.isPackage():
406
return super(PythonModule, self).walkModules(importPackages=importPackages)
408
def _subModuleName(self, mn):
410
submodules of this module are prefixed with our name.
412
return self.name + '.' + mn
414
def _packagePaths(self):
416
Yield a sequence of FilePath-like objects which represent path segments.
418
if not self.isPackage():
422
if hasattr(load, '__path__'):
423
for fn in load.__path__:
424
if fn == self.parentPath.path:
425
# this should _really_ exist.
426
assert self.parentPath.exists()
427
yield self.parentPath
429
smp = self.pathEntry.pythonPath._smartPath(fn)
433
yield self.parentPath
436
class PathEntry(_ModuleIteratorHelper):
438
I am a proxy for a single entry on sys.path.
440
@ivar filePath: a FilePath-like object pointing at the filesystem location
441
or archive file where this path entry is stored.
443
@ivar pythonPath: a PythonPath instance.
445
def __init__(self, filePath, pythonPath):
447
Create a PathEntry. This is a private constructor.
449
self.filePath = filePath
450
self.pythonPath = pythonPath
456
return 'PathEntry<%r>' % (self.filePath,)
458
def _packagePaths(self):
461
class IPathImportMapper(Interface):
463
This is an internal interface, used to map importers to factories for
464
FilePath-like objects.
466
def mapPath(self, pathLikeString):
468
Return a FilePath-like object.
470
@param pathLikeString: a path-like string, like one that might be
471
passed to an import hook.
473
@return: a L{FilePath}, or something like it (currently only a
474
L{ZipPath}, but more might be added later).
477
class _DefaultMapImpl:
478
""" Wrapper for the default importer, i.e. None. """
479
implements(IPathImportMapper)
480
def mapPath(self, fsPathString):
481
return FilePath(fsPathString)
482
_theDefaultMapper = _DefaultMapImpl()
485
""" IPathImportMapper implementation for zipimport.ZipImporter. """
486
implements(IPathImportMapper)
487
def __init__(self, importer):
488
self.importer = importer
490
def mapPath(self, fsPathString):
492
Map the given FS path to a ZipPath, by looking at the ZipImporter's
493
"archive" attribute and using it as our ZipArchive root, then walking
494
down into the archive from there.
496
@return: a L{zippath.ZipPath} or L{zippath.ZipArchive} instance.
498
za = ZipArchive(self.importer.archive)
499
myPath = FilePath(self.importer.archive)
500
itsPath = FilePath(fsPathString)
501
if myPath == itsPath:
503
# This is NOT a general-purpose rule for sys.path or __file__:
504
# zipimport specifically uses regular OS path syntax in its pathnames,
505
# even though zip files specify that slashes are always the separator,
506
# regardless of platform.
507
segs = itsPath.segmentsFrom(myPath)
513
registerAdapter(_ZipMapImpl, zipimport.zipimporter, IPathImportMapper)
515
def _defaultSysPathFactory():
517
Provide the default behavior of PythonPath's sys.path factory, which is to
518
return the current value of sys.path.
527
I represent the very top of the Python object-space, the module list in
528
sys.path and the modules list in sys.modules.
530
@ivar _sysPath: a sequence of strings like sys.path. This attribute is
533
@ivar moduleDict: a dictionary mapping string module names to module
534
objects, like sys.modules.
536
@ivar sysPathHooks: a list of PEP-302 path hooks, like sys.path_hooks.
538
@ivar moduleLoader: a function that takes a fully-qualified python name and
539
returns a module, like twisted.python.reflect.namedAny.
544
moduleDict=sys.modules,
545
sysPathHooks=sys.path_hooks,
546
importerCache=sys.path_importer_cache,
547
moduleLoader=namedAny,
548
sysPathFactory=None):
550
Create a PythonPath. You almost certainly want to use
551
modules.theSystemPath, or its aliased methods, rather than creating a
552
new instance yourself, though.
554
All parameters are optional, and if unspecified, will use 'system'
555
equivalents that makes this PythonPath like the global L{theSystemPath}
558
@param sysPath: a sys.path-like list to use for this PythonPath, to
559
specify where to load modules from.
561
@param moduleDict: a sys.modules-like dictionary to use for keeping
562
track of what modules this PythonPath has loaded.
564
@param sysPathHooks: sys.path_hooks-like list of PEP-302 path hooks to
565
be used for this PythonPath, to determie which importers should be
568
@param importerCache: a sys.path_importer_cache-like list of PEP-302
569
importers. This will be used in conjunction with the given
572
@param moduleLoader: a module loader function which takes a string and
573
returns a module. That is to say, it is like L{namedAny} - *not* like
576
@param sysPathFactory: a 0-argument callable which returns the current
577
value of a sys.path-like list of strings. Specify either this, or
578
sysPath, not both. This alternative interface is provided because the
579
way the Python import mechanism works, you can re-bind the 'sys.path'
580
name and that is what is used for current imports, so it must be a
581
factory rather than a value to deal with modification by rebinding
582
rather than modification by mutation. Note: it is not recommended to
583
rebind sys.path. Although this mechanism can deal with that, it is a
584
subtle point which some tools that it is easy for tools which interact
585
with sys.path to miss.
587
if sysPath is not None:
588
sysPathFactory = lambda : sysPath
589
elif sysPathFactory is None:
590
sysPathFactory = _defaultSysPathFactory
591
self._sysPathFactory = sysPathFactory
592
self._sysPath = sysPath
593
self.moduleDict = moduleDict
594
self.sysPathHooks = sysPathHooks
595
self.importerCache = importerCache
596
self.moduleLoader = moduleLoader
599
def _getSysPath(self):
601
Retrieve the current value of the module search path list.
603
return self._sysPathFactory()
605
sysPath = property(_getSysPath)
607
def _findEntryPathString(self, modobj):
609
Determine where a given Python module object came from by looking at path
612
topPackageObj = modobj
613
while '.' in topPackageObj.__name__:
614
topPackageObj = self.moduleDict['.'.join(
615
topPackageObj.__name__.split('.')[:-1])]
616
if _isPackagePath(FilePath(topPackageObj.__file__)):
617
# if package 'foo' is on sys.path at /a/b/foo, package 'foo's
618
# __file__ will be /a/b/foo/__init__.py, and we are looking for
619
# /a/b here, the path-entry; so go up two steps.
620
rval = dirname(dirname(topPackageObj.__file__))
622
# the module is completely top-level, not within any packages. The
623
# path entry it's on is just its dirname.
624
rval = dirname(topPackageObj.__file__)
626
# There are probably some awful tricks that an importer could pull
627
# which would break this, so let's just make sure... it's a loaded
628
# module after all, which means that its path MUST be in
629
# path_importer_cache according to PEP 302 -glyph
630
if rval not in self.importerCache:
632
"%s (for module %s) not in path importer cache "
633
"(PEP 302 violation - check your local configuration)." % (
634
rval, modobj.__name__),
639
def _smartPath(self, pathName):
641
Given a path entry from sys.path which may refer to an importer,
642
return the appropriate FilePath-like instance.
644
@param pathName: a str describing the path.
646
@return: a FilePath-like object.
648
importr = self.importerCache.get(pathName, _nothing)
649
if importr is _nothing:
650
for hook in self.sysPathHooks:
652
importr = hook(pathName)
653
except ImportError, ie:
655
if importr is _nothing: # still
657
return IPathImportMapper(importr, _theDefaultMapper).mapPath(pathName)
659
def iterEntries(self):
661
Iterate the entries on my sysPath.
663
@return: a generator yielding PathEntry objects
665
for pathName in self.sysPath:
666
fp = self._smartPath(pathName)
667
yield PathEntry(fp, self)
669
def __getitem__(self, modname):
671
Get a python module by a given fully-qualified name.
673
@return: a PythonModule object.
675
@raise: KeyError, if the module name is a module name.
677
# See if the module is already somewhere in Python-land.
678
if modname in self.moduleDict:
679
# we need 2 paths; one of the path entry and one for the module.
680
moduleObject = self.moduleDict[modname]
683
self._findEntryPathString(moduleObject)),
685
mp = self._smartPath(moduleObject.__file__)
686
return PythonModule(modname, mp, pe)
688
# Recurse if we're trying to get a submodule.
691
for name in modname.split('.'):
695
# Finally do the slowest possible thing and iterate
696
for module in self.iterModules():
697
if module.name == modname:
699
raise KeyError(modname)
703
Display my sysPath and moduleDict in a string representation.
705
return "PythonPath(%r,%r)" % (self.sysPath, self.moduleDict)
707
def iterModules(self):
709
Yield all top-level modules on my sysPath.
711
for entry in self.iterEntries():
712
for module in entry.iterModules():
715
def walkModules(self, importPackages=False):
717
Similar to L{iterModules}, this yields every module on the path, then every
718
submodule in each package or entry.
720
for package in self.iterModules():
721
for module in package.walkModules(importPackages=False):
724
theSystemPath = PythonPath()
726
def walkModules(importPackages=False):
728
Deeply iterate all modules on the global python path.
730
@param importPackages: Import packages as they are seen.
732
return theSystemPath.walkModules(importPackages=importPackages)
736
Iterate all modules and top-level packages on the global Python path, but
737
do not descend into packages.
739
@param importPackages: Import packages as they are seen.
741
return theSystemPath.iterModules()
743
def getModule(moduleName):
745
Retrieve a module from the system path.
747
return theSystemPath[moduleName]
added
removed
vendor/Twisted-10.0.0/twisted/python/modules.py
