1
"""Core implementation of path-based import.
3
This module is NOT meant to be directly imported! It has been designed such
4
that it can be bootstrapped into Python as the implementation of import. As
5
such it requires the injection of specific modules and attributes in order to
6
work. One should use importlib as the public-facing version of this module.
10
# IMPORTANT: Whenever making changes to this module, be sure to run
11
# a top-level make in order to get the frozen version of the module
12
# updated. Not doing so will result in the Makefile to fail for
13
# all others who don't have a ./python around to freeze the module
14
# in the early stages of compilation.
17
# See importlib._setup() for what is injected into the global namespace.
19
# When editing this code be aware that code executed at import time CANNOT
20
# reference any injected objects! This includes not only global code but also
21
# anything specified at the class level.
23
# Bootstrap-related code ######################################################
25
_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin'
28
def _make_relax_case():
29
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
31
"""True if filenames must be checked case-insensitively."""
32
return b'PYTHONCASEOK' in _os.environ
35
"""True if filenames must be checked case-insensitively."""
41
"""Convert a 32-bit integer to little-endian."""
42
return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
45
def _r_long(int_bytes):
46
"""Convert 4 bytes in little-endian to an integer."""
47
return int.from_bytes(int_bytes, 'little')
50
def _path_join(*path_parts):
51
"""Replacement for os.path.join()."""
52
return path_sep.join([part.rstrip(path_separators)
53
for part in path_parts if part])
56
def _path_split(path):
57
"""Replacement for os.path.split()."""
58
if len(path_separators) == 1:
59
front, _, tail = path.rpartition(path_sep)
61
for x in reversed(path):
62
if x in path_separators:
63
front, tail = path.rsplit(x, maxsplit=1)
71
Made a separate function to make it easier to override in experiments
72
(e.g. cache stat results).
78
def _path_is_mode_type(path, mode):
79
"""Test whether the path is the specified mode type."""
81
stat_info = _path_stat(path)
84
return (stat_info.st_mode & 0o170000) == mode
87
def _path_isfile(path):
88
"""Replacement for os.path.isfile."""
89
return _path_is_mode_type(path, 0o100000)
92
def _path_isdir(path):
93
"""Replacement for os.path.isdir."""
96
return _path_is_mode_type(path, 0o040000)
99
def _write_atomic(path, data, mode=0o666):
100
"""Best-effort function to write data to a path atomically.
101
Be prepared to handle a FileExistsError if concurrent writing of the
102
temporary file is attempted."""
103
# id() is used to generate a pseudo-random filename.
104
path_tmp = '{}.{}'.format(path, id(path))
105
fd = _os.open(path_tmp,
106
_os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
108
# We first write data to a temporary file, and then use os.replace() to
109
# perform an atomic rename.
110
with _io.FileIO(fd, 'wb') as file:
112
_os.replace(path_tmp, path)
121
_code_type = type(_write_atomic.__code__)
124
# Finder/loader utility code ###############################################
126
# Magic word to reject .pyc files generated by other Python versions.
127
# It should change for each incompatible change to the bytecode.
129
# The value of CR and LF is incorporated so if you ever read or write
130
# a .pyc file in text mode the magic number will be wrong; also, the
131
# Apple MPW compiler swaps their values, botching string constants.
133
# The magic numbers must be spaced apart at least 2 values, as the
134
# -U interpeter flag will cause MAGIC+1 being used. They have been
135
# odd numbers for some time now.
137
# There were a variety of old schemes for setting the magic number.
138
# The current working scheme is to increment the previous value by
141
# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
142
# number also includes a new "magic tag", i.e. a human readable string used
143
# to represent the magic number in __pycache__ directories. When you change
144
# the magic number, you must also set a new unique magic tag. Generally this
145
# can be named after the Python major version of the magic number bump, but
146
# it can really be anything, as long as it's different than anything else
147
# that's come before. The tags are included in the following table, starting
152
# Python 1.5.1: 20121
153
# Python 1.5.2: 20121
156
# Python 2.0.1: 50823
158
# Python 2.1.1: 60202
159
# Python 2.1.2: 60202
161
# Python 2.3a0: 62011
162
# Python 2.3a0: 62021
163
# Python 2.3a0: 62011 (!)
164
# Python 2.4a0: 62041
165
# Python 2.4a3: 62051
166
# Python 2.4b1: 62061
167
# Python 2.5a0: 62071
168
# Python 2.5a0: 62081 (ast-branch)
169
# Python 2.5a0: 62091 (with)
170
# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
171
# Python 2.5b3: 62101 (fix wrong code: for x, in ...)
172
# Python 2.5b3: 62111 (fix wrong code: x += yield)
173
# Python 2.5c1: 62121 (fix wrong lnotab with for loops and
174
# storing constants that should have been removed)
175
# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
176
# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
177
# Python 2.6a1: 62161 (WITH_CLEANUP optimization)
178
# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
179
# Python 2.7a0: 62181 (optimize conditional branches:
180
# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
181
# Python 2.7a0 62191 (introduce SETUP_WITH)
182
# Python 2.7a0 62201 (introduce BUILD_SET)
183
# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
185
# 3010 (removed UNARY_CONVERT)
186
# 3020 (added BUILD_SET)
187
# 3030 (added keyword-only parameters)
188
# 3040 (added signature annotations)
189
# 3050 (print becomes a function)
190
# 3060 (PEP 3115 metaclass syntax)
191
# 3061 (string literals become unicode)
192
# 3071 (PEP 3109 raise changes)
193
# 3081 (PEP 3137 make __file__ and __name__ unicode)
194
# 3091 (kill str8 interning)
195
# 3101 (merge from 2.6a0, see 62151)
196
# 3103 (__file__ points to source file)
197
# Python 3.0a4: 3111 (WITH_CLEANUP optimization).
198
# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT)
199
# Python 3.1a0: 3141 (optimize list, set and dict comprehensions:
200
# change LIST_APPEND and SET_ADD, add MAP_ADD)
201
# Python 3.1a0: 3151 (optimize conditional branches:
202
# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
203
# Python 3.2a0: 3160 (add SETUP_WITH)
205
# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
207
# Python 3.2a2 3180 (add DELETE_DEREF)
208
# Python 3.3a0 3190 __class__ super closure changed
209
# Python 3.3a0 3200 (__qualname__ added)
210
# 3210 (added size modulo 2**32 to the pyc header)
211
# Python 3.3a1 3220 (changed PEP 380 implementation)
212
# Python 3.3a4 3230 (revert changes to implicit __class__ closure)
213
# Python 3.4a1 3250 (evaluate positional default arguments before
214
# keyword-only defaults)
215
# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
217
# Python 3.4a1 3270 (various tweaks to the __class__ closure)
218
# Python 3.4a1 3280 (remove implicit class argument)
219
# Python 3.4a4 3290 (changes to __qualname__ computation)
220
# Python 3.4a4 3300 (more changes to __qualname__ computation)
221
# Python 3.4rc2 3310 (alter __qualname__ computation)
222
# Python 3.5a0 3320 (matrix multiplication operator)
223
# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations)
224
# Python 3.5b2 3340 (fix dictionary display evaluation order #11205)
225
# Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400)
227
# MAGIC must change whenever the bytecode emitted by the compiler may no
228
# longer be understood by older implementations of the eval loop (usually
229
# due to the addition of new opcodes).
231
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
232
# in PC/launcher.c must also be updated.
234
MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n'
235
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
237
_PYCACHE = '__pycache__'
240
SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed.
242
BYTECODE_SUFFIXES = ['.pyc']
244
DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
246
def cache_from_source(path, debug_override=None, *, optimization=None):
247
"""Given the path to a .py file, return the path to its .pyc file.
249
The .py file does not need to exist; this simply returns the path to the
250
.pyc file calculated as if the .py file were imported.
252
The 'optimization' parameter controls the presumed optimization level of
253
the bytecode file. If 'optimization' is not None, the string representation
254
of the argument is taken and verified to be alphanumeric (else ValueError
257
The debug_override parameter is deprecated. If debug_override is not None,
258
a True value is the same as setting 'optimization' to the empty string
259
while a False value is equivalent to setting 'optimization' to '1'.
261
If sys.implementation.cache_tag is None then NotImplementedError is raised.
264
if debug_override is not None:
265
_warnings.warn('the debug_override parameter is deprecated; use '
266
"'optimization' instead", DeprecationWarning)
267
if optimization is not None:
268
message = 'debug_override or optimization must be set to None'
269
raise TypeError(message)
270
optimization = '' if debug_override else 1
271
head, tail = _path_split(path)
272
base, sep, rest = tail.rpartition('.')
273
tag = sys.implementation.cache_tag
275
raise NotImplementedError('sys.implementation.cache_tag is None')
276
almost_filename = ''.join([(base if base else rest), sep, tag])
277
if optimization is None:
278
if sys.flags.optimize == 0:
281
optimization = sys.flags.optimize
282
optimization = str(optimization)
283
if optimization != '':
284
if not optimization.isalnum():
285
raise ValueError('{!r} is not alphanumeric'.format(optimization))
286
almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
287
return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0])
290
def source_from_cache(path):
291
"""Given the path to a .pyc. file, return the path to its .py file.
293
The .pyc file does not need to exist; this simply returns the path to
294
the .py file calculated to correspond to the .pyc file. If path does
295
not conform to PEP 3147/488 format, ValueError will be raised. If
296
sys.implementation.cache_tag is None then NotImplementedError is raised.
299
if sys.implementation.cache_tag is None:
300
raise NotImplementedError('sys.implementation.cache_tag is None')
301
head, pycache_filename = _path_split(path)
302
head, pycache = _path_split(head)
303
if pycache != _PYCACHE:
304
raise ValueError('{} not bottom-level directory in '
305
'{!r}'.format(_PYCACHE, path))
306
dot_count = pycache_filename.count('.')
307
if dot_count not in {2, 3}:
308
raise ValueError('expected only 2 or 3 dots in '
309
'{!r}'.format(pycache_filename))
311
optimization = pycache_filename.rsplit('.', 2)[-2]
312
if not optimization.startswith(_OPT):
313
raise ValueError("optimization portion of filename does not start "
314
"with {!r}".format(_OPT))
315
opt_level = optimization[len(_OPT):]
316
if not opt_level.isalnum():
317
raise ValueError("optimization level {!r} is not an alphanumeric "
318
"value".format(optimization))
319
base_filename = pycache_filename.partition('.')[0]
320
return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
323
def _get_sourcefile(bytecode_path):
324
"""Convert a bytecode file path to a source path (if possible).
326
This function exists purely for backwards-compatibility for
327
PyImport_ExecCodeModuleWithFilenames() in the C API.
330
if len(bytecode_path) == 0:
332
rest, _, extension = bytecode_path.rpartition('.')
333
if not rest or extension.lower()[-3:-1] != 'py':
336
source_path = source_from_cache(bytecode_path)
337
except (NotImplementedError, ValueError):
338
source_path = bytecode_path[:-1]
339
return source_path if _path_isfile(source_path) else bytecode_path
342
def _get_cached(filename):
343
if filename.endswith(tuple(SOURCE_SUFFIXES)):
345
return cache_from_source(filename)
346
except NotImplementedError:
348
elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
354
def _calc_mode(path):
355
"""Calculate the mode permissions for a bytecode file."""
357
mode = _path_stat(path).st_mode
360
# We always ensure write access so we can update cached files
361
# later even when the source files are read-only on Windows (#6074)
366
def _verbose_message(message, *args, verbosity=1):
367
"""Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
368
if sys.flags.verbose >= verbosity:
369
if not message.startswith(('#', 'import ')):
370
message = '# ' + message
371
print(message.format(*args), file=sys.stderr)
374
def _check_name(method):
375
"""Decorator to verify that the module being requested matches the one the
378
The first argument (self) must define _name which the second argument is
379
compared against. If the comparison fails then ImportError is raised.
382
def _check_name_wrapper(self, name=None, *args, **kwargs):
385
elif self.name != name:
386
raise ImportError('loader for %s cannot handle %s' %
387
(self.name, name), name=name)
388
return method(self, name, *args, **kwargs)
390
_wrap = _bootstrap._wrap
394
for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
395
if hasattr(old, replace):
396
setattr(new, replace, getattr(old, replace))
397
new.__dict__.update(old.__dict__)
398
_wrap(_check_name_wrapper, method)
399
return _check_name_wrapper
402
def _find_module_shim(self, fullname):
403
"""Try to find a loader for the specified module by delegating to
406
This method is deprecated in favor of finder.find_spec().
409
# Call find_loader(). If it returns a string (indicating this
410
# is a namespace package portion), generate a warning and
412
loader, portions = self.find_loader(fullname)
413
if loader is None and len(portions):
414
msg = 'Not importing directory {}: missing __init__'
415
_warnings.warn(msg.format(portions[0]), ImportWarning)
419
def _validate_bytecode_header(data, source_stats=None, name=None, path=None):
420
"""Validate the header of the passed-in bytecode against source_stats (if
421
given) and returning the bytecode that can be compiled by compile().
423
All other arguments are used to enhance error reporting.
425
ImportError is raised when the magic number is incorrect or the bytecode is
426
found to be stale. EOFError is raised when the data is found to be
432
exc_details['name'] = name
434
# To prevent having to make all messages have a conditional name.
437
exc_details['path'] = path
439
raw_timestamp = data[4:8]
440
raw_size = data[8:12]
441
if magic != MAGIC_NUMBER:
442
message = 'bad magic number in {!r}: {!r}'.format(name, magic)
443
_verbose_message('{}', message)
444
raise ImportError(message, **exc_details)
445
elif len(raw_timestamp) != 4:
446
message = 'reached EOF while reading timestamp in {!r}'.format(name)
447
_verbose_message('{}', message)
448
raise EOFError(message)
449
elif len(raw_size) != 4:
450
message = 'reached EOF while reading size of source in {!r}'.format(name)
451
_verbose_message('{}', message)
452
raise EOFError(message)
453
if source_stats is not None:
455
source_mtime = int(source_stats['mtime'])
459
if _r_long(raw_timestamp) != source_mtime:
460
message = 'bytecode is stale for {!r}'.format(name)
461
_verbose_message('{}', message)
462
raise ImportError(message, **exc_details)
464
source_size = source_stats['size'] & 0xFFFFFFFF
468
if _r_long(raw_size) != source_size:
469
raise ImportError('bytecode is stale for {!r}'.format(name),
474
def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
475
"""Compile bytecode as returned by _validate_bytecode_header()."""
476
code = marshal.loads(data)
477
if isinstance(code, _code_type):
478
_verbose_message('code object from {!r}', bytecode_path)
479
if source_path is not None:
480
_imp._fix_co_filename(code, source_path)
483
raise ImportError('Non-code object in {!r}'.format(bytecode_path),
484
name=name, path=bytecode_path)
486
def _code_to_bytecode(code, mtime=0, source_size=0):
487
"""Compile a code object into bytecode for writing out to a byte-compiled
489
data = bytearray(MAGIC_NUMBER)
490
data.extend(_w_long(mtime))
491
data.extend(_w_long(source_size))
492
data.extend(marshal.dumps(code))
496
def decode_source(source_bytes):
497
"""Decode bytes representing source code and return the string.
499
Universal newline support is used in the decoding.
501
import tokenize # To avoid bootstrap issues.
502
source_bytes_readline = _io.BytesIO(source_bytes).readline
503
encoding = tokenize.detect_encoding(source_bytes_readline)
504
newline_decoder = _io.IncrementalNewlineDecoder(None, True)
505
return newline_decoder.decode(source_bytes.decode(encoding[0]))
508
# Module specifications #######################################################
513
def spec_from_file_location(name, location=None, *, loader=None,
514
submodule_search_locations=_POPULATE):
515
"""Return a module spec based on a file location.
517
To indicate that the module is a package, set
518
submodule_search_locations to a list of directory paths. An
519
empty list is sufficient, though its not otherwise useful to the
522
The loader must take a spec as its only __init__() arg.
526
# The caller may simply want a partially populated location-
527
# oriented spec. So we set the location to a bogus value and
528
# fill in as much as we can.
529
location = '<unknown>'
530
if hasattr(loader, 'get_filename'):
533
location = loader.get_filename(name)
537
# If the location is on the filesystem, but doesn't actually exist,
538
# we could return None here, indicating that the location is not
539
# valid. However, we don't have a good way of testing since an
540
# indirect location (e.g. a zip file or URL) will look like a
541
# non-existent file relative to the filesystem.
543
spec = _bootstrap.ModuleSpec(name, loader, origin=location)
544
spec._set_fileattr = True
546
# Pick a loader if one wasn't provided.
548
for loader_class, suffixes in _get_supported_file_loaders():
549
if location.endswith(tuple(suffixes)):
550
loader = loader_class(name, location)
556
# Set submodule_search_paths appropriately.
557
if submodule_search_locations is _POPULATE:
559
if hasattr(loader, 'is_package'):
561
is_package = loader.is_package(name)
566
spec.submodule_search_locations = []
568
spec.submodule_search_locations = submodule_search_locations
569
if spec.submodule_search_locations == []:
571
dirname = _path_split(location)[0]
572
spec.submodule_search_locations.append(dirname)
577
# Loaders #####################################################################
579
class WindowsRegistryFinder:
581
"""Meta path finder for modules declared in the Windows registry."""
584
'Software\\Python\\PythonCore\\{sys_version}'
585
'\\Modules\\{fullname}')
586
REGISTRY_KEY_DEBUG = (
587
'Software\\Python\\PythonCore\\{sys_version}'
588
'\\Modules\\{fullname}\\Debug')
589
DEBUG_BUILD = False # Changed in _setup()
592
def _open_registry(cls, key):
594
return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
596
return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
599
def _search_registry(cls, fullname):
601
registry_key = cls.REGISTRY_KEY_DEBUG
603
registry_key = cls.REGISTRY_KEY
604
key = registry_key.format(fullname=fullname,
605
sys_version=sys.version[:3])
607
with cls._open_registry(key) as hkey:
608
filepath = _winreg.QueryValue(hkey, '')
614
def find_spec(cls, fullname, path=None, target=None):
615
filepath = cls._search_registry(fullname)
622
for loader, suffixes in _get_supported_file_loaders():
623
if filepath.endswith(tuple(suffixes)):
624
spec = _bootstrap.spec_from_loader(fullname,
625
loader(fullname, filepath),
630
def find_module(cls, fullname, path=None):
631
"""Find module named in the registry.
633
This method is deprecated. Use exec_module() instead.
636
spec = cls.find_spec(fullname, path)
645
"""Base class of common code needed by both SourceLoader and
646
SourcelessFileLoader."""
648
def is_package(self, fullname):
649
"""Concrete implementation of InspectLoader.is_package by checking if
650
the path returned by get_filename has a filename of '__init__.py'."""
651
filename = _path_split(self.get_filename(fullname))[1]
652
filename_base = filename.rsplit('.', 1)[0]
653
tail_name = fullname.rpartition('.')[2]
654
return filename_base == '__init__' and tail_name != '__init__'
656
def create_module(self, spec):
657
"""Use default semantics for module creation."""
659
def exec_module(self, module):
660
"""Execute the module."""
661
code = self.get_code(module.__name__)
663
raise ImportError('cannot load module {!r} when get_code() '
664
'returns None'.format(module.__name__))
665
_bootstrap._call_with_frames_removed(exec, code, module.__dict__)
667
def load_module(self, fullname):
668
return _bootstrap._load_module_shim(self, fullname)
671
class SourceLoader(_LoaderBasics):
673
def path_mtime(self, path):
674
"""Optional method that returns the modification time (an int) for the
675
specified path, where path is a str.
677
Raises IOError when the path cannot be handled.
681
def path_stats(self, path):
682
"""Optional method returning a metadata dict for the specified path
683
to by the path (str).
685
- 'mtime' (mandatory) is the numeric timestamp of last source
687
- 'size' (optional) is the size in bytes of the source code.
689
Implementing this method allows the loader to read bytecode files.
690
Raises IOError when the path cannot be handled.
692
return {'mtime': self.path_mtime(path)}
694
def _cache_bytecode(self, source_path, cache_path, data):
695
"""Optional method which writes data (bytes) to a file path (a str).
697
Implementing this method allows for the writing of bytecode files.
699
The source path is needed in order to correctly transfer permissions
701
# For backwards compatibility, we delegate to set_data()
702
return self.set_data(cache_path, data)
704
def set_data(self, path, data):
705
"""Optional method which writes data (bytes) to a file path (a str).
707
Implementing this method allows for the writing of bytecode files.
711
def get_source(self, fullname):
712
"""Concrete implementation of InspectLoader.get_source."""
713
path = self.get_filename(fullname)
715
source_bytes = self.get_data(path)
716
except OSError as exc:
717
raise ImportError('source not available through get_data()',
718
name=fullname) from exc
719
return decode_source(source_bytes)
721
def source_to_code(self, data, path, *, _optimize=-1):
722
"""Return the code object compiled from source.
724
The 'data' argument can be any object type that compile() supports.
726
return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
727
dont_inherit=True, optimize=_optimize)
729
def get_code(self, fullname):
730
"""Concrete implementation of InspectLoader.get_code.
732
Reading of bytecode requires path_stats to be implemented. To write
733
bytecode, set_data must also be implemented.
736
source_path = self.get_filename(fullname)
739
bytecode_path = cache_from_source(source_path)
740
except NotImplementedError:
744
st = self.path_stats(source_path)
748
source_mtime = int(st['mtime'])
750
data = self.get_data(bytecode_path)
755
bytes_data = _validate_bytecode_header(data,
756
source_stats=st, name=fullname,
758
except (ImportError, EOFError):
761
_verbose_message('{} matches {}', bytecode_path,
763
return _compile_bytecode(bytes_data, name=fullname,
764
bytecode_path=bytecode_path,
765
source_path=source_path)
766
source_bytes = self.get_data(source_path)
767
code_object = self.source_to_code(source_bytes, source_path)
768
_verbose_message('code object from {}', source_path)
769
if (not sys.dont_write_bytecode and bytecode_path is not None and
770
source_mtime is not None):
771
data = _code_to_bytecode(code_object, source_mtime,
774
self._cache_bytecode(source_path, bytecode_path, data)
775
_verbose_message('wrote {!r}', bytecode_path)
776
except NotImplementedError:
783
"""Base file loader class which implements the loader protocol methods that
784
require file system usage."""
786
def __init__(self, fullname, path):
787
"""Cache the module name and the path to the file found by the
792
def __eq__(self, other):
793
return (self.__class__ == other.__class__ and
794
self.__dict__ == other.__dict__)
797
return hash(self.name) ^ hash(self.path)
800
def load_module(self, fullname):
801
"""Load a module from a file.
803
This method is deprecated. Use exec_module() instead.
806
# The only reason for this method is for the name check.
807
# Issue #14857: Avoid the zero-argument form of super so the implementation
808
# of that form can be updated without breaking the frozen module
809
return super(FileLoader, self).load_module(fullname)
812
def get_filename(self, fullname):
813
"""Return the path to the source file as found by the finder."""
816
def get_data(self, path):
817
"""Return the data from path as raw bytes."""
818
with _io.FileIO(path, 'r') as file:
822
class SourceFileLoader(FileLoader, SourceLoader):
824
"""Concrete implementation of SourceLoader using the file system."""
826
def path_stats(self, path):
827
"""Return the metadata for the path."""
828
st = _path_stat(path)
829
return {'mtime': st.st_mtime, 'size': st.st_size}
831
def _cache_bytecode(self, source_path, bytecode_path, data):
832
# Adapt between the two APIs
833
mode = _calc_mode(source_path)
834
return self.set_data(bytecode_path, data, _mode=mode)
836
def set_data(self, path, data, *, _mode=0o666):
837
"""Write bytes data to a file."""
838
parent, filename = _path_split(path)
840
# Figure out what directories are missing.
841
while parent and not _path_isdir(parent):
842
parent, part = _path_split(parent)
843
path_parts.append(part)
844
# Create needed directories.
845
for part in reversed(path_parts):
846
parent = _path_join(parent, part)
849
except FileExistsError:
850
# Probably another Python process already created the dir.
852
except OSError as exc:
853
# Could be a permission error, read-only filesystem: just forget
854
# about writing the data.
855
_verbose_message('could not create {!r}: {!r}', parent, exc)
858
_write_atomic(path, data, _mode)
859
_verbose_message('created {!r}', path)
860
except OSError as exc:
861
# Same as above: just don't write the bytecode.
862
_verbose_message('could not create {!r}: {!r}', path, exc)
865
class SourcelessFileLoader(FileLoader, _LoaderBasics):
867
"""Loader which handles sourceless file imports."""
869
def get_code(self, fullname):
870
path = self.get_filename(fullname)
871
data = self.get_data(path)
872
bytes_data = _validate_bytecode_header(data, name=fullname, path=path)
873
return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path)
875
def get_source(self, fullname):
876
"""Return None as there is no source code."""
880
# Filled in by _setup().
881
EXTENSION_SUFFIXES = []
884
class ExtensionFileLoader(FileLoader, _LoaderBasics):
886
"""Loader for extension modules.
888
The constructor is designed to work with FileFinder.
892
def __init__(self, name, path):
896
def __eq__(self, other):
897
return (self.__class__ == other.__class__ and
898
self.__dict__ == other.__dict__)
901
return hash(self.name) ^ hash(self.path)
903
def create_module(self, spec):
904
"""Create an unitialized extension module"""
905
module = _bootstrap._call_with_frames_removed(
906
_imp.create_dynamic, spec)
907
_verbose_message('extension module {!r} loaded from {!r}',
908
spec.name, self.path)
911
def exec_module(self, module):
912
"""Initialize an extension module"""
913
_bootstrap._call_with_frames_removed(_imp.exec_dynamic, module)
914
_verbose_message('extension module {!r} executed from {!r}',
915
self.name, self.path)
917
def is_package(self, fullname):
918
"""Return True if the extension module is a package."""
919
file_name = _path_split(self.path)[1]
920
return any(file_name == '__init__' + suffix
921
for suffix in EXTENSION_SUFFIXES)
923
def get_code(self, fullname):
924
"""Return None as an extension module cannot create a code object."""
927
def get_source(self, fullname):
928
"""Return None as extension modules have no source code."""
932
def get_filename(self, fullname):
933
"""Return the path to the source file as found by the finder."""
937
class _NamespacePath:
938
"""Represents a namespace package's path. It uses the module name
939
to find its parent module, and from there it looks up the parent's
940
__path__. When this changes, the module's own path is recomputed,
941
using path_finder. For top-level modules, the parent module's path
944
def __init__(self, name, path, path_finder):
947
self._last_parent_path = tuple(self._get_parent_path())
948
self._path_finder = path_finder
950
def _find_parent_path_names(self):
951
"""Returns a tuple of (parent-module-name, parent-path-attr-name)"""
952
parent, dot, me = self._name.rpartition('.')
954
# This is a top-level module. sys.path contains the parent path.
956
# Not a top-level module. parent-module.__path__ contains the
958
return parent, '__path__'
960
def _get_parent_path(self):
961
parent_module_name, path_attr_name = self._find_parent_path_names()
962
return getattr(sys.modules[parent_module_name], path_attr_name)
964
def _recalculate(self):
965
# If the parent's path has changed, recalculate _path
966
parent_path = tuple(self._get_parent_path()) # Make a copy
967
if parent_path != self._last_parent_path:
968
spec = self._path_finder(self._name, parent_path)
969
# Note that no changes are made if a loader is returned, but we
970
# do remember the new parent path
971
if spec is not None and spec.loader is None:
972
if spec.submodule_search_locations:
973
self._path = spec.submodule_search_locations
974
self._last_parent_path = parent_path # Save the copy
978
return iter(self._recalculate())
981
return len(self._recalculate())
984
return '_NamespacePath({!r})'.format(self._path)
986
def __contains__(self, item):
987
return item in self._recalculate()
989
def append(self, item):
990
self._path.append(item)
993
# We use this exclusively in module_from_spec() for backward-compatibility.
994
class _NamespaceLoader:
995
def __init__(self, name, path, path_finder):
996
self._path = _NamespacePath(name, path, path_finder)
999
def module_repr(cls, module):
1000
"""Return repr for the module.
1002
The method is deprecated. The import machinery does the job itself.
1005
return '<module {!r} (namespace)>'.format(module.__name__)
1007
def is_package(self, fullname):
1010
def get_source(self, fullname):
1013
def get_code(self, fullname):
1014
return compile('', '<string>', 'exec', dont_inherit=True)
1016
def create_module(self, spec):
1017
"""Use default semantics for module creation."""
1019
def exec_module(self, module):
1022
def load_module(self, fullname):
1023
"""Load a namespace module.
1025
This method is deprecated. Use exec_module() instead.
1028
# The import system never calls this method.
1029
_verbose_message('namespace module loaded with path {!r}', self._path)
1030
return _bootstrap._load_module_shim(self, fullname)
1033
# Finders #####################################################################
1037
"""Meta path finder for sys.path and package __path__ attributes."""
1040
def invalidate_caches(cls):
1041
"""Call the invalidate_caches() method on all path entry finders
1042
stored in sys.path_importer_caches (where implemented)."""
1043
for finder in sys.path_importer_cache.values():
1044
if hasattr(finder, 'invalidate_caches'):
1045
finder.invalidate_caches()
1048
def _path_hooks(cls, path):
1049
"""Search sequence of hooks for a finder for 'path'.
1051
If 'hooks' is false then use sys.path_hooks.
1054
if sys.path_hooks is not None and not sys.path_hooks:
1055
_warnings.warn('sys.path_hooks is empty', ImportWarning)
1056
for hook in sys.path_hooks:
1065
def _path_importer_cache(cls, path):
1066
"""Get the finder for the path entry from sys.path_importer_cache.
1068
If the path entry is not in the cache, find the appropriate finder
1069
and cache it. If no finder is available, store None.
1075
except FileNotFoundError:
1076
# Don't cache the failure as the cwd can easily change to
1077
# a valid directory later on.
1080
finder = sys.path_importer_cache[path]
1082
finder = cls._path_hooks(path)
1083
sys.path_importer_cache[path] = finder
1087
def _legacy_get_spec(cls, fullname, finder):
1088
# This would be a good place for a DeprecationWarning if
1089
# we ended up going that route.
1090
if hasattr(finder, 'find_loader'):
1091
loader, portions = finder.find_loader(fullname)
1093
loader = finder.find_module(fullname)
1095
if loader is not None:
1096
return _bootstrap.spec_from_loader(fullname, loader)
1097
spec = _bootstrap.ModuleSpec(fullname, None)
1098
spec.submodule_search_locations = portions
1102
def _get_spec(cls, fullname, path, target=None):
1103
"""Find the loader or namespace_path for this module/package name."""
1104
# If this ends up being a namespace package, namespace_path is
1105
# the list of paths that will become its __path__
1108
if not isinstance(entry, (str, bytes)):
1110
finder = cls._path_importer_cache(entry)
1111
if finder is not None:
1112
if hasattr(finder, 'find_spec'):
1113
spec = finder.find_spec(fullname, target)
1115
spec = cls._legacy_get_spec(fullname, finder)
1118
if spec.loader is not None:
1120
portions = spec.submodule_search_locations
1121
if portions is None:
1122
raise ImportError('spec missing loader')
1123
# This is possibly part of a namespace package.
1124
# Remember these path entries (if any) for when we
1125
# create a namespace package, and continue iterating
1127
namespace_path.extend(portions)
1129
spec = _bootstrap.ModuleSpec(fullname, None)
1130
spec.submodule_search_locations = namespace_path
1134
def find_spec(cls, fullname, path=None, target=None):
1135
"""find the module on sys.path or 'path' based on sys.path_hooks and
1136
sys.path_importer_cache."""
1139
spec = cls._get_spec(fullname, path, target)
1142
elif spec.loader is None:
1143
namespace_path = spec.submodule_search_locations
1145
# We found at least one namespace path. Return a
1146
# spec which can create the namespace package.
1147
spec.origin = 'namespace'
1148
spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
1156
def find_module(cls, fullname, path=None):
1157
"""find the module on sys.path or 'path' based on sys.path_hooks and
1158
sys.path_importer_cache.
1160
This method is deprecated. Use find_spec() instead.
1163
spec = cls.find_spec(fullname, path)
1171
"""File-based finder.
1173
Interactions with the file system are cached for performance, being
1174
refreshed when the directory the finder is handling has been modified.
1178
def __init__(self, path, *loader_details):
1179
"""Initialize with the path to search on and a variable number of
1180
2-tuples containing the loader and the file suffixes the loader
1183
for loader, suffixes in loader_details:
1184
loaders.extend((suffix, loader) for suffix in suffixes)
1185
self._loaders = loaders
1186
# Base (directory) path
1187
self.path = path or '.'
1188
self._path_mtime = -1
1189
self._path_cache = set()
1190
self._relaxed_path_cache = set()
1192
def invalidate_caches(self):
1193
"""Invalidate the directory mtime."""
1194
self._path_mtime = -1
1196
find_module = _find_module_shim
1198
def find_loader(self, fullname):
1199
"""Try to find a loader for the specified module, or the namespace
1200
package portions. Returns (loader, list-of-portions).
1202
This method is deprecated. Use find_spec() instead.
1205
spec = self.find_spec(fullname)
1208
return spec.loader, spec.submodule_search_locations or []
1210
def _get_spec(self, loader_class, fullname, path, smsl, target):
1211
loader = loader_class(fullname, path)
1212
return spec_from_file_location(fullname, path, loader=loader,
1213
submodule_search_locations=smsl)
1215
def find_spec(self, fullname, target=None):
1216
"""Try to find a spec for the specified module. Returns the
1217
matching spec, or None if not found."""
1218
is_namespace = False
1219
tail_module = fullname.rpartition('.')[2]
1221
mtime = _path_stat(self.path or _os.getcwd()).st_mtime
1224
if mtime != self._path_mtime:
1226
self._path_mtime = mtime
1227
# tail_module keeps the original casing, for __file__ and friends
1229
cache = self._relaxed_path_cache
1230
cache_module = tail_module.lower()
1232
cache = self._path_cache
1233
cache_module = tail_module
1234
# Check if the module is the name of a directory (and thus a package).
1235
if cache_module in cache:
1236
base_path = _path_join(self.path, tail_module)
1237
for suffix, loader_class in self._loaders:
1238
init_filename = '__init__' + suffix
1239
full_path = _path_join(base_path, init_filename)
1240
if _path_isfile(full_path):
1241
return self._get_spec(loader_class, fullname, full_path, [base_path], target)
1243
# If a namespace package, return the path if we don't
1244
# find a module in the next section.
1245
is_namespace = _path_isdir(base_path)
1246
# Check for a file w/ a proper suffix exists.
1247
for suffix, loader_class in self._loaders:
1248
full_path = _path_join(self.path, tail_module + suffix)
1249
_verbose_message('trying {}'.format(full_path), verbosity=2)
1250
if cache_module + suffix in cache:
1251
if _path_isfile(full_path):
1252
return self._get_spec(loader_class, fullname, full_path, None, target)
1254
_verbose_message('possible namespace for {}'.format(base_path))
1255
spec = _bootstrap.ModuleSpec(fullname, None)
1256
spec.submodule_search_locations = [base_path]
1260
def _fill_cache(self):
1261
"""Fill the cache of potential modules and packages for this directory."""
1264
contents = _os.listdir(path or _os.getcwd())
1265
except (FileNotFoundError, PermissionError, NotADirectoryError):
1266
# Directory has either been removed, turned into a file, or made
1269
# We store two cached versions, to handle runtime changes of the
1270
# PYTHONCASEOK environment variable.
1271
if not sys.platform.startswith('win'):
1272
self._path_cache = set(contents)
1274
# Windows users can import modules with case-insensitive file
1275
# suffixes (for legacy reasons). Make the suffix lowercase here
1276
# so it's done once instead of for every import. This is safe as
1277
# the specified suffixes to check against are always specified in a
1278
# case-sensitive manner.
1279
lower_suffix_contents = set()
1280
for item in contents:
1281
name, dot, suffix = item.partition('.')
1283
new_name = '{}.{}'.format(name, suffix.lower())
1286
lower_suffix_contents.add(new_name)
1287
self._path_cache = lower_suffix_contents
1288
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
1289
self._relaxed_path_cache = {fn.lower() for fn in contents}
1292
def path_hook(cls, *loader_details):
1293
"""A class method which returns a closure to use on sys.path_hook
1294
which will return an instance using the specified loaders and the path
1295
called on the closure.
1297
If the path called on the closure is not a directory, ImportError is
1301
def path_hook_for_FileFinder(path):
1302
"""Path hook for importlib.machinery.FileFinder."""
1303
if not _path_isdir(path):
1304
raise ImportError('only directories are supported', path=path)
1305
return cls(path, *loader_details)
1307
return path_hook_for_FileFinder
1310
return 'FileFinder({!r})'.format(self.path)
1313
# Import setup ###############################################################
1315
def _fix_up_module(ns, name, pathname, cpathname=None):
1316
# This function is used by PyImport_ExecCodeModuleObject().
1317
loader = ns.get('__loader__')
1318
spec = ns.get('__spec__')
1321
loader = spec.loader
1322
elif pathname == cpathname:
1323
loader = SourcelessFileLoader(name, pathname)
1325
loader = SourceFileLoader(name, pathname)
1327
spec = spec_from_file_location(name, pathname, loader=loader)
1329
ns['__spec__'] = spec
1330
ns['__loader__'] = loader
1331
ns['__file__'] = pathname
1332
ns['__cached__'] = cpathname
1334
# Not important enough to report.
1338
def _get_supported_file_loaders():
1339
"""Returns a list of file-based module loaders.
1341
Each item is a tuple (loader, suffixes).
1343
extensions = ExtensionFileLoader, _imp.extension_suffixes()
1344
source = SourceFileLoader, SOURCE_SUFFIXES
1345
bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
1346
return [extensions, source, bytecode]
1349
def _setup(_bootstrap_module):
1350
"""Setup the path-based importers for importlib by importing needed
1351
built-in modules and injecting them into the global namespace.
1353
Other components are extracted from the core bootstrap module.
1356
global sys, _imp, _bootstrap
1357
_bootstrap = _bootstrap_module
1358
sys = _bootstrap.sys
1359
_imp = _bootstrap._imp
1361
# Directly load built-in modules needed during bootstrap.
1362
self_module = sys.modules[__name__]
1363
for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
1364
if builtin_name not in sys.modules:
1365
builtin_module = _bootstrap._builtin_from_name(builtin_name)
1367
builtin_module = sys.modules[builtin_name]
1368
setattr(self_module, builtin_name, builtin_module)
1370
# Directly load the os module (needed during bootstrap).
1371
os_details = ('posix', ['/']), ('nt', ['\\', '/'])
1372
for builtin_os, path_separators in os_details:
1373
# Assumption made in _path_join()
1374
assert all(len(sep) == 1 for sep in path_separators)
1375
path_sep = path_separators[0]
1376
if builtin_os in sys.modules:
1377
os_module = sys.modules[builtin_os]
1381
os_module = _bootstrap._builtin_from_name(builtin_os)
1386
raise ImportError('importlib requires posix or nt')
1387
setattr(self_module, '_os', os_module)
1388
setattr(self_module, 'path_sep', path_sep)
1389
setattr(self_module, 'path_separators', ''.join(path_separators))
1391
# Directly load the _thread module (needed during bootstrap).
1393
thread_module = _bootstrap._builtin_from_name('_thread')
1395
# Python was built without threads
1396
thread_module = None
1397
setattr(self_module, '_thread', thread_module)
1399
# Directly load the _weakref module (needed during bootstrap).
1400
weakref_module = _bootstrap._builtin_from_name('_weakref')
1401
setattr(self_module, '_weakref', weakref_module)
1403
# Directly load the winreg module (needed during bootstrap).
1404
if builtin_os == 'nt':
1405
winreg_module = _bootstrap._builtin_from_name('winreg')
1406
setattr(self_module, '_winreg', winreg_module)
1409
setattr(self_module, '_relax_case', _make_relax_case())
1410
EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
1411
if builtin_os == 'nt':
1412
SOURCE_SUFFIXES.append('.pyw')
1413
if '_d.pyd' in EXTENSION_SUFFIXES:
1414
WindowsRegistryFinder.DEBUG_BUILD = True
1417
def _install(_bootstrap_module):
1418
"""Install the path-based import components."""
1419
_setup(_bootstrap_module)
1420
supported_loaders = _get_supported_file_loaders()
1421
sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
1422
if _os.__name__ == 'nt':
1423
sys.meta_path.append(WindowsRegistryFinder)
1424
sys.meta_path.append(PathFinder)
1426
# XXX We expose a couple of classes in _bootstrap for the sake of
1427
# a setuptools bug (https://bitbucket.org/pypa/setuptools/issue/378).
1428
_bootstrap_module.FileFinder = FileFinder
1429
_bootstrap_module.SourceFileLoader = SourceFileLoader
added
removed
.tox/py35/lib/python3.5/importlib/_bootstrap_external.py
