1
# Copyright 2015, Kay Hayen, mailto:kay.hayen@gmail.com
3
# Part of "Nuitka", an optimizing Python compiler that is compatible and
4
# integrates with CPython, but also works on its own.
6
# Licensed under the Apache License, Version 2.0 (the "License");
7
# you may not use this file except in compliance with the License.
8
# You may obtain a copy of the License at
10
# http://www.apache.org/licenses/LICENSE-2.0
12
# Unless required by applicable law or agreed to in writing, software
13
# distributed under the License is distributed on an "AS IS" BASIS,
14
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
18
""" Locating modules and package source on disk.
20
The actual import of a module would already execute code that changes things.
21
Imagine a module that does "os.system()", it will be done. People often connect
22
to databases, and these kind of things, at import time. Not a good style, but
25
Therefore CPython exhibits the interfaces in an "imp" module in standard
26
library, which one can use those to know ahead of time, what file import would
27
load. For us unfortunately there is nothing in CPython that gives the fully
28
compatible functionality we need for packages and search paths exactly like
29
CPython does, so we implement here a multiple step search process that is
32
This approach is much safer of course and there is no loss. To determine if it's
33
from the standard library, we can abuse the attribute "__file__" of the "os"
34
module like it's done in "isStandardLibraryPath" of this module.
38
from __future__ import print_function
43
from logging import warning
45
from . import Utils, oset
47
_debug_module_finding = False
51
# Directory where the main script lives. Should attempt to import from there.
54
def setMainScriptDirectory(main_dir):
55
""" Initialize the main script directory.
57
We use this as part of the search path for modules.
59
# We need to set this from the outside, pylint: disable=W0603
64
def isPackageDir(dirname):
65
""" Decide if a directory is a package.
67
Before Python3.3 it's required to have a "__init__.py" file, but then
68
it became impossible to decide.
71
return Utils.isDir(dirname) and \
73
Utils.python_version >= 330 or
74
Utils.isFile(Utils.joinpath(dirname, "__init__.py"))
77
def findModule(source_ref, module_name, parent_package, level, warn):
78
""" Find a module with given package name as parent.
80
The package name can be None of course. Level is the same
81
as with "__import__" built-in. Warnings are optional.
83
Returns a triple of package name the module is in, module name
84
and filename of it, which can be a directory.
87
# We have many branches here, because there are a lot of cases to try.
88
# pylint: disable=R0912
90
if level > 1 and parent_package is not None:
91
parent_package = '.'.join(parent_package.split('.')[:-level+1])
93
if parent_package == "":
96
if module_name != "" or parent_package is not None:
98
module_filename, module_package_name = _findModule(
99
module_name = module_name,
100
parent_package = parent_package
103
if warn and not _isWhiteListedNotExistingModule(module_name):
104
key = module_name, parent_package, level
106
if key not in warned_about:
107
warned_about.add(key)
110
level_desc = "as absolute import"
112
level_desc = "as relative or absolute import"
114
level_desc = "%d package level up" % level
116
level_desc = "%d package levels up" % level
118
if parent_package is not None:
120
"%s: Cannot find '%s' in package '%s' %s.",
121
source_ref.getAsString(),
128
"%s: Cannot find '%s' %s.",
129
source_ref.getAsString(),
135
if '.' in module_name:
136
module_package_name = module_name[:module_name.rfind('.')]
138
module_package_name = None
140
module_filename = None
142
if '.' in module_name:
143
module_package_name = module_name[:module_name.rfind('.')]
145
module_package_name = None
147
module_filename = None
149
if _debug_module_finding:
151
"findModule: Result",
157
return module_package_name, module_name, module_filename
159
# Some platforms are case insensitive.
160
case_sensitive = not sys.platform.startswith(("win", "cygwin", "darwin"))
162
def _findModuleInPath2(module_name, search_path):
163
""" This is out own module finding low level implementation.
165
Just the full module name and search path are given. This is then
166
tasked to raise ImportError or return a path if it finds it, or
167
None, if it is a built-in.
169
# We have many branches here, because there are a lot of cases to try.
170
# pylint: disable=R0912
172
if imp.is_builtin(module_name):
175
# We may have to decide between package and module, therefore build
176
# a list of candidates.
177
candidates = oset.OrderedSet()
181
for entry in search_path:
182
# Don't try again, just with an entry of different casing or complete
184
if Utils.normcase(entry) in considered:
186
considered.add(Utils.normcase(entry))
188
package_directory = os.path.join(entry, module_name)
190
# First, check for a package with an init file, that would be the
192
if Utils.isDir(package_directory):
193
for suffix in (".py", ".pyc"):
194
package_file_name = "__init__" + suffix
196
file_path = os.path.join(package_directory, package_file_name)
198
if Utils.isFile(file_path):
200
(entry, 1, package_directory)
204
if Utils.python_version >= 330:
206
(entry, 2, package_directory)
209
# Then, check out suffixes of all kinds.
210
for suffix, _mode, _type in imp.get_suffixes():
211
file_path = Utils.joinpath(entry, module_name + suffix)
212
if Utils.isFile(file_path):
214
(entry, 1, file_path)
219
# Ignore lower priority matches, package directories without init.
220
min_prio = min(candidate[1] for candidate in candidates)
225
if candidate[1] == min_prio
228
# On case sensitive systems, no resolution needed.
230
return candidates[0][2]
232
if len(candidates) == 1:
233
# Just one finding, good.
234
return candidates[0][2]
236
for candidate in candidates:
237
dir_listing = os.listdir(candidate[0])
239
for filename in dir_listing:
240
if Utils.joinpath(candidate[0], filename) == candidate[2]:
243
# Please report this, but you may uncomment and have luck.
244
assert False, candidates
246
# If no case matches, just pick the first.
247
return candidates[0][2]
253
def _findModuleInPath(module_name, package_name):
254
# We have many branches here, because there are a lot of cases to try.
255
# pylint: disable=R0912
257
if _debug_module_finding:
258
print("_findModuleInPath: Enter", module_name, "in", package_name)
260
assert main_path is not None
261
extra_paths = [os.getcwd(), main_path]
263
if package_name is not None:
264
# Work around _findModuleInPath2 bug on at least Windows. Won't handle
265
# module name empty in find_module. And thinking of it, how could it
267
if module_name == "":
268
module_name = package_name.split('.')[-1]
269
package_name = '.'.join(package_name.split('.')[:-1])
271
def getPackageDirnames(element):
272
yield Utils.joinpath(element,*package_name.split('.')), False
274
if package_name == "win32com":
275
yield Utils.joinpath(element,"win32comext"), True
278
for element in extra_paths + sys.path:
279
for package_dir, force_package in getPackageDirnames(element):
280
if isPackageDir(package_dir) or force_package:
281
ext_path.append(package_dir)
283
if _debug_module_finding:
284
print("_findModuleInPath: Package, using extended path", ext_path)
287
module_filename = _findModuleInPath2(
288
module_name = module_name,
289
search_path = ext_path
292
if _debug_module_finding:
294
"_findModuleInPath: _findModuleInPath2 worked",
299
return module_filename, package_name
301
if _debug_module_finding:
302
print("_findModuleInPath: _findModuleInPath2 failed to locate")
304
# Warn user, as this is kind of unusual.
306
"%s: Module cannot be imported due to syntax errors.",
312
ext_path = extra_paths + sys.path
314
if _debug_module_finding:
315
print("_findModuleInPath: Non-package, using extended path", ext_path)
318
module_filename = _findModuleInPath2(
319
module_name = module_name,
320
search_path = ext_path
323
# Warn user, as this is kind of unusual.
325
"%s: Module cannot be imported due to syntax errors.",
331
if _debug_module_finding:
332
print("_findModuleInPath: _findModuleInPath2 gave", module_filename)
334
return module_filename, None
337
def _findModule(module_name, parent_package):
338
if _debug_module_finding:
339
print("_findModule: Enter", module_name, "in", parent_package)
341
# The os.path is strangely hacked into the "os" module, dispatching per
342
# platform, we either cannot look into it, or we require that we resolve it
344
if module_name == "os.path" and parent_package is None:
345
parent_package = "os"
347
module_name = Utils.basename(os.path.__file__)
348
if module_name.endswith(".pyc"):
349
module_name = module_name[:-4]
351
assert module_name != "" or parent_package is not None
353
# Built-in module names must not be searched any further.
354
if module_name in sys.builtin_module_names and parent_package is None:
357
if '.' in module_name:
358
package_part = module_name[ : module_name.rfind('.') ]
359
module_name = module_name[ module_name.rfind('.') + 1 : ]
362
if parent_package is not None:
365
module_name = module_name,
366
parent_package = parent_package + '.' + package_part
373
module_name = module_name,
374
parent_package = package_part
377
module_filename, package = _findModuleInPath(
378
module_name = module_name,
379
package_name = parent_package
385
return module_filename, package
387
def getModuleWhiteList():
389
"mac", "nt", "os2", "posix", "_emx_link", "riscos", "ce", "riscospath",
390
"riscosenviron", "Carbon.File", "org.python.core", "_sha", "_sha256",
391
"array", "_sha512", "_md5", "_subprocess", "msvcrt", "cPickle",
392
"marshal", "imp", "sys", "itertools", "cStringIO", "time", "zlib",
393
"thread", "math", "errno", "operator", "signal", "gc", "exceptions",
394
"win32process", "unicodedata", "__builtin__", "fcntl", "_socket",
395
"_ssl", "pwd", "spwd", "_random", "grp", "_io", "_string", "select",
396
"__main__", "_winreg", "_warnings", "_sre", "_functools", "_hashlib",
397
"_collections", "_locale", "_codecs", "_weakref", "_struct",
398
"_dummy_threading", "binascii", "datetime", "_ast", "xxsubtype",
399
"_bytesio", "cmath", "_fileio", "aetypes", "aepack", "MacOS", "cd",
400
"cl", "gdbm", "gl", "GL", "aetools", "_bisect", "_heapq", "_symtable",
401
"syslog", "_datetime", "_elementtree", "_pickle", "_posixsubprocess",
402
"_thread", "atexit", "pyexpat", "_imp", "_sha1", "faulthandler",
404
# Python-Qt4 does these if missing python3 parts:
405
"PyQt4.uic.port_v3.string_io", "PyQt4.uic.port_v3.load_plugin",
406
"PyQt4.uic.port_v3.ascii_upper", "PyQt4.uic.port_v3.proxy_base",
407
"PyQt4.uic.port_v3.as_string",
409
# CPython3 does these:
410
"builtins", "UserDict", "os.path", "StringIO",
412
# test_applesingle.py
416
"__package__.module", "__mangled_mod",
422
"distutils.tests", "distutils.mwerkscompiler",
428
"email.test.test_email", "email.test.test_email_renamed",
429
"email.test.test_email_codecs",
431
# test/test_dbm_ndbm.py
435
"__hello__", "__phello__", "__phello__.spam", "__phello__.foo",
438
"importlib.test.import_", "pep3147.foo", "pep3147",
441
"RAnDoM", "infinite_reload", "test_trailing_slash", "nonexistent_xyzzy",
442
"_parent_foo.bar", "_parent_foo", "test_unc_path",
444
# test_importhooks.py
445
"hooktestmodule", "hooktestpackage", "hooktestpackage.sub",
446
"reloadmodule", "hooktestpackage.sub.subber", "hooktestpackage.oldabs",
447
"hooktestpackage.newrel", "hooktestpackage.sub.subber.subest",
448
"hooktestpackage.futrel", "sub", "hooktestpackage.newabs",
463
"win32evtlog", "win32evtlogutil",
468
# test_namespace_pkgs.py
469
"foo.one", "foo.two", "parent.child.one", "parent.child.two",
470
"parent.child.three", "bar.two", "a_test",
481
"t1", "t2", "t2.sub", "t2.sub.subsub", "t3.sub.subsub", "t5", "t6",
482
"t7", "t7.sub", "t7.sub.subsub", "t8",
488
"""areallylongpackageandmodulenametotestreprtruncation.\
489
areallylongpackageandmodulenametotestreprtruncation""",
491
# test_robotparser.py
495
"test.script_helper",
504
"distutils.emxccompiler", "os2emxpath",
510
"analyze_dxp", "test_unparse",
516
"xml.parsers.expat.errors",
518
# test_zipimport_support.py
519
"test_zipped_doctest", "zip_pkg",
521
# test/test_zipimport_support.py
522
"test.test_cmd_line_script",
524
# Python3: modules that no longer exist
525
"commands", "dummy_thread", "_dummy_thread", "httplib", "Queue", "sets",
527
# Python2: modules that don't yet exit
528
"http.client", "queue", "winreg",
530
# Very old modules with older names
531
"simplejson", "sets",
533
# Standalone mode "site" import flexibilities
534
"sitecustomize", "usercustomize", "apport_python_hook",
537
# Standard library stuff that is optional
538
"comtypes.server.inprocserver", "_tkinter", "_scproxy", "EasyDialogs",
539
"SOCKS", "rourl2path", "_winapi", "win32api", "win32con", "_gestalt",
540
"java.lang", "vms_lib", "ic", "readline", "termios", "_sysconfigdata",
541
"al", "AL", "sunaudiodev", "SUNAUDIODEV", "Audio_mac", "nis",
542
"test.test_MimeWriter", "dos", "win32pipe", "Carbon", "Carbon.Files",
543
"sgi", "ctypes.macholib.dyld", "bsddb3", "_pybsddb", "_xmlrpclib",
544
"netbios", "win32wnet", "email.Parser", "elementree.cElementTree",
545
"elementree.ElementTree", "_gbdm", "resource", "crypt", "bz2", "dbm",
552
"statprof", "email.Generator", "email.Utils",
555
def _isWhiteListedNotExistingModule(module_name):
556
result = module_name in getModuleWhiteList()
558
if not result and module_name in sys.builtin_module_names:
560
Your CPython version has a built-in module '%s', that is not whitelisted
561
please report this to http://bugs.nuitka.net.""", module_name)
566
def getStandardLibraryPaths():
567
""" Get the standard library paths.
571
# Using the function object to cache its result, avoiding global variable
573
if not hasattr(getStandardLibraryPaths, "result"):
574
os_filename = os.__file__
575
if os_filename.endswith(".pyc"):
576
os_filename = os_filename[:-1]
578
os_path = Utils.normcase(Utils.dirname(os_filename))
580
stdlib_paths = set([os_path])
582
# Happens for virtualenv situation, some modules will come from the link
584
if Utils.isLink(os_filename):
585
os_filename = Utils.readLink(os_filename)
586
stdlib_paths.add(Utils.normcase(Utils.dirname(os_filename)))
588
# Another possibility is "orig-prefix.txt" file near the os.py, which
589
# points to the original install.
590
orig_prefix_filename = Utils.joinpath(os_path, "orig-prefix.txt")
592
if Utils.isFile(orig_prefix_filename):
593
# Scan upwards, until we find a "bin" folder, with "activate" to
594
# locate the structural path to be added. We do not know for sure
595
# if there is a sub-directory under "lib" to use or not. So we try
600
while os.path.splitdrive(search)[1] not in (os.path.sep, ""):
601
if Utils.isFile(Utils.joinpath(search,"bin/activate")) or \
602
Utils.isFile(Utils.joinpath(search,"scripts/activate")):
605
lib_part = Utils.joinpath(Utils.basename(search), lib_part)
607
search = Utils.dirname(search)
609
assert search and lib_part
614
open(orig_prefix_filename).read(),
620
# And yet another possibility, for MacOS Homebrew created virtualenv
621
# at least is a link ".Python", which points to the original install.
622
python_link_filename = Utils.joinpath(os_path, "..", ".Python")
623
if Utils.isLink(python_link_filename):
627
Utils.readLink(python_link_filename),
633
getStandardLibraryPaths.result = [
634
Utils.normcase(stdlib_path)
639
return getStandardLibraryPaths.result
642
def isStandardLibraryPath(path):
643
""" Check if a path is in the standard library.
647
path = Utils.normcase(path)
649
# In virtualenv, the "site.py" lives in a place that suggests it is not in
650
# standard library, although it is.
651
if os.path.basename(path) == "site.py":
654
# These never are in standard library paths.
655
if "dist-packages" in path or "site-packages" in path:
658
for candidate in getStandardLibraryPaths():
659
if path.startswith(candidate):
added
removed
nuitka/Importing.py
