1
from email.parser import FeedParser
11
from distutils.util import change_root
8
12
from pip.locations import bin_py, running_under_virtualenv
9
13
from pip.exceptions import (InstallationError, UninstallationError,
10
BestVersionAlreadyInstalled)
14
BestVersionAlreadyInstalled,
11
16
from pip.vcs import vcs
12
17
from pip.log import logger
13
from pip.util import display_path, rmtree
14
from pip.util import ask, ask_path_exists, backup_dir
15
from pip.util import is_installable_dir, is_local, dist_is_local
16
from pip.util import renames, normalize_path, egg_link_path
17
from pip.util import make_path_relative
18
from pip import call_subprocess
19
from pip.backwardcompat import (any, copytree, urlparse, urllib,
18
from pip.util import (display_path, rmtree, ask, ask_path_exists, backup_dir,
19
is_installable_dir, is_local, dist_is_local,
20
dist_in_usersite, dist_in_site_packages, renames,
21
normalize_path, egg_link_path, make_path_relative,
23
from pip.backwardcompat import (urlparse, urllib, uses_pycache,
20
24
ConfigParser, string_types, HTTPError,
21
FeedParser, get_python_version,
25
get_python_version, b)
23
26
from pip.index import Link
24
27
from pip.locations import build_prefix
25
28
from pip.download import (get_file_content, is_url, url_to_path,
34
37
class InstallRequirement(object):
36
39
def __init__(self, req, comes_from, source_dir=None, editable=False,
37
url=None, update=True):
40
url=None, as_egg=False, update=True):
39
42
if isinstance(req, string_types):
40
43
req = pkg_resources.Requirement.parse(req)
59
63
self.install_succeeded = None
60
64
# UninstallPathSet of uninstalled distribution (for possible rollback)
61
65
self.uninstalled = None
66
self.use_user_site = False
64
69
def from_editable(cls, editable_req, comes_from=None, default_vcs=None):
65
name, url = parse_editable(editable_req, default_vcs)
70
name, url, extras_override = parse_editable(editable_req, default_vcs)
66
71
if url.startswith('file:'):
67
72
source_dir = url_to_path(url)
70
return cls(name, comes_from, source_dir=source_dir, editable=True, url=url)
76
res = cls(name, comes_from, source_dir=source_dir, editable=True, url=url)
78
if extras_override is not None:
79
res.extras = extras_override
73
84
def from_line(cls, name, comes_from=None):
85
96
elif os.path.isdir(path) and (os.path.sep in name or name.startswith('.')):
86
97
if not is_installable_dir(path):
87
raise InstallationError("Directory %r is not installable. File 'setup.py' not found.", name)
98
raise InstallationError("Directory %r is not installable. File 'setup.py' not found." % name)
88
99
link = Link(path_to_url(name))
89
100
elif is_archive_file(path):
90
101
if not os.path.isfile(path):
94
105
# If the line has an egg= definition, but isn't editable, pull the requirement out.
95
106
# Otherwise, assume the name is the req for the non URL/path/archive case.
96
107
if link and req is None:
97
url = link.url_fragment
98
req = link.egg_fragment
108
url = link.url_without_fragment
109
req = link.egg_fragment #when fragment is None, this will become an 'unnamed' requirement
100
111
# Handle relative file URLs
101
112
if link.scheme == 'file' and re.search(r'\.\./', url):
235
246
_run_setup_py = """
236
247
__file__ = __SETUP_PY__
237
248
from setuptools.command import egg_info
238
251
def replacement_run(self):
239
252
self.mkpath(self.egg_info)
240
253
installer = self.distribution.fetch_build_egg
241
for ep in egg_info.iter_entry_points('egg_info.writers'):
254
for ep in pkg_resources.iter_entry_points('egg_info.writers'):
242
255
# require=False is the change we're making:
243
256
writer = ep.load(require=False)
245
writer(self, ep.name, egg_info.os.path.join(self.egg_info,ep.name))
258
writer(self, ep.name, os.path.join(self.egg_info,ep.name))
246
259
self.find_sources()
247
260
egg_info.egg_info.run = replacement_run
248
261
exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
275
288
for dir in vcs.dirnames:
291
# Iterate over a copy of ``dirs``, since mutating
292
# a list while iterating over it can cause trouble.
293
# (See https://github.com/pypa/pip/pull/462.)
294
for dir in list(dirs):
279
295
# Don't search in anything that looks like a virtualenv environment
280
296
if (os.path.exists(os.path.join(root, dir, 'bin', 'python'))
281
297
or os.path.exists(os.path.join(root, dir, 'Scripts', 'Python.exe'))):
288
304
filenames = [f for f in filenames if f.endswith('.egg-info')]
290
306
if not filenames:
291
raise InstallationError('No files/directores in %s (from %s)' % (base, filename))
307
raise InstallationError('No files/directories in %s (from %s)' % (base, filename))
292
308
assert filenames, "No files/directories in %s (from %s)" % (base, filename)
294
310
# if we have more than one match, we pick the toplevel one. This can
353
369
def assert_source_matches_version(self):
354
370
assert self.source_dir
355
if self.comes_from is None:
356
# We don't check the versions of things explicitly installed.
357
# This makes, e.g., "pip Package==dev" possible
359
371
version = self.installed_version
360
372
if version not in self.req:
362
'Source in %s has the version %s, which does not match the requirement %s'
363
% (display_path(self.source_dir), version, self))
364
raise InstallationError(
365
'Source in %s has version %s that conflicts with %s'
366
% (display_path(self.source_dir), version, self))
373
logger.warn('Requested %s, but installing version %s' % (self, self.installed_version))
368
375
logger.debug('Source in %s has version %s, which satisfies requirement %s'
369
376
% (display_path(self.source_dir), version, self))
433
440
for installed_file in dist.get_metadata('installed-files.txt').splitlines():
434
441
path = os.path.normpath(os.path.join(egg_info_path, installed_file))
435
442
paths_to_remove.add(path)
436
if dist.has_metadata('top_level.txt'):
443
#FIXME: need a test for this elif block
444
#occurs with --single-version-externally-managed/--record outside of pip
445
elif dist.has_metadata('top_level.txt'):
437
446
if dist.has_metadata('namespace_packages.txt'):
438
447
namespaces = dist.get_metadata('namespace_packages.txt')
453
462
'easy-install.pth')
454
463
paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg)
456
elif os.path.isfile(develop_egg_link):
465
elif develop_egg_link:
458
467
fh = open(develop_egg_link, 'r')
459
468
link_pointer = os.path.normcase(fh.readline().strip())
548
557
name = name.replace(os.path.sep, '/')
551
def install(self, install_options, global_options=()):
560
def install(self, install_options, global_options=(), root=None):
552
561
if self.editable:
553
562
self.install_editable(install_options, global_options)
555
565
temp_location = tempfile.mkdtemp('-record', 'pip-')
556
566
record_filename = os.path.join(temp_location, 'install-record.txt')
561
571
"exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py] +\
562
572
list(global_options) + [
564
'--single-version-externally-managed',
565
574
'--record', record_filename]
577
install_args += ['--single-version-externally-managed']
580
install_args += ['--root', root]
567
582
if running_under_virtualenv():
568
583
## FIXME: I'm not sure if this is a reasonable location; probably not
569
584
## but we can't put it in the default location, as that is a virtualenv symlink that isn't writable
581
596
logger.notify('Record file %s not found' % record_filename)
583
598
self.install_succeeded = True
600
# there's no --always-unzip option we can pass to install command
601
# so we unable to save the installed-files.txt
604
def prepend_root(path):
605
if root is None or not os.path.isabs(path):
608
return change_root(root, path)
584
610
f = open(record_filename)
586
612
line = line.strip()
587
613
if line.endswith('.egg-info'):
614
egg_info_dir = prepend_root(line)
591
617
logger.warn('Could not find .egg-info directory in install record for %s' % self)
599
625
filename = line.strip()
600
626
if os.path.isdir(filename):
601
627
filename += os.path.sep
602
new_lines.append(make_path_relative(filename, egg_info_dir))
628
new_lines.append(make_path_relative(prepend_root(filename), egg_info_dir))
604
630
f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w')
605
631
f.write('\n'.join(new_lines)+'\n')
660
687
except pkg_resources.DistributionNotFound:
662
689
except pkg_resources.VersionConflict:
663
self.conflicts_with = pkg_resources.get_distribution(self.req.project_name)
690
existing_dist = pkg_resources.get_distribution(self.req.project_name)
691
if self.use_user_site:
692
if dist_in_usersite(existing_dist):
693
self.conflicts_with = existing_dist
694
elif running_under_virtualenv() and dist_in_site_packages(existing_dist):
695
raise InstallationError("Will not install to the user site because it will lack sys.path precedence to %s in %s"
696
%(existing_dist.project_name, existing_dist.location))
698
self.conflicts_with = existing_dist
780
812
class RequirementSet(object):
782
814
def __init__(self, build_dir, src_dir, download_dir, download_cache=None,
783
upgrade=False, ignore_installed=False,
784
ignore_dependencies=False, force_reinstall=False):
815
upgrade=False, ignore_installed=False, as_egg=False,
816
ignore_dependencies=False, force_reinstall=False, use_user_site=False):
785
817
self.build_dir = build_dir
786
818
self.src_dir = src_dir
787
819
self.download_dir = download_dir
807
841
def add_requirement(self, install_req):
808
842
name = install_req.name
843
install_req.as_egg = self.as_egg
844
install_req.use_user_site = self.use_user_site
846
#url or path requirement w/o an egg fragment
810
847
self.unnamed_requirements.append(install_req)
812
849
if self.has_requirement(name):
813
850
raise InstallationError(
814
'Double requirement given: %s (aready in %s, name=%r)'
851
'Double requirement given: %s (already in %s, name=%r)'
815
852
% (install_req, self.get_requirement(name), name))
816
853
self.requirements[name] = install_req
817
854
## FIXME: what about other normalizations? E.g., _ vs. -?
863
900
req.commit_uninstall()
865
902
def locate_files(self):
866
## FIXME: duplicates code from install_files; relevant code should
903
## FIXME: duplicates code from prepare_files; relevant code should
867
904
## probably be factored out into a separate method
868
905
unnamed = list(self.unnamed_requirements)
869
906
reqs = list(self.requirements.values())
877
914
req_to_install.check_if_exists()
878
915
if req_to_install.satisfied_by:
880
req_to_install.conflicts_with = req_to_install.satisfied_by
917
#don't uninstall conflict if user install and and conflict is not user install
918
if not (self.use_user_site and not dist_in_usersite(req_to_install.satisfied_by)):
919
req_to_install.conflicts_with = req_to_install.satisfied_by
881
920
req_to_install.satisfied_by = None
883
922
install_needed = False
910
949
req_to_install = reqs.pop(0)
912
951
best_installed = False
913
953
if not self.ignore_installed and not req_to_install.editable:
914
954
req_to_install.check_if_exists()
915
955
if req_to_install.satisfied_by:
917
if not self.force_reinstall:
957
if not self.force_reinstall and not req_to_install.url:
919
959
url = finder.find_requirement(
920
960
req_to_install, self.upgrade)
921
961
except BestVersionAlreadyInstalled:
922
962
best_installed = True
964
except DistributionNotFound:
965
not_found = sys.exc_info()[1]
925
967
# Avoid the need to call find_requirement again
926
968
req_to_install.url = url.url
928
970
if not best_installed:
929
req_to_install.conflicts_with = req_to_install.satisfied_by
971
#don't uninstall conflict if user install and conflict is not user install
972
if not (self.use_user_site and not dist_in_usersite(req_to_install.satisfied_by)):
973
req_to_install.conflicts_with = req_to_install.satisfied_by
930
974
req_to_install.satisfied_by = None
968
1012
##occurs when the script attempts to unpack the
969
1013
##build directory
1015
# NB: This call can result in the creation of a temporary build directory
971
1016
location = req_to_install.build_location(self.build_dir, not self.is_download)
972
1018
## FIXME: is the existance of the checkout good enough to use it? I don't think so.
975
1021
if not os.path.exists(os.path.join(location, 'setup.py')):
976
1022
## FIXME: this won't upgrade when there's an existing package unpacked in `location`
977
1023
if req_to_install.url is None:
978
1026
url = finder.find_requirement(req_to_install, upgrade=self.upgrade)
980
1028
## FIXME: should req_to_install.url already be a link?
1021
1069
req_to_install.check_if_exists()
1022
1070
if req_to_install.satisfied_by:
1023
1071
if self.upgrade or self.ignore_installed:
1024
req_to_install.conflicts_with = req_to_install.satisfied_by
1072
#don't uninstall conflict if user install and and conflict is not user install
1073
if not (self.use_user_site and not dist_in_usersite(req_to_install.satisfied_by)):
1074
req_to_install.conflicts_with = req_to_install.satisfied_by
1025
1075
req_to_install.satisfied_by = None
1027
1077
install = False
1045
1095
subreq = InstallRequirement(req, req_to_install)
1046
1096
reqs.append(subreq)
1047
1097
self.add_requirement(subreq)
1048
if req_to_install.name not in self.requirements:
1049
self.requirements[req_to_install.name] = req_to_install
1050
if self.is_download:
1098
if not self.has_requirement(req_to_install.name):
1099
#'unnamed' requirements will get added here
1100
self.add_requirement(req_to_install)
1101
if self.is_download or req_to_install._temp_build_dir is not None:
1051
1102
self.reqs_to_cleanup.append(req_to_install)
1053
1104
self.reqs_to_cleanup.append(req_to_install)
1090
1141
target_dir = req_to_install.editable and self.src_dir or self.build_dir
1091
1142
logger.info("Copying %s to %s" % (req_to_install.name, target_dir))
1092
1143
dest = os.path.join(target_dir, req_to_install.name)
1093
copytree(req_to_install.source_dir, dest)
1144
shutil.copytree(req_to_install.source_dir, dest)
1094
1145
call_subprocess(["python", "%s/setup.py" % dest, "clean"], cwd=dest,
1095
1146
command_desc='python setup.py clean')
1102
1153
if is_vcs_url(link):
1103
1154
return unpack_vcs_link(link, loc, only_download)
1104
elif is_file_url(link):
1155
# a local file:// index could have links with hashes
1156
elif not link.hash and is_file_url(link):
1105
1157
return unpack_file_url(link, loc)
1107
1159
if self.download_cache:
1111
1163
_write_delete_marker_message(os.path.join(location, PIP_DELETE_MARKER_FILENAME))
1114
def install(self, install_options, global_options=()):
1166
def install(self, install_options, global_options=(), *args, **kwargs):
1115
1167
"""Install everything in this set (after having downloaded and unpacked the packages)"""
1116
1168
to_install = [r for r in self.requirements.values()
1117
1169
if not r.satisfied_by]
1131
1183
logger.indent -= 2
1133
requirement.install(install_options, global_options)
1185
requirement.install(install_options, global_options, *args, **kwargs)
1135
1187
# if install did not succeed, rollback previous uninstall
1136
1188
if requirement.conflicts_with and not requirement.install_succeeded:
1238
1290
def parse_requirements(filename, finder=None, comes_from=None, options=None):
1239
1291
skip_match = None
1240
skip_regex = options.skip_requirements_regex
1292
skip_regex = options.skip_requirements_regex if options else None
1242
1294
skip_match = re.compile(skip_regex)
1295
reqs_file_dir = os.path.dirname(os.path.abspath(filename))
1243
1296
filename, content = get_file_content(filename, comes_from=comes_from)
1244
1297
for line_number, line in enumerate(content.splitlines()):
1245
1298
line_number += 1
1271
1324
line = line[len('--find-links'):].strip().lstrip('=')
1272
1325
## FIXME: it would be nice to keep track of the source of
1273
1326
## the find_links:
1327
# support a find-links local path relative to a requirements file
1328
relative_to_reqs_file = os.path.join(reqs_file_dir, line)
1329
if os.path.exists(relative_to_reqs_file):
1330
line = relative_to_reqs_file
1275
1332
finder.find_links.append(line)
1276
1333
elif line.startswith('-i') or line.startswith('--index-url'):
1284
1341
line = line[len('--extra-index-url'):].strip().lstrip('=')
1286
1343
finder.index_urls.append(line)
1344
elif line.startswith('--no-index'):
1345
finder.index_urls = []
1288
1347
comes_from = '-r %s (line %s)' % (filename, line_number)
1289
1348
if line.startswith('-e') or line.startswith('--editable'):
1301
1360
def parse_editable(editable_req, default_vcs=None):
1302
1361
"""Parses svn+http://blahblah@rev#egg=Foobar into a requirement
1303
1362
(Foobar) and a URL"""
1304
1364
url = editable_req
1305
if os.path.isdir(url) and os.path.exists(os.path.join(url, 'setup.py')):
1367
# If a file path is specified with extras, strip off the extras.
1368
m = re.match(r'^(.+)(\[[^\]]+\])$', url)
1370
url_no_extras = m.group(1)
1375
if os.path.isdir(url_no_extras):
1376
if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
1377
raise InstallationError("Directory %r is not installable. File 'setup.py' not found." % url_no_extras)
1306
1378
# Treating it as code that has already been checked out
1307
url = path_to_url(url)
1308
if url.lower().startswith('file:'):
1379
url_no_extras = path_to_url(url_no_extras)
1381
if url_no_extras.lower().startswith('file:'):
1383
return None, url_no_extras, pkg_resources.Requirement.parse('__placeholder__' + extras).extras
1385
return None, url_no_extras, None
1310
1387
for version_control in vcs:
1311
1388
if url.lower().startswith('%s:' % version_control):
1312
1389
url = '%s+%s' % (version_control, url)
1315
1392
url = default_vcs + '+' + url
1317
1394
raise InstallationError(
1318
'--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req)
1395
'%s should either by a path to a local project or a VCS url beginning with svn+, git+, hg+, or bzr+' % editable_req)
1319
1396
vc_type = url.split('+', 1)[0].lower()
1320
1397
if not vcs.get_backend(vc_type):
1321
raise InstallationError(
1322
'For --editable=%s only svn (svn+URL), Git (git+URL), Mercurial (hg+URL) and Bazaar (bzr+URL) is currently supported' % editable_req)
1398
error_message = 'For --editable=%s only ' % editable_req + \
1399
', '.join([backend.name + '+URL' for backend in vcs.backends]) + \
1400
' is currently supported'
1401
raise InstallationError(error_message)
1323
1402
match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req)
1324
1403
if (not match or not match.group(1)) and vcs.get_backend(vc_type):
1325
1404
parts = [p for p in editable_req.split('#', 1)[0].split('/') if p]
1377
1456
self._refuse.add(path)
1458
# __pycache__ files can show up after 'installed-files.txt' is created, due to imports
1459
if os.path.splitext(path)[1] == '.py' and uses_pycache:
1460
self.add(imp.cache_from_source(path))
1379
1463
def add_pth(self, pth_file, entry):
1380
1464
pth_file = normalize_path(pth_file)
1381
1465
if self._permitted(pth_file):
1407
1491
``auto_confirm`` is True)."""
1408
1492
if not self._can_uninstall():
1495
logger.notify("Can't uninstall '%s'. No files were found to uninstall." % self.dist.project_name)
1410
1497
logger.notify('Uninstalling %s:' % self.dist.project_name)
1411
1498
logger.indent += 2
1412
1499
paths = sorted(self.compact(self.paths))