« back to all changes in this revision
Viewing changes to pip.py
- browse files at revision 1
- compare with another revision
- download diff
- download tarball
- view history from revision 1
- Committer: Bazaar Package Importer
- Author(s): Jeff Licquia
- Date: 2009-04-21 21:10:13 UTC
- Revision ID: james.westby@ubuntu.com-20090421211013-gw0kzfhbrkc4nhev
Tags: upstream-0.3.1
Import upstream version 0.3.1
- files added:
- docs
- docs/_build
- docs/_build/_sources
- pip.egg-info
- scripts
added
removed
pip.py
1
#!/usr/bin/env python
2
import sys
3
import os
4
import optparse
5
import pkg_resources
6
import urllib2
7
import urllib
8
import mimetypes
9
import zipfile
10
import tarfile
11
import tempfile
12
import subprocess
13
import posixpath
14
import re
15
import shutil
16
import fnmatch
17
try:
18
from hashlib import md5
19
except ImportError:
20
import md5 as md5_module
21
md5 = md5_module.new
22
import urlparse
23
from email.FeedParser import FeedParser
24
import traceback
25
from cStringIO import StringIO
26
import socket
27
from Queue import Queue
28
from Queue import Empty as QueueEmpty
29
import threading
30
import httplib
31
import time
32
import logging
33
import ConfigParser
34
35
class InstallationError(Exception):
36
"""General exception during installation"""
37
38
class DistributionNotFound(InstallationError):
39
"""Raised when a distribution cannot be found to satisfy a requirement"""
40
41
if getattr(sys, 'real_prefix', None):
42
## FIXME: is build/ a good name?
43
base_prefix = os.path.join(sys.prefix, 'build')
44
base_src_prefix = os.path.join(sys.prefix, 'src')
45
else:
46
## FIXME: this isn't a very good default
47
base_prefix = os.path.join(os.getcwd(), 'build')
48
base_src_prefix = os.path.join(os.getcwd(), 'src')
49
50
pypi_url = "http://pypi.python.org/simple"
51
52
default_timeout = 15
53
54
# Choose a Git command based on platform.
55
if sys.platform == 'win32':
56
GIT_CMD = 'git.cmd'
57
else:
58
GIT_CMD = 'git'
59
60
## FIXME: this shouldn't be a module setting
61
default_vcs = None
62
if os.environ.get('PIP_DEFAULT_VCS'):
63
default_vcs = os.environ['PIP_DEFAULT_VCS']
64
65
66
try:
67
pip_dist = pkg_resources.get_distribution('pip')
68
version = '%s from %s (python %s)' % (
69
pip_dist, pip_dist.location, sys.version[:3])
70
except pkg_resources.DistributionNotFound:
71
# when running pip.py without installing
72
version=None
73
74
75
class VcsSupport(object):
76
_registry = {}
77
# Register more schemes with urlparse for the versio control support
78
schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp']
79
80
def __init__(self):
81
urlparse.uses_netloc.extend(self.schemes)
82
urlparse.uses_fragment.extend(self.schemes)
83
super(VcsSupport, self).__init__()
84
85
def __iter__(self):
86
return self._registry.__iter__()
87
88
@property
89
def backends(self):
90
return self._registry.values()
91
92
@property
93
def dirnames(self):
94
return [backend.dirname for backend in self.backends]
95
96
def register(self, cls):
97
if not hasattr(cls, 'name'):
98
logger.warn('Cannot register VCS %s' % cls.__name__)
99
return
100
if cls.name not in self._registry:
101
self._registry[cls.name] = cls
102
103
def unregister(self, cls=None, name=None):
104
if name in self._registry:
105
del self._registry[name]
106
elif cls in self._registry.values():
107
del self._registry[cls.name]
108
else:
109
logger.warn('Cannot unregister because no class or name given')
110
111
def get_backend_name(self, location):
112
"""
113
Return the name of the version control backend if found at given
114
location, e.g. vcs.get_backend_name('/path/to/vcs/checkout')
115
"""
116
for vc_type in self._registry:
117
path = os.path.join(location, '.%s' % vc_type)
118
if os.path.exists(path):
119
return vc_type
120
return None
121
122
def get_backend(self, name):
123
if name in self._registry:
124
return self._registry[name]
125
126
def get_backend_from_location(self, location):
127
vc_type = self.get_backend_name(location)
128
if vc_type:
129
return self.get_backend(vc_type)
130
return None
131
132
133
vcs = VcsSupport()
134
135
parser = optparse.OptionParser(
136
usage='%prog COMMAND [OPTIONS]',
137
version=version,
138
add_help_option=False)
139
140
parser.add_option(
141
'-h', '--help',
142
dest='help',
143
action='store_true',
144
help='Show help')
145
parser.add_option(
146
'-E', '--environment',
147
dest='venv',
148
metavar='DIR',
149
help='virtualenv environment to run pip in (either give the '
150
'interpreter or the environment base directory)')
151
parser.add_option(
152
'-v', '--verbose',
153
dest='verbose',
154
action='count',
155
default=0,
156
help='Give more output')
157
parser.add_option(
158
'-q', '--quiet',
159
dest='quiet',
160
action='count',
161
default=0,
162
help='Give less output')
163
parser.add_option(
164
'--log',
165
dest='log',
166
metavar='FILENAME',
167
help='Log file where a complete (maximum verbosity) record will be kept')
168
parser.add_option(
169
'--proxy',
170
dest='proxy',
171
type='str',
172
default='',
173
help="Specify a proxy in the form user:passwd@proxy.server:port. "
174
"Note that the user:password@ is optional and required only if you "
175
"are behind an authenticated proxy. If you provide "
176
"user@proxy.server:port then you will be prompted for a password."
177
)
178
parser.add_option(
179
'--timeout',
180
metavar='SECONDS',
181
dest='timeout',
182
type='float',
183
default=default_timeout,
184
help='Set the socket timeout (default %s seconds)' % default_timeout)
185
186
parser.disable_interspersed_args()
187
188
189
_commands = {}
190
191
class Command(object):
192
name = None
193
usage = None
194
def __init__(self):
195
assert self.name
196
self.parser = optparse.OptionParser(
197
usage=self.usage,
198
prog='%s %s' % (sys.argv[0], self.name),
199
version=parser.version)
200
for option in parser.option_list:
201
if not option.dest or option.dest == 'help':
202
# -h, --version, etc
203
continue
204
self.parser.add_option(option)
205
_commands[self.name] = self
206
207
def merge_options(self, initial_options, options):
208
for attr in ['log', 'venv', 'proxy']:
209
setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr))
210
options.quiet += initial_options.quiet
211
options.verbose += initial_options.verbose
212
213
def main(self, complete_args, args, initial_options):
214
global logger
215
options, args = self.parser.parse_args(args)
216
self.merge_options(initial_options, options)
217
218
if args and args[-1] == '___VENV_RESTART___':
219
## FIXME: We don't do anything this this value yet:
220
venv_location = args[-2]
221
args = args[:-2]
222
options.venv = None
223
level = 1 # Notify
224
level += options.verbose
225
level -= options.quiet
226
level = Logger.level_for_integer(4-level)
227
complete_log = []
228
logger = Logger([(level, sys.stdout),
229
(Logger.DEBUG, complete_log.append)])
230
if os.environ.get('PIP_LOG_EXPLICIT_LEVELS'):
231
logger.explicit_levels = True
232
if options.venv:
233
if options.verbose > 0:
234
# The logger isn't setup yet
235
print 'Running in environment %s' % options.venv
236
restart_in_venv(options.venv, complete_args)
237
# restart_in_venv should actually never return, but for clarity...
238
return
239
## FIXME: not sure if this sure come before or after venv restart
240
if options.log:
241
log_fp = open_logfile_append(options.log)
242
logger.consumers.append((logger.DEBUG, log_fp))
243
else:
244
log_fp = None
245
246
socket.setdefaulttimeout(options.timeout or None)
247
248
setup_proxy_handler(options.proxy)
249
250
exit = 0
251
try:
252
self.run(options, args)
253
except InstallationError, e:
254
logger.fatal(str(e))
255
logger.info('Exception information:\n%s' % format_exc())
256
exit = 1
257
except:
258
logger.fatal('Exception:\n%s' % format_exc())
259
exit = 2
260
261
if log_fp is not None:
262
log_fp.close()
263
if exit:
264
log_fn = './pip-log.txt'
265
text = '\n'.join(complete_log)
266
logger.fatal('Storing complete log in %s' % log_fn)
267
log_fp = open_logfile_append(log_fn)
268
log_fp.write(text)
269
log_fp.close()
270
return exit
271
272
class HelpCommand(Command):
273
name = 'help'
274
usage = '%prog'
275
summary = 'Show available commands'
276
277
def run(self, options, args):
278
if args:
279
## FIXME: handle errors better here
280
command = args[0]
281
if command not in _commands:
282
raise InstallationError('No command with the name: %s' % command)
283
command = _commands[command]
284
command.parser.print_help()
285
return
286
parser.print_help()
287
print
288
print 'Commands available:'
289
commands = list(set(_commands.values()))
290
commands.sort(key=lambda x: x.name)
291
for command in commands:
292
print ' %s: %s' % (command.name, command.summary)
293
294
HelpCommand()
295
296
class InstallCommand(Command):
297
name = 'install'
298
usage = '%prog [OPTIONS] PACKAGE_NAMES...'
299
summary = 'Install packages'
300
bundle = False
301
302
def __init__(self):
303
super(InstallCommand, self).__init__()
304
self.parser.add_option(
305
'-e', '--editable',
306
dest='editables',
307
action='append',
308
default=[],
309
metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE',
310
help='Install a package directly from a checkout. Source will be checked '
311
'out into src/PACKAGE (lower-case) and installed in-place (using '
312
'setup.py develop). You can run this on an existing directory/checkout (like '
313
'pip install -e src/mycheckout). This option may be provided multiple times. '
314
'Possible values for VCS are: svn, git, hg and bzr.')
315
self.parser.add_option(
316
'-r', '--requirement',
317
dest='requirements',
318
action='append',
319
default=[],
320
metavar='FILENAME',
321
help='Install all the packages listed in the given requirements file. '
322
'This option can be used multiple times.')
323
self.parser.add_option(
324
'-f', '--find-links',
325
dest='find_links',
326
action='append',
327
default=[],
328
metavar='URL',
329
help='URL to look for packages at')
330
self.parser.add_option(
331
'-i', '--index-url',
332
dest='index_url',
333
metavar='URL',
334
default=pypi_url,
335
help='base URL of Python Package Index')
336
self.parser.add_option(
337
'--extra-index-url',
338
dest='extra_index_urls',
339
metavar='URL',
340
action='append',
341
default=[],
342
help='extra URLs of package indexes to use in addition to --index-url')
343
344
self.parser.add_option(
345
'-b', '--build', '--build-dir', '--build-directory',
346
dest='build_dir',
347
metavar='DIR',
348
default=None,
349
help='Unpack packages into DIR (default %s) and build from there' % base_prefix)
350
self.parser.add_option(
351
'--src', '--source',
352
dest='src_dir',
353
metavar='DIR',
354
default=None,
355
help='Check out --editable packages into DIR (default %s)' % base_src_prefix)
356
357
self.parser.add_option(
358
'-U', '--upgrade',
359
dest='upgrade',
360
action='store_true',
361
help='Upgrade all packages to the newest available version')
362
self.parser.add_option(
363
'-I', '--ignore-installed',
364
dest='ignore_installed',
365
action='store_true',
366
help='Ignore the installed packages (reinstalling instead)')
367
self.parser.add_option(
368
'--no-install',
369
dest='no_install',
370
action='store_true',
371
help="Download and unpack all packages, but don't actually install them")
372
373
self.parser.add_option(
374
'--install-option',
375
dest='install_options',
376
action='append',
377
help="Extra arguments to be supplied to the setup.py install "
378
"command (use like --install-option=\"--install-scripts=/usr/local/bin\"). "
379
"Use multiple --install-option options to pass multiple options to setup.py install"
380
)
381
382
def run(self, options, args):
383
if not options.build_dir:
384
options.build_dir = base_prefix
385
if not options.src_dir:
386
options.src_dir = base_src_prefix
387
options.build_dir = os.path.abspath(options.build_dir)
388
options.src_dir = os.path.abspath(options.src_dir)
389
install_options = options.install_options or []
390
index_urls = [options.index_url] + options.extra_index_urls
391
finder = PackageFinder(
392
find_links=options.find_links,
393
index_urls=index_urls)
394
requirement_set = RequirementSet(
395
build_dir=options.build_dir,
396
src_dir=options.src_dir,
397
upgrade=options.upgrade,
398
ignore_installed=options.ignore_installed)
399
for name in args:
400
requirement_set.add_requirement(
401
InstallRequirement.from_line(name, None))
402
for name in options.editables:
403
requirement_set.add_requirement(
404
InstallRequirement.from_editable(name))
405
for filename in options.requirements:
406
for req in parse_requirements(filename, finder=finder):
407
requirement_set.add_requirement(req)
408
requirement_set.install_files(finder, force_root_egg_info=self.bundle)
409
if not options.no_install and not self.bundle:
410
requirement_set.install(install_options)
411
logger.notify('Successfully installed %s' % requirement_set)
412
elif not self.bundle:
413
logger.notify('Successfully downloaded %s' % requirement_set)
414
return requirement_set
415
416
InstallCommand()
417
418
class BundleCommand(InstallCommand):
419
name = 'bundle'
420
usage = '%prog [OPTIONS] BUNDLE_NAME.pybundle PACKAGE_NAMES...'
421
summary = 'Create pybundles (archives containing multiple packages)'
422
bundle = True
423
424
def __init__(self):
425
super(BundleCommand, self).__init__()
426
427
def run(self, options, args):
428
if not args:
429
raise InstallationError('You must give a bundle filename')
430
if not options.build_dir:
431
options.build_dir = backup_dir(base_prefix, '-bundle')
432
if not options.src_dir:
433
options.src_dir = backup_dir(base_src_prefix, '-bundle')
434
# We have to get everything when creating a bundle:
435
options.ignore_installed = True
436
logger.notify('Putting temporary build files in %s and source/develop files in %s'
437
% (display_path(options.build_dir), display_path(options.src_dir)))
438
bundle_filename = args[0]
439
args = args[1:]
440
requirement_set = super(BundleCommand, self).run(options, args)
441
# FIXME: here it has to do something
442
requirement_set.create_bundle(bundle_filename)
443
logger.notify('Created bundle in %s' % bundle_filename)
444
return requirement_set
445
446
BundleCommand()
447
448
class FreezeCommand(Command):
449
name = 'freeze'
450
usage = '%prog [OPTIONS] FREEZE_NAME.txt'
451
summary = 'Put all currently installed packages (exact versions) into a requirements file'
452
453
def __init__(self):
454
super(FreezeCommand, self).__init__()
455
self.parser.add_option(
456
'-r', '--requirement',
457
dest='requirement',
458
action='store',
459
default=None,
460
metavar='FILENAME',
461
help='Use the given requirements file as a hint about how to generate the new frozen requirements')
462
self.parser.add_option(
463
'-f', '--find-links',
464
dest='find_links',
465
action='append',
466
default=[],
467
metavar='URL',
468
help='URL for finding packages, which will be added to the frozen requirements file')
469
470
def run(self, options, args):
471
if args:
472
filename = args[0]
473
else:
474
filename = '-'
475
requirement = options.requirement
476
find_links = options.find_links or []
477
## FIXME: Obviously this should be settable:
478
find_tags = False
479
480
if filename == '-':
481
logger.move_stdout_to_stderr()
482
dependency_links = []
483
if filename == '-':
484
f = sys.stdout
485
else:
486
## FIXME: should be possible to overwrite requirement file
487
logger.notify('Writing frozen requirements to %s' % filename)
488
f = open(filename, 'w')
489
for dist in pkg_resources.working_set:
490
if dist.has_metadata('dependency_links.txt'):
491
dependency_links.extend(dist.get_metadata_lines('dependency_links.txt'))
492
for link in find_links:
493
if '#egg=' in link:
494
dependency_links.append(link)
495
for link in find_links:
496
f.write('-f %s\n' % link)
497
installations = {}
498
for dist in pkg_resources.working_set:
499
if dist.key in ('setuptools', 'pip', 'python'):
500
## FIXME: also skip virtualenv?
501
continue
502
req = FrozenRequirement.from_dist(dist, dependency_links, find_tags=find_tags)
503
installations[req.name] = req
504
if requirement:
505
req_f = open(requirement)
506
for line in req_f:
507
if not line.strip() or line.strip().startswith('#'):
508
f.write(line)
509
continue
510
elif line.startswith('-e') or line.startswith('--editable'):
511
if line.startswith('-e'):
512
line = line[2:].strip()
513
else:
514
line = line[len('--editable'):].strip().lstrip('=')
515
line_req = InstallRequirement.from_editable(line)
516
elif (line.startswith('-r') or line.startswith('--requirement')
517
or line.startswith('-Z') or line.startswith('--always-unzip')):
518
logger.debug('Skipping line %r' % line.strip())
519
continue
520
else:
521
line_req = InstallRequirement.from_line(line)
522
if not line_req.name:
523
logger.notify("Skipping line because it's not clear what it would install: %s"
524
% line.strip())
525
continue
526
if line_req.name not in installations:
527
logger.warn("Requirement file contains %s, but that package is not installed"
528
% line.strip())
529
continue
530
f.write(str(installations[line_req.name]))
531
del installations[line_req.name]
532
f.write('## The following requirements were added by pip --freeze:\n')
533
for installation in sorted(installations.values(), key=lambda x: x.name):
534
f.write(str(installation))
535
if filename != '-':
536
logger.notify('Put requirements in %s' % filename)
537
f.close()
538
539
FreezeCommand()
540
541
class ZipCommand(Command):
542
name = 'zip'
543
usage = '%prog [OPTIONS] PACKAGE_NAMES...'
544
summary = 'Zip individual packages'
545
546
def __init__(self):
547
super(ZipCommand, self).__init__()
548
if self.name == 'zip':
549
self.parser.add_option(
550
'--unzip',
551
action='store_true',
552
dest='unzip',
553
help='Unzip (rather than zip) a package')
554
else:
555
self.parser.add_option(
556
'--zip',
557
action='store_false',
558
dest='unzip',
559
default=True,
560
help='Zip (rather than unzip) a package')
561
self.parser.add_option(
562
'--no-pyc',
563
action='store_true',
564
dest='no_pyc',
565
help='Do not include .pyc files in zip files (useful on Google App Engine)')
566
self.parser.add_option(
567
'-l', '--list',
568
action='store_true',
569
dest='list',
570
help='List the packages available, and their zip status')
571
self.parser.add_option(
572
'--sort-files',
573
action='store_true',
574
dest='sort_files',
575
help='With --list, sort packages according to how many files they contain')
576
self.parser.add_option(
577
'--path',
578
action='append',
579
dest='paths',
580
help='Restrict operations to the given paths (may include wildcards)')
581
self.parser.add_option(
582
'-n', '--simulate',
583
action='store_true',
584
help='Do not actually perform the zip/unzip operation')
585
586
def paths(self):
587
"""All the entries of sys.path, possibly restricted by --path"""
588
if not self.select_paths:
589
return sys.path
590
result = []
591
match_any = set()
592
for path in sys.path:
593
path = os.path.normcase(os.path.abspath(path))
594
for match in self.select_paths:
595
match = os.path.normcase(os.path.abspath(match))
596
if '*' in match:
597
if re.search(fnmatch.translate(match+'*'), path):
598
result.append(path)
599
match_any.add(match)
600
break
601
else:
602
if path.startswith(match):
603
result.append(path)
604
match_any.add(match)
605
break
606
else:
607
logger.debug("Skipping path %s because it doesn't match %s"
608
% (path, ', '.join(self.select_paths)))
609
for match in self.select_paths:
610
if match not in match_any and '*' not in match:
611
result.append(match)
612
logger.debug("Adding path %s because it doesn't match anything already on sys.path"
613
% match)
614
return result
615
616
def run(self, options, args):
617
self.select_paths = options.paths
618
self.simulate = options.simulate
619
if options.list:
620
return self.list(options, args)
621
if not args:
622
raise InstallationError(
623
'You must give at least one package to zip or unzip')
624
packages = []
625
for arg in args:
626
module_name, filename = self.find_package(arg)
627
if options.unzip and os.path.isdir(filename):
628
raise InstallationError(
629
'The module %s (in %s) is not a zip file; cannot be unzipped'
630
% (module_name, filename))
631
elif not options.unzip and not os.path.isdir(filename):
632
raise InstallationError(
633
'The module %s (in %s) is not a directory; cannot be zipped'
634
% (module_name, filename))
635
packages.append((module_name, filename))
636
for module_name, filename in packages:
637
if options.unzip:
638
return self.unzip_package(module_name, filename)
639
else:
640
return self.zip_package(module_name, filename, options.no_pyc)
641
642
def unzip_package(self, module_name, filename):
643
zip_filename = os.path.dirname(filename)
644
if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename):
645
raise InstallationError(
646
'Module %s (in %s) isn\'t located in a zip file in %s'
647
% (module_name, filename, zip_filename))
648
package_path = os.path.dirname(zip_filename)
649
if not package_path in self.paths():
650
logger.warn(
651
'Unpacking %s into %s, but %s is not on sys.path'
652
% (display_path(zip_filename), display_path(package_path),
653
display_path(package_path)))
654
logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename)))
655
if self.simulate:
656
logger.notify('Skipping remaining operations because of --simulate')
657
return
658
logger.indent += 2
659
try:
660
## FIXME: this should be undoable:
661
zip = zipfile.ZipFile(zip_filename)
662
to_save = []
663
for name in zip.namelist():
664
if name.startswith('%s/' % module_name):
665
content = zip.read(name)
666
dest = os.path.join(package_path, name)
667
if not os.path.exists(os.path.dirname(dest)):
668
os.makedirs(os.path.dirname(dest))
669
if not content and dest.endswith('/'):
670
if not os.path.exists(dest):
671
os.makedirs(dest)
672
else:
673
f = open(dest, 'wb')
674
f.write(content)
675
f.close()
676
else:
677
to_save.append((name, zip.read(name)))
678
zip.close()
679
if not to_save:
680
logger.info('Removing now-empty zip file %s' % display_path(zip_filename))
681
os.unlink(zip_filename)
682
self.remove_filename_from_pth(zip_filename)
683
else:
684
logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename)))
685
zip = zipfile.ZipFile(zip_filename, 'w')
686
for name, content in to_save:
687
zip.writestr(name, content)
688
zip.close()
689
finally:
690
logger.indent -= 2
691
692
def zip_package(self, module_name, filename, no_pyc):
693
orig_filename = filename
694
logger.notify('Zip %s (in %s)' % (module_name, display_path(filename)))
695
logger.indent += 2
696
if filename.endswith('.egg'):
697
dest_filename = filename
698
else:
699
dest_filename = filename + '.zip'
700
try:
701
## FIXME: I think this needs to be undoable:
702
if filename == dest_filename:
703
filename = backup_dir(orig_filename)
704
logger.notify('Moving %s aside to %s' % (orig_filename, filename))
705
if not self.simulate:
706
shutil.move(orig_filename, filename)
707
try:
708
logger.info('Creating zip file in %s' % display_path(dest_filename))
709
if not self.simulate:
710
zip = zipfile.ZipFile(dest_filename, 'w')
711
zip.writestr(module_name + '/', '')
712
for dirpath, dirnames, filenames in os.walk(filename):
713
if no_pyc:
714
filenames = [f for f in filenames
715
if not f.lower().endswith('.pyc')]
716
for fns, is_dir in [(dirnames, True), (filenames, False)]:
717
for fn in fns:
718
full = os.path.join(dirpath, fn)
719
dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn)
720
if is_dir:
721
zip.writestr(dest+'/', '')
722
else:
723
zip.write(full, dest)
724
zip.close()
725
logger.info('Removing old directory %s' % display_path(filename))
726
if not self.simulate:
727
shutil.rmtree(filename)
728
except:
729
## FIXME: need to do an undo here
730
raise
731
## FIXME: should also be undone:
732
self.add_filename_to_pth(dest_filename)
733
finally:
734
logger.indent -= 2
735
736
def remove_filename_from_pth(self, filename):
737
for pth in self.pth_files():
738
f = open(pth, 'r')
739
lines = f.readlines()
740
f.close()
741
new_lines = [
742
l for l in lines if l.strip() != filename]
743
if lines != new_lines:
744
logger.info('Removing reference to %s from .pth file %s'
745
% (display_path(filename), display_path(pth)))
746
if not filter(None, new_lines):
747
logger.info('%s file would be empty: deleting' % display_path(pth))
748
if not self.simulate:
749
os.unlink(pth)
750
else:
751
if not self.simulate:
752
f = open(pth, 'w')
753
f.writelines(new_lines)
754
f.close()
755
return
756
logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename))
757
758
def add_filename_to_pth(self, filename):
759
path = os.path.dirname(filename)
760
dest = os.path.join(path, filename + '.pth')
761
if path not in self.paths():
762
logger.warn('Adding .pth file %s, but it is not on sys.path' % display_path(dest))
763
if not self.simulate:
764
if os.path.exists(dest):
765
f = open(dest)
766
lines = f.readlines()
767
f.close()
768
if lines and not lines[-1].endswith('\n'):
769
lines[-1] += '\n'
770
lines.append(filename+'\n')
771
else:
772
lines = [filename + '\n']
773
f = open(dest, 'w')
774
f.writelines(lines)
775
f.close()
776
777
def pth_files(self):
778
for path in self.paths():
779
if not os.path.exists(path) or not os.path.isdir(path):
780
continue
781
for filename in os.listdir(path):
782
if filename.endswith('.pth'):
783
yield os.path.join(path, filename)
784
785
def find_package(self, package):
786
for path in self.paths():
787
full = os.path.join(path, package)
788
if os.path.exists(full):
789
return package, full
790
if not os.path.isdir(path) and zipfile.is_zipfile(path):
791
zip = zipfile.ZipFile(path, 'r')
792
try:
793
zip.read('%s/__init__.py' % package)
794
except KeyError:
795
pass
796
else:
797
zip.close()
798
return package, full
799
zip.close()
800
## FIXME: need special error for package.py case:
801
raise InstallationError(
802
'No package with the name %s found' % package)
803
804
def list(self, options, args):
805
if args:
806
raise InstallationError(
807
'You cannot give an argument with --list')
808
for path in sorted(self.paths()):
809
if not os.path.exists(path):
810
continue
811
basename = os.path.basename(path.rstrip(os.path.sep))
812
if os.path.isfile(path) and zipfile.is_zipfile(path):
813
if os.path.dirname(path) not in self.paths():
814
logger.notify('Zipped egg: %s' % display_path(path))
815
continue
816
if (basename != 'site-packages'
817
and not path.replace('\\', '/').endswith('lib/python')):
818
continue
819
logger.notify('In %s:' % display_path(path))
820
logger.indent += 2
821
zipped = []
822
unzipped = []
823
try:
824
for filename in sorted(os.listdir(path)):
825
ext = os.path.splitext(filename)[1].lower()
826
if ext in ('.pth', '.egg-info', '.egg-link'):
827
continue
828
if ext == '.py':
829
logger.info('Not displaying %s: not a package' % display_path(filename))
830
continue
831
full = os.path.join(path, filename)
832
if os.path.isdir(full):
833
unzipped.append((filename, self.count_package(full)))
834
elif zipfile.is_zipfile(full):
835
zipped.append(filename)
836
else:
837
logger.info('Unknown file: %s' % display_path(filename))
838
if zipped:
839
logger.notify('Zipped packages:')
840
logger.indent += 2
841
try:
842
for filename in zipped:
843
logger.notify(filename)
844
finally:
845
logger.indent -= 2
846
else:
847
logger.notify('No zipped packages.')
848
if unzipped:
849
if options.sort_files:
850
unzipped.sort(key=lambda x: -x[1])
851
logger.notify('Unzipped packages:')
852
logger.indent += 2
853
try:
854
for filename, count in unzipped:
855
logger.notify('%s (%i files)' % (filename, count))
856
finally:
857
logger.indent -= 2
858
else:
859
logger.notify('No unzipped packages.')
860
finally:
861
logger.indent -= 2
862
863
def count_package(self, path):
864
total = 0
865
for dirpath, dirnames, filenames in os.walk(path):
866
filenames = [f for f in filenames
867
if not f.lower().endswith('.pyc')]
868
total += len(filenames)
869
return total
870
871
ZipCommand()
872
873
class UnzipCommand(ZipCommand):
874
name = 'unzip'
875
summary = 'Unzip individual packages'
876
877
UnzipCommand()
878
879
880
def main(initial_args=None):
881
if initial_args is None:
882
initial_args = sys.argv[1:]
883
options, args = parser.parse_args(initial_args)
884
if options.help and not args:
885
args = ['help']
886
if not args:
887
parser.error('You must give a command (use "pip help" see a list of commands)')
888
command = args[0].lower()
889
## FIXME: search for a command match?
890
if command not in _commands:
891
parser.error('No command by the name %s %s' % (os.path.basename(sys.argv[0]), command))
892
command = _commands[command]
893
return command.main(initial_args, args[1:], options)
894
895
def get_proxy(proxystr=''):
896
"""Get the proxy given the option passed on the command line. If an
897
empty string is passed it looks at the HTTP_PROXY environment
898
variable."""
899
if not proxystr:
900
proxystr = os.environ.get('HTTP_PROXY', '')
901
if proxystr:
902
if '@' in proxystr:
903
user_password, server_port = proxystr.split('@', 1)
904
if ':' in user_password:
905
user, password = user_password.split(':', 1)
906
else:
907
user = user_password
908
import getpass
909
prompt = 'Password for %s@%s: ' % (user, server_port)
910
password = urllib.quote(getpass.getpass(prompt))
911
return '%s:%s@%s' % (user, password, server_port)
912
else:
913
return proxystr
914
else:
915
return None
916
917
def setup_proxy_handler(proxystr=''):
918
"""Set the proxy handler given the option passed on the command
919
line. If an empty string is passed it looks at the HTTP_PROXY
920
environment variable. """
921
proxy = get_proxy(proxystr)
922
if proxy:
923
proxy_support = urllib2.ProxyHandler({"http": proxy, "ftp": proxy})
924
opener = urllib2.build_opener(proxy_support, urllib2.CacheFTPHandler)
925
urllib2.install_opener(opener)
926
927
def format_exc(exc_info=None):
928
if exc_info is None:
929
exc_info = sys.exc_info()
930
out = StringIO()
931
traceback.print_exception(*exc_info, **dict(file=out))
932
return out.getvalue()
933
934
def restart_in_venv(venv, args):
935
"""
936
Restart this script using the interpreter in the given virtual environment
937
"""
938
venv = os.path.abspath(venv)
939
if not os.path.exists(venv):
940
try:
941
import virtualenv
942
except ImportError:
943
print 'The virtual environment does not exist: %s' % venv
944
print 'and virtualenv is not installed, so a new environment cannot be created'
945
sys.exit(3)
946
print 'Creating new virtualenv environment in %s' % venv
947
virtualenv.logger = logger
948
logger.indent += 2
949
## FIXME: always have no_site_packages?
950
virtualenv.create_environment(venv, site_packages=False)
951
if sys.platform == 'win32':
952
python = os.path.join(venv, 'Scripts', 'python.exe')
953
else:
954
python = os.path.join(venv, 'bin', 'python')
955
if not os.path.exists(python):
956
python = venv
957
if not os.path.exists(python):
958
raise BadCommand('Cannot find virtual environment interpreter at %s' % python)
959
base = os.path.dirname(os.path.dirname(python))
960
file = __file__
961
if file.endswith('.pyc'):
962
file = file[:-1]
963
os.execv(python, [python, file] + args + [base, '___VENV_RESTART___'])
964
965
class PackageFinder(object):
966
"""This finds packages.
967
968
This is meant to match easy_install's technique for looking for
969
packages, by reading pages and looking for appropriate links
970
"""
971
972
failure_limit = 3
973
974
def __init__(self, find_links, index_urls):
975
self.find_links = find_links
976
self.index_urls = index_urls
977
self.dependency_links = []
978
self.cache = PageCache()
979
# These are boring links that have already been logged somehow:
980
self.logged_links = set()
981
982
def add_dependency_links(self, links):
983
## FIXME: this shouldn't be global list this, it should only
984
## apply to requirements of the package that specifies the
985
## dependency_links value
986
## FIXME: also, we should track comes_from (i.e., use Link)
987
self.dependency_links.extend(links)
988
989
def find_requirement(self, req, upgrade):
990
url_name = req.url_name
991
# Check that we have the url_name correctly spelled:
992
main_index_url = Link(posixpath.join(self.index_urls[0], url_name))
993
# This will also cache the page, so it's okay that we get it again later:
994
page = self._get_page(main_index_url, req)
995
if page is None:
996
url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req)
997
if url_name is not None:
998
locations = [
999
posixpath.join(url, url_name)
1000
for url in self.index_urls] + self.find_links
1001
else:
1002
locations = list(self.find_links)
1003
locations.extend(self.dependency_links)
1004
for version in req.absolute_versions:
1005
if url_name is not None:
1006
locations = [
1007
posixpath.join(url, url_name, version)] + locations
1008
locations = [Link(url) for url in locations]
1009
logger.debug('URLs to search for versions for %s:' % req)
1010
for location in locations:
1011
logger.debug('* %s' % location)
1012
found_versions = []
1013
found_versions.extend(
1014
self._package_versions(
1015
[Link(url, '-f') for url in self.find_links], req.name.lower()))
1016
for page in self._get_pages(locations, req):
1017
logger.debug('Analyzing links from page %s' % page.url)
1018
logger.indent += 2
1019
try:
1020
found_versions.extend(self._package_versions(page.links, req.name.lower()))
1021
finally:
1022
logger.indent -= 2
1023
dependency_versions = list(self._package_versions([Link(url) for url in self.dependency_links], req.name.lower()))
1024
if dependency_versions:
1025
logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions]))
1026
found_versions.extend(dependency_versions)
1027
if not found_versions:
1028
logger.fatal('Could not find any downloads that satisfy the requirement %s' % req)
1029
raise DistributionNotFound('No distributions at all found for %s' % req)
1030
if req.satisfied_by is not None:
1031
found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version))
1032
found_versions.sort(reverse=True)
1033
applicable_versions = []
1034
for (parsed_version, link, version) in found_versions:
1035
if version not in req.req:
1036
logger.info("Ignoring link %s, version %s doesn't match %s"
1037
% (link, version, ','.join([''.join(s) for s in req.req.specs])))
1038
continue
1039
applicable_versions.append((link, version))
1040
existing_applicable = bool([link for link, version in applicable_versions if link is Inf])
1041
if not upgrade and existing_applicable:
1042
if applicable_versions[0][1] is Inf:
1043
logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement'
1044
% req.satisfied_by.version)
1045
else:
1046
logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)'
1047
% (req.satisfied_by.version, application_versions[0][2]))
1048
return None
1049
if not applicable_versions:
1050
logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)'
1051
% (req, ', '.join([version for parsed_version, link, version in found_versions])))
1052
raise DistributionNotFound('No distributions matching the version for %s' % req)
1053
if applicable_versions[0][0] is Inf:
1054
# We have an existing version, and its the best version
1055
logger.info('Installed version (%s) is most up-to-date (past versions: %s)'
1056
% (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none'))
1057
return None
1058
if len(applicable_versions) > 1:
1059
logger.info('Using version %s (newest of versions: %s)' %
1060
(applicable_versions[0][1], ', '.join([version for link, version in applicable_versions])))
1061
return applicable_versions[0][0]
1062
1063
def _find_url_name(self, index_url, url_name, req):
1064
"""Finds the true URL name of a package, when the given name isn't quite correct.
1065
This is usually used to implement case-insensitivity."""
1066
if not index_url.url.endswith('/'):
1067
# Vaguely part of the PyPI API... weird but true.
1068
## FIXME: bad to modify this?
1069
index_url.url += '/'
1070
page = self._get_page(index_url, req)
1071
if page is None:
1072
logger.fatal('Cannot fetch index base URL %s' % index_url)
1073
raise DistributionNotFound('Cannot find requirement %s, nor fetch index URL %s' % (req, index_url))
1074
norm_name = normalize_name(req.url_name)
1075
for link in page.links:
1076
base = posixpath.basename(link.path.rstrip('/'))
1077
if norm_name == normalize_name(base):
1078
logger.notify('Real name of requirement %s is %s' % (url_name, base))
1079
return base
1080
return None
1081
1082
def _get_pages(self, locations, req):
1083
"""Yields (page, page_url) from the given locations, skipping
1084
locations that have errors, and adding download/homepage links"""
1085
pending_queue = Queue()
1086
for location in locations:
1087
pending_queue.put(location)
1088
done = []
1089
seen = set()
1090
threads = []
1091
for i in range(min(10, len(locations))):
1092
t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen))
1093
t.setDaemon(True)
1094
threads.append(t)
1095
t.start()
1096
for t in threads:
1097
t.join()
1098
return done
1099
1100
_log_lock = threading.Lock()
1101
1102
def _get_queued_page(self, req, pending_queue, done, seen):
1103
while 1:
1104
try:
1105
location = pending_queue.get(False)
1106
except QueueEmpty:
1107
return
1108
if location in seen:
1109
continue
1110
seen.add(location)
1111
page = self._get_page(location, req)
1112
if page is None:
1113
continue
1114
done.append(page)
1115
for link in page.rel_links():
1116
pending_queue.put(link)
1117
1118
_egg_fragment_re = re.compile(r'#egg=([^&]*)')
1119
_egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I)
1120
_py_version_re = re.compile(r'-py([123]\.[0-9])$')
1121
1122
def _package_versions(self, links, search_name):
1123
seen_links = {}
1124
for link in links:
1125
if link.url in seen_links:
1126
continue
1127
seen_links[link.url] = None
1128
if link.egg_fragment:
1129
egg_info = link.egg_fragment
1130
else:
1131
path = link.path
1132
egg_info, ext = link.splitext()
1133
if not ext:
1134
if link not in self.logged_links:
1135
logger.debug('Skipping link %s; not a file' % link)
1136
self.logged_links.add(link)
1137
continue
1138
if egg_info.endswith('.tar'):
1139
# Special double-extension case:
1140
egg_info = egg_info[:-4]
1141
ext = '.tar' + ext
1142
if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'):
1143
if link not in self.logged_links:
1144
logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext))
1145
self.logged_links.add(link)
1146
continue
1147
version = self._egg_info_matches(egg_info, search_name, link)
1148
if version is None:
1149
logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name))
1150
continue
1151
match = self._py_version_re.search(version)
1152
if match:
1153
version = version[:match.start()]
1154
py_version = match.group(1)
1155
if py_version != sys.version[:3]:
1156
logger.debug('Skipping %s because Python version is incorrect' % link)
1157
continue
1158
logger.debug('Found link %s, version: %s' % (link, version))
1159
yield (pkg_resources.parse_version(version),
1160
link,
1161
version)
1162
1163
def _egg_info_matches(self, egg_info, search_name, link):
1164
match = self._egg_info_re.search(egg_info)
1165
if not match:
1166
logger.debug('Could not parse version from link: %s' % link)
1167
return None
1168
name = match.group(0).lower()
1169
# To match the "safe" name that pkg_resources creates:
1170
name = name.replace('_', '-')
1171
if name.startswith(search_name.lower()):
1172
return match.group(0)[len(search_name):].lstrip('-')
1173
else:
1174
return None
1175
1176
def _get_page(self, link, req):
1177
return HTMLPage.get_page(link, req, cache=self.cache)
1178
1179
1180
class InstallRequirement(object):
1181
1182
def __init__(self, req, comes_from, source_dir=None, editable=False,
1183
url=None, update=True):
1184
if isinstance(req, basestring):
1185
req = pkg_resources.Requirement.parse(req)
1186
self.req = req
1187
self.comes_from = comes_from
1188
self.source_dir = source_dir
1189
self.editable = editable
1190
self.url = url
1191
self._egg_info_path = None
1192
# This holds the pkg_resources.Distribution object if this requirement
1193
# is already available:
1194
self.satisfied_by = None
1195
self._temp_build_dir = None
1196
self._is_bundle = None
1197
# True if the editable should be updated:
1198
self.update = update
1199
1200
@classmethod
1201
def from_editable(cls, editable_req, comes_from=None):
1202
name, url = parse_editable(editable_req)
1203
if url.startswith('file:'):
1204
source_dir = url_to_filename(url)
1205
else:
1206
source_dir = None
1207
return cls(name, comes_from, source_dir=source_dir, editable=True, url=url)
1208
1209
@classmethod
1210
def from_line(cls, name, comes_from=None):
1211
"""Creates an InstallRequirement from a name, which might be a
1212
requirement, filename, or URL.
1213
"""
1214
url = None
1215
req = name
1216
if is_url(name):
1217
url = name
1218
## FIXME: I think getting the requirement here is a bad idea:
1219
#req = get_requirement_from_url(url)
1220
req = None
1221
elif is_filename(name):
1222
if not os.path.exists(name):
1223
logger.warn('Requirement %r looks like a filename, but the file does not exist'
1224
% name)
1225
url = filename_to_url(name)
1226
#req = get_requirement_from_url(url)
1227
req = None
1228
return cls(req, comes_from, url=url)
1229
1230
def __str__(self):
1231
if self.req:
1232
s = str(self.req)
1233
if self.url:
1234
s += ' from %s' % self.url
1235
else:
1236
s = self.url
1237
if self.satisfied_by is not None:
1238
s += ' in %s' % display_path(self.satisfied_by.location)
1239
if self.editable:
1240
if self.req:
1241
s += ' checkout from %s' % self.url
1242
if self.comes_from:
1243
if isinstance(self.comes_from, basestring):
1244
comes_from = self.comes_from
1245
else:
1246
comes_from = self.comes_from.from_path()
1247
if comes_from:
1248
s += ' (from %s)' % comes_from
1249
return s
1250
1251
def from_path(self):
1252
s = str(self.req)
1253
if self.comes_from:
1254
if isinstance(self.comes_from, basestring):
1255
comes_from = self.comes_from
1256
else:
1257
comes_from = self.comes_from.from_path()
1258
s += '->' + comes_from
1259
return s
1260
1261
def build_location(self, build_dir):
1262
if self._temp_build_dir is not None:
1263
return self._temp_build_dir
1264
if self.req is None:
1265
self._temp_build_dir = tempfile.mkdtemp('-build', 'pip-')
1266
self._ideal_build_dir = build_dir
1267
return self._temp_build_dir
1268
if self.editable:
1269
name = self.name.lower()
1270
else:
1271
name = self.name
1272
# FIXME: Is there a better place to create the build_dir? (hg and bzr need this)
1273
if not os.path.exists(build_dir):
1274
os.makedirs(build_dir)
1275
return os.path.join(build_dir, name)
1276
1277
def correct_build_location(self):
1278
"""If the build location was a temporary directory, this will move it
1279
to a new more permanent location"""
1280
if self.source_dir is not None:
1281
return
1282
assert self.req is not None
1283
assert self._temp_build_dir
1284
old_location = self._temp_build_dir
1285
new_build_dir = self._ideal_build_dir
1286
del self._ideal_build_dir
1287
if self.editable:
1288
name = self.name.lower()
1289
else:
1290
name = self.name
1291
new_location = os.path.join(new_build_dir, name)
1292
if not os.path.exists(new_build_dir):
1293
logger.debug('Creating directory %s' % new_build_dir)
1294
os.makedirs(new_build_dir)
1295
if os.path.exists(new_location):
1296
raise InstallationError(
1297
'A package already exists in %s; please remove it to continue'
1298
% display_path(new_location))
1299
logger.debug('Moving package %s from %s to new location %s'
1300
% (self, display_path(old_location), display_path(new_location)))
1301
shutil.move(old_location, new_location)
1302
self._temp_build_dir = new_location
1303
self.source_dir = new_location
1304
self._egg_info_path = None
1305
1306
@property
1307
def name(self):
1308
if self.req is None:
1309
return None
1310
return self.req.project_name
1311
1312
@property
1313
def url_name(self):
1314
if self.req is None:
1315
return None
1316
return urllib.quote(self.req.unsafe_name)
1317
1318
@property
1319
def setup_py(self):
1320
return os.path.join(self.source_dir, 'setup.py')
1321
1322
def run_egg_info(self, force_root_egg_info=False):
1323
assert self.source_dir
1324
if self.name:
1325
logger.notify('Running setup.py egg_info for package %s' % self.name)
1326
else:
1327
logger.notify('Running setup.py egg_info for package from %s' % self.url)
1328
logger.indent += 2
1329
try:
1330
script = self._run_setup_py
1331
script = script.replace('__SETUP_PY__', repr(self.setup_py))
1332
script = script.replace('__PKG_NAME__', repr(self.name))
1333
# We can't put the .egg-info files at the root, because then the source code will be mistaken
1334
# for an installed egg, causing problems
1335
if self.editable or force_root_egg_info:
1336
egg_base_option = []
1337
else:
1338
egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info')
1339
if not os.path.exists(egg_info_dir):
1340
os.makedirs(egg_info_dir)
1341
egg_base_option = ['--egg-base', 'pip-egg-info']
1342
call_subprocess(
1343
[sys.executable, '-c', script, 'egg_info'] + egg_base_option,
1344
cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False,
1345
command_level=Logger.VERBOSE_DEBUG,
1346
command_desc='python setup.py egg_info')
1347
finally:
1348
logger.indent -= 2
1349
if not self.req:
1350
self.req = pkg_resources.Requirement.parse(self.pkg_info()['Name'])
1351
self.correct_build_location()
1352
1353
## FIXME: This is a lame hack, entirely for PasteScript which has
1354
## a self-provided entry point that causes this awkwardness
1355
_run_setup_py = """
1356
__file__ = __SETUP_PY__
1357
from setuptools.command import egg_info
1358
def replacement_run(self):
1359
self.mkpath(self.egg_info)
1360
installer = self.distribution.fetch_build_egg
1361
for ep in egg_info.iter_entry_points('egg_info.writers'):
1362
# require=False is the change we're making:
1363
writer = ep.load(require=False)
1364
writer(self, ep.name, egg_info.os.path.join(self.egg_info,ep.name))
1365
self.find_sources()
1366
egg_info.egg_info.run = replacement_run
1367
execfile(__file__)
1368
"""
1369
1370
def egg_info_data(self, filename):
1371
if self.satisfied_by is not None:
1372
if not self.satisfied_by.has_metadata(filename):
1373
return None
1374
return self.satisfied_by.get_metadata(filename)
1375
assert self.source_dir
1376
filename = self.egg_info_path(filename)
1377
if not os.path.exists(filename):
1378
return None
1379
fp = open(filename, 'r')
1380
data = fp.read()
1381
fp.close()
1382
return data
1383
1384
def egg_info_path(self, filename):
1385
if self._egg_info_path is None:
1386
if self.editable:
1387
base = self.source_dir
1388
else:
1389
base = os.path.join(self.source_dir, 'pip-egg-info')
1390
filenames = os.listdir(base)
1391
if self.editable:
1392
filenames = []
1393
for root, dirs, files in os.walk(base):
1394
for dir in vcs.dirnames:
1395
if dir in dirs:
1396
dirs.remove(dir)
1397
filenames.extend([os.path.join(root, dir)
1398
for dir in dirs])
1399
filenames = [f for f in filenames if f.endswith('.egg-info')]
1400
assert len(filenames) == 1, "Unexpected files/directories in %s: %s" % (base, ' '.join(filenames))
1401
self._egg_info_path = os.path.join(base, filenames[0])
1402
return os.path.join(self._egg_info_path, filename)
1403
1404
def egg_info_lines(self, filename):
1405
data = self.egg_info_data(filename)
1406
if not data:
1407
return []
1408
result = []
1409
for line in data.splitlines():
1410
line = line.strip()
1411
if not line or line.startswith('#'):
1412
continue
1413
result.append(line)
1414
return result
1415
1416
def pkg_info(self):
1417
p = FeedParser()
1418
data = self.egg_info_data('PKG-INFO')
1419
if not data:
1420
logger.warn('No PKG-INFO file found in %s' % display_path(self.egg_info_path('PKG-INFO')))
1421
p.feed(data or '')
1422
return p.close()
1423
1424
@property
1425
def dependency_links(self):
1426
return self.egg_info_lines('dependency_links.txt')
1427
1428
_requirements_section_re = re.compile(r'\[(.*?)\]')
1429
1430
def requirements(self, extras=()):
1431
in_extra = None
1432
for line in self.egg_info_lines('requires.txt'):
1433
match = self._requirements_section_re.match(line)
1434
if match:
1435
in_extra = match.group(1)
1436
continue
1437
if in_extra and in_extra not in extras:
1438
# Skip requirement for an extra we aren't requiring
1439
continue
1440
yield line
1441
1442
@property
1443
def absolute_versions(self):
1444
for qualifier, version in self.req.specs:
1445
if qualifier == '==':
1446
yield version
1447
1448
@property
1449
def installed_version(self):
1450
return self.pkg_info()['version']
1451
1452
def assert_source_matches_version(self):
1453
assert self.source_dir
1454
if self.comes_from == 'command line':
1455
# We don't check the versions of things explicitly installed.
1456
# This makes, e.g., "pip Package==dev" possible
1457
return
1458
version = self.installed_version
1459
if version not in self.req:
1460
logger.fatal(
1461
'Source in %s has the version %s, which does not match the requirement %s'
1462
% (display_path(self.source_dir), version, self))
1463
raise InstallationError(
1464
'Source in %s has version %s that conflicts with %s'
1465
% (display_path(self.source_dir), version, self))
1466
else:
1467
logger.debug('Source in %s has version %s, which satisfies requirement %s'
1468
% (display_path(self.source_dir), version, self))
1469
1470
def update_editable(self):
1471
if not self.url:
1472
logger.info("Cannot update repository at %s; repository location is unknown" % self.source_dir)
1473
return
1474
assert self.editable
1475
assert self.source_dir
1476
if self.url.startswith('file:'):
1477
# Static paths don't get updated
1478
return
1479
assert '+' in self.url, "bad url: %r" % self.url
1480
if not self.update:
1481
return
1482
vc_type, url = self.url.split('+', 1)
1483
vc_type = vc_type.lower()
1484
version_control = vcs.get_backend(vc_type)
1485
if version_control:
1486
version_control(self.url).obtain(self.source_dir)
1487
else:
1488
assert 0, (
1489
'Unexpected version control type (in %s): %s'
1490
% (self.url, vc_type))
1491
1492
def install(self, install_options):
1493
if self.editable:
1494
self.install_editable()
1495
return
1496
## FIXME: this is not a useful record:
1497
## Also a bad location
1498
if sys.platform == 'win32':
1499
install_location = os.path.join(sys.prefix, 'Lib')
1500
else:
1501
install_location = os.path.join(sys.prefix, 'lib', 'python%s' % sys.version[:3])
1502
temp_location = tempfile.mkdtemp('-record', 'pip-')
1503
record_filename = os.path.join(temp_location, 'install-record.txt')
1504
## FIXME: I'm not sure if this is a reasonable location; probably not
1505
## but we can't put it in the default location, as that is a virtualenv symlink that isn't writable
1506
header_dir = os.path.join(os.path.dirname(os.path.dirname(self.source_dir)), 'lib', 'include')
1507
logger.notify('Running setup.py install for %s' % self.name)
1508
logger.indent += 2
1509
try:
1510
call_subprocess(
1511
[sys.executable, '-c',
1512
"import setuptools; __file__=%r; execfile(%r)" % (self.setup_py, self.setup_py),
1513
'install', '--single-version-externally-managed', '--record', record_filename,
1514
'--install-headers', header_dir] + install_options,
1515
cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False)
1516
finally:
1517
logger.indent -= 2
1518
f = open(record_filename)
1519
for line in f:
1520
line = line.strip()
1521
if line.endswith('.egg-info'):
1522
egg_info_dir = line
1523
break
1524
else:
1525
logger.warn('Could not find .egg-info directory in install record for %s' % self)
1526
## FIXME: put the record somewhere
1527
return
1528
f.close()
1529
new_lines = []
1530
f = open(record_filename)
1531
for line in f:
1532
filename = line.strip()
1533
if os.path.isdir(filename):
1534
filename += os.path.sep
1535
new_lines.append(make_path_relative(filename, egg_info_dir))
1536
f.close()
1537
f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w')
1538
f.write('\n'.join(new_lines)+'\n')
1539
f.close()
1540
1541
def remove_temporary_source(self):
1542
"""Remove the source files from this requirement, if they are marked
1543
for deletion"""
1544
if self.is_bundle or os.path.exists(self.delete_marker_filename):
1545
logger.info('Removing source in %s' % self.source_dir)
1546
if self.source_dir:
1547
shutil.rmtree(self.source_dir)
1548
self.source_dir = None
1549
if self._temp_build_dir and os.path.exists(self._temp_build_dir):
1550
shutil.rmtree(self._temp_build_dir)
1551
self._temp_build_dir = None
1552
1553
def install_editable(self):
1554
logger.notify('Running setup.py develop for %s' % self.name)
1555
logger.indent += 2
1556
try:
1557
## FIXME: should we do --install-headers here too?
1558
call_subprocess(
1559
[sys.executable, '-c',
1560
"import setuptools; __file__=%r; execfile(%r)" % (self.setup_py, self.setup_py),
1561
'develop', '--no-deps'], cwd=self.source_dir, filter_stdout=self._filter_install,
1562
show_stdout=False)
1563
finally:
1564
logger.indent -= 2
1565
1566
def _filter_install(self, line):
1567
level = Logger.NOTIFY
1568
for regex in [r'^running .*', r'^writing .*', '^creating .*', '^[Cc]opying .*',
1569
r'^reading .*', r"^removing .*\.egg-info' \(and everything under it\)$",
1570
r'^byte-compiling ',
1571
# Not sure what this warning is, but it seems harmless:
1572
r"^warning: manifest_maker: standard file '-c' not found$"]:
1573
if re.search(regex, line.strip()):
1574
level = Logger.INFO
1575
break
1576
return (level, line)
1577
1578
def check_if_exists(self):
1579
"""Checks if this requirement is satisfied by something already installed"""
1580
if self.req is None:
1581
return False
1582
try:
1583
dist = pkg_resources.get_distribution(self.req)
1584
except pkg_resources.DistributionNotFound:
1585
return False
1586
self.satisfied_by = dist
1587
return True
1588
1589
@property
1590
def is_bundle(self):
1591
if self._is_bundle is not None:
1592
return self._is_bundle
1593
base = self._temp_build_dir
1594
if not base:
1595
## FIXME: this doesn't seem right:
1596
return False
1597
self._is_bundle = (os.path.exists(os.path.join(base, 'pip-manifest.txt'))
1598
or os.path.exists(os.path.join(base, 'pyinstall-manifest.txt')))
1599
return self._is_bundle
1600
1601
def bundle_requirements(self):
1602
base = self._temp_build_dir
1603
assert base
1604
src_dir = os.path.join(base, 'src')
1605
build_dir = os.path.join(base, 'build')
1606
if os.path.exists(src_dir):
1607
for package in os.listdir(src_dir):
1608
## FIXME: svnism:
1609
for vcs_backend in vcs.backends:
1610
url = rev = None
1611
vcs_bundle_file = os.path.join(
1612
src_dir, package, vcs_backend.bundle_file)
1613
if os.path.exists(vcs_bundle_file):
1614
vc_type = vcs_backend.name
1615
fp = open(vcs_bundle_file)
1616
content = fp.read()
1617
fp.close()
1618
url, rev = vcs_backend().parse_checkout_text(content)
1619
break
1620
if url:
1621
url = '%s+%s@%s' % (vc_type, url, rev)
1622
else:
1623
url = None
1624
yield InstallRequirement(
1625
package, self, editable=True, url=url,
1626
update=False, source_dir=os.path.join(src_dir, package))
1627
if os.path.exists(build_dir):
1628
for package in os.listdir(build_dir):
1629
yield InstallRequirement(
1630
package, self,
1631
source_dir=os.path.join(build_dir, package))
1632
1633
def move_bundle_files(self, dest_build_dir, dest_src_dir):
1634
base = self._temp_build_dir
1635
assert base
1636
src_dir = os.path.join(base, 'src')
1637
build_dir = os.path.join(base, 'build')
1638
for source_dir, dest_dir in [(src_dir, dest_src_dir),
1639
(build_dir, dest_build_dir)]:
1640
if os.path.exists(source_dir):
1641
for dirname in os.listdir(source_dir):
1642
dest = os.path.join(dest_dir, dirname)
1643
if os.path.exists(dest):
1644
logger.warn('The directory %s (containing package %s) already exists; cannot move source from bundle %s'
1645
% (dest, dirname, self))
1646
continue
1647
if not os.path.exists(dest_dir):
1648
logger.info('Creating directory %s' % dest_dir)
1649
os.makedirs(dest_dir)
1650
shutil.move(os.path.join(source_dir, dirname), dest)
1651
1652
@property
1653
def delete_marker_filename(self):
1654
assert self.source_dir
1655
return os.path.join(self.source_dir, 'pip-delete-this-directory.txt')
1656
1657
DELETE_MARKER_MESSAGE = '''\
1658
This file is placed here by pip to indicate the source was put
1659
here by pip.
1660
1661
Once this package is successfully installed this source code will be
1662
deleted (unless you remove this file).
1663
'''
1664
1665
class RequirementSet(object):
1666
1667
def __init__(self, build_dir, src_dir, upgrade=False, ignore_installed=False):
1668
self.build_dir = build_dir
1669
self.src_dir = src_dir
1670
self.upgrade = upgrade
1671
self.ignore_installed = ignore_installed
1672
self.requirements = {}
1673
# Mapping of alias: real_name
1674
self.requirement_aliases = {}
1675
self.unnamed_requirements = []
1676
1677
def __str__(self):
1678
reqs = [req for req in self.requirements.values()
1679
if not req.comes_from]
1680
reqs.sort(key=lambda req: req.name.lower())
1681
return ' '.join([str(req.req) for req in reqs])
1682
1683
def add_requirement(self, install_req):
1684
name = install_req.name
1685
if not name:
1686
self.unnamed_requirements.append(install_req)
1687
else:
1688
if self.has_requirement(name):
1689
raise InstallationError(
1690
'Double requirement given: %s (aready in %s, name=%r)'
1691
% (install_req, self.get_requirement(name), name))
1692
self.requirements[name] = install_req
1693
## FIXME: what about other normalizations? E.g., _ vs. -?
1694
if name.lower() != name:
1695
self.requirement_aliases[name.lower()] = name
1696
1697
def has_requirement(self, project_name):
1698
for name in project_name, project_name.lower():
1699
if name in self.requirements or name in self.requirement_aliases:
1700
return True
1701
return False
1702
1703
def get_requirement(self, project_name):
1704
for name in project_name, project_name.lower():
1705
if name in self.requirements:
1706
return self.requirements[name]
1707
if name in self.requirement_aliases:
1708
return self.requirements[self.requirement_aliases[name]]
1709
raise KeyError("No project with the name %r" % project_name)
1710
1711
def install_files(self, finder, force_root_egg_info=False):
1712
unnamed = list(self.unnamed_requirements)
1713
reqs = self.requirements.values()
1714
while reqs or unnamed:
1715
if unnamed:
1716
req_to_install = unnamed.pop(0)
1717
else:
1718
req_to_install = reqs.pop(0)
1719
install = True
1720
if not self.ignore_installed and not req_to_install.editable and not self.upgrade:
1721
if req_to_install.check_if_exists():
1722
install = False
1723
if req_to_install.satisfied_by is not None and not self.upgrade:
1724
logger.notify('Requirement already satisfied: %s' % req_to_install)
1725
elif req_to_install.editable:
1726
logger.notify('Checking out %s' % req_to_install)
1727
else:
1728
if req_to_install.url and req_to_install.url.lower().startswith('file:'):
1729
logger.notify('Unpacking %s' % display_path(url_to_filename(req_to_install.url)))
1730
else:
1731
logger.notify('Downloading/unpacking %s' % req_to_install)
1732
logger.indent += 2
1733
is_bundle = False
1734
try:
1735
if req_to_install.editable:
1736
if req_to_install.source_dir is None:
1737
location = req_to_install.build_location(self.src_dir)
1738
req_to_install.source_dir = location
1739
else:
1740
location = req_to_install.source_dir
1741
req_to_install.update_editable()
1742
req_to_install.run_egg_info()
1743
elif install:
1744
location = req_to_install.build_location(self.build_dir)
1745
## FIXME: is the existance of the checkout good enough to use it? I'm don't think so.
1746
unpack = True
1747
if not os.path.exists(os.path.join(location, 'setup.py')):
1748
## FIXME: this won't upgrade when there's an existing package unpacked in `location`
1749
if req_to_install.url is None:
1750
url = finder.find_requirement(req_to_install, upgrade=self.upgrade)
1751
else:
1752
## FIXME: should req_to_install.url already be a link?
1753
url = Link(req_to_install.url)
1754
assert url
1755
if url:
1756
try:
1757
self.unpack_url(url, location)
1758
except urllib2.HTTPError, e:
1759
logger.fatal('Could not install requirement %s because of error %s'
1760
% (req_to_install, e))
1761
raise InstallationError(
1762
'Could not install requirement %s because of HTTP error %s for URL %s'
1763
% (req_to_install, e, url))
1764
else:
1765
unpack = False
1766
if unpack:
1767
is_bundle = req_to_install.is_bundle
1768
if is_bundle:
1769
for subreq in req_to_install.bundle_requirements():
1770
reqs.append(subreq)
1771
self.add_requirement(subreq)
1772
req_to_install.move_bundle_files(self.build_dir, self.src_dir)
1773
else:
1774
req_to_install.source_dir = location
1775
req_to_install.run_egg_info()
1776
if force_root_egg_info:
1777
# We need to run this to make sure that the .egg-info/
1778
# directory is created for packing in the bundle
1779
req_to_install.run_egg_info(force_root_egg_info=True)
1780
req_to_install.assert_source_matches_version()
1781
f = open(req_to_install.delete_marker_filename, 'w')
1782
f.write(DELETE_MARKER_MESSAGE)
1783
f.close()
1784
if not is_bundle:
1785
## FIXME: shouldn't be globally added:
1786
finder.add_dependency_links(req_to_install.dependency_links)
1787
## FIXME: add extras in here:
1788
for req in req_to_install.requirements():
1789
try:
1790
name = pkg_resources.Requirement.parse(req).project_name
1791
except ValueError, e:
1792
## FIXME: proper warning
1793
logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install))
1794
continue
1795
if self.has_requirement(name):
1796
## FIXME: check for conflict
1797
continue
1798
subreq = InstallRequirement(req, req_to_install)
1799
reqs.append(subreq)
1800
self.add_requirement(subreq)
1801
if req_to_install.name not in self.requirements:
1802
self.requirements[req_to_install.name] = req_to_install
1803
else:
1804
req_to_install.remove_temporary_source()
1805
finally:
1806
logger.indent -= 2
1807
1808
def unpack_url(self, link, location):
1809
for backend in vcs.backends:
1810
if link.scheme in backend.schemes:
1811
backend(link).unpack(location)
1812
return
1813
dir = tempfile.mkdtemp()
1814
if link.url.lower().startswith('file:'):
1815
source = url_to_filename(link.url)
1816
content_type = mimetypes.guess_type(source)
1817
self.unpack_file(source, location, content_type, link)
1818
return
1819
md5_hash = link.md5_hash
1820
target_url = link.url.split('#', 1)[0]
1821
target_file = None
1822
if os.environ.get('PIP_DOWNLOAD_CACHE'):
1823
target_file = os.path.join(os.environ['PIP_DOWNLOAD_CACHE'],
1824
urllib.quote(target_url, ''))
1825
if (target_file and os.path.exists(target_file)
1826
and os.path.exists(target_file+'.content-type')):
1827
fp = open(target_file+'.content-type')
1828
content_type = fp.read().strip()
1829
fp.close()
1830
if md5_hash:
1831
download_hash = md5()
1832
fp = open(target_file, 'rb')
1833
while 1:
1834
chunk = fp.read(4096)
1835
if not chunk:
1836
break
1837
download_hash.update(chunk)
1838
fp.close()
1839
temp_location = target_file
1840
logger.notify('Using download cache from %s' % target_file)
1841
else:
1842
try:
1843
resp = urllib2.urlopen(target_url)
1844
except urllib2.HTTPError, e:
1845
logger.fatal("HTTP error %s while getting %s" % (e.code, link))
1846
raise
1847
except IOError, e:
1848
# Typically an FTP error
1849
logger.fatal("Error %s while getting %s" % (e, link))
1850
raise
1851
content_type = resp.info()['content-type']
1852
filename = link.filename
1853
ext = splitext(filename)
1854
if not ext:
1855
ext = mimetypes.guess_extension(content_type)
1856
filename += ext
1857
temp_location = os.path.join(dir, filename)
1858
fp = open(temp_location, 'wb')
1859
if md5_hash:
1860
download_hash = md5()
1861
try:
1862
total_length = int(resp.info()['content-length'])
1863
except (ValueError, KeyError):
1864
total_length = 0
1865
downloaded = 0
1866
show_progress = total_length > 40*1000 or not total_length
1867
show_url = link.show_url
1868
try:
1869
if show_progress:
1870
## FIXME: the URL can get really long in this message:
1871
if total_length:
1872
logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length)))
1873
else:
1874
logger.start_progress('Downloading %s (unknown size): ' % show_url)
1875
else:
1876
logger.notify('Downloading %s' % show_url)
1877
logger.debug('Downloading from URL %s' % link)
1878
while 1:
1879
chunk = resp.read(4096)
1880
if not chunk:
1881
break
1882
downloaded += len(chunk)
1883
if show_progress:
1884
if not total_length:
1885
logger.show_progress('%s' % format_size(downloaded))
1886
else:
1887
logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded)))
1888
if md5_hash:
1889
download_hash.update(chunk)
1890
fp.write(chunk)
1891
fp.close()
1892
finally:
1893
if show_progress:
1894
logger.end_progress('%s downloaded' % format_size(downloaded))
1895
if md5_hash:
1896
download_hash = download_hash.hexdigest()
1897
if download_hash != md5_hash:
1898
logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!"
1899
% (link, download_hash, md5_hash))
1900
raise InstallationError('Bad MD5 hash for package %s' % link)
1901
self.unpack_file(temp_location, location, content_type, link)
1902
if target_file and target_file != temp_location:
1903
logger.notify('Storing download in cache at %s' % display_path(target_file))
1904
shutil.copyfile(temp_location, target_file)
1905
fp = open(target_file+'.content-type', 'w')
1906
fp.write(content_type)
1907
fp.close()
1908
os.unlink(temp_location)
1909
1910
def unpack_file(self, filename, location, content_type, link):
1911
if (content_type == 'application/zip'
1912
or filename.endswith('.zip')
1913
or filename.endswith('.pybundle')
1914
or zipfile.is_zipfile(filename)):
1915
self.unzip_file(filename, location, flatten=not filename.endswith('.pybundle'))
1916
elif (content_type == 'application/x-gzip'
1917
or tarfile.is_tarfile(filename)
1918
or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz')):
1919
self.untar_file(filename, location)
1920
elif (content_type.startswith('text/html')
1921
and is_svn_page(file_contents(filename))):
1922
# We don't really care about this
1923
Subversion('svn+' + link.url).unpack(location)
1924
else:
1925
## FIXME: handle?
1926
## FIXME: magic signatures?
1927
logger.fatal('Cannot unpack file %s (downloaded from %s, content-type: %s); cannot detect archive format'
1928
% (filename, location, content_type))
1929
raise InstallationError('Cannot determine archive format of %s' % location)
1930
1931
def unzip_file(self, filename, location, flatten=True):
1932
"""Unzip the file (zip file located at filename) to the destination
1933
location"""
1934
if not os.path.exists(location):
1935
os.makedirs(location)
1936
zipfp = open(filename, 'rb')
1937
try:
1938
zip = zipfile.ZipFile(zipfp)
1939
leading = has_leading_dir(zip.namelist()) and flatten
1940
for name in zip.namelist():
1941
data = zip.read(name)
1942
fn = name
1943
if leading:
1944
fn = split_leading_dir(name)[1]
1945
fn = os.path.join(location, fn)
1946
dir = os.path.dirname(fn)
1947
if not os.path.exists(dir):
1948
os.makedirs(dir)
1949
if fn.endswith('/'):
1950
# A directory
1951
if not os.path.exists(fn):
1952
os.makedirs(fn)
1953
else:
1954
fp = open(fn, 'wb')
1955
try:
1956
fp.write(data)
1957
finally:
1958
fp.close()
1959
finally:
1960
zipfp.close()
1961
1962
def untar_file(self, filename, location):
1963
"""Untar the file (tar file located at filename) to the destination location"""
1964
if not os.path.exists(location):
1965
os.makedirs(location)
1966
if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
1967
mode = 'r:gz'
1968
elif filename.lower().endswith('.bz2'):
1969
mode = 'r:bz2'
1970
elif filename.lower().endswith('.tar'):
1971
mode = 'r'
1972
else:
1973
logger.warn('Cannot determine compression type for file %s' % filename)
1974
mode = 'r:*'
1975
tar = tarfile.open(filename, mode)
1976
try:
1977
leading = has_leading_dir([member.name for member in tar.getmembers()])
1978
for member in tar.getmembers():
1979
fn = member.name
1980
if leading:
1981
fn = split_leading_dir(fn)[1]
1982
path = os.path.join(location, fn)
1983
if member.isdir():
1984
if not os.path.exists(path):
1985
os.makedirs(path)
1986
else:
1987
try:
1988
fp = tar.extractfile(member)
1989
except KeyError, e:
1990
# Some corrupt tar files seem to produce this
1991
# (specifically bad symlinks)
1992
logger.warn(
1993
'In the tar file %s the member %s is invalid: %s'
1994
% (filename, member.name, e))
1995
continue
1996
if not os.path.exists(os.path.dirname(path)):
1997
os.makedirs(os.path.dirname(path))
1998
destfp = open(path, 'wb')
1999
try:
2000
shutil.copyfileobj(fp, destfp)
2001
finally:
2002
destfp.close()
2003
fp.close()
2004
finally:
2005
tar.close()
2006
2007
def install(self, install_options):
2008
"""Install everything in this set (after having downloaded and unpacked the packages)"""
2009
requirements = sorted(self.requirements.values(), key=lambda p: p.name.lower())
2010
logger.notify('Installing collected packages: %s' % (', '.join([req.name for req in requirements])))
2011
logger.indent += 2
2012
try:
2013
for requirement in self.requirements.values():
2014
if requirement.satisfied_by is not None:
2015
# Already installed
2016
continue
2017
requirement.install(install_options)
2018
requirement.remove_temporary_source()
2019
finally:
2020
logger.indent -= 2
2021
2022
def create_bundle(self, bundle_filename):
2023
## FIXME: can't decide which is better; zip is easier to read
2024
## random files from, but tar.bz2 is smaller and not as lame a
2025
## format.
2026
2027
## FIXME: this file should really include a manifest of the
2028
## packages, maybe some other metadata files. It would make
2029
## it easier to detect as well.
2030
zip = zipfile.ZipFile(bundle_filename, 'w', zipfile.ZIP_DEFLATED)
2031
vcs_dirs = []
2032
for dir, basename in (self.build_dir, 'build'), (self.src_dir, 'src'):
2033
dir = os.path.normcase(os.path.abspath(dir))
2034
for dirpath, dirnames, filenames in os.walk(dir):
2035
for backend in vcs.backends:
2036
vcs_backend = backend()
2037
vcs_url = vcs_rev = None
2038
if vcs_backend.dirname in dirnames:
2039
for vcs_dir in vcs_dirs:
2040
if dirpath.startswith(vcs_dir):
2041
# vcs bundle file already in parent directory
2042
break
2043
else:
2044
vcs_url, vcs_rev = vcs_backend.get_info(
2045
os.path.join(dir, dirpath))
2046
vcs_dirs.append(dirpath)
2047
vcs_bundle_file = vcs_backend.bundle_file
2048
vcs_guide = vcs_backend.guide % {'url': vcs_url,
2049
'rev': vcs_rev}
2050
dirnames.remove(vcs_backend.dirname)
2051
break
2052
if 'pip-egg-info' in dirnames:
2053
dirnames.remove('pip-egg-info')
2054
for dirname in dirnames:
2055
dirname = os.path.join(dirpath, dirname)
2056
name = self._clean_zip_name(dirname, dir)
2057
zip.writestr(basename + '/' + name + '/', '')
2058
for filename in filenames:
2059
if filename == 'pip-delete-this-directory.txt':
2060
continue
2061
filename = os.path.join(dirpath, filename)
2062
name = self._clean_zip_name(filename, dir)
2063
zip.write(filename, basename + '/' + name)
2064
if vcs_url:
2065
name = os.path.join(dirpath, vcs_bundle_file)
2066
name = self._clean_zip_name(name, dir)
2067
zip.writestr(basename + '/' + name, vcs_guide)
2068
2069
zip.writestr('pip-manifest.txt', self.bundle_requirements())
2070
zip.close()
2071
# Unlike installation, this will always delete the build directories
2072
logger.info('Removing temporary build dir %s and source dir %s'
2073
% (self.build_dir, self.src_dir))
2074
for dir in self.build_dir, self.src_dir:
2075
if os.path.exists(dir):
2076
shutil.rmtree(dir)
2077
2078
2079
BUNDLE_HEADER = '''\
2080
# This is a pip bundle file, that contains many source packages
2081
# that can be installed as a group. You can install this like:
2082
# pip this_file.zip
2083
# The rest of the file contains a list of all the packages included:
2084
'''
2085
2086
def bundle_requirements(self):
2087
parts = [self.BUNDLE_HEADER]
2088
for req in sorted(
2089
[req for req in self.requirements.values()
2090
if not req.comes_from],
2091
key=lambda x: x.name):
2092
parts.append('%s==%s\n' % (req.name, req.installed_version))
2093
parts.append('# These packages were installed to satisfy the above requirements:\n')
2094
for req in sorted(
2095
[req for req in self.requirements.values()
2096
if req.comes_from],
2097
key=lambda x: x.name):
2098
parts.append('%s==%s\n' % (req.name, req.installed_version))
2099
## FIXME: should we do something with self.unnamed_requirements?
2100
return ''.join(parts)
2101
2102
def _clean_zip_name(self, name, prefix):
2103
assert name.startswith(prefix+'/'), (
2104
"name %r doesn't start with prefix %r" % (name, prefix))
2105
name = name[len(prefix)+1:]
2106
name = name.replace(os.path.sep, '/')
2107
return name
2108
2109
class HTMLPage(object):
2110
"""Represents one page, along with its URL"""
2111
2112
## FIXME: these regexes are horrible hacks:
2113
_homepage_re = re.compile(r'<th>\s*home\s*page', re.I)
2114
_download_re = re.compile(r'<th>\s*download\s+url', re.I)
2115
## These aren't so aweful:
2116
_rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I)
2117
_href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S)
2118
2119
def __init__(self, content, url, headers=None):
2120
self.content = content
2121
self.url = url
2122
self.headers = headers
2123
2124
def __str__(self):
2125
return self.url
2126
2127
@classmethod
2128
def get_page(cls, link, req, cache=None, skip_archives=True):
2129
url = link.url
2130
url = url.split('#', 1)[0]
2131
if cache.too_many_failures(url):
2132
return None
2133
if url.lower().startswith('svn'):
2134
logger.debug('Cannot look at svn URL %s' % link)
2135
return None
2136
if cache is not None:
2137
inst = cache.get_page(url)
2138
if inst is not None:
2139
return inst
2140
try:
2141
if skip_archives:
2142
if cache is not None:
2143
if cache.is_archive(url):
2144
return None
2145
filename = link.filename
2146
for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']:
2147
if filename.endswith(bad_ext):
2148
content_type = cls._get_content_type(url)
2149
if content_type.lower().startswith('text/html'):
2150
break
2151
else:
2152
logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type))
2153
if cache is not None:
2154
cache.set_is_archive(url)
2155
return None
2156
logger.debug('Getting page %s' % url)
2157
resp = urllib2.urlopen(url)
2158
real_url = resp.geturl()
2159
headers = resp.info()
2160
inst = cls(resp.read(), real_url, headers)
2161
except (urllib2.HTTPError, urllib2.URLError, socket.timeout, socket.error), e:
2162
desc = str(e)
2163
if isinstance(e, socket.timeout):
2164
log_meth = logger.warn
2165
level =1
2166
desc = 'timed out'
2167
elif isinstance(e, urllib2.URLError):
2168
log_meth = logger.warn
2169
if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout):
2170
desc = 'timed out'
2171
level = 1
2172
else:
2173
level = 2
2174
elif isinstance(e, urllib2.HTTPError) and e.code == 404:
2175
## FIXME: notify?
2176
log_meth = logger.info
2177
level = 2
2178
else:
2179
log_meth = logger.warn
2180
level = 1
2181
log_meth('Could not fetch URL %s: %s' % (link, desc))
2182
log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req))
2183
if cache is not None:
2184
cache.add_page_failure(url, level)
2185
return None
2186
if cache is not None:
2187
cache.add_page([url, real_url], inst)
2188
return inst
2189
2190
@staticmethod
2191
def _get_content_type(url):
2192
"""Get the Content-Type of the given url, using a HEAD request"""
2193
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
2194
if scheme == 'http':
2195
ConnClass = httplib.HTTPConnection
2196
elif scheme == 'https':
2197
ConnClass = httplib.HTTPSConnection
2198
else:
2199
## FIXME: some warning or something?
2200
## assertion error?
2201
return ''
2202
if query:
2203
path += '?' + query
2204
conn = ConnClass(netloc)
2205
try:
2206
conn.request('HEAD', path, headers={'Host': netloc})
2207
resp = conn.getresponse()
2208
if resp.status != 200:
2209
## FIXME: doesn't handle redirects
2210
return ''
2211
return resp.getheader('Content-Type') or ''
2212
finally:
2213
conn.close()
2214
2215
@property
2216
def links(self):
2217
"""Yields all links in the page"""
2218
for match in self._href_re.finditer(self.content):
2219
url = match.group(1) or match.group(2) or match.group(3)
2220
url = self.clean_link(urlparse.urljoin(self.url, url))
2221
yield Link(url, self)
2222
2223
def rel_links(self):
2224
for url in self.explicit_rel_links():
2225
yield url
2226
for url in self.scraped_rel_links():
2227
yield url
2228
2229
def explicit_rel_links(self, rels=('homepage', 'download')):
2230
"""Yields all links with the given relations"""
2231
for match in self._rel_re.finditer(self.content):
2232
found_rels = match.group(1).lower().split()
2233
for rel in rels:
2234
if rel in found_rels:
2235
break
2236
else:
2237
continue
2238
match = self._href_re.search(match.group(0))
2239
if not match:
2240
continue
2241
url = match.group(1) or match.group(2) or match.group(3)
2242
url = self.clean_link(urlparse.urljoin(self.url, url))
2243
yield Link(url, self)
2244
2245
def scraped_rel_links(self):
2246
for regex in (self._homepage_re, self._download_re):
2247
match = regex.search(self.content)
2248
if not match:
2249
continue
2250
href_match = self._href_re.search(self.content, pos=match.end())
2251
if not href_match:
2252
continue
2253
url = match.group(1) or match.group(2) or match.group(3)
2254
if not url:
2255
continue
2256
url = self.clean_link(urlparse.urljoin(self.url, url))
2257
yield Link(url, self)
2258
2259
_clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
2260
2261
def clean_link(self, url):
2262
"""Makes sure a link is fully encoded. That is, if a ' ' shows up in
2263
the link, it will be rewritten to %20 (while not over-quoting
2264
% or other characters)."""
2265
return self._clean_re.sub(
2266
lambda match: '%%%2x' % ord(match.group(0)), url)
2267
2268
class PageCache(object):
2269
"""Cache of HTML pages"""
2270
2271
failure_limit = 3
2272
2273
def __init__(self):
2274
self._failures = {}
2275
self._pages = {}
2276
self._archives = {}
2277
2278
def too_many_failures(self, url):
2279
return self._failures.get(url, 0) >= self.failure_limit
2280
2281
def get_page(self, url):
2282
return self._pages.get(url)
2283
2284
def is_archive(self, url):
2285
return self._archives.get(url, False)
2286
2287
def set_is_archive(self, url, value=True):
2288
self._archives[url] = value
2289
2290
def add_page_failure(self, url, level):
2291
self._failures[url] = self._failures.get(url, 0)+level
2292
2293
def add_page(self, urls, page):
2294
for url in urls:
2295
self._pages[url] = page
2296
2297
class Link(object):
2298
2299
def __init__(self, url, comes_from=None):
2300
self.url = url
2301
self.comes_from = comes_from
2302
2303
def __str__(self):
2304
if self.comes_from:
2305
return '%s (from %s)' % (self.url, self.comes_from)
2306
else:
2307
return self.url
2308
2309
def __repr__(self):
2310
return '<Link %s>' % self
2311
2312
@property
2313
def filename(self):
2314
url = self.url
2315
url = url.split('#', 1)[0]
2316
url = url.split('?', 1)[0]
2317
url = url.rstrip('/')
2318
name = posixpath.basename(url)
2319
assert name, (
2320
'URL %r produced no filename' % url)
2321
return name
2322
2323
@property
2324
def scheme(self):
2325
return urlparse.urlsplit(self.url)[0]
2326
2327
@property
2328
def path(self):
2329
return urlparse.urlsplit(self.url)[2]
2330
2331
def splitext(self):
2332
return splitext(posixpath.basename(self.path.rstrip('/')))
2333
2334
_egg_fragment_re = re.compile(r'#egg=([^&]*)')
2335
2336
@property
2337
def egg_fragment(self):
2338
match = self._egg_fragment_re.search(self.url)
2339
if not match:
2340
return None
2341
return match.group(1)
2342
2343
_md5_re = re.compile(r'md5=([a-f0-9]+)')
2344
2345
@property
2346
def md5_hash(self):
2347
match = self._md5_re.search(self.url)
2348
if match:
2349
return match.group(1)
2350
return None
2351
2352
@property
2353
def show_url(self):
2354
return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
2355
2356
############################################################
2357
## Writing freeze files
2358
2359
2360
class FrozenRequirement(object):
2361
2362
def __init__(self, name, req, editable, comments=()):
2363
self.name = name
2364
self.req = req
2365
self.editable = editable
2366
self.comments = comments
2367
2368
_rev_re = re.compile(r'-r(\d+)$')
2369
_date_re = re.compile(r'-(20\d\d\d\d\d\d)$')
2370
2371
@classmethod
2372
def from_dist(cls, dist, dependency_links, find_tags=False):
2373
location = os.path.normcase(os.path.abspath(dist.location))
2374
comments = []
2375
if vcs.get_backend_name(location):
2376
editable = True
2377
req = get_src_requirement(dist, location, find_tags)
2378
if req is None:
2379
logger.warn('Could not determine repository location of %s' % location)
2380
comments.append('## !! Could not determine repository location')
2381
req = dist.as_requirement()
2382
editable = False
2383
else:
2384
editable = False
2385
req = dist.as_requirement()
2386
specs = req.specs
2387
assert len(specs) == 1 and specs[0][0] == '=='
2388
version = specs[0][1]
2389
ver_match = cls._rev_re.search(version)
2390
date_match = cls._date_re.search(version)
2391
if ver_match or date_match:
2392
svn_backend = vcs.get_backend('svn')
2393
if svn_backend:
2394
svn_location = svn_backend(
2395
).get_location(dist, dependency_links)
2396
if not svn_location:
2397
logger.warn(
2398
'Warning: cannot find svn location for %s' % req)
2399
comments.append('## FIXME: could not find svn URL in dependency_links for this package:')
2400
else:
2401
comments.append('# Installing as editable to satisfy requirement %s:' % req)
2402
if ver_match:
2403
rev = ver_match.group(1)
2404
else:
2405
rev = '{%s}' % date_match.group(1)
2406
editable = True
2407
req = 'svn+%s@%s#egg=%s' % (svn_location, rev, cls.egg_name(dist))
2408
return cls(dist.project_name, req, editable, comments)
2409
2410
@staticmethod
2411
def egg_name(dist):
2412
name = dist.egg_name()
2413
match = re.search(r'-py\d\.\d$', name)
2414
if match:
2415
name = name[:match.start()]
2416
return name
2417
2418
def __str__(self):
2419
req = self.req
2420
if self.editable:
2421
req = '-e %s' % req
2422
return '\n'.join(list(self.comments)+[str(req)])+'\n'
2423
2424
class VersionControl(object):
2425
name = ''
2426
2427
def __init__(self, url=None, *args, **kwargs):
2428
self.url = url
2429
super(VersionControl, self).__init__(*args, **kwargs)
2430
2431
def _filter(self, line):
2432
return (Logger.INFO, line)
2433
2434
def get_url_rev(self):
2435
"""
2436
Returns the correct repository URL and revision by parsing the given
2437
repository URL
2438
"""
2439
url = self.url.split('+', 1)[1]
2440
scheme, netloc, path, query, frag = urlparse.urlsplit(url)
2441
if '@' in path:
2442
path, rev = path.split('@', 1)
2443
else:
2444
rev = None
2445
url = urlparse.urlunsplit((scheme, netloc, path, query, ''))
2446
return url, rev
2447
2448
def obtain(self, dest):
2449
raise NotImplementedError
2450
2451
def unpack(self, location):
2452
raise NotImplementedError
2453
2454
def get_src_requirement(self, dist, location, find_tags=False):
2455
raise NotImplementedError
2456
2457
_svn_xml_url_re = re.compile('url="([^"]+)"')
2458
_svn_rev_re = re.compile('committed-rev="(\d+)"')
2459
_svn_url_re = re.compile(r'URL: (.+)')
2460
_svn_revision_re = re.compile(r'Revision: (.+)')
2461
2462
class Subversion(VersionControl):
2463
name = 'svn'
2464
dirname = '.svn'
2465
schemes = ('svn', 'svn+ssh')
2466
bundle_file = 'svn-checkout.txt'
2467
guide = ('# This was an svn checkout; to make it a checkout again run:\n'
2468
'svn checkout --force -r %(rev)s %(url)s .\n')
2469
2470
def get_info(self, location):
2471
"""Returns (url, revision), where both are strings"""
2472
assert not location.rstrip('/').endswith('.svn'), 'Bad directory: %s' % location
2473
output = call_subprocess(
2474
['svn', 'info', location], show_stdout=False, extra_environ={'LANG': 'C'})
2475
match = _svn_url_re.search(output)
2476
if not match:
2477
logger.warn('Cannot determine URL of svn checkout %s' % display_path(location))
2478
logger.info('Output that cannot be parsed: \n%s' % output)
2479
return 'unknown', 'unknown'
2480
url = match.group(1).strip()
2481
match = _svn_revision_re.search(output)
2482
if not match:
2483
logger.warn('Cannot determine revision of svn checkout %s' % display_path(location))
2484
logger.info('Output that cannot be parsed: \n%s' % output)
2485
return url, 'unknown'
2486
return url, match.group(1)
2487
2488
def parse_checkout_text(self, text):
2489
for line in text.splitlines():
2490
if not line.strip() or line.strip().startswith('#'):
2491
continue
2492
match = re.search(r'^-r\s*([^ ])?', line)
2493
if not match:
2494
return None, None
2495
rev = match.group(1)
2496
rest = line[match.end():].strip().split(None, 1)[0]
2497
return rest, rev
2498
return None, None
2499
2500
def unpack(self, location):
2501
"""Check out the svn repository at the url to the destination location"""
2502
url, rev = self.get_url_rev()
2503
logger.notify('Checking out svn repository %s to %s' % (url, location))
2504
logger.indent += 2
2505
try:
2506
if os.path.exists(location):
2507
# Subversion doesn't like to check out over an existing directory
2508
# --force fixes this, but was only added in svn 1.5
2509
os.rmdir(location)
2510
call_subprocess(
2511
['svn', 'checkout', url, location],
2512
filter_stdout=self._filter, show_stdout=False)
2513
finally:
2514
logger.indent -= 2
2515
2516
def obtain(self, dest):
2517
url, rev = self.get_url_rev()
2518
if rev:
2519
rev_options = ['-r', rev]
2520
rev_display = ' (to revision %s)' % rev
2521
else:
2522
rev_options = []
2523
rev_display = ''
2524
checkout = True
2525
if os.path.exists(os.path.join(dest, '.svn')):
2526
existing_url = self.get_info(dest)[0]
2527
checkout = False
2528
if existing_url == url:
2529
logger.info('Checkout in %s exists, and has correct URL (%s)'
2530
% (display_path(dest), url))
2531
logger.notify('Updating checkout %s%s'
2532
% (display_path(dest), rev_display))
2533
call_subprocess(
2534
['svn', 'update'] + rev_options + [dest])
2535
else:
2536
logger.warn('svn checkout in %s exists with URL %s'
2537
% (display_path(dest), existing_url))
2538
logger.warn('The plan is to install the svn repository %s'
2539
% url)
2540
response = ask('What to do? (s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b'))
2541
if response == 's':
2542
logger.notify('Switching checkout %s to %s%s'
2543
% (display_path(dest), url, rev_display))
2544
call_subprocess(
2545
['svn', 'switch'] + rev_options + [url, dest])
2546
elif response == 'i':
2547
# do nothing
2548
pass
2549
elif response == 'w':
2550
logger.warn('Deleting %s' % display_path(dest))
2551
shutil.rmtree(dest)
2552
checkout = True
2553
elif response == 'b':
2554
dest_dir = backup_dir(dest)
2555
logger.warn('Backing up %s to %s'
2556
% display_path(dest, dest_dir))
2557
shutil.move(dest, dest_dir)
2558
checkout = True
2559
if checkout:
2560
logger.notify('Checking out %s%s to %s'
2561
% (url, rev_display, display_path(dest)))
2562
call_subprocess(
2563
['svn', 'checkout', '-q'] + rev_options + [url, dest])
2564
2565
def get_location(self, dist, dependency_links):
2566
egg_fragment_re = re.compile(r'#egg=(.*)$')
2567
for url in dependency_links:
2568
egg_fragment = Link(url).egg_fragment
2569
if not egg_fragment:
2570
continue
2571
if '-' in egg_fragment:
2572
## FIXME: will this work when a package has - in the name?
2573
key = '-'.join(egg_fragment.split('-')[:-1]).lower()
2574
else:
2575
key = egg_fragment
2576
if key == dist.key:
2577
return url.split('#', 1)[0]
2578
return None
2579
2580
def get_revision(self, location):
2581
"""
2582
Return the maximum revision for all files under a given location
2583
"""
2584
# Note: taken from setuptools.command.egg_info
2585
revision = 0
2586
2587
for base, dirs, files in os.walk(location):
2588
if '.svn' not in dirs:
2589
dirs[:] = []
2590
continue # no sense walking uncontrolled subdirs
2591
dirs.remove('.svn')
2592
entries_fn = os.path.join(base, '.svn', 'entries')
2593
if not os.path.exists(entries_fn):
2594
## FIXME: should we warn?
2595
continue
2596
f = open(entries_fn)
2597
data = f.read()
2598
f.close()
2599
2600
if data.startswith('8') or data.startswith('9'):
2601
data = map(str.splitlines,data.split('\n\x0c\n'))
2602
del data[0][0] # get rid of the '8'
2603
dirurl = data[0][3]
2604
revs = [int(d[9]) for d in data if len(d)>9 and d[9]]+[0]
2605
if revs:
2606
localrev = max(revs)
2607
else:
2608
localrev = 0
2609
elif data.startswith('<?xml'):
2610
dirurl = _svn_xml_url_re.search(data).group(1) # get repository URL
2611
revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)]+[0]
2612
if revs:
2613
localrev = max(revs)
2614
else:
2615
localrev = 0
2616
else:
2617
logger.warn("Unrecognized .svn/entries format; skipping %s", base)
2618
dirs[:] = []
2619
continue
2620
if base == location:
2621
base_url = dirurl+'/' # save the root url
2622
elif not dirurl.startswith(base_url):
2623
dirs[:] = []
2624
continue # not part of the same svn tree, skip it
2625
revision = max(revision, localrev)
2626
return revision
2627
2628
def get_url(self, location):
2629
# In cases where the source is in a subdirectory, not alongside setup.py
2630
# we have to look up in the location until we find a real setup.py
2631
orig_location = location
2632
while not os.path.exists(os.path.join(location, 'setup.py')):
2633
last_location = location
2634
location = os.path.dirname(location)
2635
if location == last_location:
2636
# We've traversed up to the root of the filesystem without finding setup.py
2637
logger.warn("Could not find setup.py for directory %s (tried all parent directories)"
2638
% orig_location)
2639
return None
2640
f = open(os.path.join(location, '.svn', 'entries'))
2641
data = f.read()
2642
f.close()
2643
if data.startswith('8') or data.startswith('9'):
2644
data = map(str.splitlines,data.split('\n\x0c\n'))
2645
del data[0][0] # get rid of the '8'
2646
return data[0][3]
2647
elif data.startswith('<?xml'):
2648
match = _svn_xml_url_re.search(data)
2649
if not match:
2650
raise ValueError('Badly formatted data: %r' % data)
2651
return match.group(1) # get repository URL
2652
else:
2653
logger.warn("Unrecognized .svn/entries format in %s" % location)
2654
# Or raise exception?
2655
return None
2656
2657
def get_tag_revs(self, svn_tag_url):
2658
stdout = call_subprocess(
2659
['svn', 'ls', '-v', svn_tag_url], show_stdout=False)
2660
results = []
2661
for line in stdout.splitlines():
2662
parts = line.split()
2663
rev = int(parts[0])
2664
tag = parts[-1].strip('/')
2665
results.append((tag, rev))
2666
return results
2667
2668
def find_tag_match(self, rev, tag_revs):
2669
best_match_rev = None
2670
best_tag = None
2671
for tag, tag_rev in tag_revs:
2672
if (tag_rev > rev and
2673
(best_match_rev is None or best_match_rev > tag_rev)):
2674
# FIXME: Is best_match > tag_rev really possible?
2675
# or is it a sign something is wacky?
2676
best_match_rev = tag_rev
2677
best_tag = tag
2678
return best_tag
2679
2680
def get_src_requirement(self, dist, location, find_tags=False):
2681
repo = self.get_url(location)
2682
if repo is None:
2683
return None
2684
parts = repo.split('/')
2685
## FIXME: why not project name?
2686
egg_project_name = dist.egg_name().split('-', 1)[0]
2687
if parts[-2] in ('tags', 'tag'):
2688
# It's a tag, perfect!
2689
return 'svn+%s#egg=%s-%s' % (repo, egg_project_name, parts[-1])
2690
elif parts[-2] in ('branches', 'branch'):
2691
# It's a branch :(
2692
rev = self.get_revision(location)
2693
return 'svn+%s@%s#egg=%s%s-r%s' % (repo, rev, dist.egg_name(), parts[-1], rev)
2694
elif parts[-1] == 'trunk':
2695
# Trunk :-/
2696
rev = self.get_revision(location)
2697
if find_tags:
2698
tag_url = '/'.join(parts[:-1]) + '/tags'
2699
tag_revs = self.get_tag_revs(tag_url)
2700
match = self.find_tag_match(rev, tag_revs)
2701
if match:
2702
logger.notify('trunk checkout %s seems to be equivalent to tag %s' % match)
2703
return 'svn+%s/%s#egg=%s-%s' % (tag_url, match, egg_project_name, match)
2704
return 'svn+%s@%s#egg=%s-dev' % (repo, rev, dist.egg_name())
2705
else:
2706
# Don't know what it is
2707
logger.warn('svn URL does not fit normal structure (tags/branches/trunk): %s' % repo)
2708
rev = self.get_revision(location)
2709
return 'svn+%s@%s#egg=%s-dev' % (repo, rev, egg_project_name)
2710
2711
vcs.register(Subversion)
2712
2713
2714
class Git(VersionControl):
2715
name = 'git'
2716
dirname = '.git'
2717
schemes = ('git', 'git+http', 'git+ssh')
2718
bundle_file = 'git-clone.txt'
2719
guide = ('# This was a Git repo; to make it a repo again run:\n'
2720
'git init\ngit remote add origin %(url)s -f\ngit checkout %(rev)s\n')
2721
2722
def get_info(self, location):
2723
"""Returns (url, revision), where both are strings"""
2724
assert not location.rstrip('/').endswith('.git'), 'Bad directory: %s' % location
2725
return self.get_url(location), self.get_revision(location)
2726
2727
def parse_clone_text(self, text):
2728
url = rev = None
2729
for line in text.splitlines():
2730
if not line.strip() or line.strip().startswith('#'):
2731
continue
2732
url_match = re.search(r'git\s*remote\s*add\s*origin(.*)\s*-f', line)
2733
if url_match:
2734
url = url_match.group(1).strip()
2735
rev_match = re.search(r'^git\s*checkout\s*-q\s*(.*)\s*', line)
2736
if rev_match:
2737
rev = rev_match.group(1).strip()
2738
if url and rev:
2739
return url, rev
2740
return None, None
2741
2742
def unpack(self, location):
2743
"""Clone the Git repository at the url to the destination location"""
2744
url, rev = self.get_url_rev()
2745
logger.notify('Cloning Git repository %s to %s' % (url, location))
2746
logger.indent += 2
2747
try:
2748
if os.path.exists(location):
2749
os.rmdir(location)
2750
call_subprocess(
2751
[GIT_CMD, 'clone', url, location],
2752
filter_stdout=self._filter, show_stdout=False)
2753
finally:
2754
logger.indent -= 2
2755
2756
def obtain(self, dest):
2757
url, rev = self.get_url_rev()
2758
if rev:
2759
rev_options = [rev]
2760
rev_display = ' (to revision %s)' % rev
2761
else:
2762
rev_options = ['master']
2763
rev_display = ''
2764
clone = True
2765
if os.path.exists(os.path.join(dest, '.git')):
2766
existing_url = self.get_url(dest)
2767
clone = False
2768
if existing_url == url:
2769
logger.info('Clone in %s exists, and has correct URL (%s)'
2770
% (display_path(dest), url))
2771
logger.notify('Updating clone %s%s'
2772
% (display_path(dest), rev_display))
2773
call_subprocess([GIT_CMD, 'fetch', '-q'], cwd=dest)
2774
call_subprocess(
2775
[GIT_CMD, 'checkout', '-q', '-f'] + rev_options, cwd=dest)
2776
else:
2777
logger.warn('Git clone in %s exists with URL %s'
2778
% (display_path(dest), existing_url))
2779
logger.warn('The plan is to install the Git repository %s'
2780
% url)
2781
response = ask('What to do? (s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b'))
2782
if response == 's':
2783
logger.notify('Switching clone %s to %s%s'
2784
% (display_path(dest), url, rev_display))
2785
call_subprocess(
2786
[GIT_CMD, 'config', 'remote.origin.url', url], cwd=dest)
2787
call_subprocess(
2788
[GIT_CMD, 'checkout', '-q'] + rev_options, cwd=dest)
2789
elif response == 'i':
2790
# do nothing
2791
pass
2792
elif response == 'w':
2793
logger.warn('Deleting %s' % display_path(dest))
2794
shutil.rmtree(dest)
2795
clone = True
2796
elif response == 'b':
2797
dest_dir = backup_dir(dest)
2798
logger.warn('Backing up %s to %s' % (display_path(dest), dest_dir))
2799
shutil.move(dest, dest_dir)
2800
clone = True
2801
if clone:
2802
logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest)))
2803
call_subprocess(
2804
[GIT_CMD, 'clone', '-q', url, dest])
2805
call_subprocess(
2806
[GIT_CMD, 'checkout', '-q'] + rev_options, cwd=dest)
2807
2808
def get_url(self, location):
2809
url = call_subprocess(
2810
[GIT_CMD, 'config', 'remote.origin.url'],
2811
show_stdout=False, cwd=location)
2812
return url.strip()
2813
2814
def get_revision(self, location):
2815
current_rev = call_subprocess(
2816
[GIT_CMD, 'rev-parse', 'HEAD'], show_stdout=False, cwd=location)
2817
return current_rev.strip()
2818
2819
def get_master_revision(self, location):
2820
master_rev = call_subprocess(
2821
[GIT_CMD, 'rev-parse', 'master'], show_stdout=False, cwd=location)
2822
return master_rev.strip()
2823
2824
def get_tag_revs(self, location):
2825
tags = call_subprocess(
2826
[GIT_CMD, 'tag'], show_stdout=False, cwd=location)
2827
tag_revs = []
2828
for line in tags.splitlines():
2829
tag = line.strip()
2830
rev = call_subprocess(
2831
[GIT_CMD, 'rev-parse', tag], show_stdout=False, cwd=location)
2832
tag_revs.append((rev.strip(), tag))
2833
tag_revs = dict(tag_revs)
2834
return tag_revs
2835
2836
def get_branch_revs(self, location):
2837
branches = call_subprocess(
2838
[GIT_CMD, 'branch', '-r'], show_stdout=False, cwd=location)
2839
branch_revs = []
2840
for line in branches.splitlines():
2841
branch = "".join([b for b in line.split() if b != '*'])
2842
rev = call_subprocess(
2843
[GIT_CMD, 'rev-parse', branch], show_stdout=False, cwd=location)
2844
branch_revs.append((rev.strip(), branch))
2845
branch_revs = dict(branch_revs)
2846
return branch_revs
2847
2848
def get_src_requirement(self, dist, location, find_tags):
2849
repo = self.get_url(location)
2850
if not repo.lower().startswith('git:'):
2851
repo = 'git+' + repo
2852
egg_project_name = dist.egg_name().split('-', 1)[0]
2853
if not repo:
2854
return None
2855
current_rev = self.get_revision(location)
2856
tag_revs = self.get_tag_revs(location)
2857
master_rev = self.get_master_revision(location)
2858
branch_revs = self.get_branch_revs(location)
2859
2860
if current_rev in tag_revs:
2861
# It's a tag, perfect!
2862
tag = tag_revs.get(current_rev, current_rev)
2863
return '%s@%s#egg=%s-%s' % (repo, tag, egg_project_name, tag)
2864
elif current_rev in branch_revs:
2865
# It's the head of a branch, nice too.
2866
branch = branch_revs.get(current_rev, current_rev)
2867
return '%s@%s#egg=%s-%s' % (repo, current_rev, dist.egg_name(), current_rev)
2868
elif current_rev == master_rev:
2869
if find_tags:
2870
if current_rev in tag_revs:
2871
tag = tag_revs.get(current_rev, current_rev)
2872
logger.notify('Revision %s seems to be equivalent to tag %s' % (current_rev, tag))
2873
return '%s@%s#egg=%s-%s' % (repo, tag, egg_project_name, tag)
2874
return '%s@%s#egg=%s-dev' % (repo, master_rev, dist.egg_name())
2875
else:
2876
# Don't know what it is
2877
logger.warn('Git URL does not fit normal structure: %s' % repo)
2878
return '%s@%s#egg=%s-dev' % (repo, current_rev, egg_project_name)
2879
2880
vcs.register(Git)
2881
2882
2883
class Mercurial(VersionControl):
2884
name = 'hg'
2885
dirname = '.hg'
2886
schemes = ('hg', 'hg+http', 'hg+ssh')
2887
bundle_file = 'hg-clone.txt'
2888
guide = ('# This was a Mercurial repo; to make it a repo again run:\n'
2889
'hg init\nhg pull %(url)s\nhg update -r %(rev)s\n')
2890
2891
def get_info(self, location):
2892
"""Returns (url, revision), where both are strings"""
2893
assert not location.rstrip('/').endswith('.hg'), 'Bad directory: %s' % location
2894
return self.get_url(location), self.get_revision(location)
2895
2896
def parse_clone_text(self, text):
2897
url = rev = None
2898
for line in text.splitlines():
2899
if not line.strip() or line.strip().startswith('#'):
2900
continue
2901
url_match = re.search(r'hg\s*pull\s*(.*)\s*', line)
2902
if url_match:
2903
url = url_match.group(1).strip()
2904
rev_match = re.search(r'^hg\s*update\s*-r\s*(.*)\s*', line)
2905
if rev_match:
2906
rev = rev_match.group(1).strip()
2907
if url and rev:
2908
return url, rev
2909
return None, None
2910
2911
def unpack(self, location):
2912
"""Clone the Hg repository at the url to the destination location"""
2913
url, rev = self.get_url_rev()
2914
logger.notify('Cloning Mercurial repository %s to %s' % (url, location))
2915
logger.indent += 2
2916
try:
2917
if os.path.exists(location):
2918
os.rmdir(location)
2919
call_subprocess(
2920
['hg', 'clone', url, location],
2921
filter_stdout=self._filter, show_stdout=False)
2922
finally:
2923
logger.indent -= 2
2924
2925
def obtain(self, dest):
2926
url, rev = self.get_url_rev()
2927
if rev:
2928
rev_options = [rev]
2929
rev_display = ' (to revision %s)' % rev
2930
else:
2931
rev_options = ['default']
2932
rev_display = ''
2933
clone = True
2934
if os.path.exists(os.path.join(dest, '.hg')):
2935
existing_url = self.get_url(dest)
2936
clone = False
2937
if existing_url == url:
2938
logger.info('Clone in %s exists, and has correct URL (%s)'
2939
% (display_path(dest), url))
2940
logger.notify('Updating clone %s%s'
2941
% (display_path(dest), rev_display))
2942
call_subprocess(['hg', 'fetch', '-q'], cwd=dest)
2943
call_subprocess(
2944
['hg', 'update', '-q'] + rev_options, cwd=dest)
2945
else:
2946
logger.warn('Mercurial clone in %s exists with URL %s'
2947
% (display_path(dest), existing_url))
2948
logger.warn('The plan is to install the Mercurial repository %s'
2949
% url)
2950
response = ask('What to do? (s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b'))
2951
if response == 's':
2952
logger.notify('Switching clone %s to %s%s'
2953
% (display_path(dest), url, rev_display))
2954
repo_config = os.path.join(dest, '.hg/hgrc')
2955
config = ConfigParser.SafeConfigParser()
2956
try:
2957
config_file = open(repo_config, 'wb')
2958
config.readfp(config_file)
2959
config.set('paths', ''.join(rev_options), url)
2960
config.write(config_file)
2961
except (OSError, ConfigParser.NoSectionError):
2962
logger.warn(
2963
'Could not switch Mercurial repository to %s: %s'
2964
% (url, e))
2965
else:
2966
call_subprocess(
2967
['hg', 'update', '-q'] + rev_options, cwd=dest)
2968
elif response == 'i':
2969
# do nothing
2970
pass
2971
elif response == 'w':
2972
logger.warn('Deleting %s' % display_path(dest))
2973
shutil.rmtree(dest)
2974
clone = True
2975
elif response == 'b':
2976
dest_dir = backup_dir(dest)
2977
logger.warn('Backing up %s to %s' % (display_path(dest), dest_dir))
2978
shutil.move(dest, dest_dir)
2979
clone = True
2980
if clone:
2981
logger.notify('Cloning hg %s%s to %s'
2982
% (url, rev_display, display_path(dest)))
2983
call_subprocess(['hg', 'clone', '-q', url, dest])
2984
call_subprocess(['hg', 'update', '-q'] + rev_options, cwd=dest)
2985
2986
def get_url(self, location):
2987
url = call_subprocess(
2988
['hg', 'showconfig', 'paths.default'],
2989
show_stdout=False, cwd=location)
2990
return url.strip()
2991
2992
def get_tip_revision(self, location):
2993
current_rev = call_subprocess(
2994
['hg', 'tip', '--template={rev}'], show_stdout=False, cwd=location)
2995
return current_rev.strip()
2996
2997
def get_tag_revs(self, location):
2998
tags = call_subprocess(
2999
['hg', 'tags'], show_stdout=False, cwd=location)
3000
tag_revs = []
3001
for line in tags.splitlines():
3002
tags_match = re.search(r'([\w-]+)\s*([\d]+):.*$', line)
3003
if tags_match:
3004
tag = tags_match.group(1)
3005
rev = tags_match.group(2)
3006
tag_revs.append((rev.strip(), tag.strip()))
3007
return dict(tag_revs)
3008
3009
def get_branch_revs(self, location):
3010
branches = call_subprocess(
3011
['hg', 'branches'], show_stdout=False, cwd=location)
3012
branch_revs = []
3013
for line in branches.splitlines():
3014
branches_match = re.search(r'([\w-]+)\s*([\d]+):.*$', line)
3015
if branches_match:
3016
branch = branches_match.group(1)
3017
rev = branches_match.group(2)
3018
branch_revs.append((rev.strip(), branch.strip()))
3019
return dict(branch_revs)
3020
3021
def get_revision(self, location):
3022
current_branch = call_subprocess(
3023
['hg', 'branch'], show_stdout=False, cwd=location).strip()
3024
branch_revs = self.get_branch_revs(location)
3025
for branch in branch_revs:
3026
if current_branch == branch_revs[branch]:
3027
return branch
3028
return self.get_tip_revision(location)
3029
3030
def get_src_requirement(self, dist, location, find_tags):
3031
repo = self.get_url(location)
3032
if not repo.lower().startswith('hg:'):
3033
repo = 'hg+' + repo
3034
egg_project_name = dist.egg_name().split('-', 1)[0]
3035
if not repo:
3036
return None
3037
current_rev = self.get_revision(location)
3038
tag_revs = self.get_tag_revs(location)
3039
branch_revs = self.get_branch_revs(location)
3040
tip_rev = self.get_tip_revision(location)
3041
if current_rev in tag_revs:
3042
# It's a tag, perfect!
3043
tag = tag_revs.get(current_rev, current_rev)
3044
return '%s@%s#egg=%s-%s' % (repo, tag, egg_project_name, tag)
3045
elif current_rev in branch_revs:
3046
# It's the tip of a branch, nice too.
3047
branch = branch_revs.get(current_rev, current_rev)
3048
return '%s@%s#egg=%s-%s' % (repo, branch, dist.egg_name(), current_rev)
3049
elif current_rev == tip_rev:
3050
if find_tags:
3051
if current_rev in tag_revs:
3052
tag = tag_revs.get(current_rev, current_rev)
3053
logger.notify('Revision %s seems to be equivalent to tag %s' % (current_rev, tag))
3054
return '%s@%s#egg=%s-%s' % (repo, tag, egg_project_name, tag)
3055
return '%s@%s#egg=%s-dev' % (repo, tip_rev, dist.egg_name())
3056
else:
3057
# Don't know what it is
3058
logger.warn('Mercurial URL does not fit normal structure: %s' % repo)
3059
return '%s@%s#egg=%s-dev' % (repo, current_rev, egg_project_name)
3060
3061
vcs.register(Mercurial)
3062
3063
3064
class Bazaar(VersionControl):
3065
name = 'bzr'
3066
dirname = '.bzr'
3067
bundle_file = 'bzr-branch.txt'
3068
schemes = ('bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp')
3069
guide = ('# This was a Bazaar branch; to make it a branch again run:\n'
3070
'bzr branch -r %(rev)s %(url)s .\n')
3071
3072
def get_info(self, location):
3073
"""Returns (url, revision), where both are strings"""
3074
assert not location.rstrip('/').endswith('.bzr'), 'Bad directory: %s' % location
3075
return self.get_url(location), self.get_revision(location)
3076
3077
def parse_clone_text(self, text):
3078
url = rev = None
3079
for line in text.splitlines():
3080
if not line.strip() or line.strip().startswith('#'):
3081
continue
3082
match = re.search(r'^bzr\s*branch\s*-r\s*(\d*)', line)
3083
if match:
3084
rev = match.group(1).strip()
3085
url = line[match.end():].strip().split(None, 1)[0]
3086
if url and rev:
3087
return url, rev
3088
return None, None
3089
3090
def unpack(self, location):
3091
"""Get the bzr branch at the url to the destination location"""
3092
url, rev = self.get_url_rev()
3093
logger.notify('Checking out bzr repository %s to %s' % (url, location))
3094
logger.indent += 2
3095
try:
3096
if os.path.exists(location):
3097
os.rmdir(location)
3098
call_subprocess(
3099
['bzr', 'branch', url, location],
3100
filter_stdout=self._filter, show_stdout=False)
3101
finally:
3102
logger.indent -= 2
3103
3104
def obtain(self, dest):
3105
url, rev = self.get_url_rev()
3106
if rev:
3107
rev_options = ['-r', rev]
3108
rev_display = ' (to revision %s)' % rev
3109
else:
3110
rev_options = []
3111
rev_display = ''
3112
branch = True
3113
update = False
3114
if os.path.exists(os.path.join(dest, '.bzr')):
3115
existing_url = self.get_url(dest)
3116
branch = False
3117
if existing_url == url:
3118
logger.info('Checkout in %s exists, and has correct URL (%s)'
3119
% (display_path(dest), url))
3120
logger.notify('Updating branch %s%s'
3121
% (display_path(dest), rev_display))
3122
branch = update = True
3123
else:
3124
logger.warn('Bazaar branch in %s exists with URL %s'
3125
% (display_path(dest), existing_url))
3126
logger.warn('The plan is to install the Bazaar repository %s'
3127
% url)
3128
response = ask('What to do? (s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b'))
3129
if response == 's':
3130
logger.notify('Switching branch %s to %s%s'
3131
% (display_path(dest), url, rev_display))
3132
call_subprocess(['bzr', 'switch', url], cwd=dest)
3133
elif response == 'i':
3134
# do nothing
3135
pass
3136
elif response == 'w':
3137
logger.warn('Deleting %s' % display_path(dest))
3138
shutil.rmtree(dest)
3139
branch = True
3140
elif response == 'b':
3141
dest_dir = backup_dir(dest)
3142
logger.warn('Backing up %s to %s' % (display_path(dest), dest_dir))
3143
shutil.move(dest, dest_dir)
3144
branch = True
3145
if branch:
3146
logger.notify('Checking out %s%s to %s'
3147
% (url, rev_display, display_path(dest)))
3148
# FIXME: find a better place to hotfix the URL scheme
3149
# after removing bzr+ from bzr+ssh:// readd it
3150
if url.startswith('ssh://'):
3151
url = 'bzr+' + url
3152
if update:
3153
call_subprocess(
3154
['bzr', 'pull', '-q'] + rev_options + [url], cwd=dest)
3155
else:
3156
call_subprocess(
3157
['bzr', 'branch', '-q'] + rev_options + [url, dest])
3158
3159
def get_url(self, location):
3160
urls = call_subprocess(
3161
['bzr', 'info'], show_stdout=False, cwd=location)
3162
for line in urls.splitlines():
3163
line = line.strip()
3164
for x in ('checkout of branch: ',
3165
'repository branch: ',
3166
'parent branch: '):
3167
if line.startswith(x):
3168
return line.split(x)[1]
3169
return None
3170
3171
def get_revision(self, location):
3172
revision = call_subprocess(
3173
['bzr', 'revno'], show_stdout=False, cwd=location)
3174
return revision.strip()
3175
3176
def get_newest_revision(self, location):
3177
url = self.get_url(location)
3178
revision = call_subprocess(
3179
['bzr', 'revno', url], show_stdout=False, cwd=location)
3180
return revision.strip()
3181
3182
def get_tag_revs(self, location):
3183
tags = call_subprocess(
3184
['bzr', 'tags'], show_stdout=False, cwd=location)
3185
tag_revs = []
3186
for line in tags.splitlines():
3187
tags_match = re.search(r'([.\w-]+)\s*(.*)$', line)
3188
if tags_match:
3189
tag = tags_match.group(1)
3190
rev = tags_match.group(2)
3191
tag_revs.append((rev.strip(), tag.strip()))
3192
return dict(tag_revs)
3193
3194
def get_src_requirement(self, dist, location, find_tags):
3195
repo = self.get_url(location)
3196
if not repo.lower().startswith('bzr:'):
3197
repo = 'bzr+' + repo
3198
egg_project_name = dist.egg_name().split('-', 1)[0]
3199
if not repo:
3200
return None
3201
current_rev = self.get_revision(location)
3202
tag_revs = self.get_tag_revs(location)
3203
newest_rev = self.get_newest_revision(location)
3204
if current_rev in tag_revs:
3205
# It's a tag, perfect!
3206
tag = tag_revs.get(current_rev, current_rev)
3207
return '%s@%s#egg=%s-%s' % (repo, tag, egg_project_name, tag)
3208
elif current_rev == newest_rev:
3209
if find_tags:
3210
if current_rev in tag_revs:
3211
tag = tag_revs.get(current_rev, current_rev)
3212
logger.notify('Revision %s seems to be equivalent to tag %s' % (current_rev, tag))
3213
return '%s@%s#egg=%s-%s' % (repo, tag, egg_project_name, tag)
3214
return '%s@%s#egg=%s-dev' % (repo, newest_rev, dist.egg_name())
3215
else:
3216
# Don't know what it is
3217
logger.warn('Bazaar URL does not fit normal structure: %s' % repo)
3218
return '%s@%s#egg=%s-dev' % (repo, current_rev, egg_project_name)
3219
3220
vcs.register(Bazaar)
3221
3222
def get_src_requirement(dist, location, find_tags):
3223
version_control = vcs.get_backend_from_location(location)
3224
if version_control:
3225
return version_control().get_src_requirement(dist, location, find_tags)
3226
logger.warn('cannot determine version of editable source in %s (is not SVN checkout, Git clone, Mercurial clone or Bazaar branch)' % location)
3227
return dist.as_requirement()
3228
3229
############################################################
3230
## Requirement files
3231
3232
_scheme_re = re.compile(r'^(http|https|file):', re.I)
3233
_drive_re = re.compile(r'/*([a-z])\|', re.I)
3234
def get_file_content(url, comes_from=None):
3235
"""Gets the content of a file; it may be a filename, file: URL, or
3236
http: URL. Returns (location, content)"""
3237
match = _scheme_re.search(url)
3238
if match:
3239
scheme = match.group(1).lower()
3240
if (scheme == 'file' and comes_from
3241
and comes_from.startswith('http')):
3242
raise InstallationError(
3243
'Requirements file %s references URL %s, which is local'
3244
% (comes_from, url))
3245
if scheme == 'file':
3246
path = url.split(':', 1)[1]
3247
path = path.replace('\\', '/')
3248
match = _drive_re.match(path)
3249
if match:
3250
path = match.group(1) + ':' + path.split('|', 1)[1]
3251
path = urllib.unquote(path)
3252
if path.startswith('/'):
3253
path = '/' + path.lstrip('/')
3254
url = path
3255
else:
3256
## FIXME: catch some errors
3257
resp = urllib2.urlopen(url)
3258
return resp.geturl(), resp.read()
3259
f = open(url)
3260
content = f.read()
3261
f.close()
3262
return url, content
3263
3264
def parse_requirements(filename, finder, comes_from=None):
3265
skip_match = None
3266
if os.environ.get('PIP_SKIP_REQUIREMENTS_REGEX'):
3267
skip_match = re.compile(os.environ['PIP_SKIP_REQUIREMENTS_REGEX'])
3268
filename, content = get_file_content(filename, comes_from=comes_from)
3269
for line_number, line in enumerate(content.splitlines()):
3270
line_number += 1
3271
line = line.strip()
3272
if not line or line.startswith('#'):
3273
continue
3274
if skip_match and skip_match.search(line):
3275
continue
3276
if line.startswith('-r') or line.startswith('--requirement'):
3277
if line.startswith('-r'):
3278
req_url = line[2:].strip()
3279
else:
3280
req_url = line[len('--requirement'):].strip().strip('=')
3281
if _scheme_re.search(filename):
3282
# Relative to a URL
3283
req_url = urlparse.urljoin(filename, url)
3284
elif not _scheme_re.search(req_url):
3285
req_url = os.path.join(os.path.dirname(filename), req_url)
3286
for item in parse_requirements(req_url, finder, comes_from=filename):
3287
yield item
3288
elif line.startswith('-Z') or line.startswith('--always-unzip'):
3289
# No longer used, but previously these were used in
3290
# requirement files, so we'll ignore.
3291
pass
3292
elif line.startswith('-f') or line.startswith('--find-links'):
3293
if line.startswith('-f'):
3294
line = line[2:].strip()
3295
else:
3296
line = line[len('--find-links'):].strip().lstrip('=')
3297
## FIXME: it would be nice to keep track of the source of
3298
## the find_links:
3299
finder.find_links.append(line)
3300
else:
3301
comes_from = '-r %s (line %s)' % (filename, line_number)
3302
if line.startswith('-e') or line.startswith('--editable'):
3303
if line.startswith('-e'):
3304
line = line[2:].strip()
3305
else:
3306
line = line[len('--editable'):].strip()
3307
req = InstallRequirement.from_editable(
3308
line, comes_from)
3309
else:
3310
req = InstallRequirement.from_line(line, comes_from)
3311
yield req
3312
3313
############################################################
3314
## Logging
3315
3316
3317
3318
class Logger(object):
3319
3320
"""
3321
Logging object for use in command-line script. Allows ranges of
3322
levels, to avoid some redundancy of displayed information.
3323
"""
3324
3325
VERBOSE_DEBUG = logging.DEBUG-1
3326
DEBUG = logging.DEBUG
3327
INFO = logging.INFO
3328
NOTIFY = (logging.INFO+logging.WARN)/2
3329
WARN = WARNING = logging.WARN
3330
ERROR = logging.ERROR
3331
FATAL = logging.FATAL
3332
3333
LEVELS = [VERBOSE_DEBUG, DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL]
3334
3335
def __init__(self, consumers):
3336
self.consumers = consumers
3337
self.indent = 0
3338
self.explicit_levels = False
3339
self.in_progress = None
3340
self.in_progress_hanging = False
3341
3342
def debug(self, msg, *args, **kw):
3343
self.log(self.DEBUG, msg, *args, **kw)
3344
def info(self, msg, *args, **kw):
3345
self.log(self.INFO, msg, *args, **kw)
3346
def notify(self, msg, *args, **kw):
3347
self.log(self.NOTIFY, msg, *args, **kw)
3348
def warn(self, msg, *args, **kw):
3349
self.log(self.WARN, msg, *args, **kw)
3350
def error(self, msg, *args, **kw):
3351
self.log(self.WARN, msg, *args, **kw)
3352
def fatal(self, msg, *args, **kw):
3353
self.log(self.FATAL, msg, *args, **kw)
3354
def log(self, level, msg, *args, **kw):
3355
if args:
3356
if kw:
3357
raise TypeError(
3358
"You may give positional or keyword arguments, not both")
3359
args = args or kw
3360
rendered = None
3361
for consumer_level, consumer in self.consumers:
3362
if self.level_matches(level, consumer_level):
3363
if (self.in_progress_hanging
3364
and consumer in (sys.stdout, sys.stderr)):
3365
self.in_progress_hanging = False
3366
sys.stdout.write('\n')
3367
sys.stdout.flush()
3368
if rendered is None:
3369
if args:
3370
rendered = msg % args
3371
else:
3372
rendered = msg
3373
rendered = ' '*self.indent + rendered
3374
if self.explicit_levels:
3375
## FIXME: should this be a name, not a level number?
3376
rendered = '%02i %s' % (level, rendered)
3377
if hasattr(consumer, 'write'):
3378
consumer.write(rendered+'\n')
3379
else:
3380
consumer(rendered)
3381
3382
def start_progress(self, msg):
3383
assert not self.in_progress, (
3384
"Tried to start_progress(%r) while in_progress %r"
3385
% (msg, self.in_progress))
3386
if self.level_matches(self.NOTIFY, self._stdout_level()):
3387
sys.stdout.write(' '*self.indent + msg)
3388
sys.stdout.flush()
3389
self.in_progress_hanging = True
3390
else:
3391
self.in_progress_hanging = False
3392
self.in_progress = msg
3393
self.last_message = None
3394
3395
def end_progress(self, msg='done.'):
3396
assert self.in_progress, (
3397
"Tried to end_progress without start_progress")
3398
if self.stdout_level_matches(self.NOTIFY):
3399
if not self.in_progress_hanging:
3400
# Some message has been printed out since start_progress
3401
sys.stdout.write('...' + self.in_progress + msg + '\n')
3402
sys.stdout.flush()
3403
else:
3404
# These erase any messages shown with show_progress (besides .'s)
3405
logger.show_progress('')
3406
logger.show_progress('')
3407
sys.stdout.write(msg + '\n')
3408
sys.stdout.flush()
3409
self.in_progress = None
3410
self.in_progress_hanging = False
3411
3412
def show_progress(self, message=None):
3413
"""If we are in a progress scope, and no log messages have been
3414
shown, write out another '.'"""
3415
if self.in_progress_hanging:
3416
if message is None:
3417
sys.stdout.write('.')
3418
sys.stdout.flush()
3419
else:
3420
if self.last_message:
3421
padding = ' ' * max(0, len(self.last_message)-len(message))
3422
else:
3423
padding = ''
3424
sys.stdout.write('\r%s%s%s%s' % (' '*self.indent, self.in_progress, message, padding))
3425
sys.stdout.flush()
3426
self.last_message = message
3427
3428
def stdout_level_matches(self, level):
3429
"""Returns true if a message at this level will go to stdout"""
3430
return self.level_matches(level, self._stdout_level())
3431
3432
def _stdout_level(self):
3433
"""Returns the level that stdout runs at"""
3434
for level, consumer in self.consumers:
3435
if consumer is sys.stdout:
3436
return level
3437
return self.FATAL
3438
3439
def level_matches(self, level, consumer_level):
3440
"""
3441
>>> l = Logger()
3442
>>> l.level_matches(3, 4)
3443
False
3444
>>> l.level_matches(3, 2)
3445
True
3446
>>> l.level_matches(slice(None, 3), 3)
3447
False
3448
>>> l.level_matches(slice(None, 3), 2)
3449
True
3450
>>> l.level_matches(slice(1, 3), 1)
3451
True
3452
>>> l.level_matches(slice(2, 3), 1)
3453
False
3454
"""
3455
if isinstance(level, slice):
3456
start, stop = level.start, level.stop
3457
if start is not None and start > consumer_level:
3458
return False
3459
if stop is not None or stop <= consumer_level:
3460
return False
3461
return True
3462
else:
3463
return level >= consumer_level
3464
3465
@classmethod
3466
def level_for_integer(cls, level):
3467
levels = cls.LEVELS
3468
if level < 0:
3469
return levels[0]
3470
if level >= len(levels):
3471
return levels[-1]
3472
return levels[level]
3473
3474
def move_stdout_to_stderr(self):
3475
to_remove = []
3476
to_add = []
3477
for consumer_level, consumer in self.consumers:
3478
if consumer == sys.stdout:
3479
to_remove.append((consumer_level, consumer))
3480
to_add.append((consumer_level, sys.stderr))
3481
for item in to_remove:
3482
self.consumers.remove(item)
3483
self.consumers.extend(to_add)
3484
3485
3486
def call_subprocess(cmd, show_stdout=True,
3487
filter_stdout=None, cwd=None,
3488
raise_on_returncode=True,
3489
command_level=Logger.DEBUG, command_desc=None,
3490
extra_environ=None):
3491
if command_desc is None:
3492
cmd_parts = []
3493
for part in cmd:
3494
if ' ' in part or '\n' in part or '"' in part or "'" in part:
3495
part = '"%s"' % part.replace('"', '\\"')
3496
cmd_parts.append(part)
3497
command_desc = ' '.join(cmd_parts)
3498
if show_stdout:
3499
stdout = None
3500
else:
3501
stdout = subprocess.PIPE
3502
logger.log(command_level, "Running command %s" % command_desc)
3503
env = os.environ.copy()
3504
if extra_environ:
3505
env.update(extra_environ)
3506
try:
3507
proc = subprocess.Popen(
3508
cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
3509
cwd=cwd, env=env)
3510
except Exception, e:
3511
logger.fatal(
3512
"Error %s while executing command %s" % (e, command_desc))
3513
raise
3514
all_output = []
3515
if stdout is not None:
3516
stdout = proc.stdout
3517
while 1:
3518
line = stdout.readline()
3519
if not line:
3520
break
3521
line = line.rstrip()
3522
all_output.append(line + '\n')
3523
if filter_stdout:
3524
level = filter_stdout(line)
3525
if isinstance(level, tuple):
3526
level, line = level
3527
logger.log(level, line)
3528
if not logger.stdout_level_matches(level):
3529
logger.show_progress()
3530
else:
3531
logger.info(line)
3532
else:
3533
returned_stdout, returned_stderr = proc.communicate()
3534
all_output = [returned_stdout or '']
3535
proc.wait()
3536
if proc.returncode:
3537
if raise_on_returncode:
3538
if all_output:
3539
logger.notify('Complete output from command %s:' % command_desc)
3540
logger.notify('\n'.join(all_output) + '\n----------------------------------------')
3541
raise InstallationError(
3542
"Command %s failed with error code %s"
3543
% (command_desc, proc.returncode))
3544
else:
3545
logger.warn(
3546
"Command %s had error code %s"
3547
% (command_desc, proc.returncode))
3548
if stdout is not None:
3549
return ''.join(all_output)
3550
3551
############################################################
3552
## Utility functions
3553
3554
def is_svn_page(html):
3555
"""Returns true if the page appears to be the index page of an svn repository"""
3556
return (re.search(r'<title>[^<]*Revision \d+:', html)
3557
and re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I))
3558
3559
def file_contents(filename):
3560
fp = open(filename, 'rb')
3561
try:
3562
return fp.read()
3563
finally:
3564
fp.close()
3565
3566
def split_leading_dir(path):
3567
path = str(path)
3568
path = path.lstrip('/').lstrip('\\')
3569
if '/' in path and (('\\' in path and path.find('/') < path.find('\\'))
3570
or '\\' not in path):
3571
return path.split('/', 1)
3572
elif '\\' in path:
3573
return path.split('\\', 1)
3574
else:
3575
return path, ''
3576
3577
def has_leading_dir(paths):
3578
"""Returns true if all the paths have the same leading path name
3579
(i.e., everything is in one subdirectory in an archive)"""
3580
common_prefix = None
3581
for path in paths:
3582
prefix, rest = split_leading_dir(path)
3583
if not prefix:
3584
return False
3585
elif common_prefix is None:
3586
common_prefix = prefix
3587
elif prefix != common_prefix:
3588
return False
3589
return True
3590
3591
def format_size(bytes):
3592
if bytes > 1000*1000:
3593
return '%.1fMb' % (bytes/1000.0/1000)
3594
elif bytes > 10*1000:
3595
return '%iKb' % (bytes/1000)
3596
elif bytes > 1000:
3597
return '%.1fKb' % (bytes/1000.0)
3598
else:
3599
return '%ibytes' % bytes
3600
3601
_normalize_re = re.compile(r'[^a-z]', re.I)
3602
3603
def normalize_name(name):
3604
return _normalize_re.sub('-', name.lower())
3605
3606
def make_path_relative(path, rel_to):
3607
"""
3608
Make a filename relative, where the filename path, and it is
3609
relative to rel_to
3610
3611
>>> make_relative_path('/usr/share/something/a-file.pth',
3612
... '/usr/share/another-place/src/Directory')
3613
'../../../something/a-file.pth'
3614
>>> make_relative_path('/usr/share/something/a-file.pth',
3615
... '/home/user/src/Directory')
3616
'../../../usr/share/something/a-file.pth'
3617
>>> make_relative_path('/usr/share/a-file.pth', '/usr/share/')
3618
'a-file.pth'
3619
"""
3620
path_filename = os.path.basename(path)
3621
path = os.path.dirname(path)
3622
path = os.path.normpath(os.path.abspath(path))
3623
rel_to = os.path.normpath(os.path.abspath(rel_to))
3624
path_parts = path.strip(os.path.sep).split(os.path.sep)
3625
rel_to_parts = rel_to.strip(os.path.sep).split(os.path.sep)
3626
while path_parts and rel_to_parts and path_parts[0] == rel_to_parts[0]:
3627
path_parts.pop(0)
3628
rel_to_parts.pop(0)
3629
full_parts = ['..']*len(rel_to_parts) + path_parts + [path_filename]
3630
if full_parts == ['']:
3631
return '.' + os.path.sep
3632
return os.path.sep.join(full_parts)
3633
3634
def display_path(path):
3635
"""Gives the display value for a given path, making it relative to cwd
3636
if possible."""
3637
path = os.path.normcase(os.path.abspath(path))
3638
if path.startswith(os.getcwd() + os.path.sep):
3639
path = '.' + path[len(os.getcwd()):]
3640
return path
3641
3642
def parse_editable(editable_req):
3643
"""Parses svn+http://blahblah@rev#egg=Foobar into a requirement
3644
(Foobar) and a URL"""
3645
url = editable_req
3646
if os.path.isdir(url) and os.path.exists(os.path.join(url, 'setup.py')):
3647
# Treating it as code that has already been checked out
3648
url = filename_to_url(url)
3649
if url.lower().startswith('file:'):
3650
return None, url
3651
for version_control in vcs:
3652
if url.lower().startswith('%s:' % version_control):
3653
url = '%s+%s' % (version_control, url)
3654
if '+' not in url:
3655
if default_vcs:
3656
url = default_vcs + '+' + url
3657
else:
3658
raise InstallationError(
3659
'--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req)
3660
vc_type = url.split('+', 1)[0].lower()
3661
if not vcs.get_backend(vc_type):
3662
raise InstallationError(
3663
'For --editable=%s only svn (svn+URL), Git (git+URL), Mercurial (hg+URL) and Bazaar (bzr+URL) is currently supported' % editable_req)
3664
match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req)
3665
if (not match or not match.group(1)) and vcs.get_backend(vc_type):
3666
parts = [p for p in editable_req.split('#', 1)[0].split('/') if p]
3667
if parts[-2] in ('tags', 'branches', 'tag', 'branch'):
3668
req = parts[-3]
3669
elif parts[-1] == 'trunk':
3670
req = parts[-2]
3671
else:
3672
raise InstallationError(
3673
'--editable=%s is not the right format; it must have #egg=Package'
3674
% editable_req)
3675
else:
3676
req = match.group(1)
3677
## FIXME: use package_to_requirement?
3678
match = re.search(r'^(.*?)(?:-dev|-\d.*)', req)
3679
if match:
3680
# Strip off -dev, -0.2, etc.
3681
req = match.group(1)
3682
return req, url
3683
3684
def backup_dir(dir, ext='.bak'):
3685
"""Figure out the name of a directory to back up the given dir to
3686
(adding .bak, .bak2, etc)"""
3687
n = 1
3688
extension = ext
3689
while os.path.exists(dir + extension):
3690
n += 1
3691
extension = ext + str(n)
3692
return dir + extension
3693
3694
def ask(message, options):
3695
"""Ask the message interactively, with the given possible responses"""
3696
while 1:
3697
response = raw_input(message)
3698
response = response.strip().lower()
3699
if response not in options:
3700
print 'Your response (%r) was not one of the expected responses: %s' % (
3701
response, ', '.join(options))
3702
else:
3703
return response
3704
3705
def open_logfile_append(filename):
3706
"""Open the named log file in append mode.
3707
3708
If the file already exists, a separator will also be printed to
3709
the file to separate past activity from current activity.
3710
"""
3711
exists = os.path.exists(filename)
3712
log_fp = open(filename, 'a')
3713
if exists:
3714
print >> log_fp, '-'*60
3715
print >> log_fp, '%s run on %s' % (sys.argv[0], time.strftime('%c'))
3716
return log_fp
3717
3718
def is_url(name):
3719
"""Returns true if the name looks like a URL"""
3720
if ':' not in name:
3721
return False
3722
scheme = name.split(':', 1)[0].lower()
3723
return scheme in ('http', 'https', 'file', 'ftp')
3724
3725
def is_filename(name):
3726
if (splitext(name)[1].lower() in ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle')
3727
and os.path.exists(name)):
3728
return True
3729
if os.path.sep not in name and '/' not in name:
3730
# Doesn't have any path components, probably a requirement like 'Foo'
3731
return False
3732
return True
3733
3734
_drive_re = re.compile('^([a-z]):', re.I)
3735
_url_drive_re = re.compile('^([a-z])[|]', re.I)
3736
3737
def filename_to_url(filename):
3738
"""
3739
Convert a path to a file: URL. The path will be made absolute.
3740
"""
3741
filename = os.path.normcase(os.path.abspath(filename))
3742
url = urllib.quote(filename)
3743
if _drive_re.match(url):
3744
url = url[0] + '|' + url[2:]
3745
url = url.replace(os.path.sep, '/')
3746
url = url.lstrip('/')
3747
return 'file:///' + url
3748
3749
def url_to_filename(url):
3750
"""
3751
Convert a file: URL to a path.
3752
"""
3753
assert url.startswith('file:'), (
3754
"You can only turn file: urls into filenames (not %r)" % url)
3755
filename = url[len('file:'):].lstrip('/')
3756
filename = urllib.unquote(filename)
3757
if _url_drive_re.match(filename):
3758
filename = filename[0] + ':' + filename[2:]
3759
else:
3760
filename = '/' + filename
3761
return filename
3762
3763
def get_requirement_from_url(url):
3764
"""Get a requirement from the URL, if possible. This looks for #egg
3765
in the URL"""
3766
link = Link(url)
3767
egg_info = link.egg_fragment
3768
if not egg_info:
3769
egg_info = splitext(link.filename)[0]
3770
return package_to_requirement(egg_info)
3771
3772
def package_to_requirement(package_name):
3773
"""Translate a name like Foo-1.2 to Foo==1.3"""
3774
match = re.search(r'^(.*?)(-dev|-\d.*)', package_name)
3775
if match:
3776
name = match.group(1)
3777
version = match.group(2)
3778
else:
3779
name = package_name
3780
version = ''
3781
if version:
3782
return '%s==%s' % (name, version)
3783
else:
3784
return name
3785
3786
def splitext(path):
3787
"""Like os.path.splitext, but take off .tar too"""
3788
base, ext = posixpath.splitext(path)
3789
if base.lower().endswith('.tar'):
3790
ext = base[-4:] + ext
3791
base = base[:-4]
3792
return base, ext
3793
3794
class _Inf(object):
3795
"""I am bigger than everything!"""
3796
def __cmp__(self, a):
3797
if self is a:
3798
return 0
3799
return 1
3800
def __repr__(self):
3801
return 'Inf'
3802
Inf = _Inf()
3803
del _Inf
3804
3805
if __name__ == '__main__':
3806
exit = main()
3807
if exit:
3808
sys.exit(exit)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1