← Back to branch summary

~abentley/juju-ci-tools/client-from-config-4

« back to all changes in this revision

Viewing changes to pipdeps.py

  • Committer: Aaron Bentley
  • Date: 2016-03-18 14:47:06 UTC
  • mto: This revision was merged to the branch mainline in revision 1324.
  • Revision ID: aaron.bentley@canonical.com-20160318144706-z7wy9c21m3psi6g5
Introduce set_model_name, update tests, check controller on bootstrap.

Show diffs side-by-side

added added

removed removed

Lines of Context:
pipdeps.py
5
5
 
6
6
import argparse
7
7
import os
8
 
import platform
9
8
import subprocess
10
9
import sys
11
10
 
17
16
 
18
17
BUCKET = "juju-pip-archives"
19
18
PREFIX = "juju-ci-tools/"
20
 
REQUIREMENTS = os.path.join(os.path.realpath(os.path.dirname(__file__)),
21
 
                            "requirements.txt")
22
 
MAC_WIN_REQS = os.path.join(os.path.realpath(os.path.dirname(__file__)),
23
 
                            "mac-win-requirements.txt")
24
 
OBSOLETE = os.path.join(os.path.realpath(os.path.dirname(__file__)),
25
 
                        "obsolete-requirements.txt")
26
 
 
27
 
 
28
 
def get_requirements():
29
 
    if platform.dist()[0] in ('Ubuntu', 'debian'):
30
 
        return REQUIREMENTS
31
 
    else:
32
 
        return MAC_WIN_REQS
33
19
 
34
20
 
35
21
def s3_anon():
50
36
    return boto.s3.connection.S3Connection(access_key, secret_key)
51
37
 
52
38
 
53
 
def run_pip_install(extra_args, requirements, verbose=False):
 
39
def run_pip_install(extra_args, verbose=False):
54
40
    """Run pip install in a subprocess with given additional arguments."""
55
41
    cmd = ["pip"]
56
42
    if not verbose:
57
43
        cmd.append("-q")
 
44
    requirements = os.path.join(os.path.realpath(os.path.dirname(__file__)),
 
45
                                "requirements.txt")
58
46
    cmd.extend(["install", "-r", requirements])
59
47
    cmd.extend(extra_args)
60
48
    subprocess.check_call(cmd)
61
49
 
62
50
 
63
 
def run_pip_uninstall(obsolete_requirements):
64
 
    """Run pip uninstall for each package version in obsolete_requirements.
65
 
 
66
 
    pip uninstall the package without regard to its version. In most cases,
67
 
    calling install with a new package version implicitly upgrades.
68
 
    There are only a few package version that cannot by upgraded, they must
69
 
    be removed before install. This function uninstalls packages only when
70
 
    their version matches the obsolete.
71
 
 
72
 
    The obsolete_requirements entries must match the output of pip list. eg:
73
 
        azure (0.8.0)
74
 
        bibbel (1.2.3)
75
 
    """
76
 
    pip_cmd = ['pip']
77
 
    list_cmd = pip_cmd + ['list']
78
 
    installed_packages = set(subprocess.check_output(list_cmd).splitlines())
79
 
    with open(obsolete_requirements, 'r') as o_file:
80
 
        obsolete_packages = o_file.read().splitlines()
81
 
    removable = installed_packages.intersection(obsolete_packages)
82
 
    for package_version in removable:
83
 
        package, version = package_version.split()
84
 
        uninstall_cmd = pip_cmd + ['uninstall', '-y', package]
85
 
        subprocess.check_call(uninstall_cmd)
86
 
 
87
 
 
88
 
def command_install(bucket, requirements, verbose=False):
 
51
def command_install(bucket, verbose=False):
89
52
    with utility.temp_dir() as archives_dir:
90
53
        for key in bucket.list(prefix=PREFIX):
91
54
            archive = key.name[len(PREFIX):]
92
55
            key.get_contents_to_filename(os.path.join(archives_dir, archive))
93
56
        archives_url = "file://" + archives_dir
94
 
        run_pip_uninstall(OBSOLETE)
95
57
        run_pip_install(["--user", "--no-index", "--find-links", archives_url],
96
 
                        requirements, verbose=verbose)
97
 
 
98
 
 
99
 
def command_update(s3, requirements, verbose=False):
 
58
                        verbose=verbose)
 
59
 
 
60
 
 
61
def command_update(s3, verbose=False):
100
62
    bucket = s3.lookup(BUCKET)
101
63
    if bucket is None:
102
64
        if verbose:
103
65
            print("Creating bucket {}".format(BUCKET))
104
66
        bucket = s3.create_bucket(BUCKET, policy="public-read")
105
67
    with utility.temp_dir() as archives_dir:
106
 
        run_pip_install(
107
 
            ["--download", archives_dir], requirements, verbose=verbose)
 
68
        run_pip_install(["--download", archives_dir], verbose=verbose)
108
69
        for archive in os.listdir(archives_dir):
109
70
            filename = os.path.join(archives_dir, archive)
110
71
            key = boto.s3.key.Key(bucket)
133
94
    parser.add_argument(
134
95
        "--cloud-city", default="~/cloud-city", type=os.path.expanduser,
135
96
        help="Location of cloud-city repository for credentials.")
136
 
    parser.add_argument(
137
 
        "--requirements", default=get_requirements(), type=os.path.expanduser,
138
 
        help="Location requirements file to use.")
139
97
    subparsers = parser.add_subparsers(dest="command")
140
98
    subparsers.add_parser("install", help="Download deps from S3 and install.")
141
99
    subparsers.add_parser(
153
111
        parser.error("Need cloud-city credentials to modify S3 cache.")
154
112
    s3 = s3_auth_with_rc(args.cloud_city) if use_auth else s3_anon()
155
113
    if args.command == "update":
156
 
        command_update(s3, args.requirements, args.verbose)
 
114
        command_update(s3, args.verbose)
157
115
    else:
158
116
        bucket = s3.get_bucket(BUCKET)
159
117
        if args.command == "install":
160
 
            command_install(bucket, args.requirements, args.verbose)
 
118
            command_install(bucket, args.verbose)
161
119
        elif args.command == "list":
162
120
            command_list(bucket, args.verbose)
163
121
        elif args.command == "delete":