← Back to branch summary

~abentley/juju-ci-tools/client-from-config-4

« back to all changes in this revision

Viewing changes to s3ci.py

  • Committer: Aaron Bentley
  • Date: 2015-11-17 14:13:35 UTC
  • mfrom: (1151.2.6 s3-get)
  • Revision ID: aaron.bentley@canonical.com-20151117141335-8thtwtluaum2xn1d
Add a get command to s3ci.

Show diffs side-by-side

added added

removed removed

Lines of Context:
s3ci.py
32
32
        Download the package for this client machine from s3.
33
33
        Extract the package files.  Print the location of the juju client.
34
34
        """)
35
 
    parser_get_juju_bin.add_argument(
36
 
        'revision_build', type=int,
37
 
        help='The revision-build to get binaries from')
38
 
    parser_get_juju_bin.add_argument(
39
 
        'workspace', nargs='?', default='.',
40
 
        help='The directory to download into')
41
 
    parser_get_juju_bin.add_argument(
42
 
        '--config', default=default_config,
43
 
        help=('s3cmd config file for credentials.  Default to juju-qa.s3cfg in'
44
 
              ' juju home.'))
45
 
    parser_get_juju_bin.add_argument(
46
 
        '--verbose', '-v', default=0, action='count',
47
 
        help='Increase verbosity')
 
35
    parser_get = subparsers.add_parser(
 
36
        'get', help='Download job files(s)',
 
37
        description="""
 
38
        Download a file from a job for this revision build.
 
39
 
 
40
        Within the revision build, the most recent version will be selected.
 
41
        """)
 
42
    for subparser in [parser_get_juju_bin, parser_get]:
 
43
        subparser.add_argument(
 
44
            'revision_build', type=int,
 
45
            help='The revision-build to use.')
 
46
    parser_get.add_argument('job', help='The job to get files from',)
 
47
    parser_get.add_argument(
 
48
        'file_pattern', help='The file pattern to use for selecting files',)
 
49
    for subparser in [parser_get_juju_bin, parser_get]:
 
50
        subparser.add_argument(
 
51
            'workspace', nargs='?', default='.',
 
52
            help='The directory to download into')
 
53
        subparser.add_argument(
 
54
            '--config', default=default_config,
 
55
            help=('s3cmd config file for credentials.  Default to '
 
56
                  'juju-qa.s3cfg in juju home.'))
 
57
        subparser.add_argument(
 
58
            '--verbose', '-v', default=0, action='count',
 
59
            help='Increase verbosity')
48
60
    return parser.parse_args(args)
49
61
 
50
62
 
57
69
    return access_key, secret_key
58
70
 
59
71
 
60
 
def get_job_path(revision_build):
61
 
    namer = JobNamer.factory()
62
 
    job = namer.get_build_binary_job()
 
72
def get_job_path(revision_build, job):
63
73
    return 'juju-ci/products/version-{}/{}'.format(revision_build, job)
64
74
 
65
75
 
68
78
 
69
79
 
70
80
def find_package_key(bucket, revision_build):
71
 
    prefix = get_job_path(revision_build)
 
81
    namer = JobNamer.factory()
 
82
    job = namer.get_build_binary_job()
 
83
    prefix = get_job_path(revision_build, job)
72
84
    keys = bucket.list(prefix)
73
85
    suffix = PackageNamer.factory().get_release_package_suffix()
74
86
    filtered = [
89
101
    return acquire_binary(package_path, workspace)
90
102
 
91
103
 
 
104
def find_file_keys(bucket, revision_build, job, file_regex):
 
105
    prefix = get_job_path(revision_build, job)
 
106
    keys = bucket.list(prefix)
 
107
    by_build = {}
 
108
    for key in keys:
 
109
        match = re.search('^/build-(\d+)', key.name[len(prefix):])
 
110
        build = int(match.group(1))
 
111
        by_build.setdefault(build, []).append(key)
 
112
    # We can't use last successful build, because we don't know what builds
 
113
    # are successful, so use last build and require it to be successful.
 
114
    last_build = max(by_build.keys())
 
115
    build_keys = by_build[last_build]
 
116
    filtered = []
 
117
    full_prefix = '{}/build-{}/'.format(prefix, last_build)
 
118
    for key in build_keys:
 
119
        path = key.name[len(full_prefix):]
 
120
        if re.match(file_regex, path):
 
121
            filtered.append(key)
 
122
    return filtered
 
123
 
 
124
 
 
125
def fetch_files(bucket, revision_build, job, file_pattern, workspace):
 
126
    file_keys = find_file_keys(bucket, revision_build, job, file_pattern)
 
127
    out_files = [os.path.join(workspace, k.name.split('/')[-1])
 
128
                 for k in file_keys]
 
129
    for key in file_keys:
 
130
        logging.info('Selected: %s', key.name)
 
131
    download_files(file_keys, workspace)
 
132
    return out_files
 
133
 
 
134
 
92
135
def main():
93
136
    args = parse_args()
94
137
    log_level = logging.WARNING - args.verbose * (
96
139
    configure_logging(log_level)
97
140
    if args.command == 'get-juju-bin':
98
141
        return get_juju_bin(args)
 
142
    elif args.command == 'get':
 
143
        return cmd_get(args)
99
144
    else:
100
145
        raise Exception('{} not implemented.'.format(args.command))
101
146
 
107
152
    print(fetch_juju_binary(bucket, args.revision_build, args.workspace))
108
153
 
109
154
 
 
155
def cmd_get(args):
 
156
    credentials = get_s3_credentials(args.config)
 
157
    conn = S3Connection(*credentials)
 
158
    bucket = conn.get_bucket('juju-qa-data')
 
159
    for path in fetch_files(bucket, args.revision_build, args.job,
 
160
                            args.file_pattern, args.workspace):
 
161
        print(path)
 
162
 
 
163
 
110
164
if __name__ == '__main__':
111
165
    sys.exit(main())