9
from simplestreams import util
13
# could support reading from other mirrors
15
# http://cloud-images-archive.ubuntu.com/
16
# file:///srv/ec2-images
18
BASE_URLS = ("http://cloud-images.ubuntu.com/",)
22
'size': 10240, 'md5': '1276481102f218c981e0324180bafd9f',
23
'sha256': '84ff92691f909a05b224e1c56abb4864f01b4f8e3c854e4bb4c7baf1d3f6d652'},
25
'size': 11264, 'md5': '820a81e0916bac82838fd7e74ab29b15',
26
'sha256': '5309e677c79cffae49a65728c61b436d3cdc2a2bab4c81bf0038415f74a56880'},
28
'size': 12288, 'md5': '4072783b8efb99a9e5817067d68f61c6',
29
'sha256': 'f3cc103136423a57975750907ebc1d367e2985ac6338976d4d5a439f50323f4a'},
32
REAL_DATA = os.environ.get("REAL_DATA", False)
33
if REAL_DATA and REAL_DATA != "0":
41
def get_cache_data(path, field):
42
dirname = os.path.dirname(path)
43
bname = os.path.basename(path)
44
return FILE_DATA.get(dirname, {}).get(bname, {}).get(field)
47
def store_cache_data(path, field, value):
48
dirname = os.path.dirname(path)
49
bname = os.path.basename(path)
50
if dirname not in FILE_DATA:
51
FILE_DATA[dirname] = {}
52
if bname not in FILE_DATA[dirname]:
53
FILE_DATA[dirname][bname] = {}
54
FILE_DATA[dirname][bname][field] = value
59
hashcache = FILE_DATA['filename']
60
with open(hashcache, "w") as hfp:
61
hfp.write(json.dumps(FILE_DATA, indent=1))
64
def get_cloud_images_file_hash(path):
65
md5 = get_cache_data(path, 'md5')
66
sha256 = get_cache_data(path, 'sha256')
68
return {'md5': md5, 'sha256': sha256}
71
dirname = os.path.dirname(path)
72
for cksum in ("md5", "sha256"):
74
for burl in BASE_URLS:
75
dir_url = burl + dirname
78
url = dir_url + "/%sSUMS" % cksum.upper()
79
sys.stderr.write("reading %s\n" % url)
80
content = util.read_url(url)
82
except Exception as error:
88
for line in content.splitlines():
89
(hexsum, fname) = line.split()
90
if fname.startswith("*"):
93
store_cache_data(dirname + "/" + fname, cksum, hexsum)
95
md5 = get_cache_data(path, 'md5')
96
sha256 = get_cache_data(path, 'sha256')
98
return {'md5': md5, 'sha256': sha256}
101
def get_url_len(url):
102
if url.startswith("file:///"):
103
path = url[len("file://"):]
104
return os.stat(path).st_size
105
if os.path.exists(url):
106
return os.stat(url).st_size
108
# http://stackoverflow.com/questions/4421170/python-head-request-with-urllib2
109
sys.stderr.write("getting size for %s\n" % url)
110
request = urllib2.Request(url)
111
request.get_method = lambda : 'HEAD'
112
response = urllib2.urlopen(request)
113
return int(response.headers.get('content-length', 0))
116
def get_cloud_images_file_size(path):
117
size = get_cache_data(path, 'size')
121
for burl in BASE_URLS:
123
size = int(get_url_len(burl + path))
125
except Exception as error:
130
store_cache_data(path, 'size', size)
135
def create_fake_file(prefix, item):
136
fpath = os.path.join(prefix, item['path'])
139
data = FAKE_DATA[item['ftype']]
141
util.mkdir_p(os.path.dirname(fpath))
142
print "creating %s" % fpath
143
with open(fpath, "w") as fp:
144
fp.truncate(data['size'])
148
for cksum in util.CHECKSUMS:
149
if cksum in item and not cksum in data:
155
def dl_load_query(path):
157
for rline in toolutil.load_query_download(path):
158
(stream, rel, build, label, serial, arch, filepath, fname) = rline
160
if stream not in tree:
161
tree[stream] = {'products': {}}
162
products = tree[stream]['products']
164
prodname = "%s:%s:%s" % (build, rel, arch)
166
if prodname not in products:
167
products[prodname] = {
169
"version": toolutil.REL2VER[rel]['version'],
174
product = products[prodname]
176
if serial not in product['versions']:
177
product['versions'][serial] = {'items': {}, "label": label}
179
name = pubname(label, rel, arch, serial, build)
180
product['versions'][serial]['pubname'] = name
182
items = product['versions'][serial]['items']
184
# ftype: finding the unique extension is not-trivial
185
# take basename of the filename, and remove up to "-<arch>?"
186
# so ubuntu-12.04-server-cloudimg-armhf.tar.gz becomes
187
# 'tar.gz' and 'ubuntu-12.04-server-cloudimg-armhf-disk1.img'
188
# becomse 'disk1.img'
189
dash_arch = "-" + arch
190
ftype = filepath[filepath.rindex(dash_arch) + len(dash_arch) + 1:]
199
def pubname(label, rel, arch, serial, build='server'):
200
version = toolutil.REL2VER[rel]['version']
203
rv_label = rel + "-daily"
204
elif label == "release":
205
rv_label = "%s-%s" % (rel, version)
206
elif label.startswith("beta"):
207
rv_label = "%s-%s-%s" % (rel, version, label)
209
rv_label = "%s-%s" % (rel, label)
210
return "ubuntu-%s-%s-%s-%s" % (rv_label, arch, build, serial)
213
def ec2_load_query(path):
227
'pv': {'instance': "pi", "ebs": "pe"},
228
'hvm': {'instance': "hi", "ebs": "he"}
231
for rline in toolutil.load_query_ec2(path):
232
(stream, rel, build, label, serial, store, arch, region,
233
iid, _kern, _rmd, vtype) = rline
235
if stream not in tree:
236
tree[stream] = {'products': {}}
237
products = tree[stream]['products']
239
prodname = "%s:%s:%s" % (build, rel, arch)
241
if prodname not in products:
242
products[prodname] = {
244
"version": toolutil.REL2VER[rel]['version'],
249
product = products[prodname]
251
if serial not in product['versions']:
252
product['versions'][serial] = {'items': {}, "label": label}
253
items = product['versions'][serial]['items']
255
name = pubname(label, rel, arch, serial, build)
256
product['versions'][serial]['pubname'] = name
258
if store == "instance-store":
260
if vtype == "paravirtual":
263
# create the item key:
264
# - 2 letter country code (us)
265
# - 2 letter direction ('nn' for north, 'nw' for northwest)
267
# - 1 char for virt type
268
# - 1 char for root-store type
269
(cc, direction, num) = region.split("-")
270
ikey = cc + dmap[direction] + num + itmap[vtype][store]
281
def printitem(item, exdata):
287
def create_image_data(query_tree, out_d, streamdir):
288
hashcache = os.path.join(query_tree, "FILE_DATA_CACHE")
289
FILE_DATA['filename'] = hashcache
290
if os.path.isfile(hashcache):
291
FILE_DATA.update(json.loads(open(hashcache).read()))
293
ts = util.timestamp()
294
tree = dl_load_query(query_tree)
296
def update_hashes(item, tree, pedigree):
297
item.update(get_cloud_images_file_hash(item['path']))
299
def update_sizes(item, tree, pedigree):
300
item.update({'size': get_cloud_images_file_size(item['path'])})
302
cid_fmt = "com.ubuntu.cloud:%s:download"
304
def create_file(item, tree, pedigree):
305
create_fake_file(os.path.join(out_d, stream), item)
307
cid = cid_fmt % stream
309
util.walk_products(tree[stream], cb_item=update_hashes)
310
util.walk_products(tree[stream], cb_item=update_sizes)
312
util.walk_products(tree[stream], cb_item=create_file)
314
tree[stream]['format'] = "products:1.0"
315
tree[stream]['updated'] = ts
316
tree[stream]['content_id'] = cid
318
outfile = os.path.join(out_d, stream, streamdir, cid + ".js")
319
util.mkdir_p(os.path.dirname(outfile))
320
with open(outfile, "w") as fp:
321
sys.stderr.write("writing %s\n" % outfile)
322
fp.write(json.dumps(tree[stream], indent=1) + "\n")
329
def create_aws_data(query_tree, out_d, streamdir):
330
tree = ec2_load_query(query_tree)
331
ts = util.timestamp()
332
cid_fmt = "com.ubuntu.cloud:%s:aws"
334
cid = cid_fmt % stream
335
# now add the '_alias' data
338
def findregions(item, tree, pedigree):
339
regions.add(item['crsn'])
341
util.walk_products(tree[stream], cb_item=findregions)
343
tree[stream]['_aliases'] = {'crsn': {}}
344
for region in regions:
345
tree[stream]['_aliases']['crsn'][region] = {
346
'endpoint': 'http://ec2.%s.amazonaws.com' % region,
349
tree[stream]['format'] = "products:1.0"
350
tree[stream]['updated'] = ts
351
tree[stream]['content_id'] = cid
352
outfile = os.path.join(out_d, stream, streamdir, cid + ".js")
353
util.mkdir_p(os.path.dirname(outfile))
354
with open(outfile, "w") as fp:
355
sys.stderr.write("writing %s\n" % outfile)
356
fp.write(json.dumps(tree[stream], indent=1) + "\n")
362
parser = argparse.ArgumentParser(description="create example content tree")
364
parser.add_argument("query_tree", metavar='query_tree',
365
help=('read in content from /query tree. Hint: ' +
366
'make exdata-query'))
368
parser.add_argument("out_d", metavar='out_d',
369
help=('create content under output_dir'))
371
parser.add_argument('--sign', action='store_true', default=False,
372
help='sign all generated files')
374
args = parser.parse_args()
375
streamdir = "streams/v1"
377
dltree = create_image_data(args.query_tree, args.out_d, streamdir)
379
aws_tree = create_aws_data(args.query_tree, args.out_d, streamdir)
381
for streamname in aws_tree:
382
index = {"index": {}, 'format': 'index:1.0',
383
'updated': util.timestamp()}
385
clouds = aws_tree[streamname]['_aliases']['crsn'].values()
386
index['index'][aws_tree[streamname]['content_id']] = {
387
'updated': aws_tree[streamname]['updated'],
388
'datatype': 'image-ids',
391
'path': '/'.join((streamdir,
392
"%s.js" % aws_tree[streamname]['content_id'],)),
393
'products': aws_tree[streamname]['products'].keys(),
394
'format': aws_tree[streamname]['format'],
396
index['index'][dltree[streamname]['content_id']] = {
397
'updated': dltree[streamname]['updated'],
398
'datatype': 'image-downloads',
399
'path': '/'.join((streamdir,
400
"%s.js" % dltree[streamname]['content_id'],)),
401
'products': dltree[streamname]['products'].keys(),
402
'format': dltree[streamname]['format']
405
outfile = os.path.join(args.out_d, streamname, streamdir, 'index.js')
406
util.mkdir_p(os.path.dirname(outfile))
407
with open(outfile, "w") as fp:
408
sys.stderr.write("writing %s\n" % outfile)
409
fp.write(json.dumps(index, indent=1) + "\n")
412
def printstatus(name, fmt):
413
sys.stderr.write("signing %s: %s\n" % (name, fmt))
414
for root, dirs, files in os.walk(args.out_d):
415
for f in [f for f in files if f.endswith(".js")]:
416
toolutil.signjs_file(os.path.join(root, f),
417
status_cb=printstatus)
421
if __name__ == '__main__':
added
removed
tools/make-test-data
