« back to all changes in this revision
Viewing changes to script/ucs
- browse files at revision 49
- compare with another revision
- download diff
- download tarball
- view history from revision 49
- Committer: Nekhelesh Ramananthan
- Author(s): Stuart Langridge
- Date: 2015-04-13 20:10:45 UTC
- mfrom: (39.1.27 community-components)
- Revision ID: krnekhelesh@gmail.com-20150413201045-dtb6a4hl0sos0quk
Migrated ucs bash script to python. Added Curated and Community Stores. Added support for binary components.
- files added:
- community-server
- curated-store
- files removed:
- script/ucs
- files renamed:
- ComponentStore.apparmor => curated-store/ComponentStore.apparmor
- ComponentStore.desktop => curated-store/ComponentStore.desktop
- ComponentStore.png => curated-store/ComponentStore.png
- ComponentStore.qmlproject => curated-store/ComponentStore.qmlproject
- ComponentStore/ => curated-store/ComponentStore/
- GallerySRC/ => curated-store/GallerySRC/
- main.qml => curated-store/main.qml
- manifest.json => curated-store/manifest.json
- tests/ => curated-store/tests/
- files modified:
- .bzrignore
added
removed
script/ucs
1
#!/usr/bin/python
2
# has to be Python2 because there's no python3 bzrlib
3
4
import sys, argparse, os, time, codecs, json, urllib, urllib2, re, subprocess
5
from gi.repository import GLib
6
from bzrlib.plugin import load_plugins
7
load_plugins()
8
from bzrlib.branch import Branch
9
10
class Project(object):
11
def __init__(self, rootfolder):
12
self.rootfolder = rootfolder
13
14
def add_to_import_path(self, component_type):
15
pass
16
17
def copy_branch_folder_contents_to(self, branch, tree, folder, destfolder):
18
# Ensure destfolder exists
19
try:
20
os.makedirs(destfolder)
21
except OSError, e:
22
if e.errno == 17:
23
pass
24
else:
25
raise
26
# Copy immediate file children of folder to it (not child folders or their contents)
27
copied_files = []
28
for path, versioned, kind, file_id, inventory_entry in tree.list_files(from_dir=folder, recursive=False):
29
if kind == "file":
30
fp = open(os.path.join(destfolder, path), "wb")
31
fp.write(tree.get_file_text(file_id))
32
fp.close()
33
copied_files.append(path)
34
return copied_files
35
36
def install_component_from_branch_folder(self, branch, tree, folder, component, component_type):
37
if component_type == "qml":
38
destfolder = os.path.join(self.rootfolder, "ubuntu_component_store",
39
component["name"] + "." + component["version"])
40
copied_files = self.copy_branch_folder_contents_to(branch, tree, folder, destfolder)
41
# create qml qmldir file
42
qmlfiles = []
43
for f in copied_files:
44
if f.endswith(".qml"):
45
qmlfiles.append("%s %s %s" % (os.path.splitext(f)[0], component["version"], f))
46
if qmlfiles:
47
username, cname = component["name"].split("/")
48
cname = cname[0].upper() + cname[1:]
49
qmldir = ["module ubuntu_component_store.%s.%s" % (username, cname)]
50
qmldir += qmlfiles
51
qmldirfn = os.path.join(destfolder, "qmldir")
52
fp = codecs.open(qmldirfn, mode="w", encoding="utf8")
53
fp.write("\n".join(qmldir))
54
fp.write("\n")
55
fp.close()
56
elif component_type == "binary":
57
fid = tree.path2id(folder)
58
if not fid:
59
raise Exception("Component has no '%s' folder" % (folder,))
60
arches = []
61
for path, versioned, kind, file_id, inventory_entry in tree.list_files(from_dir=folder, recursive=False):
62
if kind == "directory":
63
arches.append(path)
64
for arch in arches:
65
destfolder = os.path.join(self.rootfolder, "lib", arch, "ubuntu_component_store",
66
component["name"] + "." + component["version"])
67
copied_files = self.copy_branch_folder_contents_to(branch, tree, folder + "/" + arch, destfolder)
68
# create binary qmldir file
69
plugin_names = []
70
for f in copied_files:
71
m = re.match("^lib(?P<libname>[^z.]+)\.so", f)
72
if m:
73
plugin_names.append(m.groupdict()["libname"])
74
if plugin_names:
75
username, cname = component["name"].split("/")
76
cname = cname[0].upper() + cname[1:]
77
qmldir = ["module ubuntu_component_store.%s.%s" % (username, cname)]
78
qmldir += ["plugin %s" % x for x in plugin_names]
79
qmldirfn = os.path.join(destfolder, "qmldir")
80
fp = codecs.open(qmldirfn, mode="w", encoding="utf8")
81
fp.write("\n".join(qmldir))
82
fp.write("\n")
83
fp.close()
84
else:
85
raise Exception("Unknown component type '%s'" % (component_type,))
86
87
def add_to_import_path(self, component_type):
88
raise NotImplementedError
89
90
def add_to_project_specific_copy_path(self, folder):
91
raise NotImplementedError
92
93
def add_to_project_specific_import_path(self, folder):
94
raise NotImplementedError
95
96
97
class PureQMLProject(Project):
98
def add_to_project_specific_import_path(self, folder):
99
"Add the passed folder to root/projname.qmlproject:importPaths"
100
projname = os.path.split(os.path.abspath(self.rootfolder))[1]
101
qmlprojfile = os.path.join(self.rootfolder, "%s.qmlproject" % projname)
102
if not os.path.exists(qmlprojfile):
103
raise Exception("Couldn't add import path to %s" % (qmlprojfile,))
104
fp = codecs.open(qmlprojfile, encoding="utf8")
105
data = fp.read()
106
fp.close()
107
ipre = "importPaths\s*:\s*\[(?P<paths>[^]]*)\]"
108
ip = re.search(ipre, data)
109
if not ip:
110
raise Exception("Couldn't add import path to %s" % (qmlprojfile,))
111
pathsline = ip.groupdict()["paths"]
112
paths = [re.sub('^"|"$', '', x.strip()) for x in pathsline.split(",")]
113
if folder in paths:
114
# this folder is already on the importPaths, so bail
115
print "done already"
116
return
117
pathsline += ', "%s"' % (folder,)
118
newpathline = "importPaths: [%s]" % pathsline
119
data = re.sub(ipre, newpathline, data)
120
fp = codecs.open(qmlprojfile, encoding="utf8", mode="w")
121
fp.write(data)
122
fp.close()
123
124
def add_to_project_specific_copy_path(self, folder):
125
"QML projects don't copy anything anywhere; they build in place. So this is not relevant."
126
pass
127
128
def add_to_import_path(self, component_type):
129
if component_type == "binary":
130
# since this is a binary component, add the part for our arch to the Qt Creator import path
131
# FIXME it would be nice to not have to shell out for this
132
ourarch = subprocess.check_output(["dpkg-architecture", "-qDEB_BUILD_GNU_TYPE"]).strip()
133
self.add_to_project_specific_import_path("./lib/%s" % (ourarch,))
134
# FIXME may need an add_to_project_specific_copy_path here too
135
elif component_type == "qml":
136
# qml projects build in place anyway
137
pass
138
139
140
class QMakeProject(Project):
141
def add_to_import_path(self, component_type):
142
if component_type == "binary":
143
print ("This is a qmake project.\n"
144
"You will need to ensure that all the ./lib/(arch)/ paths are added to "
145
"your import path when running the project, and that they are included "
146
"in the build process so they are copied into the build folder.")
147
elif component_type == "qml":
148
print ("This is a qmake project.\n"
149
"You will need to ensure that the ubuntu_component_store folder is "
150
"added to your import path when running the project, and that it is "
151
"included in the build process so it is copied into the build folder.")
152
153
class CMakeProject(QMakeProject):
154
def add_to_import_path(self, component_type):
155
if component_type == "binary":
156
print ("This is a cmake project.\n"
157
"You will need to ensure that all the ./lib/(arch)/ paths are added to "
158
"your import path when running the project, and that they are included "
159
"in the build process so they are copied into the build folder.")
160
elif component_type == "qml":
161
print ("This is a cmake project.\n"
162
"You will need to ensure that the ubuntu_component_store folder is "
163
"added to your import path when running the project, and that it is "
164
"included in the build process so it is copied into the build folder.")
165
166
def get_project():
167
"""Identify the current project type and return an appropriate wrapper.
168
169
Pure QML projects are in a folder Foo which has a Foo.qmlproject file in it.
170
qMake-based projects are in a folder Foo which has a Foo.pro file in it.
171
CMake-based projects are in a folder Foo which has CMakeLists.txt and that
172
file contains a project(Foo ...) line in it.
173
"""
174
175
def get_class_for_folder(folder):
176
afolder = os.path.abspath(folder)
177
basename = os.path.split(afolder)[1]
178
cmakelistname = os.path.join(afolder, "CMakeLists.txt")
179
if os.path.exists(os.path.join(afolder, "%s.%s" % (basename, "qmlproject"))):
180
return PureQMLProject, afolder
181
elif os.path.exists(os.path.join(afolder, "%s.%s" % (basename, "pro"))):
182
return QMakeProject, afolder
183
elif os.path.exists(cmakelistname):
184
fp = codecs.open(cmakelistname, encoding="utf8")
185
lines = fp.read().split("\n")
186
fp.close()
187
for line in lines:
188
m = re.match("^\s*project\s*\(\s*(?P<project>[^\s]+)((\s+[^\s]+)*\s*)\s*$", line)
189
if m:
190
if m.groupdict()["project"] == basename:
191
return CMakeProject, afolder
192
elif os.path.abspath(afolder) == "/":
193
# got to the root and found nothing
194
return None, None
195
return get_class_for_folder(os.path.join(afolder, ".."))
196
197
project_class, root_folder = get_class_for_folder(os.curdir)
198
if not project_class:
199
raise Exception("Not in an Ubuntu SDK app: %s" % (os.curdir))
200
return project_class(root_folder)
201
202
class Repository(object):
203
def install_component_from_branch_folder(self, branch, tree, folder, component_type, component):
204
""" Download a thing from an LP folder and install it.
205
206
To install a component we need to:
207
1. fetch it from LP
208
2. put it in the correct place.
209
For pure QML components, the component will be one or more QML files, which
210
are in the branch folder that we are passed, and they get put in:
211
<project root>/ubuntu_component_store/username/componentname/
212
For binary components, the component will be a libFoo.so file for each arch,
213
and the branch folder we are passed should contain files <arch>/libFoo.so
214
for each arch. We then set up:
215
<project root>/lib/$arch/ubuntu_component_store/username/componentname/
216
for each arch and copy the appropriate libFoo.so into each.
217
3. Set up qmldir files.
218
Pure QML components get a qmldir file
219
<project root>/ubuntu_component_store/username/componentname/qmldir
220
with content
221
module RedRectangle
222
RedRectangle 1.0 RedRectangle.qml
223
Binary components (libFoo.so, in component myname/BarPlugin) get a qmldir file
224
<project root>/lib/$arch/ubuntu_component_store/myname/BarPlugin/qmldir
225
with content
226
module ubuntu_component_store.myname.BarPlugin
227
plugin Foo
228
4. Make sure the components are on the import path.
229
Pure QML components do this automatically; they are on the import path because
230
their folder (ubuntu_component_store/username/componentname/) is in the project
231
root, and that's on the import path. Binary components do not, because the
232
<project root>/lib/$arch folder is *not* on the import path for running in Qt Creator.
233
(It *is* on the import path when packaged as a click and installed on Ubuntu;
234
the click-apparmor wrapper /usr/bin/aa-exec-click sets it.) So it's important to
235
ensure when installing a binary component that it's on the import path. This happens
236
in different ways depending on the type of project:
237
238
Pure QML projects
239
These have a projectname.qmlproject file in the project root, with an importPaths key. We add
240
"./lib/$arch" to it, for the arch that this machine is currently using. This is a
241
bit dodgy (adding arch-specific paths), but this importPaths key *already* contains
242
arch-specific paths (frex, "/usr/lib/x86_64-linux-gnu/qt5/qml") and so adding a project-specific
243
arch-specific path can't make that any worse.
244
245
CMake-based projects
246
FIXME: dunno yet
247
248
qMake-based projects
249
FIXME: dunno yet
250
251
"""
252
project = get_project()
253
project.install_component_from_branch_folder(branch, tree, folder, component, component_type)
254
project.add_to_import_path(component_type)
255
256
def search_component_list(self, query, md):
257
matches = [{"componentname": x[0], "lcn": x[0].lower(),
258
"description": x[1].get("description", "(none)"),
259
"lcd": x[1].get("description", "").lower()}
260
for x in md.items()]
261
for word in query:
262
word = word.lower()
263
matches2 = []
264
for pkg in matches:
265
if word in pkg["lcn"] or word in pkg["lcd"]:
266
matches2.append(pkg)
267
matches = matches2
268
return matches
269
270
class CuratedRepository(Repository):
271
UCS_BRANCH = "lp:~ubuntu-touch-community-dev/component-store/trunk.14.10"
272
COMPONENTS_ROOT = "curated-store/ComponentStore"
273
274
def __init__(self, args):
275
self.__refresh = args.refresh
276
277
def __get_branch(self):
278
b = Branch.open(self.UCS_BRANCH)
279
tree = b.basis_tree()
280
return b, tree
281
282
def __update_metadata_if_required(self):
283
cache_dir = GLib.get_user_cache_dir()
284
my_cache_dir = os.path.join(cache_dir, "ubuntu_component_store")
285
try:
286
os.makedirs(my_cache_dir)
287
except OSError, e:
288
if e.errno != 17:
289
raise
290
291
cached_curated_file = os.path.join(my_cache_dir, "ucs-curated-components-metadata.json")
292
293
branch, tree = self.__get_branch()
294
fetch_packages = False
295
try:
296
fp = codecs.open(cached_curated_file, encoding="utf8")
297
component_metadata = json.load(fp)
298
if branch.revno() > component_metadata.get("cached_revno", 0):
299
fetch_packages = True
300
except:
301
fetch_packages = True
302
303
if self.__refresh:
304
fetch_packages = True
305
self.__refresh = False
306
307
if fetch_packages:
308
component_metadata = {"cached_revno": branch.revno(), "components": {}}
309
cns = [path
310
for (path, versioned, kind, file_id, inventory_entry)
311
in tree.list_files(from_dir=self.COMPONENTS_ROOT)
312
if kind == "directory"
313
]
314
for c in cns:
315
try:
316
mdpath = "%s/%s/%s" % (self.COMPONENTS_ROOT, c, "ubuntu_component_store.json")
317
file_id = tree.path2id(mdpath)
318
mdjson = tree.get_file_text(file_id)
319
md = json.loads(mdjson)
320
component_metadata["components"][c] = md
321
except:
322
print "(unable to get metadata for curated component %s)" % (c,)
323
fp = codecs.open(cached_curated_file, mode="w", encoding="utf8")
324
json.dump(component_metadata, fp)
325
fp.close()
326
return component_metadata
327
328
def install(self, cn):
329
print("(installing {0} from curated repository)".format(cn))
330
component_metadata = self.__update_metadata_if_required()
331
component = component_metadata["components"].get(cn)
332
if not component:
333
raise Exception("Unable to find curated component '%s'." % (cn,))
334
b, tree = self.__get_branch()
335
component_path = "/".join([self.COMPONENTS_ROOT, cn])
336
file_id = tree.path2id(component_path)
337
if not file_id: # should not happen because it's in the metadata
338
raise Exception("Unable to find curated component '%s'." % (cn,))
339
# Curated components are all QML-only
340
component["name"] = "Curated/%s" % (component["name"],)
341
self.install_component_from_branch_folder(b, tree, component_path, "qml", component)
342
343
def update(self, cn):
344
raise NotImplementedError
345
346
def search(self, query):
347
component_metadata = self.__update_metadata_if_required()
348
return self.search_component_list(query, component_metadata.get("components", {}))
349
350
class CommunityRepository(Repository):
351
PACKAGES_FILE_BRANCH = "lp:~ubuntu-touch-community-dev/component-store/community-components-json"
352
PACKAGES_FILE_NAME = "community_components.json"
353
354
def __init__(self, args):
355
self.__refresh = args.refresh
356
357
def __update_packages_file_if_required(self):
358
cache_dir = GLib.get_user_cache_dir()
359
my_cache_dir = os.path.join(cache_dir, "ubuntu_component_store")
360
try:
361
os.makedirs(my_cache_dir)
362
except OSError, e:
363
if e.errno != 17:
364
raise
365
366
cached_packages_file = os.path.join(my_cache_dir, "ucs-packages-cache.json")
367
368
fetch_packages = False
369
try:
370
age = time.time() - os.stat(cached_packages_file).st_mtime
371
if age > 3600: fetch_packages = True
372
except OSError:
373
fetch_packages = True
374
if self.__refresh:
375
fetch_packages = True
376
self.__refresh = False
377
378
if fetch_packages:
379
packages_data = self.__fetch_packages_file(cached_packages_file)
380
else:
381
fp = codecs.open(cached_packages_file, encoding="utf8")
382
try:
383
packages_data = json.load(fp)
384
except:
385
raise Exception("Local package cache seems corrupt")
386
fp.close()
387
return packages_data
388
389
def __fetch_packages_file(self, cached_packages_file):
390
print "(getting packages file)"
391
b = Branch.open(self.PACKAGES_FILE_BRANCH)
392
tree = b.basis_tree()
393
file_id = tree.path2id(self.PACKAGES_FILE_NAME)
394
if not file_id:
395
raise Exception("Unable to find the packages file.")
396
try:
397
packages_data = tree.get_file_text(file_id)
398
except:
399
raise Exception("Unable to read the packages file.")
400
try:
401
packages_data = json.loads(packages_data)
402
except:
403
raise Exception("Master packages file seems corrupt")
404
fp = codecs.open(cached_packages_file, mode="w", encoding="utf8")
405
json.dump(packages_data, fp)
406
fp.close()
407
return packages_data
408
409
def install(self, cn):
410
packages_data = self.__update_packages_file_if_required()
411
print("(installing {0} from community repository)".format(cn))
412
pkg = packages_data.get("components", {}).get(cn)
413
if not pkg:
414
print("Package {0} not found".format(cn))
415
return
416
print("OK installing.")
417
b = Branch.open(pkg["lpurl"])
418
tree = b.repository.revision_tree(b.get_rev_id(pkg["revno"]))
419
if "qml" in pkg["type"]:
420
self.install_component_from_branch_folder(b, tree, "qml", "qml", pkg)
421
if "binary" in pkg["type"]:
422
self.install_component_from_branch_folder(b, tree, "qmllib", "binary", pkg)
423
424
def update(self, cn):
425
raise NotImplementedError
426
427
def search(self, query):
428
packages_data = self.__update_packages_file_if_required()
429
return self.search_component_list(query, packages_data.get("components", {}))
430
431
def submit(self, lpurl):
432
433
def chunk_read(response):
434
read = ""
435
436
while 1:
437
chunk = response.read(1)
438
read += chunk
439
440
if "\n" in read:
441
lines = read.split("\n")
442
read = lines.pop(-1)
443
lines = [x.split(":", 1) for x in lines]
444
if lines:
445
for x in lines:
446
if x[0] != "ok":
447
print "Error: %s" % x[1]
448
return False
449
print "\n".join([x[1] for x in lines])
450
451
if not chunk:
452
print read
453
if not read.startswith("ok:"): return False
454
return True
455
return True
456
457
try:
458
response = urllib2.urlopen('https://sil.pythonanywhere.com/submit', data=urllib.urlencode({"lpurl": lpurl}));
459
except urllib2.HTTPError, e:
460
print("Error while submitting:")
461
print(e.read())
462
return
463
ok = chunk_read(response)
464
if not ok: sys.exit(1)
465
466
def get_repository_from_component_name(args):
467
if "/" in args.componentname:
468
return CommunityRepository(args)
469
else:
470
return CuratedRepository(args)
471
472
def cmd_install(args):
473
get_repository_from_component_name(args).install(args.componentname)
474
def cmd_update(args):
475
get_repository_from_component_name(args).update(args.componentname)
476
def cmd_search(args):
477
print("Matching curated components")
478
print("===========================")
479
cu = CuratedRepository(args)
480
results = cu.search(args.query)
481
for result in results:
482
print(" {componentname:>30} {description}".format(**result))
483
if not results:
484
print(" {0:>30}".format("(no matches)"))
485
print
486
print("Matching community components")
487
print("=============================")
488
co = CommunityRepository(args)
489
results = co.search(args.query)
490
for result in results:
491
print(" {componentname:>30} {description}".format(**result))
492
if not results:
493
print(" {0:>30}".format("(no matches)"))
494
495
def cmd_submit(args):
496
print("(submitting to community repository)")
497
args.refresh = False
498
co = CommunityRepository(args)
499
co.submit(args.launchpadBranchURL)
500
501
if __name__ == "__main__":
502
parser = argparse.ArgumentParser(description="Install and manage third-party components into your Ubuntu SDK app")
503
subparsers = parser.add_subparsers(dest="subparser_name")
504
parser_install = subparsers.add_parser("install", help="Install a component into this app")
505
parser_install.add_argument("componentname", type=str,
506
help="ComponentName for curated components, developer/ComponentName for community components")
507
parser_install.add_argument("--refresh", action="store_true", help="Refresh the packages list")
508
parser_install.set_defaults(func=cmd_install)
509
parser_update = subparsers.add_parser("update", help="Update an installed component in this app")
510
parser_update.add_argument("componentname", type=str,
511
help="ComponentName for curated components, developer/ComponentName for community components")
512
parser_update.add_argument("--refresh", action="store_true", help="Refresh the packages list")
513
parser_update.set_defaults(func=cmd_update)
514
parser_search = subparsers.add_parser("search", help="Search names and descriptions of existing components")
515
parser_search.add_argument("query", type=str, nargs="*", help="Search terms")
516
parser_search.add_argument("--refresh", action="store_true", help="Refresh the packages list")
517
parser_search.set_defaults(func=cmd_search)
518
parser_submit = subparsers.add_parser("submit", help="Publish a component to the community store")
519
parser_submit.add_argument("launchpadBranchURL", type=str, help="lp:~username/project/branch")
520
parser_submit.set_defaults(func=cmd_submit)
521
parser_help = subparsers.add_parser("help", help="Help on subcommands")
522
parser_help.add_argument("subcommand", nargs="?", type=str)
523
524
args = parser.parse_args()
525
if args.subparser_name == "help":
526
if args.subcommand:
527
parser.parse_args([args.subcommand, "-h"])
528
else:
529
parser.parse_args(["-h"])
530
else:
531
args.func(args)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1