~vauxoo/openerp-tools/openerprunbot-add_many_branches-dev-moylop260

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
"""
Main data structures for the Runbot.
"""
import copy
import optparse
import os
import re
import signal
import simplejson
import subprocess
import sys
import threading
import time
import traceback

import mako.template

import openerprunbot
from openerprunbot.misc import *
from openerprunbot.jobs.install_all import InstallAllJob

# list of managed series, used to complement monitored branches
# (i.e. to create branch groups).
SERIES_BRANCHES = ['6.0', '6.1', '7.0', 'saas-1', 'trunk']

# Number of build 'slots' per branch group.
POINTS = 5

# Job states
STATE_ALLOCATED = 'allocated'
STATE_BROKEN = 'broken'
STATE_PULLING = 'pulling'
STATE_RUNNING = 'running'
STATE_TESTING = 'testing'

# This constant matches community_dashboard.py.
NEW_MERGE_STATUS = ['Needs review', 'Code failed to merge', 'Approved']

# Hack to allow more ports than strictly necessary.
ADDITIONAL_NUMBER = 50

#----------------------------------------------------------
# OpenERP RunBot Branch
#----------------------------------------------------------

class RunBotBranch(object):
    def __init__(self, runbot, branch, group=None,
        repo_path=None, project_name=None, trigger_build=False):
        """
        Represent a single Launchpad branch, e.g.
        `~openerp-dev/openobject-server/trunk-foo-bar`.
        """
        self.runbot = runbot
        self.group = group
        self.trigger_build = trigger_build

        # e.g. trunk
        self.name = branch.name
        self.name_underscore = underscore(self.name)
        self.name_dashed = dashes(self.name)
        # e.g. ~openerp/openobject-server/trunk
        self.unique_name = branch.unique_name
        self.unique_name_underscore = underscore(self.unique_name)
        self.unique_name_dashes = dashes(self.unique_name)
        # server, addons, or web (both for openobject-client-web and openerp-web)
        if project_name:
            self.project_name = project_name
        else:
            self.project_name = project_name_from_unique_name(self.unique_name)

        self.lp_date_last_modified = branch.date_last_modified
        self.lp_revision_count = branch.revision_count

        self.local_date_last_modified = 0
        self.local_revision_count = 0
        self.merge_count = 0

        # Repository path <Root>/repo/<branchuniquename>
        if repo_path:
            self.repo_path=repo_path
        else:
            self.repo_path=os.path.join(self.runbot.wd,'repo',self.unique_name_underscore)

        self.committer_name = None
        self.committer_xgram = None
        self.committer_email = None

    def age(self):
        return time.time()-int(self.local_date_last_modified.strftime("%s"))

    def update_launchpad(self,branch):
        self.lp_date_last_modified=branch.date_last_modified
        self.lp_revision_count=branch.revision_count
        if self.local_revision_count != self.lp_revision_count:
            self.local_date_last_modified = self.lp_date_last_modified
            self.local_revision_count = self.lp_revision_count
            if self.trigger_build:
                name = self.project_name
                self.group.need_run_reason.append(name)

#----------------------------------------------------------
# OpenERP RunBot Grouped Branch
#----------------------------------------------------------

class RunBotGroupedBranch(object):
    """
    A single 'branch group' represents a single branch name for all our
    projects. E.g. trunk is available in server, addons, web, and client-web.

    When the corresponding project doesn't have that branch, trunk is used.
    For instance, a branch group can represent
        `~openerp-dev/openobject-server/trunk-foo-bar`
        `~openerp-dev/openobject-addons/trunk-foo-bar`
    and
        `~openerp/openobject-web-client/6.0`
        `~openerp/openerp-web/trunk`
    will be used to complete the group, building the four branches together.

    Alternatively a 'branch group' can be configured. In that case, all
    branches are manually specified; no attempt is done to complete the group.
    """
    def __init__(self, runbot, team_name, name, version, sticky,
            server_branch=None, client_web_branch=None,
            web_branch=None, addons_branches=None, modules=None, job_type=None):
        self.runbot = runbot
        self.sticky = sticky
        self.job_type = job_type

        # Points are build 'slots'
        self.points = [None for x in xrange(POINTS)]

        self.team_name = team_name
        self.name = name # group name or manually chosen name.
        self.name_underscore = underscore(self.name)
        self.name_dashes = dashes(self.name)
        self.version = version # '6.0', '6.1', '7.0'  or 'trunk'

        # Reason to do a build, e.g. because 'addons' was commited to.
        self.need_run_reason = []

        self.server_branch = server_branch
        self.web_branch = client_web_branch or web_branch
        self.addons_branches = addons_branches

        self.server = None
        self.web = None
        self.addons = []

        self.json_path=os.path.join(runbot.wd,'static',"%s-%s.json"%(team_name, self.name.replace('_','-').replace('.','-')))

        # List of modules to install instead of `all`.
        self.modules = modules.split(',') if modules else []

        self.configured = False

        # A normal build_number is maxint, a build_number is assigned when a
        # build is manually requested or a triggering branch has a new revision.
        # The build number is used to assign a position in the queue.
        # The value will be reset to maxint after the build is done.
        self.build_number = sys.maxint

        # True when it is not possible to provide default branches based on
        # the name.
        self.wrong_matching = False

    def add_point(self, j):
        assert j.state == STATE_ALLOCATED
        job0 = self.points[0]
        if job0:
            job0.stop()
            try:
                del self.runbot.jobs[job0.job_id]
            except KeyError:
                pass
        self.points = self.points[1:] + [j]

    def complete_point(self, j):
        assert j.state in (STATE_RUNNING, STATE_BROKEN)
        for p in self.points + [None]:
            if p and p.job_id == j.job_id:
                break
        if p and p.state != j.state:
            p.save_json()

    def all_points_completed(self):
        for p in self.points:
            if p is not None and p.state not in (STATE_RUNNING, STATE_BROKEN):
                return False
        return True

    def is_sticky_job(self, j):
        """ Return True if the job is the latest running point of a sticky branch. """
        runnings = filter(lambda x: x and x.state == STATE_RUNNING, self.points)
        return self.sticky and runnings and runnings[-1].job_id == j.job_id

    def update_state(self, state):
        """Copy some values taken from the state.runbot file."""
        if self.team_name == 'openerp-dev' and self.name in SERIES_BRANCHES:
            # Don't change stickiness for the hard-coded branches.
            return
        previous = self.sticky
        self.sticky = state.get(self.team_name, {}).get(self.name, {}).get('sticky', False)
        if self.sticky != previous:
            log("stickiness changed", team=self.team_name, branch=self.name, now=self.sticky)

    def repo_updates(self):
        return [copy.copy(x) for x in self.all_branches()]

    def all_branches(self):
        r = []
        r.append(self.server)
        r.extend([x for x in self.addons])
        r.append(self.web)
        return r

    def lp_date_last_modified(self):
        return max(p.lp_date_last_modified for p in self.all_branches() if p and p.trigger_build)

    def is_ok(self):
        """
        Test whether the group is useable, i.e. if self.configure_branches()
        didn't early return.
        """
        return self.server and self.web and self.addons

    # TODO all the xxxx_branch argument in __init__ can be moved here.
    def configure_branches(self, launchpad):
        """ Fetch the configured branches. """
        log("runbot-populate-configured-branches")
        self.configured = True
        repo = os.path.join(self.runbot.wd, 'repo')
        filters = {'status': NEW_MERGE_STATUS}

        path = os.path.join(repo, '__configured_' + self.name + '_server')
        b = launchpad.get_branch(unique_name=self.server_branch)
        if not b:
            log("WARNING:no such unique name", name=self.server_branch)
            return
        self.server = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='server', trigger_build=True)
        self.server.update_launchpad(b)
        #self.server.merge_count = len(list(b.getMergeProposals(**filters)))

        if self.web_branch:
            path = os.path.join(repo, '__configured_' + self.name + '_web')
            b = launchpad.get_branch(unique_name=self.web_branch)
            if not b:
                log("WARNING:no such unique name", name=self.web_branch)
                return
            self.web = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='web', trigger_build=True)
            self.web.update_launchpad(b)
            #self.web.merge_count = len(list(b.getMergeProposals(**filters)))
        else:
            self.web = None

        self.addons = []
        for x in xrange(len(self.addons_branches)):
            path = os.path.join(repo, '__configured_' + self.name + '_addons_' + str(x+1) + 'of' + str(len(self.addons)))
            b = launchpad.get_branch(unique_name=self.addons_branches[x])
            if not b:
                log("WARNING:no such unique name", name=self.addons_branches[x])
                self.addons = []
                return
            bb = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='addons', trigger_build=True)
            bb.update_launchpad(b)
            #bb.merge_count = len(list(b.getMergeProposals(**filters)))
            self.addons.append(bb)

    def add_branch(self, b):
        """ Add a single branch (when not using configure_branches(). """
        assert b.name == self.name
        b.group = self
        if b.project_name == 'addons':
            self.addons = [b]
        if b.project_name == 'server':
            self.server = b
        if b.project_name == 'web':
            self.web = b

    # Note: any access to launchpad is done in the same thread.
    def add_main_branches(self):
        """
        Add trunk or 6.{0,1} branches when the complementary branches are missing.
        Also do it when the addons branch is not complete, i.e. when missing
        one of the official addons.
        """
        repo = os.path.join(self.runbot.wd,'repo')

        matched = [s for s in SERIES_BRANCHES if self.name.startswith(s)]
        if matched:
            self.version = matched[0]
            ending = '_' + self.version
        else:
            self.wrong_matching = True
            ending = "_trunk"
            self.version = 'trunk' # TODO this is not accurate

        paths = {
            'addons': os.path.join(repo, 'openerp_openobject-addons_' + self.name_underscore),
            'server': os.path.join(repo, 'openerp_openobject-server_' + self.name_underscore),
            'web': os.path.join(repo, 'openerp_openerp-web_' + self.name_underscore),
        }
        if self.version == '6.0':
            paths['web'] = os.path.join(repo, 'openerp_openobject-client-web_' + self.name_underscore)

        for p in ('server', 'web'):
            if not getattr(self, p):
                b = self.runbot.main_branches[p+ending]
                setattr(self, p, RunBotBranch(self.runbot,b,group=self,repo_path=paths[p]))
                getattr(self, p).update_launchpad(b)
            elif not getattr(self, p).trigger_build:
                b = self.runbot.main_branches[p+ending]
                getattr(self, p).update_launchpad(b)
        if not self.addons:
            b = self.runbot.main_branches['addons'+ending]
            self.addons = [RunBotBranch(self.runbot,b,group=self,repo_path=paths['addons'])]
            self.addons[0].update_launchpad(b)
        elif not self.addons[0].trigger_build:
            b = self.runbot.main_branches['addons'+ending]
            self.addons[0].update_launchpad(b)

#----------------------------------------------------------
# OpenERP RunBot Engine
#----------------------------------------------------------

class RunBot(object):
    """Used as a singleton, to manage almost all the Runbot state and logic."""
    def __init__(self, wd, poll, server_net_port, server_xml_port,
        client_web_port, server_longpolling_port, number, nginx_port, domain, test, workers,
        current_job_id, debug, lp, default_job_type='install_all'):
        self.wd=wd
        self.server_net_port=int(server_net_port)
        self.server_xml_port=int(server_xml_port)
        self.client_web_port=int(client_web_port)
        self.server_longpolling_port=int(server_longpolling_port)
        self.number=int(number)
        self.nginx_port=int(nginx_port)
        self.domain=domain
        self.test=int(test)
        self.workers = int(workers)
        self.branches={}
        self.groups={}
        self.main_branches = {}
        self.allocated_port = 0
        self.jobs = {}
        # current_job_id is used to assign build number to groups (thus
        # realizing a queue of groups to be processed).
        self.current_job_id = current_job_id
        self.debug = debug
        self.next_build_number = 0
        self.launchpad = lp
        self.checked_date_last_modified = None
        self.default_job_type = default_job_type

    def get_by_host(self, host):
        # Not thread-safe...
        print "Searching for", host
        try:
            d = {}
            for g in self.groups.values():
                if not any(g.points): continue
                for i in g.points:
                    print i
                    if i is not None and \
                        i and (i.job_id > self.current_job_id - self.number or g.is_sticky_job(i)) and \
                        i.running_since and \
                        i.db_prefix == host:
                        print "Found."
                        # Via nginx:
                        d['port'] = str(self.nginx_port)
                        d['name'] = i.db_prefix + '-all'
                        # Directly:
                        d['port'] = str(self.server_xml_port + i.port) # valid for >= 6.1
                        d['name'] = i.db_prefix + '-all'
                        break
                if d: break
            j = simplejson.dumps(d, sort_keys=True, indent=4)
            return j
        except Exception, e:
            print "Exception in get_by_host():", e
            return simplejson.dumps({}, sort_keys=True, indent=4)

    def registered_teams(self):
        return openerprunbot.state.get('registered-teams', []) + ['openerp-dev']

    def nginx_reload(self):
        nginx_pid_path = os.path.join(self.wd,'nginx','nginx.pid')
        if os.path.isfile(nginx_pid_path):
            pid=int(open(nginx_pid_path).read())
            os.kill(pid,1)
        else:
            run(["nginx","-p",self.wd,"-c",os.path.join(self.wd,"nginx/nginx.conf")])

    def nginx_config(self):
        return openerprunbot.templates.render_template('nginx.conf.mako',r=self)

    def registration_page(self):
        return openerprunbot.templates.render_template('registration.html.mako',r=self)

    def nginx_index_time(self,t):
        for m,u in [(86400,'d'),(3600,'h'),(60,'m')]:
            if t>=m:
                return str(int(t/m))+u
        return str(int(t))+"s"

    def nginx_index_sticky(self):
        #branches = [(b.subdomain,b) for b in self.running if b.sticky]
        #branches.sort()
        #return [b for (x,b) in branches]
        return []

    def nginx_groups_sticky(self,team_name):
        gs = [(g.name,g) for g in self.groups.values() if any(g.points) and g.team_name==team_name and g.sticky]
        gs.sort()
        return [g for (x,g) in gs]

    def nginx_groups_others(self,team_name):
        gs = [(g.sticky, g.lp_date_last_modified(), g) for g in self.groups.values() if any(g.points) and g.team_name==team_name and not g.sticky]
        gs.sort(reverse=1)
        gs = [g for (x,y,g) in gs]
        return gs

    def nginx_groups_registered(self,team_name):
        gs = [(g.name,g) for g in self.groups.values() if not any(g.points) and g.team_name==team_name and g.build_number == sys.maxint]
        gs.sort()
        return [g for (x,g) in gs]

    def nginx_index_others(self):
        #branches = [(b.local_date_last_modified,b) for b in self.running if not b.sticky]
        #branches.sort(reverse=1)
        #return [b for (x,b) in branches]
        return []

    def nginx_index(self, team_name):
        return openerprunbot.templates.render_template('branches.html.mako',
            r=self, t=time.time(), re=re, team_name=team_name, sys=sys)

    def nginx_update(self):
        for team_name in self.teams_with_branches():
            try:
                f = None
                if team_name == 'openerp-dev':
                    fn = 'index.html'
                else:
                    fn = team_name + '.html'
                f = open(os.path.join(self.wd,'static',fn),"w")
                content = self.nginx_index(team_name)
                f.write(content)
            except Exception, e:
                log("WARNING: exception when templating %s:" % fn)
                print traceback.format_exc()
                if f: f.close()
                break

        try:
            f = open(os.path.join(self.wd,'static','register.html'),"w")
            content = self.registration_page()
            f.write(content)
        except Exception, e:
            log("WARNING: exception when templating register.html:")
            print traceback.format_exc()
            if f: f.close()

        try:
            f = open(os.path.join(self.wd,'nginx','nginx.conf'),"w")
            content = self.nginx_config()
            f.write(content)
        except Exception, e:
            log("WARNING: exception when templating nginx.conf:")
            print traceback.format_exc()
            if f: f.close()
        self.nginx_reload()

    def teams_with_branches(self):
        ts = [g.team_name for g in self.groups.values()]
        ts = list(set(ts))
        ts.sort()
        return ts

    def allocate_port(self):
        if self.allocated_port >= self.number + ADDITIONAL_NUMBER:
            self.allocated_port = 0
        self.allocated_port += 1
        return self.allocated_port

    def process_add(self, b, sticky, team_name):
        log("runbot-process-add", b.unique_name)
        if not project_name_from_unique_name(b.unique_name):
            log("WARNING: can't add branch: project name is not standard.", unique_name=b.unique_name)
            return
        if b.unique_name not in self.branches:
            self.branches[b.unique_name] = RunBotBranch(self, b, trigger_build=True)
        bb = self.branches[b.unique_name]
        if (team_name, bb.name) not in self.groups:
            self.groups[(team_name, bb.name)] = RunBotGroupedBranch(self, team_name, bb.name, None, sticky, job_type=self.default_job_type)
        self.groups[(team_name, bb.name)].add_branch(bb)
        bb.update_launchpad(b)
        #filters = {'status': NEW_MERGE_STATUS}
        #try:
        #    bb.merge_count = len(list(b.getMergeProposals(**filters)))
        #except Exception, e:
        #    bb.merge_count = 0

    def register_configured_branches(self):
        for team, v in openerprunbot.state.get('configured-branches', {}).items():
            if not team: continue
            if self.quit: return
            for name, c in v.items():
                if not name: continue
                g = RunBotGroupedBranch(self, team, name, c['version'], 0,
                    server_branch=c['server_branch'],
                    client_web_branch=c.get('client_web_branch'),
                    web_branch=c.get('web_branch'),
                    addons_branches=c['addons_branches'],
                    modules=c['modules'])

                # TODO or if it is already there but with different branches.
                if (g.team_name, g.name) not in self.groups:
                    g.configure_branches(self.launchpad)
                    if g.is_ok():
                        log("adding configured group", team=g.team_name, name=g.name)
                        self.groups[(g.team_name, g.name)] = g
                    else:
                        log("WARNING: misconfigured group", team=g.team_name, name=g.name)

    # Note: any access to launchpad is done in the same thread.
    def populate_branches(self):
        """Return all LP branches matching our teams and projects."""
        log("runbot-populate-branches")
        # Register main sticky branches
        for s in SERIES_BRANCHES:
            parts = {
                'server': '~openerp/openobject-server/' + s,
                'addons': '~openerp/openobject-addons/' + s,
                'web': '~openerp/' + ('openobject-client-web/' if s == '6.0' else 'openerp-web/') + s,
            }
            for p, v in parts.items():
                k = p + '_' + s
                if self.quit: return []
                b = self.launchpad.get_branch(unique_name=v)
                if self.quit: return []
                self.process_add(b, 1, 'openerp-dev')
                self.main_branches[k] = b

        # Register other branches
        for v in openerprunbot.state.get('registered-branches', []):
            if self.debug: break
            if self.quit: return []
            m = openerprunbot.branch_input_re.match(v)
            if m:
                if self.quit: return []
                b = self.launchpad.get_branch(unique_name=v)
                if self.quit: return []
                if not b:
                    # TODO better handling of launchpad connection problem
                    log("WARNING: can't get branch:", name=v)
                    continue
                self.process_add(b, 0, m.group(1))
            else:
                log("WARNING: malformed branch name:", v)

        # Register team branches
        for team_name in self.registered_teams():
            if self.debug: break
            if self.quit: return []
            log("Register team branches:", team=team_name)
            team_branches = self.launchpad.get_team_branches(team_name)
            for b in team_branches:
                if project_name_from_unique_name(b.unique_name):
                    self.process_add(b, 0, team_name)

        # Register configured branches
        self.register_configured_branches()

        for g in self.groups.values():
            if self.quit: return []
            if not g.configured:
                g.add_main_branches()

        self.checked_date_last_modified = max(x.lp_date_last_modified() for x in self.groups.values())
        self.assign_positions()
        return self.get_queue()

    def assign_positions(self):
        # Sort by last modification date, then assign a position in the queue.
        # Consider only newly modified groups.
        # Don't assign a new position if the position was not yet handled.
        gs = sorted(self.groups.values(), key=lambda x: (-x.sticky, x.lp_date_last_modified()))
        for g in gs:
            if not g.sticky and g.lp_date_last_modified() < self.checked_date_last_modified:
                continue
            if not g.sticky:
                self.checked_date_last_modified = g.lp_date_last_modified()
            if g.need_run_reason and g.build_number == sys.maxint:
                self.next_build_number +=1
                g.build_number = self.next_build_number

    def get_queue(self):
        gs = sorted(self.groups.values(), key=lambda x: x.build_number)
        return filter(lambda g: g.build_number != sys.maxint, gs)

    def available_workers(self):
        return self.workers - len([t for t in threading.enumerate() if t.name.startswith('runbot-group-worker-')])

    def process_command_queue(self):
        while not openerprunbot.queue.empty():
            command, params = openerprunbot.queue.get()
            if command == 'build':
                team_name, group_name = params
                if (team_name, group_name) in self.groups and self.groups[(team_name, group_name)].build_number == sys.maxint:
                    self.next_build_number += 1
                    self.groups[(team_name, group_name)].build_number = self.next_build_number
                    self.groups[(team_name, group_name)].need_run_reason.append('build')
            else:
                log("WARNING: unknown command", command)

    def reset_build_numbers(self):
        gs = [(g.build_number, g) for g in self.groups.values()]
        gs.sort()
        gs = [g for (x,g) in gs]
        self.next_build_number = 0
        sticky_branches = 0
        for g in gs:
            if g.sticky:
                sticky_branches += 1
            if g.build_number == sys.maxint:
                break
            self.next_build_number += 1
            g.build_number = self.next_build_number
        self.number = max(sticky_branches + 1, self.number)

    def complete_jobs(self):
        """Update all slots with the completed jobs."""
        for job in self.jobs.values():
            if job.state in (STATE_RUNNING, STATE_BROKEN):
                for g in self.groups.values():
                    if g.name == job.name:
                        g.complete_point(job)
                        break

    def is_sticky_job(self, job):
        """ Return True if the job is the latest point of a sticky branch. """
        for g in self.groups.values():
            if g.name == job.name:
                return g.is_sticky_job(job)

    def reap_oldest_job(self):
        """Kill some job to limit the number of processes."""
        if len(self.jobs) > self.number:
            victim = None
            for job in self.jobs.itervalues():
                if self.is_sticky_job(job):
                    continue
                if not job.running_since:
                    continue
                if not victim or job.job_id < victim.job_id:
                    victim = job
            if victim:
                victim.stop()
                try:
                    del self.jobs[victim.job_id]
                except KeyError:
                    pass

    def fetch_groups(self):
        """
        Fetch branch information from Launchpad.
        """
        log("runbot-process-grouped-branches")

        self.launchpad.connect()
        gs = self.populate_branches()

        log("runbot-run-group", groups=len(gs))
        return gs

    def run_groups(self, gs):
        for g in gs:
            if not self.available_workers():
                break
            if len(self.jobs) > self.number:
                break
            # Don't run multiple jobs at the same time for the same group.
            if g.need_run_reason and g.all_points_completed():
                port = self.allocate_port()
                self.current_job_id += 1
                job_class = openerprunbot.jobs.JOBS[g.job_type] if g.job_type else openerprunbot.jobs.JOBS[self.default_job_type]
                job = job_class(g, port, self.test, self.current_job_id, self.debug)
                g.add_point(job)
                g.need_run_reason = []
                g.build_number = sys.maxint
                self.jobs[job.job_id] = job
                job.spawn()

    def loop(self):
        """
        Repeatedly fetch branch information from Launchpad, run jobs, update
        HTML pages, ...
        """
        self.quit = False
        def signal_handler(sig, frame):
            log("SIGINT or SIGTERM received, exiting...")
            openerprunbot.server.stop_server()
            for i in self.jobs.values():
                i.stop()
            self.quit = True
        signal.signal(signal.SIGINT, signal_handler)
        signal.signal(signal.SIGTERM, signal_handler)
        while not self.quit:
            gs = []
            try:
                gs = self.fetch_groups()
            except Exception, e:
                log("WARNING: exception:")
                log(traceback.format_exc())

            for i in xrange(12): # 2 minutes
                if self.quit: break
                try: # Make sure time.sleep() is always called.
                    self.process_command_queue()
                    self.run_groups(gs)
                    self.complete_jobs()
                    self.reap_oldest_job()
                    self.reset_build_numbers()
                    self.nginx_update()
                    for g in self.groups.values():
                        g.update_state(openerprunbot.state)
                except Exception, e:
                    log("WARNING: exception:")
                    log(traceback.format_exc())
                time.sleep(10)