← Back to branch summary

~ubuntu-branches/ubuntu/karmic/pypy/karmic

« back to all changes in this revision

Viewing changes to py/test/rsession/hostmanage.py

  • Committer: Bazaar Package Importer
  • Author(s): Alexandre Fayolle
  • Date: 2007-04-13 09:33:09 UTC
  • Revision ID: james.westby@ubuntu.com-20070413093309-yoojh4jcoocu2krz
Tags: upstream-1.0.0
Import upstream version 1.0.0

Show diffs side-by-side

added added

removed removed

Lines of Context:
py/test/rsession/hostmanage.py
 
1
import sys, os
 
2
import py
 
3
import time
 
4
import thread, threading 
 
5
from py.__.test.rsession.master import MasterNode
 
6
from py.__.test.rsession.slave import setup_slave
 
7
 
 
8
from py.__.test.rsession import repevent
 
9
 
 
10
class HostInfo(object):
 
11
    """ Class trying to store all necessary attributes
 
12
    for host
 
13
    """
 
14
    _hostname2list = {}
 
15
    
 
16
    def __init__(self, spec, addrel=""):
 
17
        parts = spec.split(':', 1)
 
18
        self.hostname = parts.pop(0)
 
19
        self.relpath = parts and parts.pop(0) or ""
 
20
        if self.hostname == "localhost" and not self.relpath:
 
21
            self.inplacelocal = True
 
22
            addrel = "" # inplace localhosts cannot have additions
 
23
        else:
 
24
            self.inplacelocal = False 
 
25
            if not self.relpath:
 
26
                self.relpath = "pytestcache-%s" % self.hostname
 
27
        if addrel:
 
28
            self.relpath += "/" + addrel # XXX too os-dependent
 
29
        assert not parts
 
30
        assert self.inplacelocal or self.relpath
 
31
        self.hostid = self._getuniqueid(self.hostname) 
 
32
 
 
33
    def _getuniqueid(self, hostname):
 
34
        l = self._hostname2list.setdefault(hostname, [])
 
35
        hostid = hostname + "[%d]" % len(l)
 
36
        l.append(hostid)
 
37
        return hostid
 
38
 
 
39
    def initgateway(self, python="python"):
 
40
        if self.hostname == "localhost":
 
41
            self.gw = py.execnet.PopenGateway(python=python)
 
42
        else:
 
43
            self.gw = py.execnet.SshGateway(self.hostname, 
 
44
                                            remotepython=python)
 
45
        if self.inplacelocal:
 
46
            self.gw.remote_exec(py.code.Source(
 
47
                sethomedir, "sethomedir()"
 
48
            )).waitclose()
 
49
            self.gw_remotepath = None
 
50
        else:
 
51
            assert self.relpath
 
52
            channel = self.gw.remote_exec(py.code.Source(
 
53
                gethomedir,
 
54
                sethomedir, "sethomedir()", 
 
55
                getpath_relto_home, """
 
56
                channel.send(getpath_relto_home(%r))
 
57
            """ % self.relpath,
 
58
            ))
 
59
            self.gw_remotepath = channel.receive()
 
60
 
 
61
    def __str__(self):
 
62
        return "<HostInfo %s:%s>" % (self.hostname, self.relpath)
 
63
    __repr__ = __str__
 
64
 
 
65
    def __hash__(self):
 
66
        return hash(self.hostid)
 
67
 
 
68
    def __eq__(self, other):
 
69
        return self.hostid == other.hostid
 
70
 
 
71
    def __ne__(self, other):
 
72
        return not self.hostid == other.hostid
 
73
 
 
74
class HostRSync(py.execnet.RSync):
 
75
    """ RSyncer that filters out common files 
 
76
    """
 
77
    def __init__(self, sourcedir, *args, **kwargs):
 
78
        self._synced = {}
 
79
        ignores= None
 
80
        if 'ignores' in kwargs:
 
81
            ignores = kwargs.pop('ignores')
 
82
        self._ignores = ignores or []
 
83
        super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
 
84
 
 
85
    def filter(self, path):
 
86
        path = py.path.local(path)
 
87
        if not path.ext in ('.pyc', '.pyo'):
 
88
            if not path.basename.endswith('~'): 
 
89
                if path.check(dotfile=0):
 
90
                    for x in self._ignores:
 
91
                        if path == x:
 
92
                            break
 
93
                    else:
 
94
                        return True
 
95
 
 
96
    def add_target_host(self, host, destrelpath="", reporter=lambda x: None):
 
97
        remotepath = host.gw_remotepath 
 
98
        key = host.hostname, host.relpath
 
99
        if host.inplacelocal: 
 
100
            remotepath = self._sourcedir
 
101
            self._synced[key] = True
 
102
        elif destrelpath:
 
103
            remotepath = os.path.join(remotepath, destrelpath)
 
104
        synced = key in self._synced 
 
105
        reporter(repevent.HostRSyncing(host, py.path.local(self._sourcedir), 
 
106
                                       remotepath, synced))
 
107
        def hostrsynced(host=host):
 
108
            reporter(repevent.HostRSyncRootReady(host, self._sourcedir))
 
109
        if key in self._synced:
 
110
            hostrsynced()
 
111
            return
 
112
        self._synced[key] = True
 
113
        super(HostRSync, self).add_target(host.gw, remotepath, 
 
114
                                          finishedcallback=hostrsynced,
 
115
                                          delete=True,
 
116
                                          )
 
117
        return remotepath 
 
118
 
 
119
class HostManager(object):
 
120
    def __init__(self, config, hosts=None):
 
121
        self.config = config
 
122
        roots = self.config.getvalue_pathlist("dist_rsync_roots")
 
123
        addrel = ""
 
124
        if roots is None:
 
125
            roots = [self.config.topdir]
 
126
            addrel = self.config.topdir.basename 
 
127
        self._addrel = addrel
 
128
        self.roots = roots
 
129
        if hosts is None:
 
130
            hosts = self.config.getvalue("dist_hosts")
 
131
            hosts = [HostInfo(x, addrel) for x in hosts]
 
132
        self.hosts = hosts
 
133
 
 
134
    def prepare_gateways(self, reporter):
 
135
        python = self.config.getvalue("dist_remotepython")
 
136
        for host in self.hosts:
 
137
            host.initgateway(python=python)
 
138
            reporter(repevent.HostGatewayReady(host, self.roots))
 
139
            host.gw.host = host
 
140
 
 
141
    def init_rsync(self, reporter):
 
142
        ignores = self.config.getvalue_pathlist("dist_rsync_ignore")
 
143
        self.prepare_gateways(reporter)
 
144
        # send each rsync root
 
145
        for root in self.roots:
 
146
            rsync = HostRSync(root, ignores=ignores, 
 
147
                              verbose=self.config.option.verbose)
 
148
            if self._addrel: 
 
149
                destrelpath = ""
 
150
            else:
 
151
                destrelpath = root.basename
 
152
            for host in self.hosts:
 
153
                rsync.add_target_host(host, destrelpath, reporter)
 
154
            rsync.send(raises=False)
 
155
 
 
156
    def setup_hosts(self, reporter):
 
157
        self.init_rsync(reporter)
 
158
        nodes = []
 
159
        for host in self.hosts:
 
160
            if hasattr(host.gw, 'remote_exec'): # otherwise dummy for tests :/
 
161
                ch = setup_slave(host, self.config)
 
162
                nodes.append(MasterNode(ch, reporter))
 
163
        return nodes
 
164
 
 
165
    def teardown_hosts(self, reporter, channels, nodes,
 
166
                       waiter=lambda : time.sleep(.1), exitfirst=False):
 
167
        for channel in channels:
 
168
            channel.send(None)
 
169
    
 
170
        clean = exitfirst
 
171
        while not clean:
 
172
            clean = True
 
173
            for node in nodes:
 
174
                if node.pending:
 
175
                    clean = False
 
176
            waiter()
 
177
        self.teardown_gateways(reporter, channels)
 
178
 
 
179
    def kill_channels(self, channels):
 
180
        for channel in channels:
 
181
            channel.send(42)
 
182
 
 
183
    def teardown_gateways(self, reporter, channels):
 
184
        for channel in channels:
 
185
            #try:
 
186
            try:
 
187
                repevent.wrapcall(reporter, channel.waitclose, 1)
 
188
            except IOError: # timeout
 
189
                # force closing
 
190
                channel.close()
 
191
            channel.gateway.exit()
 
192
 
 
193
def gethomedir():
 
194
    import os
 
195
    homedir = os.environ.get('HOME', '')
 
196
    if not homedir:
 
197
        homedir = os.environ.get('HOMEPATH', '.')
 
198
    return os.path.abspath(homedir)
 
199
 
 
200
def getpath_relto_home(targetpath):
 
201
    import os
 
202
    if not os.path.isabs(targetpath):
 
203
        homedir = gethomedir()
 
204
        targetpath = os.path.join(homedir, targetpath)
 
205
    return os.path.normpath(targetpath)
 
206
 
 
207
def sethomedir():
 
208
    import os
 
209
    homedir = os.environ.get('HOME', '')
 
210
    if not homedir:
 
211
        homedir = os.environ.get('HOMEPATH', '.')
 
212
    os.chdir(homedir)