← Back to branch summary

~david-goetz/swift/wal

« back to all changes in this revision

Viewing changes to test/unit/common/test_manager.py

  • Committer: David Goetz
  • Date: 2011-05-02 19:59:51 UTC
  • mfrom: (203.2.83 swift)
  • Revision ID: david.goetz@rackspace.com-20110502195951-w15psa53urkqhoml
update to trunk

Show diffs side-by-side

added added

removed removed

Lines of Context:
test/unit/common/test_manager.py
 
1
# Copyright (c) 2010-2011 OpenStack, LLC.
 
2
#
 
3
# Licensed under the Apache License, Version 2.0 (the "License");
 
4
# you may not use this file except in compliance with the License.
 
5
# You may obtain a copy of the License at
 
6
#
 
7
#    http://www.apache.org/licenses/LICENSE-2.0
 
8
#
 
9
# Unless required by applicable law or agreed to in writing, software
 
10
# distributed under the License is distributed on an "AS IS" BASIS,
 
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 
12
# implied.
 
13
# See the License for the specific language governing permissions and
 
14
# limitations under the License.
 
15
 
 
16
import unittest
 
17
from nose import SkipTest
 
18
from test.unit import temptree
 
19
 
 
20
import os
 
21
import sys
 
22
import resource
 
23
import signal
 
24
import errno
 
25
from contextlib import contextmanager
 
26
from collections import defaultdict
 
27
from threading import Thread
 
28
from time import sleep, time
 
29
 
 
30
from swift.common import manager
 
31
 
 
32
DUMMY_SIG = 1
 
33
 
 
34
 
 
35
class MockOs():
 
36
 
 
37
    def __init__(self, pids):
 
38
        self.running_pids = pids
 
39
        self.pid_sigs = defaultdict(list)
 
40
        self.closed_fds = []
 
41
        self.child_pid = 9999  # fork defaults to test parent process path
 
42
        self.execlp_called = False
 
43
 
 
44
    def kill(self, pid, sig):
 
45
        if pid not in self.running_pids:
 
46
            raise OSError(3, 'No such process')
 
47
        self.pid_sigs[pid].append(sig)
 
48
 
 
49
    def __getattr__(self, name):
 
50
        # I only over-ride portions of the os module
 
51
        try:
 
52
            return object.__getattr__(self, name)
 
53
        except AttributeError:
 
54
            return getattr(os, name)
 
55
 
 
56
 
 
57
def pop_stream(f):
 
58
    """read everything out of file from the top and clear it out
 
59
    """
 
60
    f.flush()
 
61
    f.seek(0)
 
62
    output = f.read()
 
63
    f.seek(0)
 
64
    f.truncate()
 
65
    #print >> sys.stderr, output
 
66
    return output
 
67
 
 
68
 
 
69
class TestManagerModule(unittest.TestCase):
 
70
 
 
71
    def test_servers(self):
 
72
        main_plus_rest = set(manager.MAIN_SERVERS + manager.REST_SERVERS)
 
73
        self.assertEquals(set(manager.ALL_SERVERS), main_plus_rest)
 
74
        # make sure there's no server listed in both
 
75
        self.assertEquals(len(main_plus_rest), len(manager.MAIN_SERVERS) +
 
76
                          len(manager.REST_SERVERS))
 
77
 
 
78
    def test_setup_env(self):
 
79
        class MockResource():
 
80
            def __init__(self, error=None):
 
81
                self.error = error
 
82
                self.called_with_args = []
 
83
 
 
84
            def setrlimit(self, resource, limits):
 
85
                if self.error:
 
86
                    raise self.error
 
87
                self.called_with_args.append((resource, limits))
 
88
 
 
89
            def __getattr__(self, name):
 
90
                # I only over-ride portions of the resource module
 
91
                try:
 
92
                    return object.__getattr__(self, name)
 
93
                except AttributeError:
 
94
                    return getattr(resource, name)
 
95
 
 
96
        _orig_resource = manager.resource
 
97
        _orig_environ = os.environ
 
98
        try:
 
99
            manager.resource = MockResource()
 
100
            manager.os.environ = {}
 
101
            manager.setup_env()
 
102
            expected = [
 
103
                (resource.RLIMIT_NOFILE, (manager.MAX_DESCRIPTORS,
 
104
                                          manager.MAX_DESCRIPTORS)),
 
105
                (resource.RLIMIT_DATA, (manager.MAX_MEMORY,
 
106
                                        manager.MAX_MEMORY)),
 
107
            ]
 
108
            self.assertEquals(manager.resource.called_with_args, expected)
 
109
            self.assertEquals(manager.os.environ['PYTHON_EGG_CACHE'], '/tmp')
 
110
 
 
111
            # test error condition
 
112
            manager.resource = MockResource(error=ValueError())
 
113
            manager.os.environ = {}
 
114
            manager.setup_env()
 
115
            self.assertEquals(manager.resource.called_with_args, [])
 
116
            self.assertEquals(manager.os.environ['PYTHON_EGG_CACHE'], '/tmp')
 
117
 
 
118
            manager.resource = MockResource(error=OSError())
 
119
            manager.os.environ = {}
 
120
            self.assertRaises(OSError, manager.setup_env)
 
121
            self.assertEquals(manager.os.environ.get('PYTHON_EGG_CACHE'), None)
 
122
        finally:
 
123
            manager.resource = _orig_resource
 
124
            os.environ = _orig_environ
 
125
 
 
126
    def test_command_wrapper(self):
 
127
        @manager.command
 
128
        def myfunc(arg1):
 
129
            """test doc
 
130
            """
 
131
            return arg1
 
132
 
 
133
        self.assertEquals(myfunc.__doc__.strip(), 'test doc')
 
134
        self.assertEquals(myfunc(1), 1)
 
135
        self.assertEquals(myfunc(0), 0)
 
136
        self.assertEquals(myfunc(True), 1)
 
137
        self.assertEquals(myfunc(False), 0)
 
138
        self.assert_(hasattr(myfunc, 'publicly_accessible'))
 
139
        self.assert_(myfunc.publicly_accessible)
 
140
 
 
141
    def test_watch_server_pids(self):
 
142
        class MockOs():
 
143
            WNOHANG = os.WNOHANG
 
144
 
 
145
            def __init__(self, pid_map={}):
 
146
                self.pid_map = {}
 
147
                for pid, v in pid_map.items():
 
148
                    self.pid_map[pid] = (x for x in v)
 
149
 
 
150
            def waitpid(self, pid, options):
 
151
                try:
 
152
                    rv = self.pid_map[pid].next()
 
153
                except StopIteration:
 
154
                    raise OSError(errno.ECHILD, os.strerror(errno.ECHILD))
 
155
                except KeyError:
 
156
                    raise OSError(errno.ESRCH, os.strerror(errno.ESRCH))
 
157
                if isinstance(rv, Exception):
 
158
                    raise rv
 
159
                else:
 
160
                    return rv
 
161
 
 
162
        class MockTime():
 
163
            def __init__(self, ticks=None):
 
164
                self.tock = time()
 
165
                if not ticks:
 
166
                    ticks = []
 
167
 
 
168
                self.ticks = (t for t in ticks)
 
169
 
 
170
            def time(self):
 
171
                try:
 
172
                    self.tock += self.ticks.next()
 
173
                except StopIteration:
 
174
                    self.tock += 1
 
175
                return self.tock
 
176
 
 
177
            def sleep(*args):
 
178
                return
 
179
 
 
180
        class MockServer():
 
181
 
 
182
            def __init__(self, pids, zombie=0):
 
183
                self.heartbeat = (pids for _ in range(zombie))
 
184
 
 
185
            def get_running_pids(self):
 
186
                try:
 
187
                    rv = self.heartbeat.next()
 
188
                    return rv
 
189
                except StopIteration:
 
190
                    return {}
 
191
 
 
192
        _orig_os = manager.os
 
193
        _orig_time = manager.time
 
194
        _orig_server = manager.Server
 
195
        try:
 
196
            manager.time = MockTime()
 
197
            manager.os = MockOs()
 
198
            # this server always says it's dead when you ask for running pids
 
199
            server = MockServer([1])
 
200
            # list of pids keyed on servers to watch
 
201
            server_pids = {
 
202
                server: [1],
 
203
            }
 
204
            # basic test, server dies
 
205
            gen = manager.watch_server_pids(server_pids)
 
206
            expected = [(server, 1)]
 
207
            self.assertEquals([x for x in gen], expected)
 
208
            # start long running server and short interval
 
209
            server = MockServer([1], zombie=15)
 
210
            server_pids = {
 
211
                server: [1],
 
212
            }
 
213
            gen = manager.watch_server_pids(server_pids)
 
214
            self.assertEquals([x for x in gen], [])
 
215
            # wait a little longer
 
216
            gen = manager.watch_server_pids(server_pids, interval=15)
 
217
            self.assertEquals([x for x in gen], [(server, 1)])
 
218
            # zombie process
 
219
            server = MockServer([1], zombie=200)
 
220
            server_pids = {
 
221
                server: [1],
 
222
            }
 
223
            # test weird os error
 
224
            manager.os = MockOs({1: [OSError()]})
 
225
            gen = manager.watch_server_pids(server_pids)
 
226
            self.assertRaises(OSError, lambda: [x for x in gen])
 
227
            # test multi-server
 
228
            server1 = MockServer([1, 10], zombie=200)
 
229
            server2 = MockServer([2, 20], zombie=8)
 
230
            server_pids = {
 
231
                server1: [1, 10],
 
232
                server2: [2, 20],
 
233
            }
 
234
            pid_map = {
 
235
                1: [None for _ in range(10)],
 
236
                2: [None for _ in range(8)],
 
237
                20: [None for _ in range(4)],
 
238
            }
 
239
            manager.os = MockOs(pid_map)
 
240
            gen = manager.watch_server_pids(server_pids,
 
241
                                            interval=manager.KILL_WAIT)
 
242
            expected = [
 
243
                (server2, 2),
 
244
                (server2, 20),
 
245
            ]
 
246
            self.assertEquals([x for x in gen], expected)
 
247
 
 
248
        finally:
 
249
            manager.os = _orig_os
 
250
            manager.time = _orig_time
 
251
            manager.Server = _orig_server
 
252
 
 
253
    def test_exc(self):
 
254
        self.assert_(issubclass(manager.UnknownCommandError, Exception))
 
255
 
 
256
 
 
257
class TestServer(unittest.TestCase):
 
258
 
 
259
    def tearDown(self):
 
260
        reload(manager)
 
261
 
 
262
    def join_swift_dir(self, path):
 
263
        return os.path.join(manager.SWIFT_DIR, path)
 
264
 
 
265
    def join_run_dir(self, path):
 
266
        return os.path.join(manager.RUN_DIR, path)
 
267
 
 
268
    def test_create_server(self):
 
269
        server = manager.Server('proxy')
 
270
        self.assertEquals(server.server, 'proxy-server')
 
271
        self.assertEquals(server.type, 'proxy')
 
272
        self.assertEquals(server.cmd, 'swift-proxy-server')
 
273
        server = manager.Server('object-replicator')
 
274
        self.assertEquals(server.server, 'object-replicator')
 
275
        self.assertEquals(server.type, 'object')
 
276
        self.assertEquals(server.cmd, 'swift-object-replicator')
 
277
 
 
278
    def test_server_to_string(self):
 
279
        server = manager.Server('Proxy')
 
280
        self.assertEquals(str(server), 'proxy-server')
 
281
        server = manager.Server('object-replicator')
 
282
        self.assertEquals(str(server), 'object-replicator')
 
283
 
 
284
    def test_server_repr(self):
 
285
        server = manager.Server('proxy')
 
286
        self.assert_(server.__class__.__name__ in repr(server))
 
287
        self.assert_(str(server) in repr(server))
 
288
 
 
289
    def test_server_equality(self):
 
290
        server1 = manager.Server('Proxy')
 
291
        server2 = manager.Server('proxy-server')
 
292
        self.assertEquals(server1, server2)
 
293
        # it is NOT a string
 
294
        self.assertNotEquals(server1, 'proxy-server')
 
295
 
 
296
    def test_get_pid_file_name(self):
 
297
        server = manager.Server('proxy')
 
298
        conf_file = self.join_swift_dir('proxy-server.conf')
 
299
        pid_file = self.join_run_dir('proxy-server.pid')
 
300
        self.assertEquals(pid_file, server.get_pid_file_name(conf_file))
 
301
        server = manager.Server('object-replicator')
 
302
        conf_file = self.join_swift_dir('object-server/1.conf')
 
303
        pid_file = self.join_run_dir('object-replicator/1.pid')
 
304
        self.assertEquals(pid_file, server.get_pid_file_name(conf_file))
 
305
        server = manager.Server('container-auditor')
 
306
        conf_file = self.join_swift_dir(
 
307
            'container-server/1/container-auditor.conf')
 
308
        pid_file = self.join_run_dir(
 
309
            'container-auditor/1/container-auditor.pid')
 
310
        self.assertEquals(pid_file, server.get_pid_file_name(conf_file))
 
311
 
 
312
    def test_get_conf_file_name(self):
 
313
        server = manager.Server('proxy')
 
314
        conf_file = self.join_swift_dir('proxy-server.conf')
 
315
        pid_file = self.join_run_dir('proxy-server.pid')
 
316
        self.assertEquals(conf_file, server.get_conf_file_name(pid_file))
 
317
        server = manager.Server('object-replicator')
 
318
        conf_file = self.join_swift_dir('object-server/1.conf')
 
319
        pid_file = self.join_run_dir('object-replicator/1.pid')
 
320
        self.assertEquals(conf_file, server.get_conf_file_name(pid_file))
 
321
        server = manager.Server('container-auditor')
 
322
        conf_file = self.join_swift_dir(
 
323
            'container-server/1/container-auditor.conf')
 
324
        pid_file = self.join_run_dir(
 
325
            'container-auditor/1/container-auditor.pid')
 
326
        self.assertEquals(conf_file, server.get_conf_file_name(pid_file))
 
327
 
 
328
    def test_conf_files(self):
 
329
        # test get single conf file
 
330
        conf_files = (
 
331
            'proxy-server.conf',
 
332
            'proxy-server.ini',
 
333
            'auth-server.conf',
 
334
        )
 
335
        with temptree(conf_files) as t:
 
336
            manager.SWIFT_DIR = t
 
337
            server = manager.Server('proxy')
 
338
            conf_files = server.conf_files()
 
339
            self.assertEquals(len(conf_files), 1)
 
340
            conf_file = conf_files[0]
 
341
            proxy_conf = self.join_swift_dir('proxy-server.conf')
 
342
            self.assertEquals(conf_file, proxy_conf)
 
343
 
 
344
        # test multi server conf files & grouping of server-type config
 
345
        conf_files = (
 
346
            'object-server1.conf',
 
347
            'object-server/2.conf',
 
348
            'object-server/object3.conf',
 
349
            'object-server/conf/server4.conf',
 
350
            'object-server.txt',
 
351
            'proxy-server.conf',
 
352
        )
 
353
        with temptree(conf_files) as t:
 
354
            manager.SWIFT_DIR = t
 
355
            server = manager.Server('object-replicator')
 
356
            conf_files = server.conf_files()
 
357
            self.assertEquals(len(conf_files), 4)
 
358
            c1 = self.join_swift_dir('object-server1.conf')
 
359
            c2 = self.join_swift_dir('object-server/2.conf')
 
360
            c3 = self.join_swift_dir('object-server/object3.conf')
 
361
            c4 = self.join_swift_dir('object-server/conf/server4.conf')
 
362
            for c in [c1, c2, c3, c4]:
 
363
                self.assert_(c in conf_files)
 
364
            # test configs returned sorted
 
365
            sorted_confs = sorted([c1, c2, c3, c4])
 
366
            self.assertEquals(conf_files, sorted_confs)
 
367
 
 
368
        # test get single numbered conf
 
369
        conf_files = (
 
370
            'account-server/1.conf',
 
371
            'account-server/2.conf',
 
372
            'account-server/3.conf',
 
373
            'account-server/4.conf',
 
374
        )
 
375
        with temptree(conf_files) as t:
 
376
            manager.SWIFT_DIR = t
 
377
            server = manager.Server('account')
 
378
            conf_files = server.conf_files(number=2)
 
379
            self.assertEquals(len(conf_files), 1)
 
380
            conf_file = conf_files[0]
 
381
            self.assertEquals(conf_file,
 
382
                              self.join_swift_dir('account-server/2.conf'))
 
383
            # test missing config number
 
384
            conf_files = server.conf_files(number=5)
 
385
            self.assertFalse(conf_files)
 
386
 
 
387
        # test verbose & quiet
 
388
        conf_files = (
 
389
            'auth-server.ini',
 
390
            'container-server/1.conf',
 
391
        )
 
392
        with temptree(conf_files) as t:
 
393
            manager.SWIFT_DIR = t
 
394
            old_stdout = sys.stdout
 
395
            try:
 
396
                with open(os.path.join(t, 'output'), 'w+') as f:
 
397
                    sys.stdout = f
 
398
                    server = manager.Server('auth')
 
399
                    # check warn "unable to locate"
 
400
                    conf_files = server.conf_files()
 
401
                    self.assertFalse(conf_files)
 
402
                    self.assert_('unable to locate' in pop_stream(f).lower())
 
403
                    # check quiet will silence warning
 
404
                    conf_files = server.conf_files(verbose=True, quiet=True)
 
405
                    self.assertEquals(pop_stream(f), '')
 
406
                    # check found config no warning
 
407
                    server = manager.Server('container-auditor')
 
408
                    conf_files = server.conf_files()
 
409
                    self.assertEquals(pop_stream(f), '')
 
410
                    # check missing config number warn "unable to locate"
 
411
                    conf_files = server.conf_files(number=2)
 
412
                    self.assert_('unable to locate' in pop_stream(f).lower())
 
413
                    # check verbose lists configs
 
414
                    conf_files = server.conf_files(number=2, verbose=True)
 
415
                    c1 = self.join_swift_dir('container-server/1.conf')
 
416
                    self.assert_(c1 in pop_stream(f))
 
417
            finally:
 
418
                sys.stdout = old_stdout
 
419
 
 
420
    def test_iter_pid_files(self):
 
421
        """
 
422
        Server.iter_pid_files is kinda boring, test the
 
423
        Server.pid_files stuff here as well
 
424
        """
 
425
        pid_files = (
 
426
            ('proxy-server.pid', 1),
 
427
            ('auth-server.pid', 'blah'),
 
428
            ('object-replicator/1.pid', 11),
 
429
            ('object-replicator/2.pid', 12),
 
430
        )
 
431
        files, contents = zip(*pid_files)
 
432
        with temptree(files, contents) as t:
 
433
            manager.RUN_DIR = t
 
434
            server = manager.Server('proxy')
 
435
            # test get one file
 
436
            iter = server.iter_pid_files()
 
437
            pid_file, pid = iter.next()
 
438
            self.assertEquals(pid_file, self.join_run_dir('proxy-server.pid'))
 
439
            self.assertEquals(pid, 1)
 
440
            # ... and only one file
 
441
            self.assertRaises(StopIteration, iter.next)
 
442
            # test invalid value in pid file
 
443
            server = manager.Server('auth')
 
444
            self.assertRaises(ValueError, server.iter_pid_files().next)
 
445
            # test object-server doesn't steal pids from object-replicator
 
446
            server = manager.Server('object')
 
447
            self.assertRaises(StopIteration, server.iter_pid_files().next)
 
448
            # test multi-pid iter
 
449
            server = manager.Server('object-replicator')
 
450
            real_map = {
 
451
                11: self.join_run_dir('object-replicator/1.pid'),
 
452
                12: self.join_run_dir('object-replicator/2.pid'),
 
453
            }
 
454
            pid_map = {}
 
455
            for pid_file, pid in server.iter_pid_files():
 
456
                pid_map[pid] = pid_file
 
457
            self.assertEquals(pid_map, real_map)
 
458
 
 
459
        # test get pid_files by number
 
460
        conf_files = (
 
461
            'object-server/1.conf',
 
462
            'object-server/2.conf',
 
463
            'object-server/3.conf',
 
464
            'object-server/4.conf',
 
465
        )
 
466
 
 
467
        pid_files = (
 
468
            ('object-server/1.pid', 1),
 
469
            ('object-server/2.pid', 2),
 
470
            ('object-server/5.pid', 5),
 
471
        )
 
472
 
 
473
        with temptree(conf_files) as swift_dir:
 
474
            manager.SWIFT_DIR = swift_dir
 
475
            files, pids = zip(*pid_files)
 
476
            with temptree(files, pids) as t:
 
477
                manager.RUN_DIR = t
 
478
                server = manager.Server('object')
 
479
                # test get all pid files
 
480
                real_map = {
 
481
                    1: self.join_run_dir('object-server/1.pid'),
 
482
                    2: self.join_run_dir('object-server/2.pid'),
 
483
                    5: self.join_run_dir('object-server/5.pid'),
 
484
                }
 
485
                pid_map = {}
 
486
                for pid_file, pid in server.iter_pid_files():
 
487
                    pid_map[pid] = pid_file
 
488
                self.assertEquals(pid_map, real_map)
 
489
                # test get pid with matching conf
 
490
                pids = list(server.iter_pid_files(number=2))
 
491
                self.assertEquals(len(pids), 1)
 
492
                pid_file, pid = pids[0]
 
493
                self.assertEquals(pid, 2)
 
494
                pid_two = self.join_run_dir('object-server/2.pid')
 
495
                self.assertEquals(pid_file, pid_two)
 
496
                # try to iter on a pid number with a matching conf but no pid
 
497
                pids = list(server.iter_pid_files(number=3))
 
498
                self.assertFalse(pids)
 
499
                # test get pids w/o matching conf
 
500
                pids = list(server.iter_pid_files(number=5))
 
501
                self.assertFalse(pids)
 
502
 
 
503
    def test_signal_pids(self):
 
504
        pid_files = (
 
505
            ('proxy-server.pid', 1),
 
506
            ('auth-server.pid', 2),
 
507
        )
 
508
        files, pids = zip(*pid_files)
 
509
        with temptree(files, pids) as t:
 
510
            manager.RUN_DIR = t
 
511
            # mock os with both pids running
 
512
            manager.os = MockOs([1, 2])
 
513
            server = manager.Server('proxy')
 
514
            pids = server.signal_pids(DUMMY_SIG)
 
515
            self.assertEquals(len(pids), 1)
 
516
            self.assert_(1 in pids)
 
517
            self.assertEquals(manager.os.pid_sigs[1], [DUMMY_SIG])
 
518
            # make sure other process not signaled
 
519
            self.assertFalse(2 in pids)
 
520
            self.assertFalse(2 in manager.os.pid_sigs)
 
521
            # capture stdio
 
522
            old_stdout = sys.stdout
 
523
            try:
 
524
                with open(os.path.join(t, 'output'), 'w+') as f:
 
525
                    sys.stdout = f
 
526
                    #test print details
 
527
                    pids = server.signal_pids(DUMMY_SIG)
 
528
                    output = pop_stream(f)
 
529
                    self.assert_('pid: %s' % 1 in output)
 
530
                    self.assert_('signal: %s' % DUMMY_SIG in output)
 
531
                    # test no details on signal.SIG_DFL
 
532
                    pids = server.signal_pids(signal.SIG_DFL)
 
533
                    self.assertEquals(pop_stream(f), '')
 
534
                    # reset mock os so only the other server is running
 
535
                    manager.os = MockOs([2])
 
536
                    # test pid not running
 
537
                    pids = server.signal_pids(signal.SIG_DFL)
 
538
                    self.assert_(1 not in pids)
 
539
                    self.assert_(1 not in manager.os.pid_sigs)
 
540
                    # test remove stale pid file
 
541
                    self.assertFalse(os.path.exists(
 
542
                        self.join_run_dir('proxy-server.pid')))
 
543
                    # reset mock os with no running pids
 
544
                    manager.os = MockOs([])
 
545
                    server = manager.Server('auth')
 
546
                    # test verbose warns on removing pid file
 
547
                    pids = server.signal_pids(signal.SIG_DFL, verbose=True)
 
548
                    output = pop_stream(f)
 
549
                    self.assert_('stale pid' in output.lower())
 
550
                    auth_pid = self.join_run_dir('auth-server.pid')
 
551
                    self.assert_(auth_pid in output)
 
552
            finally:
 
553
                sys.stdout = old_stdout
 
554
 
 
555
    def test_get_running_pids(self):
 
556
        # test only gets running pids
 
557
        pid_files = (
 
558
            ('test-server1.pid', 1),
 
559
            ('test-server2.pid', 2),
 
560
        )
 
561
        with temptree(*zip(*pid_files)) as t:
 
562
            manager.RUN_DIR = t
 
563
            server = manager.Server('test-server')
 
564
            # mock os, only pid '1' is running
 
565
            manager.os = MockOs([1])
 
566
            running_pids = server.get_running_pids()
 
567
            self.assertEquals(len(running_pids), 1)
 
568
            self.assert_(1 in running_pids)
 
569
            self.assert_(2 not in running_pids)
 
570
            # test persistant running pid files
 
571
            self.assert_(os.path.exists(os.path.join(t, 'test-server1.pid')))
 
572
            # test clean up stale pids
 
573
            pid_two = self.join_swift_dir('test-server2.pid')
 
574
            self.assertFalse(os.path.exists(pid_two))
 
575
            # reset mock os, no pids running
 
576
            manager.os = MockOs([])
 
577
            running_pids = server.get_running_pids()
 
578
            self.assertFalse(running_pids)
 
579
            # and now all pid files are cleaned out
 
580
            pid_one = self.join_run_dir('test-server1.pid')
 
581
            self.assertFalse(os.path.exists(pid_one))
 
582
            all_pids = os.listdir(t)
 
583
            self.assertEquals(len(all_pids), 0)
 
584
 
 
585
        # test only get pids for right server
 
586
        pid_files = (
 
587
            ('thing-doer.pid', 1),
 
588
            ('thing-sayer.pid', 2),
 
589
            ('other-doer.pid', 3),
 
590
            ('other-sayer.pid', 4),
 
591
        )
 
592
        files, pids = zip(*pid_files)
 
593
        with temptree(files, pids) as t:
 
594
            manager.RUN_DIR = t
 
595
            # all pids are running
 
596
            manager.os = MockOs(pids)
 
597
            server = manager.Server('thing-doer')
 
598
            running_pids = server.get_running_pids()
 
599
            # only thing-doer.pid, 1
 
600
            self.assertEquals(len(running_pids), 1)
 
601
            self.assert_(1 in running_pids)
 
602
            # no other pids returned
 
603
            for n in (2, 3, 4):
 
604
                self.assert_(n not in running_pids)
 
605
            # assert stale pids for other servers ignored
 
606
            manager.os = MockOs([1])  # only thing-doer is running
 
607
            running_pids = server.get_running_pids()
 
608
            for f in ('thing-sayer.pid', 'other-doer.pid', 'other-sayer.pid'):
 
609
                # other server pid files persist
 
610
                self.assert_(os.path.exists, os.path.join(t, f))
 
611
            # verify that servers are in fact not running
 
612
            for server_name in ('thing-sayer', 'other-doer', 'other-sayer'):
 
613
                server = manager.Server(server_name)
 
614
                running_pids = server.get_running_pids()
 
615
                self.assertFalse(running_pids)
 
616
            # and now all OTHER pid files are cleaned out
 
617
            all_pids = os.listdir(t)
 
618
            self.assertEquals(len(all_pids), 1)
 
619
            self.assert_(os.path.exists(os.path.join(t, 'thing-doer.pid')))
 
620
 
 
621
    def test_kill_running_pids(self):
 
622
        pid_files = (
 
623
            ('object-server.pid', 1),
 
624
            ('object-replicator1.pid', 11),
 
625
            ('object-replicator2.pid', 12),
 
626
        )
 
627
        files, running_pids = zip(*pid_files)
 
628
        with temptree(files, running_pids) as t:
 
629
            manager.RUN_DIR = t
 
630
            server = manager.Server('object')
 
631
            # test no servers running
 
632
            manager.os = MockOs([])
 
633
            pids = server.kill_running_pids()
 
634
            self.assertFalse(pids, pids)
 
635
        files, running_pids = zip(*pid_files)
 
636
        with temptree(files, running_pids) as t:
 
637
            manager.RUN_DIR = t
 
638
            # start up pid
 
639
            manager.os = MockOs([1])
 
640
            # test kill one pid
 
641
            pids = server.kill_running_pids()
 
642
            self.assertEquals(len(pids), 1)
 
643
            self.assert_(1 in pids)
 
644
            self.assertEquals(manager.os.pid_sigs[1], [signal.SIGTERM])
 
645
            # reset os mock
 
646
            manager.os = MockOs([1])
 
647
            # test shutdown
 
648
            self.assert_('object-server' in
 
649
                         manager.GRACEFUL_SHUTDOWN_SERVERS)
 
650
            pids = server.kill_running_pids(graceful=True)
 
651
            self.assertEquals(len(pids), 1)
 
652
            self.assert_(1 in pids)
 
653
            self.assertEquals(manager.os.pid_sigs[1], [signal.SIGHUP])
 
654
            # start up other servers
 
655
            manager.os = MockOs([11, 12])
 
656
            # test multi server kill & ignore graceful on unsupported server
 
657
            self.assertFalse('object-replicator' in
 
658
                             manager.GRACEFUL_SHUTDOWN_SERVERS)
 
659
            server = manager.Server('object-replicator')
 
660
            pids = server.kill_running_pids(graceful=True)
 
661
            self.assertEquals(len(pids), 2)
 
662
            for pid in (11, 12):
 
663
                self.assert_(pid in pids)
 
664
                self.assertEquals(manager.os.pid_sigs[pid],
 
665
                                  [signal.SIGTERM])
 
666
            # and the other pid is of course not signaled
 
667
            self.assert_(1 not in manager.os.pid_sigs)
 
668
 
 
669
    def test_status(self):
 
670
        conf_files = (
 
671
            'test-server/1.conf',
 
672
            'test-server/2.conf',
 
673
            'test-server/3.conf',
 
674
            'test-server/4.conf',
 
675
        )
 
676
 
 
677
        pid_files = (
 
678
            ('test-server/1.pid', 1),
 
679
            ('test-server/2.pid', 2),
 
680
            ('test-server/3.pid', 3),
 
681
            ('test-server/4.pid', 4),
 
682
        )
 
683
 
 
684
        with temptree(conf_files) as swift_dir:
 
685
            manager.SWIFT_DIR = swift_dir
 
686
            files, pids = zip(*pid_files)
 
687
            with temptree(files, pids) as t:
 
688
                manager.RUN_DIR = t
 
689
                # setup running servers
 
690
                server = manager.Server('test')
 
691
                # capture stdio
 
692
                old_stdout = sys.stdout
 
693
                try:
 
694
                    with open(os.path.join(t, 'output'), 'w+') as f:
 
695
                        sys.stdout = f
 
696
                        # test status for all running
 
697
                        manager.os = MockOs(pids)
 
698
                        self.assertEquals(server.status(), 0)
 
699
                        output = pop_stream(f).strip().splitlines()
 
700
                        self.assertEquals(len(output), 4)
 
701
                        for line in output:
 
702
                            self.assert_('test-server running' in line)
 
703
                        # test get single server by number
 
704
                        self.assertEquals(server.status(number=4), 0)
 
705
                        output = pop_stream(f).strip().splitlines()
 
706
                        self.assertEquals(len(output), 1)
 
707
                        line = output[0]
 
708
                        self.assert_('test-server running' in line)
 
709
                        conf_four = self.join_swift_dir(conf_files[3])
 
710
                        self.assert_('4 - %s' % conf_four in line)
 
711
                        # test some servers not running
 
712
                        manager.os = MockOs([1, 2, 3])
 
713
                        self.assertEquals(server.status(), 0)
 
714
                        output = pop_stream(f).strip().splitlines()
 
715
                        self.assertEquals(len(output), 3)
 
716
                        for line in output:
 
717
                            self.assert_('test-server running' in line)
 
718
                        # test single server not running
 
719
                        manager.os = MockOs([1, 2])
 
720
                        self.assertEquals(server.status(number=3), 1)
 
721
                        output = pop_stream(f).strip().splitlines()
 
722
                        self.assertEquals(len(output), 1)
 
723
                        line = output[0]
 
724
                        self.assert_('not running' in line)
 
725
                        conf_three = self.join_swift_dir(conf_files[2])
 
726
                        self.assert_(conf_three in line)
 
727
                        # test no running pids
 
728
                        manager.os = MockOs([])
 
729
                        self.assertEquals(server.status(), 1)
 
730
                        output = pop_stream(f).lower()
 
731
                        self.assert_('no test-server running' in output)
 
732
                        # test use provided pids
 
733
                        pids = {
 
734
                            1: '1.pid',
 
735
                            2: '2.pid',
 
736
                        }
 
737
                        # shouldn't call get_running_pids
 
738
                        called = []
 
739
 
 
740
                        def mock(*args, **kwargs):
 
741
                            called.append(True)
 
742
                        server.get_running_pids = mock
 
743
                        status = server.status(pids=pids)
 
744
                        self.assertEquals(status, 0)
 
745
                        self.assertFalse(called)
 
746
                        output = pop_stream(f).strip().splitlines()
 
747
                        self.assertEquals(len(output), 2)
 
748
                        for line in output:
 
749
                            self.assert_('test-server running' in line)
 
750
                finally:
 
751
                    sys.stdout = old_stdout
 
752
 
 
753
    def test_spawn(self):
 
754
 
 
755
        # mocks
 
756
        class MockProcess():
 
757
 
 
758
            NOTHING = 'default besides None'
 
759
            STDOUT = 'stdout'
 
760
            PIPE = 'pipe'
 
761
 
 
762
            def __init__(self, pids=None):
 
763
                if pids is None:
 
764
                    pids = []
 
765
                self.pids = (p for p in pids)
 
766
 
 
767
            def Popen(self, args, **kwargs):
 
768
                return MockProc(self.pids.next(), args, **kwargs)
 
769
 
 
770
        class MockProc():
 
771
 
 
772
            def __init__(self, pid, args, stdout=MockProcess.NOTHING,
 
773
                         stderr=MockProcess.NOTHING):
 
774
                self.pid = pid
 
775
                self.args = args
 
776
                self.stdout = stdout
 
777
                if stderr == MockProcess.STDOUT:
 
778
                    self.stderr = self.stdout
 
779
                else:
 
780
                    self.stderr = stderr
 
781
 
 
782
        # setup running servers
 
783
        server = manager.Server('test')
 
784
 
 
785
        with temptree(['test-server.conf']) as swift_dir:
 
786
            manager.SWIFT_DIR = swift_dir
 
787
            with temptree([]) as t:
 
788
                manager.RUN_DIR = t
 
789
                old_subprocess = manager.subprocess
 
790
                try:
 
791
                    # test single server process calls spawn once
 
792
                    manager.subprocess = MockProcess([1])
 
793
                    conf_file = self.join_swift_dir('test-server.conf')
 
794
                    # spawn server no kwargs
 
795
                    server.spawn(conf_file)
 
796
                    # test pid file
 
797
                    pid_file = self.join_run_dir('test-server.pid')
 
798
                    self.assert_(os.path.exists(pid_file))
 
799
                    pid_on_disk = int(open(pid_file).read().strip())
 
800
                    self.assertEquals(pid_on_disk, 1)
 
801
                    # assert procs args
 
802
                    self.assert_(server.procs)
 
803
                    self.assertEquals(len(server.procs), 1)
 
804
                    proc = server.procs[0]
 
805
                    expected_args = [
 
806
                        'swift-test-server',
 
807
                        conf_file,
 
808
                    ]
 
809
                    self.assertEquals(proc.args, expected_args)
 
810
                    # assert stdout is piped
 
811
                    self.assertEquals(proc.stdout, MockProcess.PIPE)
 
812
                    self.assertEquals(proc.stderr, proc.stdout)
 
813
                    # test multi server process calls spawn multiple times
 
814
                    manager.subprocess = MockProcess([11, 12, 13, 14])
 
815
                    conf1 = self.join_swift_dir('test-server/1.conf')
 
816
                    conf2 = self.join_swift_dir('test-server/2.conf')
 
817
                    conf3 = self.join_swift_dir('test-server/3.conf')
 
818
                    conf4 = self.join_swift_dir('test-server/4.conf')
 
819
                    server = manager.Server('test')
 
820
                    # test server run once
 
821
                    server.spawn(conf1, once=True)
 
822
                    self.assert_(server.procs)
 
823
                    self.assertEquals(len(server.procs), 1)
 
824
                    proc = server.procs[0]
 
825
                    expected_args = ['swift-test-server', conf1, 'once']
 
826
                    # assert stdout is piped
 
827
                    self.assertEquals(proc.stdout, MockProcess.PIPE)
 
828
                    self.assertEquals(proc.stderr, proc.stdout)
 
829
                    # test server not daemon
 
830
                    server.spawn(conf2, daemon=False)
 
831
                    self.assert_(server.procs)
 
832
                    self.assertEquals(len(server.procs), 2)
 
833
                    proc = server.procs[1]
 
834
                    expected_args = ['swift-test-server', conf2, 'verbose']
 
835
                    self.assertEquals(proc.args, expected_args)
 
836
                    # assert stdout is not changed
 
837
                    self.assertEquals(proc.stdout, None)
 
838
                    self.assertEquals(proc.stderr, None)
 
839
                    # test server wait
 
840
                    server.spawn(conf3, wait=False)
 
841
                    self.assert_(server.procs)
 
842
                    self.assertEquals(len(server.procs), 3)
 
843
                    proc = server.procs[2]
 
844
                    # assert stdout is /dev/null
 
845
                    self.assert_(isinstance(proc.stdout, file))
 
846
                    self.assertEquals(proc.stdout.name, os.devnull)
 
847
                    self.assertEquals(proc.stdout.mode, 'w+b')
 
848
                    self.assertEquals(proc.stderr, proc.stdout)
 
849
                    # test not daemon over-rides wait
 
850
                    server.spawn(conf4, wait=False, daemon=False, once=True)
 
851
                    self.assert_(server.procs)
 
852
                    self.assertEquals(len(server.procs), 4)
 
853
                    proc = server.procs[3]
 
854
                    expected_args = ['swift-test-server', conf4, 'once',
 
855
                                     'verbose']
 
856
                    self.assertEquals(proc.args, expected_args)
 
857
                    # daemon behavior should trump wait, once shouldn't matter
 
858
                    self.assertEquals(proc.stdout, None)
 
859
                    self.assertEquals(proc.stderr, None)
 
860
                    # assert pids
 
861
                    for i, proc in enumerate(server.procs):
 
862
                        pid_file = self.join_run_dir('test-server/%d.pid' %
 
863
                                                     (i + 1))
 
864
                        pid_on_disk = int(open(pid_file).read().strip())
 
865
                        self.assertEquals(pid_on_disk, proc.pid)
 
866
                finally:
 
867
                    manager.subprocess = old_subprocess
 
868
 
 
869
    def test_wait(self):
 
870
        server = manager.Server('test')
 
871
        self.assertEquals(server.wait(), 0)
 
872
 
 
873
        class MockProcess(Thread):
 
874
            def __init__(self, delay=0.1, fail_to_start=False):
 
875
                Thread.__init__(self)
 
876
                # setup pipe
 
877
                rfd, wfd = os.pipe()
 
878
                # subprocess connection to read stdout
 
879
                self.stdout = os.fdopen(rfd)
 
880
                # real process connection to write stdout
 
881
                self._stdout = os.fdopen(wfd, 'w')
 
882
                self.delay = delay
 
883
                self.finished = False
 
884
                self.returncode = None
 
885
                if fail_to_start:
 
886
                    self._returncode = 1
 
887
                    self.run = self.fail
 
888
                else:
 
889
                    self._returncode = 0
 
890
 
 
891
            def __enter__(self):
 
892
                self.start()
 
893
                return self
 
894
 
 
895
            def __exit__(self, *args):
 
896
                if self.isAlive():
 
897
                    self.join()
 
898
 
 
899
            def close_stdout(self):
 
900
                self._stdout.flush()
 
901
                with open(os.devnull, 'wb') as nullfile:
 
902
                    try:
 
903
                        os.dup2(nullfile.fileno(), self._stdout.fileno())
 
904
                    except OSError:
 
905
                        pass
 
906
 
 
907
            def fail(self):
 
908
                print >>self._stdout, 'mock process started'
 
909
                sleep(self.delay)  # perform setup processing
 
910
                print >>self._stdout, 'mock process failed to start'
 
911
                self.close_stdout()
 
912
 
 
913
            def poll(self):
 
914
                self.returncode = self._returncode
 
915
                return self.returncode or None
 
916
 
 
917
            def run(self):
 
918
                print >>self._stdout, 'mock process started'
 
919
                sleep(self.delay)  # perform setup processing
 
920
                print >>self._stdout, 'setup complete!'
 
921
                self.close_stdout()
 
922
                sleep(self.delay)  # do some more processing
 
923
                print >>self._stdout, 'mock process finished'
 
924
                self.finished = True
 
925
 
 
926
        class MockTime():
 
927
 
 
928
            def time(self):
 
929
                return time()
 
930
 
 
931
            def sleep(self, *args, **kwargs):
 
932
                pass
 
933
 
 
934
        with temptree([]) as t:
 
935
            old_stdout = sys.stdout
 
936
            old_wait = manager.WARNING_WAIT
 
937
            old_time = manager.time
 
938
            try:
 
939
                manager.WARNING_WAIT = 0.01
 
940
                manager.time = MockTime()
 
941
                with open(os.path.join(t, 'output'), 'w+') as f:
 
942
                    # acctually capture the read stdout (for prints)
 
943
                    sys.stdout = f
 
944
                    # test closing pipe in subprocess unblocks read
 
945
                    with MockProcess() as proc:
 
946
                        server.procs = [proc]
 
947
                        status = server.wait()
 
948
                        self.assertEquals(status, 0)
 
949
                        # wait should return before process exits
 
950
                        self.assert_(proc.isAlive())
 
951
                        self.assertFalse(proc.finished)
 
952
                    self.assert_(proc.finished)  # make sure it did finish...
 
953
                    # test output kwarg prints subprocess output
 
954
                    with MockProcess() as proc:
 
955
                        server.procs = [proc]
 
956
                        status = server.wait(output=True)
 
957
                    output = pop_stream(f)
 
958
                    self.assert_('mock process started' in output)
 
959
                    self.assert_('setup complete' in output)
 
960
                    # make sure we don't get prints after stdout was closed
 
961
                    self.assert_('mock process finished' not in output)
 
962
                    # test process which fails to start
 
963
                    with MockProcess(fail_to_start=True) as proc:
 
964
                        server.procs = [proc]
 
965
                        status = server.wait()
 
966
                        self.assertEquals(status, 1)
 
967
                    self.assert_('failed' in pop_stream(f))
 
968
                    # test multiple procs
 
969
                    procs = [MockProcess() for i in range(3)]
 
970
                    for proc in procs:
 
971
                        proc.start()
 
972
                    server.procs = procs
 
973
                    status = server.wait()
 
974
                    self.assertEquals(status, 0)
 
975
                    for proc in procs:
 
976
                        self.assert_(proc.isAlive())
 
977
                    for proc in procs:
 
978
                        proc.join()
 
979
            finally:
 
980
                sys.stdout = old_stdout
 
981
                manager.WARNING_WAIT = old_wait
 
982
                manager.time = old_time
 
983
 
 
984
    def test_interact(self):
 
985
        class MockProcess():
 
986
 
 
987
            def __init__(self, fail=False):
 
988
                self.returncode = None
 
989
                if fail:
 
990
                    self._returncode = 1
 
991
                else:
 
992
                    self._returncode = 0
 
993
 
 
994
            def communicate(self):
 
995
                self.returncode = self._returncode
 
996
                return '', ''
 
997
 
 
998
        server = manager.Server('test')
 
999
        server.procs = [MockProcess()]
 
1000
        self.assertEquals(server.interact(), 0)
 
1001
        server.procs = [MockProcess(fail=True)]
 
1002
        self.assertEquals(server.interact(), 1)
 
1003
        procs = []
 
1004
        for fail in (False, True, True):
 
1005
            procs.append(MockProcess(fail=fail))
 
1006
        server.procs = procs
 
1007
        self.assert_(server.interact() > 0)
 
1008
 
 
1009
    def test_launch(self):
 
1010
        # stubs
 
1011
        conf_files = (
 
1012
            'proxy-server.conf',
 
1013
            'auth-server.conf',
 
1014
            'object-server/1.conf',
 
1015
            'object-server/2.conf',
 
1016
            'object-server/3.conf',
 
1017
            'object-server/4.conf',
 
1018
        )
 
1019
        pid_files = (
 
1020
            ('proxy-server.pid', 1),
 
1021
            ('proxy-server/2.pid', 2),
 
1022
        )
 
1023
 
 
1024
        #mocks
 
1025
        class MockSpawn():
 
1026
 
 
1027
            def __init__(self, pids=None):
 
1028
                self.conf_files = []
 
1029
                self.kwargs = []
 
1030
                if not pids:
 
1031
                    def one_forever():
 
1032
                        while True:
 
1033
                            yield 1
 
1034
                    self.pids = one_forever()
 
1035
                else:
 
1036
                    self.pids = (x for x in pids)
 
1037
 
 
1038
            def __call__(self, conf_file, **kwargs):
 
1039
                self.conf_files.append(conf_file)
 
1040
                self.kwargs.append(kwargs)
 
1041
                rv = self.pids.next()
 
1042
                if isinstance(rv, Exception):
 
1043
                    raise rv
 
1044
                else:
 
1045
                    return rv
 
1046
 
 
1047
        with temptree(conf_files) as swift_dir:
 
1048
            manager.SWIFT_DIR = swift_dir
 
1049
            files, pids = zip(*pid_files)
 
1050
            with temptree(files, pids) as t:
 
1051
                manager.RUN_DIR = t
 
1052
                old_stdout = sys.stdout
 
1053
                try:
 
1054
                    with open(os.path.join(t, 'output'), 'w+') as f:
 
1055
                        sys.stdout = f
 
1056
                        # can't start server w/o an conf
 
1057
                        server = manager.Server('test')
 
1058
                        self.assertFalse(server.launch())
 
1059
                        # start mock os running all pids
 
1060
                        manager.os = MockOs(pids)
 
1061
                        server = manager.Server('proxy')
 
1062
                        # can't start server if it's already running
 
1063
                        self.assertFalse(server.launch())
 
1064
                        output = pop_stream(f)
 
1065
                        self.assert_('running' in output)
 
1066
                        conf_file = self.join_swift_dir('proxy-server.conf')
 
1067
                        self.assert_(conf_file in output)
 
1068
                        pid_file = self.join_run_dir('proxy-server/2.pid')
 
1069
                        self.assert_(pid_file in output)
 
1070
                        self.assert_('already started' in output)
 
1071
                        # no running pids
 
1072
                        manager.os = MockOs([])
 
1073
                        # test ignore once for non-start-once server
 
1074
                        mock_spawn = MockSpawn([1])
 
1075
                        server.spawn = mock_spawn
 
1076
                        conf_file = self.join_swift_dir('proxy-server.conf')
 
1077
                        expected = {
 
1078
                            1: conf_file,
 
1079
                        }
 
1080
                        self.assertEquals(server.launch(once=True), expected)
 
1081
                        self.assertEquals(mock_spawn.conf_files, [conf_file])
 
1082
                        expected = {
 
1083
                            'once': False,
 
1084
                        }
 
1085
                        self.assertEquals(mock_spawn.kwargs, [expected])
 
1086
                        output = pop_stream(f)
 
1087
                        self.assert_('Starting' in output)
 
1088
                        self.assert_('once' not in output)
 
1089
                        # test multi-server kwarg once
 
1090
                        server = manager.Server('object-replicator')
 
1091
                        mock_spawn = MockSpawn([1, 2, 3, 4])
 
1092
                        server.spawn = mock_spawn
 
1093
                        conf1 = self.join_swift_dir('object-server/1.conf')
 
1094
                        conf2 = self.join_swift_dir('object-server/2.conf')
 
1095
                        conf3 = self.join_swift_dir('object-server/3.conf')
 
1096
                        conf4 = self.join_swift_dir('object-server/4.conf')
 
1097
                        expected = {
 
1098
                            1: conf1,
 
1099
                            2: conf2,
 
1100
                            3: conf3,
 
1101
                            4: conf4,
 
1102
                        }
 
1103
                        self.assertEquals(server.launch(once=True), expected)
 
1104
                        self.assertEquals(mock_spawn.conf_files, [conf1, conf2,
 
1105
                                                                 conf3, conf4])
 
1106
                        expected = {
 
1107
                            'once': True,
 
1108
                        }
 
1109
                        self.assertEquals(len(mock_spawn.kwargs), 4)
 
1110
                        for kwargs in mock_spawn.kwargs:
 
1111
                            self.assertEquals(kwargs, expected)
 
1112
                        # test number kwarg
 
1113
                        mock_spawn = MockSpawn([4])
 
1114
                        server.spawn = mock_spawn
 
1115
                        expected = {
 
1116
                            4: conf4,
 
1117
                        }
 
1118
                        self.assertEquals(server.launch(number=4), expected)
 
1119
                        self.assertEquals(mock_spawn.conf_files, [conf4])
 
1120
                        expected = {
 
1121
                            'number': 4
 
1122
                        }
 
1123
                        self.assertEquals(mock_spawn.kwargs, [expected])
 
1124
                        # test cmd does not exist
 
1125
                        server = manager.Server('auth')
 
1126
                        mock_spawn = MockSpawn([OSError(errno.ENOENT, 'blah')])
 
1127
                        server.spawn = mock_spawn
 
1128
                        self.assertEquals(server.launch(), {})
 
1129
                        self.assert_('swift-auth-server does not exist' in
 
1130
                                     pop_stream(f))
 
1131
                finally:
 
1132
                    sys.stdout = old_stdout
 
1133
 
 
1134
    def test_stop(self):
 
1135
        conf_files = (
 
1136
            'account-server/1.conf',
 
1137
            'account-server/2.conf',
 
1138
            'account-server/3.conf',
 
1139
            'account-server/4.conf',
 
1140
        )
 
1141
        pid_files = (
 
1142
            ('account-reaper/1.pid', 1),
 
1143
            ('account-reaper/2.pid', 2),
 
1144
            ('account-reaper/3.pid', 3),
 
1145
            ('account-reaper/4.pid', 4),
 
1146
        )
 
1147
 
 
1148
        with temptree(conf_files) as swift_dir:
 
1149
            manager.SWIFT_DIR = swift_dir
 
1150
            files, pids = zip(*pid_files)
 
1151
            with temptree(files, pids) as t:
 
1152
                manager.RUN_DIR = t
 
1153
                # start all pids in mock os
 
1154
                manager.os = MockOs(pids)
 
1155
                server = manager.Server('account-reaper')
 
1156
                # test kill all running pids
 
1157
                pids = server.stop()
 
1158
                self.assertEquals(len(pids), 4)
 
1159
                for pid in (1, 2, 3, 4):
 
1160
                    self.assert_(pid in pids)
 
1161
                    self.assertEquals(manager.os.pid_sigs[pid],
 
1162
                                      [signal.SIGTERM])
 
1163
                conf1 = self.join_swift_dir('account-reaper/1.conf')
 
1164
                conf2 = self.join_swift_dir('account-reaper/2.conf')
 
1165
                conf3 = self.join_swift_dir('account-reaper/3.conf')
 
1166
                conf4 = self.join_swift_dir('account-reaper/4.conf')
 
1167
                # reset mock os with only 2 running pids
 
1168
                manager.os = MockOs([3, 4])
 
1169
                pids = server.stop()
 
1170
                self.assertEquals(len(pids), 2)
 
1171
                for pid in (3, 4):
 
1172
                    self.assert_(pid in pids)
 
1173
                    self.assertEquals(manager.os.pid_sigs[pid],
 
1174
                                      [signal.SIGTERM])
 
1175
                self.assertFalse(os.path.exists(conf1))
 
1176
                self.assertFalse(os.path.exists(conf2))
 
1177
                # test number kwarg
 
1178
                manager.os = MockOs([3, 4])
 
1179
                pids = server.stop(number=3)
 
1180
                self.assertEquals(len(pids), 1)
 
1181
                expected = {
 
1182
                    3: conf3,
 
1183
                }
 
1184
                self.assert_(pids, expected)
 
1185
                self.assertEquals(manager.os.pid_sigs[3], [signal.SIGTERM])
 
1186
                self.assertFalse(os.path.exists(conf4))
 
1187
                self.assertFalse(os.path.exists(conf3))
 
1188
 
 
1189
 
 
1190
class TestManager(unittest.TestCase):
 
1191
 
 
1192
    def test_create(self):
 
1193
        m = manager.Manager(['test'])
 
1194
        self.assertEquals(len(m.servers), 1)
 
1195
        server = m.servers.pop()
 
1196
        self.assert_(isinstance(server, manager.Server))
 
1197
        self.assertEquals(server.server, 'test-server')
 
1198
        # test multi-server and simple dedupe
 
1199
        servers = ['object-replicator', 'object-auditor', 'object-replicator']
 
1200
        m = manager.Manager(servers)
 
1201
        self.assertEquals(len(m.servers), 2)
 
1202
        for server in m.servers:
 
1203
            self.assert_(server.server in servers)
 
1204
        # test all
 
1205
        m = manager.Manager(['all'])
 
1206
        self.assertEquals(len(m.servers), len(manager.ALL_SERVERS))
 
1207
        for server in m.servers:
 
1208
            self.assert_(server.server in manager.ALL_SERVERS)
 
1209
        # test main
 
1210
        m = manager.Manager(['main'])
 
1211
        self.assertEquals(len(m.servers), len(manager.MAIN_SERVERS))
 
1212
        for server in m.servers:
 
1213
            self.assert_(server.server in manager.MAIN_SERVERS)
 
1214
        # test rest
 
1215
        m = manager.Manager(['rest'])
 
1216
        self.assertEquals(len(m.servers), len(manager.REST_SERVERS))
 
1217
        for server in m.servers:
 
1218
            self.assert_(server.server in manager.REST_SERVERS)
 
1219
        # test main + rest == all
 
1220
        m = manager.Manager(['main', 'rest'])
 
1221
        self.assertEquals(len(m.servers), len(manager.ALL_SERVERS))
 
1222
        for server in m.servers:
 
1223
            self.assert_(server.server in manager.ALL_SERVERS)
 
1224
        # test dedupe
 
1225
        m = manager.Manager(['main', 'rest', 'proxy', 'object',
 
1226
                                           'container', 'account'])
 
1227
        self.assertEquals(len(m.servers), len(manager.ALL_SERVERS))
 
1228
        for server in m.servers:
 
1229
            self.assert_(server.server in manager.ALL_SERVERS)
 
1230
        # test glob
 
1231
        m = manager.Manager(['object-*'])
 
1232
        object_servers = [s for s in manager.ALL_SERVERS if
 
1233
                          s.startswith('object')]
 
1234
        self.assertEquals(len(m.servers), len(object_servers))
 
1235
        for s in m.servers:
 
1236
            self.assert_(str(s) in object_servers)
 
1237
        m = manager.Manager(['*-replicator'])
 
1238
        replicators = [s for s in manager.ALL_SERVERS if
 
1239
                       s.endswith('replicator')]
 
1240
        for s in m.servers:
 
1241
            self.assert_(str(s) in replicators)
 
1242
 
 
1243
    def test_status(self):
 
1244
        class MockServer():
 
1245
 
 
1246
            def __init__(self, server):
 
1247
                self.server = server
 
1248
                self.called_kwargs = []
 
1249
 
 
1250
            def status(self, **kwargs):
 
1251
                self.called_kwargs.append(kwargs)
 
1252
                if 'error' in self.server:
 
1253
                    return 1
 
1254
                else:
 
1255
                    return 0
 
1256
 
 
1257
        old_server_class = manager.Server
 
1258
        try:
 
1259
            manager.Server = MockServer
 
1260
            m = manager.Manager(['test'])
 
1261
            status = m.status()
 
1262
            self.assertEquals(status, 0)
 
1263
            m = manager.Manager(['error'])
 
1264
            status = m.status()
 
1265
            self.assertEquals(status, 1)
 
1266
            # test multi-server
 
1267
            m = manager.Manager(['test', 'error'])
 
1268
            kwargs = {'key': 'value'}
 
1269
            status = m.status(**kwargs)
 
1270
            self.assertEquals(status, 1)
 
1271
            for server in m.servers:
 
1272
                self.assertEquals(server.called_kwargs, [kwargs])
 
1273
        finally:
 
1274
            manager.Server = old_server_class
 
1275
 
 
1276
    def test_start(self):
 
1277
        def mock_setup_env():
 
1278
            getattr(mock_setup_env, 'called', []).append(True)
 
1279
 
 
1280
        class MockServer():
 
1281
            def __init__(self, server):
 
1282
                self.server = server
 
1283
                self.called = defaultdict(list)
 
1284
 
 
1285
            def launch(self, **kwargs):
 
1286
                self.called['launch'].append(kwargs)
 
1287
 
 
1288
            def wait(self, **kwargs):
 
1289
                self.called['wait'].append(kwargs)
 
1290
                return int('error' in self.server)
 
1291
 
 
1292
            def stop(self, **kwargs):
 
1293
                self.called['stop'].append(kwargs)
 
1294
 
 
1295
            def interact(self, **kwargs):
 
1296
                self.called['interact'].append(kwargs)
 
1297
                if 'raise' in self.server:
 
1298
                    raise KeyboardInterrupt
 
1299
                elif 'error' in self.server:
 
1300
                    return 1
 
1301
                else:
 
1302
                    return 0
 
1303
 
 
1304
        old_setup_env = manager.setup_env
 
1305
        old_swift_server = manager.Server
 
1306
        try:
 
1307
            manager.setup_env = mock_setup_env
 
1308
            manager.Server = MockServer
 
1309
 
 
1310
            # test no errors on launch
 
1311
            m = manager.Manager(['proxy'])
 
1312
            status = m.start()
 
1313
            self.assertEquals(status, 0)
 
1314
            for server in m.servers:
 
1315
                self.assertEquals(server.called['launch'], [{}])
 
1316
 
 
1317
            # test error on launch
 
1318
            m = manager.Manager(['proxy', 'error'])
 
1319
            status = m.start()
 
1320
            self.assertEquals(status, 1)
 
1321
            for server in m.servers:
 
1322
                self.assertEquals(server.called['launch'], [{}])
 
1323
                self.assertEquals(server.called['wait'], [{}])
 
1324
 
 
1325
            # test interact
 
1326
            m = manager.Manager(['proxy', 'error'])
 
1327
            kwargs = {'daemon': False}
 
1328
            status = m.start(**kwargs)
 
1329
            self.assertEquals(status, 1)
 
1330
            for server in m.servers:
 
1331
                self.assertEquals(server.called['launch'], [kwargs])
 
1332
                self.assertEquals(server.called['interact'], [kwargs])
 
1333
            m = manager.Manager(['raise'])
 
1334
            kwargs = {'daemon': False}
 
1335
            status = m.start(**kwargs)
 
1336
 
 
1337
        finally:
 
1338
            manager.setup_env = old_setup_env
 
1339
            manager.Server = old_swift_server
 
1340
 
 
1341
    def test_no_wait(self):
 
1342
        class MockServer():
 
1343
            def __init__(self, server):
 
1344
                self.server = server
 
1345
                self.called = defaultdict(list)
 
1346
 
 
1347
            def launch(self, **kwargs):
 
1348
                self.called['launch'].append(kwargs)
 
1349
 
 
1350
            def wait(self, **kwargs):
 
1351
                self.called['wait'].append(kwargs)
 
1352
                return int('error' in self.server)
 
1353
 
 
1354
        orig_swift_server = manager.Server
 
1355
        try:
 
1356
            manager.Server = MockServer
 
1357
            # test success
 
1358
            init = manager.Manager(['proxy'])
 
1359
            status = init.no_wait()
 
1360
            self.assertEquals(status, 0)
 
1361
            for server in init.servers:
 
1362
                self.assertEquals(len(server.called['launch']), 1)
 
1363
                called_kwargs = server.called['launch'][0]
 
1364
                self.assertFalse(called_kwargs['wait'])
 
1365
                self.assertFalse(server.called['wait'])
 
1366
            # test no errocode status even on error
 
1367
            init = manager.Manager(['error'])
 
1368
            status = init.no_wait()
 
1369
            self.assertEquals(status, 0)
 
1370
            for server in init.servers:
 
1371
                self.assertEquals(len(server.called['launch']), 1)
 
1372
                called_kwargs = server.called['launch'][0]
 
1373
                self.assert_('wait' in called_kwargs)
 
1374
                self.assertFalse(called_kwargs['wait'])
 
1375
                self.assertFalse(server.called['wait'])
 
1376
            # test wait with once option
 
1377
            init = manager.Manager(['updater', 'replicator-error'])
 
1378
            status = init.no_wait(once=True)
 
1379
            self.assertEquals(status, 0)
 
1380
            for server in init.servers:
 
1381
                self.assertEquals(len(server.called['launch']), 1)
 
1382
                called_kwargs = server.called['launch'][0]
 
1383
                self.assert_('wait' in called_kwargs)
 
1384
                self.assertFalse(called_kwargs['wait'])
 
1385
                self.assert_('once' in called_kwargs)
 
1386
                self.assert_(called_kwargs['once'])
 
1387
                self.assertFalse(server.called['wait'])
 
1388
        finally:
 
1389
            manager.Server = orig_swift_server
 
1390
 
 
1391
    def test_no_daemon(self):
 
1392
        class MockServer():
 
1393
 
 
1394
            def __init__(self, server):
 
1395
                self.server = server
 
1396
                self.called = defaultdict(list)
 
1397
 
 
1398
            def launch(self, **kwargs):
 
1399
                self.called['launch'].append(kwargs)
 
1400
 
 
1401
            def interact(self, **kwargs):
 
1402
                self.called['interact'].append(kwargs)
 
1403
                return int('error' in self.server)
 
1404
 
 
1405
        orig_swift_server = manager.Server
 
1406
        try:
 
1407
            manager.Server = MockServer
 
1408
            # test success
 
1409
            init = manager.Manager(['proxy'])
 
1410
            stats = init.no_daemon()
 
1411
            self.assertEquals(stats, 0)
 
1412
            # test error
 
1413
            init = manager.Manager(['proxy', 'object-error'])
 
1414
            stats = init.no_daemon()
 
1415
            self.assertEquals(stats, 1)
 
1416
            # test once
 
1417
            init = manager.Manager(['proxy', 'object-error'])
 
1418
            stats = init.no_daemon()
 
1419
            for server in init.servers:
 
1420
                self.assertEquals(len(server.called['launch']), 1)
 
1421
                self.assertEquals(len(server.called['wait']), 0)
 
1422
                self.assertEquals(len(server.called['interact']), 1)
 
1423
        finally:
 
1424
            manager.Server = orig_swift_server
 
1425
 
 
1426
    def test_once(self):
 
1427
        class MockServer():
 
1428
 
 
1429
            def __init__(self, server):
 
1430
                self.server = server
 
1431
                self.called = defaultdict(list)
 
1432
 
 
1433
            def wait(self, **kwargs):
 
1434
                self.called['wait'].append(kwargs)
 
1435
                if 'error' in self.server:
 
1436
                    return 1
 
1437
                else:
 
1438
                    return 0
 
1439
 
 
1440
            def launch(self, **kwargs):
 
1441
                return self.called['launch'].append(kwargs)
 
1442
 
 
1443
        orig_swift_server = manager.Server
 
1444
        try:
 
1445
            manager.Server = MockServer
 
1446
            # test no errors
 
1447
            init = manager.Manager(['account-reaper'])
 
1448
            status = init.once()
 
1449
            self.assertEquals(status, 0)
 
1450
            # test error code on error
 
1451
            init = manager.Manager(['error-reaper'])
 
1452
            status = init.once()
 
1453
            self.assertEquals(status, 1)
 
1454
            for server in init.servers:
 
1455
                self.assertEquals(len(server.called['launch']), 1)
 
1456
                called_kwargs = server.called['launch'][0]
 
1457
                self.assertEquals(called_kwargs, {'once': True})
 
1458
                self.assertEquals(len(server.called['wait']), 1)
 
1459
                self.assertEquals(len(server.called['interact']), 0)
 
1460
        finally:
 
1461
            manager.Server = orig_swift_server
 
1462
 
 
1463
    def test_stop(self):
 
1464
        class MockServerFactory():
 
1465
            class MockServer():
 
1466
                def __init__(self, pids):
 
1467
                    self.pids = pids
 
1468
 
 
1469
                def stop(self, **kwargs):
 
1470
                    return self.pids
 
1471
 
 
1472
            def __init__(self, server_pids):
 
1473
                self.server_pids = server_pids
 
1474
 
 
1475
            def __call__(self, server):
 
1476
                return MockServerFactory.MockServer(self.server_pids[server])
 
1477
 
 
1478
        def mock_watch_server_pids(server_pids, **kwargs):
 
1479
            for server, pids in server_pids.items():
 
1480
                for pid in pids:
 
1481
                    if pid is None:
 
1482
                        continue
 
1483
                    yield server, pid
 
1484
 
 
1485
        _orig_server = manager.Server
 
1486
        _orig_watch_server_pids = manager.watch_server_pids
 
1487
        try:
 
1488
            manager.watch_server_pids = mock_watch_server_pids
 
1489
            # test stop one server
 
1490
            server_pids = {
 
1491
                'test': [1]
 
1492
            }
 
1493
            manager.Server = MockServerFactory(server_pids)
 
1494
            m = manager.Manager(['test'])
 
1495
            status = m.stop()
 
1496
            self.assertEquals(status, 0)
 
1497
            # test not running
 
1498
            server_pids = {
 
1499
                'test': []
 
1500
            }
 
1501
            manager.Server = MockServerFactory(server_pids)
 
1502
            m = manager.Manager(['test'])
 
1503
            status = m.stop()
 
1504
            self.assertEquals(status, 1)
 
1505
            # test won't die
 
1506
            server_pids = {
 
1507
                'test': [None]
 
1508
            }
 
1509
            manager.Server = MockServerFactory(server_pids)
 
1510
            m = manager.Manager(['test'])
 
1511
            status = m.stop()
 
1512
            self.assertEquals(status, 1)
 
1513
 
 
1514
        finally:
 
1515
            manager.Server = _orig_server
 
1516
            manager.watch_server_pids = _orig_watch_server_pids
 
1517
 
 
1518
    # TODO: more tests
 
1519
    def test_shutdown(self):
 
1520
        m = manager.Manager(['test'])
 
1521
        m.stop_was_called = False
 
1522
 
 
1523
        def mock_stop(*args, **kwargs):
 
1524
            m.stop_was_called = True
 
1525
            expected = {'graceful': True}
 
1526
            self.assertEquals(kwargs, expected)
 
1527
            return 0
 
1528
        m.stop = mock_stop
 
1529
        status = m.shutdown()
 
1530
        self.assertEquals(status, 0)
 
1531
        self.assertEquals(m.stop_was_called, True)
 
1532
 
 
1533
    def test_restart(self):
 
1534
        m = manager.Manager(['test'])
 
1535
        m.stop_was_called = False
 
1536
 
 
1537
        def mock_stop(*args, **kwargs):
 
1538
            m.stop_was_called = True
 
1539
            return 0
 
1540
        m.start_was_called = False
 
1541
 
 
1542
        def mock_start(*args, **kwargs):
 
1543
            m.start_was_called = True
 
1544
            return 0
 
1545
        m.stop = mock_stop
 
1546
        m.start = mock_start
 
1547
        status = m.restart()
 
1548
        self.assertEquals(status, 0)
 
1549
        self.assertEquals(m.stop_was_called, True)
 
1550
        self.assertEquals(m.start_was_called, True)
 
1551
 
 
1552
    def test_reload(self):
 
1553
        class MockManager():
 
1554
            called = defaultdict(list)
 
1555
 
 
1556
            def __init__(self, servers):
 
1557
                pass
 
1558
 
 
1559
            @classmethod
 
1560
            def reset_called(cls):
 
1561
                cls.called = defaultdict(list)
 
1562
 
 
1563
            def stop(self, **kwargs):
 
1564
                MockManager.called['stop'].append(kwargs)
 
1565
                return 0
 
1566
 
 
1567
            def start(self, **kwargs):
 
1568
                MockManager.called['start'].append(kwargs)
 
1569
                return 0
 
1570
 
 
1571
        _orig_manager = manager.Manager
 
1572
        try:
 
1573
            m = _orig_manager(['auth'])
 
1574
            for server in m.servers:
 
1575
                self.assert_(server.server in
 
1576
                             manager.GRACEFUL_SHUTDOWN_SERVERS)
 
1577
            manager.Manager = MockManager
 
1578
            status = m.reload()
 
1579
            self.assertEquals(status, 0)
 
1580
            expected = {
 
1581
                'start': [{'graceful': True}],
 
1582
                'stop': [{'graceful': True}],
 
1583
            }
 
1584
            self.assertEquals(MockManager.called, expected)
 
1585
            # test force graceful
 
1586
            MockManager.reset_called()
 
1587
            m = _orig_manager(['*-server'])
 
1588
            self.assert_(len(m.servers), 4)
 
1589
            for server in m.servers:
 
1590
                self.assert_(server.server in
 
1591
                             manager.GRACEFUL_SHUTDOWN_SERVERS)
 
1592
            manager.Manager = MockManager
 
1593
            status = m.reload(graceful=False)
 
1594
            self.assertEquals(status, 0)
 
1595
            expected = {
 
1596
                'start': [{'graceful': True}] * 4,
 
1597
                'stop': [{'graceful': True}] * 4,
 
1598
            }
 
1599
            self.assertEquals(MockManager.called, expected)
 
1600
 
 
1601
        finally:
 
1602
            manager.Manager = _orig_manager
 
1603
 
 
1604
    def test_force_reload(self):
 
1605
        m = manager.Manager(['test'])
 
1606
        m.reload_was_called = False
 
1607
 
 
1608
        def mock_reload(*args, **kwargs):
 
1609
            m.reload_was_called = True
 
1610
            return 0
 
1611
        m.reload = mock_reload
 
1612
        status = m.force_reload()
 
1613
        self.assertEquals(status, 0)
 
1614
        self.assertEquals(m.reload_was_called, True)
 
1615
 
 
1616
    def test_get_command(self):
 
1617
        m = manager.Manager(['test'])
 
1618
        self.assertEquals(m.start, m.get_command('start'))
 
1619
        self.assertEquals(m.force_reload, m.get_command('force-reload'))
 
1620
        self.assertEquals(m.get_command('force-reload'),
 
1621
                          m.get_command('force_reload'))
 
1622
        self.assertRaises(manager.UnknownCommandError, m.get_command,
 
1623
                          'no_command')
 
1624
        self.assertRaises(manager.UnknownCommandError, m.get_command,
 
1625
                          '__init__')
 
1626
 
 
1627
    def test_list_commands(self):
 
1628
        for cmd, help in manager.Manager.list_commands():
 
1629
            method = getattr(manager.Manager, cmd.replace('-', '_'), None)
 
1630
            self.assert_(method, '%s is not a command' % cmd)
 
1631
            self.assert_(getattr(method, 'publicly_accessible', False))
 
1632
            self.assertEquals(method.__doc__.strip(), help)
 
1633
 
 
1634
    def test_run_command(self):
 
1635
        m = manager.Manager(['test'])
 
1636
        m.cmd_was_called = False
 
1637
 
 
1638
        def mock_cmd(*args, **kwargs):
 
1639
            m.cmd_was_called = True
 
1640
            expected = {'kw1': True, 'kw2': False}
 
1641
            self.assertEquals(kwargs, expected)
 
1642
            return 0
 
1643
        mock_cmd.publicly_accessible = True
 
1644
        m.mock_cmd = mock_cmd
 
1645
        kwargs = {'kw1': True, 'kw2': False}
 
1646
        status = m.run_command('mock_cmd', **kwargs)
 
1647
        self.assertEquals(status, 0)
 
1648
        self.assertEquals(m.cmd_was_called, True)
 
1649
 
 
1650
if __name__ == '__main__':
 
1651
    unittest.main()