← Back to branch summary

~christopher-hunt08/maus/maus_integrated_kalman

« back to all changes in this revision

Viewing changes to tests/integration/test_load/test_load.py

  • Committer: Durga Rajaram
  • Date: 2014-07-16 15:13:05 UTC
  • mfrom: (659.1.92 cand)
  • Revision ID: durga@fnal.gov-20140716151305-q27rv1y9p03v9lks
Tags: MAUS-v0.9, MAUS-v0.9.0
MAUS-v0.9.0

Show diffs side-by-side

added added

removed removed

Lines of Context:
tests/integration/test_load/test_load.py
25
25
import subprocess
26
26
import signal
27
27
import sys
 
28
import argparse
28
29
 
29
30
import process_monitor #pylint: disable = F0401
30
31
 
31
 
MAX_RUN_TIME = 60*10*1 # 10 minutes
32
32
PLOT_DIR = os.path.expandvars("${MAUS_ROOT_DIR}/tests/integration/plots/")
33
33
TMP_DIR = os.path.expandvars("${MAUS_TMP_DIR}")
34
34
MRD = os.path.expandvars("${MAUS_ROOT_DIR}")
37
37
 
38
38
def get_data(run_number):
39
39
    """Get data for run number"""
 
40
 
40
41
    proc = subprocess.Popen(["bash",
41
42
                             MTP+"/third_party/bash/71test_data.bash",
42
43
                             str(run_number)])
43
44
    proc.wait()
44
45
 
45
46
 
46
 
def get_sim_proc():
 
47
def get_sim_proc(valgrind):
47
48
    """Start up a simulate_mice process"""
48
49
    sim_path = MRD+"/bin/simulate_mice.py"  
49
50
    config = TEST_DIR+"/test_load_configuration"
50
51
    sim_options = [sim_path, "--configuration_file", config]
51
52
    sim_log = open(TMP_DIR+"/test_load_simulate_mice.log", "w")
 
53
    if valgrind:
 
54
        sim_options = ['valgrind', '--leak-check=yes', 'python']+sim_options
52
55
    proc = subprocess.Popen(sim_options, stdout=sim_log,
53
56
                                         stderr=subprocess.STDOUT)
54
57
    return proc
55
58
 
56
 
def get_rec_proc(run_number):
 
59
def get_rec_proc(run_number, valgrind):
57
60
    """Start up a analyze_data_offline process"""
58
61
    rec_path = MRD+"/bin/analyze_data_offline.py"
59
62
    config = TEST_DIR+"/test_load_configuration"
60
63
    rec_options = [rec_path, "--configuration_file", config,
61
64
                   "--daq_data_file", run_number]
62
 
    rec_log = open(TMP_DIR+"/test_load_analyze_data_offline.log", "w")
63
 
    proc = subprocess.Popen(rec_options, stdout=rec_log,
 
65
    if valgrind:
 
66
        rec_options = ['valgrind', '--leak-check=yes', 'python']+rec_options
 
67
    rec_log = TMP_DIR+"/test_load_analyze_data_offline_"+run_number+".log"
 
68
    proc = subprocess.Popen(rec_options, stdout=open(rec_log, "w"),
64
69
                                         stderr=subprocess.STDOUT)
65
70
    return proc
66
71
 
 
72
def arg_parser():
 
73
    """
 
74
    Parse command line arguments.
 
75
 
 
76
    Use -h switch at the command line for information on command line args used.
 
77
    """
 
78
    parser = argparse.ArgumentParser(description="Runs MAUS reconstruction "+\
 
79
                          "and checks that memory usage is below a threshold.")
 
80
    parser.add_argument('--run-number', dest='run_number',
 
81
                        help='Run number to test against. '+\
 
82
                        'Set to 0 to run against a default scenario, '+\
 
83
                        'including a bit of MC',
 
84
                        default=0, type=int)
 
85
    parser.add_argument('--max-mem-size', dest='max_mem_size',
 
86
                        help='Fail if more memory is used by a process',
 
87
                        default=300000, type=int)
 
88
    parser.add_argument('--max-time', dest='max_time',
 
89
                        help='Kill the process if it takes more time [s]',
 
90
                        default=600, type=int)
 
91
    parser.add_argument('--time-step', dest='time_step',
 
92
                        help='Time step for checking the process memory size',
 
93
                        default=5, type=int)
 
94
    parser.add_argument('--valgrind', dest='valgrind',
 
95
                        help='Set to 1 to run valgrind leak-check=yes',
 
96
                        default=0, type=int)
 
97
    return parser
67
98
 
68
99
class LoadTest(unittest.TestCase): # pylint: disable = R0904
69
100
    """
70
101
    Look for memory usage over a long duration MC
71
102
    """
 
103
    @classmethod
 
104
    def setUpClass(cls): # pylint: disable = C0103
 
105
        """
 
106
        Class setup
 
107
        """
 
108
        if not cls.args:
 
109
            cls.args = arg_parser().parse_args([]) # defaults
 
110
 
72
111
    def check_resource_usage(self,
73
112
                             resource_usage_list,
74
 
                             fraction_of_time,
75
 
                             mem_threshold_factor):
 
113
                             max_mem_size):
76
114
        """
77
115
        Check that resource usage does not grow excessively
 
116
 
 
117
        We want to check that the resource usage for MAUS does not get out of
 
118
        hand
 
119
        - resource_usage_list output from the process_monitor; list looks like
 
120
          [
 
121
            [
 
122
                {resource usage for process A at time step},
 
123
                {resource usage for process B at time step},
 
124
                ...
 
125
            ]
 
126
          ]
 
127
          so the first array indexes the time, the second indexes the process
 
128
        - max_mem_size - fail the test if mem size is ever greater than max
78
129
        """
79
 
        pass
 
130
        for resource_usage in resource_usage_list:
 
131
            for proc_usage in resource_usage:
 
132
                if "sz" in proc_usage and \
 
133
                   float(proc_usage["sz"]) > max_mem_size:
 
134
                    print "Failed", proc_usage
 
135
                    self.assertLess(float(proc_usage["sz"]),
 
136
                                    max_mem_size)
80
137
 
81
138
    def test_simulate_mice(self):
82
139
        """
83
140
        Test that we can simulate mice for a long duration without memory 
84
141
        problems
85
142
        """
86
 
        sim_proc = get_sim_proc()
87
 
        rec_proc = get_rec_proc(self.run_number)
 
143
        do_valgrind = self.args.valgrind == 1
 
144
        if self.args.run_number == 0:
 
145
            self.run_number = self.default_run_number
 
146
            get_data(self.run_number)
 
147
            procs = [get_sim_proc(do_valgrind), get_rec_proc(self.run_number,
 
148
                                                             do_valgrind)]
 
149
            title = PLOT_DIR+"maus"
 
150
        else:
 
151
            self.run_number = str(self.args.run_number).rjust(5, "0")
 
152
            get_data(self.run_number)
 
153
            procs = [get_rec_proc(self.run_number, do_valgrind)]
 
154
            title = PLOT_DIR+"run_"+str(self.run_number)
88
155
        try:
89
156
            resource_usage_list = process_monitor.main(
90
 
                                [sim_proc.pid, rec_proc.pid], 
91
 
                                PLOT_DIR+"maus",
92
 
                                5,
93
 
                                MAX_RUN_TIME)
94
 
            self.assertFalse(sim_proc.returncode) # no crash during running
95
 
            self.assertFalse(rec_proc.returncode) # no crash during running
96
 
            self.check_resource_usage(resource_usage_list, 0.5, 2.)
 
157
                                [proc.pid for proc in procs],
 
158
                                title,
 
159
                                self.args.time_step,
 
160
                                self.args.max_time)
 
161
            for proc in procs:
 
162
                self.assertFalse(proc.returncode) # no crash during running
 
163
            self.check_resource_usage(resource_usage_list,
 
164
                                      self.args.max_mem_size)
97
165
        except Exception:
98
166
            raise
99
167
        finally:
100
 
            for proc in sim_proc, rec_proc:
 
168
            for proc in procs:
101
169
                print "Killing process", proc.pid
102
170
                proc.send_signal(signal.SIGINT)
103
 
            time.sleep(5)
104
 
            for proc in sim_proc, rec_proc:
 
171
            time.sleep(max(5, self.args.max_time/100))
 
172
            for proc in procs:
105
173
                while proc.poll() == None:
106
174
                    proc.send_signal(signal.SIGKILL)
107
175
                    time.sleep(5)
108
 
            print "killed with return code", proc.returncode
 
176
                print "killed with return code", proc.returncode
109
177
 
110
 
    run_number = "04258"
 
178
    args = None
 
179
    run_number = None
 
180
    default_run_number = "04258"
111
181
 
112
182
if __name__ == "__main__":
113
 
    if len(sys.argv) > 1:
114
 
        LoadTest.run_number = sys.argv[1]
115
 
        sys.argv = [sys.argv[0]]
116
 
        get_data(LoadTest.run_number)
 
183
    LoadTest.args = arg_parser().parse_args(sys.argv[1:])
 
184
    sys.argv = [sys.argv[0]]
117
185
    unittest.main()
118
186