← Back to branch summary

~ubuntu-branches/ubuntu/saucy/drizzle/saucy-proposed

« back to all changes in this revision

Viewing changes to tests/kewpie/lib/server_mgmt/mysqld.py

  • Committer: Package Import Robot
  • Author(s): Clint Byrum
  • Date: 2012-06-19 10:46:49 UTC
  • mfrom: (1.1.6)
  • mto: This revision was merged to the branch mainline in revision 29.
  • Revision ID: package-import@ubuntu.com-20120619104649-e2l0ggd4oz3um0f4
Tags: upstream-7.1.36-stable
Import upstream version 7.1.36-stable

Show diffs side-by-side

added added

removed removed

Lines of Context:
tests/kewpie/lib/server_mgmt/mysqld.py
 
1
#! /usr/bin/env python
 
2
# -*- mode: python; indent-tabs-mode: nil; -*-
 
3
# vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
 
4
#
 
5
# Copyright (C) 2010,2011 Patrick Crews
 
6
#
 
7
# This program is free software; you can redistribute it and/or modify
 
8
# it under the terms of the GNU General Public License as published by
 
9
# the Free Software Foundation; either version 2 of the License, or
 
10
# (at your option) any later version.
 
11
#
 
12
# This program is distributed in the hope that it will be useful,
 
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
15
# GNU General Public License for more details.
 
16
#
 
17
# You should have received a copy of the GNU General Public License
 
18
# along with this program; if not, write to the Free Software
 
19
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
20
 
 
21
 
 
22
""" mysqld.py:  code to allow a serverManager
 
23
    to provision and start up a mysqld server object
 
24
    for test execution
 
25
 
 
26
"""
 
27
 
 
28
# imports
 
29
import os
 
30
import sys
 
31
import time
 
32
import subprocess
 
33
import platform
 
34
 
 
35
from ConfigParser import RawConfigParser
 
36
 
 
37
import MySQLdb
 
38
 
 
39
from lib.server_mgmt.server import Server
 
40
from lib.util.mysql_methods import execute_query
 
41
 
 
42
class mysqlServer(Server):
 
43
    """ represents a mysql server, its possessions
 
44
        (datadir, ports, etc), and methods for controlling
 
45
        and querying it
 
46
 
 
47
        TODO: create a base server class that contains
 
48
              standard methods from which we can inherit
 
49
              Currently there are definitely methods / attr
 
50
              which are general
 
51
 
 
52
    """
 
53
 
 
54
    def __init__( self, name, server_manager, code_tree, default_storage_engine
 
55
                , server_options, requester, test_executor, workdir_root):
 
56
        super(mysqlServer, self).__init__( name
 
57
                                           , server_manager
 
58
                                           , code_tree
 
59
                                           , default_storage_engine
 
60
                                           , server_options
 
61
                                           , requester
 
62
                                           , test_executor
 
63
                                           , workdir_root)
 
64
        self.preferred_base_port = 9306
 
65
        
 
66
        # client files
 
67
        self.mysqldump = self.code_tree.mysqldump
 
68
        self.mysqladmin = self.code_tree.mysqladmin
 
69
        self.mysql_client = self.code_tree.mysql_client
 
70
        self.mysqlimport = self.code_tree.mysqlimport
 
71
        self.mysqlslap = self.code_tree.mysqlslap
 
72
        self.mysql_upgrade = self.code_tree.mysql_upgrade
 
73
        self.server_path = self.code_tree.mysql_server
 
74
        self.mysql_client_path = self.code_tree.mysql_client
 
75
      
 
76
        # important stuff
 
77
        self.langdir = self.code_tree.langdir
 
78
        self.charsetdir = self.code_tree.charsetdir
 
79
        self.bootstrap_file = self.code_tree.bootstrap_path
 
80
        self.bootstrap_cmd = None
 
81
 
 
82
        # MySQL 5.0 and 5.1 servers use --language argument vs. lc-message-dir
 
83
        # We check and set things appropriately for such cases
 
84
        self.langstring = "--lc-messages-dir=%s" %(self.langdir)
 
85
        if self.version.startswith('5.1') or self.version.startswith('5.0'):
 
86
            self.langstring = "--language=%s" %(os.path.join(self.langdir
 
87
                                                            ,'english')
 
88
                                               )
 
89
 
 
90
        # Get our ports
 
91
        self.port_block = self.system_manager.port_manager.get_port_block( self.name
 
92
                                                                         , self.preferred_base_port
 
93
                                                                         , 1 )
 
94
        self.master_port = self.port_block[0]
 
95
 
 
96
        # Generate our working directories
 
97
        self.dirset = { 'var_%s' %(self.name): {'std_data_ln':( os.path.join(self.code_tree.testdir,'std_data'))
 
98
                                               ,'log':None
 
99
                                               ,'run':None
 
100
                                               ,'tmp':None
 
101
                                               ,'master-data': { 'test':None
 
102
                                                               , 'mysql':None
 
103
                                                               }
 
104
                                               }  
 
105
                      }
 
106
        self.workdir = self.system_manager.create_dirset( workdir_root
 
107
                                                        , self.dirset)
 
108
        self.vardir = self.workdir
 
109
        self.tmpdir = os.path.join(self.vardir,'tmp')
 
110
        self.rundir = os.path.join(self.vardir,'run')
 
111
        self.logdir = os.path.join(self.vardir,'log')
 
112
        self.std_data = os.path.join(self.vardir,'std_data_ln')
 
113
        self.datadir = os.path.join(self.vardir,'master-data')
 
114
 
 
115
        self.error_log = os.path.join(self.logdir,('error.log'))
 
116
        self.bootstrap_log = os.path.join(self.logdir,('bootstrap.log'))
 
117
        self.pid_file = os.path.join(self.rundir,('%s.pid' %(self.name)))
 
118
        self.socket_file = os.path.join(self.vardir, ('%s.sock' %(self.name)))
 
119
        if len(self.socket_file) > 107:
 
120
            # MySQL has a limitation of 107 characters for socket file path
 
121
            # we copy the mtr workaround of creating one in /tmp
 
122
            self.logging.verbose("Default socket file path: %s" %(self.socket_file))
 
123
            self.socket_file = "/tmp/%s_%s.%s.sock" %(self.system_manager.uuid
 
124
                                                    ,self.owner
 
125
                                                    ,self.name)
 
126
            self.logging.verbose("Changing to alternate: %s" %(self.socket_file))
 
127
        self.timer_file = os.path.join(self.logdir,('timer'))
 
128
        self.general_log_file = os.path.join(self.logdir,'mysqld.log')
 
129
        self.slow_query_log_file = os.path.join(self.logdir,'mysqld-slow.log')
 
130
        self.cnf_file = os.path.join(self.vardir,'my.cnf')
 
131
 
 
132
        self.snapshot_path = os.path.join(self.tmpdir,('snapshot_%s' %(self.master_port)))
 
133
        # We want to use --secure-file-priv = $vardir by default
 
134
        # but there are times / tools when we need to shut this off
 
135
        if self.no_secure_file_priv:
 
136
            self.secure_file_string = ''
 
137
        else:
 
138
            self.secure_file_string = "--secure-file-priv='%s'" %(self.vardir)
 
139
 
 
140
        self.initialize_databases()
 
141
        self.take_db_snapshot()
 
142
 
 
143
        self.logging.debug_class(self)
 
144
 
 
145
 
 
146
    def report(self):
 
147
        """ We print out some general useful info """
 
148
        report_values = [ 'name'
 
149
                        , 'master_port'
 
150
                        , 'socket_file'
 
151
                        , 'vardir'
 
152
                        , 'status'
 
153
                        ]
 
154
        self.logging.info("%s server:" %(self.owner))
 
155
        for key in report_values:
 
156
          value = vars(self)[key] 
 
157
          self.logging.info("%s: %s" %(key.upper(), value))
 
158
 
 
159
    def initialize_databases(self):
 
160
        """ Do the voodoo required to have a working database setup.
 
161
            For MySQL, this is calling the server with the 
 
162
            --bootstrap argument.  We generate the bootstrap
 
163
            file during codeTree intialization as the file is standard for
 
164
            all MySQL servers that are spawned from a single codeTree
 
165
 
 
166
        """
 
167
  
 
168
        # generate the bootstrap startup command
 
169
        if not self.bootstrap_cmd:
 
170
            mysqld_args = [ "--no-defaults"
 
171
                          , "--bootstrap"
 
172
                          , "--basedir=%s" %(self.code_tree.basedir)
 
173
                          , "--datadir=%s" %(self.datadir)
 
174
                          , "--loose-skip-falcon"
 
175
                          , "--loose-skip-ndbcluster"
 
176
                          , "--tmpdir=%s" %(self.tmpdir)
 
177
                          , "--core-file"
 
178
                          , self.langstring
 
179
                          , "--character-sets-dir=%s" %(self.charsetdir)
 
180
                          ]
 
181
            # We add server_path into the mix this way as we
 
182
            # may alter how we store / handle server args later
 
183
            mysqld_args.insert(0,self.server_path)
 
184
            self.bootstrap_cmd = " ".join(mysqld_args)
 
185
        # execute our command
 
186
        bootstrap_log = open(self.bootstrap_log,'w')
 
187
        # open our bootstrap file
 
188
        bootstrap_in = open(self.bootstrap_file,'r')
 
189
        bootstrap_subproc = subprocess.Popen( self.bootstrap_cmd
 
190
                                            , shell=True
 
191
                                            , stdin=bootstrap_in
 
192
                                            , stdout=bootstrap_log
 
193
                                            , stderr=bootstrap_log
 
194
                                            )
 
195
        bootstrap_subproc.wait()
 
196
        bootstrap_in.close()
 
197
        bootstrap_log.close()
 
198
        bootstrap_retcode = bootstrap_subproc.returncode
 
199
        if bootstrap_retcode:
 
200
            self.logging.error("Received retcode: %s executing command: %s"
 
201
                               %(bootstrap_retcode, self.bootstrap_cmd))
 
202
            self.logging.error("Check the bootstrap log: %s" %(self.bootstrap_log))
 
203
            sys.exit(1)
 
204
 
 
205
 
 
206
    def get_start_cmd(self):
 
207
        """ Return the command string that will start up the server 
 
208
            as desired / intended
 
209
 
 
210
        """
 
211
 
 
212
        # We make ourselves bug-compatible with MTR / xtrabackup
 
213
        # test runners
 
214
        if platform.system() != 'Windows' and os.geteuid() == 0:
 
215
            self.server_options.append("--user=root")
 
216
        server_args = [ "--no-defaults" 
 
217
                      , self.process_server_options()
 
218
                      , "--open-files-limit=1024"
 
219
                      , "--local-infile"
 
220
                      , "--character-set-server=latin1"
 
221
                      , "--connect-timeout=60"
 
222
                      , "--log-bin-trust-function-creators=1"
 
223
                      , "--key_buffer_size=1M"
 
224
                      , "--sort_buffer=256K"
 
225
                      , "--max_heap_table_size=1M"
 
226
                      , "--loose-innodb_data_file_path=ibdata1:10M:autoextend"
 
227
                      , "--loose-innodb_buffer_pool_size=8M"
 
228
                      , "--loose-innodb_write_io_threads=2"
 
229
                      , "--loose-innodb_read_io_threads=2"
 
230
                      , "--loose-innodb_log_buffer_size=1M"
 
231
                      , "--loose-innodb_log_file_size=5M"
 
232
                      , "--loose-innodb_additional_mem_pool_size=1M"
 
233
                      , "--loose-innodb_log_files_in_group=2"
 
234
                      , "--slave-net-timeout=120"
 
235
                      , "--log-bin=%s" %(os.path.join(self.logdir,"mysqld-bin"))
 
236
                      , "--binlog-format=ROW"
 
237
                      , "--loose-enable-performance-schema"
 
238
                      , "--loose-performance-schema-max-mutex-instances=10000"
 
239
                      , "--loose-performance-schema-max-rwlock-instances=10000"
 
240
                      , "--loose-performance-schema-max-table-instances=500"
 
241
                      , "--loose-performance-schema-max-table-handles=1000"
 
242
                      , "--loose-enable-performance-schema"
 
243
                      , "--basedir=%s" %(self.code_tree.basedir)
 
244
                      , "--datadir=%s" %(self.datadir)
 
245
                      , "--tmpdir=%s"  %(self.tmpdir)
 
246
                      , "--character-sets-dir=%s" %(self.charsetdir)
 
247
                      , self.langstring
 
248
                      , "--ssl-ca=%s" %(os.path.join(self.std_data,'cacert.pem'))
 
249
                      , "--ssl-cert=%s" %(os.path.join(self.std_data,'server-cert.pem'))
 
250
                      , "--ssl-key=%s" %(os.path.join(self.std_data,'server-key.pem'))
 
251
                      , "--port=%d" %(self.master_port)
 
252
                      , "--socket=%s" %(self.socket_file)
 
253
                      , "--pid-file=%s" %(self.pid_file)
 
254
                      , "--default-storage-engine=%s" %(self.default_storage_engine)
 
255
                      # server-id maybe needs fixing, but we want to avoid
 
256
                      # the server-id=0 and no replication thing...
 
257
                      , "--server-id=%d" %(self.get_numeric_server_id()+1)
 
258
                      , self.secure_file_string
 
259
                      ]
 
260
 
 
261
        if not self.version.startswith('5.0'):
 
262
            server_args += [ "--binlog-direct-non-transactional-updates"
 
263
                           , "--general_log=1"
 
264
                           , "--general_log_file=%s" %(self.general_log_file)
 
265
                           , "--slow_query_log=1"
 
266
                           , "--slow_query_log_file=%s" %(self.slow_query_log_file)
 
267
                           ]
 
268
        self.gen_cnf_file(server_args)
 
269
 
 
270
        if self.gdb:
 
271
            server_args.append('--gdb')
 
272
            return self.system_manager.handle_gdb_reqs(self, server_args)
 
273
        else:
 
274
            return "%s %s %s & " % ( self.cmd_prefix
 
275
                                   , self.server_path
 
276
                                   , " ".join(server_args)
 
277
                                   )
 
278
 
 
279
 
 
280
    def get_stop_cmd(self):
 
281
        """ Return the command that will shut us down """
 
282
        
 
283
        return "%s --no-defaults --user=root --port=%d --host=127.0.0.1 --protocol=tcp shutdown " %(self.mysqladmin, self.master_port)
 
284
           
 
285
 
 
286
    def get_ping_cmd(self):
 
287
        """Return the command string that will 
 
288
           ping / check if the server is alive 
 
289
 
 
290
        """
 
291
 
 
292
        return '%s --no-defaults --user=root --port=%d --host=127.0.0.1 --protocol=tcp ping' % (self.mysqladmin, self.master_port)
 
293
 
 
294
    def is_started(self):
 
295
        """ Determine if the server is up and running - 
 
296
            this may vary from server type to server type
 
297
 
 
298
        """
 
299
 
 
300
        # We experiment with waiting for a pid file to be created vs. pinging
 
301
        # This is what test-run.pl does and it helps us pass logging_stats tests
 
302
        # while not self.ping_server(server, quiet=True) and timer != timeout:
 
303
 
 
304
        return self.system_manager.find_path( [self.pid_file]
 
305
                                            , required=0)
 
306
 
 
307
    def gen_cnf_file(self, server_args):
 
308
        """ We generate a .cnf file for the server based
 
309
            on the arguments.  We currently don't use
 
310
            this for much, but xtrabackup uses it, so 
 
311
            we must produce one.  This could also be
 
312
            helpful for testing / etc
 
313
 
 
314
        """
 
315
 
 
316
        config_file = open(self.cnf_file,'w')
 
317
        config_file.write('[mysqld]') 
 
318
        for server_arg in server_args:
 
319
            # We currently have a list of string values
 
320
            # We need to remove any '--' stuff 
 
321
            server_arg = server_arg.replace('--','')+'\n'
 
322
            config_file.write(server_arg)
 
323
        config_file.close() 
 
324
 
 
325
    def set_master(self, master_server, get_cur_log_pos = True):
 
326
        """ We do what is needed to set the master_server
 
327
            as the replication master
 
328
 
 
329
        """
 
330
        msg = None
 
331
        if self.status:  # we are running and can do things!
 
332
            # Get master binlog info
 
333
            retcode, master_binlog_file, master_binlog_pos = master_server.get_binlog_info()
 
334
            if not retcode:
 
335
                if not get_cur_log_pos:
 
336
                    master_binlog_pos = 0
 
337
            else:
 
338
                return retcode, master_binlog_file #contains error msg on failure
 
339
            
 
340
            # update our slave's master info
 
341
            query = ("CHANGE MASTER TO "
 
342
                     "MASTER_HOST='127.0.0.1',"
 
343
                     "MASTER_USER='root',"
 
344
                     "MASTER_PASSWORD='',"
 
345
                     "MASTER_PORT=%d,"
 
346
                     "MASTER_LOG_FILE='%s',"
 
347
                     "MASTER_LOG_POS=%d" % ( master_server.master_port
 
348
                                           , master_binlog_file
 
349
                                           , int(master_binlog_pos)))
 
350
            retcode, result_set = execute_query(query, self)
 
351
            if retcode:
 
352
                msg = ("Could not set slave: %s.%s\n"
 
353
                       "With query: %s\n."
 
354
                       "Returned result: %s" %( self.owner
 
355
                                              , self.name
 
356
                                              , query
 
357
                                              , result_set)
 
358
                      )
 
359
                return 1, msg
 
360
            # start the slave
 
361
            self.slave_start()
 
362
            self.need_to_set_master = False
 
363
        else:
 
364
            self.need_to_set_master = True 
 
365
            self.master = master_server
 
366
        return 0,msg
 
367
 
 
368
    def get_slave_status(self):
 
369
        query = "SHOW SLAVE STATUS"
 
370
        retcode, result_set = execute_query(query, self)
 
371
        result_set = result_set[0]
 
372
        if not retcode:
 
373
            slave_data = { 'slave_io_state':result_set[0]
 
374
                         , 'master_host':result_set[1]
 
375
                         , 'master_user':result_set[2]
 
376
                         , 'master_port':result_set[3]
 
377
                         , 'connect_retry':result_set[4]
 
378
                         , 'master_log_file':result_set[5]
 
379
                         , 'read_master_log_pos':result_set[6]
 
380
                         , 'relay_log_file':result_set[7]
 
381
                         , 'relay_log_pos':result_set[8]
 
382
                         , 'relay_master_log_file':result_set[9]
 
383
                         , 'slave_io_running':result_set[10]
 
384
                         , 'slave_sql_running':result_set[11]
 
385
                         , 'replicate_do_db':result_set[12]
 
386
                         , 'replicate_ignore_db':result_set[13]
 
387
                         , 'replicate_do_table':result_set[14]
 
388
                         , 'replicate_ignore_table':result_set[15]
 
389
                         , 'replicate_wild_do_table':result_set[16]
 
390
                         , 'replicate_wild_ignore_table':result_set[17]
 
391
                         , 'last_errno':result_set[18]
 
392
                         , 'last_error':result_set[19]
 
393
                         , 'skip_counter':result_set[20]
 
394
                         , 'exec_master_log_pos':result_set[21]
 
395
                         , 'relay_log_space':result_set[22]
 
396
                         , 'until_condition':result_set[23]
 
397
                         , 'until_log_file':result_set[24]
 
398
                         , 'until_log_pos':result_set[25]
 
399
                         , 'master_ssl_allowed':result_set[26]
 
400
                         , 'master_ssl_ca_file':result_set[27]
 
401
                         , 'master_ssl_ca_path':result_set[28]
 
402
                         , 'master_ssl_cert':result_set[29]
 
403
                         , 'master_ssl_cipher':result_set[30]
 
404
                         , 'master_ssl_key':result_set[31]
 
405
                         , 'seconds_behind_master':result_set[32]
 
406
                         , 'master_ssl_verify_server_cert':result_set[33]
 
407
                         , 'last_io_errno':result_set[34]
 
408
                         , 'last_io_error':result_set[35]
 
409
                         , 'last_sql_errno':result_set[36]
 
410
                         , 'last_sql_error':result_set[37]
 
411
                         #, 'replicate_ignore_server_ids':result_set[38]
 
412
                         }
 
413
            return slave_data
 
414
        else:
 
415
            return None
 
416
     
 
417
         
 
418
    def get_binlog_info(self):
 
419
        """ We try to get binlog information for the server """
 
420
        query = "SHOW MASTER STATUS"
 
421
        retcode, result_set = execute_query(query, self)
 
422
        if not retcode:
 
423
            binlog_file = result_set[0][0]
 
424
            binlog_pos = result_set[0][1]
 
425
        else:
 
426
            binlog_file = result_set
 
427
            binlog_pos = result_set
 
428
        return retcode, binlog_file, binlog_pos
 
429
 
 
430
    def slave_stop(self):
 
431
        """ We issue STOP SLAVE and wait for IO and SQL threads to stop """
 
432
        
 
433
        query = "STOP SLAVE"
 
434
        retcode, result = execute_query(query, self)
 
435
        slave_status = self.get_slave_status()
 
436
        while slave_status['slave_io_running'] == 'Yes' or slave_status['slave_sql_running'] == 'Yes': 
 
437
            slave_status = self.get_slave_status()
 
438
 
 
439
    def slave_start(self):
 
440
        """ We issue START SLAVE and wait for IO and SQL threads to start """
 
441
        query = "START SLAVE"
 
442
        decrement = .5
 
443
        retcode, result = execute_query(query, self)
 
444
        slave_status = self.get_slave_status()
 
445
        while slave_status['slave_io_running'] == 'No' or slave_status['slave_sql_running'] == 'No':
 
446
            time.sleep(decrement)
 
447
            slave_status = self.get_slave_status()
 
448
 
 
449
 
 
450
    def wait_sync_with_slaves(self, slave_list, timeout=60):
 
451
        """ We scan through our slave list and make sure we are synched up
 
452
            before moving on with the test
 
453
 
 
454
        """
 
455
 
 
456
        retcode, master_binlog_file, master_binlog_pos = self.get_binlog_info()
 
457
        while slave_list:
 
458
            for idx, slave_server in enumerate(slave_list):
 
459
                slave_status_data = slave_server.get_slave_status()
 
460
                read_master_log_pos = slave_status_data['read_master_log_pos']
 
461
                exec_master_log_pos = slave_status_data['exec_master_log_pos']
 
462
                seconds_behind_master = slave_status_data['seconds_behind_master']
 
463
                slave_io_state = slave_status_data['slave_io_state']
 
464
                # This test subject to change as I learn more ; )
 
465
                #if master_binlog_pos == exec_master_log_pos:
 
466
                if exec_master_log_pos == read_master_log_pos and slave_io_state == 'Waiting for master to send event':
 
467
                    #print 'Server: %s.%s' %(self.owner, self.name)
 
468
                    #print 'Master_binlog_file: %s' %master_binlog_file
 
469
                    #print 'Master_binlog_pos: %s' %master_binlog_pos
 
470
                    #print 'Exec_master_log_pos: %s' %exec_master_log_pos
 
471
                    #print 'Read_master_log_pos: %s' %read_master_log_pos
 
472
                    #if read_master_log_pos > master_binlog_pos:
 
473
                    #for key, item in slave_status_data.items():
 
474
                    #        print "%s:%s" %(key, item)
 
475
                    #print '#'*80
 
476
                    # It is normally a no-no to remove items from a list
 
477
                    # mid-iteration, but this is a chance to break that rule
 
478
                    # >: )
 
479
                    slave_list.pop(idx)
 
480
 
 
481
    def slave_ready(self, master_server):
 
482
       return True 
 
483
                            
 
484
 
 
485
    def get_innodb_version(self):
 
486
        """ SHOW VARIABLES LIKE innodb_version 
 
487
            mostly used as a check to ensure if a 
 
488
            test should/shouldn't be executed
 
489
 
 
490
        """ 
 
491
 
 
492
        query = "SHOW VARIABLES LIKE 'innodb_version'"
 
493
        retcode, result = execute_query(query, self)
 
494
        return retcode, result 
 
495
 
 
496
    def get_xtradb_version(self):
 
497
        """ Return the xtradb version or None """
 
498
 
 
499
        retcode, result = self.get_innodb_version()
 
500
        # result format = (('innodb_version', '1.1.6-20.1'),)
 
501
        if result:
 
502
            innodb_version = result[0][1]
 
503
            split_data = innodb_version.split('-')
 
504
            if len(split_data) > 1:
 
505
                return split_data[-1]
 
506
        return None