← Back to branch summary

~longbow/percona-pam-for-mysql/pamtest_py

« back to all changes in this revision

Viewing changes to percona_tests/xtrabackup_main/xb_export_test.py

  • Committer: Patrick Crews
  • Date: 2012-01-05 02:48:07 UTC
  • Revision ID: gleebix@gmail.com-20120105024807-iob2emmrylinos8d
Ported the last of the xtrabackup tests...huzzah \o/

Show diffs side-by-side

added added

removed removed

Lines of Context:
percona_tests/xtrabackup_main/xb_export_test.py
 
1
#! /usr/bin/env python
 
2
# -*- mode: python; indent-tabs-mode: nil; -*-
 
3
# vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
 
4
#
 
5
# Copyright (C) 2011 Patrick Crews
 
6
#
 
7
#
 
8
# This program is free software; you can redistribute it and/or modify
 
9
# it under the terms of the GNU General Public License as published by
 
10
# the Free Software Foundation; either version 2 of the License, or
 
11
# (at your option) any later version.
 
12
#
 
13
# This program is distributed in the hope that it will be useful,
 
14
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
15
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
16
# GNU General Public License for more details.
 
17
#
 
18
# You should have received a copy of the GNU General Public License
 
19
# along with this program; if not, write to the Free Software
 
20
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
21
 
 
22
import os
 
23
import shutil
 
24
 
 
25
from lib.util.mysqlBaseTestCase import mysqlBaseTestCase
 
26
 
 
27
server_requirements = [['--innodb-file-per-table --innodb_file_format=Barracuda']]
 
28
servers = []
 
29
server_manager = None
 
30
test_executor = None
 
31
# we explicitly use the --no-timestamp option
 
32
# here.  We will be using a generic / vanilla backup dir
 
33
backup_path = None
 
34
 
 
35
class basicTest(mysqlBaseTestCase):
 
36
 
 
37
    def setUp(self):
 
38
        master_server = servers[0] # assumption that this is 'master'
 
39
        backup_path = os.path.join(master_server.vardir, 'backup')
 
40
        # remove backup paths
 
41
        for del_path in [backup_path]:
 
42
            if os.path.exists(del_path):
 
43
                shutil.rmtree(del_path)
 
44
 
 
45
    def create_test_table(self, table_name, server):
 
46
        queries = ["DROP TABLE IF EXISTS %s" %(table_name)
 
47
                  ,("CREATE TABLE %s "
 
48
                    "(`a` int(11) DEFAULT NULL, "
 
49
                    "`number` int(11) DEFAULT NULL) "
 
50
                    " ENGINE=InnoDB DEFAULT CHARSET=latin1"
 
51
                    %(table_name)
 
52
                   )
 
53
                  ]
 
54
        retcode, result = self.execute_queries(queries, server)
 
55
        self.assertEqual(retcode, 0, msg = result) 
 
56
 
 
57
    def load_table(self, table_name, row_count, server):
 
58
        queries = []
 
59
        for i in range(row_count):
 
60
            queries.append("INSERT INTO %s VALUES (%d, %d)" %(table_name,i, row_count))
 
61
        retcode, result = self.execute_queries(queries, server)
 
62
        self.assertEqual(retcode, 0, msg=result)
 
63
 
 
64
 
 
65
    def test_xb_export(self):
 
66
        self.servers = servers
 
67
        logging = test_executor.logging
 
68
        server_comment = servers[0].code_tree.server_compile_comment
 
69
        if server_comment and "with XtraDB" not in server_comment:
 
70
            logging.warning("Test requires XtraDB, skipping test...")
 
71
            return
 
72
        else:
 
73
            innobackupex = test_executor.system_manager.innobackupex_path
 
74
            xtrabackup = test_executor.system_manager.xtrabackup_path
 
75
            master_server = servers[0] # assumption that this is 'master'
 
76
            backup_path = os.path.join(master_server.vardir, 'backup')
 
77
            output_path = os.path.join(master_server.vardir, 'innobackupex.out')
 
78
            exec_path = os.path.dirname(innobackupex)
 
79
            table_name = "`test`"
 
80
            schema_name = "test"
 
81
 
 
82
 
 
83
            # This is a bit hacky.  We have a version-dependent server
 
84
            # option and no clean mechanism for doing this at test-collection
 
85
            # time
 
86
            import_option = "--innodb_expand_import=1"
 
87
            if master_server.version.startswith("5.5"):
 
88
                import_option = "--innodb_import_table_from_xtrabackup=1"
 
89
            master_server.server_options.append(import_option)
 
90
            master_server.stop()
 
91
            master_server.start()
 
92
 
 
93
            # populate our server with a test bed
 
94
            self.create_test_table(table_name, master_server)
 
95
            row_count = 100
 
96
            self.load_table(table_name, row_count, master_server)
 
97
        
 
98
            # take a backup
 
99
            cmd = [ xtrabackup
 
100
                  , "--defaults-file=%s" %master_server.cnf_file
 
101
                  , "--datadir=%s" %master_server.datadir
 
102
                  , "--backup"
 
103
                  , "--target-dir=%s" %backup_path
 
104
                  ]
 
105
            cmd = " ".join(cmd)
 
106
            retcode, output = self.execute_cmd(cmd, output_path, exec_path, True)
 
107
            self.assertTrue(retcode==0,output)
 
108
 
 
109
            # load more data
 
110
            row_count = 100
 
111
            self.load_table(table_name, row_count, master_server)
 
112
 
 
113
            # Get a checksum for our table
 
114
            query = "CHECKSUM TABLE %s" %table_name
 
115
            retcode, checksum1 = self.execute_query(query, master_server)
 
116
            self.assertEqual(retcode, 0, msg=checksum1)
 
117
            logging.test_debug("Checksum1: %s" %checksum1)
 
118
 
 
119
            # Reset the server to treat it as the 'importing' server
 
120
            # We clean the datadir + create a table w/ similar
 
121
            # structure
 
122
            # TODO: see how things fail if we skip / mess up this step
 
123
            master_server.stop()
 
124
            master_server.restore_snapshot()
 
125
            master_server.start()
 
126
        
 
127
            # recreate the table:
 
128
            self.create_test_table(table_name, master_server)
 
129
            logging.test_debug("Server reinitialized")
 
130
 
 
131
            # discard the tablespace
 
132
            query = "ALTER TABLE %s DISCARD TABLESPACE" %(table_name)
 
133
            retcode, result = self.execute_query(query, master_server)
 
134
            self.assertEqual(retcode,0,msg=result)
 
135
 
 
136
            # prepare our main backup
 
137
            # Test the with innodb_file_per_table=0 --export bails out with an error
 
138
            # Bug #758888
 
139
            cmd = [ xtrabackup
 
140
                  , "--prepare"
 
141
                  , "--datadir=%s" %master_server.datadir
 
142
                  , "--use-memory=500M"
 
143
                  , "--target-dir=%s" %backup_path
 
144
                  , "--export"
 
145
                  , "--innodb-file-per-table=0"
 
146
                  ]
 
147
            cmd = " ".join(cmd)
 
148
            retcode, output = self.execute_cmd(cmd, output_path, exec_path, True)
 
149
            self.assertEqual(retcode,1,output)
 
150
 
 
151
            # prepare our main backup
 
152
            cmd = [ xtrabackup
 
153
                  , "--prepare"
 
154
                  , "--datadir=%s" %master_server.datadir
 
155
                  , "--use-memory=500M"
 
156
                  , "--target-dir=%s" %backup_path
 
157
                  , "--export"
 
158
                  , "--innodb-file-per-table=1"
 
159
                  ]
 
160
            cmd = " ".join(cmd)
 
161
            retcode, output = self.execute_cmd(cmd, output_path, exec_path, True)
 
162
            self.assertEqual(retcode,0,output)
 
163
 
 
164
            # copy our data files back
 
165
            db_path = os.path.join(backup_path,schema_name)
 
166
            for bkp_file in os.listdir(db_path):
 
167
                if bkp_file.startswith('test'):
 
168
                    shutil.copy(os.path.join(db_path,bkp_file)
 
169
                               ,os.path.join(master_server.datadir,schema_name)
 
170
                               )
 
171
 
 
172
            # import the tablespace
 
173
            query = "ALTER TABLE %s IMPORT TABLESPACE" %(table_name)
 
174
            retcode, result = self.execute_query(query, master_server)
 
175
            self.assertEqual(retcode,0,msg=result)
 
176
            logging.test_debug("Tablespace imported...")
 
177
 
 
178
            # Tablespace import is asynchronous, so shutdown the server to have
 
179
            # consistent backup results. Otherwise we risk ending up with no test.ibd
 
180
            # in the backup in case importing has not finished before taking backup
 
181
            master_server.stop()
 
182
            master_server.start()
 
183
            self.assertTrue(master_server.status==1, 'Server failed restart from restored datadir...')
 
184
 
 
185
            query="SELECT COUNT(*) FROM %s" %(table_name)
 
186
            retcode, result = self.execute_query(query, master_server)
 
187
            self.assertEqual(retcode,0,result)
 
188
            expected_result = ((100L,),)
 
189
            self.assertEqual(expected_result, result, msg = "%s || %s" %(expected_result, result))
 
190
 
 
191
            query = "SHOW CREATE TABLE %s" %(table_name)
 
192
            retcode, result = self.execute_query(query, master_server)
 
193
            self.assertEqual(retcode,0,result)
 
194
            expected_result = (('test', 'CREATE TABLE `test` (\n  `a` int(11) DEFAULT NULL,\n  `number` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=latin1'),)
 
195
            self.assertEqual(expected_result, result, msg = "%s || %s" %(expected_result, result))
 
196
 
 
197
            # Get a checksum for our table
 
198
            query = "CHECKSUM TABLE %s" %table_name
 
199
            retcode, checksum2 = self.execute_query(query, master_server)
 
200
            self.assertEqual(retcode, 0, msg=checksum2)
 
201
            logging.test_debug("Checksum2: %s" %checksum2)
 
202
            if checksum1 != checksum2:
 
203
                logging.warning("Initial and exported/restored checksums do not match!")
 
204
                logging.warning("Checksum1:  %s" %checksum1)
 
205
                logging.warning("Checksum2:  %s" %checksum2)
 
206
            #self.assertEqual(checksum1,checksum2,msg="%s || %s" %(checksum1,checksum2))
 
207
 
 
208
            # create a dir to hold the new backup
 
209
            backup_path = os.path.join(backup_path,'new_backup')
 
210
            # take a backup of the imported table
 
211
            cmd = [ xtrabackup
 
212
                  , "--defaults-file=%s" %master_server.cnf_file
 
213
                  , "--datadir=%s" %master_server.datadir
 
214
                  , "--backup"
 
215
                  , "--target-dir=%s" %backup_path
 
216
                  ]
 
217
            cmd = " ".join(cmd)
 
218
            retcode, output = self.execute_cmd(cmd, output_path, exec_path, True)
 
219
            self.assertTrue(retcode==0,output)
 
220
 
 
221
            # Clear our table so we know the backup restored
 
222
            for del_table in [table_name]:
 
223
                query = "DELETE FROM %s" %del_table
 
224
                retcode, result = self.execute_query(query,master_server)
 
225
                self.assertEqual(retcode, 0, result) 
 
226
        
 
227
            # shutdown our server
 
228
            master_server.stop()
 
229
 
 
230
            # prepare our main backup
 
231
            cmd = [ xtrabackup
 
232
                  , "--prepare"
 
233
                  , "--apply-log-only"
 
234
                  , "--datadir=%s" %master_server.datadir
 
235
                  , "--use-memory=500M"
 
236
                  , "--target-dir=%s" %backup_path
 
237
                  ]
 
238
            cmd = " ".join(cmd)
 
239
            retcode, output = self.execute_cmd(cmd, output_path, exec_path, True)
 
240
            self.assertTrue(retcode==0,output)
 
241
 
 
242
            # do final prepare on main backup
 
243
            cmd = [ xtrabackup
 
244
                  , "--prepare"
 
245
                  , "--datadir=%s" %master_server.datadir
 
246
                  , "--use-memory=500M"
 
247
                  , "--target-dir=%s" %backup_path
 
248
                  ]
 
249
            cmd = " ".join(cmd)
 
250
            retcode, output = self.execute_cmd(cmd, output_path, exec_path, True)
 
251
            self.assertTrue(retcode==0,output)
 
252
        
 
253
            # copy our data files back
 
254
            for root, dirs, files in os.walk(backup_path):
 
255
                if files:
 
256
                    file_info = root.split(backup_path)[1]
 
257
                    for file_name in files:
 
258
                        # We do a quick check to make sure
 
259
                        # no names start with '/' as os.path
 
260
                        # throws a hissy when it sees such things
 
261
                        if file_info.startswith('/'):
 
262
                            file_info = file_info[1:]
 
263
                        if file_name.startswith('/'):
 
264
                            file_name = file_name[1:]
 
265
                        to_path = os.path.join(master_server.datadir
 
266
                                              , file_info
 
267
                                              , file_name)
 
268
                        new_dir = os.path.dirname(to_path)
 
269
                        try:
 
270
                           if not os.path.exists(new_dir):
 
271
                               os.makedirs(new_dir)
 
272
                        except OSError, e:
 
273
                            logging.error("Could not create directory: %s | %s" %(new_dir, e))
 
274
                        try:
 
275
                            shutil.copy(os.path.join(root,file_name),to_path)
 
276
                        except IOError, e:
 
277
                            logging.error( "ERROR:  Could not copy file: %s | %s" %(file_name, e))
 
278
 
 
279
            # restart server (and ensure it doesn't crash)
 
280
            master_server.start()
 
281
            self.assertTrue(master_server.status==1, 'Server failed restart from restored datadir...')
 
282
 
 
283
            # Get a checksum for our table
 
284
            query = "CHECKSUM TABLE %s" %table_name
 
285
            retcode, checksum3 = self.execute_query(query, master_server)
 
286
            self.assertEqual(retcode, 0, msg=checksum3)
 
287
            logging.test_debug("Checksum3: %s" %checksum3)
 
288
            self.assertEqual(checksum2,checksum3,msg="%s || %s" %(checksum2,checksum3))
 
289
 
 
290
 
 
291
 
 
292
    def tearDown(self):
 
293
            server_manager.reset_servers(test_executor.name)