← Back to branch summary

~openerp-dev/openobject-server/trunk-imp-onchange-behave-darshan

« back to all changes in this revision

Viewing changes to openerp/service/db.py

  • Committer: Darshan Kalola(OpenERP)
  • Date: 2014-04-09 08:44:52 UTC
  • mfrom: (4936.1.232 openobject-server)
  • Revision ID: dka@tinyerp.com-20140409084452-w1e499j21i3eli9d
[MERGE]sync with trunk.

Show diffs side-by-side

added added

removed removed

Lines of Context:
openerp/service/db.py
1
1
# -*- coding: utf-8 -*-
2
 
 
3
 
import base64
4
 
import contextlib
 
2
from contextlib import closing
 
3
from functools import wraps
5
4
import logging
6
5
import os
 
6
import shutil
7
7
import threading
8
8
import traceback
9
 
from contextlib import contextmanager, closing
 
9
import tempfile
 
10
import zipfile
 
11
 
 
12
import psycopg2
10
13
 
11
14
import openerp
12
15
from openerp import SUPERUSER_ID
28
31
        self_actions[id]['progress'] = 0
29
32
        db = openerp.sql_db.db_connect(db_name)
30
33
        with closing(db.cursor()) as cr:
31
 
            openerp.modules.db.initialize(cr) # TODO this should be removed as it is done by RegistryManager.new().
 
34
            # TODO this should be removed as it is done by RegistryManager.new().
 
35
            openerp.modules.db.initialize(cr)
32
36
            openerp.tools.config['lang'] = lang
33
37
            cr.commit()
34
38
 
55
59
        self_actions[id]['traceback'] = traceback.format_exc()
56
60
 
57
61
def dispatch(method, params):
58
 
    if method in [ 'create', 'get_progress', 'drop', 'dump',
59
 
        'restore', 'rename',
60
 
        'change_admin_password', 'migrate_databases',
61
 
        'create_database', 'duplicate_database' ]:
 
62
    if method in ['create', 'get_progress', 'drop', 'dump', 'restore', 'rename',
 
63
                  'change_admin_password', 'migrate_databases',
 
64
                  'create_database', 'duplicate_database']:
62
65
        passwd = params[0]
63
66
        params = params[1:]
64
67
        security.check_super(passwd)
65
 
    elif method in [ 'db_exist', 'list', 'list_lang', 'server_version' ]:
 
68
    elif method in ['db_exist', 'list', 'list_lang', 'server_version']:
66
69
        # params = params
67
70
        # No security check for these methods
68
71
        pass
78
81
        cr.execute("SELECT datname FROM pg_database WHERE datname = %s",
79
82
                   (name,))
80
83
        if cr.fetchall():
81
 
            raise openerp.exceptions.Warning(" %s database already exists!" % name )
 
84
            raise openerp.exceptions.Warning("database %r already exists!" % (name,))
82
85
        else:
83
 
            cr.autocommit(True) # avoid transaction block
 
86
            cr.autocommit(True)     # avoid transaction block
84
87
            cr.execute("""CREATE DATABASE "%s" ENCODING 'unicode' TEMPLATE "%s" """ % (name, chosen_template))
85
88
 
86
89
def exp_create(db_name, demo, lang, user_password='admin'):
96
99
 
97
100
    _logger.info('CREATE DATABASE %s', db_name.lower())
98
101
    create_thread = threading.Thread(target=_initialize_db,
99
 
            args=(id, db_name, demo, lang, user_password))
 
102
                                     args=(id, db_name, demo, lang, user_password))
100
103
    create_thread.start()
101
104
    self_actions[id]['thread'] = create_thread
102
105
    return id
121
124
    openerp.sql_db.close_db(db_original_name)
122
125
    db = openerp.sql_db.db_connect('postgres')
123
126
    with closing(db.cursor()) as cr:
124
 
        cr.autocommit(True) # avoid transaction block
 
127
        cr.autocommit(True)     # avoid transaction block
125
128
        cr.execute("""CREATE DATABASE "%s" ENCODING 'unicode' TEMPLATE "%s" """ % (db_name, db_original_name))
 
129
 
 
130
    from_fs = openerp.tools.config.filestore(db_original_name)
 
131
    to_fs = openerp.tools.config.filestore(db_name)
 
132
    if os.path.exists(from_fs) and not os.path.exists(to_fs):
 
133
        shutil.copy(from_fs, to_fs)
126
134
    return True
127
135
 
128
136
def exp_get_progress(id):
129
137
    if self_actions[id]['thread'].isAlive():
130
138
#       return openerp.modules.init_progress[db_name]
131
 
        return min(self_actions[id].get('progress', 0),0.95), []
 
139
        return min(self_actions[id].get('progress', 0), 0.95), []
132
140
    else:
133
141
        clean = self_actions[id]['clean']
134
142
        if clean:
140
148
            self_actions.pop(id)
141
149
            return 1.0, users
142
150
        else:
143
 
            e = self_actions[id]['exception'] # TODO this seems wrong: actions[id]['traceback'] is set, but not 'exception'.
144
 
            self_actions.pop(id)
145
 
            raise Exception, e
 
151
            a = self_actions.pop(id)
 
152
            exc, tb = a['exception'], a['traceback']
 
153
            raise Exception, exc, tb
146
154
 
147
155
def exp_drop(db_name):
148
156
    if db_name not in exp_list(True):
152
160
 
153
161
    db = openerp.sql_db.db_connect('postgres')
154
162
    with closing(db.cursor()) as cr:
155
 
        cr.autocommit(True) # avoid transaction block
 
163
        cr.autocommit(True)     # avoid transaction block
156
164
        # Try to terminate all other connections that might prevent
157
165
        # dropping the database
158
166
        try:
159
 
 
160
167
            # PostgreSQL 9.2 renamed pg_stat_activity.procpid to pid:
161
168
            # http://www.postgresql.org/docs/9.2/static/release-9-2.html#AEN110389
162
169
            pid_col = 'pid' if cr._cnx.server_version >= 90200 else 'procpid'
163
170
 
164
171
            cr.execute("""SELECT pg_terminate_backend(%(pid_col)s)
165
172
                          FROM pg_stat_activity
166
 
                          WHERE datname = %%s AND 
 
173
                          WHERE datname = %%s AND
167
174
                                %(pid_col)s != pg_backend_pid()""" % {'pid_col': pid_col},
168
175
                       (db_name,))
169
176
        except Exception:
176
183
            raise Exception("Couldn't drop database %s: %s" % (db_name, e))
177
184
        else:
178
185
            _logger.info('DROP DB: %s', db_name)
 
186
 
 
187
    fs = openerp.tools.config.filestore(db_name)
 
188
    if os.path.exists(fs):
 
189
        shutil.rmtree(fs)
179
190
    return True
180
191
 
181
 
@contextlib.contextmanager
182
 
def _set_pg_password_in_environment():
 
192
def _set_pg_password_in_environment(func):
183
193
    """ On systems where pg_restore/pg_dump require an explicit
184
194
    password (i.e. when not connecting via unix sockets, and most
185
195
    importantly on Windows), it is necessary to pass the PG user
186
196
    password in the environment or in a special .pgpass file.
187
197
 
188
 
    This context management method handles setting
 
198
    This decorator handles setting
189
199
    :envvar:`PGPASSWORD` if it is not already
190
200
    set, and removing it afterwards.
191
201
 
192
202
    See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html
193
 
    
 
203
 
194
204
    .. note:: This is not thread-safe, and should never be enabled for
195
205
         SaaS (giving SaaS users the super-admin password is not a good idea
196
206
         anyway)
197
207
    """
198
 
    if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
199
 
        yield
200
 
    else:
201
 
        os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
202
 
        try:
203
 
            yield
204
 
        finally:
205
 
            del os.environ['PGPASSWORD']
206
 
 
 
208
    @wraps(func)
 
209
    def wrapper(*args, **kwargs):
 
210
        if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
 
211
            return func(*args, **kwargs)
 
212
        else:
 
213
            os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
 
214
            try:
 
215
                return func(*args, **kwargs)
 
216
            finally:
 
217
                del os.environ['PGPASSWORD']
 
218
    return wrapper
207
219
 
208
220
def exp_dump(db_name):
209
 
    with _set_pg_password_in_environment():
210
 
        cmd = ['pg_dump', '--format=c', '--no-owner']
211
 
        if openerp.tools.config['db_user']:
212
 
            cmd.append('--username=' + openerp.tools.config['db_user'])
213
 
        if openerp.tools.config['db_host']:
214
 
            cmd.append('--host=' + openerp.tools.config['db_host'])
215
 
        if openerp.tools.config['db_port']:
216
 
            cmd.append('--port=' + str(openerp.tools.config['db_port']))
217
 
        cmd.append(db_name)
218
 
 
219
 
        stdin, stdout = openerp.tools.exec_pg_command_pipe(*tuple(cmd))
220
 
        stdin.close()
221
 
        data = stdout.read()
222
 
        res = stdout.close()
223
 
 
224
 
        if not data or res:
225
 
            _logger.error(
226
 
                    'DUMP DB: %s failed! Please verify the configuration of the database password on the server. '
227
 
                    'You may need to create a .pgpass file for authentication, or specify `db_password` in the '
228
 
                    'server configuration file.\n %s', db_name, data)
229
 
            raise Exception, "Couldn't dump database"
230
 
        _logger.info('DUMP DB successful: %s', db_name)
231
 
 
232
 
        return base64.encodestring(data)
233
 
 
234
 
def exp_restore(db_name, data):
235
 
    with _set_pg_password_in_environment():
236
 
        if exp_db_exist(db_name):
237
 
            _logger.warning('RESTORE DB: %s already exists', db_name)
238
 
            raise Exception, "Database already exists"
239
 
 
240
 
        _create_empty_database(db_name)
241
 
 
242
 
        cmd = ['pg_restore', '--no-owner']
243
 
        if openerp.tools.config['db_user']:
244
 
            cmd.append('--username=' + openerp.tools.config['db_user'])
245
 
        if openerp.tools.config['db_host']:
246
 
            cmd.append('--host=' + openerp.tools.config['db_host'])
247
 
        if openerp.tools.config['db_port']:
248
 
            cmd.append('--port=' + str(openerp.tools.config['db_port']))
249
 
        cmd.append('--dbname=' + db_name)
250
 
        args2 = tuple(cmd)
251
 
 
252
 
        buf=base64.decodestring(data)
253
 
        if os.name == "nt":
254
 
            tmpfile = (os.environ['TMP'] or 'C:\\') + os.tmpnam()
255
 
            file(tmpfile, 'wb').write(buf)
256
 
            args2=list(args2)
257
 
            args2.append(tmpfile)
258
 
            args2=tuple(args2)
259
 
        stdin, stdout = openerp.tools.exec_pg_command_pipe(*args2)
260
 
        if not os.name == "nt":
261
 
            stdin.write(base64.decodestring(data))
262
 
        stdin.close()
263
 
        res = stdout.close()
264
 
        if res:
265
 
            raise Exception, "Couldn't restore database"
266
 
        _logger.info('RESTORE DB: %s', db_name)
267
 
 
268
 
        return True
 
221
    with tempfile.TemporaryFile() as t:
 
222
        dump_db(db_name, t)
 
223
        t.seek(0)
 
224
        return t.read().encode('base64')
 
225
 
 
226
@_set_pg_password_in_environment
 
227
def dump_db(db, stream):
 
228
    """Dump database `db` into file-like object `stream`"""
 
229
    with openerp.tools.osutil.tempdir() as dump_dir:
 
230
        registry = openerp.modules.registry.RegistryManager.get(db)
 
231
        with registry.cursor() as cr:
 
232
            filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
 
233
            if os.path.exists(filestore):
 
234
                shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
 
235
 
 
236
        dump_file = os.path.join(dump_dir, 'dump.sql')
 
237
        cmd = ['pg_dump', '--format=p', '--no-owner', '--file=' + dump_file]
 
238
        if openerp.tools.config['db_user']:
 
239
            cmd.append('--username=' + openerp.tools.config['db_user'])
 
240
        if openerp.tools.config['db_host']:
 
241
            cmd.append('--host=' + openerp.tools.config['db_host'])
 
242
        if openerp.tools.config['db_port']:
 
243
            cmd.append('--port=' + str(openerp.tools.config['db_port']))
 
244
        cmd.append(db)
 
245
 
 
246
        if openerp.tools.exec_pg_command(*cmd):
 
247
            _logger.error('DUMP DB: %s failed! Please verify the configuration of the database '
 
248
                          'password on the server. You may need to create a .pgpass file for '
 
249
                          'authentication, or specify `db_password` in the server configuration '
 
250
                          'file.', db)
 
251
            raise Exception("Couldn't dump database")
 
252
 
 
253
        openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
 
254
 
 
255
    _logger.info('DUMP DB successful: %s', db)
 
256
 
 
257
def exp_restore(db_name, data, copy=False):
 
258
    data_file = tempfile.NamedTemporaryFile(delete=False)
 
259
    try:
 
260
        data_file.write(data.decode('base64'))
 
261
        data_file.close()
 
262
        restore_db(db_name, data_file.name, copy=copy)
 
263
    finally:
 
264
        os.unlink(data_file.name)
 
265
    return True
 
266
 
 
267
@_set_pg_password_in_environment
 
268
def restore_db(db, dump_file, copy=False):
 
269
    assert isinstance(db, basestring)
 
270
    if exp_db_exist(db):
 
271
        _logger.warning('RESTORE DB: %s already exists', db)
 
272
        raise Exception("Database already exists")
 
273
 
 
274
    _create_empty_database(db)
 
275
 
 
276
    filestore_path = None
 
277
    with openerp.tools.osutil.tempdir() as dump_dir:
 
278
        if zipfile.is_zipfile(dump_file):
 
279
            # v8 format
 
280
            with zipfile.ZipFile(dump_file, 'r') as z:
 
281
                # only extract known members!
 
282
                filestore = [m for m in z.namelist() if m.startswith('filestore/')]
 
283
                z.extractall(dump_dir, ['dump.sql'] + filestore)
 
284
 
 
285
                if filestore:
 
286
                    filestore_path = os.path.join(dump_dir, 'filestore')
 
287
 
 
288
            pg_cmd = 'psql'
 
289
            pg_args = ['-q', '-f', os.path.join(dump_dir, 'dump.sql')]
 
290
 
 
291
        else:
 
292
            # <= 7.0 format (raw pg_dump output)
 
293
            pg_cmd = 'pg_restore'
 
294
            pg_args = ['--no-owner', dump_file]
 
295
 
 
296
        args = []
 
297
        if openerp.tools.config['db_user']:
 
298
            args.append('--username=' + openerp.tools.config['db_user'])
 
299
        if openerp.tools.config['db_host']:
 
300
            args.append('--host=' + openerp.tools.config['db_host'])
 
301
        if openerp.tools.config['db_port']:
 
302
            args.append('--port=' + str(openerp.tools.config['db_port']))
 
303
        args.append('--dbname=' + db)
 
304
        pg_args = args + pg_args
 
305
 
 
306
        if openerp.tools.exec_pg_command(pg_cmd, *pg_args):
 
307
            raise Exception("Couldn't restore database")
 
308
 
 
309
        registry = openerp.modules.registry.RegistryManager.new(db)
 
310
        with registry.cursor() as cr:
 
311
            if copy:
 
312
                # if it's a copy of a database, force generation of a new dbuuid
 
313
                registry['ir.config_parameter'].init(cr, force=True)
 
314
            if filestore_path:
 
315
                filestore_dest = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
 
316
                shutil.move(filestore_path, filestore_dest)
 
317
 
 
318
            if openerp.tools.config['unaccent']:
 
319
                try:
 
320
                    with cr.savepoint():
 
321
                        cr.execute("CREATE EXTENSION unaccent")
 
322
                except psycopg2.Error:
 
323
                    pass
 
324
 
 
325
    _logger.info('RESTORE DB: %s', db)
269
326
 
270
327
def exp_rename(old_name, new_name):
271
328
    openerp.modules.registry.RegistryManager.delete(old_name)
273
330
 
274
331
    db = openerp.sql_db.db_connect('postgres')
275
332
    with closing(db.cursor()) as cr:
276
 
        cr.autocommit(True) # avoid transaction block
 
333
        cr.autocommit(True)     # avoid transaction block
277
334
        try:
278
335
            cr.execute('ALTER DATABASE "%s" RENAME TO "%s"' % (old_name, new_name))
279
336
            _logger.info('RENAME DB: %s -> %s', old_name, new_name)
280
337
        except Exception, e:
281
338
            _logger.error('RENAME DB: %s -> %s failed:\n%s', old_name, new_name, e)
282
339
            raise Exception("Couldn't rename database %s to %s: %s" % (old_name, new_name, e))
 
340
 
 
341
    old_fs = openerp.tools.config.filestore(old_name)
 
342
    new_fs = openerp.tools.config.filestore(new_name)
 
343
    if os.path.exists(old_fs) and not os.path.exists(new_fs):
 
344
        shutil.move(old_fs, new_fs)
283
345
    return True
284
346
 
 
347
@openerp.tools.mute_logger('openerp.sql_db')
285
348
def exp_db_exist(db_name):
286
349
    ## Not True: in fact, check if connection to database is possible. The database may exists
287
350
    return bool(openerp.sql_db.db_connect(db_name))