1
# Copyright 2011 OpenStack Foundation.
4
# Licensed under the Apache License, Version 2.0 (the "License"); you may
5
# not use this file except in compliance with the License. You may obtain
6
# a copy of the License at
8
# http://www.apache.org/licenses/LICENSE-2.0
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
# License for the specific language governing permissions and limitations
29
from oslo.config import cfg
31
from ironic.openstack.common import fileutils
32
from ironic.openstack.common.gettextutils import _, _LE, _LI
35
LOG = logging.getLogger(__name__)
39
cfg.BoolOpt('disable_process_locking', default=False,
40
help='Enables or disables inter-process locks.'),
41
cfg.StrOpt('lock_path',
42
default=os.environ.get("IRONIC_LOCK_PATH"),
43
help='Directory to use for lock files.')
48
CONF.register_opts(util_opts)
51
def set_defaults(lock_path):
52
cfg.set_defaults(util_opts, lock_path=lock_path)
55
class _FileLock(object):
56
"""Lock implementation which allows multiple locks, working around
57
issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
58
not require any cleanup. Since the lock is always held on a file
59
descriptor rather than outside of the process, the lock gets dropped
60
automatically if the process crashes, even if __exit__ is not executed.
62
There are no guarantees regarding usage by multiple green threads in a
63
single process here. This lock works only between processes. Exclusive
64
access between local threads should be achieved using the semaphores
65
in the @synchronized decorator.
67
Note these locks are released when the descriptor is closed, so it's not
68
safe to close the file descriptor while another green thread holds the
69
lock. Just opening and closing the lock file can break synchronisation,
70
so lock files must be accessed only using this abstraction.
73
def __init__(self, name):
78
basedir = os.path.dirname(self.fname)
80
if not os.path.exists(basedir):
81
fileutils.ensure_tree(basedir)
82
LOG.info(_LI('Created lock path: %s'), basedir)
84
self.lockfile = open(self.fname, 'w')
88
# Using non-blocking locks since green threads are not
89
# patched to deal with blocking locking calls.
90
# Also upon reading the MSDN docs for locking(), it seems
91
# to have a laughable 10 attempts "blocking" mechanism.
93
LOG.debug('Got file lock "%s"', self.fname)
96
if e.errno in (errno.EACCES, errno.EAGAIN):
97
# external locks synchronise things like iptables
98
# updates - give it some time to prevent busy spinning
101
raise threading.ThreadError(_("Unable to acquire lock on"
102
" `%(filename)s` due to"
105
'filename': self.fname,
116
self.lockfile.close()
117
LOG.debug('Released file lock "%s"', self.fname)
119
LOG.exception(_LE("Could not release the acquired lock `%s`"),
122
def __exit__(self, exc_type, exc_val, exc_tb):
126
return os.path.exists(self.fname)
129
raise NotImplementedError()
132
raise NotImplementedError()
135
class _WindowsLock(_FileLock):
137
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
140
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
143
class _FcntlLock(_FileLock):
145
fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
148
fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
151
class _PosixLock(object):
152
def __init__(self, name):
153
# Hash the name because it's not valid to have POSIX semaphore
154
# names with things like / in them. Then use base64 to encode
155
# the digest() instead taking the hexdigest() because the
156
# result is shorter and most systems can't have shm sempahore
157
# names longer than 31 characters.
159
h.update(name.encode('ascii'))
160
self.name = str((b'/' + base64.urlsafe_b64encode(
161
h.digest())).decode('ascii'))
163
def acquire(self, timeout=None):
164
self.semaphore = posix_ipc.Semaphore(self.name,
165
flags=posix_ipc.O_CREAT,
167
self.semaphore.acquire(timeout)
175
self.semaphore.release()
176
self.semaphore.close()
178
def __exit__(self, exc_type, exc_val, exc_tb):
183
semaphore = posix_ipc.Semaphore(self.name)
184
except posix_ipc.ExistentialError:
193
InterProcessLock = _WindowsLock
194
FileLock = _WindowsLock
201
InterProcessLock = _PosixLock
202
FileLock = _FcntlLock
204
_semaphores = weakref.WeakValueDictionary()
205
_semaphores_lock = threading.Lock()
208
def _get_lock_path(name, lock_file_prefix, lock_path=None):
209
# NOTE(mikal): the lock name cannot contain directory
211
name = name.replace(os.sep, '_')
213
sep = '' if lock_file_prefix.endswith('-') else '-'
214
name = '%s%s%s' % (lock_file_prefix, sep, name)
216
local_lock_path = lock_path or CONF.lock_path
218
if not local_lock_path:
219
# NOTE(bnemec): Create a fake lock path for posix locks so we don't
220
# unnecessarily raise the RequiredOptError below.
221
if InterProcessLock is not _PosixLock:
222
raise cfg.RequiredOptError('lock_path')
223
local_lock_path = 'posixlock:/'
225
return os.path.join(local_lock_path, name)
228
def external_lock(name, lock_file_prefix=None, lock_path=None):
229
LOG.debug('Attempting to grab external lock "%(lock)s"',
232
lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
234
# NOTE(bnemec): If an explicit lock_path was passed to us then it
235
# means the caller is relying on file-based locking behavior, so
236
# we can't use posix locks for those calls.
238
return FileLock(lock_file_path)
239
return InterProcessLock(lock_file_path)
242
def remove_external_lock_file(name, lock_file_prefix=None):
243
"""Remove an external lock file when it's not used anymore
244
This will be helpful when we have a lot of lock files
246
with internal_lock(name):
247
lock_file_path = _get_lock_path(name, lock_file_prefix)
249
os.remove(lock_file_path)
251
LOG.info(_LI('Failed to remove file %(file)s'),
252
{'file': lock_file_path})
255
def internal_lock(name):
256
with _semaphores_lock:
258
sem = _semaphores[name]
260
sem = threading.Semaphore()
261
_semaphores[name] = sem
263
LOG.debug('Got semaphore "%(lock)s"', {'lock': name})
267
@contextlib.contextmanager
268
def lock(name, lock_file_prefix=None, external=False, lock_path=None):
269
"""Context based lock
271
This function yields a `threading.Semaphore` instance (if we don't use
272
eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
273
True, in which case, it'll yield an InterProcessLock instance.
275
:param lock_file_prefix: The lock_file_prefix argument is used to provide
276
lock files on disk with a meaningful prefix.
278
:param external: The external keyword argument denotes whether this lock
279
should work across multiple processes. This means that if two different
280
workers both run a method decorated with @synchronized('mylock',
281
external=True), only one of them will execute at a time.
283
int_lock = internal_lock(name)
285
if external and not CONF.disable_process_locking:
286
ext_lock = external_lock(name, lock_file_prefix, lock_path)
291
LOG.debug('Released semaphore "%(lock)s"', {'lock': name})
294
def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
295
"""Synchronization decorator.
297
Decorating a method like so::
299
@synchronized('mylock')
300
def foo(self, *args):
303
ensures that only one thread will execute the foo method at a time.
305
Different methods can share the same lock::
307
@synchronized('mylock')
308
def foo(self, *args):
311
@synchronized('mylock')
312
def bar(self, *args):
315
This way only one of either foo or bar can be executing at a time.
320
def inner(*args, **kwargs):
322
with lock(name, lock_file_prefix, external, lock_path):
323
LOG.debug('Got semaphore / lock "%(function)s"',
324
{'function': f.__name__})
325
return f(*args, **kwargs)
327
LOG.debug('Semaphore / lock released "%(function)s"',
328
{'function': f.__name__})
333
def synchronized_with_prefix(lock_file_prefix):
334
"""Partial object generator for the synchronization decorator.
336
Redefine @synchronized in each project like so::
339
from nova.openstack.common import lockutils
341
synchronized = lockutils.synchronized_with_prefix('nova-')
345
from nova import utils
347
@utils.synchronized('mylock')
348
def bar(self, *args):
351
The lock_file_prefix argument is used to provide lock files on disk with a
355
return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
359
"""Create a dir for locks and pass it to command from arguments
362
python -m openstack.common.lockutils python setup.py testr <etc>
364
a temporary directory will be created for all your locks and passed to all
365
your tests in an environment variable. The temporary dir will be deleted
366
afterwards and the return value will be preserved.
369
lock_dir = tempfile.mkdtemp()
370
os.environ["IRONIC_LOCK_PATH"] = lock_dir
372
ret_val = subprocess.call(argv[1:])
374
shutil.rmtree(lock_dir, ignore_errors=True)
378
if __name__ == '__main__':
379
sys.exit(main(sys.argv))
added
removed
ironic/openstack/common/lockutils.py
