2
# -*- coding: utf-8 -*-
4
# Copyright 2014-2015 Canonical Limited.
6
# Licensed under the Apache License, Version 2.0 (the "License");
7
# you may not use this file except in compliance with the License.
8
# You may obtain a copy of the License at
10
# http://www.apache.org/licenses/LICENSE-2.0
12
# Unless required by applicable law or agreed to in writing, software
13
# distributed under the License is distributed on an "AS IS" BASIS,
14
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
19
# Kapil Thangavelu <kapil.foss@gmail.com>
25
A simple way to store state in units. This provides a key value
26
storage with support for versioned, transactional operation,
27
and can calculate deltas from previous values to simplify unit logic
28
when processing changes.
34
There are several extant frameworks for hook execution, including
36
- charmhelpers.core.hookenv.Hooks
37
- charmhelpers.core.services.ServiceManager
39
The storage classes are framework agnostic, one simple integration is
40
via the HookData contextmanager. It will record the current hook
41
execution environment (including relation data, config data, etc.),
42
setup a transaction and allow easy access to the changes from
43
previously seen values. One consequence of the integration is the
44
reservation of particular keys ('rels', 'unit', 'env', 'config',
45
'charm_revisions') for their respective values.
47
Here's a fully worked integration example using hookenv.Hooks::
49
from charmhelper.core import hookenv, unitdata
51
hook_data = unitdata.HookData()
53
hooks = hookenv.Hooks()
57
# Print all changes to configuration from previously seen
59
for changed, (prev, cur) in hook_data.conf.items():
60
print('config changed', changed,
61
'previous value', prev,
64
# Get some unit specific bookeeping
65
if not db.get('pkg_key'):
66
key = urllib.urlopen('https://example.com/pkg_key').read()
67
db.set('pkg_key', key)
69
# Directly access all charm config as a mapping.
70
conf = db.getrange('config', True)
72
# Directly access all relation data as a mapping
73
rels = db.getrange('rels', True)
75
if __name__ == '__main__':
80
A more basic integration is via the hook_scope context manager which simply
81
manages transaction scope (and records hook name, and timestamp)::
83
>>> from unitdata import kv
85
>>> with db.hook_scope('install'):
86
... # do work, in transactional scope.
95
Values are automatically json de/serialized to preserve basic typing
96
and complex data struct capabilities (dicts, lists, ints, booleans, etc).
98
Individual values can be manipulated via get/set::
100
>>> kv.set('y', True)
104
# We can set complex values (dicts, lists) as a single key.
105
>>> kv.set('config', {'a': 1, 'b': True'})
107
# Also supports returning dictionaries as a record which
108
# provides attribute access.
109
>>> config = kv.get('config', record=True)
114
Groups of keys can be manipulated with update/getrange::
116
>>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
117
>>> kv.getrange('gui.', strip=True)
120
When updating values, its very helpful to understand which values
121
have actually changed and how have they changed. The storage
122
provides a delta method to provide for this::
124
>>> data = {'debug': True, 'option': 2}
125
>>> delta = kv.delta(data, 'config.')
126
>>> delta.debug.previous
128
>>> delta.debug.current
131
{'debug': (None, True), 'option': (None, 2)}
133
Note the delta method does not persist the actual change, it needs to
134
be explicitly saved via 'update' method::
136
>>> kv.update(data, 'config.')
138
Values modified in the context of a hook scope retain historical values
139
associated to the hookname.
141
>>> with db.hook_scope('config-changed'):
143
>>> db.gethistory('x')
144
[(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
145
(2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
159
__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>'
162
class Storage(object):
163
"""Simple key value database for local unit state within charms.
165
Modifications are not persisted unless :meth:`flush` is called.
167
To support dicts, lists, integer, floats, and booleans values
168
are automatically json encoded/decoded.
170
def __init__(self, path=None):
173
if 'UNIT_STATE_DB' in os.environ:
174
self.db_path = os.environ['UNIT_STATE_DB']
176
self.db_path = os.path.join(
177
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
178
self.conn = sqlite3.connect('%s' % self.db_path)
179
self.cursor = self.conn.cursor()
192
def get(self, key, default=None, record=False):
193
self.cursor.execute('select data from kv where key=?', [key])
194
result = self.cursor.fetchone()
198
return Record(json.loads(result[0]))
199
return json.loads(result[0])
201
def getrange(self, key_prefix, strip=False):
203
Get a range of keys starting with a common prefix as a mapping of
206
:param str key_prefix: Common prefix among all keys
207
:param bool strip: Optionally strip the common prefix from the key
208
names in the returned dict
209
:return dict: A (possibly empty) dict of key-value mappings
211
self.cursor.execute("select key, data from kv where key like ?",
212
['%s%%' % key_prefix])
213
result = self.cursor.fetchall()
220
(k[len(key_prefix):], json.loads(v)) for k, v in result])
222
def update(self, mapping, prefix=""):
224
Set the values of multiple keys at once.
226
:param dict mapping: Mapping of keys to values
227
:param str prefix: Optional prefix to apply to all keys in `mapping`
230
for k, v in mapping.items():
231
self.set("%s%s" % (prefix, k), v)
233
def unset(self, key):
235
Remove a key from the database entirely.
237
self.cursor.execute('delete from kv where key=?', [key])
238
if self.revision and self.cursor.rowcount:
240
'insert into kv_revisions values (?, ?, ?)',
241
[key, self.revision, json.dumps('DELETED')])
243
def unsetrange(self, keys=None, prefix=""):
245
Remove a range of keys starting with a common prefix, from the database
248
:param list keys: List of keys to remove.
249
:param str prefix: Optional prefix to apply to all keys in ``keys``
253
keys = ['%s%s' % (prefix, key) for key in keys]
254
self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
255
if self.revision and self.cursor.rowcount:
257
'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
258
list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
260
self.cursor.execute('delete from kv where key like ?',
262
if self.revision and self.cursor.rowcount:
264
'insert into kv_revisions values (?, ?, ?)',
265
['%s%%' % prefix, self.revision, json.dumps('DELETED')])
267
def set(self, key, value):
269
Set a value in the database.
271
:param str key: Key to set the value for
272
:param value: Any JSON-serializable value to be set
274
serialized = json.dumps(value)
276
self.cursor.execute('select data from kv where key=?', [key])
277
exists = self.cursor.fetchone()
279
# Skip mutations to the same value
281
if exists[0] == serialized:
286
'insert into kv (key, data) values (?, ?)',
289
self.cursor.execute('''
292
where key = ?''', [serialized, key])
295
if not self.revision:
299
'select 1 from kv_revisions where key=? and revision=?',
300
[key, self.revision])
301
exists = self.cursor.fetchone()
305
'''insert into kv_revisions (
306
revision, key, data) values (?, ?, ?)''',
307
(self.revision, key, serialized))
315
[serialized, key, self.revision])
319
def delta(self, mapping, prefix):
321
return a delta containing values that have changed.
323
previous = self.getrange(prefix, strip=True)
327
pk = set(previous.keys())
328
ck = set(mapping.keys())
332
for k in ck.difference(pk):
333
delta[k] = Delta(None, mapping[k])
336
for k in pk.difference(ck):
337
delta[k] = Delta(previous[k], None)
340
for k in pk.intersection(ck):
344
delta[k] = Delta(p, c)
348
@contextlib.contextmanager
349
def hook_scope(self, name=""):
350
"""Scope all future interactions to the current hook execution
352
assert not self.revision
354
'insert into hooks (hook, date) values (?, ?)',
355
(name or sys.argv[0],
356
datetime.datetime.utcnow().isoformat()))
357
self.revision = self.cursor.lastrowid
368
def flush(self, save=True):
377
self.cursor.execute('''
378
create table if not exists kv (
383
self.cursor.execute('''
384
create table if not exists kv_revisions (
388
primary key (key, revision)
390
self.cursor.execute('''
391
create table if not exists hooks (
392
version integer primary key autoincrement,
398
def gethistory(self, key, deserialize=False):
401
select kv.revision, kv.key, kv.data, h.hook, h.date
402
from kv_revisions kv,
405
and kv.revision = h.version
407
if deserialize is False:
408
return self.cursor.fetchall()
409
return map(_parse_history, self.cursor.fetchall())
411
def debug(self, fh=sys.stderr):
412
self.cursor.execute('select * from kv')
413
pprint.pprint(self.cursor.fetchall(), stream=fh)
414
self.cursor.execute('select * from kv_revisions')
415
pprint.pprint(self.cursor.fetchall(), stream=fh)
418
def _parse_history(d):
419
return (d[0], d[1], json.loads(d[2]), d[3],
420
datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
423
class HookData(object):
424
"""Simple integration for existing hook exec frameworks.
426
Records all unit information, and stores deltas for processing
431
from charmhelper.core import hookenv, unitdata
433
changes = unitdata.HookData()
435
hooks = hookenv.Hooks()
438
def config_changed():
439
# View all changes to configuration
440
for changed, (prev, cur) in changes.conf.items():
441
print('config changed', changed,
442
'previous value', prev,
443
'current value', cur)
445
# Get some unit specific bookeeping
446
if not db.get('pkg_key'):
447
key = urllib.urlopen('https://example.com/pkg_key').read()
448
db.set('pkg_key', key)
450
if __name__ == '__main__':
460
@contextlib.contextmanager
462
from charmhelpers.core import hookenv
463
hook_name = hookenv.hook_name()
465
with self.kv.hook_scope(hook_name):
466
self._record_charm_version(hookenv.charm_dir())
467
delta_config, delta_relation = self._record_hook(hookenv)
468
yield self.kv, delta_config, delta_relation
470
def _record_charm_version(self, charm_dir):
471
# Record revisions.. charm revisions are meaningless
472
# to charm authors as they don't control the revision.
473
# so logic dependnent on revision is not particularly
474
# useful, however it is useful for debugging analysis.
476
os.path.join(charm_dir, 'revision')).read().strip()
477
charm_rev = charm_rev or '0'
478
revs = self.kv.get('charm_revisions', [])
479
if charm_rev not in revs:
480
revs.append(charm_rev.strip() or '0')
481
self.kv.set('charm_revisions', revs)
483
def _record_hook(self, hookenv):
484
data = hookenv.execution_environment()
485
self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
486
self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
487
self.kv.set('env', dict(data['env']))
488
self.kv.set('unit', data['unit'])
489
self.kv.set('relid', data.get('relid'))
490
return conf_delta, rels_delta
497
def __getattr__(self, k):
500
raise AttributeError(k)
503
class DeltaSet(Record):
508
Delta = collections.namedtuple('Delta', ['previous', 'current'])
added
removed
tests/charmhelpers/core/unitdata.py
