2
# -*- coding: utf-8 -*-
4
# Copyright 2014-2015 Canonical Limited.
6
# This file is part of charm-helpers.
8
# charm-helpers is free software: you can redistribute it and/or modify
9
# it under the terms of the GNU Lesser General Public License version 3 as
10
# published by the Free Software Foundation.
12
# charm-helpers is distributed in the hope that it will be useful,
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU Lesser General Public License for more details.
17
# You should have received a copy of the GNU Lesser General Public License
18
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
22
# Kapil Thangavelu <kapil.foss@gmail.com>
28
A simple way to store state in units. This provides a key value
29
storage with support for versioned, transactional operation,
30
and can calculate deltas from previous values to simplify unit logic
31
when processing changes.
37
There are several extant frameworks for hook execution, including
39
- charmhelpers.core.hookenv.Hooks
40
- charmhelpers.core.services.ServiceManager
42
The storage classes are framework agnostic, one simple integration is
43
via the HookData contextmanager. It will record the current hook
44
execution environment (including relation data, config data, etc.),
45
setup a transaction and allow easy access to the changes from
46
previously seen values. One consequence of the integration is the
47
reservation of particular keys ('rels', 'unit', 'env', 'config',
48
'charm_revisions') for their respective values.
50
Here's a fully worked integration example using hookenv.Hooks::
52
from charmhelper.core import hookenv, unitdata
54
hook_data = unitdata.HookData()
56
hooks = hookenv.Hooks()
60
# Print all changes to configuration from previously seen
62
for changed, (prev, cur) in hook_data.conf.items():
63
print('config changed', changed,
64
'previous value', prev,
67
# Get some unit specific bookeeping
68
if not db.get('pkg_key'):
69
key = urllib.urlopen('https://example.com/pkg_key').read()
70
db.set('pkg_key', key)
72
# Directly access all charm config as a mapping.
73
conf = db.getrange('config', True)
75
# Directly access all relation data as a mapping
76
rels = db.getrange('rels', True)
78
if __name__ == '__main__':
83
A more basic integration is via the hook_scope context manager which simply
84
manages transaction scope (and records hook name, and timestamp)::
86
>>> from unitdata import kv
88
>>> with db.hook_scope('install'):
89
... # do work, in transactional scope.
98
Values are automatically json de/serialized to preserve basic typing
99
and complex data struct capabilities (dicts, lists, ints, booleans, etc).
101
Individual values can be manipulated via get/set::
103
>>> kv.set('y', True)
107
# We can set complex values (dicts, lists) as a single key.
108
>>> kv.set('config', {'a': 1, 'b': True'})
110
# Also supports returning dictionaries as a record which
111
# provides attribute access.
112
>>> config = kv.get('config', record=True)
117
Groups of keys can be manipulated with update/getrange::
119
>>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
120
>>> kv.getrange('gui.', strip=True)
123
When updating values, its very helpful to understand which values
124
have actually changed and how have they changed. The storage
125
provides a delta method to provide for this::
127
>>> data = {'debug': True, 'option': 2}
128
>>> delta = kv.delta(data, 'config.')
129
>>> delta.debug.previous
131
>>> delta.debug.current
134
{'debug': (None, True), 'option': (None, 2)}
136
Note the delta method does not persist the actual change, it needs to
137
be explicitly saved via 'update' method::
139
>>> kv.update(data, 'config.')
141
Values modified in the context of a hook scope retain historical values
142
associated to the hookname.
144
>>> with db.hook_scope('config-changed'):
146
>>> db.gethistory('x')
147
[(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
148
(2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
162
__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>'
165
class Storage(object):
166
"""Simple key value database for local unit state within charms.
168
Modifications are not persisted unless :meth:`flush` is called.
170
To support dicts, lists, integer, floats, and booleans values
171
are automatically json encoded/decoded.
173
def __init__(self, path=None):
176
if 'UNIT_STATE_DB' in os.environ:
177
self.db_path = os.environ['UNIT_STATE_DB']
179
self.db_path = os.path.join(
180
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
181
self.conn = sqlite3.connect('%s' % self.db_path)
182
self.cursor = self.conn.cursor()
195
def get(self, key, default=None, record=False):
196
self.cursor.execute('select data from kv where key=?', [key])
197
result = self.cursor.fetchone()
201
return Record(json.loads(result[0]))
202
return json.loads(result[0])
204
def getrange(self, key_prefix, strip=False):
206
Get a range of keys starting with a common prefix as a mapping of
209
:param str key_prefix: Common prefix among all keys
210
:param bool strip: Optionally strip the common prefix from the key
211
names in the returned dict
212
:return dict: A (possibly empty) dict of key-value mappings
214
self.cursor.execute("select key, data from kv where key like ?",
215
['%s%%' % key_prefix])
216
result = self.cursor.fetchall()
223
(k[len(key_prefix):], json.loads(v)) for k, v in result])
225
def update(self, mapping, prefix=""):
227
Set the values of multiple keys at once.
229
:param dict mapping: Mapping of keys to values
230
:param str prefix: Optional prefix to apply to all keys in `mapping`
233
for k, v in mapping.items():
234
self.set("%s%s" % (prefix, k), v)
236
def unset(self, key):
238
Remove a key from the database entirely.
240
self.cursor.execute('delete from kv where key=?', [key])
241
if self.revision and self.cursor.rowcount:
243
'insert into kv_revisions values (?, ?, ?)',
244
[key, self.revision, json.dumps('DELETED')])
246
def unsetrange(self, keys=None, prefix=""):
248
Remove a range of keys starting with a common prefix, from the database
251
:param list keys: List of keys to remove.
252
:param str prefix: Optional prefix to apply to all keys in ``keys``
256
keys = ['%s%s' % (prefix, key) for key in keys]
257
self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
258
if self.revision and self.cursor.rowcount:
260
'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
261
list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
263
self.cursor.execute('delete from kv where key like ?',
265
if self.revision and self.cursor.rowcount:
267
'insert into kv_revisions values (?, ?, ?)',
268
['%s%%' % prefix, self.revision, json.dumps('DELETED')])
270
def set(self, key, value):
272
Set a value in the database.
274
:param str key: Key to set the value for
275
:param value: Any JSON-serializable value to be set
277
serialized = json.dumps(value)
279
self.cursor.execute('select data from kv where key=?', [key])
280
exists = self.cursor.fetchone()
282
# Skip mutations to the same value
284
if exists[0] == serialized:
289
'insert into kv (key, data) values (?, ?)',
292
self.cursor.execute('''
295
where key = ?''', [serialized, key])
298
if not self.revision:
302
'select 1 from kv_revisions where key=? and revision=?',
303
[key, self.revision])
304
exists = self.cursor.fetchone()
308
'''insert into kv_revisions (
309
revision, key, data) values (?, ?, ?)''',
310
(self.revision, key, serialized))
318
[serialized, key, self.revision])
322
def delta(self, mapping, prefix):
324
return a delta containing values that have changed.
326
previous = self.getrange(prefix, strip=True)
330
pk = set(previous.keys())
331
ck = set(mapping.keys())
335
for k in ck.difference(pk):
336
delta[k] = Delta(None, mapping[k])
339
for k in pk.difference(ck):
340
delta[k] = Delta(previous[k], None)
343
for k in pk.intersection(ck):
347
delta[k] = Delta(p, c)
351
@contextlib.contextmanager
352
def hook_scope(self, name=""):
353
"""Scope all future interactions to the current hook execution
355
assert not self.revision
357
'insert into hooks (hook, date) values (?, ?)',
358
(name or sys.argv[0],
359
datetime.datetime.utcnow().isoformat()))
360
self.revision = self.cursor.lastrowid
371
def flush(self, save=True):
380
self.cursor.execute('''
381
create table if not exists kv (
386
self.cursor.execute('''
387
create table if not exists kv_revisions (
391
primary key (key, revision)
393
self.cursor.execute('''
394
create table if not exists hooks (
395
version integer primary key autoincrement,
401
def gethistory(self, key, deserialize=False):
404
select kv.revision, kv.key, kv.data, h.hook, h.date
405
from kv_revisions kv,
408
and kv.revision = h.version
410
if deserialize is False:
411
return self.cursor.fetchall()
412
return map(_parse_history, self.cursor.fetchall())
414
def debug(self, fh=sys.stderr):
415
self.cursor.execute('select * from kv')
416
pprint.pprint(self.cursor.fetchall(), stream=fh)
417
self.cursor.execute('select * from kv_revisions')
418
pprint.pprint(self.cursor.fetchall(), stream=fh)
421
def _parse_history(d):
422
return (d[0], d[1], json.loads(d[2]), d[3],
423
datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
426
class HookData(object):
427
"""Simple integration for existing hook exec frameworks.
429
Records all unit information, and stores deltas for processing
434
from charmhelper.core import hookenv, unitdata
436
changes = unitdata.HookData()
438
hooks = hookenv.Hooks()
441
def config_changed():
442
# View all changes to configuration
443
for changed, (prev, cur) in changes.conf.items():
444
print('config changed', changed,
445
'previous value', prev,
446
'current value', cur)
448
# Get some unit specific bookeeping
449
if not db.get('pkg_key'):
450
key = urllib.urlopen('https://example.com/pkg_key').read()
451
db.set('pkg_key', key)
453
if __name__ == '__main__':
463
@contextlib.contextmanager
465
from charmhelpers.core import hookenv
466
hook_name = hookenv.hook_name()
468
with self.kv.hook_scope(hook_name):
469
self._record_charm_version(hookenv.charm_dir())
470
delta_config, delta_relation = self._record_hook(hookenv)
471
yield self.kv, delta_config, delta_relation
473
def _record_charm_version(self, charm_dir):
474
# Record revisions.. charm revisions are meaningless
475
# to charm authors as they don't control the revision.
476
# so logic dependnent on revision is not particularly
477
# useful, however it is useful for debugging analysis.
479
os.path.join(charm_dir, 'revision')).read().strip()
480
charm_rev = charm_rev or '0'
481
revs = self.kv.get('charm_revisions', [])
482
if charm_rev not in revs:
483
revs.append(charm_rev.strip() or '0')
484
self.kv.set('charm_revisions', revs)
486
def _record_hook(self, hookenv):
487
data = hookenv.execution_environment()
488
self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
489
self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
490
self.kv.set('env', dict(data['env']))
491
self.kv.set('unit', data['unit'])
492
self.kv.set('relid', data.get('relid'))
493
return conf_delta, rels_delta
500
def __getattr__(self, k):
503
raise AttributeError(k)
506
class DeltaSet(Record):
511
Delta = collections.namedtuple('Delta', ['previous', 'current'])
added
removed
hooks/charmhelpers/core/unitdata.py
