2
# -*- coding: utf-8 -*-
4
# Copyright 2014-2015 Canonical Limited.
6
# This file is part of charm-helpers.
8
# charm-helpers is free software: you can redistribute it and/or modify
9
# it under the terms of the GNU Lesser General Public License version 3 as
10
# published by the Free Software Foundation.
12
# charm-helpers is distributed in the hope that it will be useful,
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU Lesser General Public License for more details.
17
# You should have received a copy of the GNU Lesser General Public License
18
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
22
# Kapil Thangavelu <kapil.foss@gmail.com>
28
A simple way to store state in units. This provides a key value
29
storage with support for versioned, transactional operation,
30
and can calculate deltas from previous values to simplify unit logic
31
when processing changes.
37
There are several extant frameworks for hook execution, including
39
- charmhelpers.core.hookenv.Hooks
40
- charmhelpers.core.services.ServiceManager
42
The storage classes are framework agnostic, one simple integration is
43
via the HookData contextmanager. It will record the current hook
44
execution environment (including relation data, config data, etc.),
45
setup a transaction and allow easy access to the changes from
46
previously seen values. One consequence of the integration is the
47
reservation of particular keys ('rels', 'unit', 'env', 'config',
48
'charm_revisions') for their respective values.
50
Here's a fully worked integration example using hookenv.Hooks::
52
from charmhelper.core import hookenv, unitdata
54
hook_data = unitdata.HookData()
56
hooks = hookenv.Hooks()
60
# Print all changes to configuration from previously seen
62
for changed, (prev, cur) in hook_data.conf.items():
63
print('config changed', changed,
64
'previous value', prev,
67
# Get some unit specific bookeeping
68
if not db.get('pkg_key'):
69
key = urllib.urlopen('https://example.com/pkg_key').read()
70
db.set('pkg_key', key)
72
# Directly access all charm config as a mapping.
73
conf = db.getrange('config', True)
75
# Directly access all relation data as a mapping
76
rels = db.getrange('rels', True)
78
if __name__ == '__main__':
83
A more basic integration is via the hook_scope context manager which simply
84
manages transaction scope (and records hook name, and timestamp)::
86
>>> from unitdata import kv
88
>>> with db.hook_scope('install'):
89
... # do work, in transactional scope.
98
Values are automatically json de/serialized to preserve basic typing
99
and complex data struct capabilities (dicts, lists, ints, booleans, etc).
101
Individual values can be manipulated via get/set::
103
>>> kv.set('y', True)
107
# We can set complex values (dicts, lists) as a single key.
108
>>> kv.set('config', {'a': 1, 'b': True'})
110
# Also supports returning dictionaries as a record which
111
# provides attribute access.
112
>>> config = kv.get('config', record=True)
117
Groups of keys can be manipulated with update/getrange::
119
>>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
120
>>> kv.getrange('gui.', strip=True)
123
When updating values, its very helpful to understand which values
124
have actually changed and how have they changed. The storage
125
provides a delta method to provide for this::
127
>>> data = {'debug': True, 'option': 2}
128
>>> delta = kv.delta(data, 'config.')
129
>>> delta.debug.previous
131
>>> delta.debug.current
134
{'debug': (None, True), 'option': (None, 2)}
136
Note the delta method does not persist the actual change, it needs to
137
be explicitly saved via 'update' method::
139
>>> kv.update(data, 'config.')
141
Values modified in the context of a hook scope retain historical values
142
associated to the hookname.
144
>>> with db.hook_scope('config-changed'):
146
>>> db.gethistory('x')
147
[(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
148
(2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
161
__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>'
164
class Storage(object):
165
"""Simple key value database for local unit state within charms.
167
Modifications are automatically committed at hook exit. That's
168
currently regardless of exit code.
170
To support dicts, lists, integer, floats, and booleans values
171
are automatically json encoded/decoded.
173
def __init__(self, path=None):
176
self.db_path = os.path.join(
177
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
178
self.conn = sqlite3.connect('%s' % self.db_path)
179
self.cursor = self.conn.cursor()
192
def _scoped_query(self, stmt, params=None):
197
def get(self, key, default=None, record=False):
200
'select data from kv where key=?', [key]))
201
result = self.cursor.fetchone()
205
return Record(json.loads(result[0]))
206
return json.loads(result[0])
208
def getrange(self, key_prefix, strip=False):
209
stmt = "select key, data from kv where key like '%s%%'" % key_prefix
210
self.cursor.execute(*self._scoped_query(stmt))
211
result = self.cursor.fetchall()
218
(k[len(key_prefix):], json.loads(v)) for k, v in result])
220
def update(self, mapping, prefix=""):
221
for k, v in mapping.items():
222
self.set("%s%s" % (prefix, k), v)
224
def unset(self, key):
225
self.cursor.execute('delete from kv where key=?', [key])
226
if self.revision and self.cursor.rowcount:
228
'insert into kv_revisions values (?, ?, ?)',
229
[key, self.revision, json.dumps('DELETED')])
231
def set(self, key, value):
232
serialized = json.dumps(value)
235
'select data from kv where key=?', [key])
236
exists = self.cursor.fetchone()
238
# Skip mutations to the same value
240
if exists[0] == serialized:
245
'insert into kv (key, data) values (?, ?)',
248
self.cursor.execute('''
251
where key = ?''', [serialized, key])
254
if not self.revision:
258
'select 1 from kv_revisions where key=? and revision=?',
259
[key, self.revision])
260
exists = self.cursor.fetchone()
264
'''insert into kv_revisions (
265
revision, key, data) values (?, ?, ?)''',
266
(self.revision, key, serialized))
274
[serialized, key, self.revision])
278
def delta(self, mapping, prefix):
280
return a delta containing values that have changed.
282
previous = self.getrange(prefix, strip=True)
286
pk = set(previous.keys())
287
ck = set(mapping.keys())
291
for k in ck.difference(pk):
292
delta[k] = Delta(None, mapping[k])
295
for k in pk.difference(ck):
296
delta[k] = Delta(previous[k], None)
299
for k in pk.intersection(ck):
303
delta[k] = Delta(p, c)
307
@contextlib.contextmanager
308
def hook_scope(self, name=""):
309
"""Scope all future interactions to the current hook execution
311
assert not self.revision
313
'insert into hooks (hook, date) values (?, ?)',
314
(name or sys.argv[0],
315
datetime.datetime.utcnow().isoformat()))
316
self.revision = self.cursor.lastrowid
327
def flush(self, save=True):
336
self.cursor.execute('''
337
create table if not exists kv (
342
self.cursor.execute('''
343
create table if not exists kv_revisions (
347
primary key (key, revision)
349
self.cursor.execute('''
350
create table if not exists hooks (
351
version integer primary key autoincrement,
357
def gethistory(self, key, deserialize=False):
360
select kv.revision, kv.key, kv.data, h.hook, h.date
361
from kv_revisions kv,
364
and kv.revision = h.version
366
if deserialize is False:
367
return self.cursor.fetchall()
368
return map(_parse_history, self.cursor.fetchall())
370
def debug(self, fh=sys.stderr):
371
self.cursor.execute('select * from kv')
372
pprint.pprint(self.cursor.fetchall(), stream=fh)
373
self.cursor.execute('select * from kv_revisions')
374
pprint.pprint(self.cursor.fetchall(), stream=fh)
377
def _parse_history(d):
378
return (d[0], d[1], json.loads(d[2]), d[3],
379
datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
382
class HookData(object):
383
"""Simple integration for existing hook exec frameworks.
385
Records all unit information, and stores deltas for processing
390
from charmhelper.core import hookenv, unitdata
392
changes = unitdata.HookData()
394
hooks = hookenv.Hooks()
397
def config_changed():
398
# View all changes to configuration
399
for changed, (prev, cur) in changes.conf.items():
400
print('config changed', changed,
401
'previous value', prev,
402
'current value', cur)
404
# Get some unit specific bookeeping
405
if not db.get('pkg_key'):
406
key = urllib.urlopen('https://example.com/pkg_key').read()
407
db.set('pkg_key', key)
409
if __name__ == '__main__':
419
@contextlib.contextmanager
421
from charmhelpers.core import hookenv
422
hook_name = hookenv.hook_name()
424
with self.kv.hook_scope(hook_name):
425
self._record_charm_version(hookenv.charm_dir())
426
delta_config, delta_relation = self._record_hook(hookenv)
427
yield self.kv, delta_config, delta_relation
429
def _record_charm_version(self, charm_dir):
430
# Record revisions.. charm revisions are meaningless
431
# to charm authors as they don't control the revision.
432
# so logic dependnent on revision is not particularly
433
# useful, however it is useful for debugging analysis.
435
os.path.join(charm_dir, 'revision')).read().strip()
436
charm_rev = charm_rev or '0'
437
revs = self.kv.get('charm_revisions', [])
438
if charm_rev not in revs:
439
revs.append(charm_rev.strip() or '0')
440
self.kv.set('charm_revisions', revs)
442
def _record_hook(self, hookenv):
443
data = hookenv.execution_environment()
444
self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
445
self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
446
self.kv.set('env', data['env'])
447
self.kv.set('unit', data['unit'])
448
self.kv.set('relid', data.get('relid'))
449
return conf_delta, rels_delta
456
def __getattr__(self, k):
459
raise AttributeError(k)
462
class DeltaSet(Record):
467
Delta = collections.namedtuple('Delta', ['previous', 'current'])
added
removed
hooks/charmhelpers/core/unitdata.py
