1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
# Copyright 2012, 2013 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
from charmworld.testing import (
factory,
MongoTestBase,
)
from charmworld.migrations import migrate
from charmworld.migrations.migrate import (
DataStore,
Versions,
)
from charmworld.migrations.versions.tests.data import migration_027 as mdata
from charmworld.models import CharmSource
from pyelasticsearch.exceptions import ElasticHttpError
import logging
import mock
class MigrationTestBase(MongoTestBase):
def setUp(self):
super(MigrationTestBase, self).setUp()
self.versions = Versions(migrate.get_migration_path())
self.use_context(mock.patch.object(migrate, 'configure_logging'))
class TestExodus026(MigrationTestBase):
version = 26
exodus = '026_convert_comma_constraints.py'
def setUp(self):
super(TestExodus026, self).setUp()
self.use_index_client()
self.source = CharmSource.from_request(self)
self.source_db = self.source.collection.database
self.exodus_update = self.versions.get_exodus(
self.exodus)
self.target = self.versions.get_pending_source(
self.source, self.version)
self.target_db = self.target.collection.database
# Test converting service constraints to space-separated.
def test_converts_constraints(self):
services = dict(
svc1=dict(constraints='cpu-cores=4,mem=2G,arch=i686'),
svc2=dict(constraints='cpu-cores=44 arch=i386 mem=9G'),
)
self.assertEqual(0, self.source_db.bundles.count())
self.assertEqual(0, self.target_db.bundles.count())
_, bundle_data = factory.make_bundle(
self.source_db, services=services)
# Add in 'inherits' property to exercise its deletion.
bundle_data['data']['inherits'] = 'super-charm'
self.source_db.bundles.save(bundle_data)
self.assertEqual(1, self.source_db.bundles.count())
self.assertEqual(1, self.target_db.bundles.count())
# Run the exodus.
self.exodus_update(self.source, self.target, self.version)
# Ensure there is still just the one bundle.
self.assertEqual(1, self.target_db.bundles.count())
converted = self.target_db.bundles.find_one()
services = converted['data']['services']
# Constraints are now space-separated and sorted.
self.assertEqual(services['svc1']['constraints'],
'arch=i686 cpu-cores=4 mem=2G')
self.assertEqual(services['svc2']['constraints'],
'arch=i386 cpu-cores=44 mem=9G')
def test_skips_invalid_constraints(self):
services = dict(
# 'cpu' is an obsolete constraint name.
svc1=dict(constraints='cpu=4,mem=2G,arch=i686'),
)
self.assertEqual(0, self.source_db.bundles.count())
self.assertEqual(0, self.target_db.bundles.count())
_, bundle_data = factory.make_bundle(
self.source_db, services=services)
self.source_db.bundles.save(bundle_data)
self.assertEqual(1, self.source_db.bundles.count())
self.assertEqual(1, self.target_db.bundles.count())
# Run the exodus.
log = logging.getLogger('charm.exodus026')
with mock.patch.object(log, 'warning') as mock_warn:
self.exodus_update(self.source, self.target, self.version)
# There are now no bundles, as the invalid one was removed from the
# target database.
self.assertEqual(0, self.target_db.bundles.count())
expected_warnings = [
'Bundle has invalid constraint. Skipping {}'.format(
bundle_data['_id']),
]
mock_warn.assert_has_calls(
[mock.call(msg) for msg in expected_warnings])
def test_no_promulgated(self):
services = dict(
# 'cpu' is an obsolete constraint name.
svc1=dict(constraints='cpu-cores=4,mem=2G,arch=i686'),
)
self.assertEqual(0, self.source_db.bundles.count())
self.assertEqual(0, self.target_db.bundles.count())
_, bundle_data = factory.make_bundle(
self.source_db, services=services)
del bundle_data['promulgated']
self.source_db.bundles.save(bundle_data)
self.target_db.bundles.save(bundle_data)
self.assertEqual(1, self.source_db.bundles.count())
self.assertEqual(1, self.target_db.bundles.count())
# Run the exodus.
self.exodus_update(self.source, self.target, self.version)
# Ensure there is still just the one bundle.
self.assertEqual(1, self.target_db.bundles.count())
converted = self.target_db.bundles.find_one()
services = converted['data']['services']
# Constraints are now space-separated and sorted.
self.assertEqual(services['svc1']['constraints'],
'arch=i686 cpu-cores=4 mem=2G')
class TestExodusRestrictions(MigrationTestBase):
def test_exodus_restrictions(self):
"""Ensure that deploy-from-scratch does not violate exodus rules."""
self.versions._check_exodus(self.versions.list_pending(0))
class TestUseNgrams027(MigrationTestBase):
version = 27
module_name = '027_use_ngrams.py'
def setUp(self):
super(TestUseNgrams027, self).setUp()
self.use_index_client()
self.data_store = DataStore(self.db)
self.charm_source = CharmSource(self.db, self.index_client)
def exists_in_index(self, id_):
return self.index_client.get(id_) is not None
def sync_index(self):
charm_ids = self.charm_source.sync_index()
while True:
try:
next(charm_ids)
except StopIteration:
break
def put_old_mapping(self):
# Delete the temp_index.
self.index_client.delete_index()
# Create an empty index.
self.index_client._client.create_index(self.index_client.index_name)
self.index_client.wait_for_green_status()
# Put up the old mappings.
for type_ in ('charm', 'bundle'):
self.index_client._client.put_mapping(
self.index_client.index_name,
type_,
getattr(mdata, 'old_{}_mapping'.format(type_)))
def test_index_updated(self):
"""Ensure that running upgrade reindexes charms."""
self.put_old_mapping()
self.versions.ensure_initialized(self.data_store, True)
## Pretend upgrades through 26 are complete.
self.data_store.update_version(26)
charm = factory.get_charm_json()
self.db.charms.insert(charm)
self.assertFalse(self.exists_in_index(charm['_id']))
self.sync_index()
self.assertTrue(self.exists_in_index(charm['_id']))
self.versions.upgrade(self.data_store, self.index_client, True)
self.assertEquals(self.data_store.current_version, self.version)
self.assertTrue(self.exists_in_index(charm['_id']))
def test_exception_raised(self):
"""Putting new mapping without creating the filter and analyzer
raises the expected exception.
"""
self.versions.ensure_initialized(self.data_store, True)
## Pretend upgrades through 26 are complete.
self.data_store.update_version(26)
self.put_old_mapping()
charm = factory.get_charm_json()
self.assertFalse(self.exists_in_index(charm['_id']))
self.index_client.index_charm(charm)
self.assertTrue(self.exists_in_index(charm['_id']))
with self.assertRaises(ElasticHttpError) as e:
self.index_client.put_mapping()
self.assertEquals(e.status_code, 400)
self.assertEquals(e.error,
u'MapperParsingException[Analyzer [n3_20grams] '
u'not found for field [ngrams]]')
|