← Back to branch summary

~ubuntu-branches/ubuntu/vivid/sahara/vivid-proposed

« back to all changes in this revision

Viewing changes to sahara/tests/integration/tests/gating/test_hdp2_gating.py

  • Committer: Package Import Robot
  • Author(s): Thomas Goirand
  • Date: 2014-09-24 16:34:46 UTC
  • Revision ID: package-import@ubuntu.com-20140924163446-8gu3zscu5e3n9lr2
Tags: upstream-2014.2~b3
Import upstream version 2014.2~b3

Show diffs side-by-side

added added

removed removed

Lines of Context:
sahara/tests/integration/tests/gating/test_hdp2_gating.py
 
1
# Copyright (c) 2014 Mirantis Inc.
 
2
#
 
3
# Licensed under the Apache License, Version 2.0 (the "License");
 
4
# you may not use this file except in compliance with the License.
 
5
# You may obtain a copy of the License at
 
6
#
 
7
#    http://www.apache.org/licenses/LICENSE-2.0
 
8
#
 
9
# Unless required by applicable law or agreed to in writing, software
 
10
# distributed under the License is distributed on an "AS IS" BASIS,
 
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 
12
# implied.
 
13
# See the License for the specific language governing permissions and
 
14
# limitations under the License.
 
15
 
 
16
from testtools import testcase
 
17
 
 
18
from sahara.tests.integration.configs import config as cfg
 
19
from sahara.tests.integration.tests import base as b
 
20
from sahara.tests.integration.tests import edp
 
21
from sahara.tests.integration.tests import scaling
 
22
from sahara.tests.integration.tests import swift
 
23
from sahara.utils import edp as utils_edp
 
24
 
 
25
 
 
26
class HDP2GatingTest(swift.SwiftTest, scaling.ScalingTest,
 
27
                     edp.EDPTest):
 
28
 
 
29
    config = cfg.ITConfig().hdp2_config
 
30
    SKIP_EDP_TEST = config.SKIP_EDP_TEST
 
31
    SKIP_SWIFT_TEST = config.SKIP_SWIFT_TEST
 
32
    SKIP_SCALING_TEST = config.SKIP_SCALING_TEST
 
33
 
 
34
    def setUp(self):
 
35
        super(HDP2GatingTest, self).setUp()
 
36
        self.cluster_id = None
 
37
        self.cluster_template_id = None
 
38
        self.ng_template_ids = []
 
39
 
 
40
    def _prepare_test(self):
 
41
        self.hdp2_config = cfg.ITConfig().hdp2_config
 
42
        self.floating_ip_pool = self.common_config.FLOATING_IP_POOL
 
43
        self.internal_neutron_net = None
 
44
        if self.common_config.NEUTRON_ENABLED:
 
45
            self.internal_neutron_net = self.get_internal_neutron_net_id()
 
46
            self.floating_ip_pool = (
 
47
                self.get_floating_ip_pool_id_for_neutron_net())
 
48
 
 
49
        self.hdp2_config.IMAGE_ID, self.hdp2_config.SSH_USERNAME = (
 
50
            self.get_image_id_and_ssh_username(self.hdp2_config))
 
51
 
 
52
    @b.errormsg("Failure while 'rm-nn' node group template creation: ")
 
53
    def _create_rm_nn_ng_template(self):
 
54
        template = {
 
55
            'name': 'test-node-group-template-hdp2-rm-nn',
 
56
            'plugin_config': self.hdp2_config,
 
57
            'description': 'test node group template for HDP plugin',
 
58
            'node_processes': self.hdp2_config.MASTER_NODE_PROCESSES,
 
59
            'floating_ip_pool': self.floating_ip_pool,
 
60
            'node_configs': {}
 
61
        }
 
62
        self.ng_tmpl_rm_nn_id = self.create_node_group_template(**template)
 
63
        self.ng_template_ids.append(self.ng_tmpl_rm_nn_id)
 
64
 
 
65
    @b.errormsg("Failure while 'nm-dn' node group template creation: ")
 
66
    def _create_nm_dn_ng_template(self):
 
67
        template = {
 
68
            'name': 'test-node-group-template-hdp2-nm-dn',
 
69
            'plugin_config': self.hdp2_config,
 
70
            'description': 'test node group template for HDP plugin',
 
71
            'node_processes': self.hdp2_config.WORKER_NODE_PROCESSES,
 
72
            'floating_ip_pool': self.floating_ip_pool,
 
73
            'node_configs': {}
 
74
        }
 
75
        self.ng_tmpl_nm_dn_id = self.create_node_group_template(**template)
 
76
        self.ng_template_ids.append(self.ng_tmpl_nm_dn_id)
 
77
 
 
78
    @b.errormsg("Failure while cluster template creation: ")
 
79
    def _create_cluster_template(self):
 
80
        template = {
 
81
            'name': 'test-cluster-template-hdp2',
 
82
            'plugin_config': self.hdp2_config,
 
83
            'description': 'test cluster template for HDP plugin',
 
84
            'cluster_configs': {
 
85
                'YARN': {
 
86
                    'yarn.log-aggregation-enable': False
 
87
                }
 
88
            },
 
89
            'node_groups': [
 
90
                {
 
91
                    'name': 'master-node-dn',
 
92
                    'node_group_template_id': self.ng_tmpl_rm_nn_id,
 
93
                    'count': 1
 
94
                },
 
95
                {
 
96
                    'name': 'worker-node-nm',
 
97
                    'node_group_template_id': self.ng_tmpl_nm_dn_id,
 
98
                    'count': 3
 
99
                }
 
100
            ],
 
101
            'net_id': self.internal_neutron_net
 
102
        }
 
103
        self.cluster_template_id = self.create_cluster_template(**template)
 
104
 
 
105
    @b.errormsg("Failure while cluster creation: ")
 
106
    def _create_cluster(self):
 
107
        cluster_name = '%s-%s-v2' % (self.common_config.CLUSTER_NAME,
 
108
                                     self.hdp2_config.PLUGIN_NAME)
 
109
        cluster = {
 
110
            'name': cluster_name,
 
111
            'plugin_config': self.hdp2_config,
 
112
            'cluster_template_id': self.cluster_template_id,
 
113
            'description': 'test cluster',
 
114
            'cluster_configs': {}
 
115
        }
 
116
        self.create_cluster(**cluster)
 
117
        self.cluster_info = self.get_cluster_info(self.hdp2_config)
 
118
        self.await_active_workers_for_namenode(self.cluster_info['node_info'],
 
119
                                               self.hdp2_config)
 
120
 
 
121
    @b.errormsg("Failure during check of Swift availability: ")
 
122
    def _check_swift(self):
 
123
        self.check_swift_availability(self.cluster_info)
 
124
 
 
125
    @b.errormsg("Failure while EDP testing: ")
 
126
    def _check_edp(self):
 
127
        self._edp_test()
 
128
 
 
129
    def _edp_test(self):
 
130
        # check pig
 
131
        pig_job = self.edp_info.read_pig_example_script()
 
132
        pig_lib = self.edp_info.read_pig_example_jar()
 
133
 
 
134
        self.edp_testing(job_type=utils_edp.JOB_TYPE_PIG,
 
135
                         job_data_list=[{'pig': pig_job}],
 
136
                         lib_data_list=[{'jar': pig_lib}],
 
137
                         swift_binaries=True,
 
138
                         hdfs_local_output=True)
 
139
 
 
140
        # check mapreduce
 
141
        mapreduce_jar = self.edp_info.read_mapreduce_example_jar()
 
142
        mapreduce_configs = self.edp_info.mapreduce_example_configs()
 
143
        self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE,
 
144
                         job_data_list=[],
 
145
                         lib_data_list=[{'jar': mapreduce_jar}],
 
146
                         configs=mapreduce_configs,
 
147
                         swift_binaries=True,
 
148
                         hdfs_local_output=True)
 
149
 
 
150
        # check mapreduce streaming
 
151
        self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE_STREAMING,
 
152
                         job_data_list=[],
 
153
                         lib_data_list=[],
 
154
                         configs=self.edp_info.mapreduce_streaming_configs())
 
155
 
 
156
        # check java
 
157
        java_jar = self.edp_info.read_java_example_lib(2)
 
158
        java_configs = self.edp_info.java_example_configs(2)
 
159
        self.edp_testing(utils_edp.JOB_TYPE_JAVA,
 
160
                         job_data_list=[],
 
161
                         lib_data_list=[{'jar': java_jar}],
 
162
                         configs=java_configs)
 
163
 
 
164
    @b.errormsg("Failure while cluster scaling: ")
 
165
    def _check_scaling(self):
 
166
        datanode_count_after_resizing = (
 
167
            self.cluster_info['node_info']['datanode_count']
 
168
            + self.hdp_config.SCALE_EXISTING_NG_COUNT)
 
169
        change_list = [
 
170
            {
 
171
                'operation': 'resize',
 
172
                'info': ['worker-node-nm',
 
173
                         datanode_count_after_resizing]
 
174
            },
 
175
            {
 
176
                'operation': 'add',
 
177
                'info': ['new-worker-node-tt-dn',
 
178
                         self.hdp2_config.SCALE_NEW_NG_COUNT,
 
179
                         '%s' % self.ng_tmpl_nm_dn_id]
 
180
            }
 
181
        ]
 
182
 
 
183
        self.cluster_info = self.cluster_scaling(self.cluster_info,
 
184
                                                 change_list)
 
185
        self.await_active_workers_for_namenode(self.cluster_info['node_info'],
 
186
                                               self.hdp2_config)
 
187
 
 
188
    @b.errormsg(
 
189
        "Failure during check of Swift availability after cluster scaling: ")
 
190
    def _check_swift_after_scaling(self):
 
191
        self.check_swift_availability(self.cluster_info)
 
192
 
 
193
    @b.errormsg("Failure while EDP testing after cluster scaling: ")
 
194
    def _check_edp_after_scaling(self):
 
195
        self._edp_test()
 
196
 
 
197
    @testcase.attr('hdp2')
 
198
    @testcase.skipIf(config.SKIP_ALL_TESTS_FOR_PLUGIN,
 
199
                     'All tests for HDP2 plugin were skipped')
 
200
    def test_hdp2_plugin_gating(self):
 
201
        self._prepare_test()
 
202
        self._create_rm_nn_ng_template()
 
203
        self._create_nm_dn_ng_template()
 
204
        self._create_cluster_template()
 
205
        self._create_cluster()
 
206
 
 
207
        self._check_swift()
 
208
        self._check_edp()
 
209
 
 
210
        if not self.hdp2_config.SKIP_SCALING_TEST:
 
211
            self._check_scaling()
 
212
            self._check_swift_after_scaling()
 
213
            self._check_edp_after_scaling()
 
214
 
 
215
    def tearDown(self):
 
216
        self.delete_objects(self.cluster_id, self.cluster_template_id,
 
217
                            self.ng_template_ids)
 
218
        super(HDP2GatingTest, self).tearDown()