← Back to branch summary

~ubuntu-branches/ubuntu/vivid/sahara/vivid-proposed

« back to all changes in this revision

Viewing changes to sahara/topology/topology_helper.py

  • Committer: Package Import Robot
  • Author(s): Thomas Goirand
  • Date: 2014-09-24 16:34:46 UTC
  • Revision ID: package-import@ubuntu.com-20140924163446-8gu3zscu5e3n9lr2
Tags: upstream-2014.2~b3
Import upstream version 2014.2~b3

Show diffs side-by-side

added added

removed removed

Lines of Context:
sahara/topology/topology_helper.py
 
1
# Copyright (c) 2013 Mirantis Inc.
 
2
#
 
3
# Licensed under the Apache License, Version 2.0 (the "License");
 
4
# you may not use this file except in compliance with the License.
 
5
# You may obtain a copy of the License at
 
6
#
 
7
#    http://www.apache.org/licenses/LICENSE-2.0
 
8
#
 
9
# Unless required by applicable law or agreed to in writing, software
 
10
# distributed under the License is distributed on an "AS IS" BASIS,
 
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 
12
# implied.
 
13
# See the License for the specific language governing permissions and
 
14
# limitations under the License.
 
15
 
 
16
import hashlib
 
17
 
 
18
from oslo.config import cfg
 
19
 
 
20
from sahara import context
 
21
from sahara import exceptions as ex
 
22
from sahara.i18n import _
 
23
from sahara.i18n import _LI
 
24
from sahara.openstack.common import log
 
25
from sahara.utils.openstack import nova
 
26
from sahara.utils import xmlutils as x
 
27
 
 
28
 
 
29
TOPOLOGY_CONFIG = {
 
30
    "topology.node.switch.mapping.impl":
 
31
    "org.apache.hadoop.net.ScriptBasedMapping",
 
32
    "topology.script.file.name":
 
33
    "/etc/hadoop/topology.sh"
 
34
}
 
35
 
 
36
LOG = log.getLogger(__name__)
 
37
 
 
38
opts = [
 
39
    cfg.BoolOpt('enable_data_locality',
 
40
                default=False,
 
41
                help="""Enables data locality for hadoop cluster.
 
42
                 Also enables data locality for Swift used by hadoop.
 
43
                 If enabled, 'compute_topology' and 'swift_topology'
 
44
                 configuration parameters should point to OpenStack and Swift
 
45
                 topology correspondingly."""),
 
46
    cfg.BoolOpt('enable_hypervisor_awareness',
 
47
                default=True,
 
48
                help="""Enables four-level topology for data locality.
 
49
                Works only if corresponding plugin supports such mode."""),
 
50
    cfg.StrOpt('compute_topology_file',
 
51
               default='etc/sahara/compute.topology',
 
52
               help="""File with nova compute topology.
 
53
                It should contain mapping between nova computes and racks.
 
54
                File format:
 
55
                compute1 /rack1
 
56
                compute2 /rack2
 
57
                compute3 /rack2"""),
 
58
    cfg.StrOpt('swift_topology_file',
 
59
               default='etc/sahara/swift.topology',
 
60
               help="""File with Swift topology.
 
61
                It should contain mapping between Swift nodes and racks.
 
62
                File format:
 
63
                node1 /rack1
 
64
                node2 /rack2
 
65
                node3 /rack2""")
 
66
]
 
67
 
 
68
CONF = cfg.CONF
 
69
CONF.register_opts(opts)
 
70
 
 
71
 
 
72
def _read_swift_topology():
 
73
    LOG.debug("Reading Swift nodes topology from %s", CONF.swift_topology_file)
 
74
    topology = {}
 
75
    try:
 
76
        with open(CONF.swift_topology_file) as f:
 
77
            for line in f:
 
78
                line = line.strip()
 
79
                if not line:
 
80
                    continue
 
81
                (host, path) = line.split()
 
82
                topology[host] = path
 
83
    except IOError:
 
84
        LOG.debug("Unable to read Swift nodes topology from %s",
 
85
                  CONF.swift_topology_file)
 
86
        return {}
 
87
 
 
88
    return topology
 
89
 
 
90
 
 
91
def _read_compute_topology():
 
92
    LOG.debug("Reading compute nodes topology from %s",
 
93
              CONF.compute_topology_file)
 
94
    ctx = context.ctx()
 
95
    tenant_id = str(ctx.tenant_id)
 
96
    topology = {}
 
97
    try:
 
98
        with open(CONF.compute_topology_file) as f:
 
99
            for line in f:
 
100
                line = line.strip()
 
101
                if not line:
 
102
                    continue
 
103
                (host, path) = line.split()
 
104
                # Calulating host id based on tenant id and host
 
105
                # using the same algorithm as in nova
 
106
                # see nova/api/openstack/compute/views/servers.py
 
107
                # def _get_host_id(instance):
 
108
                sha_hash = hashlib.sha224(tenant_id + host)
 
109
                topology[sha_hash.hexdigest()] = path
 
110
    except IOError:
 
111
        raise ex.NotFoundException(
 
112
            CONF.compute_topology_file,
 
113
            _("Unable to find file %s with compute topology"))
 
114
    return topology
 
115
 
 
116
 
 
117
def generate_topology_map(cluster, is_node_awareness):
 
118
    mapping = _read_compute_topology()
 
119
    nova_client = nova.client()
 
120
    topology_mapping = {}
 
121
    for ng in cluster.node_groups:
 
122
        for i in ng.instances:
 
123
            # TODO(alazarev) get all servers info with one request
 
124
            ni = nova_client.servers.get(i.instance_id)
 
125
            hostId = ni.hostId
 
126
            if hostId not in mapping:
 
127
                raise ex.NotFoundException(
 
128
                    i.instance_id,
 
129
                    _("Was not able to find compute node topology for VM %s"))
 
130
            rack = mapping[hostId]
 
131
            if is_node_awareness:
 
132
                rack += "/" + hostId
 
133
 
 
134
            topology_mapping[i.instance_name] = rack
 
135
            topology_mapping[i.management_ip] = rack
 
136
            topology_mapping[i.internal_ip] = rack
 
137
 
 
138
    topology_mapping.update(_read_swift_topology())
 
139
 
 
140
    return topology_mapping
 
141
 
 
142
 
 
143
def vm_awareness_core_config():
 
144
    c = x.load_hadoop_xml_defaults('topology/resources/core-template.xml')
 
145
    result = [cfg for cfg in c if cfg['value']]
 
146
 
 
147
    if not CONF.enable_hypervisor_awareness:
 
148
        # not leveraging 4-layer approach so override template value
 
149
        param = next((prop for prop in result
 
150
                      if prop['name'] == 'net.topology.impl'), None)
 
151
        if param:
 
152
            param['value'] = 'org.apache.hadoop.net.NetworkTopology'
 
153
 
 
154
    LOG.info(_LI("Vm awareness will add following configs in core-site "
 
155
             "params: %s"), result)
 
156
    return result
 
157
 
 
158
 
 
159
def vm_awareness_mapred_config():
 
160
    c = x.load_hadoop_xml_defaults('topology/resources/mapred-template.xml')
 
161
    result = [cfg for cfg in c if cfg['value']]
 
162
    LOG.info(_LI("Vm awareness will add following configs in map-red "
 
163
             "params: %s"), result)
 
164
    return result
 
165
 
 
166
 
 
167
def vm_awareness_all_config():
 
168
    return vm_awareness_core_config() + vm_awareness_mapred_config()