243
static HYD_status cache_to_cpu_type(int cache_depth, hwloc_obj_type_t * cpu_type)
245
hwloc_obj_t cache_obj, cpu_obj;
246
HYD_status status = HYD_SUCCESS;
250
cache_obj = hwloc_get_root_obj(topology);
251
HYDU_ASSERT(cache_obj, status);
253
while (cache_obj && cache_obj->type != HWLOC_OBJ_CACHE &&
254
cache_obj->attr->cache.depth != cache_depth)
255
cache_obj = cache_obj->first_child;
256
if (cache_obj == NULL) {
257
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "cache level %d not found\n",
261
cpu_obj = hwloc_get_root_obj(topology);
262
HYDU_ASSERT(cpu_obj, status);
264
while (cpu_obj && cpu_obj->type == HWLOC_OBJ_CACHE &&
265
!hwloc_bitmap_isequal(cpu_obj->cpuset, cache_obj->cpuset))
266
cpu_obj = cpu_obj->first_child;
267
if (cpu_obj == NULL) {
268
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
269
"could not find cpu type that maps cache\n");
272
*cpu_type = cpu_obj->type;
282
static HYD_status obj_type_to_map_str(hwloc_obj_type_t type, int cache_depth, char **map)
284
hwloc_obj_type_t cpu_type;
285
HYD_status status = HYD_SUCCESS;
289
if (type == HWLOC_OBJ_MACHINE) {
290
*map = HYDU_strdup("BTCSN");
293
else if (type == HWLOC_OBJ_NODE) {
294
*map = HYDU_strdup("NTCSB");
297
else if (type == HWLOC_OBJ_SOCKET) {
298
*map = HYDU_strdup("STCNB");
301
else if (type == HWLOC_OBJ_CORE) {
302
*map = HYDU_strdup("CTSNB");
305
else if (type == HWLOC_OBJ_PU) {
306
*map = HYDU_strdup("TCSNB");
310
HYDU_ASSERT(type == HWLOC_OBJ_CACHE, status);
312
status = cache_to_cpu_type(cache_depth, &cpu_type);
313
HYDU_ERR_POP(status, "error while mapping cache to cpu object\n");
315
status = obj_type_to_map_str(cpu_type, cache_depth, map);
316
HYDU_ERR_POP(status, "error while mapping object to map string\n");
326
static int balance_obj_idx(int *obj_idx, int *nbobjs_per_parent)
331
for (i = 0; i < MAP_LENGTH - 1; i++) {
332
while (obj_idx[i] >= nbobjs_per_parent[i]) {
333
obj_idx[i] -= nbobjs_per_parent[i];
336
while (obj_idx[i] < 0) {
337
obj_idx[i] += nbobjs_per_parent[i];
341
while (obj_idx[MAP_LENGTH - 1] >= nbobjs_per_parent[MAP_LENGTH - 1]) {
342
obj_idx[MAP_LENGTH - 1] -= nbobjs_per_parent[MAP_LENGTH - 1];
349
167
static HYD_status handle_bitmap_binding(const char *binding, const char *mapping)
351
int i, j, k, idx, bind_count, map_count, cache_depth = 0;
353
hwloc_obj_type_t bind_obj_type;
354
int total_nbobjs[MAP_LENGTH], obj_idx[MAP_LENGTH], nbpu_per_obj[MAP_LENGTH];
355
int nbobjs_per_parent[MAP_LENGTH];
169
int i, j, k, bind_count, map_count, cache_depth = 0, bind_depth = 0, map_depth = 0;
170
int total_map_objs, total_bind_objs, num_pus_in_map_domain, num_pus_in_bind_domain,
172
hwloc_obj_t map_obj, bind_obj, *start_pu;
173
hwloc_cpuset_t *map_domains;
356
174
char *bind_str, *map_str;
357
175
HYD_status status = HYD_SUCCESS;
384
202
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
385
203
"unrecognized binding string \"%s\"\n", binding);
387
bind_obj_type = HWLOC_OBJ_CACHE;
391
/* get the mapping string */
392
if (!strcmp(map_str, "board")) {
394
obj_type_to_map_str(HWLOC_OBJ_MACHINE, 0, &map_str);
396
else if (!strcmp(map_str, "numa")) {
398
obj_type_to_map_str(HWLOC_OBJ_NODE, 0, &map_str);
400
else if (!strcmp(map_str, "socket")) {
402
obj_type_to_map_str(HWLOC_OBJ_SOCKET, 0, &map_str);
404
else if (!strcmp(map_str, "core")) {
406
obj_type_to_map_str(HWLOC_OBJ_CORE, 0, &map_str);
408
else if (!strcmp(map_str, "hwthread")) {
410
obj_type_to_map_str(HWLOC_OBJ_PU, 0, &map_str);
205
bind_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1);
208
/* get the mapping */
209
if (!strcmp(map_str, "board"))
210
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE);
211
else if (!strcmp(map_str, "numa"))
212
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE);
213
else if (!strcmp(map_str, "socket"))
214
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET);
215
else if (!strcmp(map_str, "core"))
216
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE);
217
else if (!strcmp(map_str, "hwthread"))
218
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU);
413
220
cache_depth = parse_cache_string(map_str);
416
obj_type_to_map_str(HWLOC_OBJ_CACHE, cache_depth, &map_str);
419
for (i = 0; i < strlen(map_str); i++) {
420
if (map_str[i] >= 'a' && map_str[i] <= 'z')
421
map_str[i] += ('A' - 'a');
423
/* If any of the characters are not in the form, we
424
* want, return an error */
425
if (map_str[i] != 'T' && map_str[i] != 'C' && map_str[i] != 'S' &&
426
map_str[i] != 'N' && map_str[i] != 'B') {
427
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
428
"unrecognized mapping string \"%s\"\n", mapping);
435
for (i = 0; i < MAP_LENGTH; i++) {
436
if (map_str[i] == 'T')
437
status = get_nbobjs_by_type(HWLOC_OBJ_PU, &total_nbobjs[i], &nbobjs_per_parent[i]);
438
else if (map_str[i] == 'C')
440
get_nbobjs_by_type(HWLOC_OBJ_CORE, &total_nbobjs[i], &nbobjs_per_parent[i]);
441
else if (map_str[i] == 'S')
443
get_nbobjs_by_type(HWLOC_OBJ_SOCKET, &total_nbobjs[i], &nbobjs_per_parent[i]);
444
else if (map_str[i] == 'N')
446
get_nbobjs_by_type(HWLOC_OBJ_NODE, &total_nbobjs[i], &nbobjs_per_parent[i]);
447
else if (map_str[i] == 'B')
449
get_nbobjs_by_type(HWLOC_OBJ_MACHINE, &total_nbobjs[i], &nbobjs_per_parent[i]);
450
HYDU_ERR_POP(status, "unable to get number of objects\n");
452
nbpu_per_obj[i] = HYDT_topo_hwloc_info.num_bitmaps / total_nbobjs[i];
222
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
223
"unrecognized mapping string \"%s\"\n", mapping);
225
map_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1);
229
* Process Affinity Algorithm:
231
* The code below works in 3 stages. The end result is an array of all the possible
232
* binding bitmaps for a system, based on the options specified.
234
* 1. Define all possible mapping "domains" in a system. A mapping domain is a group
235
* of hardware elements found by traversing the topology. Each traversal skips the
236
* number of elements the user specified in the mapping string. The traversal ends
237
* when the next mapping domain == the first mapping domain. Note that if the
238
* mapping string defines a domain that is larger than the system size, we exit
241
* 2. Define the number of possible binding domains within a mapping domain. This
242
* process is similar to step 1, in that we traverse the mapping domain finding
243
* all possible bind combinations, stopping when a duplicate of the first binding
244
* is reached. If a binding is larger (in # of PUs) than the mapping domain,
245
* the number of possible bindings for that domain is 1. In this stage, we also
246
* locate the first PU in each mapping domain for use later during binding.
248
* 3. Create the binding bitmaps. We allocate an array of bitmaps and fill them in
249
* with all possible bindings. The starting PU in each mapping domain is advanced
250
* if and when we wrap around to the beginning of the mapping domains. This ensures
251
* that we do not repeat.
255
/* calculate the number of map domains */
256
total_map_objs = hwloc_get_nbobjs_by_depth(topology, map_depth);
257
num_pus_in_map_domain = (HYDT_topo_hwloc_info.total_num_pus / total_map_objs) * map_count;
258
HYDU_ERR_CHKANDJUMP(status, num_pus_in_map_domain > HYDT_topo_hwloc_info.total_num_pus,
259
HYD_INTERNAL_ERROR, "mapping option \"%s\" larger than total system size\n",
262
/* The number of total_map_domains should be large enough to
263
* contain all contiguous map object collections of length
264
* map_count. For example, if the map object is "socket" and the
265
* map_count is 3, on a system with 4 sockets, the following map
266
* domains should be included: (0,1,2), (3,0,1), (2,3,0), (1,2,3).
267
* We do this by finding how many times we need to replicate the
268
* list of the map objects so that an integral number of map
269
* domains can map to them. In the above case, the list of map
270
* objects is replicated 3 times. */
271
for (i = 1; (i * total_map_objs) % map_count; i++);
272
total_map_domains = (i * total_map_objs) / map_count;
274
/* initialize the map domains */
275
HYDU_MALLOC(map_domains, hwloc_bitmap_t *, total_map_domains * sizeof(hwloc_bitmap_t), status);
276
HYDU_MALLOC(start_pu, hwloc_obj_t *, total_map_domains * sizeof(hwloc_obj_t), status);
278
/* For each map domain, find the next map object (first map object
279
* for the first map domain) and add the following "map_count"
280
* number of contiguous map objects, wrapping to the first one if
281
* needed, to the map domain. Store the first PU in the first map
282
* object of the map domain as "start_pu". This is needed later
283
* for the actual binding. */
285
for (i = 0; i < total_map_domains; i++) {
286
map_domains[i] = hwloc_bitmap_alloc();
287
hwloc_bitmap_zero(map_domains[i]);
289
for (j = 0; j < map_count; j++) {
290
map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj);
291
/* map_obj will be NULL if it reaches the end. call again to wrap around */
293
map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj);
297
hwloc_get_obj_inside_cpuset_by_type(topology, map_obj->cpuset, HWLOC_OBJ_PU, 0);
299
hwloc_bitmap_or(map_domains[i], map_domains[i], map_obj->cpuset);
304
/* Find the possible binding domains is similar to that of map
305
* domains. But if a binding domain is larger (in # of PUs) than
306
* the mapping domain, the number of possible bindings for that
309
/* calculate the number of possible bindings and allocate bitmaps for them */
310
total_bind_objs = hwloc_get_nbobjs_by_depth(topology, bind_depth);
311
num_pus_in_bind_domain = (HYDT_topo_hwloc_info.total_num_pus / total_bind_objs) * bind_count;
313
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
314
for (i = 1; (i * num_pus_in_map_domain) % num_pus_in_bind_domain; i++);
315
HYDT_topo_hwloc_info.num_bitmaps =
316
(i * num_pus_in_map_domain * total_map_domains) / num_pus_in_bind_domain;
319
HYDT_topo_hwloc_info.num_bitmaps = total_map_domains;
322
/* initialize bitmaps */
323
HYDU_MALLOC(HYDT_topo_hwloc_info.bitmap, hwloc_bitmap_t *,
324
HYDT_topo_hwloc_info.num_bitmaps * sizeof(hwloc_bitmap_t), status);
326
for (i = 0; i < HYDT_topo_hwloc_info.num_bitmaps; i++) {
327
HYDT_topo_hwloc_info.bitmap[i] = hwloc_bitmap_alloc();
328
hwloc_bitmap_zero(HYDT_topo_hwloc_info.bitmap[i]);
457
333
while (i < HYDT_topo_hwloc_info.num_bitmaps) {
458
for (j = 0; j < bind_count; j++) {
459
for (idx = 0, k = 0; k < MAP_LENGTH; k++)
460
idx += (obj_idx[k] * nbpu_per_obj[k]);
462
obj = find_obj_containing_pu(bind_obj_type, idx++, cache_depth);
466
hwloc_bitmap_or(HYDT_topo_hwloc_info.bitmap[i], HYDT_topo_hwloc_info.bitmap[i],
469
obj_idx[0] += map_count;
470
balance_obj_idx(obj_idx, nbobjs_per_parent);
334
for (j = 0; j < total_map_domains; j++) {
335
bind_obj = hwloc_get_ancestor_obj_by_depth(topology, bind_depth, start_pu[j]);
337
for (k = 0; k < bind_count; k++) {
338
hwloc_bitmap_or(HYDT_topo_hwloc_info.bitmap[i], HYDT_topo_hwloc_info.bitmap[i],
341
/* if the binding is smaller than the mapping domain, wrap around inside that domain */
342
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
344
hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j],
345
bind_depth, bind_obj);
348
hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j],
349
bind_depth, bind_obj);
352
bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj);
354
bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj);
360
/* advance the starting position for this map domain, if needed */
361
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
362
for (k = 0; k < num_pus_in_bind_domain; k++) {
363
start_pu[j] = hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j],
368
hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j],
369
HWLOC_OBJ_PU, start_pu[j]);
475
/* reset the number of bitmaps available to what we actually set */
476
HYDT_topo_hwloc_info.num_bitmaps = i;
375
/* free temporary memory */
376
HYDU_FREE(map_domains);
479
380
HYDU_FUNC_EXIT();
added
removed
src/pm/hydra/tools/topo/hwloc/topo_hwloc.c
