← Back to branch summary

~ubuntu-branches/ubuntu/precise/slurm-llnl/precise

« back to all changes in this revision

Viewing changes to src/slurmctld/node_mgr.c

  • Committer: Bazaar Package Importer
  • Author(s): Gennaro Oliva
  • Date: 2011-04-08 11:21:17 UTC
  • mfrom: (3.3.16 sid)
  • Revision ID: james.westby@ubuntu.com-20110408112117-nfnyq9dtm55hqoaw
Tags: 2.2.4-1
http://bugs.debian.org/575822
* New upstream releases 
* Cleaning spare file and directories, not belonging to the sources
  generated by the building process and not removed by distclean.
  Added debian/clean with spare files and rm -rf inside debian/rules
  for directories.
* Added new packages libslurm-perl, libslurmdb-perl, slurm-llnl-torque
  (Closes: #575822) thanks to Julien Blache

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/slurmctld/node_mgr.c
5
5
 *      configuration list (config_list)
6
6
 *****************************************************************************
7
7
 *  Copyright (C) 2002-2007 The Regents of the University of California.
8
 
 *  Copyright (C) 2008-2009 Lawrence Livermore National Security.
 
8
 *  Copyright (C) 2008-2010 Lawrence Livermore National Security.
9
9
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
10
10
 *  Written by Morris Jette <jette1@llnl.gov>, et. al.
11
11
 *  CODE-OCEC-09-009. All rights reserved.
55
55
#include <sys/stat.h>
56
56
#include <fcntl.h>
57
57
 
 
58
#include "src/common/gres.h"
58
59
#include "src/common/hostlist.h"
59
60
#include "src/common/macros.h"
60
61
#include "src/common/pack.h"
78
79
#define MAX_RETRIES     10
79
80
 
80
81
/* Change NODE_STATE_VERSION value when changing the state save format */
81
 
#define NODE_STATE_VERSION      "VER003"
 
82
#define NODE_STATE_VERSION      "VER004"
 
83
#define NODE_2_2_STATE_VERSION  "VER004"        /* SLURM version 2.2 */
 
84
#define NODE_2_1_STATE_VERSION  "VER003"        /* SLURM version 2.1 */
82
85
 
83
86
/* Global variables */
84
87
bitstr_t *avail_node_bitmap = NULL;     /* bitmap of available nodes */
 
88
bitstr_t *cg_node_bitmap    = NULL;     /* bitmap of completing nodes */
85
89
bitstr_t *idle_node_bitmap  = NULL;     /* bitmap of idle nodes */
86
90
bitstr_t *power_node_bitmap = NULL;     /* bitmap of powered down nodes */
87
91
bitstr_t *share_node_bitmap = NULL;     /* bitmap of sharable nodes */
94
98
static void     _node_did_resp(struct node_record *node_ptr);
95
99
static bool     _node_is_hidden(struct node_record *node_ptr);
96
100
static int      _open_node_state_file(char **state_file);
97
 
static void     _pack_node (struct node_record *dump_node_ptr, Buf buffer);
 
101
static void     _pack_node (struct node_record *dump_node_ptr, Buf buffer,
 
102
                            uint16_t protocol_version);
98
103
static void     _sync_bitmaps(struct node_record *node_ptr, int job_count);
99
104
static void     _update_config_ptr(bitstr_t *bitmap,
100
105
                                struct config_record *config_ptr);
101
106
static int      _update_node_features(char *node_names, char *features);
 
107
static int      _update_node_gres(char *node_names, char *gres);
102
108
static int      _update_node_weight(char *node_names, uint32_t weight);
103
109
static bool     _valid_node_state_change(uint16_t old, uint16_t new);
104
110
#ifndef HAVE_FRONT_END
113
119
        static int high_buffer_size = (1024 * 1024);
114
120
        int error_code = 0, inx, log_fd;
115
121
        char *old_file, *new_file, *reg_file;
 
122
        struct node_record *node_ptr;
116
123
        /* Locks: Read config and node */
117
124
        slurmctld_lock_t node_read_lock = { READ_LOCK, NO_LOCK, READ_LOCK,
118
 
                                                NO_LOCK };
 
125
                                            NO_LOCK };
119
126
        Buf buffer = init_buf(high_buffer_size);
120
127
        DEF_TIMERS;
121
128
 
126
133
 
127
134
        /* write node records to buffer */
128
135
        lock_slurmctld (node_read_lock);
129
 
        for (inx = 0; inx < node_record_count; inx++) {
130
 
                xassert (node_record_table_ptr[inx].magic == NODE_MAGIC);
131
 
                xassert (node_record_table_ptr[inx].config_ptr->magic ==
132
 
                         CONFIG_MAGIC);
133
 
 
134
 
                _dump_node_state (&node_record_table_ptr[inx], buffer);
 
136
        for (inx = 0, node_ptr = node_record_table_ptr; inx < node_record_count;
 
137
             inx++, node_ptr++) {
 
138
                xassert (node_ptr->magic == NODE_MAGIC);
 
139
                xassert (node_ptr->config_ptr->magic == CONFIG_MAGIC);
 
140
                _dump_node_state (node_ptr, buffer);
135
141
        }
136
 
        unlock_slurmctld (node_read_lock);
137
142
 
138
 
        /* write the buffer to file */
139
143
        old_file = xstrdup (slurmctld_conf.state_save_location);
140
144
        xstrcat (old_file, "/node_state.old");
141
145
        reg_file = xstrdup (slurmctld_conf.state_save_location);
142
146
        xstrcat (reg_file, "/node_state");
143
147
        new_file = xstrdup (slurmctld_conf.state_save_location);
144
148
        xstrcat (new_file, "/node_state.new");
145
 
        lock_state_files ();
 
149
        unlock_slurmctld (node_read_lock);
 
150
 
 
151
        /* write the buffer to file */
 
152
        lock_state_files();
146
153
        log_fd = creat (new_file, 0600);
147
154
        if (log_fd < 0) {
148
 
                error ("Can't save state, error creating file %s %m",
149
 
                       new_file);
 
155
                error ("Can't save state, error creating file %s %m", new_file);
150
156
                error_code = errno;
151
157
        } else {
152
158
                int pos = 0, nwrite = get_buf_offset(buffer), amount, rc;
200
206
{
201
207
        packstr (dump_node_ptr->name, buffer);
202
208
        packstr (dump_node_ptr->reason, buffer);
203
 
        packstr (dump_node_ptr->config_ptr->feature, buffer);
 
209
        packstr (dump_node_ptr->features, buffer);
 
210
        packstr (dump_node_ptr->gres, buffer);
204
211
        pack16  (dump_node_ptr->node_state, buffer);
205
212
        pack16  (dump_node_ptr->cpus, buffer);
206
213
        pack16  (dump_node_ptr->sockets, buffer);
208
215
        pack16  (dump_node_ptr->threads, buffer);
209
216
        pack32  (dump_node_ptr->real_memory, buffer);
210
217
        pack32  (dump_node_ptr->tmp_disk, buffer);
 
218
        pack32  (dump_node_ptr->reason_uid, buffer);
 
219
        pack_time(dump_node_ptr->reason_time, buffer);
 
220
        (void) gres_plugin_node_state_pack(dump_node_ptr->gres_list, buffer,
 
221
                                           dump_node_ptr->name);
211
222
}
212
223
 
213
224
 
244
255
 * load_all_node_state - Load the node state from file, recover on slurmctld
245
256
 *      restart. Execute this after loading the configuration file data.
246
257
 *      Data goes into common storage.
247
 
 * IN state_only - if true, overwrite only node state, features and reason
 
258
 * IN state_only - if true, overwrite only node state and reason
248
259
 *      Use this to overwrite the "UNKNOWN state typically used in slurm.conf
249
260
 * RET 0 or error code
250
261
 * NOTE: READ lock_slurmctld config before entry
251
262
 */
252
263
extern int load_all_node_state ( bool state_only )
253
264
{
254
 
        char *node_name, *reason = NULL, *data = NULL, *state_file, *features;
 
265
        char *node_name = NULL, *reason = NULL, *data = NULL, *state_file;
 
266
        char *features = NULL, *gres = NULL;
255
267
        int data_allocated, data_read = 0, error_code = 0, node_cnt = 0;
256
268
        uint16_t node_state;
257
269
        uint16_t cpus = 1, sockets = 1, cores = 1, threads = 1;
258
270
        uint32_t real_memory, tmp_disk, data_size = 0, name_len;
 
271
        uint32_t reason_uid = NO_VAL;
 
272
        time_t reason_time = 0;
 
273
        List gres_list = NULL;
259
274
        struct node_record *node_ptr;
260
275
        int state_fd;
261
276
        time_t time_stamp, now = time(NULL);
262
277
        Buf buffer;
263
278
        char *ver_str = NULL;
264
279
        hostset_t hs = NULL;
265
 
        slurm_ctl_conf_t *conf = slurm_conf_lock();
266
280
        bool power_save_mode = false;
 
281
        uint16_t protocol_version = (uint16_t)NO_VAL;
267
282
 
268
 
        if (conf->suspend_program && conf->resume_program)
 
283
        if (slurmctld_conf.suspend_program && slurmctld_conf.resume_program)
269
284
                power_save_mode = true;
270
 
        slurm_conf_unlock();
271
285
 
272
286
        /* read the file */
273
 
        state_file = xstrdup (slurmctld_conf.state_save_location);
274
 
        xstrcat (state_file, "/node_state");
275
287
        lock_state_files ();
276
288
        state_fd = _open_node_state_file(&state_file);
277
289
        if (state_fd < 0) {
306
318
 
307
319
        safe_unpackstr_xmalloc( &ver_str, &name_len, buffer);
308
320
        debug3("Version string in node_state header is %s", ver_str);
309
 
        if ((!ver_str) || (strcmp(ver_str, NODE_STATE_VERSION) != 0)) {
 
321
        if(ver_str) {
 
322
                if(!strcmp(ver_str, NODE_STATE_VERSION)) {
 
323
                        protocol_version = SLURM_PROTOCOL_VERSION;
 
324
                } else if(!strcmp(ver_str, NODE_2_1_STATE_VERSION)) {
 
325
                        protocol_version = SLURM_2_1_PROTOCOL_VERSION;
 
326
                }
 
327
        }
 
328
 
 
329
        if (protocol_version == (uint16_t)NO_VAL) {
310
330
                error("*****************************************************");
311
331
                error("Can not recover node state, data version incompatible");
312
332
                error("*****************************************************");
320
340
 
321
341
        while (remaining_buf (buffer) > 0) {
322
342
                uint16_t base_state;
323
 
                safe_unpackstr_xmalloc (&node_name, &name_len, buffer);
324
 
                safe_unpackstr_xmalloc (&reason,    &name_len, buffer);
325
 
                safe_unpackstr_xmalloc (&features,  &name_len, buffer);
326
 
                safe_unpack16 (&node_state,  buffer);
327
 
                safe_unpack16 (&cpus,        buffer);
328
 
                safe_unpack16 (&sockets,     buffer);
329
 
                safe_unpack16 (&cores,       buffer);
330
 
                safe_unpack16 (&threads,     buffer);
331
 
                safe_unpack32 (&real_memory, buffer);
332
 
                safe_unpack32 (&tmp_disk,    buffer);
333
 
                base_state = node_state & NODE_STATE_BASE;
 
343
                if(protocol_version >= SLURM_2_2_PROTOCOL_VERSION) {
 
344
                        safe_unpackstr_xmalloc (&node_name, &name_len, buffer);
 
345
                        safe_unpackstr_xmalloc (&reason,    &name_len, buffer);
 
346
                        safe_unpackstr_xmalloc (&features,  &name_len, buffer);
 
347
                        safe_unpackstr_xmalloc (&gres,      &name_len, buffer);
 
348
                        safe_unpack16 (&node_state,  buffer);
 
349
                        safe_unpack16 (&cpus,        buffer);
 
350
                        safe_unpack16 (&sockets,     buffer);
 
351
                        safe_unpack16 (&cores,       buffer);
 
352
                        safe_unpack16 (&threads,     buffer);
 
353
                        safe_unpack32 (&real_memory, buffer);
 
354
                        safe_unpack32 (&tmp_disk,    buffer);
 
355
                        safe_unpack32 (&reason_uid,  buffer);
 
356
                        safe_unpack_time (&reason_time, buffer);
 
357
                        if (gres_plugin_node_state_unpack(
 
358
                                    &gres_list, buffer, node_name,
 
359
                                    protocol_version) != SLURM_SUCCESS)
 
360
                                goto unpack_error;
 
361
                        base_state = node_state & NODE_STATE_BASE;
 
362
                } else if(protocol_version >= SLURM_2_1_PROTOCOL_VERSION) {
 
363
                        safe_unpackstr_xmalloc (&node_name, &name_len, buffer);
 
364
                        safe_unpackstr_xmalloc (&reason,    &name_len, buffer);
 
365
                        safe_unpackstr_xmalloc (&features,  &name_len, buffer);
 
366
                        safe_unpack16 (&node_state,  buffer);
 
367
                        safe_unpack16 (&cpus,        buffer);
 
368
                        safe_unpack16 (&sockets,     buffer);
 
369
                        safe_unpack16 (&cores,       buffer);
 
370
                        safe_unpack16 (&threads,     buffer);
 
371
                        safe_unpack32 (&real_memory, buffer);
 
372
                        safe_unpack32 (&tmp_disk,    buffer);
 
373
                        gres_list = NULL;
 
374
 
 
375
                        base_state = node_state & NODE_STATE_BASE;
 
376
                } else
 
377
                        goto unpack_error;
334
378
 
335
379
                /* validity test as possible */
336
380
                if ((cpus == 0) ||
344
388
                                sockets, cores, threads, node_state);
345
389
                        error ("No more node data will be processed from the "
346
390
                                "checkpoint file");
347
 
                        xfree (node_name);
348
 
                        error_code = EINVAL;
349
 
                        break;
 
391
                        goto unpack_error;
350
392
 
351
393
                }
352
394
 
355
397
                if (node_ptr == NULL) {
356
398
                        error ("Node %s has vanished from configuration",
357
399
                               node_name);
358
 
                        xfree(features);
359
 
                        xfree(reason);
360
400
                } else if (state_only) {
361
401
                        uint16_t orig_flags;
362
402
                        orig_flags = node_ptr->node_state & NODE_STATE_FLAGS;
391
431
                                                hs = hostset_create(node_name);
392
432
                                }
393
433
                        }
394
 
                        if (node_ptr->reason == NULL)
 
434
                        if (node_ptr->reason == NULL) {
395
435
                                node_ptr->reason = reason;
396
 
                        else
397
 
                                xfree(reason);
398
 
                        xfree(node_ptr->features);
399
 
                        node_ptr->features = features;
 
436
                                reason = NULL;  /* Nothing to free */
 
437
                                node_ptr->reason_time = reason_time;
 
438
                                node_ptr->reason_uid = reason_uid;
 
439
                        }
 
440
                        node_ptr->gres_list     = gres_list;
 
441
                        gres_list               = NULL; /* Nothing to free */
400
442
                } else {
401
443
                        node_cnt++;
402
444
                        if ((!power_save_mode) &&
411
453
                        }
412
454
                        node_ptr->node_state    = node_state;
413
455
                        xfree(node_ptr->reason);
414
 
                        node_ptr->reason        = reason;
 
456
                        node_ptr->reason        = reason;
 
457
                        reason                  = NULL; /* Nothing to free */
 
458
                        node_ptr->reason_time   = reason_time;
 
459
                        node_ptr->reason_uid    = reason_uid;
415
460
                        xfree(node_ptr->features);
416
 
                        node_ptr->features = features;
 
461
                        node_ptr->features      = features;
 
462
                        features                = NULL; /* Nothing to free */
 
463
                        xfree(node_ptr->gres);
 
464
                        node_ptr->gres          = gres;
 
465
                        gres                    = NULL; /* Nothing to free */
 
466
                        node_ptr->gres_list     = gres_list;
 
467
                        gres_list               = NULL; /* Nothing to free */
417
468
                        node_ptr->part_cnt      = 0;
418
469
                        xfree(node_ptr->part_pptr);
419
470
                        node_ptr->cpus          = cpus;
425
476
                        node_ptr->last_response = (time_t) 0;
426
477
                        node_ptr->last_idle     = now;
427
478
                }
 
479
 
 
480
                if (node_ptr) {
 
481
                        select_g_update_node_state(
 
482
                                (node_ptr - node_record_table_ptr),
 
483
                                node_ptr->node_state);
 
484
                }
 
485
                xfree(features);
 
486
                xfree(gres);
 
487
                if (gres_list) {
 
488
                        list_destroy(gres_list);
 
489
                        gres_list = NULL;
 
490
                }
428
491
                xfree (node_name);
429
 
 
430
 
                if(node_ptr)
431
 
                        select_g_update_node_state(
432
 
                                (node_ptr - node_record_table_ptr),
433
 
                                node_ptr->node_state);
 
492
                xfree(reason);
434
493
        }
435
494
 
436
495
fini:   info("Recovered state of %d nodes", node_cnt);
446
505
unpack_error:
447
506
        error("Incomplete node data checkpoint file");
448
507
        error_code = EFAULT;
 
508
        xfree(features);
 
509
        xfree(gres);
 
510
        if (gres_list) {
 
511
                list_destroy(gres_list);
 
512
                gres_list = NULL;
 
513
        }
 
514
        xfree (node_name);
 
515
        xfree(reason);
449
516
        goto fini;
450
517
}
451
518
 
467
534
        bool shown = false;
468
535
 
469
536
        for (i=0; i<node_ptr->part_cnt; i++) {
470
 
                if (node_ptr->part_pptr[i]->hidden == 0) {
 
537
                if (!(node_ptr->part_pptr[i]->flags & PART_FLAG_HIDDEN)) {
471
538
                        shown = true;
472
539
                        break;
473
540
                }
485
552
 * OUT buffer_size - set to size of the buffer in bytes
486
553
 * IN show_flags - node filtering options
487
554
 * IN uid - uid of user making request (for partition filtering)
 
555
 * IN protocol_version - slurm protocol version of client
488
556
 * global: node_record_table_ptr - pointer to global node table
489
557
 * NOTE: the caller must xfree the buffer at *buffer_ptr
490
558
 * NOTE: change slurm_load_node() in api/node_info.c when data format changes
491
559
 * NOTE: READ lock_slurmctld config before entry
492
560
 */
493
561
extern void pack_all_node (char **buffer_ptr, int *buffer_size,
494
 
                uint16_t show_flags, uid_t uid)
 
562
                           uint16_t show_flags, uid_t uid,
 
563
                           uint16_t protocol_version)
495
564
{
496
565
        int inx;
497
566
        uint32_t nodes_packed, tmp_offset, node_scaling;
504
573
        *buffer_size = 0;
505
574
 
506
575
        buffer = init_buf (BUF_SIZE*16);
507
 
 
508
 
        /* write header: version and time */
509
 
        nodes_packed = 0 ;
510
 
        pack32  (nodes_packed, buffer);
511
 
        select_g_alter_node_cnt(SELECT_GET_NODE_SCALING,
512
 
                                &node_scaling);
513
 
        pack32  (node_scaling, buffer);
514
 
 
515
 
        pack_time  (now, buffer);
516
 
 
517
 
        /* write node records */
518
 
        part_filter_set(uid);
519
 
        for (inx = 0; inx < node_record_count; inx++, node_ptr++) {
520
 
                xassert (node_ptr->magic == NODE_MAGIC);
521
 
                xassert (node_ptr->config_ptr->magic ==
522
 
                         CONFIG_MAGIC);
523
 
 
524
 
                /* We can't avoid packing node records without breaking
525
 
                 * the node index pointers. So pack a node with a name of
526
 
                 * NULL and let the caller deal with it. */
527
 
                hidden = false;
528
 
                if (((show_flags & SHOW_ALL) == 0) && (uid != 0) &&
529
 
                    (_node_is_hidden(node_ptr)))
530
 
                        hidden = true;
531
 
                else if (IS_NODE_FUTURE(node_ptr))
532
 
                        hidden = true;
533
 
                else if ((node_ptr->name == NULL) ||
534
 
                         (node_ptr->name[0] == '\0'))
535
 
                        hidden = true;
536
 
 
537
 
                if (hidden) {
538
 
                        char *orig_name = node_ptr->name;
539
 
                        node_ptr->name = NULL;
540
 
                        _pack_node(node_ptr, buffer);
541
 
                        node_ptr->name = orig_name;
542
 
                } else
543
 
                        _pack_node(node_ptr, buffer);
544
 
                nodes_packed ++ ;
 
576
        nodes_packed = 0;
 
577
 
 
578
        if(protocol_version >= SLURM_2_1_PROTOCOL_VERSION) {
 
579
                /* write header: version and time */
 
580
                pack32(nodes_packed, buffer);
 
581
                select_g_alter_node_cnt(SELECT_GET_NODE_SCALING,
 
582
                                        &node_scaling);
 
583
                pack32(node_scaling, buffer);
 
584
 
 
585
                pack_time(now, buffer);
 
586
 
 
587
                /* write node records */
 
588
                part_filter_set(uid);
 
589
                for (inx = 0; inx < node_record_count; inx++, node_ptr++) {
 
590
                        xassert (node_ptr->magic == NODE_MAGIC);
 
591
                        xassert (node_ptr->config_ptr->magic ==
 
592
                                 CONFIG_MAGIC);
 
593
 
 
594
                        /* We can't avoid packing node records without breaking
 
595
                         * the node index pointers. So pack a node
 
596
                         * with a name of NULL and let the caller deal
 
597
                         * with it. */
 
598
                        hidden = false;
 
599
                        if (((show_flags & SHOW_ALL) == 0) && (uid != 0) &&
 
600
                            (_node_is_hidden(node_ptr)))
 
601
                                hidden = true;
 
602
                        else if (IS_NODE_FUTURE(node_ptr))
 
603
                                hidden = true;
 
604
                        else if ((node_ptr->name == NULL) ||
 
605
                                 (node_ptr->name[0] == '\0'))
 
606
                                hidden = true;
 
607
 
 
608
                        if (hidden) {
 
609
                                char *orig_name = node_ptr->name;
 
610
                                node_ptr->name = NULL;
 
611
                                _pack_node(node_ptr, buffer, protocol_version);
 
612
                                node_ptr->name = orig_name;
 
613
                        } else
 
614
                                _pack_node(node_ptr, buffer, protocol_version);
 
615
                        nodes_packed++;
 
616
                }
 
617
                part_filter_clear();
 
618
        } else {
 
619
                error("pack_all_node: Unsupported slurm version %u",
 
620
                      protocol_version);
545
621
        }
546
 
        part_filter_clear();
547
622
 
548
623
        tmp_offset = get_buf_offset (buffer);
549
624
        set_buf_offset (buffer, 0);
560
635
 *      machine independent form (for network transmission)
561
636
 * IN dump_node_ptr - pointer to node for which information is requested
562
637
 * IN/OUT buffer - buffer where data is placed, pointers automatically updated
 
638
 * IN protocol_version - slurm protocol version of client
563
639
 * NOTE: if you make any changes here be sure to make the corresponding
564
640
 *      changes to load_node_config in api/node_info.c
565
641
 * NOTE: READ lock_slurmctld config before entry
566
642
 */
567
 
static void _pack_node (struct node_record *dump_node_ptr, Buf buffer)
 
643
static void _pack_node (struct node_record *dump_node_ptr, Buf buffer,
 
644
                        uint16_t protocol_version)
568
645
{
569
 
        packstr (dump_node_ptr->name, buffer);
570
 
        pack16  (dump_node_ptr->node_state, buffer);
571
 
        if (slurmctld_conf.fast_schedule) {
572
 
                /* Only data from config_record used for scheduling */
573
 
                pack16  (dump_node_ptr->config_ptr->cpus, buffer);
574
 
                pack16  (dump_node_ptr->config_ptr->sockets, buffer);
575
 
                pack16  (dump_node_ptr->config_ptr->cores, buffer);
576
 
                pack16  (dump_node_ptr->config_ptr->threads, buffer);
577
 
                pack32  (dump_node_ptr->config_ptr->real_memory, buffer);
578
 
                pack32  (dump_node_ptr->config_ptr->tmp_disk, buffer);
 
646
        if(protocol_version >= SLURM_2_2_PROTOCOL_VERSION) {
 
647
                packstr (dump_node_ptr->name, buffer);
 
648
                pack16  (dump_node_ptr->node_state, buffer);
 
649
                if (slurmctld_conf.fast_schedule) {
 
650
                        /* Only data from config_record used for scheduling */
 
651
                        pack16(dump_node_ptr->config_ptr->cpus, buffer);
 
652
                        pack16(dump_node_ptr->config_ptr->sockets, buffer);
 
653
                        pack16(dump_node_ptr->config_ptr->cores, buffer);
 
654
                        pack16(dump_node_ptr->config_ptr->threads, buffer);
 
655
                        pack32(dump_node_ptr->config_ptr->real_memory, buffer);
 
656
                        pack32(dump_node_ptr->config_ptr->tmp_disk, buffer);
 
657
                } else {
 
658
                        /* Individual node data used for scheduling */
 
659
                        pack16(dump_node_ptr->cpus, buffer);
 
660
                        pack16(dump_node_ptr->sockets, buffer);
 
661
                        pack16(dump_node_ptr->cores, buffer);
 
662
                        pack16(dump_node_ptr->threads, buffer);
 
663
                        pack32(dump_node_ptr->real_memory, buffer);
 
664
                        pack32(dump_node_ptr->tmp_disk, buffer);
 
665
                }
 
666
                pack32(dump_node_ptr->config_ptr->weight, buffer);
 
667
                pack32(dump_node_ptr->reason_uid, buffer);
 
668
 
 
669
                pack_time(dump_node_ptr->boot_time, buffer);
 
670
                pack_time(dump_node_ptr->reason_time, buffer);
 
671
                pack_time(dump_node_ptr->slurmd_start_time, buffer);
 
672
 
 
673
                select_g_select_nodeinfo_pack(dump_node_ptr->select_nodeinfo,
 
674
                                              buffer, protocol_version);
 
675
 
 
676
                packstr(dump_node_ptr->arch, buffer);
 
677
                packstr(dump_node_ptr->features, buffer);
 
678
                if (dump_node_ptr->gres)
 
679
                        packstr(dump_node_ptr->gres, buffer);
 
680
                else
 
681
                        packstr(dump_node_ptr->config_ptr->gres, buffer);
 
682
                packstr(dump_node_ptr->os, buffer);
 
683
                packstr(dump_node_ptr->reason, buffer);
 
684
        } else if(protocol_version >= SLURM_2_1_PROTOCOL_VERSION) {
 
685
                packstr (dump_node_ptr->name, buffer);
 
686
                pack16  (dump_node_ptr->node_state, buffer);
 
687
                if (slurmctld_conf.fast_schedule) {
 
688
                        /* Only data from config_record used for scheduling */
 
689
                        pack16(dump_node_ptr->config_ptr->cpus, buffer);
 
690
                        pack16(dump_node_ptr->config_ptr->sockets, buffer);
 
691
                        pack16(dump_node_ptr->config_ptr->cores, buffer);
 
692
                        pack16(dump_node_ptr->config_ptr->threads, buffer);
 
693
                        pack32(dump_node_ptr->config_ptr->real_memory, buffer);
 
694
                        pack32(dump_node_ptr->config_ptr->tmp_disk, buffer);
 
695
                } else {
 
696
                        /* Individual node data used for scheduling */
 
697
                        pack16(dump_node_ptr->cpus, buffer);
 
698
                        pack16(dump_node_ptr->sockets, buffer);
 
699
                        pack16(dump_node_ptr->cores, buffer);
 
700
                        pack16(dump_node_ptr->threads, buffer);
 
701
                        pack32(dump_node_ptr->real_memory, buffer);
 
702
                        pack32(dump_node_ptr->tmp_disk, buffer);
 
703
                }
 
704
                pack32(dump_node_ptr->config_ptr->weight, buffer);
 
705
 
 
706
                select_g_select_nodeinfo_pack(dump_node_ptr->select_nodeinfo,
 
707
                                              buffer, protocol_version);
 
708
 
 
709
                packstr(dump_node_ptr->arch, buffer);
 
710
                packstr(dump_node_ptr->features, buffer);
 
711
                packstr(dump_node_ptr->os, buffer);
 
712
                packstr(dump_node_ptr->reason, buffer);
579
713
        } else {
580
 
                /* Individual node data used for scheduling */
581
 
                pack16  (dump_node_ptr->cpus, buffer);
582
 
                pack16  (dump_node_ptr->sockets, buffer);
583
 
                pack16  (dump_node_ptr->cores, buffer);
584
 
                pack16  (dump_node_ptr->threads, buffer);
585
 
                pack32  (dump_node_ptr->real_memory, buffer);
586
 
                pack32  (dump_node_ptr->tmp_disk, buffer);
 
714
                error("_pack_node: Unsupported slurm version %u",
 
715
                      protocol_version);
587
716
        }
588
 
        pack32  (dump_node_ptr->config_ptr->weight, buffer);
589
 
 
590
 
        select_g_select_nodeinfo_pack(dump_node_ptr->select_nodeinfo, buffer);
591
 
 
592
 
        packstr (dump_node_ptr->arch, buffer);
593
 
        packstr (dump_node_ptr->config_ptr->feature, buffer);
594
 
        packstr (dump_node_ptr->os, buffer);
595
 
        packstr (dump_node_ptr->reason, buffer);
596
717
}
597
718
 
598
719
 
599
720
/*
600
 
 * set_slurmd_addr - establish the slurm_addr for the slurmd on each node
 
721
 * set_slurmd_addr - establish the slurm_addr_t for the slurmd on each node
601
722
 *      Uses common data structures.
602
723
 * NOTE: READ lock_slurmctld config before entry
603
724
 */
671
792
                        break;
672
793
                }
673
794
 
 
795
                if (update_node_msg->features) {
 
796
                        xfree(node_ptr->features);
 
797
                        if (update_node_msg->features[0])
 
798
                                node_ptr->features = xstrdup(update_node_msg->
 
799
                                                             features);
 
800
                        /* _update_node_features() logs and updates config */
 
801
                }
 
802
 
 
803
                if (update_node_msg->gres) {
 
804
                        xfree(node_ptr->gres);
 
805
                        if (update_node_msg->gres[0])
 
806
                                node_ptr->gres = xstrdup(update_node_msg->gres);
 
807
                        /* _update_node_gres() logs and updates config */
 
808
                }
 
809
 
674
810
                if ((update_node_msg -> reason) &&
675
811
                    (update_node_msg -> reason[0])) {
676
812
                        xfree(node_ptr->reason);
677
813
                        node_ptr->reason = xstrdup(update_node_msg->reason);
 
814
                        node_ptr->reason_time = now;
 
815
                        node_ptr->reason_uid = update_node_msg->reason_uid;
678
816
                        info ("update_node: node %s reason set to: %s",
679
817
                                this_node_name, node_ptr->reason);
680
818
                }
699
837
                                     IS_NODE_FAIL(node_ptr))) {
700
838
                                        clusteracct_storage_g_node_up(
701
839
                                                acct_db_conn,
702
 
                                                slurmctld_cluster_name,
703
840
                                                node_ptr,
704
841
                                                now);
705
842
                                }
746
883
                                        trigger_node_up(node_ptr);
747
884
                                        clusteracct_storage_g_node_up(
748
885
                                                acct_db_conn,
749
 
                                                slurmctld_cluster_name,
750
886
                                                node_ptr,
751
887
                                                now);
752
888
                                } else if (IS_NODE_IDLE(node_ptr)   &&
754
890
                                            IS_NODE_FAIL(node_ptr))) {
755
891
                                        clusteracct_storage_g_node_up(
756
892
                                                acct_db_conn,
757
 
                                                slurmctld_cluster_name,
758
893
                                                node_ptr,
759
894
                                                now);
760
895
                                }       /* else already fully available */
783
918
                                        trigger_node_drained(node_ptr);
784
919
                                        clusteracct_storage_g_node_down(
785
920
                                                acct_db_conn,
786
 
                                                slurmctld_cluster_name,
787
 
                                                node_ptr, now, NULL);
 
921
                                                node_ptr, now, NULL,
 
922
                                                node_ptr->reason_uid);
788
923
                                }
789
924
                        } else if (state_val == NODE_STATE_POWER_SAVE) {
790
925
                                if (IS_NODE_POWER_SAVE(node_ptr)) {
831
966
                }
832
967
 
833
968
                if (!IS_NODE_DOWN(node_ptr) &&
834
 
                    !IS_NODE_DRAIN(node_ptr) && !IS_NODE_FAIL(node_ptr))
 
969
                    !IS_NODE_DRAIN(node_ptr) && !IS_NODE_FAIL(node_ptr)) {
835
970
                        xfree(node_ptr->reason);
 
971
                        node_ptr->reason_time = 0;
 
972
                        node_ptr->reason_uid = NO_VAL;
 
973
                }
836
974
 
837
975
                free (this_node_name);
838
976
        }
839
977
        hostlist_destroy (host_list);
840
978
 
841
979
        if ((error_code == 0) && (update_node_msg->features)) {
842
 
                error_code = _update_node_features(
843
 
                        update_node_msg->node_names,
844
 
                        update_node_msg->features);
845
 
        }
846
 
 
 
980
                error_code = _update_node_features(update_node_msg->node_names,
 
981
                                                   update_node_msg->features);
 
982
        }
 
983
        if ((error_code == 0) && (update_node_msg->gres)) {
 
984
                error_code = _update_node_gres(update_node_msg->node_names,
 
985
                                               update_node_msg->gres);
 
986
        }
847
987
 
848
988
        /* Update weight. Weight is part of config_ptr,
849
 
         * hence do the splitting if required */
 
989
         * hence split config records if required */
850
990
        if ((error_code == 0) && (update_node_msg->weight != NO_VAL))   {
851
991
                error_code = _update_node_weight(update_node_msg->node_names,
852
992
                                                 update_node_msg->weight);
859
999
        return error_code;
860
1000
}
861
1001
 
 
1002
/* variation of strcmp that accepts NULL pointers */
 
1003
static int _strcmp(char *str1, char *str2)
 
1004
{
 
1005
        if (!str1 && !str2)
 
1006
                return 0;
 
1007
        if (str1 && !str2)
 
1008
                return 1;
 
1009
        if (!str1 && str2)
 
1010
                return -1;
 
1011
        return strcmp(str1, str2);
 
1012
}
 
1013
 
862
1014
/*
863
 
 * restore_node_features - Restore node features based upon state
864
 
 *      saved (preserves interactive updates)
 
1015
 * restore_node_features - Make node and config (from slurm.conf) fields
 
1016
 *      consistent for Features, Gres and Weight
 
1017
 * IN recover -
 
1018
 *              0, 1 - use data from config record, built using slurm.conf
 
1019
 *              2 = use data from node record, built from saved state
865
1020
 */
866
 
extern void restore_node_features(void)
 
1021
extern void restore_node_features(int recover)
867
1022
{
868
 
        int i, j;
869
 
        char *node_list;
870
 
        struct node_record *node_ptr1, *node_ptr2;
871
 
        hostlist_t hl;
872
 
 
873
 
        for (i=0, node_ptr1=node_record_table_ptr; i<node_record_count; 
874
 
             i++, node_ptr1++) {
875
 
 
876
 
                if (!node_ptr1->features)
877
 
                        continue;       /* No feature to preserve */
878
 
                if (node_ptr1->config_ptr->feature &&
879
 
                    !strcmp(node_ptr1->config_ptr->feature,
880
 
                            node_ptr1->features)) {
881
 
                        continue;       /* Identical feature value */
882
 
                }
883
 
 
884
 
                hl = hostlist_create(node_ptr1->name);
885
 
                for (j=(i+1), node_ptr2=(node_ptr1+1); j<node_record_count; 
886
 
                     j++, node_ptr2++) {
887
 
                        if (!node_ptr2->features ||
888
 
                            strcmp(node_ptr1->features, node_ptr2->features))
889
 
                                continue;
890
 
                        hostlist_push(hl, node_ptr2->name);
891
 
                }
892
 
 
893
 
                node_list = xmalloc(2048);
894
 
                hostlist_ranged_string(hl, 2048, node_list);
895
 
                error("Node %s Features(%s) differ from slurm.conf",
896
 
                      node_list, node_ptr1->features);
897
 
                _update_node_features(node_list, node_ptr1->features);
898
 
                xfree(node_ptr1->features);
899
 
                xfree(node_list);
900
 
                hostlist_destroy(hl);
 
1023
        int i;
 
1024
        struct node_record *node_ptr;
 
1025
 
 
1026
        for (i=0, node_ptr=node_record_table_ptr; i<node_record_count;
 
1027
             i++, node_ptr++) {
 
1028
 
 
1029
                if (node_ptr->weight != node_ptr->config_ptr->weight) {
 
1030
                        error("Node %s Weight(%u) differ from slurm.conf",
 
1031
                              node_ptr->name, node_ptr->weight);
 
1032
                        if (recover == 2) {
 
1033
                                _update_node_weight(node_ptr->name,
 
1034
                                                    node_ptr->weight);
 
1035
                        } else {
 
1036
                                node_ptr->weight = node_ptr->config_ptr->
 
1037
                                                   weight;
 
1038
                        }
 
1039
                }
 
1040
 
 
1041
                if (_strcmp(node_ptr->config_ptr->feature, node_ptr->features)){
 
1042
                        error("Node %s Features(%s) differ from slurm.conf",
 
1043
                              node_ptr->name, node_ptr->features);
 
1044
                        if (recover == 2) {
 
1045
                                _update_node_features(node_ptr->name,
 
1046
                                                      node_ptr->features);
 
1047
                        } else {
 
1048
                                xfree(node_ptr->features);
 
1049
                                node_ptr->features = xstrdup(node_ptr->
 
1050
                                                             config_ptr->
 
1051
                                                             feature);
 
1052
                        }
 
1053
                }
 
1054
 
 
1055
                /* We lose the gres information updated manually and always
 
1056
                 * use the information from slurm.conf */
 
1057
                (void) gres_plugin_node_reconfig(node_ptr->name,
 
1058
                                                 node_ptr->config_ptr->gres,
 
1059
                                                 &node_ptr->gres,
 
1060
                                                 &node_ptr->gres_list,
 
1061
                                                 slurmctld_conf.fast_schedule);
 
1062
                gres_plugin_node_state_log(node_ptr->gres_list, node_ptr->name);
901
1063
        }
902
1064
}
903
1065
 
 
1066
/* Duplicate a configuration record except for the node names & bitmap */
 
1067
struct config_record * _dup_config(struct config_record *config_ptr)
 
1068
{
 
1069
        struct config_record *new_config_ptr;
 
1070
 
 
1071
        new_config_ptr = create_config_record();
 
1072
        new_config_ptr->magic       = config_ptr->magic;
 
1073
        new_config_ptr->cpus        = config_ptr->cpus;
 
1074
        new_config_ptr->sockets     = config_ptr->sockets;
 
1075
        new_config_ptr->cores       = config_ptr->cores;
 
1076
        new_config_ptr->threads     = config_ptr->threads;
 
1077
        new_config_ptr->real_memory = config_ptr->real_memory;
 
1078
        new_config_ptr->tmp_disk    = config_ptr->tmp_disk;
 
1079
        new_config_ptr->weight      = config_ptr->weight;
 
1080
        new_config_ptr->feature     = xstrdup(config_ptr->feature);
 
1081
        new_config_ptr->gres        = xstrdup(config_ptr->gres);
 
1082
 
 
1083
        return new_config_ptr;
 
1084
}
 
1085
 
904
1086
/*
905
1087
 * _update_node_weight - Update weight associated with nodes
906
1088
 *      build new config list records as needed
944
1126
                        config_ptr->weight = weight;
945
1127
                } else {
946
1128
                        /* partial update, split config_record */
947
 
                        new_config_ptr = create_config_record();
 
1129
                        new_config_ptr = _dup_config(config_ptr);
948
1130
                        if (first_new == NULL);
949
1131
                                first_new = new_config_ptr;
950
 
                        new_config_ptr->magic       = config_ptr->magic;
951
 
                        new_config_ptr->cpus        = config_ptr->cpus;
952
 
                        new_config_ptr->sockets     = config_ptr->sockets;
953
 
                        new_config_ptr->cores       = config_ptr->cores;
954
 
                        new_config_ptr->threads     = config_ptr->threads;
955
 
                        new_config_ptr->real_memory = config_ptr->real_memory;
956
 
                        new_config_ptr->tmp_disk    = config_ptr->tmp_disk;
957
1132
                        /* Change weight for the given node */
958
1133
                        new_config_ptr->weight      = weight;
959
 
                        if (config_ptr->feature) {
960
 
                                new_config_ptr->feature = xstrdup(config_ptr->
961
 
                                                                  feature);
962
 
                        }
963
1134
                        new_config_ptr->node_bitmap = bit_copy(tmp_bitmap);
964
 
                        new_config_ptr->nodes =
965
 
                                bitmap2node_name(tmp_bitmap);
 
1135
                        new_config_ptr->nodes = bitmap2node_name(tmp_bitmap);
 
1136
 
966
1137
                        build_config_feature_list(new_config_ptr);
967
1138
                        _update_config_ptr(tmp_bitmap, new_config_ptr);
968
1139
 
973
1144
                        config_ptr->nodes = bitmap2node_name(
974
1145
                                config_ptr->node_bitmap);
975
1146
                }
976
 
                bit_free(tmp_bitmap);
 
1147
                FREE_NULL_BITMAP(tmp_bitmap);
977
1148
        }
978
1149
        list_iterator_destroy(config_iterator);
979
 
        bit_free(node_bitmap);
 
1150
        FREE_NULL_BITMAP(node_bitmap);
980
1151
 
981
1152
        info("_update_node_weight: nodes %s weight set to: %u",
982
1153
                node_names, weight);
1024
1195
                } else if (tmp_cnt == config_cnt) {
1025
1196
                        /* all nodes changed, update in situ */
1026
1197
                        xfree(config_ptr->feature);
1027
 
                        if (features[0])
 
1198
                        if (features && features[0])
1028
1199
                                config_ptr->feature = xstrdup(features);
1029
 
                        else
1030
 
                                config_ptr->feature = NULL;
1031
1200
                        build_config_feature_list(config_ptr);
1032
1201
                } else {
1033
1202
                        /* partial update, split config_record */
1034
 
                        new_config_ptr = create_config_record();
 
1203
                        new_config_ptr = _dup_config(config_ptr);
1035
1204
                        if (first_new == NULL);
1036
1205
                                first_new = new_config_ptr;
1037
 
                        new_config_ptr->magic       = config_ptr->magic;
1038
 
                        new_config_ptr->cpus        = config_ptr->cpus;
1039
 
                        new_config_ptr->sockets     = config_ptr->sockets;
1040
 
                        new_config_ptr->cores       = config_ptr->cores;
1041
 
                        new_config_ptr->threads     = config_ptr->threads;
1042
 
                        new_config_ptr->real_memory = config_ptr->real_memory;
1043
 
                        new_config_ptr->tmp_disk    = config_ptr->tmp_disk;
1044
 
                        new_config_ptr->weight      = config_ptr->weight;
1045
 
                        if (features[0])
 
1206
                        xfree(new_config_ptr->feature);
 
1207
                        if (features && features[0])
1046
1208
                                new_config_ptr->feature = xstrdup(features);
1047
1209
                        new_config_ptr->node_bitmap = bit_copy(tmp_bitmap);
1048
 
                        new_config_ptr->nodes =
1049
 
                                bitmap2node_name(tmp_bitmap);
 
1210
                        new_config_ptr->nodes = bitmap2node_name(tmp_bitmap);
 
1211
 
1050
1212
                        build_config_feature_list(new_config_ptr);
1051
1213
                        _update_config_ptr(tmp_bitmap, new_config_ptr);
1052
1214
 
1054
1216
                        bit_not(tmp_bitmap);
1055
1217
                        bit_and(config_ptr->node_bitmap, tmp_bitmap);
1056
1218
                        xfree(config_ptr->nodes);
1057
 
                        config_ptr->nodes = bitmap2node_name(
1058
 
                                config_ptr->node_bitmap);
 
1219
                        config_ptr->nodes = bitmap2node_name(config_ptr->
 
1220
                                                             node_bitmap);
1059
1221
                }
1060
 
                bit_free(tmp_bitmap);
 
1222
                FREE_NULL_BITMAP(tmp_bitmap);
1061
1223
        }
1062
1224
        list_iterator_destroy(config_iterator);
1063
 
        bit_free(node_bitmap);
 
1225
        FREE_NULL_BITMAP(node_bitmap);
1064
1226
 
1065
1227
        info("_update_node_features: nodes %s features set to: %s",
1066
1228
                node_names, features);
1067
1229
        return SLURM_SUCCESS;
1068
1230
}
1069
1231
 
 
1232
/*
 
1233
 * _update_node_gres - Update generic resources associated with nodes
 
1234
 *      build new config list records as needed
 
1235
 * IN node_names - List of nodes to update
 
1236
 * IN gres - New gres value
 
1237
 * RET: SLURM_SUCCESS or error code
 
1238
 */
 
1239
static int _update_node_gres(char *node_names, char *gres)
 
1240
{
 
1241
        bitstr_t *node_bitmap = NULL, *tmp_bitmap;
 
1242
        ListIterator config_iterator;
 
1243
        struct config_record *config_ptr, *new_config_ptr;
 
1244
        struct config_record *first_new = NULL;
 
1245
        struct node_record *node_ptr;
 
1246
        int rc, config_cnt, tmp_cnt;
 
1247
        int i, i_first, i_last;
 
1248
 
 
1249
        rc = node_name2bitmap(node_names, false, &node_bitmap);
 
1250
        if (rc) {
 
1251
                info("_update_node_gres: invalid node_name");
 
1252
                return rc;
 
1253
        }
 
1254
 
 
1255
        /* For each config_record with one of these nodes,
 
1256
         * update it (if all nodes updated) or split it into
 
1257
         * a new entry */
 
1258
        config_iterator = list_iterator_create(config_list);
 
1259
        if (config_iterator == NULL)
 
1260
                fatal("list_iterator_create malloc failure");
 
1261
        while ((config_ptr = (struct config_record *)
 
1262
                        list_next(config_iterator))) {
 
1263
                if (config_ptr == first_new)
 
1264
                        break;  /* done with all original records */
 
1265
 
 
1266
                tmp_bitmap = bit_copy(node_bitmap);
 
1267
                bit_and(tmp_bitmap, config_ptr->node_bitmap);
 
1268
                config_cnt = bit_set_count(config_ptr->node_bitmap);
 
1269
                tmp_cnt = bit_set_count(tmp_bitmap);
 
1270
                if (tmp_cnt == 0) {
 
1271
                        /* no overlap, leave alone */
 
1272
                } else if (tmp_cnt == config_cnt) {
 
1273
                        /* all nodes changed, update in situ */
 
1274
                        xfree(config_ptr->gres);
 
1275
                        if (gres && gres[0])
 
1276
                                config_ptr->gres = xstrdup(gres);
 
1277
                } else {
 
1278
                        /* partial update, split config_record */
 
1279
                        new_config_ptr = _dup_config(config_ptr);
 
1280
                        if (first_new == NULL);
 
1281
                                first_new = new_config_ptr;
 
1282
                        xfree(new_config_ptr->gres);
 
1283
                        if (gres && gres[0])
 
1284
                                new_config_ptr->gres = xstrdup(gres);
 
1285
                        new_config_ptr->node_bitmap = bit_copy(tmp_bitmap);
 
1286
                        new_config_ptr->nodes = bitmap2node_name(tmp_bitmap);
 
1287
 
 
1288
                        _update_config_ptr(tmp_bitmap, new_config_ptr);
 
1289
 
 
1290
                        /* Update remaining records */
 
1291
                        bit_not(tmp_bitmap);
 
1292
                        bit_and(config_ptr->node_bitmap, tmp_bitmap);
 
1293
                        xfree(config_ptr->nodes);
 
1294
                        config_ptr->nodes = bitmap2node_name(config_ptr->
 
1295
                                                             node_bitmap);
 
1296
                }
 
1297
                FREE_NULL_BITMAP(tmp_bitmap);
 
1298
        }
 
1299
        list_iterator_destroy(config_iterator);
 
1300
 
 
1301
        i_first = bit_ffs(node_bitmap);
 
1302
        i_last  = bit_fls(node_bitmap);
 
1303
        for (i=i_first; i<=i_last; i++) {
 
1304
                node_ptr = node_record_table_ptr + i;
 
1305
                (void) gres_plugin_node_reconfig(node_ptr->name,
 
1306
                                                 node_ptr->config_ptr->gres,
 
1307
                                                 &node_ptr->gres,
 
1308
                                                 &node_ptr->gres_list,
 
1309
                                                 slurmctld_conf.fast_schedule);
 
1310
                gres_plugin_node_state_log(node_ptr->gres_list, node_ptr->name);
 
1311
        }
 
1312
        FREE_NULL_BITMAP(node_bitmap);
 
1313
 
 
1314
        info("_update_node_gres: nodes %s gres set to: %s", node_names, gres);
 
1315
        return SLURM_SUCCESS;
 
1316
}
 
1317
 
1070
1318
/* Reset the config pointer for updated jobs */
1071
1319
static void _update_config_ptr(bitstr_t *bitmap,
1072
1320
                struct config_record *config_ptr)
1088
1336
 * RET SLURM_SUCCESS or error code
1089
1337
 * global: node_record_table_ptr - pointer to global node table
1090
1338
 */
1091
 
extern int drain_nodes ( char *nodes, char *reason )
 
1339
extern int drain_nodes ( char *nodes, char *reason, uint32_t reason_uid )
1092
1340
{
1093
1341
        int error_code = 0, node_inx;
1094
1342
        struct node_record *node_ptr;
1131
1379
 
1132
1380
                xfree(node_ptr->reason);
1133
1381
                node_ptr->reason = xstrdup(reason);
 
1382
                node_ptr->reason_time = now;
 
1383
                node_ptr->reason_uid = reason_uid;
1134
1384
                if ((node_ptr->run_job_cnt  == 0) &&
1135
1385
                    (node_ptr->comp_job_cnt == 0)) {
1136
1386
                        /* no jobs, node is drained */
1137
1387
                        trigger_node_drained(node_ptr);
1138
1388
                        clusteracct_storage_g_node_down(acct_db_conn,
1139
 
                                                        slurmctld_cluster_name,
1140
 
                                                        node_ptr, now, NULL);
 
1389
                                                        node_ptr, now, NULL,
 
1390
                                                        reason_uid);
1141
1391
                }
1142
1392
 
1143
1393
                select_g_update_node_state(node_inx, node_ptr->node_state);
1204
1454
 */
1205
1455
extern int validate_node_specs(slurm_node_registration_status_msg_t *reg_msg)
1206
1456
{
1207
 
        int error_code, i;
 
1457
        int error_code, i, node_inx;
1208
1458
        struct config_record *config_ptr;
1209
1459
        struct node_record *node_ptr;
1210
1460
        char *reason_down = NULL;
1212
1462
        time_t now = time(NULL);
1213
1463
        bool gang_flag = false;
1214
1464
        static uint32_t cr_flag = NO_VAL;
1215
 
        time_t last_restart = now - reg_msg->up_time;
1216
1465
 
1217
1466
        node_ptr = find_node_record (reg_msg->node_name);
1218
1467
        if (node_ptr == NULL)
1219
1468
                return ENOENT;
 
1469
        node_inx = node_ptr - node_record_table_ptr;
1220
1470
 
1221
1471
        config_ptr = node_ptr->config_ptr;
1222
1472
        error_code = SLURM_SUCCESS;
1231
1481
        if (slurm_get_preempt_mode() != PREEMPT_MODE_OFF)
1232
1482
                gang_flag = true;
1233
1483
 
 
1484
        if (gres_plugin_node_config_unpack(reg_msg->gres_info,
 
1485
                                           node_ptr->name) != SLURM_SUCCESS) {
 
1486
                error_code = SLURM_ERROR;
 
1487
                reason_down = "Could not unpack gres data";
 
1488
        } else if (gres_plugin_node_config_validate(
 
1489
                           node_ptr->name, config_ptr->gres,
 
1490
                           &node_ptr->gres, &node_ptr->gres_list,
 
1491
                           slurmctld_conf.fast_schedule, &reason_down)
 
1492
                   != SLURM_SUCCESS) {
 
1493
                error_code = EINVAL;
 
1494
                /* reason_down set in function above */
 
1495
        }
 
1496
        gres_plugin_node_state_log(node_ptr->gres_list, node_ptr->name);
 
1497
 
1234
1498
        if (slurmctld_conf.fast_schedule != 2) {
1235
1499
                int sockets1, sockets2; /* total sockets on node */
1236
1500
                int cores1, cores2;     /* total cores on node */
1244
1508
                threads2 = cores2   * config_ptr->threads;
1245
1509
 
1246
1510
                if (threads1 < threads2) {
1247
 
                        error("Node %s has low socket*core*thread count %u",
1248
 
                                reg_msg->node_name, threads1);
 
1511
                        error("Node %s has low socket*core*thread count "
 
1512
                              "(%d < %d)",
 
1513
                              reg_msg->node_name, threads1, threads2);
1249
1514
                        error_code = EINVAL;
1250
1515
                        reason_down = "Low socket*core*thread count";
1251
1516
                } else if ((slurmctld_conf.fast_schedule == 0) &&
1252
1517
                           ((cr_flag == 1) || gang_flag) && (cores1 < cores2)) {
1253
 
                        error("Node %s has low socket*core count %u",
1254
 
                              reg_msg->node_name, cores1);
 
1518
                        error("Node %s has low socket*core count (%d < %d)",
 
1519
                              reg_msg->node_name, cores1, cores2);
1255
1520
                        error_code = EINVAL;
1256
1521
                        reason_down = "Low socket*core count";
1257
1522
                } else if ((slurmctld_conf.fast_schedule == 0) &&
1258
1523
                           ((cr_flag == 1) || gang_flag) &&
1259
1524
                           ((sockets1 > sockets2) || (cores1 > cores2) ||
1260
1525
                            (threads1 > threads2))) {
1261
 
                        error("Node %s has high socket*core*thread count %u, "
1262
 
                              "extra resources ignored",
1263
 
                              reg_msg->node_name, threads1);
 
1526
                        error("Node %s has high socket,core,thread count "
 
1527
                              "(%d,%d,%d > %d,%d,%d), extra resources ignored",
 
1528
                              reg_msg->node_name, sockets1, cores1, threads1,
 
1529
                              sockets2, cores2, threads2);
1264
1530
                        /* Preserve configured values */
1265
1531
                        reg_msg->sockets = config_ptr->sockets;
1266
1532
                        reg_msg->cores   = config_ptr->cores;
1268
1534
                }
1269
1535
 
1270
1536
                if (reg_msg->cpus < config_ptr->cpus) {
1271
 
                        error ("Node %s has low cpu count %u",
1272
 
                                reg_msg->node_name, reg_msg->cpus);
 
1537
                        error("Node %s has low cpu count (%u < %u)",
 
1538
                              reg_msg->node_name, reg_msg->cpus,
 
1539
                              config_ptr->cpus);
1273
1540
                        error_code  = EINVAL;
1274
1541
                        reason_down = "Low CPUs";
1275
1542
                } else if ((slurmctld_conf.fast_schedule == 0) &&
1276
1543
                           ((cr_flag == 1) || gang_flag) &&
1277
1544
                           (reg_msg->cpus > config_ptr->cpus)) {
1278
 
                        error("Node %s has high CPU count %u, "
 
1545
                        error("Node %s has high CPU count (%u > %u), "
1279
1546
                              "extra resources ignored",
1280
 
                              reg_msg->node_name, reg_msg->cpus);
 
1547
                              reg_msg->node_name, reg_msg->cpus,
 
1548
                              config_ptr->cpus);
1281
1549
                        reg_msg->cpus    = config_ptr->cpus;
1282
1550
                }
1283
1551
        }
1299
1567
 
1300
1568
        if ((slurmctld_conf.fast_schedule != 2) &&
1301
1569
            (reg_msg->real_memory < config_ptr->real_memory)) {
1302
 
                error ("Node %s has low real_memory size %u",
1303
 
                       reg_msg->node_name, reg_msg->real_memory);
 
1570
                error("Node %s has low real_memory size (%u < %u)",
 
1571
                      reg_msg->node_name, reg_msg->real_memory,
 
1572
                      config_ptr->real_memory);
1304
1573
                error_code  = EINVAL;
1305
1574
                reason_down = "Low RealMemory";
1306
1575
        }
1308
1577
 
1309
1578
        if ((slurmctld_conf.fast_schedule != 2) &&
1310
1579
            (reg_msg->tmp_disk < config_ptr->tmp_disk)) {
1311
 
                error ("Node %s has low tmp_disk size %u",
1312
 
                       reg_msg->node_name, reg_msg->tmp_disk);
 
1580
                error("Node %s has low tmp_disk size (%u < %u)",
 
1581
                      reg_msg->node_name, reg_msg->tmp_disk,
 
1582
                      config_ptr->tmp_disk);
1313
1583
                error_code = EINVAL;
1314
1584
                reason_down = "Low TmpDisk";
1315
1585
        }
1365
1635
                        if (!IS_NODE_DRAIN(node_ptr)
1366
1636
                            && !IS_NODE_FAIL(node_ptr)) {
1367
1637
                                xfree(node_ptr->reason);
 
1638
                                node_ptr->reason_time = 0;
 
1639
                                node_ptr->reason_uid = NO_VAL;
1368
1640
                                clusteracct_storage_g_node_up(
1369
 
                                        acct_db_conn, slurmctld_cluster_name,
1370
 
                                        node_ptr, now);
 
1641
                                        acct_db_conn, node_ptr, now);
1371
1642
                        }
1372
1643
                } else if (IS_NODE_DOWN(node_ptr) &&
1373
1644
                           ((slurmctld_conf.ret2service == 2) ||
1374
 
                            ((slurmctld_conf.ret2service == 1) &&
 
1645
                            ((slurmctld_conf.ret2service == 1) &&
1375
1646
                             (node_ptr->reason != NULL) &&
1376
1647
                             (strncmp(node_ptr->reason, "Not responding", 14)
1377
1648
                                        == 0)))) {
1391
1662
                        if (!IS_NODE_DRAIN(node_ptr)
1392
1663
                            && !IS_NODE_FAIL(node_ptr)) {
1393
1664
                                xfree(node_ptr->reason);
 
1665
                                node_ptr->reason_time = 0;
 
1666
                                node_ptr->reason_uid = NO_VAL;
1394
1667
                                clusteracct_storage_g_node_up(
1395
 
                                        acct_db_conn, slurmctld_cluster_name,
1396
 
                                        node_ptr, now);
 
1668
                                        acct_db_conn, node_ptr, now);
1397
1669
                        }
1398
1670
                } else if (node_ptr->last_response
1399
 
                           && (last_restart > node_ptr->last_response)
 
1671
                           && (node_ptr->boot_time > node_ptr->last_response)
1400
1672
                           && (slurmctld_conf.ret2service != 2)) {
1401
 
                        char time_str[32];
1402
1673
                        last_node_update = now;
1403
 
                        slurm_make_time_str(&now, time_str, sizeof(time_str));
1404
 
                        if(!node_ptr->reason)
1405
 
                                node_ptr->reason = xstrdup_printf(
1406
 
                                        "Node silently failed and "
1407
 
                                        "came back [slurm@%s]",
1408
 
                                        time_str);
 
1674
                        if(!node_ptr->reason) {
 
1675
                                node_ptr->reason_time = now;
 
1676
                                node_ptr->reason_uid =
 
1677
                                        slurm_get_slurm_user_id();
 
1678
                                node_ptr->reason = xstrdup(
 
1679
                                        "Node silently failed and came back");
 
1680
                        }
1409
1681
                        info("Node %s silently failed and came back",
1410
1682
                             reg_msg->node_name);
1411
1683
                        _make_node_down(node_ptr, last_node_update);
1420
1692
                           (reg_msg->job_count == 0)) { /* job already done */
1421
1693
                        last_node_update = now;
1422
1694
                        node_ptr->node_state &= (~NODE_STATE_COMPLETING);
 
1695
                        bit_clear(cg_node_bitmap, node_inx);
1423
1696
                } else if (IS_NODE_IDLE(node_ptr) &&
1424
1697
                           (reg_msg->job_count != 0)) {
1425
1698
                        last_node_update = now;
1426
 
                        node_ptr->node_state = NODE_STATE_ALLOCATED |
1427
 
                                               node_flags;
1428
 
                        error("Invalid state for node %s, was IDLE with %u "
1429
 
                              "running jobs",
1430
 
                              node_ptr->name, reg_msg->job_count);
 
1699
                        if (node_ptr->run_job_cnt != 0) {
 
1700
                                node_ptr->node_state = NODE_STATE_ALLOCATED |
 
1701
                                                       node_flags;
 
1702
                                error("Invalid state for node %s, was IDLE "
 
1703
                                      "with %u running jobs",
 
1704
                                      node_ptr->name, reg_msg->job_count);
 
1705
                        }
 
1706
                        /* 
 
1707
                         * there must be completing job(s) on this node since
 
1708
                         * reg_msg->job_count was set (run_job_cnt +
 
1709
                         * comp_job_cnt) in validate_jobs_on_node()
 
1710
                         */
 
1711
                        if (node_ptr->comp_job_cnt != 0) { 
 
1712
                                node_ptr->node_state |= NODE_STATE_COMPLETING;
 
1713
                                bit_set(cg_node_bitmap, node_inx);
 
1714
                        }
1431
1715
                }
1432
1716
 
1433
 
                select_g_update_node_config((node_ptr-node_record_table_ptr));
1434
 
                select_g_update_node_state((node_ptr - node_record_table_ptr),
1435
 
                                           node_ptr->node_state);
 
1717
                select_g_update_node_config(node_inx);
 
1718
                select_g_update_node_state(node_inx, node_ptr->node_state);
1436
1719
                _sync_bitmaps(node_ptr, reg_msg->job_count);
1437
1720
        }
1438
1721
 
1453
1736
                slurm_node_registration_status_msg_t *reg_msg)
1454
1737
{
1455
1738
        int error_code = 0, i, jobs_on_node;
1456
 
        bool updated_job = false;
 
1739
        bool update_node_state = false;
1457
1740
#ifdef HAVE_BG
1458
1741
        bool failure_logged = false;
1459
1742
#endif
1464
1747
        ListIterator job_iterator;
1465
1748
        hostlist_t return_hostlist = NULL, reg_hostlist = NULL;
1466
1749
        hostlist_t prolog_hostlist = NULL;
1467
 
        char host_str[64];
 
1750
        char *host_str = NULL;
1468
1751
        uint16_t node_flags;
1469
1752
 
 
1753
        if (reg_msg->up_time > now) {
 
1754
                error("Node up_time is invalid: %u>%u", reg_msg->up_time,
 
1755
                      (uint32_t) now);
 
1756
                reg_msg->up_time = 0;
 
1757
        }
 
1758
 
1470
1759
        /* First validate the job info */
1471
 
        node_ptr = &node_record_table_ptr[0];   /* All msg send to node zero,
1472
 
                                 * the front-end for the wholel cluster */
 
1760
        node_ptr = node_record_table_ptr;       /* All msg send to node zero,
 
1761
                                 * the front-end for the whole cluster */
1473
1762
        for (i = 0; i < reg_msg->job_count; i++) {
1474
1763
                if ( (reg_msg->job_id[i] >= MIN_NOALLOC_JOBID) &&
1475
1764
                     (reg_msg->job_id[i] <= MAX_NOALLOC_JOBID) ) {
1544
1833
#endif
1545
1834
 
1546
1835
                info("Killing orphan batch job %u", job_ptr->job_id);
1547
 
                job_complete(job_ptr->job_id, 0, false, 0);
 
1836
                job_complete(job_ptr->job_id, 0, false, false, 0);
1548
1837
        }
1549
1838
        list_iterator_destroy(job_iterator);
1550
1839
 
1551
 
        /* Now validate the node info */
 
1840
        (void) gres_plugin_node_config_unpack(reg_msg->gres_info,
 
1841
                                              node_record_table_ptr->name);
1552
1842
        for (i=0; i<node_record_count; i++) {
1553
1843
                node_ptr = &node_record_table_ptr[i];
1554
1844
                config_ptr = node_ptr->config_ptr;
1555
1845
                jobs_on_node = node_ptr->run_job_cnt + node_ptr->comp_job_cnt;
1556
1846
                node_ptr->last_response = time (NULL);
1557
1847
 
 
1848
                (void) gres_plugin_node_config_validate(node_ptr->name,
 
1849
                                                        config_ptr->gres,
 
1850
                                                        &node_ptr->gres,
 
1851
                                                        &node_ptr->gres_list,
 
1852
                                                        slurmctld_conf.
 
1853
                                                        fast_schedule,
 
1854
                                                        NULL);
 
1855
                gres_plugin_node_state_log(node_ptr->gres_list, node_ptr->name);
 
1856
 
 
1857
                if (reg_msg->up_time) {
 
1858
                        node_ptr->up_time = reg_msg->up_time;
 
1859
                        node_ptr->boot_time = now - reg_msg->up_time;
 
1860
                }
 
1861
                node_ptr->slurmd_start_time = reg_msg->slurmd_start_time;
 
1862
 
1558
1863
                if (IS_NODE_NO_RESPOND(node_ptr)) {
1559
 
                        updated_job = true;
 
1864
                        update_node_state = true;
1560
1865
                        node_ptr->node_state &= (~NODE_STATE_NO_RESPOND);
1561
1866
                        node_ptr->node_state &= (~NODE_STATE_POWER_UP);
1562
1867
                }
1570
1875
                                        failure_logged = true;
1571
1876
                                }
1572
1877
#else
1573
 
                                updated_job = true;
 
1878
                                update_node_state = true;
1574
1879
                                if (prolog_hostlist)
1575
1880
                                        (void) hostlist_push_host(
1576
1881
                                                prolog_hostlist,
1583
1888
                        }
1584
1889
                } else {
1585
1890
                        if (reg_hostlist)
1586
 
                                (void) hostlist_push_host(
1587
 
                                        reg_hostlist, node_ptr->name);
 
1891
                                (void) hostlist_push_host(reg_hostlist,
 
1892
                                                          node_ptr->name);
1588
1893
                        else
1589
1894
                                reg_hostlist = hostlist_create(node_ptr->name);
1590
1895
 
1591
1896
                        node_flags = node_ptr->node_state & NODE_STATE_FLAGS;
1592
1897
                        if (IS_NODE_UNKNOWN(node_ptr)) {
1593
 
                                updated_job = true;
 
1898
                                update_node_state = true;
1594
1899
                                if (jobs_on_node) {
1595
1900
                                        node_ptr->node_state =
1596
1901
                                                NODE_STATE_ALLOCATED |
1604
1909
                                if (!IS_NODE_DRAIN(node_ptr)
1605
1910
                                    && !IS_NODE_FAIL(node_ptr)) {
1606
1911
                                        xfree(node_ptr->reason);
 
1912
                                        node_ptr->reason_time = 0;
 
1913
                                        node_ptr->reason_uid = NO_VAL;
1607
1914
                                        clusteracct_storage_g_node_up(
1608
1915
                                                acct_db_conn,
1609
 
                                                slurmctld_cluster_name,
1610
1916
                                                node_ptr, now);
1611
1917
                                }
1612
1918
                        } else if (IS_NODE_DOWN(node_ptr) &&
1613
1919
                                   ((slurmctld_conf.ret2service == 2) ||
1614
 
                                    ((slurmctld_conf.ret2service == 1) &&
 
1920
                                    ((slurmctld_conf.ret2service == 1) &&
1615
1921
                                     (node_ptr->reason != NULL) &&
1616
 
                                     (strncmp(node_ptr->reason, 
 
1922
                                     (strncmp(node_ptr->reason,
1617
1923
                                              "Not responding", 14) == 0)))) {
1618
 
                                updated_job = true;
 
1924
                                update_node_state = true;
1619
1925
                                if (jobs_on_node) {
1620
1926
                                        node_ptr->node_state =
1621
1927
                                                NODE_STATE_ALLOCATED |
1630
1936
                                if (!IS_NODE_DRAIN(node_ptr)
1631
1937
                                    && !IS_NODE_FAIL(node_ptr)) {
1632
1938
                                        xfree(node_ptr->reason);
 
1939
                                        node_ptr->reason_time = 0;
 
1940
                                        node_ptr->reason_uid = NO_VAL;
1633
1941
                                        clusteracct_storage_g_node_up(
1634
1942
                                                acct_db_conn,
1635
 
                                                slurmctld_cluster_name,
1636
1943
                                                node_ptr, now);
1637
1944
                                }
1638
1945
                        } else if (IS_NODE_ALLOCATED(node_ptr) &&
1639
1946
                                   (jobs_on_node == 0)) {
1640
1947
                                /* job vanished */
1641
 
                                updated_job = true;
 
1948
                                update_node_state = true;
1642
1949
                                node_ptr->node_state = NODE_STATE_IDLE |
1643
1950
                                        node_flags;
1644
1951
                                node_ptr->last_idle = now;
1645
1952
                        } else if (IS_NODE_COMPLETING(node_ptr) &&
1646
 
                                   (jobs_on_node == 0)) {
 
1953
                                   (jobs_on_node == 0)) {
1647
1954
                                /* job already done */
1648
 
                                updated_job = true;
 
1955
                                update_node_state = true;
1649
1956
                                node_ptr->node_state &=
1650
1957
                                        (~NODE_STATE_COMPLETING);
 
1958
                                bit_clear(cg_node_bitmap, i);
1651
1959
                        } else if (IS_NODE_IDLE(node_ptr) &&
1652
1960
                                   (jobs_on_node != 0)) {
1653
 
                                updated_job = true;
 
1961
                                update_node_state = true;
1654
1962
                                node_ptr->node_state = NODE_STATE_ALLOCATED |
1655
1963
                                                       node_flags;
1656
1964
                                error("Invalid state for node %s, was IDLE "
1658
1966
                                      node_ptr->name, reg_msg->job_count);
1659
1967
                        }
1660
1968
 
1661
 
                        select_g_update_node_config(
1662
 
                                (node_ptr - node_record_table_ptr));
1663
 
                        select_g_update_node_state(
1664
 
                                (node_ptr - node_record_table_ptr),
1665
 
                                node_ptr->node_state);
 
1969
                        select_g_update_node_config(i);
 
1970
                        select_g_update_node_state(i, node_ptr->node_state);
1666
1971
                        _sync_bitmaps(node_ptr, jobs_on_node);
1667
1972
                }
1668
1973
        }
1669
1974
 
1670
1975
        if (prolog_hostlist) {
1671
1976
                hostlist_uniq(prolog_hostlist);
1672
 
                hostlist_ranged_string(prolog_hostlist, sizeof(host_str),
1673
 
                        host_str);
 
1977
                host_str = hostlist_ranged_string_xmalloc(prolog_hostlist);
1674
1978
                error("Prolog failure on nodes %s, set to DOWN", host_str);
 
1979
                xfree(host_str);
1675
1980
                hostlist_destroy(prolog_hostlist);
1676
1981
        }
1677
1982
        if (reg_hostlist) {
1678
1983
                hostlist_uniq(reg_hostlist);
1679
 
                hostlist_ranged_string(reg_hostlist, sizeof(host_str),
1680
 
                                       host_str);
 
1984
                host_str = hostlist_ranged_string_xmalloc(reg_hostlist);
1681
1985
                debug("Nodes %s have registered", host_str);
 
1986
                xfree(host_str);
1682
1987
                hostlist_destroy(reg_hostlist);
1683
1988
        }
1684
1989
        if (return_hostlist) {
1685
1990
                hostlist_uniq(return_hostlist);
1686
 
                hostlist_ranged_string(return_hostlist, sizeof(host_str),
1687
 
                        host_str);
 
1991
                host_str = hostlist_ranged_string_xmalloc(return_hostlist);
1688
1992
                info("Nodes %s returned to service", host_str);
 
1993
                xfree(host_str);
1689
1994
                hostlist_destroy(return_hostlist);
1690
1995
        }
1691
1996
 
1692
 
        if (updated_job) {
 
1997
        if (update_node_state) {
1693
1998
                last_node_update = time (NULL);
1694
1999
                reset_job_priority();
1695
2000
        }
1770
2075
                        node_ptr->node_state = NODE_STATE_IDLE | node_flags;
1771
2076
                if (!IS_NODE_DRAIN(node_ptr) && !IS_NODE_FAIL(node_ptr)) {
1772
2077
                        clusteracct_storage_g_node_up(acct_db_conn,
1773
 
                                                      slurmctld_cluster_name,
1774
2078
                                                      node_ptr, now);
1775
2079
                }
1776
2080
        }
1786
2090
                trigger_node_up(node_ptr);
1787
2091
                if (!IS_NODE_DRAIN(node_ptr) && !IS_NODE_FAIL(node_ptr)) {
1788
2092
                        xfree(node_ptr->reason);
 
2093
                        node_ptr->reason_time = 0;
 
2094
                        node_ptr->reason_uid = NO_VAL;
1789
2095
                        clusteracct_storage_g_node_up(acct_db_conn,
1790
 
                                                      slurmctld_cluster_name,
1791
2096
                                                      node_ptr, now);
1792
2097
                }
1793
2098
        }
1848
2153
{
1849
2154
        int i;
1850
2155
        struct node_record *node_ptr;
1851
 
        char host_str[1024];
 
2156
        char *host_str = NULL;
1852
2157
        hostlist_t no_resp_hostlist = NULL;
1853
2158
 
1854
2159
        for (i=0; i<node_record_count; i++) {
1864
2169
        }
1865
2170
        if (no_resp_hostlist) {
1866
2171
                hostlist_uniq(no_resp_hostlist);
1867
 
                hostlist_ranged_string(no_resp_hostlist,
1868
 
                                       sizeof(host_str), host_str);
 
2172
                host_str = hostlist_ranged_string_xmalloc(no_resp_hostlist);
1869
2173
                error("Nodes %s not responding", host_str);
 
2174
                xfree(host_str);
1870
2175
                hostlist_destroy(no_resp_hostlist);
1871
2176
        }
1872
2177
}
1905
2210
 
1906
2211
        node_ptr = find_node_record (name);
1907
2212
        if (node_ptr == NULL) {
1908
 
                error ("node_not_resp unable to find node %s", name);
 
2213
                error ("set_node_down unable to find node %s", name);
1909
2214
                return;
1910
2215
        }
1911
2216
 
1912
2217
        if ((node_ptr->reason == NULL) ||
1913
2218
            (strncmp(node_ptr->reason, "Not responding", 14) == 0)) {
1914
 
                time_t now;
1915
 
                char time_buf[64], time_str[32];
1916
 
 
1917
 
                now = time (NULL);
1918
 
                slurm_make_time_str(&now, time_str, sizeof(time_str));
1919
 
                snprintf(time_buf, sizeof(time_buf), " [slurm@%s]",
1920
 
                        time_str);
1921
2219
                xfree(node_ptr->reason);
1922
2220
                node_ptr->reason = xstrdup(reason);
1923
 
                xstrcat(node_ptr->reason, time_buf);
 
2221
                node_ptr->reason_time = time(NULL);
 
2222
                node_ptr->reason_uid = slurm_get_slurm_user_id();
1924
2223
        }
1925
2224
        _make_node_down(node_ptr, now);
1926
2225
        (void) kill_running_job_by_node_name(name);
2035
2334
 * IN job_ptr  - pointer to job that is starting
2036
2335
 */
2037
2336
extern void make_node_alloc(struct node_record *node_ptr,
2038
 
                            struct job_record *job_ptr)
 
2337
                            struct job_record *job_ptr)
2039
2338
{
2040
2339
        int inx = node_ptr - node_record_table_ptr;
2041
2340
        uint16_t node_flags;
2052
2351
        node_flags = node_ptr->node_state & NODE_STATE_FLAGS;
2053
2352
        node_ptr->node_state = NODE_STATE_ALLOCATED | node_flags;
2054
2353
        xfree(node_ptr->reason);
 
2354
        node_ptr->reason_time = 0;
 
2355
        node_ptr->reason_uid = NO_VAL;
2055
2356
}
2056
2357
 
2057
2358
/* make_node_comp - flag specified node as completing a job
2068
2369
 
2069
2370
        xassert(node_ptr);
2070
2371
        last_node_update = now;
2071
 
        if (!suspended) {
 
2372
        if (suspended) {
 
2373
                if (node_ptr->sus_job_cnt)
 
2374
                        (node_ptr->sus_job_cnt)--;
 
2375
                else
 
2376
                        error("Node %s sus_job_cnt underflow in "
 
2377
                                "make_node_comp", node_ptr->name);
 
2378
        } else {
2072
2379
                if (node_ptr->run_job_cnt)
2073
2380
                        (node_ptr->run_job_cnt)--;
2074
2381
                else
2090
2397
                /* Don't verify  RPC if DOWN */
2091
2398
                (node_ptr->comp_job_cnt)++;
2092
2399
                node_ptr->node_state |= NODE_STATE_COMPLETING;
 
2400
                bit_set(cg_node_bitmap, inx);
2093
2401
        }
2094
2402
        node_flags = node_ptr->node_state & NODE_STATE_FLAGS;
2095
2403
 
2098
2406
                bit_set(idle_node_bitmap, inx);
2099
2407
                if (IS_NODE_DRAIN(node_ptr) || IS_NODE_FAIL(node_ptr)) {
2100
2408
                        trigger_node_drained(node_ptr);
2101
 
                        clusteracct_storage_g_node_down(acct_db_conn,
2102
 
                                                        slurmctld_cluster_name,
2103
 
                                                        node_ptr, now, NULL);
 
2409
                        clusteracct_storage_g_node_down(
 
2410
                                acct_db_conn,
 
2411
                                node_ptr, now, NULL,
 
2412
                                slurm_get_slurm_user_id());
2104
2413
                }
2105
2414
        }
2106
2415
 
2127
2436
        node_flags &= (~NODE_STATE_COMPLETING);
2128
2437
        node_ptr->node_state = NODE_STATE_DOWN | node_flags;
2129
2438
        bit_clear (avail_node_bitmap, inx);
 
2439
        bit_clear (cg_node_bitmap,    inx);
2130
2440
        bit_set   (idle_node_bitmap,  inx);
2131
2441
        bit_set   (share_node_bitmap, inx);
2132
2442
        bit_clear (up_node_bitmap,    inx);
2133
2443
        select_g_update_node_state(inx, node_ptr->node_state);
2134
2444
        trigger_node_down(node_ptr);
2135
2445
        clusteracct_storage_g_node_down(acct_db_conn,
2136
 
                                        slurmctld_cluster_name,
2137
 
                                        node_ptr, event_time, NULL);
 
2446
                                        node_ptr, event_time, NULL,
 
2447
                                        node_ptr->reason_uid);
2138
2448
}
2139
2449
 
2140
2450
/*
2148
2458
        int inx = node_ptr - node_record_table_ptr;
2149
2459
        uint16_t node_flags;
2150
2460
        time_t now = time(NULL);
 
2461
        bitstr_t *node_bitmap = NULL;
 
2462
 
 
2463
        if (job_ptr) { /* Specific job completed */
 
2464
                if(job_ptr->node_bitmap_cg)
 
2465
                        node_bitmap = job_ptr->node_bitmap_cg;
 
2466
                else
 
2467
                        node_bitmap = job_ptr->node_bitmap;
 
2468
        }
2151
2469
 
2152
2470
        xassert(node_ptr);
2153
 
        if (job_ptr &&                  /* Specific job completed */
2154
 
            (bit_test(job_ptr->node_bitmap, inx))) {    /* Not a replay */
 
2471
        if (node_bitmap && (bit_test(node_bitmap, inx))) {
 
2472
                /* Not a replay */
2155
2473
                last_job_update = now;
2156
 
                bit_clear(job_ptr->node_bitmap, inx);
 
2474
                bit_clear(node_bitmap, inx);
2157
2475
 
2158
2476
                job_update_cpu_cnt(job_ptr, inx);
2159
2477
 
2174
2492
                              job_ptr->job_id);
2175
2493
                }
2176
2494
 
2177
 
                if (IS_JOB_RUNNING(job_ptr)) {
 
2495
                if (IS_JOB_SUSPENDED(job_ptr)) {
 
2496
                        /* Remove node from suspended job */
 
2497
                        if (node_ptr->sus_job_cnt)
 
2498
                                (node_ptr->sus_job_cnt)--;
 
2499
                        else
 
2500
                                error("Node %s sus_job_cnt underflow in "
 
2501
                                        "make_node_idle", node_ptr->name);
 
2502
                } else if (IS_JOB_RUNNING(job_ptr)) {
2178
2503
                        /* Remove node from running job */
2179
2504
                        if (node_ptr->run_job_cnt)
2180
2505
                                (node_ptr->run_job_cnt)--;
2194
2519
        }
2195
2520
 
2196
2521
        last_node_update = now;
2197
 
        if (node_ptr->comp_job_cnt == 0)
 
2522
        if (node_ptr->comp_job_cnt == 0) {
2198
2523
                node_ptr->node_state &= (~NODE_STATE_COMPLETING);
 
2524
                bit_clear(cg_node_bitmap, inx);
 
2525
        }
2199
2526
        node_flags = node_ptr->node_state & NODE_STATE_FLAGS;
2200
2527
        if (IS_NODE_DOWN(node_ptr)) {
2201
2528
                debug3("make_node_idle: Node %s being left DOWN",
2202
2529
                        node_ptr->name);
2203
2530
        } else if ((IS_NODE_DRAIN(node_ptr) || IS_NODE_FAIL(node_ptr)) &&
2204
 
                   (node_ptr->run_job_cnt == 0) &&
2205
 
                   (node_ptr->comp_job_cnt == 0)) {
 
2531
                   (node_ptr->run_job_cnt == 0) &&
 
2532
                   (node_ptr->comp_job_cnt == 0)) {
2206
2533
                node_ptr->node_state = NODE_STATE_IDLE | node_flags;
2207
 
                bit_set(idle_node_bitmap, inx);
2208
2534
                bit_clear(avail_node_bitmap, inx);
 
2535
                bit_set(idle_node_bitmap, inx);
 
2536
                bit_set(up_node_bitmap, inx);
2209
2537
                debug3("make_node_idle: Node %s is DRAINED",
2210
2538
                       node_ptr->name);
2211
2539
                node_ptr->last_idle = now;
2212
2540
                trigger_node_drained(node_ptr);
2213
2541
                clusteracct_storage_g_node_down(acct_db_conn,
2214
 
                                                slurmctld_cluster_name,
2215
 
                                                node_ptr, now, NULL);
 
2542
                                                node_ptr, now, NULL,
 
2543
                                                slurm_get_slurm_user_id());
2216
2544
        } else if (node_ptr->run_job_cnt) {
2217
2545
                node_ptr->node_state = NODE_STATE_ALLOCATED | node_flags;
 
2546
                if (!IS_NODE_NO_RESPOND(node_ptr) &&
 
2547
                     !IS_NODE_FAIL(node_ptr) && !IS_NODE_DRAIN(node_ptr))
 
2548
                        bit_set(avail_node_bitmap, inx);
 
2549
                bit_set(up_node_bitmap, inx);
2218
2550
        } else {
2219
2551
                node_ptr->node_state = NODE_STATE_IDLE | node_flags;
2220
2552
                if (!IS_NODE_NO_RESPOND(node_ptr) &&
 
2553
                     !IS_NODE_FAIL(node_ptr) && !IS_NODE_DRAIN(node_ptr))
 
2554
                        bit_set(avail_node_bitmap, inx);
 
2555
                if (!IS_NODE_NO_RESPOND(node_ptr) &&
2221
2556
                    !IS_NODE_COMPLETING(node_ptr))
2222
2557
                        bit_set(idle_node_bitmap, inx);
 
2558
                bit_set(up_node_bitmap, inx);
2223
2559
                node_ptr->last_idle = now;
2224
2560
        }
2225
2561
}
2282
2618
 
2283
2619
                                        rc = clusteracct_storage_g_node_down(
2284
2620
                                                acct_db_conn,
2285
 
                                                slurmctld_cluster_name,
2286
2621
                                                &send_node, event_time,
2287
 
                                                reason);
 
2622
                                                reason,
 
2623
                                                slurm_get_slurm_user_id());
2288
2624
                                }
2289
2625
                        }
2290
2626
                } else
2291
2627
                        rc = clusteracct_storage_g_node_down(
2292
2628
                                acct_db_conn,
2293
 
                                slurmctld_cluster_name,
2294
2629
                                node_ptr, event_time,
2295
 
                                reason);
 
2630
                                reason,
 
2631
                                slurm_get_slurm_user_id());
2296
2632
                if (rc == SLURM_ERROR)
2297
2633
                        break;
2298
2634
        }
2303
2639
/* node_fini - free all memory associated with node records */
2304
2640
extern void node_fini (void)
2305
2641
{
 
2642
        FREE_NULL_BITMAP(avail_node_bitmap);
 
2643
        FREE_NULL_BITMAP(cg_node_bitmap);
2306
2644
        FREE_NULL_BITMAP(idle_node_bitmap);
2307
 
        FREE_NULL_BITMAP(avail_node_bitmap);
2308
2645
        FREE_NULL_BITMAP(power_node_bitmap);
2309
2646
        FREE_NULL_BITMAP(share_node_bitmap);
2310
2647
        FREE_NULL_BITMAP(up_node_bitmap);
2311
2648
        node_fini2();
2312
2649
}
2313