← Back to branch summary

~ubuntu-branches/ubuntu/quantal/slurm-llnl/quantal

« back to all changes in this revision

Viewing changes to src/plugins/select/cons_res/select_cons_res.c

  • Committer: Bazaar Package Importer
  • Author(s): Gennaro Oliva
  • Date: 2011-04-08 11:21:17 UTC
  • mfrom: (3.3.16 sid)
  • Revision ID: james.westby@ubuntu.com-20110408112117-nfnyq9dtm55hqoaw
Tags: 2.2.4-1
http://bugs.debian.org/575822
* New upstream releases 
* Cleaning spare file and directories, not belonging to the sources
  generated by the building process and not removed by distclean.
  Added debian/clean with spare files and rm -rf inside debian/rules
  for directories.
* Added new packages libslurm-perl, libslurmdb-perl, slurm-llnl-torque
  (Closes: #575822) thanks to Julien Blache

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/plugins/select/cons_res/select_cons_res.c
39
39
 *  Once Job 2 finishes, Job 5, which was pending, is allocated
40
40
 *  available resources and is then running as illustrated below:
41
41
 *
42
 
 *  [<snip>]# squeue4
 
42
 *  [<snip>]# squeue
43
43
 *   JOBID PARTITION    NAME     USER  ST       TIME  NODES NODELIST(REASON)
44
44
 *     3        lsf    sleep     root   R       1:58      3 linux[01-03]
45
45
 *     4        lsf    sleep     root   R       1:54      1 linux04
48
48
 *
49
49
 *  Job 3, Job 4, and Job 5 are now running concurrently on the cluster.
50
50
 *
51
 
 *  [<snip>]#  squeue4
 
51
 *  [<snip>]#  squeue
52
52
 *  JOBID PARTITION     NAME     USER  ST       TIME  NODES NODELIST(REASON)
53
 
 *     5        lsf    sleep     root   R       1:52      3 xc14n[13-15]
 
53
 *     5        lsf    sleep     root   R       1:52      3 linux[01-03]
54
54
 *  [<snip>]#
55
55
 *
56
56
 * The advantage of the consumable resource scheduling policy is that
106
106
#include "dist_tasks.h"
107
107
#include "job_test.h"
108
108
 
109
 
#if(0)
110
 
#define CR_DEBUG 1
111
 
#endif
112
109
 
113
110
#define NODEINFO_MAGIC 0x82aa
114
111
 
117
114
 * overwritten when linking with the slurmctld.
118
115
 */
119
116
#if defined (__APPLE__)
120
 
        slurm_ctl_conf_t slurmctld_conf __attribute__((weak_import));
121
 
        struct node_record *node_record_table_ptr __attribute__((weak_import));
122
 
        List part_list __attribute__((weak_import));
123
 
        List job_list __attribute__((weak_import));
124
 
        int node_record_count __attribute__((weak_import));
125
 
        time_t last_node_update __attribute__((weak_import));
126
 
        struct switch_record *switch_record_table __attribute__((weak_import));
127
 
        int switch_record_cnt __attribute__((weak_import));
128
 
        bitstr_t *avail_node_bitmap __attribute__((weak_import));
129
 
        bitstr_t *idle_node_bitmap __attribute__((weak_import));
130
 
        List slurm_find_preemptable_jobs(struct job_record *job_ptr)
131
 
                        __attribute__((weak_import));
 
117
slurm_ctl_conf_t slurmctld_conf __attribute__((weak_import));
 
118
struct node_record *node_record_table_ptr __attribute__((weak_import));
 
119
List part_list __attribute__((weak_import));
 
120
List job_list __attribute__((weak_import));
 
121
int node_record_count __attribute__((weak_import));
 
122
time_t last_node_update __attribute__((weak_import));
 
123
struct switch_record *switch_record_table __attribute__((weak_import));
 
124
int switch_record_cnt __attribute__((weak_import));
 
125
bitstr_t *avail_node_bitmap __attribute__((weak_import));
 
126
bitstr_t *idle_node_bitmap __attribute__((weak_import));
132
127
#else
133
 
        slurm_ctl_conf_t slurmctld_conf;
134
 
        struct node_record *node_record_table_ptr;
135
 
        List part_list;
136
 
        List job_list;
137
 
        int node_record_count;
138
 
        time_t last_node_update;
139
 
        struct switch_record *switch_record_table;
140
 
        int switch_record_cnt;
141
 
        bitstr_t *avail_node_bitmap;
142
 
        bitstr_t *idle_node_bitmap;
143
 
        List slurm_find_preemptable_jobs(struct job_record *job_ptr);
 
128
slurm_ctl_conf_t slurmctld_conf;
 
129
struct node_record *node_record_table_ptr;
 
130
List part_list;
 
131
List job_list;
 
132
int node_record_count;
 
133
time_t last_node_update;
 
134
struct switch_record *switch_record_table;
 
135
int switch_record_cnt;
 
136
bitstr_t *avail_node_bitmap;
 
137
bitstr_t *idle_node_bitmap;
144
138
#endif
145
139
 
146
140
/*
169
163
 * of the plugin.  If major and minor revisions are desired, the major
170
164
 * version number may be multiplied by a suitable magnitude constant such
171
165
 * as 100 or 1000.  Various SLURM versions will likely require a certain
172
 
 * minimum versions for their plugins as the node selection API matures.
 
166
 * minimum version for their plugins as the node selection API matures.
173
167
 */
174
168
const char plugin_name[] = "Consumable Resources (CR) Node Selection plugin";
175
169
const char plugin_type[] = "select/cons_res";
 
170
const uint32_t plugin_id      = 101;
176
171
const uint32_t plugin_version = 91;
177
172
const uint32_t pstate_version = 7;      /* version control on saved state */
178
173
 
179
 
select_type_plugin_info_t cr_type = CR_CPU; /* cr_type is overwritten in init() */
 
174
uint16_t cr_type = CR_CPU; /* cr_type is overwritten in init() */
180
175
 
 
176
uint32_t select_debug_flags;
181
177
uint16_t select_fast_schedule;
182
178
 
183
179
uint16_t *cr_node_num_cores = NULL;
184
 
uint32_t *cr_num_core_count = NULL;
 
180
uint32_t *cr_node_cores_offset = NULL;
185
181
struct part_res_record *select_part_record = NULL;
186
182
struct node_res_record *select_node_record = NULL;
187
183
struct node_use_record *select_node_usage  = NULL;
 
184
static bool select_state_initializing = true;
188
185
static int select_node_cnt = 0;
189
186
static bool job_preemption_enabled = false;
190
187
static bool job_preemption_killing = false;
199
196
extern int select_p_select_nodeinfo_free(select_nodeinfo_t *nodeinfo);
200
197
 
201
198
/* Procedure Declarations */
 
199
static int _rm_job_from_one_node(struct job_record *job_ptr,
 
200
                                 struct node_record *node_ptr);
202
201
static int _run_now(struct job_record *job_ptr, bitstr_t *bitmap,
203
202
                    uint32_t min_nodes, uint32_t max_nodes,
204
203
                    uint32_t req_nodes, uint16_t job_node_req,
207
206
                      uint32_t min_nodes, uint32_t max_nodes,
208
207
                      uint32_t req_nodes, uint16_t job_node_req);
209
208
static int _will_run_test(struct job_record *job_ptr, bitstr_t *bitmap,
210
 
                        uint32_t min_nodes, uint32_t max_nodes,
211
 
                        uint32_t req_nodes, uint16_t job_node_req,
212
 
                        List preemptee_candidates, List *preemptee_job_list);
213
 
 
214
 
#if (CR_DEBUG)
 
209
                          uint32_t min_nodes, uint32_t max_nodes,
 
210
                          uint32_t req_nodes, uint16_t job_node_req,
 
211
                          List preemptee_candidates, List *preemptee_job_list);
215
212
 
216
213
static void _dump_job_res(struct job_resources *job) {
217
214
        char str[64];
223
220
        info("DEBUG: Dump job_resources: nhosts %u cb %s", job->nhosts, str);
224
221
}
225
222
 
226
 
static void _dump_nodes()
 
223
static void _dump_nodes(void)
227
224
{
 
225
        struct node_record *node_ptr;
 
226
        List gres_list;
228
227
        int i;
229
228
 
230
229
        for (i=0; i<select_node_cnt; i++) {
 
230
                node_ptr = select_node_record[i].node_ptr;
231
231
                info("node:%s cpus:%u c:%u s:%u t:%u mem:%u a_mem:%u state:%d",
232
 
                        select_node_record[i].node_ptr->name,
233
 
                        select_node_record[i].cpus,
234
 
                        select_node_record[i].cores,
235
 
                        select_node_record[i].sockets,
236
 
                        select_node_record[i].vpus,
237
 
                        select_node_record[i].real_memory,
238
 
                        select_node_usage[i].alloc_memory,
239
 
                        select_node_usage[i].node_state);
 
232
                     node_ptr->name,
 
233
                     select_node_record[i].cpus,
 
234
                     select_node_record[i].cores,
 
235
                     select_node_record[i].sockets,
 
236
                     select_node_record[i].vpus,
 
237
                     select_node_record[i].real_memory,
 
238
                     select_node_usage[i].alloc_memory,
 
239
                     select_node_usage[i].node_state);
 
240
 
 
241
                if (select_node_usage[i].gres_list)
 
242
                        gres_list = select_node_usage[i].gres_list;
 
243
                else
 
244
                        gres_list = node_ptr->gres_list;
 
245
                if (gres_list)
 
246
                        gres_plugin_node_state_log(gres_list, node_ptr->name);
240
247
        }
241
248
}
242
249
 
244
251
{
245
252
        uint16_t i;
246
253
        info("part:%s rows:%u pri:%u ", p_ptr->part_ptr->name, p_ptr->num_rows,
247
 
                p_ptr->part_ptr->priority);
 
254
             p_ptr->part_ptr->priority);
248
255
        if (!p_ptr->row)
249
256
                return;
250
257
 
256
263
                        sprintf(str, "[no row_bitmap]");
257
264
                }
258
265
                info("  row%u: num_jobs %u: bitmap: %s", i,
259
 
                        p_ptr->row[i].num_jobs, str);
 
266
                     p_ptr->row[i].num_jobs, str);
260
267
        }
261
268
}
262
269
 
270
277
        }
271
278
        return;
272
279
}
273
 
#endif
274
 
 
275
 
/*  */
276
 
extern bool cr_preemption_enabled(void)
277
 
{
278
 
        if (!job_preemption_tested) {
279
 
                uint16_t mode = slurm_get_preempt_mode();
280
 
                mode &= ~PREEMPT_MODE_GANG;
281
 
                if (mode == PREEMPT_MODE_SUSPEND)
282
 
                        job_preemption_enabled = true;
283
 
                else if ((mode == PREEMPT_MODE_CANCEL)     ||
284
 
                         (mode == PREEMPT_MODE_CHECKPOINT) ||
285
 
                         (mode == PREEMPT_MODE_REQUEUE)) {
286
 
                        job_preemption_enabled = true;
287
 
                        job_preemption_killing = true;
288
 
                }
289
 
                job_preemption_tested = true;
290
 
        }
291
 
        return job_preemption_enabled;
292
 
}
293
 
extern bool cr_preemption_killing(void)
294
 
{
295
 
        (void) cr_preemption_enabled();
296
 
        return job_preemption_killing;
297
 
 
298
 
}
299
 
 
300
 
#define CR_NUM_CORE_ARRAY_INCREMENT 8
301
 
 
302
 
/* (re)set cr_node_num_cores and cr_num_core_count arrays */
 
280
 
 
281
/* (re)set cr_node_num_cores arrays */
303
282
static void _init_global_core_data(struct node_record *node_ptr, int node_cnt)
304
283
{
305
 
        uint32_t i, n, array_size = CR_NUM_CORE_ARRAY_INCREMENT;
 
284
        uint32_t n;
306
285
 
307
 
        xfree(cr_num_core_count);
308
286
        xfree(cr_node_num_cores);
309
 
        cr_node_num_cores = xmalloc(array_size * sizeof(uint16_t));
310
 
        cr_num_core_count = xmalloc(array_size * sizeof(uint32_t));
311
 
 
312
 
        for (i = 0, n = 0; n < node_cnt; n++) {
 
287
        cr_node_num_cores = xmalloc(node_cnt * sizeof(uint16_t));
 
288
 
 
289
        xfree(cr_node_cores_offset);
 
290
        cr_node_cores_offset = xmalloc((node_cnt+1) * sizeof(uint32_t));
 
291
 
 
292
        for (n = 0; n < node_cnt; n++) {
313
293
                uint16_t cores;
314
294
                if (select_fast_schedule) {
315
295
                        cores  = node_ptr[n].config_ptr->cores;
318
298
                        cores  = node_ptr[n].cores;
319
299
                        cores *= node_ptr[n].sockets;
320
300
                }
321
 
                if (cr_node_num_cores[i] == cores) {
322
 
                        cr_num_core_count[i]++;
323
 
                        continue;
324
 
                }
325
 
                if (cr_num_core_count[i] > 0) {
326
 
                        if (++i >= array_size) {
327
 
                                array_size += CR_NUM_CORE_ARRAY_INCREMENT;
328
 
                                xrealloc(cr_node_num_cores,
329
 
                                        array_size * sizeof(uint16_t));
330
 
                                xrealloc(cr_num_core_count,
331
 
                                        array_size * sizeof(uint32_t));
332
 
                        }
333
 
                }
334
 
                cr_node_num_cores[i] = cores;
335
 
                cr_num_core_count[i] = 1;
336
 
        }
337
 
        /* make sure we have '0'-terminate the arrays */
338
 
        if (++i >= array_size) {
339
 
                array_size = i + 1;
340
 
                xrealloc(cr_node_num_cores, array_size * sizeof(uint16_t));
341
 
                xrealloc(cr_num_core_count, array_size * sizeof(uint32_t));
342
 
        }
 
301
                cr_node_num_cores[n] = cores;
 
302
                if (n > 0) {
 
303
                        cr_node_cores_offset[n] = cr_node_cores_offset[n-1] +
 
304
                                                  cr_node_num_cores[n-1] ;
 
305
                } else
 
306
                        cr_node_cores_offset[0] = 0;
 
307
        }
 
308
 
 
309
        /* an extra value is added to get the total number of cores */
 
310
        /* as cr_get_coremap_offset is sometimes used to get the total */
 
311
        /* number of cores in the cluster */
 
312
        cr_node_cores_offset[node_cnt] = cr_node_cores_offset[node_cnt-1] +
 
313
                                         cr_node_num_cores[node_cnt-1] ;
 
314
 
343
315
}
344
316
 
345
317
 
346
318
/* return the coremap index to the first core of the given node */
347
319
extern uint32_t cr_get_coremap_offset(uint32_t node_index)
348
320
{
349
 
        uint32_t i;
350
 
        uint32_t cindex = 0;
351
 
        uint32_t n = cr_num_core_count[0];
352
 
        for (i = 0; cr_num_core_count[i] && node_index > n; i++) {
353
 
                cindex += cr_node_num_cores[i] * cr_num_core_count[i];
354
 
                n += cr_num_core_count[i+1];
355
 
        }
356
 
        if (!cr_num_core_count[i])
357
 
                return cindex;
358
 
        n -= cr_num_core_count[i];
359
 
 
360
 
        cindex += cr_node_num_cores[i] * (node_index-n);
361
 
        return cindex;
362
 
}
363
 
 
364
 
 
365
 
/* return the total number of cores in a given node */
366
 
extern uint32_t cr_get_node_num_cores(uint32_t node_index)
367
 
{
368
 
        uint32_t i = 0;
369
 
        uint32_t pos = cr_num_core_count[i++];
370
 
        while (node_index >= pos) {
371
 
                pos += cr_num_core_count[i++];
372
 
        }
373
 
        return cr_node_num_cores[i-1];
 
321
        return cr_node_cores_offset[node_index];
374
322
}
375
323
 
376
324
 
395
343
                        continue;
396
344
                /* copy the job list */
397
345
                new_row[i].job_list = xmalloc(new_row[i].job_list_size *
398
 
                                                        sizeof(bitstr_t *));
 
346
                                              sizeof(bitstr_t *));
399
347
                for (j = 0; j < new_row[i].num_jobs; j++) {
400
348
                        new_row[i].job_list[j] = orig_row[i].job_list[j];
401
349
                }
434
382
static struct node_use_record *_dup_node_usage(struct node_use_record *orig_ptr)
435
383
{
436
384
        struct node_use_record *new_use_ptr, *new_ptr;
 
385
        List gres_list;
437
386
        uint32_t i;
438
387
 
439
388
        if (orig_ptr == NULL)
440
389
                return NULL;
441
390
 
442
 
        new_use_ptr = xmalloc(select_node_cnt *
443
 
                              sizeof(struct node_use_record));
 
391
        new_use_ptr = xmalloc(select_node_cnt * sizeof(struct node_use_record));
444
392
        new_ptr = new_use_ptr;
445
393
 
446
394
        for (i = 0; i < select_node_cnt; i++) {
447
395
                new_ptr[i].node_state   = orig_ptr[i].node_state;
448
396
                new_ptr[i].alloc_memory = orig_ptr[i].alloc_memory;
 
397
                if (orig_ptr[i].gres_list)
 
398
                        gres_list = orig_ptr[i].gres_list;
 
399
                else
 
400
                        gres_list = node_record_table_ptr[i].gres_list;
 
401
                new_ptr[i].gres_list = gres_plugin_node_state_dup(gres_list);
449
402
        }
450
403
        return new_use_ptr;
451
404
}
483
436
 
484
437
 
485
438
/* (re)create the global select_part_record array */
486
 
static void _create_part_data()
 
439
static void _create_part_data(void)
487
440
{
488
441
        ListIterator part_iterator;
489
442
        struct part_record *p_ptr;
538
491
 
539
492
/* delete the given select_node_record and select_node_usage arrays */
540
493
static void _destroy_node_data(struct node_use_record *node_usage,
541
 
                                struct node_res_record *node_data)
 
494
                               struct node_res_record *node_data)
542
495
{
543
496
        xfree(node_data);
544
 
        xfree(node_usage);
 
497
        if (node_usage) {
 
498
                if (node_usage->gres_list)
 
499
                        list_destroy(node_usage->gres_list);
 
500
                xfree(node_usage);
 
501
        }
545
502
}
546
503
 
547
504
 
554
511
                uint32_t size = bit_size(r_ptr->row_bitmap);
555
512
                bit_nclear(r_ptr->row_bitmap, 0, size-1);
556
513
        }
557
 
        add_job_to_cores(job, &(r_ptr->row_bitmap), cr_node_num_cores,
558
 
                                cr_num_core_count);
 
514
        add_job_to_cores(job, &(r_ptr->row_bitmap), cr_node_num_cores);
559
515
 
560
516
        /*  add the job to the job_list */
561
517
        if (r_ptr->num_jobs >= r_ptr->job_list_size) {
562
518
                r_ptr->job_list_size += 8;
563
519
                xrealloc(r_ptr->job_list, r_ptr->job_list_size *
564
 
                                        sizeof(struct job_resources *));
 
520
                         sizeof(struct job_resources *));
565
521
        }
566
522
        r_ptr->job_list[r_ptr->num_jobs++] = job;
567
523
}
571
527
static int _can_job_fit_in_row(struct job_resources *job,
572
528
                               struct part_row_data *r_ptr)
573
529
{
574
 
        if (r_ptr->num_jobs == 0 || !r_ptr->row_bitmap)
 
530
        if ((r_ptr->num_jobs == 0) || !r_ptr->row_bitmap)
575
531
                return 1;
576
 
        return job_fits_into_cores(job, r_ptr->row_bitmap,
577
 
                                        cr_node_num_cores, cr_num_core_count);
 
532
 
 
533
        return job_fits_into_cores(job, r_ptr->row_bitmap, cr_node_num_cores);
578
534
}
579
535
 
580
536
 
689
645
                return;
690
646
        }
691
647
 
692
 
#if (CR_DEBUG)
693
 
        info("DEBUG: _build_row_bitmaps (before):");
694
 
        _dump_part(p_ptr);
695
 
#endif
 
648
        if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
 
649
                info("DEBUG: _build_row_bitmaps (before):");
 
650
                _dump_part(p_ptr);
 
651
        }
696
652
        debug3("cons_res: build_row_bitmaps reshuffling %u jobs", num_jobs);
697
653
 
698
654
        /* make a copy, in case we cannot do better than this */
734
690
         */
735
691
        for (i = 0; i < num_jobs; i++) {
736
692
                for (j = i+1; j < num_jobs; j++) {
737
 
                        if (jstart[j] < jstart[i] || (jstart[j] == jstart[i] &&
738
 
                            tmpjobs[j]->nprocs > tmpjobs[i]->nprocs)) {
 
693
                        if (jstart[j] < jstart[i]
 
694
                            || (jstart[j] == jstart[i] &&
 
695
                                tmpjobs[j]->ncpus > tmpjobs[i]->ncpus)) {
739
696
                                x = jstart[i];
740
697
                                jstart[i] = jstart[j];
741
698
                                jstart[j] = x;
746
703
                }
747
704
        }
748
705
 
749
 
#if (CR_DEBUG)
750
 
        for (i = 0; i < num_jobs; i++) {
751
 
                char cstr[64], nstr[64];
752
 
                if (tmpjobs[i]->core_bitmap) {
753
 
                        bit_fmt(cstr, (sizeof(cstr)-1) ,
754
 
                                tmpjobs[i]->core_bitmap);
755
 
                } else
756
 
                        sprintf(cstr, "[no core_bitmap]");
757
 
                if (tmpjobs[i]->node_bitmap) {
758
 
                        bit_fmt(nstr, (sizeof(nstr)-1),
759
 
                                tmpjobs[i]->node_bitmap);
760
 
                } else
761
 
                        sprintf(nstr, "[no node_bitmap]");
762
 
                info ("DEBUG:  jstart %d job nb %s cb %s", jstart[i], nstr,
763
 
                        cstr);
 
706
        if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
 
707
                for (i = 0; i < num_jobs; i++) {
 
708
                        char cstr[64], nstr[64];
 
709
                        if (tmpjobs[i]->core_bitmap) {
 
710
                                bit_fmt(cstr, (sizeof(cstr)-1) ,
 
711
                                        tmpjobs[i]->core_bitmap);
 
712
                        } else
 
713
                                sprintf(cstr, "[no core_bitmap]");
 
714
                        if (tmpjobs[i]->node_bitmap) {
 
715
                                bit_fmt(nstr, (sizeof(nstr)-1),
 
716
                                        tmpjobs[i]->node_bitmap);
 
717
                        } else
 
718
                                sprintf(nstr, "[no node_bitmap]");
 
719
                        info("DEBUG:  jstart %d job nb %s cb %s", jstart[i],
 
720
                             nstr, cstr);
 
721
                }
764
722
        }
765
 
#endif
766
723
 
767
724
        /* add jobs to the rows */
768
725
        for (j = 0; j < num_jobs; j++) {
788
745
                 * algorithm couldn't improve apon the existing layout.
789
746
                 * Thus, we'll restore the original layout here */
790
747
                debug3("cons_res: build_row_bitmap: dangling job found");
791
 
#if (CR_DEBUG)
792
 
                info("DEBUG: _build_row_bitmaps (post-algorithm):");
793
 
                _dump_part(p_ptr);
794
 
#endif
 
748
 
 
749
                if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
 
750
                        info("DEBUG: _build_row_bitmaps (post-algorithm):");
 
751
                        _dump_part(p_ptr);
 
752
                }
 
753
 
795
754
                _destroy_row_data(p_ptr->row, p_ptr->num_rows);
796
755
                p_ptr->row = orig_row;
797
756
                orig_row = NULL;
804
763
                        if (p_ptr->row[i].num_jobs == 0)
805
764
                                continue;
806
765
                        for (j = 0; j < p_ptr->row[i].num_jobs; j++) {
807
 
                                add_job_to_cores(
808
 
                                                p_ptr->row[i].job_list[j],
809
 
                                                &(p_ptr->row[i].row_bitmap),
810
 
                                                cr_node_num_cores,
811
 
                                                cr_num_core_count);
 
766
                                add_job_to_cores(p_ptr->row[i].job_list[j],
 
767
                                                 &(p_ptr->row[i].row_bitmap),
 
768
                                                 cr_node_num_cores);
812
769
                        }
813
770
                }
814
771
        }
815
772
 
816
 
#if (CR_DEBUG)
817
 
        info("DEBUG: _build_row_bitmaps (after):");
818
 
        _dump_part(p_ptr);
819
 
#endif
 
773
        if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
 
774
                info("DEBUG: _build_row_bitmaps (after):");
 
775
                _dump_part(p_ptr);
 
776
        }
820
777
 
821
778
        if (orig_row)
822
779
                _destroy_row_data(orig_row, p_ptr->num_rows);
859
816
 * - add 'struct job_resources' resources to 'struct part_res_record'
860
817
 * - add job's memory requirements to 'struct node_res_record'
861
818
 *
862
 
 * if action = 0 then add cores and memory
863
 
 * if action = 1 then only add memory (job is suspended)
864
 
 * if action = 2 then only add cores (job is resumed)
 
819
 * if action = 0 then add cores and memory (starting new job)
 
820
 * if action = 1 then only add memory (adding suspended job)
 
821
 * if action = 2 then only add cores (suspended job is resumed)
865
822
 */
866
823
static int _add_job_to_res(struct job_record *job_ptr, int action)
867
824
{
868
825
        struct job_resources *job = job_ptr->job_resrcs;
 
826
        struct node_record *node_ptr;
869
827
        struct part_res_record *p_ptr;
 
828
        List gres_list;
870
829
        int i, n;
871
830
 
872
831
        if (!job || !job->core_bitmap) {
875
834
        }
876
835
 
877
836
        debug3("cons_res: _add_job_to_res: job %u act %d ", job_ptr->job_id,
878
 
                action);
879
 
 
880
 
#if (CR_DEBUG)
881
 
        _dump_job_res(job);
882
 
#endif
883
 
 
884
 
        /* add memory */
885
 
        if (action != 2) {
886
 
                for (i = 0, n = 0; i < select_node_cnt; i++) {
887
 
                        if (!bit_test(job->node_bitmap, i))
888
 
                                continue;
 
837
               action);
 
838
 
 
839
        if (select_debug_flags & DEBUG_FLAG_CPU_BIND)
 
840
                _dump_job_res(job);
 
841
 
 
842
        for (i = 0, n = -1; i < select_node_cnt; i++) {
 
843
                if (!bit_test(job->node_bitmap, i))
 
844
                        continue;
 
845
                n++;
 
846
 
 
847
                node_ptr = select_node_record[i].node_ptr;
 
848
                if (action != 2) {
 
849
                        if (select_node_usage[i].gres_list)
 
850
                                gres_list = select_node_usage[i].gres_list;
 
851
                        else
 
852
                                gres_list = node_ptr->gres_list;
 
853
                        gres_plugin_job_alloc(job_ptr->gres_list, gres_list,
 
854
                                              job->nhosts, n, job->cpus[n],
 
855
                                              job_ptr->job_id, node_ptr->name);
 
856
                        gres_plugin_node_state_log(gres_list, node_ptr->name);
 
857
                }
 
858
 
 
859
                if (action != 2) {
 
860
                        if (job->memory_allocated[n] == 0)
 
861
                                continue;       /* node lost by job resizing */
889
862
                        select_node_usage[i].alloc_memory +=
890
 
                                                job->memory_allocated[n];
 
863
                                job->memory_allocated[n];
891
864
                        if ((select_node_usage[i].alloc_memory >
892
865
                             select_node_record[i].real_memory)) {
893
 
                                error("error: node %s mem is overallocated "
894
 
                                      "(%u) for job %u",
895
 
                                      select_node_record[i].node_ptr->name,
 
866
                                error("cons_res: node %s memory is "
 
867
                                      "overallocated (%u) for job %u",
 
868
                                      node_ptr->name,
896
869
                                      select_node_usage[i].alloc_memory,
897
870
                                      job_ptr->job_id);
898
 
 
899
871
                        }
900
 
                        n++;
901
872
                }
902
873
        }
903
874
 
909
880
                }
910
881
                if (!p_ptr) {
911
882
                        error("cons_res: could not find cr partition %s",
912
 
                                job_ptr->part_ptr->name);
 
883
                              job_ptr->part_ptr->name);
913
884
                        return SLURM_ERROR;
914
885
                }
915
886
                if (!p_ptr->row) {
916
887
                        p_ptr->row = xmalloc(p_ptr->num_rows *
917
 
                                                sizeof(struct part_row_data));
 
888
                                             sizeof(struct part_row_data));
918
889
                }
919
890
 
920
891
                /* find a row to add this job */
922
893
                        if (!_can_job_fit_in_row(job, &(p_ptr->row[i])))
923
894
                                continue;
924
895
                        debug3("cons_res: adding job %u to part %s row %u",
925
 
                        job_ptr->job_id, p_ptr->part_ptr->name, i);
 
896
                               job_ptr->job_id, p_ptr->part_ptr->name, i);
926
897
                        _add_job_to_row(job, &(p_ptr->row[i]));
927
898
                        break;
928
899
                }
936
907
                /* update the node state */
937
908
                for (i = 0; i < select_node_cnt; i++) {
938
909
                        if (bit_test(job->node_bitmap, i))
939
 
                                select_node_usage[i].node_state +=job->
940
 
                                                                  node_req;
941
 
                }
942
 
#if (CR_DEBUG)
943
 
                info("DEBUG: _add_job_to_res (after):");
944
 
                _dump_part(p_ptr);
945
 
#endif
 
910
                                select_node_usage[i].node_state +=
 
911
                                        job->node_req;
 
912
                }
 
913
                if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
 
914
                        info("DEBUG: _add_job_to_res (after):");
 
915
                        _dump_part(p_ptr);
 
916
                }
946
917
        }
947
918
 
948
919
        return SLURM_SUCCESS;
953
924
 * - subtract 'struct job_resources' resources from 'struct part_res_record'
954
925
 * - subtract job's memory requirements from 'struct node_res_record'
955
926
 *
956
 
 * if action = 0 then subtract cores and memory
 
927
 * if action = 0 then subtract cores and memory (running job was terminated)
957
928
 * if action = 1 then only subtract memory (suspended job was terminated)
958
929
 * if action = 2 then only subtract cores (job is suspended)
959
930
 *
963
934
                            struct job_record *job_ptr, int action)
964
935
{
965
936
        struct job_resources *job = job_ptr->job_resrcs;
 
937
        struct node_record *node_ptr;
 
938
        int first_bit, last_bit;
966
939
        int i, n;
 
940
        List gres_list;
967
941
 
 
942
        if (select_state_initializing) {
 
943
                /* Ignore job removal until select/cons_res data structures
 
944
                 * values are set by select_p_reconfigure() */
 
945
                return SLURM_SUCCESS;
 
946
        }
968
947
        if (!job || !job->core_bitmap) {
969
948
                error("job %u has no select data", job_ptr->job_id);
970
949
                return SLURM_ERROR;
971
950
        }
972
951
 
973
 
        debug3("cons_res: _rm_job_from_res: job %u act %d", job_ptr->job_id,
974
 
                action);
975
 
#if (CR_DEBUG)
976
 
        _dump_job_res(job);
977
 
#endif
978
 
 
979
 
        /* subtract memory */
980
 
        if (action != 2) {
981
 
                for (i = 0, n = 0; i < select_node_cnt; i++) {
982
 
                        if (!bit_test(job->node_bitmap, i))
983
 
                                continue;
 
952
        debug3("cons_res: _rm_job_from_res: job %u action %d", job_ptr->job_id,
 
953
               action);
 
954
        if (select_debug_flags & DEBUG_FLAG_CPU_BIND)
 
955
                _dump_job_res(job);
 
956
 
 
957
        first_bit = bit_ffs(job->node_bitmap);
 
958
        last_bit =  bit_fls(job->node_bitmap);
 
959
        for (i = first_bit, n = -1; i <= last_bit; i++) {
 
960
                if (!bit_test(job->node_bitmap, i))
 
961
                        continue;
 
962
                n++;
 
963
 
 
964
                node_ptr = node_record_table_ptr + i;
 
965
                if (action != 2) {
 
966
                        if (node_usage[i].gres_list)
 
967
                                gres_list = node_usage[i].gres_list;
 
968
                        else
 
969
                                gres_list = node_ptr->gres_list;
 
970
                        gres_plugin_job_dealloc(job_ptr->gres_list, gres_list,
 
971
                                                n, job_ptr->job_id,
 
972
                                                node_ptr->name);
 
973
                        gres_plugin_node_state_log(gres_list, node_ptr->name);
 
974
                }
 
975
 
 
976
                if (action != 2) {
 
977
                        if (job->memory_allocated[n] == 0)
 
978
                                continue;       /* no memory allocated */
984
979
                        if (node_usage[i].alloc_memory <
985
980
                            job->memory_allocated[n]) {
986
 
                                error("error: node %s mem is underallocated "
987
 
                                      "(%u-%u) for job %u",
988
 
                                      select_node_record[i].node_ptr->name,
 
981
                                error("cons_res: node %s memory is "
 
982
                                      "underallocated (%u-%u) for job %u",
 
983
                                      node_ptr->name,
989
984
                                      node_usage[i].alloc_memory,
990
985
                                      job->memory_allocated[n],
991
986
                                      job_ptr->job_id);
992
987
                                node_usage[i].alloc_memory = 0;
993
988
                        } else {
994
989
                                node_usage[i].alloc_memory -=
995
 
                                                job->memory_allocated[n];
 
990
                                        job->memory_allocated[n];
996
991
                        }
997
 
                        n++;
998
992
                }
999
993
        }
1000
994
 
1003
997
                /* reconstruct rows with remaining jobs */
1004
998
                struct part_res_record *p_ptr;
1005
999
 
1006
 
                if(!job_ptr->part_ptr) {
1007
 
                        error("error: 'rm' job %u does not have a "
 
1000
                if (!job_ptr->part_ptr) {
 
1001
                        error("cons_res: removed job %u does not have a "
1008
1002
                              "partition assigned",
1009
1003
                              job_ptr->job_id);
1010
1004
                        return SLURM_ERROR;
1015
1009
                                break;
1016
1010
                }
1017
1011
                if (!p_ptr) {
1018
 
                        error("error: 'rm' could not find part %s",
1019
 
                                job_ptr->part_ptr->name);
 
1012
                        error("cons_res: removed job %u could not find part %s",
 
1013
                              job_ptr->job_id, job_ptr->part_ptr->name);
1020
1014
                        return SLURM_ERROR;
1021
1015
                }
1022
1016
 
1023
 
                if (!p_ptr->row) {
 
1017
                if (!p_ptr->row)
1024
1018
                        return SLURM_SUCCESS;
1025
 
                }
1026
1019
 
1027
1020
                /* remove the job from the job_list */
1028
1021
                n = 0;
1031
1024
                        for (j = 0; j < p_ptr->row[i].num_jobs; j++) {
1032
1025
                                if (p_ptr->row[i].job_list[j] != job)
1033
1026
                                        continue;
1034
 
                                debug3("cons_res: removing job %u from "
 
1027
                                debug3("cons_res: removed job %u from "
1035
1028
                                       "part %s row %u",
1036
1029
                                       job_ptr->job_id,
1037
1030
                                       p_ptr->part_ptr->name, i);
1056
1049
                         * the removal of this job. If all cores are now
1057
1050
                         * available, set node_state = NODE_CR_AVAILABLE
1058
1051
                         */
1059
 
                        for (n = 0; n < select_node_cnt; n++) {
1060
 
                                if (bit_test(job->node_bitmap, n) == 0)
 
1052
                        for (i = 0, n = -1; i < select_node_cnt; i++) {
 
1053
                                if (bit_test(job->node_bitmap, i) == 0)
1061
1054
                                        continue;
1062
 
                                if (node_usage[n].node_state >=
 
1055
                                n++;
 
1056
                                if (job->cpus[n] == 0)
 
1057
                                        continue;  /* node lost by job resize */
 
1058
                                if (node_usage[i].node_state >=
1063
1059
                                    job->node_req) {
1064
 
                                        node_usage[n].node_state -=
1065
 
                                                                job->node_req;
 
1060
                                        node_usage[i].node_state -=
 
1061
                                                job->node_req;
1066
1062
                                } else {
1067
1063
                                        error("cons_res:_rm_job_from_res: "
1068
 
                                                "node_state mis-count");
1069
 
                                        node_usage[n].node_state =
1070
 
                                                        NODE_CR_AVAILABLE;
 
1064
                                              "node_state mis-count");
 
1065
                                        node_usage[i].node_state =
 
1066
                                                NODE_CR_AVAILABLE;
1071
1067
                                }
1072
1068
                        }
1073
1069
                }
1076
1072
        return SLURM_SUCCESS;
1077
1073
}
1078
1074
 
 
1075
static int _rm_job_from_one_node(struct job_record *job_ptr,
 
1076
                                 struct node_record *node_ptr)
 
1077
{
 
1078
        struct part_res_record *part_record_ptr = select_part_record;
 
1079
        struct node_use_record *node_usage = select_node_usage;
 
1080
        struct job_resources *job = job_ptr->job_resrcs;
 
1081
        struct part_res_record *p_ptr;
 
1082
        int first_bit, last_bit;
 
1083
        int i, node_inx, n;
 
1084
        List gres_list;
 
1085
 
 
1086
        if (!job || !job->core_bitmap) {
 
1087
                error("job %u has no select data", job_ptr->job_id);
 
1088
                return SLURM_ERROR;
 
1089
        }
 
1090
 
 
1091
        debug3("cons_res: _rm_job_from_one_node: job %u node %s",
 
1092
               job_ptr->job_id, node_ptr->name);
 
1093
        if (select_debug_flags & DEBUG_FLAG_CPU_BIND)
 
1094
                _dump_job_res(job);
 
1095
 
 
1096
        /* subtract memory */
 
1097
        node_inx  = node_ptr - node_record_table_ptr;
 
1098
        first_bit = bit_ffs(job->node_bitmap);
 
1099
        last_bit  = bit_fls(job->node_bitmap);
 
1100
        for (i = first_bit, n = -1; i <= last_bit; i++) {
 
1101
                if (!bit_test(job->node_bitmap, i))
 
1102
                        continue;
 
1103
                n++;
 
1104
                if (i != node_inx)
 
1105
                        continue;
 
1106
 
 
1107
                if (job->cpus[n] == 0) {
 
1108
                        info("attempt to remove node %s from job %u again",
 
1109
                             node_ptr->name, job_ptr->job_id);
 
1110
                        return SLURM_SUCCESS;
 
1111
                }
 
1112
 
 
1113
                if (node_usage[i].gres_list)
 
1114
                        gres_list = node_usage[i].gres_list;
 
1115
                else
 
1116
                        gres_list = node_ptr->gres_list;
 
1117
                gres_plugin_job_dealloc(job_ptr->gres_list, gres_list, n,
 
1118
                                        job_ptr->job_id, node_ptr->name);
 
1119
                gres_plugin_node_state_log(gres_list, node_ptr->name);
 
1120
 
 
1121
                job->cpus[n] = 0;
 
1122
                job->ncpus = build_job_resources_cpu_array(job);
 
1123
                clear_job_resources_node(job, n);
 
1124
                if (node_usage[i].alloc_memory < job->memory_allocated[n]) {
 
1125
                        error("cons_res: node %s memory is underallocated "
 
1126
                              "(%u-%u) for job %u",
 
1127
                              node_ptr->name, node_usage[i].alloc_memory,
 
1128
                              job->memory_allocated[n], job_ptr->job_id);
 
1129
                        node_usage[i].alloc_memory = 0;
 
1130
                } else
 
1131
                        node_usage[i].alloc_memory -= job->memory_allocated[n];
 
1132
                job->memory_allocated[n] = 0;
 
1133
                break;
 
1134
        }
 
1135
 
 
1136
        if (IS_JOB_SUSPENDED(job_ptr))
 
1137
                return SLURM_SUCCESS;   /* No cores allocated to the job now */
 
1138
 
 
1139
        /* subtract cores, reconstruct rows with remaining jobs */
 
1140
        if (!job_ptr->part_ptr) {
 
1141
                error("cons_res: removed job %u does not have a partition "
 
1142
                      "assigned", job_ptr->job_id);
 
1143
                return SLURM_ERROR;
 
1144
        }
 
1145
 
 
1146
        for (p_ptr = part_record_ptr; p_ptr; p_ptr = p_ptr->next) {
 
1147
                if (p_ptr->part_ptr == job_ptr->part_ptr)
 
1148
                        break;
 
1149
        }
 
1150
        if (!p_ptr) {
 
1151
                error("cons_res: removed job %u could not find part %s",
 
1152
                      job_ptr->job_id, job_ptr->part_ptr->name);
 
1153
                return SLURM_ERROR;
 
1154
        }
 
1155
 
 
1156
        if (!p_ptr->row)
 
1157
                return SLURM_SUCCESS;
 
1158
 
 
1159
        /* look for the job in the partition's job_list */
 
1160
        n = 0;
 
1161
        for (i = 0; i < p_ptr->num_rows; i++) {
 
1162
                uint32_t j;
 
1163
                for (j = 0; j < p_ptr->row[i].num_jobs; j++) {
 
1164
                        if (p_ptr->row[i].job_list[j] != job)
 
1165
                                continue;
 
1166
                        debug3("cons_res: found job %u in part %s row %u",
 
1167
                               job_ptr->job_id, p_ptr->part_ptr->name, i);
 
1168
                        /* found job - we're done, don't actually remove */
 
1169
                        n = 1;
 
1170
                        i = p_ptr->num_rows;
 
1171
                        break;
 
1172
                }
 
1173
        }
 
1174
        if (n == 0) {
 
1175
                error("cons_res: could not find job %u in partition %s",
 
1176
                      job_ptr->job_id, p_ptr->part_ptr->name);
 
1177
                return SLURM_ERROR;
 
1178
        }
 
1179
 
 
1180
 
 
1181
        /* job was found and removed from core-bitmap, so refresh CR bitmaps */
 
1182
        _build_row_bitmaps(p_ptr);
 
1183
 
 
1184
        /* Adjust the node_state of the node removed from this job.
 
1185
         * If all cores are now available, set node_state = NODE_CR_AVAILABLE */
 
1186
        if (node_usage[node_inx].node_state >= job->node_req) {
 
1187
                node_usage[node_inx].node_state -= job->node_req;
 
1188
        } else {
 
1189
                error("cons_res:_rm_job_from_one_node: node_state miscount");
 
1190
                node_usage[node_inx].node_state = NODE_CR_AVAILABLE;
 
1191
        }
 
1192
 
 
1193
        return SLURM_SUCCESS;
 
1194
}
 
1195
 
1079
1196
static struct multi_core_data * _create_default_mc(void)
1080
1197
{
1081
1198
        struct multi_core_data *mc_ptr;
1082
1199
        mc_ptr = xmalloc(sizeof(struct multi_core_data));
1083
 
        mc_ptr->min_sockets = (uint16_t) NO_VAL;
1084
 
        mc_ptr->min_cores   = (uint16_t) NO_VAL;
1085
 
        mc_ptr->min_threads = (uint16_t) NO_VAL;
 
1200
        mc_ptr->sockets_per_node = (uint16_t) NO_VAL;
 
1201
        mc_ptr->cores_per_socket = (uint16_t) NO_VAL;
 
1202
        mc_ptr->threads_per_core = (uint16_t) NO_VAL;
1086
1203
/*      mc_ptr is initialized to zero by xmalloc*/
1087
1204
/*      mc_ptr->ntasks_per_socket = 0; */
1088
1205
/*      mc_ptr->ntasks_per_core   = 0; */
1162
1279
        ListIterator job_iterator, preemptee_iterator;
1163
1280
        struct part_res_record *future_part;
1164
1281
        struct node_use_record *future_usage;
 
1282
        bool remove_some_jobs = false;
 
1283
        uint16_t mode;
1165
1284
 
1166
1285
        orig_map = bit_copy(bitmap);
1167
1286
        if (!orig_map)
1172
1291
                         select_node_cnt, select_part_record,
1173
1292
                         select_node_usage);
1174
1293
 
1175
 
        if ((rc != SLURM_SUCCESS) && cr_preemption_killing() &&
1176
 
            preemptee_candidates) {
 
1294
        if ((rc != SLURM_SUCCESS) && preemptee_candidates) {
1177
1295
                /* Remove preemptable jobs from simulated environment */
1178
1296
                future_part = _dup_part_data(select_part_record);
1179
1297
                if (future_part == NULL) {
1180
 
                        bit_free(orig_map);
 
1298
                        FREE_NULL_BITMAP(orig_map);
1181
1299
                        return SLURM_ERROR;
1182
1300
                }
1183
1301
                future_usage = _dup_node_usage(select_node_usage);
1184
1302
                if (future_usage == NULL) {
1185
1303
                        _destroy_part_data(future_part);
1186
 
                        bit_free(orig_map);
 
1304
                        FREE_NULL_BITMAP(orig_map);
1187
1305
                        return SLURM_ERROR;
1188
1306
                }
1189
1307
 
1190
1308
                job_iterator = list_iterator_create(job_list);
 
1309
                if (job_iterator == NULL)
 
1310
                        fatal ("memory allocation failure");
1191
1311
                while ((tmp_job_ptr = (struct job_record *)
1192
 
                                list_next(job_iterator))) {
 
1312
                        list_next(job_iterator))) {
1193
1313
                        if (!IS_JOB_RUNNING(tmp_job_ptr) &&
1194
1314
                            !IS_JOB_SUSPENDED(tmp_job_ptr))
1195
1315
                                continue;
 
1316
                        mode = slurm_job_preempt_mode(tmp_job_ptr);
 
1317
                        if ((mode != PREEMPT_MODE_REQUEUE)    &&
 
1318
                            (mode != PREEMPT_MODE_CHECKPOINT) &&
 
1319
                            (mode != PREEMPT_MODE_CANCEL))
 
1320
                                continue;       /* can't remove job */
1196
1321
                        if (_is_preemptable(tmp_job_ptr,
1197
1322
                                            preemptee_candidates)) {
1198
1323
                                /* Remove preemptable job now */
1212
1337
                list_iterator_destroy(job_iterator);
1213
1338
 
1214
1339
                if ((rc == SLURM_SUCCESS) && preemptee_job_list &&
1215
 
                    preemptee_candidates && cr_preemption_killing()) {
 
1340
                    preemptee_candidates) {
1216
1341
                        /* Build list of preemptee jobs whose resources are
1217
1342
                         * actually used */
1218
1343
                        if (*preemptee_job_list == NULL) {
1221
1346
                                        fatal("list_create malloc failure");
1222
1347
                        }
1223
1348
                        preemptee_iterator = list_iterator_create(
1224
 
                                                preemptee_candidates);
 
1349
                                preemptee_candidates);
 
1350
                        if (preemptee_iterator == NULL)
 
1351
                                fatal ("memory allocation failure");
1225
1352
                        while ((tmp_job_ptr = (struct job_record *)
1226
 
                                        list_next(preemptee_iterator))) {
 
1353
                                list_next(preemptee_iterator))) {
 
1354
                                mode = slurm_job_preempt_mode(tmp_job_ptr);
 
1355
                                if ((mode != PREEMPT_MODE_REQUEUE)    &&
 
1356
                                    (mode != PREEMPT_MODE_CHECKPOINT) &&
 
1357
                                    (mode != PREEMPT_MODE_CANCEL))
 
1358
                                        continue;
1227
1359
                                if (bit_overlap(bitmap,
1228
1360
                                                tmp_job_ptr->node_bitmap) == 0)
1229
1361
                                        continue;
1230
 
 
1231
1362
                                list_append(*preemptee_job_list,
1232
1363
                                            tmp_job_ptr);
 
1364
                                remove_some_jobs = true;
1233
1365
                        }
1234
1366
                        list_iterator_destroy(preemptee_iterator);
 
1367
                        if (!remove_some_jobs) {
 
1368
                                list_destroy(*preemptee_job_list);
 
1369
                                *preemptee_job_list = NULL;
 
1370
                        }
1235
1371
                }
1236
1372
 
1237
1373
                _destroy_part_data(future_part);
1238
1374
                _destroy_node_data(future_usage, NULL);
1239
1375
        }
1240
 
        bit_free(orig_map);
 
1376
        FREE_NULL_BITMAP(orig_map);
1241
1377
 
1242
1378
        return rc;
1243
1379
}
1269
1405
                         select_node_cnt, select_part_record,
1270
1406
                         select_node_usage);
1271
1407
        if (rc == SLURM_SUCCESS) {
1272
 
                bit_free(orig_map);
 
1408
                FREE_NULL_BITMAP(orig_map);
1273
1409
                job_ptr->start_time = time(NULL);
1274
1410
                return SLURM_SUCCESS;
1275
1411
        }
1278
1414
         * to determine when and where the job can start. */
1279
1415
        future_part = _dup_part_data(select_part_record);
1280
1416
        if (future_part == NULL) {
1281
 
                bit_free(orig_map);
 
1417
                FREE_NULL_BITMAP(orig_map);
1282
1418
                return SLURM_ERROR;
1283
1419
        }
1284
1420
        future_usage = _dup_node_usage(select_node_usage);
1285
1421
        if (future_usage == NULL) {
1286
1422
                _destroy_part_data(future_part);
1287
 
                bit_free(orig_map);
 
1423
                FREE_NULL_BITMAP(orig_map);
1288
1424
                return SLURM_ERROR;
1289
1425
        }
1290
1426
 
1292
1428
        cr_job_list = list_create(NULL);
1293
1429
        if (!cr_job_list)
1294
1430
                fatal("list_create: memory allocation error");
1295
 
        if (cr_preemption_killing())
1296
 
                action = 0;     /* remove cores and memory */
1297
 
        else
1298
 
                action = 2;     /* remove cores only, suspend job */
1299
1431
        job_iterator = list_iterator_create(job_list);
 
1432
        if (job_iterator == NULL)
 
1433
                fatal ("memory allocation failure");
1300
1434
        while ((tmp_job_ptr = (struct job_record *) list_next(job_iterator))) {
1301
1435
                if (!IS_JOB_RUNNING(tmp_job_ptr) &&
1302
1436
                    !IS_JOB_SUSPENDED(tmp_job_ptr))
1306
1440
                        continue;
1307
1441
                }
1308
1442
                if (_is_preemptable(tmp_job_ptr, preemptee_candidates)) {
 
1443
                        uint16_t mode = slurm_job_preempt_mode(tmp_job_ptr);
 
1444
                        if (mode == PREEMPT_MODE_OFF)
 
1445
                                continue;
 
1446
                        if (mode == PREEMPT_MODE_SUSPEND)
 
1447
                                action = 2;     /* remove cores, keep memory */
 
1448
                        else
 
1449
                                action = 0;     /* remove cores and memory */
1309
1450
                        /* Remove preemptable job now */
1310
1451
                        _rm_job_from_res(future_part, future_usage,
1311
1452
                                         tmp_job_ptr, action);
1326
1467
        }
1327
1468
 
1328
1469
        /* Remove the running jobs one at a time from exp_node_cr and try
1329
 
         * scheduling the pending job after each one */
 
1470
         * scheduling the pending job after each one. */
1330
1471
        if (rc != SLURM_SUCCESS) {
1331
1472
                list_sort(cr_job_list, _cr_job_list_sort);
1332
1473
                job_iterator = list_iterator_create(cr_job_list);
 
1474
                if (job_iterator == NULL)
 
1475
                        fatal ("memory allocation failure");
1333
1476
                while ((tmp_job_ptr = list_next(job_iterator))) {
 
1477
                        int ovrlap;
 
1478
                        bit_or(bitmap, orig_map);
 
1479
                        ovrlap = bit_overlap(bitmap, tmp_job_ptr->node_bitmap);
 
1480
                        if (ovrlap == 0)        /* job has no usable nodes */
 
1481
                                continue;       /* skip it */
 
1482
                        debug2("cons_res: _will_run_test, job %u: overlap=%d",
 
1483
                               tmp_job_ptr->job_id, ovrlap);
1334
1484
                        _rm_job_from_res(future_part, future_usage,
1335
1485
                                         tmp_job_ptr, 0);
1336
 
                        bit_or(bitmap, orig_map);
1337
1486
                        rc = cr_job_test(job_ptr, bitmap, min_nodes,
1338
1487
                                         max_nodes, req_nodes,
1339
1488
                                         SELECT_MODE_WILL_RUN, cr_type,
1344
1493
                                        job_ptr->start_time = now + 1;
1345
1494
                                else
1346
1495
                                        job_ptr->start_time = tmp_job_ptr->
1347
 
                                                              end_time;
 
1496
                                                end_time;
1348
1497
                                break;
1349
1498
                        }
1350
1499
                }
1362
1511
                                fatal("list_create malloc failure");
1363
1512
                }
1364
1513
                preemptee_iterator =list_iterator_create(preemptee_candidates);
 
1514
                if (preemptee_iterator == NULL)
 
1515
                        fatal ("memory allocation failure");
1365
1516
                while ((tmp_job_ptr = (struct job_record *)
1366
 
                                list_next(preemptee_iterator))) {
 
1517
                        list_next(preemptee_iterator))) {
1367
1518
                        if (bit_overlap(bitmap,
1368
1519
                                        tmp_job_ptr->node_bitmap) == 0)
1369
1520
                                continue;
1375
1526
        list_destroy(cr_job_list);
1376
1527
        _destroy_part_data(future_part);
1377
1528
        _destroy_node_data(future_usage, NULL);
1378
 
        bit_free(orig_map);
 
1529
        FREE_NULL_BITMAP(orig_map);
1379
1530
        return rc;
1380
1531
}
1381
1532
 
1426
1577
static int _synchronize_bitmaps(struct job_record *job_ptr,
1427
1578
                                bitstr_t ** partially_idle_bitmap)
1428
1579
{
1429
 
        int size, i, idlecpus = bit_set_count(avail_node_bitmap);
 
1580
        int size, i;
1430
1581
        struct part_res_record *p_ptr;
1431
1582
        size = bit_size(avail_node_bitmap);
1432
1583
        bitstr_t *bitmap = bit_alloc(size);
 
1584
 
1433
1585
        if (bitmap == NULL)
1434
1586
                return SLURM_ERROR;
1435
1587
 
1436
1588
        debug3("cons_res: synch_bm: avail %d of %d set, idle %d of %d set",
1437
 
               idlecpus, size, bit_set_count(idle_node_bitmap), size);
 
1589
               bit_set_count(avail_node_bitmap), size,
 
1590
               bit_set_count(idle_node_bitmap), size);
1438
1591
 
1439
1592
        if (!job_ptr)
1440
1593
                fatal("cons_res: error: don't know what job I'm sync'ing");
1445
1598
        }
1446
1599
 
1447
1600
        for (i = 0; i < select_node_cnt; i++) {
1448
 
                if (bit_test(avail_node_bitmap, i) == 0)
 
1601
                if (!bit_test(avail_node_bitmap, i))
1449
1602
                        continue;
1450
1603
 
1451
 
                if (bit_test(idle_node_bitmap, i) == 1) {
 
1604
                if (bit_test(idle_node_bitmap, i)) {
1452
1605
                        bit_set(bitmap, i);
1453
1606
                        continue;
1454
1607
                }
1455
1608
 
1456
 
                if(!p_ptr || _is_node_avail(p_ptr, i))
 
1609
                if (!p_ptr || _is_node_avail(p_ptr, i))
1457
1610
                        bit_set(bitmap, i);
1458
1611
        }
1459
 
        idlecpus = bit_set_count(bitmap);
1460
 
        if (p_ptr)
 
1612
        if (p_ptr) {
1461
1613
                debug3("cons_res: found %d partially idle nodes in part %s",
1462
 
                        idlecpus, p_ptr->part_ptr->name);
1463
 
        else
 
1614
                       bit_set_count(bitmap), p_ptr->part_ptr->name);
 
1615
        } else {
1464
1616
                debug3("cons_res: found %d partially idle nodes",
1465
 
                        idlecpus);
 
1617
                       bit_set_count(bitmap));
 
1618
        }
1466
1619
 
1467
1620
        *partially_idle_bitmap = bitmap;
1468
1621
        return SLURM_SUCCESS;
1474
1627
 */
1475
1628
extern int init(void)
1476
1629
{
1477
 
#ifdef HAVE_XCPU
1478
 
        error("%s is incompatible with XCPU use", plugin_name);
1479
 
        fatal("Use SelectType=select/linear");
1480
 
#endif
1481
 
#ifdef HAVE_BG
1482
 
        error("%s is incompatible with BlueGene", plugin_name);
1483
 
        fatal("Use SelectType=select/bluegene");
1484
 
#endif
1485
 
        cr_type = (select_type_plugin_info_t)slurmctld_conf.select_type_param;
1486
 
        verbose("%s loaded with argument %d ", plugin_name, cr_type);
 
1630
        cr_type = slurmctld_conf.select_type_param;
 
1631
        if (cr_type)
 
1632
                verbose("%s loaded with argument %u", plugin_name, cr_type);
 
1633
        select_debug_flags = slurm_get_debug_flags();
1487
1634
 
1488
1635
        return SLURM_SUCCESS;
1489
1636
}
1496
1643
        _destroy_part_data(select_part_record);
1497
1644
        select_part_record = NULL;
1498
1645
        xfree(cr_node_num_cores);
1499
 
        xfree(cr_num_core_count);
1500
 
        cr_node_num_cores = NULL;
1501
 
        cr_num_core_count = NULL;
 
1646
        xfree(cr_node_cores_offset);
1502
1647
 
1503
 
        verbose("%s shutting down ...", plugin_name);
 
1648
        if (cr_type)
 
1649
                verbose("%s shutting down ...", plugin_name);
1504
1650
 
1505
1651
        return SLURM_SUCCESS;
1506
1652
}
1539
1685
/* This is Part 1 of a 4-part procedure which can be found in
1540
1686
 * src/slurmctld/read_config.c. The whole story goes like this:
1541
1687
 *
1542
 
 * Step 1: select_g_node_init       : initializes the global node arrays
1543
 
 * Step 2: select_g_state_restore   : NO-OP - nothing to restore
1544
 
 * Step 3: select_g_job_init        : NO-OP - nothing to initialize
1545
 
 * Step 4: select_g_update_nodeinfo : called from reset_job_bitmaps() with
1546
 
 *                                    each valid recovered job_ptr AND from
1547
 
 *                                    select_nodes(), this procedure adds job
1548
 
 *                                    data to the 'select_part_record' global
1549
 
 *                                    array
 
1688
 * Step 1: select_g_node_init          : initializes the global node arrays
 
1689
 * Step 2: select_g_state_restore      : NO-OP - nothing to restore
 
1690
 * Step 3: select_g_job_init           : NO-OP - nothing to initialize
 
1691
 * Step 4: select_g_select_nodeinfo_set: called from reset_job_bitmaps() with
 
1692
 *                                       each valid recovered job_ptr AND from
 
1693
 *                                       select_nodes(), this procedure adds
 
1694
 *                                       job data to the 'select_part_record'
 
1695
 *                                       global array
1550
1696
 */
1551
1697
extern int select_p_node_init(struct node_record *node_ptr, int node_cnt)
1552
1698
{
1553
 
        int i;
 
1699
        int i, tot_core;
1554
1700
 
1555
1701
        info("cons_res: select_p_node_init");
1556
1702
        if (node_ptr == NULL) {
1563
1709
        }
1564
1710
 
1565
1711
        /* initial global core data structures */
 
1712
        select_state_initializing = true;
1566
1713
        select_fast_schedule = slurm_get_fast_schedule();
1567
1714
        _init_global_core_data(node_ptr, node_cnt);
1568
1715
 
1583
1730
                        select_node_record[i].cores   = config_ptr->cores;
1584
1731
                        select_node_record[i].vpus    = config_ptr->threads;
1585
1732
                        select_node_record[i].real_memory = config_ptr->
1586
 
                                                            real_memory;
 
1733
                                real_memory;
1587
1734
                } else {
1588
1735
                        select_node_record[i].cpus    = node_ptr[i].cpus;
1589
1736
                        select_node_record[i].sockets = node_ptr[i].sockets;
1590
1737
                        select_node_record[i].cores   = node_ptr[i].cores;
1591
1738
                        select_node_record[i].vpus    = node_ptr[i].threads;
1592
1739
                        select_node_record[i].real_memory = node_ptr[i].
1593
 
                                                            real_memory;
 
1740
                                real_memory;
1594
1741
                }
 
1742
                tot_core = select_node_record[i].sockets *
 
1743
                           select_node_record[i].cores;
 
1744
                if (tot_core >= select_node_record[i].cpus)
 
1745
                        select_node_record[i].vpus = 1;
1595
1746
                select_node_usage[i].node_state = NODE_CR_AVAILABLE;
 
1747
                gres_plugin_node_state_dealloc_all(select_node_record[i].
 
1748
                                                   node_ptr->gres_list);
1596
1749
        }
1597
1750
        _create_part_data();
1598
1751
 
1631
1784
 * NOTE: the job information that is considered for scheduling includes:
1632
1785
 *      req_node_bitmap: bitmap of specific nodes required by the job
1633
1786
 *      contiguous: allocated nodes must be sequentially located
1634
 
 *      num_procs: minimum number of processors required by the job
 
1787
 *      num_cpus: minimum number of processors required by the job
1635
1788
 * NOTE: bitmap must be a superset of req_nodes at the time that
1636
1789
 *      select_p_job_test is called
1637
1790
 */
1660
1813
                job_ptr->details->mc_ptr = _create_default_mc();
1661
1814
        job_node_req = _get_job_node_req(job_ptr);
1662
1815
 
1663
 
        debug3("cons_res: select_p_job_test: job %u node_req %u, mode %d",
1664
 
               job_ptr->job_id, job_node_req, mode);
1665
 
        debug3("cons_res: select_p_job_test: min_n %u max_n %u req_n %u nb %u",
1666
 
               min_nodes, max_nodes, req_nodes, bit_set_count(bitmap));
1667
 
 
1668
 
#if (CR_DEBUG)
1669
 
        _dump_state(select_part_record);
1670
 
#endif
 
1816
        if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
 
1817
                info("cons_res: select_p_job_test: job %u node_req %u mode %d",
 
1818
                     job_ptr->job_id, job_node_req, mode);
 
1819
                info("cons_res: select_p_job_test: min_n %u max_n %u req_n %u "
 
1820
                     "avail_n %u",
 
1821
                     min_nodes, max_nodes, req_nodes, bit_set_count(bitmap));
 
1822
                _dump_state(select_part_record);
 
1823
        }
1671
1824
        if (mode == SELECT_MODE_WILL_RUN) {
1672
1825
                rc = _will_run_test(job_ptr, bitmap, min_nodes, max_nodes,
1673
1826
                                    req_nodes, job_node_req,
1682
1835
        } else
1683
1836
                fatal("select_p_job_test: Mode %d is invalid", mode);
1684
1837
 
1685
 
#if (CR_DEBUG)
1686
 
        if (job_ptr->job_resrcs)
1687
 
                log_job_resources(job_ptr->job_id, job_ptr->job_resrcs);
1688
 
        else
1689
 
                info("no job_resources info for job %u",
1690
 
                     job_ptr->job_id);
1691
 
#else
1692
 
        if (debug_cpu_bind && job_ptr->job_resrcs)
1693
 
                log_job_resources(job_ptr->job_id, job_ptr->job_resrcs);
1694
 
#endif
 
1838
        if (select_debug_flags & DEBUG_FLAG_CPU_BIND) {
 
1839
                if (job_ptr->job_resrcs)
 
1840
                        log_job_resources(job_ptr->job_id, job_ptr->job_resrcs);
 
1841
                else {
 
1842
                        info("no job_resources info for job %u",
 
1843
                             job_ptr->job_id);
 
1844
                }
 
1845
        } else if (debug_cpu_bind && job_ptr->job_resrcs) {
 
1846
                log_job_resources(job_ptr->job_id, job_ptr->job_resrcs);
 
1847
        }
1695
1848
 
1696
1849
        return rc;
1697
1850
}
1701
1854
        return SLURM_SUCCESS;
1702
1855
}
1703
1856
 
 
1857
/* Determine if allocated nodes are usable (powered up) */
1704
1858
extern int select_p_job_ready(struct job_record *job_ptr)
1705
1859
{
 
1860
        int i, i_first, i_last;
 
1861
        struct node_record *node_ptr;
 
1862
 
 
1863
        if (!IS_JOB_RUNNING(job_ptr) && !IS_JOB_SUSPENDED(job_ptr)) {
 
1864
                /* Gang scheduling might suspend job immediately */
 
1865
                return 0;
 
1866
        }
 
1867
 
 
1868
        if ((job_ptr->node_bitmap == NULL) ||
 
1869
            ((i_first = bit_ffs(job_ptr->node_bitmap)) == -1))
 
1870
                return READY_NODE_STATE;
 
1871
        i_last  = bit_fls(job_ptr->node_bitmap);
 
1872
 
 
1873
        for (i=i_first; i<=i_last; i++) {
 
1874
                if (bit_test(job_ptr->node_bitmap, i) == 0)
 
1875
                        continue;
 
1876
                node_ptr = node_record_table_ptr + i;
 
1877
                if (IS_NODE_POWER_SAVE(node_ptr) || IS_NODE_POWER_UP(node_ptr))
 
1878
                        return 0;
 
1879
        }
 
1880
 
 
1881
        return READY_NODE_STATE;
 
1882
}
 
1883
 
 
1884
extern int select_p_job_resized(struct job_record *job_ptr,
 
1885
                                struct node_record *node_ptr)
 
1886
{
 
1887
        xassert(job_ptr);
 
1888
        xassert(job_ptr->magic == JOB_MAGIC);
 
1889
 
 
1890
        _rm_job_from_one_node(job_ptr, node_ptr);
1706
1891
        return SLURM_SUCCESS;
1707
1892
}
1708
1893
 
1736
1921
}
1737
1922
 
1738
1923
 
1739
 
extern int select_p_pack_select_info(time_t last_query_time, Buf *buffer_ptr)
 
1924
extern int select_p_pack_select_info(time_t last_query_time,
 
1925
                                     uint16_t show_flags, Buf *buffer_ptr,
 
1926
                                     uint16_t protocol_version)
1740
1927
{
1741
1928
        /* This function is always invalid on normal Linux clusters */
1742
1929
        return SLURM_ERROR;
1743
1930
}
1744
1931
 
1745
1932
extern int select_p_select_nodeinfo_pack(select_nodeinfo_t *nodeinfo,
1746
 
                                         Buf buffer)
 
1933
                                         Buf buffer,
 
1934
                                         uint16_t protocol_version)
1747
1935
{
1748
1936
        pack16(nodeinfo->alloc_cpus, buffer);
1749
1937
 
1751
1939
}
1752
1940
 
1753
1941
extern int select_p_select_nodeinfo_unpack(select_nodeinfo_t **nodeinfo,
1754
 
                                           Buf buffer)
 
1942
                                           Buf buffer,
 
1943
                                           uint16_t protocol_version)
1755
1944
{
1756
1945
        select_nodeinfo_t *nodeinfo_ptr = NULL;
1757
1946
 
1801
1990
        uint16_t tmp, tmp_16 = 0;
1802
1991
        static time_t last_set_all = 0;
1803
1992
        uint32_t node_threads, node_cpus;
 
1993
        select_nodeinfo_t *nodeinfo = NULL;
1804
1994
 
1805
1995
        /* only set this once when the last_node_update is newer than
1806
1996
           the last time we set things up. */
1807
1997
        if(last_set_all && (last_node_update < last_set_all)) {
1808
1998
                debug2("Node select info for set all hasn't "
1809
 
                       "changed since %d",
1810
 
                       last_set_all);
 
1999
                       "changed since %ld",
 
2000
                       (long)last_set_all);
1811
2001
                return SLURM_NO_CHANGE_IN_DATA;
1812
2002
        }
1813
2003
        last_set_all = last_node_update;
1814
2004
 
1815
 
        for (n=0; n < node_record_count; n++) {
 
2005
        for (n=0; n < select_node_cnt; n++) {
1816
2006
                node_ptr = &(node_record_table_ptr[n]);
 
2007
 
 
2008
                /* We have to use the '_g_' here to make sure we get
 
2009
                   the correct data to work on.  i.e. cray calls this
 
2010
                   plugin from within select/cray which has it's own
 
2011
                   struct.
 
2012
                */
 
2013
                select_g_select_nodeinfo_get(node_ptr->select_nodeinfo,
 
2014
                                             SELECT_NODEDATA_PTR, 0,
 
2015
                                             (void *)&nodeinfo);
 
2016
                if(!nodeinfo) {
 
2017
                        error("no nodeinfo returned from structure");
 
2018
                        continue;
 
2019
                }
 
2020
 
1817
2021
                if (slurmctld_conf.fast_schedule) {
1818
2022
                        node_cpus    = node_ptr->config_ptr->cpus;
1819
2023
                        node_threads = node_ptr->config_ptr->threads;
1849
2053
                if ((end - start) < node_cpus)
1850
2054
                        tmp_16 *= node_threads;
1851
2055
 
1852
 
                node_ptr->select_nodeinfo->alloc_cpus = tmp_16;
 
2056
                nodeinfo->alloc_cpus = tmp_16;
1853
2057
        }
1854
2058
 
1855
2059
        return SLURM_SUCCESS;
1857
2061
 
1858
2062
extern int select_p_select_nodeinfo_set(struct job_record *job_ptr)
1859
2063
{
 
2064
        int rc;
1860
2065
        xassert(job_ptr);
1861
2066
        xassert(job_ptr->magic == JOB_MAGIC);
1862
2067
 
1863
2068
        if (!IS_JOB_RUNNING(job_ptr) && !IS_JOB_SUSPENDED(job_ptr))
1864
2069
                return SLURM_SUCCESS;
1865
2070
 
1866
 
        return _add_job_to_res(job_ptr, 0);
 
2071
        rc = _add_job_to_res(job_ptr, 0);
 
2072
        gres_plugin_job_state_log(job_ptr->gres_list, job_ptr->job_id);
 
2073
 
 
2074
        return rc;
1867
2075
}
1868
2076
 
1869
2077
extern int select_p_select_nodeinfo_get(select_nodeinfo_t *nodeinfo,
1873
2081
{
1874
2082
        int rc = SLURM_SUCCESS;
1875
2083
        uint16_t *uint16 = (uint16_t *) data;
 
2084
        select_nodeinfo_t **select_nodeinfo = (select_nodeinfo_t **) data;
1876
2085
 
1877
2086
        if (nodeinfo == NULL) {
1878
2087
                error("get_nodeinfo: nodeinfo not set");
1889
2098
                *uint16 = 0;
1890
2099
                break;
1891
2100
        case SELECT_NODEDATA_SUBCNT:
1892
 
                if(state == NODE_STATE_ALLOCATED)
 
2101
                if (state == NODE_STATE_ALLOCATED)
1893
2102
                        *uint16 = nodeinfo->alloc_cpus;
 
2103
                else
 
2104
                        *uint16 = 0;
 
2105
                break;
 
2106
        case SELECT_NODEDATA_PTR:
 
2107
                *select_nodeinfo = nodeinfo;
1894
2108
                break;
1895
2109
        default:
1896
2110
                error("Unsupported option %d for get_nodeinfo.", dinfo);
1900
2114
        return rc;
1901
2115
}
1902
2116
 
1903
 
extern int select_p_select_jobinfo_alloc (select_jobinfo_t *jobinfo)
 
2117
extern int select_p_select_jobinfo_alloc(void)
1904
2118
{
1905
2119
        return SLURM_SUCCESS;
1906
2120
}
1911
2125
}
1912
2126
 
1913
2127
extern int select_p_select_jobinfo_set(select_jobinfo_t *jobinfo,
1914
 
                enum select_jobdata_type data_type, void *data)
 
2128
                                       enum select_jobdata_type data_type,
 
2129
                                       void *data)
1915
2130
{
1916
2131
        return SLURM_SUCCESS;
1917
2132
}
1918
2133
 
1919
2134
extern int select_p_select_jobinfo_get(select_jobinfo_t *jobinfo,
1920
 
                enum select_jobdata_type data_type, void *data)
 
2135
                                       enum select_jobdata_type data_type,
 
2136
                                       void *data)
1921
2137
{
1922
2138
        return SLURM_ERROR;
1923
2139
}
1924
2140
 
1925
2141
extern select_jobinfo_t *select_p_select_jobinfo_copy(
1926
 
                                        select_jobinfo_t *jobinfo)
 
2142
        select_jobinfo_t *jobinfo)
1927
2143
{
1928
2144
        return NULL;
1929
2145
}
1930
2146
 
1931
 
extern int select_p_select_jobinfo_pack(select_jobinfo_t *jobinfo, Buf buffer)
 
2147
extern int select_p_select_jobinfo_pack(select_jobinfo_t *jobinfo, Buf buffer,
 
2148
                                        uint16_t protocol_version)
1932
2149
{
1933
2150
        return SLURM_SUCCESS;
1934
2151
}
1935
2152
 
1936
2153
extern int select_p_select_jobinfo_unpack(select_jobinfo_t *jobinfo,
1937
 
                                          Buf buffer)
 
2154
                                          Buf buffer,
 
2155
                                          uint16_t protocol_version)
1938
2156
{
1939
2157
        return SLURM_SUCCESS;
1940
2158
}
2016
2234
                return SLURM_SUCCESS;
2017
2235
 
2018
2236
        select_node_record[index].real_memory = select_node_record[index].
2019
 
                                                node_ptr->real_memory;
 
2237
                node_ptr->real_memory;
2020
2238
        return SLURM_SUCCESS;
2021
2239
}
2022
2240
 
2037
2255
        int rc = SLURM_SUCCESS;
2038
2256
 
2039
2257
        info("cons_res: select_p_reconfigure");
 
2258
        select_debug_flags = slurm_get_debug_flags();
2040
2259
 
2041
2260
        /* Rebuild the global data structures */
2042
2261
        job_preemption_enabled = false;
2048
2267
 
2049
2268
        /* reload job data */
2050
2269
        job_iterator = list_iterator_create(job_list);
 
2270
        if (job_iterator == NULL)
 
2271
                fatal ("memory allocation failure");
2051
2272
        while ((job_ptr = (struct job_record *) list_next(job_iterator))) {
2052
2273
                if (IS_JOB_RUNNING(job_ptr)) {
2053
2274
                        /* add the job */
2058
2279
                }
2059
2280
        }
2060
2281
        list_iterator_destroy(job_iterator);
 
2282
        select_state_initializing = false;
2061
2283
 
2062
2284
        return SLURM_SUCCESS;
2063
2285
}