← Back to branch summary

~ubuntu-branches/ubuntu/vivid/slurm-llnl/vivid

« back to all changes in this revision

Viewing changes to src/slurmd/slurmstepd/mgr.c

  • Committer: Bazaar Package Importer
  • Author(s): Gennaro Oliva
  • Date: 2009-09-24 23:28:15 UTC
  • mfrom: (1.1.11 upstream) (3.2.4 sid)
  • Revision ID: james.westby@ubuntu.com-20090924232815-enh65jn32q1ebg07
Tags: 2.0.5-1
http://bugs.debian.org/541252
* New upstream release 
* Changed dependecy from lib-mysqlclient15 to lib-mysqlclient 
* Added Default-Start for runlevel 2 and 4 and $remote_fs requirement in
  init.d scripts (Closes: #541252)
* Postinst checks for wrong runlevels 2 and 4 links
* Upgraded to standard version 3.8.3
* Add lintian overrides for missing slurm-llnl-configurator.html in doc
  base registration
* modified postrm scripts to ignore pkill return value in order to avoid
  postrm failure when no slurm process is running
* Checking for slurmctld.pid before cancelling running and pending
  jobs during package removal 

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/slurmd/slurmstepd/mgr.c
1
1
/*****************************************************************************\
2
2
 *  src/slurmd/slurmstepd/mgr.c - job manager functions for slurmstepd
3
 
 *  $Id: mgr.c 17040 2009-03-26 15:03:18Z jette $
 
3
 *  $Id: mgr.c 18638 2009-09-08 21:54:27Z jette $
4
4
 *****************************************************************************
5
5
 *  Copyright (C) 2002-2007 The Regents of the University of California.
6
 
 *  Copyright (C) 2008 Lawrence Livermore National Security.
 
6
 *  Copyright (C) 2008-2009 Lawrence Livermore National Security.
7
7
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
8
8
 *  Written by Mark Grondona <mgrondona@llnl.gov>.
9
 
 *  LLNL-CODE-402394.
 
9
 *  CODE-OCEC-09-009. All rights reserved.
10
10
 *  
11
11
 *  This file is part of SLURM, a resource management program.
12
 
 *  For details, see <http://www.llnl.gov/linux/slurm/>.
 
12
 *  For details, see <https://computing.llnl.gov/linux/slurm/>.
 
13
 *  Please also read the included file: DISCLAIMER.
13
14
 *  
14
15
 *  SLURM is free software; you can redistribute it and/or modify it under
15
16
 *  the terms of the GNU General Public License as published by the Free
76
77
 
77
78
#include <slurm/slurm_errno.h>
78
79
 
 
80
#include "src/common/basil_resv_conf.h"
79
81
#include "src/common/cbuf.h"
80
82
#include "src/common/env.h"
 
83
#include "src/common/fd.h"
 
84
#include "src/common/forward.h"
81
85
#include "src/common/hostlist.h"
82
86
#include "src/common/log.h"
 
87
#include "src/common/mpi.h"
83
88
#include "src/common/node_select.h"
84
 
#include "src/common/fd.h"
 
89
#include "src/common/plugstack.h"
85
90
#include "src/common/safeopen.h"
86
91
#include "src/common/slurm_jobacct_gather.h"
87
92
#include "src/common/switch.h"
 
93
#include "src/common/util-net.h"
 
94
#include "src/common/xmalloc.h"
88
95
#include "src/common/xsignal.h"
89
96
#include "src/common/xstring.h"
90
 
#include "src/common/xmalloc.h"
91
 
#include "src/common/util-net.h"
92
 
#include "src/common/forward.h"
93
 
#include "src/common/plugstack.h"
94
 
#include "src/common/mpi.h"
95
97
 
96
98
#include "src/slurmd/slurmd/slurmd.h"
97
99
 
100
102
#include "src/slurmd/common/task_plugin.h"
101
103
#include "src/slurmd/common/run_script.h"
102
104
#include "src/slurmd/common/reverse_tree.h"
 
105
#include "src/slurmd/common/set_oomadj.h"
103
106
 
104
107
#include "src/slurmd/slurmstepd/slurmstepd.h"
105
108
#include "src/slurmd/slurmstepd/mgr.h"
124
127
};
125
128
 
126
129
struct priv_state {
127
 
        uid_t           saved_uid;
128
 
        gid_t           saved_gid;
129
 
        gid_t *         gid_list;
130
 
        int             ngids;
131
 
        char            saved_cwd [4096];
 
130
        uid_t   saved_uid;
 
131
        gid_t   saved_gid;
 
132
        gid_t * gid_list;
 
133
        int     ngids;
 
134
        char    saved_cwd [4096];
132
135
};
133
136
 
134
137
step_complete_t step_complete = {
143
146
        true,
144
147
        (bitstr_t *)NULL,
145
148
        0,
146
 
        NULL
 
149
        NULL
147
150
};
148
151
 
149
152
typedef struct kill_thread {
161
164
 */
162
165
static int  _access(const char *path, int modes, uid_t uid, gid_t gid);
163
166
static void _send_launch_failure(launch_tasks_request_msg_t *, 
164
 
                                 slurm_addr *, int);
 
167
                                 slurm_addr *, int);
165
168
static int  _fork_all_tasks(slurmd_job_t *job);
166
169
static int  _become_user(slurmd_job_t *job, struct priv_state *ps);
167
170
static void  _set_prio_process (slurmd_job_t *job);
168
171
static void _set_job_log_prefix(slurmd_job_t *job);
169
172
static int  _setup_normal_io(slurmd_job_t *job);
170
173
static int  _drop_privileges(slurmd_job_t *job, bool do_setuid,
171
 
                                struct priv_state *state);
 
174
                             struct priv_state *state);
172
175
static int  _reclaim_privileges(struct priv_state *state);
173
176
static void _send_launch_resp(slurmd_job_t *job, int rc);
174
 
static void _slurmd_job_log_init(slurmd_job_t *job);
 
177
static int  _slurmd_job_log_init(slurmd_job_t *job);
175
178
static void _wait_for_io(slurmd_job_t *job);
176
179
static int  _send_exit_msg(slurmd_job_t *job, uint32_t *tid, int n, 
177
 
                int status);
 
180
                           int status);
178
181
static void _wait_for_children_slurmstepd(slurmd_job_t *job);
179
182
static int  _send_pending_exit_msgs(slurmd_job_t *job);
180
183
static void _send_step_complete_msgs(slurmd_job_t *job);
287
290
mgr_launch_batch_job_setup(batch_job_launch_msg_t *msg, slurm_addr *cli)
288
291
{
289
292
        slurmd_job_t *job = NULL;
290
 
        
 
293
 
291
294
        if (!(job = job_batch_job_create(msg))) {
292
295
                error("job_batch_job_create() failed: %m");
293
296
                return NULL;
329
332
        if (job->aborted)
330
333
                verbose("job %u abort complete", job->jobid);
331
334
        else if (msg->step_id == SLURM_BATCH_SCRIPT) {
332
 
                _send_complete_batch_script_msg(job, 
333
 
                        ESLURMD_CREATE_BATCH_DIR_ERROR, -1);
 
335
                _send_complete_batch_script_msg(
 
336
                        job, ESLURMD_CREATE_BATCH_DIR_ERROR, -1);
334
337
        } else
335
338
                _send_step_complete_msgs(job);
336
339
 
356
359
static int
357
360
_setup_normal_io(slurmd_job_t *job)
358
361
{
359
 
        int            rc   = 0;
 
362
        int rc = 0, ii = 0;
360
363
        struct priv_state sprivs;
361
364
 
362
365
        debug2("Entering _setup_normal_io");
369
372
        if (_drop_privileges(job, true, &sprivs) < 0)
370
373
                return ESLURMD_SET_UID_OR_GID_ERROR;
371
374
 
372
 
        /* FIXME - need to check a return code for failures */
373
 
        io_init_tasks_stdio(job);
374
 
 
375
 
        if (_reclaim_privileges(&sprivs) < 0)
376
 
                error("sete{u/g}id(%lu/%lu): %m", 
377
 
                      (u_long) sprivs.saved_uid, (u_long) sprivs.saved_gid);
 
375
        if (io_init_tasks_stdio(job) != SLURM_SUCCESS) {
 
376
                rc = ESLURMD_IO_ERROR;
 
377
                goto claim;
 
378
        }
378
379
 
379
380
        /*
380
381
         * MUST create the initial client object before starting
382
383
         */
383
384
        if (!job->batch) {
384
385
                srun_info_t *srun = list_peek(job->sruns);
 
386
 
 
387
                /* local id of task that sends to srun, -1 for all tasks,
 
388
                   any other value for no tasks */
 
389
                int srun_stdout_tasks = -1;
 
390
                int srun_stderr_tasks = -1;
 
391
 
385
392
                xassert(srun != NULL);
386
 
                rc = io_initial_client_connect(srun, job);
387
 
                if (rc < 0) 
388
 
                        return ESLURMD_IO_ERROR;
389
 
        }
390
 
 
391
 
        if (!job->batch)
 
393
 
 
394
                /* If I/O is labelled with task num, and if a separate file is
 
395
                   written per node or per task, the I/O needs to be sent 
 
396
                   back to the stepd, get a label appended, and written from
 
397
                   the stepd rather than sent back to srun or written directly
 
398
                   from the node.  When a task has ofname or efname == NULL, it
 
399
                   means data gets sent back to the client. */
 
400
 
 
401
                if (job->labelio) {
 
402
                        slurmd_filename_pattern_t outpattern, errpattern;
 
403
                        bool same = false;
 
404
                        int file_flags;
 
405
 
 
406
                        io_find_filename_pattern(job, &outpattern, &errpattern,
 
407
                                                 &same);
 
408
                        file_flags = io_get_file_flags(job);
 
409
 
 
410
                        /* Make eio objects to write from the slurmstepd */
 
411
                        if (outpattern == SLURMD_ALL_UNIQUE) {
 
412
                                /* Open a separate file per task */
 
413
                                for (ii = 0; ii < job->ntasks; ii++) {
 
414
                                        rc = io_create_local_client( 
 
415
                                                job->task[ii]->ofname, 
 
416
                                                file_flags, job, job->labelio,
 
417
                                                job->task[ii]->id,
 
418
                                                same ? job->task[ii]->id : -2);
 
419
                                        if (rc != SLURM_SUCCESS) {
 
420
                                                error("Could not open output "
 
421
                                                      "file %s: %m", 
 
422
                                                      job->task[ii]->ofname);
 
423
                                                rc = ESLURMD_IO_ERROR;
 
424
                                                goto claim;
 
425
                                        }
 
426
                                }
 
427
                                srun_stdout_tasks = -2;
 
428
                                if (same)
 
429
                                        srun_stderr_tasks = -2;
 
430
                        } else if (outpattern == SLURMD_ALL_SAME) {
 
431
                                /* Open a file for all tasks */
 
432
                                rc = io_create_local_client( 
 
433
                                        job->task[0]->ofname, 
 
434
                                        file_flags, job, job->labelio,
 
435
                                        -1, same ? -1 : -2);
 
436
                                if (rc != SLURM_SUCCESS) {
 
437
                                        error("Could not open output "
 
438
                                              "file %s: %m", 
 
439
                                              job->task[0]->ofname);
 
440
                                        rc = ESLURMD_IO_ERROR;
 
441
                                        goto claim;
 
442
                                }
 
443
                                srun_stdout_tasks = -2;
 
444
                                if (same)
 
445
                                        srun_stderr_tasks = -2;
 
446
                        }
 
447
 
 
448
                        if (!same) {
 
449
                                if (errpattern == SLURMD_ALL_UNIQUE) {
 
450
                                        /* Open a separate file per task */
 
451
                                        for (ii = 0; ii < job->ntasks; ii++) {
 
452
                                                rc = io_create_local_client( 
 
453
                                                        job->task[ii]->efname, 
 
454
                                                        file_flags, job, 
 
455
                                                        job->labelio,
 
456
                                                        -2, job->task[ii]->id);
 
457
                                                if (rc != SLURM_SUCCESS) {
 
458
                                                        error("Could not "
 
459
                                                              "open error "
 
460
                                                              "file %s: %m", 
 
461
                                                              job->task[ii]->
 
462
                                                              efname);
 
463
                                                        rc = ESLURMD_IO_ERROR;
 
464
                                                        goto claim;
 
465
                                                }
 
466
                                        }
 
467
                                        srun_stderr_tasks = -2;
 
468
                                } else if (errpattern == SLURMD_ALL_SAME) {
 
469
                                        /* Open a file for all tasks */
 
470
                                        rc = io_create_local_client( 
 
471
                                                job->task[0]->efname, 
 
472
                                                file_flags, job, job->labelio,
 
473
                                                -2, -1);
 
474
                                        if (rc != SLURM_SUCCESS) {
 
475
                                                error("Could not open error "
 
476
                                                      "file %s: %m", 
 
477
                                                      job->task[0]->efname);
 
478
                                                rc = ESLURMD_IO_ERROR;
 
479
                                                goto claim;
 
480
                                        }
 
481
                                        srun_stderr_tasks = -2;
 
482
                                }
 
483
                        }
 
484
                }
 
485
                
 
486
                if(io_initial_client_connect(srun, job, srun_stdout_tasks, 
 
487
                                             srun_stderr_tasks) < 0) {
 
488
                        rc = ESLURMD_IO_ERROR;
 
489
                        goto claim;
 
490
                }
 
491
        }
 
492
 
 
493
claim:
 
494
        if (_reclaim_privileges(&sprivs) < 0) {
 
495
                error("sete{u/g}id(%lu/%lu): %m",
 
496
                      (u_long) sprivs.saved_uid, (u_long) sprivs.saved_gid);
 
497
        }
 
498
 
 
499
        if (!rc && !job->batch) {
392
500
                if (io_thread_start(job) < 0)
393
 
                        return ESLURMD_IO_ERROR;
 
501
                        rc = ESLURMD_IO_ERROR;
 
502
        }
 
503
        
394
504
        debug2("Leaving  _setup_normal_io");
395
 
        return SLURM_SUCCESS;
 
505
        return rc;
396
506
}
397
507
 
398
508
static int
436
546
 
437
547
        debug3("sending task exit msg for %d tasks", n);
438
548
 
439
 
        msg.task_id_list = tid;
440
 
        msg.num_tasks    = n;
441
 
        msg.return_code  = status;
442
 
        msg.job_id       = job->jobid;
443
 
        msg.step_id      = job->stepid;
 
549
        msg.task_id_list        = tid;
 
550
        msg.num_tasks           = n;
 
551
        msg.return_code         = status;
 
552
        msg.job_id              = job->jobid;
 
553
        msg.step_id             = job->stepid;
444
554
        slurm_msg_t_init(&resp);
445
 
        resp.data        = &msg;
446
 
        resp.msg_type    = MESSAGE_TASK_EXIT;
 
555
        resp.data               = &msg;
 
556
        resp.msg_type           = MESSAGE_TASK_EXIT;
447
557
        
448
558
        /*
449
559
         *  XXX Hack for TCP timeouts on exit of large, synchronized
544
654
        static bool acct_sent = false;
545
655
 
546
656
        debug2("_one_step_complete_msg: first=%d, last=%d", first, last);
 
657
 
 
658
        memset(&msg, 0, sizeof(step_complete_msg_t));
547
659
        msg.job_id = job->jobid;
548
660
        msg.job_step_id = job->stepid;
549
661
        msg.range_first = first;
684
796
        pthread_mutex_unlock(&step_complete.lock);
685
797
}
686
798
 
 
799
/* This dummy function is provided so that the checkpoint functions can
 
800
 *      resolve this symbol name (as needed for some of the checkpoint  
 
801
 *      functions used by slurmctld). */
 
802
extern void agent_queue_request(void *dummy)
 
803
{
 
804
        fatal("Invalid agent_queue_request function call, likely from "
 
805
              "checkpoint plugin");
 
806
}
 
807
 
687
808
/* 
688
809
 * Executes the functions of the slurmd job manager process,
689
810
 * which runs as root and performs shared memory and interconnect
697
818
{
698
819
        int  rc = 0;
699
820
        bool io_initialized = false;
 
821
        char *ckpt_type = slurm_get_checkpoint_type();
700
822
 
701
823
        debug3("Entered job_manager for %u.%u pid=%lu",
702
824
               job->jobid, job->stepid, (unsigned long) job->jmgr_pid);
703
825
        /*
704
826
         * Preload plugins.
705
827
         */
706
 
        if (switch_init() != SLURM_SUCCESS
707
 
            || slurmd_task_init() != SLURM_SUCCESS
708
 
            || slurm_proctrack_init() != SLURM_SUCCESS
709
 
            || slurm_jobacct_gather_init() != SLURM_SUCCESS) {
 
828
        if ((switch_init() != SLURM_SUCCESS)                    ||
 
829
            (slurmd_task_init() != SLURM_SUCCESS)               ||
 
830
            (slurm_proctrack_init() != SLURM_SUCCESS)           ||
 
831
            (checkpoint_init(ckpt_type) != SLURM_SUCCESS)       ||
 
832
            (slurm_jobacct_gather_init() != SLURM_SUCCESS)) {
710
833
                rc = SLURM_PLUGIN_NAME_INVALID;
711
834
                goto fail1;
712
835
        }
721
844
                goto fail1;
722
845
        }
723
846
        
 
847
#ifndef NDEBUG
 
848
#  ifdef PR_SET_DUMPABLE
 
849
        if (prctl(PR_SET_DUMPABLE, 1) < 0)
 
850
                debug ("Unable to set dumpable to 1");
 
851
#  endif /* PR_SET_DUMPABLE */
 
852
#endif   /* !NDEBUG      */
 
853
 
724
854
        set_umask(job);         /* set umask for stdout/err files */
725
855
        if (job->user_managed_io)
726
856
                rc = _setup_user_managed_io(job);
729
859
        /*
730
860
         * Initialize log facility to copy errors back to srun
731
861
         */
732
 
        _slurmd_job_log_init(job);
 
862
        if(!rc)
 
863
                rc = _slurmd_job_log_init(job);
733
864
        
734
 
#ifndef NDEBUG
735
 
#  ifdef PR_SET_DUMPABLE
736
 
        if (prctl(PR_SET_DUMPABLE, 1) < 0)
737
 
                debug ("Unable to set dumpable to 1");
738
 
#  endif /* PR_SET_DUMPABLE */
739
 
#endif   /* !NDEBUG         */
740
 
 
741
865
        if (rc) {
742
866
                error("IO setup failed: %m");
 
867
                rc = SLURM_SUCCESS;     /* drains node otherwise */
743
868
                goto fail2;
744
869
        } else {
745
870
                io_initialized = true;
754
879
                goto fail2;
755
880
        }
756
881
 
 
882
        /* fork necessary threads for checkpoint */
 
883
        if (checkpoint_stepd_prefork(job) != SLURM_SUCCESS) {
 
884
                error("Failed checkpoint_stepd_prefork");
 
885
                rc = SLURM_FAILURE;
 
886
                io_close_task_fds(job);
 
887
                goto fail2;
 
888
        }
 
889
        
757
890
        /* calls pam_setup() and requires pam_finish() if successful */
758
891
        if (_fork_all_tasks(job) < 0) {
759
892
                debug("_fork_all_tasks failed");
820
953
        /*
821
954
         * Wait for io thread to complete (if there is one)
822
955
         */
823
 
        if (!job->batch && !job->user_managed_io && io_initialized) {
824
 
                eio_signal_shutdown(job->eio);
 
956
        if (!job->batch && !job->user_managed_io && io_initialized) 
825
957
                _wait_for_io(job);
826
 
        }
827
 
 
 
958
        
828
959
        debug2("Before call to spank_fini()");
829
960
        if (spank_fini (job)  < 0) {
830
961
                error ("spank_fini failed\n");
847
978
                _send_step_complete_msgs(job);
848
979
        }
849
980
 
 
981
        xfree(ckpt_type);
850
982
        return(rc);
851
983
}
852
984
 
 
985
static int
 
986
_spank_task_privileged(slurmd_job_t *job, int taskid, struct priv_state *sp)
 
987
{
 
988
        if (_reclaim_privileges(sp) < 0)
 
989
                return SLURM_ERROR;
 
990
 
 
991
        if (spank_task_privileged (job, taskid) < 0)
 
992
                return error("spank_task_init_privileged failed");
 
993
 
 
994
        return(_drop_privileges (job, true, sp));
 
995
}
 
996
 
853
997
 
854
998
/* fork and exec N tasks
855
999
 */ 
863
1007
        int fdpair[2];
864
1008
        struct priv_state sprivs;
865
1009
        jobacct_id_t jobacct_id;
 
1010
        char *oom_value;
866
1011
 
867
1012
        xassert(job != NULL);
868
1013
 
871
1016
                return SLURM_ERROR;
872
1017
        }
873
1018
 
 
1019
#ifdef HAVE_CRAY_XT
 
1020
        if (basil_resv_conf(job->resv_id, job->jobid)) {
 
1021
                error("could not confirm reservation");
 
1022
                return SLURM_ERROR;
 
1023
        }
 
1024
#endif
 
1025
 
874
1026
        debug2("Before call to spank_init()");
875
1027
        if (spank_init (job) < 0) {
876
1028
                error ("Plugin stack initialization failed.\n");
908
1060
                writefds[i] = fdpair[1];
909
1061
        }
910
1062
 
 
1063
        set_oom_adj(0); /* the tasks may be killed by OOM */
 
1064
        if (pre_setuid(job)) {
 
1065
                error("Failed task affinity setup");
 
1066
                return SLURM_ERROR;
 
1067
        }
 
1068
 
911
1069
        /* Temporarily drop effective privileges, except for the euid.
912
1070
         * We need to wait until after pam_setup() to drop euid.
913
1071
         */
965
1123
                        if (conf->propagate_prio == 1)
966
1124
                                _set_prio_process(job);
967
1125
 
968
 
                        (void) pre_setuid(job);
 
1126
                        /*
 
1127
                         *  Reclaim privileges and call any plugin hooks
 
1128
                         *   that may require elevated privs
 
1129
                         */
 
1130
                        if (_spank_task_privileged(job, i, &sprivs) < 0)
 
1131
                                exit(1);
 
1132
 
969
1133
                        if (_become_user(job, &sprivs) < 0) {
970
1134
                                error("_become_user failed: %m");
971
1135
                                /* child process, should not return */
1006
1170
                /* Don't bother erroring out here */
1007
1171
        }
1008
1172
 
 
1173
        if ((oom_value = getenv("SLURMSTEPD_OOM_ADJ"))) {
 
1174
                int i = atoi(oom_value);
 
1175
                debug("Setting slurmstepd oom_adj to %d", i);
 
1176
                set_oom_adj(i);
 
1177
        }
 
1178
 
1009
1179
        if (chdir (sprivs.saved_cwd) < 0) {
1010
1180
                error ("Unable to return to working directory");
1011
1181
        }
1024
1194
                                i, job->task[i]->pid, job->pgid);
1025
1195
                }
1026
1196
 
1027
 
                if (slurm_container_add(job, job->task[i]->pid)
 
1197
                if (slurm_container_add(job, job->task[i]->pid)
1028
1198
                    == SLURM_ERROR) {
1029
 
                        error("slurm_container_add: %m");
 
1199
                        error("slurm_container_add: %m");
1030
1200
                        goto fail1;
1031
 
                }
 
1201
                }
1032
1202
                jobacct_id.nodeid = job->nodeid;
1033
1203
                jobacct_id.taskid = job->task[i]->gtid;
1034
1204
                jobacct_gather_g_add_task(job->task[i]->pid, 
1121
1291
        return nsent;
1122
1292
}
1123
1293
 
 
1294
static inline void
 
1295
_log_task_exit(unsigned long taskid, unsigned long pid, int status)
 
1296
{
 
1297
        /*
 
1298
         *  Print a nice message to the log describing the task exit status.
 
1299
         *
 
1300
         *  The final else is there just in case there is ever an exit status
 
1301
         *   that isn't WIFEXITED || WIFSIGNALED. We'll probably never reach
 
1302
         *   that code, but it is better than dropping a potentially useful
 
1303
         *   exit status.
 
1304
         */
 
1305
        if (WIFEXITED(status))
 
1306
                verbose("task %lu (%lu) exited with exit code %d.",
 
1307
                        taskid, pid, WEXITSTATUS(status));
 
1308
        else if (WIFSIGNALED(status))
 
1309
                /* WCOREDUMP isn't available on AIX */
 
1310
                verbose("task %lu (%lu) exited. Killed by signal %d%s.",
 
1311
                        taskid, pid, WTERMSIG(status),
 
1312
#ifdef WCOREDUMP
 
1313
                        WCOREDUMP(status) ? " (core dumped)" : ""
 
1314
#else
 
1315
                        ""
 
1316
#endif
 
1317
                        );
 
1318
        else
 
1319
                verbose("task %lu (%lu) exited with status 0x%04x.",
 
1320
                        taskid, pid, status);
 
1321
}
 
1322
 
1124
1323
/*
1125
1324
 * If waitflag is true, perform a blocking wait for a single process
1126
1325
 * and then return.
1179
1378
                        }
1180
1379
                }
1181
1380
                if (t != NULL) {
1182
 
                        verbose("task %lu (%lu) exited status 0x%04x %M",
1183
 
                                (unsigned long)job->task[i]->gtid,
1184
 
                                (unsigned long)pid, status);
 
1381
                        _log_task_exit(job->task[i]->gtid, pid, status);
 
1382
 
1185
1383
                        t->exited  = true;
1186
1384
                        t->estatus = status;
1187
1385
                        job->envtp->env = job->env;
1189
1387
                        job->envtp->localid = job->task[i]->id;
1190
1388
                        
1191
1389
                        job->envtp->distribution = -1;
1192
 
                        setup_env(job->envtp);
 
1390
                        job->envtp->batch_flag = job->batch;
 
1391
                        setup_env(job->envtp, false);
1193
1392
                        job->env = job->envtp->env;
1194
1393
                        if (job->task_epilog) {
1195
1394
                                _run_script_as_user("user task_epilog",
1305
1504
        } else
1306
1505
                info("_wait_for_io: ioid==0");
1307
1506
 
 
1507
        /* Close any files for stdout/stderr opened by the stepd */
 
1508
        io_close_local_fds(job);
 
1509
 
1308
1510
        return;
1309
1511
}
1310
1512
 
1431
1633
        debug("Sending launch resp rc=%d", rc);
1432
1634
 
1433
1635
        slurm_msg_t_init(&resp_msg);
1434
 
        resp_msg.address      = srun->resp_addr;
1435
 
        resp_msg.data         = &resp;
1436
 
        resp_msg.msg_type     = RESPONSE_LAUNCH_TASKS;
 
1636
        resp_msg.address        = srun->resp_addr;
 
1637
        resp_msg.data           = &resp;
 
1638
        resp_msg.msg_type       = RESPONSE_LAUNCH_TASKS;
1437
1639
        
1438
 
        resp.node_name        = xstrdup(job->node_name);
1439
 
        resp.return_code      = rc;
1440
 
        resp.count_of_pids    = job->ntasks;
 
1640
        resp.node_name          = xstrdup(job->node_name);
 
1641
        resp.return_code        = rc;
 
1642
        resp.count_of_pids      = job->ntasks;
1441
1643
 
1442
1644
        resp.local_pids = xmalloc(job->ntasks * sizeof(*resp.local_pids));
1443
1645
        resp.task_ids = xmalloc(job->ntasks * sizeof(*resp.task_ids));
1457
1659
static int
1458
1660
_send_complete_batch_script_msg(slurmd_job_t *job, int err, int status)
1459
1661
{
1460
 
        int                      rc, i;
1461
 
        slurm_msg_t              req_msg;
1462
 
        complete_batch_script_msg_t  req;
 
1662
        int             rc, i;
 
1663
        slurm_msg_t     req_msg;
 
1664
        complete_batch_script_msg_t req;
1463
1665
 
1464
1666
        req.job_id      = job->jobid;
1465
1667
        req.job_rc      = status;
1509
1711
 
1510
1712
        ps->gid_list = (gid_t *) xmalloc(ps->ngids * sizeof(gid_t));
1511
1713
 
1512
 
        getgroups(ps->ngids, ps->gid_list);
 
1714
        if(getgroups(ps->ngids, ps->gid_list) == -1) {
 
1715
                error("_drop_privileges: couldn't get %d groups: %m",
 
1716
                      ps->ngids);
 
1717
                xfree(ps->gid_list);
 
1718
                return -1;
 
1719
        }
1513
1720
 
1514
1721
        /*
1515
1722
         * No need to drop privileges if we're not running as root
1561
1768
}
1562
1769
 
1563
1770
 
1564
 
static void
 
1771
static int
1565
1772
_slurmd_job_log_init(slurmd_job_t *job) 
1566
1773
{
1567
1774
        char argv0[64];
1588
1795
        
1589
1796
        log_alter(conf->log_opts, 0, NULL);
1590
1797
        log_set_argv0(argv0);
1591
 
        
 
1798
       
1592
1799
        /* Connect slurmd stderr to job's stderr */
1593
1800
        if (!job->user_managed_io && job->task != NULL) {
1594
1801
                if (dup2(job->task[0]->stderr_fd, STDERR_FILENO) < 0) {
1595
1802
                        error("job_log_init: dup2(stderr): %m");
1596
 
                        return;
 
1803
                        return ESLURMD_IO_ERROR;
1597
1804
                }
1598
1805
        }
1599
1806
        verbose("debug level = %d", conf->log_opts.stderr_level);
 
1807
        return SLURM_SUCCESS;
1600
1808
}
1601
1809
 
1602
1810
 
1794
2002
                        exit(127);
1795
2003
                }
1796
2004
 
1797
 
                chdir(job->cwd);
 
2005
                if(chdir(job->cwd) == -1)
 
2006
                        error("run_script_as_user: couldn't "
 
2007
                              "change working dir to %s: %m", job->cwd);
1798
2008
#ifdef SETPGRP_TWO_ARGS
1799
2009
                setpgrp(0, 0);
1800
2010
#else