← Back to branch summary

~ubuntu-branches/ubuntu/saucy/slurm-llnl/saucy

« back to all changes in this revision

Viewing changes to src/slurmd/slurmd/req.c

  • Committer: Bazaar Package Importer
  • Author(s): Gennaro Oliva
  • Date: 2008-05-30 13:11:30 UTC
  • mfrom: (1.1.3 upstream)
  • Revision ID: james.westby@ubuntu.com-20080530131130-l6ko6aie7xhrlmxe
Tags: 1.3.3-1
* New upstream release
* Removed patches to src/slurmctd/controller.c src/slurmdbd/slurmdbd.c
  doc/man/man1/sacctmgr.1 included to upstream
* Edited watch file to seek for 1.3 releases
* doc/man/man1/salloc.1 doc/man/man1/sbatch.1 doc/man/man5/slurm.conf.5
  patched to improve formatting and avoid manual warnings 

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/slurmd/slurmd/req.c
1
1
/*****************************************************************************\
2
2
 *  src/slurmd/slurmd/req.c - slurmd request handling
3
 
 *  $Id: req.c 13326 2008-02-21 20:37:56Z jette $
4
3
 *****************************************************************************
5
4
 *  Copyright (C) 2002-2006 The Regents of the University of California.
 
5
 *  Copyright (C) 2008 Lawrence Livermore National Security.
6
6
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
7
7
 *  Written by Mark Grondona <mgrondona@llnl.gov>.
8
 
 *  UCRL-CODE-226842.
 
8
 *  LLNL-CODE-402394.
9
9
 *  
10
10
 *  This file is part of SLURM, a resource management program.
11
11
 *  For details, see <http://www.llnl.gov/linux/slurm/>.
56
56
#include <grp.h>
57
57
 
58
58
#include "src/common/hostlist.h"
 
59
#include "src/common/jobacct_common.h"
59
60
#include "src/common/log.h"
60
61
#include "src/common/macros.h"
61
62
#include "src/common/node_select.h"
62
63
#include "src/common/slurm_auth.h"
63
64
#include "src/common/slurm_cred.h"
64
 
#include "src/common/slurm_jobacct.h"
 
65
#include "src/common/slurm_jobacct_gather.h"
65
66
#include "src/common/slurm_protocol_defs.h"
66
67
#include "src/common/slurm_protocol_api.h"
67
68
#include "src/common/slurm_protocol_interface.h"
93
94
        gid_t *gids;
94
95
} gids_t;
95
96
 
 
97
typedef struct {
 
98
        uint32_t job_id;
 
99
        uint32_t job_mem;
 
100
} job_mem_limits_t;
 
101
 
96
102
static int  _abort_job(uint32_t job_id);
97
103
static int  _abort_step(uint32_t job_id, uint32_t step_id);
98
104
static char ** _build_env(uint32_t jobid, uid_t uid, char *bg_part_id);
99
105
static void _delay_rpc(int host_inx, int host_cnt, int usec_per_rpc);
100
106
static void _destroy_env(char **env);
101
107
static bool _slurm_authorized_user(uid_t uid);
 
108
static void _job_limits_free(void *x);
 
109
static int  _job_limits_match(void *x, void *key);
102
110
static bool _job_still_running(uint32_t job_id);
103
111
static int  _kill_all_active_steps(uint32_t jobid, int sig, bool batch);
104
112
static int  _terminate_all_steps(uint32_t jobid, bool batch);
105
113
static void _rpc_launch_tasks(slurm_msg_t *);
106
114
static void _rpc_batch_job(slurm_msg_t *);
107
115
static void _rpc_signal_tasks(slurm_msg_t *);
 
116
static void _rpc_checkpoint_tasks(slurm_msg_t *);
108
117
static void _rpc_terminate_tasks(slurm_msg_t *);
109
118
static void _rpc_timelimit(slurm_msg_t *);
110
119
static void _rpc_reattach_tasks(slurm_msg_t *);
117
126
static void _rpc_pid2jid(slurm_msg_t *msg);
118
127
static int  _rpc_file_bcast(slurm_msg_t *msg);
119
128
static int  _rpc_ping(slurm_msg_t *);
 
129
static int  _rpc_health_check(slurm_msg_t *);
120
130
static int  _rpc_step_complete(slurm_msg_t *msg);
121
131
static int  _rpc_stat_jobacct(slurm_msg_t *msg);
122
132
static int  _rpc_daemon_status(slurm_msg_t *msg);
144
154
static time_t booted = 0;
145
155
static time_t last_slurmctld_msg = 0;
146
156
 
 
157
static pthread_mutex_t job_limits_mutex = PTHREAD_MUTEX_INITIALIZER;
 
158
static List job_limits_list = NULL;
 
159
static bool job_limits_loaded = false;
 
160
 
 
161
/* NUM_PARALLEL_SUSPEND controls the number of jobs suspended/resumed
 
162
 * at one time as well as the number of jobsteps per job that can be
 
163
 * suspended at one time */
 
164
#define NUM_PARALLEL_SUSPEND 8
 
165
static pthread_mutex_t suspend_mutex = PTHREAD_MUTEX_INITIALIZER;
 
166
static uint32_t job_suspend_array[NUM_PARALLEL_SUSPEND];
 
167
static int job_suspend_size = 0;
 
168
 
147
169
void
148
170
slurmd_req(slurm_msg_t *msg)
149
171
{
156
178
                        list_destroy(waiters);
157
179
                        waiters = NULL;
158
180
                }
 
181
                slurm_mutex_lock(&job_limits_mutex);
 
182
                if (job_limits_list) {
 
183
                        list_destroy(job_limits_list);
 
184
                        job_limits_list = NULL;
 
185
                        job_limits_loaded = false;
 
186
                }
 
187
                slurm_mutex_unlock(&job_limits_mutex);
159
188
                return;
160
189
        }
161
190
 
180
209
                _rpc_signal_tasks(msg);
181
210
                slurm_free_kill_tasks_msg(msg->data);
182
211
                break;
 
212
        case REQUEST_CHECKPOINT_TASKS:
 
213
                debug2("Processing RPC: REQUEST_CHECKPOINT_TASKS");
 
214
                _rpc_checkpoint_tasks(msg);
 
215
                slurm_free_checkpoint_tasks_msg(msg->data);
 
216
                break;
183
217
        case REQUEST_TERMINATE_TASKS:
184
218
                debug2("Processing RPC: REQUEST_TERMINATE_TASKS");
185
219
                _rpc_terminate_tasks(msg);
240
274
                last_slurmctld_msg = time(NULL);
241
275
                /* No body to free */
242
276
                break;
 
277
        case REQUEST_HEALTH_CHECK:
 
278
                _rpc_health_check(msg);
 
279
                last_slurmctld_msg = time(NULL);
 
280
                /* No body to free */
 
281
                break;
243
282
        case REQUEST_JOB_ID:
244
283
                _rpc_pid2jid(msg);
245
284
                slurm_free_job_id_request_msg(msg->data);
561
600
/*
562
601
 * The job(step) credential is the only place to get a definitive
563
602
 * list of the nodes allocated to a job step.  We need to return
564
 
 * a hostset_t of the nodes.
565
 
 *
566
 
 * FIXME - Rewrite this to only take a slurm_cred_t and only return a
567
 
 *         slurm_cred_arg_t.  The other parameters, jobid, stepid, etc.
568
 
 *         should be checked one caller layer higher.
 
603
 * a hostset_t of the nodes. Validate the incoming RPC, updating 
 
604
 * job_mem and task_mem as needed.
569
605
 */
570
606
static int
571
 
_check_job_credential(slurm_cred_t cred, uint32_t jobid, 
572
 
                      uint32_t stepid, uid_t uid, int tasks_to_launch,
573
 
                      hostset_t *step_hset)
 
607
_check_job_credential(launch_tasks_request_msg_t *req, uid_t uid,
 
608
                      int tasks_to_launch, hostset_t *step_hset)
574
609
{
575
610
        slurm_cred_arg_t arg;
576
611
        hostset_t        hset    = NULL;
577
612
        bool             user_ok = _slurm_authorized_user(uid); 
578
613
        int              host_index = -1;
579
614
        int              rc;
 
615
        slurm_cred_t     cred = req->cred;
 
616
        uint32_t         jobid = req->job_id;
 
617
        uint32_t         stepid = req->job_step_id;
580
618
 
581
619
        /*
582
620
         * First call slurm_cred_verify() so that all valid
591
629
                }
592
630
        }
593
631
 
 
632
        /* Overwrite any memory limits in the RPC with 
 
633
         * contents of the credential */
 
634
        req->job_mem  = arg.job_mem;
 
635
        req->task_mem = arg.task_mem;
 
636
 
594
637
        /*
595
638
         * If uid is the slurm user id or root, do not bother
596
639
         * performing validity check of the credential
689
732
        char     host[MAXHOSTNAMELEN];
690
733
        uid_t    req_uid;
691
734
        launch_tasks_request_msg_t *req = msg->data;
692
 
        uint32_t jobid  = req->job_id;
693
 
        uint32_t stepid = req->job_step_id;
694
735
        bool     super_user = false;
695
736
        bool     first_job_run;
696
737
        slurm_addr self;
697
738
        slurm_addr *cli = &msg->orig_addr;
698
739
        socklen_t adlen;
699
740
        hostset_t step_hset = NULL;
 
741
        job_mem_limits_t *job_limits_ptr;
700
742
        int nodeid = nodelist_find(req->complete_nodelist, conf->node_name);
701
743
 
702
 
        req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
744
        req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
703
745
        memcpy(&req->orig_addr, &msg->orig_addr, sizeof(slurm_addr));
704
746
 
705
 
        slurmd_launch_request(jobid, req, nodeid);
 
747
        slurmd_launch_request(req->job_id, req, nodeid);
706
748
 
707
749
        super_user = _slurm_authorized_user(req_uid);
708
750
 
718
760
             req->job_step_id, req->uid, req->gid, host, port);
719
761
 
720
762
        first_job_run = !slurm_cred_jobid_cached(conf->vctx, req->job_id);
721
 
        if (_check_job_credential(req->cred, jobid, stepid, req_uid,
722
 
                                  req->tasks_to_launch[nodeid],
 
763
        if (_check_job_credential(req, req_uid, req->tasks_to_launch[nodeid],
723
764
                                  &step_hset) < 0) {
724
765
                errnum = errno;
725
766
                error("Invalid job credential from %ld@%s: %m", 
726
767
                      (long) req_uid, host);
727
768
                goto done;
728
769
        }
729
 
 
 
770
        
730
771
#ifndef HAVE_FRONT_END
731
772
        if (first_job_run) {
732
 
                if (_run_prolog(req->job_id, req->uid, NULL) != 0) {
733
 
                        error("[job %u] prolog failed", req->job_id);
 
773
                int rc;
 
774
                rc =  _run_prolog(req->job_id, req->uid, NULL);
 
775
                if (rc) {
 
776
                        int term_sig, exit_status;
 
777
                        if (WIFSIGNALED(rc)) {
 
778
                                exit_status = 0;
 
779
                                term_sig    = WTERMSIG(rc);
 
780
                        } else {
 
781
                                exit_status = WEXITSTATUS(rc);
 
782
                                term_sig    = 0;
 
783
                        }
 
784
                        error("[job %u] prolog failed status=%d:%d", 
 
785
                              req->job_id, exit_status, term_sig);
734
786
                        errnum = ESLURMD_PROLOG_FAILED;
735
787
                        goto done;
736
788
                }
737
789
        }
738
790
#endif
 
791
 
 
792
        if (req->job_mem) {
 
793
                slurm_mutex_lock(&job_limits_mutex);
 
794
                if (!job_limits_list)
 
795
                        job_limits_list = list_create(_job_limits_free);
 
796
                job_limits_ptr = list_find_first (job_limits_list, 
 
797
                                                  _job_limits_match, 
 
798
                                                  &req->job_id);
 
799
                if (!job_limits_ptr) {
 
800
                        //info("AddLim job:%u mem:%u",req->job_id,req->job_mem);
 
801
                        job_limits_ptr = xmalloc(sizeof(job_mem_limits_t));
 
802
                        job_limits_ptr->job_id = req->job_id;
 
803
                        list_append(job_limits_list, job_limits_ptr);
 
804
                }
 
805
                job_limits_ptr->job_mem = req->job_mem; /* reset limit */
 
806
                slurm_mutex_unlock(&job_limits_mutex);
 
807
        }
 
808
 
739
809
        adlen = sizeof(self);
740
810
        _slurm_getsockname(msg->conn_fd, (struct sockaddr *)&self, &adlen);
741
811
 
750
820
 
751
821
        if (slurm_send_rc_msg(msg, errnum) < 0) {
752
822
 
753
 
                error("launch_tasks: unable to send return code: %m");
 
823
                error("_rpc_launch_tasks: unable to send return code: %m");
754
824
 
755
825
                /*
756
826
                 * Rewind credential so that srun may perform retry
806
876
        batch_job_launch_msg_t *req = (batch_job_launch_msg_t *)msg->data;
807
877
        bool     first_job_run = true;
808
878
        int      rc = SLURM_SUCCESS;
809
 
        uid_t    req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
879
        uid_t    req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
810
880
        char    *bg_part_id = NULL;
811
881
        bool    replied = false;
812
882
        slurm_addr *cli = &msg->orig_addr;
851
921
 
852
922
                rc = _run_prolog(req->job_id, req->uid, bg_part_id);
853
923
                xfree(bg_part_id);
854
 
                if (rc != 0) {
855
 
                        error("[job %u] prolog failed", req->job_id);
 
924
                if (rc) {
 
925
                        int term_sig, exit_status;
 
926
                        if (WIFSIGNALED(rc)) {
 
927
                                exit_status = 0;
 
928
                                term_sig    = WTERMSIG(rc);
 
929
                        } else {
 
930
                                exit_status = WEXITSTATUS(rc);
 
931
                                term_sig    = 0;
 
932
                        }
 
933
                        error("[job %u] prolog failed status=%d:%d", 
 
934
                              req->job_id, exit_status, term_sig);
856
935
                        _prolog_error(req, rc);
857
936
                        rc = ESLURMD_PROLOG_FAILED;
858
937
                        goto done;
934
1013
        resp.range_first  = 0;
935
1014
        resp.range_last   = 0;
936
1015
        resp.step_rc      = 1;
937
 
        resp.jobacct      = jobacct_g_alloc(NULL);
 
1016
        resp.jobacct      = jobacct_gather_g_create(NULL);
938
1017
        resp_msg.msg_type = REQUEST_STEP_COMPLETE;
939
1018
        resp_msg.data     = &resp;
940
1019
        return slurm_send_recv_controller_rc_msg(&resp_msg, &rc);
943
1022
static void
944
1023
_rpc_reconfig(slurm_msg_t *msg)
945
1024
{
946
 
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
1025
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
947
1026
 
948
1027
        if (!_slurm_authorized_user(req_uid))
949
1028
                error("Security violation, reconfig RPC from uid %u",
957
1036
static void
958
1037
_rpc_shutdown(slurm_msg_t *msg)
959
1038
{
960
 
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
1039
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
961
1040
        
962
1041
        forward_wait(msg);
963
1042
        if (!_slurm_authorized_user(req_uid))
971
1050
        /* Never return a message, slurmctld does not expect one */
972
1051
}
973
1052
 
 
1053
static void _job_limits_free(void *x)
 
1054
{
 
1055
        xfree(x);
 
1056
}
 
1057
 
 
1058
 
 
1059
static int _job_limits_match(void *x, void *key)
 
1060
{
 
1061
        job_mem_limits_t *job_limits_ptr = (job_mem_limits_t *) x;
 
1062
        uint32_t *job_id = (uint32_t *) key;
 
1063
        if (job_limits_ptr->job_id == *job_id)
 
1064
                return 1;
 
1065
        return 0;
 
1066
}
 
1067
 
 
1068
/* Call only with job_limits_mutex locked */
 
1069
static void
 
1070
_load_job_limits(void)
 
1071
{
 
1072
        List steps;
 
1073
        ListIterator step_iter;
 
1074
        step_loc_t *stepd;
 
1075
        int fd;
 
1076
        job_mem_limits_t *job_limits_ptr;
 
1077
        slurmstepd_info_t *stepd_info_ptr;
 
1078
 
 
1079
        if (!job_limits_list)
 
1080
                job_limits_list = list_create(_job_limits_free);
 
1081
        job_limits_loaded = true;
 
1082
 
 
1083
        steps = stepd_available(conf->spooldir, conf->node_name);
 
1084
        step_iter = list_iterator_create(steps);
 
1085
        while ((stepd = list_next(step_iter))) {
 
1086
                job_limits_ptr = list_find_first(job_limits_list,
 
1087
                                                 _job_limits_match,
 
1088
                                                 &stepd->jobid);
 
1089
                if (job_limits_ptr)     /* already processed */
 
1090
                        continue;
 
1091
                fd = stepd_connect(stepd->directory, stepd->nodename,
 
1092
                                   stepd->jobid, stepd->stepid);
 
1093
                if (fd == -1)
 
1094
                        continue;       /* step completed */
 
1095
                stepd_info_ptr = stepd_get_info(fd);
 
1096
                if (stepd_info_ptr && stepd_info_ptr->job_mem_limit) {
 
1097
                        /* create entry for this job */
 
1098
                        job_limits_ptr = xmalloc(sizeof(job_mem_limits_t));
 
1099
                        job_limits_ptr->job_id  = stepd->jobid;
 
1100
                        job_limits_ptr->job_mem = stepd_info_ptr->job_mem_limit;
 
1101
                        debug("RecLim job:%u mem:%u", 
 
1102
                              stepd->jobid, stepd_info_ptr->job_mem_limit);
 
1103
                        list_append(job_limits_list, job_limits_ptr);
 
1104
                }
 
1105
                xfree(stepd_info_ptr);
 
1106
                close(fd);
 
1107
        }
 
1108
        list_iterator_destroy(step_iter);
 
1109
        list_destroy(steps);
 
1110
}
 
1111
 
 
1112
static void
 
1113
_enforce_job_mem_limit(void)
 
1114
{
 
1115
        List steps;
 
1116
        ListIterator step_iter, job_limits_iter;
 
1117
        job_mem_limits_t *job_limits_ptr;
 
1118
        step_loc_t *stepd;
 
1119
        int fd, i, job_inx, job_cnt = 0;
 
1120
        uint32_t step_rss;
 
1121
        stat_jobacct_msg_t acct_req;
 
1122
        stat_jobacct_msg_t *resp = NULL;
 
1123
        struct job_mem_info {
 
1124
                uint32_t job_id;
 
1125
                uint32_t mem_limit;     /* MB */
 
1126
                uint32_t mem_used;      /* KB */
 
1127
        };
 
1128
        struct job_mem_info *job_mem_info_ptr = NULL;
 
1129
        slurm_msg_t msg;
 
1130
        job_notify_msg_t notify_req;
 
1131
        job_step_kill_msg_t kill_req;
 
1132
 
 
1133
        slurm_mutex_lock(&job_limits_mutex);
 
1134
        if (!job_limits_loaded)
 
1135
                _load_job_limits();
 
1136
        if (list_count(job_limits_list) == 0) {
 
1137
                slurm_mutex_unlock(&job_limits_mutex);
 
1138
                return;
 
1139
        }
 
1140
 
 
1141
        job_mem_info_ptr = xmalloc((list_count(job_limits_list) + 1) * 
 
1142
                           sizeof(struct job_mem_info));
 
1143
        job_cnt = 0;
 
1144
        job_limits_iter = list_iterator_create(job_limits_list);
 
1145
        while ((job_limits_ptr = list_next(job_limits_iter))) {
 
1146
                job_mem_info_ptr[job_cnt].job_id    = job_limits_ptr->job_id; 
 
1147
                job_mem_info_ptr[job_cnt].mem_limit = job_limits_ptr->job_mem;
 
1148
                job_cnt++;
 
1149
        }
 
1150
        list_iterator_destroy(job_limits_iter);
 
1151
        slurm_mutex_unlock(&job_limits_mutex);
 
1152
 
 
1153
        steps = stepd_available(conf->spooldir, conf->node_name);
 
1154
        step_iter = list_iterator_create(steps);
 
1155
        while ((stepd = list_next(step_iter))) {
 
1156
                for (job_inx=0; job_inx<job_cnt; job_inx++) {
 
1157
                        if (job_mem_info_ptr[job_inx].job_id == stepd->jobid)
 
1158
                                break;
 
1159
                }
 
1160
                if (job_inx >= job_cnt)
 
1161
                        continue;       /* job not being tracked */
 
1162
 
 
1163
                fd = stepd_connect(stepd->directory, stepd->nodename,
 
1164
                                   stepd->jobid, stepd->stepid);
 
1165
                if (fd == -1)
 
1166
                        continue;       /* step completed */
 
1167
                acct_req.job_id  = stepd->jobid;
 
1168
                acct_req.step_id = stepd->stepid;
 
1169
                resp = xmalloc(sizeof(stat_jobacct_msg_t));
 
1170
                if ((!stepd_stat_jobacct(fd, &acct_req, resp)) &&
 
1171
                    (resp->jobacct)) {
 
1172
                        /* resp->jobacct is NULL if account is disabled */
 
1173
                        jobacct_common_getinfo((struct jobacctinfo *)
 
1174
                                               resp->jobacct,
 
1175
                                               JOBACCT_DATA_TOT_RSS,
 
1176
                                               &step_rss);
 
1177
                        //info("job %u.%u rss:%u",stepd->jobid, stepd->stepid, step_rss);
 
1178
                        step_rss = MAX(step_rss, 1);
 
1179
                        job_mem_info_ptr[job_inx].mem_used += step_rss;
 
1180
                }
 
1181
                slurm_free_stat_jobacct_msg(resp);
 
1182
                close(fd);
 
1183
        }
 
1184
        list_iterator_destroy(step_iter);
 
1185
        list_destroy(steps);
 
1186
 
 
1187
        for (i=0; i<job_cnt; i++) {
 
1188
                if ((job_mem_info_ptr[i].mem_limit == 0) ||
 
1189
                    (job_mem_info_ptr[i].mem_used == 0)) {
 
1190
                        /* no memory limit or no steps found, purge record */
 
1191
                        slurm_mutex_lock(&job_limits_mutex);
 
1192
                        list_delete_all(job_limits_list, _job_limits_match, 
 
1193
                                        &job_mem_info_ptr[i].job_id);
 
1194
                        slurm_mutex_unlock(&job_limits_mutex);
 
1195
                        break;
 
1196
                }
 
1197
                job_mem_info_ptr[i].mem_used /= 1024;   /* KB to MB */
 
1198
                if (job_mem_info_ptr[i].mem_used <=
 
1199
                    job_mem_info_ptr[i].mem_limit)
 
1200
                        continue;
 
1201
 
 
1202
                info("Job %u exceeded memory limit (%u>%u), cancelling it",
 
1203
                    job_mem_info_ptr[i].job_id, job_mem_info_ptr[i].mem_used,
 
1204
                    job_mem_info_ptr[i].mem_limit);
 
1205
                /* NOTE: Batch jobs may have no srun to get this message */
 
1206
                slurm_msg_t_init(&msg);
 
1207
                notify_req.job_id      = job_mem_info_ptr[i].job_id;
 
1208
                notify_req.job_step_id = NO_VAL;
 
1209
                notify_req.message     = "Exceeded job memory limit";
 
1210
                msg.msg_type    = REQUEST_JOB_NOTIFY;
 
1211
                msg.data        = &notify_req;
 
1212
                slurm_send_only_controller_msg(&msg);
 
1213
 
 
1214
                kill_req.job_id      = job_mem_info_ptr[i].job_id;
 
1215
                kill_req.job_step_id = NO_VAL;
 
1216
                kill_req.signal      = SIGKILL;
 
1217
                kill_req.batch_flag  = (uint16_t) 0;
 
1218
                msg.msg_type    = REQUEST_CANCEL_JOB_STEP;
 
1219
                msg.data        = &kill_req;
 
1220
                slurm_send_only_controller_msg(&msg);
 
1221
        }
 
1222
        xfree(job_mem_info_ptr);
 
1223
}
 
1224
 
974
1225
static int
975
1226
_rpc_ping(slurm_msg_t *msg)
976
1227
{
977
1228
        int        rc = SLURM_SUCCESS;
978
 
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred);
979
 
 
980
 
        if (!_slurm_authorized_user(req_uid)) {
981
 
                error("Security violation, ping RPC from uid %u",
982
 
                      (unsigned int) req_uid);
983
 
                rc = ESLURM_USER_ID_MISSING;    /* or bad in this case */
984
 
        }
985
 
 
986
 
        /* Return result. If the reply can't be sent this indicates that
987
 
         * 1. The network is broken OR
988
 
         * 2. slurmctld has died    OR
989
 
         * 3. slurmd was paged out due to full memory
990
 
         * If the reply request fails, we send an registration message to 
991
 
         * slurmctld in hopes of avoiding having the node set DOWN due to
992
 
         * slurmd paging and not being able to respond in a timely fashion. */
993
 
        if (slurm_send_rc_msg(msg, rc) < 0) {
994
 
                error("Error responding to ping: %m");
995
 
                send_registration_msg(SLURM_SUCCESS, false);
996
 
        }
 
1229
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
 
1230
 
 
1231
        if (!_slurm_authorized_user(req_uid)) {
 
1232
                error("Security violation, ping RPC from uid %u",
 
1233
                      (unsigned int) req_uid);
 
1234
                rc = ESLURM_USER_ID_MISSING;    /* or bad in this case */
 
1235
        }
 
1236
 
 
1237
        /* Return result. If the reply can't be sent this indicates that
 
1238
         * 1. The network is broken OR
 
1239
         * 2. slurmctld has died    OR
 
1240
         * 3. slurmd was paged out due to full memory
 
1241
         * If the reply request fails, we send an registration message to 
 
1242
         * slurmctld in hopes of avoiding having the node set DOWN due to
 
1243
         * slurmd paging and not being able to respond in a timely fashion. */
 
1244
        if (slurm_send_rc_msg(msg, rc) < 0) {
 
1245
                error("Error responding to ping: %m");
 
1246
                send_registration_msg(SLURM_SUCCESS, false);
 
1247
        }
 
1248
 
 
1249
        /* Take this opportunity to enforce any job memory limits */
 
1250
        _enforce_job_mem_limit();
 
1251
        return rc;
 
1252
}
 
1253
 
 
1254
static int
 
1255
_rpc_health_check(slurm_msg_t *msg)
 
1256
{
 
1257
        int        rc = SLURM_SUCCESS;
 
1258
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
 
1259
 
 
1260
        if (!_slurm_authorized_user(req_uid)) {
 
1261
                error("Security violation, ping RPC from uid %u",
 
1262
                      (unsigned int) req_uid);
 
1263
                rc = ESLURM_USER_ID_MISSING;    /* or bad in this case */
 
1264
        }
 
1265
 
 
1266
        /* Return result. If the reply can't be sent this indicates that
 
1267
         * 1. The network is broken OR
 
1268
         * 2. slurmctld has died    OR
 
1269
         * 3. slurmd was paged out due to full memory
 
1270
         * If the reply request fails, we send an registration message to 
 
1271
         * slurmctld in hopes of avoiding having the node set DOWN due to
 
1272
         * slurmd paging and not being able to respond in a timely fashion. */
 
1273
        if (slurm_send_rc_msg(msg, rc) < 0) {
 
1274
                error("Error responding to ping: %m");
 
1275
                send_registration_msg(SLURM_SUCCESS, false);
 
1276
        }
 
1277
 
 
1278
        if ((rc == SLURM_SUCCESS) && (conf->health_check_program)) {
 
1279
                char *env[1] = { NULL };
 
1280
                rc = run_script("health_check", conf->health_check_program, 
 
1281
                                0, 60, env);
 
1282
        }
 
1283
 
 
1284
        /* Take this opportunity to enforce any job memory limits */
 
1285
        _enforce_job_mem_limit();
997
1286
        return rc;
998
1287
}
999
1288
 
1002
1291
{
1003
1292
        int               fd;
1004
1293
        int               rc = SLURM_SUCCESS;
1005
 
        uid_t             req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
1294
        uid_t             req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
1006
1295
        kill_tasks_msg_t *req = (kill_tasks_msg_t *) msg->data;
1007
1296
        slurmstepd_info_t *step;
1008
1297
 
1064
1353
}
1065
1354
 
1066
1355
static void
 
1356
_rpc_checkpoint_tasks(slurm_msg_t *msg)
 
1357
{
 
1358
        int               fd;
 
1359
        int               rc = SLURM_SUCCESS;
 
1360
        uid_t             req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
 
1361
        checkpoint_tasks_msg_t *req = (checkpoint_tasks_msg_t *) msg->data;
 
1362
        slurmstepd_info_t *step;
 
1363
 
 
1364
        fd = stepd_connect(conf->spooldir, conf->node_name,
 
1365
                           req->job_id, req->job_step_id);
 
1366
        if (fd == -1) {
 
1367
                debug("checkpoint for nonexistant %u.%u stepd_connect failed: %m",
 
1368
                      req->job_id, req->job_step_id);
 
1369
                rc = ESLURM_INVALID_JOB_ID;
 
1370
                goto done;
 
1371
        }
 
1372
        if ((step = stepd_get_info(fd)) == NULL) {
 
1373
                debug("checkpoint for nonexistent job %u.%u requested",
 
1374
                      req->job_id, req->job_step_id);
 
1375
                rc = ESLURM_INVALID_JOB_ID;
 
1376
                goto done2;
 
1377
        }
 
1378
 
 
1379
        if ((req_uid != step->uid) && (!_slurm_authorized_user(req_uid))) {
 
1380
                debug("checkpoint req from uid %ld for job %u.%u owned by uid %ld",
 
1381
                      (long) req_uid, req->job_id, req->job_step_id,
 
1382
                      (long) step->uid);
 
1383
                rc = ESLURM_USER_ID_MISSING;     /* or bad in this case */
 
1384
                goto done3;
 
1385
        }
 
1386
 
 
1387
        rc = stepd_checkpoint(fd, req->signal, req->timestamp);
 
1388
        if (rc == -1)
 
1389
                rc = ESLURMD_JOB_NOTRUNNING;
 
1390
 
 
1391
 done3:
 
1392
        xfree(step);
 
1393
 done2:
 
1394
        close(fd);
 
1395
 done:
 
1396
        slurm_send_rc_msg(msg, rc);
 
1397
}
 
1398
 
 
1399
static void
1067
1400
_rpc_terminate_tasks(slurm_msg_t *msg)
1068
1401
{
1069
1402
        kill_tasks_msg_t *req = (kill_tasks_msg_t *) msg->data;
1088
1421
                goto done2;
1089
1422
        } 
1090
1423
 
1091
 
        req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
1424
        req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
1092
1425
        if ((req_uid != step->uid) && (!_slurm_authorized_user(req_uid))) {
1093
1426
                debug("kill req from uid %ld for job %u.%u owned by uid %ld",
1094
1427
                      (long) req_uid, req->job_id, req->job_step_id, 
1127
1460
                goto done;
1128
1461
        }
1129
1462
 
1130
 
        /* step completionmessages are only allowed from other slurmstepd,
 
1463
        /* step completion messages are only allowed from other slurmstepd,
1131
1464
           so only root or SlurmUser is allowed here */
1132
 
        req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
1465
        req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
1133
1466
        if (!_slurm_authorized_user(req_uid)) {
1134
1467
                debug("step completion from uid %ld for job %u.%u",
1135
1468
                      (long) req_uid, req->job_id, req->job_step_id);
1238
1571
        debug3("Entering _rpc_stat_jobacct");
1239
1572
        /* step completion messages are only allowed from other slurmstepd,
1240
1573
           so only root or SlurmUser is allowed here */
1241
 
        req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
1574
        req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
1242
1575
        
1243
1576
        job_uid = _get_job_uid(req->job_id);
1244
1577
        if (job_uid < 0) {
1279
1612
                
1280
1613
        }
1281
1614
        if (stepd_stat_jobacct(fd, req, resp) == SLURM_ERROR) {
1282
 
                debug("kill for nonexistent job %u.%u requested",
 
1615
                debug("accounting for nonexistent job %u.%u requested",
1283
1616
                      req->job_id, req->step_id);
1284
1617
        } 
1285
1618
        close(fd);
1296
1629
/* 
1297
1630
 *  For the specified job_id: reply to slurmctld, 
1298
1631
 *   sleep(configured kill_wait), then send SIGKILL 
1299
 
 *  FIXME! - Perhaps we should send SIGXCPU first?
1300
1632
 */
1301
1633
static void
1302
1634
_rpc_timelimit(slurm_msg_t *msg)
1303
1635
{
1304
 
        uid_t           uid = g_slurm_auth_get_uid(msg->auth_cred);
 
1636
        uid_t           uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
1305
1637
        kill_job_msg_t *req = msg->data;
1306
1638
        int             nsteps;
1307
1639
 
1323
1655
                _kill_all_active_steps(req->job_id, SIGTERM, false);
1324
1656
        verbose( "Job %u: timeout: sent SIGTERM to %d active steps", 
1325
1657
                 req->job_id, nsteps );
 
1658
        _kill_all_active_steps(req->job_id, SIGXCPU, true);
1326
1659
 
1327
1660
        /* Revoke credential, send SIGKILL, run epilog, etc. */
1328
1661
        _rpc_terminate_job(msg); 
1378
1711
_rpc_file_bcast(slurm_msg_t *msg)
1379
1712
{
1380
1713
        file_bcast_msg_t *req = msg->data;
1381
 
        int i, fd, flags, offset, inx, rc;
1382
 
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred);
1383
 
        uid_t req_gid = g_slurm_auth_get_gid(msg->auth_cred);
 
1714
        int fd, flags, offset, inx, rc;
 
1715
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
 
1716
        uid_t req_gid = g_slurm_auth_get_gid(msg->auth_cred, NULL);
1384
1717
        pid_t child;
1385
1718
 
1386
1719
#if 0
1433
1766
                exit(errno);
1434
1767
        }
1435
1768
 
1436
 
        for (i=0; i<FILE_BLOCKS; i++) {
1437
 
                offset = 0;
1438
 
                while (req->block_len[i] - offset) {
1439
 
                        inx = write(fd, &req->block[i][offset], 
1440
 
                                (req->block_len[i] - offset));
1441
 
                        if (inx == -1) {
1442
 
                                if ((errno == EINTR) || (errno == EAGAIN))
1443
 
                                        continue;
1444
 
                                error("sbcast: uid:%u can't write `%s`: %s",
1445
 
                                        req_uid, req->fname, strerror(errno));
1446
 
                                close(fd);
1447
 
                                exit(errno);
1448
 
                        }
1449
 
                        offset += inx;
 
1769
        offset = 0;
 
1770
        while (req->block_len - offset) {
 
1771
                inx = write(fd, &req->block[offset], (req->block_len - offset));
 
1772
                if (inx == -1) {
 
1773
                        if ((errno == EINTR) || (errno == EAGAIN))
 
1774
                                continue;
 
1775
                        error("sbcast: uid:%u can't write `%s`: %s",
 
1776
                                req_uid, req->fname, strerror(errno));
 
1777
                        close(fd);
 
1778
                        exit(errno);
1450
1779
                }
 
1780
                offset += inx;
1451
1781
        }
1452
1782
        if (req->last_block && fchmod(fd, (req->modes & 0777))) {
1453
1783
                error("sbcast: uid:%u can't chmod `%s`: %s",
1508
1838
        nodeid = step->nodeid;
1509
1839
        debug2("_rpc_reattach_tasks: nodeid %d in the job step", nodeid);
1510
1840
 
1511
 
        req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
1841
        req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
1512
1842
        if ((req_uid != step->uid) && (!_slurm_authorized_user(req_uid))) {
1513
1843
                error("uid %ld attempt to attach to job %u.%u owned by %ld",
1514
1844
                      (long) req_uid, req->job_id, req->job_step_id,
1852
2182
_rpc_signal_job(slurm_msg_t *msg)
1853
2183
{
1854
2184
        signal_job_msg_t *req = msg->data;
1855
 
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
2185
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
1856
2186
        long job_uid;
1857
2187
        List steps;
1858
2188
        ListIterator i;
1951
2281
        }
1952
2282
}
1953
2283
 
 
2284
/* if a lock is granted to the job then return 1; else return 0 if
 
2285
 * the lock for the job is already taken or there's no more locks */
 
2286
static int
 
2287
_get_suspend_job_lock(uint32_t jobid)
 
2288
{
 
2289
        int i, spot = -1;
 
2290
        pthread_mutex_lock(&suspend_mutex);
 
2291
 
 
2292
        for (i = 0; i < job_suspend_size; i++) {
 
2293
                if (job_suspend_array[i] == -1) {
 
2294
                        spot = i;
 
2295
                        continue;
 
2296
                }
 
2297
                if (job_suspend_array[i] == jobid) {
 
2298
                        /* another thread already has the lock */
 
2299
                        pthread_mutex_unlock(&suspend_mutex);
 
2300
                        return 0;
 
2301
                }
 
2302
        }
 
2303
        i = 0;
 
2304
        if (spot != -1) {
 
2305
                /* nobody has the lock and here's an available used lock */
 
2306
                job_suspend_array[spot] = jobid;
 
2307
                i = 1;
 
2308
        } else if (job_suspend_size < NUM_PARALLEL_SUSPEND) {
 
2309
                /* a new lock is available */
 
2310
                job_suspend_array[job_suspend_size++] = jobid;
 
2311
                i = 1;
 
2312
        }
 
2313
        pthread_mutex_unlock(&suspend_mutex);
 
2314
        return i;
 
2315
}
 
2316
 
 
2317
static void
 
2318
_unlock_suspend_job(uint32_t jobid)
 
2319
{
 
2320
        int i;
 
2321
        pthread_mutex_lock(&suspend_mutex);
 
2322
        for (i = 0; i < job_suspend_size; i++) {
 
2323
                if (job_suspend_array[i] == jobid)
 
2324
                        job_suspend_array[i] = -1;
 
2325
        }
 
2326
        pthread_mutex_unlock(&suspend_mutex);
 
2327
}
 
2328
 
1954
2329
/*
1955
2330
 * Send a job suspend/resume request through the appropriate slurmstepds for 
1956
2331
 * each job step belonging to a given job allocation.
1959
2334
_rpc_suspend_job(slurm_msg_t *msg)
1960
2335
{
1961
2336
        suspend_msg_t *req = msg->data;
1962
 
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred);
1963
 
        long job_uid;
 
2337
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
1964
2338
        List steps;
1965
2339
        ListIterator i;
1966
2340
        step_loc_t *stepd;
1967
2341
        int step_cnt  = 0;  
1968
 
        int fd, rc = SLURM_SUCCESS;
 
2342
        int first_time, rc = SLURM_SUCCESS;
1969
2343
 
1970
2344
        if (req->op != SUSPEND_JOB && req->op != RESUME_JOB) {
1971
2345
                error("REQUEST_SUSPEND: bad op code %u", req->op);
1972
2346
                rc = ESLURM_NOT_SUPPORTED;
1973
 
                goto fini;
1974
2347
        }
1975
 
        debug("_rpc_suspend_job jobid=%u uid=%d", 
1976
 
                req->job_id, req_uid);
1977
 
        job_uid = _get_job_uid(req->job_id);
1978
 
        if (job_uid < 0)
1979
 
                goto no_job;
 
2348
 
1980
2349
        /* 
1981
2350
         * check that requesting user ID is the SLURM UID or root
1982
2351
         */
1983
2352
        if (!_slurm_authorized_user(req_uid)) {
1984
 
                error("Security violation: signal_job(%u) from uid %ld",
 
2353
                error("Security violation: suspend_job(%u) from uid %ld",
1985
2354
                      req->job_id, (long) req_uid);
1986
2355
                rc =  ESLURM_USER_ID_MISSING;
1987
 
                goto fini;
1988
 
        } 
 
2356
        }
 
2357
        
 
2358
        /* send a response now, which will include any errors
 
2359
         * detected with the request */
 
2360
        if (msg->conn_fd >= 0) {
 
2361
                slurm_send_rc_msg(msg, rc);
 
2362
                if (slurm_close_accepted_conn(msg->conn_fd) < 0)
 
2363
                        error ("_rpc_suspend_job: close(%d): %m", msg->conn_fd);
 
2364
                msg->conn_fd = -1;
 
2365
        }
 
2366
        if (rc != SLURM_SUCCESS)
 
2367
                return;
 
2368
 
 
2369
        /* now we can focus on performing the requested action,
 
2370
         * which could take a few seconds to complete */
 
2371
        debug("_rpc_suspend_job jobid=%u uid=%d action=%s", req->job_id,
 
2372
                req_uid, req->op == SUSPEND_JOB ? "suspend" : "resume");
 
2373
 
 
2374
        /* Try to get a thread lock for this job. If the lock
 
2375
         * is not available then sleep and try again */
 
2376
         first_time = 1;
 
2377
         while (!_get_suspend_job_lock(req->job_id)) {
 
2378
                first_time = 0;
 
2379
                debug3("suspend lock sleep for %u", req->job_id);
 
2380
                sleep(1);
 
2381
         }
 
2382
 
 
2383
        /* If suspending and you got the lock on the first try then
 
2384
         * sleep for 1 second to give any launch requests a chance
 
2385
         * to get started and avoid a race condition that would
 
2386
         * effectively cause the suspend request to get ignored
 
2387
         * because "there's no job to suspend" */
 
2388
        if (first_time && req->op == SUSPEND_JOB) {
 
2389
                debug3("suspend first sleep for %u", req->job_id);
 
2390
                sleep(1);
 
2391
        }
 
2392
 
 
2393
        /* Release or reclaim resources bound to these tasks (task affinity) */
 
2394
        if (req->op == SUSPEND_JOB)
 
2395
                (void) slurmd_suspend_job(req->job_id);
 
2396
        else
 
2397
                (void) slurmd_resume_job(req->job_id);
1989
2398
 
1990
2399
        /*
1991
2400
         * Loop through all job steps and call stepd_suspend or stepd_resume
1992
 
         * as appropriate.
 
2401
         * as appropriate. Since the "suspend" action contains a 'sleep 1',
 
2402
         * suspend multiple jobsteps in parallel.
1993
2403
         */
1994
2404
        steps = stepd_available(conf->spooldir, conf->node_name);
1995
2405
        i = list_iterator_create(steps);
1996
 
        while ((stepd = list_next(i))) {
1997
 
                if (stepd->jobid != req->job_id) {
1998
 
                        /* multiple jobs expected on shared nodes */
1999
 
                        debug3("Step from other job: jobid=%u (this jobid=%u)",
2000
 
                              stepd->jobid, req->job_id);
2001
 
                        continue;
2002
 
                }
2003
 
                step_cnt++;
2004
 
 
2005
 
                fd = stepd_connect(stepd->directory, stepd->nodename,
2006
 
                                   stepd->jobid, stepd->stepid);
2007
 
                if (fd == -1) {
2008
 
                        debug3("Unable to connect to step %u.%u",
2009
 
                               stepd->jobid, stepd->stepid);
2010
 
                        continue;
2011
 
                }
 
2406
 
 
2407
        while (1) {
 
2408
                int x, fdi, fd[NUM_PARALLEL_SUSPEND];
 
2409
                fdi = 0;
 
2410
                while ((stepd = list_next(i))) {
 
2411
                        if (stepd->jobid != req->job_id) {
 
2412
                                /* multiple jobs expected on shared nodes */
 
2413
                                debug3("Step from other job: jobid=%u (this jobid=%u)",
 
2414
                                      stepd->jobid, req->job_id);
 
2415
                                continue;
 
2416
                        }
 
2417
                        step_cnt++;
 
2418
 
 
2419
                        fd[fdi] = stepd_connect(stepd->directory,
 
2420
                                                stepd->nodename, stepd->jobid,
 
2421
                                                stepd->stepid);
 
2422
                        if (fd[fdi] == -1) {
 
2423
                                debug3("Unable to connect to step %u.%u",
 
2424
                                        stepd->jobid, stepd->stepid);
 
2425
                                continue;
 
2426
                        }
 
2427
                        
 
2428
 
 
2429
                        fdi++;
 
2430
                        if (fdi >= NUM_PARALLEL_SUSPEND)
 
2431
                                break;
 
2432
                }
 
2433
                /* check for open connections */
 
2434
                if (fdi == 0)
 
2435
                        break;
2012
2436
 
2013
2437
                if (req->op == SUSPEND_JOB) {
2014
 
                        debug2("Suspending job step %u.%u",
2015
 
                               stepd->jobid, stepd->stepid);
2016
 
                        if (stepd_suspend(fd) < 0)
2017
 
                                debug("  suspend failed: %m", stepd->jobid);
 
2438
                        stepd_suspend(fd, fdi, req->job_id);
2018
2439
                } else {
2019
 
                        debug2("Resuming job step %u.%u",
2020
 
                               stepd->jobid, stepd->stepid);
2021
 
                        if (stepd_resume(fd) < 0)
2022
 
                                debug("  resume failed: %m", stepd->jobid);
 
2440
                        /* "resume" remains a serial action (for now) */
 
2441
                        for (x = 0; x < fdi; x++) {
 
2442
                                debug2("Resuming job %u (cached step count %d)",
 
2443
                                        req->job_id, x);
 
2444
                                if (stepd_resume(fd[x]) < 0)
 
2445
                                        debug("  resume failed: %m");
 
2446
                        }
2023
2447
                }
 
2448
                for (x = 0; x < fdi; x++)
 
2449
                        /* fd may have been closed by stepd_suspend */
 
2450
                        if (fd[x] != -1)
 
2451
                                close(fd[x]);
2024
2452
 
2025
 
                close(fd);
 
2453
                /* check for no more jobs */
 
2454
                if (fdi < NUM_PARALLEL_SUSPEND)
 
2455
                        break;
2026
2456
        }
2027
2457
        list_iterator_destroy(i);
2028
2458
        list_destroy(steps);
 
2459
        _unlock_suspend_job(req->job_id);
2029
2460
 
2030
 
 no_job:
2031
2461
        if (step_cnt == 0) {
2032
2462
                debug2("No steps in jobid %u to suspend/resume", 
2033
2463
                        req->job_id);
2034
2464
        }
2035
 
 
2036
 
        /*
2037
 
         *  At this point, if connection still open, we send controller
2038
 
         *  a reply.
2039
 
         */
2040
 
 fini:  if (msg->conn_fd >= 0) {
2041
 
                slurm_send_rc_msg(msg, rc);
2042
 
                if (slurm_close_accepted_conn(msg->conn_fd) < 0)
2043
 
                        error ("_rpc_signal_job: close(%d): %m", msg->conn_fd);
2044
 
                msg->conn_fd = -1;
2045
 
        }
2046
2465
}
2047
2466
 
2048
2467
static void 
2050
2469
{
2051
2470
        int             rc     = SLURM_SUCCESS;
2052
2471
        kill_job_msg_t *req    = msg->data;
2053
 
        uid_t           uid    = g_slurm_auth_get_uid(msg->auth_cred);
 
2472
        uid_t           uid    = g_slurm_auth_get_uid(msg->auth_cred, NULL);
2054
2473
        int             nsteps = 0;
2055
2474
        int             delay;
2056
2475
        char           *bg_part_id = NULL;
 
2476
        uint16_t        base_job_state = req->job_state & (~JOB_COMPLETING);
2057
2477
        slurm_ctl_conf_t *cf;
2058
2478
 
2059
2479
        debug("_rpc_terminate_job, uid = %d", uid);
2095
2515
                debug("credential for job %u revoked", req->job_id);
2096
2516
        }
2097
2517
 
 
2518
        if ((base_job_state == JOB_NODE_FAIL) || 
 
2519
            (base_job_state == JOB_PENDING))            /* requeued */
 
2520
                _kill_all_active_steps(req->job_id, SIG_NODE_FAIL, true);
 
2521
        else if (base_job_state == JOB_FAILED)
 
2522
                _kill_all_active_steps(req->job_id, SIG_FAILURE, true);
 
2523
 
2098
2524
        /*
2099
2525
         * Tasks might be stopped (possibly by a debugger)
2100
2526
         * so send SIGCONT first.
2178
2604
        rc = _run_epilog(req->job_id, req->job_uid, bg_part_id);
2179
2605
        xfree(bg_part_id);
2180
2606
        
2181
 
        if (rc != 0) {
2182
 
                error ("[job %u] epilog failed", req->job_id);
 
2607
        if (rc) {
 
2608
                int term_sig, exit_status;
 
2609
                if (WIFSIGNALED(rc)) {
 
2610
                        exit_status = 0;
 
2611
                        term_sig    = WTERMSIG(rc);
 
2612
                } else {
 
2613
                        exit_status = WEXITSTATUS(rc);
 
2614
                        term_sig    = 0;
 
2615
                }
 
2616
                error("[job %u] epilog failed status=%d:%d", 
 
2617
                      req->job_id, exit_status, term_sig);
2183
2618
                rc = ESLURMD_EPILOG_FAILED;
2184
2619
        } else
2185
2620
                debug("completed epilog for jobid %u", req->job_id);
2201
2636
        int host_cnt, host_inx;
2202
2637
        char *host;
2203
2638
        hostset_t hosts;
 
2639
        int epilog_msg_time;
2204
2640
 
2205
2641
        hosts = hostset_create(req->nodes);
2206
2642
        host_cnt = hostset_count(hosts);
2207
 
        if (host_cnt <= 32)
 
2643
        if (host_cnt <= 64)
2208
2644
                goto fini;
2209
2645
        if (conf->hostname == NULL)
2210
2646
                goto fini;      /* should never happen */
2219
2655
                }
2220
2656
                free(host);
2221
2657
        }
2222
 
        _delay_rpc(host_inx, host_cnt, 10000);
 
2658
        epilog_msg_time = slurm_get_epilog_msg_time();
 
2659
        _delay_rpc(host_inx, host_cnt, epilog_msg_time);
2223
2660
 
2224
2661
 fini:  hostset_destroy(hosts);
2225
2662
}
2354
2791
_rpc_update_time(slurm_msg_t *msg)
2355
2792
{
2356
2793
        int   rc      = SLURM_SUCCESS;
2357
 
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred);
 
2794
        uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred, NULL);
2358
2795
 
2359
2796
        if ((req_uid != conf->slurm_user_id) && (req_uid != 0)) {
2360
2797
                rc = ESLURM_USER_ID_MISSING;