← Back to branch summary

~ubuntu-branches/ubuntu/vivid/slurm-llnl/vivid

« back to all changes in this revision

Viewing changes to src/plugins/accounting_storage/mysql/mysql_jobacct_process.c

  • Committer: Bazaar Package Importer
  • Author(s): Gennaro Oliva
  • Date: 2009-09-24 23:28:15 UTC
  • mfrom: (1.1.11 upstream) (3.2.4 sid)
  • Revision ID: james.westby@ubuntu.com-20090924232815-enh65jn32q1ebg07
Tags: 2.0.5-1
http://bugs.debian.org/541252
* New upstream release 
* Changed dependecy from lib-mysqlclient15 to lib-mysqlclient 
* Added Default-Start for runlevel 2 and 4 and $remote_fs requirement in
  init.d scripts (Closes: #541252)
* Postinst checks for wrong runlevels 2 and 4 links
* Upgraded to standard version 3.8.3
* Add lintian overrides for missing slurm-llnl-configurator.html in doc
  base registration
* modified postrm scripts to ignore pkill return value in order to avoid
  postrm failure when no slurm process is running
* Checking for slurmctld.pid before cancelling running and pending
  jobs during package removal 

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/plugins/accounting_storage/mysql/mysql_jobacct_process.c
4
4
 *                               storage.
5
5
 *****************************************************************************
6
6
 *
 
7
 *  Copyright (C) 2008-2009 Lawrence Livermore National Security.
7
8
 *  Copyright (C) 2004-2007 The Regents of the University of California.
8
9
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
9
10
 *  Written by Danny Auble <da@llnl.gov>
10
11
 *  
11
12
 *  This file is part of SLURM, a resource management program.
12
 
 *  For details, see <http://www.llnl.gov/linux/slurm/>.
 
13
 *  For details, see <https://computing.llnl.gov/linux/slurm/>.
 
14
 *  Please also read the included file: DISCLAIMER.
13
15
 *  
14
16
 *  SLURM is free software; you can redistribute it and/or modify it under
15
17
 *  the terms of the GNU General Public License as published by the Free
46
48
#include "mysql_jobacct_process.h"
47
49
#include <fcntl.h>
48
50
 
49
 
#ifdef HAVE_MYSQL
50
51
static pthread_mutex_t local_file_lock = PTHREAD_MUTEX_INITIALIZER;
51
52
 
 
53
typedef struct {
 
54
        hostlist_t hl;
 
55
        time_t start;
 
56
        time_t end;
 
57
        bitstr_t *asked_bitmap;
 
58
} local_cluster_t;
 
59
 
 
60
static void _destroy_local_cluster(void *object)
 
61
{
 
62
        local_cluster_t *local_cluster = (local_cluster_t *)object;
 
63
        if(local_cluster) {
 
64
                if(local_cluster->hl)
 
65
                        hostlist_destroy(local_cluster->hl);
 
66
                FREE_NULL_BITMAP(local_cluster->asked_bitmap);
 
67
                xfree(local_cluster);
 
68
        }
 
69
}
 
70
 
52
71
static int _write_to_file(int fd, char *data)
53
72
{
54
73
        int pos = 0, nwrite = strlen(data), amount;
67
86
        return rc;
68
87
}
69
88
 
 
89
static int _write_archive_file(MYSQL_RES *result, int start_col, int col_count,
 
90
                               time_t curr_end, char *arch_dir,
 
91
                               char *arch_type, char *insert,
 
92
                               bool with_deleted)
 
93
{
 
94
        time_t period_start = 0;
 
95
        int fd = 0;
 
96
        int rc = SLURM_SUCCESS;
 
97
        MYSQL_ROW row;
 
98
        struct tm time_tm;
 
99
        char *old_file = NULL, *new_file = NULL, *reg_file = NULL;
 
100
        char *values = NULL;
 
101
        char start_char[32];
 
102
        char end_char[32];
 
103
        int i=0;
 
104
 
 
105
        xassert(result);
 
106
 
 
107
        //START_TIMER;
 
108
        slurm_mutex_lock(&local_file_lock);
 
109
        while((row = mysql_fetch_row(result))) {
 
110
                if(period_start) {
 
111
                        xstrcat(values, ",\n(");
 
112
                } else {
 
113
                        period_start = atoi(row[start_col]);
 
114
                        localtime_r((time_t *)&period_start, &time_tm);
 
115
                        time_tm.tm_sec = 0;
 
116
                        time_tm.tm_min = 0;
 
117
                        time_tm.tm_hour = 0;
 
118
                        time_tm.tm_mday = 1;
 
119
                        snprintf(start_char, sizeof(start_char),
 
120
                                 "%4.4u-%2.2u-%2.2u"
 
121
                                 "T%2.2u:%2.2u:%2.2u",
 
122
                                 (time_tm.tm_year + 1900),
 
123
                                 (time_tm.tm_mon+1), 
 
124
                                 time_tm.tm_mday,
 
125
                                 time_tm.tm_hour,
 
126
                                 time_tm.tm_min, 
 
127
                                 time_tm.tm_sec);
 
128
 
 
129
                        localtime_r((time_t *)&curr_end, &time_tm);
 
130
                        snprintf(end_char, sizeof(end_char),
 
131
                                 "%4.4u-%2.2u-%2.2u"
 
132
                                 "T%2.2u:%2.2u:%2.2u",
 
133
                                 (time_tm.tm_year + 1900),
 
134
                                 (time_tm.tm_mon+1), 
 
135
                                 time_tm.tm_mday,
 
136
                                 time_tm.tm_hour,
 
137
                                 time_tm.tm_min, 
 
138
                                 time_tm.tm_sec);
 
139
 
 
140
                        /* write the buffer to file */
 
141
                        reg_file = xstrdup_printf(
 
142
                                "%s/%s_archive_%s_%s.sql",
 
143
                                arch_dir, arch_type,
 
144
                                start_char, end_char);
 
145
                        debug("Storing event archive at %s", reg_file);
 
146
                        old_file = xstrdup_printf("%s.old", reg_file);
 
147
                        new_file = xstrdup_printf("%s.new", reg_file);
 
148
                                        
 
149
                        fd = creat(new_file, 0600);
 
150
                        if (fd == 0) {
 
151
                                error("Can't save archive, "
 
152
                                      "create file %s error %m",
 
153
                                      new_file);
 
154
                                rc = errno;
 
155
                                xfree(insert);
 
156
                                break;
 
157
                        } 
 
158
                        values = xstrdup_printf("%s\nvalues\n(", insert);
 
159
                }
 
160
        
 
161
                xstrfmtcat(values, "'%s'", row[0]);
 
162
                for(i=1; i<col_count; i++) {
 
163
                        xstrfmtcat(values, ", '%s'", row[i]);
 
164
                }
 
165
 
 
166
                if(with_deleted)
 
167
                        xstrcat(values, ", '1')");
 
168
                else
 
169
                        xstrcat(values, ")");
 
170
                                
 
171
                if(!fd 
 
172
                   || ((rc = _write_to_file(fd, values)) != SLURM_SUCCESS)) {
 
173
                        xfree(values);
 
174
                        break;
 
175
                }
 
176
                xfree(values);
 
177
        }
 
178
 
 
179
        if(with_deleted)
 
180
                rc = _write_to_file(fd,
 
181
                            " on duplicate key update "
 
182
                            "deleted=1;");
 
183
        else
 
184
                rc = _write_to_file(fd,
 
185
                            " on duplicate key update "
 
186
                            "period_end=VALUES(period_end);");
 
187
//                      END_TIMER2("write file");
 
188
//                      info("write file took %s", TIME_STR);
 
189
 
 
190
        fsync(fd);
 
191
        close(fd);
 
192
                        
 
193
        if (rc)
 
194
                (void) unlink(new_file);
 
195
        else {                  /* file shuffle */
 
196
                int ign;        /* avoid warning */
 
197
                (void) unlink(old_file);
 
198
                ign =  link(reg_file, old_file);
 
199
                (void) unlink(reg_file);
 
200
                ign =  link(new_file, reg_file);
 
201
                (void) unlink(new_file);
 
202
        }
 
203
        xfree(old_file);
 
204
        xfree(reg_file);
 
205
        xfree(new_file);
 
206
        slurm_mutex_unlock(&local_file_lock);
 
207
 
 
208
        return rc;
 
209
}
 
210
 
70
211
static int _archive_script(acct_archive_cond_t *arch_cond, time_t last_submit)
71
212
{
72
213
        char * args[] = {arch_cond->archive_script, NULL};
105
246
 
106
247
        env = env_array_create();
107
248
 
108
 
        if(arch_cond->step_purge) {
 
249
        if(arch_cond->purge_event) {
109
250
                /* use localtime to avoid any daylight savings issues */
110
251
                if(!localtime_r(&last_submit, &time_tm)) {
111
 
                        error("Couldn't get localtime from first step start %d",
 
252
                        error("Couldn't get localtime from "
 
253
                              "first event start %d",
112
254
                              last_submit);
113
255
                        return SLURM_ERROR;
114
256
                }
115
 
                time_tm.tm_mon -= arch_cond->step_purge;
 
257
                time_tm.tm_mon -= arch_cond->purge_step;
116
258
                time_tm.tm_isdst = -1;
117
259
                curr_end = mktime(&time_tm);
118
 
                env_array_append_fmt(&env, "SLURM_ARCHIVE_STEPS", "%u",
119
 
                                     arch_cond->archive_steps);
120
 
                env_array_append_fmt(&env, "SLURM_ARCHIVE_LAST_STEP", "%d",
 
260
                env_array_append_fmt(&env, "SLURM_ARCHIVE_EVENTS", "%u",
 
261
                                     arch_cond->archive_events);
 
262
                env_array_append_fmt(&env, "SLURM_ARCHIVE_LAST_EVENT", "%d",
121
263
                                     curr_end);
122
264
        }
123
265
 
124
 
        if(arch_cond->job_purge) {
 
266
        if(arch_cond->purge_job) {
125
267
                /* use localtime to avoid any daylight savings issues */
126
268
                if(!localtime_r(&last_submit, &time_tm)) {
127
269
                        error("Couldn't get localtime from first start %d",
128
270
                              last_submit);
129
271
                        return SLURM_ERROR;
130
272
                }
131
 
                time_tm.tm_mon -= arch_cond->job_purge;
 
273
                time_tm.tm_mon -= arch_cond->purge_job;
132
274
                time_tm.tm_isdst = -1;
133
275
                curr_end = mktime(&time_tm);
134
276
                
138
280
                                      curr_end);
139
281
        }
140
282
 
 
283
        if(arch_cond->purge_step) {
 
284
                /* use localtime to avoid any daylight savings issues */
 
285
                if(!localtime_r(&last_submit, &time_tm)) {
 
286
                        error("Couldn't get localtime from first step start %d",
 
287
                              last_submit);
 
288
                        return SLURM_ERROR;
 
289
                }
 
290
                time_tm.tm_mon -= arch_cond->purge_step;
 
291
                time_tm.tm_isdst = -1;
 
292
                curr_end = mktime(&time_tm);
 
293
                env_array_append_fmt(&env, "SLURM_ARCHIVE_STEPS", "%u",
 
294
                                     arch_cond->archive_steps);
 
295
                env_array_append_fmt(&env, "SLURM_ARCHIVE_LAST_STEP", "%d",
 
296
                                     curr_end);
 
297
        }
 
298
 
 
299
        if(arch_cond->purge_suspend) {
 
300
                /* use localtime to avoid any daylight savings issues */
 
301
                if(!localtime_r(&last_submit, &time_tm)) {
 
302
                        error("Couldn't get localtime from first "
 
303
                              "suspend start %d",
 
304
                              last_submit);
 
305
                        return SLURM_ERROR;
 
306
                }
 
307
                time_tm.tm_mon -= arch_cond->purge_step;
 
308
                time_tm.tm_isdst = -1;
 
309
                curr_end = mktime(&time_tm);
 
310
                env_array_append_fmt(&env, "SLURM_ARCHIVE_SUSPEND", "%u",
 
311
                                     arch_cond->archive_steps);
 
312
                env_array_append_fmt(&env, "SLURM_ARCHIVE_LAST_SUSPEND", "%d",
 
313
                                     curr_end);
 
314
        }
 
315
 
141
316
#ifdef _PATH_STDPATH
142
317
        env_array_append (&env, "PATH", _PATH_STDPATH);
143
318
#else
150
325
        return SLURM_SUCCESS;
151
326
}
152
327
 
153
 
 
154
 
extern int setup_job_cond_limits(acct_job_cond_t *job_cond, char **extra)
 
328
extern List setup_cluster_list_with_inx(mysql_conn_t *mysql_conn,
 
329
                                        acct_job_cond_t *job_cond,
 
330
                                        void **curr_cluster)
 
331
{
 
332
        List local_cluster_list = NULL;
 
333
        time_t now = time(NULL);
 
334
        MYSQL_RES *result = NULL;
 
335
        MYSQL_ROW row;
 
336
        hostlist_t temp_hl = NULL;
 
337
        hostlist_iterator_t h_itr = NULL;
 
338
        char *object = NULL;
 
339
        char *query = NULL;
 
340
        
 
341
        if(!job_cond || !job_cond->used_nodes) 
 
342
                return NULL;
 
343
 
 
344
        if(!job_cond->cluster_list || list_count(job_cond->cluster_list) != 1) {
 
345
                error("If you are doing a query against nodes "
 
346
                      "you must only have 1 cluster "
 
347
                      "you are asking for.");
 
348
                return NULL;
 
349
        }
 
350
        
 
351
        temp_hl = hostlist_create(job_cond->used_nodes);
 
352
        if(!hostlist_count(temp_hl)) {
 
353
                error("we didn't get any real hosts to look for.");
 
354
                goto no_hosts;
 
355
        }
 
356
        h_itr = hostlist_iterator_create(temp_hl);
 
357
 
 
358
        query = xstrdup_printf("select cluster_nodes, period_start, "
 
359
                               "period_end from %s where node_name='' "
 
360
                               "&& cluster_nodes !=''",
 
361
                               event_table);
 
362
 
 
363
        if((object = list_peek(job_cond->cluster_list)))
 
364
                xstrfmtcat(query, " && cluster='%s'", object);
 
365
 
 
366
        if(job_cond->usage_start) {
 
367
                if(!job_cond->usage_end)
 
368
                        job_cond->usage_end = now;
 
369
 
 
370
                xstrfmtcat(query, 
 
371
                           " && ((period_start < %d) "
 
372
                           "&& (period_end >= %d || period_end = 0))",
 
373
                           job_cond->usage_end, job_cond->usage_start);
 
374
        }
 
375
                
 
376
        debug3("%d(%d) query\n%s", mysql_conn->conn, __LINE__, query);
 
377
        if(!(result = mysql_db_query_ret(mysql_conn->db_conn, query, 0))) {
 
378
                xfree(query);
 
379
                hostlist_destroy(temp_hl);
 
380
                return NULL;
 
381
        }
 
382
        xfree(query);
 
383
 
 
384
        local_cluster_list = list_create(_destroy_local_cluster);
 
385
        while((row = mysql_fetch_row(result))) {
 
386
                char *host = NULL;
 
387
                int loc = 0;
 
388
                local_cluster_t *local_cluster =
 
389
                        xmalloc(sizeof(local_cluster_t));
 
390
                local_cluster->hl = hostlist_create(row[0]);
 
391
                local_cluster->start = atoi(row[1]);
 
392
                local_cluster->end   = atoi(row[2]);
 
393
                local_cluster->asked_bitmap = 
 
394
                        bit_alloc(hostlist_count(local_cluster->hl));
 
395
                while((host = hostlist_next(h_itr))) {
 
396
                        if((loc = hostlist_find(
 
397
                                    local_cluster->hl, host)) != -1) 
 
398
                                bit_set(local_cluster->asked_bitmap, loc);
 
399
                        free(host);
 
400
                }
 
401
                hostlist_iterator_reset(h_itr);
 
402
                if(bit_ffs(local_cluster->asked_bitmap) != -1) {
 
403
                        list_append(local_cluster_list, local_cluster);
 
404
                        if(local_cluster->end == 0) {
 
405
                                local_cluster->end = now;
 
406
                                (*curr_cluster) = local_cluster;
 
407
                        }
 
408
                } else 
 
409
                        _destroy_local_cluster(local_cluster);
 
410
        }
 
411
        mysql_free_result(result);
 
412
        hostlist_iterator_destroy(h_itr);
 
413
        if(!list_count(local_cluster_list)) {
 
414
                hostlist_destroy(temp_hl);
 
415
                list_destroy(local_cluster_list);
 
416
                return NULL;
 
417
        }
 
418
no_hosts:
 
419
 
 
420
        hostlist_destroy(temp_hl);
 
421
 
 
422
        return local_cluster_list;
 
423
}
 
424
 
 
425
extern int good_nodes_from_inx(List local_cluster_list, 
 
426
                               void **object, char *node_inx,
 
427
                               int submit)
 
428
{
 
429
        local_cluster_t **curr_cluster = (local_cluster_t **)object;
 
430
 
 
431
        /* check the bitmap to see if this is one of the jobs
 
432
           we are looking for */
 
433
        if(*curr_cluster) {
 
434
                bitstr_t *job_bitmap = NULL;
 
435
                if(!node_inx || !node_inx[0])
 
436
                        return 0;
 
437
                if((submit < (*curr_cluster)->start)
 
438
                   || (submit > (*curr_cluster)->end)) {
 
439
                        local_cluster_t *local_cluster = NULL;
 
440
                        
 
441
                        ListIterator itr =
 
442
                                list_iterator_create(local_cluster_list);
 
443
                        while((local_cluster = list_next(itr))) {
 
444
                                if((submit >= local_cluster->start)
 
445
                                   && (submit <= local_cluster->end)) {
 
446
                                        *curr_cluster = local_cluster;
 
447
                                                break;
 
448
                                }
 
449
                        }
 
450
                        list_iterator_destroy(itr);
 
451
                        if(!local_cluster)
 
452
                                return 0;
 
453
                }
 
454
                job_bitmap = bit_alloc(hostlist_count((*curr_cluster)->hl));
 
455
                bit_unfmt(job_bitmap, node_inx);
 
456
                if(!bit_overlap((*curr_cluster)->asked_bitmap, job_bitmap)) {
 
457
                        FREE_NULL_BITMAP(job_bitmap);
 
458
                        return 0;
 
459
                }
 
460
                FREE_NULL_BITMAP(job_bitmap);
 
461
        }
 
462
        return 1;
 
463
}
 
464
 
 
465
extern int setup_job_cond_limits(mysql_conn_t *mysql_conn,
 
466
                                 acct_job_cond_t *job_cond, char **extra)
155
467
{
156
468
        int set = 0;
157
469
        ListIterator itr = NULL;
158
470
        char *object = NULL;
159
471
        char *table_level = "t2";
160
472
        jobacct_selected_step_t *selected_step = NULL;
161
 
        time_t now = time(NULL);
162
473
 
163
474
        if(!job_cond)
164
475
                return 0;
254
565
                xstrcat(*extra, ")");
255
566
        }
256
567
 
 
568
        /* this must be done before resvid_list since we set
 
569
           resvid_list up here */
 
570
        if(job_cond->resv_list && list_count(job_cond->resv_list)) {
 
571
                char *query = xstrdup_printf(
 
572
                        "select distinct id from %s where (");
 
573
                int my_set = 0;
 
574
                MYSQL_RES *result = NULL;
 
575
                MYSQL_ROW row;
 
576
 
 
577
                if(job_cond->cluster_list
 
578
                   && list_count(job_cond->cluster_list)) {
 
579
                        
 
580
                        itr = list_iterator_create(job_cond->cluster_list);
 
581
                        while((object = list_next(itr))) {
 
582
                                if(my_set) 
 
583
                                        xstrcat(query, " || ");
 
584
                                xstrfmtcat(query, "cluster='%s'", object);
 
585
                                my_set = 1;
 
586
                        }
 
587
                        list_iterator_destroy(itr);
 
588
                } 
 
589
 
 
590
                if(my_set)
 
591
                        xstrcat(query, ") && (");
 
592
                
 
593
                itr = list_iterator_create(job_cond->resv_list);
 
594
                while((object = list_next(itr))) {
 
595
                        if(my_set)
 
596
                                xstrcat(query, " || ");
 
597
                        xstrfmtcat(query, "name='%s'", object);
 
598
                        my_set = 1;
 
599
                }
 
600
                list_iterator_destroy(itr);
 
601
                xstrcat(query, ")");
 
602
                if(!(result = mysql_db_query_ret(
 
603
                             mysql_conn->db_conn, query, 0))) {
 
604
                        xfree(query);
 
605
                        error("couldn't query the database");
 
606
                        goto no_resv;
 
607
                }
 
608
                xfree(query);
 
609
                if(!job_cond->resvid_list) 
 
610
                        job_cond->resvid_list = list_create(slurm_destroy_char);
 
611
                while((row = mysql_fetch_row(result))) {
 
612
                        list_append(job_cond->resvid_list, xstrdup(row[0]));
 
613
                }
 
614
                mysql_free_result(result);
 
615
        }
 
616
        no_resv:
 
617
 
 
618
        if(job_cond->resvid_list && list_count(job_cond->resvid_list)) {
 
619
                set = 0;
 
620
                if(*extra)
 
621
                        xstrcat(*extra, " && (");
 
622
                else
 
623
                        xstrcat(*extra, " where (");
 
624
                itr = list_iterator_create(job_cond->resvid_list);
 
625
                while((object = list_next(itr))) {
 
626
                        if(set)
 
627
                                xstrcat(*extra, " || ");
 
628
                        xstrfmtcat(*extra, "t1.resvid='%s'", object);
 
629
                        set = 1;
 
630
                }
 
631
                list_iterator_destroy(itr);
 
632
                xstrcat(*extra, ")");
 
633
        }
 
634
 
257
635
        if(job_cond->step_list && list_count(job_cond->step_list)) {
258
636
                set = 0;
259
637
                if(*extra)
272
650
        }
273
651
 
274
652
        if(job_cond->usage_start) {
275
 
                if(!job_cond->usage_end)
276
 
                        job_cond->usage_end = now;
277
 
 
278
653
                if(*extra)
279
654
                        xstrcat(*extra, " && (");
280
655
                else
281
656
                        xstrcat(*extra, " where (");
282
 
                xstrfmtcat(*extra, 
283
 
                           "(t1.eligible < %d "
284
 
                           "&& (t1.end >= %d || t1.end = 0)))",
285
 
                           job_cond->usage_end, job_cond->usage_start);
 
657
 
 
658
                if(!job_cond->usage_end)
 
659
                        xstrfmtcat(*extra, 
 
660
                                   "t1.end >= %d || t1.end = 0)",
 
661
                                   job_cond->usage_start);
 
662
                else
 
663
                        xstrfmtcat(*extra, 
 
664
                                   "(t1.eligible < %d "
 
665
                                   "&& (t1.end >= %d || t1.end = 0)))",
 
666
                                   job_cond->usage_end, job_cond->usage_start);
286
667
        } else if(job_cond->usage_end) {
287
668
                if(*extra)
288
669
                        xstrcat(*extra, " && (");
371
752
        List job_list = list_create(destroy_jobacct_job_rec);
372
753
        uint16_t private_data = 0;
373
754
        acct_user_rec_t user;
374
 
                
 
755
        local_cluster_t *curr_cluster = NULL;
 
756
        List local_cluster_list = NULL;
 
757
 
375
758
        /* if this changes you will need to edit the corresponding 
376
759
         * enum below also t1 is job_table */
377
760
        char *job_req_inx[] = {
382
765
                "t1.wckeyid",
383
766
                "t1.uid",
384
767
                "t1.gid",
 
768
                "t1.resvid",
385
769
                "t1.partition",
386
770
                "t1.blockid",
387
771
                "t1.cluster",
398
782
                "t1.priority",
399
783
                "t1.req_cpus",
400
784
                "t1.alloc_cpus",
 
785
                "t1.alloc_nodes",
401
786
                "t1.nodelist",
 
787
                "t1.node_inx",
402
788
                "t1.kill_requid",
403
789
                "t1.qos",
404
790
                "t2.user",
407
793
                "t2.lft"
408
794
        };
409
795
 
 
796
        enum {
 
797
                JOB_REQ_ID,
 
798
                JOB_REQ_JOBID,
 
799
                JOB_REQ_ASSOCID,
 
800
                JOB_REQ_WCKEY,
 
801
                JOB_REQ_WCKEYID,
 
802
                JOB_REQ_UID,
 
803
                JOB_REQ_GID,
 
804
                JOB_REQ_RESVID,
 
805
                JOB_REQ_PARTITION,
 
806
                JOB_REQ_BLOCKID,
 
807
                JOB_REQ_CLUSTER1,
 
808
                JOB_REQ_ACCOUNT1,
 
809
                JOB_REQ_ELIGIBLE,
 
810
                JOB_REQ_SUBMIT,
 
811
                JOB_REQ_START,
 
812
                JOB_REQ_END,
 
813
                JOB_REQ_SUSPENDED,
 
814
                JOB_REQ_NAME,
 
815
                JOB_REQ_TRACKSTEPS,
 
816
                JOB_REQ_STATE,
 
817
                JOB_REQ_COMP_CODE,
 
818
                JOB_REQ_PRIORITY,
 
819
                JOB_REQ_REQ_CPUS,
 
820
                JOB_REQ_ALLOC_CPUS,
 
821
                JOB_REQ_ALLOC_NODES,
 
822
                JOB_REQ_NODELIST,
 
823
                JOB_REQ_NODE_INX,
 
824
                JOB_REQ_KILL_REQUID,
 
825
                JOB_REQ_QOS,
 
826
                JOB_REQ_USER_NAME,
 
827
                JOB_REQ_CLUSTER,
 
828
                JOB_REQ_ACCOUNT,
 
829
                JOB_REQ_LFT,
 
830
                JOB_REQ_COUNT           
 
831
        };
 
832
 
410
833
        /* if this changes you will need to edit the corresponding 
411
834
         * enum below also t1 is step_table */
412
835
        char *step_req_inx[] = {
416
839
                "t1.suspended",
417
840
                "t1.name",
418
841
                "t1.nodelist",
 
842
                "t1.node_inx",
419
843
                "t1.state",
420
844
                "t1.kill_requid",
421
845
                "t1.comp_code",
 
846
                "t1.nodes",
422
847
                "t1.cpus",
 
848
                "t1.tasks",
 
849
                "t1.task_dist",
423
850
                "t1.user_sec",
424
851
                "t1.user_usec",
425
852
                "t1.sys_sec",
443
870
        };
444
871
 
445
872
        enum {
446
 
                JOB_REQ_ID,
447
 
                JOB_REQ_JOBID,
448
 
                JOB_REQ_ASSOCID,
449
 
                JOB_REQ_WCKEY,
450
 
                JOB_REQ_WCKEYID,
451
 
                JOB_REQ_UID,
452
 
                JOB_REQ_GID,
453
 
                JOB_REQ_PARTITION,
454
 
                JOB_REQ_BLOCKID,
455
 
                JOB_REQ_CLUSTER1,
456
 
                JOB_REQ_ACCOUNT1,
457
 
                JOB_REQ_ELIGIBLE,
458
 
                JOB_REQ_SUBMIT,
459
 
                JOB_REQ_START,
460
 
                JOB_REQ_END,
461
 
                JOB_REQ_SUSPENDED,
462
 
                JOB_REQ_NAME,
463
 
                JOB_REQ_TRACKSTEPS,
464
 
                JOB_REQ_STATE,
465
 
                JOB_REQ_COMP_CODE,
466
 
                JOB_REQ_PRIORITY,
467
 
                JOB_REQ_REQ_CPUS,
468
 
                JOB_REQ_ALLOC_CPUS,
469
 
                JOB_REQ_NODELIST,
470
 
                JOB_REQ_KILL_REQUID,
471
 
                JOB_REQ_QOS,
472
 
                JOB_REQ_USER_NAME,
473
 
                JOB_REQ_CLUSTER,
474
 
                JOB_REQ_ACCOUNT,
475
 
                JOB_REQ_LFT,
476
 
                JOB_REQ_COUNT           
477
 
        };
478
 
        enum {
479
873
                STEP_REQ_STEPID,
480
874
                STEP_REQ_START,
481
875
                STEP_REQ_END,
482
876
                STEP_REQ_SUSPENDED,
483
877
                STEP_REQ_NAME,
484
878
                STEP_REQ_NODELIST,
 
879
                STEP_REQ_NODE_INX,
485
880
                STEP_REQ_STATE,
486
881
                STEP_REQ_KILL_REQUID,
487
882
                STEP_REQ_COMP_CODE,
 
883
                STEP_REQ_NODES,
488
884
                STEP_REQ_CPUS,
 
885
                STEP_REQ_TASKS,
 
886
                STEP_REQ_TASKDIST,
489
887
                STEP_REQ_USER_SEC,
490
888
                STEP_REQ_USER_USEC,
491
889
                STEP_REQ_SYS_SEC,
531
929
                           >= ACCT_ADMIN_OPERATOR) 
532
930
                                is_admin = 1;   
533
931
                        else {
534
 
                                assoc_mgr_fill_in_user(mysql_conn, &user, 1);
 
932
                                assoc_mgr_fill_in_user(mysql_conn, &user, 1,
 
933
                                                       NULL);
535
934
                        }
536
935
                }
537
936
        }
538
937
 
539
 
        setup_job_cond_limits(job_cond, &extra);
 
938
 
 
939
        /* Here we set up environment to check used nodes of jobs.
 
940
           Since we store the bitmap of the entire cluster we can use
 
941
           that to set up a hostlist and set up the bitmap to make
 
942
           things work.  This should go before the setup of conds
 
943
           since we could update the start/end time.
 
944
        */
 
945
        if(job_cond && job_cond->used_nodes) {
 
946
                local_cluster_list = setup_cluster_list_with_inx(
 
947
                        mysql_conn, job_cond, (void **)&curr_cluster);
 
948
                if(!local_cluster_list) {
 
949
                        list_destroy(job_list);
 
950
                        return NULL;
 
951
                }
 
952
        }
 
953
 
 
954
        setup_job_cond_limits(mysql_conn, job_cond, &extra);
540
955
 
541
956
        xfree(tmp);
542
957
        xstrfmtcat(tmp, "%s", job_req_inx[0]);
565
980
                             mysql_conn->db_conn, query, 0))) {
566
981
                        xfree(extra);
567
982
                        xfree(query);
 
983
                        list_destroy(job_list);
 
984
                        if(local_cluster_list)
 
985
                                list_destroy(local_cluster_list);
568
986
                        return NULL;
569
987
                }
570
988
                xfree(query);
603
1021
                xstrcat(query, extra);
604
1022
                xfree(extra);
605
1023
        }
 
1024
        
606
1025
        /* Here we want to order them this way in such a way so it is
607
1026
           easy to look for duplicates 
608
1027
        */
610
1029
                xstrcat(query, " order by t1.cluster, jobid, submit desc");
611
1030
        else
612
1031
                xstrcat(query, " order by t1.cluster, submit desc");
613
 
                
614
1032
 
615
1033
        debug3("%d(%d) query\n%s", mysql_conn->conn, __LINE__, query);
616
 
        if(!(result = mysql_db_query_ret(
617
 
                     mysql_conn->db_conn, query, 0))) {
 
1034
        if(!(result = mysql_db_query_ret(mysql_conn->db_conn, query, 0))) {
618
1035
                xfree(query);
619
1036
                list_destroy(job_list);
 
1037
                if(local_cluster_list)
 
1038
                        list_destroy(local_cluster_list);
620
1039
                return NULL;
621
1040
        }
622
1041
        xfree(query);
624
1043
        while((row = mysql_fetch_row(result))) {
625
1044
                char *id = row[JOB_REQ_ID];
626
1045
                bool job_ended = 0;
 
1046
                int submit = atoi(row[JOB_REQ_SUBMIT]);
627
1047
 
628
1048
                curr_id = atoi(row[JOB_REQ_JOBID]);
629
1049
 
632
1052
                
633
1053
                last_id = curr_id;
634
1054
 
 
1055
                /* check the bitmap to see if this is one of the jobs
 
1056
                   we are looking for */
 
1057
                if(!good_nodes_from_inx(local_cluster_list,
 
1058
                                        (void **)&curr_cluster,
 
1059
                                        row[JOB_REQ_NODE_INX], submit))
 
1060
                        continue;
 
1061
                
635
1062
                job = create_jobacct_job_rec();
 
1063
                list_append(job_list, job);
636
1064
 
637
1065
                job->alloc_cpus = atoi(row[JOB_REQ_ALLOC_CPUS]);
 
1066
                job->alloc_nodes = atoi(row[JOB_REQ_ALLOC_NODES]);
638
1067
                job->associd = atoi(row[JOB_REQ_ASSOCID]);
 
1068
                job->resvid = atoi(row[JOB_REQ_RESVID]);
639
1069
 
640
 
                if(row[JOB_REQ_WCKEY] && row[JOB_REQ_WCKEY][0])
 
1070
                /* we want a blank wckey if the name is null */
 
1071
                if(row[JOB_REQ_WCKEY])
641
1072
                        job->wckey = xstrdup(row[JOB_REQ_WCKEY]);
 
1073
                else
 
1074
                        job->wckey = xstrdup("");
642
1075
                job->wckeyid = atoi(row[JOB_REQ_WCKEYID]);
643
1076
 
644
1077
                if(row[JOB_REQ_CLUSTER] && row[JOB_REQ_CLUSTER][0])
663
1096
                        job->blockid = xstrdup(row[JOB_REQ_BLOCKID]);
664
1097
 
665
1098
                job->eligible = atoi(row[JOB_REQ_ELIGIBLE]);
666
 
                job->submit = atoi(row[JOB_REQ_SUBMIT]);
 
1099
                job->submit = submit;
667
1100
                job->start = atoi(row[JOB_REQ_START]);
668
1101
                job->end = atoi(row[JOB_REQ_END]);
 
1102
 
669
1103
                /* since the job->end could be set later end it here */
670
 
                if(job->end)
 
1104
                if(job->end) {
671
1105
                        job_ended = 1;
 
1106
                        if(!job->start || (job->start > job->end))
 
1107
                                job->start = job->end;
 
1108
                }
672
1109
 
673
 
                if(job_cond && job_cond->usage_start) {
 
1110
                if(job_cond && !job_cond->without_usage_truncation
 
1111
                   && job_cond->usage_start) {
674
1112
                        if(job->start && (job->start < job_cond->usage_start))
675
1113
                                job->start = job_cond->usage_start;
676
1114
 
677
 
                        if(!job->start && job->end)
678
 
                                job->start = job->end;
679
 
 
680
1115
                        if(!job->end || job->end > job_cond->usage_end) 
681
1116
                                job->end = job_cond->usage_end;
682
1117
 
 
1118
                        if(!job->start)
 
1119
                                job->start = job->end;
 
1120
                        
683
1121
                        job->elapsed = job->end - job->start;
684
1122
 
685
1123
                        if(row[JOB_REQ_SUSPENDED]) {
772
1210
                job->qos = atoi(row[JOB_REQ_QOS]);
773
1211
                job->show_full = 1;
774
1212
                                        
775
 
                list_append(job_list, job);
776
 
 
777
1213
                if(job_cond && job_cond->step_list
778
1214
                   && list_count(job_cond->step_list)) {
779
1215
                        set = 0;
820
1256
                             mysql_conn->db_conn, query, 0))) {
821
1257
                        xfree(query);
822
1258
                        list_destroy(job_list);
 
1259
                        if(local_cluster_list)
 
1260
                                list_destroy(local_cluster_list);
823
1261
                        return NULL;
824
1262
                }
825
1263
                xfree(query);
 
1264
                
 
1265
                /* Querying the steps in the fashion was faster than
 
1266
                   doing only 1 query and then matching the steps up
 
1267
                   later with the job.  
 
1268
                */
826
1269
                while ((step_row = mysql_fetch_row(step_result))) {
 
1270
                        /* check the bitmap to see if this is one of the steps
 
1271
                           we are looking for */
 
1272
                        if(!good_nodes_from_inx(local_cluster_list,
 
1273
                                                (void **)&curr_cluster,
 
1274
                                                step_row[STEP_REQ_NODE_INX],
 
1275
                                                submit))
 
1276
                                continue;
 
1277
                
827
1278
                        step = create_jobacct_step_rec();
828
 
                        step->jobid = job->jobid;
 
1279
                        step->job_ptr = job;
 
1280
                        if(!job->first_step_ptr)
 
1281
                                job->first_step_ptr = step;
829
1282
                        list_append(job->steps, step);
830
1283
                        step->stepid = atoi(step_row[STEP_REQ_STEPID]);
831
1284
                        /* info("got step %u.%u", */
833
1286
                        step->state = atoi(step_row[STEP_REQ_STATE]);
834
1287
                        step->exitcode = atoi(step_row[STEP_REQ_COMP_CODE]);
835
1288
                        step->ncpus = atoi(step_row[STEP_REQ_CPUS]);
 
1289
                        step->nnodes = atoi(step_row[STEP_REQ_NODES]);
 
1290
 
 
1291
                        step->ntasks = atoi(step_row[STEP_REQ_TASKS]);
 
1292
                        step->task_dist = atoi(step_row[STEP_REQ_TASKDIST]);
 
1293
                        if(!step->ntasks)
 
1294
                                step->ntasks = step->ncpus;
 
1295
 
836
1296
                        step->start = atoi(step_row[STEP_REQ_START]);
837
1297
                        
838
1298
                        step->end = atoi(step_row[STEP_REQ_END]);
842
1302
                                step->state = job->state;
843
1303
                        }
844
1304
 
845
 
                        if(job_cond && job_cond->usage_start) {
 
1305
                        if(job_cond && !job_cond->without_usage_truncation
 
1306
                           && job_cond->usage_start) {
846
1307
                                if(step->start 
847
1308
                                   && (step->start < job_cond->usage_start))
848
1309
                                        step->start = job_cond->usage_start;
855
1316
                                        step->end = job_cond->usage_end;
856
1317
                        }
857
1318
 
858
 
                        step->elapsed = step->end - step->start;
859
1319
                        /* figure this out by start stop */
860
1320
                        step->suspended = atoi(step_row[STEP_REQ_SUSPENDED]);
861
1321
                        if(!step->end) {
928
1388
                        if(list_count(job->steps) > 1) 
929
1389
                                job->track_steps = 1;
930
1390
                        else if(step && step->stepname && job->jobname) {
931
 
                                if(strcmp(step->stepname, job->jobname))
 
1391
                                if(strcmp(step->stepname, job->jobname)) 
932
1392
                                        job->track_steps = 1;
933
1393
                        }
934
 
               }
 
1394
                }
935
1395
                /* need to reset here to make the above test valid */
936
1396
                step = NULL;
937
1397
        }
938
1398
        mysql_free_result(result);
 
1399
        if(local_cluster_list)
 
1400
                list_destroy(local_cluster_list);
939
1401
 
940
1402
        return job_list;
941
1403
}
943
1405
extern int mysql_jobacct_process_archive(mysql_conn_t *mysql_conn,
944
1406
                                         acct_archive_cond_t *arch_cond)
945
1407
{
946
 
        int rc = SLURM_SUCCESS, fd = 0;
 
1408
        int rc = SLURM_SUCCESS;
947
1409
        char *query = NULL;
948
1410
        time_t last_submit = time(NULL);
949
1411
        time_t curr_end;
950
1412
        char *tmp = NULL;
951
1413
        int i=0;
952
 
        char *old_file = NULL, *new_file = NULL, *reg_file = NULL;
953
1414
        struct tm time_tm;
954
 
        char start_char[32];
955
 
        char end_char[32];
 
1415
 
956
1416
//      DEF_TIMERS;
957
1417
 
958
1418
        /* if this changes you will need to edit the corresponding 
959
1419
         * enum below */
 
1420
        char *event_req_inx[] = {
 
1421
                "node_name",
 
1422
                "cluster",
 
1423
                "cpu_count",
 
1424
                "state",
 
1425
                "period_start",
 
1426
                "period_end",
 
1427
                "reason",
 
1428
                "cluster_nodes",
 
1429
        };
 
1430
 
 
1431
        /* if this changes you will need to edit the corresponding 
 
1432
         * enum below */
960
1433
        char *job_req_inx[] = {
961
1434
                "id",
962
1435
                "jobid",
965
1438
                "wckeyid",
966
1439
                "uid",
967
1440
                "gid",
 
1441
                "resvid",
968
1442
                "partition",
969
1443
                "blockid",
970
1444
                "cluster",
981
1455
                "priority",
982
1456
                "req_cpus",
983
1457
                "alloc_cpus",
 
1458
                "alloc_nodes",
984
1459
                "nodelist",
 
1460
                "node_inx",
985
1461
                "kill_requid",
986
1462
                "qos"
987
1463
        };
996
1472
                "suspended",
997
1473
                "name",
998
1474
                "nodelist",
 
1475
                "node_inx",
999
1476
                "state",
1000
1477
                "kill_requid",
1001
1478
                "comp_code",
 
1479
                "nodes",
1002
1480
                "cpus",
 
1481
                "tasks",
 
1482
                "task_dist",
1003
1483
                "user_sec",
1004
1484
                "user_usec",
1005
1485
                "sys_sec",
1022
1502
                "ave_cpu"
1023
1503
        };
1024
1504
 
 
1505
 
 
1506
        /* if this changes you will need to edit the corresponding 
 
1507
         * enum below */
 
1508
        char *suspend_req_inx[] = {
 
1509
                "id",
 
1510
                "associd",
 
1511
                "start",
 
1512
                "end",
 
1513
        };
 
1514
 
 
1515
        enum {
 
1516
                EVENT_REQ_NODE,
 
1517
                EVENT_REQ_CLUSTER,
 
1518
                EVENT_REQ_CPUS,
 
1519
                EVENT_REQ_STATE,
 
1520
                EVENT_REQ_START,
 
1521
                EVENT_REQ_END,
 
1522
                EVENT_REQ_REASON,
 
1523
                EVENT_REQ_NODES,
 
1524
                EVENT_REQ_COUNT
 
1525
        };
 
1526
 
1025
1527
        enum {
1026
1528
                JOB_REQ_ID,
1027
1529
                JOB_REQ_JOBID,
1030
1532
                JOB_REQ_WCKEYID,
1031
1533
                JOB_REQ_UID,
1032
1534
                JOB_REQ_GID,
 
1535
                JOB_REQ_RESVID,
1033
1536
                JOB_REQ_PARTITION,
1034
1537
                JOB_REQ_BLOCKID,
1035
1538
                JOB_REQ_CLUSTER,
1046
1549
                JOB_REQ_PRIORITY,
1047
1550
                JOB_REQ_REQ_CPUS,
1048
1551
                JOB_REQ_ALLOC_CPUS,
 
1552
                JOB_REQ_ALLOC_NODES,
1049
1553
                JOB_REQ_NODELIST,
 
1554
                JOB_REQ_NODE_INX,
1050
1555
                JOB_REQ_KILL_REQUID,
1051
1556
                JOB_REQ_QOS,
1052
1557
                JOB_REQ_COUNT           
1053
1558
        };
 
1559
 
1054
1560
        enum {
1055
1561
                STEP_REQ_ID,
1056
1562
                STEP_REQ_STEPID,
1059
1565
                STEP_REQ_SUSPENDED,
1060
1566
                STEP_REQ_NAME,
1061
1567
                STEP_REQ_NODELIST,
 
1568
                STEP_REQ_NODE_INX,
1062
1569
                STEP_REQ_STATE,
1063
1570
                STEP_REQ_KILL_REQUID,
1064
1571
                STEP_REQ_COMP_CODE,
 
1572
                STEP_REQ_NODES,
1065
1573
                STEP_REQ_CPUS,
 
1574
                STEP_REQ_TASKS,
 
1575
                STEP_REQ_TASKDIST,
1066
1576
                STEP_REQ_USER_SEC,
1067
1577
                STEP_REQ_USER_USEC,
1068
1578
                STEP_REQ_SYS_SEC,
1086
1596
                STEP_REQ_COUNT
1087
1597
        };
1088
1598
 
 
1599
        enum {
 
1600
                SUSPEND_REQ_ID,
 
1601
                SUSPEND_REQ_ASSOCID,
 
1602
                SUSPEND_REQ_START,
 
1603
                SUSPEND_REQ_END,
 
1604
                SUSPEND_REQ_COUNT
 
1605
        };
 
1606
 
1089
1607
        if(!arch_cond) {
1090
1608
                error("No arch_cond was given to archive from.  returning");
1091
1609
                return SLURM_ERROR;
1103
1621
        time_tm.tm_isdst = -1;
1104
1622
        last_submit = mktime(&time_tm);
1105
1623
        last_submit--;
1106
 
        debug("adjusted last submit is (%d)", last_submit);
 
1624
        debug("archive: adjusted last submit is (%d)", last_submit);
1107
1625
        
1108
1626
        if(arch_cond->archive_script)
1109
1627
                return _archive_script(arch_cond, last_submit);
1112
1630
                return SLURM_ERROR;
1113
1631
        }
1114
1632
 
1115
 
        if(arch_cond->step_purge) {
1116
 
                /* remove all data from step table that was older than
1117
 
                 * start * arch_cond->step_purge. 
 
1633
        if(arch_cond->purge_event) {
 
1634
                /* remove all data from step table that was older than
 
1635
                 * period_start * arch_cond->purge_event. 
 
1636
                 */
 
1637
                /* use localtime to avoid any daylight savings issues */
 
1638
                if(!localtime_r(&last_submit, &time_tm)) {
 
1639
                        error("Couldn't get localtime from first submit %d",
 
1640
                              last_submit);
 
1641
                        return SLURM_ERROR;
 
1642
                }
 
1643
                time_tm.tm_mday = 1;
 
1644
                time_tm.tm_mon -= arch_cond->purge_event;
 
1645
                time_tm.tm_isdst = -1;
 
1646
                curr_end = mktime(&time_tm);
 
1647
 
 
1648
                debug4("from %d - %d months purging events from before %d", 
 
1649
                       last_submit, arch_cond->purge_event, curr_end);
 
1650
                
 
1651
                if(arch_cond->archive_events) {
 
1652
                        char *insert = NULL;
 
1653
                        MYSQL_RES *result = NULL;
 
1654
                        
 
1655
                        xfree(tmp);
 
1656
                        xstrfmtcat(tmp, "%s", event_req_inx[0]);
 
1657
                        for(i=1; i<EVENT_REQ_COUNT; i++) {
 
1658
                                xstrfmtcat(tmp, ", %s", event_req_inx[i]);
 
1659
                        }
 
1660
 
 
1661
                        /* get all the events started before this time
 
1662
                           listed */
 
1663
                        query = xstrdup_printf("select %s from %s where "
 
1664
                                               "period_start <= %d "
 
1665
                                               "&& period_end != 0 "
 
1666
                                               "order by period_start asc",
 
1667
                                               tmp, event_table, curr_end);
 
1668
 
 
1669
                        insert = xstrdup_printf("insert into %s (%s) ",
 
1670
                                                event_table, tmp);
 
1671
                        xfree(tmp);
 
1672
                        
 
1673
//                      START_TIMER;
 
1674
                        debug3("%d(%d) query\n%s", mysql_conn->conn,
 
1675
                               __LINE__, query);
 
1676
                        if(!(result = mysql_db_query_ret(
 
1677
                                     mysql_conn->db_conn, query, 0))) {
 
1678
                                xfree(insert);
 
1679
                                xfree(query);
 
1680
                                return SLURM_ERROR;
 
1681
                        }
 
1682
                        xfree(query);
 
1683
//                      END_TIMER2("step query");
 
1684
//                      info("event query took %s", TIME_STR);
 
1685
 
 
1686
                        if(!mysql_num_rows(result)) {
 
1687
                                xfree(insert);
 
1688
                                mysql_free_result(result);
 
1689
                                goto exit_events;
 
1690
                        }
 
1691
 
 
1692
                        rc = _write_archive_file(
 
1693
                                result, EVENT_REQ_START, EVENT_REQ_COUNT,
 
1694
                                curr_end, arch_cond->archive_dir, 
 
1695
                                "event", insert, false);
 
1696
                        
 
1697
                        xfree(insert);
 
1698
                        mysql_free_result(result);
 
1699
 
 
1700
                        if(rc != SLURM_SUCCESS)
 
1701
                                return rc;
 
1702
                }
 
1703
                query = xstrdup_printf("delete from %s where "
 
1704
                                       "period_start <= %d && period_end != 0",
 
1705
                                       event_table, curr_end);
 
1706
                debug3("%d(%d) query\n%s", mysql_conn->conn, __LINE__, query);
 
1707
                rc = mysql_db_query(mysql_conn->db_conn, query);
 
1708
                xfree(query);
 
1709
                if(rc != SLURM_SUCCESS) {
 
1710
                        error("Couldn't remove old event data");
 
1711
                        return SLURM_ERROR;
 
1712
                }
 
1713
        }
 
1714
 
 
1715
exit_events:
 
1716
 
 
1717
        if(arch_cond->purge_suspend) {
 
1718
                /* remove all data from step table that was older than
 
1719
                 * period_start * arch_cond->purge_suspend. 
 
1720
                 */
 
1721
                /* use localtime to avoid any daylight savings issues */
 
1722
                if(!localtime_r(&last_submit, &time_tm)) {
 
1723
                        error("Couldn't get localtime from first submit %d",
 
1724
                              last_submit);
 
1725
                        return SLURM_ERROR;
 
1726
                }
 
1727
                time_tm.tm_mday = 1;
 
1728
                time_tm.tm_mon -= arch_cond->purge_suspend;
 
1729
                time_tm.tm_isdst = -1;
 
1730
                curr_end = mktime(&time_tm);
 
1731
 
 
1732
                debug4("from %d - %d months purging suspend from before %d", 
 
1733
                       last_submit, arch_cond->purge_suspend, curr_end);
 
1734
                
 
1735
                if(arch_cond->archive_suspend) {
 
1736
                        char *insert = NULL;
 
1737
                        MYSQL_RES *result = NULL;
 
1738
                        
 
1739
                        xfree(tmp);
 
1740
                        xstrfmtcat(tmp, "%s", suspend_req_inx[0]);
 
1741
                        for(i=1; i<SUSPEND_REQ_COUNT; i++) {
 
1742
                                xstrfmtcat(tmp, ", %s", suspend_req_inx[i]);
 
1743
                        }
 
1744
 
 
1745
                        /* get all the suspend started before this time
 
1746
                           listed */
 
1747
                        query = xstrdup_printf("select %s from %s where "
 
1748
                                               "start <= %d && end != 0 "
 
1749
                                               "order by start asc",
 
1750
                                               tmp, suspend_table, curr_end);
 
1751
 
 
1752
                        insert = xstrdup_printf("insert into %s (%s) ",
 
1753
                                                suspend_table, tmp);
 
1754
                        xfree(tmp);
 
1755
                        
 
1756
//                      START_TIMER;
 
1757
                        debug3("%d(%d) query\n%s", mysql_conn->conn,
 
1758
                               __LINE__, query);
 
1759
                        if(!(result = mysql_db_query_ret(
 
1760
                                     mysql_conn->db_conn, query, 0))) {
 
1761
                                xfree(insert);
 
1762
                                xfree(query);
 
1763
                                return SLURM_ERROR;
 
1764
                        }
 
1765
                        xfree(query);
 
1766
//                      END_TIMER2("step query");
 
1767
//                      info("suspend query took %s", TIME_STR);
 
1768
 
 
1769
                        if(!mysql_num_rows(result)) {
 
1770
                                xfree(insert);
 
1771
                                mysql_free_result(result);
 
1772
                                goto exit_suspend;
 
1773
                        }
 
1774
 
 
1775
                        rc = _write_archive_file(
 
1776
                                result, SUSPEND_REQ_START, SUSPEND_REQ_COUNT,
 
1777
                                curr_end, arch_cond->archive_dir, 
 
1778
                                "suspend", insert, false);
 
1779
                        
 
1780
                        xfree(insert);
 
1781
                        mysql_free_result(result);
 
1782
 
 
1783
                        if(rc != SLURM_SUCCESS)
 
1784
                                return rc;
 
1785
                }
 
1786
                query = xstrdup_printf("delete from %s where start <= %d "
 
1787
                                       "&& end != 0",
 
1788
                                       suspend_table, curr_end);
 
1789
                debug3("%d(%d) query\n%s", mysql_conn->conn, __LINE__, query);
 
1790
                rc = mysql_db_query(mysql_conn->db_conn, query);
 
1791
                xfree(query);
 
1792
                if(rc != SLURM_SUCCESS) {
 
1793
                        error("Couldn't remove old suspend data");
 
1794
                        return SLURM_ERROR;
 
1795
                }
 
1796
        }
 
1797
 
 
1798
exit_suspend:
 
1799
 
 
1800
        if(arch_cond->purge_step) {
 
1801
                /* remove all data from step table that was older than
 
1802
                 * start * arch_cond->purge_step. 
1118
1803
                 */
1119
1804
                /* use localtime to avoid any daylight savings issues */
1120
1805
                if(!localtime_r(&last_submit, &time_tm)) {
1122
1807
                              last_submit);
1123
1808
                        return SLURM_ERROR;
1124
1809
                }
1125
 
                time_tm.tm_mon -= arch_cond->step_purge;
 
1810
                time_tm.tm_mon -= arch_cond->purge_step;
1126
1811
                time_tm.tm_isdst = -1;
1127
1812
                curr_end = mktime(&time_tm);
1128
1813
 
1129
1814
                debug4("from %d - %d months purging steps from before %d", 
1130
 
                       last_submit, arch_cond->step_purge, curr_end);
 
1815
                       last_submit, arch_cond->purge_step, curr_end);
1131
1816
                
1132
1817
                if(arch_cond->archive_steps) {
1133
1818
                        char *insert = NULL;
1134
 
                        char *values = NULL;
1135
 
                        int period_start = 0;
1136
1819
                        MYSQL_RES *result = NULL;
1137
 
                        MYSQL_ROW row;
1138
1820
 
1139
1821
                        xfree(tmp);
1140
1822
                        xstrfmtcat(tmp, "%s", step_req_inx[0]);
1173
1855
                                mysql_free_result(result);
1174
1856
                                goto exit_steps;
1175
1857
                        }
1176
 
 
1177
 
//                      START_TIMER;
1178
 
                        slurm_mutex_lock(&local_file_lock);
1179
 
                        while((row = mysql_fetch_row(result))) {
1180
 
                                if(period_start) {
1181
 
                                        xstrcat(values, ",\n(");
1182
 
                                } else {
1183
 
                                        period_start = 
1184
 
                                                atoi(row[STEP_REQ_START]);
1185
 
                                        localtime_r((time_t *)&period_start,
1186
 
                                                    &time_tm);
1187
 
                                        time_tm.tm_sec = 0;
1188
 
                                        time_tm.tm_min = 0;
1189
 
                                        time_tm.tm_hour = 0;
1190
 
                                        time_tm.tm_mday = 1;
1191
 
                                        time_tm.tm_isdst = -1;
1192
 
                                        period_start = mktime(&time_tm);
1193
 
                                        localtime_r((time_t *)&period_start,
1194
 
                                                    &time_tm);
1195
 
                                        snprintf(start_char, sizeof(start_char),
1196
 
                                                 "%4.4u-%2.2u-%2.2u"
1197
 
                                                 "T%2.2u:%2.2u:%2.2u",
1198
 
                                                 (time_tm.tm_year + 1900),
1199
 
                                                 (time_tm.tm_mon+1), 
1200
 
                                                 time_tm.tm_mday,
1201
 
                                                 time_tm.tm_hour,
1202
 
                                                 time_tm.tm_min, 
1203
 
                                                 time_tm.tm_sec);
1204
 
 
1205
 
                                        localtime_r((time_t *)&curr_end,
1206
 
                                                    &time_tm);
1207
 
                                        snprintf(end_char, sizeof(end_char),
1208
 
                                                 "%4.4u-%2.2u-%2.2u"
1209
 
                                                 "T%2.2u:%2.2u:%2.2u",
1210
 
                                                 (time_tm.tm_year + 1900),
1211
 
                                                 (time_tm.tm_mon+1), 
1212
 
                                                 time_tm.tm_mday,
1213
 
                                                 time_tm.tm_hour,
1214
 
                                                 time_tm.tm_min, 
1215
 
                                                 time_tm.tm_sec);
1216
 
 
1217
 
                                        /* write the buffer to file */
1218
 
                                        reg_file = xstrdup_printf(
1219
 
                                                "%s/step_archive_%s_%s.sql",
1220
 
                                                arch_cond->archive_dir,
1221
 
                                                start_char, end_char);
1222
 
                                        debug("Storing step archive at %s",
1223
 
                                              reg_file);
1224
 
                                        old_file = xstrdup_printf(
1225
 
                                                "%s.old", reg_file);
1226
 
                                        new_file = xstrdup_printf(
1227
 
                                                "%s.new", reg_file);
1228
 
                                        
1229
 
                                        fd = creat(new_file, 0600);
1230
 
                                        if (fd == 0) {
1231
 
                                                error("Can't save archive, "
1232
 
                                                      "create file %s error %m",
1233
 
                                                      new_file);
1234
 
                                                rc = errno;
1235
 
                                                xfree(insert);
1236
 
                                                break;
1237
 
                                        } 
1238
 
                                        values = xstrdup_printf("%s\nvalues\n(",
1239
 
                                                                insert);
1240
 
                                        xfree(insert);
1241
 
                                }
1242
 
        
1243
 
                                xstrfmtcat(values, "'%s'", row[0]);
1244
 
                                for(i=1; i<STEP_REQ_COUNT; i++) {
1245
 
                                        xstrfmtcat(values, ", '%s'", row[i]);
1246
 
                                }
1247
 
                                xstrcat(values, ", '1')");
1248
 
                                
1249
 
                                if(!fd || ((rc = _write_to_file(fd, values))
1250
 
                                           != SLURM_SUCCESS)) {
1251
 
                                        xfree(values);
1252
 
                                        break;
1253
 
                                }
1254
 
                                xfree(values);
1255
 
                        }
 
1858
                        
 
1859
                        rc = _write_archive_file(
 
1860
                                result, STEP_REQ_START, STEP_REQ_COUNT,
 
1861
                                curr_end, arch_cond->archive_dir, 
 
1862
                                "step", insert, true);
 
1863
                        
 
1864
                        xfree(insert);
1256
1865
                        mysql_free_result(result);
1257
 
                        rc = _write_to_file(
1258
 
                                fd, " on duplicate key update deleted=1;");
1259
 
//                      END_TIMER2("write file");
1260
 
//                      info("write file took %s", TIME_STR);
1261
 
 
1262
 
                        fsync(fd);
1263
 
                        close(fd);
1264
 
                        
1265
 
                        if (rc)
1266
 
                                (void) unlink(new_file);
1267
 
                        else {                  /* file shuffle */
1268
 
                                int ign;        /* avoid warning */
1269
 
                                (void) unlink(old_file);
1270
 
                                ign =  link(reg_file, old_file);
1271
 
                                (void) unlink(reg_file);
1272
 
                                ign =   link(new_file, reg_file);
1273
 
                                (void) unlink(new_file);
1274
 
                        }
1275
 
                        xfree(old_file);
1276
 
                        xfree(reg_file);
1277
 
                        xfree(new_file);
1278
 
                        slurm_mutex_unlock(&local_file_lock);
1279
 
 
1280
 
                        period_start = 0;
 
1866
 
 
1867
                        if(rc != SLURM_SUCCESS)
 
1868
                                return rc;
1281
1869
                }
1282
1870
 
1283
 
                if(rc != SLURM_SUCCESS) 
1284
 
                        return rc;
1285
 
 
1286
1871
                query = xstrdup_printf("delete from %s where start <= %d "
1287
1872
                                       "&& end != 0",
1288
1873
                                       step_table, curr_end);
1296
1881
        }
1297
1882
exit_steps:
1298
1883
        
1299
 
        if(arch_cond->job_purge) {
 
1884
        if(arch_cond->purge_job) {
1300
1885
                /* remove all data from step table that was older than
1301
 
                 * last_submit * arch_cond->job_purge. 
 
1886
                 * last_submit * arch_cond->purge_job. 
1302
1887
                 */
1303
1888
                /* use localtime to avoid any daylight savings issues */
1304
1889
                if(!localtime_r(&last_submit, &time_tm)) {
1307
1892
                        return SLURM_ERROR;
1308
1893
                }
1309
1894
                time_tm.tm_mday = 1;
1310
 
                time_tm.tm_mon -= arch_cond->job_purge;
 
1895
                time_tm.tm_mon -= arch_cond->purge_job;
1311
1896
                time_tm.tm_isdst = -1;
1312
1897
                curr_end = mktime(&time_tm);
1313
1898
 
1314
1899
                debug4("from %d - %d months purging jobs from before %d", 
1315
 
                       last_submit, arch_cond->job_purge, curr_end);
 
1900
                       last_submit, arch_cond->purge_job, curr_end);
1316
1901
 
1317
1902
                if(arch_cond->archive_jobs) {
1318
1903
                        char *insert = NULL;
1319
 
                        char *values = NULL;
1320
 
                        int period_start = 0;
1321
1904
                        MYSQL_RES *result = NULL;
1322
 
                        MYSQL_ROW row;
1323
 
                        
 
1905
                                                
1324
1906
                        xfree(tmp);
1325
1907
                        xstrfmtcat(tmp, "%s", job_req_inx[0]);
1326
1908
                        for(i=1; i<JOB_REQ_COUNT; i++) {
1358
1940
                                goto exit_jobs;
1359
1941
                        }
1360
1942
                        
1361
 
//                      START_TIMER;
1362
 
                        slurm_mutex_lock(&local_file_lock);
1363
 
                        while((row = mysql_fetch_row(result))) {
1364
 
                                if(period_start) {
1365
 
                                        xstrcat(values, ",\n(");
1366
 
                                } else {
1367
 
                                        period_start = 
1368
 
                                                atoi(row[JOB_REQ_SUBMIT]);
1369
 
                                        localtime_r((time_t *)&period_start,
1370
 
                                                    &time_tm);
1371
 
                                        time_tm.tm_sec = 0;
1372
 
                                        time_tm.tm_min = 0;
1373
 
                                        time_tm.tm_hour = 0;
1374
 
                                        time_tm.tm_mday = 1;
1375
 
                                        time_tm.tm_isdst = -1;
1376
 
                                        period_start = mktime(&time_tm);
1377
 
                                        localtime_r((time_t *)&period_start,
1378
 
                                                    &time_tm);
1379
 
                                        snprintf(start_char, sizeof(start_char),
1380
 
                                                 "%4.4u-%2.2u-%2.2u"
1381
 
                                                 "T%2.2u:%2.2u:%2.2u",
1382
 
                                                 (time_tm.tm_year + 1900),
1383
 
                                                 (time_tm.tm_mon+1), 
1384
 
                                                 time_tm.tm_mday,
1385
 
                                                 time_tm.tm_hour,
1386
 
                                                 time_tm.tm_min, 
1387
 
                                                 time_tm.tm_sec);
1388
 
 
1389
 
                                        localtime_r((time_t *)&curr_end,
1390
 
                                                    &time_tm);
1391
 
 
1392
 
                                        snprintf(end_char, sizeof(end_char),
1393
 
                                                 "%4.4u-%2.2u-%2.2u"
1394
 
                                                 "T%2.2u:%2.2u:%2.2u",
1395
 
                                                 (time_tm.tm_year + 1900),
1396
 
                                                 (time_tm.tm_mon+1), 
1397
 
                                                 time_tm.tm_mday,
1398
 
                                                 time_tm.tm_hour,
1399
 
                                                 time_tm.tm_min, 
1400
 
                                                 time_tm.tm_sec);
1401
 
 
1402
 
                                        /* write the buffer to file */
1403
 
                                        reg_file = xstrdup_printf(
1404
 
                                                "%s/job_archive_%s_%s.sql",
1405
 
                                                arch_cond->archive_dir,
1406
 
                                                start_char, end_char);
1407
 
                                        debug("Storing job archive at %s",
1408
 
                                              reg_file);
1409
 
                                        old_file = xstrdup_printf(
1410
 
                                                "%s.old", reg_file);
1411
 
                                        new_file = xstrdup_printf(
1412
 
                                                "%s.new", reg_file);
1413
 
                                        
1414
 
                                        fd = creat(new_file, 0600);
1415
 
                                        if (fd == 0) {
1416
 
                                                error("Can't save archive, "
1417
 
                                                      "create file %s error %m",
1418
 
                                                      new_file);
1419
 
                                                rc = errno;
1420
 
                                                xfree(insert);
1421
 
                                                break;
1422
 
                                        } 
1423
 
                                        values = xstrdup_printf("%s\nvalues\n(",
1424
 
                                                                insert);
1425
 
                                        xfree(insert);
1426
 
                                }
1427
 
                                
1428
 
                                xstrfmtcat(values, "'%s'", row[0]);
1429
 
                                for(i=1; i<JOB_REQ_COUNT; i++) {
1430
 
                                        xstrfmtcat(values, ", '%s'", row[i]);
1431
 
                                }
1432
 
                                xstrcat(values, ", '1')");
1433
 
                                
1434
 
                                if(!fd || ((rc = _write_to_file(fd, values))
1435
 
                                           != SLURM_SUCCESS)) {
1436
 
                                        xfree(values);
1437
 
                                        break;
1438
 
                                }
1439
 
                                xfree(values);
1440
 
                        }
 
1943
                        rc = _write_archive_file(
 
1944
                                result, JOB_REQ_SUBMIT, JOB_REQ_COUNT,
 
1945
                                curr_end, arch_cond->archive_dir, 
 
1946
                                "job", insert, true);
 
1947
                        
 
1948
                        xfree(insert);
1441
1949
                        mysql_free_result(result);
1442
1950
 
1443
 
                        rc = _write_to_file(
1444
 
                                fd, " on duplicate key update deleted=1;");
1445
 
//                      END_TIMER2("write file");
1446
 
//                      info("write file took %s", TIME_STR);
1447
 
                        
1448
 
                        
1449
 
                        fsync(fd);
1450
 
                        close(fd);
1451
 
 
1452
 
                        if (rc)
1453
 
                                (void) unlink(new_file);
1454
 
                        else {                  /* file shuffle */
1455
 
                                int ign;        /* avoid warning */
1456
 
                                (void) unlink(old_file);
1457
 
                                ign =  link(reg_file, old_file);
1458
 
                                (void) unlink(reg_file);
1459
 
                                ign =  link(new_file, reg_file);
1460
 
                                (void) unlink(new_file);
1461
 
                        }
1462
 
                        xfree(old_file);
1463
 
                        xfree(reg_file);
1464
 
                        xfree(new_file);
1465
 
                        slurm_mutex_unlock(&local_file_lock);
1466
 
 
1467
 
                        period_start = 0;
 
1951
                        if(rc != SLURM_SUCCESS)
 
1952
                                return rc;
1468
1953
                }
 
1954
 
1469
1955
                query = xstrdup_printf("delete from %s where submit <= %d "
1470
1956
                                       "&& end != 0",
1471
1957
                                       job_table, curr_end);
1478
1964
                }
1479
1965
        }
1480
1966
exit_jobs:
1481
 
        
 
1967
 
1482
1968
        return SLURM_SUCCESS;
1483
1969
}
1484
1970
 
1550
2036
 
1551
2037
        return SLURM_SUCCESS;
1552
2038
}
1553
 
 
1554
 
#endif