1
/*****************************************************************************\
2
* common_as.c - common functions for accounting storage
4
* $Id: common_as.c 13061 2008-01-22 21:23:56Z da $
5
*****************************************************************************
6
* Copyright (C) 2004-2007 The Regents of the University of California.
7
* Copyright (C) 2008-2010 Lawrence Livermore National Security.
8
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
9
* Written by Danny Auble <da@llnl.gov>
11
* This file is part of SLURM, a resource management program.
12
* For details, see <https://computing.llnl.gov/linux/slurm/>.
13
* Please also read the included file: DISCLAIMER.
15
* SLURM is free software; you can redistribute it and/or modify it under
16
* the terms of the GNU General Public License as published by the Free
17
* Software Foundation; either version 2 of the License, or (at your option)
20
* In addition, as a special exception, the copyright holders give permission
21
* to link the code of portions of this program with the OpenSSL library under
22
* certain conditions as described in each individual source file, and
23
* distribute linked combinations including the two. You must obey the GNU
24
* General Public License in all respects for all of the code used other than
25
* OpenSSL. If you modify file(s) with this exception, you may extend this
26
* exception to your version of the file(s), but you are not obligated to do
27
* so. If you do not wish to do so, delete this exception statement from your
28
* version. If you delete this exception statement from all source files in
29
* the program, then also delete it here.
31
* SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
32
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
33
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
36
* You should have received a copy of the GNU General Public License along
37
* with SLURM; if not, write to the Free Software Foundation, Inc.,
38
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
39
\*****************************************************************************/
42
#include <sys/types.h>
46
#include "src/common/slurmdbd_defs.h"
47
#include "src/common/slurm_auth.h"
48
#include "src/common/xstring.h"
49
#include "src/common/env.h"
50
#include "src/slurmdbd/read_config.h"
51
#include "common_as.h"
53
extern char *assoc_hour_table;
54
extern char *assoc_day_table;
55
extern char *assoc_month_table;
57
extern char *cluster_hour_table;
58
extern char *cluster_day_table;
59
extern char *cluster_month_table;
61
extern char *wckey_hour_table;
62
extern char *wckey_day_table;
63
extern char *wckey_month_table;
66
* We want SLURMDB_MODIFY_ASSOC always to be the last
68
static int _sort_update_object_dec(slurmdb_update_object_t *object_a,
69
slurmdb_update_object_t *object_b)
71
if ((object_a->type == SLURMDB_MODIFY_ASSOC)
72
&& (object_b->type != SLURMDB_MODIFY_ASSOC))
74
else if((object_b->type == SLURMDB_MODIFY_ASSOC)
75
&& (object_a->type != SLURMDB_MODIFY_ASSOC))
80
static void _dump_slurmdb_assoc_records(List assoc_list)
82
slurmdb_association_rec_t *assoc = NULL;
83
ListIterator itr = NULL;
85
itr = list_iterator_create(assoc_list);
86
while((assoc = list_next(itr))) {
87
debug("\t\tid=%d", assoc->id);
89
list_iterator_destroy(itr);
93
* addto_update_list - add object updated to list
94
* IN/OUT update_list: list of updated objects
95
* IN type: update type
96
* IN object: object updated
99
* NOTE: This function will take the object given and free it later so it
100
* needed to be removed from a list if in one before.
102
extern int addto_update_list(List update_list, slurmdb_update_type_t type,
105
slurmdb_update_object_t *update_object = NULL;
106
slurmdb_association_rec_t *assoc = object;
107
slurmdb_qos_rec_t *qos = object;
108
ListIterator itr = NULL;
110
error("no update list given");
114
itr = list_iterator_create(update_list);
115
while((update_object = list_next(itr))) {
116
if(update_object->type == type)
119
list_iterator_destroy(itr);
122
/* here we prepend primarly for remove association
123
since parents need to be removed last, and they are
124
removed first in the calling code */
125
list_prepend(update_object->objects, object);
126
return SLURM_SUCCESS;
128
update_object = xmalloc(sizeof(slurmdb_update_object_t));
130
list_append(update_list, update_object);
132
update_object->type = type;
134
list_sort(update_list, (ListCmpF)_sort_update_object_dec);
137
case SLURMDB_MODIFY_USER:
138
case SLURMDB_ADD_USER:
139
case SLURMDB_REMOVE_USER:
140
case SLURMDB_ADD_COORD:
141
case SLURMDB_REMOVE_COORD:
142
update_object->objects = list_create(slurmdb_destroy_user_rec);
144
case SLURMDB_ADD_ASSOC:
145
/* We are going to send these to the slurmctld's so
146
lets set up the correct limits to INIFINITE instead
148
if(assoc->grp_cpu_mins == (uint64_t)NO_VAL)
149
assoc->grp_cpu_mins = (uint64_t)INFINITE;
150
if(assoc->grp_cpu_run_mins == (uint64_t)NO_VAL)
151
assoc->grp_cpu_run_mins = (uint64_t)INFINITE;
152
if(assoc->grp_cpus == NO_VAL)
153
assoc->grp_cpus = INFINITE;
154
if(assoc->grp_jobs == NO_VAL)
155
assoc->grp_jobs = INFINITE;
156
if(assoc->grp_nodes == NO_VAL)
157
assoc->grp_nodes = INFINITE;
158
if(assoc->grp_submit_jobs == NO_VAL)
159
assoc->grp_submit_jobs = INFINITE;
160
if(assoc->grp_wall == NO_VAL)
161
assoc->grp_wall = INFINITE;
163
if(assoc->max_cpu_mins_pj == (uint64_t)NO_VAL)
164
assoc->max_cpu_mins_pj = (uint64_t)INFINITE;
165
if(assoc->max_cpu_run_mins == (uint64_t)NO_VAL)
166
assoc->max_cpu_run_mins = (uint64_t)INFINITE;
167
if(assoc->max_cpus_pj == NO_VAL)
168
assoc->max_cpus_pj = INFINITE;
169
if(assoc->max_jobs == NO_VAL)
170
assoc->max_jobs = INFINITE;
171
if(assoc->max_nodes_pj == NO_VAL)
172
assoc->max_nodes_pj = INFINITE;
173
if(assoc->max_submit_jobs == NO_VAL)
174
assoc->max_submit_jobs = INFINITE;
175
if(assoc->max_wall_pj == NO_VAL)
176
assoc->max_wall_pj = INFINITE;
177
case SLURMDB_MODIFY_ASSOC:
178
case SLURMDB_REMOVE_ASSOC:
179
xassert(((slurmdb_association_rec_t *)object)->cluster);
180
update_object->objects = list_create(
181
slurmdb_destroy_association_rec);
183
case SLURMDB_ADD_QOS:
184
/* We are going to send these to the slurmctld's so
185
lets set up the correct limits to INIFINITE instead
187
if(qos->grp_cpu_mins == (uint64_t)NO_VAL)
188
qos->grp_cpu_mins = (uint64_t)INFINITE;
189
if(qos->grp_cpu_run_mins == (uint64_t)NO_VAL)
190
qos->grp_cpu_run_mins = (uint64_t)INFINITE;
191
if(qos->grp_cpus == NO_VAL)
192
qos->grp_cpus = INFINITE;
193
if(qos->grp_jobs == NO_VAL)
194
qos->grp_jobs = INFINITE;
195
if(qos->grp_nodes == NO_VAL)
196
qos->grp_nodes = INFINITE;
197
if(qos->grp_submit_jobs == NO_VAL)
198
qos->grp_submit_jobs = INFINITE;
199
if(qos->grp_wall == NO_VAL)
200
qos->grp_wall = INFINITE;
202
if(qos->max_cpu_mins_pj == (uint64_t)NO_VAL)
203
qos->max_cpu_mins_pj = (uint64_t)INFINITE;
204
if(qos->max_cpu_run_mins_pu == (uint64_t)NO_VAL)
205
qos->max_cpu_run_mins_pu = (uint64_t)INFINITE;
206
if(qos->max_cpus_pj == NO_VAL)
207
qos->max_cpus_pj = INFINITE;
208
if(qos->max_jobs_pu == NO_VAL)
209
qos->max_jobs_pu = INFINITE;
210
if(qos->max_nodes_pj == NO_VAL)
211
qos->max_nodes_pj = INFINITE;
212
if(qos->max_submit_jobs_pu == NO_VAL)
213
qos->max_submit_jobs_pu = INFINITE;
214
if(qos->max_wall_pj == NO_VAL)
215
qos->max_wall_pj = INFINITE;
216
case SLURMDB_MODIFY_QOS:
217
case SLURMDB_REMOVE_QOS:
218
update_object->objects = list_create(
219
slurmdb_destroy_qos_rec);
221
case SLURMDB_ADD_WCKEY:
222
case SLURMDB_MODIFY_WCKEY:
223
case SLURMDB_REMOVE_WCKEY:
224
xassert(((slurmdb_wckey_rec_t *)object)->cluster);
225
update_object->objects = list_create(
226
slurmdb_destroy_wckey_rec);
228
case SLURMDB_ADD_CLUSTER:
229
case SLURMDB_REMOVE_CLUSTER:
230
/* This should only be the name of the cluster, and is
231
only used in the plugin for rollback purposes.
233
update_object->objects = list_create(slurm_destroy_char);
235
case SLURMDB_UPDATE_NOTSET:
237
error("unknown type set in update_object: %d", type);
240
debug4("XXX: update object with type %d added", type);
241
list_append(update_object->objects, object);
242
return SLURM_SUCCESS;
246
* dump_update_list - dump contents of updates
247
* IN update_list: updates to perform
249
extern void dump_update_list(List update_list)
251
ListIterator itr = NULL;
252
slurmdb_update_object_t *object = NULL;
254
debug3("========== DUMP UPDATE LIST ==========");
255
itr = list_iterator_create(update_list);
256
while((object = list_next(itr))) {
257
if(!object->objects || !list_count(object->objects)) {
258
debug3("\tUPDATE OBJECT WITH NO RECORDS, type: %d",
262
switch(object->type) {
263
case SLURMDB_MODIFY_USER:
264
case SLURMDB_ADD_USER:
265
case SLURMDB_REMOVE_USER:
266
case SLURMDB_ADD_COORD:
267
case SLURMDB_REMOVE_COORD:
268
debug3("\tUSER RECORDS");
270
case SLURMDB_ADD_ASSOC:
271
case SLURMDB_MODIFY_ASSOC:
272
case SLURMDB_REMOVE_ASSOC:
273
debug3("\tASSOC RECORDS");
274
_dump_slurmdb_assoc_records(object->objects);
276
case SLURMDB_ADD_QOS:
277
case SLURMDB_MODIFY_QOS:
278
case SLURMDB_REMOVE_QOS:
279
debug3("\tQOS RECORDS");
281
case SLURMDB_ADD_WCKEY:
282
case SLURMDB_MODIFY_WCKEY:
283
case SLURMDB_REMOVE_WCKEY:
284
debug3("\tWCKEY RECORDS");
286
case SLURMDB_UPDATE_NOTSET:
288
error("unknown type set in "
294
list_iterator_destroy(itr);
299
* cluster_first_reg - ask for controller to send nodes in a down state
300
* and jobs pending or running on first registration.
302
* IN host: controller host
303
* IN port: controller port
304
* IN rpc_version: controller rpc version
307
extern int cluster_first_reg(char *host, uint16_t port, uint16_t rpc_version)
309
slurm_addr_t ctld_address;
311
int rc = SLURM_SUCCESS;
313
info("First time to register cluster requesting "
314
"running jobs and system information.");
316
slurm_set_addr_char(&ctld_address, port, host);
317
fd = slurm_open_msg_conn(&ctld_address);
319
error("can not open socket back to slurmctld "
320
"%s(%u): %m", host, port);
324
accounting_update_msg_t update;
325
/* We have to put this update message here so
326
we can tell the sender to send the correct
329
memset(&update, 0, sizeof(accounting_update_msg_t));
330
update.rpc_version = rpc_version;
331
slurm_msg_t_init(&out_msg);
332
out_msg.msg_type = ACCOUNTING_FIRST_REG;
333
out_msg.flags = SLURM_GLOBAL_AUTH_KEY;
334
out_msg.data = &update;
335
slurm_send_node_msg(fd, &out_msg);
336
/* We probably need to add matching recv_msg function
337
* for an arbitray fd or should these be fire
338
* and forget? For this, that we can probably
340
slurm_close_stream(fd);
346
* set_usage_information - set time and table information for getting usage
348
* OUT usage_table: which usage table to query
349
* IN type: usage type to get
350
* IN/OUT usage_start: start time
351
* IN/OUT usage_end: end time
354
extern int set_usage_information(char **usage_table, slurmdbd_msg_type_t type,
355
time_t *usage_start, time_t *usage_end)
357
time_t start = (*usage_start), end = (*usage_end);
358
time_t my_time = time(NULL);
361
char *my_usage_table = (*usage_table);
363
/* Default is going to be the last day */
365
if(!localtime_r(&my_time, &end_tm)) {
366
error("Couldn't get localtime from end %ld",
372
if(!localtime_r(&end, &end_tm)) {
373
error("Couldn't get localtime from user end %ld",
380
end_tm.tm_isdst = -1;
381
end = mktime(&end_tm);
384
if(!localtime_r(&my_time, &start_tm)) {
385
error("Couldn't get localtime from start %ld",
389
start_tm.tm_hour = 0;
392
if(!localtime_r(&start, &start_tm)) {
393
error("Couldn't get localtime from user start %ld",
400
start_tm.tm_isdst = -1;
401
start = mktime(&start_tm);
403
if(end-start < 3600) {
405
if(!localtime_r(&end, &end_tm)) {
406
error("2 Couldn't get localtime from user end %ld",
411
/* check to see if we are off day boundaries or on month
412
* boundaries other wise use the day table.
414
//info("%d %d %d", start_tm.tm_hour, end_tm.tm_hour, end-start);
415
if(start_tm.tm_hour || end_tm.tm_hour || (end-start < 86400)
416
|| (end > my_time)) {
418
case DBD_GET_ASSOC_USAGE:
419
my_usage_table = assoc_hour_table;
421
case DBD_GET_WCKEY_USAGE:
422
my_usage_table = wckey_hour_table;
424
case DBD_GET_CLUSTER_USAGE:
425
my_usage_table = cluster_hour_table;
428
error("Bad type given for hour usage %d %s", type,
429
slurmdbd_msg_type_2_str(type, 1));
432
} else if(start_tm.tm_mday == 0 && end_tm.tm_mday == 0
433
&& (end-start > 86400)) {
435
case DBD_GET_ASSOC_USAGE:
436
my_usage_table = assoc_month_table;
438
case DBD_GET_WCKEY_USAGE:
439
my_usage_table = wckey_month_table;
441
case DBD_GET_CLUSTER_USAGE:
442
my_usage_table = cluster_month_table;
445
error("Bad type given for month usage %d %s", type,
446
slurmdbd_msg_type_2_str(type, 1));
451
(*usage_start) = start;
453
(*usage_table) = my_usage_table;
454
return SLURM_SUCCESS;
459
* merge_delta_qos_list - apply delta_qos_list to qos_list
461
* IN/OUT qos_list: list of QOS'es
462
* IN delta_qos_list: list of delta QOS'es
464
extern void merge_delta_qos_list(List qos_list, List delta_qos_list)
466
ListIterator curr_itr = list_iterator_create(qos_list);
467
ListIterator new_itr = list_iterator_create(delta_qos_list);
468
char *new_qos = NULL, *curr_qos = NULL;
470
while((new_qos = list_next(new_itr))) {
471
if(new_qos[0] == '-') {
472
while((curr_qos = list_next(curr_itr))) {
473
if(!strcmp(curr_qos, new_qos+1)) {
474
list_delete_item(curr_itr);
478
list_iterator_reset(curr_itr);
479
} else if(new_qos[0] == '+') {
480
while((curr_qos = list_next(curr_itr))) {
481
if(!strcmp(curr_qos, new_qos+1)) {
486
list_append(qos_list, xstrdup(new_qos+1));
488
list_iterator_reset(curr_itr);
491
list_iterator_destroy(new_itr);
492
list_iterator_destroy(curr_itr);
495
extern bool is_user_min_admin_level(void *db_conn, uid_t uid,
496
slurmdb_admin_level_t min_level)
499
/* This only works when running though the slurmdbd.
500
* THERE IS NO AUTHENTICATION WHEN RUNNNING OUT OF THE
504
/* We have to check the authentication here in the
505
* plugin since we don't know what accounts are being
506
* referenced until after the query.
508
if((uid != slurmdbd_conf->slurm_user_id && uid != 0)
509
&& assoc_mgr_get_admin_level(db_conn, uid) < min_level)
515
extern bool is_user_coord(slurmdb_user_rec_t *user, char *account)
518
slurmdb_coord_rec_t *coord;
523
if (!user->coord_accts || !list_count(user->coord_accts))
526
itr = list_iterator_create(user->coord_accts);
527
while((coord = list_next(itr))) {
528
if(!strcasecmp(coord->name, account))
531
list_iterator_destroy(itr);
532
return coord ? 1 : 0;
535
extern bool is_user_any_coord(void *db_conn, slurmdb_user_rec_t *user)
538
if(assoc_mgr_fill_in_user(db_conn, user, 1, NULL) != SLURM_SUCCESS) {
539
error("couldn't get information for this user %s(%d)",
540
user->name, user->uid);
543
return (user->coord_accts && list_count(user->coord_accts));
547
* acct_get_db_name - get database name of accouting storage
548
* RET: database name, should be free-ed by caller
550
extern char *acct_get_db_name(void)
552
char *db_name = NULL;
553
char *location = slurm_get_accounting_storage_loc();
556
db_name = xstrdup(DEFAULT_ACCOUNTING_DB);
560
if(location[i] == '.' || location[i] == '/') {
561
debug("%s doesn't look like a database "
563
location, DEFAULT_ACCOUNTING_DB);
569
db_name = xstrdup(DEFAULT_ACCOUNTING_DB);
577
extern time_t archive_setup_end_time(time_t last_submit, uint32_t purge)
582
if(purge == NO_VAL) {
583
error("Invalid purge set");
587
units = SLURMDB_PURGE_GET_UNITS(purge);
589
error("invalid units from purge '%d'", units);
593
/* use localtime to avoid any daylight savings issues */
594
if(!localtime_r(&last_submit, &time_tm)) {
595
error("Couldn't get localtime from first "
596
"suspend start %ld", (long)last_submit);
603
if(SLURMDB_PURGE_IN_HOURS(purge))
604
time_tm.tm_hour -= units;
605
else if(SLURMDB_PURGE_IN_DAYS(purge)) {
607
time_tm.tm_mday -= units;
608
} else if(SLURMDB_PURGE_IN_MONTHS(purge)) {
611
time_tm.tm_mon -= units;
614
error("No known unit given for purge, "
615
"we are guessing mistake and returning error");
619
time_tm.tm_isdst = -1;
620
return (mktime(&time_tm) - 1);
624
/* execute archive script */
625
extern int archive_run_script(slurmdb_archive_cond_t *arch_cond,
626
char *cluster_name, time_t last_submit)
628
char * args[] = {arch_cond->archive_script, NULL};
639
if (stat(arch_cond->archive_script, &st) < 0) {
641
error("archive_run_script: failed to stat %s: %m",
642
arch_cond->archive_script);
646
if (!(st.st_mode & S_IFREG)) {
648
error("archive_run_script: %s isn't a regular file",
649
arch_cond->archive_script);
653
if (access(arch_cond->archive_script, X_OK) < 0) {
655
error("archive_run_script: %s is not executable",
656
arch_cond->archive_script);
660
env = env_array_create();
661
env_array_append_fmt(&env, "SLURM_ARCHIVE_CLUSTER", "%s",
664
if(arch_cond->purge_event != NO_VAL) {
665
if(!(curr_end = archive_setup_end_time(
666
last_submit, arch_cond->purge_event))) {
667
error("Parsing purge events failed");
671
env_array_append_fmt(&env, "SLURM_ARCHIVE_EVENTS", "%u",
672
SLURMDB_PURGE_ARCHIVE_SET(
673
arch_cond->purge_event));
674
env_array_append_fmt(&env, "SLURM_ARCHIVE_LAST_EVENT", "%ld",
678
if(arch_cond->purge_job != NO_VAL) {
679
if(!(curr_end = archive_setup_end_time(
680
last_submit, arch_cond->purge_job))) {
681
error("Parsing purge job failed");
685
env_array_append_fmt(&env, "SLURM_ARCHIVE_JOBS", "%u",
686
SLURMDB_PURGE_ARCHIVE_SET(
687
arch_cond->purge_job));
688
env_array_append_fmt(&env, "SLURM_ARCHIVE_LAST_JOB", "%ld",
692
if(arch_cond->purge_step != NO_VAL) {
693
if(!(curr_end = archive_setup_end_time(
694
last_submit, arch_cond->purge_step))) {
695
error("Parsing purge step");
699
env_array_append_fmt(&env, "SLURM_ARCHIVE_STEPS", "%u",
700
SLURMDB_PURGE_ARCHIVE_SET(
701
arch_cond->purge_step));
702
env_array_append_fmt(&env, "SLURM_ARCHIVE_LAST_STEP", "%ld",
706
if(arch_cond->purge_suspend != NO_VAL) {
707
if(!(curr_end = archive_setup_end_time(
708
last_submit, arch_cond->purge_suspend))) {
709
error("Parsing purge suspend");
713
env_array_append_fmt(&env, "SLURM_ARCHIVE_SUSPEND", "%u",
714
SLURMDB_PURGE_ARCHIVE_SET(
715
arch_cond->purge_suspend));
716
env_array_append_fmt(&env, "SLURM_ARCHIVE_LAST_SUSPEND", "%ld",
721
env_array_append (&env, "PATH", _PATH_STDPATH);
723
env_array_append (&env, "PATH", "/bin:/usr/bin");
725
execve(arch_cond->archive_script, args, env);
729
return SLURM_SUCCESS;
732
static char *_make_archive_name(time_t period_start, time_t period_end,
733
char *cluster_name, char *arch_dir,
734
char *arch_type, uint32_t archive_period)
740
localtime_r((time_t *)&period_start, &time_tm);
744
/* set up the start time based off the period we are purging */
745
if(SLURMDB_PURGE_IN_HOURS(archive_period)) {
746
} else if(SLURMDB_PURGE_IN_DAYS(archive_period)) {
753
snprintf(start_char, sizeof(start_char),
755
"T%2.2u:%2.2u:%2.2u",
756
(time_tm.tm_year + 1900),
763
localtime_r((time_t *)&period_end, &time_tm);
764
snprintf(end_char, sizeof(end_char),
766
"T%2.2u:%2.2u:%2.2u",
767
(time_tm.tm_year + 1900),
774
/* write the buffer to file */
775
return xstrdup_printf("%s/%s_%s_archive_%s_%s",
776
arch_dir, cluster_name, arch_type,
777
start_char, end_char);
780
extern int archive_write_file(Buf buffer, char *cluster_name,
781
time_t period_start, time_t period_end,
782
char *arch_dir, char *arch_type,
783
uint32_t archive_period)
786
int rc = SLURM_SUCCESS;
787
char *old_file = NULL, *new_file = NULL, *reg_file = NULL;
788
static int high_buffer_size = (1024 * 1024);
789
static pthread_mutex_t local_file_lock = PTHREAD_MUTEX_INITIALIZER;
793
slurm_mutex_lock(&local_file_lock);
795
/* write the buffer to file */
796
reg_file = _make_archive_name(period_start, period_end,
797
cluster_name, arch_dir,
798
arch_type, archive_period);
800
debug("Storing %s archive for %s at %s",
801
arch_type, cluster_name, reg_file);
802
old_file = xstrdup_printf("%s.old", reg_file);
803
new_file = xstrdup_printf("%s.new", reg_file);
805
fd = creat(new_file, 0600);
807
error("Can't save archive, create file %s error %m", new_file);
810
int pos = 0, nwrite = get_buf_offset(buffer), amount;
811
char *data = (char *)get_buf_data(buffer);
812
high_buffer_size = MAX(nwrite, high_buffer_size);
814
amount = write(fd, &data[pos], nwrite);
815
if ((amount < 0) && (errno != EINTR)) {
816
error("Error writing file %s, %m", new_file);
828
(void) unlink(new_file);
829
else { /* file shuffle */
830
int ign; /* avoid warning */
831
(void) unlink(old_file);
832
ign = link(reg_file, old_file);
833
(void) unlink(reg_file);
834
ign = link(new_file, reg_file);
835
(void) unlink(new_file);
840
slurm_mutex_unlock(&local_file_lock);