2
Copyright (C) 2003-2004 Douglas Thain and the University of Wisconsin
3
Copyright (C) 2005- The University of Notre Dame
4
This software is distributed under the GNU General Public License.
5
See the file COPYING for details.
9
The chirp_global module provides a "global" view of the Chirp
10
storage space, presenting multiple servers as one big file tree.
11
It uses the catalog_query module to obtain the global list of servers
12
and then uses chirp_reli to access the underlying file servers.
14
So as to avoid many slow queries to the catalog server,
15
queries are cached and sonsulted repeatedly.
16
Each query pulls off the details of each server in the
17
form of name-value pairs (nvpairs) that are placed into
18
a hash table according to the server name and port.
19
The catalog is not queried above once per minute.
20
Note that no matter how often the catalog is queried,
21
the data will be stale due to the propagation delay
22
from servers to the catalog. If you are using a catalog
23
other than the default, set the environment variable CATALOG_HOST
26
Directory lists simply iterate through the hash table to obtain
27
the global list. Stat operations on file servers query the hash
28
table in order to determine a few key stats, such as total storage
29
in use and last time heard from. This allows an ls -l through
30
Parrot to show the last message time and the space used (in MB.)
33
#include "chirp_global.h"
34
#include "chirp_reli.h"
35
#include "chirp_protocol.h"
36
#include "chirp_multi.h"
37
#include "chirp_client.h"
38
#include "chirp_protocol.h"
39
#include "catalog_query.h"
40
#include "catalog_server.h"
45
#include "sleeptools.h"
46
#include "hash_table.h"
55
static struct hash_table *server_table = 0;
56
static time_t last_update = 0;
57
static time_t update_interval = 60;
58
static int inhibit_catalog_queries = 0;
60
static int not_empty( const char *str )
62
if(!str || !str[0] || !strcmp(str,"/")) {
69
static int is_multi_path( const char *host )
71
return !strcmp(host,"multi") || !strcmp(host,"multi:9094");
74
static void chirp_nvpair_to_stat( struct nvpair *nv, struct chirp_stat *info )
76
memset(info,0,sizeof(*info));
77
info->cst_atime = info->cst_mtime = info->cst_ctime = nvpair_lookup_integer(nv,"lastheardfrom");
78
info->cst_size = nvpair_lookup_integer(nv,"total")-nvpair_lookup_integer(nv,"avail");
79
info->cst_size /= 1024*1024;
80
info->cst_mode = S_IFDIR|0555;
83
static void chirp_blank_stat( struct chirp_stat *info )
85
memset(info,0,sizeof(*info));
86
info->cst_mode = S_IFDIR|0555;
89
static void parse_multi_path( const char *path, char *mhost, char *mpath )
94
sscanf(path,"/%[^/]%s",mhost,mpath);
96
if(mhost[0] && !mpath[0]) {
101
static int server_table_load( time_t stoptime )
103
struct catalog_query *q;
108
if( (last_update+update_interval) > time(0) ) {
113
server_table = hash_table_create(0,0);
114
if(!server_table) return 0;
117
if(inhibit_catalog_queries) {
118
debug(D_CHIRP,"catalog queries disabled\n");
122
hash_table_firstkey(server_table);
123
while(hash_table_nextkey(server_table,&key,&item)) {
124
hash_table_remove(server_table,key);
128
debug(D_CHIRP,"querying catalog at %s:%d",CATALOG_HOST,CATALOG_PORT);
130
q = catalog_query_create(CATALOG_HOST,CATALOG_PORT,stoptime);
133
while((n=catalog_query_read(q,stoptime))) {
134
char name[CHIRP_PATH_MAX];
135
const char *type, *hname;
138
type = nvpair_lookup_string(n,"type");
139
if(type && !strcmp(type,"chirp")) {
140
hname = nvpair_lookup_string(n,"name");
142
port = nvpair_lookup_integer(n,"port");
143
if(!port) port=CHIRP_PORT;
144
sprintf(name,"%s:%d",hname,port);
145
hash_table_insert(server_table,name,n);
153
catalog_query_delete(q);
154
last_update = time(0);
159
static struct nvpair * server_lookup( const char *host, time_t stoptime )
161
if(server_table_load(stoptime)) {
162
return hash_table_lookup(server_table,host);
168
void chirp_global_inhibit_catalog( int onoff )
170
inhibit_catalog_queries = onoff;
173
struct chirp_file * chirp_global_open( const char *host, const char *path, INT64_T flags, INT64_T mode, time_t stoptime )
175
if(is_multi_path(host)) {
176
char mhost[CHIRP_PATH_MAX];
177
char mpath[CHIRP_PATH_MAX];
178
parse_multi_path(path,mhost,mpath);
179
return chirp_multi_open(mhost,mpath,flags,mode,stoptime);
180
} else if(not_empty(path)) {
181
return chirp_reli_open(host,path,flags,mode,stoptime);
182
} else if(not_empty(host)) {
183
if(server_lookup(host,stoptime)) {
200
INT64_T chirp_global_close( struct chirp_file *file, time_t stoptime )
202
return chirp_reli_close(file,stoptime);
205
INT64_T chirp_global_pread( struct chirp_file *file, void *buffer, INT64_T length, INT64_T offset, time_t stoptime )
207
return chirp_reli_pread(file,buffer,length,offset,stoptime);
210
INT64_T chirp_global_pwrite( struct chirp_file *file, const void *buffer, INT64_T length, INT64_T offset, time_t stoptime )
212
return chirp_reli_pwrite(file,buffer,length,offset,stoptime);
215
INT64_T chirp_global_sread( struct chirp_file *file, void *buffer, INT64_T length, INT64_T stride_length, INT64_T stride_skip, INT64_T offset, time_t stoptime )
217
return chirp_reli_sread(file,buffer,length,stride_length,stride_skip,offset,stoptime);
220
INT64_T chirp_global_swrite( struct chirp_file *file, const void *buffer, INT64_T length, INT64_T stride_length, INT64_T stride_skip, INT64_T offset, time_t stoptime )
222
return chirp_reli_swrite(file,buffer,length,stride_length,stride_skip,offset,stoptime);
225
INT64_T chirp_global_fstat( struct chirp_file *file, struct chirp_stat *buf, time_t stoptime )
227
return chirp_reli_fstat(file,buf,stoptime);
230
INT64_T chirp_global_fstatfs( struct chirp_file *file, struct chirp_statfs *buf, time_t stoptime )
232
return chirp_reli_fstatfs(file,buf,stoptime);
235
INT64_T chirp_global_fchown( struct chirp_file *file, INT64_T uid, INT64_T gid, time_t stoptime )
237
return chirp_reli_fchown(file,uid,gid,stoptime);
240
INT64_T chirp_global_fchmod( struct chirp_file *file, INT64_T mode, time_t stoptime )
242
return chirp_reli_fchmod(file,mode,stoptime);
245
INT64_T chirp_global_ftruncate( struct chirp_file *file, INT64_T length, time_t stoptime )
247
return chirp_reli_ftruncate(file,length,stoptime);
250
INT64_T chirp_global_flush( struct chirp_file *file, time_t stoptime )
252
return chirp_reli_flush(file,stoptime);
255
INT64_T chirp_global_getfile( const char *host, const char *path, FILE *stream, time_t stoptime )
257
if(is_multi_path(host)) {
258
char mhost[CHIRP_PATH_MAX];
259
char mpath[CHIRP_PATH_MAX];
260
parse_multi_path(path,mhost,mpath);
261
return chirp_multi_getfile(mhost,mpath,stream,stoptime);
262
} else if(not_empty(path)) {
263
return chirp_reli_getfile(host,path,stream,stoptime);
264
} else if(not_empty(host)) {
265
if(server_lookup(host,stoptime)) {
278
INT64_T chirp_global_getfile_buffer( const char *host, const char *path, char **buffer, time_t stoptime )
280
if(is_multi_path(host)) {
281
char mhost[CHIRP_PATH_MAX];
282
char mpath[CHIRP_PATH_MAX];
283
parse_multi_path(path,mhost,mpath);
284
return chirp_multi_getfile_buffer(mhost,mpath,buffer,stoptime);
285
} else if(not_empty(path)) {
286
return chirp_reli_getfile_buffer(host,path,buffer,stoptime);
287
} else if(not_empty(host)) {
288
if(server_lookup(host,stoptime)) {
301
INT64_T chirp_global_putfile( const char *host, const char *path, FILE *stream, INT64_T mode, INT64_T length, time_t stoptime )
303
if(is_multi_path(host)) {
304
char mhost[CHIRP_PATH_MAX];
305
char mpath[CHIRP_PATH_MAX];
306
parse_multi_path(path,mhost,mpath);
307
return chirp_multi_putfile(mhost,mpath,stream,mode,length,stoptime);
308
} else if(not_empty(path)) {
309
return chirp_reli_putfile(host,path,stream,mode,length,stoptime);
310
} else if(not_empty(host)) {
311
if(server_lookup(host,stoptime)) {
324
INT64_T chirp_global_putfile_buffer( const char *host, const char *path, const char *buffer, INT64_T mode, INT64_T length, time_t stoptime )
326
if(is_multi_path(host)) {
327
char mhost[CHIRP_PATH_MAX];
328
char mpath[CHIRP_PATH_MAX];
329
parse_multi_path(path,mhost,mpath);
330
return chirp_multi_putfile_buffer(mhost,mpath,buffer,mode,length,stoptime);
331
} else if(not_empty(path)) {
332
return chirp_reli_putfile_buffer(host,path,buffer,mode,length,stoptime);
333
} else if(not_empty(host)) {
334
if(server_lookup(host,stoptime)) {
347
INT64_T chirp_global_getlongdir( const char *host, const char *path, chirp_longdir_t callback, void *arg, time_t stoptime )
349
if(is_multi_path(host)) {
352
} else if(not_empty(path)) {
353
return chirp_reli_getlongdir(host,path,callback,arg,stoptime);
354
} else if(not_empty(host)) {
355
return chirp_reli_getlongdir(host,"/",callback,arg,stoptime);
357
if(server_table_load(stoptime)) {
360
struct chirp_stat info;
362
hash_table_firstkey(server_table);
363
while(hash_table_nextkey(server_table,&key,&item)) {
364
chirp_nvpair_to_stat(item,&info);
365
callback(key,&info,arg);
367
chirp_blank_stat(&info);
368
callback("multi",&info,arg);
377
INT64_T chirp_global_getdir( const char *host, const char *path, chirp_dir_t callback, void *arg, time_t stoptime )
379
if(is_multi_path(host)) {
380
char mhost[CHIRP_PATH_MAX];
381
char mpath[CHIRP_PATH_MAX];
382
parse_multi_path(path,mhost,mpath);
383
return chirp_multi_getdir(mhost,mpath,callback,arg,stoptime);
384
} else if(not_empty(path)) {
385
return chirp_reli_getdir(host,path,callback,arg,stoptime);
386
} else if(not_empty(host)) {
387
return chirp_reli_getdir(host,"/",callback,arg,stoptime);
389
if(server_table_load(stoptime)) {
392
hash_table_firstkey(server_table);
393
while(hash_table_nextkey(server_table,&key,&item)) {
396
callback("multi",arg);
405
INT64_T chirp_global_getacl( const char *host, const char *path, chirp_dir_t callback, void *arg, time_t stoptime )
407
if(is_multi_path(host)) {
408
char mhost[CHIRP_PATH_MAX];
409
char mpath[CHIRP_PATH_MAX];
410
parse_multi_path(path,mhost,mpath);
411
return chirp_multi_getacl(mhost,mpath,callback,arg,stoptime);
412
} else if(not_empty(path)) {
413
return chirp_reli_getacl(host,path,callback,arg,stoptime);
414
} else if(not_empty(host)) {
415
return chirp_reli_getacl(host,"/",callback,arg,stoptime);
422
INT64_T chirp_global_setacl( const char *host, const char *path, const char *subject, const char *rights, time_t stoptime )
424
if(is_multi_path(host)) {
425
char mhost[CHIRP_PATH_MAX];
426
char mpath[CHIRP_PATH_MAX];
427
parse_multi_path(path,mhost,mpath);
428
return chirp_multi_setacl(mhost,mpath,subject,rights,stoptime);
429
} else if(not_empty(path)) {
430
return chirp_reli_setacl(host,path,subject,rights,stoptime);
431
} else if(not_empty(host)) {
432
return chirp_reli_setacl(host,"/",subject,rights,stoptime);
439
INT64_T chirp_global_whoami( const char *host, const char *path, char *buf, INT64_T length, time_t stoptime )
441
if(is_multi_path(host)) {
442
char mhost[CHIRP_PATH_MAX];
443
char mpath[CHIRP_PATH_MAX];
444
parse_multi_path(path,mhost,mpath);
445
return chirp_multi_whoami(mhost,buf,length,stoptime);
446
} else if(not_empty(host)) {
447
return chirp_reli_whoami(host,buf,length,stoptime);
454
INT64_T chirp_global_locate( const char *host, const char *path, chirp_loc_t callback, void *arg, time_t stoptime)
456
if(is_multi_path(host)) {
459
/*char mhost[CHIRP_PATH_MAX];
460
char mpath[CHIRP_PATH_MAX];
461
parse_multi_path(path,mhost,mpath);
462
return chirp_multi_locate(mhost,mpath,callback,arg,stoptime);
464
} else if(not_empty(host)) {
465
return chirp_reli_locate(host,path,callback,arg,stoptime);
472
INT64_T chirp_global_unlink( const char *host, const char *path, time_t stoptime )
474
if(is_multi_path(host)) {
475
char mhost[CHIRP_PATH_MAX];
476
char mpath[CHIRP_PATH_MAX];
477
parse_multi_path(path,mhost,mpath);
478
return chirp_multi_unlink(mhost,mpath,stoptime);
479
} else if(not_empty(path)) {
480
return chirp_reli_unlink(host,path,stoptime);
481
} else if(not_empty(host)) {
482
if(server_lookup(host,stoptime)) {
495
INT64_T chirp_global_rename( const char *host, const char *path, const char *newpath, time_t stoptime )
497
if(is_multi_path(host)) {
498
char mhost[CHIRP_PATH_MAX];
499
char mpath[CHIRP_PATH_MAX];
501
char mnewhost[CHIRP_PATH_MAX];
502
char mnewpath[CHIRP_PATH_MAX];
504
parse_multi_path(path,mhost,mpath);
505
parse_multi_path(path,mnewhost,mnewpath);
507
if(!strcmp(mhost,mnewhost)) {
508
return chirp_multi_rename(mhost,mpath,mnewpath,stoptime);
513
} else if(not_empty(path)) {
514
return chirp_reli_rename(host,path,newpath,stoptime);
521
INT64_T chirp_global_link( const char *host, const char *path, const char *newpath, time_t stoptime )
523
if(is_multi_path(host)) {
524
char mhost[CHIRP_PATH_MAX];
525
char mpath[CHIRP_PATH_MAX];
526
parse_multi_path(path,mhost,mpath);
527
return chirp_multi_symlink(mhost,mpath,newpath,stoptime);
528
} else if(not_empty(path)) {
529
return chirp_reli_link(host,path,newpath,stoptime);
536
INT64_T chirp_global_symlink( const char *host, const char *path, const char *newpath, time_t stoptime )
538
if(is_multi_path(host)) {
539
char mhost[CHIRP_PATH_MAX];
540
char mpath[CHIRP_PATH_MAX];
541
parse_multi_path(path,mhost,mpath);
542
return chirp_multi_symlink(mhost,mpath,newpath,stoptime);
543
} else if(not_empty(path)) {
544
return chirp_reli_symlink(host,path,newpath,stoptime);
551
INT64_T chirp_global_readlink( const char *host, const char *path, char *buf, INT64_T length, time_t stoptime )
553
if(is_multi_path(host)) {
554
char mhost[CHIRP_PATH_MAX];
555
char mpath[CHIRP_PATH_MAX];
556
parse_multi_path(path,mhost,mpath);
557
return chirp_multi_readlink(mhost,mpath,buf,length,stoptime);
558
} else if(not_empty(path)) {
559
return chirp_reli_readlink(host,path,buf,length,stoptime);
560
} else if(not_empty(host)) {
561
if(server_lookup(host,stoptime)) {
574
INT64_T chirp_global_mkdir( const char *host, const char *path, INT64_T mode, time_t stoptime )
576
if(is_multi_path(host)) {
577
char mhost[CHIRP_PATH_MAX];
578
char mpath[CHIRP_PATH_MAX];
579
parse_multi_path(path,mhost,mpath);
580
return chirp_multi_mkdir(mhost,mpath,mode,stoptime);
581
} else if(not_empty(path)) {
582
return chirp_reli_mkdir(host,path,mode,stoptime);
589
INT64_T chirp_global_rmdir( const char *host, const char *path, time_t stoptime )
591
if(is_multi_path(host)) {
592
char mhost[CHIRP_PATH_MAX];
593
char mpath[CHIRP_PATH_MAX];
594
parse_multi_path(path,mhost,mpath);
595
return chirp_multi_rmdir(mhost,mpath,stoptime);
596
} else if(not_empty(path)) {
597
return chirp_reli_rmdir(host,path,stoptime);
598
} else if(not_empty(host)) {
599
if(server_lookup(host,stoptime)) {
612
INT64_T chirp_global_rmall( const char *host, const char *path, time_t stoptime )
614
if(is_multi_path(host)) {
617
} else if(not_empty(path)) {
618
return chirp_reli_rmall(host,path,stoptime);
619
} else if(not_empty(host)) {
620
if(server_lookup(host,stoptime)) {
633
INT64_T chirp_global_stat( const char *host, const char *path, struct chirp_stat *buf, time_t stoptime )
635
if(is_multi_path(host)) {
636
char mhost[CHIRP_PATH_MAX];
637
char mpath[CHIRP_PATH_MAX];
638
parse_multi_path(path,mhost,mpath);
639
return chirp_multi_stat(mhost,mpath,buf,stoptime);
640
} else if(not_empty(path)) {
641
return chirp_reli_stat(host,path,buf,stoptime);
642
} else if(not_empty(host)) {
643
struct nvpair *nv = server_lookup(host,stoptime);
645
chirp_nvpair_to_stat(nv,buf);
648
return chirp_reli_stat(host,"/",buf,stoptime);
651
chirp_blank_stat(buf);
656
INT64_T chirp_global_lstat( const char *host, const char *path, struct chirp_stat *buf, time_t stoptime )
658
if(is_multi_path(host)) {
659
char mhost[CHIRP_PATH_MAX];
660
char mpath[CHIRP_PATH_MAX];
661
parse_multi_path(path,mhost,mpath);
662
return chirp_multi_lstat(mhost,mpath,buf,stoptime);
663
} else if(not_empty(path)) {
664
return chirp_reli_lstat(host,path,buf,stoptime);
665
} else if(not_empty(host)) {
666
struct nvpair *nv = server_lookup(host,stoptime);
668
chirp_nvpair_to_stat(nv,buf);
671
return chirp_reli_lstat(host,"/",buf,stoptime);
674
chirp_blank_stat(buf);
679
INT64_T chirp_global_statfs( const char *host, const char *path, struct chirp_statfs *buf, time_t stoptime )
681
if(is_multi_path(host)) {
682
char mhost[CHIRP_PATH_MAX];
683
char mpath[CHIRP_PATH_MAX];
684
parse_multi_path(path,mhost,mpath);
685
return chirp_multi_statfs(mhost,mpath,buf,stoptime);
686
} else if(not_empty(path)) {
687
return chirp_reli_statfs(host,path,buf,stoptime);
688
} else if(not_empty(host)) {
689
return chirp_reli_statfs(host,"/",buf,stoptime);
691
memset(buf,0,sizeof(*buf));
697
INT64_T chirp_global_access( const char *host, const char *path, INT64_T mode, time_t stoptime )
699
if(is_multi_path(host)) {
700
char mhost[CHIRP_PATH_MAX];
701
char mpath[CHIRP_PATH_MAX];
702
parse_multi_path(path,mhost,mpath);
703
return chirp_multi_access(mhost,mpath,mode,stoptime);
704
} else if(not_empty(path)) {
705
return chirp_reli_access(host,path,mode,stoptime);
706
} else if(not_empty(host)) {
707
if(server_lookup(host,stoptime)) {
710
return chirp_reli_access(host,path,mode,stoptime);
717
INT64_T chirp_global_chmod( const char *host, const char *path, INT64_T mode, time_t stoptime )
719
if(is_multi_path(host)) {
720
char mhost[CHIRP_PATH_MAX];
721
char mpath[CHIRP_PATH_MAX];
722
parse_multi_path(path,mhost,mpath);
723
return chirp_multi_chmod(mhost,mpath,mode,stoptime);
724
} else if(not_empty(path)) {
725
return chirp_reli_chmod(host,path,mode,stoptime);
726
} else if(not_empty(host)) {
727
if(server_lookup(host,stoptime)) {
741
INT64_T chirp_global_chown( const char *host, const char *path, INT64_T uid, INT64_T gid, time_t stoptime )
743
if(is_multi_path(host)) {
744
char mhost[CHIRP_PATH_MAX];
745
char mpath[CHIRP_PATH_MAX];
746
parse_multi_path(path,mhost,mpath);
747
return chirp_multi_chown(mhost,mpath,uid,gid,stoptime);
748
} else if(not_empty(path)) {
749
return chirp_reli_chown(host,path,uid,gid,stoptime);
750
} else if(not_empty(host)) {
751
if(server_lookup(host,stoptime)) {
764
INT64_T chirp_global_lchown( const char *host, const char *path, INT64_T uid, INT64_T gid, time_t stoptime )
766
if(is_multi_path(host)) {
767
char mhost[CHIRP_PATH_MAX];
768
char mpath[CHIRP_PATH_MAX];
769
parse_multi_path(path,mhost,mpath);
770
return chirp_multi_lchown(mhost,mpath,uid,gid,stoptime);
771
} else if(not_empty(path)) {
772
return chirp_reli_lchown(host,path,uid,gid,stoptime);
773
} else if(not_empty(host)) {
774
if(server_lookup(host,stoptime)) {
787
INT64_T chirp_global_truncate( const char *host, const char *path, INT64_T length, time_t stoptime )
789
if(is_multi_path(host)) {
790
char mhost[CHIRP_PATH_MAX];
791
char mpath[CHIRP_PATH_MAX];
792
parse_multi_path(path,mhost,mpath);
793
return chirp_multi_truncate(mhost,mpath,length,stoptime);
794
} else if(not_empty(path)) {
795
return chirp_reli_truncate(host,path,length,stoptime);
796
} else if(not_empty(host)) {
797
if(server_lookup(host,stoptime)) {
810
INT64_T chirp_global_utime( const char *host, const char *path, time_t actime, time_t modtime, time_t stoptime )
812
if(is_multi_path(host)) {
813
char mhost[CHIRP_PATH_MAX];
814
char mpath[CHIRP_PATH_MAX];
815
parse_multi_path(path,mhost,mpath);
816
return chirp_multi_utime(mhost,mpath,actime,modtime,stoptime);
817
} else if(not_empty(path)) {
818
return chirp_reli_utime(host,path,actime,modtime,stoptime);
819
} else if(not_empty(host)) {
820
if(server_lookup(host,stoptime)) {
833
INT64_T chirp_global_thirdput( const char *host, const char *path, const char *thirdhost, const char *thirdpath, time_t stoptime )
835
if(is_multi_path(host)) {
838
} else if(not_empty(host)) {
839
return chirp_reli_thirdput(host,path,thirdhost,thirdpath,stoptime);
846
INT64_T chirp_global_md5( const char *host, const char *path, unsigned char *digest, time_t stoptime )
848
if(is_multi_path(host)) {
849
char mhost[CHIRP_PATH_MAX];
850
char mpath[CHIRP_PATH_MAX];
851
parse_multi_path(path,mhost,mpath);
852
return chirp_multi_md5(mhost,mpath,digest,stoptime);
853
} else if(not_empty(host)) {
854
return chirp_reli_md5(host,path,digest,stoptime);
861
INT64_T chirp_global_lsalloc( const char *host, const char *path, char *alloc_path, INT64_T *size, INT64_T *inuse, time_t stoptime )
863
if(is_multi_path(host)) {
866
} else if(not_empty(host)) {
867
return chirp_reli_lsalloc(host,path,alloc_path,size,inuse,stoptime);
874
INT64_T chirp_global_mkalloc( const char *host, const char *path, INT64_T size, INT64_T mode, time_t stoptime )
876
if(is_multi_path(host)) {
879
} else if(not_empty(path)) {
880
return chirp_reli_mkalloc(host,path,size,mode,stoptime);