5
/* $Id: shmem.c,v 1.87.2.2 2007-09-10 23:31:32 manoj Exp $ */
6
/* System V shared memory allocation and managment
10
* char *Create_Shared_Region(long *idlist, long size, long *offset)
11
* . to be called by just one process.
12
* . calls kr_malloc, malloc-like memory allocator from the K&R book.
13
* kr_malloc inturn calls armci_allocate() that does shmget() and shmat().
14
* . idlist might be just a pointer to integer or a true array in the
15
* MULTIPLE_REGIONS versions (calling routine has to take care of it)
16
* char *Attach_Shared_Region(long *idlist, long size, long offset)
17
* . called by any other process to attach to existing shmem region or
18
* if attached just calculate the address based on the offset
19
* . has to be called after shmem region was created
20
* void Free_Shmem_Ptr(long id, long size, char* addr)
21
* . called ONLY by the process that created shmem region (id) to return
22
* pointer to kr_malloc (shmem is not destroyed)
23
* void Delete_All_Regions()
24
* . destroys all shared memory regions
25
* . can be called by any process assuming that all processes attached
26
* to alllocated shmem regions
27
* . needs to by called by cleanup procedure(s)
29
* Jarek Nieplocha, 06.13.94
40
/* For debugging purposes at the beginning of the shared memory region
41
* creator process can write a stamp which then is read by attaching processes
42
* NOTE: on clusters we cannot use it anymore since ARMCI node master
43
* uses it since Nov 99 to write the value of address it attached at
44
* This feature is used in the ARMCI memlock table.
49
#include <sys/types.h>
53
#include <sys/param.h>
58
#include "kr_malloc.h"
66
static size_t pagesize=0;
67
static int logpagesize=0;
68
/* allow only that big shared memory segment (in MB)- incresed from 128 11/02 */
69
#define MAX_ALLOC_MUNMAP 128
70
#define MAX_ALLOC_MUNMAP_ 368
71
static long max_alloc_munmap=MAX_ALLOC_MUNMAP;
78
#define SHM_UNIT (1024)
81
/* Need to determine the max shmem segment size. There are 2 alternatives:
82
* 1. use predefined SHMMAX if available or set some reasonable values, or
83
* 2. trial-and-error search for a max value (default)
84
* case a) fork a process to determine shmmax size (more accurate)
85
* case b) search w/o forking until success (less accurate)
88
/* under Myrinet GM, we cannot fork */
89
#if defined(GM) || defined(VAPI)
90
# define SHMMAX_SEARCH_NO_FORK
92
#if defined(LAPI) || defined(AIX) || defined(SHMMAX_SEARCH_NO_FORK) || defined(CRAY_XT)
93
# define NO_SHMMAX_SEARCH
96
/* Limits for the largest shmem segment are in Kilobytes to avoid passing
97
* Gigavalues to kr_malloc
98
* the limit for the KSR is lower than SHMMAX in sys/param.h because
99
* shmat would fail -- SHMMAX cannot be trusted (a bug)
101
#define _SHMMAX 4*1024
103
#if defined(SUN)||defined(SOLARIS)
105
# define _SHMMAX (1024) /* memory in KB */
106
#elif defined(SGI64) || defined(AIX) || defined(CONVEX)
108
# define _SHMMAX ((unsigned long)512*1024)
109
#elif defined(SGI) && !defined(SGI64)
111
# define _SHMMAX ((unsigned long)128*1024)
114
# define _SHMMAX ((unsigned long)512*1024)
117
# define _SHMMAX ((unsigned long)64*1024)
118
#elif defined(__FreeBSD__)
120
# define _SHMMAX ((unsigned long)3*1024)
122
# if !defined(SHMMAX) /* Red Hat does not define SHMMAX */
124
# if defined(__sparc__) || defined(__powerpc__)
125
# define _SHMMAX ((unsigned long)16*1024)
126
# elif defined(__alpha__)
127
# define _SHMMAX ((unsigned long)4072)
130
# define _SHMMAX ((unsigned long)32*1024)
133
#elif defined(SHMMAX)
135
# define _SHMMAX (((unsigned long)SHMMAX)>>10)
138
static unsigned long MinShmem_per_core = 0;
139
static unsigned long MaxShmem_per_core = 0;
140
static unsigned long MinShmem = _SHMMAX;
141
static unsigned long MaxShmem = MAX_REGIONS*_SHMMAX;
142
static context_t ctx_shmem; /* kr_malloc context */
143
static context_t *ctx_shmem_global; /* kr_malloc context stored in shmem */
144
static int create_call=0;
146
#ifdef SHMMAX_SEARCH_NO_FORK
147
static char *ptr_search_no_fork = (char*)0;
148
static int id_search_no_fork=0;
153
#define CLEANUP_CMD(command) sprintf(command,"/usr/bin/ipcrm shm %d",id);
154
#elif defined(SOLARIS)
155
#define CLEANUP_CMD(command) sprintf(command,"/bin/ipcrm -m %d",id);
157
#define CLEANUP_CMD(command) sprintf(command,"/usr/sbin/ipcrm -m %d",id);
159
#define CLEANUP_CMD(command) sprintf(command,"/usr/bin/ipcrm -m %d",id);
165
# include <elan/elan.h>
166
# include <elan3/elan3.h>
167
static char *armci_elan_starting_address = (char*)0;
170
# define ALLOC_MUNMAP_ALIGN 1024*1024
172
# define ALLOC_MUNMAP_ALIGN 64*1024
175
# define ALGN_MALLOC(s,a) elan_allocMain(elan_base->state, (a), (s))
177
# define ALGN_MALLOC(s,a) malloc((s))
180
static char* alloc_munmap(size_t size)
184
size_t bytes = size+pagesize-1;
186
if(armci_elan_starting_address){
187
tmp = armci_elan_starting_address;
188
armci_elan_starting_address += size;
189
# ifdef ALLOC_MUNMAP_ALIGN
190
armci_elan_starting_address += ALLOC_MUNMAP_ALIGN;
192
if(DEBUG_) {printf("%d: address for shm attachment is %p size=%ld\n",
193
armci_me,tmp,(long)size); fflush(stdout); }
195
tmp = ALGN_MALLOC(bytes, getpagesize());
197
iptr = (unsigned long)tmp + pagesize-1;
198
iptr >>= logpagesize; iptr <<= logpagesize;
199
if(DEBUG_) printf("%d:unmap ptr=%p->%p size=%d pagesize=%d\n",armci_me,
200
tmp,(char*)iptr,(int)size,pagesize);
202
if(munmap(tmp, size) == -1) armci_die("munmap failed",0);
203
if(DEBUG_){printf("%d: unmap OK\n",armci_me); fflush(stdout);}
204
}else armci_die("alloc_munmap: malloc failed",(int)size);
210
/*\ A wrapper to shmget. Just to be sure that ID is not 0.
212
int armci_shmget(size_t size,char *from)
216
id = shmget(IPC_PRIVATE, size, (IPC_CREAT | 00600));
218
/*attaching with id 0 somehow fails (Seen on pentium4+linux24+gm163)
219
*so if id=0, shmget again. */
221
/* free id=0 and get a new one */
222
if(shmctl((int)id,IPC_RMID,(struct shmid_ds *)NULL)) {
223
fprintf(stderr,"id=%d \n",id);
224
armci_die("allocate: failed to _delete_ shared region ",id);
226
id = shmget(IPC_PRIVATE, size, (IPC_CREAT | 00600));
229
printf("\n%d:armci_shmget sz=%ld caller=%s id=%d\n",armci_me,(long)size,
237
Semget(key_t key,int nsems,int semflg) {
240
if((ret = semget(key,nsems,semflg)) == -1) {
242
case EACCES: fprintf(stdout," semget errno=EACCES.\n"); break;
243
case EINVAL: fprintf(stdout," semget errno=EINVAL.\n"); break;
244
case ENOENT: fprintf(stdout," semget errno=ENOENT.\n"); break;
245
case ENOSPC: fprintf(stdout," semget errno=ENOSPC -- check system limit for sysv semaphores.\n"); break;
246
case ENOMEM: fprintf(stdout," semget errno=ENOMEM.\n"); break;
247
case EEXIST: fprintf(stdout," semget errno=EEXIST.\n"); break;
249
fprintf(stdout," unusual semget errno=%d\n",errno); break;
251
armci_die("semget failed",errno);
258
int armci_semget(int count)
261
id = Semget(IPC_PRIVATE,2,0600);
265
int armci_semrm(int id)
267
semctl(id,0,IPC_RMID);
270
int armci_shmrm(int id)
273
if((ret = shmctl(id,IPC_RMID,NULL)) != 0) {
274
fprintf(stdout,"[cp]: shmctl return an error.\n");
277
fprintf(stdout," Error EINVAL: shmid is not a valid shared memory segment.\n");
280
fprintf(stdout," Error EFAULT: argument 3 is not a valid struct shmid_ds.\n");
283
fprintf(stdout," Error EPREM: permission to access/change shared mem segment denied.\n");
286
fprintf(stdout," unusual shmctl errno=%d\n",errno); break;
289
armci_die("error deleting shmid",id);
294
/*\ test is a shared memory region of a specified size can be allocated
295
* return 0 (no) or 1 (yes)
297
int armci_test_allocate(long size)
300
int id = armci_shmget((size_t)size,"armci_test_allocate");
303
/* attach to segment */
304
ptr = shmat(id, (char *) NULL, 0);
306
/* delete segment id */
307
if(shmctl( id, IPC_RMID, (struct shmid_ds *)NULL))
308
fprintf(stderr,"failed to remove shm id=%d\n",id);
311
if (((long)ptr) == -1L) return 0;
316
/*\ try to allocate a shared memory region of a specified size; return pointer
318
static int armci_shmalloc_try(long size)
320
#ifdef SHMMAX_SEARCH_NO_FORK
322
int id = armci_shmget((size_t) size,"armci_shmalloc_try");
325
/* attach to segment */
326
ptr = shmat(id, (char *) NULL, 0);
329
if (((long)ptr) == -1L) return 0;
331
ptr_search_no_fork = ptr;
332
id_search_no_fork = id;
340
/* parameters that define range and granularity of search for shm segment size
341
* UBOUND is chosen to be < 2GB to avoid overflowing on 32-bit systems
342
* smaller PAGE gives more accurate results but with more search steps
343
* LBOUND is set to minimum amount for our purposes
344
* change UBOUND=512MB if you need larger arrays than 512 MB
346
#define PAGE (16*65536L)
347
#define LBOUND 1048576L
348
#if defined(MULTI_CTX) && defined(QUADRICS)
349
#define UBOUND 256*LBOUND
351
#define UBOUND 512*LBOUND
354
static long get_user_shmmax()
358
uval = getenv("ARMCI_DEFAULT_SHMMAX");
360
sscanf(uval,"%ld",&x);
361
if(x<1L || x> 2048L){
362
fprintf(stderr,"incorrect ARMCI_DEFAULT_SHMMAX should be <1,2048>mb and 2^N Found=%ld\n",x);
366
return x*1048576; /* return value in bytes */
369
/*\ determine the max shmem segment size using bisection
371
int armci_shmem_test()
375
long upper_bound=UBOUND;
378
x = get_user_shmmax();
379
if(!x) x = upper_bound;
382
if(DEBUG_){printf("%d: x = %ld upper_bound=%ld\n",armci_me, x, upper_bound); fflush(stdout);}
386
rc = armci_test_allocate(x);
388
printf("%d:test %d size=%ld bytes status=%d\n",armci_me,i,x,rc);
391
step = (upper_bound -x)>>1;
392
if(step < PAGE) break;
396
step = (x-lower_bound)>>1;
400
/* round it up to a full base-2 MB */
407
/* try if can get LBOUND - necessary if search starts from UBOUND */
409
rc = armci_test_allocate(lower_bound);
413
if(DEBUG_) printf("%ld bytes segment size, %d calls \n",lower_bound,i);
414
return (int)( lower_bound>>20); /* return shmmax in mb */
418
#ifdef SHMMAX_SEARCH_NO_FORK
419
/*\ determine the max shmem segment size by halving
421
static int armci_shmem_test_no_fork()
425
long lower_bound=_SHMMAX*SHM_UNIT;
426
#define UBOUND_SEARCH_NO_FORK (256*SHM_UNIT*SHM_UNIT)
428
x = get_user_shmmax();
429
if(!x) x = UBOUND_SEARCH_NO_FORK;
433
rc = armci_shmalloc_try(x);
435
printf("%d:test by halving size=%ld bytes rc=%d\n",armci_me,x,rc);
442
if(x<lower_bound) break;
446
if(DEBUG_) printf("%ld: shmax test no fork: bytes segment size, %d calls \n",lower_bound,i);
447
return (int)( lower_bound>>20); /* return shmmax in mb */
453
void armci_nattach_preallocate_info(int* segments, int *segsize)
457
uval = getenv("LIBELAN_NATTACH");
459
sscanf(uval,"%d",&x);
460
if(x<2 || x>8) armci_die("Error in LIBELAN_NATTACH <8, >1 ",(int)x);
462
armci_die("Inconsistent configuration: ARMCI needs LIBELAN_NATTACH",0);
464
*segsize = (int) (SHM_UNIT * MinShmem);
469
/* Create shared region to store kr_malloc context in shared memory */
470
void armci_krmalloc_init_ctxshmem() {
472
long idlist[SHMIDLEN];
474
int offset = sizeof(void*)/sizeof(int);
476
/* to store shared memory context and myptr */
477
size = SHMEM_CTX_MEM;
479
if(armci_me == armci_master ){
480
myptr = Create_Shared_Region(idlist+1,size,idlist);
481
if(!myptr && size>0 ) armci_die("armci_krmalloc_init_ctxshmem: could not create", (int)(size>>10));
482
if(size) *(volatile void**)myptr = myptr;
484
printf("%d:armci_krmalloc_init_ctxshmem addr mptr=%p ref=%p size=%ld\n", armci_me, myptr, *(void**)myptr, size);
488
/* Bootstrapping: allocate storage for ctx_shmem_global. NOTE:there is
489
offset,as master places its address at begining for others to see */
490
ctx_shmem_global = (context_t*) ( ((int*)myptr)+offset );
491
*ctx_shmem_global = ctx_shmem; /*master copies ctx into shared region */
494
/* broadcast shmem id to other processes on the same cluster node */
495
armci_msg_clus_brdcst(idlist, SHMIDLEN*sizeof(long));
497
if(armci_me != armci_master){
498
myptr=(double*)Attach_Shared_Region(idlist+1,size,idlist[0]);
499
if(!myptr)armci_die("armci_krmalloc_init_ctxshmem: could not attach", (int)(size>>10));
501
/* now every process in a SMP node needs to find out its offset
502
* w.r.t. master - this offset is necessary to use memlock table
504
if(size) armci_set_mem_offset(myptr);
506
printf("%d:armci_krmalloc_init_ctxshmem attached addr mptr=%p ref=%p size=%ld\n", armci_me,myptr, *(void**)myptr,size); fflush(stdout);
508
/* store context info */
509
ctx_shmem_global = (context_t*) ( ((int*)myptr)+offset );
511
printf("%d:armci_krmalloc_init_ctxshmem: shmid=%d off=%ld size=%ld\n", armci_me, ctx_shmem_global->shmid, ctx_shmem_global->shmoffset,
512
(long)ctx_shmem_global->shmsize);
518
void armci_shmem_init()
523
#if defined(QUADRICS)
524
# if (defined(__ia64__) || defined(__alpha)) && !defined(DECOSF)
526
/* this is to determine size of Elan Main memory allocator for munmap */
529
uval = getenv("LIBELAN_ALLOC_SIZE");
531
sscanf(uval,"%ld",&x);
532
if((x>80000000) && (x< 4*1024*1024*1024L)){
533
max_alloc_munmap = (x>>20) - 72;
535
printf("%d: max_alloc_munmap is %ld\n",armci_me,max_alloc_munmap);
541
/* an alternative approach is to use MMAP area where we get
542
the address from the Elan environment variable in qsnetlibs 1.4+ */
543
uval = getenv("LIBELAN3_MMAPBASE");
545
sscanf(uval,"%p",&armci_elan_starting_address);
549
# if defined(__ia64__)
550
/* need aligment on 1MB boundary rather than the actual pagesize */
551
pagesize = 1024*1024;
554
/* determine log2(pagesize) needed for address alignment */
557
pagesize = getpagesize();
558
if(tp>pagesize)armci_die("armci_shmem_init:pagesize",pagesize);
564
if(tp!=pagesize)armci_die("armci_shmem_init:pagesize pow 2",pagesize);
568
printf("page size =%d log=%d\n",pagesize,logpagesize); fflush(stdout); }
573
if(armci_me == armci_master){
574
#if !defined(NO_SHMMAX_SEARCH) || defined(SHMMAX_SEARCH_NO_FORK)
575
# ifdef SHMMAX_SEARCH_NO_FORK
576
int x = armci_shmem_test_no_fork();
578
int x = armci_child_shmem_init();
582
armci_die("no usable amount of shared memory available: only got \n",
585
# if defined(ALLOC_MUNMAP)
586
/* cap down for special memory allocator unless ARMCI_DEFAULT_SHMMAX
587
not set - the user knows what is doing*/
588
# if !defined(REGION_ALLOC)
589
if(!getenv("ARMCI_DEFAULT_SHMMAX"))
590
if(x>max_alloc_munmap && !armci_elan_starting_address) x=max_alloc_munmap;
597
printf("%d:shmem_init: %d mbytes max segment size\n",armci_me,x);fflush(stdout);}
599
MinShmem = (long)(x<<10); /* make sure it is in kb: mb <<10 */
600
MaxShmem = MAX_REGIONS*MinShmem;
601
# ifdef REPORT_SHMMAX
602
printf("%d using x=%d SHMMAX=%ldKB\n", armci_me,x, MinShmem);
607
/* nothing to do here - limits were given */
612
armci_krmalloc_init_ctxshmem();
613
if(DEBUG_)printf("%d: out of shmem_init\n",armci_me);
616
void armci_set_shmem_limit_per_node(int nslaves)
618
if (MaxShmem_per_core > 0) MaxShmem = nslaves*MaxShmem_per_core;
619
if (MinShmem_per_core > 0) MinShmem = nslaves*MinShmem_per_core;
622
void armci_set_shmem_limit_per_core(unsigned long shmemlimit)
624
MaxShmem_per_core = (shmemlimit + SHM_UNIT - 1)/SHM_UNIT;
625
MinShmem_per_core = (shmemlimit + SHM_UNIT - 1)/SHM_UNIT;
628
/*\ application can reset the upper limit (bytes) for memory allocation
630
void armci_set_shmem_limit(unsigned long shmemlimit)
632
unsigned long kbytes;
633
kbytes = (shmemlimit + SHM_UNIT -1)/SHM_UNIT;
634
if(MaxShmem > kbytes) MaxShmem = kbytes;
635
if(MinShmem > kbytes) MinShmem = kbytes;
639
static void shmem_errmsg(size_t size)
642
printf("******************* ARMCI INFO ************************\n");
643
printf("The application attempted to allocate a shared memory segment ");
644
printf("of %ld bytes in size. This might be in addition to segments ",sz);
645
printf("that were allocated succesfully previously. ");
646
printf("The current system configuration does not allow enough ");
647
printf("shared memory to be allocated to the application.\n");
648
printf("This is most often caused by:\n1) system parameter SHMMAX ");
649
printf("(largest shared memory segment) being too small or\n");
650
printf("2) insufficient swap space.\n");
651
printf("Please ask your system administrator to verify if SHMMAX ");
652
printf("matches the amount of memory needed by your application and ");
653
printf("the system has sufficient amount of swap space. ");
654
printf("Most UNIX systems can be easily reconfigured ");
655
printf("to allow larger shared memory segments,\n");
656
printf("see http://www.emsl.pnl.gov/docs/global/support.shtml\n");
657
printf("In some cases, the problem might be caused by insufficient swap space.\n");
658
printf("*******************************************************\n");
662
static struct shm_region_list{
667
}region_list[MAX_REGIONS];
668
static int alloc_regions=0;
669
static long occup_blocks=0;
672
* region - actual piece of shared memory allocated from OS
673
* block - a part of allocated shmem that is given to the requesting process
677
static int last_allocated=-1;
680
unsigned long armci_max_region()
686
int find_regions(char *addrp, long* id, int *region)
690
if(last_allocated!=-1){
695
for(reg=-1,nreg=0;nreg<alloc_regions; nreg++)
697
if(addrp >= region_list[nreg].addr &&
698
addrp < (region_list[nreg].addr + region_list[nreg].sz))
706
armci_die("find_regions: failed to locate shared region", 0L);
710
*id = region_list[reg].id;
715
/* returns the shmem info based on the addr */
716
int armci_get_shmem_info(char *addrp, int* shmid, long *shmoffset,
721
find_regions(addrp, &id, ®ion);
723
*shmoffset = (long)(addrp - region_list[region].addr);
724
*shmsize = region_list[region].sz;
729
long armci_shm_reg_size(int i, long id)
731
if(i<0 || i>= MAX_REGIONS)armci_die("armci_shmem_reg_size: bad i",i);
732
return region_list[i].sz;
735
void* armci_shm_reg_ptr(int i)
737
if(i<0 || i>= MAX_REGIONS)armci_die("armci_shmem_reg_ptr: bad i",i);
738
return (void *)region_list[i].addr;
741
Header *armci_get_shmem_ptr(int shmid, long shmoffset, size_t shmsize)
743
/* returns, address of the shared memory region based on shmid, offset.
744
* (i.e. return_addr = stating address of shmid + offset)*/
745
long idlist[SHMIDLEN];
748
idlist[1] = (long)shmid;
749
idlist[0] = shmoffset;
750
idlist[IDLOC+1] = shmsize; /* CHECK : idlist in CreateShmem????*/
752
if(!(p=(Header*)Attach_Shared_Region(idlist+1, shmsize, idlist[0])))
753
armci_die("kr_malloc:could not attach",(int)(p->s.shmsize>>10));
755
printf("%d: armci_get_shmem_ptr: %d %ld %ld %p\n",
756
armci_me, idlist[1], idlist[0], shmsize, p);
763
char *Attach_Shared_Region(id, size, offset)
764
long *id, offset, size;
766
int reg, found, shmflag=0;
769
if(alloc_regions>=MAX_REGIONS)
770
armci_die("Attach_Shared_Region: to many regions ",0);
773
printf("%d:AttachSharedRegion %d:size=%ld id=%ld\n",
774
armci_me, create_call++, size,*id);
779
/* under Linux we can get valid id=0 */
781
if(!*id) armci_die("Attach_Shared_Region: shmem ID=0 ",(int)*id);
784
/* first time needs to initialize region_list structure */
786
for(reg=0;reg<MAX_REGIONS;reg++){
787
region_list[reg].addr=(char*)0;
788
region_list[reg].attached=0;
789
region_list[reg].id=0;
791
MinShmem= id[SHMIDLEN-2];
793
printf("%d:attach: allocation unit: %ldK\n",armci_me,MinShmem);
798
/* search region_list for the current shmem id */
799
for(found = 0, reg=0; reg < MAX_REGIONS;reg++)
800
if((found=(region_list[reg].id == *id)))break;
804
region_list[reg].id =*id;
808
/* we need to use the actual shared memory segment not user req size */
811
/* attach if not attached yet */
812
if(!region_list[reg].attached){
815
char *pref_addr = alloc_munmap((size_t) (size));
817
char *pref_addr = (char*)0;
819
if ( (long) (temp = shmat((int) *id, pref_addr, shmflag)) == -1L){
820
fprintf(stderr,"%d:attach error:id=%ld off=%ld seg=%ld\n",armci_me,*id,offset,MinShmem);
821
shmem_errmsg((size_t)MinShmem*1024);
822
armci_die("Attach_Shared_Region:failed to attach to segment id=",(int)*id);
825
printf("%d:attached: id=%d address=%p\n",armci_me,(int)*id, temp);
828
POST_ALLOC_CHECK(temp,size);
829
region_list[reg].addr = temp;
830
region_list[reg].attached = 1;
831
region_list[reg].sz= size;
833
printf("\n%d:%s:new memory region\n",armci_me,FUNCTION_NAME);fflush(stdout);
839
/* check stamp to make sure that we are attached in the right place */
840
if(*((int*)(region_list[reg].addr+ offset))!= alloc_regions-1)
841
armci_die("Attach_Shared_Region: wrong stamp value !",
842
*((int*)(region_list[reg].addr+ offset)));
844
return (region_list[reg].addr+ offset);
848
extern void armci_region_register_shm(void *start, long size);
851
/*\ allocates shmem, to be called by krmalloc that is called by process that
852
* creates shmem region
854
void *armci_allocate(long size)
858
size_t sz = (size_t)size;
860
char *pref_addr = alloc_munmap((size_t) (MinShmem*SHM_UNIT));
862
char *pref_addr = (char*)0;
864
#if defined(SGI_N32) && defined(SHM_SGI_ANYADDR)
865
shmflag= SHM_SGI_ANYADDR;
869
printf("%d:allocate: Shmem allocate size %ld bytes\n",armci_me,size);
873
if( alloc_regions >= MAX_REGIONS)
874
armci_die("Create_Shared_Region:allocate:too many regions allocated ",0);
876
last_allocated = alloc_regions;
878
#ifdef SHMMAX_SEARCH_NO_FORK
879
if (ptr_search_no_fork){
880
temp = ptr_search_no_fork;
881
id = id_search_no_fork;
882
ptr_search_no_fork = (char*)0; /* do not look at it again */
886
if ( (id = armci_shmget(sz,"armci_allocate")) < 0 ) {
887
fprintf(stderr,"id=%d size=%ld\n",id, size);
889
armci_die("allocate: failed to create shared region ",id);
892
if ( (long)( (temp = shmat(id, pref_addr, shmflag))) == -1L){
894
CLEANUP_CMD(command);
895
if(system(command) == -1)
896
printf("Please clean shared memory (id=%d): see man ipcrm\n",id);
897
armci_die("allocate: failed to attach to shared region id=",id);
900
printf("%d:allocate:attach:id=%d paddr=%p size=%ld\n",armci_me,id,temp,size);
904
/* delete segment id so that OS cleans it when all attached processes are gone */
905
if(shmctl( id, IPC_RMID, (struct shmid_ds *)NULL))
906
fprintf(stderr,"failed to remove shm id=%d\n",id);
910
POST_ALLOC_CHECK(temp,sz);
912
region_list[alloc_regions].addr = temp;
913
region_list[alloc_regions].id = id;
914
region_list[alloc_regions].attached=1;
915
region_list[alloc_regions].sz=sz;
919
printf("%d:allocate:id=%d addr=%p size=%ld\n",armci_me,id,temp,size);
923
return (void*) (temp);
926
/******************** common code for the two versions *********************/
929
/*\ Allocate a block of shared memory - called by master process
931
char *Create_Shared_Region(long *id, long size, long *offset)
934
int reg, refreg=0,nreg;
936
if(alloc_regions>=MAX_REGIONS)
937
armci_die("Create_Shared_Region: to many regions ",0);
940
printf("%d:CreateSharedRegion %d:size=%ld\n",armci_me,create_call++,size);
944
/*initialization: 1st allocation request */
946
for(reg=0;reg<MAX_REGIONS;reg++){
947
region_list[reg].addr=(char*)0;
948
region_list[reg].attached=0;
949
region_list[reg].id=0;
952
printf("%d:1st CreateSharedRegion: allocation unit:%ldK,shmax:%ldK\n",
953
armci_me,MinShmem,MaxShmem);
957
kr_malloc_init(SHM_UNIT, (size_t)MinShmem, (size_t)MaxShmem,
958
armci_allocate, 0, &ctx_shmem);
959
ctx_shmem.ctx_type = KR_CTX_SHMEM;
960
id[SHMIDLEN-2]=MinShmem;
963
if(!alloc_regions) temp = kr_malloc((size_t)size, &ctx_shmem,0, NULL, NULL);
964
else temp = kr_malloc((size_t)size, ctx_shmem_global,0,NULL,NULL);
966
if(temp == (char*)0 )
967
armci_die("CreateSharedRegion:kr_malloc failed KB=",(int)size>>10);
969
if(!(nreg=find_regions(temp,id,®)))
970
armci_die("CreateSharedRegion: allocation inconsitent",0);
972
#ifndef MULTIPLE_REGIONS
976
if(STAMP) *((int*)temp) = alloc_regions-1;
977
*offset = (long) (temp - region_list[refreg].addr);
978
id[IDLOC]=region_list[reg].sz; /* elan post check */
982
printf("%d:CreateShmReg:reg=%d id=%ld off=%ld ptr=%p adr=%p s=%d n=%d sz=%ld\n",
983
armci_me,reg,region_list[reg].id,*offset,region_list[reg].addr,
984
temp,(int)size,nreg,id[IDLOC]);
993
/*\ only process that created shared region returns the pointer to kr_malloc
995
void Free_Shmem_Ptr( id, size, addr)
999
kr_free(addr, ctx_shmem_global);
1003
void Delete_All_Regions()
1007
extern int armci_me;
1009
for(reg = 0; reg < MAX_REGIONS; reg++){
1010
if(region_list[reg].addr != (char*)0){
1011
code += shmctl((int)region_list[reg].id,IPC_RMID,(struct shmid_ds *)NULL);
1012
region_list[reg].addr = (char*)0;
1013
region_list[reg].attached = 0;
1015
fprintf(stderr,"%d Delete_All_Regions id=%d code=%d\n",armci_me,
1016
(int)region_list[reg].id, code);
1023
Semop(int semid,struct sembuf *opers,size_t nops) {
1026
if((ret = semop(semid,opers,nops)) == -1) {
1027
printf("%s: semop error performing %i operation(s) on semid %i.\n",Portals_ID(),(int) nops,semid);
1029
case EFBIG: printf(" semop errno=EFBIG.\n"); break;
1030
case E2BIG: printf(" semop errno=E2BIG.\n"); break;
1031
case EINTR: printf(" semop errno=EINTR.\n"); break;
1032
case EINVAL: printf(" semop errno=EINVAL.\n"); break;
1033
case EACCES: printf(" semop errno=EACCES.\n"); break;
1034
case EAGAIN: printf(" semop errno=EAGAIN.\n"); break;
1035
case ENOSPC: printf(" semop errno=ENOSPC.\n"); break;
1036
case ERANGE: printf(" semop errno=ERANGE.\n"); break;
1037
case EFAULT: printf(" semop errno=EFAULT.\n"); break;
1039
printf(" unusual semop errno=%d\n",errno); break;
1041
armci_die("semaphore error",911);
1049
semaphoreOperation(int semid,int semnum,int semop) {
1052
op.sem_num = semnum;
1054
return Semop(semid,&op,1);
1059
semaphoreAcquire(int semid, int semnum,int access) {
1061
op.sem_op = -access;
1062
op.sem_num = semnum;
1069
semaphoreRelease(int semid,int semnum,int access) {
1072
op.sem_num = semnum;
1078
what are doing here ?