2
/* ARMCI header file */
9
#if defined(__cplusplus) || defined(c_plusplus)
13
typedef unsigned long long u64Int;
14
typedef long long s64Int;
16
extern int armci_sameclusnode(int proc);
24
typedef long armci_size_t;
25
extern int armci_notify(int proc);
26
extern int armci_notify_wait(int proc,int *pval);
27
extern int ARMCI_Init(void); /* initialize ARMCI */
28
extern int ARMCI_Init_args(int *argc, char ***argv);
29
extern void ARMCI_Barrier(void); /* ARMCI Barrier*/
31
extern int ARMCI_Put(void *src, void* dst, int bytes, int proc);
32
extern int ARMCI_Put_flag(void *src, void* dst,int bytes,int *f,int v,int proc);
34
#define ARMCI_Put1(_s,_d,_b,_p) memcpy(_d,_s,_b), 0
36
extern int ARMCI_PutS( /* strided put */
37
void *src_ptr, /* pointer to 1st segment at source*/
38
int src_stride_arr[], /* array of strides at source */
39
void* dst_ptr, /* pointer to 1st segment at destination*/
40
int dst_stride_arr[], /* array of strides at destination */
41
int count[], /* number of units at each stride level count[0]=bytes */
42
int stride_levels, /* number of stride levels */
43
int proc /* remote process(or) ID */
46
extern int ARMCI_PutS_flag_dir( /* put with flag that uses direct put */
47
void *src_ptr, /* pointer to 1st segment at source*/
48
int src_stride_arr[], /* array of strides at source */
49
void* dst_ptr, /* pointer to 1st segment at destination*/
50
int dst_stride_arr[], /* array of strides at destination */
51
int count[], /* number of segments at each stride
52
levels: count[0]=bytes*/
53
int stride_levels, /* number of stride levels */
54
int *flag, /* pointer to remote flag */
55
int val, /* value to set flag upon completion of
57
int proc /* remote process(or) ID */
60
extern int ARMCI_PutS_flag(
61
void *src_ptr, /* pointer to 1st segment at source*/
62
int src_stride_arr[], /* array of strides at source */
63
void* dst_ptr, /* pointer to 1st segment at destination*/
64
int dst_stride_arr[], /* array of strides at destination */
65
int count[], /* number of segments at each stride
66
levels: count[0]=bytes*/
67
int stride_levels, /* number of stride levels */
68
int *flag, /* pointer to remote flag */
69
int val, /* value to set flag upon completion of
71
int proc /* remote process(or) ID */
74
extern int ARMCI_Acc(int optype, void *scale, void *src, void *dst, int bytes, int proc);
76
extern int ARMCI_AccS( /* strided accumulate */
77
int optype, /* operation */
78
void *scale, /* scale factor x += scale*y */
79
void *src_ptr, /* pointer to 1st segment at source*/
80
int src_stride_arr[], /* array of strides at source */
81
void* dst_ptr, /* pointer to 1st segment at destination*/
82
int dst_stride_arr[], /* array of strides at destination */
83
int count[], /* number of units at each stride level count[0]=bytes */
84
int stride_levels, /* number of stride levels */
85
int proc /* remote process(or) ID */
89
extern int ARMCI_Get(void *src, void* dst, int bytes, int proc);
91
extern int ARMCI_GetS( /* strided get */
92
void *src_ptr, /* pointer to 1st segment at source*/
93
int src_stride_arr[], /* array of strides at source */
94
void* dst_ptr, /* pointer to 1st segment at destination*/
95
int dst_stride_arr[], /* array of strides at destination */
96
int count[], /* number of units at each stride level count[0]=bytes */
97
int stride_levels, /* number of stride levels */
98
int proc /* remote process(or) ID */
101
extern int ARMCI_GetV( armci_giov_t darr[], /* descriptor array */
102
int len, /* length of descriptor array */
103
int proc /* remote process(or) ID */
106
extern int ARMCI_PutV( armci_giov_t darr[], /* descriptor array */
107
int len, /* length of descriptor array */
108
int proc /* remote process(or) ID */
111
extern int ARMCI_AccV( int op, /* operation code */
112
void *scale, /* scaling factor for accumulate */
113
armci_giov_t darr[], /* descriptor array */
114
int len, /* length of descriptor array */
115
int proc /* remote process(or) ID */
118
extern int ARMCI_PutValueInt(int src, /* value in a register to put */
119
void *dst, /* dest starting addr to put data */
120
int proc /* remote process (or) ID */
123
extern int ARMCI_PutValueLong(long src, /* value in a register to put */
124
void *dst, /* dest starting addr to put data */
125
int proc /* remote process (or) ID */
128
extern int ARMCI_PutValueFloat(float src, /* value in a register to put */
129
void *dst, /* dest starting addr to put data */
130
int proc /* remote process (or) ID */
133
extern int ARMCI_PutValueDouble(double src,/* value in a register to put */
134
void *dst, /* dest starting addr to put data */
135
int proc /* remote process (or) ID */
138
extern int ARMCI_GetValueInt(void *src, int proc);
139
extern long ARMCI_GetValueLong(void *src, int proc);
140
extern float ARMCI_GetValueFloat(void *src, int proc);
141
extern double ARMCI_GetValueDouble(void *src, int proc);
144
extern int ARMCI_Malloc(void* ptr_arr[], armci_size_t bytes);
145
extern int ARMCI_Free(void *ptr);
146
extern void* ARMCI_Malloc_local(armci_size_t bytes);
147
extern int ARMCI_Free_local(void *ptr);
148
extern int ARMCI_Same_node(int proc);
150
extern void ARMCI_Finalize(); /* terminate ARMCI */
151
extern void ARMCI_Error(char *msg, int code);
152
extern void ARMCI_Fence(int proc);
153
extern void ARMCI_DoFence(int proc);
154
extern void ARMCI_AllFence(void);
155
extern int ARMCI_Rmw(int op, void *ploc, void *prem, int extra, int proc);
156
extern void ARMCI_Cleanup(void);
157
extern int ARMCI_Create_mutexes(int num);
158
extern int ARMCI_Destroy_mutexes(void);
159
extern void ARMCI_Lock(int mutex, int proc);
160
extern void ARMCI_Unlock(int mutex, int proc);
161
extern void ARMCI_Set_shm_limit(unsigned long shmemlimit);
162
extern int ARMCI_Uses_shm();
163
extern void ARMCI_Copy(void *src, void *dst, int n);
174
#define ARMCI_SWAP 10
175
#define ARMCI_SWAP_LONG 11
176
#define ARMCI_FETCH_AND_ADD 12
177
#define ARMCI_FETCH_AND_ADD_LONG 13
179
#define ARMCI_ACC_OFF 36
180
#define ARMCI_ACC_INT (ARMCI_ACC_OFF + 1)
181
#define ARMCI_ACC_DBL (ARMCI_ACC_OFF + 2)
182
#define ARMCI_ACC_FLT (ARMCI_ACC_OFF + 3)
183
#define ARMCI_ACC_CPL (ARMCI_ACC_OFF + 4)
184
#define ARMCI_ACC_DCP (ARMCI_ACC_OFF + 5)
185
#define ARMCI_ACC_LNG (ARMCI_ACC_OFF + 6)
186
#define ARMCI_ACC_RA (ARMCI_ACC_OFF + 7)
188
#define ARMCI_MAX_STRIDE_LEVEL 8
191
#define ARMCI_CRITICAL_SECTION_ENTER() BGML_CriticalSection_enter();
192
#define ARMCI_CRITICAL_SECTION_EXIT() BGML_CriticalSection_exit();
194
#define ARMCI_CRITICAL_SECTION_ENTER()
195
#define ARMCI_CRITICAL_SECTION_EXIT()
198
/************ locality information **********************************************/
199
typedef int armci_domain_t;
200
#define ARMCI_DOMAIN_SMP 0 /* SMP node domain for armci_domain_XXX calls */
201
extern int armci_domain_nprocs(armci_domain_t domain, int id);
202
extern int armci_domain_id(armci_domain_t domain, int glob_proc_id);
203
extern int armci_domain_glob_proc_id(armci_domain_t domain, int id, int loc_proc_id);
204
extern int armci_domain_my_id(armci_domain_t domain);
205
extern int armci_domain_count(armci_domain_t domain);
206
extern int armci_domain_same_id(armci_domain_t domain, int proc);
207
extern int armci_smp_master(int);
211
* On CrayT3E: the default group is the global group which is (char *)NULL
212
* It is the only working group.
213
* On Workstations: the default group is "mp_working_group". User can set
214
* the group name by calling the ARMCI_PVM_init (defined
215
* in message.c) and passing the group name to the library.
218
extern char *mp_group_name;
220
/*********************stuff for non-blocking API******************************/
221
/*\ the request structure for non-blocking api.
225
int data[4]; /* tag, bufid, agg_flag, op, proc */
226
double dummy[72]; /* bg1s_t, count, extra */
230
# if defined(__64BIT__)
231
double dummy[27]; /*lapi_cntr_t is 200 bytes, using 216 just to be safe*/
233
double dummy[24]; /*lapi_cntr_t is 148 bytes, using 166 just to be safe*/
235
#elif defined(ALLOW_PIN)
236
void *dummy[2];/*2 cause itshould be aligned after we cast hdl_t to ihdl_t*/
243
#define armci_req_t armci_hdl_t
245
typedef int ARMCI_Group;
247
extern void ARMCI_Group_create(int n, int *pid_list, ARMCI_Group *group_out);
248
extern void ARMCI_Group_create_child(int n, int *pid_list,
249
ARMCI_Group *group_out, ARMCI_Group *group_parent);
250
extern void ARMCI_Group_free(ARMCI_Group *group);
251
extern int ARMCI_Group_rank(ARMCI_Group *group, int *rank);
252
extern void ARMCI_Group_size(ARMCI_Group *group, int *size);
253
extern void ARMCI_Group_set_default(ARMCI_Group *group);
254
extern void ARMCI_Group_get_default(ARMCI_Group *group_out);
255
extern void ARMCI_Group_get_world(ARMCI_Group *group_out);
257
extern int ARMCI_Malloc_group(void *ptr_arr[], armci_size_t bytes,ARMCI_Group *group);
258
extern int ARMCI_Free_group(void *ptr, ARMCI_Group *group);
260
extern int ARMCI_NbPut(void *src, void* dst, int bytes, int proc,armci_hdl_t* nb_handle);
262
extern int ARMCI_NbPutS( /* strided put */
263
void *src_ptr, /* pointer to 1st segment at source*/
264
int src_stride_arr[], /* array of strides at source */
265
void* dst_ptr, /* pointer to 1st segment at destination*/
266
int dst_stride_arr[], /* array of strides at destination */
267
int count[], /* number of units at each stride level count[0]=bytes */
268
int stride_levels, /* number of stride levels */
269
int proc, /* remote process(or) ID */
270
armci_hdl_t* nb_handle /*armci_non-blocking request handle*/
273
extern int ARMCI_NbAccS( /* strided accumulate */
274
int optype, /* operation */
275
void *scale, /* scale factor x += scale*y */
276
void *src_ptr, /* pointer to 1st segment at source*/
277
int src_stride_arr[], /* array of strides at source */
278
void* dst_ptr, /* pointer to 1st segment at destination*/
279
int dst_stride_arr[], /* array of strides at destination */
280
int count[], /* number of units at each stride level count[0]=bytes */
281
int stride_levels, /* number of stride levels */
282
int proc, /* remote process(or) ID */
283
armci_hdl_t* nb_handle /*armci_non-blocking request handle*/
286
extern int ARMCI_NbGet(void *src, void* dst, int bytes, int proc,armci_hdl_t* nb_handle);
288
extern int ARMCI_NbGetS( /* strided get */
289
void *src_ptr, /* pointer to 1st segment at source*/
290
int src_stride_arr[], /* array of strides at source */
291
void* dst_ptr, /* pointer to 1st segment at destination*/
292
int dst_stride_arr[], /* array of strides at destination */
293
int count[], /* number of units at each stride level count[0]=bytes */
294
int stride_levels, /* number of stride levels */
295
int proc, /* remote process(or) ID */
296
armci_hdl_t* nb_handler/*armci_non-blocking request handle*/
299
extern int ARMCI_NbGetV( armci_giov_t darr[], /* descriptor array */
300
int len, /* length of descriptor array */
301
int proc, /* remote process(or) ID */
302
armci_hdl_t* nb_handle /*armci_non-blocking request handle*/
305
extern int ARMCI_NbPutV( armci_giov_t darr[], /* descriptor array */
306
int len, /* length of descriptor array */
307
int proc, /* remote process(or) ID */
308
armci_hdl_t* nb_handle /*armci_non-blocking request handle*/
311
extern int ARMCI_NbAccV( int op, /* operation code */
312
void *scale, /* scaling factor for accumulate */
313
armci_giov_t darr[], /* descriptor array */
314
int len, /* length of descriptor array */
315
int proc, /* remote process(or) ID */
316
armci_hdl_t* nb_handle /*armci_non-blocking request handle*/
319
extern int ARMCI_NbPutValueInt(int src, /* value in a register to put */
320
void *dst, /* dest starting addr to put data */
321
int proc, /* remote process (or) ID */
322
armci_hdl_t* nb_handle /*armci_non-blocking
326
extern int ARMCI_NbPutValueLong(long src, /* value in a register to put */
327
void *dst, /* dest starting addr to put data */
328
int proc, /* remote process (or) ID */
329
armci_hdl_t* nb_handle /*armci_non-blocking
333
extern int ARMCI_NbPutValueFloat(float src,/* value in a register to put */
334
void *dst,/* dest starting addr to put data */
335
int proc, /* remote process (or) ID */
336
armci_hdl_t* nb_handle /*armci_non-blocking
340
extern int ARMCI_NbPutValueDouble(double src,/* value in a register to put */
341
void *dst,/* dest starting addr to put data*/
342
int proc, /* remote process (or) ID */
343
armci_hdl_t* nb_handle /*armci_non-blocking
347
extern int ARMCI_Wait(armci_hdl_t* nb_handle); /*non-blocking request handle*/
349
extern int ARMCI_Test(armci_hdl_t* nb_handle); /*non-blocking request handle*/
351
extern int ARMCI_WaitAll (void);
353
extern int ARMCI_WaitProc (int proc);
355
extern void ARMCI_SET_AGGREGATE_HANDLE(armci_hdl_t* nb_handle);
357
extern void ARMCI_UNSET_AGGREGATE_HANDLE(armci_hdl_t* nb_handle);
359
#define ARMCI_INIT_HANDLE(hdl) do {((double *)((hdl)->data))[0]=0; \
360
((double *)((hdl)->data))[1]=0; }while(0)
362
/* -------------- ARMCI Non-collective memory allocator ------------- */
363
typedef struct armci_meminfo_ds {
364
char * armci_addr; /* remote address of the creator which can be
365
used in ARMCI communication */
366
char *addr; /* local address of creator which can be used in
367
to set SMP memoffset, armci_set_mem_offset() */
368
size_t size; /* size of remote pid's segment (bytes) */
369
int cpid; /* armci pid of creator */
373
extern void ARMCI_Memget(size_t bytes, armci_meminfo_t *meminfo, int memflg);
375
extern void* ARMCI_Memat(armci_meminfo_t *meminfo, long offset);
377
extern void ARMCI_Memdt(armci_meminfo_t *meminfo, int memflg);
379
extern void ARMCI_Memctl(armci_meminfo_t *meminfo);
381
/* ------------------- ARMCI Checkpointing/Recovery ----------------- */
384
#define ARMCI_RESTART 1
391
void ARMCI_Ckpt_create_ds(armci_ckpt_ds_t *ckptds, int count);
392
int ARMCI_Ckpt_init(char *filename, ARMCI_Group *grp, int savestack, int saveheap, armci_ckpt_ds_t *ckptds);
393
int ARMCI_Ckpt(int rid);
394
void ARMCI_Ckpt_finalize(int rid);
395
#define ARMCI_Restart_simulate armci_irecover
397
ARMCI_Group * ARMCI_Get_ft_group();
401
/* ------------------------------------------------------------------ */
404
#if defined(__cplusplus) || defined(c_plusplus)
408
#endif /* _ARMCI_H */