2
ADIOS is freely available under the terms of the BSD license described
3
in the COPYING file in the top level directory of this source distribution.
5
Copyright (c) 2008 - 2009. UT-BATTELLE, LLC. All rights reserved.
8
This is a cython file. To generate a CPP file, use the following command:
9
$ cython --cplus adios.pyx
15
import mpi4py.MPI as MPI
16
cimport mpi4py.MPI as MPI
21
## ====================
22
## ADIOS Exported Functions
23
## ====================
25
from libc.stdint cimport uint32_t, int64_t, uint64_t
26
from libc.stdlib cimport malloc, free
28
cdef extern from "adios_types.h":
29
ctypedef enum ADIOS_DATATYPES:
37
adios_unsigned_integer
46
ctypedef enum ADIOS_BUFFER_ALLOC_WHEN:
47
ADIOS_BUFFER_ALLOC_UNKNOWN
48
ADIOS_BUFFER_ALLOC_NOW
49
ADIOS_BUFFER_ALLOC_LATER
51
ctypedef enum ADIOS_FLAG:
54
cdef extern from "adios.h":
55
ctypedef struct MPI_Comm:
58
ctypedef char* const_char_ptr "const char*"
60
cdef int adios_init (char * config, MPI_Comm)
62
cdef int adios_finalize (int mype)
64
cdef int adios_open (int64_t * fd,
70
cdef int adios_group_size (int64_t fd_p,
72
uint64_t * total_size)
74
cdef int adios_write (int64_t fd_p,
78
cdef int adios_read (int64_t fd_p,
83
cdef int adios_close(int64_t fd_p)
85
cdef int adios_init_noxml (MPI_Comm)
87
cdef int adios_allocate_buffer (ADIOS_BUFFER_ALLOC_WHEN when,
90
cdef int adios_declare_group (int64_t * id,
95
cdef int adios_define_var (int64_t group_id,
100
char * global_dimensions,
101
char * local_offsets)
103
cdef int adios_define_attribute (int64_t group,
106
ADIOS_DATATYPES type,
110
cdef int adios_select_method (int64_t group,
115
cdef extern from "adios_selection.h":
116
ctypedef enum ADIOS_SELECTION_TYPE:
117
ADIOS_SELECTION_BOUNDINGBOX
118
ADIOS_SELECTION_POINTS
119
ADIOS_SELECTION_WRITEBLOCK
122
ctypedef struct ADIOS_SELECTION_BOUNDINGBOX_STRUCT:
127
ctypedef struct ADIOS_SELECTION_POINTS_STRUCT:
130
ctypedef struct ADIOS_SELECTION_WRITEBLOCK_STRUCT:
133
ctypedef struct ADIOS_SELECTION_AUTO_STRUCT:
136
cdef union ADIOS_SELECTION_UNION:
137
ADIOS_SELECTION_BOUNDINGBOX_STRUCT bb
138
ADIOS_SELECTION_POINTS_STRUCT points
139
ADIOS_SELECTION_WRITEBLOCK_STRUCT block
140
ADIOS_SELECTION_AUTO_STRUCT autosel
142
ctypedef struct ADIOS_SELECTION:
143
ADIOS_SELECTION_TYPE type
144
ADIOS_SELECTION_UNION u
146
cdef ADIOS_SELECTION * adios_selection_boundingbox (int ndim,
147
const uint64_t *start,
148
const uint64_t *count)
150
cdef extern from "adios_read.h":
151
ctypedef enum ADIOS_READ_METHOD:
153
ADIOS_READ_METHOD_BP_AGGREGATE
156
ctypedef enum ADIOS_LOCKMODE:
158
ADIOS_LOCKMODE_CURRENT
161
ctypedef struct ADIOS_FILE:
166
char ** attr_namelist
168
char ** mesh_namelist
176
ctypedef struct ADIOS_VARINFO:
186
cdef int adios_read_init_method (ADIOS_READ_METHOD method,
189
cdef int adios_read_finalize_method(ADIOS_READ_METHOD method)
190
cdef ADIOS_FILE * adios_read_open (const char * fname,
191
ADIOS_READ_METHOD method,
193
ADIOS_LOCKMODE lock_mode,
195
cdef ADIOS_FILE * adios_read_open_file (const char * fname,
196
ADIOS_READ_METHOD method,
198
cdef int adios_read_close (ADIOS_FILE *fp)
199
cdef int adios_advance_step (ADIOS_FILE *fp, int last, float timeout_sec)
200
cdef void adios_release_step (ADIOS_FILE *fp)
201
cdef ADIOS_VARINFO * adios_inq_var (ADIOS_FILE *fp, const char * varname)
202
cdef ADIOS_VARINFO * adios_inq_var_byid (ADIOS_FILE *fp, int varid)
203
cdef void adios_free_varinfo (ADIOS_VARINFO *cp)
204
cdef int adios_schedule_read (const ADIOS_FILE * fp,
205
const ADIOS_SELECTION * sel,
206
const char * varname,
210
cdef int adios_schedule_read_byid (const ADIOS_FILE * fp,
211
const ADIOS_SELECTION * sel,
216
cdef int adios_perform_reads (const ADIOS_FILE *fp, int blocking)
219
## ====================
220
## ADIOS Enum (public)
221
## ====================
231
unsigned_integer = 52
245
class BUFFER_ALLOC_WHEN:
250
## ====================
252
## ====================
254
cpdef init(char * config, MPI.Comm comm = MPI.COMM_WORLD):
255
return adios_init(config, comm.ob_mpi)
257
cpdef int64_t open(char * group_name,
260
MPI.Comm comm = MPI.COMM_WORLD):
263
result = adios_open(&fd, group_name, name, mode, comm.ob_mpi)
266
cpdef int64_t set_group_size(int64_t fd_p, uint64_t data_size):
267
cdef uint64_t total_size
269
result = adios_group_size(fd_p, data_size, &total_size)
272
cpdef int write (int64_t fd_p, char * name, np.ndarray val):
274
if val.flags.contiguous:
277
val_ = np.array(val, copy=True)
279
return adios_write (fd_p, name, <void *> val_.data)
281
cpdef int write_int (int64_t fd_p, char * name, int val):
282
return adios_write (fd_p, name, &val)
284
cpdef int write_long (int64_t fd_p, char * name, long val):
285
return adios_write (fd_p, name, &val)
287
cpdef int write_float (int64_t fd_p, char * name, float val):
288
return adios_write (fd_p, name, &val)
290
cpdef int read(int64_t fd_p, char * name, np.ndarray val):
291
assert val.flags.contiguous, 'Only contiguous arrays are supported.'
292
print "Reading ... ", val.itemsize * val.size, "(bytes)"
293
return adios_read(fd_p, name, <void *> val.data, val.itemsize * val.size)
295
cpdef int close(int64_t fd_p):
296
return adios_close(fd_p)
298
cpdef finalize(int mype = 0):
299
return adios_finalize(mype)
301
## ====================
303
## ====================
304
cpdef int init_noxml(MPI.Comm comm = MPI.COMM_WORLD):
305
return adios_init_noxml(comm.ob_mpi)
307
cpdef int allocate_buffer(int when,
308
uint64_t buffer_size):
309
return adios_allocate_buffer(<ADIOS_BUFFER_ALLOC_WHEN> when,
312
cpdef int64_t declare_group(char * name,
316
adios_declare_group (&id,
322
cpdef int define_var(int64_t group_id,
327
char * global_dimensions,
328
char * local_offsets):
329
return adios_define_var(group_id,
331
<ADIOS_DATATYPES> type,
336
cpdef int define_attribute (int64_t group,
342
return adios_define_attribute (group,
345
<ADIOS_DATATYPES> type,
349
cpdef int select_method (int64_t group,
353
return adios_select_method (group,
359
## ====================
360
## ADIOS Read API (V2)
361
## ====================
363
cdef type adios2nptype(ADIOS_DATATYPES t):
364
cdef type ntype = None
367
elif t == adios_short:
369
elif t == adios_integer:
371
elif t == adios_long:
373
elif t == adios_unsigned_byte:
375
elif t == adios_unsigned_short:
377
elif t == adios_unsigned_integer:
379
elif t == adios_unsigned_long:
381
elif t == adios_real:
383
elif t == adios_double:
385
elif t == adios_long_double:
387
elif t == adios_complex:
389
elif t == adios_double_complex:
390
ntype = np.complex128
396
cdef printfile(ADIOS_FILE * f):
397
print '%15s : %lu' % ('fh', f.fh)
398
print '%15s : %d' % ('nvars', f.nvars)
399
print '%15s : %s' % ('var_namelist', [f.var_namelist[i] for i in range(f.nvars)])
400
print '%15s : %d' % ('nattrs', f.nattrs)
401
print '%15s : %s' % ('attr_namelist', [f.attr_namelist[i] for i in range(f.nattrs)])
402
print '%15s : %d' % ('current_step', f.current_step)
403
print '%15s : %d' % ('last_step', f.last_step)
404
print '%15s : %s' % ('path', f.path)
405
print '%15s : %d' % ('endianness', f.endianness)
406
print '%15s : %d' % ('version', f.version)
407
print '%15s : %lu' % ('file_size', f.file_size)
409
cdef printvar(ADIOS_VARINFO * v):
410
print '%15s : %d' % ('varid', v.varid)
411
print '%15s : %s' % ('type', adios2nptype(v.type))
412
print '%15s : %d' % ('ndim', v.ndim)
413
print '%15s : %s' % ('dims', [v.dims[i] for i in range(v.ndim)])
414
print '%15s : %d' % ('nsteps', v.nsteps)
416
## ====================
417
## ADIOS Class Definitions for Read
418
## ====================
420
""" Call adios_read_init_method """
421
cpdef read_init(ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP,
422
MPI.Comm comm = MPI.COMM_WORLD,
423
char * parameters = ""):
424
return adios_read_init_method (method, comm.ob_mpi, parameters)
427
""" Call adios_read_finalize_method """
428
cpdef read_finalize(ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP):
429
return adios_read_finalize_method (method)
431
""" Python class for ADIOS_FILE structure """
433
""" Private Memeber """
434
cpdef ADIOS_FILE * fp
436
""" Public Memeber """
437
cpdef public bytes name
438
cpdef public int nvars
439
cpdef public int nattrs
440
cpdef public int current_step
441
cpdef public int last_step
442
cpdef public int endianness
443
cpdef public int version
444
cpdef public int file_size
446
cpdef public dict var
447
cpdef public dict attr
449
""" Initialization. Call adios_read_open and populate public members """
450
def __init__(self, char * fname,
451
ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP,
452
MPI.Comm comm = MPI.COMM_WORLD):
457
self.fp = adios_read_open_file(fname, method, comm.ob_mpi)
458
assert self.fp != NULL, 'Not an open file'
460
self.name = fname.split('/')[-1] ## basename
461
self.nvars = self.fp.nvars
462
self.nattrs = self.fp.nattrs
463
self.current_step = self.fp.current_step
464
self.last_step = self.fp.last_step
465
self.endianness = self.fp.endianness
466
self.version = self.fp.version
467
self.file_size = self.fp.file_size
469
for varname in [self.fp.var_namelist[i] for i in range(self.nvars)]:
470
self.var[varname] = var(self, varname)
475
""" Call adios_read_close """
477
assert self.fp != NULL, 'Not an open file'
478
adios_read_close(self.fp)
482
cpdef printself(self):
483
assert self.fp != NULL, 'Not an open file'
484
print '=== AdiosFile ==='
485
print '%15s : %lu' % ('fp', <unsigned long> self.fp)
488
""" Python class for ADIOS_VARINFO structure """
490
""" Private Memeber """
492
cdef ADIOS_VARINFO * vp
494
""" Public Memeber """
495
cpdef public bytes name
496
cpdef public int varid
497
cpdef public type type
498
cpdef public int ndim
499
cpdef public tuple dims
500
cpdef public int nsteps
502
""" Initialization. Call adios_inq_var and populate public members """
503
def __init__(self, file file, char * name):
507
assert self.file.fp != NULL, 'Not an open file'
508
self.vp = adios_inq_var(self.file.fp, name)
509
assert self.vp != NULL, 'Not a valid var'
512
self.varid = self.vp.varid
513
self.type = adios2nptype(self.vp.type)
514
self.ndim = self.vp.ndim
515
self.dims = tuple([self.vp.dims[i] for i in range(self.vp.ndim)])
516
self.nsteps = self.vp.nsteps
521
""" Call adios_free_varinfo """
523
assert self.vp != NULL, 'Not an open var'
524
adios_free_varinfo(self.vp)
527
""" Call adios_schedule_read and adios_perform_reads """
528
cpdef read(self, tuple offset = (), tuple count = (), from_steps = 0, nsteps = 1):
529
assert self.type is not None, 'Data type is not supported yet'
530
assert from_steps + nsteps <= self.nsteps, 'Step index is out of range'
532
cdef list lshape = [self.vp.dims[i] for i in range(self.vp.ndim)]
533
cdef np.ndarray npshape = np.array(lshape, dtype=np.int64)
535
cdef np.ndarray npoffset
537
npoffset = npshape.copy()
540
npoffset = np.array(offset, dtype=np.int64)
542
cdef np.ndarray npcount
544
npcount = npshape - npoffset
546
npcount = np.array(count, dtype=np.int64)
548
assert npshape.ndim == npoffset.ndim, 'Offset dimension mismatch'
549
assert npshape.ndim == npcount.ndim, 'Count dimension mismatch.'
550
assert (npshape - npoffset >= npcount).all(), 'Count is larger than shape.'
552
shape = list(npcount)
554
shape.insert(0, nsteps)
555
cdef np.ndarray var = np.zeros(shape, dtype=self.type)
557
cdef ADIOS_SELECTION * sel
558
sel = adios_selection_boundingbox (self.vp.ndim, <uint64_t *> npoffset.data, <uint64_t *> npcount.data)
560
adios_schedule_read_byid (self.file.fp, sel, self.vp.varid, from_steps, nsteps, <void *> var.data)
561
adios_perform_reads(self.file.fp, 1)
566
cpdef printself(self):
567
assert self.vp != NULL, 'Not an open variable'
568
print '=== AdiosVariable ==='
569
print '%15s : %lu' % ('vp', <unsigned long> self.vp)
570
print '%15s : %lu' % ('fp', <unsigned long> self.file.fp)
573
## ====================
574
## ADIOS Global functions
575
## ====================
577
""" Read data in a BP file and return as a numpy array """
578
def readvar(fname, varname):
579
f = file(fname, comm=MPI.COMM_SELF)
580
if not f.var.has_key(varname):
581
print "No valid variable"
585
return v.read(from_steps=0, nsteps=v.nsteps)
587
""" List attributes of a BP file """
589
f = file(fname, comm=MPI.COMM_SELF)
590
return {'nvars': f.nvars,
592
'vars': tuple([ k for k in f.var.iterkeys() ]),
593
'attrs': tuple([ k for k in f.attr.iterkeys() ]),
594
'time_steps': (f.current_step, f.last_step),
595
'file_size': f.file_size}
added
removed
wrappers/numpy/adios_mpi.pyx
