207
208
BP_PROC * p = (BP_PROC *) fp->fh;
208
209
BP_FILE * fh = (BP_FILE *) p->fh;
209
210
struct adios_index_var_struct_v1 * v;
211
int start_time, stop_time, start_idx, stop_idx, f_idx;
212
int ndim, time, has_subfile, file_is_fortran;
213
uint64_t size, * dims = 0;
212
int start_idx, stop_idx;
213
int ndim, time, file_is_fortran;
214
215
uint64_t ldims[32], gdims[32], offsets[32];
215
216
uint64_t * start, * count;
216
217
int nsteps, is_global = 0, flag;
218
219
file_is_fortran = is_fortran_file (fh);
219
has_subfile = has_subfiles (fh);
220
220
v = bp_find_var_byid (fh, varid);
222
222
/* Get dimensions and flip if caller != writer language */
512
513
char * b = 0, * recv_buff = 0;
513
514
int * sizes = 0, * offsets = 0;
516
516
read_request * r = p->local_read_request_list;
518
518
if (pvt->rank == pvt->aggregator_rank)
555
555
static void parse_buffer (BP_PROC * p, char * b, int src)
557
read_request * h = p->local_read_request_list;
558
int i, j, type, count, varid, ndims, size = calc_data_size (p);
560
558
read_request * r;
561
559
rr_pvt_struct * rr_pvt;
629
627
BP_PROC * p = (BP_PROC *) fp->fh;
630
628
BP_FILE * fh = (BP_FILE *) p->fh;
631
629
struct adios_index_var_struct_v1 * v;
632
int i, j, k, idx, t, varid, nsteps, time, dummy;
633
int start_time, stop_time, start_idx, stop_idx, f_idx;
634
int ndim, has_subfile, file_is_fortran;
635
uint64_t size, * dims = 0;
630
int i, j, idx, t, varid, nsteps, time, dummy;
631
int start_idx, stop_idx;
632
int ndim, file_is_fortran;
636
634
uint64_t ldims[32], gdims[32], offsets[32];
637
635
uint64_t * start, * count;
638
636
int is_global = 0, flag;
639
ADIOS_SELECTION * sel;
640
637
read_request * h = 0;
641
638
rr_pvt_struct * rr_pvt = (rr_pvt_struct *) r->priv;
644
641
varid = r->varid;
646
642
start = r->sel->u.bb.start;
647
643
count = r->sel->u.bb.count;
648
644
file_is_fortran = is_fortran_file (fh);
649
has_subfile = has_subfiles (fh);
650
645
v = bp_find_var_byid (fh, varid);
652
647
bp_get_and_swap_dimensions (fh, v, file_is_fortran,
812
807
else if (hole_break == 0)
815
uint64_t size_in_dset = 0;
816
uint64_t offset_in_dset = 0;
818
811
isize = offsets[0] + ldims[0];
819
812
if (start[0] >= offsets[0])
846
838
n->sel->u.bb.count[0] = count[0] + start[0] - offsets[0];
848
840
n->sel->u.bb.start[0] = offsets[0];
849
//offset_in_dset = 0;
855
uint64_t stride_offset = 0;
857
847
uint64_t size_in_dset[10];
858
848
uint64_t offset_in_dset[10];
979
969
new_h->file_index = file_idx;
981
if (ch = strrchr (fh->fname, '/'))
971
if ( (ch = strrchr (fh->fname, '/')))
983
973
name_no_path = malloc (strlen (ch + 1) + 1);
984
974
strcpy (name_no_path, ch + 1);
1039
1029
fh->b->offset = 0;
1041
gettimeofday (&t1, NULL);
1043
// printf ("read chunk time = %f \n", t1.tv_sec - t0.tv_sec + (double)(t1.tv_usec - t0.tv_usec)/1000000 );
1031
//gettimeofday (&t1, NULL);
1032
//printf ("read chunk time = %f \n", t1.tv_sec - t0.tv_sec + (double)(t1.tv_usec - t0.tv_usec)/1000000 );
1049
1038
static void do_read (const ADIOS_FILE * fp)
1051
1040
BP_PROC * p = (BP_PROC *) fp->fh;
1052
BP_FILE * fh = (BP_FILE *) p->fh;
1053
1041
bp_proc_pvt_struct * pvt = (bp_proc_pvt_struct *) p->priv;
1057
1043
uint64_t offset, payload_size;
1060
double t2, t3, t4, t5;
1044
//struct timeval t0;
1045
//struct timeval t1;
1046
//double t2, t3, t4, t5;
1062
1048
read_request * s = pvt->split_read_request_list, * f_start = s, * f_end = s;
1063
1049
read_request * o_start = s, * o_end = s, * o_prev_end = 0, * parent = 0;
1065
gettimeofday (&t0, NULL);
1051
//gettimeofday (&t0, NULL);
1069
1055
while (f_start)
1102
1088
//printf ("o_start.offset = %llu\n", o_start->ra->offset);
1103
1089
//printf ("o_prev_end.offset = %llu\n", o_prev_end->ra->offset);
1091
//t4 = MPI_Wtime ();
1106
1092
// read a chunk from file into internal buffer
1107
1093
read_chunk (fp,
1108
1094
((rr_pvt_struct *) o_start->priv)->file_idx,
1131
1117
f_start = f_end;
1134
gettimeofday (&t1, NULL);
1120
//gettimeofday (&t1, NULL);
1121
//t3 = MPI_Wtime ();
1137
// printf ("while time = %f \n", t1.tv_sec - t0.tv_sec + (double)(t1.tv_usec - t0.tv_usec)/1000000 );
1138
// printf ("while time = %f \n", t3 - t2);
1123
//printf ("while time = %f \n", t1.tv_sec - t0.tv_sec + (double)(t1.tv_usec - t0.tv_usec)/1000000 );
1124
//printf ("while time = %f \n", t3 - t2);
1141
1127
/* r - the original read rquest
1153
1139
struct adios_index_var_struct_v1 * v;
1154
1140
uint64_t * s_start, * s_count;
1155
1141
uint64_t * start, * count;
1156
int i, j, k, idx, t, time, ndim, dummy;
1157
int varid, start_idx, stop_idx, has_subfile, file_is_fortran;
1142
int i, j, idx, t, time, ndim, dummy;
1143
int varid, start_idx, stop_idx, file_is_fortran;
1158
1144
uint64_t ldims[MAX_DIMS], gdims[MAX_DIMS], offsets[MAX_DIMS];
1159
1145
uint64_t datasize, dset_stride, var_stride, total_size = 0, items_read;
1160
1146
int is_global = 0, size_unit, break_dim, idx_check1, idx_check2;
1169
1155
s_start = s->sel->u.bb.start;
1170
1156
s_count = s->sel->u.bb.count;
1171
1157
file_is_fortran = is_fortran_file (fh);
1172
has_subfile = has_subfiles (fh);
1173
1158
ndim = r->sel->u.bb.ndim;
1174
1159
size_unit = bp_get_type_size (v->type,
1175
1160
v->characteristics [0].value
1530
1514
BP_FILE * fh = (BP_FILE *) p->fh;
1531
1515
struct bp_index_pg_struct_v1 * pgs_root = fh->pgs_root, * pg;
1532
1516
struct adios_index_var_struct_v1 * vars_root = fh->vars_root, * v;
1533
struct adios_index_attribute_struct_v1 * attrs_root = fh->attrs_root;
1535
1518
uint64_t buffer_size, buffer_offset = 0;
1536
1519
int i, j, nsteps;
1554
1537
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1555
1538
&fh->mfooter.pgs_count, 8); //vars_count
1556
1539
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1557
&fh->mfooter.vars_count, 2); //vars_count
1540
&fh->mfooter.vars_count, 4); //vars_count
1558
1541
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1559
&fh->mfooter.attrs_count, 2); //attrs_count
1542
&fh->mfooter.attrs_count, 4); //attrs_count
1561
1544
for (i = 0; i < fh->gvar_h->group_count; i++)
1566
1549
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1567
1550
fh->gvar_h->namelist[i], len); // namelist
1568
1551
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1569
&fh->gvar_h->var_counts_per_group[i], 2); // var_counts_per_group
1552
&fh->gvar_h->var_counts_per_group[i], sizeof(uint32_t)); // var_counts_per_group
1572
1555
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1574
1557
for (i = 0; i < fh->gattr_h->group_count; i++)
1576
1559
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1577
&fh->gattr_h->attr_counts_per_group[i], 2); // attr_counts_per_group
1560
&fh->gattr_h->attr_counts_per_group[i], sizeof(uint32_t)); // attr_counts_per_group
1580
1563
for (i = 0; i < fh->mfooter.vars_count; i++)
1626
1609
vars_root = fh->vars_root;
1627
1610
while (vars_root)
1629
uint64_t bo = buffer_offset;
1630
1612
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1631
&vars_root->id, 2); // id
1613
&vars_root->id, 4); // id
1633
1615
len = strlen (vars_root->group_name);
1634
1616
_buffer_write (&buffer, &buffer_size, &buffer_offset,
1723
1705
_buffer_read (buffer, &buffer_offset, &fh->gvar_h->group_count, 2); //group_count
1724
1706
_buffer_read (buffer, &buffer_offset, &fh->mfooter.pgs_count, 8); //pgs_count
1725
_buffer_read (buffer, &buffer_offset, &fh->mfooter.vars_count, 2); //vars_count
1726
_buffer_read (buffer, &buffer_offset, &fh->mfooter.attrs_count, 2); //attrs_count
1707
_buffer_read (buffer, &buffer_offset, &fh->mfooter.vars_count, 4); //vars_count
1708
_buffer_read (buffer, &buffer_offset, &fh->mfooter.attrs_count, 4); //attrs_count
1728
1710
fh->gvar_h->namelist = (char **) malloc (fh->gvar_h->group_count * sizeof (char *));
1729
fh->gvar_h->var_counts_per_group = (uint16_t *) malloc (fh->gvar_h->group_count * 2);
1711
fh->gvar_h->var_counts_per_group = (uint32_t *) malloc (fh->gvar_h->group_count * sizeof(uint32_t));
1731
1713
for (i = 0; i < fh->gvar_h->group_count; i++)
1735
1717
_buffer_read (buffer, &buffer_offset, fh->gvar_h->namelist[i], len); // namelist
1736
1718
fh->gvar_h->namelist[i][len] = '\0';
1738
_buffer_read (buffer, &buffer_offset, &fh->gvar_h->var_counts_per_group[i], 2); // var_counts_per_group
1720
_buffer_read (buffer, &buffer_offset, &fh->gvar_h->var_counts_per_group[i], sizeof(uint32_t)); // var_counts_per_group
1741
1723
fh->gattr_h = (struct BP_GROUP_ATTR *) malloc (sizeof (struct BP_GROUP_ATTR));
1745
1727
_buffer_read (buffer, &buffer_offset, &fh->gattr_h->group_count, 2); //group_count
1747
fh->gattr_h->attr_counts_per_group = (uint16_t *) malloc (fh->gattr_h->group_count * 2);
1729
fh->gattr_h->attr_counts_per_group = (uint32_t *) malloc (fh->gattr_h->group_count * sizeof(uint32_t));
1748
1730
fh->gattr_h->namelist = fh->gvar_h->namelist;
1750
1732
for (i = 0; i < fh->gattr_h->group_count; i++)
1752
_buffer_read (buffer, &buffer_offset, &fh->gattr_h->attr_counts_per_group[i], 2); // attr_counts_per_group
1734
_buffer_read (buffer, &buffer_offset, &fh->gattr_h->attr_counts_per_group[i], sizeof(uint32_t)); // attr_counts_per_group
1755
1737
fh->gvar_h->var_namelist = (char **) malloc (fh->mfooter.vars_count * sizeof (char *));
1819
1801
fh->vars_table[i] = v;
1821
_buffer_read (buffer, &buffer_offset, &v->id, 2);
1803
_buffer_read (buffer, &buffer_offset, &v->id, 4);
1823
1805
_buffer_read (buffer, &buffer_offset, &len, 2);
1824
1806
v->group_name = (char *) malloc (len + 1);
2020
2002
static void free_proc_struct (BP_PROC * p)
2022
2004
bp_proc_pvt_struct * pvt = (bp_proc_pvt_struct *) p->priv;
2023
read_request * h = p->local_read_request_list, * n;
2025
2006
/* We need to free split_read_request_list cause
2026
2007
* people might do two back-to-back sets of
2329
2310
int adios_read_bp_staged_perform_reads (const ADIOS_FILE *fp, int blocking)
2331
2312
BP_PROC * p = (BP_PROC *) fp->fh;
2332
BP_FILE * fh = (BP_FILE *) p->fh;
2333
2313
bp_proc_pvt_struct * pvt = (bp_proc_pvt_struct *) p->priv;
2334
2314
read_request * r, * h;
2335
2315
int i, count, varid, ndims, total_size, size;
2336
rr_pvt_struct * rrs;
2339
2318
// First populate the read request private struct for
2488
2467
adios_read_bp_reset_dimension_order (fp, is_fortran);
2491
void adios_read_bp_staged_get_groupinfo (const ADIOS_FILE *fp, int *ngroups, char ***group_namelist, int **nvars_per_group, int **nattrs_per_group)
2470
void adios_read_bp_staged_get_groupinfo (const ADIOS_FILE *fp, int *ngroups, char ***group_namelist, uint32_t **nvars_per_group, uint32_t **nattrs_per_group)
2495
int i, j, k, offset;
2497
2476
p = (BP_PROC *) fp->fh;
2498
2477
fh = (BP_FILE *) p->fh;
2500
2479
* ngroups = fh->gvar_h->group_count;
2502
alloc_namelist (group_namelist, fh->gvar_h->group_count);
2481
*group_namelist = (char **) malloc (sizeof (char *) * fh->gvar_h->group_count);
2503
2482
for (i = 0; i < fh->gvar_h->group_count; i++)
2505
2484
(*group_namelist)[i] = malloc (strlen (fh->gvar_h->namelist[i]) + 1);
2509
2488
namelist[i]) + 1);
2512
* nvars_per_group = (int *) malloc (fh->gvar_h->group_count * sizeof (int));
2491
* nvars_per_group = (uint32_t *) malloc (fh->gvar_h->group_count * sizeof (uint32_t));
2513
2492
assert (* nvars_per_group);
2515
2494
for (i = 0; i < fh->gvar_h->group_count; i++)
2517
2496
(* nvars_per_group)[i] = fh->gvar_h->var_counts_per_group[i];
2520
* nattrs_per_group = (int *) malloc (fh->gattr_h->group_count * sizeof (int));
2499
* nattrs_per_group = (uint32_t *) malloc (fh->gattr_h->group_count * sizeof (uint32_t));
2521
2500
assert (* nattrs_per_group);
2523
2502
for (i = 0; i < fh->gvar_h->group_count; i++)
2552
2531
struct adios_index_var_struct_v1 * v;
2553
2532
struct adios_index_characteristic_struct_v1 ch;
2554
int retval = 0, ndim, k, dummy, file_is_fortran;
2533
int retval = 0, ndim, k;
2555
2534
uint64_t gdims[32];
2557
2536
p = (BP_PROC *) fp->fh;
added
removed
src/read/read_bp_staged.c
