« back to all changes in this revision
Viewing changes to test/test_tables.py
- browse files at revision 2.1.1
- compare with another revision
- download diff
- download tarball
- view history from revision 2.1.1
- Committer: Bazaar Package Importer
- Author(s): Alexandre Fayolle
- Date: 2006-06-28 10:45:03 UTC
- mfrom: (1.2.1 upstream)
- mto: This revision was merged to the branch mainline in revision 5.
- Revision ID: james.westby@ubuntu.com-20060628104503-cc251q5o5j3e2k10
* Fixed call to pyversions in debian/rules which failed on recent versions
of pyversions
* Fixed clean rule in debian/rules which left the stamp files behind
* Acknowledge NMU
* Added Alexandre Fayolle to uploaders
of pyversions
* Fixed clean rule in debian/rules which left the stamp files behind
* Acknowledge NMU
* Added Alexandre Fayolle to uploaders
- files added:
- ANNOUNCE.txt.in
- LICENSES
- doc/text
- tables/scripts
- tables/tests
- files removed:
- LICENSE
- test
- files modified:
- ANNOUNCE.txt
added
removed
test/test_tables.py
1
import sys
2
import unittest
3
import os
4
import tempfile
5
6
import numarray
7
from numarray import *
8
import numarray.records as records
9
from tables import *
10
from tables.hdf5Extension import getIndices
11
12
from test_all import verbose, allequal, heavy
13
14
# Test Record class
15
class Record(IsDescription):
16
var1 = StringCol(length=4, dflt="abcd", pos=0) # 4-character String
17
var2 = IntCol(1, pos=1) # integer
18
var3 = Int16Col(2, pos=2) # short integer
19
var4 = Float64Col(3.1, pos=3) # double (double-precision)
20
var5 = Float32Col(4.2, pos=4) # float (single-precision)
21
var6 = UInt16Col(5, pos=5) # unsigned short integer
22
var7 = StringCol(length=1, dflt="e", pos=6) # 1-character String
23
var8 = BoolCol(1, pos=7) # boolean
24
var9 = Complex32Col((0.+1.j), pos=8) # Complex single precision
25
var10 = Complex64Col((1.-0.j), pos=9) # Complex double precision
26
27
# From 0.3 on, you can dynamically define the tables with a dictionary
28
RecordDescriptionDict = {
29
'var1': StringCol(4, "abcd", pos=0), # 4-character String
30
'var2': IntCol(1, pos=1), # integer
31
'var3': Int16Col(2, pos=2), # short integer
32
'var4': FloatCol(3.1, pos=3), # double (double-precision)
33
'var5': Float32Col(4.2, pos=4), # float (single-precision)
34
'var6': UInt16Col(5, pos=5), # unsigned short integer
35
'var7': StringCol(1, "e", pos=6), # 1-character String
36
'var8': BoolCol(1, pos=7), # boolean
37
'var9': Complex32Col((0.+1.j), pos=8), # Complex single precision
38
'var10': Complex64Col((1.-0.j), pos=9), # Complex double precision
39
}
40
41
# Old fashion of defining tables (for testing backward compatibility)
42
class OldRecord(IsDescription):
43
var1 = Col("CharType", shape=4, dflt="abcd", pos=0)
44
var2 = Col(Int32, 1, 1, pos=1)
45
var3 = Col(Int16, 1, 2, pos=2)
46
var4 = Col("Float64", 1, 3.1, pos=3)
47
var5 = Col("Float32", 1, 4.2, pos=4)
48
var6 = Col("UInt16", 1, 5, pos=5)
49
var7 = Col("CharType", shape=1, dflt="e", pos=6)
50
var8 = Col("Bool", shape=1, dflt=1, pos=7)
51
var9 = Col("Complex32", shape=1, dflt=(0.+1.j), pos=8)
52
var10 = Col("Complex64", shape=1, dflt=(1.-0.j), pos = 9)
53
54
class BasicTestCase(unittest.TestCase):
55
#file = "test.h5"
56
mode = "w"
57
title = "This is the table title"
58
expectedrows = 100
59
appendrows = 20
60
compress = 0
61
shuffle = 0
62
fletcher32 = 0
63
complib = "zlib" # Default compression library
64
record = Record
65
recarrayinit = 0
66
maxshort = 1 << 15
67
68
def setUp(self):
69
70
# Create an instance of an HDF5 Table
71
self.file = tempfile.mktemp(".h5")
72
self.fileh = openFile(self.file, self.mode)
73
self.rootgroup = self.fileh.root
74
self.populateFile()
75
self.fileh.close()
76
77
def initRecArray(self):
78
record = self.recordtemplate
79
row = record[0]
80
buflist = []
81
# Fill the recarray
82
for i in xrange(self.expectedrows):
83
tmplist = []
84
var1 = '%04d' % (self.expectedrows - i)
85
tmplist.append(var1)
86
var2 = i
87
tmplist.append(var2)
88
var3 = i % self.maxshort
89
tmplist.append(var3)
90
if isinstance(row.field('var4'), NumArray):
91
tmplist.append([float(i), float(i*i)])
92
else:
93
tmplist.append(float(i))
94
if isinstance(row.field('var5'), NumArray):
95
tmplist.append(array((float(i),)*4))
96
else:
97
tmplist.append(float(i))
98
# var6 will be like var3 but byteswaped
99
tmplist.append(((var3>>8) & 0xff) + ((var3<<8) & 0xff00))
100
var7 = var1[-1]
101
tmplist.append(var7)
102
if isinstance(row.field('var8'), NumArray):
103
tmplist.append([0, 10]) # should be equivalent to [0,1]
104
else:
105
tmplist.append(10) # should be equivalent to 1
106
if isinstance(row.field('var9'), NumArray):
107
tmplist.append([0.+float(i)*1j, float(i)+0.j])
108
else:
109
tmplist.append(float(i)+0j)
110
if isinstance(row.field('var10'), NumArray):
111
tmplist.append([float(i)+0j, 1+float(i)*1j])
112
else:
113
tmplist.append(1+float(i)*1j)
114
buflist.append(tmplist)
115
116
self.record=records.array(buflist, formats=record._formats,
117
names=record._names,
118
shape = self.expectedrows)
119
return
120
121
def populateFile(self):
122
group = self.rootgroup
123
if self.recarrayinit:
124
# Initialize an starting buffer, if any
125
self.initRecArray()
126
for j in range(3):
127
# Create a table
128
filterprops = Filters(complevel = self.compress,
129
shuffle = self.shuffle,
130
fletcher32 = self.fletcher32,
131
complib = self.complib)
132
table = self.fileh.createTable(group, 'table'+str(j), self.record,
133
title = self.title,
134
filters = filterprops,
135
expectedrows = self.expectedrows)
136
if not self.recarrayinit:
137
# Get the row object associated with the new table
138
row = table.row
139
140
# Fill the table
141
for i in xrange(self.expectedrows):
142
row['var1'] = '%04d' % (self.expectedrows - i)
143
row['var7'] = row['var1'][-1]
144
row['var2'] = i
145
row['var3'] = i % self.maxshort
146
if isinstance(row['var4'], NumArray):
147
row['var4'] = [float(i), float(i*i)]
148
else:
149
row['var4'] = float(i)
150
if isinstance(row['var8'], NumArray):
151
row['var8'] = [0, 1]
152
else:
153
row['var8'] = 1
154
if isinstance(row['var9'], NumArray):
155
row['var9'] = [0.+float(i)*1j, float(i)+0.j]
156
else:
157
row['var9'] = float(i)+0.j
158
if isinstance(row['var10'], NumArray):
159
row['var10'] = [float(i)+0.j, 1.+float(i)*1j]
160
else:
161
row['var10'] = 1.+float(i)*1j
162
if isinstance(row['var5'], NumArray):
163
row['var5'] = array((float(i),)*4)
164
else:
165
row['var5'] = float(i)
166
# var6 will be like var3 but byteswaped
167
row['var6'] = ((row['var3']>>8) & 0xff) + \
168
((row['var3']<<8) & 0xff00)
169
#print("Saving -->", row)
170
row.append()
171
172
# Flush the buffer for this table
173
table.flush()
174
# Create a new group (descendant of group)
175
group2 = self.fileh.createGroup(group, 'group'+str(j))
176
# Iterate over this new group (group2)
177
group = group2
178
179
180
def tearDown(self):
181
self.fileh.close()
182
#del self.fileh, self.rootgroup
183
os.remove(self.file)
184
185
#----------------------------------------
186
187
def test01_readTable(self):
188
"""Checking table read"""
189
190
rootgroup = self.rootgroup
191
if verbose:
192
print '\n', '-=' * 30
193
print "Running %s.test01_readTable..." % self.__class__.__name__
194
195
# Create an instance of an HDF5 Table
196
self.fileh = openFile(self.file, "r")
197
table = self.fileh.getNode("/table0")
198
199
# Choose a small value for buffer size
200
table._v_maxTuples = 3
201
# Read the records and select those with "var2" file less than 20
202
result = [ rec['var2'] for rec in table.iterrows()
203
if rec['var2'] < 20 ]
204
if verbose:
205
print "Nrows in", table._v_pathname, ":", table.nrows
206
print "Last record in table ==>", rec
207
print "Total selected records in table ==> ", len(result)
208
nrows = self.expectedrows - 1
209
assert (rec['var1'], rec['var2'], rec['var7']) == ("0001", nrows,"1")
210
if isinstance(rec['var5'], NumArray):
211
assert allequal(rec['var5'], array((float(nrows),)*4, Float32))
212
else:
213
assert rec['var5'] == float(nrows)
214
if isinstance(rec['var9'], NumArray):
215
assert allequal(rec['var9'],array([0.+float(nrows)*1.j,float(nrows)+0.j], Complex32))
216
else:
217
assert (rec['var9']) == float(nrows)+0.j
218
assert len(result) == 20
219
220
def test01b_readTable(self):
221
"""Checking table read and cuts (multidimensional columns case)"""
222
223
rootgroup = self.rootgroup
224
if verbose:
225
print '\n', '-=' * 30
226
print "Running %s.test01b_readTable..." % self.__class__.__name__
227
228
# Create an instance of an HDF5 Table
229
self.fileh = openFile(self.file, "r")
230
table = self.fileh.getNode("/table0")
231
232
# Choose a small value for buffer size
233
table._v_maxTuples = 3
234
# Read the records and select those with "var2" file less than 20
235
result = [ rec['var5'] for rec in table.iterrows()
236
if rec['var2'] < 20 ]
237
if verbose:
238
print "Nrows in", table._v_pathname, ":", table.nrows
239
print "Last record in table ==>", rec
240
print "rec['var5'] ==>", rec['var5'],
241
print "nrows ==>", table.row.nrow()
242
print "Total selected records in table ==> ", len(result)
243
nrows = table.row.nrow()
244
if isinstance(rec['var5'], NumArray):
245
assert allequal(result[0], array((float(0),)*4, Float32))
246
assert allequal(result[1], array((float(1),)*4, Float32))
247
assert allequal(result[2], array((float(2),)*4, Float32))
248
assert allequal(result[3], array((float(3),)*4, Float32))
249
assert allequal(result[10], array((float(10),)*4, Float32))
250
assert allequal(rec['var5'], array((float(nrows),)*4, Float32))
251
else:
252
assert rec['var5'] == float(nrows)
253
# Read the records and select those with "var2" file less than 20
254
result = [ rec['var10'] for rec in table.iterrows()
255
if rec['var2'] < 20 ]
256
if isinstance(rec['var10'], NumArray):
257
assert allequal(result[0], array([float(0)+0.j, 1.+float(0)*1j],
258
Complex64))
259
assert allequal(result[1], array([float(1)+0.j, 1.+float(1)*1j],
260
Complex64))
261
assert allequal(result[2], array([float(2)+0.j, 1.+float(2)*1j],
262
Complex64))
263
assert allequal(result[3], array([float(3)+0.j, 1.+float(3)*1j],
264
Complex64))
265
assert allequal(result[10], array([float(10)+0.j, 1.+float(10)*1j],
266
Complex64))
267
assert allequal(rec['var10'], array([float(nrows)+0.j,
268
1.+float(nrows)*1j],
269
Complex64))
270
else:
271
assert rec['var10'] == 1.+float(nrows)*1j
272
assert len(result) == 20
273
274
def test02_AppendRows(self):
275
"""Checking whether appending record rows works or not"""
276
277
# Now, open it, but in "append" mode
278
self.fileh = openFile(self.file, mode = "a")
279
self.rootgroup = self.fileh.root
280
if verbose:
281
print '\n', '-=' * 30
282
print "Running %s.test02_AppendRows..." % self.__class__.__name__
283
284
# Get a table
285
table = self.fileh.getNode("/group0/table1")
286
# Get their row object
287
row = table.row
288
if verbose:
289
print "Nrows in old", table._v_pathname, ":", table.nrows
290
print "Record Format ==>", table._v_fmt
291
print "Record Size ==>", table.rowsize
292
# Append some rows
293
for i in xrange(self.appendrows):
294
row['var1'] = '%04d' % (self.appendrows - i)
295
row['var7'] = row['var1'][-1]
296
row['var2'] = i
297
row['var3'] = i % self.maxshort
298
if isinstance(row['var4'], NumArray):
299
row['var4'] = [float(i), float(i*i)]
300
else:
301
row['var4'] = float(i)
302
if isinstance(row['var8'], NumArray):
303
row['var8'] = [0, 1]
304
else:
305
row['var8'] = 1
306
if isinstance(row['var9'], NumArray):
307
row['var9'] = [0.+float(i)*1j, float(i)+0.j]
308
else:
309
row['var9'] = float(i)+0.j
310
if isinstance(row['var10'], NumArray):
311
row['var10'] = [float(i)+0.j, 1.+float(i)*1j]
312
else:
313
row['var10'] = 1.+float(i)*1j
314
if isinstance(row['var5'], NumArray):
315
row['var5'] = array((float(i),)*4)
316
else:
317
row['var5'] = float(i)
318
row.append()
319
320
# Flush the buffer for this table and read it
321
table.flush()
322
result = [ row['var2'] for row in table.iterrows()
323
if row['var2'] < 20 ]
324
325
nrows = self.appendrows - 1
326
assert (row['var1'], row['var2'], row['var7']) == ("0001", nrows, "1")
327
if isinstance(row['var5'], NumArray):
328
assert allequal(row['var5'], array((float(nrows),)*4, Float32))
329
else:
330
assert row['var5'] == float(nrows)
331
if self.appendrows <= 20:
332
add = self.appendrows
333
else:
334
add = 20
335
assert len(result) == 20 + add # because we appended new rows
336
337
# CAVEAT: The next test only works for tables with rows < 2**15
338
def test03_endianess(self):
339
"""Checking if table is endianess aware"""
340
341
rootgroup = self.rootgroup
342
if verbose:
343
print '\n', '-=' * 30
344
print "Running %s.test03_endianess..." % self.__class__.__name__
345
346
# Create an instance of an HDF5 Table
347
self.fileh = openFile(self.file, "r")
348
table = self.fileh.getNode("/table0")
349
350
# Manually change the byteorder property for this table
351
table.byteorder = {"little":"big","big":"little"}[table.byteorder]
352
353
# Read the records and select the ones with "var6" column less than 20
354
result = [ rec['var2'] for rec in table.iterrows() if rec['var6'] < 20]
355
if verbose:
356
print "Nrows in", table._v_pathname, ":", table.nrows
357
print "Last record in table ==>", rec
358
print "Selected records ==>", result
359
print "Total selected records in table ==>", len(result)
360
nrows = self.expectedrows - 1
361
assert (rec['var1'], rec['var6']) == ("0001", nrows)
362
assert len(result) == 20
363
364
def test04_delete(self):
365
"""Checking whether a single row can be deleted"""
366
367
if verbose:
368
print '\n', '-=' * 30
369
print "Running %s.test04_delete..." % self.__class__.__name__
370
371
# Create an instance of an HDF5 Table
372
self.fileh = openFile(self.file, "a")
373
table = self.fileh.getNode("/table0")
374
375
# Read the records and select the ones with "var2" column less than 20
376
result = [ r['var2'] for r in table.iterrows() if r['var2'] < 20]
377
378
if verbose:
379
print "Nrows in", table._v_pathname, ":", table.nrows
380
print "Last selected value ==>", result[-1]
381
print "Total selected records in table ==>", len(result)
382
383
nrows = table.nrows
384
table._v_maxTuples = 3 # small value of the buffer
385
# Delete the twenty-th row
386
table.removeRows(19)
387
388
# Re-read the records
389
result2 = [ r['var2'] for r in table.iterrows() if r['var2'] < 20]
390
391
if verbose:
392
print "Nrows in", table._v_pathname, ":", table.nrows
393
print "Last selected value ==>", result2[-1]
394
print "Total selected records in table ==>", len(result2)
395
396
assert table.nrows == nrows - 1
397
assert table.shape == (nrows - 1,)
398
# Check that the new list is smaller than the original one
399
assert len(result) == len(result2) + 1
400
assert result[:-1] == result2
401
402
def test04b_delete(self):
403
"""Checking whether a range of rows can be deleted"""
404
405
if verbose:
406
print '\n', '-=' * 30
407
print "Running %s.test04b_delete..." % self.__class__.__name__
408
409
# Create an instance of an HDF5 Table
410
self.fileh = openFile(self.file, "a")
411
table = self.fileh.getNode("/table0")
412
413
# Read the records and select the ones with "var2" column less than 20
414
result = [ r['var2'] for r in table.iterrows() if r['var2'] < 20]
415
416
if verbose:
417
print "Nrows in", table._v_pathname, ":", table.nrows
418
print "Last selected value ==>", result[-1]
419
print "Total selected records in table ==>", len(result)
420
421
nrows = table.nrows
422
table._v_maxTuples = 4 # small value of the buffer
423
# Delete the last ten rows
424
table.removeRows(10, 20)
425
426
# Re-read the records
427
result2 = [ r['var2'] for r in table.iterrows() if r['var2'] < 20]
428
429
if verbose:
430
print "Nrows in", table._v_pathname, ":", table.nrows
431
print "Last selected value ==>", result2[-1]
432
print "Total selected records in table ==>", len(result2)
433
434
assert table.nrows == nrows - 10
435
assert table.shape == (nrows - 10,)
436
# Check that the new list is smaller than the original one
437
assert len(result) == len(result2) + 10
438
assert result[:10] == result2
439
440
def test04c_delete(self):
441
"""Checking whether removing a bad range of rows is detected"""
442
443
if verbose:
444
print '\n', '-=' * 30
445
print "Running %s.test04c_delete..." % self.__class__.__name__
446
447
# Create an instance of an HDF5 Table
448
self.fileh = openFile(self.file, "a")
449
table = self.fileh.getNode("/table0")
450
451
# Read the records and select the ones with "var2" column less than 20
452
result = [ r['var2'] for r in table.iterrows() if r['var2'] < 20]
453
454
nrows = table.nrows
455
table._v_maxTuples = 5 # small value of the buffer
456
# Delete a too large range of rows
457
table.removeRows(10, nrows + 100)
458
459
# Re-read the records
460
result2 = [ r['var2'] for r in table.iterrows() if r['var2'] < 20]
461
462
if verbose:
463
print "Nrows in", table._v_pathname, ":", table.nrows
464
print "Last selected value ==>", result2[-1]
465
print "Total selected records in table ==>", len(result2)
466
467
assert table.nrows == 10
468
assert table.shape == (10,)
469
# Check that the new list is smaller than the original one
470
assert len(result) == len(result2) + 10
471
assert result[:10] == result2
472
473
def test04d_delete(self):
474
"""Checking whether removing rows several times at once is working"""
475
476
if verbose:
477
print '\n', '-=' * 30
478
print "Running %s.test04d_delete..." % self.__class__.__name__
479
480
# Create an instance of an HDF5 Table
481
self.fileh = openFile(self.file, "a")
482
table = self.fileh.getNode("/table0")
483
484
# Read the records and select the ones with "var2" column less than 20
485
result = [ r['var2'] for r in table if r['var2'] < 20]
486
487
nrows = table.nrows
488
maxTuples = table._v_maxTuples
489
table._v_maxTuples = 6 # small value of the buffer
490
# Delete some rows
491
table.removeRows(10, 15)
492
# It's necessary to restore the value of buffer to use the row object
493
# afterwards...
494
table._v_maxTuples = maxTuples
495
496
# Append some rows
497
row = table.row
498
for i in xrange(10, 15):
499
row['var1'] = '%04d' % (self.appendrows - i)
500
# This line gives problems on Windows. Why?
501
#row['var7'] = row['var1'][-1]
502
row['var2'] = i
503
row['var3'] = i % self.maxshort
504
if isinstance(row['var4'], NumArray):
505
row['var4'] = [float(i), float(i*i)]
506
else:
507
row['var4'] = float(i)
508
if isinstance(row['var8'], NumArray):
509
row['var8'] = [0, 1]
510
else:
511
row['var8'] = 1
512
if isinstance(row['var9'], NumArray):
513
row['var9'] = [0.+float(i)*1j, float(i)+0.j]
514
else:
515
row['var9'] = float(i)+0.j
516
if isinstance(row['var10'], NumArray):
517
row['var10'] = [float(i)+0.j, 1.+float(i)*1j]
518
else:
519
row['var10'] = 1.+float(i)*1j
520
if isinstance(row['var5'], NumArray):
521
row['var5'] = array((float(i),)*4)
522
else:
523
row['var5'] = float(i)
524
row.append()
525
# Flush the buffer for this table
526
table.flush()
527
528
# Delete 5 rows more
529
table.removeRows(5, 10)
530
531
# Re-read the records
532
result2 = [ r['var2'] for r in table if r['var2'] < 20 ]
533
534
if verbose:
535
print "Nrows in", table._v_pathname, ":", table.nrows
536
print "Last selected value ==>", result2[-1]
537
print "Total selected records in table ==>", len(result2)
538
539
assert table.nrows == nrows - 5
540
assert table.shape == (nrows - 5,)
541
# Check that the new list is smaller than the original one
542
assert len(result) == len(result2) + 5
543
# The last values has to be equal
544
assert result[10:15] == result2[10:15]
545
546
def test05_filtersTable(self):
547
"""Checking tablefilters"""
548
549
rootgroup = self.rootgroup
550
if verbose:
551
print '\n', '-=' * 30
552
print "Running %s.test05_filtersTable..." % self.__class__.__name__
553
554
# Create an instance of an HDF5 Table
555
self.fileh = openFile(self.file, "r")
556
table = self.fileh.getNode("/table0")
557
558
# Check filters:
559
if self.compress <> table.filters.complevel and verbose:
560
print "Error in compress. Class:", self.__class__.__name__
561
print "self, table:", self.compress, table.filters.complevel
562
tinfo = whichLibVersion(self.complib)
563
if tinfo[0] == 0:
564
self.complib = "zlib"
565
assert table.filters.complib == self.complib
566
assert table.filters.complevel == self.compress
567
if self.shuffle <> table.filters.shuffle and verbose:
568
print "Error in shuffle. Class:", self.__class__.__name__
569
print "self, table:", self.shuffle, table.filters.shuffle
570
assert self.shuffle == table.filters.shuffle
571
if self.fletcher32 <> table.filters.fletcher32 and verbose:
572
print "Error in fletcher32. Class:", self.__class__.__name__
573
print "self, table:", self.fletcher32, table.filters.fletcher32
574
assert self.fletcher32 == table.filters.fletcher32
575
576
class BasicWriteTestCase(BasicTestCase):
577
title = "BasicWrite"
578
579
class OldRecordBasicWriteTestCase(BasicTestCase):
580
title = "OldRecordBasicWrite"
581
record = OldRecord
582
583
class DictWriteTestCase(BasicTestCase):
584
# This checks also unidimensional arrays as columns
585
title = "DictWrite"
586
record = RecordDescriptionDict
587
nrows = 21
588
maxTuples = 3 # Choose a small value for the buffer size
589
start = 0
590
stop = 10
591
step = 3
592
593
class RecArrayOneWriteTestCase(BasicTestCase):
594
title = "RecArrayOneWrite"
595
record=records.array(formats="a4,i4,i2,2f8,f4,i2,a1,b1,c8,c16",
596
names='var1,var2,var3,var4,var5,var6,var7,var8,var9,var10')
597
598
class RecArrayTwoWriteTestCase(BasicTestCase):
599
title = "RecArrayTwoWrite"
600
expectedrows = 100
601
recarrayinit = 1
602
recordtemplate=records.array(formats="a4,i4,i2,f8,f4,i2,a1,b1,2c8,c16",
603
names='var1,var2,var3,var4,var5,var6,var7,var8,var9,var10',
604
shape=1)
605
606
class RecArrayThreeWriteTestCase(BasicTestCase):
607
title = "RecArrayThreeWrite"
608
expectedrows = 100
609
recarrayinit = 1
610
recordtemplate=records.array(formats="a4,i4,i2,2f8,4f4,i2,a1,2b1,c8,c16",
611
names='var1,var2,var3,var4,var5,var6,var7,var8,var9,var10',
612
shape=1)
613
614
class CompressLZOTablesTestCase(BasicTestCase):
615
title = "CompressLZOTables"
616
compress = 1
617
complib = "lzo"
618
619
class CompressLZOShuffleTablesTestCase(BasicTestCase):
620
title = "CompressLZOTables"
621
compress = 1
622
shuffle = 1
623
complib = "lzo"
624
625
class CompressUCLTablesTestCase(BasicTestCase):
626
title = "CompressUCLTables"
627
compress = 1
628
complib = "ucl"
629
630
class CompressUCLShuffleTablesTestCase(BasicTestCase):
631
title = "CompressUCLTables"
632
compress = 1
633
shuffle = 1
634
complib = "ucl"
635
636
class CompressZLIBTablesTestCase(BasicTestCase):
637
title = "CompressOneTables"
638
compress = 1
639
complib = "zlib"
640
641
class CompressZLIBShuffleTablesTestCase(BasicTestCase):
642
title = "CompressOneTables"
643
compress = 1
644
shuffle = 1
645
complib = "zlib"
646
647
class Fletcher32TablesTestCase(BasicTestCase):
648
title = "Fletcher32Tables"
649
fletcher32 = 1
650
shuffle = 0
651
complib = "zlib"
652
653
class AllFiltersTablesTestCase(BasicTestCase):
654
title = "AllFiltersTables"
655
compress = 1
656
fletcher32 = 1
657
shuffle = 1
658
complib = "zlib"
659
660
class CompressTwoTablesTestCase(BasicTestCase):
661
title = "CompressTwoTables"
662
compress = 1
663
# This checks also unidimensional arrays as columns
664
record = RecordDescriptionDict
665
666
class BigTablesTestCase(BasicTestCase):
667
title = "BigTables"
668
# 10000 rows takes much more time than we can afford for tests
669
# reducing to 1000 would be more than enough
670
# F. Alted 2004-01-19
671
# Will be executed only in heavy mode
672
expectedrows = 10000
673
appendrows = 100
674
675
class BasicRangeTestCase(unittest.TestCase):
676
#file = "test.h5"
677
mode = "w"
678
title = "This is the table title"
679
record = Record
680
maxshort = 1 << 15
681
expectedrows = 100
682
compress = 0
683
shuffle = 1
684
# Default values
685
nrows = 20
686
maxTuples = 3 # Choose a small value for the buffer size
687
start = 1
688
stop = nrows
689
checkrecarray = 0
690
checkgetCol = 0
691
692
def setUp(self):
693
# Create an instance of an HDF5 Table
694
self.file = tempfile.mktemp(".h5")
695
self.fileh = openFile(self.file, self.mode)
696
self.rootgroup = self.fileh.root
697
self.populateFile()
698
self.fileh.close()
699
700
def populateFile(self):
701
group = self.rootgroup
702
for j in range(3):
703
# Create a table
704
filterprops = Filters(complevel = self.compress,
705
shuffle = self.shuffle)
706
table = self.fileh.createTable(group, 'table'+str(j), self.record,
707
title = self.title,
708
filters = filterprops,
709
expectedrows = self.expectedrows)
710
# Get the row object associated with the new table
711
row = table.row
712
713
# Fill the table
714
for i in xrange(self.expectedrows):
715
row['var1'] = '%04d' % (self.expectedrows - i)
716
row['var7'] = row['var1'][-1]
717
row['var2'] = i
718
row['var3'] = i % self.maxshort
719
if isinstance(row['var4'], NumArray):
720
row['var4'] = [float(i), float(i*i)]
721
else:
722
row['var4'] = float(i)
723
if isinstance(row['var5'], NumArray):
724
row['var5'] = array((float(i),)*4)
725
else:
726
row['var5'] = float(i)
727
# var6 will be like var3 but byteswaped
728
row['var6'] = ((row['var3'] >> 8) & 0xff) + \
729
((row['var3'] << 8) & 0xff00)
730
row.append()
731
732
# Flush the buffer for this table
733
table.flush()
734
# Create a new group (descendant of group)
735
group2 = self.fileh.createGroup(group, 'group'+str(j))
736
# Iterate over this new group (group2)
737
group = group2
738
739
740
def tearDown(self):
741
if self.fileh.isopen:
742
self.fileh.close()
743
os.remove(self.file)
744
745
#----------------------------------------
746
747
def check_range(self):
748
749
# Create an instance of an HDF5 Table
750
self.fileh = openFile(self.file, "r")
751
table = self.fileh.getNode("/table0")
752
753
table._v_maxTuples = self.maxTuples
754
r = slice(self.start, self.stop, self.step)
755
resrange = getIndices(r,table.nrows)
756
reslength = len(range(*resrange))
757
if self.checkrecarray:
758
recarray = table.read(self.start, self.stop, self.step)
759
result = []
760
for nrec in range(len(recarray)):
761
if recarray.field('var2')[nrec] < self.nrows:
762
result.append(recarray.field('var2')[nrec])
763
elif self.checkgetCol:
764
column = table.read(self.start, self.stop, self.step, 'var2')
765
result = []
766
for nrec in range(len(column)):
767
if column[nrec] < self.nrows:
768
result.append(column[nrec])
769
else:
770
result = [ rec['var2'] for rec in
771
table.iterrows(self.start, self.stop, self.step)
772
if rec['var2'] < self.nrows ]
773
774
if self.start < 0:
775
startr = self.expectedrows + self.start
776
else:
777
startr = self.start
778
779
if self.stop == None:
780
stopr = startr+1
781
elif self.stop < 0:
782
stopr = self.expectedrows + self.stop
783
else:
784
stopr = self.stop
785
786
if self.nrows < stopr:
787
stopr = self.nrows
788
789
if verbose:
790
print "Nrows in", table._v_pathname, ":", table.nrows
791
if reslength:
792
if self.checkrecarray:
793
print "Last record *read* in recarray ==>", recarray[-1]
794
elif self.checkgetCol:
795
print "Last value *read* in getCol ==>", column[-1]
796
else:
797
print "Last record *read* in table range ==>", rec
798
print "Total number of selected records ==>", len(result)
799
print "Selected records:\n", result
800
print "Selected records should look like:\n", \
801
range(startr, stopr, self.step)
802
print "start, stop, step ==>", self.start, self.stop, self.step
803
print "startr, stopr, step ==>", startr, stopr, self.step
804
805
assert result == range(startr, stopr, self.step)
806
if startr < stopr and not (self.checkrecarray or self.checkgetCol):
807
if self.nrows < self.expectedrows:
808
assert rec['var2'] == range(self.start, self.stop, self.step)[-1]
809
else:
810
assert rec['var2'] == range(startr, stopr, self.step)[-1]
811
812
# Close the file
813
self.fileh.close()
814
815
def test01_range(self):
816
"""Checking ranges in table iterators (case1)"""
817
818
if verbose:
819
print '\n', '-=' * 30
820
print "Running %s.test01_range..." % self.__class__.__name__
821
822
# Case where step < maxTuples < 2*step
823
self.nrows = 21
824
self.maxTuples = 3
825
self.start = 0
826
self.stop = self.expectedrows
827
self.step = 2
828
829
self.check_range()
830
831
def test02_range(self):
832
"""Checking ranges in table iterators (case2)"""
833
834
if verbose:
835
print '\n', '-=' * 30
836
print "Running %s.test02_range..." % self.__class__.__name__
837
838
# Case where step < maxTuples < 10*step
839
self.nrows = 21
840
self.maxTuples = 31
841
self.start = 11
842
self.stop = self.expectedrows
843
self.step = 3
844
845
self.check_range()
846
847
def test03_range(self):
848
"""Checking ranges in table iterators (case3)"""
849
850
if verbose:
851
print '\n', '-=' * 30
852
print "Running %s.test03_range..." % self.__class__.__name__
853
854
# Case where step < maxTuples < 1.1*step
855
self.nrows = self.expectedrows
856
self.maxTuples = 11 # Choose a small value for the buffer size
857
self.start = 0
858
self.stop = self.expectedrows
859
self.step = 10
860
861
self.check_range()
862
863
def test04_range(self):
864
"""Checking ranges in table iterators (case4)"""
865
866
if verbose:
867
print '\n', '-=' * 30
868
print "Running %s.test04_range..." % self.__class__.__name__
869
870
# Case where step == maxTuples
871
self.nrows = self.expectedrows
872
self.maxTuples = 11 # Choose a small value for the buffer size
873
self.start = 1
874
self.stop = self.expectedrows
875
self.step = 11
876
877
self.check_range()
878
879
def test05_range(self):
880
"""Checking ranges in table iterators (case5)"""
881
882
if verbose:
883
print '\n', '-=' * 30
884
print "Running %s.test05_range..." % self.__class__.__name__
885
886
# Case where step > 1.1*maxTuples
887
self.nrows = 21
888
self.maxTuples = 10 # Choose a small value for the buffer size
889
self.start = 1
890
self.stop = self.expectedrows
891
self.step = 11
892
893
self.check_range()
894
895
def test06_range(self):
896
"""Checking ranges in table iterators (case6)"""
897
898
if verbose:
899
print '\n', '-=' * 30
900
print "Running %s.test06_range..." % self.__class__.__name__
901
902
# Case where step > 3*maxTuples
903
self.nrows = 3
904
self.maxTuples = 3 # Choose a small value for the buffer size
905
self.start = 2
906
self.stop = self.expectedrows
907
self.step = 10
908
909
self.check_range()
910
911
def test07_range(self):
912
"""Checking ranges in table iterators (case7)"""
913
914
if verbose:
915
print '\n', '-=' * 30
916
print "Running %s.test07_range..." % self.__class__.__name__
917
918
# Case where start == stop
919
self.nrows = 2
920
self.maxTuples = 3 # Choose a small value for the buffer size
921
self.start = self.nrows
922
self.stop = self.nrows
923
self.step = 10
924
925
self.check_range()
926
927
def test08_range(self):
928
"""Checking ranges in table iterators (case8)"""
929
930
if verbose:
931
print '\n', '-=' * 30
932
print "Running %s.test08_range..." % self.__class__.__name__
933
934
# Case where start > stop
935
self.nrows = 2
936
self.maxTuples = 3 # Choose a small value for the buffer size
937
self.start = self.nrows + 1
938
self.stop = self.nrows
939
self.step = 1
940
941
self.check_range()
942
943
def test09_range(self):
944
"""Checking ranges in table iterators (case9)"""
945
946
if verbose:
947
print '\n', '-=' * 30
948
print "Running %s.test09_range..." % self.__class__.__name__
949
950
# Case where stop = None (last row)
951
self.nrows = 100
952
self.maxTuples = 3 # Choose a small value for the buffer size
953
self.start = 1
954
self.stop = None
955
self.step = 1
956
957
self.check_range()
958
959
def test10_range(self):
960
"""Checking ranges in table iterators (case10)"""
961
962
if verbose:
963
print '\n', '-=' * 30
964
print "Running %s.test10_range..." % self.__class__.__name__
965
966
# Case where start < 0 and stop = None (last row)
967
self.nrows = self.expectedrows
968
self.maxTuples = 5 # Choose a small value for the buffer size
969
self.start = -6
970
self.startr = self.expectedrows + self.start
971
self.stop = 0
972
self.stop = None
973
self.stopr = self.expectedrows
974
self.step = 2
975
976
self.check_range()
977
978
def test10a_range(self):
979
"""Checking ranges in table iterators (case10a)"""
980
981
if verbose:
982
print '\n', '-=' * 30
983
print "Running %s.test10a_range..." % self.__class__.__name__
984
985
# Case where start < 0 and stop = 0
986
self.nrows = self.expectedrows
987
self.maxTuples = 5 # Choose a small value for the buffer size
988
self.start = -6
989
self.startr = self.expectedrows + self.start
990
self.stop = 0
991
self.stopr = self.expectedrows
992
self.step = 2
993
994
self.check_range()
995
996
def test11_range(self):
997
"""Checking ranges in table iterators (case11)"""
998
999
if verbose:
1000
print '\n', '-=' * 30
1001
print "Running %s.test11_range..." % self.__class__.__name__
1002
1003
# Case where start < 0 and stop < 0
1004
self.nrows = self.expectedrows
1005
self.maxTuples = 5 # Choose a small value for the buffer size
1006
self.start = -6
1007
self.startr = self.expectedrows + self.start
1008
self.stop = -2
1009
self.stopr = self.expectedrows + self.stop
1010
self.step = 1
1011
1012
self.check_range()
1013
1014
def test12_range(self):
1015
"""Checking ranges in table iterators (case12)"""
1016
1017
if verbose:
1018
print '\n', '-=' * 30
1019
print "Running %s.test12_range..." % self.__class__.__name__
1020
1021
# Case where start < 0 and stop < 0 and start > stop
1022
self.nrows = self.expectedrows
1023
self.maxTuples = 5 # Choose a small value for the buffer size
1024
self.start = -1
1025
self.startr = self.expectedrows + self.start
1026
self.stop = -2
1027
self.stopr = self.expectedrows + self.stop
1028
self.step = 1
1029
1030
self.check_range()
1031
1032
def test13_range(self):
1033
"""Checking ranges in table iterators (case13)"""
1034
1035
if verbose:
1036
print '\n', '-=' * 30
1037
print "Running %s.test13_range..." % self.__class__.__name__
1038
1039
# Case where step < 0
1040
self.step = -11
1041
try:
1042
self.check_range()
1043
except ValueError:
1044
if verbose:
1045
(type, value, traceback) = sys.exc_info()
1046
print "\nGreat!, the next ValueError was catched!"
1047
print value
1048
self.fileh.close()
1049
else:
1050
print rec
1051
self.fail("expected a ValueError")
1052
1053
# Case where step == 0
1054
self.step = 0
1055
try:
1056
self.check_range()
1057
except ValueError:
1058
if verbose:
1059
(type, value, traceback) = sys.exc_info()
1060
print "\nGreat!, the next ValueError was catched!"
1061
print value
1062
self.fileh.close()
1063
else:
1064
print rec
1065
self.fail("expected a ValueError")
1066
1067
1068
class IterRangeTestCase(BasicRangeTestCase):
1069
pass
1070
1071
class RecArrayRangeTestCase(BasicRangeTestCase):
1072
checkrecarray = 1
1073
1074
class getColRangeTestCase(BasicRangeTestCase):
1075
checkgetCol = 1
1076
1077
def test01_nonexistentField(self):
1078
"""Checking non-existing Field in getCol method """
1079
1080
if verbose:
1081
print '\n', '-=' * 30
1082
print "Running %s.test01_nonexistentField..." % self.__class__.__name__
1083
1084
# Create an instance of an HDF5 Table
1085
self.fileh = openFile(self.file, "r")
1086
self.root = self.fileh.root
1087
table = self.fileh.getNode("/table0")
1088
1089
try:
1090
#column = table.read(field='non-existent-column')
1091
column = table['non-existent-column']
1092
except LookupError:
1093
if verbose:
1094
(type, value, traceback) = sys.exc_info()
1095
print "\nGreat!, the next LookupError was catched!"
1096
print value
1097
pass
1098
else:
1099
print rec
1100
self.fail("expected a LookupError")
1101
1102
1103
class getItemTestCase(unittest.TestCase):
1104
mode = "w"
1105
title = "This is the table title"
1106
record = Record
1107
maxshort = 1 << 15
1108
expectedrows = 100
1109
compress = 0
1110
shuffle = 1
1111
# Default values
1112
nrows = 20
1113
maxTuples = 3 # Choose a small value for the buffer size
1114
start = 1
1115
stop = nrows
1116
checkrecarray = 0
1117
checkgetCol = 0
1118
1119
def setUp(self):
1120
# Create an instance of an HDF5 Table
1121
self.file = tempfile.mktemp(".h5")
1122
self.fileh = openFile(self.file, self.mode)
1123
self.rootgroup = self.fileh.root
1124
self.populateFile()
1125
self.fileh.close()
1126
1127
def populateFile(self):
1128
group = self.rootgroup
1129
for j in range(3):
1130
# Create a table
1131
filterprops = Filters(complevel = self.compress,
1132
shuffle = self.shuffle)
1133
table = self.fileh.createTable(group, 'table'+str(j), self.record,
1134
title = self.title,
1135
filters = filterprops,
1136
expectedrows = self.expectedrows)
1137
# Get the row object associated with the new table
1138
row = table.row
1139
1140
# Fill the table
1141
for i in xrange(self.expectedrows):
1142
row['var1'] = '%04d' % (self.expectedrows - i)
1143
row['var7'] = row['var1'][-1]
1144
row['var2'] = i
1145
row['var3'] = i % self.maxshort
1146
if isinstance(row['var4'], NumArray):
1147
row['var4'] = [float(i), float(i*i)]
1148
else:
1149
row['var4'] = float(i)
1150
if isinstance(row['var5'], NumArray):
1151
row['var5'] = array((float(i),)*4)
1152
else:
1153
row['var5'] = float(i)
1154
# var6 will be like var3 but byteswaped
1155
row['var6'] = ((row['var3'] >> 8) & 0xff) + \
1156
((row['var3'] << 8) & 0xff00)
1157
row.append()
1158
1159
# Flush the buffer for this table
1160
table.flush()
1161
# Create a new group (descendant of group)
1162
group2 = self.fileh.createGroup(group, 'group'+str(j))
1163
# Iterate over this new group (group2)
1164
group = group2
1165
1166
1167
def tearDown(self):
1168
if self.fileh.isopen:
1169
self.fileh.close()
1170
os.remove(self.file)
1171
1172
#----------------------------------------
1173
1174
def test01a_singleItem(self):
1175
"""Checking __getitem__ method with single parameter """
1176
1177
if verbose:
1178
print '\n', '-=' * 30
1179
print "Running %s.test01a_singleItem..." % self.__class__.__name__
1180
1181
self.fileh = openFile(self.file, "r")
1182
table = self.fileh.root.table0
1183
result = table[2]
1184
#assert result.field("var2") == 2
1185
assert result[1] == 2
1186
result = table[25]
1187
#assert result.field("var2") == 25
1188
assert result[1] == 25
1189
result = table[self.expectedrows-1]
1190
#assert result.field("var2") == self.expectedrows - 1
1191
assert result[1] == self.expectedrows - 1
1192
1193
def test01b_singleItem(self):
1194
"""Checking __getitem__ method with single parameter (negative)"""
1195
1196
if verbose:
1197
print '\n', '-=' * 30
1198
print "Running %s.test01b_singleItem..." % self.__class__.__name__
1199
1200
self.fileh = openFile(self.file, "r")
1201
table = self.fileh.root.table0
1202
result = table[-5]
1203
#assert result.field("var2") == self.expectedrows - 5
1204
assert result[1] == self.expectedrows - 5
1205
result = table[-1]
1206
#assert result.field("var2") == self.expectedrows - 1
1207
assert result[1] == self.expectedrows - 1
1208
result = table[-self.expectedrows]
1209
#assert result.field("var2") == 0
1210
assert result[1] == 0
1211
1212
def test02_twoItems(self):
1213
"""Checking __getitem__ method with start, stop parameters """
1214
1215
if verbose:
1216
print '\n', '-=' * 30
1217
print "Running %s.test02_twoItem..." % self.__class__.__name__
1218
1219
self.fileh = openFile(self.file, "r")
1220
table = self.fileh.root.table0
1221
result = table[2:6]
1222
assert result.field("var2").tolist() == range(2,6)
1223
result = table[2:-6]
1224
assert result.field("var2").tolist() == range(2,self.expectedrows-6)
1225
result = table[2:]
1226
assert result.field("var2").tolist() == range(2,self.expectedrows)
1227
result = table[-2:]
1228
assert result.field("var2").tolist() == range(self.expectedrows-2,self.expectedrows)
1229
1230
def test03_threeItems(self):
1231
"""Checking __getitem__ method with start, stop, step parameters """
1232
1233
if verbose:
1234
print '\n', '-=' * 30
1235
print "Running %s.test03_threeItem..." % self.__class__.__name__
1236
1237
self.fileh = openFile(self.file, "r")
1238
table = self.fileh.root.table0
1239
result = table[2:6:3]
1240
assert result.field("var2").tolist() == range(2,6,3)
1241
result = table[2::3]
1242
assert result.field("var2").tolist() == range(2,self.expectedrows,3)
1243
result = table[:6:2]
1244
assert result.field("var2").tolist() == range(0,6,2)
1245
result = table[::]
1246
assert result.field("var2").tolist() == range(0,self.expectedrows,1)
1247
1248
def test04_negativeStep(self):
1249
"""Checking __getitem__ method with negative step parameter"""
1250
1251
if verbose:
1252
print '\n', '-=' * 30
1253
print "Running %s.test04_negativeStep..." % self.__class__.__name__
1254
1255
self.fileh = openFile(self.file, "r")
1256
table = self.fileh.root.table0
1257
try:
1258
table[2:3:-3]
1259
except ValueError:
1260
if verbose:
1261
(type, value, traceback) = sys.exc_info()
1262
print "\nGreat!, the next RuntimeError was catched!"
1263
print value
1264
else:
1265
self.fail("expected a RuntimeError")
1266
1267
1268
def test06a_singleItemCol(self):
1269
"""Checking __getitem__ method in Col with single parameter """
1270
1271
if verbose:
1272
print '\n', '-=' * 30
1273
print "Running %s.test06a_singleItemCol..." % self.__class__.__name__
1274
1275
self.fileh = openFile(self.file, "r")
1276
colvar2 = self.fileh.root.table0.cols.var2
1277
assert colvar2[2] == 2
1278
assert colvar2[25] == 25
1279
assert colvar2[self.expectedrows-1] == self.expectedrows - 1
1280
1281
def test06b_singleItemCol(self):
1282
"""Checking __getitem__ method in Col with single parameter (negative)"""
1283
1284
if verbose:
1285
print '\n', '-=' * 30
1286
print "Running %s.test06b_singleItem..." % self.__class__.__name__
1287
1288
self.fileh = openFile(self.file, "r")
1289
colvar2 = self.fileh.root.table0.cols.var2
1290
assert colvar2[-5] == self.expectedrows - 5
1291
assert colvar2[-1] == self.expectedrows - 1
1292
assert colvar2[-self.expectedrows] == 0
1293
1294
def test07_twoItemsCol(self):
1295
"""Checking __getitem__ method in Col with start, stop parameters """
1296
1297
if verbose:
1298
print '\n', '-=' * 30
1299
print "Running %s.test07_twoItemCol..." % self.__class__.__name__
1300
1301
self.fileh = openFile(self.file, "r")
1302
colvar2 = self.fileh.root.table0.cols.var2
1303
assert colvar2[2:6].tolist() == range(2,6)
1304
assert colvar2[2:-6].tolist() == range(2,self.expectedrows-6)
1305
assert colvar2[2:].tolist() == range(2,self.expectedrows)
1306
assert colvar2[-2:].tolist() == range(self.expectedrows-2,self.expectedrows)
1307
1308
def test08_threeItemsCol(self):
1309
"""Checking __getitem__ method in Col with start, stop, step parameters """
1310
1311
if verbose:
1312
print '\n', '-=' * 30
1313
print "Running %s.test08_threeItemCol..." % self.__class__.__name__
1314
1315
self.fileh = openFile(self.file, "r")
1316
colvar2 = self.fileh.root.table0.cols.var2
1317
assert colvar2[2:6:3].tolist() == range(2,6,3)
1318
assert colvar2[2::3].tolist() == range(2,self.expectedrows,3)
1319
assert colvar2[:6:2].tolist() == range(0,6,2)
1320
assert colvar2[::].tolist() == range(0,self.expectedrows,1)
1321
1322
def test09_negativeStep(self):
1323
"""Checking __getitem__ method in Col with negative step parameter"""
1324
1325
if verbose:
1326
print '\n', '-=' * 30
1327
print "Running %s.test09_negativeStep..." % self.__class__.__name__
1328
1329
self.fileh = openFile(self.file, "r")
1330
colvar2 = self.fileh.root.table0.cols.var2
1331
try:
1332
colvar2[2:3:-3]
1333
except ValueError:
1334
if verbose:
1335
(type, value, traceback) = sys.exc_info()
1336
print "\nGreat!, the next RuntimeError was catched!"
1337
print value
1338
else:
1339
self.fail("expected a RuntimeError")
1340
1341
1342
class Rec(IsDescription):
1343
col1 = IntCol(pos=1)
1344
col2 = StringCol(3, pos=2)
1345
col3 = FloatCol(pos=3)
1346
1347
class setItem(unittest.TestCase):
1348
1349
def test01(self):
1350
"Checking modifying one table row with __setitem__"
1351
1352
if verbose:
1353
print '\n', '-=' * 30
1354
print "Running %s.test01..." % self.__class__.__name__
1355
1356
file = tempfile.mktemp(".h5")
1357
fileh = openFile(file, "w")
1358
1359
# Create a new table:
1360
table = fileh.createTable(fileh.root, 'recarray', Rec)
1361
table._v_maxTuples = self.buffersize # set buffer value
1362
1363
# append new rows
1364
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1365
table.append(r)
1366
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1367
1368
# Modify just one existing row
1369
table[2] = [456,'db2',1.2]
1370
# Create the modified recarray
1371
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
1372
[456,'db2',1.2],[5,'de1',1.3]],
1373
formats="i4,a3,f8",
1374
names = "col1,col2,col3")
1375
# Read the modified table
1376
if self.reopen:
1377
fileh.close()
1378
fileh = openFile(file, "r")
1379
table = fileh.root.recarray
1380
table._v_maxTuples = self.buffersize # set buffer value
1381
r2 = table.read()
1382
if verbose:
1383
print "Original table-->", repr(r2)
1384
print "Should look like-->", repr(r1)
1385
assert r1.tostring() == r2.tostring()
1386
assert table.nrows == 4
1387
1388
fileh.close()
1389
os.remove(file)
1390
1391
def test02(self):
1392
"Modifying one row, with a step (__setitem__)"
1393
1394
if verbose:
1395
print '\n', '-=' * 30
1396
print "Running %s.test02..." % self.__class__.__name__
1397
1398
file = tempfile.mktemp(".h5")
1399
fileh = openFile(file, "w")
1400
1401
# Create a new table:
1402
table = fileh.createTable(fileh.root, 'recarray', Rec)
1403
table._v_maxTuples = self.buffersize # set buffer value
1404
1405
# append new rows
1406
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1407
table.append(r)
1408
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1409
1410
# Modify two existing rows
1411
rows = records.array([[457,'db1',1.2],[6,'de2',1.3]],
1412
formats="i4,a3,f8")
1413
table[1:3:2] = rows
1414
# Create the modified recarray
1415
r1=records.array([[456,'dbe',1.2],[457,'db1',1.2],
1416
[457,'db1',1.2],[5,'de1',1.3]],
1417
formats="i4,a3,f8",
1418
names = "col1,col2,col3")
1419
# Read the modified table
1420
if self.reopen:
1421
fileh.close()
1422
fileh = openFile(file, "r")
1423
table = fileh.root.recarray
1424
table._v_maxTuples = self.buffersize # set buffer value
1425
r2 = table.read()
1426
if verbose:
1427
print "Original table-->", repr(r2)
1428
print "Should look like-->", repr(r1)
1429
assert r1.tostring() == r2.tostring()
1430
assert table.nrows == 4
1431
1432
fileh.close()
1433
os.remove(file)
1434
1435
def test03(self):
1436
"Checking modifying several rows at once (__setitem__)"
1437
1438
if verbose:
1439
print '\n', '-=' * 30
1440
print "Running %s.test03..." % self.__class__.__name__
1441
1442
file = tempfile.mktemp(".h5")
1443
fileh = openFile(file, "w")
1444
1445
# Create a new table:
1446
table = fileh.createTable(fileh.root, 'recarray', Rec)
1447
table._v_maxTuples = self.buffersize # set buffer value
1448
1449
# append new rows
1450
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1451
table.append(r)
1452
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1453
1454
# Modify two existing rows
1455
rows = records.array([[457,'db1',1.2],[5,'de1',1.3]],
1456
formats="i4,a3,f8")
1457
#table.modifyRows(start=1, rows=rows)
1458
table[1:3] = rows
1459
# Create the modified recarray
1460
r1=records.array([[456,'dbe',1.2],[457,'db1',1.2],
1461
[5,'de1',1.3],[5,'de1',1.3]],
1462
formats="i4,a3,f8",
1463
names = "col1,col2,col3")
1464
# Read the modified table
1465
if self.reopen:
1466
fileh.close()
1467
fileh = openFile(file, "r")
1468
table = fileh.root.recarray
1469
table._v_maxTuples = self.buffersize # set buffer value
1470
r2 = table.read()
1471
if verbose:
1472
print "Original table-->", repr(r2)
1473
print "Should look like-->", repr(r1)
1474
assert r1.tostring() == r2.tostring()
1475
assert table.nrows == 4
1476
1477
fileh.close()
1478
os.remove(file)
1479
1480
def test04(self):
1481
"Modifying several rows at once, with a step (__setitem__)"
1482
1483
if verbose:
1484
print '\n', '-=' * 30
1485
print "Running %s.test04..." % self.__class__.__name__
1486
1487
file = tempfile.mktemp(".h5")
1488
fileh = openFile(file, "w")
1489
1490
# Create a new table:
1491
table = fileh.createTable(fileh.root, 'recarray', Rec)
1492
table._v_maxTuples = self.buffersize # set buffer value
1493
1494
# append new rows
1495
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1496
table.append(r)
1497
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1498
1499
# Modify two existing rows
1500
rows = records.array([[457,'db1',1.2],[6,'de2',1.3]],
1501
formats="i4,a3,f8")
1502
#table[1:4:2] = rows
1503
table[1::2] = rows
1504
# Create the modified recarray
1505
r1=records.array([[456,'dbe',1.2],[457,'db1',1.2],
1506
[457,'db1',1.2],[6,'de2',1.3]],
1507
formats="i4,a3,f8",
1508
names = "col1,col2,col3")
1509
# Read the modified table
1510
if self.reopen:
1511
fileh.close()
1512
fileh = openFile(file, "r")
1513
table = fileh.root.recarray
1514
table._v_maxTuples = self.buffersize # set buffer value
1515
r2 = table.read()
1516
if verbose:
1517
print "Original table-->", repr(r2)
1518
print "Should look like-->", repr(r1)
1519
assert r1.tostring() == r2.tostring()
1520
assert table.nrows == 4
1521
1522
fileh.close()
1523
os.remove(file)
1524
1525
def test05(self):
1526
"Checking modifying one column (single element, __setitem__)"
1527
1528
if verbose:
1529
print '\n', '-=' * 30
1530
print "Running %s.test05..." % self.__class__.__name__
1531
1532
file = tempfile.mktemp(".h5")
1533
fileh = openFile(file, "w")
1534
1535
# Create a new table:
1536
table = fileh.createTable(fileh.root, 'recarray', Rec)
1537
table._v_maxTuples = self.buffersize # set buffer value
1538
1539
# append new rows
1540
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1541
table.append(r)
1542
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1543
1544
# Modify just one existing column
1545
table.cols.col1[1] = -1
1546
# Create the modified recarray
1547
r1=records.array([[456,'dbe',1.2],[-1,'ded',1.3],
1548
[457,'db1',1.2],[5,'de1',1.3]],
1549
formats="i4,a3,f8",
1550
names = "col1,col2,col3")
1551
# Read the modified table
1552
if self.reopen:
1553
fileh.close()
1554
fileh = openFile(file, "r")
1555
table = fileh.root.recarray
1556
table._v_maxTuples = self.buffersize # set buffer value
1557
r2 = table.read()
1558
if verbose:
1559
print "Original table-->", repr(r2)
1560
print "Should look like-->", repr(r1)
1561
assert r1.tostring() == r2.tostring()
1562
assert table.nrows == 4
1563
1564
fileh.close()
1565
os.remove(file)
1566
1567
def test06(self):
1568
"Checking modifying one column (several elements, __setitem__)"
1569
1570
if verbose:
1571
print '\n', '-=' * 30
1572
print "Running %s.test06..." % self.__class__.__name__
1573
1574
file = tempfile.mktemp(".h5")
1575
fileh = openFile(file, "w")
1576
1577
# Create a new table:
1578
table = fileh.createTable(fileh.root, 'recarray', Rec)
1579
table._v_maxTuples = self.buffersize # set buffer value
1580
1581
# append new rows
1582
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1583
table.append(r)
1584
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1585
1586
# Modify just one existing column
1587
table.cols.col1[1:4] = [2,3,4]
1588
# Create the modified recarray
1589
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
1590
[3,'db1',1.2],[4,'de1',1.3]],
1591
formats="i4,a3,f8",
1592
names = "col1,col2,col3")
1593
# Read the modified table
1594
if self.reopen:
1595
fileh.close()
1596
fileh = openFile(file, "r")
1597
table = fileh.root.recarray
1598
table._v_maxTuples = self.buffersize # set buffer value
1599
r2 = table.read()
1600
if verbose:
1601
print "Original table-->", repr(r2)
1602
print "Should look like-->", repr(r1)
1603
assert r1.tostring() == r2.tostring()
1604
assert table.nrows == 4
1605
1606
fileh.close()
1607
os.remove(file)
1608
1609
def test07(self):
1610
"Modifying one column (several elements, __setitem__, step)"
1611
1612
if verbose:
1613
print '\n', '-=' * 30
1614
print "Running %s.test07..." % self.__class__.__name__
1615
1616
file = tempfile.mktemp(".h5")
1617
fileh = openFile(file, "w")
1618
1619
# Create a new table:
1620
table = fileh.createTable(fileh.root, 'recarray', Rec)
1621
table._v_maxTuples = self.buffersize # set buffer value
1622
1623
# append new rows
1624
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1625
table.append(r)
1626
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1627
1628
# Modify just one existing column
1629
table.cols.col1[1:4:2] = [2,3]
1630
# Create the modified recarray
1631
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
1632
[457,'db1',1.2],[3,'de1',1.3]],
1633
formats="i4,a3,f8",
1634
names = "col1,col2,col3")
1635
# Read the modified table
1636
if self.reopen:
1637
fileh.close()
1638
fileh = openFile(file, "r")
1639
table = fileh.root.recarray
1640
table._v_maxTuples = self.buffersize # set buffer value
1641
r2 = table.read()
1642
if verbose:
1643
print "Original table-->", repr(r2)
1644
print "Should look like-->", repr(r1)
1645
assert r1.tostring() == r2.tostring()
1646
assert table.nrows == 4
1647
1648
fileh.close()
1649
os.remove(file)
1650
1651
def test08(self):
1652
"Modifying one column (one element, __setitem__, step)"
1653
1654
if verbose:
1655
print '\n', '-=' * 30
1656
print "Running %s.test08..." % self.__class__.__name__
1657
1658
file = tempfile.mktemp(".h5")
1659
fileh = openFile(file, "w")
1660
1661
# Create a new table:
1662
table = fileh.createTable(fileh.root, 'recarray', Rec)
1663
table._v_maxTuples = self.buffersize # set buffer value
1664
1665
# append new rows
1666
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1667
table.append(r)
1668
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1669
1670
# Modify just one existing column
1671
table.cols.col1[1:4:3] = [2]
1672
# Create the modified recarray
1673
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
1674
[457,'db1',1.2],[5,'de1',1.3]],
1675
formats="i4,a3,f8",
1676
names = "col1,col2,col3")
1677
# Read the modified table
1678
if self.reopen:
1679
fileh.close()
1680
fileh = openFile(file, "r")
1681
table = fileh.root.recarray
1682
table._v_maxTuples = self.buffersize # set buffer value
1683
r2 = table.read()
1684
if verbose:
1685
print "Original table-->", repr(r2)
1686
print "Should look like-->", repr(r1)
1687
assert r1.tostring() == r2.tostring()
1688
assert table.nrows == 4
1689
1690
fileh.close()
1691
os.remove(file)
1692
1693
def test09(self):
1694
"Modifying beyond the table extend (__setitem__, step)"
1695
1696
if verbose:
1697
print '\n', '-=' * 30
1698
print "Running %s.test09..." % self.__class__.__name__
1699
1700
file = tempfile.mktemp(".h5")
1701
fileh = openFile(file, "w")
1702
1703
# Create a new table:
1704
table = fileh.createTable(fileh.root, 'recarray', Rec)
1705
table._v_maxTuples = self.buffersize # set buffer value
1706
1707
# append new rows
1708
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1709
table.append(r)
1710
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1711
1712
# Try to modify beyond the extend
1713
# This will silently exclude the non-fitting rows
1714
rows = records.array([[457,'db1',1.2],[6,'de2',1.3],[457,'db1',1.2]],
1715
formats="i4,a3,f8")
1716
table[1:6:2] = rows
1717
# How it should look like
1718
r1 = records.array([[456,'dbe',1.2],[457,'db1',1.2],
1719
[457,'db1',1.2],[6,'de2',1.3]],
1720
formats="i4,a3,f8")
1721
1722
# Read the modified table
1723
if self.reopen:
1724
fileh.close()
1725
fileh = openFile(file, "r")
1726
table = fileh.root.recarray
1727
table._v_maxTuples = self.buffersize # set buffer value
1728
r2 = table.read()
1729
if verbose:
1730
print "Original table-->", repr(r2)
1731
print "Should look like-->", repr(r1)
1732
assert r1.tostring() == r2.tostring()
1733
assert table.nrows == 4
1734
1735
fileh.close()
1736
os.remove(file)
1737
1738
1739
class setItem1(setItem):
1740
reopen=0
1741
buffersize = 1
1742
1743
class setItem2(setItem):
1744
reopen=1
1745
buffersize = 2
1746
1747
class setItem3(setItem):
1748
reopen=1
1749
buffersize = 1000
1750
1751
class RecArrayIO(unittest.TestCase):
1752
1753
def test00(self):
1754
"Checking saving a regular recarray"
1755
1756
if verbose:
1757
print '\n', '-=' * 30
1758
print "Running %s.test00..." % self.__class__.__name__
1759
1760
file = tempfile.mktemp(".h5")
1761
fileh = openFile(file, "w")
1762
1763
# Create a recarray
1764
r=records.array([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
1765
1766
# Save it in a table:
1767
fileh.createTable(fileh.root, 'recarray', r)
1768
1769
# Read it again
1770
if self.reopen:
1771
fileh.close()
1772
fileh = openFile(file, "r")
1773
r2 = fileh.root.recarray.read()
1774
assert r.tostring() == r2.tostring()
1775
1776
fileh.close()
1777
os.remove(file)
1778
1779
def test01(self):
1780
"Checking saving a recarray with an offset in its buffer"
1781
1782
if verbose:
1783
print '\n', '-=' * 30
1784
print "Running %s.test01..." % self.__class__.__name__
1785
1786
file = tempfile.mktemp(".h5")
1787
fileh = openFile(file, "w")
1788
1789
# Create a recarray
1790
r=records.array([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
1791
1792
# Get an offsetted bytearray
1793
r1 = r[1:]
1794
assert r1._byteoffset > 0
1795
1796
# Save it in a table:
1797
fileh.createTable(fileh.root, 'recarray', r1)
1798
1799
# Read it again
1800
if self.reopen:
1801
fileh.close()
1802
fileh = openFile(file, "r")
1803
r2 = fileh.root.recarray.read()
1804
1805
assert r1.tostring() == r2.tostring()
1806
1807
fileh.close()
1808
os.remove(file)
1809
1810
def test02(self):
1811
"Checking saving a large recarray with an offset in its buffer"
1812
1813
if verbose:
1814
print '\n', '-=' * 30
1815
print "Running %s.test02..." % self.__class__.__name__
1816
1817
file = tempfile.mktemp(".h5")
1818
fileh = openFile(file, "w")
1819
1820
# Create a recarray
1821
r=records.array('a'*200000,'f4,3i4,a5,i2',3000)
1822
1823
# Get an offsetted bytearray
1824
r1 = r[2000:]
1825
assert r1._byteoffset > 0
1826
1827
# Save it in a table:
1828
fileh.createTable(fileh.root, 'recarray', r1)
1829
1830
# Read it again
1831
if self.reopen:
1832
fileh.close()
1833
fileh = openFile(file, "r")
1834
r2 = fileh.root.recarray.read()
1835
1836
assert r1.tostring() == r2.tostring()
1837
1838
fileh.close()
1839
os.remove(file)
1840
1841
def test03(self):
1842
"Checking saving a strided recarray with an offset in its buffer"
1843
1844
if verbose:
1845
print '\n', '-=' * 30
1846
print "Running %s.test03..." % self.__class__.__name__
1847
1848
file = tempfile.mktemp(".h5")
1849
fileh = openFile(file, "w")
1850
1851
# Create a recarray
1852
r=records.array('a'*200000,'f4,3i4,a5,i2',3000)
1853
1854
# Get an strided recarray
1855
r2 = r[::2]
1856
1857
# Get an offsetted bytearray
1858
r1 = r2[1200:]
1859
assert r1._byteoffset > 0
1860
1861
# Save it in a table:
1862
fileh.createTable(fileh.root, 'recarray', r1)
1863
1864
# Read it again
1865
if self.reopen:
1866
fileh.close()
1867
fileh = openFile(file, "r")
1868
r2 = fileh.root.recarray.read()
1869
1870
assert r1.tostring() == r2.tostring()
1871
1872
fileh.close()
1873
os.remove(file)
1874
1875
def test04(self):
1876
"Checking appending several rows at once"
1877
1878
if verbose:
1879
print '\n', '-=' * 30
1880
print "Running %s.test04..." % self.__class__.__name__
1881
1882
file = tempfile.mktemp(".h5")
1883
fileh = openFile(file, "w")
1884
1885
class Rec(IsDescription):
1886
col1 = IntCol(pos=1)
1887
col2 = StringCol(3, pos=2)
1888
col3 = FloatCol(pos=3)
1889
1890
# Save it in a table:
1891
table = fileh.createTable(fileh.root, 'recarray', Rec)
1892
1893
# append new rows
1894
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1895
table.append(r)
1896
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1897
# Create the complete table
1898
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
1899
[457,'db1',1.2],[5,'de1',1.3]],
1900
formats="i4,a3,f8",
1901
names = "col1,col2,col3")
1902
# Read the original table
1903
if self.reopen:
1904
fileh.close()
1905
fileh = openFile(file, "r")
1906
table = fileh.root.recarray
1907
r2 = fileh.root.recarray.read()
1908
if verbose:
1909
print "Original table-->", repr(r2)
1910
print "Should look like-->", repr(r1)
1911
assert r1.tostring() == r2.tostring()
1912
assert table.nrows == 4
1913
1914
fileh.close()
1915
os.remove(file)
1916
1917
def test05(self):
1918
"Checking appending several rows at once (close file version)"
1919
1920
if verbose:
1921
print '\n', '-=' * 30
1922
print "Running %s.test05..." % self.__class__.__name__
1923
1924
file = tempfile.mktemp(".h5")
1925
fileh = openFile(file, "w")
1926
1927
# Save it in a table:
1928
table = fileh.createTable(fileh.root, 'recarray', Rec)
1929
1930
# append new rows
1931
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1932
table.append(r)
1933
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1934
1935
fileh.close()
1936
fileh = openFile(file, "r")
1937
table = fileh.root.recarray
1938
1939
# Create the complete table
1940
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
1941
[457,'db1',1.2],[5,'de1',1.3]],
1942
formats="i4,a3,f8",
1943
names = "col1,col2,col3")
1944
# Read the original table
1945
if self.reopen:
1946
fileh.close()
1947
fileh = openFile(file, "r")
1948
table = fileh.root.recarray
1949
r2 = fileh.root.recarray.read()
1950
if verbose:
1951
print "Original table-->", repr(r2)
1952
print "Should look like-->", repr(r1)
1953
assert r1.tostring() == r2.tostring()
1954
assert table.nrows == 4
1955
1956
fileh.close()
1957
os.remove(file)
1958
1959
def test06a(self):
1960
"Checking modifying one table row (list version)"
1961
1962
if verbose:
1963
print '\n', '-=' * 30
1964
print "Running %s.test06a..." % self.__class__.__name__
1965
1966
file = tempfile.mktemp(".h5")
1967
fileh = openFile(file, "w")
1968
1969
# Create a new table:
1970
table = fileh.createTable(fileh.root, 'recarray', Rec)
1971
1972
# append new rows
1973
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
1974
table.append(r)
1975
table.append([[457,'db1',1.2],[5,'de1',1.3]])
1976
1977
# Modify just one existing rows
1978
table.modifyRows(start=1, rows=[[456,'db1',1.2]])
1979
# Create the modified recarray
1980
r1=records.array([[456,'dbe',1.2],[456,'db1',1.2],
1981
[457,'db1',1.2],[5,'de1',1.3]],
1982
formats="i4,a3,f8",
1983
names = "col1,col2,col3")
1984
# Read the modified table
1985
if self.reopen:
1986
fileh.close()
1987
fileh = openFile(file, "r")
1988
table = fileh.root.recarray
1989
r2 = table.read()
1990
if verbose:
1991
print "Original table-->", repr(r2)
1992
print "Should look like-->", repr(r1)
1993
assert r1.tostring() == r2.tostring()
1994
assert table.nrows == 4
1995
1996
fileh.close()
1997
os.remove(file)
1998
1999
def test06b(self):
2000
"Checking modifying one table row (recarray version)"
2001
2002
if verbose:
2003
print '\n', '-=' * 30
2004
print "Running %s.test06b..." % self.__class__.__name__
2005
2006
file = tempfile.mktemp(".h5")
2007
fileh = openFile(file, "w")
2008
2009
# Create a new table:
2010
table = fileh.createTable(fileh.root, 'recarray', Rec)
2011
2012
# append new rows
2013
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2014
table.append(r)
2015
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2016
2017
# Modify just one existing rows
2018
table.modifyRows(start=2, rows=records.array([[456,'db2',1.2]],
2019
formats="i4,a3,f8"))
2020
# Create the modified recarray
2021
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
2022
[456,'db2',1.2],[5,'de1',1.3]],
2023
formats="i4,a3,f8",
2024
names = "col1,col2,col3")
2025
# Read the modified table
2026
if self.reopen:
2027
fileh.close()
2028
fileh = openFile(file, "r")
2029
table = fileh.root.recarray
2030
r2 = table.read()
2031
if verbose:
2032
print "Original table-->", repr(r2)
2033
print "Should look like-->", repr(r1)
2034
assert r1.tostring() == r2.tostring()
2035
assert table.nrows == 4
2036
2037
fileh.close()
2038
os.remove(file)
2039
2040
def test07a(self):
2041
"Checking modifying several rows at once (list version)"
2042
2043
if verbose:
2044
print '\n', '-=' * 30
2045
print "Running %s.test07a..." % self.__class__.__name__
2046
2047
file = tempfile.mktemp(".h5")
2048
fileh = openFile(file, "w")
2049
2050
# Create a new table:
2051
table = fileh.createTable(fileh.root, 'recarray', Rec)
2052
2053
# append new rows
2054
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2055
table.append(r)
2056
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2057
2058
# Modify two existing rows
2059
table.modifyRows(start=1, rows=[[457,'db1',1.2],[5,'de1',1.3]])
2060
# Create the modified recarray
2061
r1=records.array([[456,'dbe',1.2],[457,'db1',1.2],
2062
[5,'de1',1.3],[5,'de1',1.3]],
2063
formats="i4,a3,f8",
2064
names = "col1,col2,col3")
2065
# Read the modified table
2066
if self.reopen:
2067
fileh.close()
2068
fileh = openFile(file, "r")
2069
table = fileh.root.recarray
2070
r2 = table.read()
2071
if verbose:
2072
print "Original table-->", repr(r2)
2073
print "Should look like-->", repr(r1)
2074
assert r1.tostring() == r2.tostring()
2075
assert table.nrows == 4
2076
2077
fileh.close()
2078
os.remove(file)
2079
2080
def test07b(self):
2081
"Checking modifying several rows at once (recarray version)"
2082
2083
if verbose:
2084
print '\n', '-=' * 30
2085
print "Running %s.test07b..." % self.__class__.__name__
2086
2087
file = tempfile.mktemp(".h5")
2088
fileh = openFile(file, "w")
2089
2090
# Create a new table:
2091
table = fileh.createTable(fileh.root, 'recarray', Rec)
2092
2093
# append new rows
2094
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2095
table.append(r)
2096
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2097
2098
# Modify two existing rows
2099
rows = records.array([[457,'db1',1.2],[5,'de1',1.3]],
2100
formats="i4,a3,f8")
2101
table.modifyRows(start=1, rows=rows)
2102
# Create the modified recarray
2103
r1=records.array([[456,'dbe',1.2],[457,'db1',1.2],
2104
[5,'de1',1.3],[5,'de1',1.3]],
2105
formats="i4,a3,f8",
2106
names = "col1,col2,col3")
2107
# Read the modified table
2108
if self.reopen:
2109
fileh.close()
2110
fileh = openFile(file, "r")
2111
table = fileh.root.recarray
2112
r2 = table.read()
2113
if verbose:
2114
print "Original table-->", repr(r2)
2115
print "Should look like-->", repr(r1)
2116
assert r1.tostring() == r2.tostring()
2117
assert table.nrows == 4
2118
2119
fileh.close()
2120
os.remove(file)
2121
2122
def test08a(self):
2123
"Checking modifying one column (single column version)"
2124
2125
if verbose:
2126
print '\n', '-=' * 30
2127
print "Running %s.test08a..." % self.__class__.__name__
2128
2129
file = tempfile.mktemp(".h5")
2130
fileh = openFile(file, "w")
2131
2132
# Create a new table:
2133
table = fileh.createTable(fileh.root, 'recarray', Rec)
2134
2135
# append new rows
2136
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2137
table.append(r)
2138
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2139
2140
# Modify just one existing column
2141
table.modifyColumns(start=1, columns=[[2,3,4]], names=["col1"])
2142
# Create the modified recarray
2143
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
2144
[3,'db1',1.2],[4,'de1',1.3]],
2145
formats="i4,a3,f8",
2146
names = "col1,col2,col3")
2147
# Read the modified table
2148
if self.reopen:
2149
fileh.close()
2150
fileh = openFile(file, "r")
2151
table = fileh.root.recarray
2152
r2 = table.read()
2153
if verbose:
2154
print "Original table-->", repr(r2)
2155
print "Should look like-->", repr(r1)
2156
assert r1.tostring() == r2.tostring()
2157
assert table.nrows == 4
2158
2159
fileh.close()
2160
os.remove(file)
2161
2162
def test08b(self):
2163
"Checking modifying one column (single column version, recarray)"
2164
2165
if verbose:
2166
print '\n', '-=' * 30
2167
print "Running %s.test08b..." % self.__class__.__name__
2168
2169
file = tempfile.mktemp(".h5")
2170
fileh = openFile(file, "w")
2171
2172
# Create a new table:
2173
table = fileh.createTable(fileh.root, 'recarray', Rec)
2174
2175
# append new rows
2176
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2177
table.append(r)
2178
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2179
2180
# Modify just one existing column
2181
columns = records.fromarrays([[2,3,4]], formats="i4")
2182
table.modifyColumns(start=1, columns=columns, names=["col1"])
2183
#table.cols.col1[1:] = [2,3,4]
2184
# Create the modified recarray
2185
r1=records.array([[456,'dbe',1.2],[2,'ded',1.3],
2186
[3,'db1',1.2],[4,'de1',1.3]],
2187
formats="i4,a3,f8",
2188
names = "col1,col2,col3")
2189
# Read the modified table
2190
if self.reopen:
2191
fileh.close()
2192
fileh = openFile(file, "r")
2193
table = fileh.root.recarray
2194
r2 = table.read()
2195
if verbose:
2196
print "Original table-->", repr(r2)
2197
print "Should look like-->", repr(r1)
2198
assert r1.tostring() == r2.tostring()
2199
assert table.nrows == 4
2200
2201
fileh.close()
2202
os.remove(file)
2203
2204
def test08c(self):
2205
"Checking modifying one column (single column version, single element)"
2206
2207
if verbose:
2208
print '\n', '-=' * 30
2209
print "Running %s.test08c..." % self.__class__.__name__
2210
2211
file = tempfile.mktemp(".h5")
2212
fileh = openFile(file, "w")
2213
2214
# Create a new table:
2215
table = fileh.createTable(fileh.root, 'recarray', Rec)
2216
2217
# append new rows
2218
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2219
table.append(r)
2220
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2221
2222
# Modify just one existing column
2223
columns = records.fromarrays([[4]], formats="i4")
2224
#table.modifyColumns(start=1, columns=columns, names=["col1"])
2225
table.modifyColumns(start=1, columns=[[4]], names=["col1"])
2226
# Create the modified recarray
2227
r1=records.array([[456,'dbe',1.2],[4,'ded',1.3],
2228
[457,'db1',1.2],[5,'de1',1.3]],
2229
formats="i4,a3,f8",
2230
names = "col1,col2,col3")
2231
# Read the modified table
2232
if self.reopen:
2233
fileh.close()
2234
fileh = openFile(file, "r")
2235
table = fileh.root.recarray
2236
r2 = table.read()
2237
if verbose:
2238
print "Original table-->", repr(r2)
2239
print "Should look like-->", repr(r1)
2240
assert r1.tostring() == r2.tostring()
2241
assert table.nrows == 4
2242
2243
fileh.close()
2244
os.remove(file)
2245
2246
def test09a(self):
2247
"Checking modifying table columns (multiple column version)"
2248
2249
if verbose:
2250
print '\n', '-=' * 30
2251
print "Running %s.test09a..." % self.__class__.__name__
2252
2253
file = tempfile.mktemp(".h5")
2254
fileh = openFile(file, "w")
2255
2256
# Create a new table:
2257
table = fileh.createTable(fileh.root, 'recarray', Rec)
2258
2259
# append new rows
2260
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2261
table.append(r)
2262
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2263
2264
# Modify a couple of columns
2265
columns = [["aaa","bbb","ccc"], [1.2, .1, .3]]
2266
table.modifyColumns(start=1, columns=columns, names=["col2", "col3"])
2267
# Create the modified recarray
2268
r1=records.array([[456,'dbe',1.2],[2,'aaa',1.2],
2269
[457,'bbb',.1],[5,'ccc',.3]],
2270
formats="i4,a3,f8",
2271
names = "col1,col2,col3")
2272
2273
# Read the modified table
2274
if self.reopen:
2275
fileh.close()
2276
fileh = openFile(file, "r")
2277
table = fileh.root.recarray
2278
r2 = table.read()
2279
if verbose:
2280
print "Original table-->", repr(r2)
2281
print "Should look like-->", repr(r1)
2282
assert r1.tostring() == r2.tostring()
2283
assert table.nrows == 4
2284
2285
fileh.close()
2286
os.remove(file)
2287
2288
def test09b(self):
2289
"Checking modifying table columns (multiple columns, recarray)"
2290
2291
if verbose:
2292
print '\n', '-=' * 30
2293
print "Running %s.test09b..." % self.__class__.__name__
2294
2295
file = tempfile.mktemp(".h5")
2296
fileh = openFile(file, "w")
2297
2298
# Create a new table:
2299
table = fileh.createTable(fileh.root, 'recarray', Rec)
2300
2301
# append new rows
2302
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2303
table.append(r)
2304
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2305
2306
# Modify a couple of columns
2307
columns = records.array([["aaa",1.2],["bbb", .1], ["ccc", .3]],
2308
formats="a3,f8")
2309
table.modifyColumns(start=1, columns=columns, names=["col2", "col3"])
2310
# Create the modified recarray
2311
r1=records.array([[456,'dbe',1.2],[2,'aaa',1.2],
2312
[457,'bbb',.1],[5,'ccc',.3]],
2313
formats="i4,a3,f8",
2314
names = "col1,col2,col3")
2315
# Read the modified table
2316
if self.reopen:
2317
fileh.close()
2318
fileh = openFile(file, "r")
2319
table = fileh.root.recarray
2320
r2 = table.read()
2321
if verbose:
2322
print "Original table-->", repr(r2)
2323
print "Should look like-->", repr(r1)
2324
assert r1.tostring() == r2.tostring()
2325
assert table.nrows == 4
2326
2327
fileh.close()
2328
os.remove(file)
2329
2330
def test09c(self):
2331
"Checking modifying table columns (single column, step)"
2332
2333
if verbose:
2334
print '\n', '-=' * 30
2335
print "Running %s.test09c..." % self.__class__.__name__
2336
2337
file = tempfile.mktemp(".h5")
2338
fileh = openFile(file, "w")
2339
2340
# Create a new table:
2341
table = fileh.createTable(fileh.root, 'recarray', Rec)
2342
2343
# append new rows
2344
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2345
table.append(r)
2346
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2347
2348
# Modify a couple of columns
2349
columns = records.array([["aaa",1.2],["bbb", .1]],
2350
formats="a3,f8")
2351
table.modifyColumns(start=1, step=2, columns=columns,
2352
names=["col2", "col3"])
2353
# Create the modified recarray
2354
r1=records.array([[456,'dbe',1.2],[2,'aaa',1.2],
2355
[457,'db1',1.2],[5,'bbb',.1]],
2356
formats="i4,a3,f8",
2357
names = "col1,col2,col3")
2358
# Read the modified table
2359
if self.reopen:
2360
fileh.close()
2361
fileh = openFile(file, "r")
2362
table = fileh.root.recarray
2363
r2 = table.read()
2364
if verbose:
2365
print "Original table-->", repr(r2)
2366
print "Should look like-->", repr(r1)
2367
assert r1.tostring() == r2.tostring()
2368
assert table.nrows == 4
2369
2370
fileh.close()
2371
os.remove(file)
2372
2373
def test09d(self):
2374
"Checking modifying table columns (multiple columns, step)"
2375
2376
if verbose:
2377
print '\n', '-=' * 30
2378
print "Running %s.test09d..." % self.__class__.__name__
2379
2380
file = tempfile.mktemp(".h5")
2381
fileh = openFile(file, "w")
2382
2383
# Create a new table:
2384
table = fileh.createTable(fileh.root, 'recarray', Rec)
2385
2386
# append new rows
2387
r=records.array([[456,'dbe',1.2],[2,'ded',1.3]], formats="i4,a3,f8")
2388
table.append(r)
2389
table.append([[457,'db1',1.2],[5,'de1',1.3]])
2390
2391
# Modify a couple of columns
2392
columns = records.array([["aaa",1.3],["bbb", .1]],
2393
formats="a3,f8")
2394
table.modifyColumns(start=0, step=2, columns=columns,
2395
names=["col2", "col3"])
2396
# Create the modified recarray
2397
r1=records.array([[456,'aaa',1.3],[2,'ded',1.3],
2398
[457,'bbb',.1],[5,'de1',1.3]],
2399
formats="i4,a3,f8",
2400
names = "col1,col2,col3")
2401
# Read the modified table
2402
if self.reopen:
2403
fileh.close()
2404
fileh = openFile(file, "r")
2405
table = fileh.root.recarray
2406
r2 = table.read()
2407
if verbose:
2408
print "Original table-->", repr(r2)
2409
print "Should look like-->", repr(r1)
2410
assert r1.tostring() == r2.tostring()
2411
assert table.nrows == 4
2412
2413
fileh.close()
2414
os.remove(file)
2415
2416
2417
class RecArrayIO1(RecArrayIO):
2418
reopen=0
2419
2420
class RecArrayIO2(RecArrayIO):
2421
reopen=1
2422
2423
class CopyTestCase(unittest.TestCase):
2424
2425
def test01_copy(self):
2426
"""Checking Table.copy() method """
2427
2428
if verbose:
2429
print '\n', '-=' * 30
2430
print "Running %s.test01_copy..." % self.__class__.__name__
2431
2432
# Create an instance of an HDF5 Table
2433
file = tempfile.mktemp(".h5")
2434
fileh = openFile(file, "w")
2435
2436
# Create a recarray
2437
r=records.array([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
2438
# Save it in a table:
2439
table1 = fileh.createTable(fileh.root, 'table1', r, "title table1")
2440
2441
if self.close:
2442
if verbose:
2443
print "(closing file version)"
2444
fileh.close()
2445
fileh = openFile(file, mode = "a")
2446
table1 = fileh.root.table1
2447
2448
# Copy to another table
2449
table2, size = table1.copy('/', 'table2')
2450
2451
if self.close:
2452
if verbose:
2453
print "(closing file version)"
2454
fileh.close()
2455
fileh = openFile(file, mode = "a")
2456
table1 = fileh.root.table1
2457
table2 = fileh.root.table2
2458
2459
if verbose:
2460
print "table1-->", table1.read()
2461
print "table2-->", table2.read()
2462
#print "dirs-->", dir(table1), dir(table2)
2463
print "attrs table1-->", repr(table1.attrs)
2464
print "attrs table2-->", repr(table2.attrs)
2465
2466
# Check that all the elements are equal
2467
for row1 in table1:
2468
nrow = row1.nrow() # current row
2469
# row1 is a Row instance, while table2[] is a
2470
# RecArray.Record instance
2471
#print "reprs-->", repr(row1), repr(table2.read(nrow))
2472
for colname in table1.colnames:
2473
# Both ways to compare works well
2474
#assert row1[colname] == table2[nrow].field(colname)
2475
assert row1[colname] == table2.read(nrow, field=colname)[0]
2476
2477
# Assert other properties in table
2478
assert table1.nrows == table2.nrows
2479
assert table1.shape == table2.shape
2480
assert table1.colnames == table2.colnames
2481
assert table1.coltypes == table2.coltypes
2482
assert table1.colshapes == table2.colshapes
2483
assert repr(table1.description) == repr(table2.description)
2484
2485
# This could be not the same when re-opening the file
2486
#assert table1.description._v_ColObjects == table2.description._v_ColObjects
2487
# Leaf attributes
2488
assert table1.title == table2.title
2489
assert table1.filters.complevel == table2.filters.complevel
2490
assert table1.filters.complib == table2.filters.complib
2491
assert table1.filters.shuffle == table2.filters.shuffle
2492
assert table1.filters.fletcher32 == table2.filters.fletcher32
2493
2494
# Close the file
2495
fileh.close()
2496
os.remove(file)
2497
2498
def test02_copy(self):
2499
"""Checking Table.copy() method (where specified)"""
2500
2501
if verbose:
2502
print '\n', '-=' * 30
2503
print "Running %s.test02_copy..." % self.__class__.__name__
2504
2505
# Create an instance of an HDF5 Table
2506
file = tempfile.mktemp(".h5")
2507
fileh = openFile(file, "w")
2508
2509
# Create a recarray
2510
r=records.array([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
2511
# Save it in a table:
2512
table1 = fileh.createTable(fileh.root, 'table1', r, "title table1")
2513
2514
if self.close:
2515
if verbose:
2516
print "(closing file version)"
2517
fileh.close()
2518
fileh = openFile(file, mode = "a")
2519
table1 = fileh.root.table1
2520
2521
# Copy to another table in another group
2522
group1 = fileh.createGroup("/", "group1")
2523
table2, size = table1.copy(group1, 'table2')
2524
2525
if self.close:
2526
if verbose:
2527
print "(closing file version)"
2528
fileh.close()
2529
fileh = openFile(file, mode = "r")
2530
table1 = fileh.root.table1
2531
table2 = fileh.root.group1.table2
2532
2533
if verbose:
2534
print "table1-->", table1.read()
2535
print "table2-->", table2.read()
2536
print "attrs table1-->", repr(table1.attrs)
2537
print "attrs table2-->", repr(table2.attrs)
2538
2539
# Check that all the elements are equal
2540
for row1 in table1:
2541
nrow = row1.nrow() # current row
2542
for colname in table1.colnames:
2543
# Both ways to compare works well
2544
#assert row1[colname] == table2[nrow].field(colname)
2545
assert row1[colname] == table2.read(nrow, field=colname)[0]
2546
2547
# Assert other properties in table
2548
assert table1.nrows == table2.nrows
2549
assert table1.shape == table2.shape
2550
assert table1.colnames == table2.colnames
2551
assert table1.coltypes == table2.coltypes
2552
assert table1.colshapes == table2.colshapes
2553
assert repr(table1.description) == repr(table2.description)
2554
2555
# Leaf attributes
2556
assert table1.title == table2.title
2557
assert table1.filters.complevel == table2.filters.complevel
2558
assert table1.filters.complib == table2.filters.complib
2559
assert table1.filters.shuffle == table2.filters.shuffle
2560
assert table1.filters.fletcher32 == table2.filters.fletcher32
2561
2562
# Close the file
2563
fileh.close()
2564
os.remove(file)
2565
2566
def test03_copy(self):
2567
"""Checking Table.copy() method (table larger than buffer)"""
2568
2569
if verbose:
2570
print '\n', '-=' * 30
2571
print "Running %s.test03_copy..." % self.__class__.__name__
2572
2573
# Create an instance of an HDF5 Table
2574
file = tempfile.mktemp(".h5")
2575
fileh = openFile(file, "w")
2576
2577
# Create a recarray exceeding buffers capability
2578
# This works, but takes too much CPU for a test
2579
# It is better to reduce the buffer size (table1._v_maxTuples)
2580
# r=records.array('aaaabbbbccccddddeeeeffffgggg'*20000,
2581
# formats='2i2,i4, (2,3)u2, (1,)f4, f8',shape=700)
2582
r=records.array('aaaabbbbccccddddeeeeffffgggg'*200,
2583
formats='2i2,i4, (2,3)u2, (1,)f4, f8',shape=7)
2584
# Save it in a table:
2585
table1 = fileh.createTable(fileh.root, 'table1', r, "title table1")
2586
2587
if self.close:
2588
if verbose:
2589
print "(closing file version)"
2590
fileh.close()
2591
fileh = openFile(file, mode = "a")
2592
table1 = fileh.root.table1
2593
2594
# Copy to another table in another group and other title
2595
group1 = fileh.createGroup("/", "group1")
2596
table1._v_maxTuples = 2 # small value of buffer
2597
table2, size = table1.copy(group1, 'table2', title="title table2")
2598
if self.close:
2599
if verbose:
2600
print "(closing file version)"
2601
fileh.close()
2602
fileh = openFile(file, mode = "r")
2603
table1 = fileh.root.table1
2604
table2 = fileh.root.group1.table2
2605
2606
if verbose:
2607
print "table1-->", table1.read()
2608
print "table2-->", table2.read()
2609
print "attrs table1-->", repr(table1.attrs)
2610
print "attrs table2-->", repr(table2.attrs)
2611
2612
# Check that all the elements are equal
2613
for row1 in table1:
2614
nrow = row1.nrow() # current row
2615
for colname in table1.colnames:
2616
#assert allequal(row1[colname], table2[nrow].field(colname))
2617
assert allequal(row1[colname], table2.read(nrow, field=colname)[0])
2618
2619
# Assert other properties in table
2620
assert table1.nrows == table2.nrows
2621
assert table1.shape == table2.shape
2622
assert table1.colnames == table2.colnames
2623
assert table1.coltypes == table2.coltypes
2624
assert table1.colshapes == table2.colshapes
2625
assert repr(table1.description) == repr(table2.description)
2626
2627
# Leaf attributes
2628
assert "title table2" == table2.title
2629
assert table1.filters.complevel == table2.filters.complevel
2630
assert table1.filters.complib == table2.filters.complib
2631
assert table1.filters.shuffle == table2.filters.shuffle
2632
assert table1.filters.fletcher32 == table2.filters.fletcher32
2633
2634
# Close the file
2635
fileh.close()
2636
os.remove(file)
2637
2638
def test04_copy(self):
2639
"""Checking Table.copy() method (different compress level)"""
2640
2641
if verbose:
2642
print '\n', '-=' * 30
2643
print "Running %s.test04_copy..." % self.__class__.__name__
2644
2645
# Create an instance of an HDF5 Table
2646
file = tempfile.mktemp(".h5")
2647
fileh = openFile(file, "w")
2648
2649
# Create a recarray
2650
r=records.array([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
2651
# Save it in a table:
2652
table1 = fileh.createTable(fileh.root, 'table1', r, "title table1")
2653
2654
if self.close:
2655
if verbose:
2656
print "(closing file version)"
2657
fileh.close()
2658
fileh = openFile(file, mode = "a")
2659
table1 = fileh.root.table1
2660
2661
# Copy to another table in another group
2662
group1 = fileh.createGroup("/", "group1")
2663
table2, size = table1.copy(group1, 'table2',
2664
filters=Filters(complevel=6))
2665
2666
if self.close:
2667
if verbose:
2668
print "(closing file version)"
2669
fileh.close()
2670
fileh = openFile(file, mode = "r")
2671
table1 = fileh.root.table1
2672
table2 = fileh.root.group1.table2
2673
2674
if verbose:
2675
print "table1-->", table1.read()
2676
print "table2-->", table2.read()
2677
print "attrs table1-->", repr(table1.attrs)
2678
print "attrs table2-->", repr(table2.attrs)
2679
2680
# Check that all the elements are equal
2681
for row1 in table1:
2682
nrow = row1.nrow() # current row
2683
for colname in table1.colnames:
2684
# Both ways to compare works well
2685
#assert row1[colname] == table2[nrow].field(colname)
2686
assert row1[colname] == table2.read(nrow, field=colname)[0]
2687
2688
# Assert other properties in table
2689
assert table1.nrows == table2.nrows
2690
assert table1.shape == table2.shape
2691
assert table1.colnames == table2.colnames
2692
assert table1.coltypes == table2.coltypes
2693
assert table1.colshapes == table2.colshapes
2694
assert repr(table1.description) == repr(table2.description)
2695
2696
# Leaf attributes
2697
assert table1.title == table2.title
2698
assert 6 == table2.filters.complevel
2699
assert table1.filters.complib == table2.filters.complib
2700
assert 1 == table2.filters.shuffle
2701
assert table1.filters.fletcher32 == table2.filters.fletcher32
2702
2703
# Close the file
2704
fileh.close()
2705
os.remove(file)
2706
2707
def test05_copy(self):
2708
"""Checking Table.copy() method (user attributes copied)"""
2709
2710
if verbose:
2711
print '\n', '-=' * 30
2712
print "Running %s.test05_copy..." % self.__class__.__name__
2713
2714
# Create an instance of an HDF5 Table
2715
file = tempfile.mktemp(".h5")
2716
fileh = openFile(file, "w")
2717
2718
# Create a recarray
2719
r=records.array([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
2720
# Save it in a table:
2721
table1 = fileh.createTable(fileh.root, 'table1', r, "title table1")
2722
# Add some user attributes
2723
table1.attrs.attr1 = "attr1"
2724
table1.attrs.attr2 = 2
2725
2726
if self.close:
2727
if verbose:
2728
print "(closing file version)"
2729
fileh.close()
2730
fileh = openFile(file, mode = "a")
2731
table1 = fileh.root.table1
2732
2733
# Copy to another table in another group
2734
group1 = fileh.createGroup("/", "group1")
2735
table2, size = table1.copy(group1, 'table2',
2736
copyuserattrs=1,
2737
filters=Filters(complevel=6))
2738
2739
if self.close:
2740
if verbose:
2741
print "(closing file version)"
2742
fileh.close()
2743
fileh = openFile(file, mode = "r")
2744
table1 = fileh.root.table1
2745
table2 = fileh.root.group1.table2
2746
2747
if verbose:
2748
print "table1-->", table1.read()
2749
print "table2-->", table2.read()
2750
print "attrs table1-->", repr(table1.attrs)
2751
print "attrs table2-->", repr(table2.attrs)
2752
2753
# Check that all the elements are equal
2754
for row1 in table1:
2755
nrow = row1.nrow() # current row
2756
for colname in table1.colnames:
2757
#assert row1[colname] == table2[nrow].field(colname)
2758
assert row1[colname] == table2.read(nrow, field=colname)[0]
2759
2760
# Assert other properties in table
2761
assert table1.nrows == table2.nrows
2762
assert table1.shape == table2.shape
2763
assert table1.colnames == table2.colnames
2764
assert table1.coltypes == table2.coltypes
2765
assert table1.colshapes == table2.colshapes
2766
assert repr(table1.description) == repr(table2.description)
2767
2768
# Leaf attributes
2769
assert table1.title == table2.title
2770
assert 6 == table2.filters.complevel
2771
assert table1.filters.complib == table2.filters.complib
2772
assert 1 == table2.filters.shuffle
2773
assert table1.filters.fletcher32 == table2.filters.fletcher32
2774
# User attributes
2775
table2.attrs.attr1 == "attr1"
2776
table2.attrs.attr2 == 2
2777
2778
# Close the file
2779
fileh.close()
2780
os.remove(file)
2781
2782
def test05b_copy(self):
2783
"""Checking Table.copy() method (user attributes not copied)"""
2784
2785
if verbose:
2786
print '\n', '-=' * 30
2787
print "Running %s.test05b_copy..." % self.__class__.__name__
2788
2789
# Create an instance of an HDF5 Table
2790
file = tempfile.mktemp(".h5")
2791
fileh = openFile(file, "w")
2792
2793
# Create a recarray
2794
r=records.array([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
2795
# Save it in a table:
2796
table1 = fileh.createTable(fileh.root, 'table1', r, "title table1")
2797
# Add some user attributes
2798
table1.attrs.attr1 = "attr1"
2799
table1.attrs.attr2 = 2
2800
2801
if self.close:
2802
if verbose:
2803
print "(closing file version)"
2804
fileh.close()
2805
fileh = openFile(file, mode = "a")
2806
table1 = fileh.root.table1
2807
2808
# Copy to another table in another group
2809
group1 = fileh.createGroup("/", "group1")
2810
table2, size = table1.copy(group1, 'table2',
2811
copyuserattrs=0,
2812
filters=Filters(complevel=6))
2813
2814
if self.close:
2815
if verbose:
2816
print "(closing file version)"
2817
fileh.close()
2818
fileh = openFile(file, mode = "r")
2819
table1 = fileh.root.table1
2820
table2 = fileh.root.group1.table2
2821
2822
if verbose:
2823
print "table1-->", table1.read()
2824
print "table2-->", table2.read()
2825
print "attrs table1-->", repr(table1.attrs)
2826
print "attrs table2-->", repr(table2.attrs)
2827
2828
# Check that all the elements are equal
2829
for row1 in table1:
2830
nrow = row1.nrow() # current row
2831
for colname in table1.colnames:
2832
#assert row1[colname] == table2[nrow].field(colname)
2833
assert row1[colname] == table2.read(nrow, field=colname)[0]
2834
2835
# Assert other properties in table
2836
assert table1.nrows == table2.nrows
2837
assert table1.shape == table2.shape
2838
assert table1.colnames == table2.colnames
2839
assert table1.coltypes == table2.coltypes
2840
assert table1.colshapes == table2.colshapes
2841
assert repr(table1.description) == repr(table2.description)
2842
2843
# Leaf attributes
2844
assert table1.title == table2.title
2845
assert 6 == table2.filters.complevel
2846
assert table1.filters.complib == table2.filters.complib
2847
assert 1 == table2.filters.shuffle
2848
assert table1.filters.fletcher32 == table2.filters.fletcher32
2849
# User attributes
2850
# table2.attrs.attr1 == None
2851
# table2.attrs.attr2 == None
2852
hasattr(table2.attrs, "attr1") == 0
2853
hasattr(table2.attrs, "attr2") == 0
2854
2855
# Close the file
2856
fileh.close()
2857
os.remove(file)
2858
2859
class CloseCopyTestCase(CopyTestCase):
2860
close = 1
2861
2862
class OpenCopyTestCase(CopyTestCase):
2863
close = 0
2864
2865
class CopyIndexTestCase(unittest.TestCase):
2866
2867
def test01_index(self):
2868
"""Checking Table.copy() method with indexes"""
2869
2870
if verbose:
2871
print '\n', '-=' * 30
2872
print "Running %s.test01_index..." % self.__class__.__name__
2873
2874
# Create an instance of an HDF5 Table
2875
file = tempfile.mktemp(".h5")
2876
fileh = openFile(file, "w")
2877
2878
# Create a recarray exceeding buffers capability
2879
r=records.array('aaaabbbbccccddddeeeeffffgggg'*200,
2880
#formats='2i2,i4, (2,3)u2, (1,)f4, f8',shape=10)
2881
# to avoid an ugly warning
2882
formats='2i2,i4, (2,3)u2, f4, f8',shape=10)
2883
# Save it in a table:
2884
table1 = fileh.createTable(fileh.root, 'table1', r, "title table1")
2885
2886
if self.close:
2887
if verbose:
2888
print "(closing file version)"
2889
fileh.close()
2890
fileh = openFile(file, mode = "a")
2891
table1 = fileh.root.table1
2892
2893
# Copy to another table
2894
table1._v_maxTuples = self.maxTuples
2895
table2, size = table1.copy("/", 'table2',
2896
start=self.start,
2897
stop=self.stop,
2898
step=self.step)
2899
if verbose:
2900
print "table1-->", table1.read()
2901
print "table2-->", table2.read()
2902
print "attrs table1-->", repr(table1.attrs)
2903
print "attrs table2-->", repr(table2.attrs)
2904
2905
# Check that all the elements are equal
2906
r2 = r[self.start:self.stop:self.step]
2907
for nrow in range(r2.shape[0]):
2908
icol = 0
2909
for colname in table1.colnames:
2910
assert allequal(r2[nrow].field(colname),
2911
table2[nrow][icol])
2912
icol += 1
2913
# for colname in table1.colnames:
2914
# assert allequal(r2[nrow].field(colname),
2915
# table2[nrow].field(colname))
2916
2917
# Assert the number of rows in table
2918
if verbose:
2919
print "nrows in table2-->", table2.nrows
2920
print "and it should be-->", r2.shape[0]
2921
assert r2.shape[0] == table2.nrows
2922
2923
# Close the file
2924
fileh.close()
2925
os.remove(file)
2926
2927
def test02_indexclosef(self):
2928
"""Checking Table.copy() method with indexes (close file version)"""
2929
2930
if verbose:
2931
print '\n', '-=' * 30
2932
print "Running %s.test02_indexclosef..." % self.__class__.__name__
2933
2934
# Create an instance of an HDF5 Table
2935
file = tempfile.mktemp(".h5")
2936
fileh = openFile(file, "w")
2937
2938
# Create a recarray exceeding buffers capability
2939
r=records.array('aaaabbbbccccddddeeeeffffgggg'*200,
2940
formats='2i2,i4, (2,3)u2, f4, f8',shape=10)
2941
# Save it in a table:
2942
table1 = fileh.createTable(fileh.root, 'table1', r, "title table1")
2943
2944
if self.close:
2945
if verbose:
2946
print "(closing file version)"
2947
fileh.close()
2948
fileh = openFile(file, mode = "a")
2949
table1 = fileh.root.table1
2950
2951
# Copy to another table
2952
table1._v_maxTuples = self.maxTuples
2953
table2, size = table1.copy("/", 'table2',
2954
start=self.start,
2955
stop=self.stop,
2956
step=self.step)
2957
2958
fileh.close()
2959
fileh = openFile(file, mode = "r")
2960
table1 = fileh.root.table1
2961
table2 = fileh.root.table2
2962
2963
if verbose:
2964
print "table1-->", table1.read()
2965
print "table2-->", table2.read()
2966
print "attrs table1-->", repr(table1.attrs)
2967
print "attrs table2-->", repr(table2.attrs)
2968
2969
# Check that all the elements are equal
2970
r2 = r[self.start:self.stop:self.step]
2971
for nrow in range(r2.shape[0]):
2972
icol = 0
2973
for colname in table1.colnames:
2974
assert allequal(r2[nrow].field(colname),
2975
table2[nrow][icol])
2976
icol += 1
2977
# for colname in table1.colnames:
2978
# assert allequal(r2[nrow].field(colname),
2979
# table2[nrow].field(colname))
2980
2981
# Assert the number of rows in table
2982
if verbose:
2983
print "nrows in table2-->", table2.nrows
2984
print "and it should be-->", r2.shape[0]
2985
assert r2.shape[0] == table2.nrows
2986
2987
# Close the file
2988
fileh.close()
2989
os.remove(file)
2990
2991
class CopyIndex1TestCase(CopyIndexTestCase):
2992
maxTuples = 2
2993
close = 1
2994
start = 0
2995
stop = 7
2996
step = 1
2997
2998
class CopyIndex2TestCase(CopyIndexTestCase):
2999
maxTuples = 2
3000
close = 0
3001
start = 0
3002
stop = -1
3003
step = 1
3004
3005
class CopyIndex3TestCase(CopyIndexTestCase):
3006
maxTuples = 3
3007
close = 1
3008
start = 1
3009
stop = 7
3010
step = 1
3011
3012
class CopyIndex4TestCase(CopyIndexTestCase):
3013
maxTuples = 4
3014
close = 0
3015
start = 0
3016
stop = 6
3017
step = 1
3018
3019
class CopyIndex5TestCase(CopyIndexTestCase):
3020
maxTuples = 2
3021
close = 1
3022
start = 3
3023
stop = 7
3024
step = 1
3025
3026
class CopyIndex6TestCase(CopyIndexTestCase):
3027
maxTuples = 2
3028
close = 0
3029
start = 3
3030
stop = 6
3031
step = 2
3032
3033
class CopyIndex7TestCase(CopyIndexTestCase):
3034
maxTuples = 2
3035
close = 1
3036
start = 0
3037
stop = 7
3038
step = 10
3039
3040
class CopyIndex8TestCase(CopyIndexTestCase):
3041
maxTuples = 2
3042
close = 0
3043
start = 6
3044
stop = 3
3045
step = 1
3046
3047
class CopyIndex9TestCase(CopyIndexTestCase):
3048
maxTuples = 2
3049
close = 1
3050
start = 3
3051
stop = 4
3052
step = 1
3053
3054
class CopyIndex10TestCase(CopyIndexTestCase):
3055
maxTuples = 1
3056
close = 0
3057
start = 3
3058
stop = 4
3059
step = 2
3060
3061
class CopyIndex11TestCase(CopyIndexTestCase):
3062
maxTuples = 2
3063
close = 1
3064
start = -3
3065
stop = -1
3066
step = 2
3067
3068
class CopyIndex12TestCase(CopyIndexTestCase):
3069
maxTuples = 3
3070
close = 0
3071
start = -1 # Should point to the last element
3072
stop = None # None should mean the last element (including it)
3073
step = 1
3074
3075
class LargeRowSize(unittest.TestCase):
3076
3077
def test00(self):
3078
"Checking saving a Table with a moderately large rowsize"
3079
file = tempfile.mktemp(".h5")
3080
fileh = openFile(file, "w")
3081
3082
# Create a recarray
3083
r=records.array([[arange(100)]*2])
3084
3085
# Save it in a table:
3086
fileh.createTable(fileh.root, 'largerow', r)
3087
3088
# Read it again
3089
r2 = fileh.root.largerow.read()
3090
3091
assert r.tostring() == r2.tostring()
3092
3093
fileh.close()
3094
os.remove(file)
3095
3096
def test01(self):
3097
"Checking saving a Table with an extremely large rowsize"
3098
file = tempfile.mktemp(".h5")
3099
fileh = openFile(file, "w")
3100
3101
# Create a recarray
3102
r=records.array([[arange(40000)]*4]) # 640 KB
3103
3104
# Save it in a table:
3105
try:
3106
fileh.createTable(fileh.root, 'largerow', r)
3107
except RuntimeError:
3108
if verbose:
3109
(type, value, traceback) = sys.exc_info()
3110
print "\nGreat!, the next RuntimeError was catched!"
3111
print value
3112
else:
3113
self.fail("expected a RuntimeError")
3114
3115
fileh.close()
3116
os.remove(file)
3117
3118
3119
class DefaultValues(unittest.TestCase):
3120
record = Record
3121
3122
def test00(self):
3123
"Checking saving a Table with default values"
3124
file = tempfile.mktemp(".h5")
3125
#file = "/tmp/test.h5"
3126
fileh = openFile(file, "w")
3127
3128
# Create a table
3129
table = fileh.createTable(fileh.root, 'table', self.record)
3130
3131
# Take a number of records a bit greater
3132
nrows = int(table._v_maxTuples * 1.1)
3133
# Fill the table with nrows records
3134
for i in xrange(nrows):
3135
if i == 3 or i == 4:
3136
table.row['var2'] = 2
3137
# This injects the row values.
3138
table.row.append()
3139
3140
# We need to flush the buffers in table in order to get an
3141
# accurate number of records on it.
3142
table.flush()
3143
3144
# Create a recarray with the same default values
3145
r=records.array([["abcd", 1, 2, 3.1, 4.2, 5, "e", 1, 1j, 1+0j]]*nrows,
3146
formats='a4,i4,i2,f8,f4,i2,a1,b1,c8,c16')
3147
3148
# Assign the value exceptions
3149
r.field("c2")[3] = 2
3150
r.field("c2")[4] = 2
3151
3152
# Read the table in another recarray
3153
#r2 = table.read()
3154
r2 = table[::] # Equivalent to table.read()
3155
3156
# This generates too much output. Activate only when
3157
# self._v_maxTuples is very small (<10)
3158
if verbose and 1:
3159
print "First 10 table values:"
3160
for row in table.iterrows(0, 10):
3161
print row
3162
print "The first 10 read recarray values:"
3163
print r2[:10]
3164
print "Records should look like:"
3165
print r[:10]
3166
3167
assert r.tostring() == r2.tostring()
3168
3169
fileh.close()
3170
os.remove(file)
3171
3172
class OldRecordDefaultValues(DefaultValues):
3173
title = "OldRecordDefaultValues"
3174
record = OldRecord
3175
3176
class Record2(IsDescription):
3177
var1 = StringCol(length=4, dflt="abcd") # 4-character String
3178
var2 = IntCol(1) # integer
3179
var3 = Int16Col(2) # short integer
3180
var4 = Float64Col(3.1) # double (double-precision)
3181
3182
class LengthTestCase(unittest.TestCase):
3183
record = Record
3184
nrows = 20
3185
3186
def setUp(self):
3187
# Create an instance of an HDF5 Table
3188
self.file = tempfile.mktemp(".h5")
3189
self.fileh = openFile(self.file, "w")
3190
self.rootgroup = self.fileh.root
3191
self.populateFile()
3192
3193
def populateFile(self):
3194
group = self.rootgroup
3195
# Create a table
3196
table = self.fileh.createTable(self.fileh.root, 'table',
3197
self.record, title = "__length__ test")
3198
# Get the row object associated with the new table
3199
row = table.row
3200
3201
# Fill the table
3202
for i in xrange(self.nrows):
3203
row.append()
3204
3205
# Flush the buffer for this table
3206
table.flush()
3207
self.table = table
3208
3209
def tearDown(self):
3210
if self.fileh.isopen:
3211
self.fileh.close()
3212
os.remove(self.file)
3213
3214
#----------------------------------------
3215
3216
def test01_lengthrows(self):
3217
"""Checking __length__ in Table"""
3218
3219
if verbose:
3220
print '\n', '-=' * 30
3221
print "Running %s.test01_lengthrows..." % self.__class__.__name__
3222
3223
# Number of rows
3224
len(self.table) == self.nrows
3225
3226
def test02_lengthcols(self):
3227
"""Checking __length__ in Cols"""
3228
3229
if verbose:
3230
print '\n', '-=' * 30
3231
print "Running %s.test02_lengthcols..." % self.__class__.__name__
3232
3233
# Number of columns
3234
if self.record is Record:
3235
len(self.table.cols) == 8
3236
elif self.record is Record2:
3237
len(self.table.cols) == 4
3238
3239
def test03_lengthcol(self):
3240
"""Checking __length__ in Column"""
3241
3242
if verbose:
3243
print '\n', '-=' * 30
3244
print "Running %s.test03_lengthcol..." % self.__class__.__name__
3245
3246
# Number of rows for all columns column
3247
for colname in self.table.colnames:
3248
len(getattr(self.table.cols, colname)) == self.nrows
3249
3250
3251
class Length1TestCase(LengthTestCase):
3252
record = Record
3253
nrows = 20
3254
3255
class Length2TestCase(LengthTestCase):
3256
record = Record2
3257
nrows = 100
3258
3259
3260
#----------------------------------------------------------------------
3261
3262
def suite():
3263
theSuite = unittest.TestSuite()
3264
niter = 1
3265
#heavy = 1 # uncomment this only for testing purposes
3266
3267
#theSuite.addTest(unittest.makeSuite(RecArrayIO1))
3268
#theSuite.addTest(unittest.makeSuite(setItem1))
3269
#theSuite.addTest(unittest.makeSuite(BasicWriteTestCase))
3270
for n in range(niter):
3271
theSuite.addTest(unittest.makeSuite(BasicWriteTestCase))
3272
theSuite.addTest(unittest.makeSuite(OldRecordBasicWriteTestCase))
3273
theSuite.addTest(unittest.makeSuite(DictWriteTestCase))
3274
theSuite.addTest(unittest.makeSuite(RecArrayOneWriteTestCase))
3275
theSuite.addTest(unittest.makeSuite(RecArrayTwoWriteTestCase))
3276
theSuite.addTest(unittest.makeSuite(RecArrayThreeWriteTestCase))
3277
theSuite.addTest(unittest.makeSuite(CompressLZOTablesTestCase))
3278
theSuite.addTest(unittest.makeSuite(CompressLZOShuffleTablesTestCase))
3279
theSuite.addTest(unittest.makeSuite(CompressUCLTablesTestCase))
3280
theSuite.addTest(unittest.makeSuite(CompressUCLShuffleTablesTestCase))
3281
theSuite.addTest(unittest.makeSuite(CompressZLIBTablesTestCase))
3282
theSuite.addTest(unittest.makeSuite(CompressZLIBShuffleTablesTestCase))
3283
theSuite.addTest(unittest.makeSuite(Fletcher32TablesTestCase))
3284
theSuite.addTest(unittest.makeSuite(AllFiltersTablesTestCase))
3285
theSuite.addTest(unittest.makeSuite(CompressTwoTablesTestCase))
3286
theSuite.addTest(unittest.makeSuite(IterRangeTestCase))
3287
theSuite.addTest(unittest.makeSuite(RecArrayRangeTestCase))
3288
theSuite.addTest(unittest.makeSuite(getColRangeTestCase))
3289
theSuite.addTest(unittest.makeSuite(getItemTestCase))
3290
theSuite.addTest(unittest.makeSuite(setItem1))
3291
theSuite.addTest(unittest.makeSuite(setItem2))
3292
theSuite.addTest(unittest.makeSuite(setItem3))
3293
theSuite.addTest(unittest.makeSuite(RecArrayIO1))
3294
theSuite.addTest(unittest.makeSuite(RecArrayIO2))
3295
theSuite.addTest(unittest.makeSuite(OpenCopyTestCase))
3296
theSuite.addTest(unittest.makeSuite(CloseCopyTestCase))
3297
theSuite.addTest(unittest.makeSuite(CopyIndex1TestCase))
3298
theSuite.addTest(unittest.makeSuite(CopyIndex2TestCase))
3299
theSuite.addTest(unittest.makeSuite(CopyIndex3TestCase))
3300
theSuite.addTest(unittest.makeSuite(CopyIndex4TestCase))
3301
theSuite.addTest(unittest.makeSuite(CopyIndex5TestCase))
3302
theSuite.addTest(unittest.makeSuite(CopyIndex6TestCase))
3303
theSuite.addTest(unittest.makeSuite(CopyIndex7TestCase))
3304
theSuite.addTest(unittest.makeSuite(CopyIndex8TestCase))
3305
theSuite.addTest(unittest.makeSuite(CopyIndex9TestCase))
3306
theSuite.addTest(unittest.makeSuite(CopyIndex10TestCase))
3307
theSuite.addTest(unittest.makeSuite(CopyIndex11TestCase))
3308
theSuite.addTest(unittest.makeSuite(CopyIndex12TestCase))
3309
theSuite.addTest(unittest.makeSuite(LargeRowSize))
3310
theSuite.addTest(unittest.makeSuite(DefaultValues))
3311
theSuite.addTest(unittest.makeSuite(OldRecordDefaultValues))
3312
theSuite.addTest(unittest.makeSuite(Length1TestCase))
3313
theSuite.addTest(unittest.makeSuite(Length2TestCase))
3314
3315
if heavy:
3316
theSuite.addTest(unittest.makeSuite(BigTablesTestCase))
3317
3318
return theSuite
3319
3320
3321
if __name__ == '__main__':
3322
unittest.main( defaultTest='suite' )
Loggerhead is a web-based interface for Breezy
Version: 2.0.1