8
from tables.Index import Index
9
from tables.IndexArray import calcChunksize, minRowIndex
10
from common import verbose, allequal, heavy, cleanup
11
# To delete the internal attributes automagically
12
unittest.TestCase.tearDown = cleanup
16
class Small(IsDescription):
17
var1 = StringCol(length=4, dflt="", pos=1)
18
var2 = BoolCol(0, pos=2)
19
var3 = IntCol(0, pos=3)
20
var4 = FloatCol(0, pos=4)
22
class BasicTestCase(unittest.TestCase):
30
# Create an instance of an HDF5 Table
31
self.file = tempfile.mktemp(".h5")
32
self.fileh = openFile(self.file, "w")
33
self.rootgroup = self.fileh.root
38
def populateFile(self):
39
group = self.rootgroup
41
title = "This is the IndexArray title"
43
self.filters = Filters(complevel = self.compress,
44
complib = self.complib,
45
shuffle = self.shuffle,
46
fletcher32 = self.fletcher32)
47
table = self.fileh.createTable(group, 'table', Small, title,
48
self.filters, self.nrows)
49
for i in range(self.nrows):
50
table.row['var1'] = str(i)
51
# table.row['var2'] = i > 2
52
table.row['var2'] = i % 2
54
table.row['var4'] = float(self.nrows - i - 1)
58
indexrows = table.cols.var1.createIndex(testmode=1)
59
indexrows = table.cols.var2.createIndex(testmode=1)
60
indexrows = table.cols.var3.createIndex(testmode=1)
61
indexrows = table.cols.var4.createIndex(testmode=1)
63
print "Number of written rows:", self.nrows
64
print "Number of indexed rows:", indexrows
70
#print "File %s not removed!" % self.file
74
#----------------------------------------
76
def test01_readIndex(self):
77
"""Checking reading an Index (string flavor)"""
81
print "Running %s.test01_readIndex..." % self.__class__.__name__
83
# Open the HDF5 file in read-only mode
84
self.fileh = openFile(self.file, mode = "r")
85
table = self.fileh.root.table
86
idxcol = table.cols.var1.index
88
print "Max rows in buf:", table._v_maxTuples
89
print "Number of elements per slice:", idxcol.nelemslice
90
print "Chunk size:", idxcol.sorted.chunksize
93
results = [p["var1"] for p in table.where(table.cols.var1 == "1")]
94
assert len(results) == 1
96
def test02_readIndex(self):
97
"""Checking reading an Index (bool flavor)"""
100
print '\n', '-=' * 30
101
print "Running %s.test02_readIndex..." % self.__class__.__name__
103
# Open the HDF5 file in read-only mode
104
self.fileh = openFile(self.file, mode = "r")
105
table = self.fileh.root.table
106
idxcol = table.cols.var2.index
108
print "Rows in table:", table.nrows
109
print "Max rows in buf:", table._v_maxTuples
110
print "Number of elements per slice:", idxcol.nelemslice
111
print "Chunk size:", idxcol.sorted.chunksize
114
results = [p["var2"] for p in table.where(table.cols.var2 == 1)]
116
print "Selected values:", results
117
assert len(results) == self.nrows // 2
119
def test03_readIndex(self):
120
"""Checking reading an Index (int flavor)"""
123
print '\n', '-=' * 30
124
print "Running %s.test03_readIndex..." % self.__class__.__name__
126
# Open the HDF5 file in read-only mode
127
self.fileh = openFile(self.file, mode = "r")
128
table = self.fileh.root.table
129
idxcol = table.cols.var3.index
131
print "Max rows in buf:", table._v_maxTuples
132
print "Number of elements per slice:", idxcol.nelemslice
133
print "Chunk size:", idxcol.sorted.chunksize
136
results = [p["var3"] for p in table.where(1< table.cols.var3 < 10)]
138
print "Selected values:", results
139
assert len(results) == 8
141
def test04_readIndex(self):
142
"""Checking reading an Index (float flavor)"""
145
print '\n', '-=' * 30
146
print "Running %s.test04_readIndex..." % self.__class__.__name__
148
# Open the HDF5 file in read-only mode
149
self.fileh = openFile(self.file, mode = "r")
150
table = self.fileh.root.table
151
idxcol = table.cols.var4.index
153
print "Max rows in buf:", table._v_maxTuples
154
print "Number of elements per slice:", idxcol.nelemslice
155
print "Chunk size:", idxcol.sorted.chunksize
158
results = [p["var4"] for p in table.where(table.cols.var4 < 10)]
160
print "Selected values:", results
161
assert len(results) == 10
163
def test05_getWhereList(self):
164
"""Checking reading an Index with getWhereList (string flavor)"""
167
print '\n', '-=' * 30
168
print "Running %s.test05_getWhereList..." % self.__class__.__name__
170
# Open the HDF5 file in read-only mode
171
self.fileh = openFile(self.file, mode = "r")
172
table = self.fileh.root.table
173
idxcol = table.cols.var4.index
175
print "Max rows in buf:", table._v_maxTuples
176
print "Number of elements per slice:", idxcol.nelemslice
177
print "Chunk size:", idxcol.sorted.chunksize
180
rowList1 = table.getWhereList(table.cols.var1 < "10", "python")
181
rowList2 = [p.nrow for p in table if p['var1'] < "10"]
183
print "Selected values:", rowList1
184
print "Should look like:", rowList2
185
assert len(rowList1) == len(rowList2)
186
assert rowList1 == rowList2
188
def test06_getWhereList(self):
189
"""Checking reading an Index with getWhereList (bool flavor)"""
192
print '\n', '-=' * 30
193
print "Running %s.test06_getWhereList..." % self.__class__.__name__
195
# Open the HDF5 file in read-only mode
196
self.fileh = openFile(self.file, mode = "r")
197
table = self.fileh.root.table
198
idxcol = table.cols.var4.index
200
print "Max rows in buf:", table._v_maxTuples
201
print "Number of elements per slice:", idxcol.nelemslice
202
print "Chunk size:", idxcol.sorted.chunksize
205
rowList1 = table.getWhereList(table.cols.var2 == 0, "numarray")
206
rowList2 = [p.nrow for p in table if p['var2'] == 0]
207
# Convert to a numarray object
208
rowList2 = numarray.array(rowList2, numarray.Int64)
210
print "Selected values:", rowList1
211
print "Should look like:", rowList2
212
assert len(rowList1) == len(rowList2)
213
assert allequal(rowList1, rowList2)
215
def test07_getWhereList(self):
216
"""Checking reading an Index with getWhereList (int flavor)"""
219
print '\n', '-=' * 30
220
print "Running %s.test07_getWhereList..." % self.__class__.__name__
222
# Open the HDF5 file in read-only mode
223
self.fileh = openFile(self.file, mode = "r")
224
table = self.fileh.root.table
225
idxcol = table.cols.var4.index
227
print "Max rows in buf:", table._v_maxTuples
228
print "Number of elements per slice:", idxcol.nelemslice
229
print "Chunk size:", idxcol.sorted.chunksize
232
rowList1 = table.getWhereList(table.cols.var3 < 15, "python")
233
rowList2 = [p.nrow for p in table if p["var3"] < 15]
235
print "Selected values:", rowList1
236
print "Should look like:", rowList2
237
assert len(rowList1) == len(rowList2)
238
assert rowList1 == rowList2
240
def test08_getWhereList(self):
241
"""Checking reading an Index with getWhereList (float flavor)"""
244
print '\n', '-=' * 30
245
print "Running %s.test08_getWhereList..." % self.__class__.__name__
247
# Open the HDF5 file in read-only mode
248
self.fileh = openFile(self.file, mode = "r")
249
table = self.fileh.root.table
250
idxcol = table.cols.var4.index
252
print "Max rows in buf:", table._v_maxTuples
253
print "Number of elements per slice:", idxcol.nelemslice
254
print "Chunk size:", idxcol.sorted.chunksize
257
rowList1 = table.getWhereList(table.cols.var4 < 10, "python")
258
rowList2 = [p.nrow for p in table if p['var4'] < 10]
260
print "Selected values:", rowList1
261
print "Should look like:", rowList2
262
assert len(rowList1) == len(rowList2)
263
assert rowList1 == rowList2
265
def test09a_removeIndex(self):
266
"""Checking removing an index"""
269
print '\n', '-=' * 30
270
print "Running %s.test09a_removeIndex..." % self.__class__.__name__
272
# Open the HDF5 file in read-only mode
273
self.fileh = openFile(self.file, mode = "a")
274
table = self.fileh.root.table
275
idxcol = table.cols.var1.index
277
print "Before deletion"
278
print "var1 column:", table.cols.var1
279
assert table.colindexed["var1"] == 1
280
assert idxcol is not None
283
table.removeIndex(idxcol)
285
print "After deletion"
286
print "var1 column:", table.cols.var1
287
assert table.cols.var1.index is None
288
assert table.colindexed["var1"] == 0
290
# re-create the index again
291
indexrows = table.cols.var1.createIndex(testmode=1)
292
idxcol = table.cols.var1.index
294
print "After re-creation"
295
print "var1 column:", table.cols.var1
296
assert idxcol is not None
297
assert table.colindexed["var1"] == 1
299
def test09b_removeIndex(self):
300
"""Checking removing an index (persistent version)"""
303
print '\n', '-=' * 30
304
print "Running %s.test09b_removeIndex..." % self.__class__.__name__
306
# Open the HDF5 file in read-only mode
307
self.fileh = openFile(self.file, mode = "a")
308
table = self.fileh.root.table
309
idxcol = table.cols.var1.index
311
print "Before deletion"
312
print "var1 index column:", table.cols.var1
313
assert idxcol is not None
314
assert table.colindexed["var1"] == 1
316
table.removeIndex(idxcol)
318
# close and reopen the file
320
self.fileh = openFile(self.file, mode = "a")
321
table = self.fileh.root.table
322
idxcol = table.cols.var1.index
325
print "After deletion"
326
print "var1 column:", table.cols.var1
327
assert table.cols.var1.index is None
328
assert table.colindexed["var1"] == 0
330
# re-create the index again
331
indexrows = table.cols.var1.createIndex(testmode=1)
332
idxcol = table.cols.var1.index
334
print "After re-creation"
335
print "var1 column:", table.cols.var1
336
assert idxcol is not None
337
assert table.colindexed["var1"] == 1
339
def test10a_moveIndex(self):
340
"""Checking moving a table with an index"""
343
print '\n', '-=' * 30
344
print "Running %s.test10a_moveIndex..." % self.__class__.__name__
346
# Open the HDF5 file in read-only mode
347
self.fileh = openFile(self.file, mode = "a")
348
table = self.fileh.root.table
349
idxcol = table.cols.var1.index
352
print "var1 column:", idxcol
353
assert table.colindexed["var1"] == 1
354
assert idxcol is not None
356
# Create a new group called "agroup"
357
agroup = self.fileh.createGroup("/", "agroup")
359
# move the table to "agroup"
360
table.move(agroup, "table2")
363
print "var1 column:", idxcol
364
assert table.cols.var1.index is not None
365
assert table.colindexed["var1"] == 1
368
rowList1 = table.getWhereList(table.cols.var1 < "10", "python")
369
rowList2 = [p.nrow for p in table if p['var1'] < "10"]
371
print "Selected values:", rowList1
372
print "Should look like:", rowList2
373
assert len(rowList1) == len(rowList2)
374
assert rowList1 == rowList2
376
def test10b_moveIndex(self):
377
"""Checking moving a table with an index (persistent version)"""
380
print '\n', '-=' * 30
381
print "Running %s.test10b_moveIndex..." % self.__class__.__name__
383
# Open the HDF5 file in read-only mode
384
self.fileh = openFile(self.file, mode = "a")
385
table = self.fileh.root.table
386
idxcol = table.cols.var1.index
389
print "var1 index column:", idxcol
390
assert idxcol is not None
391
assert table.colindexed["var1"] == 1
392
# Create a new group called "agroup"
393
agroup = self.fileh.createGroup("/", "agroup")
395
# move the table to "agroup"
396
table.move(agroup, "table2")
398
# close and reopen the file
400
self.fileh = openFile(self.file, mode = "a")
401
table = self.fileh.root.agroup.table2
402
idxcol = table.cols.var1.index
406
print "var1 column:", idxcol
407
assert table.cols.var1.index is not None
408
assert table.colindexed["var1"] == 1
411
rowList1 = table.getWhereList(table.cols.var1 < "10", "python")
412
rowList2 = [p.nrow for p in table if p['var1'] < "10"]
414
print "Selected values:", rowList1
415
print "Should look like:", rowList2
416
assert len(rowList1) == len(rowList2)
417
assert rowList1 == rowList2
420
def test11a_removeTableWithIndex(self):
421
"""Checking removing a table with indexes"""
424
print '\n', '-=' * 30
425
print "Running %s.test11a_removeTableWithIndex..." % self.__class__.__name__
427
# Open the HDF5 file in read-only mode
428
self.fileh = openFile(self.file, mode = "a")
429
table = self.fileh.root.table
430
idxcol = table.cols.var1.index
432
print "Before deletion"
433
print "var1 column:", table.cols.var1
434
assert table.colindexed["var1"] == 1
435
assert idxcol is not None
438
self.fileh.removeNode("/table")
440
print "After deletion"
441
assert "table" not in self.fileh.root
443
# re-create the table and the index again
444
table = self.fileh.createTable("/", 'table', Small, "New table",
445
self.filters, self.nrows)
446
for i in range(self.nrows):
447
table.row['var1'] = str(i)
448
table.row['var2'] = i % 2
449
table.row['var3'] = i
450
table.row['var4'] = float(self.nrows - i - 1)
454
indexrows = table.cols.var1.createIndex(testmode=1)
455
indexrows = table.cols.var2.createIndex(testmode=1)
456
indexrows = table.cols.var3.createIndex(testmode=1)
457
indexrows = table.cols.var4.createIndex(testmode=1)
458
idxcol = table.cols.var1.index
460
print "After re-creation"
461
print "var1 column:", table.cols.var1
462
assert idxcol is not None
463
assert table.colindexed["var1"] == 1
465
def test11b_removeTableWithIndex(self):
466
"""Checking removing a table with indexes (persistent version 2)"""
469
print '\n', '-=' * 30
470
print "Running %s.test11b_removeTableWithIndex..." % self.__class__.__name__
472
self.fileh = openFile(self.file, mode = "a")
473
table = self.fileh.root.table
474
idxcol = table.cols.var1.index
476
print "Before deletion"
477
print "var1 column:", table.cols.var1
478
assert table.colindexed["var1"] == 1
479
assert idxcol is not None
482
self.fileh.removeNode("/table")
484
print "After deletion"
485
assert "table" not in self.fileh.root
487
# close and reopen the file
489
self.fileh = openFile(self.file, mode = "r+")
491
# re-create the table and the index again
492
table = self.fileh.createTable("/", 'table', Small, "New table",
493
self.filters, self.nrows)
494
for i in range(self.nrows):
495
table.row['var1'] = str(i)
496
table.row['var2'] = i % 2
497
table.row['var3'] = i
498
table.row['var4'] = float(self.nrows - i - 1)
502
indexrows = table.cols.var1.createIndex(testmode=1)
503
indexrows = table.cols.var2.createIndex(testmode=1)
504
indexrows = table.cols.var3.createIndex(testmode=1)
505
indexrows = table.cols.var4.createIndex(testmode=1)
506
idxcol = table.cols.var1.index
508
print "After re-creation"
509
print "var1 column:", table.cols.var1
510
assert idxcol is not None
511
assert table.colindexed["var1"] == 1
513
# Test provided by Andrew Straw
514
def test11c_removeTableWithIndex(self):
515
"""Checking removing a table with indexes (persistent version 2)"""
518
print '\n', '-=' * 30
519
print "Running %s.test11c_removeTableWithIndex..." % self.__class__.__name__
521
class Distance(IsDescription):
522
frame = Int32Col(pos=0, indexed=True)
523
distance = FloatCol(pos=1)
525
self.file = tempfile.mktemp(".h5")
526
self.fileh = openFile(self.file, mode='w')
527
table = self.fileh.createTable(self.fileh.root, 'distance_table', Distance)
531
r['distance']=float(i**2)
536
self.fileh = openFile(self.file, mode='r+')
537
self.fileh.removeNode(self.fileh.root.distance_table)
540
class BasicReadTestCase(BasicTestCase):
545
ns, cs = calcChunksize(minRowIndex, testmode=1)
548
class ZlibReadTestCase(BasicTestCase):
553
ns, cs = calcChunksize(minRowIndex, testmode=1)
556
class LZOReadTestCase(BasicTestCase):
561
ns, cs = calcChunksize(minRowIndex, testmode=1)
564
class BZIP2ReadTestCase(BasicTestCase):
569
ns, cs = calcChunksize(minRowIndex, testmode=1)
572
class ShuffleReadTestCase(BasicTestCase):
577
ns, cs = calcChunksize(minRowIndex, testmode=1)
580
class Fletcher32ReadTestCase(BasicTestCase):
585
ns, cs = calcChunksize(minRowIndex, testmode=1)
588
class ShuffleFletcher32ReadTestCase(BasicTestCase):
593
ns, cs = calcChunksize(minRowIndex, testmode=1)
596
class OneHalfTestCase(BasicTestCase):
597
ns, cs = calcChunksize(minRowIndex, testmode=1)
600
class UpperBoundTestCase(BasicTestCase):
601
ns, cs = calcChunksize(minRowIndex, testmode=1)
604
class LowerBoundTestCase(BasicTestCase):
605
ns, cs = calcChunksize(minRowIndex, testmode=1)
608
# This warning has non-sense now in the PyTables Pro version, as *all*
609
# the rows can be indexed there
610
class WarningTestCase(unittest.TestCase):
611
nrows = 100 # Small enough to raise the warning
614
"Checking the user warning for too few entries to index"
615
# Create an instance of an HDF5 Table
616
self.file = tempfile.mktemp(".h5")
617
self.fileh = openFile(self.file, "w")
618
self.rootgroup = self.fileh.root
619
group = self.rootgroup
621
title = "This is the IndexArray title"
623
table = self.fileh.createTable(group, 'table', Small, title,
625
for i in range(self.nrows):
626
# Fill rows with defaults
629
# try to index one entry
630
warnings.filterwarnings("error", category=UserWarning)
632
indexrows = table.cols.var1.createIndex()
635
(type, value, traceback) = sys.exc_info()
636
print "\nGreat!, the next UserWarning was catched!"
639
self.fail("expected an UserWarning")
641
warnings.filterwarnings("default", category=UserWarning)
646
class DeepTableIndexTestCase(unittest.TestCase):
650
"Checking the indexing of a table in a 2nd level hierarchy"
651
# Create an instance of an HDF5 Table
652
self.file = tempfile.mktemp(".h5")
653
self.fileh = openFile(self.file, "w")
654
group = self.fileh.createGroup(self.fileh.root,"agroup")
656
title = "This is the IndexArray title"
658
table = self.fileh.createTable(group, 'table', Small, title,
660
for i in range(self.nrows):
661
# Fill rows with defaults
665
indexrows = table.cols.var1.createIndex(testmode=1)
666
idxcol = table.cols.var1.index
668
assert table.colindexed["var1"] == 1
669
assert idxcol is not None
670
assert idxcol.nelements == self.nrows
676
"Checking the indexing of a table in 2nd level (persistent version)"
677
# Create an instance of an HDF5 Table
678
self.file = tempfile.mktemp(".h5")
679
self.fileh = openFile(self.file, "w")
680
group = self.fileh.createGroup(self.fileh.root,"agroup")
682
title = "This is the IndexArray title"
684
table = self.fileh.createTable(group, 'table', Small, title,
686
for i in range(self.nrows):
687
# Fill rows with defaults
691
indexrows = table.cols.var1.createIndex(testmode=1)
692
idxcol = table.cols.var1.index
693
# Close and re-open this file
695
self.fileh = openFile(self.file, "a")
696
table = self.fileh.root.agroup.table
697
idxcol = table.cols.var1.index
699
assert table.colindexed["var1"] == 1
700
assert idxcol is not None
701
assert idxcol.nelements == self.nrows
707
"Checking the indexing of a table in a 4th level hierarchy"
708
# Create an instance of an HDF5 Table
709
self.file = tempfile.mktemp(".h5")
710
self.fileh = openFile(self.file, "w")
711
group = self.fileh.createGroup(self.fileh.root,"agroup")
712
group = self.fileh.createGroup(group,"agroup")
713
group = self.fileh.createGroup(group,"agroup")
715
title = "This is the IndexArray title"
717
table = self.fileh.createTable(group, 'table', Small, title,
719
for i in range(self.nrows):
720
# Fill rows with defaults
724
indexrows = table.cols.var1.createIndex(testmode=1)
725
idxcol = table.cols.var1.index
727
assert table.colindexed["var1"] == 1
728
assert idxcol is not None
729
assert idxcol.nelements == self.nrows
735
"Checking the indexing of a table in a 4th level (persistent version)"
736
# Create an instance of an HDF5 Table
737
self.file = tempfile.mktemp(".h5")
738
self.fileh = openFile(self.file, "w")
739
group = self.fileh.createGroup(self.fileh.root,"agroup")
740
group = self.fileh.createGroup(group,"agroup")
741
group = self.fileh.createGroup(group,"agroup")
743
title = "This is the IndexArray title"
745
table = self.fileh.createTable(group, 'table', Small, title,
747
for i in range(self.nrows):
748
# Fill rows with defaults
752
indexrows = table.cols.var1.createIndex(testmode=1)
753
idxcol = table.cols.var1.index
754
# Close and re-open this file
756
self.fileh = openFile(self.file, "a")
757
table = self.fileh.root.agroup.agroup.agroup.table
758
idxcol = table.cols.var1.index
760
assert table.colindexed["var1"] == 1
761
assert idxcol is not None
762
assert idxcol.nelements == self.nrows
768
"Checking the indexing of a table in a 100th level hierarchy"
769
# Create an instance of an HDF5 Table
770
self.file = tempfile.mktemp(".h5")
771
self.fileh = openFile(self.file, "w")
772
group = self.fileh.root
774
group = self.fileh.createGroup(group,"agroup")
776
title = "This is the IndexArray title"
778
table = self.fileh.createTable(group, 'table', Small, title,
780
for i in range(self.nrows):
781
# Fill rows with defaults
785
indexrows = table.cols.var1.createIndex(testmode=1)
786
idxcol = table.cols.var1.index
788
assert table.colindexed["var1"] == 1
789
assert idxcol is not None
790
assert idxcol.nelements == self.nrows
795
class NoAuto(IsDescription):
796
_v_indexprops = IndexProps(auto=0)
797
var1 = StringCol(length=4, dflt="", pos=1, indexed=1)
798
var2 = BoolCol(0, indexed=1, pos = 2)
799
var3 = IntCol(0, indexed=1, pos = 3)
800
var4 = FloatCol(0, indexed=0, pos = 4)
802
class NoReindex(IsDescription):
803
_v_indexprops = IndexProps(reindex=0)
804
var1 = StringCol(length=4, dflt="", indexed=1, pos=1)
805
var2 = BoolCol(0, indexed=1, pos=2)
806
var3 = IntCol(0, indexed=1, pos=3)
807
var4 = FloatCol(0, indexed=0, pos=4)
809
class ChangeFilters(IsDescription):
810
_v_indexprops = IndexProps(filters=Filters(complevel=6, complib="zlib",
811
shuffle=0, fletcher32=1))
812
var1 = StringCol(length=4, dflt="", indexed=1, pos=1)
813
var2 = BoolCol(0, indexed=1, pos=2)
814
var3 = IntCol(0, indexed=1, pos=3)
815
var4 = FloatCol(0, indexed=0, pos=4)
818
class AutomaticIndexingTestCase(unittest.TestCase):
823
# Create an instance of an HDF5 Table
824
self.file = tempfile.mktemp(".h5")
825
self.fileh = openFile(self.file, "w")
827
title = "This is the IndexArray title"
829
root = self.fileh.root
830
self.table = self.fileh.createTable(root, 'table', self.klass, title,
832
for i in range(self.nrows):
833
# Fill rows with defaults
834
self.table.row.append()
838
self.fileh = openFile(self.file, "a")
839
self.table = self.fileh.root.table
846
def test01_attrs(self):
847
"Checking indexing attributes (part1)"
849
print '\n', '-=' * 30
850
print "Running %s.test01_attrs..." % self.__class__.__name__
853
if self.klass is Small:
854
assert table.indexed == 0
856
assert table.indexed == 1
857
if self.klass is Small:
858
assert table.colindexed["var1"] == 0
859
assert table.cols.var1.index is None
860
assert table.colindexed["var2"] == 0
861
assert table.cols.var2.index is None
862
assert table.colindexed["var3"] == 0
863
assert table.cols.var3.index is None
864
assert table.colindexed["var4"] == 0
865
assert table.cols.var4.index is None
867
# Check that the var1, var2 and var3 (and only these)
869
assert table.colindexed["var1"] == 1
870
assert table.cols.var1.index is not None
871
assert table.colindexed["var2"] == 1
872
assert table.cols.var2.index is not None
873
assert table.colindexed["var3"] == 1
874
assert table.cols.var3.index is not None
875
assert table.colindexed["var4"] == 0
876
assert table.cols.var4.index is None
878
def test02_attrs(self):
879
"Checking indexing attributes (part2)"
881
print '\n', '-=' * 30
882
print "Running %s.test02_attrs..." % self.__class__.__name__
885
# Check the policy parameters
888
print "indexprops:", table.indexprops
890
print "Table is not indexed"
891
# Check non-default values for index saving policy
892
if self.klass is Small:
893
assert table.indexprops is not None
894
elif self.klass is NoAuto:
895
assert table.indexprops.auto == 0
896
assert table.indexprops.reindex == 1
897
filters = Filters(complevel=1, complib="zlib",
898
shuffle=1, fletcher32=0)
899
assert str(table.indexprops.filters) == str(filters)
900
elif self.klass is NoReindex:
901
assert table.indexprops.auto == 1
902
assert table.indexprops.reindex == 0
903
filters = Filters(complevel=1, complib="zlib",
904
shuffle=1, fletcher32=0)
905
assert str(table.indexprops.filters) == str(filters)
906
elif self.klass is ChangeFilters:
907
assert table.indexprops.auto == 1
908
assert table.indexprops.reindex == 1
909
filters = Filters(complevel=6, complib="zlib",
910
shuffle=0, fletcher32=1)
911
assert str(table.indexprops.filters) == str(filters)
913
# Check Index() objects exists and are properly placed
914
if self.klass is Small:
915
assert table.cols.var1.index == None
916
assert table.cols.var2.index == None
917
assert table.cols.var3.index == None
918
assert table.cols.var4.index == None
920
assert isinstance(table.cols.var1.index, Index)
921
assert isinstance(table.cols.var2.index, Index)
922
assert isinstance(table.cols.var3.index, Index)
923
assert table.cols.var4.index == None
925
def test03_counters(self):
926
"Checking indexing counters"
928
print '\n', '-=' * 30
929
print "Running %s.test03_counters..." % self.__class__.__name__
931
# Check the counters for indexes
934
print "indexedrows:", table._indexedrows
935
print "unsavedindexedrows:", table._unsaved_indexedrows
936
index = table.cols.var1.index
937
print "table rows:", table.nrows
938
print "computed indexed rows:", index.nrows * index.nelemslice
940
print "Table is not indexed"
941
if self.klass is not Small:
942
index = table.cols.var1.index
943
indexedrows = index.nelements
944
assert table._indexedrows == indexedrows
945
indexedrows = index.nelements
946
assert table._unsaved_indexedrows == self.nrows - indexedrows
948
def test04_noauto(self):
949
"Checking indexing counters (non-automatic mode)"
951
print '\n', '-=' * 30
952
print "Running %s.test04_noauto..." % self.__class__.__name__
954
# Force a sync in indexes
955
table.flushRowsToIndex()
956
# Check the counters for indexes
959
print "indexedrows:", table._indexedrows
960
print "unsavedindexedrows:", table._unsaved_indexedrows
961
index = table.cols.var1.index
962
indexedrows = index.nrows * index.nelemslice
963
print "computed indexed rows:", index.nelements
965
print "Table is not indexed"
967
# No unindexated rows should remain
968
index = table.cols.var1.index
969
if self.klass is Small:
972
indexedrows = index.nrows * index.nelemslice
973
assert table._indexedrows == indexedrows
974
assert table._unsaved_indexedrows == self.nrows - indexedrows
976
# Check non-default values for index saving policy
977
if self.klass is Small:
978
assert table.indexprops is not None
979
elif self.klass is NoAuto:
980
assert table.indexprops.auto == 0
981
assert table.indexprops.reindex == 1
982
filters = Filters(complevel=1, complib="zlib",
983
shuffle=1, fletcher32=0)
984
assert str(table.indexprops.filters) == str(filters)
985
elif self.klass is NoReindex:
986
assert table.indexprops.auto == 1
987
assert table.indexprops.reindex == 0
988
filters = Filters(complevel=1, complib="zlib",
989
shuffle=1, fletcher32=0)
990
assert str(table.indexprops.filters) == str(filters)
991
elif self.klass is ChangeFilters:
992
assert table.indexprops.auto == 1
993
assert table.indexprops.reindex == 1
994
filters = Filters(complevel=6, complib="zlib",
995
shuffle=0, fletcher32=1)
996
assert str(table.indexprops.filters) == str(filters)
999
def test05_icounters(self):
1000
"Checking indexing counters (removeRows)"
1002
print '\n', '-=' * 30
1003
print "Running %s.test05_icounters..." % self.__class__.__name__
1005
# Force a sync in indexes
1006
table.flushRowsToIndex()
1007
# Non indexated rows should remain here
1008
if self.klass is not Small:
1009
indexedrows = table._indexedrows
1010
unsavedindexedrows = table._unsaved_indexedrows
1011
# Now, remove some rows:
1012
table.removeRows(3,5)
1015
self.fileh = openFile(self.file, "a")
1016
table = self.fileh.root.table
1017
# Check the counters for indexes
1020
print "indexedrows:", table._indexedrows
1021
print "original indexedrows:", indexedrows
1022
print "unsavedindexedrows:", table._unsaved_indexedrows
1023
print "original unsavedindexedrows:", unsavedindexedrows
1024
index = table.cols.var1.index
1025
print "index dirty:", table.cols.var1.dirty
1027
print "Table is not indexed"
1029
# Check the counters
1030
assert table.nrows == self.nrows - 2
1031
if self.klass is NoReindex:
1032
# I'm not sure that the results below are what we want...
1033
# But I don't think this is going to be important
1034
# the important thing is that dirtiness is working right
1035
# Francesc Altet 2004-12-31
1037
# assert table._indexedrows == indexedrows - 2
1038
# assert table._unsaved_indexedrows == unsavedindexedrows + 2
1040
# assert table._indexedrows == indexedrows
1041
# The next values should be more consistent... 2005-01-03
1042
# assert table._indexedrows == 0
1043
# assert table._unsaved_indexedrows == table.nrows
1045
elif self.klass is NoAuto:
1046
index = table.cols.var1.index
1047
indexedrows = index.nrows * index.nelemslice
1048
assert table._indexedrows == indexedrows
1049
assert table._indexedrows == index.nelements
1050
assert table._unsaved_indexedrows == self.nrows - indexedrows - 2
1052
# Check non-default values for index saving policy
1053
if self.klass is Small:
1054
assert table.indexprops is not None
1055
elif self.klass is NoAuto:
1056
assert table.indexprops.auto == 0
1057
assert table.indexprops.reindex == 1
1058
filters = Filters(complevel=1, complib="zlib",
1059
shuffle=1, fletcher32=0)
1060
assert str(table.indexprops.filters) == str(filters)
1061
elif self.klass is NoReindex:
1062
assert table.indexprops.auto == 1
1063
assert table.indexprops.reindex == 0
1064
filters = Filters(complevel=1, complib="zlib",
1065
shuffle=1, fletcher32=0)
1066
assert str(table.indexprops.filters) == str(filters)
1067
elif self.klass is ChangeFilters:
1068
assert table.indexprops.auto == 1
1069
assert table.indexprops.reindex == 1
1070
filters = Filters(complevel=6, complib="zlib",
1071
shuffle=0, fletcher32=1)
1072
assert str(table.indexprops.filters) == str(filters)
1075
def test06_dirty(self):
1076
"Checking dirty flags (removeRows action)"
1078
print '\n', '-=' * 30
1079
print "Running %s.test06_dirty..." % self.__class__.__name__
1081
# Force a sync in indexes
1082
table.flushRowsToIndex()
1083
# Now, remove some rows:
1084
table.removeRows(3,5)
1087
self.fileh = openFile(self.file, "a")
1088
table = self.fileh.root.table
1089
# Check the dirty flag for indexes
1091
for colname in table.colnames:
1092
print "dirty flag col %s: %s" % \
1093
(colname, table.cols._f_col(colname).dirty)
1095
for colname in table.colnames:
1096
if (table.cols._f_col(colname).index and
1097
not table.indexprops.reindex):
1098
assert table.cols._f_col(colname).dirty == 1
1100
assert table.cols._f_col(colname).dirty == 0
1102
def test07_noreindex(self):
1103
"Checking indexing counters (modifyRows, no-reindex mode)"
1105
print '\n', '-=' * 30
1106
print "Running %s.test07_noreindex..." % self.__class__.__name__
1108
# Force a sync in indexes
1109
table.flushRowsToIndex()
1110
# No unindexated rows should remain here
1111
if self.klass is not Small:
1112
indexedrows = table._indexedrows
1113
unsavedindexedrows = table._unsaved_indexedrows
1114
# Now, modify just one row:
1115
table.modifyRows(3, None, 1, [["asa",0,3,3.1]])
1118
self.fileh = openFile(self.file, "a")
1119
table = self.fileh.root.table
1120
# Check the counters for indexes
1123
print "indexedrows:", table._indexedrows
1124
print "original indexedrows:", indexedrows
1125
print "unsavedindexedrows:", table._unsaved_indexedrows
1126
print "original unsavedindexedrows:", unsavedindexedrows
1127
index = table.cols.var1.index
1128
print "computed indexed rows:", index.nelements
1130
print "Table is not indexed"
1131
# Check the counters
1132
assert table.nrows == self.nrows
1133
if self.klass is NoReindex:
1134
# The unsaved indexed rows counter should be unchanged
1135
# assert table._indexedrows == indexedrows
1136
# assert table._unsaved_indexedrows == unsavedindexedrows
1137
# I'm not sure that the results below are what we want...
1138
# But I don't think this is going to be important
1139
# the important thing is that dirtiness is working right
1140
# Francesc Altet 2004-12-31
1142
# assert table._indexedrows == indexedrows - 2
1143
# assert table._unsaved_indexedrows == unsavedindexedrows + 2
1145
# assert table._indexedrows == indexedrows
1147
elif self.klass is NoAuto:
1148
index = table.cols.var1.index
1149
indexedrows = index.nrows * index.nelemslice
1150
assert table._indexedrows == indexedrows
1151
indexedrows = index.nelements
1152
assert table._indexedrows == indexedrows
1153
assert table._unsaved_indexedrows == self.nrows - indexedrows
1155
# Check the dirty flag for indexes
1157
for colname in table.colnames:
1158
print "dirty flag col %s: %s" % \
1159
(colname, table.cols._f_col(colname).dirty)
1160
for colname in table.colnames:
1161
if (table.cols._f_col(colname).index and
1162
not table.indexprops.reindex):
1163
assert table.cols._f_col(colname).dirty == 1
1165
assert table.cols._f_col(colname).dirty == 0
1167
def test08_dirty(self):
1168
"Checking dirty flags (modifyColumns)"
1170
print '\n', '-=' * 30
1171
print "Running %s.test08_dirty..." % self.__class__.__name__
1173
# Force a sync in indexes
1174
table.flushRowsToIndex()
1175
# Non indexated rows should remain here
1176
if self.klass is not Small:
1177
indexedrows = table._indexedrows
1178
unsavedindexedrows = table._unsaved_indexedrows
1179
# Now, modify a couple of rows:
1180
table.modifyColumns(1, columns=[["asa","asb"],[1.,2.]],
1181
names=["var1", "var4"])
1184
self.fileh = openFile(self.file, "a")
1185
table = self.fileh.root.table
1187
# Check the counters
1188
assert table.nrows == self.nrows
1189
if self.klass is NoReindex:
1190
# The unsaved indexed rows counter should be unchanged
1191
assert table._indexedrows == indexedrows
1192
assert table._unsaved_indexedrows == unsavedindexedrows
1193
elif self.klass is NoAuto:
1194
index = table.cols.var1.index
1195
indexedrows = index.nrows * index.nelemslice
1196
assert table._indexedrows == indexedrows
1197
assert table._unsaved_indexedrows == self.nrows - indexedrows
1199
# Check the dirty flag for indexes
1201
for colname in table.colnames:
1202
print "dirty flag col %s: %s" % \
1203
(colname, table.cols._f_col(colname).dirty)
1204
for colname in table.colnames:
1205
if (table.cols._f_col(colname).index and
1206
not table.indexprops.reindex):
1207
if colname in ["var1"]:
1208
assert table.cols._f_col(colname).dirty == 1
1210
assert table.cols._f_col(colname).dirty == 0
1212
assert table.cols._f_col(colname).dirty == 0
1214
def test09_copyIndex(self):
1215
"Checking copy Index feature in copyTable (attrs)"
1217
print '\n', '-=' * 30
1218
print "Running %s.test09_copyIndex..." % self.__class__.__name__
1220
# Don't force a sync in indexes
1221
#table.flushRowsToIndex()
1222
# Non indexated rows should remain here
1223
if self.klass is not Small:
1224
indexedrows = table._indexedrows
1225
unsavedindexedrows = table._unsaved_indexedrows
1226
# Now, remove some rows to make columns dirty
1227
#table.removeRows(3,5)
1228
# Copy a Table to another location
1229
warnings.filterwarnings("ignore", category=UserWarning)
1230
table2 = table.copy("/", 'table2')
1231
warnings.filterwarnings("default", category=UserWarning)
1234
self.fileh = openFile(self.file, "a")
1235
table = self.fileh.root.table
1236
table2 = self.fileh.root.table2
1238
index1 = table.cols.var1.index
1239
index2 = table2.cols.var1.index
1241
print "Copied index:", index2
1242
print "Original index:", index1
1244
print "Elements in copied index:", index2.nelements
1245
print "Elements in original index:", index1.nelements
1246
# Check the counters
1247
assert table.nrows == table2.nrows
1249
assert table2.indexed
1250
assert table._indexedrows == table2._indexedrows
1251
assert table._unsaved_indexedrows == table2._unsaved_indexedrows
1252
if self.klass is Small:
1253
# No index: the index should not exist
1254
assert index1 is None
1255
assert index2 is None
1256
elif self.klass is NoAuto:
1257
# No auto: the index should exists, but be empty
1258
assert index2 is not None
1259
assert index2.nelements == 0
1260
elif self.klass is NoReindex:
1261
# Auto: the index should exists, and have elements
1262
assert index2 is not None
1263
assert index2.nelements == index1.nelements
1265
# Check the dirty flag for indexes
1267
for colname in table2.colnames:
1268
print "dirty flag col %s: %s" % \
1269
(colname, table2.cols._f_col(colname).dirty)
1270
for colname in table2.colnames:
1271
assert table2.cols._f_col(colname).dirty == 0
1273
def test10_copyIndex(self):
1274
"Checking copy Index feature in copyTable (values)"
1276
print '\n', '-=' * 30
1277
print "Running %s.test10_copyIndex..." % self.__class__.__name__
1279
# Don't force a sync in indexes
1280
#table.flushRowsToIndex()
1281
# Non indexated rows should remain here
1282
if self.klass is not Small:
1283
indexedrows = table._indexedrows
1284
unsavedindexedrows = table._unsaved_indexedrows
1285
# Now, remove some rows to make columns dirty
1286
#table.removeRows(3,5)
1287
# Copy a Table to another location
1288
warnings.filterwarnings("ignore", category=UserWarning)
1289
table2 = table.copy("/", 'table2')
1290
warnings.filterwarnings("default", category=UserWarning)
1293
self.fileh = openFile(self.file, "a")
1294
table = self.fileh.root.table
1295
table2 = self.fileh.root.table2
1297
index1 = table.cols.var3.index
1298
index2 = table2.cols.var3.index
1300
print "Copied index:", index2
1301
print "Original index:", index1
1303
print "Elements in copied index:", index2.nelements
1304
print "Elements in original index:", index1.nelements
1305
if index2.nelements > 10:
1306
print "First 10 elements in copied index (sorted):\n", \
1307
index2.sorted[0,:10]
1308
print "First 10 elements in orig index (sorted):\n", \
1309
index1.sorted[0,:10]
1310
print "First 10 elements in copied index (indices):\n", \
1311
index2.indices[0,:10]
1312
print "First 10 elements in orig index (indices):\n", \
1313
index1.indices[0,:10]
1314
if self.klass is NoReindex:
1315
# Auto: the index should exists, and have equal elements
1316
assert allequal(index2.sorted.read(), index1.sorted.read())
1317
# The next assertion cannot be guaranteed. Why?
1318
# sorting algorithm in numarray is not deterministic?
1319
#assert allequal(index2.indices.read(), index1.indices.read())
1321
def test11_copyIndex(self):
1322
"Checking copy Index feature in copyTable (dirty flags)"
1324
print '\n', '-=' * 30
1325
print "Running %s.test11_copyIndex..." % self.__class__.__name__
1327
# Force a sync in indexes
1328
table.flushRowsToIndex()
1329
# Non indexated rows should remain here
1330
if self.klass is not Small:
1331
indexedrows = table._indexedrows
1332
unsavedindexedrows = table._unsaved_indexedrows
1333
# Now, modify an indexed column and an unindexed one
1334
# to make the "var1" dirty
1335
table.modifyColumns(1, columns=[["asa","asb"],[1.,2.]],
1336
names=["var1", "var4"])
1337
# Copy a Table to another location
1338
warnings.filterwarnings("ignore", category=UserWarning)
1339
table2 = table.copy("/", 'table2')
1340
warnings.filterwarnings("default", category=UserWarning)
1343
self.fileh = openFile(self.file, "a")
1344
table = self.fileh.root.table
1345
table2 = self.fileh.root.table2
1347
index1 = table.cols.var1.index
1348
index2 = table2.cols.var1.index
1350
print "Copied index:", index2
1351
print "Original index:", index1
1353
print "Elements in copied index:", index2.nelements
1354
print "Elements in original index:", index1.nelements
1356
# Check the dirty flag for indexes
1358
for colname in table2.colnames:
1359
print "dirty flag col %s: %s" % \
1360
(colname, table2.cols._f_col(colname).dirty)
1361
for colname in table2.colnames:
1362
if (table2.cols._f_col(colname).index and
1363
not table2.indexprops.reindex):
1364
if colname in ["var1"]:
1365
# All the destination columns should be non-dirty because
1366
# the copy removes the dirty state and puts the
1367
# index in a sane state
1368
assert table.cols._f_col(colname).dirty == 1
1369
assert table2.cols._f_col(colname).dirty == 0
1371
assert table2.cols._f_col(colname).dirty == 0
1373
assert table2.cols._f_col(colname).dirty == 0
1376
# minRowIndex = 10000 # just if one wants more indexed rows to be checked
1377
class AI1TestCase(AutomaticIndexingTestCase):
1383
class AI2TestCase(AutomaticIndexingTestCase):
1389
class AI3TestCase(AutomaticIndexingTestCase):
1395
class AI4aTestCase(AutomaticIndexingTestCase):
1401
class AI4bTestCase(AutomaticIndexingTestCase):
1407
class AI5TestCase(AutomaticIndexingTestCase):
1408
ns, cs = calcChunksize(minRowIndex, testmode=0)
1413
class AI6TestCase(AutomaticIndexingTestCase):
1414
ns, cs = calcChunksize(minRowIndex, testmode=0)
1419
class AI7TestCase(AutomaticIndexingTestCase):
1420
ns, cs = calcChunksize(minRowIndex, testmode=0)
1425
class AI8TestCase(AutomaticIndexingTestCase):
1426
ns, cs = calcChunksize(minRowIndex, testmode=0)
1431
class AI9TestCase(AutomaticIndexingTestCase):
1432
ns, cs = calcChunksize(minRowIndex, testmode=1)
1437
class AI10TestCase(AutomaticIndexingTestCase):
1443
class AI11TestCase(AutomaticIndexingTestCase):
1447
klass = ChangeFilters
1449
class AI12TestCase(AutomaticIndexingTestCase):
1453
klass = ChangeFilters
1456
#----------------------------------------------------------------------
1459
theSuite = unittest.TestSuite()
1461
#heavy = 1 # Uncomment this only for testing purposes!
1463
# theSuite.addTest(unittest.makeSuite(AI5TestCase))
1464
# theSuite.addTest(unittest.makeSuite(AI6TestCase))
1465
for n in range(niter):
1466
theSuite.addTest(unittest.makeSuite(BasicReadTestCase))
1467
theSuite.addTest(unittest.makeSuite(ZlibReadTestCase))
1468
theSuite.addTest(unittest.makeSuite(LZOReadTestCase))
1469
theSuite.addTest(unittest.makeSuite(BZIP2ReadTestCase))
1470
theSuite.addTest(unittest.makeSuite(ShuffleReadTestCase))
1471
theSuite.addTest(unittest.makeSuite(Fletcher32ReadTestCase))
1472
theSuite.addTest(unittest.makeSuite(ShuffleFletcher32ReadTestCase))
1473
theSuite.addTest(unittest.makeSuite(OneHalfTestCase))
1474
theSuite.addTest(unittest.makeSuite(UpperBoundTestCase))
1475
theSuite.addTest(unittest.makeSuite(LowerBoundTestCase))
1476
theSuite.addTest(unittest.makeSuite(AI1TestCase))
1477
theSuite.addTest(unittest.makeSuite(AI2TestCase))
1478
theSuite.addTest(unittest.makeSuite(AI9TestCase))
1479
theSuite.addTest(unittest.makeSuite(DeepTableIndexTestCase))
1481
# These are too heavy for normal testing
1482
theSuite.addTest(unittest.makeSuite(AI3TestCase))
1483
theSuite.addTest(unittest.makeSuite(AI4aTestCase))
1484
theSuite.addTest(unittest.makeSuite(AI4bTestCase))
1485
theSuite.addTest(unittest.makeSuite(AI5TestCase))
1486
theSuite.addTest(unittest.makeSuite(AI6TestCase))
1487
theSuite.addTest(unittest.makeSuite(AI7TestCase))
1488
theSuite.addTest(unittest.makeSuite(AI8TestCase))
1489
theSuite.addTest(unittest.makeSuite(AI10TestCase))
1490
theSuite.addTest(unittest.makeSuite(AI11TestCase))
1491
theSuite.addTest(unittest.makeSuite(AI12TestCase))
1495
if __name__ == '__main__':
1496
unittest.main( defaultTest='suite' )
added
removed
tables/tests/test_indexes.py
