8
from tables.Index import Index
9
from tables.IndexArray import calcChunksize
10
from test_all import verbose, allequal, heavy
13
# The minimum number of rows that can be indexed
14
# Remember to change that if the number is changed in
15
# IndexArray._calcChunksize
18
class Small(IsDescription):
19
var1 = StringCol(length=4, dflt="", pos=1)
20
var2 = BoolCol(0, pos=2)
21
var3 = IntCol(0, pos=3)
22
var4 = FloatCol(0, pos=4)
24
class BasicTestCase(unittest.TestCase):
32
# Create an instance of an HDF5 Table
33
self.file = tempfile.mktemp(".h5")
34
self.fileh = openFile(self.file, "w")
35
self.rootgroup = self.fileh.root
40
def populateFile(self):
41
group = self.rootgroup
43
title = "This is the IndexArray title"
45
filters = Filters(complevel = self.compress,
46
complib = self.complib,
47
shuffle = self.shuffle,
48
fletcher32 = self.fletcher32)
49
table = self.fileh.createTable(group, 'table', Small, title,
51
for i in range(self.nrows):
52
table.row['var1'] = str(i)
53
# table.row['var2'] = i > 2
54
table.row['var2'] = i % 2
56
table.row['var4'] = float(self.nrows - i - 1)
60
indexrows = table.cols.var1.createIndex(testmode=1)
61
indexrows = table.cols.var2.createIndex(testmode=1)
62
indexrows = table.cols.var3.createIndex(testmode=1)
63
indexrows = table.cols.var4.createIndex(testmode=1)
65
print "Number of written rows:", self.nrows
66
print "Number of indexed rows:", indexrows
74
#----------------------------------------
76
def test01_readIndex(self):
77
"""Checking reading an Index (string flavor)"""
81
print "Running %s.test01_readIndex..." % self.__class__.__name__
83
# Open the HDF5 file in read-only mode
84
self.fileh = openFile(self.file, mode = "r")
85
table = self.fileh.root.table
86
idxcol = table.cols.var1.index
88
print "Max rows in buf:", table._v_maxTuples
89
print "Number of elements per slice:", idxcol.nelemslice
90
print "Chunk size:", idxcol.sorted.chunksize
93
results = [p["var1"] for p in table.where(table.cols.var1 == "1")]
94
#results = [p["var1"] for p in table(where=table.cols.var1 == "1")]
95
assert len(results) == 1
97
def test02_readIndex(self):
98
"""Checking reading an Index (bool flavor)"""
101
print '\n', '-=' * 30
102
print "Running %s.test02_readIndex..." % self.__class__.__name__
104
# Open the HDF5 file in read-only mode
105
self.fileh = openFile(self.file, mode = "r")
106
table = self.fileh.root.table
107
idxcol = table.cols.var2.index
109
print "Max rows in buf:", table._v_maxTuples
110
print "Number of elements per slice:", idxcol.nelemslice
111
print "Chunk size:", idxcol.sorted.chunksize
114
results = [p["var2"] for p in table.where(table.cols.var2 == 1)]
116
print "Selected values:", results
117
assert len(results) == self.nrows // 2
119
def test03_readIndex(self):
120
"""Checking reading an Index (int flavor)"""
123
print '\n', '-=' * 30
124
print "Running %s.test03_readIndex..." % self.__class__.__name__
126
# Open the HDF5 file in read-only mode
127
self.fileh = openFile(self.file, mode = "r")
128
table = self.fileh.root.table
129
idxcol = table.cols.var3.index
131
print "Max rows in buf:", table._v_maxTuples
132
print "Number of elements per slice:", idxcol.nelemslice
133
print "Chunk size:", idxcol.sorted.chunksize
136
results = [p["var3"] for p in table.where(1< table.cols.var3 < 10)]
138
print "Selected values:", results
139
assert len(results) == 8
141
def test04_readIndex(self):
142
"""Checking reading an Index (float flavor)"""
145
print '\n', '-=' * 30
146
print "Running %s.test04_readIndex..." % self.__class__.__name__
148
# Open the HDF5 file in read-only mode
149
self.fileh = openFile(self.file, mode = "r")
150
table = self.fileh.root.table
151
idxcol = table.cols.var4.index
153
print "Max rows in buf:", table._v_maxTuples
154
print "Number of elements per slice:", idxcol.nelemslice
155
print "Chunk size:", idxcol.sorted.chunksize
158
results = [p["var4"] for p in table.where(table.cols.var4 < 10)]
160
print "Selected values:", results
161
assert len(results) == 10
163
def test05_getWhereList(self):
164
"""Checking reading an Index with getWhereList (string flavor)"""
167
print '\n', '-=' * 30
168
print "Running %s.test05_getWhereList..." % self.__class__.__name__
170
# Open the HDF5 file in read-only mode
171
self.fileh = openFile(self.file, mode = "r")
172
table = self.fileh.root.table
173
idxcol = table.cols.var4.index
175
print "Max rows in buf:", table._v_maxTuples
176
print "Number of elements per slice:", idxcol.nelemslice
177
print "Chunk size:", idxcol.sorted.chunksize
180
rowList1 = table.getWhereList(table.cols.var1 < "10", "List")
181
#rowList2 = [p.nrow() for p in table.where(table.cols.var1 < "10")]
182
rowList2 = [p.nrow() for p in table if p['var1'] < "10"]
184
print "Selected values:", rowList1
185
print "Should look like:", rowList2
186
assert len(rowList1) == len(rowList2)
187
assert rowList1 == rowList2
189
def test06_getWhereList(self):
190
"""Checking reading an Index with getWhereList (bool flavor)"""
193
print '\n', '-=' * 30
194
print "Running %s.test06_getWhereList..." % self.__class__.__name__
196
# Open the HDF5 file in read-only mode
197
self.fileh = openFile(self.file, mode = "r")
198
table = self.fileh.root.table
199
idxcol = table.cols.var4.index
201
print "Max rows in buf:", table._v_maxTuples
202
print "Number of elements per slice:", idxcol.nelemslice
203
print "Chunk size:", idxcol.sorted.chunksize
206
rowList1 = table.getWhereList(table.cols.var2 == 0, "NumArray")
207
#rowList2 = [p.nrow() for p in table.where(table.cols.var2 == 0)]
208
rowList2 = [p.nrow() for p in table if p['var2'] == 0]
209
# Convert to a numarray object
210
rowList2 = numarray.array(rowList2, numarray.Int64)
212
print "Selected values:", rowList1
213
print "Should look like:", rowList2
214
assert len(rowList1) == len(rowList2)
215
assert allequal(rowList1, rowList2)
217
def test07_getWhereList(self):
218
"""Checking reading an Index with getWhereList (int flavor)"""
221
print '\n', '-=' * 30
222
print "Running %s.test07_getWhereList..." % self.__class__.__name__
224
# Open the HDF5 file in read-only mode
225
self.fileh = openFile(self.file, mode = "r")
226
table = self.fileh.root.table
227
idxcol = table.cols.var4.index
229
print "Max rows in buf:", table._v_maxTuples
230
print "Number of elements per slice:", idxcol.nelemslice
231
print "Chunk size:", idxcol.sorted.chunksize
234
rowList1 = table.getWhereList(table.cols.var3 < 15, "Tuple")
235
rowList2 = tuple([p.nrow() for p in table if p["var3"] < 15])
237
print "Selected values:", rowList1
238
print "Should look like:", rowList2
239
assert len(rowList1) == len(rowList2)
240
assert rowList1 == rowList2
242
def test08_getWhereList(self):
243
"""Checking reading an Index with getWhereList (float flavor)"""
246
print '\n', '-=' * 30
247
print "Running %s.test08_getWhereList..." % self.__class__.__name__
249
# Open the HDF5 file in read-only mode
250
self.fileh = openFile(self.file, mode = "r")
251
table = self.fileh.root.table
252
idxcol = table.cols.var4.index
254
print "Max rows in buf:", table._v_maxTuples
255
print "Number of elements per slice:", idxcol.nelemslice
256
print "Chunk size:", idxcol.sorted.chunksize
259
rowList1 = table.getWhereList(table.cols.var4 < 10, "List")
260
rowList2 = [p.nrow() for p in table if p['var4'] < 10]
262
print "Selected values:", rowList1
263
print "Should look like:", rowList2
264
assert len(rowList1) == len(rowList2)
265
assert rowList1 == rowList2
267
def test09_removeIndex(self):
268
"""Checking removing an index"""
271
print '\n', '-=' * 30
272
print "Running %s.test09_removeIndex..." % self.__class__.__name__
274
# Open the HDF5 file in read-only mode
275
self.fileh = openFile(self.file, mode = "a")
276
table = self.fileh.root.table
277
idxcol = table.cols.var1.index
279
print "Before deletion"
280
print "var1 column:", idxcol
281
assert idxcol is not None
282
assert table.colindexed["var1"] == 1
285
table.removeIndex(idxcol)
287
print "After deletion"
288
print "var1 column:", idxcol
289
assert table.cols.var1.index is None
290
assert table.colindexed["var1"] == 0
292
# re-create the index again
293
indexrows = table.cols.var1.createIndex(testmode=1)
294
idxcol = table.cols.var1.index
296
print "After re-creation"
297
print "var1 column:", idxcol
298
assert idxcol is not None
299
assert table.colindexed["var1"] == 1
301
def test10_removeIndex(self):
302
"""Checking removing an index (persistent version)"""
305
print '\n', '-=' * 30
306
print "Running %s.test10_removeIndex..." % self.__class__.__name__
308
# Open the HDF5 file in read-only mode
309
self.fileh = openFile(self.file, mode = "a")
310
table = self.fileh.root.table
311
idxcol = table.cols.var1.index
313
print "Before deletion"
314
print "var1 column:", idxcol
315
assert idxcol is not None
316
assert table.colindexed["var1"] == 1
318
table.removeIndex(idxcol)
320
# close and reopen the file
322
self.fileh = openFile(self.file, mode = "a")
323
table = self.fileh.root.table
324
idxcol = table.cols.var1.index
327
print "After deletion"
328
print "var1 column:", idxcol
329
assert table.cols.var1.index is None
330
assert table.colindexed["var1"] == 0
332
# re-create the index again
333
indexrows = table.cols.var1.createIndex(testmode=1)
334
idxcol = table.cols.var1.index
336
print "After re-creation"
337
print "var1 column:", idxcol
338
assert idxcol is not None
339
assert table.colindexed["var1"] == 1
341
class BasicReadTestCase(BasicTestCase):
346
ns, cs = calcChunksize(minRowIndex, testmode=1)
349
class ZlibReadTestCase(BasicTestCase):
354
ns, cs = calcChunksize(minRowIndex, testmode=1)
357
class LZOReadTestCase(BasicTestCase):
362
ns, cs = calcChunksize(minRowIndex, testmode=1)
365
class UCLReadTestCase(BasicTestCase):
370
ns, cs = calcChunksize(minRowIndex, testmode=1)
373
class ShuffleReadTestCase(BasicTestCase):
378
ns, cs = calcChunksize(minRowIndex, testmode=1)
381
class Fletcher32ReadTestCase(BasicTestCase):
386
ns, cs = calcChunksize(minRowIndex, testmode=1)
389
class ShuffleFletcher32ReadTestCase(BasicTestCase):
394
ns, cs = calcChunksize(minRowIndex, testmode=1)
397
class OneHalfTestCase(BasicTestCase):
398
ns, cs = calcChunksize(minRowIndex, testmode=1)
401
class UpperBoundTestCase(BasicTestCase):
402
ns, cs = calcChunksize(minRowIndex, testmode=1)
405
class LowerBoundTestCase(BasicTestCase):
406
ns, cs = calcChunksize(minRowIndex, testmode=1)
409
class WarningTestCase(unittest.TestCase):
410
nrows = 100 # Small enough to raise the warning
413
"Checking the user waring for too few entries to index"
414
# Create an instance of an HDF5 Table
415
self.file = tempfile.mktemp(".h5")
416
self.fileh = openFile(self.file, "w")
417
self.rootgroup = self.fileh.root
418
group = self.rootgroup
420
title = "This is the IndexArray title"
422
table = self.fileh.createTable(group, 'table', Small, title,
424
for i in range(self.nrows):
425
# Fill rows with defaults
428
# try to index one entry
429
warnings.filterwarnings("error", category=UserWarning)
431
indexrows = table.cols.var1.createIndex()
434
(type, value, traceback) = sys.exc_info()
435
print "\nGreat!, the next UserWarning was catched!"
438
self.fail("expected an UserWarning")
440
warnings.filterwarnings("default", category=UserWarning)
445
class Small2(IsDescription):
446
_v_indexprops = IndexProps(auto=0)
447
var1 = StringCol(length=4, dflt="", pos=1, indexed=1)
448
var2 = BoolCol(0, indexed=1, pos = 2)
449
var3 = IntCol(0, indexed=1, pos = 3)
450
var4 = FloatCol(0, indexed=0, pos = 4)
452
class Small3(IsDescription):
453
_v_indexprops = IndexProps(reindex=0)
454
var1 = StringCol(length=4, dflt="", indexed=1, pos=1)
455
var2 = BoolCol(0, indexed=1, pos=2)
456
var3 = IntCol(0, indexed=1, pos=3)
457
var4 = FloatCol(0, indexed=0, pos=4)
459
class Small4(IsDescription):
460
_v_indexprops = IndexProps(filters=Filters(complevel=6, complib="zlib",
461
shuffle=0, fletcher32=1))
462
var1 = StringCol(length=4, dflt="", indexed=1, pos=1)
463
var2 = BoolCol(0, indexed=1, pos=2)
464
var3 = IntCol(0, indexed=1, pos=3)
465
var4 = FloatCol(0, indexed=0, pos=4)
468
class AutomaticIndexingTestCase(unittest.TestCase):
473
# Create an instance of an HDF5 Table
474
self.file = tempfile.mktemp(".h5")
475
self.fileh = openFile(self.file, "w")
477
title = "This is the IndexArray title"
479
root = self.fileh.root
480
self.table = self.fileh.createTable(root, 'table', self.klass, title,
482
for i in range(self.nrows):
483
# Fill rows with defaults
484
self.table.row.append()
488
self.fileh = openFile(self.file, "a")
489
self.table = self.fileh.root.table
495
def test01_attrs(self):
496
"Checking indexing attributes (part1)"
498
print '\n', '-=' * 30
499
print "Running %s.test01_attrs..." % self.__class__.__name__
502
if self.klass is Small:
503
assert table.indexed == 0
505
assert table.indexed == 1
506
if self.klass is Small:
507
assert table.colindexed["var1"] == 0
508
assert table.cols.var1.index is None
509
assert table.colindexed["var2"] == 0
510
assert table.cols.var2.index is None
511
assert table.colindexed["var3"] == 0
512
assert table.cols.var3.index is None
513
assert table.colindexed["var4"] == 0
514
assert table.cols.var4.index is None
516
# Check that the var1, var2 and var3 (and only these)
518
assert table.colindexed["var1"] == 1
519
assert table.cols.var1.index is not None
520
assert table.colindexed["var2"] == 1
521
assert table.cols.var2.index is not None
522
assert table.colindexed["var3"] == 1
523
assert table.cols.var3.index is not None
524
assert table.colindexed["var4"] == 0
525
assert table.cols.var4.index is None
527
def test02_attrs(self):
528
"Checking indexing attributes (part2)"
530
print '\n', '-=' * 30
531
print "Running %s.test02_attrs..." % self.__class__.__name__
534
# Check the policy parameters
537
print "indexprops:", table.indexprops
539
print "Table is not indexed"
540
# Check non-default values for index saving policy
541
if self.klass is Small:
542
assert not hasattr(table, "indexprops")
543
elif self.klass is Small2:
544
assert table.indexprops.auto == 0
545
assert table.indexprops.reindex == 1
546
filters = Filters(complevel=1, complib="zlib",
547
shuffle=1, fletcher32=0)
548
assert str(table.indexprops.filters) == str(filters)
549
elif self.klass is Small3:
550
assert table.indexprops.auto == 1
551
assert table.indexprops.reindex == 0
552
filters = Filters(complevel=1, complib="zlib",
553
shuffle=1, fletcher32=0)
554
assert str(table.indexprops.filters) == str(filters)
555
elif self.klass is Small4:
556
assert table.indexprops.auto == 1
557
assert table.indexprops.reindex == 1
558
filters = Filters(complevel=6, complib="zlib",
559
shuffle=0, fletcher32=1)
560
assert str(table.indexprops.filters) == str(filters)
562
# Check Index() objects exists and are properly placed
563
if self.klass is Small:
564
assert table.cols.var1.index == None
565
assert table.cols.var2.index == None
566
assert table.cols.var3.index == None
567
assert table.cols.var4.index == None
569
assert isinstance(table.cols.var1.index, Index)
570
assert isinstance(table.cols.var2.index, Index)
571
assert isinstance(table.cols.var3.index, Index)
572
assert table.cols.var4.index == None
574
def test03_counters(self):
575
"Checking indexing counters"
577
print '\n', '-=' * 30
578
print "Running %s.test03_counters..." % self.__class__.__name__
580
# Check the counters for indexes
583
print "indexedrows:", table._indexedrows
584
print "unsavedindexedrows:", table._unsaved_indexedrows
585
index = table.cols.var1.index
586
indexedrows = index.nrows * index.nelemslice
587
print "computed indexed rows:", indexedrows
589
print "Table is not indexed"
590
if self.klass is not Small:
591
index = table.cols.var1.index
592
indexedrows = index.nrows * index.nelemslice
593
assert table._indexedrows == indexedrows
594
indexedrows = index.nelements
595
assert table._indexedrows == indexedrows
596
assert table._unsaved_indexedrows == self.nrows - indexedrows
598
def test04_noauto(self):
599
"Checking indexing counters (non-automatic mode)"
601
print '\n', '-=' * 30
602
print "Running %s.test04_noauto..." % self.__class__.__name__
604
# Force a sync in indexes
605
table.flushRowsToIndex()
606
# Check the counters for indexes
609
print "indexedrows:", table._indexedrows
610
print "unsavedindexedrows:", table._unsaved_indexedrows
611
index = table.cols.var1.index
612
indexedrows = index.nrows * index.nelemslice
613
print "computed indexed rows:", indexedrows
615
print "Table is not indexed"
617
# No unindexated rows should remain
618
index = table.cols.var1.index
619
if self.klass is Small:
622
indexedrows = index.nrows * index.nelemslice
623
assert table._indexedrows == indexedrows
624
indexedrows = index.nelements
625
assert table._indexedrows == indexedrows
626
assert table._unsaved_indexedrows == self.nrows - indexedrows
628
# Check non-default values for index saving policy
629
if self.klass is Small:
630
assert not hasattr(table, "indexprops")
631
elif self.klass is Small2:
632
assert table.indexprops.auto == 0
633
assert table.indexprops.reindex == 1
634
filters = Filters(complevel=1, complib="zlib",
635
shuffle=1, fletcher32=0)
636
assert str(table.indexprops.filters) == str(filters)
637
elif self.klass is Small3:
638
assert table.indexprops.auto == 1
639
assert table.indexprops.reindex == 0
640
filters = Filters(complevel=1, complib="zlib",
641
shuffle=1, fletcher32=0)
642
assert str(table.indexprops.filters) == str(filters)
643
elif self.klass is Small4:
644
assert table.indexprops.auto == 1
645
assert table.indexprops.reindex == 1
646
filters = Filters(complevel=6, complib="zlib",
647
shuffle=0, fletcher32=1)
648
assert str(table.indexprops.filters) == str(filters)
651
def test05_icounters(self):
652
"Checking indexing counters (removeRows)"
654
print '\n', '-=' * 30
655
print "Running %s.test05_icounters..." % self.__class__.__name__
657
# Force a sync in indexes
658
table.flushRowsToIndex()
659
# No unidexated rows should remain here
660
if self.klass is not Small:
661
indexedrows = table._indexedrows
662
unsavedindexedrows = table._unsaved_indexedrows
663
# Now, remove some rows:
664
table.removeRows(3,5)
667
self.fileh = openFile(self.file, "a")
668
table = self.fileh.root.table
669
# Check the counters for indexes
672
print "indexedrows:", table._indexedrows
673
print "original indexedrows:", indexedrows
674
print "unsavedindexedrows:", table._unsaved_indexedrows
675
print "original unsavedindexedrows:", unsavedindexedrows
676
index = table.cols.var1.index
677
indexedrows = index.nelements
678
print "computed indexed rows:", indexedrows
679
print "index dirty:", table.cols.var1.dirty
681
print "Table is not indexed"
684
assert table.nrows == self.nrows - 2
685
if self.klass is Small3:
686
# The unsaved indexed rows counter should be unchanged
687
assert table._indexedrows == indexedrows
689
assert table._unsaved_indexedrows == unsavedindexedrows - 2
691
assert table._unsaved_indexedrows == unsavedindexedrows
692
elif self.klass is Small2:
693
index = table.cols.var1.index
694
indexedrows = index.nrows * index.nelemslice
695
assert table._indexedrows == indexedrows
696
indexedrows = index.nelements
697
assert table._indexedrows == indexedrows
698
assert table._unsaved_indexedrows == self.nrows - indexedrows - 2
700
# Check non-default values for index saving policy
701
if self.klass is Small:
702
assert not hasattr(table, "indexprops")
703
elif self.klass is Small2:
704
assert table.indexprops.auto == 0
705
assert table.indexprops.reindex == 1
706
filters = Filters(complevel=1, complib="zlib",
707
shuffle=1, fletcher32=0)
708
assert str(table.indexprops.filters) == str(filters)
709
elif self.klass is Small3:
710
assert table.indexprops.auto == 1
711
assert table.indexprops.reindex == 0
712
filters = Filters(complevel=1, complib="zlib",
713
shuffle=1, fletcher32=0)
714
assert str(table.indexprops.filters) == str(filters)
715
elif self.klass is Small4:
716
assert table.indexprops.auto == 1
717
assert table.indexprops.reindex == 1
718
filters = Filters(complevel=6, complib="zlib",
719
shuffle=0, fletcher32=1)
720
assert str(table.indexprops.filters) == str(filters)
723
def test06_dirty(self):
724
"Checking dirty flags (removeRows action)"
726
print '\n', '-=' * 30
727
print "Running %s.test06_dirty..." % self.__class__.__name__
729
# Force a sync in indexes
730
table.flushRowsToIndex()
731
# Now, remove some rows:
732
table.removeRows(3,5)
735
self.fileh = openFile(self.file, "a")
736
table = self.fileh.root.table
737
# Check the dirty flag for indexes
739
for colname in table.colnames:
740
print "dirty flag col %s: %s" % \
741
(colname, table.cols[colname].dirty)
743
for colname in table.colnames:
744
if (table.cols[colname].index and not table.indexprops.reindex):
745
assert table.cols[colname].dirty == 1
747
assert table.cols[colname].dirty == 0
749
def test07_noreindex(self):
750
"Checking indexing counters (modifyRows, no-reindex mode)"
752
print '\n', '-=' * 30
753
print "Running %s.test07_noreindex..." % self.__class__.__name__
755
# Force a sync in indexes
756
table.flushRowsToIndex()
757
# Non indexated rows should remain here
758
if self.klass is not Small:
759
indexedrows = table._indexedrows
760
unsavedindexedrows = table._unsaved_indexedrows
761
# Now, modify just one row:
762
table.modifyRows(3, None, 1, [["asa",0,3,3.1]])
765
self.fileh = openFile(self.file, "a")
766
table = self.fileh.root.table
767
# Check the counters for indexes
770
print "indexedrows:", table._indexedrows
771
print "unsavedindexedrows:", table._unsaved_indexedrows
772
index = table.cols.var1.index
773
indexedrows = index.nelements
774
print "computed indexed rows:", indexedrows
776
print "Table is not indexed"
778
assert table.nrows == self.nrows
779
if self.klass is Small3:
780
# The unsaved indexed rows counter should be unchanged
781
assert table._indexedrows == indexedrows
782
assert table._unsaved_indexedrows == unsavedindexedrows
783
elif self.klass is Small2:
784
index = table.cols.var1.index
785
indexedrows = index.nrows * index.nelemslice
786
assert table._indexedrows == indexedrows
787
indexedrows = index.nelements
788
assert table._indexedrows == indexedrows
789
assert table._unsaved_indexedrows == self.nrows - indexedrows
791
# Check the dirty flag for indexes
793
for colname in table.colnames:
794
print "dirty flag col %s: %s" % \
795
(colname, table.cols[colname].dirty)
796
for colname in table.colnames:
797
if (table.cols[colname].index and not table.indexprops.reindex):
798
assert table.cols[colname].dirty == 1
800
assert table.cols[colname].dirty == 0
802
def test08_dirty(self):
803
"Checking dirty flags (modifyColumns)"
805
print '\n', '-=' * 30
806
print "Running %s.test08_dirty..." % self.__class__.__name__
808
# Force a sync in indexes
809
table.flushRowsToIndex()
810
# Non indexated rows should remain here
811
if self.klass is not Small:
812
indexedrows = table._indexedrows
813
unsavedindexedrows = table._unsaved_indexedrows
814
# Now, modify a couple of rows:
815
table.modifyColumns(1, columns=[["asa","asb"],[1.,2.]],
816
names=["var1", "var4"])
819
self.fileh = openFile(self.file, "a")
820
table = self.fileh.root.table
823
assert table.nrows == self.nrows
824
if self.klass is Small3:
825
# The unsaved indexed rows counter should be unchanged
826
assert table._indexedrows == indexedrows
827
assert table._unsaved_indexedrows == unsavedindexedrows
828
elif self.klass is Small2:
829
index = table.cols.var1.index
830
indexedrows = index.nrows * index.nelemslice
831
assert table._indexedrows == indexedrows
832
indexedrows = index.nelements
833
assert table._indexedrows == indexedrows
834
assert table._unsaved_indexedrows == self.nrows - indexedrows
836
# Check the dirty flag for indexes
838
for colname in table.colnames:
839
print "dirty flag col %s: %s" % \
840
(colname, table.cols[colname].dirty)
841
for colname in table.colnames:
842
if (table.cols[colname].index and
843
not table.indexprops.reindex):
844
if colname in ["var1"]:
845
assert table.cols[colname].dirty == 1
847
assert table.cols[colname].dirty == 0
849
assert table.cols[colname].dirty == 0
851
def test09_copyIndex(self):
852
"Checking copy Index feature in copyTable (attrs)"
854
print '\n', '-=' * 30
855
print "Running %s.test09_copyIndex..." % self.__class__.__name__
857
# Don't force a sync in indexes
858
#table.flushRowsToIndex()
859
# Non indexated rows should remain here
860
if self.klass is not Small:
861
indexedrows = table._indexedrows
862
unsavedindexedrows = table._unsaved_indexedrows
863
# Now, remove some rows to make columns dirty
864
#table.removeRows(3,5)
865
# Copy a Table to another location
866
table2, size = table.copy("/", 'table2')
869
self.fileh = openFile(self.file, "a")
870
table = self.fileh.root.table
871
table2 = self.fileh.root.table2
873
index1 = table.cols.var1.index
874
index2 = table2.cols.var1.index
876
print "Copied index:", index2
877
print "Original index:", index1
879
print "Elements in copied index:", index2.nelements
880
print "Elements in original index:", index1.nelements
882
assert table.nrows == table2.nrows
884
assert table2.indexed
885
assert table._indexedrows == table2._indexedrows
886
assert table._unsaved_indexedrows == table2._unsaved_indexedrows
887
if self.klass is Small:
888
# No index: the index should not exist
889
assert index1 is None
890
assert index2 is None
891
elif self.klass is Small2:
892
# No auto: the index should exists, but be empty
893
assert index2 is not None
894
assert index2.nelements == 0
895
elif self.klass is Small3:
896
# Auto: the index should exists, and have elements
897
assert index2 is not None
898
assert index2.nelements == index1.nelements
900
# Check the dirty flag for indexes
902
for colname in table2.colnames:
903
print "dirty flag col %s: %s" % \
904
(colname, table2.cols[colname].dirty)
905
for colname in table2.colnames:
906
assert table2.cols[colname].dirty == 0
908
def test10_copyIndex(self):
909
"Checking copy Index feature in copyTable (values)"
911
print '\n', '-=' * 30
912
print "Running %s.test10_copyIndex..." % self.__class__.__name__
914
# Don't force a sync in indexes
915
#table.flushRowsToIndex()
916
# Non indexated rows should remain here
917
if self.klass is not Small:
918
indexedrows = table._indexedrows
919
unsavedindexedrows = table._unsaved_indexedrows
920
# Now, remove some rows to make columns dirty
921
#table.removeRows(3,5)
922
# Copy a Table to another location
923
table2, size = table.copy("/", 'table2')
926
self.fileh = openFile(self.file, "a")
927
table = self.fileh.root.table
928
table2 = self.fileh.root.table2
930
index1 = table.cols.var3.index
931
index2 = table2.cols.var3.index
933
print "Copied index:", index2
934
print "Original index:", index1
936
print "Elements in copied index:", index2.nelements
937
print "Elements in original index:", index1.nelements
938
if index2.nelements > 10:
939
print "First 10 elements in copied index (sorted):\n", \
941
print "First 10 elements in orig index (sorted):\n", \
943
print "First 10 elements in copied index (indices):\n", \
944
index2.indices[0,:10]
945
print "First 10 elements in orig index (indices):\n", \
946
index1.indices[0,:10]
947
if self.klass is Small3:
948
# Auto: the index should exists, and have equal elements
949
assert allequal(index2.sorted.read(), index1.sorted.read())
950
# The next assertion cannot be guaranteed. Why?
951
# sorting algorithm in numarray is not deterministic?
952
#assert allequal(index2.indices.read(), index1.indices.read())
954
def test11_copyIndex(self):
955
"Checking copy Index feature in copyTable (dirty flags)"
957
print '\n', '-=' * 30
958
print "Running %s.test11_copyIndex..." % self.__class__.__name__
960
# Force a sync in indexes
961
table.flushRowsToIndex()
962
# Non indexated rows should remain here
963
if self.klass is not Small:
964
indexedrows = table._indexedrows
965
unsavedindexedrows = table._unsaved_indexedrows
966
# Now, modify an indexed column and an unindexed one
967
# to make the "var1" dirty
968
table.modifyColumns(1, columns=[["asa","asb"],[1.,2.]],
969
names=["var1", "var4"])
970
# Copy a Table to another location
971
table2, size = table.copy("/", 'table2')
974
self.fileh = openFile(self.file, "a")
975
table = self.fileh.root.table
976
table2 = self.fileh.root.table2
978
index1 = table.cols.var1.index
979
index2 = table2.cols.var1.index
981
print "Copied index:", index2
982
print "Original index:", index1
984
print "Elements in copied index:", index2.nelements
985
print "Elements in original index:", index1.nelements
987
# Check the dirty flag for indexes
989
for colname in table2.colnames:
990
print "dirty flag col %s: %s" % \
991
(colname, table2.cols[colname].dirty)
992
for colname in table2.colnames:
993
if (table2.cols[colname].index and
994
not table2.indexprops.reindex):
995
if colname in ["var1"]:
996
#print "-->", index2.sorted[:]
997
# All the destination columns should be non-dirty because
998
# the copy removes the dirty state and puts the
999
# index in a sane state
1000
assert table.cols[colname].dirty == 1
1001
assert table2.cols[colname].dirty == 0
1003
assert table2.cols[colname].dirty == 0
1005
assert table2.cols[colname].dirty == 0
1009
class AI1TestCase(AutomaticIndexingTestCase):
1014
class AI2TestCase(AutomaticIndexingTestCase):
1019
class AI3TestCase(AutomaticIndexingTestCase):
1024
class AI4aTestCase(AutomaticIndexingTestCase):
1029
class AI4bTestCase(AutomaticIndexingTestCase):
1034
class AI5TestCase(AutomaticIndexingTestCase):
1035
ns, cs = calcChunksize(minRowIndex, testmode=0)
1040
class AI6TestCase(AutomaticIndexingTestCase):
1041
ns, cs = calcChunksize(minRowIndex, testmode=0)
1046
class AI7TestCase(AutomaticIndexingTestCase):
1047
ns, cs = calcChunksize(minRowIndex, testmode=0)
1052
class AI8TestCase(AutomaticIndexingTestCase):
1053
ns, cs = calcChunksize(minRowIndex, testmode=0)
1058
class AI9TestCase(AutomaticIndexingTestCase):
1059
ns, cs = calcChunksize(minRowIndex, testmode=1)
1064
class AI10TestCase(AutomaticIndexingTestCase):
1069
class AI11TestCase(AutomaticIndexingTestCase):
1074
class AI12TestCase(AutomaticIndexingTestCase):
1080
#----------------------------------------------------------------------
1083
theSuite = unittest.TestSuite()
1085
#heavy = 1 # Uncomment this only for testing purposes!
1087
#theSuite.addTest(unittest.makeSuite(AI3TestCase))
1088
#theSuite.addTest(unittest.makeSuite(AI4aTestCase))
1089
#theSuite.addTest(unittest.makeSuite(AI4aTestCase))
1090
#theSuite.addTest(unittest.makeSuite(BasicReadTestCase))
1091
#theSuite.addTest(unittest.makeSuite(OneHalfTestCase))
1092
#theSuite.addTest(unittest.makeSuite(UpperBoundTestCase))
1093
#theSuite.addTest(unittest.makeSuite(LowerBoundTestCase))
1095
for n in range(niter):
1096
theSuite.addTest(unittest.makeSuite(BasicReadTestCase))
1097
theSuite.addTest(unittest.makeSuite(ZlibReadTestCase))
1098
theSuite.addTest(unittest.makeSuite(LZOReadTestCase))
1099
theSuite.addTest(unittest.makeSuite(UCLReadTestCase))
1100
theSuite.addTest(unittest.makeSuite(ShuffleReadTestCase))
1101
theSuite.addTest(unittest.makeSuite(Fletcher32ReadTestCase))
1102
theSuite.addTest(unittest.makeSuite(ShuffleFletcher32ReadTestCase))
1103
theSuite.addTest(unittest.makeSuite(OneHalfTestCase))
1104
theSuite.addTest(unittest.makeSuite(UpperBoundTestCase))
1105
theSuite.addTest(unittest.makeSuite(LowerBoundTestCase))
1106
theSuite.addTest(unittest.makeSuite(WarningTestCase))
1107
theSuite.addTest(unittest.makeSuite(AI1TestCase))
1108
theSuite.addTest(unittest.makeSuite(AI2TestCase))
1109
theSuite.addTest(unittest.makeSuite(AI3TestCase))
1110
theSuite.addTest(unittest.makeSuite(AI4aTestCase))
1111
theSuite.addTest(unittest.makeSuite(AI9TestCase))
1112
theSuite.addTest(unittest.makeSuite(AI10TestCase))
1115
# These are too heavy for normal testing
1116
theSuite.addTest(unittest.makeSuite(AI4bTestCase))
1117
theSuite.addTest(unittest.makeSuite(AI5TestCase))
1118
theSuite.addTest(unittest.makeSuite(AI6TestCase))
1119
theSuite.addTest(unittest.makeSuite(AI7TestCase))
1120
theSuite.addTest(unittest.makeSuite(AI8TestCase))
1121
theSuite.addTest(unittest.makeSuite(AI11TestCase))
1122
theSuite.addTest(unittest.makeSuite(AI12TestCase))
1126
if __name__ == '__main__':
1127
unittest.main( defaultTest='suite' )
added
removed
test/test_indexes.py
