← Back to branch summary

~isviridov/magnetodb/master

« back to all changes in this revision

Viewing changes to magnetodb/storage/impl/cassandra_impl.py

  • Committer: ikhudoshyn
  • Date: 2014-02-28 14:49:28 UTC
  • mfrom: (110.1.1)
  • Revision ID: git-v1:a44af23b923d02e182c4b9ca1e31bbc304ec059d
Merge pull request #106 from dukhlov/master

conditions support implemented, cassandra storage backend refactoring done

Show diffs side-by-side

added added

removed removed

Lines of Context:
magnetodb/storage/impl/cassandra_impl.py
32
32
LOG = logging.getLogger(__name__)
33
33
 
34
34
 
35
 
class CassandraStorageImpl():
 
35
class CassandraStorageImpl(object):
36
36
 
37
37
    STORAGE_TO_CASSANDRA_TYPES = {
38
38
        models.ATTRIBUTE_TYPE_STRING: 'text',
222
222
        cas_table_name = self.USER_TABLE_PREFIX + table_schema.table_name
223
223
 
224
224
        user_columns = [
225
 
            "\"{}{}\" {}".format(
226
 
                self.USER_COLUMN_PREFIX, attr_def.name,
227
 
                self.STORAGE_TO_CASSANDRA_TYPES[attr_def.type]
 
225
            '"{}{}" {}'.format(
 
226
                self.USER_COLUMN_PREFIX, attr_name,
 
227
                self.STORAGE_TO_CASSANDRA_TYPES[attr_type]
228
228
            )
229
 
            for attr_def in table_schema.attribute_defs
 
229
            for attr_name, attr_type in
 
230
            table_schema.attribute_type_map.iteritems()
230
231
        ]
231
232
 
232
233
        hash_name = table_schema.key_attributes[0]
233
 
        hash_type = [
234
 
            attr.type for attr in table_schema.attribute_defs
235
 
            if attr.name == hash_name
236
 
        ][0]
 
234
        hash_type = table_schema.attribute_type_map[hash_name]
237
235
 
238
236
        cassandra_hash_type = self.STORAGE_TO_CASSANDRA_TYPES[hash_type]
239
237
 
280
278
 
281
279
            LOG.debug("Waiting for schema agreement... Done")
282
280
 
283
 
            for index_def in table_schema.index_defs:
 
281
            for index_name, index_def in (
 
282
                    table_schema.index_def_map.iteritems()):
284
283
                self._create_index(context.tenant, cas_table_name,
285
284
                                   self.USER_COLUMN_PREFIX +
286
285
                                   index_def.attribute_to_index,
287
 
                                   index_def.index_name)
 
286
                                   index_name)
288
287
 
289
288
            self._create_index(
290
289
                context.tenant, cas_table_name, self.SYSTEM_COLUMN_HASH,
382
381
                        in table_meta.columns.iteritems()
383
382
                        if key.startswith(self.USER_COLUMN_PREFIX)]
384
383
 
385
 
        attr_defs = set()
386
 
        index_defs = set()
 
384
        attribute_type_map = {}
 
385
        index_def_map = {}
387
386
 
388
387
        for column in user_columns:
389
388
            name = column.name[prefix_len:]
390
389
            storage_type = self.CASSANDRA_TO_STORAGE_TYPES[column.typestring]
391
 
            attr_defs.add(models.AttributeDefinition(name, storage_type))
 
390
            attribute_type_map[name] = storage_type
392
391
            if column.index:
393
 
                index_defs.add(models.IndexDefinition(
394
 
                    column.index.name[len(table_name) + 1:], name)
 
392
                index_def_map[column.index.name[len(table_name) + 1:]] = (
 
393
                    models.IndexDefinition(name)
395
394
                )
396
395
 
397
396
        hash_key_name = table_meta.partition_key[0].name[prefix_len:]
402
401
            range_key_name = table_meta.clustering_key[0].name[prefix_len:]
403
402
            key_attrs.append(range_key_name)
404
403
 
405
 
        table_schema = models.TableSchema(table_name, attr_defs,
406
 
                                          key_attrs, index_defs)
 
404
        table_schema = models.TableSchema(table_name, attribute_type_map,
 
405
                                          key_attrs, index_def_map)
407
406
 
408
407
        self._save_table_schema_to_cache(context.tenant, table_name,
409
408
                                         table_schema)
422
421
        """
423
422
 
424
423
        query_builder = [
425
 
            "SELECT \"columnfamily_name\"",
426
 
            " FROM \"system\".\"schema_columnfamilies\"",
427
 
            " WHERE \"keyspace_name\"='", context.tenant, "'"
 
424
            "SELECT columnfamily_name",
 
425
            " FROM system.schema_columnfamilies",
 
426
            " WHERE keyspace_name='", context.tenant, "'"
428
427
        ]
429
428
 
430
429
        if exclusive_start_table_name:
431
430
            query_builder += (
432
 
                " AND \"columnfamily_name\" > '",
 
431
                " AND columnfamily_name > '",
433
432
                exclusive_start_table_name, "'"
434
433
            )
435
434
 
436
435
        if limit:
437
436
            query_builder += (
438
 
                " AND \"columnfamily_name\" > '",
 
437
                " AND columnfamily_name > '",
439
438
                exclusive_start_table_name, "'"
440
439
            )
441
440
            query_builder += (" LIMIT ", str(limit))
476
475
            put_attr_map[hash_key_name]
477
476
        )
478
477
 
479
 
        not_processed_predefined_attr_names = set()
480
 
        for attr in schema.attribute_defs:
481
 
            not_processed_predefined_attr_names.add(attr.name)
 
478
        not_processed_predefined_attr_names = set(
 
479
            schema.attribute_type_map.keys()
 
480
        )
 
481
 
 
482
        query_builder = None
482
483
 
483
484
        if expected_condition_map:
484
485
            query_builder = [
485
 
                "UPDATE \"", context.tenant, "\".\"", self.USER_TABLE_PREFIX,
486
 
                put_request.table_name, "\" SET "]
 
486
                'UPDATE "', context.tenant, '"."', self.USER_TABLE_PREFIX,
 
487
                put_request.table_name, '" SET ']
487
488
 
488
489
            dynamic_attr_names = []
489
490
            dynamic_attr_values = []
 
491
 
490
492
            for name, val in put_attr_map.iteritems():
491
493
                if name in key_attr_names:
492
494
                    not_processed_predefined_attr_names.remove(name)
493
495
                elif name in not_processed_predefined_attr_names:
494
496
                    query_builder += (
495
 
                        "\"", self.USER_COLUMN_PREFIX, name, "\"=", val, ","
 
497
                        '"', self.USER_COLUMN_PREFIX, name, '"=',
 
498
                        self._encode_predefined_attr_value(val), ","
496
499
                    )
497
500
                    not_processed_predefined_attr_names.remove(name)
498
501
                else:
501
504
                        self._encode_dynamic_attr_value(val)
502
505
                    )
503
506
 
504
 
            query_builder.append("{")
505
 
            dynamic_value_iter = iter(dynamic_attr_values)
506
 
            for name in dynamic_attr_names:
507
 
                query_builder += (
508
 
                    "'", name, "':" + dynamic_value_iter.next(), ","
509
 
                )
510
 
            query_builder.pop()
511
 
            query_builder.append("}, ")
 
507
            query_builder += (self.SYSTEM_COLUMN_ATTRS, "={")
 
508
            if dynamic_attr_values:
 
509
                dynamic_value_iter = iter(dynamic_attr_values)
 
510
                for name in dynamic_attr_names:
 
511
                    query_builder += (
 
512
                        "'", name, "':", dynamic_value_iter.next(), ","
 
513
                    )
 
514
                query_builder.pop()
 
515
            query_builder.append("},")
512
516
 
513
517
            for name in not_processed_predefined_attr_names:
514
518
                query_builder += (
515
 
                    "\"", self.USER_COLUMN_PREFIX, name, "\"=null,"
 
519
                    '"', self.USER_COLUMN_PREFIX, name,
 
520
                    '"=null,'
516
521
                )
517
522
 
518
523
            query_builder += (
519
524
                self.SYSTEM_COLUMN_ATTR_TYPES, "={", types, "},",
520
525
                self.SYSTEM_COLUMN_ATTR_EXIST, "={", exists, "},",
521
526
                self.SYSTEM_COLUMN_HASH, "=", encoded_hash_key_value,
522
 
                " WHERE ", self.USER_COLUMN_PREFIX, hash_key_name, "=",
 
527
                ' WHERE "', self.USER_COLUMN_PREFIX, hash_key_name, '"=',
523
528
                encoded_hash_key_value
524
529
            )
525
530
 
526
 
            if key_attr_names == 2:
 
531
            if len(key_attr_names) == 2:
527
532
                range_key_name = key_attr_names[1]
528
533
                encoded_range_key_value = self._encode_predefined_attr_value(
529
534
                    put_attr_map[range_key_name]
530
535
                )
531
536
 
532
537
                query_builder += (
533
 
                    " AND ", self.USER_COLUMN_PREFIX, range_key_name, "=",
 
538
                    ' AND "', self.USER_COLUMN_PREFIX, range_key_name, '"=',
534
539
                    encoded_range_key_value
535
540
                )
536
541
 
537
 
            if_clause = self._conditions_as_string(expected_condition_map)
538
 
            query_builder += (" IF ", if_clause)
539
 
 
540
 
            self._execute_query("".join(query_builder), consistent=True)
 
542
            if expected_condition_map:
 
543
                query_builder.append(" IF ")
 
544
                self._append_expected_conditions(
 
545
                    expected_condition_map, schema, query_builder
 
546
                )
541
547
        else:
542
548
            query_builder = [
543
 
                "INSERT INTO \"", context.tenant, "\".\"",
544
 
                self.USER_TABLE_PREFIX, put_request.table_name, "\" ("
 
549
                'INSERT INTO "', context.tenant, '"."',
 
550
                self.USER_TABLE_PREFIX, put_request.table_name, '" ('
545
551
            ]
546
552
            attr_values = []
547
553
            dynamic_attr_names = []
549
555
            for name, val in put_attr_map.iteritems():
550
556
                if name in not_processed_predefined_attr_names:
551
557
                    query_builder += (
552
 
                        "\"", self.USER_COLUMN_PREFIX, name, "\","
 
558
                        '"', self.USER_COLUMN_PREFIX, name, '",'
553
559
                    )
554
560
                    attr_values.append(self._encode_predefined_attr_value(val))
555
561
                    not_processed_predefined_attr_names.remove(name)
571
577
                query_builder += (attr_value, ",")
572
578
 
573
579
            query_builder.append("{")
574
 
            dynamic_value_iter = iter(dynamic_attr_values)
575
 
            for name in dynamic_attr_names:
576
 
                query_builder += ("'", name, "':" + dynamic_value_iter.next())
 
580
 
 
581
            if dynamic_attr_values:
 
582
                dynamic_value_iter = iter(dynamic_attr_values)
 
583
                for name in dynamic_attr_names:
 
584
                    query_builder += (
 
585
                        "'", name, "':" + dynamic_value_iter.next(), ","
 
586
                    )
 
587
                query_builder.pop()
577
588
 
578
589
            query_builder += (
579
590
                "},{", types, "},{" + exists + "},", encoded_hash_key_value
582
593
            if if_not_exist:
583
594
                query_builder.append(" IF NOT EXISTS")
584
595
 
585
 
            self._execute_query("".join(query_builder), consistent=True)
 
596
        result = self._execute_query("".join(query_builder), consistent=True)
586
597
 
587
 
        return True
 
598
        return (result is None) or result[0]['[applied]']
588
599
 
589
600
    def _put_types(self, attribute_map):
590
601
        return ','.join((
615
626
 
616
627
        @raise BackendInteractionException
617
628
        """
618
 
        query = "DELETE FROM \"{}\".\"{}{}\" WHERE ".format(
619
 
            context.tenant, self.USER_TABLE_PREFIX, delete_request.table_name)
620
 
 
621
 
        where = self._primary_key_as_string(delete_request.key_attribute_map)
622
 
 
623
 
        query += where
 
629
        query_builder = [
 
630
            'DELETE FROM "', context.tenant, '"."', self.USER_TABLE_PREFIX,
 
631
            delete_request.table_name, '" WHERE '
 
632
        ]
 
633
 
 
634
        query_builder.append(
 
635
            self._primary_key_as_string(delete_request.key_attribute_map)
 
636
        )
624
637
 
625
638
        if expected_condition_map:
626
 
            if_clause = self._conditions_as_string(expected_condition_map)
627
 
 
628
 
            query += " IF " + if_clause
629
 
 
630
 
        self._execute_query(query, consistent=True)
631
 
 
632
 
        return True
633
 
 
634
 
    def _condition_as_string(self, attr, condition):
635
 
        name = self.USER_COLUMN_PREFIX + attr
636
 
 
 
639
            schema = self.describe_table(context, delete_request.table_name)
 
640
            query_builder.append(" IF ")
 
641
            self._append_expected_conditions(
 
642
                expected_condition_map, schema, query_builder
 
643
            )
 
644
 
 
645
        result = self._execute_query("".join(query_builder), consistent=True)
 
646
 
 
647
        return (result is None) or result[0]['[applied]']
 
648
 
 
649
    def _compact_indexed_condition(self, cond_list):
 
650
        left_condition = None
 
651
        right_condition = None
 
652
        exact_condition = None
 
653
 
 
654
        assert cond_list
 
655
 
 
656
        for condition in cond_list:
 
657
            if condition.type == models.IndexedCondition.CONDITION_TYPE_EQUAL:
 
658
                if (exact_condition is not None and
 
659
                        condition.arg.value != exact_condition.arg.value):
 
660
                    return None
 
661
                exact_condition = condition
 
662
            elif condition.is_left_border():
 
663
                if left_condition is None:
 
664
                    left_condition = condition
 
665
                elif condition.is_strict_border():
 
666
                    if condition.arg.value >= left_condition.arg.value:
 
667
                        left_condition = condition
 
668
                else:
 
669
                    if condition.arg.value > left_condition.arg.value:
 
670
                        left_condition = condition
 
671
            elif condition.is_right_border():
 
672
                if right_condition is None:
 
673
                    right_condition = condition
 
674
                elif condition.is_strict():
 
675
                    if condition.arg.value <= right_condition.arg.value:
 
676
                        right_condition = condition
 
677
                else:
 
678
                    if condition.arg.value < right_condition.arg.value:
 
679
                        right_condition = condition
 
680
 
 
681
        if exact_condition is not None:
 
682
            if left_condition is not None:
 
683
                if left_condition.is_strict():
 
684
                    if left_condition.arg.value >= exact_condition.arg.value:
 
685
                        return None
 
686
                else:
 
687
                    if left_condition.arg.value > exact_condition.arg.value:
 
688
                        return None
 
689
            if right_condition is not None:
 
690
                if right_condition.is_strict():
 
691
                    if right_condition.arg.value <= exact_condition.arg.value:
 
692
                        return None
 
693
                else:
 
694
                    if right_condition.arg.value < exact_condition.arg.value:
 
695
                        return None
 
696
            return [exact_condition]
 
697
        elif left_condition is not None:
 
698
            if right_condition is not None:
 
699
                if (left_condition.is_strict_border() or
 
700
                        right_condition.is_strict_border()):
 
701
                    if left_condition.arg.value >= right_condition.arg.value:
 
702
                        return None
 
703
                else:
 
704
                    if left_condition.arg.value > right_condition.arg.value:
 
705
                        return None
 
706
                return [left_condition, right_condition]
 
707
            else:
 
708
                return [left_condition]
 
709
 
 
710
        assert right_condition is not None
 
711
 
 
712
        return [right_condition]
 
713
 
 
714
    def _append_indexed_condition(self, attr_name, condition, query_builder,
 
715
                                  column_prefix=USER_COLUMN_PREFIX):
 
716
        op = self.CONDITION_TO_OP[condition.type]
 
717
        query_builder += (
 
718
            '"', column_prefix, attr_name, '"', op,
 
719
            self._encode_predefined_attr_value(condition.arg)
 
720
        )
 
721
 
 
722
    def _append_hash_key_indexed_condition(
 
723
            self, attr_name, condition, query_builder,
 
724
            column_prefix=USER_COLUMN_PREFIX):
 
725
        if condition.type == models.IndexedCondition.CONDITION_TYPE_EQUAL:
 
726
            self._append_indexed_condition(
 
727
                attr_name, condition, query_builder, column_prefix
 
728
            )
 
729
        else:
 
730
            op = self.CONDITION_TO_OP[condition.type]
 
731
            query_builder += (
 
732
                'token("', column_prefix, attr_name, '")', op, "token(",
 
733
                self._encode_predefined_attr_value(condition.arg), ")"
 
734
            )
 
735
 
 
736
    def _append_expected_conditions(self, expected_condition_map, schema,
 
737
                                    query_builder):
 
738
        init_length = len(query_builder)
 
739
 
 
740
        for attr_name, cond_list in expected_condition_map.iteritems():
 
741
            for condition in cond_list:
 
742
                self._append_expected_condition(
 
743
                    attr_name, condition, query_builder,
 
744
                    attr_name in schema.attribute_type_map
 
745
                )
 
746
                query_builder.append(" AND ")
 
747
 
 
748
        if len(query_builder) > init_length:
 
749
            query_builder.pop()
 
750
 
 
751
    def _append_expected_condition(self, attr, condition, query_builder,
 
752
                                   is_predefined):
637
753
        if condition.type == models.ExpectedCondition.CONDITION_TYPE_EXISTS:
638
754
            if condition.arg:
639
 
                return "\"{}\"={{\"{}\"}}".format(
640
 
                    self.SYSTEM_COLUMN_ATTR_EXIST, attr)
641
 
            else:
642
 
                return "\"{}\"=null".format(name)
643
 
        elif condition.type == models.IndexedCondition.CONDITION_TYPE_BETWEEN:
644
 
            first, second = condition.arg
645
 
            val1 = self._encode_predefined_attr_value(first)
646
 
            val2 = self._encode_predefined_attr_value(second)
647
 
            return " \"{}\" >= {} AND \"{}\" <= {}".format(
648
 
                name, val1, name, val2)
649
 
        elif (condition.type ==
650
 
              models.IndexedCondition.CONDITION_TYPE_BEGINS_WITH):
651
 
            first = condition.arg
652
 
            second = first.value[:-1] + chr(ord(first.value[-1]) + 1)
653
 
            second = models.AttributeValue(condition.arg.type, second)
654
 
            val1 = self._encode_predefined_attr_value(first)
655
 
            val2 = self._encode_predefined_attr_value(second)
656
 
            return " \"{}\" >= {} AND \"{}\" < {}".format(
657
 
                name, val1, name, val2)
 
755
                query_builder += (
 
756
                    self.SYSTEM_COLUMN_ATTR_EXIST, "={'", attr, "'}"
 
757
                )
 
758
            else:
 
759
                if is_predefined:
 
760
                    query_builder += (
 
761
                        '"', self.USER_COLUMN_PREFIX, attr, '"=null'
 
762
                    )
 
763
                else:
 
764
                    query_builder += (
 
765
                        self.SYSTEM_COLUMN_ATTRS, "['", attr, "']=null"
 
766
                    )
 
767
        elif condition.type == models.ExpectedCondition.CONDITION_TYPE_EQUAL:
 
768
            if is_predefined:
 
769
                query_builder += (
 
770
                    '"', self.USER_COLUMN_PREFIX, attr, '"=',
 
771
                    self._encode_predefined_attr_value(condition.arg)
 
772
                )
 
773
            else:
 
774
                query_builder += (
 
775
                    self.SYSTEM_COLUMN_ATTRS, "['", attr, "']=",
 
776
                    self._encode_dynamic_attr_value(condition.arg)
 
777
                )
658
778
        else:
659
 
            op = self.CONDITION_TO_OP[condition.type]
660
 
            return '"' + name + '"' + op + self._encode_predefined_attr_value(
661
 
                condition.arg
662
 
            )
663
 
 
664
 
    def _conditions_as_string(self, condition_map):
665
 
        return " AND ".join((self._condition_as_string(attr, cond)
666
 
                             for attr, cond
667
 
                             in condition_map.iteritems()))
 
779
            assert False
668
780
 
669
781
    def _primary_key_as_string(self, key_map):
670
782
        return " AND ".join((
710
822
 
711
823
        where = self._primary_key_as_string(key_attribute_map)
712
824
 
713
 
        query = "UPDATE \"{}\".\"{}{}\" SET {} WHERE {}".format(
714
 
            context.tenant, self.USER_TABLE_PREFIX, table_name,
715
 
            set_clause, where
716
 
        )
 
825
        query_builder = [
 
826
            'UPDATE "', context.tenant, '"."', self.USER_TABLE_PREFIX,
 
827
            table_name, '" SET ', set_clause, " WHERE ", where
 
828
        ]
717
829
 
718
830
        if expected_condition_map:
719
 
            if_clause = self._conditions_as_string(expected_condition_map)
720
 
            query += " IF {}".format(if_clause)
721
 
 
722
 
        self._execute_query(query, consistent=True)
723
 
 
724
 
        return True
 
831
            query_builder.append(" IF ")
 
832
            self._append_expected_conditions(
 
833
                expected_condition_map, schema, query_builder
 
834
            )
 
835
 
 
836
        result = self._execute_query("".join(query_builder), consistent=True)
 
837
 
 
838
        return (result is None) or result[0]['[applied]']
725
839
 
726
840
    def _updates_as_string(self, schema, key_attribute_map, update_map):
727
 
        predefined_attrs = [attr.name for attr in schema.attribute_defs]
728
 
 
729
841
        set_clause = ", ".join({
730
 
            self._update_as_string(attr, update, attr in predefined_attrs)
 
842
            self._update_as_string(attr, update,
 
843
                                   attr in schema.attribute_type_map)
731
844
            for attr, update in update_map.iteritems()})
732
845
 
733
846
        #update system_hash
815
928
                lambda el: self._encode_single_value_as_dynamic_attr(
816
929
                    el, attr_value.type.element_type
817
930
                ),
818
 
                val)
 
931
                val
 
932
            )
 
933
            val.sort()
819
934
        else:
820
935
            val = self._encode_single_value_as_dynamic_attr(
821
936
                val, attr_value.type.element_type)
888
1003
 
889
1004
        select_type = select_type or models.SelectType.all()
890
1005
 
891
 
        select = 'COUNT(*)' if select_type.is_count else '*'
892
 
 
893
1006
        query_builder = [
894
 
            "SELECT ", select, " FROM \"", context.tenant, "\".\"",
895
 
            self.USER_TABLE_PREFIX, table_name, "\""
 
1007
            "SELECT ", 'COUNT(*)' if select_type.is_count else '*', ' FROM "',
 
1008
            context.tenant, '"."', self.USER_TABLE_PREFIX, table_name, '"'
896
1009
        ]
897
1010
 
898
 
        indexed_condition_map = indexed_condition_map or {}
899
 
 
900
 
        token_cond = None
901
 
        fixed_range_cond = None
902
 
 
903
1011
        if exclusive_start_key:
904
 
            if range_name in exclusive_start_key:
905
 
 
906
 
                fixed_range_cond = self._fixed_range_condition(
907
 
                    range_name, indexed_condition_map,
908
 
                    exclusive_start_key)
909
 
 
910
 
                indexed_condition_map[hash_name] = models.Condition.eq(
911
 
                    exclusive_start_key[hash_name])
912
 
 
913
 
            else:
914
 
                token_cond = 'token(\"{}\")>token({})'.format(
915
 
                    self.USER_COLUMN_PREFIX + hash_name,
916
 
                    self._encode_predefined_attr_value(
917
 
                        exclusive_start_key[hash_name]))
918
 
 
919
 
                if hash_name in indexed_condition_map:
920
 
                    del indexed_condition_map[hash_name]
921
 
 
922
 
        where = self._conditions_as_string(indexed_condition_map)
923
 
 
924
 
        if token_cond:
925
 
            if where:
926
 
                where += ' AND ' + token_cond
927
 
            else:
928
 
                where = token_cond
929
 
 
930
 
        if fixed_range_cond:
931
 
            if where:
932
 
                where += ' AND ' + fixed_range_cond
933
 
            else:
934
 
                where = fixed_range_cond
935
 
 
936
 
        if where:
937
 
            query_builder += (" WHERE ", where)
938
 
 
939
 
        add_filtering, add_system_hash = self._add_filtering_and_sys_hash(
940
 
            schema, indexed_condition_map)
941
 
 
942
 
        if add_system_hash:
943
 
            hash_value = self._encode_predefined_attr_value(
944
 
                indexed_condition_map[hash_name].arg
945
 
            )
946
 
 
947
 
            query_builder += (
948
 
                " AND \"", self.SYSTEM_COLUMN_HASH, "\"=", hash_value
949
 
            )
 
1012
            indexed_condition_map = indexed_condition_map or {}
 
1013
 
 
1014
            exclusive_hash_key_value = exclusive_start_key[hash_name]
 
1015
            exclusive_range_key_value = exclusive_start_key.get(range_name,
 
1016
                                                                None)
 
1017
            if exclusive_range_key_value:
 
1018
                range_key_cond_list = indexed_condition_map.get(
 
1019
                    range_name, None
 
1020
                )
 
1021
                if range_key_cond_list is None:
 
1022
                    range_key_cond_list = []
 
1023
                    indexed_condition_map[range_name] = range_key_cond_list
 
1024
 
 
1025
                range_key_cond_list.append(
 
1026
                    models.IndexedCondition.lt(exclusive_range_key_value)
 
1027
                    if order_type == models.ORDER_TYPE_DESC else
 
1028
                    models.IndexedCondition.gt(exclusive_range_key_value)
 
1029
                )
 
1030
 
 
1031
                hash_key_cond_list = indexed_condition_map.get(
 
1032
                    hash_name, None
 
1033
                )
 
1034
                if hash_key_cond_list is None:
 
1035
                    hash_key_cond_list = []
 
1036
                    indexed_condition_map[hash_name] = hash_key_cond_list
 
1037
 
 
1038
                hash_key_cond_list.append(
 
1039
                    models.IndexedCondition.eq(exclusive_hash_key_value)
 
1040
                )
 
1041
            else:
 
1042
                hash_key_cond_list = indexed_condition_map.get(
 
1043
                    hash_name, None
 
1044
                )
 
1045
                if hash_key_cond_list is None:
 
1046
                    hash_key_cond_list = []
 
1047
                    indexed_condition_map[hash_name] = hash_key_cond_list
 
1048
 
 
1049
                hash_key_cond_list.append(
 
1050
                    models.IndexedCondition.lt(exclusive_hash_key_value)
 
1051
                    if order_type == models.ORDER_TYPE_DESC else
 
1052
                    models.IndexedCondition.gt(exclusive_hash_key_value)
 
1053
                )
 
1054
 
 
1055
        pre_condition_str = " WHERE "
 
1056
 
 
1057
        if indexed_condition_map:
 
1058
            hash_cond_list = None
 
1059
            for attr, cond_list in indexed_condition_map.iteritems():
 
1060
                active_cond_list = self._compact_indexed_condition(cond_list)
 
1061
                if active_cond_list is None:
 
1062
                    return models.SelectResult(count=0)
 
1063
 
 
1064
                if attr == hash_name:
 
1065
                    hash_cond_list = active_cond_list
 
1066
                    for active_cond in active_cond_list:
 
1067
                        query_builder.append(pre_condition_str)
 
1068
                        pre_condition_str = " AND "
 
1069
                        self._append_hash_key_indexed_condition(
 
1070
                            attr, active_cond, query_builder
 
1071
                        )
 
1072
                else:
 
1073
                    for active_cond in active_cond_list:
 
1074
                        query_builder.append(pre_condition_str)
 
1075
                        pre_condition_str = " AND "
 
1076
                        self._append_indexed_condition(
 
1077
                            attr, active_cond, query_builder
 
1078
                        )
 
1079
 
 
1080
            if (hash_cond_list is not None and
 
1081
                    len(hash_cond_list) == 1 and
 
1082
                    hash_cond_list[0].type ==
 
1083
                    models.IndexedCondition.CONDITION_TYPE_EQUAL):
 
1084
                query_builder.append(pre_condition_str)
 
1085
                self._append_indexed_condition(
 
1086
                    self.SYSTEM_COLUMN_HASH, hash_cond_list[0],
 
1087
                    query_builder, column_prefix="")
950
1088
 
951
1089
        #add limit
952
1090
        if limit:
955
1093
        #add ordering
956
1094
        if order_type and range_name:
957
1095
            query_builder += (
958
 
                " ORDER BY \"", self.USER_COLUMN_PREFIX, range_name, "\" ",
 
1096
                ' ORDER BY "', self.USER_COLUMN_PREFIX, range_name, '" ',
959
1097
                order_type
960
1098
            )
961
1099
 
962
1100
        #add allow filtering
963
 
        if add_filtering:
964
 
            query_builder.append(" ALLOW FILTERING")
 
1101
        query_builder.append(" ALLOW FILTERING")
965
1102
 
966
1103
        rows = self._execute_query("".join(query_builder), consistent)
967
1104
 
971
1108
        # process results
972
1109
 
973
1110
        prefix_len = len(self.USER_COLUMN_PREFIX)
974
 
        attr_defs = {attr.name: attr.type for attr in schema.attribute_defs}
975
1111
        result = []
976
1112
 
977
1113
        # TODO ikhudoshyn: if select_type.is_all_projected,
987
1123
                if key.startswith(self.USER_COLUMN_PREFIX) and val:
988
1124
                    name = key[prefix_len:]
989
1125
                    if not attributes_to_get or name in attributes_to_get:
990
 
                        storage_type = attr_defs[name]
 
1126
                        storage_type = schema.attribute_type_map[name]
991
1127
                        record[name] = self._decode_value(
992
1128
                            val, storage_type, True)
993
1129
 
1016
1152
                                   last_evaluated_key=last_evaluated_key,
1017
1153
                                   count=count)
1018
1154
 
1019
 
    def _fixed_range_condition(self, range_name,
1020
 
                               condition_map, exclusive_start_key):
1021
 
 
1022
 
        if range_name not in exclusive_start_key:
1023
 
            return None
1024
 
 
1025
 
        right_cond = None
1026
 
 
1027
 
        if range_name in condition_map:
1028
 
            condition = condition_map.pop(range_name)
1029
 
 
1030
 
            if condition.type == models.Condition.CONDITION_TYPE_EQUAL:
1031
 
                condition_map[range_name] = condition
1032
 
                return None
1033
 
 
1034
 
            elif condition.type in (
1035
 
                models.IndexedCondition.CONDITION_TYPE_LESS,
1036
 
                models.IndexedCondition.CONDITION_TYPE_LESS_OR_EQUAL
1037
 
            ):
1038
 
 
1039
 
                right_cond = condition
1040
 
 
1041
 
            elif condition.type == (models.IndexedCondition.
1042
 
                                    CONDITION_TYPE_BETWEEN):
1043
 
 
1044
 
                first, second = condition.arg
1045
 
 
1046
 
                right_cond = models.IndexedCondition.le(second)
1047
 
 
1048
 
            elif condition.type == (models.IndexedCondition.
1049
 
                                    CONDITION_TYPE_BEGINS_WITH):
1050
 
 
1051
 
                first = condition.arg
1052
 
                second_value = first.value[:-1] + chr(
1053
 
                    ord(first.value[-1]) + 1)
1054
 
                second = models.AttributeValue(
1055
 
                    condition.arg.type, second_value)
1056
 
 
1057
 
                right_cond = models.IndexedCondition.lt(second)
1058
 
 
1059
 
        left_cond = models.IndexedCondition.gt(
1060
 
            exclusive_start_key[range_name])
1061
 
 
1062
 
        fixed = self._condition_as_string(range_name, left_cond)
1063
 
 
1064
 
        if right_cond:
1065
 
            fixed += ' AND ' + self._condition_as_string(
1066
 
                range_name, right_cond)
1067
 
 
1068
 
        return fixed
1069
 
 
1070
1155
    def scan(self, context, table_name, condition_map, attributes_to_get=None,
1071
1156
             limit=None, exclusive_start_key=None, consistent=False):
1072
1157
        """
1160
1245
 
1161
1246
        return filtered
1162
1247
 
1163
 
    @staticmethod
1164
 
    def _add_filtering_and_sys_hash(schema, condition_map={}):
1165
 
 
1166
 
        condition_map = condition_map or {}
1167
 
 
1168
 
        hash_name = schema.key_attributes[0]
1169
 
 
1170
 
        if hash_name in condition_map:
1171
 
            assert (condition_map[hash_name].type
1172
 
                    == models.Condition.CONDITION_TYPE_EQUAL)
1173
 
 
1174
 
        try:
1175
 
            range_name = schema.key_attributes[1]
1176
 
        except IndexError:
1177
 
            range_name = None
1178
 
 
1179
 
        non_pk_attrs = [
1180
 
            key
1181
 
            for key in condition_map.iterkeys()
1182
 
            if key != hash_name and key != range_name
1183
 
        ]
1184
 
 
1185
 
        non_pk_attrs_count = len(non_pk_attrs)
1186
 
 
1187
 
        if non_pk_attrs_count == 0:
1188
 
            return False, False
1189
 
 
1190
 
        indexed_attrs = [
1191
 
            ind_def.attribute_to_index
1192
 
            for ind_def in schema.index_defs
1193
 
        ]
1194
 
 
1195
 
        has_one_indexed_eq = any(
1196
 
            [(attr in indexed_attrs) and
1197
 
             (condition_map[attr].type ==
1198
 
              models.Condition.CONDITION_TYPE_EQUAL)
1199
 
             for attr in non_pk_attrs])
1200
 
 
1201
 
        add_sys_hash = not has_one_indexed_eq
1202
 
        add_filtering = non_pk_attrs_count > 1 or add_sys_hash
1203
 
 
1204
 
        return add_filtering, add_sys_hash
1205
 
 
1206
 
    def _conditions_satisfied(self, row, cond_map={}):
1207
 
        cond_map = cond_map or {}
1208
 
        return all([self._condition_satisfied(row.get(attr, None), cond)
1209
 
                    for attr, cond in cond_map.iteritems()])
 
1248
    def _conditions_satisfied(self, row, cond_map=None):
 
1249
        if not cond_map:
 
1250
            return True
 
1251
 
 
1252
        for attr_name, cond_list in cond_map.iteritems():
 
1253
            for cond in cond_list:
 
1254
                if not self._condition_satisfied(
 
1255
                        row.get(attr_name, None), cond):
 
1256
                    return False
 
1257
        return True
1210
1258
 
1211
1259
    @staticmethod
1212
1260
    def _condition_satisfied(attr_val, cond):
1238
1286
            return (attr_val.type == cond.arg.type and
1239
1287
                    attr_val.value >= cond.arg.value)
1240
1288
 
1241
 
        if cond.type == models.IndexedCondition.CONDITION_TYPE_BETWEEN:
1242
 
            first, second = cond.arg
1243
 
            return (attr_val.type == first.type and
1244
 
                    second.type == first.type and
1245
 
                    first.value <= attr_val.value <= second.value)
1246
 
 
1247
 
        if cond.type == models.IndexedCondition.CONDITION_TYPE_BEGINS_WITH:
1248
 
            return (attr_val.type == cond.arg.type and
1249
 
                    attr_val.value.startswith(cond.arg.value))
1250
 
 
1251
1289
        if cond.type == models.ScanCondition.CONDITION_TYPE_NOT_EQUAL:
1252
1290
            return (attr_val.type != cond.arg.type or
1253
1291
                    attr_val.value != cond.arg.value)