~swift/swift/omega-1.3.0-4 : revision 243

165

self.logger = logger or logging.getLogger()

166

self.account = account

167

self.container = container

168

self._db_version = -1

168

169

170

def initialize(self, put_timestamp=None):

170

171

"""

607

608

conn.executescript("""

608

609

CREATE TABLE object (

609

610

ROWID INTEGER PRIMARY KEY AUTOINCREMENT,

610

name TEXT UNIQUE,

611

name TEXT,

611

612

created_at TEXT,

612

613

size INTEGER,

613

614

content_type TEXT,

615

616

deleted INTEGER DEFAULT 0

616

617

);

617

618

CREATE INDEX ix_object_deleted ON object (deleted);

619

CREATE INDEX ix_object_deleted_name ON object (deleted, name);

619

620

621

CREATE TRIGGER object_insert AFTER INSERT ON object

621

622

BEGIN

678

679

''', (self.account, self.container, normalize_timestamp(time.time()),

679

680

str(uuid4()), put_timestamp))

680

681

682

def get_db_version(self, conn):

683

if self._db_version == -1:

684

self._db_version = 0

685

for row in conn.execute('''

686

SELECT name FROM sqlite_master

687

WHERE name = 'ix_object_deleted_name' '''):

688

self._db_version = 1

689

return self._db_version

690

681

691

def _newid(self, conn):

682

692

conn.execute('''

683

693

UPDATE container_stat

910

920

''', (put_timestamp, delete_timestamp, object_count, bytes_used))

911

921

conn.commit()

912

922

913

def get_random_objects(self, max_count=100):

914

"""

915

Get random objects from the DB. This is used by the container_auditor

916

when testing random objects for existence.

917

918

:param max_count: maximum number of objects to get

919

920

:returns: list of object names

921

"""

922

try:

923

self._commit_puts()

924

except LockTimeout:

925

if not self.stale_reads_ok:

926

raise

927

rv = []

928

with self.get() as conn:

929

row = conn.execute('''

930

SELECT ROWID FROM object ORDER BY ROWID DESC LIMIT 1

931

''').fetchone()

932

if not row:

933

return []

934

max_rowid = row['ROWID']

935

for _junk in xrange(min(max_count, max_rowid)):

936

row = conn.execute('''

937

SELECT name FROM object WHERE ROWID >= ? AND +deleted = 0

938

LIMIT 1

939

''', (randint(0, max_rowid),)).fetchone()

940

if row:

941

rv.append(row['name'])

942

return list(set(rv))

943

944

923

def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,

945

924

path=None, format=None):

946

925

"""

988

967

elif prefix:

989

968

query += ' name >= ? AND'

990

969

query_args.append(prefix)

991

query += ' +deleted = 0 ORDER BY name LIMIT ?'

970

if self.get_db_version(conn) < 1:

971

query += ' +deleted = 0'

972

else:

973

query += ' deleted = 0'

974

query += ' ORDER BY name LIMIT ?'

992

975

query_args.append(limit - len(results))

993

976

curs = conn.execute(query, query_args)

994

977

curs.row_factory = None

1035

1018

with self.get() as conn:

1036

1019

max_rowid = -1

1037

1020

for rec in item_list:

1038

conn.execute('''

1039

DELETE FROM object WHERE name = ? AND

1040

(created_at < ?)

1041

''', (rec['name'], rec['created_at']))

1042

try:

1021

query = '''

1022

DELETE FROM object

1023

WHERE name = ? AND (created_at < ?)

1024

'''

1025

if self.get_db_version(conn) >= 1:

1026

query += ' AND deleted IN (0, 1)'

1027

conn.execute(query, (rec['name'], rec['created_at']))

1028

query = 'SELECT 1 FROM object WHERE name = ?'

1029

if self.get_db_version(conn) >= 1:

1030

query += ' AND deleted IN (0, 1)'

1031

if not conn.execute(query, (rec['name'],)).fetchall():

1043

1032

conn.execute('''

1044

1033

INSERT INTO object (name, created_at, size,

1045

1034

content_type, etag, deleted)

1046

1035

VALUES (?, ?, ?, ?, ?, ?)

1047

1036

''', ([rec['name'], rec['created_at'], rec['size'],

1048

1037

rec['content_type'], rec['etag'], rec['deleted']]))

1049

except sqlite3.IntegrityError:

1050

pass

1051

1038

if source:

1052

1039

max_rowid = max(max_rowid, rec['ROWID'])

1053

1040

if source:

1091

1078

conn.executescript("""

1092

1079

CREATE TABLE container (

1093

1080

ROWID INTEGER PRIMARY KEY AUTOINCREMENT,

1094

name TEXT UNIQUE,

1081

name TEXT,

1095

1082

put_timestamp TEXT,

1096

1083

delete_timestamp TEXT,

1097

1084

object_count INTEGER,

1099

1086

deleted INTEGER DEFAULT 0

1100

1087

);

1101

1088

1102

CREATE INDEX ix_container_deleted ON container (deleted);

1103

CREATE INDEX ix_container_name ON container (name);

1089

CREATE INDEX ix_container_deleted_name ON

1090

container (deleted, name);

1091

1104

1092

CREATE TRIGGER container_insert AFTER INSERT ON container

1105

1093

BEGIN

1106

1094

UPDATE account_stat

1164

1152

''', (self.account, normalize_timestamp(time.time()), str(uuid4()),

1165

1153

put_timestamp))

1166

1154

1155

def get_db_version(self, conn):

1156

if self._db_version == -1:

1157

self._db_version = 0

1158

for row in conn.execute('''

1159

SELECT name FROM sqlite_master

1160

WHERE name = 'ix_container_deleted_name' '''):

1161

self._db_version = 1

1162

return self._db_version

1163

1167

1164

def update_put_timestamp(self, timestamp):

1168

1165

"""

1169

1166

Update the put_timestamp. Only modifies it if it is greater than

1413

1410

FROM account_stat

1414

1411

''').fetchone()

1415

1412

1416

def get_random_containers(self, max_count=100):

1417

"""

1418

Get random containers from the DB. This is used by the

1419

account_auditor when testing random containerss for existence.

1420

1421

:param max_count: maximum number of containers to get

1422

1423

:returns: list of container names

1424

"""

1425

try:

1426

self._commit_puts()

1427

except LockTimeout:

1428

if not self.stale_reads_ok:

1429

raise

1430

rv = []

1431

with self.get() as conn:

1432

row = conn.execute('''

1433

SELECT ROWID FROM container ORDER BY ROWID DESC LIMIT 1

1434

''').fetchone()

1435

if not row:

1436

return []

1437

max_rowid = row['ROWID']

1438

for _junk in xrange(min(max_count, max_rowid)):

1439

row = conn.execute('''

1440

SELECT name FROM container WHERE

1441

ROWID >= ? AND +deleted = 0

1442

LIMIT 1

1443

''', (randint(0, max_rowid),)).fetchone()

1444

if row:

1445

rv.append(row['name'])

1446

return list(set(rv))

1447

1448

1413

def list_containers_iter(self, limit, marker, end_marker, prefix,

1449

1414

delimiter):

1450

1415

"""

1485

1450

elif prefix:

1486

1451

query += ' name >= ? AND'

1487

1452

query_args.append(prefix)

1488

query += ' +deleted = 0 ORDER BY name LIMIT ?'

1453

if self.get_db_version(conn) < 1:

1454

query += ' +deleted = 0'

1455

else:

1456

query += ' deleted = 0'

1457

query += ' ORDER BY name LIMIT ?'

1489

1458

query_args.append(limit - len(results))

1490

1459

curs = conn.execute(query, query_args)

1491

1460

curs.row_factory = None

1529

1498

record = [rec['name'], rec['put_timestamp'],

1530

1499

rec['delete_timestamp'], rec['object_count'],

1531

1500

rec['bytes_used'], rec['deleted']]

1532

try:

1533

conn.execute('''

1534

INSERT INTO container (name, put_timestamp,

1535

delete_timestamp, object_count, bytes_used,

1536

deleted)

1537

VALUES (?, ?, ?, ?, ?, ?)

1538

''', record)

1539

except sqlite3.IntegrityError:

1540

curs = conn.execute('''

1541

SELECT name, put_timestamp, delete_timestamp,

1542

object_count, bytes_used, deleted

1543

FROM container WHERE name = ? AND

1544

(put_timestamp < ? OR delete_timestamp < ? OR

1545

object_count != ? OR bytes_used != ?)''',

1546

(rec['name'], rec['put_timestamp'],

1547

rec['delete_timestamp'], rec['object_count'],

1548

rec['bytes_used']))

1549

curs.row_factory = None

1550

row = curs.fetchone()

1551

if row:

1552

row = list(row)

1553

for i in xrange(5):

1554

if record[i] is None and row[i] is not None:

1555

record[i] = row[i]

1556

if row[1] > record[1]: # Keep newest put_timestamp

1557

record[1] = row[1]

1558

if row[2] > record[2]: # Keep newest delete_timestamp

1559

record[2] = row[2]

1560

conn.execute('DELETE FROM container WHERE name = ?',

1561

(record[0],))

1562

# If deleted, mark as such

1563

if record[2] > record[1] and \

1564

record[3] in (None, '', 0, '0'):

1565

record[5] = 1

1566

else:

1567

record[5] = 0

1568

try:

1569

conn.execute('''

1570

INSERT INTO container (name, put_timestamp,

1571

delete_timestamp, object_count, bytes_used,

1572

deleted)

1573

VALUES (?, ?, ?, ?, ?, ?)

1574

''', record)

1575

except sqlite3.IntegrityError:

1576

continue

1501

query = '''

1502

SELECT name, put_timestamp, delete_timestamp,

1503

object_count, bytes_used, deleted

1504

FROM container WHERE name = ?

1505

'''

1506

if self.get_db_version(conn) >= 1:

1507

query += ' AND deleted IN (0, 1)'

1508

curs = conn.execute(query, (rec['name'],))

1509

curs.row_factory = None

1510

row = curs.fetchone()

1511

if row:

1512

row = list(row)

1513

for i in xrange(5):

1514

if record[i] is None and row[i] is not None:

1515

record[i] = row[i]

1516

if row[1] > record[1]: # Keep newest put_timestamp

1517

record[1] = row[1]

1518

if row[2] > record[2]: # Keep newest delete_timestamp

1519

record[2] = row[2]

1520

# If deleted, mark as such

1521

if record[2] > record[1] and \

1522

record[3] in (None, '', 0, '0'):

1523

record[5] = 1

1524

else:

1525

record[5] = 0

1526

conn.execute('''

1527

DELETE FROM container WHERE name = ? AND

1528

deleted IN (0, 1)

1529

''', (record[0],))

1530

conn.execute('''

1531

INSERT INTO container (name, put_timestamp,

1532

delete_timestamp, object_count, bytes_used,

1533

deleted)

1534

VALUES (?, ?, ?, ?, ?, ?)

1535

''', record)

1577

1536

if source:

1578

1537

max_rowid = max(max_rowid, rec['ROWID'])

1579

1538

if source: