~ubuntu-branches/ubuntu/trusty/swift/trusty-updates

« back to all changes in this revision

Viewing changes to swift/common/middleware/slo.py

• browse files at revision 50
• compare with another revision
• download diff
• download tarball
• view history from revision 50

Show diffs side-by-side

added

removed

swift/common/middleware/slo.py

61

61

the included segments' size_bytes. This extra parameter will be hidden from

62

62

the user.

63

63

 

64

 

Manifest files can reference objects in separate containers, which

65

 

will improve concurrent upload speed. Objects can be referenced by

66

 

multiple manifests.

 

64

Manifest files can reference objects in separate containers, which will improve

 

65

concurrent upload speed. Objects can be referenced by multiple manifests. The

 

66

segments of a SLO manifest can even be other SLO manifests. Treat them as any

 

67

other object i.e., use the Etag and Content-Length given on the PUT of the

 

68

sub-SLO in the manifest to the parent SLO.

67

69

 

68

70

-------------------------

69

71

Retrieving a Large Object

107

109

 

108

110

    ?multipart-manifest=delete

109

111

 

110

 

will delete all the segments referenced in the manifest and then, if

111

 

successful, the manifest itself. The failure response will be similar to

112

 

the bulk delete middleware.

 

112

will delete all the segments referenced in the manifest and then the manifest

 

113

itself. The failure response will be similar to the bulk delete middleware.

113

114

 

114

115

------------------------

115

116

Modifying a Large Object

141

142

    HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \

142

143

    HTTPOk, HTTPPreconditionFailed, wsgify

143

144

from swift.common.utils import json, get_logger, config_true_value

144

 

from swift.common.constraints import check_utf8

 

145

from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS

 

146

from swift.common.http import HTTP_NOT_FOUND

145

147

from swift.common.middleware.bulk import get_response_body, \

146

148

    ACCEPTABLE_FORMATS, Bulk

147

149

 

254

256

                '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT')

255

257

            head_seg_resp = \

256

258

                Request.blank(obj_path, new_env).get_response(self.app)

257

 

            if head_seg_resp.status_int // 100 == 2:

 

259

            if head_seg_resp.is_success:

258

260

                total_size += seg_size

259

261

                if seg_size != head_seg_resp.content_length:

260

262

                    problem_segments.append([quote(obj_path), 'Size Mismatch'])

261

263

                if seg_dict['etag'] != head_seg_resp.etag:

262

264

                    problem_segments.append([quote(obj_path), 'Etag Mismatch'])

263

 

                if 'X-Static-Large-Object' in head_seg_resp.headers or \

264

 

                        'X-Object-Manifest' in head_seg_resp.headers:

265

 

                    problem_segments.append(

266

 

                        [quote(obj_path), 'Segments cannot be Large Objects'])

267

265

                if head_seg_resp.last_modified:

268

266

                    last_modified = head_seg_resp.last_modified

269

267

                else:

272

270

 

273

271

                last_modified_formatted = \

274

272

                    last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f')

275

 

                data_for_storage.append(

276

 

                    {'name': '/' + seg_dict['path'].lstrip('/'),

277

 

                     'bytes': seg_size,

278

 

                     'hash': seg_dict['etag'],

279

 

                     'content_type': head_seg_resp.content_type,

280

 

                     'last_modified': last_modified_formatted})

 

273

                seg_data = {'name': '/' + seg_dict['path'].lstrip('/'),

 

274

                            'bytes': seg_size,

 

275

                            'hash': seg_dict['etag'],

 

276

                            'content_type': head_seg_resp.content_type,

 

277

                            'last_modified': last_modified_formatted}

 

278

                if config_true_value(

 

279

                        head_seg_resp.headers.get('X-Static-Large-Object')):

 

280

                    seg_data['sub_slo'] = True

 

281

                data_for_storage.append(seg_data)

281

282

 

282

283

            else:

283

284

                problem_segments.append([quote(obj_path),

299

300

        env['wsgi.input'] = StringIO(json_data)

300

301

        return self.app

301

302

 

 

303

    def get_segments_to_delete_iter(self, req):

 

304

        """

 

305

        A generator function to be used to delete all the segments and

 

306

        sub-segments referenced in a manifest.

 

307

 

 

308

        :raises HTTPBadRequest: on sub manifest not manifest anymore or

 

309

                                on too many buffered sub segments

 

310

        :raises HTTPServerError: on unable to load manifest

 

311

        """

 

312

        try:

 

313

            vrs, account, container, obj = req.split_path(4, 4, True)

 

314

        except ValueError:

 

315

            raise HTTPBadRequest('Not a SLO manifest')

 

316

        sub_segments = [{

 

317

            'sub_slo': True,

 

318

            'name': ('/%s/%s' % (container, obj)).decode('utf-8')}]

 

319

        while sub_segments:

 

320

            if len(sub_segments) > MAX_BUFFERED_SLO_SEGMENTS:

 

321

                raise HTTPBadRequest(

 

322

                    'Too many buffered slo segments to delete.')

 

323

            if sub_segments:

 

324

                seg_data = sub_segments.pop(0)

 

325

            if seg_data.get('sub_slo'):

 

326

                new_env = req.environ.copy()

 

327

                new_env['REQUEST_METHOD'] = 'GET'

 

328

                del(new_env['wsgi.input'])

 

329

                new_env['QUERY_STRING'] = 'multipart-manifest=get'

 

330

                new_env['CONTENT_LENGTH'] = 0

 

331

                new_env['HTTP_USER_AGENT'] = \

 

332

                    '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT')

 

333

                new_env['swift.source'] = 'SLO'

 

334

                new_env['PATH_INFO'] = (

 

335

                    '/%s/%s/%s' % (

 

336

                    vrs, account,

 

337

                    seg_data['name'].lstrip('/'))).encode('utf-8')

 

338

                sub_resp = Request.blank('', new_env).get_response(self.app)

 

339

                if sub_resp.is_success:

 

340

                    try:

 

341

                        # if its still a SLO, load its segments

 

342

                        if config_true_value(

 

343

                                sub_resp.headers.get('X-Static-Large-Object')):

 

344

                            sub_segments.extend(json.loads(sub_resp.body))

 

345

                    except ValueError:

 

346

                        raise HTTPServerError('Unable to load SLO manifest')

 

347

                    # add sub-manifest back to be deleted after sub segments

 

348

                    # (even if obj is not a SLO)

 

349

                    seg_data['sub_slo'] = False

 

350

                    sub_segments.append(seg_data)

 

351

                elif sub_resp.status_int != HTTP_NOT_FOUND:

 

352

                    # on deletes treat not found as success

 

353

                    raise HTTPServerError('Sub SLO unable to load.')

 

354

            else:

 

355

                yield seg_data['name'].encode('utf-8')

 

356

 

302

357

    def handle_multipart_delete(self, req):

303

358

        """

304

359

        Will delete all the segments in the SLO manifest and then, if

310

365

        if not check_utf8(req.path_info):

311

366

            raise HTTPPreconditionFailed(

312

367

                request=req, body='Invalid UTF8 or contains NULL')

313

 

        try:

314

 

            vrs, account, container, obj = req.split_path(4, 4, True)

315

 

        except ValueError:

316

 

            raise HTTPBadRequest('Not an SLO manifest')

317

 

        new_env = req.environ.copy()

318

 

        new_env['REQUEST_METHOD'] = 'GET'

319

 

        del(new_env['wsgi.input'])

320

 

        new_env['QUERY_STRING'] = 'multipart-manifest=get'

321

 

        new_env['CONTENT_LENGTH'] = 0

322

 

        new_env['HTTP_USER_AGENT'] = \

323

 

            '%s MultipartDELETE' % req.environ.get('HTTP_USER_AGENT')

324

 

        new_env['swift.source'] = 'SLO'

325

 

        get_man_resp = \

326

 

            Request.blank('', new_env).get_response(self.app)

327

 

        if get_man_resp.status_int // 100 == 2:

328

 

            if not config_true_value(

329

 

                    get_man_resp.headers.get('X-Static-Large-Object')):

330

 

                raise HTTPBadRequest('Not an SLO manifest')

331

 

            try:

332

 

                manifest = json.loads(get_man_resp.body)

333

 

                # append the manifest file for deletion at the end

334

 

                manifest.append(

335

 

                    {'name': '/'.join(['', container, obj]).decode('utf-8')})

336

 

            except ValueError:

337

 

                raise HTTPServerError('Invalid manifest file')

338

 

            resp = HTTPOk(request=req)

339

 

            resp.app_iter = self.bulk_deleter.handle_delete_iter(

340

 

                req,

341

 

                objs_to_delete=[o['name'].encode('utf-8') for o in manifest],

342

 

                user_agent='MultipartDELETE', swift_source='SLO')

343

 

            return resp

344

 

        return get_man_resp

 

368

 

 

369

        resp = HTTPOk(request=req)

 

370

        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)

 

371

        if out_content_type:

 

372

            resp.content_type = out_content_type

 

373

        resp.app_iter = self.bulk_deleter.handle_delete_iter(

 

374

            req, objs_to_delete=self.get_segments_to_delete_iter(req),

 

375

            user_agent='MultipartDELETE', swift_source='SLO',

 

376

            out_content_type=out_content_type)

 

377

        return resp

345

378

 

346

379

    @wsgify

347

380

    def __call__(self, req):

• Older »

Loggerhead is a web-based interface for Breezy
Version: 2.0.1