← Back to branch summary

~david-goetz/swift/obj_server_file_checker

« back to all changes in this revision

Viewing changes to swift/obj/server.py

  • Committer: David Goetz
  • Date: 2011-03-29 02:29:20 UTC
  • mfrom: (238.2.18 swift)
  • Revision ID: david.goetz@rackspace.com-20110329022920-549jbs9vssv7qjd0
merge to trunk conflicts

Show diffs side-by-side

added added

removed removed

Lines of Context:
swift/obj/server.py
44
44
    check_float, check_utf8
45
45
from swift.common.exceptions import ConnectionTimeout, DiskFileError, \
46
46
    DiskFileNotExist
47
 
from swift.obj.replicator import get_hashes, invalidate_hash, \
48
 
    recalculate_hashes
 
47
from swift.obj.replicator import get_hashes, invalidate_hash
49
48
 
50
49
 
51
50
DATADIR = 'objects'
54
53
METADATA_KEY = 'user.swift.metadata'
55
54
MAX_OBJECT_NAME_LENGTH = 1024
56
55
KEEP_CACHE_SIZE = (5 * 1024 * 1024)
 
56
# keep these lower-case
 
57
DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split())
57
58
 
58
59
 
59
60
def read_metadata(fd):
171
172
        if self.meta_file:
172
173
            with open(self.meta_file) as mfp:
173
174
                for key in self.metadata.keys():
174
 
                    if key.lower() not in ('content-type', 'content-encoding',
175
 
                                'deleted', 'content-length', 'etag'):
 
175
                    if key.lower() not in DISALLOWED_HEADERS:
176
176
                        del self.metadata[key]
177
177
                self.metadata.update(read_metadata(mfp))
178
178
 
388
388
        self.max_upload_time = int(conf.get('max_upload_time', 86400))
389
389
        self.slow = int(conf.get('slow', 0))
390
390
        self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024
 
391
        default_allowed_headers = 'content-encoding, x-object-manifest, ' \
 
392
                                  'content-disposition'
 
393
        self.allowed_headers = set(i.strip().lower() for i in \
 
394
                conf.get('allowed_headers', \
 
395
                default_allowed_headers).split(',') if i.strip() and \
 
396
                i.strip().lower() not in DISALLOWED_HEADERS)
391
397
 
392
398
    def container_update(self, op, account, container, obj, headers_in,
393
399
                         headers_out, objdevice):
467
473
        metadata = {'X-Timestamp': request.headers['x-timestamp']}
468
474
        metadata.update(val for val in request.headers.iteritems()
469
475
                if val[0].lower().startswith('x-object-meta-'))
 
476
        for header_key in self.allowed_headers:
 
477
            if header_key in request.headers:
 
478
                header_caps = header_key.title()
 
479
                metadata[header_caps] = request.headers[header_key]
470
480
        with file.mkstemp() as (fd, tmppath):
471
481
            file.put(fd, tmppath, metadata, extension='.meta')
472
482
        return response_class(request=request)
525
535
                'ETag': etag,
526
536
                'Content-Length': str(os.fstat(fd).st_size),
527
537
            }
528
 
            if 'x-object-manifest' in request.headers:
529
 
                metadata['X-Object-Manifest'] = \
530
 
                    request.headers['x-object-manifest']
531
538
            metadata.update(val for val in request.headers.iteritems()
532
539
                    if val[0].lower().startswith('x-object-meta-') and
533
540
                    len(val[0]) > 14)
534
 
            if 'content-encoding' in request.headers:
535
 
                metadata['Content-Encoding'] = \
536
 
                    request.headers['Content-Encoding']
 
541
            for header_key in self.allowed_headers:
 
542
                if header_key in request.headers:
 
543
                    header_caps = header_key.title()
 
544
                    metadata[header_caps] = request.headers[header_key]
537
545
            file.put(fd, tmppath, metadata)
538
546
        file.unlinkold(metadata['X-Timestamp'])
539
547
        self.container_update('PUT', account, container, obj, request.headers,
603
611
                        'application/octet-stream'), app_iter=file,
604
612
                        request=request, conditional_response=True)
605
613
        for key, value in file.metadata.iteritems():
606
 
            if key == 'X-Object-Manifest' or \
607
 
                    key.lower().startswith('x-object-meta-'):
 
614
            if key.lower().startswith('x-object-meta-') or \
 
615
                    key.lower() in self.allowed_headers:
608
616
                response.headers[key] = value
609
617
        response.etag = file.metadata['ETag']
610
618
        response.last_modified = float(file.metadata['X-Timestamp'])
699
707
        path = os.path.join(self.devices, device, DATADIR, partition)
700
708
        if not os.path.exists(path):
701
709
            mkdirs(path)
702
 
        if suffix:
703
 
            recalculate_hashes(path, suffix.split('-'))
704
 
            return Response()
705
 
        _junk, hashes = get_hashes(path, do_listdir=False)
 
710
        suffixes = suffix.split('-') if suffix else []
 
711
        _junk, hashes = tpool.execute(get_hashes, path, recalculate=suffixes)
706
712
        return Response(body=pickle.dumps(hashes))
707
713
 
708
714
    def __call__(self, env, start_response):