← Back to branch summary

~notmyname/swift/saio_reference

« back to all changes in this revision

Viewing changes to swift/obj/auditor.py

  • Committer: Tarmac
  • Author(s): David Goetz
  • Date: 2011-05-24 15:32:26 UTC
  • mfrom: (297.1.2 zero_logger)
  • Revision ID: tarmac-20110524153226-3ubiaj3xpirwwqcm
Changing auditor logging to show zero byte files rate and amount of time doing list dirs verses actual auditing.

Show diffs side-by-side

added added

removed removed

Lines of Context:
swift/obj/auditor.py
63
63
        begin = reported = time.time()
64
64
        self.total_bytes_processed = 0
65
65
        self.total_files_processed = 0
 
66
        total_quarantines = 0
 
67
        total_errors = 0
66
68
        files_running_time = 0
 
69
        time_auditing = 0
67
70
        all_locs = audit_location_generator(self.devices,
68
71
                                            object_server.DATADIR,
69
72
                                            mount_check=self.mount_check,
70
73
                                            logger=self.logger)
71
74
        for path, device, partition in all_locs:
 
75
            loop_time = time.time()
72
76
            self.object_audit(path, device, partition)
73
77
            self.files_running_time = ratelimit_sleep(
74
78
                self.files_running_time, self.max_files_per_second)
75
79
            self.total_files_processed += 1
76
 
            if time.time() - reported >= self.log_time:
 
80
            now = time.time()
 
81
            if now - reported >= self.log_time:
77
82
                self.logger.info(_(
78
83
                    'Object audit (%(type)s). '
79
84
                    'Since %(start_time)s: Locally: %(passes)d passed, '
80
85
                    '%(quars)d quarantined, %(errors)d errors '
81
 
                    'files/sec: %(frate).2f , bytes/sec: %(brate).2f') % {
 
86
                    'files/sec: %(frate).2f , bytes/sec: %(brate).2f, '
 
87
                    'Total time: %(total).2f, Auditing time: %(audit).2f, '
 
88
                    'Rate: %(audit_rate).2f') % {
82
89
                            'type': self.auditor_type,
83
90
                            'start_time': time.ctime(reported),
84
 
                            'passes': self.passes,
85
 
                            'quars': self.quarantines,
 
91
                            'passes': self.passes, 'quars': self.quarantines,
86
92
                            'errors': self.errors,
87
 
                            'frate': self.passes / (time.time() - reported),
88
 
                            'brate': self.bytes_processed /
89
 
                                     (time.time() - reported)})
90
 
                reported = time.time()
 
93
                            'frate': self.passes / (now - reported),
 
94
                            'brate': self.bytes_processed / (now - reported),
 
95
                            'total': (now - begin), 'audit': time_auditing,
 
96
                            'audit_rate': time_auditing / (now - begin)})
 
97
                reported = now
 
98
                total_quarantines += self.quarantines
 
99
                total_errors += self.errors
91
100
                self.passes = 0
92
101
                self.quarantines = 0
93
102
                self.errors = 0
94
103
                self.bytes_processed = 0
 
104
            time_auditing += (now - loop_time)
95
105
        elapsed = time.time() - begin
96
106
        self.logger.info(_(
97
 
                'Object audit (%(type)s) "%(mode)s" mode '
98
 
                'completed: %(elapsed).02fs. '
99
 
                'Total files/sec: %(frate).2f , '
100
 
                'Total bytes/sec: %(brate).2f ') % {
101
 
                    'type': self.auditor_type,
102
 
                    'mode': mode,
103
 
                    'elapsed': elapsed,
104
 
                    'frate': self.total_files_processed / elapsed,
105
 
                    'brate': self.total_bytes_processed / elapsed})
 
107
            'Object audit (%(type)s) "%(mode)s" mode '
 
108
            'completed: %(elapsed).02fs. Total quarantined: %(quars)d, '
 
109
            'Total errors: %(errors)d, Total files/sec: %(frate).2f , '
 
110
            'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, '
 
111
            'Rate: %(audit_rate).2f') % {
 
112
                'type': self.auditor_type, 'mode': mode, 'elapsed': elapsed,
 
113
                'quars': total_quarantines, 'errors': total_errors,
 
114
                'frate': self.total_files_processed / elapsed,
 
115
                'brate': self.total_bytes_processed / elapsed,
 
116
                'audit': time_auditing, 'audit_rate': time_auditing / elapsed})
106
117
 
107
118
    def object_audit(self, path, device, partition):
108
119
        """
133
144
            except DiskFileNotExist:
134
145
                return
135
146
            if self.zero_byte_only_at_fps and obj_size:
 
147
                self.passes += 1
136
148
                return
137
149
            for chunk in df:
138
150
                self.bytes_running_time = ratelimit_sleep(