1
# Copyright (c) 2010-2011 OpenStack, LLC.
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
7
# http://www.apache.org/licenses/LICENSE-2.0
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
18
from paste.deploy import appconfig
23
from swift.account.server import DATADIR as account_server_data_dir
24
from swift.container.server import DATADIR as container_server_data_dir
25
from swift.common.db import AccountBroker, ContainerBroker
26
from swift.common.utils import renamer, get_logger, readconf, mkdirs, \
27
TRUE_VALUES, remove_file
28
from swift.common.constraints import check_mount
29
from swift.common.daemon import Daemon
32
class DatabaseStatsCollector(Daemon):
34
Extract storage stats from account databases on the account
37
Any subclasses must define the function get_data.
40
def __init__(self, stats_conf, stats_type, data_dir, filename_format):
41
super(DatabaseStatsCollector, self).__init__(stats_conf)
42
self.stats_type = stats_type
43
self.data_dir = data_dir
44
self.filename_format = filename_format
45
self.devices = stats_conf.get('devices', '/srv/node')
46
self.mount_check = stats_conf.get('mount_check',
47
'true').lower() in TRUE_VALUES
48
self.target_dir = stats_conf.get('log_dir', '/var/log/swift')
49
mkdirs(self.target_dir)
50
self.logger = get_logger(stats_conf,
51
log_route='%s-stats' % stats_type)
53
def run_once(self, *args, **kwargs):
54
self.logger.info(_("Gathering %s stats" % self.stats_type))
56
self.find_and_process()
57
self.logger.info(_("Gathering %s stats complete (%0.2f minutes)") %
58
(self.stats_type, (time.time() - start) / 60))
61
raise NotImplementedError('Subclasses must override')
64
raise NotImplementedError('Subclasses must override')
66
def find_and_process(self):
67
src_filename = time.strftime(self.filename_format)
68
working_dir = os.path.join(self.target_dir,
69
'.%-stats_tmp' % self.stats_type)
70
shutil.rmtree(working_dir, ignore_errors=True)
72
tmp_filename = os.path.join(working_dir, src_filename)
73
hasher = hashlib.md5()
75
with open(tmp_filename, 'wb') as statfile:
76
statfile.write(self.get_header())
77
for device in os.listdir(self.devices):
78
if self.mount_check and not check_mount(self.devices,
81
_("Device %s is not mounted, skipping.") % device)
83
db_dir = os.path.join(self.devices, device, self.data_dir)
84
if not os.path.exists(db_dir):
86
_("Path %s does not exist, skipping.") % db_dir)
88
for root, dirs, files in os.walk(db_dir, topdown=False):
89
for filename in files:
90
if filename.endswith('.db'):
91
db_path = os.path.join(root, filename)
92
line_data = self.get_data(db_path)
94
statfile.write(line_data)
95
hasher.update(line_data)
97
src_filename += hasher.hexdigest()
98
renamer(tmp_filename, os.path.join(self.target_dir, src_filename))
100
shutil.rmtree(working_dir, ignore_errors=True)
103
class AccountStatsCollector(DatabaseStatsCollector):
105
Extract storage stats from account databases on the account
109
def __init__(self, stats_conf):
110
super(AccountStatsCollector, self).__init__(stats_conf, 'account',
111
account_server_data_dir,
114
def get_data(self, db_path):
116
Data for generated csv has the following columns:
117
Account Hash, Container Count, Object Count, Bytes Used
120
broker = AccountBroker(db_path)
121
if not broker.is_deleted():
122
info = broker.get_info()
123
line_data = '"%s",%d,%d,%d\n' % (info['account'],
124
info['container_count'],
125
info['object_count'],
129
def get_header(self):
133
class ContainerStatsCollector(DatabaseStatsCollector):
135
Extract storage stats from container databases on the container
139
def __init__(self, stats_conf):
140
super(ContainerStatsCollector, self).__init__(stats_conf, 'container',
141
container_server_data_dir,
142
'container-stats-%Y%m%d%H_')
143
# webob calls title on all the header keys
144
self.metadata_keys = ['X-Container-Meta-%s' % mkey.strip().title()
145
for mkey in stats_conf.get('metadata_keys', '').split(',')
148
def get_header(self):
149
header = 'Account Hash,Container Name,Object Count,Bytes Used'
150
if self.metadata_keys:
151
xtra_headers = ','.join(self.metadata_keys)
152
header += ',%s' % xtra_headers
156
def get_data(self, db_path):
158
Data for generated csv has the following columns:
159
Account Hash, Container Name, Object Count, Bytes Used
160
This will just collect whether or not the metadata is set
164
broker = ContainerBroker(db_path)
165
if not broker.is_deleted():
166
info = broker.get_info(include_metadata=bool(self.metadata_keys))
167
encoded_container_name = urllib.quote(info['container'])
168
line_data = '"%s","%s",%d,%d' % (
169
info['account'], encoded_container_name,
170
info['object_count'], info['bytes_used'])
171
if self.metadata_keys:
172
metadata_results = ','.join(
173
[info['metadata'].get(mkey) and '1' or ''
174
for mkey in self.metadata_keys])
175
line_data += ',%s' % metadata_results
added
removed
swift/stats/db_stats_collector.py
