2
* collection.indexStats({...}) command
5
/* Copyright 2012 10gen Inc.
7
* Licensed under the Apache License, Version 2.0 (the "License");
8
* you may not use this file except in compliance with the License.
9
* You may obtain a copy of the License at
11
* http://www.apache.org/licenses/LICENSE-2.0
13
* Unless required by applicable law or agreed to in writing, software
14
* distributed under the License is distributed on an "AS IS" BASIS,
15
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
* See the License for the specific language governing permissions and
17
* limitations under the License.
20
#include "mongo/pch.h"
26
#include <boost/noncopyable.hpp>
27
#include <boost/optional.hpp>
29
#include "mongo/base/init.h"
30
#include "mongo/db/auth/action_set.h"
31
#include "mongo/db/auth/action_type.h"
32
#include "mongo/db/auth/privilege.h"
33
#include "mongo/db/btree.h"
34
#include "mongo/db/commands.h"
35
#include "mongo/db/db.h"
36
#include "mongo/db/index.h"
37
#include "mongo/db/jsobj.h"
38
#include "mongo/db/kill_current_op.h"
39
#include "mongo/db/namespace_details.h"
40
#include "mongo/util/descriptive_stats.h"
45
* Holds operation parameters.
47
struct IndexStatsParams {
49
vector<int> expandNodes;
53
* Holds information about a single btree bucket (not its subtree).
56
NodeInfo() : childNum(0), keyCount(0), usedKeyCount(0), depth(0), fillRatio(0) {}
58
boost::optional<BSONObj> firstKey;
59
boost::optional<BSONObj> lastKey;
61
unsigned int childNum;
62
unsigned int keyCount;
63
unsigned int usedKeyCount;
69
* Aggregates and statistics for some part of the tree:
70
* the entire tree, a level or a certain subtree.
74
static const int quantiles = 99;
76
boost::optional<NodeInfo> nodeInfo;
78
unsigned int numBuckets;
79
SummaryEstimators<double, quantiles> bsonRatio;
80
SummaryEstimators<double, quantiles> fillRatio;
81
SummaryEstimators<double, quantiles> keyNodeRatio;
82
SummaryEstimators<unsigned int, quantiles> keyCount;
83
SummaryEstimators<unsigned int, quantiles> usedKeyCount;
85
AreaStats() : numBuckets(0) {
88
virtual ~AreaStats() {
92
* add the provided values as a sample to the computed statistics for this tree / level /
95
* @param keyCount number of keys in the bucket
96
* @param usedKeyCount number of used (non-empty) keys in the bucket
97
* @param bucket current bucket
98
* @param keyNodeBytes size (number of bytes) of a KeyNode
100
template<class Version>
101
void addStats(int keyCount, int usedKeyCount, const BtreeBucket<Version>* bucket,
104
this->keyCount << keyCount;
105
this->usedKeyCount << usedKeyCount;
106
this->bsonRatio << (static_cast<double>(bucket->getTopSize()) / bucket->bodySize());
107
this->keyNodeRatio <<
108
(static_cast<double>(keyNodeBytes * keyCount) / bucket->bodySize());
110
(1.0 - static_cast<double>(bucket->getEmptySize()) / bucket->bodySize());
113
void appendTo(BSONObjBuilder& builder) const {
115
BSONObjBuilder nodeInfoBuilder(builder.subobjStart("nodeInfo"));
116
nodeInfoBuilder << "childNum" << nodeInfo->childNum
117
<< "keyCount" << nodeInfo->keyCount
118
<< "usedKeyCount" << nodeInfo->usedKeyCount
119
<< "diskLoc" << nodeInfo->diskLoc
120
<< "depth" << nodeInfo->depth
121
<< "fillRatio" << nodeInfo->fillRatio;
122
if (nodeInfo->firstKey) nodeInfoBuilder << "firstKey" << *(nodeInfo->firstKey);
123
if (nodeInfo->lastKey) nodeInfoBuilder << "lastKey" << *(nodeInfo->lastKey);
126
builder << "numBuckets" << numBuckets
127
<< "keyCount" << keyCount.statisticSummaryToBSONObj()
128
<< "usedKeyCount" << usedKeyCount.statisticSummaryToBSONObj()
129
<< "bsonRatio" << bsonRatio.statisticSummaryToBSONObj()
130
<< "keyNodeRatio" << keyNodeRatio.statisticSummaryToBSONObj()
131
<< "fillRatio" << fillRatio.statisticSummaryToBSONObj();
136
* Holds statistics and aggregates for the entire tree and its parts.
140
unsigned int bucketBodyBytes;
143
vector<AreaStats> perLevel;
144
vector<vector<AreaStats> > branch;
146
BtreeStats() : bucketBodyBytes(0), depth(0) {
147
branch.push_back(vector<AreaStats>(1));
150
AreaStats& nodeAt(unsigned int nodeDepth, unsigned int childNum) {
151
verify(branch.size() > nodeDepth);
152
verify(branch[nodeDepth].size() > childNum);
153
return branch[nodeDepth][childNum];
156
void newBranchLevel(unsigned int childrenCount) {
157
branch.push_back(vector<AreaStats>(childrenCount));
160
void appendTo(BSONObjBuilder& builder) const {
161
builder << "bucketBodyBytes" << bucketBodyBytes;
162
builder << "depth" << depth;
164
BSONObjBuilder wholeTreeBuilder(builder.subobjStart("overall"));
165
wholeTree.appendTo(wholeTreeBuilder);
166
wholeTreeBuilder.doneFast();
168
BSONArrayBuilder perLevelArrayBuilder(builder.subarrayStart("perLevel"));
169
for (vector<AreaStats>::const_iterator it = perLevel.begin();
170
it != perLevel.end();
172
BSONObjBuilder levelBuilder(perLevelArrayBuilder.subobjStart());
173
it->appendTo(levelBuilder);
174
levelBuilder.doneFast();
176
perLevelArrayBuilder.doneFast();
178
if (branch.size() > 1) {
179
BSONArrayBuilder expandedNodesArrayBuilder(builder.subarrayStart("expandedNodes"));
180
for (unsigned int depth = 0; depth < branch.size(); ++depth) {
182
BSONArrayBuilder childrenArrayBuilder(
183
expandedNodesArrayBuilder.subarrayStart());
184
const vector<AreaStats>& children = branch[depth];
185
for (unsigned int child = 0; child < children.size(); ++child) {
186
BSONObjBuilder childBuilder(childrenArrayBuilder.subobjStart());
187
children[child].appendTo(childBuilder);
188
childBuilder.doneFast();
191
expandedNodesArrayBuilder.doneFast();
197
* Performs the btree analysis for a generic btree version. After inspect() is called on the
198
* tree root, statistics are available through stats().
200
* Template-based implementation in BtreeInspectorImpl.
202
class BtreeInspector : boost::noncopyable {
204
virtual ~BtreeInspector() {}
205
virtual bool inspect(const DiskLoc& head) = 0;
206
virtual BtreeStats& stats() = 0;
209
// See BtreeInspector.
210
template <class Version>
211
class BtreeInspectorImpl : public BtreeInspector {
213
typedef typename mongo::BucketBasics<Version> BucketBasics;
214
typedef typename mongo::BucketBasics<Version>::_KeyNode _KeyNode;
215
typedef typename mongo::BucketBasics<Version>::KeyNode KeyNode;
216
typedef typename mongo::BucketBasics<Version>::Key Key;
218
BtreeInspectorImpl(vector<int> expandNodes) : _expandNodes(expandNodes) {
221
virtual bool inspect(const DiskLoc& head) {
222
_stats.bucketBodyBytes = BucketBasics::bodySize();
223
vector<int> expandedAncestors;
224
return this->inspectBucket(head, 0, 0, true, expandedAncestors);
227
virtual BtreeStats& stats() {
233
* Recursively inspect btree buckets.
234
* @param dl DiskLoc for the current bucket
235
* @param depth depth for the current bucket (root is 0)
236
* @param childNum so that the current bucket is the childNum-th child of its parent
237
* (the right child is numbered as the last left child + 1)
238
* @param parentIsExpanded bucket expansion was requested for the parent bucket so the
239
* statistics and information for this bucket will appear in the
241
* @param expandedAncestors if the d-th element is k, the k-th child of an expanded parent
242
* at depth d is expanded
243
* [0, 4, 1] means that root is expanded, its 4th child is expanded
244
* and, in turn, the first child of the 4th child of the root is
246
* @return true on success, false otherwise
248
bool inspectBucket(const DiskLoc& dl, unsigned int depth, int childNum,
249
bool parentIsExpanded, vector<int> expandedAncestors) {
251
if (dl.isNull()) return true;
252
killCurrentOp.checkForInterrupt();
254
const BtreeBucket<Version>* bucket = dl.btree<Version>();
255
int usedKeyCount = 0; // number of used keys in this bucket
257
int keyCount = bucket->getN();
258
int childrenCount = keyCount + 1; // maximum number of children of this bucket
259
// including the right child
261
if (depth > _stats.depth) _stats.depth = depth;
263
bool curNodeIsExpanded = false;
264
if (parentIsExpanded) {
265
// if the parent node is expanded, statistics and info will be outputted for this
267
expandedAncestors.push_back(childNum);
269
// if the expansion of this node was requested
270
if (depth < _expandNodes.size() && _expandNodes[depth] == childNum) {
271
verify(_stats.branch.size() == depth + 1);
272
_stats.newBranchLevel(childrenCount);
273
curNodeIsExpanded = true;
277
const _KeyNode* firstKeyNode = NULL;
278
const _KeyNode* lastKeyNode = NULL;
279
for (int i = 0; i < keyCount; i++ ) {
280
const _KeyNode& kn = bucket->k(i);
289
this->inspectBucket(kn.prevChildBucket, depth + 1, i, curNodeIsExpanded,
293
this->inspectBucket(bucket->getNextChild(), depth + 1, keyCount, curNodeIsExpanded,
296
killCurrentOp.checkForInterrupt();
298
if (parentIsExpanded) {
299
// stats for the children of this bucket have been added in the recursive calls,
300
// avoid including the current bucket in the stats for its subtree
301
expandedAncestors.pop_back();
305
// add the stats for the current bucket to the aggregates for all its ancestors and
307
for (unsigned int d = 0; d < expandedAncestors.size(); ++d) {
308
AreaStats& nodeStats = _stats.nodeAt(d, expandedAncestors[d]);
309
nodeStats.addStats(keyCount, usedKeyCount, bucket, sizeof(_KeyNode));
311
_stats.wholeTree.addStats(keyCount, usedKeyCount, bucket, sizeof(_KeyNode));
313
if (parentIsExpanded) {
315
if (firstKeyNode != NULL) {
316
nodeInfo.firstKey = KeyNode(*bucket, *firstKeyNode).key.toBson();
318
if (lastKeyNode != NULL) {
319
nodeInfo.lastKey = KeyNode(*bucket, *lastKeyNode).key.toBson();
322
nodeInfo.childNum = childNum;
323
nodeInfo.depth = depth;
324
nodeInfo.diskLoc = dl.toBSONObj();
325
nodeInfo.keyCount = keyCount;
326
nodeInfo.usedKeyCount = bucket->getN();
328
(1.0 - static_cast<double>(bucket->getEmptySize()) / BucketBasics::bodySize());
330
_stats.nodeAt(depth, childNum).nodeInfo = nodeInfo;
333
// add the stats for this bucket to the aggregate for a certain depth
334
while (_stats.perLevel.size() < depth + 1)
335
_stats.perLevel.push_back(AreaStats());
336
verify(_stats.perLevel.size() > depth);
337
AreaStats& level = _stats.perLevel[depth];
338
level.addStats(keyCount, usedKeyCount, bucket, sizeof(_KeyNode));
343
vector<int> _expandNodes;
347
typedef BtreeInspectorImpl<V0> BtreeInspectorV0;
348
typedef BtreeInspectorImpl<V1> BtreeInspectorV1;
351
* Run analysis with the provided parameters. See IndexStatsCmd for in-depth expanation of
354
* @return true on success, false otherwise
356
bool runInternal(const NamespaceDetails* nsd, IndexStatsParams params, string& errmsg,
357
BSONObjBuilder& result) {
359
const IndexDetails* details = NULL;
361
// casting away const, we are not going to modify NamespaceDetails
362
// but ii() is not marked const, see SERVER-7619
363
for (NamespaceDetails::IndexIterator it = const_cast<NamespaceDetails*>(nsd)->ii();
365
IndexDetails& cur = it.next();
366
if (cur.indexName() == params.indexName) details = &cur;
369
if (details == NULL) {
370
errmsg = "the requested index does not exist";
374
result << "index" << details->indexName()
375
<< "version" << details->version()
376
<< "isIdIndex" << details->isIdIndex()
377
<< "keyPattern" << details->keyPattern()
378
<< "storageNs" << details->indexNamespace();
380
scoped_ptr<BtreeInspector> inspector(NULL);
381
switch (details->version()) {
382
case 1: inspector.reset(new BtreeInspectorV1(params.expandNodes)); break;
383
case 0: inspector.reset(new BtreeInspectorV0(params.expandNodes)); break;
385
errmsg = str::stream() << "index version " << details->version() << " is "
390
inspector->inspect(details->head);
392
inspector->stats().appendTo(result);
400
* This command provides detailed and aggregate information and statistics for a btree.
401
* Stats are aggregated for the entire tree, per-depth and, if requested through the expandNodes
402
* option, per-subtree.
403
* The entire btree is walked depth-first on every call. This command takes a read lock and may
404
* be slow for large indexes if the underlying extents arent't already in physical memory.
406
* The output has the form:
407
* { index: <index name>,
408
* version: <index version (0 or 1),
409
* isIdKey: <true if this is the default _id index>,
410
* keyPattern: <bson object describing the key pattern>,
411
* storageNs: <namespace of the index's underlying storage>,
412
* bucketBodyBytes: <bytes available for keynodes and bson objects in the bucket's body>,
413
* depth: <index depth (root excluded)>
414
* overall: { (statistics for the entire tree)
415
* numBuckets: <number of buckets (samples)>
416
* keyCount: { (stats about the number of keys in a bucket)
417
* count: <number of samples>,
419
* (optional) stddev: <standard deviation>
420
* (optional) min: <minimum value (number of keys for the bucket that has the least)>
421
* (optional) max: <maximum value (number of keys for the bucket that has the most)>
422
* (optional) quantiles: {
423
* 0.01: <1st percentile>, 0.02: ..., 0.09: ..., 0.25: <1st quartile>,
424
* 0.5: <median>, 0.75: <3rd quartile>, 0.91: ..., 0.98: ..., 0.99: ...
426
* (optional fields are only present if there are enough samples to compute sensible
429
* usedKeyCount: <stats about the number of used keys in a bucket>
430
* (same structure as keyCount)
431
* bsonRatio: <stats about how much of the bucket body is occupied by bson objects>
432
* (same structure as keyCount)
433
* keyNodeRatio: <stats about how much of the bucket body is occupied by KeyNodes>
434
* (same structure as keyCount)
435
* fillRatio: <stats about how full is the bucket body (bson objects + KeyNodes)>
436
* (same structure as keyCount)
438
* perLevel: [ (statistics aggregated per depth)
439
* (one element with the same structure as 'overall' for each btree level,
440
* the first refers to the root)
444
* If 'expandNodes: [array]' was specified in the parameters, an additional field named
445
* 'expandedNodes' is included in the output. It contains two nested arrays, such that the
446
* n-th element of the outer array contains stats for nodes at depth n (root is included) and
447
* the i-th element (0-based) of the inner array at depth n contains stats for the subtree
448
* rooted at the i-th child of the expanend node at depth (n - 1).
449
* Each element of the inner array has the same structure as 'overall' in the description above:
450
* it includes the aggregate stats for all the nodes in the subtree excluding the current
452
* It also contains an additional field 'nodeInfo' representing information for the current
454
* { childNum: <i so that this is the (i + 1)-th child of the parent node>
455
* keyCount: <number of keys in this bucket>
456
* usedKeyCount: <number of non-empty KeyNodes>
457
* diskLoc: { (bson representation of the disk location for this bucket)
461
* depth: <depth of this bucket, root is at depth 0>
462
* fillRatio: <a value between 0 and 1 representing how full this bucket is>
463
* firstKey: <bson object containing the value for the first key>
464
* lastKey: <bson object containing the value for the last key>
468
class IndexStatsCmd : public Command {
470
IndexStatsCmd() : Command("indexStats") {}
472
virtual bool slaveOk() const {
476
virtual void help(stringstream& h) const {
477
h << "EXPERIMENTAL (UNSUPPORTED). "
478
<< "Provides detailed and aggregate information and statistics for a btree. "
479
<< "The entire btree is walked on every call. This command takes a read lock, "
480
<< "requires the entire btree storage to be paged-in and will be slow on large "
481
<< "indexes. Requires an index name in {index: '_name'}. Accepts and optional array "
482
<< "of the nodes to be expanded, {expandNodes: [...]}. "
483
<< "For example, {indexStats: 'collection', index: '_id', expandNodes: [0, 4]} "
484
<< "aggregates statistics for the _id index for 'collection' and expands root "
485
<< "and the fifth child of root.";
488
virtual LockType locktype() const { return READ; }
490
virtual void addRequiredPrivileges(const std::string& dbname,
491
const BSONObj& cmdObj,
492
std::vector<Privilege>* out) {
494
actions.addAction(ActionType::indexStats);
495
out->push_back(Privilege(parseNs(dbname, cmdObj), actions));
498
bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg,
499
BSONObjBuilder& result, bool fromRepl) {
501
string ns = dbname + "." + cmdObj.firstElement().valuestrsafe();
502
const NamespaceDetails* nsd = nsdetails(ns);
503
if (!cmdLine.quiet) {
504
tlog() << "CMD: indexStats " << ns << endl;
507
errmsg = "ns not found";
511
IndexStatsParams params;
513
// { index: _index_name }
514
BSONElement indexName = cmdObj["index"];
515
if (!indexName.ok() || indexName.type() != String) {
516
errmsg = "an index name is required, use {index: \"indexname\"}";
519
params.indexName = indexName.String();
521
BSONElement expandNodes = cmdObj["expandNodes"];
522
if (expandNodes.ok()) {
523
if (expandNodes.type() != mongo::Array) {
524
errmsg = "expandNodes must be an array of numbers";
527
vector<BSONElement> arr = expandNodes.Array();
528
for (vector<BSONElement>::const_iterator it = arr.begin(); it != arr.end(); ++it) {
529
if (!it->isNumber()) {
530
errmsg = "expandNodes must be an array of numbers";
533
params.expandNodes.push_back(int(it->Number()));
537
BSONObjBuilder resultBuilder;
538
if (!runInternal(nsd, params, errmsg, resultBuilder))
540
result.appendElements(resultBuilder.obj());
546
MONGO_INITIALIZER(IndexStatsCmd)(InitializerContext* context) {
547
if (cmdLine.experimental.indexStatsCmdEnabled) {
548
// Leaked intentionally: a Command registers itself when constructed.