diff --git a/packages/shell-api/src/collection.spec.ts b/packages/shell-api/src/collection.spec.ts index 003e02761..b7f54b128 100644 --- a/packages/shell-api/src/collection.spec.ts +++ b/packages/shell-api/src/collection.spec.ts @@ -1081,49 +1081,206 @@ describe('Collection', () => { }); describe('stats', () => { - it('calls serviceProvider.runCommandWithCheck on the database with no options', async() => { + beforeEach(() => { + const serviceProviderCursor = stubInterface(); + serviceProviderCursor.limit.returns(serviceProviderCursor); + serviceProviderCursor.tryNext.returns(); + serviceProvider.find.returns(serviceProviderCursor); + + const tryNext = sinon.stub(); + tryNext.onCall(0).resolves({ storageStats: {} }); + tryNext.onCall(1).resolves(null); + serviceProvider.aggregate.returns({ tryNext } as any); + }); + + it('calls serviceProvider.aggregate on the database with no options', async() => { await collection.stats(); - expect(serviceProvider.runCommandWithCheck).to.have.been.calledWith( - database._name, - { collStats: 'coll1', scale: 1 } // ensure simple collname - ); + expect(serviceProvider.aggregate).to.have.been.calledOnce; + expect(serviceProvider.aggregate.firstCall.args[0]).to.equal(database._name); + expect(serviceProvider.aggregate.firstCall.args[1]).to.equal(collection._name); + expect(serviceProvider.aggregate.firstCall.args[2][0]).to.deep.equal({ + '$collStats': { + storageStats: { + scale: 1 + } + } + }); }); - it('calls serviceProvider.runCommandWithCheck on the database with scale option', async() => { + it('calls serviceProvider.aggregate on the database with the default scale option', async() => { await collection.stats({ scale: 2 }); - expect(serviceProvider.runCommandWithCheck).to.have.been.calledWith( - database._name, - { collStats: collection._name, scale: 2 } - ); + expect(serviceProvider.aggregate).to.have.been.calledOnce; + expect(serviceProvider.aggregate.firstCall.args[0]).to.equal(database._name); + expect(serviceProvider.aggregate.firstCall.args[1]).to.equal(collection._name); + expect(serviceProvider.aggregate.firstCall.args[2][0]).to.deep.equal({ + '$collStats': { + storageStats: { + // We scale the results ourselves, this checks we are passing the default scale. + scale: 1 + } + } + }); }); - it('calls serviceProvider.runCommandWithCheck on the database with legacy scale', async() => { + it('calls serviceProvider.aggregate on the database with default scale when legacy scale is passed', async() => { await collection.stats(2); - expect(serviceProvider.runCommandWithCheck).to.have.been.calledWith( - database._name, - { collStats: collection._name, scale: 2 } - ); + expect(serviceProvider.aggregate).to.have.been.calledOnce; + expect(serviceProvider.aggregate.firstCall.args[0]).to.equal(database._name); + expect(serviceProvider.aggregate.firstCall.args[1]).to.equal(collection._name); + expect(serviceProvider.aggregate.firstCall.args[2][0]).to.deep.equal({ + '$collStats': { + storageStats: { + // We scale the results ourselves, this checks we are passing the default scale. + scale: 1 + } + } + }); + }); + + context('when the user lacks permissions to check for the sharding cluster collection in config', () => { + beforeEach(() => { + const serviceProviderCursor = stubInterface(); + serviceProviderCursor.limit.returns(serviceProviderCursor); + serviceProviderCursor.tryNext.returns(); + // Throw an error when attempting to check permissions. + serviceProvider.find.onCall(0).returns(false as any); + serviceProvider.find.onCall(1).returns(serviceProviderCursor); + }); + + context('when there is more than one collStats document returned', () => { + beforeEach(() => { + const tryNext = sinon.stub(); + tryNext.onCall(0).resolves({ storageStats: {} }); + tryNext.onCall(1).resolves({ storageStats: {} }); + tryNext.onCall(2).resolves({ storageStats: {} }); + tryNext.onCall(3).resolves(null); + serviceProvider.aggregate.returns({ tryNext } as any); + }); + + it('returns sharded `true`', async() => { + const stats = await collection.stats(2); + expect(stats.sharded).to.equal(true); + }); + }); + + context('when there is one collStats document returned', () => { + it('returns sharded `false`', async() => { + const stats = await collection.stats(2); + expect(stats.sharded).to.equal(false); + }); + }); + }); + + context('deprecated fallback', () => { + context('when the aggregation fails with error code that is not `13388`', () => { + beforeEach(() => { + const tryNext = sinon.stub(); + const mockError: any = new Error('test error'); + mockError.code = 123; + tryNext.onCall(0).rejects(mockError); + serviceProvider.aggregate.returns({ tryNext } as any); + }); + + it('does not run the deprecated collStats command', async() => { + const error = await collection.stats().catch(e => e); + + expect(serviceProvider.runCommandWithCheck).to.not.have.been.called; + expect(error.message).to.equal('test error'); + }); + }); + + context('when the aggregation fails with error code `13388`', () => { + beforeEach(() => { + const tryNext = sinon.stub(); + const mockError: any = new Error('test error'); + mockError.code = 13388; + tryNext.onCall(0).rejects(mockError); + serviceProvider.aggregate.returns({ tryNext } as any); + }); + + it('runs the deprecated collStats command with the default scale', async() => { + await collection.stats(); + + expect(serviceProvider.runCommandWithCheck).to.have.been.calledWith( + database._name, + { collStats: collection._name, scale: 1 } + ); + }); + + it('runs the deprecated collStats command with a custom scale', async() => { + await collection.stats({ + scale: 1024 // Scale to kilobytes. + }); + + expect(serviceProvider.runCommandWithCheck).to.have.been.calledWith( + database._name, + { collStats: collection._name, scale: 1024 } + ); + }); + + it('runs the deprecated collStats command with the legacy scale parameter', async() => { + await collection.stats(2); + + expect(serviceProvider.runCommandWithCheck).to.have.been.calledWith( + database._name, + { collStats: collection._name, scale: 2 } + ); + }); + + context('when the fallback collStats command fails', () => { + beforeEach(() => { + serviceProvider.runCommandWithCheck.rejects(new Error('not our error')); + }); + + it('surfaces the original aggregation error', async() => { + const error = await collection.stats().catch(e => e); + + expect(serviceProvider.runCommandWithCheck).to.have.been.called; + expect(error.message).to.equal('test error'); + }); + }); + }); }); context('indexDetails', () => { let expectedResult; let indexesResult; + beforeEach(() => { - expectedResult = { ok: 1, indexDetails: { k1_1: { details: 1 }, k2_1: { details: 2 } } }; + expectedResult = { + avgObjSize: 0, + indexSizes: {}, + nindexes: 0, + indexDetails: { k1_1: { details: 1 }, k2_1: { details: 2 } }, + ok: 1, + ns: 'db1.coll1', + sharded: false, + }; indexesResult = [ { v: 2, key: { k1: 1 }, name: 'k1_1' }, { v: 2, key: { k2: 1 }, name: 'k2_1' }]; - serviceProvider.runCommandWithCheck.resolves(expectedResult); + const tryNext = sinon.stub(); + tryNext.onCall(0).resolves({ + storageStats: { + indexDetails: expectedResult.indexDetails + } + }); + tryNext.onCall(1).resolves(null); + serviceProvider.aggregate.returns({ tryNext } as any); serviceProvider.getIndexes.resolves(indexesResult); }); it('not returned when no args', async() => { const result = await collection.stats(); - expect(result).to.deep.equal({ ok: 1 }); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { indexDetails, ...expectedResultWithoutIndexDetails } = expectedResult; + expect(result).to.deep.equal(expectedResultWithoutIndexDetails); }); it('not returned when options indexDetails: false', async() => { const result = await collection.stats({ indexDetails: false }); - expect(result).to.deep.equal({ ok: 1 }); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { indexDetails, ...expectedResultWithoutIndexDetails } = expectedResult; + expect(result).to.deep.equal(expectedResultWithoutIndexDetails); }); it('returned all when true, even if no key/name set', async() => { const result = await collection.stats({ indexDetails: true }); @@ -1131,7 +1288,7 @@ describe('Collection', () => { }); it('returned only 1 when indexDetailsName set', async() => { const result = await collection.stats({ indexDetails: true, indexDetailsName: 'k2_1' }); - expect(result).to.deep.equal({ ok: 1, indexDetails: { 'k2_1': expectedResult.indexDetails.k2_1 } }); + expect(result).to.deep.equal({ ...expectedResult, indexDetails: { 'k2_1': expectedResult.indexDetails.k2_1 } }); }); it('returned all when indexDetailsName set but not found', async() => { const result = await collection.stats({ indexDetails: true, indexDetailsName: 'k3_1' }); @@ -1139,7 +1296,7 @@ describe('Collection', () => { }); it('returned only 1 when indexDetailsKey set', async() => { const result = await collection.stats({ indexDetails: true, indexDetailsKey: indexesResult[1].key }); - expect(result).to.deep.equal({ ok: 1, indexDetails: { 'k2_1': expectedResult.indexDetails.k2_1 } }); + expect(result).to.deep.equal({ ...expectedResult, indexDetails: { 'k2_1': expectedResult.indexDetails.k2_1 } }); }); it('returned all when indexDetailsKey set but not found', async() => { const result = await collection.stats({ indexDetails: true, indexDetailsKey: { other: 1 } }); @@ -1174,22 +1331,28 @@ describe('Collection', () => { }); }); - it('throws if serviceProvider.runCommandWithCheck rejects', async() => { + it('throws if serviceProvider.aggregate rejects', async() => { const expectedError = new Error(); - serviceProvider.runCommandWithCheck.rejects(expectedError); + const tryNext = sinon.stub(); + tryNext.onCall(0).rejects(expectedError); + tryNext.onCall(1).resolves(null); + serviceProvider.aggregate.returns({ tryNext } as any); const caughtError = await collection.stats() .catch(e => e); expect(caughtError).to.equal(expectedError); }); - it('throws is serviceProvider.runCommandWithCheck returns undefined', async() => { - serviceProvider.runCommandWithCheck.resolves(undefined); + it('throws if serviceProvider.aggregate returns undefined', async() => { + const tryNext = sinon.stub(); + tryNext.onCall(0).resolves(undefined); + tryNext.onCall(1).resolves(null); + serviceProvider.aggregate.returns({ tryNext } as any); const error = await collection.stats( { indexDetails: true, indexDetailsName: 'k2_1' } ).catch(e => e); expect(error).to.be.instanceOf(MongoshRuntimeError); - expect(error.message).to.contain('Error running collStats command'); + expect(error.message).to.contain('Error running $collStats aggregation stage'); }); }); @@ -1713,7 +1876,7 @@ describe('Collection', () => { expect(caughtError).to.equal(expectedError); }); - it('throws if optiosn is an object and options.out is not defined', async() => { + it('throws if options is an object and options.out is not defined', async() => { const error = await collection.mapReduce(mapFn, reduceFn, {}).catch(e => e); expect(error).to.be.instanceOf(MongoshInvalidInputError); expect(error.message).to.contain('Missing \'out\' option'); diff --git a/packages/shell-api/src/collection.ts b/packages/shell-api/src/collection.ts index 398a949c6..0b9b43d53 100644 --- a/packages/shell-api/src/collection.ts +++ b/packages/shell-api/src/collection.ts @@ -30,6 +30,7 @@ import { markAsExplainOutput, assertArgsDefinedType, isValidCollectionName, + scaleIndividualShardStatistics, shouldRunAggregationImmediately, coerceToJSNumber } from './helpers'; @@ -1475,6 +1476,235 @@ export default class Collection extends ShellApiWithMongoClass { return new Explainable(this._mongo, this, verbosity); } + /** + * Running the $collStats stage on sharded timeseries clusters + * fails on some versions of MongoDB. SERVER-72686 + * This function provides the deprecated fallback in those instances. + */ + async _getLegacyCollStats(scale: number) { + const result = await this._database._runCommand( + { + collStats: this._name, + scale: scale || 1 + } + ); + + if (!result) { + throw new MongoshRuntimeError( + `Error running collStats command on ${this.getFullName()}`, + CommonErrors.CommandFailed + ); + } + + return result; + } + + /** + * Build a single scaled collection stats result document from the + * potentially multiple documents returned from the `$collStats` aggregation + * result. We run the aggregation stage with scale 1 and scale it here + * in order to accurately scale the summation of stats across various shards. + */ + async _aggregateAndScaleCollStats(collStats: Document[], scale: number) { + const result: Document = { + ok: 1, + }; + + const shardStats: { + [shardId: string]: Document + } = {}; + const counts: { + [fieldName: string]: number; + } = {}; + const indexSizes: { + [indexName: string]: number; + } = {}; + const clusterTimeseriesStats: { + [statName: string]: number + } = {}; + + let maxSize = 0; + let unscaledCollSize = 0; + + let nindexes = 0; + let timeseriesBucketsNs: string | undefined; + let timeseriesTotalBucketSize = 0; + + for (const shardResult of collStats) { + const shardStorageStats = shardResult.storageStats; + + // We don't know the order that we will encounter the count and size, so we save them + // until we've iterated through all the fields before updating unscaledCollSize + // Timeseries bucket collection does not provide 'count' or 'avgObjSize'. + const countField = shardStorageStats.count; + const shardObjCount = (typeof countField !== 'undefined') ? countField : 0; + + for (const fieldName of Object.keys(shardStorageStats)) { + if (['ns', 'ok', 'lastExtentSize', 'paddingFactor'].includes(fieldName)) { + continue; + } + if ( + [ + 'userFlags', 'capped', 'max', 'paddingFactorNote', 'indexDetails', 'wiredTiger' + ].includes(fieldName) + ) { + // Fields that are copied from the first shard only, because they need to + // match across shards. + result[fieldName] ??= shardStorageStats[fieldName]; + } else if (fieldName === 'timeseries') { + const shardTimeseriesStats: Document = shardStorageStats[fieldName]; + for (const [ timeseriesStatName, timeseriesStat ] of Object.entries(shardTimeseriesStats)) { + if (typeof timeseriesStat === 'string') { + if (!timeseriesBucketsNs) { + timeseriesBucketsNs = timeseriesStat; + } + } else if (timeseriesStatName === 'avgBucketSize') { + timeseriesTotalBucketSize += coerceToJSNumber(shardTimeseriesStats.bucketCount) * coerceToJSNumber(timeseriesStat); + } else { + // Simple summation for other types of stats. + if (clusterTimeseriesStats[timeseriesStatName] === undefined) { + clusterTimeseriesStats[timeseriesStatName] = 0; + } + clusterTimeseriesStats[timeseriesStatName] += coerceToJSNumber(timeseriesStat); + } + } + } else if ( + // NOTE: `numOrphanDocs` is new in 6.0. `totalSize` is new in 4.4. + ['count', 'size', 'storageSize', 'totalIndexSize', 'totalSize', 'numOrphanDocs'].includes(fieldName) + ) { + if (counts[fieldName] === undefined) { + counts[fieldName] = 0; + } + counts[fieldName] += coerceToJSNumber(shardStorageStats[fieldName]); + } else if (fieldName === 'avgObjSize') { + const shardAvgObjSize = coerceToJSNumber(shardStorageStats[fieldName]); + unscaledCollSize += shardAvgObjSize * shardObjCount; + } else if (fieldName === 'maxSize') { + const shardMaxSize = coerceToJSNumber(shardStorageStats[fieldName]); + maxSize = Math.max(maxSize, shardMaxSize); + } else if (fieldName === 'indexSizes') { + for (const indexName of Object.keys(shardStorageStats[fieldName])) { + if (indexSizes[indexName] === undefined) { + indexSizes[indexName] = 0; + } + indexSizes[indexName] += coerceToJSNumber(shardStorageStats[fieldName][indexName]); + } + } else if (fieldName === 'nindexes') { + const shardIndexes = shardStorageStats[fieldName]; + + if (nindexes === 0) { + nindexes = shardIndexes; + } else if (shardIndexes > nindexes) { + // This hopefully means we're building an index. + nindexes = shardIndexes; + } + } + } + + if (shardResult.shard) { + shardStats[shardResult.shard] = scaleIndividualShardStatistics(shardStorageStats, scale); + } + } + + const ns = `${this._database._name}.${this._name}`; + const config = this._mongo.getDB('config'); + if (collStats[0].shard) { + result.shards = shardStats; + } + + try { + result.sharded = !!(await config.getCollection('collections').findOne({ + _id: ns, + // Dropped is gone on newer server versions, so check for !== true + // rather than for === false (SERVER-51880 and related). + dropped: { $ne: true } + })); + } catch (e) { + // A user might not have permissions to check the config. In which + // case we default to the potentially inaccurate check for multiple + // shard response documents to determine if the collection is sharded. + result.sharded = collStats.length > 1; + } + + for (const [ countField, count ] of Object.entries(counts)) { + if (['size', 'storageSize', 'totalIndexSize', 'totalSize'].includes(countField)) { + result[countField] = count / scale; + } else { + result[countField] = count; + } + } + if (timeseriesBucketsNs && Object.keys(clusterTimeseriesStats).length > 0) { + result.timeseries = { + ...clusterTimeseriesStats, + // Average across all the shards. + avgBucketSize: clusterTimeseriesStats.bucketCount + ? timeseriesTotalBucketSize / clusterTimeseriesStats.bucketCount + : 0, + bucketsNs: timeseriesBucketsNs + }; + } + result.indexSizes = {}; + for (const [ indexName, indexSize ] of Object.entries(indexSizes)) { + // Scale the index sizes with the scale option passed by the user. + result.indexSizes[indexName] = indexSize / scale; + } + // The unscaled avgObjSize for each shard is used to get the unscaledCollSize because the + // raw size returned by the shard is affected by the command's scale parameter + if (counts.count > 0) { + result.avgObjSize = unscaledCollSize / counts.count; + } else { + result.avgObjSize = 0; + } + if (result.capped) { + result.maxSize = maxSize / scale; + } + result.ns = ns; + result.nindexes = nindexes; + if (collStats[0].storageStats.scaleFactor !== undefined) { + // The `scaleFactor` property started being returned in 4.2. + result.scaleFactor = scale; + } + result.ok = 1; + + return result; + } + + async _getAggregatedCollStats(scale: number) { + try { + const collStats = await (await this.aggregate([{ + $collStats: { + storageStats: { + // We pass scale `1` and we scale the response ourselves. + // We do this because we create one document response based on the multiple + // documents the `$collStats` stage returns for sharded collections. + scale: 1, + } + } + }])).toArray(); + + if (!collStats || collStats[0] === undefined) { + throw new MongoshRuntimeError( + `Error running $collStats aggregation stage on ${this.getFullName()}`, + CommonErrors.CommandFailed + ); + } + + return await this._aggregateAndScaleCollStats(collStats, scale); + } catch (e: any) { + if (e?.code === 13388) { + // Fallback to the deprecated way of fetching that folks can still + // fetch the stats of sharded timeseries collections. SERVER-72686 + try { + return await this._getLegacyCollStats(scale); + } catch (legacyCollStatsError) { + // Surface the original error when the fallback. + throw e; + } + } + throw e; + } + } + @returnsPromise @apiVersions([]) async stats(originalOptions: CollStatsShellOptions | number = {}): Promise { @@ -1503,19 +1733,8 @@ export default class Collection extends ShellApiWithMongoClass { options.indexDetails = options.indexDetails || false; this._emitCollectionApiCall('stats', { options }); - // TODO(MONGOSH-1157): Adjust along the lines of the mongos code in - // https://github.com/mongodb/mongo/blob/master/src/mongo/s/commands/cluster_coll_stats_cmd.cpp - const result = await this._database._runCommand( - { - collStats: this._name, scale: options.scale - } - ); - if (!result) { - throw new MongoshRuntimeError( - `Error running collStats command on ${this.getFullName()}`, - CommonErrors.CommandFailed - ); - } + + const result = await this._getAggregatedCollStats(options.scale); let filterIndexName = options.indexDetailsName; if (!filterIndexName && options.indexDetailsKey) { @@ -1529,9 +1748,8 @@ export default class Collection extends ShellApiWithMongoClass { /** * Remove indexDetails if options.indexDetails is true. From the old shell code. - * @param stats */ - const updateStats = (stats: any): void => { + const updateStats = (stats: Document): void => { if (!stats.indexDetails) { return; } diff --git a/packages/shell-api/src/database.spec.ts b/packages/shell-api/src/database.spec.ts index 3637702e0..f1a7a3987 100644 --- a/packages/shell-api/src/database.spec.ts +++ b/packages/shell-api/src/database.spec.ts @@ -9,9 +9,10 @@ import Collection from './collection'; import Mongo from './mongo'; import { AggregationCursor as ServiceProviderAggCursor, + FindCursor as ServiceProviderCursor, ServiceProvider, bson, - ClientSession as ServiceProviderSession + ClientSession as ServiceProviderSession, } from '@mongosh/service-provider-core'; import ShellInstanceState from './shell-instance-state'; import crypto from 'crypto'; @@ -2056,9 +2057,25 @@ describe('Database', () => { }); it('returns an object with per-collection stats', async() => { serviceProvider.listCollections.resolves([{ name: 'abc' }]); - serviceProvider.runCommandWithCheck.resolves({ ok: 1, totalSize: 1000 }); + const collStatsResult = { storageStats: { totalSize: 1000 } }; + const tryNext = sinon.stub(); + tryNext.onCall(0).resolves(collStatsResult); + tryNext.onCall(1).resolves(null); + serviceProvider.aggregate.returns({ tryNext } as any); + const serviceProviderCursor = stubInterface(); + serviceProviderCursor.limit.returns(serviceProviderCursor); + serviceProviderCursor.tryNext.returns(); + serviceProvider.find.returns(serviceProviderCursor); const result = await database.printCollectionStats(1); - expect(result.value.abc).to.deep.equal({ ok: 1, totalSize: 1000 }); + expect(result.value.abc).to.deep.equal({ + ok: 1, + avgObjSize: 0, + indexSizes: {}, + nindexes: 0, + ns: 'db1.abc', + sharded: false, + totalSize: 1000 + }); }); }); diff --git a/packages/shell-api/src/helpers.spec.ts b/packages/shell-api/src/helpers.spec.ts index b34ad8d29..9c49fd7aa 100644 --- a/packages/shell-api/src/helpers.spec.ts +++ b/packages/shell-api/src/helpers.spec.ts @@ -1,4 +1,10 @@ -import { assertArgsDefinedType, coerceToJSNumber, dataFormat, getPrintableShardStatus } from './helpers'; +import { + assertArgsDefinedType, + coerceToJSNumber, + dataFormat, + getPrintableShardStatus, + scaleIndividualShardStatistics +} from './helpers'; import { Database, Mongo, ShellInstanceState } from './index'; import constructShellBson from './shell-bson'; import { ServiceProvider, bson } from '@mongosh/service-provider-core'; @@ -229,3 +235,71 @@ describe('coerceToJSNumber', () => { expect(coerceToJSNumber(new bson.Double(1e30))).to.equal(1e30); }); }); + +describe('scaleIndividualShardStatistics', () => { + it('scales the individual shard statistics according to the scale 10', () => { + const result = scaleIndividualShardStatistics({ + size: 200, + maxSize: 2000, // Capped collection. + storageSize: 180, // Can be smaller than `size` when the data is compressed. + totalIndexSize: 50, + totalSize: 230, // New in 4.4. (sum of storageSize and totalIndexSize) + scaleFactor: 1, + capped: true, + wiredTiger: {}, + ns: 'test.test', + indexSizes: { + _id: 20, + name: 30 + } + }, 10); + + expect(result).to.deep.equal({ + size: 20, + maxSize: 200, // Capped collection. + storageSize: 18, // Can be smaller than `size` when the data is compressed. + totalIndexSize: 5, + totalSize: 23, // New in 4.4. (sum of storageSize and totalIndexSize) + scaleFactor: 10, + capped: true, + wiredTiger: {}, + ns: 'test.test', + indexSizes: { + _id: 2, + name: 3 + } + }); + }); + + it('scales the individual shard statistics according to the scale 1', () => { + const result = scaleIndividualShardStatistics({ + size: 200, + storageSize: 180, // Can be smaller than `size` when the data is compressed. + totalIndexSize: 50, + totalSize: 230, // New in 4.4. (sum of storageSize and totalIndexSize) + scaleFactor: 1, + capped: true, + wiredTiger: {}, + ns: 'test.test', + indexSizes: { + _id: 20, + name: 30 + } + }, 1); + + expect(result).to.deep.equal({ + size: 200, + storageSize: 180, // Can be smaller than `size` when the data is compressed. + totalIndexSize: 50, + totalSize: 230, // New in 4.4. (sum of storageSize and totalIndexSize) + scaleFactor: 1, + capped: true, + wiredTiger: {}, + ns: 'test.test', + indexSizes: { + _id: 20, + name: 30 + } + }); + }); +}); diff --git a/packages/shell-api/src/helpers.ts b/packages/shell-api/src/helpers.ts index 7ad9b32cf..e2cad0f47 100644 --- a/packages/shell-api/src/helpers.ts +++ b/packages/shell-api/src/helpers.ts @@ -522,6 +522,31 @@ export function dataFormat(bytes?: number): string { return Math.floor((Math.floor(bytes / (1024 * 1024)) / 1024) * 100) / 100 + 'GiB'; } +export function scaleIndividualShardStatistics(shardStats: Document, scale: number) { + const scaledStats: Document = {}; + + for (const fieldName of Object.keys(shardStats)) { + if (['size', 'maxSize', 'storageSize', 'totalIndexSize', 'totalSize'].includes(fieldName)) { + scaledStats[fieldName] = coerceToJSNumber(shardStats[fieldName]) / scale; + } else if (fieldName === 'scaleFactor') { + // Explicitly change the scale factor as we removed the scaling before getting the + // individual shards statistics. This started being returned in 4.2. + scaledStats[fieldName] = scale; + } else if (fieldName === 'indexSizes') { + const scaledIndexSizes: Document = {}; + for (const indexKey of Object.keys(shardStats[fieldName])) { + scaledIndexSizes[indexKey] = coerceToJSNumber(shardStats[fieldName][indexKey]) / scale; + } + scaledStats[fieldName] = scaledIndexSizes; + } else { + // All the other fields that do not require further scaling. + scaledStats[fieldName] = shardStats[fieldName]; + } + } + + return scaledStats; +} + export function tsToSeconds(x: any): number { if (x.t && x.i) { return x.t; diff --git a/packages/shell-api/src/integration.spec.ts b/packages/shell-api/src/integration.spec.ts index db12e754b..b50f7f25f 100644 --- a/packages/shell-api/src/integration.spec.ts +++ b/packages/shell-api/src/integration.spec.ts @@ -664,43 +664,168 @@ describe('Shell API (integration)', function() { describe('stats', () => { skipIfApiStrict(); - beforeEach(async() => { - await serviceProvider.createCollection(dbName, collectionName); - await serviceProvider.insertOne(dbName, collectionName, { x: 1 }); + context('with a default collection', () => { + let hasTotalSize: boolean; + + beforeEach(async() => { + await serviceProvider.createCollection(dbName, collectionName); + await serviceProvider.insertOne(dbName, collectionName, { x: 1 }); + hasTotalSize = !(await database.version()).match(/^4\.[0123]\./); + }); + + it('returns the expected stats', async() => { + const stats = await collection.stats(); + + expect(stats.shard).to.equal(undefined); + expect(stats.shards).to.equal(undefined); + expect(stats.timeseries).to.equal(undefined); + expect(stats.maxSize).to.equal(undefined); + expect(stats.max).to.equal(undefined); + expect(stats.capped).to.equal(false); + expect(stats.count).to.equal(1); + expect(stats.ns).to.equal(`${dbName}.${collectionName}`); + expect(stats.ok).to.equal(1); + expect(stats.nindexes).to.equal(1); + expect(stats.avgObjSize).to.be.a('number'); + expect(stats.size).to.be.a('number'); + expect(stats.storageSize).to.be.a('number'); + expect(stats.totalIndexSize).to.be.a('number'); + expect(stats.indexSizes).to.contain.keys('_id_'); + expect(stats.indexSizes._id_).to.be.a('number'); + expect(stats).to.contain.keys('wiredTiger'); + if (hasTotalSize) { + // Added in 4.4. + expect(stats.totalSize).to.be.a('number'); + } else { + expect(stats.totalSize).to.equal(undefined); + } + }); + + it('returns stats without indexDetails', async() => { + const stats = await collection.stats(); + expect(stats).to.contain.keys( + 'avgObjSize', + 'capped', + 'count', + 'indexSizes', + 'nindexes', + 'ns', + 'ok', + 'size', + 'storageSize', + 'totalIndexSize', + 'wiredTiger' + ); + }); + it('returns stats with indexDetails', async() => { + const stats = await collection.stats({ indexDetails: true }); + expect(stats).to.contain.keys( + 'avgObjSize', + 'capped', + 'count', + 'indexDetails', + 'indexSizes', + 'nindexes', + 'ns', + 'ok', + 'size', + 'storageSize', + 'totalIndexSize', + 'wiredTiger' + ); + }); }); - it('returns stats without indexDetails', async() => { - const stats = await collection.stats(); - expect(stats).to.contain.keys( - 'avgObjSize', - 'capped', - 'count', - 'indexSizes', - 'nindexes', - 'ns', - 'ok', - 'size', - 'storageSize', - 'totalIndexSize', - 'wiredTiger' - ); + context('with a capped collection', () => { + beforeEach(async() => { + await serviceProvider.createCollection( + dbName, + collectionName, + { + capped: true, + size: 8192, + max: 5000 + } + ); + await serviceProvider.insertOne(dbName, collectionName, { x: 1 }); + }); + + it('returns the unscaled maxSize', async() => { + const stats = await collection.stats(); + + expect(stats.maxSize).to.equal(8192); + expect(stats.max).to.equal(5000); + }); + + it('returns the scaled maxSize', async() => { + const stats = await collection.stats({ scale: 1024 }); + + expect(stats.capped).to.equal(true); + expect(stats.timeseries).to.equal(undefined); + expect(stats.shards).to.equal(undefined); + expect(stats.count).to.equal(1); + expect(stats.maxSize).to.equal(8); + expect(stats.max).to.equal(5000); + }); }); - it('returns stats with indexDetails', async() => { - const stats = await collection.stats({ indexDetails: true }); - expect(stats).to.contain.keys( - 'avgObjSize', - 'capped', - 'count', - 'indexDetails', - 'indexSizes', - 'nindexes', - 'ns', - 'ok', - 'size', - 'storageSize', - 'totalIndexSize', - 'wiredTiger' - ); + + context('with a timeseries collection', () => { + skipIfServerVersion(testServer, '< 5.0'); + + beforeEach(async() => { + await serviceProvider.createCollection( + dbName, + collectionName, + { + timeseries: { + timeField: 'timestamp', + metaField: 'metadata', + granularity: 'hours' + } + } + ); + await serviceProvider.insertOne( + dbName, + collectionName, + { + timestamp: new Date(), + metadata: { + test: true + } + } + ); + }); + + it('returns the timeseries stats', async() => { + const stats = await collection.stats({ scale: 1024 }); + + // Timeseries bucket collection does not provide 'count' or 'avgObjSize'. + expect(stats.count).to.equal(undefined); + expect(stats.maxSize).to.equal(undefined); + expect(stats.capped).to.equal(false); + expect(stats.timeseries.bucketsNs).to.equal(`${dbName}.system.buckets.${collectionName}`); + expect(stats.timeseries.bucketCount).to.equal(1); + expect(stats.timeseries.numBucketInserts).to.equal(1); + expect(stats.timeseries.numBucketUpdates).to.equal(0); + + expect(stats.timeseries).to.contain.keys( + 'bucketsNs', + 'bucketCount', + 'avgBucketSize', + 'numBucketInserts', + 'numBucketUpdates', + 'numBucketsOpenedDueToMetadata', + 'numBucketsClosedDueToCount', + 'numBucketsClosedDueToSize', + 'numBucketsClosedDueToTimeForward', + 'numBucketsClosedDueToTimeBackward', + 'numBucketsClosedDueToMemoryThreshold', + 'numCommits', + 'numWaits', + 'numMeasurementsCommitted', + 'avgNumMeasurementsPerCommit' + ); + }); }); }); @@ -1173,8 +1298,10 @@ describe('Shell API (integration)', function() { it('creates a collection without options', async() => { await database.createCollection('newcoll'); - const stats = await serviceProvider.runCommand(dbName, { collStats: 'newcoll' }); - expect(stats.nindexes).to.equal(1); + const stats = (await ( + serviceProvider.aggregate(dbName, 'newcoll', [{ $collStats: { storageStats: {} } }]) + ).toArray())[0]; + expect(stats.storageStats.nindexes).to.equal(1); }); it('creates a collection with options', async() => { @@ -1183,11 +1310,13 @@ describe('Shell API (integration)', function() { size: 1024, max: 5000 }); - const stats = await serviceProvider.runCommand(dbName, { collStats: 'newcoll' }); - expect(stats.nindexes).to.equal(1); - expect(stats.capped).to.equal(true); - expect(stats.maxSize).to.equal(1024); - expect(stats.max).to.equal(5000); + const stats = (await ( + serviceProvider.aggregate(dbName, 'newcoll', [{ $collStats: { storageStats: {} } }]) + ).toArray())[0]; + expect(stats.storageStats.nindexes).to.equal(1); + expect(stats.storageStats.capped).to.equal(true); + expect(stats.storageStats.maxSize).to.equal(1024); + expect(stats.storageStats.max).to.equal(5000); }); }); describe('createView', () => { diff --git a/packages/shell-api/src/shard.spec.ts b/packages/shell-api/src/shard.spec.ts index 5d10e2f89..802562833 100644 --- a/packages/shell-api/src/shard.spec.ts +++ b/packages/shell-api/src/shard.spec.ts @@ -1451,6 +1451,7 @@ describe('Shard', () => { describe('collection.stats()', () => { let db: Database; let hasTotalSize: boolean; + let hasScaleFactorIncluded: boolean; const dbName = 'shard-stats-test'; const ns = `${dbName}.test`; @@ -1458,7 +1459,9 @@ describe('Shard', () => { db = sh._database.getSiblingDB(dbName); await db.getCollection('test').insertOne({ key: 1 }); await db.getCollection('test').createIndex({ key: 1 }); - hasTotalSize = !(await db.version()).match(/^4\.[0123]\./); + const dbVersion = await db.version(); + hasTotalSize = !(dbVersion).match(/^4\.[0123]\./); + hasScaleFactorIncluded = !(dbVersion).match(/^4\.[01]\./); }); afterEach(async() => { await db.dropDatabase(); @@ -1469,13 +1472,19 @@ describe('Shard', () => { expect(result.sharded).to.equal(false); expect(result.count).to.equal(1); if (hasTotalSize) { - expect(result.shards[result.primary].totalSize).to.be.a('number'); + for (const shardId of Object.keys(result.shards)) { + expect(result.shards[shardId].totalSize).to.be.a('number'); + } + } + for (const shardId of Object.keys(result.shards)) { + expect(result.shards[shardId].indexDetails).to.equal(undefined); } - expect(result.shards[result.primary].indexDetails).to.equal(undefined); }); it('works with indexDetails', async() => { const result = await db.getCollection('test').stats({ indexDetails: true }); - expect(result.shards[result.primary].indexDetails._id_.metadata.formatVersion).to.be.a('number'); + for (const shardId of Object.keys(result.shards)) { + expect(result.shards[shardId].indexDetails._id_.metadata.formatVersion).to.be.a('number'); + } }); }); context('sharded collections', () => { @@ -1504,6 +1513,107 @@ describe('Shard', () => { expect(shard.indexDetails._id_.metadata.formatVersion).to.be.a('number'); } }); + // eslint-disable-next-line complexity + it('returns scaled output', async() => { + const scaleFactor = 1024; + const unscaledResult = await db.getCollection('test').stats(); + const scaledResult = await db.getCollection('test').stats(scaleFactor); + + const scaledProperties = ['size', 'storageSize', 'totalIndexSize', 'totalSize']; + for (const scaledProperty of scaledProperties) { + if (scaledProperty === 'totalSize' && !hasTotalSize) { + expect(unscaledResult[scaledProperty]).to.equal(undefined); + expect(scaledResult[scaledProperty]).to.equal(undefined); + continue; + } + expect( + unscaledResult[scaledProperty] / scaledResult[scaledProperty] + ).to.equal( + scaleFactor, + `Expected scaled property "${scaledProperty}" to be scaled. ${unscaledResult[scaledProperty]} should equal ${scaleFactor}*${scaledResult[scaledProperty]}` + ); + for (const shardId of Object.keys(scaledResult.shards)) { + expect( + unscaledResult.shards[shardId][scaledProperty] / scaledResult.shards[shardId][scaledProperty] + ).to.equal(scaleFactor); + } + } + + for (const indexId of Object.keys(scaledResult.indexSizes)) { + expect(unscaledResult.indexSizes[indexId] / scaledResult.indexSizes[indexId]).to.equal(scaleFactor); + } + for (const shardId of Object.keys(scaledResult.shards)) { + for (const indexId of Object.keys(scaledResult.shards[shardId].indexSizes)) { + expect(unscaledResult.shards[shardId].indexSizes[indexId] / scaledResult.shards[shardId].indexSizes[indexId]).to.equal(scaleFactor); + } + } + + // The `scaleFactor` property started being returned in 4.2. + expect(unscaledResult.scaleFactor).to.equal(hasScaleFactorIncluded ? 1 : undefined); + expect(scaledResult.scaleFactor).to.equal(hasScaleFactorIncluded ? 1024 : undefined); + + for (const shardStats of Object.values(unscaledResult.shards as { + scaleFactor: number + }[])) { + expect(shardStats.scaleFactor).to.equal(hasScaleFactorIncluded ? 1 : undefined); + } + for (const shardStats of Object.values(scaledResult.shards as { + scaleFactor: number + }[])) { + expect(shardStats.scaleFactor).to.equal(hasScaleFactorIncluded ? 1024 : undefined); + } + }); + }); + + // We explicitly test sharded time series collections as it uses the legacy + // `collStats` command as a fallback instead of the aggregation format. SERVER-72686 + context('sharded timeseries collections', () => { + skipIfServerVersion(mongos, '< 5.1'); + + const timeseriesCollectionName = 'testTS'; + const timeseriesNS = `${dbName}.${timeseriesCollectionName}`; + + beforeEach(async() => { + expect((await sh.enableSharding(dbName)).ok).to.equal(1); + + expect((await sh.shardCollection( + timeseriesNS, + { 'metadata.bucketId': 1 }, + { + timeseries: { + timeField: 'timestamp', + metaField: 'metadata', + granularity: 'hours' + } + } + )).collectionsharded).to.equal(timeseriesNS); + await db.getCollection(timeseriesCollectionName).insertOne({ + metadata: { + bucketId: 1, + type: 'temperature' + }, + timestamp: new Date('2021-05-18T00:00:00.000Z'), + temp: 12 + }); + }); + + it('returns the collection stats', async() => { + const result = await db.getCollection(timeseriesCollectionName).stats(); + expect(result.sharded).to.equal(true); + // Timeseries bucket collection does not provide 'count' or 'avgObjSize'. + expect(result.count).to.equal(undefined); + expect(result.primary).to.equal(undefined); + for (const shard of Object.values(result.shards) as any[]) { + expect(shard.totalSize).to.be.a('number'); + expect(shard.indexDetails).to.equal(undefined); + expect(shard.timeseries.bucketsNs).to.equal(`${dbName}.system.buckets.${timeseriesCollectionName}`); + expect(shard.timeseries.numBucketUpdates).to.equal(0); + expect(typeof result.timeseries.bucketCount).to.equal('number'); + } + expect(result.timeseries.bucketsNs).to.equal(`${dbName}.system.buckets.${timeseriesCollectionName}`); + expect(result.timeseries.bucketCount).to.equal(1); + expect(result.timeseries.numBucketInserts).to.equal(1); + }); }); }); describe('collection.isCapped', () => {