Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SER-2122] Dbviewer improvements #5762

Merged
merged 4 commits into from
Nov 13, 2024
Merged

[SER-2122] Dbviewer improvements #5762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1a64122
[SER-2122] Dbviewer improvements
Cookiezaurs Nov 12, 2024
716797a
eslint
Cookiezaurs Nov 12, 2024
bd52f1f
Merge branch 'next' into anna/next
Cookiezaurs Nov 12, 2024
13caf58
Merge branch 'next' into anna/next
Cookiezaurs Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions api/utils/requestProcessor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ const Promise = require('bluebird');
const url = require('url');
const common = require('./common.js');
const countlyCommon = require('../lib/countly.common.js');
const { validateAppAdmin, validateUser, validateRead, validateUserForRead, validateUserForWrite, validateGlobalAdmin, dbUserHasAccessToCollection, validateUpdate, validateDelete, validateCreate } = require('./rights.js');
const { validateAppAdmin, validateUser, validateRead, validateUserForRead, validateUserForWrite, validateGlobalAdmin, dbUserHasAccessToCollection, validateUpdate, validateDelete, validateCreate, getBaseAppFilter } = require('./rights.js');
const authorize = require('./authorizer.js');
const taskmanager = require('./taskmanager.js');
const plugins = require('../../plugins/pluginManager.js');
Expand Down Expand Up @@ -2131,7 +2131,7 @@ const processRequest = (params) => {
}

dbUserHasAccessToCollection(params, params.qstring.collection, (hasAccess) => {
if (hasAccess) {
if (hasAccess || (params.qstring.db === "countly_drill" && params.qstring.collection === "drill_events") || (params.qstring.db === "countly" && params.qstring.collection === "events_data")) {
var dbs = { countly: common.db, countly_drill: common.drillDb, countly_out: common.outDb, countly_fs: countlyFs.gridfs.getHandler() };
var db = "";
if (params.qstring.db && dbs[params.qstring.db]) {
Expand All @@ -2140,6 +2140,23 @@ const processRequest = (params) => {
else {
db = common.db;
}
if (!params.member.global_admin && params.qstring.collection === "drill_events" || params.qstring.collection === "events_data") {
var base_filter = getBaseAppFilter(params.member, params.qstring.db, params.qstring.collection);
if (base_filter && Object.keys(base_filter).length > 0) {
params.qstring.query = params.qstring.query || {};
for (var key in base_filter) {
if (params.qstring.query[key]) {
params.qstring.query.$and = params.qstring.query.$and || [];
params.qstring.query.$and.push({[key]: base_filter[key]});
params.qstring.query.$and.push({[key]: params.qstring.query[key]});
delete params.qstring.query[key];
}
else {
params.qstring.query[key] = base_filter[key];
}
}
}
}
countlyApi.data.exports.fromDatabase({
db: db,
params: params,
Expand Down
27 changes: 26 additions & 1 deletion api/utils/rights.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1083,7 +1083,32 @@ function validateWrite(params, feature, accessType, callback, callbackParam) {
});
});
}

/**
* Creates filter object to filter by member allowed collections
* @param {object} member - members object from params
* @param {string} dbName - database name as string
* @param {string} collectionName - collection Name
* @returns {object} filter object
*/
exports.getBaseAppFilter = function(member, dbName, collectionName) {
var base_filter = {};
var apps = exports.getUserApps(member);
if (dbName === "countly_drill" && collectionName === "drill_events") {
if (Array.isArray(apps) && apps.length > 0) {
base_filter.a = {"$in": apps};
}
}
else if (dbName === "countly" && collectionName === "events_data") {
var in_array = [];
if (Array.isArray(apps) && apps.length > 0) {
for (var i = 0; i < apps.length; i++) {
in_array.push(new RegExp("^" + apps[i] + "_.*"));
}
base_filter = {"_id": {"$in": in_array}};
}
}
return base_filter;
};
/**
* Validate user for create access by api_key for provided app_id (both required parameters for the request).
* @param {params} params - {@link params} object
Expand Down
119 changes: 111 additions & 8 deletions plugins/dbviewer/api/api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,88 @@ var common = require('../../../api/utils/common.js'),
countlyFs = require('../../../api/utils/countlyFs.js'),
_ = require('underscore'),
taskManager = require('../../../api/utils/taskmanager.js'),
{ getCollectionName, dbUserHasAccessToCollection, dbLoadEventsData, validateUser, getUserApps, validateGlobalAdmin, hasReadRight } = require('../../../api/utils/rights.js'),
{ getCollectionName, dbUserHasAccessToCollection, dbLoadEventsData, validateUser, getUserApps, validateGlobalAdmin, hasReadRight, getBaseAppFilter } = require('../../../api/utils/rights.js'),
exported = {};
const { MongoInvalidArgumentError } = require('mongodb');

const { EJSON } = require('bson');

const FEATURE_NAME = 'dbviewer';
const whiteListedAggregationStages = {
"$addFields": true,
"$bucket": true,
"$bucketAuto": true,
//"$changeStream": false,
//"$changeStreamSplitLargeEvents": false,
//"$collStats": false,
"$count": true,
//"$currentOp": false,
"$densify": true,
//"$documents": false
"$facet": true,
"$fill": true,
"$geoNear": true,
"$graphLookup": true,
"$group": true,
//"$indexStats": false,
"$limit": true,
//"$listLocalSessions": false
//"$listSampledQueries": false
//"$listSearchIndexes": false
//"$listSessions": false
//"$lookup": false
"$match": true,
//"$merge": false
//"$mergeCursors": false
//"$out": false
//"$planCacheStats": false,
"$project": true,
"$querySettings": true,
"$redact": true,
"$replaceRoot": true,
"$replaceWith": true,
"$sample": true,
"$search": true,
"$searchMeta": true,
"$set": true,
"$setWindowFields": true,
//"$sharedDataDistribution": false,
"$skip": true,
"$sort": true,
"$sortByCount": true,
//"$unionWith": false,
"$unset": true,
"$unwind": true,
"$vectorSearch": true //atlas specific
};
var spawn = require('child_process').spawn,
child;

(function() {
plugins.register("/permissions/features", function(ob) {
ob.features.push(FEATURE_NAME);
});
/**
* Function removes not allowed aggregation stages from the pipeline
* @param {array} aggregation - current aggregation pipeline
* @returns {object} changes - object with information which operations were removed
*/
function escapeNotAllowedAggregationStages(aggregation) {
var changes = {};
for (var z = 0; z < aggregation.length; z++) {
for (var key in aggregation[z]) {
if (!whiteListedAggregationStages[key]) {
changes[key] = true;
delete aggregation[z][key];
}
}
if (Object.keys(aggregation[z]).length === 0) {
aggregation.splice(z, 1);
z--;
}
}
return changes;
}

/**
* @api {get} /o/db Access database
Expand Down Expand Up @@ -179,6 +248,25 @@ var spawn = require('child_process').spawn,
filter = {};
}

var base_filter = {};
if (!params.member.global_admin) {
base_filter = getBaseAppFilter(params.member, dbNameOnParam, params.qstring.collection);
}

if (base_filter && Object.keys(base_filter).length > 0) {
for (var key in base_filter) {
if (filter[key]) {
filter.$and = filter.$and || [];
filter.$and.push({[key]: base_filter[key]});
filter.$and.push({[key]: filter[key]});
delete filter[key];
}
else {
filter[key] = base_filter[key];
}
}
}

if (dbs[dbNameOnParam]) {
try {
var cursor = dbs[dbNameOnParam].collection(params.qstring.collection).find(filter, { projection });
Expand All @@ -191,6 +279,7 @@ var spawn = require('child_process').spawn,
common.returnMessage(params, 400, "Invalid collection name: Collection names can not contain '$' or other invalid characters");
}
else {
log.e(error);
common.returnMessage(params, 500, "An unexpected error occurred.");
}
return false;
Expand Down Expand Up @@ -291,7 +380,7 @@ var spawn = require('child_process').spawn,
async.each(results, function(col, done) {
if (col.collectionName.indexOf("system.indexes") === -1 && col.collectionName.indexOf("sessions_") === -1) {
userHasAccess(params, col.collectionName, params.qstring.app_id, function(hasAccess) {
if (hasAccess) {
if (hasAccess || col.collectionName === "events_data" || col.collectionName === "drill_events") {
ob = parseCollectionName(col.collectionName, lookup);
db.collections[ob.pretty] = ob.name;
}
Expand All @@ -318,8 +407,9 @@ var spawn = require('child_process').spawn,
* Get aggregated result by the parameter on the url
* @param {string} collection - collection will be applied related query
* @param {object} aggregation - aggregation object
* @param {object} changes - object referencing removed stages from pipeline
* */
function aggregate(collection, aggregation) {
function aggregate(collection, aggregation, changes) {
if (params.qstring.iDisplayLength) {
aggregation.push({ "$limit": parseInt(params.qstring.iDisplayLength) });
}
Expand All @@ -339,6 +429,10 @@ var spawn = require('child_process').spawn,
else if (collection === 'auth_tokens') {
aggregation.splice(addProjectionAt, 0, {"$addFields": {"_id": "***redacted***"}});
}
else if ((collection === "events_data" || collection === "drill_events") && !params.member.global_admin) {
var base_filter = getBaseAppFilter(params.member, dbNameOnParam, params.qstring.collection);
aggregation.splice(0, 0, {"$match": base_filter});
}
// check task is already running?
taskManager.checkIfRunning({
db: dbs[dbNameOnParam],
Expand Down Expand Up @@ -375,7 +469,7 @@ var spawn = require('child_process').spawn,
},
outputData: function(aggregationErr, result) {
if (!aggregationErr) {
common.returnOutput(params, { sEcho: params.qstring.sEcho, iTotalRecords: 0, iTotalDisplayRecords: 0, "aaData": result });
common.returnOutput(params, { sEcho: params.qstring.sEcho, iTotalRecords: 0, iTotalDisplayRecords: 0, "aaData": result, "removed": (changes || {}) });
}
else {
common.returnMessage(params, 500, aggregationErr);
Expand Down Expand Up @@ -409,7 +503,12 @@ var spawn = require('child_process').spawn,

if (appId) {
if (hasReadRight(FEATURE_NAME, appId, parameters.member)) {
return dbUserHasAccessToCollection(parameters, collection, appId, callback);
if (collection === "events_data" || collection === "drill_events") {
return callback(true);
}
else {
return dbUserHasAccessToCollection(parameters, collection, appId, callback);
}
}
}
else {
Expand Down Expand Up @@ -485,10 +584,14 @@ var spawn = require('child_process').spawn,
}
else {
userHasAccess(params, params.qstring.collection, function(hasAccess) {
if (hasAccess) {
if (hasAccess || params.qstring.collection === "events_data" || params.qstring.collection === "drill_events") {
try {
let aggregation = EJSON.parse(params.qstring.aggregation);
aggregate(params.qstring.collection, aggregation);
var changes = escapeNotAllowedAggregationStages(aggregation);
if (changes && Object.keys(changes).length > 0) {
log.d("Removed stages from pipeline: ", JSON.stringify(changes));
}
aggregate(params.qstring.collection, aggregation, changes);
}
catch (e) {
common.returnMessage(params, 500, 'Aggregation object is not valid.');
Expand All @@ -508,7 +611,7 @@ var spawn = require('child_process').spawn,
}
else {
userHasAccess(params, params.qstring.collection, function(hasAccess) {
if (hasAccess) {
if (hasAccess || params.qstring.collection === "events_data" || params.qstring.collection === "drill_events") {
dbGetCollection();
}
else {
Expand Down
9 changes: 8 additions & 1 deletion plugins/dbviewer/frontend/public/javascripts/countly.views.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,13 @@
if (res.aaData.length) {
self.fields = Object.keys(map);
}
if (res.removed && typeof res.removed === 'object' && Object.keys(res.removed).length > 0) {
self.removed = CV.i18n('dbviewer.removed-warning') + Object.keys(res.removed).join(", ");

}
else {
self.removed = "";
}
}
if (err) {
var message = CV.i18n('dbviewer.server-error');
Expand All @@ -559,7 +566,7 @@
}
},
updatePath: function(query) {
window.location.hash = "#/manage/db/aggregate/" + this.db + "/" + this.collection + "/" + query;
app.navigate("#/manage/db/aggregate/" + this.db + "/" + this.collection + "/" + query);
},
getCollectionName: function() {
var self = this;
Expand Down
1 change: 1 addition & 0 deletions plugins/dbviewer/frontend/public/localization/dbviewer.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ dbviewer.generate-aggregate-report= Generate aggregate report
dbviewer.back-to-dbviewer = Back to DB Viewer
dbviewer.invalid-pipeline = Invalid pipeline object, please check pipeline input.
dbviewer.server-error = There was a server error. There might be more information in logs.
dbviewer.removed-warning = Some stages are removed from aggregation pipleine. Following stages are allowed only with global admin rights:
dbviewer.not-found-data = Couldn't find any results
dbviewer.execute-aggregation = Execute Aggregation on {0}
dbviewer.prepare-new-aggregation = Prepare New Aggregation
Expand Down
1 change: 1 addition & 0 deletions plugins/dbviewer/frontend/public/templates/aggregate.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
</el-button>
</div>
</cly-section>
<cly-notification v-if="removed" class="bu-mb-5 cly-vue-events-all__alerts" :text="removed" ></cly-notification>
<cly-section>
<cly-datatable-n :force-loading="queryLoading" :rows="aggregationResult" :prevent-default-sort="true">
<template v-slot="scope">
Expand Down
Loading