WIP: new style access policy framework for Cube

.gitignore

@@ -23,4 +23,4 @@ testings/
 rust/cubesql/profile.json
 .cubestore
 .env
-
+.vimspector.json

packages/cubejs-api-gateway/src/gateway.ts

@@ -234,7 +234,7 @@ class ApiGateway {
       const { query, variables } = req.body;
       const compilerApi = await this.getCompilerApi(req.context);
 
-      const metaConfig = await compilerApi.metaConfig({
+      const metaConfig = await compilerApi.metaConfig(req.context, {
         requestId: req.context.requestId,
       });
 
@@ -267,7 +267,7 @@ class ApiGateway {
         const compilerApi = await this.getCompilerApi(req.context);
         let schema = compilerApi.getGraphQLSchema();
         if (!schema) {
-          let metaConfig = await compilerApi.metaConfig({
+          let metaConfig = await compilerApi.metaConfig(req.context, {
             requestId: req.context.requestId,
           });
           metaConfig = this.filterVisibleItemsInMeta(req.context, metaConfig);
@@ -551,7 +551,7 @@ class ApiGateway {
     try {
       await this.assertApiScope('meta', context.securityContext);
       const compilerApi = await this.getCompilerApi(context);
-      const metaConfig = await compilerApi.metaConfig({
+      const metaConfig = await compilerApi.metaConfig(context, {
         requestId: context.requestId,
         includeCompilerId: includeCompilerId || onlyCompilerId
       });
@@ -587,7 +587,7 @@ class ApiGateway {
     try {
       await this.assertApiScope('meta', context.securityContext);
       const compilerApi = await this.getCompilerApi(context);
-      const metaConfigExtended = await compilerApi.metaConfigExtended({
+      const metaConfigExtended = await compilerApi.metaConfigExtended(context, {
         requestId: context.requestId,
       });
       const { metaConfig, cubeDefinitions } = metaConfigExtended;
@@ -1010,7 +1010,7 @@ class ApiGateway {
           } else {
             const metaCacheKey = JSON.stringify(ctx);
             if (!metaCache.has(metaCacheKey)) {
-              metaCache.set(metaCacheKey, await compiler.metaConfigExtended(ctx));
+              metaCache.set(metaCacheKey, await compiler.metaConfigExtended(context, ctx));
             }
 
             // checking and fetching result status
@@ -1195,8 +1195,14 @@ class ApiGateway {
           }
 
           const normalizedQuery = normalizeQuery(currentQuery, persistent);
-          let rewrittenQuery = await this.queryRewrite(
+          // First apply cube/view level security policies
+          let rewrittenQuery = (await this.compilerApi(context)).applyRowLevelSecurity(
             normalizedQuery,
+            context
+          );
+          // Then apply user-supplied queryRewrite
+          rewrittenQuery = await this.queryRewrite(
+            rewrittenQuery,
             context,
           );
 
@@ -1693,7 +1699,7 @@ class ApiGateway {
         await this.getNormalizedQueries(query, context);
 
       let metaConfigResult = await (await this
-        .getCompilerApi(context)).metaConfig({
+        .getCompilerApi(context)).metaConfig(request.context, {
         requestId: context.requestId
       });
 
@@ -1803,7 +1809,7 @@ class ApiGateway {
         await this.getNormalizedQueries(query, context, request.streaming, request.memberExpressions);
 
       const compilerApi = await this.getCompilerApi(context);
-      let metaConfigResult = await compilerApi.metaConfig({
+      let metaConfigResult = await compilerApi.metaConfig(request.context, {
         requestId: context.requestId
       });
 

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -112,9 +112,71 @@ export class CubeEvaluator extends CubeSymbols {
 
     this.prepareHierarchies(cube);
 
+    this.prepareAccessPolicy(cube, errorReporter);
+
     return cube;
   }
 
+  private allMembersOrList(cube: any, specifier: string | string[]): string[] {
+    const types = ['measures', 'dimensions'];
+    if (specifier === '*') {
+      const allMembers = R.unnest(types.map(type => Object.keys(cube[type] || {})));
+      return allMembers;
+    } else {
+      return specifier as string[] || [];
+    }
+  }
+
+  private prepareAccessPolicy(cube: any, errorReporter: ErrorReporter) {
+    if (!cube.accessPolicy) {
+      return;
+    }
+
+    const memberMapper = (memberType: string) => (member: string) => {
+      if (member.indexOf('.') !== -1) {
+        const cubeName = member.split('.')[0];
+        if (cubeName !== cube.name) {
+          errorReporter.error(
+            `Paths aren't allowed in the accessPolicy policy but '${member}' provided as ${memberType} for ${cube.name}`
+          );
+        }
+        return member;
+      }
+      return this.pathFromArray([cube.name, member]);
+    };
+
+    const filterEvaluator = (filter: any) => {
+      if (filter.member) {
+        filter.memberReference = this.evaluateReferences(cube.name, filter.member);
+        filter.memberReference = memberMapper('a filter member reference')(filter.memberReference);
+      } else {
+        if (filter.and) {
+          filter.and.forEach(filterEvaluator);
+        }
+        if (filter.or) {
+          filter.or.forEach(filterEvaluator);
+        }
+      }
+    };
+
+    for (const policy of cube.accessPolicy) {
+      for (const filter of policy?.rowLevel?.filters || []) {
+        filterEvaluator(filter);
+      }
+
+      if (policy.memberLevel) {
+        policy.memberLevel.includesMembers = this.allMembersOrList(
+          cube,
+          policy.memberLevel.includes
+        ).map(memberMapper('an includes member'));
+        policy.memberLevel.excludesMembers = this.allMembersOrList(
+          cube,
+          policy.memberLevel.excludes || []
+        ).map(memberMapper('an excludes member'));
+      }
+    }
+  }
+
   private prepareHierarchies(cube: any) {
     if (Array.isArray(cube.hierarchies)) {
       cube.hierarchies = cube.hierarchies.map(hierarchy => ({

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -10,6 +10,7 @@ import { BaseQuery } from '../adapter';
 const FunctionRegex = /function\s+\w+\(([A-Za-z0-9_,]*)|\(([\s\S]*?)\)\s*=>|\(?(\w+)\)?\s*=>/;
 const CONTEXT_SYMBOLS = {
   SECURITY_CONTEXT: 'securityContext',
+  security_context: 'securityContext',
   FILTER_PARAMS: 'filterParams',
   FILTER_GROUP: 'filterGroup',
   SQL_UTILS: 'sqlUtils'
@@ -139,6 +140,7 @@ export class CubeSymbols {
     this.camelCaseTypes(cube.dimensions);
     this.camelCaseTypes(cube.segments);
     this.camelCaseTypes(cube.preAggregations);
+    this.camelCaseTypes(cube.accessPolicy);
 
     if (cube.preAggregations) {
       this.transformPreAggregations(cube.preAggregations);
@@ -406,6 +408,27 @@ export class CubeSymbols {
     });
   }
 
+  // Used to evaluate access policies to allow referencing security_context at query time
+  evaluateContextFunction(cube, contextFn, context = {}) {
+    const cubeEvaluator = this;
+
+    const res = cubeEvaluator.resolveSymbolsCall(contextFn, (name) => {
+      const resolvedSymbol = this.resolveSymbol(cube, name);
+      if (resolvedSymbol) {
+        return resolvedSymbol;
+      }
+      throw new UserError(
+        `Cube references are not allowed when evaluating RLS conditions or filters. Found: ${name} in ${cube.name}`
+      );
+    }, {
+      contextSymbols: {
+        securityContext: context.securityContext,
+      }
+    });
+
+    return res;
+  }
+
   evaluateReferences(cube, referencesFn, options = {}) {
     const cubeEvaluator = this;
 
@@ -437,7 +460,7 @@ export class CubeSymbols {
       collectJoinHints: options.collectJoinHints,
     });
     if (!Array.isArray(arrayOrSingle)) {
-      return arrayOrSingle.toString();
+      return arrayOrSingle && arrayOrSingle.toString();
     }
 
     const references = arrayOrSingle.map(p => p.toString());
@@ -458,6 +481,10 @@ export class CubeSymbols {
         res = res.fn.apply(null, res.memberNames.map((id) => nameResolver(id.trim())));
       }
       return res;
+    } catch (e) {
+      // TODO(maxim): should we keep this log?
+      console.log('Error while resolving Cube symbols: ', e);
+      console.error(e);
     } finally {
       this.resolveSymbolsCallContext = oldContext;
     }

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js

@@ -105,6 +105,7 @@ export class CubeToMetaTransformer {
           })),
           R.toPairs
         )(cube.segments || {}),
+        accessPolicy: cube.accessPolicy || [],
         hierarchies: cube.hierarchies || []
       },
     };

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -25,7 +25,8 @@ export const nonStringFields = new Set([
   'external',
   'useOriginalSqlPreAggregations',
   'readOnly',
-  'prefix'
+  'prefix',
+  'if',
 ]);
 
 const identifierRegex = /^[_a-zA-Z][_a-zA-Z0-9]*$/;
@@ -615,6 +616,63 @@ const SegmentsSchema = Joi.object().pattern(identifierRegex, Joi.object().keys({
   public: Joi.boolean().strict(),
 }));
 
+const PolicyFilterSchema = Joi.object().keys({
+  member: Joi.func().required(),
+  memberReference: Joi.string(),
+  operator: Joi.any().valid(
+    'equals',
+    'notEquals',
+    'contains',
+    'notContains',
+    'startsWith',
+    'notStartsWith',
+    'endsWith',
+    'notEndsWith',
+    'gt',
+    'gte',
+    'lt',
+    'lte',
+    'inDateRange',
+    'notInDateRange',
+    'beforeDate',
+    'beforeOrOnDate',
+    'afterDate',
+    'afterOrOnDate',
+  ).required(),
+  values: Joi.func().required(),
+});
+
+const PolicyFilterConditionSchema = Joi.object().keys({
+  or: Joi.array().items(PolicyFilterSchema, Joi.link('...').description('Filter Condition schema')),
+  and: Joi.array().items(PolicyFilterSchema, Joi.link('...').description('Filter Condition schema')),
+}).xor('or', 'and');
+
+const MemberLevelPolicySchema = Joi.object().keys({
+  // TODO(maxim): these should be .func()? Should they even allow references?
+  includes: Joi.alternatives([
+    Joi.string().valid('*'),
+    Joi.array().items(Joi.string().required())
+  ]).required(),
+  excludes: Joi.alternatives([
+    Joi.array().items(Joi.string().required())
+  ]),
+  includesMembers: Joi.array().items(Joi.string().required()),
+  excludesMembers: Joi.array().items(Joi.string().required()),
+});
+
+const RowLevelPolicySchema = Joi.object().keys({
+  filters: Joi.array().items(PolicyFilterSchema, PolicyFilterConditionSchema).required(),
+});
+
+const RolePolicySchema = Joi.object().keys({
+  role: Joi.string().required(),
+  memberLevel: MemberLevelPolicySchema,
+  rowLevel: RowLevelPolicySchema,
+  conditions: Joi.array().items(Joi.object().keys({
+    if: Joi.func().required(),
+  })),
+});
+
 /* *****************************
  * ATTENTION:
  * In case of adding/removing/changing any Joi.func() field that needs to be transpiled,
@@ -692,6 +750,7 @@ const baseSchema = {
     title: Joi.string(),
     levels: Joi.func()
   })),
+  accessPolicy: Joi.array().items(RolePolicySchema),
 };
 
 const cubeSchema = inherit(baseSchema, {

packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts

@@ -25,6 +25,10 @@ export const transpiledFieldsPatterns: Array<RegExp> = [
   /^excludes$/,
   /^hierarchies\.[0-9]+\.levels$/,
   /^cubes\.[0-9]+\.(joinPath|join_path)$/,
+  /^(accessPolicy|access_policy)\.[0-9]+\.(rowLevel|row_level)\.filters\.[0-9]+.*\.member$/,
+  /^(accessPolicy|access_policy)\.[0-9]+\.(rowLevel|row_level)\.filters\.[0-9]+.*\.values$/,
+  /^(accessPolicy|access_policy)\.[0-9]+\.conditions.[0-9]+\.if$/,
+  // /^(accessPolicy|access_policy)\.[0-9]+\.(memberLevel|member_level)\.(includes|excludes)$/,
 ];
 
 export const transpiledFields: Set<String> = new Set<String>();

packages/cubejs-schema-compiler/src/compiler/utils.ts

@@ -48,6 +48,7 @@ export function camelizeCube(cube: any): unknown {
   camelizeObjectPart(cube.dimensions, false);
   camelizeObjectPart(cube.preAggregations, false);
   camelizeObjectPart(cube.cubes, false);
+  camelizeObjectPart(cube.accessPolicy, false);
 
   return cube;
 }

packages/cubejs-schema-compiler/test/unit/schema.test.ts

@@ -1,5 +1,5 @@
 import { prepareCompiler } from './PrepareCompiler';
-import { createCubeSchema, createCubeSchemaWithCustomGranularities } from './utils';
+import { createCubeSchema, createCubeSchemaWithCustomGranularities, createCubeSchemaWithAccessPolicy } from './utils';
 
 describe('Schema Testing', () => {
   const schemaCompile = async () => {
@@ -367,4 +367,15 @@ describe('Schema Testing', () => {
       CubeD: { relationship: 'belongsTo' }
     });
   });
+
+  it('valid schema with accessPolicy', async () => {
+    const { compiler, metaTransformer } = prepareCompiler([
+      createCubeSchemaWithAccessPolicy('ProtectedCube'),
+    ]);
+    await compiler.compile();
+    compiler.throwIfAnyErrors();
+
+    // TODO(maxim): this should be further validated
+    expect(metaTransformer.cubes[0].config.accessPolicy).toBeDefined();
+  });
 });