Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: expedite benchmark tests #159

Closed
wants to merge 2 commits into from
Closed

fix: expedite benchmark tests #159

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
192215a
fix: expedite benchmark tests
noob20000405 Nov 1, 2021
9105df5
fix: expedite benchmark tests
noob20000405 Nov 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 2 additions & 48 deletions tests/benchmarks/enforcer_cached_b.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ BENCHMARK(BenchmarkCachedRBACModel);

// BENCHMARK(BenchmarkCachedRaw);

static void BenchmarkCachedRBACModelSmall(benchmark::State& state) {
static void BenchmarkCachedRBACModelLoop(benchmark::State& state) {
casbin::CachedEnforcer e(rbac_model_path, "", false);
// 100 roles, 10 resources.
for (int i = 0; i < 100; ++i)
Expand All @@ -60,53 +60,7 @@ static void BenchmarkCachedRBACModelSmall(benchmark::State& state) {
e.Enforce(params);
}

BENCHMARK(BenchmarkCachedRBACModelSmall);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think commenting this out is better than deleting it right away. If we comment out only this specific line, the benchmark for BenchmarkCachedRBACModelSmall won't run. However, the user has the option to run it anyway.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you think about this @noob20000405?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@EmperorYP7 Totally agree with you, I'll change it now.


static void BenchmarkCachedRBACModelMedium(benchmark::State& state) {
casbin::CachedEnforcer e(rbac_model_path, "", false);
std::vector<std::vector<std::string>> p_policies(1000);
// 1000 roles, 100 resources.
for (int i = 0; i < 1000; ++i)
p_policies[i] = { "group" + std::to_string(i), "data" + std::to_string(i / 10), "read" };

e.AddPolicies(p_policies);

// 10000 users.
std::vector<std::vector<std::string>> g_policies(10000);
for (int i = 0; i < 10000; ++i)
g_policies[i] = { "user" + std::to_string(i), "group" + std::to_string(i/10) };

e.AddGroupingPolicies(g_policies);
casbin::DataList params = {"user5001", "data150", "read"};
for (auto _ : state)
e.Enforce(params);
}

// BENCHMARK(BenchmarkCachedRBACModelMedium);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Likewise


static void BenchmarkCachedRBACModelLarge(benchmark::State& state) {
casbin::CachedEnforcer e(rbac_model_path, "", false);

// 10000 roles, 1000 resources.
std::vector<std::vector<std::string>> p_policies(10000);
for (int i = 0; i < 10000; ++i)
p_policies[i] = {"group", std::to_string(i), "data", std::to_string(i / 10), "read"};
e.AddPolicies(p_policies);

// 100000 users.
std::vector<std::vector<std::string>> g_policies(100000);
for (int i = 0; i < 100000; ++i) {
g_policies[i] = {"user" + std::to_string(i), "group", std::to_string(i / 10)};
}
e.AddGroupingPolicies(g_policies);
casbin::DataList params = {"user50001", "data1500", "read"};
for (auto _ : state)
{
e.Enforce(params);
}
}

// BENCHMARK(BenchmarkCachedRBACModelLarge);
BENCHMARK(BenchmarkCachedRBACModelLoop);

static void BenchmarkCachedRBACModelWithResourceRoles(benchmark::State& state) {
casbin::CachedEnforcer e(rbac_with_resource_roles_model_path, rbac_with_resource_roles_policy_path, false);
Expand Down
94 changes: 6 additions & 88 deletions tests/benchmarks/management_api_b.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ static void BenchmarkVectorOperations(benchmark::State& state) {

BENCHMARK(BenchmarkVectorOperations);

static void BenchmarkHasPolicySmall(benchmark::State& state) {
static void BenchmarkHasPolicyLoop(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 100 roles, 10 resources.
Expand All @@ -60,37 +60,9 @@ static void BenchmarkHasPolicySmall(benchmark::State& state) {
params = { "user" + std::to_string(GetRandom100()), "data" + std::to_string(GetRandom100()/10), "read" }, e.HasPolicy(params);
}

BENCHMARK(BenchmarkHasPolicySmall);
BENCHMARK(BenchmarkHasPolicyLoop);

static void BenchmarkHasPolicyMedium(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 1000 roles, 100 resources.
// std::vector<std::vector<std::string>> p_policies(1000);
for (int i = 0; i < 1000; ++i)
params = {"user" + std::to_string(i), "data" + std::to_string(i / 10), "read"}, e.AddPolicy(params);
// e.AddPolicies(p_policies);
for (auto _ : state)
params = { "user" + std::to_string(GetRandom1000()), "data" + std::to_string(GetRandom1000()/10), "read" }, e.HasPolicy(params);
}

BENCHMARK(BenchmarkHasPolicyMedium);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Likewise


static void BenchmarkHasPolicyLarge(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 10000 roles, 1000 resources.
for (int i = 0; i < 10000; i++)
params = {"user" + std::to_string(i), "data" + std::to_string(i / 10), "read"}, e.AddPolicy(params);

for(auto _ : state) {
params = {"user" + std::to_string(GetRandom10000()), "data" + std::to_string(GetRandom10000()/10), "read"}, e.HasPolicy(params);
}
}

BENCHMARK(BenchmarkHasPolicyLarge);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Likewise


static void BenchmarkAddPolicySmall(benchmark::State& state) {
static void BenchmarkAddPolicyLoop(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 100 roles, 10 resources.
Expand All @@ -101,38 +73,9 @@ static void BenchmarkAddPolicySmall(benchmark::State& state) {
params = {"user" + std::to_string(GetRandom100() + 100), "data" + std::to_string((GetRandom100() + 100)/10), "read"}, e.AddPolicy(params);
}

BENCHMARK(BenchmarkAddPolicySmall);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here as well


static void BenchmarkAddPolicyMedium(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 1000 roles, 100 resources.
for(int i = 0; i < 1000; ++i)
params = {"user" + std::to_string(i), "data" + std::to_string(i / 10), "read"}, e.AddPolicy(params);
// _, err := e.AddPolicies(pPolicies)

for(auto _ : state) {
params = {"user" + std::to_string(GetRandom1000() + 1000), "data" + std::to_string((GetRandom1000() + 1000) / 10), "read"}, e.AddPolicy(params);
}
}

BENCHMARK(BenchmarkAddPolicyMedium);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and so on. Hope you got the pattern


static void BenchmarkAddPolicyLarge(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 10000 roles, 1000 resources.
for(int i = 0; i < 10000; ++i)
params = { "user" + std::to_string(i), "data" + std::to_string(i/10), "read" }, e.AddPolicy(params);

for(auto _ : state) {
params = { "user" + std::to_string(GetRandom10000() + 10000), "data" + std::to_string((GetRandom10000() + 10000) / 10), "read" }, e.AddPolicy(params);
}
}

BENCHMARK(BenchmarkAddPolicyLarge);
BENCHMARK(BenchmarkAddPolicyLoop);

static void BenchmarkRemovePolicySmall(benchmark::State& state) {
static void BenchmarkRemovePolicyLoop(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 100 roles, 10 resources.
Expand All @@ -143,30 +86,5 @@ static void BenchmarkRemovePolicySmall(benchmark::State& state) {
params = { "user" + std::to_string(GetRandom100()), "data" + std::to_string(GetRandom100() / 10), "read" }, e.RemovePolicy(params);
}

BENCHMARK(BenchmarkRemovePolicySmall);

static void BenchmarkRemovePolicyMedium(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 1000 roles, 100 resources.
for(int i = 0; i < 1000; ++i)
params = {"user" + std::to_string(i), "data" + std::to_string(i / 10), "read"}, e.AddPolicy(params);

for(auto _ : state)
params = { "user" + std::to_string(GetRandom1000()), "data" + std::to_string(GetRandom1000() / 10), "read" }, e.RemovePolicy(params);
}

BENCHMARK(BenchmarkRemovePolicyMedium);

static void BenchmarkRemovePolicyLarge(benchmark::State& state) {
casbin::Enforcer e(basic_model_path);

// 10000 roles, 1000 resources.
for(int i = 0; i < 10000; ++i)
params = { "user" + std::to_string(i), "data" + std::to_string(i / 10), "read" }, e.AddPolicy(params);

for(auto _ : state)
params = { "user" + std::to_string(GetRandom10000()), "data" + std::to_string(GetRandom1000()), "read" }, e.RemovePolicy(params);
}
BENCHMARK(BenchmarkRemovePolicyLoop);

BENCHMARK(BenchmarkRemovePolicyLarge);
43 changes: 3 additions & 40 deletions tests/benchmarks/model_b.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ static void BenchmarkRBACModel(benchmark::State& state) {

BENCHMARK(BenchmarkRBACModel);

static void BenchmarkRBACModelSmall(benchmark::State& state) {
static void BenchmarkRBACModelLoop(benchmark::State& state) {
casbin::Enforcer e(rbac_model_path);

// 100 roles, 10 resources.
Expand All @@ -74,44 +74,7 @@ static void BenchmarkRBACModelSmall(benchmark::State& state) {
e.Enforce(params);
}

BENCHMARK(BenchmarkRBACModelSmall);

static void BenchmarkRBACModelMedium(benchmark::State& state) {
casbin::Enforcer e(rbac_model_path);

// 1000 roles, 100 resources.
for (int i = 0; i < 1000; ++i)
e.AddPolicy({"group" + std::to_string(i), "data" + std::to_string(i / 10), "read"});

// 10000 users.
for (int i = 0; i < 10000; ++i)
e.AddGroupingPolicy({"user" + std::to_string(i), "group" + std::to_string(i / 10)});

casbin::DataList params = {"user5001", "data99", "read"};
for (auto _ : state)
e.Enforce(params);
}

BENCHMARK(BenchmarkRBACModelMedium);

static void BenchmarkRBACModelLarge(benchmark::State& state) {
casbin::Enforcer e(rbac_model_path);

// 10000 roles, 1000 resources.
for(int i = 0; i < 10000; ++i)
e.AddPolicy({"group" + std::to_string(i), "data" + std::to_string(i / 10), "read"});

// 100000 users.
for(int i = 0; i < 100000; i++)
e.AddGroupingPolicy({"user" + std::to_string(i), "group" + std::to_string(i / 10)});

casbin::DataList params = {"user50001", "data999", "read"};

for(auto _ : state)
e.Enforce(params);
}

// BENCHMARK(BenchmarkRBACModelLarge);
BENCHMARK(BenchmarkRBACModelLoop);

static void BenchmarkRBACModelWithResourceRoles(benchmark::State& state) {
casbin::Enforcer e(rbac_with_resource_roles_model_path, rbac_with_resource_roles_policy_path);
Expand All @@ -120,7 +83,7 @@ static void BenchmarkRBACModelWithResourceRoles(benchmark::State& state) {
e.Enforce(params);
}

// BENCHMARK(BenchmarkRBACModelWithResourceRoles);
BENCHMARK(BenchmarkRBACModelWithResourceRoles);

static void BenchmarkRBACModelWithDomains(benchmark::State& state) {
casbin::Enforcer e(rbac_with_domains_model_path, rbac_with_domains_policy_path);
Expand Down
54 changes: 2 additions & 52 deletions tests/benchmarks/role_manager_b.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
static std::vector<std::string> params(3);
static std::vector<std::string> g_params(2);

static void BenchmarkRoleManagerSmall(benchmark::State& state) {
static void BenchmarkRoleManagerLoop(benchmark::State& state) {
casbin::Enforcer e(rbac_model_path);
// Do not rebuild the role inheritance relations for every AddGroupingPolicy() call.
e.EnableAutoBuildRoleLinks(false);
Expand All @@ -44,54 +44,4 @@ static void BenchmarkRoleManagerSmall(benchmark::State& state) {
}
}

BENCHMARK(BenchmarkRoleManagerSmall);

static void BenchmarkRoleManagerMedium(benchmark::State& state) {
casbin::Enforcer e(rbac_model_path);
// Do not rebuild the role inheritance relations for every AddGroupingPolicy() call.
e.EnableAutoBuildRoleLinks(false);

// 1000 roles, 100 resources.

for (int i = 0; i < 1000; ++i)
params = {"group" + std::to_string(i), "data" + std::to_string(i / 10), "read"}, e.AddPolicy(params);

// 10000 users.

for (int i = 0; i < 10000; ++i)
g_params = {"user" + std::to_string(i), "group" + std::to_string(i / 10)}, e.AddGroupingPolicy(g_params);

e.BuildRoleLinks();

auto rm = e.GetRoleManager();

for(auto _ : state) {
for(int j = 0; j < 1000; ++j)
rm->HasLink("user501", "group" + std::to_string(j));
}
}

BENCHMARK(BenchmarkRoleManagerMedium);

static void BenchmarkRoleManagerLarge(benchmark::State& state) {
casbin::Enforcer e(rbac_model_path);

// 10000 roles, 1000 resources.

for (int i = 0; i < 10000; ++i)
params = {"group" + std::to_string(i), "data" + std::to_string(i / 10), "read"}, e.AddPolicy(params);

// 100000 users.

for (int i = 0; i < 100000; ++i)
g_params = {"user" + std::to_string(i), "group" + std::to_string(i / 10)}, e.AddGroupingPolicy(g_params);

auto rm = e.GetRoleManager();

for(auto _ : state) {
for(int j = 0; j < 10000; ++j)
rm->HasLink("user501", "group" + std::to_string(j));
}
}

// BENCHMARK(BenchmarkRoleManagerLarge);
BENCHMARK(BenchmarkRoleManagerLoop);