From d9983aa8cd336cffed87e472212bd391b2c4d94a Mon Sep 17 00:00:00 2001 From: Abdul Qadir Date: Wed, 20 Dec 2023 19:17:05 +0500 Subject: [PATCH] Add api to fetch api usage data --- .../lowcoder/infra/serverlog/ServerLog.java | 4 ++- .../infra/serverlog/ServerLogRepository.java | 5 +++ .../infra/serverlog/ServerLogService.java | 14 ++++++++ .../framework/filter/GlobalContextFilter.java | 34 ++++++++++++------- .../api/usermanagement/OrgApiService.java | 3 ++ .../api/usermanagement/OrgApiServiceImpl.java | 10 ++++++ .../OrganizationController.java | 6 ++++ .../usermanagement/OrganizationEndpoints.java | 9 +++++ .../runner/migrations/DatabaseChangelog.java | 7 ++++ 9 files changed, 79 insertions(+), 13 deletions(-) diff --git a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLog.java b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLog.java index 4ce13d0c4..addf42bc3 100644 --- a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLog.java +++ b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLog.java @@ -15,6 +15,7 @@ @Builder public class ServerLog { private String userId; + private String orgId; private String urlPath; private String httpMethod; private String requestBody; @@ -22,8 +23,9 @@ public class ServerLog { private long createTime; @JsonCreator - private ServerLog(String userId, String urlPath, String httpMethod, String requestBody, Map queryParameters, long createTime) { + private ServerLog(String userId, String orgId, String urlPath, String httpMethod, String requestBody, Map queryParameters, long createTime) { this.userId = userId; + this.orgId = orgId; this.urlPath = urlPath; this.createTime = createTime; this.httpMethod = httpMethod; diff --git a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogRepository.java b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogRepository.java index bb6ecae8b..7e09a9151 100644 --- a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogRepository.java +++ b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogRepository.java @@ -1,8 +1,13 @@ package org.lowcoder.infra.serverlog; import org.springframework.data.mongodb.repository.ReactiveMongoRepository; +import reactor.core.publisher.Mono; public interface ServerLogRepository extends ReactiveMongoRepository { + Mono countByOrgId(String orgId); + + Mono countByOrgIdAndCreateTimeBetween(String orgId, Long startMonthEpoch, Long endMonthEpoch); + } diff --git a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogService.java b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogService.java index 8b9a38c64..e975ba407 100644 --- a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogService.java +++ b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogService.java @@ -2,6 +2,9 @@ import static org.lowcoder.infra.perf.PerfEvent.SERVER_LOG_BATCH_INSERT; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.temporal.TemporalAdjusters; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; @@ -13,6 +16,7 @@ import org.springframework.stereotype.Service; import io.micrometer.core.instrument.Tags; +import reactor.core.publisher.Mono; @Service public class ServerLogService { @@ -42,4 +46,14 @@ private void scheduledInsert() { perfHelper.count(SERVER_LOG_BATCH_INSERT, Tags.of("size", String.valueOf(result.size()))); }); } + + public Mono getApiUsageCount(String orgId, Boolean lastMonthOnly) { + if(lastMonthOnly != null && lastMonthOnly) { + Long startMonthEpoch = LocalDateTime.now().minusMonths(1).with(TemporalAdjusters.firstDayOfMonth()).toEpochSecond(ZoneOffset.UTC)*1000; + Long endMonthEpoch = LocalDateTime.now().minusMonths(1).with(TemporalAdjusters.lastDayOfMonth()).toEpochSecond(ZoneOffset.UTC)*1000; + return serverLogRepository.countByOrgIdAndCreateTimeBetween(orgId, startMonthEpoch, endMonthEpoch); + } + return serverLogRepository.countByOrgId(orgId); + } + } diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/framework/filter/GlobalContextFilter.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/framework/filter/GlobalContextFilter.java index 3d30f5c89..fe7c889ea 100644 --- a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/framework/filter/GlobalContextFilter.java +++ b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/framework/filter/GlobalContextFilter.java @@ -71,18 +71,7 @@ public class GlobalContextFilter implements WebFilter, Ordered { public Mono filter(@Nonnull ServerWebExchange exchange, @Nonnull WebFilterChain chain) { return sessionUserService.getVisitorId() - .doOnNext(visitorId -> { - if (isAnonymousUser(visitorId)) { - return; - } - ServerLog serverLog = ServerLog.builder() - .userId(visitorId) - .urlPath(exchange.getRequest().getPath().toString()) - .httpMethod(Optional.ofNullable(exchange.getRequest().getMethod()).map(HttpMethod::name).orElse("")) - .createTime(System.currentTimeMillis()) - .build(); - serverLogService.record(serverLog); - }) + .flatMap(visitorId -> saveServerLog(exchange, visitorId)) .flatMap(visitorId -> chain.filter(exchange) .contextWrite(ctx -> { Map contextMap = buildContextMap(exchange, visitorId); @@ -95,6 +84,27 @@ public Mono filter(@Nonnull ServerWebExchange exchange, @Nonnull WebFilter })); } + private Mono saveServerLog(ServerWebExchange exchange, String visitorId) { + if (isAnonymousUser(visitorId)) { + return Mono.just(visitorId); + } + + return orgMemberService + .getCurrentOrgMember(visitorId) + .map(orgMember -> { + ServerLog serverLog = ServerLog.builder() + .orgId(orgMember.getOrgId()) + .userId(visitorId) + .urlPath(exchange.getRequest().getPath().toString()) + .httpMethod(Optional.ofNullable(exchange.getRequest().getMethod()).map(HttpMethod::name).orElse("")) + .createTime(System.currentTimeMillis()) + .build(); + serverLogService.record(serverLog); + return visitorId; + }); + + } + private Map buildContextMap(ServerWebExchange serverWebExchange, String visitorId) { ServerHttpRequest request = serverWebExchange.getRequest(); Map contextMap = request.getHeaders().toSingleValueMap().entrySet() diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiService.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiService.java index 2990d8047..a62ca6d75 100644 --- a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiService.java +++ b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiService.java @@ -13,6 +13,7 @@ import reactor.core.publisher.Mono; + public interface OrgApiService { Mono leaveOrganization(String orgId); @@ -45,5 +46,7 @@ public interface OrgApiService { Mono tryAddUserToOrgAndSwitchOrg(String orgId, String userId); Mono getOrganizationConfigs(String orgId); + + Mono getApiUsageCount(String orgId, Boolean lastMonthOnly); } diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiServiceImpl.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiServiceImpl.java index f50b0018b..6663e09cb 100644 --- a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiServiceImpl.java +++ b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiServiceImpl.java @@ -38,6 +38,7 @@ import org.lowcoder.domain.user.model.Connection; import org.lowcoder.domain.user.model.User; import org.lowcoder.domain.user.service.UserService; +import org.lowcoder.infra.serverlog.ServerLogService; import org.lowcoder.sdk.auth.AbstractAuthConfig; import org.lowcoder.sdk.config.CommonConfig; import org.lowcoder.sdk.config.CommonConfig.Workspace; @@ -76,6 +77,9 @@ public class OrgApiServiceImpl implements OrgApiService { @Autowired private AuthenticationService authenticationService; + @Autowired + private ServerLogService serverLogService; + @Override public Mono getOrganizationMembers(String orgId, int page, int count) { return sessionUserService.getVisitorId() @@ -373,6 +377,12 @@ public Mono getOrganizationConfigs(String orgId) { }); } + @Override + public Mono getApiUsageCount(String orgId, Boolean lastMonthOnly) { + return checkVisitorAdminRole(orgId) + .flatMap(orgMember -> serverLogService.getApiUsageCount(orgId, lastMonthOnly)); + } + private Mono checkIfSaasMode() { return Mono.defer(() -> { if (commonConfig.getWorkspace().getMode() == WorkspaceMode.ENTERPRISE) { diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationController.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationController.java index 919d230e0..484b1aead 100644 --- a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationController.java +++ b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationController.java @@ -126,4 +126,10 @@ public Mono> updateOrgCommonSettings(@PathVariable String .map(ResponseView::success); } + @Override + public Mono> getOrgApiUsageCount(String orgId, Boolean lastMonthOnly) { + return orgApiService.getApiUsageCount(orgId, lastMonthOnly) + .map(ResponseView::success); + } + } \ No newline at end of file diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationEndpoints.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationEndpoints.java index 93fcfe3a4..a5ba2774c 100644 --- a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationEndpoints.java +++ b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationEndpoints.java @@ -160,6 +160,15 @@ public Mono> removeUserFromOrg(@PathVariable String orgId, @PutMapping("/{orgId}/common-settings") public Mono> updateOrgCommonSettings(@PathVariable String orgId, @RequestBody UpdateOrgCommonSettingsRequest request); + @Operation( + tags = TAG_ORGANIZATION_MANAGEMENT, + operationId = "getOrgApiUsageCount", + summary = "Get the api usage count for the org", + description = "Calculate the used api calls for this organization and return the count" + ) + @GetMapping("/{orgId}/api-usage") + public Mono> getOrgApiUsageCount(@PathVariable String orgId, @RequestParam(required = false) Boolean lastMonthOnly); + public record UpdateOrgCommonSettingsRequest(String key, Object value) { } diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/runner/migrations/DatabaseChangelog.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/runner/migrations/DatabaseChangelog.java index 67d1a0d9f..90d62e98b 100644 --- a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/runner/migrations/DatabaseChangelog.java +++ b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/runner/migrations/DatabaseChangelog.java @@ -175,6 +175,13 @@ public void completeAuthType(CompleteAuthType completeAuthType) { completeAuthType.complete(); } + @ChangeSet(order = "019", id = "add-org-id-index-on-server-log", author = "") + public void addOrgIdIndexOnServerLog(MongockTemplate mongoTemplate) { + ensureIndexes(mongoTemplate, ServerLog.class, + makeIndex("orgId") + ); + } + public static Index makeIndex(String... fields) { if (fields.length == 1) { return new Index(fields[0], Sort.Direction.ASC).named(fields[0]);