From 60948bad01e5491f8df7f3a84bdb7735534fd552 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Mon, 29 May 2023 22:18:33 +0300 Subject: [PATCH] Use background job to calculate admin statistics Add db migration Fix tests --- .../org/eclipse/openvsx/admin/AdminAPI.java | 42 +++- .../eclipse/openvsx/admin/AdminService.java | 162 +++++---------- .../admin/AdminStatisticsJobRequest.java | 47 +++++ .../AdminStatisticsJobRequestHandler.java | 123 +++++++++++ .../openvsx/admin/AdminStatisticsService.java | 30 +++ ...nthlyAdminStatisticsJobRequestHandler.java | 30 +++ .../openvsx/entities/AdminStatistics.java | 1 + ...sionVersionSignatureJobRequestHandler.java | 4 + .../AdminStatisticsRepository.java | 4 + .../repositories/RepositoryService.java | 4 + .../openvsx/security/SecurityConfig.java | 4 +- .../V1_38__Unique_AdminStatistics.sql | 29 +++ .../eclipse/openvsx/admin/AdminAPITest.java | 194 +----------------- .../RepositoryServiceSmokeTest.java | 3 +- 14 files changed, 375 insertions(+), 302 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequest.java create mode 100644 server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequestHandler.java create mode 100644 server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsService.java create mode 100644 server/src/main/java/org/eclipse/openvsx/admin/MonthlyAdminStatisticsJobRequestHandler.java create mode 100644 server/src/main/resources/db/migration/V1_38__Unique_AdminStatistics.sql diff --git a/server/src/main/java/org/eclipse/openvsx/admin/AdminAPI.java b/server/src/main/java/org/eclipse/openvsx/admin/AdminAPI.java index 45074f7c2..2995b7481 100644 --- a/server/src/main/java/org/eclipse/openvsx/admin/AdminAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/admin/AdminAPI.java @@ -15,6 +15,7 @@ import java.util.stream.Collectors; import java.net.URI; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Strings; import org.eclipse.openvsx.LocalRegistryService; @@ -42,6 +43,7 @@ @RestController public class AdminAPI { + @Autowired RepositoryService repositories; @@ -57,6 +59,41 @@ public class AdminAPI { @Autowired SearchUtilService search; + @GetMapping( + path = "/admin/reports", + produces = MediaType.APPLICATION_JSON_VALUE + ) + public ResponseEntity>> getReports( + @RequestParam("token") String tokenValue + ) { + try { + validateToken(tokenValue); + return ResponseEntity.ok(admins.getReports()); + } catch (ErrorResultException exc) { + return ResponseEntity.status(exc.getStatus()).build(); + } + } + + @PostMapping( + path = "/admin/report/schedule", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE + ) + public ResponseEntity scheduleReport( + @RequestParam String token, + @RequestBody JsonNode json + ) { + try { + validateToken(token); + var year = json.get("year").asInt(); + var month = json.get("month").asInt(); + admins.scheduleReport(year, month); + return ResponseEntity.accepted().build(); + } catch (ErrorResultException exc) { + return exc.toResponseEntity(ResultJson.class); + } + } + @GetMapping( path = "/admin/report", produces = MediaType.APPLICATION_JSON_VALUE @@ -91,12 +128,15 @@ public ResponseEntity getReportCsv( } } - private AdminStatistics getReport(String tokenValue, int year, int month) { + private void validateToken(String tokenValue) { var accessToken = repositories.findAccessToken(tokenValue); if(accessToken == null || !accessToken.isActive() || accessToken.getUser() == null || !ROLE_ADMIN.equals(accessToken.getUser().getRole())) { throw new ErrorResultException("Invalid access token", HttpStatus.FORBIDDEN); } + } + private AdminStatistics getReport(String tokenValue, int year, int month) { + validateToken(tokenValue); return admins.getAdminStatistics(year, month); } diff --git a/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java b/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java index f3d1d6bef..5f052292c 100644 --- a/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java +++ b/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java @@ -17,25 +17,26 @@ import org.eclipse.openvsx.eclipse.EclipseService; import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.json.*; +import org.eclipse.openvsx.migration.HandlerJobRequest; +import org.eclipse.openvsx.migration.MigrationRunner; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.storage.StorageUtilService; import org.eclipse.openvsx.util.*; import org.jobrunr.scheduling.JobRequestScheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.jobrunr.scheduling.cron.Cron; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.event.EventListener; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; -import org.springframework.util.StopWatch; import javax.persistence.EntityManager; import javax.transaction.Transactional; -import java.time.DateTimeException; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; -import java.util.Comparator; -import java.util.LinkedHashSet; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.ZoneId; +import java.util.*; import java.util.stream.Collectors; import static org.eclipse.openvsx.entities.FileResource.*; @@ -43,7 +44,6 @@ @Component public class AdminService { - private static final Logger LOGGER = LoggerFactory.getLogger(AdminService.class); @Autowired RepositoryService repositories; @@ -77,6 +77,12 @@ public class AdminService { @Autowired JobRequestScheduler scheduler; + @EventListener + public void applicationStarted(ApplicationStartedEvent event) { + var jobRequest = new HandlerJobRequest<>(MonthlyAdminStatisticsJobRequestHandler.class); + scheduler.scheduleRecurrently("MonthlyAdminStatistics", Cron.monthly(1, 0, 3), ZoneId.of("UTC"), jobRequest); + } + @Transactional(rollbackOn = ErrorResultException.class) public ResultJson deleteExtension(String namespaceName, String extensionName, UserData admin) throws ErrorResultException { @@ -347,8 +353,28 @@ public void logAdminAction(UserData admin, ResultJson result) { } } - @Transactional public AdminStatistics getAdminStatistics(int year, int month) throws ErrorResultException { + validateYearAndMonth(year, month); + var statistics = repositories.findAdminStatisticsByYearAndMonth(year, month); + if(statistics == null) { + throw new NotFoundException(); + } + + return statistics; + } + + public void scheduleReport(int year, int month) { + validateYearAndMonth(year, month); + if(repositories.findAdminStatisticsByYearAndMonth(year, month) != null) { + throw new ErrorResultException("Report for " + year + "/" + month + " already exists"); + } + + var jobIdText = "AdminStatistics::year=" + year + ",month=" + month; + var jobId = UUID.nameUUIDFromBytes(jobIdText.getBytes(StandardCharsets.UTF_8)); + scheduler.enqueue(jobId, new AdminStatisticsJobRequest(year, month)); + } + + private void validateYearAndMonth(int year, int month) { if(year < 0) { throw new ErrorResultException("Year can't be negative", HttpStatus.BAD_REQUEST); } @@ -356,112 +382,20 @@ public AdminStatistics getAdminStatistics(int year, int month) throws ErrorResul throw new ErrorResultException("Month must be a value between 1 and 12", HttpStatus.BAD_REQUEST); } - var now = LocalDateTime.now(); - if(year > now.getYear() || (year == now.getYear() && month > now.getMonthValue())) { + var now = TimeUtil.getCurrentUTC(); + if(year > now.getYear() || (year == now.getYear() && month >= now.getMonthValue())) { throw new ErrorResultException("Combination of year and month lies in the future", HttpStatus.BAD_REQUEST); } + } - var statistics = repositories.findAdminStatisticsByYearAndMonth(year, month); - if(statistics == null) { - LocalDateTime startInclusive; - try { - startInclusive = LocalDateTime.of(year, month, 1, 0, 0); - } catch(DateTimeException e) { - throw new ErrorResultException("Invalid month or year", HttpStatus.BAD_REQUEST); - } - - var currentYearAndMonth = now.getYear() == year && now.getMonthValue() == month; - var endExclusive = currentYearAndMonth - ? now.truncatedTo(ChronoUnit.MINUTES) - : startInclusive.plusMonths(1); - - LOGGER.info(">> ADMIN REPORT STATS"); - var stopwatch = new StopWatch(); - stopwatch.start("repositories.countActiveExtensions"); - var extensions = repositories.countActiveExtensions(endExclusive); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.downloadsBetween"); - var downloads = repositories.downloadsBetween(startInclusive, endExclusive); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.downloadsUntil"); - var downloadsTotal = repositories.downloadsUntil(endExclusive); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.countActiveExtensionPublishers"); - var publishers = repositories.countActiveExtensionPublishers(endExclusive); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.averageNumberOfActiveReviewsPerActiveExtension"); - var averageReviewsPerExtension = repositories.averageNumberOfActiveReviewsPerActiveExtension(endExclusive); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.countPublishersThatClaimedNamespaceOwnership"); - var namespaceOwners = repositories.countPublishersThatClaimedNamespaceOwnership(endExclusive); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.countActiveExtensionsGroupedByExtensionReviewRating"); - var extensionsByRating = repositories.countActiveExtensionsGroupedByExtensionReviewRating(endExclusive); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.countActiveExtensionPublishersGroupedByExtensionsPublished"); - var publishersByExtensionsPublished = repositories.countActiveExtensionPublishersGroupedByExtensionsPublished(endExclusive); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - var limit = 10; - - stopwatch.start("repositories.topMostActivePublishingUsers"); - var topMostActivePublishingUsers = repositories.topMostActivePublishingUsers(endExclusive, limit); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.topNamespaceExtensions"); - var topNamespaceExtensions = repositories.topNamespaceExtensions(endExclusive, limit); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.topNamespaceExtensionVersions"); - var topNamespaceExtensionVersions = repositories.topNamespaceExtensionVersions(endExclusive, limit); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - - stopwatch.start("repositories.topMostDownloadedExtensions"); - var topMostDownloadedExtensions = repositories.topMostDownloadedExtensions(endExclusive, limit); - stopwatch.stop(); - LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); - LOGGER.info("<< ADMIN REPORT STATS"); - - statistics = new AdminStatistics(); - statistics.setYear(year); - statistics.setMonth(month); - statistics.setExtensions(extensions); - statistics.setDownloads(downloads); - statistics.setDownloadsTotal(downloadsTotal); - statistics.setPublishers(publishers); - statistics.setAverageReviewsPerExtension(averageReviewsPerExtension); - statistics.setNamespaceOwners(namespaceOwners); - statistics.setExtensionsByRating(extensionsByRating); - statistics.setPublishersByExtensionsPublished(publishersByExtensionsPublished); - statistics.setTopMostActivePublishingUsers(topMostActivePublishingUsers); - statistics.setTopNamespaceExtensions(topNamespaceExtensions); - statistics.setTopNamespaceExtensionVersions(topNamespaceExtensionVersions); - statistics.setTopMostDownloadedExtensions(topMostDownloadedExtensions); - - if(!currentYearAndMonth) { - // archive statistics for quicker lookup next time - entityManager.persist(statistics); - } - } - - return statistics; + public Map> getReports() { + return repositories.findAllAdminStatistics().stream() + .sorted(Comparator.comparingInt(AdminStatistics::getYear).thenComparing(AdminStatistics::getMonth)) + .map(stat -> { + var yearText = String.valueOf(stat.getYear()); + var monthText = String.valueOf(stat.getMonth()); + return new AbstractMap.SimpleEntry<>(yearText, monthText); + }) + .collect(Collectors.groupingBy(Map.Entry::getKey, () -> new LinkedHashMap<>(), Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequest.java b/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequest.java new file mode 100644 index 000000000..8ce85285d --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequest.java @@ -0,0 +1,47 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.admin; + +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.jobs.lambdas.JobRequestHandler; + +public class AdminStatisticsJobRequest implements JobRequest { + + private int year; + private int month; + + public AdminStatisticsJobRequest() {} + + public AdminStatisticsJobRequest(int year, int month) { + this.year = year; + this.month = month; + } + + public int getYear() { + return year; + } + + public void setYear(int year) { + this.year = year; + } + + public int getMonth() { + return month; + } + + public void setMonth(int month) { + this.month = month; + } + + @Override + public Class getJobRequestHandler() { + return AdminStatisticsJobRequestHandler.class; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequestHandler.java new file mode 100644 index 000000000..7cb8711b9 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequestHandler.java @@ -0,0 +1,123 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.admin; + +import org.eclipse.openvsx.entities.AdminStatistics; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.StopWatch; + +import java.time.LocalDateTime; + +@Component +public class AdminStatisticsJobRequestHandler implements JobRequestHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(AdminStatisticsJobRequestHandler.class); + + @Autowired + RepositoryService repositories; + + @Autowired + AdminStatisticsService service; + + @Override + public void run(AdminStatisticsJobRequest jobRequest) throws Exception { + var year = jobRequest.getYear(); + var month = jobRequest.getMonth(); + var startInclusive = LocalDateTime.of(year, month, 1, 0, 0); + var endExclusive = startInclusive.plusMonths(1); + + LOGGER.info(">> ADMIN REPORT STATS {} {}", year, month); + var stopwatch = new StopWatch(); + stopwatch.start("repositories.countActiveExtensions"); + var extensions = repositories.countActiveExtensions(endExclusive); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.downloadsBetween"); + var downloads = repositories.downloadsBetween(startInclusive, endExclusive); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.downloadsUntil"); + var downloadsTotal = repositories.downloadsUntil(endExclusive); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.countActiveExtensionPublishers"); + var publishers = repositories.countActiveExtensionPublishers(endExclusive); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.averageNumberOfActiveReviewsPerActiveExtension"); + var averageReviewsPerExtension = repositories.averageNumberOfActiveReviewsPerActiveExtension(endExclusive); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.countPublishersThatClaimedNamespaceOwnership"); + var namespaceOwners = repositories.countPublishersThatClaimedNamespaceOwnership(endExclusive); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.countActiveExtensionsGroupedByExtensionReviewRating"); + var extensionsByRating = repositories.countActiveExtensionsGroupedByExtensionReviewRating(endExclusive); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.countActiveExtensionPublishersGroupedByExtensionsPublished"); + var publishersByExtensionsPublished = repositories.countActiveExtensionPublishersGroupedByExtensionsPublished(endExclusive); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + var limit = 10; + + stopwatch.start("repositories.topMostActivePublishingUsers"); + var topMostActivePublishingUsers = repositories.topMostActivePublishingUsers(endExclusive, limit); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.topNamespaceExtensions"); + var topNamespaceExtensions = repositories.topNamespaceExtensions(endExclusive, limit); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.topNamespaceExtensionVersions"); + var topNamespaceExtensionVersions = repositories.topNamespaceExtensionVersions(endExclusive, limit); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + + stopwatch.start("repositories.topMostDownloadedExtensions"); + var topMostDownloadedExtensions = repositories.topMostDownloadedExtensions(endExclusive, limit); + stopwatch.stop(); + LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis()); + LOGGER.info("<< ADMIN REPORT STATS {} {}", year, month); + + var statistics = new AdminStatistics(); + statistics.setYear(year); + statistics.setMonth(month); + statistics.setExtensions(extensions); + statistics.setDownloads(downloads); + statistics.setDownloadsTotal(downloadsTotal); + statistics.setPublishers(publishers); + statistics.setAverageReviewsPerExtension(averageReviewsPerExtension); + statistics.setNamespaceOwners(namespaceOwners); + statistics.setExtensionsByRating(extensionsByRating); + statistics.setPublishersByExtensionsPublished(publishersByExtensionsPublished); + statistics.setTopMostActivePublishingUsers(topMostActivePublishingUsers); + statistics.setTopNamespaceExtensions(topNamespaceExtensions); + statistics.setTopNamespaceExtensionVersions(topNamespaceExtensionVersions); + statistics.setTopMostDownloadedExtensions(topMostDownloadedExtensions); + service.saveAdminStatistics(statistics); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsService.java b/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsService.java new file mode 100644 index 000000000..0db4d3b9f --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsService.java @@ -0,0 +1,30 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.admin; + +import org.eclipse.openvsx.entities.AdminStatistics; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.persistence.EntityManager; +import javax.transaction.Transactional; +import java.util.Map; + +@Component +public class AdminStatisticsService { + + @Autowired + EntityManager entityManager; + + @Transactional + public void saveAdminStatistics(AdminStatistics statistics) { + entityManager.persist(statistics); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/admin/MonthlyAdminStatisticsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/admin/MonthlyAdminStatisticsJobRequestHandler.java new file mode 100644 index 000000000..005bd8e8a --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/admin/MonthlyAdminStatisticsJobRequestHandler.java @@ -0,0 +1,30 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.admin; + +import org.eclipse.openvsx.migration.HandlerJobRequest; +import org.eclipse.openvsx.util.TimeUtil; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.jobrunr.scheduling.JobRequestScheduler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class MonthlyAdminStatisticsJobRequestHandler implements JobRequestHandler> { + + @Autowired + AdminService admins; + + @Override + public void run(HandlerJobRequest jobRequest) throws Exception { + var lastMonth = TimeUtil.getCurrentUTC().minusMonths(1); + admins.scheduleReport(lastMonth.getYear(), lastMonth.getMonthValue()); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/entities/AdminStatistics.java b/server/src/main/java/org/eclipse/openvsx/entities/AdminStatistics.java index f31d75bd6..54f37af98 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/AdminStatistics.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/AdminStatistics.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; @Entity +@Table(uniqueConstraints = { @UniqueConstraint(columnNames = { "year", "month"})}) public class AdminStatistics { @Id diff --git a/server/src/main/java/org/eclipse/openvsx/migration/ExtensionVersionSignatureJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/ExtensionVersionSignatureJobRequestHandler.java index 665705a4a..4a75a50c8 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/ExtensionVersionSignatureJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/ExtensionVersionSignatureJobRequestHandler.java @@ -45,6 +45,10 @@ public class ExtensionVersionSignatureJobRequestHandler implements JobRequestHan @Job(name = "Generate signature for extension version", retries = 3) public void run(MigrationJobRequest jobRequest) throws Exception { var extVersion = migrations.getExtension(jobRequest.getEntityId()); + if(extVersion == null) { + return; + } + logger.info("Generating signature for: {}", NamingUtil.toLogFormat(extVersion)); var existingSignature = migrations.getFileResource(extVersion, FileResource.DOWNLOAD_SIG); diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/AdminStatisticsRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/AdminStatisticsRepository.java index f1cbead96..6df977557 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/AdminStatisticsRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/AdminStatisticsRepository.java @@ -11,7 +11,11 @@ import org.eclipse.openvsx.entities.AdminStatistics; import org.springframework.data.repository.Repository; +import org.springframework.data.util.Streamable; public interface AdminStatisticsRepository extends Repository { + + Streamable findAll(); + AdminStatistics findByYearAndMonth(int year, int month); } diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index a72ed5c00..8a2a64cf7 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -489,4 +489,8 @@ public void deleteAllKeyPairs() { public SignatureKeyPair findKeyPair(String publicId) { return signatureKeyPairRepo.findByPublicId(publicId); } + + public Streamable findAllAdminStatistics() { + return adminStatisticsRepo.findAll(); + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java b/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java index 7fbf5214d..00a628e5e 100644 --- a/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java +++ b/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java @@ -41,7 +41,7 @@ protected void configure(HttpSecurity http) throws Exception { .permitAll() .antMatchers("/api/*/*/review", "/api/*/*/review/delete", "/api/user/publish", "/api/user/namespace/create") .authenticated() - .antMatchers("/api/**", "/vscode/**", "/documents/**", "/admin/report") + .antMatchers("/api/**", "/vscode/**", "/documents/**", "/admin/report", "/admin/reports", "/admin/report/schedule") .permitAll() .antMatchers("/admin/**") .hasAuthority("ROLE_ADMIN") @@ -53,7 +53,7 @@ protected void configure(HttpSecurity http) throws Exception { .cors() .and() .csrf() - .ignoringAntMatchers("/api/-/publish", "/api/-/namespace/create", "/api/-/query", "/vscode/**") + .ignoringAntMatchers("/api/-/publish", "/api/-/namespace/create", "/api/-/query", "/admin/report/schedule", "/vscode/**") .and() .exceptionHandling() // Respond with 403 status when the user is not logged in diff --git a/server/src/main/resources/db/migration/V1_38__Unique_AdminStatistics.sql b/server/src/main/resources/db/migration/V1_38__Unique_AdminStatistics.sql new file mode 100644 index 000000000..ee33ec1c3 --- /dev/null +++ b/server/src/main/resources/db/migration/V1_38__Unique_AdminStatistics.sql @@ -0,0 +1,29 @@ +CREATE TEMP TABLE tmp_admin_statistics AS +SELECT year, month, MIN(id) id +FROM admin_statistics +GROUP BY year, month; + +DELETE FROM admin_statistics_extensions_by_rating +WHERE admin_statistics_id NOT IN(SELECT id FROM tmp_admin_statistics); + +DELETE FROM admin_statistics_publishers_by_extensions_published +WHERE admin_statistics_id NOT IN(SELECT id FROM tmp_admin_statistics); + +DELETE FROM admin_statistics_top_most_active_publishing_users +WHERE admin_statistics_id NOT IN(SELECT id FROM tmp_admin_statistics); + +DELETE FROM admin_statistics_top_most_downloaded_extensions +WHERE admin_statistics_id NOT IN(SELECT id FROM tmp_admin_statistics); + +DELETE FROM admin_statistics_top_namespace_extension_versions +WHERE admin_statistics_id NOT IN(SELECT id FROM tmp_admin_statistics); + +DELETE FROM admin_statistics_top_namespace_extensions +WHERE admin_statistics_id NOT IN(SELECT id FROM tmp_admin_statistics); + +DELETE FROM admin_statistics +WHERE id NOT IN(SELECT id FROM tmp_admin_statistics); + +DROP TABLE tmp_admin_statistics; + +CREATE UNIQUE INDEX unique_admin_statistics ON admin_statistics(year, month); diff --git a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java index 6f98c0034..9376ee912 100644 --- a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java @@ -836,197 +836,23 @@ public void testArchivedReportJson() throws Exception { } @Test - public void testAdminOnTheFlyReportCsv() throws Exception { + public void testCurrentMonthAdminReportCsv() throws Exception { var token = mockAdminToken(); - var year = 2021; - var month = 7; - var extensions = 9123L; - var downloads = 2145L; - var downloadsTotal = 57199L; - var publishers = 846L; - var averageReviewsPerExtension = 8.75; - var namespaceOwners = 623L; - var extensionsByRating = Map.of(3, 8000, 5, 1123); - var publishersByExtensionsPublished = Map.of(1, 6590, 3, 815); - var topMostActivePublishingUsers = Map.of("u_foo", 93, "u_bar", 543, "u_baz", 82); - var topNamespaceExtensions = Map.of("n_foo", 9, "n_bar", 48, "n_baz", 1239); - var topNamespaceExtensionVersions = Map.of("nv_foo", 234, "nv_bar", 67, "nv_baz", 932); - var topMostDownloadedExtensions = Map.of("foo.bar", 3847L, "bar.foo", 1237L, "foo.baz", 4378L); - - var stats = new AdminStatistics(); - stats.setYear(year); - stats.setMonth(month); - stats.setExtensions(extensions); - stats.setDownloads(downloads); - stats.setDownloadsTotal(downloadsTotal); - stats.setPublishers(publishers); - stats.setAverageReviewsPerExtension(averageReviewsPerExtension); - stats.setNamespaceOwners(namespaceOwners); - stats.setExtensionsByRating(extensionsByRating); - stats.setPublishersByExtensionsPublished(publishersByExtensionsPublished); - stats.setTopMostActivePublishingUsers(topMostActivePublishingUsers); - stats.setTopNamespaceExtensions(topNamespaceExtensions); - stats.setTopNamespaceExtensionVersions(topNamespaceExtensionVersions); - stats.setTopMostDownloadedExtensions(topMostDownloadedExtensions); - - Mockito.when(repositories.findAdminStatisticsByYearAndMonth(year, month)).thenReturn(null); - - var startInclusive = LocalDateTime.of(year, month, 1, 0, 0); - var endExclusive = startInclusive.plusMonths(1); - Mockito.when(repositories.countActiveExtensions(endExclusive)).thenReturn(extensions); - Mockito.when(repositories.downloadsBetween(startInclusive, endExclusive)).thenReturn(downloads); - Mockito.when(repositories.downloadsUntil(endExclusive)).thenReturn(downloadsTotal); - Mockito.when(repositories.countActiveExtensionPublishers(endExclusive)).thenReturn(publishers); - Mockito.when(repositories.averageNumberOfActiveReviewsPerActiveExtension(endExclusive)).thenReturn(averageReviewsPerExtension); - Mockito.when(repositories.countPublishersThatClaimedNamespaceOwnership(endExclusive)).thenReturn(namespaceOwners); - Mockito.when(repositories.countActiveExtensionsGroupedByExtensionReviewRating(endExclusive)).thenReturn(extensionsByRating); - Mockito.when(repositories.countActiveExtensionPublishersGroupedByExtensionsPublished(endExclusive)).thenReturn(publishersByExtensionsPublished); - Mockito.when(repositories.topMostActivePublishingUsers(endExclusive, 10)).thenReturn(topMostActivePublishingUsers); - Mockito.when(repositories.topNamespaceExtensions(endExclusive, 10)).thenReturn(topNamespaceExtensions); - Mockito.when(repositories.topNamespaceExtensionVersions(endExclusive, 10)).thenReturn(topNamespaceExtensionVersions); - Mockito.when(repositories.topMostDownloadedExtensions(endExclusive, 10)).thenReturn(topMostDownloadedExtensions); - - var values = List.of(year, month, extensions, downloads, downloadsTotal, publishers, - averageReviewsPerExtension, namespaceOwners, 0, 0, 8000, 0, 1123, 6590, 815, 543, 93, 82, - 1239, 48, 9, 932, 234, 67, 4378, 3847, 1237); - mockMvc.perform(get("/admin/report?token={token}&year={year}&month={month}", token.getValue(), year, month) + var now = LocalDateTime.now(); + mockMvc.perform(get("/admin/report?token={token}&year={year}&month={month}", token.getValue(), now.getYear(), now.getMonthValue()) .header(HttpHeaders.ACCEPT, "text/csv")) - .andExpect(status().isOk()) - .andExpect(content().string("year,month,extensions,downloads,downloads_total,publishers," + - "average_reviews_per_extension,namespace_owners,extensions_by_rating_1,extensions_by_rating_2," + - "extensions_by_rating_3,extensions_by_rating_4,extensions_by_rating_5," + - "publishers_published_extensions_1,publishers_published_extensions_3," + - "most_active_publishing_users_u_bar,most_active_publishing_users_u_foo,most_active_publishing_users_u_baz," + - "namespace_extensions_n_baz,namespace_extensions_n_bar,namespace_extensions_n_foo," + - "namespace_extension_versions_nv_baz,namespace_extension_versions_nv_foo,namespace_extension_versions_nv_bar," + - "most_downloaded_extensions_foo.baz,most_downloaded_extensions_foo.bar,most_downloaded_extensions_bar.foo\n" + - values.stream().map(Object::toString).collect(Collectors.joining(",")))); + .andExpect(status().isBadRequest()) + .andExpect(content().string("Combination of year and month lies in the future")); } @Test - public void testAdminOnTheFlyReportJson() throws Exception { + public void testCurrentMonthAdminReportJson() throws Exception { var token = mockAdminToken(); - var year = 2021; - var month = 7; - var extensions = 9123L; - var downloads = 2145L; - var downloadsTotal = 57199L; - var publishers = 846L; - var averageReviewsPerExtension = 8.75; - var namespaceOwners = 623L; - var extensionsByRating = Map.of(3, 8000, 5, 1123); - var publishersByExtensionsPublished = Map.of(1, 6590, 3, 815); - var topMostActivePublishingUsers = Map.of("u_foo", 93, "u_bar", 543, "u_baz", 82); - var topNamespaceExtensions = Map.of("n_foo", 9, "n_bar", 48, "n_baz", 1239); - var topNamespaceExtensionVersions = Map.of("nv_foo", 234, "nv_bar", 67, "nv_baz", 932); - var topMostDownloadedExtensions = Map.of("foo.bar", 3847L, "bar.foo", 1237L, "foo.baz", 4378L); - - var stats = new AdminStatistics(); - stats.setYear(year); - stats.setMonth(month); - stats.setExtensions(extensions); - stats.setDownloads(downloads); - stats.setDownloadsTotal(downloadsTotal); - stats.setPublishers(publishers); - stats.setAverageReviewsPerExtension(averageReviewsPerExtension); - stats.setNamespaceOwners(namespaceOwners); - stats.setExtensionsByRating(extensionsByRating); - stats.setPublishersByExtensionsPublished(publishersByExtensionsPublished); - stats.setTopMostActivePublishingUsers(topMostActivePublishingUsers); - stats.setTopNamespaceExtensions(topNamespaceExtensions); - stats.setTopNamespaceExtensionVersions(topNamespaceExtensionVersions); - stats.setTopMostDownloadedExtensions(topMostDownloadedExtensions); - - Mockito.when(repositories.findAdminStatisticsByYearAndMonth(year, month)).thenReturn(null); - - var startInclusive = LocalDateTime.of(year, month, 1, 0, 0); - var endExclusive = startInclusive.plusMonths(1); - Mockito.when(repositories.countActiveExtensions(endExclusive)).thenReturn(extensions); - Mockito.when(repositories.downloadsBetween(startInclusive, endExclusive)).thenReturn(downloads); - Mockito.when(repositories.downloadsUntil(endExclusive)).thenReturn(downloadsTotal); - Mockito.when(repositories.countActiveExtensionPublishers(endExclusive)).thenReturn(publishers); - Mockito.when(repositories.averageNumberOfActiveReviewsPerActiveExtension(endExclusive)).thenReturn(averageReviewsPerExtension); - Mockito.when(repositories.countPublishersThatClaimedNamespaceOwnership(endExclusive)).thenReturn(namespaceOwners); - Mockito.when(repositories.countActiveExtensionsGroupedByExtensionReviewRating(endExclusive)).thenReturn(extensionsByRating); - Mockito.when(repositories.countActiveExtensionPublishersGroupedByExtensionsPublished(endExclusive)).thenReturn(publishersByExtensionsPublished); - Mockito.when(repositories.topMostActivePublishingUsers(endExclusive, 10)).thenReturn(topMostActivePublishingUsers); - Mockito.when(repositories.topNamespaceExtensions(endExclusive, 10)).thenReturn(topNamespaceExtensions); - Mockito.when(repositories.topNamespaceExtensionVersions(endExclusive, 10)).thenReturn(topNamespaceExtensionVersions); - Mockito.when(repositories.topMostDownloadedExtensions(endExclusive, 10)).thenReturn(topMostDownloadedExtensions); - - mockMvc.perform(get("/admin/report?token={token}&year={year}&month={month}", token.getValue(), year, month) + var now = LocalDateTime.now(); + mockMvc.perform(get("/admin/report?token={token}&year={year}&month={month}", token.getValue(), now.getYear(), now.getMonthValue()) .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)) - .andExpect(status().isOk()) - .andExpect(content().json(adminStatisticsJson(s -> { - s.year = year; - s.month = month; - s.extensions = extensions; - s.downloads = downloads; - s.downloadsTotal = downloadsTotal; - s.publishers = publishers; - s.averageReviewsPerExtension = averageReviewsPerExtension; - s.namespaceOwners = namespaceOwners; - - var rating5 = new AdminStatisticsJson.ExtensionsByRating(); - rating5.rating = 5; - rating5.extensions = 1123; - var rating3 = new AdminStatisticsJson.ExtensionsByRating(); - rating3.rating = 3; - rating3.extensions = 8000; - s.extensionsByRating = List.of(rating5, rating3); - - var publishers3 = new AdminStatisticsJson.PublishersByExtensionsPublished(); - publishers3.extensionsPublished = 3; - publishers3.publishers = 815; - var publishers1 = new AdminStatisticsJson.PublishersByExtensionsPublished(); - publishers1.extensionsPublished = 1; - publishers1.publishers = 6590; - s.publishersByExtensionsPublished = List.of(publishers3, publishers1); - - var activePublisher1 = new AdminStatisticsJson.TopMostActivePublishingUsers(); - activePublisher1.userLoginName = "u_bar"; - activePublisher1.publishedExtensionVersions = 543; - var activePublisher2 = new AdminStatisticsJson.TopMostActivePublishingUsers(); - activePublisher2.userLoginName = "u_foo"; - activePublisher2.publishedExtensionVersions = 93; - var activePublisher3 = new AdminStatisticsJson.TopMostActivePublishingUsers(); - activePublisher3.userLoginName = "u_baz"; - activePublisher3.publishedExtensionVersions = 82; - s.topMostActivePublishingUsers = List.of(activePublisher1, activePublisher2, activePublisher3); - - var namespaceExtensions1 = new AdminStatisticsJson.TopNamespaceExtensions(); - namespaceExtensions1.namespace = "n_baz"; - namespaceExtensions1.extensions = 1239; - var namespaceExtensions2 = new AdminStatisticsJson.TopNamespaceExtensions(); - namespaceExtensions2.namespace = "n_bar"; - namespaceExtensions2.extensions = 48; - var namespaceExtensions3 = new AdminStatisticsJson.TopNamespaceExtensions(); - namespaceExtensions3.namespace = "n_foo"; - namespaceExtensions3.extensions = 9; - s.topNamespaceExtensions = List.of(namespaceExtensions1, namespaceExtensions2, namespaceExtensions3); - - var namespaceExtensionVersions1 = new AdminStatisticsJson.TopNamespaceExtensionVersions(); - namespaceExtensionVersions1.namespace = "nv_baz"; - namespaceExtensionVersions1.extensionVersions = 932; - var namespaceExtensionVersions2 = new AdminStatisticsJson.TopNamespaceExtensionVersions(); - namespaceExtensionVersions2.namespace = "nv_foo"; - namespaceExtensionVersions2.extensionVersions = 234; - var namespaceExtensionVersions3 = new AdminStatisticsJson.TopNamespaceExtensionVersions(); - namespaceExtensionVersions3.namespace = "nv_bar"; - namespaceExtensionVersions3.extensionVersions = 67; - s.topNamespaceExtensionVersions = List.of(namespaceExtensionVersions1, namespaceExtensionVersions2, namespaceExtensionVersions3); - - var mostDownloadedExtensions1 = new AdminStatisticsJson.TopMostDownloadedExtensions(); - mostDownloadedExtensions1.extensionIdentifier = "foo.baz"; - mostDownloadedExtensions1.downloads = 4378L; - var mostDownloadedExtensions2 = new AdminStatisticsJson.TopMostDownloadedExtensions(); - mostDownloadedExtensions2.extensionIdentifier = "foo.bar"; - mostDownloadedExtensions2.downloads = 3847L; - var mostDownloadedExtensions3 = new AdminStatisticsJson.TopMostDownloadedExtensions(); - mostDownloadedExtensions3.extensionIdentifier = "bar.foo"; - mostDownloadedExtensions3.downloads = 1237L; - s.topMostDownloadedExtensions = List.of(mostDownloadedExtensions1, mostDownloadedExtensions2, mostDownloadedExtensions3); - }))); + .andExpect(status().isBadRequest()) + .andExpect(content().json(errorJson("Combination of year and month lies in the future"))); } @Test diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index 0c1be35a5..b6a2dc040 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -182,7 +182,8 @@ void testExecuteQueries() { () -> repositories.findVersionStringsSorted(extension, "targetPlatform", true), () -> repositories.findActiveVersions(new QueryRequest()), () -> repositories.findActiveVersionStringsSorted(LONG_LIST,"targetPlatform"), - () -> repositories.findActiveVersionReferencesSorted(List.of(extension)) + () -> repositories.findActiveVersionReferencesSorted(List.of(extension)), + () -> repositories.findAllAdminStatistics() ); // check that we did not miss anything