diff --git a/pom.xml b/pom.xml index 5549eafe93..dceb186f37 100644 --- a/pom.xml +++ b/pom.xml @@ -127,6 +127,7 @@ 4.4.1 3.4.14 20200713.1 + 2.4.0 5.6.3 @@ -589,6 +590,13 @@ ${elasticsearch.version} + + + net.sf.supercsv + super-csv + ${super-csv.version} + + org.cache2k diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java index 6ccb6b1d79..6a14d68b83 100644 --- a/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java +++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java @@ -84,6 +84,8 @@ import java.net.URI; import java.util.UUID; +import org.apache.ibatis.type.LocalDateTimeTypeHandler; +import org.apache.ibatis.type.LocalDateTypeHandler; import org.mybatis.spring.boot.autoconfigure.ConfigurationCustomizer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -229,6 +231,7 @@ ConfigurationCustomizer mybatisConfigCustomizer() { configuration .getTypeAliasRegistry() .registerAlias("IDigBioCollectionDto", IDigBioCollectionDto.class); + }; } } diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.java index 669550be8a..5a0780e6ba 100644 --- a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.java +++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.java @@ -19,6 +19,7 @@ import org.gbif.api.model.common.paging.Pageable; import org.gbif.api.model.common.search.Facet; import org.gbif.api.model.occurrence.Download; +import org.gbif.api.model.occurrence.DownloadStatistics; import java.util.Date; import java.util.List; @@ -81,4 +82,19 @@ List getDownloadsByDataset( @Nullable @Param("publishingCountry") String publishingCountry, @Nullable @Param("datasetKey") UUID datasetKey, @Nullable @Param("publishingOrgKey") UUID publishingOrgKey); + + List getDownloadStatistics( + @Nullable @Param("fromDate") Date fromDate, + @Nullable @Param("toDate") Date toDate, + @Nullable @Param("publishingCountry") String publishingCountry, + @Nullable @Param("datasetKey") UUID datasetKey, + @Nullable @Param("publishingOrgKey") UUID publishingOrgKey, + @Nullable @Param("page") Pageable page); + + long countDownloadStatistics( + @Nullable @Param("fromDate") Date fromDate, + @Nullable @Param("toDate") Date toDate, + @Nullable @Param("publishingCountry") String publishingCountry, + @Nullable @Param("datasetKey") UUID datasetKey, + @Nullable @Param("publishingOrgKey") UUID publishingOrgKey); } diff --git a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.xml b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.xml index c64793b2cc..8fc1517485 100644 --- a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.xml +++ b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.xml @@ -64,6 +64,8 @@ + + key,doi,license,filter,status,download_link,size,total_records,notification_addresses,created_by,send_notification,format,created,modified,erase_after @@ -259,4 +261,37 @@ GROUP BY year_month ORDER BY year_month DESC; + + + + diff --git a/registry-ws/pom.xml b/registry-ws/pom.xml index 16d52962bc..dc777d65d1 100644 --- a/registry-ws/pom.xml +++ b/registry-ws/pom.xml @@ -15,6 +15,12 @@ + + + org.projectlombok + lombok + + org.springframework.boot @@ -76,6 +82,12 @@ springdoc-openapi-ui + + + net.sf.supercsv + super-csv + + org.gbif.registry diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java b/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java new file mode 100644 index 0000000000..47ac5e5284 --- /dev/null +++ b/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java @@ -0,0 +1,89 @@ +package org.gbif.registry.ws.export; + +import org.gbif.api.model.common.export.ExportFormat; +import org.gbif.api.model.common.paging.Pageable; +import org.gbif.api.model.occurrence.DownloadStatistics; +import org.gbif.api.service.registry.OccurrenceDownloadService; +import org.gbif.api.vocabulary.Country; + +import java.io.Writer; +import java.util.Date; +import java.util.UUID; + + +import lombok.Builder; +import lombok.Data; +import lombok.SneakyThrows; +import org.supercsv.cellprocessor.Optional; +import org.supercsv.cellprocessor.ParseInt; +import org.supercsv.cellprocessor.ift.CellProcessor; +import org.supercsv.io.CsvBeanWriter; +import org.supercsv.io.ICsvBeanWriter; +import org.supercsv.prefs.CsvPreference; +import org.supercsv.util.CsvContext; + +@Data +@Builder +public class CsvWriter { + + private final String[] header; + + private final String[] fields; + + private final CellProcessor[] processors; + + private final Iterable pager; + + private final ExportFormat preference; + + + private CsvPreference csvPreference() { + if (ExportFormat.CSV == preference) { + return CsvPreference.STANDARD_PREFERENCE; + } else if (ExportFormat.TSV == preference) { + return CsvPreference.TAB_PREFERENCE; + } + throw new IllegalArgumentException("Export format not supported " + preference); + } + + @SneakyThrows + public void export(Writer writer) { + try (ICsvBeanWriter beanWriter = new CsvBeanWriter(writer, csvPreference())) { + beanWriter.writeHeader(header); + for (T o : pager) { + beanWriter.write(o, fields, processors); + } + writer.flush(); + } + } + + /** + * Creates and CsvWriter/exporter DownloadStatistics. + */ + public static CsvWriter downloadStatisticsTsvWriter(Iterable pager, + ExportFormat preference) { + + return CsvWriter.builder() + .fields(new String[]{"datasetKey", "totalRecords", "numberDownloads", "year", "month"}) + .header(new String[]{"dataset_key", "total_records", "number_downloads", "year", "month"}) + .processors(new CellProcessor[]{new UUIDProcessor(), + new Optional(new ParseInt()), + new Optional(new ParseInt()), + new Optional(new ParseInt()), + new Optional(new ParseInt())}) + .preference(preference) + .pager(pager) + .build(); + } + + /** + * Null aware UUID processor. + */ + private static class UUIDProcessor implements CellProcessor { + @Override + public String execute(Object value, CsvContext csvContext) { + return value != null ? ((UUID) value).toString() : ""; + } + } + +} diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java index 222fb3e446..0484f22b3b 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java @@ -19,24 +19,29 @@ import org.gbif.api.annotation.Trim; import org.gbif.api.model.common.DOI; import org.gbif.api.model.common.GbifUser; +import org.gbif.api.model.common.export.ExportFormat; import org.gbif.api.model.common.paging.Pageable; import org.gbif.api.model.common.paging.PagingResponse; import org.gbif.api.model.common.search.Facet; import org.gbif.api.model.occurrence.Download; +import org.gbif.api.model.occurrence.DownloadStatistics; import org.gbif.api.model.registry.DatasetOccurrenceDownloadUsage; import org.gbif.api.model.registry.PostPersist; import org.gbif.api.model.registry.PrePersist; import org.gbif.api.service.common.IdentityAccessService; import org.gbif.api.service.registry.OccurrenceDownloadService; +import org.gbif.api.util.iterables.Iterables; import org.gbif.api.vocabulary.Country; import org.gbif.api.vocabulary.License; import org.gbif.registry.doi.DoiIssuingService; import org.gbif.registry.doi.DownloadDoiDataCiteHandlingService; import org.gbif.registry.persistence.mapper.DatasetOccurrenceDownloadMapper; import org.gbif.registry.persistence.mapper.OccurrenceDownloadMapper; +import org.gbif.registry.ws.export.CsvWriter; import org.gbif.registry.ws.provider.PartialDate; import org.gbif.ws.WebApplicationException; +import java.io.IOException; import java.util.Date; import java.util.List; import java.util.Map; @@ -47,6 +52,7 @@ import java.util.UUID; import java.util.stream.Collectors; +import javax.servlet.http.HttpServletResponse; import javax.validation.constraints.NotNull; import javax.validation.groups.Default; @@ -56,8 +62,13 @@ import org.slf4j.MarkerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Lazy; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.annotation.Secured; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; @@ -343,6 +354,57 @@ public Map> getDownloadsByDataset( publishingOrgKey)); } + @GetMapping("statistics") + @Override + public PagingResponse getDownloadStatistics( + @PartialDate Date fromDate, + @PartialDate Date toDate, + Country publishingCountry, + @RequestParam(value = "datasetKey", required = false) UUID datasetKey, + @RequestParam(value = "publishingOrgKey", required = false) UUID publishingOrgKey, + Pageable page + ) { + String country = Optional.ofNullable(publishingCountry).map(Country::getIso2LetterCode).orElse(null); + return new PagingResponse<>(page, + occurrenceDownloadMapper.countDownloadStatistics(fromDate, + toDate, + country, + datasetKey, + publishingOrgKey), + occurrenceDownloadMapper.getDownloadStatistics(fromDate, + toDate, + country, + datasetKey, + publishingOrgKey, + page)); + } + + @GetMapping("statistics/export") + public void getDownloadStatistics( + HttpServletResponse response, + @RequestParam(value = "format", defaultValue = "TSV") ExportFormat format, + @PartialDate Date fromDate, + @PartialDate Date toDate, + Country publishingCountry, + @RequestParam(value = "datasetKey", required = false) UUID datasetKey, + @RequestParam(value = "publishingOrgKey", required = false) UUID publishingOrgKey) throws + IOException { + + String headerValue = String.format("attachment; filename=\"download_statistics.%s\"", + format.name().toLowerCase()); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, headerValue); + + + CsvWriter.downloadStatisticsTsvWriter(Iterables.downloadStatistics(this, + fromDate, + toDate, + publishingCountry, + datasetKey, + publishingOrgKey), + format) + .export(response.getWriter()); + } + /** Aggregates the download statistics in tree structure of month grouped by year. */ private Map> groupByYear(List counts) { Map> yearsGrouping = new TreeMap<>(); diff --git a/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java b/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java new file mode 100644 index 0000000000..509d47559a --- /dev/null +++ b/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java @@ -0,0 +1,47 @@ +package org.gbif.registry.ws.export; + +import org.gbif.api.model.common.export.ExportFormat; +import org.gbif.api.model.occurrence.DownloadStatistics; + +import java.io.StringWriter; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CsvWriterTest { + + @Test + public void downloadStatisticsTest() { + + List stats = Arrays.asList( + new DownloadStatistics(UUID.randomUUID(), 10, 10, LocalDate.of(2020,1,1)), + new DownloadStatistics(UUID.randomUUID(), 10, 10, LocalDate.of(2021, 2,1))); + + StringWriter writer = new StringWriter(); + + CsvWriter.downloadStatisticsTsvWriter(stats, ExportFormat.TSV) + .export(writer); + + String export = writer.toString(); + String[] lines = export.split("\\n"); + + //Number of lines is header + list.size + assertEquals(stats.size() + 1, lines.length); + + //Each line has 4 tabs + assertEquals((stats.size() + 1) * 4, export.chars().filter(ch -> ch == '\t').count()); + + //Year test + assertEquals("2020", lines[1].split("\\t")[3]); + assertEquals("2021", lines[2].split("\\t")[3]); + + //Month test + assertEquals("1", lines[1].split("\\t")[4]); + assertEquals("2", lines[2].split("\\t")[4]); + } +}