diff --git a/pom.xml b/pom.xml
index 5549eafe93..dceb186f37 100644
--- a/pom.xml
+++ b/pom.xml
@@ -127,6 +127,7 @@
4.4.1
3.4.14
20200713.1
+ 2.4.0
5.6.3
@@ -589,6 +590,13 @@
${elasticsearch.version}
+
+
+ net.sf.supercsv
+ super-csv
+ ${super-csv.version}
+
+
org.cache2k
diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java
index 6ccb6b1d79..6a14d68b83 100644
--- a/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java
+++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java
@@ -84,6 +84,8 @@
import java.net.URI;
import java.util.UUID;
+import org.apache.ibatis.type.LocalDateTimeTypeHandler;
+import org.apache.ibatis.type.LocalDateTypeHandler;
import org.mybatis.spring.boot.autoconfigure.ConfigurationCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -229,6 +231,7 @@ ConfigurationCustomizer mybatisConfigCustomizer() {
configuration
.getTypeAliasRegistry()
.registerAlias("IDigBioCollectionDto", IDigBioCollectionDto.class);
+
};
}
}
diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.java
index 669550be8a..5a0780e6ba 100644
--- a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.java
+++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.java
@@ -19,6 +19,7 @@
import org.gbif.api.model.common.paging.Pageable;
import org.gbif.api.model.common.search.Facet;
import org.gbif.api.model.occurrence.Download;
+import org.gbif.api.model.occurrence.DownloadStatistics;
import java.util.Date;
import java.util.List;
@@ -81,4 +82,19 @@ List getDownloadsByDataset(
@Nullable @Param("publishingCountry") String publishingCountry,
@Nullable @Param("datasetKey") UUID datasetKey,
@Nullable @Param("publishingOrgKey") UUID publishingOrgKey);
+
+ List getDownloadStatistics(
+ @Nullable @Param("fromDate") Date fromDate,
+ @Nullable @Param("toDate") Date toDate,
+ @Nullable @Param("publishingCountry") String publishingCountry,
+ @Nullable @Param("datasetKey") UUID datasetKey,
+ @Nullable @Param("publishingOrgKey") UUID publishingOrgKey,
+ @Nullable @Param("page") Pageable page);
+
+ long countDownloadStatistics(
+ @Nullable @Param("fromDate") Date fromDate,
+ @Nullable @Param("toDate") Date toDate,
+ @Nullable @Param("publishingCountry") String publishingCountry,
+ @Nullable @Param("datasetKey") UUID datasetKey,
+ @Nullable @Param("publishingOrgKey") UUID publishingOrgKey);
}
diff --git a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.xml b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.xml
index c64793b2cc..8fc1517485 100644
--- a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.xml
+++ b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/OccurrenceDownloadMapper.xml
@@ -64,6 +64,8 @@
+
+
key,doi,license,filter,status,download_link,size,total_records,notification_addresses,created_by,send_notification,format,created,modified,erase_after
@@ -259,4 +261,37 @@
GROUP BY year_month
ORDER BY year_month DESC;
+
+
+
+
diff --git a/registry-ws/pom.xml b/registry-ws/pom.xml
index 16d52962bc..dc777d65d1 100644
--- a/registry-ws/pom.xml
+++ b/registry-ws/pom.xml
@@ -15,6 +15,12 @@
+
+
+ org.projectlombok
+ lombok
+
+
org.springframework.boot
@@ -76,6 +82,12 @@
springdoc-openapi-ui
+
+
+ net.sf.supercsv
+ super-csv
+
+
org.gbif.registry
diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java b/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java
new file mode 100644
index 0000000000..47ac5e5284
--- /dev/null
+++ b/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java
@@ -0,0 +1,89 @@
+package org.gbif.registry.ws.export;
+
+import org.gbif.api.model.common.export.ExportFormat;
+import org.gbif.api.model.common.paging.Pageable;
+import org.gbif.api.model.occurrence.DownloadStatistics;
+import org.gbif.api.service.registry.OccurrenceDownloadService;
+import org.gbif.api.vocabulary.Country;
+
+import java.io.Writer;
+import java.util.Date;
+import java.util.UUID;
+
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.SneakyThrows;
+import org.supercsv.cellprocessor.Optional;
+import org.supercsv.cellprocessor.ParseInt;
+import org.supercsv.cellprocessor.ift.CellProcessor;
+import org.supercsv.io.CsvBeanWriter;
+import org.supercsv.io.ICsvBeanWriter;
+import org.supercsv.prefs.CsvPreference;
+import org.supercsv.util.CsvContext;
+
+@Data
+@Builder
+public class CsvWriter {
+
+ private final String[] header;
+
+ private final String[] fields;
+
+ private final CellProcessor[] processors;
+
+ private final Iterable pager;
+
+ private final ExportFormat preference;
+
+
+ private CsvPreference csvPreference() {
+ if (ExportFormat.CSV == preference) {
+ return CsvPreference.STANDARD_PREFERENCE;
+ } else if (ExportFormat.TSV == preference) {
+ return CsvPreference.TAB_PREFERENCE;
+ }
+ throw new IllegalArgumentException("Export format not supported " + preference);
+ }
+
+ @SneakyThrows
+ public void export(Writer writer) {
+ try (ICsvBeanWriter beanWriter = new CsvBeanWriter(writer, csvPreference())) {
+ beanWriter.writeHeader(header);
+ for (T o : pager) {
+ beanWriter.write(o, fields, processors);
+ }
+ writer.flush();
+ }
+ }
+
+ /**
+ * Creates and CsvWriter/exporter DownloadStatistics.
+ */
+ public static CsvWriter downloadStatisticsTsvWriter(Iterable pager,
+ ExportFormat preference) {
+
+ return CsvWriter.builder()
+ .fields(new String[]{"datasetKey", "totalRecords", "numberDownloads", "year", "month"})
+ .header(new String[]{"dataset_key", "total_records", "number_downloads", "year", "month"})
+ .processors(new CellProcessor[]{new UUIDProcessor(),
+ new Optional(new ParseInt()),
+ new Optional(new ParseInt()),
+ new Optional(new ParseInt()),
+ new Optional(new ParseInt())})
+ .preference(preference)
+ .pager(pager)
+ .build();
+ }
+
+ /**
+ * Null aware UUID processor.
+ */
+ private static class UUIDProcessor implements CellProcessor {
+ @Override
+ public String execute(Object value, CsvContext csvContext) {
+ return value != null ? ((UUID) value).toString() : "";
+ }
+ }
+
+}
diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java
index 222fb3e446..0484f22b3b 100644
--- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java
+++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java
@@ -19,24 +19,29 @@
import org.gbif.api.annotation.Trim;
import org.gbif.api.model.common.DOI;
import org.gbif.api.model.common.GbifUser;
+import org.gbif.api.model.common.export.ExportFormat;
import org.gbif.api.model.common.paging.Pageable;
import org.gbif.api.model.common.paging.PagingResponse;
import org.gbif.api.model.common.search.Facet;
import org.gbif.api.model.occurrence.Download;
+import org.gbif.api.model.occurrence.DownloadStatistics;
import org.gbif.api.model.registry.DatasetOccurrenceDownloadUsage;
import org.gbif.api.model.registry.PostPersist;
import org.gbif.api.model.registry.PrePersist;
import org.gbif.api.service.common.IdentityAccessService;
import org.gbif.api.service.registry.OccurrenceDownloadService;
+import org.gbif.api.util.iterables.Iterables;
import org.gbif.api.vocabulary.Country;
import org.gbif.api.vocabulary.License;
import org.gbif.registry.doi.DoiIssuingService;
import org.gbif.registry.doi.DownloadDoiDataCiteHandlingService;
import org.gbif.registry.persistence.mapper.DatasetOccurrenceDownloadMapper;
import org.gbif.registry.persistence.mapper.OccurrenceDownloadMapper;
+import org.gbif.registry.ws.export.CsvWriter;
import org.gbif.registry.ws.provider.PartialDate;
import org.gbif.ws.WebApplicationException;
+import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -47,6 +52,7 @@
import java.util.UUID;
import java.util.stream.Collectors;
+import javax.servlet.http.HttpServletResponse;
import javax.validation.constraints.NotNull;
import javax.validation.groups.Default;
@@ -56,8 +62,13 @@
import org.slf4j.MarkerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Lazy;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.core.io.Resource;
+import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
import org.springframework.security.access.annotation.Secured;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
@@ -343,6 +354,57 @@ public Map> getDownloadsByDataset(
publishingOrgKey));
}
+ @GetMapping("statistics")
+ @Override
+ public PagingResponse getDownloadStatistics(
+ @PartialDate Date fromDate,
+ @PartialDate Date toDate,
+ Country publishingCountry,
+ @RequestParam(value = "datasetKey", required = false) UUID datasetKey,
+ @RequestParam(value = "publishingOrgKey", required = false) UUID publishingOrgKey,
+ Pageable page
+ ) {
+ String country = Optional.ofNullable(publishingCountry).map(Country::getIso2LetterCode).orElse(null);
+ return new PagingResponse<>(page,
+ occurrenceDownloadMapper.countDownloadStatistics(fromDate,
+ toDate,
+ country,
+ datasetKey,
+ publishingOrgKey),
+ occurrenceDownloadMapper.getDownloadStatistics(fromDate,
+ toDate,
+ country,
+ datasetKey,
+ publishingOrgKey,
+ page));
+ }
+
+ @GetMapping("statistics/export")
+ public void getDownloadStatistics(
+ HttpServletResponse response,
+ @RequestParam(value = "format", defaultValue = "TSV") ExportFormat format,
+ @PartialDate Date fromDate,
+ @PartialDate Date toDate,
+ Country publishingCountry,
+ @RequestParam(value = "datasetKey", required = false) UUID datasetKey,
+ @RequestParam(value = "publishingOrgKey", required = false) UUID publishingOrgKey) throws
+ IOException {
+
+ String headerValue = String.format("attachment; filename=\"download_statistics.%s\"",
+ format.name().toLowerCase());
+ response.setHeader(HttpHeaders.CONTENT_DISPOSITION, headerValue);
+
+
+ CsvWriter.downloadStatisticsTsvWriter(Iterables.downloadStatistics(this,
+ fromDate,
+ toDate,
+ publishingCountry,
+ datasetKey,
+ publishingOrgKey),
+ format)
+ .export(response.getWriter());
+ }
+
/** Aggregates the download statistics in tree structure of month grouped by year. */
private Map> groupByYear(List counts) {
Map> yearsGrouping = new TreeMap<>();
diff --git a/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java b/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java
new file mode 100644
index 0000000000..509d47559a
--- /dev/null
+++ b/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java
@@ -0,0 +1,47 @@
+package org.gbif.registry.ws.export;
+
+import org.gbif.api.model.common.export.ExportFormat;
+import org.gbif.api.model.occurrence.DownloadStatistics;
+
+import java.io.StringWriter;
+import java.time.LocalDate;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class CsvWriterTest {
+
+ @Test
+ public void downloadStatisticsTest() {
+
+ List stats = Arrays.asList(
+ new DownloadStatistics(UUID.randomUUID(), 10, 10, LocalDate.of(2020,1,1)),
+ new DownloadStatistics(UUID.randomUUID(), 10, 10, LocalDate.of(2021, 2,1)));
+
+ StringWriter writer = new StringWriter();
+
+ CsvWriter.downloadStatisticsTsvWriter(stats, ExportFormat.TSV)
+ .export(writer);
+
+ String export = writer.toString();
+ String[] lines = export.split("\\n");
+
+ //Number of lines is header + list.size
+ assertEquals(stats.size() + 1, lines.length);
+
+ //Each line has 4 tabs
+ assertEquals((stats.size() + 1) * 4, export.chars().filter(ch -> ch == '\t').count());
+
+ //Year test
+ assertEquals("2020", lines[1].split("\\t")[3]);
+ assertEquals("2021", lines[2].split("\\t")[3]);
+
+ //Month test
+ assertEquals("1", lines[1].split("\\t")[4]);
+ assertEquals("2", lines[2].split("\\t")[4]);
+ }
+}