From 34521ccf60168db57cbab8e029b95dd17f2636ef Mon Sep 17 00:00:00 2001 From: fmendezh Date: Thu, 27 May 2021 14:57:42 +0200 Subject: [PATCH] reducing the dataset usages export to only the dataset fields and using the download key as part of the file name --- .../gbif/registry/ws/export/CsvWriter.java | 74 +++++++++---------- .../resources/OccurrenceDownloadResource.java | 4 +- .../registry/ws/export/CsvWriterTest.java | 25 ++----- 3 files changed, 46 insertions(+), 57 deletions(-) diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java b/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java index 9f8ab595ae..b200ff1ef7 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java @@ -90,17 +90,17 @@ public static CsvWriter downloadStatisticsCsvWriter(Iterable ExportFormat preference) { return CsvWriter.builder() - .fields(new String[]{"datasetKey", "totalRecords", "numberDownloads", "year", "month"}) - .header(new String[]{"dataset_key", "total_records", "number_downloads", "year", "month"}) - .processors(new CellProcessor[]{new UUIDProcessor(), //datasetKey - new Optional(new ParseLong()), //totalRecords - new Optional(new ParseLong()), //numberDowloads - new Optional(new ParseInt()), //year - new Optional(new ParseInt()) //month - }) - .preference(preference) - .pager(pager) - .build(); + .fields(new String[]{"datasetKey", "totalRecords", "numberDownloads", "year", "month"}) + .header(new String[]{"dataset_key", "total_records", "number_downloads", "year", "month"}) + .processors(new CellProcessor[]{new UUIDProcessor(), //datasetKey + new Optional(new ParseLong()), //totalRecords + new Optional(new ParseLong()), //numberDowloads + new Optional(new ParseInt()), //year + new Optional(new ParseInt()) //month + }) + .preference(preference) + .pager(pager) + .build(); } /** @@ -113,23 +113,23 @@ public static CsvWriter datasetSearchResultCsvWriter(Iterab .header(new String[]{"dataset_key", "title", "doi", "license", "type", "sub_type", "hosting_organization_Key", "hosting_organization_title", "hosting_country","publishing_organization_key", "publishing_organization_title", "publishing_country", "endorsing_node_key", "network_keys", "project_identifier", "occurrence_records_count", "name_usages_count"}) // "recordCount", "nameUsagesCount" .processors(new CellProcessor[]{new UUIDProcessor(), //key - new CleanStringProcessor(), //title - new DOIProcessor(), //doi - new Optional(new ParseEnum(License.class)), //license - new Optional(new ParseEnum(DatasetType.class)), //type - new Optional(new ParseEnum(DatasetSubtype.class)),//subType - new UUIDProcessor(), //hostingOrganizationKey - new CleanStringProcessor(), //hostingOrganizationTitle - new CountryProcessor(), //hostingCountry - new UUIDProcessor(), //publishingOrganizationKey - new CleanStringProcessor(), //publishingOrganizationTitle - new CountryProcessor(), //publishingCountry - new UUIDProcessor(), //endorsingNodeKey - new ListUUIDProcessor(), //networkKeys - new CleanStringProcessor(), //projectIdentifier - new Optional(new ParseInt()), //recordCount - new Optional(new ParseInt()) //nameUsagesCount - }) + new CleanStringProcessor(), //title + new DOIProcessor(), //doi + new Optional(new ParseEnum(License.class)), //license + new Optional(new ParseEnum(DatasetType.class)), //type + new Optional(new ParseEnum(DatasetSubtype.class)),//subType + new UUIDProcessor(), //hostingOrganizationKey + new CleanStringProcessor(), //hostingOrganizationTitle + new CountryProcessor(), //hostingCountry + new UUIDProcessor(), //publishingOrganizationKey + new CleanStringProcessor(), //publishingOrganizationTitle + new CountryProcessor(), //publishingCountry + new UUIDProcessor(), //endorsingNodeKey + new ListUUIDProcessor(), //networkKeys + new CleanStringProcessor(), //projectIdentifier + new Optional(new ParseInt()), //recordCount + new Optional(new ParseInt()) //nameUsagesCount + }) .preference(preference) .pager(pager) .build(); @@ -141,16 +141,14 @@ public static CsvWriter datasetSearchResultCsvWriter(Iterab public static CsvWriter datasetOccurrenceDownloadUsageCsvWriter(Iterable pager, ExportFormat preference) { return CsvWriter.builder() - .fields(new String[]{"download", "downloadKey", "datasetDOI", "datasetKey", "datasetTitle", "datasetCitation", "numberRecords"}) - .header(new String[]{"download_doi", "download_key", "dataset_doi", "dataset_key", "dataset_title", "dataset_citation", "number_records"}) - .processors(new CellProcessor[]{new DownloadDOIProcessor(), //download.doi - null, //downloadKey - new DOIProcessor(), //datasetDOI - new UUIDProcessor(), //datasetKey - new CleanStringProcessor(), //datasetTitle - new CleanStringProcessor(), //datasetCitation - new Optional(new ParseLong()) //numberRecords - }) + .fields(new String[]{"datasetDOI", "datasetKey", "datasetTitle", "datasetCitation", "numberRecords"}) + .header(new String[]{"dataset_doi", "dataset_key", "dataset_title", "dataset_citation", "number_records"}) + .processors(new CellProcessor[]{new DOIProcessor(), //datasetDOI + new UUIDProcessor(), //datasetKey + new CleanStringProcessor(), //datasetTitle + new CleanStringProcessor(), //datasetCitation + new Optional(new ParseLong()) //numberRecords + }) .preference(preference) .pager(pager) .build(); diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java index f7695af3e2..b9c56e6e71 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java @@ -111,7 +111,7 @@ public class OccurrenceDownloadResource implements OccurrenceDownloadService { private static final int EXPORT_LIMIT = 5_000; //Export header prefix - private static final String FILE_HEADER_PRE = "attachment; filename=datasets_download_usage."; + private static final String FILE_HEADER_PRE = "attachment; filename=datasets_download_usage_"; //Download stats file header private static final String EXPORT_FILE_HEADER_PRE = "attachment; filename=download_statistics."; @@ -257,7 +257,7 @@ public void exportListDatasetUsagesByKey( @PathVariable("key") String key, @RequestParam(value = "format", defaultValue = "TSV") ExportFormat format) throws IOException { - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, FILE_HEADER_PRE + format.name().toLowerCase()); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, FILE_HEADER_PRE + key + '.' + format.name().toLowerCase()); try (Writer writer = new BufferedWriter(new OutputStreamWriter(response.getOutputStream()))) { CsvWriter.datasetOccurrenceDownloadUsageCsvWriter(Iterables.datasetOccurrenceDownloadUsages(this, diff --git a/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java b/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java index 17444bd090..7d3c1e9f29 100644 --- a/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java +++ b/registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java @@ -172,20 +172,17 @@ public void datasetSearchTest() { * Test one DatasetOccurrenceDownloadUsage against its expected exported data. */ private void assertDatasetOccurrenceDownloadUsage(DatasetOccurrenceDownloadUsage downloadUsage, String[] line) { - //"download", "downloadKey", "datasetDOI", "datasetKey", "datasetTitle", "datasetCitation", "numberRecords" - assertEquals(downloadUsage.getDownload().getDoi().toString(), line[0]); - assertEquals(downloadUsage.getDownloadKey(), line[1]); - assertEquals(downloadUsage.getDatasetDOI().toString(), line[2]); - assertEquals(downloadUsage.getDatasetKey().toString(), line[3]); - assertEquals(downloadUsage.getDatasetTitle(), line[4]); - assertEquals(downloadUsage.getDatasetCitation(), line[5]); - assertEquals(downloadUsage.getNumberRecords(), Long.parseLong(line[6].replace("\r",""))); + assertEquals(downloadUsage.getDatasetDOI().toString(), line[0]); + assertEquals(downloadUsage.getDatasetKey().toString(), line[1]); + assertEquals(downloadUsage.getDatasetTitle(), line[2]); + assertEquals(downloadUsage.getDatasetCitation(), line[3]); + assertEquals(downloadUsage.getNumberRecords(), Long.parseLong(line[4].replace("\r",""))); } /** * Generates test instances of DatasetOccurrenceDownloadUsage. */ - private static DatasetOccurrenceDownloadUsage newDatasetOccurrenceDownloadUsageTest(int consecutive, Download download) { + private static DatasetOccurrenceDownloadUsage newDatasetOccurrenceDownloadUsageTest(int consecutive) { DatasetOccurrenceDownloadUsage downloadUsage = new DatasetOccurrenceDownloadUsage(); downloadUsage.setDatasetKey(UUID.randomUUID()); @@ -193,8 +190,6 @@ private static DatasetOccurrenceDownloadUsage newDatasetOccurrenceDownloadUsageT downloadUsage.setDatasetTitle("UsageTitle" + consecutive); downloadUsage.setDatasetCitation("Citation" + consecutive); downloadUsage.setNumberRecords(consecutive); - downloadUsage.setDownloadKey(download.getKey()); - downloadUsage.setDownload(download); return downloadUsage; } @@ -203,12 +198,8 @@ private static DatasetOccurrenceDownloadUsage newDatasetOccurrenceDownloadUsageT public void datasetOccurrenceDownloadUsageTest() { //Test data - Download download = new Download(); - download.setKey("0220580-200613084148143"); - download.setDoi(new DOI("10.21373/6m9yw0")); - - List downloadUsages = Arrays.asList(newDatasetOccurrenceDownloadUsageTest(1, download), - newDatasetOccurrenceDownloadUsageTest(2, download)); + List downloadUsages = Arrays.asList(newDatasetOccurrenceDownloadUsageTest(1), + newDatasetOccurrenceDownloadUsageTest(2)); StringWriter writer = new StringWriter();