Skip to content

Commit

Permalink
reducing the dataset usages export to only the dataset fields and usi…
Browse files Browse the repository at this point in the history 
…ng the download key as part of the file name
  • Loading branch information
@fmendezh
fmendezh committed May 27, 2021
1 parent 00df497 commit 34521cc
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 57 deletions.
74 changes: 36 additions & 38 deletions registry-ws/src/main/java/org/gbif/registry/ws/export/CsvWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,17 +90,17 @@ public static CsvWriter<DownloadStatistics> downloadStatisticsCsvWriter(Iterable
ExportFormat preference) {

return CsvWriter.<DownloadStatistics>builder()
.fields(new String[]{"datasetKey", "totalRecords", "numberDownloads", "year", "month"})
.header(new String[]{"dataset_key", "total_records", "number_downloads", "year", "month"})
.processors(new CellProcessor[]{new UUIDProcessor(), //datasetKey
new Optional(new ParseLong()), //totalRecords
new Optional(new ParseLong()), //numberDowloads
new Optional(new ParseInt()), //year
new Optional(new ParseInt()) //month
})
.preference(preference)
.pager(pager)
.build();
.fields(new String[]{"datasetKey", "totalRecords", "numberDownloads", "year", "month"})
.header(new String[]{"dataset_key", "total_records", "number_downloads", "year", "month"})
.processors(new CellProcessor[]{new UUIDProcessor(), //datasetKey
new Optional(new ParseLong()), //totalRecords
new Optional(new ParseLong()), //numberDowloads
new Optional(new ParseInt()), //year
new Optional(new ParseInt()) //month
})
.preference(preference)
.pager(pager)
.build();
}

/**
Expand All @@ -113,23 +113,23 @@ public static CsvWriter<DatasetSearchResult> datasetSearchResultCsvWriter(Iterab
.header(new String[]{"dataset_key", "title", "doi", "license", "type", "sub_type", "hosting_organization_Key", "hosting_organization_title", "hosting_country","publishing_organization_key", "publishing_organization_title", "publishing_country", "endorsing_node_key", "network_keys", "project_identifier", "occurrence_records_count", "name_usages_count"})
// "recordCount", "nameUsagesCount"
.processors(new CellProcessor[]{new UUIDProcessor(), //key
new CleanStringProcessor(), //title
new DOIProcessor(), //doi
new Optional(new ParseEnum(License.class)), //license
new Optional(new ParseEnum(DatasetType.class)), //type
new Optional(new ParseEnum(DatasetSubtype.class)),//subType
new UUIDProcessor(), //hostingOrganizationKey
new CleanStringProcessor(), //hostingOrganizationTitle
new CountryProcessor(), //hostingCountry
new UUIDProcessor(), //publishingOrganizationKey
new CleanStringProcessor(), //publishingOrganizationTitle
new CountryProcessor(), //publishingCountry
new UUIDProcessor(), //endorsingNodeKey
new ListUUIDProcessor(), //networkKeys
new CleanStringProcessor(), //projectIdentifier
new Optional(new ParseInt()), //recordCount
new Optional(new ParseInt()) //nameUsagesCount
})
new CleanStringProcessor(), //title
new DOIProcessor(), //doi
new Optional(new ParseEnum(License.class)), //license
new Optional(new ParseEnum(DatasetType.class)), //type
new Optional(new ParseEnum(DatasetSubtype.class)),//subType
new UUIDProcessor(), //hostingOrganizationKey
new CleanStringProcessor(), //hostingOrganizationTitle
new CountryProcessor(), //hostingCountry
new UUIDProcessor(), //publishingOrganizationKey
new CleanStringProcessor(), //publishingOrganizationTitle
new CountryProcessor(), //publishingCountry
new UUIDProcessor(), //endorsingNodeKey
new ListUUIDProcessor(), //networkKeys
new CleanStringProcessor(), //projectIdentifier
new Optional(new ParseInt()), //recordCount
new Optional(new ParseInt()) //nameUsagesCount
})
.preference(preference)
.pager(pager)
.build();
Expand All @@ -141,16 +141,14 @@ public static CsvWriter<DatasetSearchResult> datasetSearchResultCsvWriter(Iterab
public static CsvWriter<DatasetOccurrenceDownloadUsage> datasetOccurrenceDownloadUsageCsvWriter(Iterable<DatasetOccurrenceDownloadUsage> pager,
ExportFormat preference) {
return CsvWriter.<DatasetOccurrenceDownloadUsage>builder()
.fields(new String[]{"download", "downloadKey", "datasetDOI", "datasetKey", "datasetTitle", "datasetCitation", "numberRecords"})
.header(new String[]{"download_doi", "download_key", "dataset_doi", "dataset_key", "dataset_title", "dataset_citation", "number_records"})
.processors(new CellProcessor[]{new DownloadDOIProcessor(), //download.doi
null, //downloadKey
new DOIProcessor(), //datasetDOI
new UUIDProcessor(), //datasetKey
new CleanStringProcessor(), //datasetTitle
new CleanStringProcessor(), //datasetCitation
new Optional(new ParseLong()) //numberRecords
})
.fields(new String[]{"datasetDOI", "datasetKey", "datasetTitle", "datasetCitation", "numberRecords"})
.header(new String[]{"dataset_doi", "dataset_key", "dataset_title", "dataset_citation", "number_records"})
.processors(new CellProcessor[]{new DOIProcessor(), //datasetDOI
new UUIDProcessor(), //datasetKey
new CleanStringProcessor(), //datasetTitle
new CleanStringProcessor(), //datasetCitation
new Optional(new ParseLong()) //numberRecords
})
.preference(preference)
.pager(pager)
.build();
Expand Down
4 changes: 2 additions & 2 deletions registry-ws/src/main/java/org/gbif/registry/ws/resources/OccurrenceDownloadResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public class OccurrenceDownloadResource implements OccurrenceDownloadService {
private static final int EXPORT_LIMIT = 5_000;

//Export header prefix
private static final String FILE_HEADER_PRE = "attachment; filename=datasets_download_usage.";
private static final String FILE_HEADER_PRE = "attachment; filename=datasets_download_usage_";

//Download stats file header
private static final String EXPORT_FILE_HEADER_PRE = "attachment; filename=download_statistics.";
Expand Down Expand Up @@ -257,7 +257,7 @@ public void exportListDatasetUsagesByKey(
@PathVariable("key") String key,
@RequestParam(value = "format", defaultValue = "TSV") ExportFormat format) throws IOException {

response.setHeader(HttpHeaders.CONTENT_DISPOSITION, FILE_HEADER_PRE + format.name().toLowerCase());
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, FILE_HEADER_PRE + key + '.' + format.name().toLowerCase());

try (Writer writer = new BufferedWriter(new OutputStreamWriter(response.getOutputStream()))) {
CsvWriter.datasetOccurrenceDownloadUsageCsvWriter(Iterables.datasetOccurrenceDownloadUsages(this,
Expand Down
25 changes: 8 additions & 17 deletions registry-ws/src/test/java/org/gbif/registry/ws/export/CsvWriterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,29 +172,24 @@ public void datasetSearchTest() {
* Test one DatasetOccurrenceDownloadUsage against its expected exported data.
*/
private void assertDatasetOccurrenceDownloadUsage(DatasetOccurrenceDownloadUsage downloadUsage, String[] line) {
//"download", "downloadKey", "datasetDOI", "datasetKey", "datasetTitle", "datasetCitation", "numberRecords"
assertEquals(downloadUsage.getDownload().getDoi().toString(), line[0]);
assertEquals(downloadUsage.getDownloadKey(), line[1]);
assertEquals(downloadUsage.getDatasetDOI().toString(), line[2]);
assertEquals(downloadUsage.getDatasetKey().toString(), line[3]);
assertEquals(downloadUsage.getDatasetTitle(), line[4]);
assertEquals(downloadUsage.getDatasetCitation(), line[5]);
assertEquals(downloadUsage.getNumberRecords(), Long.parseLong(line[6].replace("\r","")));
assertEquals(downloadUsage.getDatasetDOI().toString(), line[0]);
assertEquals(downloadUsage.getDatasetKey().toString(), line[1]);
assertEquals(downloadUsage.getDatasetTitle(), line[2]);
assertEquals(downloadUsage.getDatasetCitation(), line[3]);
assertEquals(downloadUsage.getNumberRecords(), Long.parseLong(line[4].replace("\r","")));
}

/**
* Generates test instances of DatasetOccurrenceDownloadUsage.
*/
private static DatasetOccurrenceDownloadUsage newDatasetOccurrenceDownloadUsageTest(int consecutive, Download download) {
private static DatasetOccurrenceDownloadUsage newDatasetOccurrenceDownloadUsageTest(int consecutive) {
DatasetOccurrenceDownloadUsage downloadUsage = new DatasetOccurrenceDownloadUsage();

downloadUsage.setDatasetKey(UUID.randomUUID());
downloadUsage.setDatasetDOI(new DOI("10.21373/6m9yw" + consecutive));
downloadUsage.setDatasetTitle("UsageTitle" + consecutive);
downloadUsage.setDatasetCitation("Citation" + consecutive);
downloadUsage.setNumberRecords(consecutive);
downloadUsage.setDownloadKey(download.getKey());
downloadUsage.setDownload(download);

return downloadUsage;
}
Expand All @@ -203,12 +198,8 @@ private static DatasetOccurrenceDownloadUsage newDatasetOccurrenceDownloadUsageT
public void datasetOccurrenceDownloadUsageTest() {

//Test data
Download download = new Download();
download.setKey("0220580-200613084148143");
download.setDoi(new DOI("10.21373/6m9yw0"));

List<DatasetOccurrenceDownloadUsage> downloadUsages = Arrays.asList(newDatasetOccurrenceDownloadUsageTest(1, download),
newDatasetOccurrenceDownloadUsageTest(2, download));
List<DatasetOccurrenceDownloadUsage> downloadUsages = Arrays.asList(newDatasetOccurrenceDownloadUsageTest(1),
newDatasetOccurrenceDownloadUsageTest(2));

StringWriter writer = new StringWriter();

Expand Down

0 comments on commit 34521cc

Please sign in to comment.