support for SIARDDK (#431)

dev/codeserver/pom.xml

@@ -62,13 +62,13 @@
         <dependency>
             <groupId>com.databasepreservation</groupId>
             <artifactId>dbptk-bindings</artifactId>
-            <version>2.0.0</version>
+            <version>2.1.0-SNAPSHOT</version>
             <type>pom</type>
         </dependency>
         <dependency>
             <groupId>com.databasepreservation</groupId>
             <artifactId>dbptk-bindings-siard2-1</artifactId>
-            <version>2.0.0</version>
+            <version>2.1.0-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>com.databasepreservation</groupId>

pom.xml

@@ -26,7 +26,7 @@
 
         <version.gwt>2.11.0</version.gwt>
         <scope.gwt-dev>provided</scope.gwt-dev>
-        <version.dbptk>3.0.0</version.dbptk>
+        <version.dbptk>3.1.0-SNAPSHOT</version.dbptk>
         <version.roda>5.4.0</version.roda>
         <version.jersey>3.1.6</version.jersey>
         <version.swagger>2.2.20</version.swagger>
@@ -486,13 +486,23 @@
         <dependency>
             <groupId>com.databasepreservation</groupId>
             <artifactId>dbptk-bindings</artifactId>
-            <version>2.0.0</version>
+            <version>2.1.0-SNAPSHOT</version>
             <type>pom</type>
         </dependency>
         <dependency>
             <groupId>com.databasepreservation</groupId>
             <artifactId>dbptk-bindings-siard2-1</artifactId>
-            <version>2.0.0</version>
+            <version>2.1.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>com.databasepreservation</groupId>
+            <artifactId>dbptk-bindings-siarddk-128</artifactId>
+            <version>2.1.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>com.databasepreservation</groupId>
+            <artifactId>dbptk-bindings-siarddk-1007</artifactId>
+            <version>2.1.0-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>com.databasepreservation</groupId>

src/main/java/com/databasepreservation/DBVTK.java

@@ -132,7 +132,7 @@ public FilterRegistrationBean<OnOffFilter> casSingleSignOutFilter() {
     FilterRegistrationBean<OnOffFilter> registrationBean = new FilterRegistrationBean<>();
     registrationBean.setFilter(new OnOffFilter());
     registrationBean.setName("CasSingleSignOutFilter");
-    registrationBean.addInitParameter("inner-filter-class", "org.jasig.cas.client.session.SingleSignOutFilter");
+    registrationBean.addInitParameter("inner-filter-class", "org.apereo.cas.client.session.SingleSignOutFilter");
     registrationBean.addInitParameter("config-prefix", "ui.filter.cas");
     registrationBean.addInitParameter("casServerUrlPrefix", "http://localhost:8888/cas");
     registrationBean.addUrlPatterns("/*");
@@ -151,7 +151,7 @@ public FilterRegistrationBean<OnOffFilter> casValidationFilter() {
     registrationBean.setFilter(new OnOffFilter());
     registrationBean.setName("CasValidationFilter");
     registrationBean.addInitParameter("inner-filter-class",
-      "org.jasig.cas.client.validation.Cas30ProxyReceivingTicketValidationFilter");
+      "org.apereo.cas.client.validation.Cas30ProxyReceivingTicketValidationFilter");
     registrationBean.addInitParameter("config-prefix", "ui.filter.cas");
     registrationBean.addInitParameter("casServerUrlPrefix", "https://localhost:8443/cas");
     registrationBean.addInitParameter("serverName", "https://localhost:8888");
@@ -170,7 +170,7 @@ public FilterRegistrationBean<OnOffFilter> casAuthenticationFilter() {
     FilterRegistrationBean<OnOffFilter> registrationBean = new FilterRegistrationBean<>();
     registrationBean.setFilter(new OnOffFilter());
     registrationBean.setName("CasAuthenticationFilter");
-    registrationBean.addInitParameter("inner-filter-class", "org.jasig.cas.client.authentication.AuthenticationFilter");
+    registrationBean.addInitParameter("inner-filter-class", "org.apereo.cas.client.authentication.AuthenticationFilter");
     registrationBean.addInitParameter("config-prefix", "ui.filter.cas");
     registrationBean.addInitParameter("casServerLoginUrl", "https://localhost:8443/cas/login");
     registrationBean.addUrlPatterns("/login");
@@ -184,7 +184,7 @@ public FilterRegistrationBean<OnOffFilter> casRequestWrapperFilter() {
     registrationBean.setFilter(new OnOffFilter());
     registrationBean.setName("CasRequestWrapperFilter");
     registrationBean.addInitParameter("inner-filter-class",
-      "org.jasig.cas.client.util.HttpServletRequestWrapperFilter");
+      "org.apereo.cas.client.util.HttpServletRequestWrapperFilter");
     registrationBean.addInitParameter("config-prefix", "ui.filter.cas");
     registrationBean.addUrlPatterns("/*");
 

src/main/java/com/databasepreservation/common/api/utils/ExtraMediaType.java

@@ -28,6 +28,7 @@ private ExtraMediaType() {
    */
   public static final String TEXT_CSV = "text/csv";
   public static final String APPLICATION_ZIP = "application/zip";
+  public static final String ZIP_FILE_EXTENSION = ".zip";
   public static final String APPLICATION_JAVASCRIPT = "application/javascript";
 
 }

src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java

@@ -13,6 +13,7 @@
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -24,9 +25,14 @@
 import java.util.List;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
+import java.util.zip.ZipOutputStream;
 
 import com.databasepreservation.common.api.exceptions.RESTException;
 import com.databasepreservation.common.exceptions.AuthorizationException;
+import com.databasepreservation.common.client.models.structure.ViewerCell;
+import com.databasepreservation.model.data.Cell;
+import com.databasepreservation.common.api.utils.ExtraMediaType;
+import com.databasepreservation.model.exception.ModuleException;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang.StringUtils;
 import org.roda.core.data.exceptions.*;
@@ -197,9 +203,9 @@ public StringResponse createCollection(String databaseUUID) {
       }
 
       final ViewerDatabase database = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID);
-      StringResponse collection = new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID));
 
-      return collection;
+      return new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID, database.getVersion()));
+
     } catch (GenericException | AuthorizationDeniedException | NotFoundException | AuthorizationException e) {
       state = LogEntryState.FAILURE;
       throw new RESTException(e);
@@ -529,7 +535,8 @@ public ResponseEntity<StreamingResponseBody> exportLOB(
           .equals(row.getCells().get(configTable.getColumnByIndex(columnIndex).getId()).getStoreType())) {
           return handleExternalLobDownload(configTable, row, columnIndex);
         } else {
-          return handleInternalLobDownload(database.getPath(), configTable, row, columnIndex);
+          String version = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID).getVersion();
+          return handleInternalLobDownload(database.getPath(), configTable, row, columnIndex, version);
         }
       }
     } catch (NotFoundException | GenericException | IOException | AuthorizationException e) {
@@ -598,37 +605,53 @@ private ResponseEntity<StreamingResponseBody> handleExternalLobDownload(TableSta
   }
 
   private ResponseEntity<StreamingResponseBody> handleInternalLobDownload(String databasePath,
-    TableStatus tableConfiguration, ViewerRow row, int columnIndex) throws IOException, GenericException {
+    TableStatus tableConfiguration, ViewerRow row, int columnIndex, String version)
+    throws IOException, GenericException {
     String handlebarsFilename = HandlebarsUtils.applyExportTemplate(row, tableConfiguration, columnIndex);
 
     if (ViewerStringUtils.isBlank(handlebarsFilename)) {
       handlebarsFilename = ViewerConstants.SIARD_RECORD_PREFIX + row.getUuid()
         + ViewerConstants.SIARD_LOB_FILE_EXTENSION;
     }
-    String handlebarsMimeType = HandlebarsUtils.applyMimeTypeTemplate(row, tableConfiguration, columnIndex);
 
+    String handlebarsMimeType = HandlebarsUtils.applyMimeTypeTemplate(row, tableConfiguration, columnIndex);
     if (ViewerStringUtils.isBlank(handlebarsMimeType)) {
       handlebarsMimeType = tableConfiguration.getColumnByIndex(columnIndex).getApplicationType();
     }
 
-    if (LobManagerUtils.isLobEmbedded(tableConfiguration, row, columnIndex)) {
-      // handle lob as embedded
-      String lobCellValue = LobManagerUtils.getLobCellValue(tableConfiguration, row, columnIndex);
-      lobCellValue = lobCellValue.replace(ViewerConstants.SIARD_EMBEDDED_LOB_PREFIX, "");
-      String decodedString = new String(Base64.decodeBase64(lobCellValue.getBytes()));
+    if (version.equals(ViewerConstants.SIARD_DK_1007) || version.equals(ViewerConstants.SIARD_DK_128)) {
+      String filePath = row.getCells().get(row.getCells().keySet().toArray()[row.getCells().size() - 1]).getValue();
+      Path path = Paths.get(filePath);
+      // if the lob is a directory zip it
+      if (path.toFile().isDirectory()) {
+        Path zipFile = LobManagerUtils.zipDirectory(path, databasePath, handlebarsFilename);
 
-      return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
-        DownloadUtils.stream(new BufferedInputStream(new ByteArrayInputStream(decodedString.getBytes())))));
-    } else {
-      // handle lob as internal on separated folder
-      ZipFile zipFile = new ZipFile(databasePath);
-      final ZipEntry entry = zipFile.getEntry(LobManagerUtils.getZipFilePath(tableConfiguration, columnIndex, row));
-      if (entry == null) {
-        throw new GenericException("Zip archive entry is missing");
+        return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
+          DownloadUtils.stream(new BufferedInputStream(new FileInputStream(zipFile.toFile())))));
+      } else {
+        return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
+          DownloadUtils.stream(new BufferedInputStream(new FileInputStream(filePath)))));
       }
+    } else {
+      if (LobManagerUtils.isLobEmbedded(tableConfiguration, row, columnIndex)) {
+        // handle lob as embedded
+        String lobCellValue = LobManagerUtils.getLobCellValue(tableConfiguration, row, columnIndex);
+        lobCellValue = lobCellValue.replace(ViewerConstants.SIARD_EMBEDDED_LOB_PREFIX, "");
+        String decodedString = new String(Base64.decodeBase64(lobCellValue.getBytes()));
+
+        return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
+          DownloadUtils.stream(new BufferedInputStream(new ByteArrayInputStream(decodedString.getBytes())))));
+      } else {
+        // handle lob as internal on separated folder
+        ZipFile zipFile = new ZipFile(databasePath);
+        final ZipEntry entry = zipFile.getEntry(LobManagerUtils.getZipFilePath(tableConfiguration, columnIndex, row));
+        if (entry == null) {
+          throw new GenericException("Zip archive entry is missing");
+        }
 
-      return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
-        DownloadUtils.stream(new BufferedInputStream(zipFile.getInputStream(entry)))));
+        return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType,
+          DownloadUtils.stream(new BufferedInputStream(zipFile.getInputStream(entry)))));
+      }
     }
   }
 

src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java

@@ -125,15 +125,15 @@ public IndexResult<ViewerDatabase> findAll(FindRequest findRequest, String local
   }
 
   @Override
-  public StringResponse create(String path) {
+  public StringResponse create(String path, ViewerConstants.SiardVersion siardVersion) {
     final ControllerAssistant controllerAssistant = new ControllerAssistant() {};
 
     LogEntryState state = LogEntryState.SUCCESS;
     User user = new User();
 
     try {
       user = controllerAssistant.checkRoles(request);
-      return new StringResponse(SIARDController.loadMetadataFromLocal(path));
+      return new StringResponse(SIARDController.loadMetadataFromLocal(path, siardVersion));
     } catch (GenericException | AuthorizationException e) {
       state = LogEntryState.FAILURE;
       throw new RESTException(e);

src/main/java/com/databasepreservation/common/api/v1/FileResource.java

@@ -16,6 +16,7 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
+import com.databasepreservation.common.utils.LobManagerUtils;
 import org.roda.core.data.exceptions.AlreadyExistsException;
 import org.roda.core.data.exceptions.GenericException;
 import org.roda.core.data.exceptions.NotFoundException;
@@ -90,9 +91,13 @@ public ResponseEntity<Resource> getSIARDFile(String filename) {
       java.nio.file.Path basePath = Paths.get(ViewerConfiguration.getInstance().getViewerConfigurationAsString("/",
         ViewerConfiguration.PROPERTY_BASE_UPLOAD_PATH));
       java.nio.file.Path siardPath = siardFilesPath.resolve(filename);
-      if (java.nio.file.Files.exists(siardPath) && !java.nio.file.Files.isDirectory(siardPath)
-        && (ViewerConfiguration.checkPathIsWithin(siardPath, siardFilesPath)
-          || ViewerConfiguration.checkPathIsWithin(siardPath, basePath))) {
+
+      if (java.nio.file.Files.isDirectory(siardPath)) {
+        siardPath = LobManagerUtils.zipDirectory(siardPath);
+      }
+
+      if (java.nio.file.Files.exists(siardPath) && (ViewerConfiguration.checkPathIsWithin(siardPath, siardFilesPath)
+        || ViewerConfiguration.checkPathIsWithin(siardPath, basePath))) {
 
         InputStreamResource resource = new InputStreamResource(new FileInputStream(siardPath.toFile()));
         return ResponseEntity.ok()
@@ -101,7 +106,7 @@ public ResponseEntity<Resource> getSIARDFile(String filename) {
       } else {
         throw new NotFoundException("SIARD file not found");
       }
-    } catch (NotFoundException | FileNotFoundException | AuthorizationException e) {
+    } catch (NotFoundException | AuthorizationException | IOException e) {
       state = LogEntryState.FAILURE;
       throw new RESTException(e);
     } finally {
@@ -149,7 +154,7 @@ public ResponseEntity<ApiResponseMessage> createSIARDFile(MultipartFile resource
 
       if (!fileExtension.equals(ViewerConstants.SIARD)) {
         return ResponseEntity.status(HttpStatus.BAD_REQUEST)
-                .body(new ApiResponseMessage(ApiResponseMessage.ERROR, "Must be a SIARD file"));
+          .body(new ApiResponseMessage(ApiResponseMessage.ERROR, "Must be a SIARD file"));
       }
 
       java.nio.file.Path path = Paths.get(ViewerConfiguration.getInstance().getSIARDFilesPath().toString(), filename);

src/main/java/com/databasepreservation/common/client/ViewerConstants.java

@@ -91,6 +91,7 @@ public class ViewerConstants {
   public static final String DENORMALIZATION_STATUS_PREFIX = "denormalization-";
 
   public static final String INTERNAL_ZIP_LOB_FOLDER = "lobs/";
+  public static final String SIARDDK_DEFAULT_SCHEMA_NAME = "public";
 
   /*
    * SOLR CONFIGSETS
@@ -441,7 +442,9 @@ public class ViewerConstants {
   public static final String SIARD_V10 = "1.0";
   public static final String SIARD_V20 = "2.0";
   public static final String SIARD_V21 = "2.1";
-
+  public static final String SIARD_DK = "dk";
+  public static final String SIARD_DK_1007 = "dk-1007";
+  public static final String SIARD_DK_128 = "dk-128";
   /*
    * DBPTK Metadata
    */
@@ -552,6 +555,7 @@ public class ViewerConstants {
   public static final String PROPERTY_PLUGIN_LOAD_ON_ACCESS = "ui.plugin.loadOnAccess";
 
   public static final String ALIAS_PREFIX = "alias-";
+  public static final String TEMP_PREFIX = "temp-";
 
   public static final String EMPTY_SEARCH = "";
 
@@ -566,6 +570,10 @@ public class ViewerConstants {
    */
   public static final String PROPERTY_REFERENCE_TABLE_SHOW_SCHEMA_NAME = "ui.reference.table.show.schema.name";
 
+  public enum SiardVersion {
+    V1_0, V2_0, V2_1, DK, DK_1007, DK_128;
+  }
+
   /**
    * private constructor
    */

src/main/java/com/databasepreservation/common/client/common/helpers/HelperUploadSIARDFile.java

@@ -108,7 +108,7 @@ private void uploadMetadataSIARD(String path, FlowPanel panel) {
       Dialogs.showErrors(messages.errorMessagesOpenFile(path), errorMessage, messages.basicActionClose());
       // Toast.showError(messages.errorMessagesOpenFile(), errorMessage);
       panel.remove(loading);
-    }).create(path);
+    }).create(path, ViewerConstants.SiardVersion.V2_1);
   }
 
   private void successHandler(Boolean confirm, FlowPanel panel, String databaseUUID, String path) {

src/main/java/com/databasepreservation/common/client/common/lists/DatabaseList.java

@@ -86,7 +86,7 @@ public SafeHtml getValue(ViewerDatabase database) {
     Column<ViewerDatabase, SafeHtml> description = new TooltipColumn<ViewerDatabase>() {
       @Override
       public SafeHtml getValue(ViewerDatabase database) {
-        return database != null && database.getMetadata() != null
+        return database != null && database.getMetadata() != null && database.getMetadata().getDescription() != null
           ? SafeHtmlUtils.fromString(database.getMetadata().getDescription())
           : SafeHtmlUtils.fromString("unknown");
       }
@@ -104,7 +104,7 @@ public SafeHtml getValue(ViewerDatabase database) {
     Column<ViewerDatabase, SafeHtml> dataOwnerColumn = new TooltipColumn<ViewerDatabase>() {
       @Override
       public SafeHtml getValue(ViewerDatabase database) {
-        return database != null && database.getMetadata() != null
+        return database != null && database.getMetadata() != null && database.getMetadata().getDataOwner() != null
           ? SafeHtmlUtils.fromString(database.getMetadata().getDataOwner())
           : SafeHtmlUtils.fromString("unknown");
       }

src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/SIARDManagerPage.java

@@ -143,7 +143,8 @@ private void populateNavigationPanels() {
     if (ApplicationType.getType().equals(ViewerConstants.APPLICATION_ENV_SERVER)) {
       DatabaseService.Util.call((Set<String> databasePermissions) -> {
         ContextService.Util.call((Set<AuthorizationGroup> authorizationGroups) -> {
-          permissionsNavigationPanel = PermissionsNavigationPanel.getInstance(database, databasePermissions, authorizationGroups);
+          permissionsNavigationPanel = PermissionsNavigationPanel.getInstance(database, databasePermissions,
+            authorizationGroups);
           if (permissionsNavigationPanel.hasPermissionsOrGroups()) {
             navigationPanels.add(permissionsNavigationPanel.build());
           }
@@ -207,16 +208,23 @@ private void setupFooterButtons() {
         if (ApplicationType.getType().equals(ViewerConstants.APPLICATION_ENV_DESKTOP)) {
           message = messages.SIARDHomePageTextForDeleteAllFromDesktop();
         }
-        CommonDialogs.showConfirmDialog(messages.SIARDHomePageDialogTitleForDelete(), message,
-          messages.basicActionCancel(), messages.basicActionConfirm(), CommonDialogs.Level.DANGER, "500px",
-          new DefaultAsyncCallback<Boolean>() {
-            @Override
-            public void onSuccess(Boolean result) {
-              if (result) {
-                deleteAll();
+        if (database.getVersion().equals(ViewerConstants.SIARD_DK_1007)
+          || database.getVersion().equals(ViewerConstants.SIARD_DK_128)) {
+          Dialogs.showInformationDialog(messages.SIARDHomePageDialogTitleForDelete(),
+            messages.SIARDHomePageDialogTextForDeleteNotAvailable(ViewerConstants.SIARD_V21),
+            messages.basicActionUnderstood(), "btn btn-link");
+        } else {
+          CommonDialogs.showConfirmDialog(messages.SIARDHomePageDialogTitleForDelete(), message,
+            messages.basicActionCancel(), messages.basicActionConfirm(), CommonDialogs.Level.DANGER, "500px",
+            new DefaultAsyncCallback<Boolean>() {
+              @Override
+              public void onSuccess(Boolean result) {
+                if (result) {
+                  deleteAll();
+                }
               }
-            }
-          });
+            });
+        }
       } else if (ViewerDatabaseStatus.INGESTING.equals(database.getStatus())) {
         Dialogs.showInformationDialog(messages.SIARDManagerPageInformationDialogTitle(),
           messages.SIARDManagerPageTextForWaitForFinishing(), messages.basicActionClose(), "btn btn-link");