Merge pull request #551 from pdowler/vos2

minoc: filename override mechanism

cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java

@@ -73,6 +73,7 @@
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -99,10 +100,16 @@ public void testPrioritizePullFromSites() throws Exception {
         for (int i = 0; i < 10; i++) {
             sites.add(new StorageSite(URI.create("ivo://site" + i), "site1" + i, true, rd.nextBoolean()));
         }
-        ProtocolsGenerator.prioritizePullFromSites(sites);
-        for (StorageSite s : sites) {
-            log.info("found: " + s.getID() + " aka " +  s.getResourceID());
-        }
+        List<StorageSite> result1 = ProtocolsGenerator.prioritizePullFromSites(sites);
+        Assert.assertEquals(sites.size(), result1.size());
+        Assert.assertTrue(result1.containsAll(sites));
+
+        List<StorageSite> result2 = ProtocolsGenerator.prioritizePullFromSites(sites);
+        Assert.assertEquals(sites.size(), result2.size());
+        Assert.assertTrue(result2.containsAll(sites));
+
+        // test random order
+        Assert.assertNotEquals(result1, result2);
     }
 
     @Test

cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java

@@ -384,6 +384,7 @@ public static void assertValidPathComponent(Class caller, String name, String te
         boolean slash = (test.indexOf('/') >= 0);
         boolean escape = (test.indexOf('\\') >= 0);
         boolean percent = (test.indexOf('%') >= 0);
+        boolean colon = (test.indexOf(":") >= 0);
         boolean semic = (test.indexOf(';') >= 0);
         boolean amp = (test.indexOf('&') >= 0);
         boolean dollar = (test.indexOf('$') >= 0);
@@ -398,7 +399,7 @@ public static void assertValidPathComponent(Class caller, String name, String te
             }
             throw new IllegalArgumentException(s + name + ": " + test
                     + " reason: path component may not contain space ( ), slash (/), escape (\\), percent (%),"
-                    + " semi-colon (;), ampersand (&), or dollar ($), question (?), or square brackets ([])");
+                    + " colon (:), semi-colon (;), ampersand (&), dollar ($), question (?), or square brackets ([])");
         }
     }
 

minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java

@@ -146,10 +146,12 @@ public void testPutGetUpdateHeadDelete() {
             long contentLength = get.getContentLength();
             String contentType = get.getContentType();
             String contentEncoding = get.getContentEncoding();
+            String contentDisposition = get.getResponseHeader("content-disposition");
             Assert.assertEquals(computeChecksumURI(data), checksumURI);
             Assert.assertEquals(data.length, contentLength);
             Assert.assertEquals(type, contentType);
             Assert.assertEquals(encoding, contentEncoding);
+            Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("file.txt"));
             Date lastModified = get.getLastModified(); 
             Assert.assertNotNull(lastModified);
 
@@ -181,10 +183,12 @@ public void testPutGetUpdateHeadDelete() {
             contentLength = head.getContentLength();
             contentType = head.getContentType();
             contentEncoding = head.getContentEncoding();
+            contentDisposition = head.getResponseHeader("content-disposition");
             Assert.assertEquals(computeChecksumURI(data), checksumURI);
             Assert.assertEquals(data.length, contentLength);
             Assert.assertEquals(newType, contentType);
             Assert.assertEquals(newEncoding, contentEncoding);
+            Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("file.txt"));
             lastModified = head.getLastModified(); 
             Assert.assertNotNull(lastModified);
 
@@ -331,6 +335,114 @@ public void testGetRanges() {
         }
     }
 
+    @Test
+    public void testFilenameOverride() {
+        try {
+            URI artifactURI = URI.create("cadc:TEST/testFilenameOverride.txt");
+            URL artifactURL = new URL(filesURL + "/" + artifactURI.toString());
+
+            String content = "abcdefghijklmnopqrstuvwxyz";
+            String encoding = "test-encoding";
+            String type = "text/plain";
+            byte[] data = content.getBytes();
+            URI expectedChecksum = computeChecksumURI(data);
+
+            // put: no length or checksum
+            InputStream in = new ByteArrayInputStream(data);
+            HttpUpload put = new HttpUpload(in, artifactURL);
+            put.setRequestProperty(HttpTransfer.CONTENT_TYPE, type);
+            put.setRequestProperty(HttpTransfer.CONTENT_ENCODING, encoding);
+            put.setDigest(expectedChecksum);
+
+            Subject.doAs(userSubject, new RunnableAction(put));
+            log.info("put: " + put.getResponseCode() + " " + put.getThrowable());
+            log.info("headers: " + put.getResponseHeader("content-length") + " " + put.getResponseHeader("digest"));
+            Assert.assertNull(put.getThrowable());
+            Assert.assertEquals("Created", 201, put.getResponseCode());
+
+            // head
+            ByteArrayOutputStream bos = new ByteArrayOutputStream();
+            HttpGet head = new HttpGet(artifactURL, bos);
+            head.setHeadOnly(true);
+            log.info("head: " + artifactURL.toExternalForm());
+            Subject.doAs(userSubject, new RunnableAction(head));
+            log.info("head: " + head.getResponseCode() + " " + head.getThrowable());
+            log.info("headers: " + head.getResponseHeader("content-length") + " " + head.getResponseHeader("digest"));
+            log.warn("head output: " + bos.toString());
+            Assert.assertNull(head.getThrowable());
+            URI checksumURI = head.getDigest();
+            long contentLength = head.getContentLength();
+            String contentType = head.getContentType();
+            String contentEncoding = head.getContentEncoding();
+            String contentDisposition = head.getResponseHeader("content-disposition");
+            Assert.assertEquals(computeChecksumURI(data), checksumURI);
+            Assert.assertEquals(data.length, contentLength);
+            Assert.assertEquals(type, contentType);
+            Assert.assertEquals(encoding, contentEncoding);
+            log.info("content-disposition: " + contentDisposition);
+            Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("testFilenameOverride.txt"));
+            Date lastModified = head.getLastModified(); 
+            Assert.assertNotNull(lastModified);
+
+            URL foURL = new URL(artifactURL.toExternalForm() + ":fo/alternate.txt");
+            head = new HttpGet(foURL, bos);
+            head.setHeadOnly(true);
+            log.info("head: " + foURL.toExternalForm());
+            Subject.doAs(userSubject, new RunnableAction(head));
+            log.info("head: " + head.getResponseCode() + " " + head.getThrowable());
+            log.info("headers: " + head.getResponseHeader("content-length") + " " + head.getResponseHeader("digest"));
+            log.warn("head output: " + bos.toString());
+            Assert.assertNull(head.getThrowable());
+            checksumURI = head.getDigest();
+            contentLength = head.getContentLength();
+            contentType = head.getContentType();
+            contentEncoding = head.getContentEncoding();
+            contentDisposition = head.getResponseHeader("content-disposition");
+            Assert.assertEquals(computeChecksumURI(data), checksumURI);
+            Assert.assertEquals(data.length, contentLength);
+            Assert.assertEquals(type, contentType);
+            Assert.assertEquals(encoding, contentEncoding);
+            log.info("content-disposition: " + contentDisposition);
+            Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("alternate.txt"));
+            Date lastModified2 = head.getLastModified(); 
+            Assert.assertEquals(lastModified, lastModified2);
+
+            // get
+            bos = new ByteArrayOutputStream();
+            log.info("get: " + foURL.toExternalForm());
+            HttpGet get = new HttpGet(foURL, bos);
+            Subject.doAs(userSubject, new RunnableAction(get));
+            log.info("get: " + get.getResponseCode() + " " + get.getThrowable());
+            log.info("headers: " + get.getResponseHeader("content-length") + " " + get.getResponseHeader("digest"));
+            log.warn("get output: " + bos.toString());
+            Assert.assertNull(get.getThrowable());
+            checksumURI = get.getDigest();
+            contentLength = get.getContentLength();
+            contentType = get.getContentType();
+            contentEncoding = get.getContentEncoding();
+            contentDisposition = get.getResponseHeader("content-disposition");
+            Assert.assertEquals(computeChecksumURI(data), checksumURI);
+            Assert.assertEquals(data.length, contentLength);
+            Assert.assertEquals(type, contentType);
+            Assert.assertEquals(encoding, contentEncoding);
+            log.info("content-disposition: " + contentDisposition);
+            Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("alternate.txt"));
+            Date lastModified3 = get.getLastModified(); 
+            Assert.assertEquals(lastModified, lastModified3);
+
+            // delete
+            HttpDelete delete = new HttpDelete(artifactURL, false);
+            Subject.doAs(userSubject, new RunnableAction(delete));
+            log.info("delete: " + delete.getResponseCode() + " " + delete.getThrowable());
+            Assert.assertNull(delete.getThrowable());
+            Assert.assertEquals("no content", 204, delete.getResponseCode());
+
+        } catch (Exception t) {
+            log.error("unexpected throwable", t);
+            Assert.fail("unexpected throwable: " + t);
+        }
+    }
+
     @Test
     public void testGetNotFound() {
         try {

minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java

@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2023.                            (c) 2023.
+*  (c) 2024.                            (c) 2024.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -116,6 +116,11 @@ public abstract class ArtifactAction extends RestAction {
 
     // The target artifact
     URI artifactURI;
+    String errMsg;
+
+    // alternmate filename for content-disposition header, usually null
+    boolean extractFilenameOverride = false;
+    String filenameOverride;
 
     // The (possibly null) authentication token.
     String authToken;
@@ -260,6 +265,10 @@ protected void initAndAuthorize(Class<? extends Grant> grantClass, boolean allow
 
     void init() {
         if (this.artifactURI == null) {
+            if (errMsg != null) {
+                throw new IllegalArgumentException(errMsg);
+            }
+            // generic
             throw new IllegalArgumentException("missing or invalid artifact URI");
         }
     }
@@ -285,16 +294,29 @@ void parsePath() {
         String path = this.syncInput.getPath();
         log.debug("path: " + path);
         if (path != null) {
-            int colonIndex = path.indexOf(":");
-            int firstSlashIndex = path.indexOf("/");
-            if (colonIndex != -1) {
-                if (firstSlashIndex < 0 || firstSlashIndex > colonIndex) {
-                    // no auth token--artifact URI is complete path
-                    this.artifactURI = createArtifactURI(path);
-                } else {
-                    this.artifactURI = createArtifactURI(path.substring(firstSlashIndex + 1));
-                    this.authToken = path.substring(0, firstSlashIndex);
-                    log.debug("authToken: " + this.authToken);
+            int colon1 = path.indexOf(":");
+            int slash1 = path.indexOf("/");
+            if (colon1 != -1) {
+                if (slash1 >= 0 && slash1 < colon1) {
+                    // auth token in front
+                    this.authToken = path.substring(0, slash1);
+                    path = path.substring(slash1 + 1);
+                }
+                try {
+                    int foi = path.indexOf(":fo/");
+                    if (foi > 0 && extractFilenameOverride) {
+                        // filename override appended
+                        this.filenameOverride = path.substring(foi + 4);
+                        path = path.substring(0, foi);
+                    } else if (foi > 0) {
+                        throw new IllegalArgumentException("detected misuse of :fo/ filename override");
+                    }
+                    URI auri = new URI(path);
+                    InventoryUtil.validateArtifactURI(ArtifactAction.class, auri);
+                    this.artifactURI = auri;
+                } catch (URISyntaxException | IllegalArgumentException e) {
+                    this.errMsg = "illegal artifact URI: " + path + " reason: " + e.getMessage();
+                    log.debug(errMsg, e);
                 }
             }
         }
@@ -307,22 +329,4 @@ Artifact getArtifact(URI artifactURI) throws ResourceNotFoundException {
         }
         return artifact;
     }
-
-    /**
-     * Create a valid artifact uri.
-     * @param uri The input string.
-     * @return The artifact uri object.
-     */
-    private URI createArtifactURI(String uri) {
-        log.debug("artifact URI: " + uri);
-        URI ret;
-        try {
-            ret = new URI(uri);
-            InventoryUtil.validateArtifactURI(ArtifactAction.class, ret);
-        } catch (URISyntaxException | IllegalArgumentException e) {
-            ret = null;
-            log.debug("illegal artifact URI: " + uri, e);
-        }
-        return ret;
-    }
 }

minoc/src/main/java/org/opencadc/minoc/GetAction.java

@@ -124,13 +124,15 @@ public class GetAction extends ArtifactAction {
     // constructor for unit tests with no config/init
     GetAction(boolean init) {
         super(init);
+        this.extractFilenameOverride = true;
     }
 
     /**
      * Default, no-arg constructor.
      */
     public GetAction() {
         super();
+        this.extractFilenameOverride = true;
     }
 
     /**
@@ -194,7 +196,7 @@ public void doAction() throws Exception {
             }
 
             // default: complete download
-            HeadAction.setHeaders(artifact, syncOutput);
+            HeadAction.setHeaders(artifact, filenameOverride, syncOutput);
             bcos = new ByteCountOutputStream(syncOutput.getOutputStream());
 
             // create tmp StorageLocation with expected checksum so adapter can potentially
@@ -238,7 +240,7 @@ public void doAction() throws Exception {
     private ByteCountOutputStream doByteRangeRequest(Artifact artifact, ByteRange byteRange) 
             throws InterruptedException, IOException, ResourceNotFoundException, 
                 ReadException, WriteException, StorageEngageException, TransientException {
-        HeadAction.setHeaders(artifact, syncOutput);
+        HeadAction.setHeaders(artifact, filenameOverride, syncOutput);
         syncOutput.setCode(206);
         long lastByte = byteRange.getOffset() + byteRange.getLength() - 1;
         syncOutput.setHeader(CONTENT_RANGE, "bytes " + byteRange.getOffset() + "-"

minoc/src/main/java/org/opencadc/minoc/HeadAction.java

@@ -93,6 +93,7 @@ public class HeadAction extends ArtifactAction {
      */
     public HeadAction() {
         super();
+        this.extractFilenameOverride = true;
     }
 
     /**
@@ -131,7 +132,7 @@ public void doAction() throws Exception {
             artifact = getArtifact(artifactURI);
         }
         if (artifact != null) {
-            setHeaders(artifact, syncOutput);
+            setHeaders(artifact, filenameOverride, syncOutput);
         }
     }
 
@@ -140,7 +141,7 @@ public void doAction() throws Exception {
      * @param artifact The artifact with metadata
      * @param syncOutput The target response
      */
-    static void setHeaders(Artifact artifact, SyncOutput syncOutput) {
+    static void setHeaders(Artifact artifact, String filenameOverride, SyncOutput syncOutput) {
         syncOutput.setHeader(ARTIFACT_ID_HDR, artifact.getID().toString());
         syncOutput.setDigest(artifact.getContentChecksum());
         syncOutput.setLastModified(artifact.getContentLastModified());
@@ -149,7 +150,10 @@ static void setHeaders(Artifact artifact, SyncOutput syncOutput) {
         DateFormat df = DateUtil.getDateFormat(DateUtil.HTTP_DATE_FORMAT, DateUtil.GMT);
         syncOutput.setHeader("Last-Modified", df.format(artifact.getContentLastModified()));
 
-        String filename = InventoryUtil.computeArtifactFilename(artifact.getURI());
+        String filename = filenameOverride;
+        if (filename == null) {
+            filename = InventoryUtil.computeArtifactFilename(artifact.getURI());
+        }
         syncOutput.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\"");
 
         if (artifact.contentEncoding != null) {

minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java

@@ -196,9 +196,13 @@ private void initStorageSite() {
         if (name.charAt(0) == '/') {
             name = name.substring(1);
         }
+
+        // possibly temporary hack: advertise readable and writable if this service
+        // is configured to accept preauth tokens
+        boolean trustPreauth = !config.getTrustedServices().isEmpty();
 
-        boolean allowRead = !config.getReadGrantServices().isEmpty();
-        boolean allowWrite = !config.getWriteGrantServices().isEmpty();
+        boolean allowRead = trustPreauth || !config.getReadGrantServices().isEmpty();
+        boolean allowWrite = trustPreauth || !config.getWriteGrantServices().isEmpty();
 
         StorageSite self = null;
         if (curlist.isEmpty()) {

minoc/src/main/java/org/opencadc/minoc/PostAction.java

@@ -193,7 +193,7 @@ public void doAction() throws Exception {
             log.debug("commit txn: OK");
 
             syncOutput.setCode(202); // Accepted
-            HeadAction.setHeaders(existing, syncOutput);
+            HeadAction.setHeaders(existing, null, syncOutput);
             syncOutput.setHeader("content-length", 0);
         } catch (Exception e) {
             log.error("failed to persist " + artifactURI, e);