diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index 0307e7c1d..d8fca2f08 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -163,12 +163,8 @@ public class ProtocolsGenerator { public ProtocolsGenerator(ArtifactDAO artifactDAO, Map siteAvailabilities, Map siteRules) { this.artifactDAO = artifactDAO; this.deletedArtifactEventDAO = new DeletedArtifactEventDAO(this.artifactDAO); - this.user = user; - this.tokenGen = tokenGen; this.siteAvailabilities = siteAvailabilities; this.siteRules = siteRules; - this.preventNotFound = preventNotFound; - this.storageResolver = storageResolver; } public boolean getStorageResolverAdded() { @@ -176,6 +172,10 @@ public boolean getStorageResolverAdded() { } public List getProtocols(Transfer transfer) throws ResourceNotFoundException, IOException { + return getProtocols(transfer, null); + } + + public List getProtocols(Transfer transfer, String filenameOverride) throws ResourceNotFoundException, IOException { String authToken = null; URI artifactURI = transfer.getTargets().get(0); // see PostAction line ~127 if (tokenGen != null) { @@ -189,9 +189,13 @@ public List getProtocols(Transfer transfer) throws ResourceNotFoundExc List protos = null; if (Direction.pullFromVoSpace.equals(transfer.getDirection())) { - protos = doPullFrom(artifactURI, transfer, authToken); - } else { + // filename override only on GET + protos = doPullFrom(artifactURI, transfer, authToken, filenameOverride); + } else if (Direction.pushToVoSpace.equals(transfer.getDirection())) { protos = doPushTo(artifactURI, transfer, authToken); + } else { + throw new UnsupportedOperationException("unexpected transfer direction: " + transfer.getDirection().getValue()); + } return protos; } @@ -269,16 +273,6 @@ public Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set storageSites) { - // contains the algorithm for prioritizing storage sites to pull from. - - // was: prefer read/write sites to put less load on a read-only "seeder" site during migration - //storageSites.sort((site1, site2) -> Boolean.compare(!site1.getAllowWrite(), !site2.getAllowWrite())); - - // random - Collections.shuffle(storageSites); - } - Artifact getRemoteArtifact(URL location, URI artifactURI) { try { HttpGet head = new HttpGet(location, true); @@ -322,9 +316,25 @@ private Capability getFilesCapability(StorageSite storageSite) { return filesCap; } + // contains the algorithm for prioritizing storage sites to get file + static List prioritizePullFromSites(List storageSites) { + // filter out non-readble + List ret = new ArrayList<>(storageSites.size()); + for (StorageSite s : storageSites) { + if (s.getAllowRead()) { + ret.add(s); + } else { + log.debug("storage site is not readable: " + s.getResourceID()); + } + } + + // random + Collections.shuffle(ret); + return ret; + } - - List doPullFrom(URI artifactURI, Transfer transfer, String authToken) throws ResourceNotFoundException, IOException { + List doPullFrom(URI artifactURI, Transfer transfer, String authToken, String filenameOverride) + throws ResourceNotFoundException, IOException { StorageSiteDAO storageSiteDAO = new StorageSiteDAO(artifactDAO); Set sites = storageSiteDAO.list(); // this set could be cached @@ -337,7 +347,8 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) artifact = getUnsyncedArtifact(artifactURI, transfer, sites, authToken); } } - + log.debug(artifactURI + " found: " + artifact); + List storageSites = new ArrayList<>(); if (artifact != null) { if (artifact.storageLocation != null) { @@ -357,64 +368,70 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) } } } - - prioritizePullFromSites(storageSites); - for (StorageSite storageSite : storageSites) { + + List readableSites = prioritizePullFromSites(storageSites); + log.debug("pullFrom: known sites " + storageSites.size() + " -> readableSites " + readableSites.size()); + for (StorageSite storageSite : readableSites) { + log.debug("trying site: " + storageSite.getResourceID() + " allowRead=" + storageSite.getAllowRead()); Capability filesCap = getFilesCapability(storageSite); - if (filesCap != null) { + if (filesCap != null && storageSite.getAllowRead()) { for (Protocol proto : transfer.getProtocols()) { - if (storageSite.getAllowRead()) { - // less generic request for service that implements an API - // HACK: this is filesCap specific in here - if (proto.getUri().equals(filesCap.getStandardID())) { + log.debug("\tprotocol: " + proto); + // less generic request for service that implements an API + // HACK: this is filesCap specific in here + if (proto.getUri().equals(filesCap.getStandardID())) { + Protocol p = new Protocol(proto.getUri()); + p.setEndpoint(storageSite.getResourceID().toASCIIString()); + protos.add(p); + } + URI sec = proto.getSecurityMethod(); + if (sec == null) { + sec = Standards.SECURITY_METHOD_ANON; + } + Interface iface = filesCap.findInterface(sec); + if (iface != null) { + URL baseURL = iface.getAccessURL().getURL(); + log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); + if (protocolCompat(proto, baseURL)) { + StringBuilder sb = new StringBuilder(); + sb.append(baseURL.toExternalForm()).append("/"); + if (authToken != null && Standards.SECURITY_METHOD_ANON.equals(sec)) { + sb.append(authToken).append("/"); + } + sb.append(artifactURI.toASCIIString()); + if (filenameOverride != null) { + sb.append(":fo/").append(filenameOverride); + } Protocol p = new Protocol(proto.getUri()); - p.setEndpoint(storageSite.getResourceID().toASCIIString()); + if (transfer.version == VOS.VOSPACE_21) { + p.setSecurityMethod(proto.getSecurityMethod()); + } + p.setEndpoint(sb.toString()); protos.add(p); - } - URI sec = proto.getSecurityMethod(); - if (sec == null) { - sec = Standards.SECURITY_METHOD_ANON; - } - Interface iface = filesCap.findInterface(sec); - if (iface != null) { - URL baseURL = iface.getAccessURL().getURL(); - log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); - if (protocolCompat(proto, baseURL)) { - StringBuilder sb = new StringBuilder(); + log.debug("added: " + p); + + // add a plain anon URL + if (authToken != null && !requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) { + sb = new StringBuilder(); sb.append(baseURL.toExternalForm()).append("/"); - if (authToken != null && Standards.SECURITY_METHOD_ANON.equals(sec)) { - sb.append(authToken).append("/"); - } sb.append(artifactURI.toASCIIString()); - Protocol p = new Protocol(proto.getUri()); - if (transfer.version == VOS.VOSPACE_21) { - p.setSecurityMethod(proto.getSecurityMethod()); + p = new Protocol(proto.getUri()); + if (filenameOverride != null) { + sb.append(":fo/").append(filenameOverride); } p.setEndpoint(sb.toString()); protos.add(p); log.debug("added: " + p); - - // add a plain anon URL - if (authToken != null && !requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) { - sb = new StringBuilder(); - sb.append(baseURL.toExternalForm()).append("/"); - sb.append(artifactURI.toASCIIString()); - p = new Protocol(proto.getUri()); - p.setEndpoint(sb.toString()); - protos.add(p); - log.debug("added: " + p); - } - } else { - log.debug("reject protocol: " + proto - + " reason: no compatible URL protocol"); } } else { log.debug("reject protocol: " + proto - + " reason: unsupported security method: " + proto.getSecurityMethod()); + + " reason: no compatible URL protocol"); } } else { - log.debug("Storage not allowed read " + storageSite.getName()); + log.debug("reject protocol: " + proto + + " reason: unsupported security method: " + proto.getSecurityMethod()); } + } } } @@ -446,13 +463,16 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) return protos; } + // the algorithm for prioritizing storage sites to put file static SortedSet prioritizePushToSites(Set storageSites, URI artifactURI, Map siteRules) { PrioritizingStorageSiteComparator comparator = new PrioritizingStorageSiteComparator(siteRules, artifactURI, null); TreeSet orderedSet = new TreeSet<>(comparator); - for (StorageSite storageSite : storageSites) { - if (storageSite.getAllowWrite()) { - orderedSet.add(storageSite); + for (StorageSite s : storageSites) { + if (s.getAllowWrite()) { + orderedSet.add(s); + } else { + log.debug("storage site is not writable: " + s.getResourceID()); } } return orderedSet; @@ -464,8 +484,10 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT Set storageSites = storageSiteDAO.list(); // this set could be cached List protos = new ArrayList<>(); - SortedSet orderedSites = prioritizePushToSites(storageSites, artifactURI, this.siteRules); + // prioritize also filters out non-writable sites + Set orderedSites = prioritizePushToSites(storageSites, artifactURI, this.siteRules); // produce URLs for all writable sites + log.debug("pushTo: known sites " + storageSites.size() + " -> writableSites " + orderedSites.size()); for (StorageSite storageSite : orderedSites) { // check if site is currently offline if (!isAvailable(storageSite.getResourceID())) { @@ -473,7 +495,7 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT continue; } - //log.warn("PUT: " + storageSite); + log.debug("pushTo: trying site " + storageSite.getResourceID()); Capability filesCap = null; try { Capabilities caps = regClient.getCapabilities(storageSite.getResourceID()); @@ -486,52 +508,50 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT } if (filesCap != null) { for (Protocol proto : transfer.getProtocols()) { - //log.warn("PUT: " + storageSite + " proto: " + proto); - if (storageSite.getAllowWrite()) { - // less generic request for service that implements - // HACK: this is filesCap specific in here - if (proto.getUri().equals(filesCap.getStandardID())) { - Protocol p = new Protocol(proto.getUri()); - p.setEndpoint(storageSite.getResourceID().toASCIIString()); - protos.add(p); - } - URI sec = proto.getSecurityMethod(); - if (sec == null) { - sec = Standards.SECURITY_METHOD_ANON; - } - boolean anon = Standards.SECURITY_METHOD_ANON.equals(sec); - Interface iface = filesCap.findInterface(sec); - log.debug("PUT: " + storageSite + " proto: " + proto + " iface: " + iface); - if (iface != null) { - URL baseURL = iface.getAccessURL().getURL(); - //log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); - if (protocolCompat(proto, baseURL)) { - // // no plain anon URL for put: !anon or anon+token - boolean gen = (!anon || (anon && authToken != null)); - if (gen) { - StringBuilder sb = new StringBuilder(); - sb.append(baseURL.toExternalForm()).append("/"); - if (authToken != null && anon) { - sb.append(authToken).append("/"); - } - sb.append(artifactURI.toASCIIString()); - Protocol p = new Protocol(proto.getUri()); - if (transfer.version == VOS.VOSPACE_21) { - p.setSecurityMethod(proto.getSecurityMethod()); - } - p.setEndpoint(sb.toString()); - protos.add(p); - log.debug("added: " + p); + log.debug("pushTo: " + storageSite + " proto: " + proto); + // less generic request for service that implements + // HACK: this is filesCap specific in here + if (proto.getUri().equals(filesCap.getStandardID())) { + Protocol p = new Protocol(proto.getUri()); + p.setEndpoint(storageSite.getResourceID().toASCIIString()); + protos.add(p); + } + URI sec = proto.getSecurityMethod(); + if (sec == null) { + sec = Standards.SECURITY_METHOD_ANON; + } + boolean anon = Standards.SECURITY_METHOD_ANON.equals(sec); + Interface iface = filesCap.findInterface(sec); + log.debug("pushTo: " + storageSite + " proto: " + proto + " iface: " + iface); + if (iface != null) { + URL baseURL = iface.getAccessURL().getURL(); + //log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); + if (protocolCompat(proto, baseURL)) { + // // no plain anon URL for put: !anon or anon+token + boolean gen = (!anon || (anon && authToken != null)); + if (gen) { + StringBuilder sb = new StringBuilder(); + sb.append(baseURL.toExternalForm()).append("/"); + if (authToken != null && anon) { + sb.append(authToken).append("/"); + } + sb.append(artifactURI.toASCIIString()); + Protocol p = new Protocol(proto.getUri()); + if (transfer.version == VOS.VOSPACE_21) { + p.setSecurityMethod(proto.getSecurityMethod()); } - - } else { - log.debug("PUT: " + storageSite + "PUT: reject protocol: " + proto - + " reason: no compatible URL protocol"); + p.setEndpoint(sb.toString()); + protos.add(p); + log.debug("added: " + p); } + } else { log.debug("PUT: " + storageSite + "PUT: reject protocol: " + proto - + " reason: unsupported security method: " + proto.getSecurityMethod()); + + " reason: no compatible URL protocol"); } + } else { + log.debug("PUT: " + storageSite + "PUT: reject protocol: " + proto + + " reason: unsupported security method: " + proto.getSecurityMethod()); } } } diff --git a/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java b/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java index 2cc21f3f9..190aad14c 100644 --- a/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java +++ b/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java @@ -73,6 +73,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; @@ -99,10 +100,16 @@ public void testPrioritizePullFromSites() throws Exception { for (int i = 0; i < 10; i++) { sites.add(new StorageSite(URI.create("ivo://site" + i), "site1" + i, true, rd.nextBoolean())); } - ProtocolsGenerator.prioritizePullFromSites(sites); - for (StorageSite s : sites) { - log.info("found: " + s.getID() + " aka " + s.getResourceID()); - } + List result1 = ProtocolsGenerator.prioritizePullFromSites(sites); + Assert.assertEquals(sites.size(), result1.size()); + Assert.assertTrue(result1.containsAll(sites)); + + List result2 = ProtocolsGenerator.prioritizePullFromSites(sites); + Assert.assertEquals(sites.size(), result2.size()); + Assert.assertTrue(result2.containsAll(sites)); + + // test random order + Assert.assertNotEquals(result1, result2); } @Test diff --git a/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java b/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java index a302179a3..adc5065f6 100644 --- a/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java +++ b/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java @@ -384,6 +384,7 @@ public static void assertValidPathComponent(Class caller, String name, String te boolean slash = (test.indexOf('/') >= 0); boolean escape = (test.indexOf('\\') >= 0); boolean percent = (test.indexOf('%') >= 0); + boolean colon = (test.indexOf(":") >= 0); boolean semic = (test.indexOf(';') >= 0); boolean amp = (test.indexOf('&') >= 0); boolean dollar = (test.indexOf('$') >= 0); @@ -398,7 +399,7 @@ public static void assertValidPathComponent(Class caller, String name, String te } throw new IllegalArgumentException(s + name + ": " + test + " reason: path component may not contain space ( ), slash (/), escape (\\), percent (%)," - + " semi-colon (;), ampersand (&), or dollar ($), question (?), or square brackets ([])"); + + " colon (:), semi-colon (;), ampersand (&), dollar ($), question (?), or square brackets ([])"); } } diff --git a/minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java b/minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java index d13de5e76..36a6552fd 100644 --- a/minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java +++ b/minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java @@ -146,10 +146,12 @@ public void testPutGetUpdateHeadDelete() { long contentLength = get.getContentLength(); String contentType = get.getContentType(); String contentEncoding = get.getContentEncoding(); + String contentDisposition = get.getResponseHeader("content-disposition"); Assert.assertEquals(computeChecksumURI(data), checksumURI); Assert.assertEquals(data.length, contentLength); Assert.assertEquals(type, contentType); Assert.assertEquals(encoding, contentEncoding); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("file.txt")); Date lastModified = get.getLastModified(); Assert.assertNotNull(lastModified); @@ -181,10 +183,12 @@ public void testPutGetUpdateHeadDelete() { contentLength = head.getContentLength(); contentType = head.getContentType(); contentEncoding = head.getContentEncoding(); + contentDisposition = head.getResponseHeader("content-disposition"); Assert.assertEquals(computeChecksumURI(data), checksumURI); Assert.assertEquals(data.length, contentLength); Assert.assertEquals(newType, contentType); Assert.assertEquals(newEncoding, contentEncoding); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("file.txt")); lastModified = head.getLastModified(); Assert.assertNotNull(lastModified); @@ -331,6 +335,114 @@ public void testGetRanges() { } } + @Test + public void testFilenameOverride() { + try { + URI artifactURI = URI.create("cadc:TEST/testFilenameOverride.txt"); + URL artifactURL = new URL(filesURL + "/" + artifactURI.toString()); + + String content = "abcdefghijklmnopqrstuvwxyz"; + String encoding = "test-encoding"; + String type = "text/plain"; + byte[] data = content.getBytes(); + URI expectedChecksum = computeChecksumURI(data); + + // put: no length or checksum + InputStream in = new ByteArrayInputStream(data); + HttpUpload put = new HttpUpload(in, artifactURL); + put.setRequestProperty(HttpTransfer.CONTENT_TYPE, type); + put.setRequestProperty(HttpTransfer.CONTENT_ENCODING, encoding); + put.setDigest(expectedChecksum); + + Subject.doAs(userSubject, new RunnableAction(put)); + log.info("put: " + put.getResponseCode() + " " + put.getThrowable()); + log.info("headers: " + put.getResponseHeader("content-length") + " " + put.getResponseHeader("digest")); + Assert.assertNull(put.getThrowable()); + Assert.assertEquals("Created", 201, put.getResponseCode()); + + // head + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet head = new HttpGet(artifactURL, bos); + head.setHeadOnly(true); + log.info("head: " + artifactURL.toExternalForm()); + Subject.doAs(userSubject, new RunnableAction(head)); + log.info("head: " + head.getResponseCode() + " " + head.getThrowable()); + log.info("headers: " + head.getResponseHeader("content-length") + " " + head.getResponseHeader("digest")); + log.warn("head output: " + bos.toString()); + Assert.assertNull(head.getThrowable()); + URI checksumURI = head.getDigest(); + long contentLength = head.getContentLength(); + String contentType = head.getContentType(); + String contentEncoding = head.getContentEncoding(); + String contentDisposition = head.getResponseHeader("content-disposition"); + Assert.assertEquals(computeChecksumURI(data), checksumURI); + Assert.assertEquals(data.length, contentLength); + Assert.assertEquals(type, contentType); + Assert.assertEquals(encoding, contentEncoding); + log.info("content-disposition: " + contentDisposition); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("testFilenameOverride.txt")); + Date lastModified = head.getLastModified(); + Assert.assertNotNull(lastModified); + + URL foURL = new URL(artifactURL.toExternalForm() + ":fo/alternate.txt"); + head = new HttpGet(foURL, bos); + head.setHeadOnly(true); + log.info("head: " + foURL.toExternalForm()); + Subject.doAs(userSubject, new RunnableAction(head)); + log.info("head: " + head.getResponseCode() + " " + head.getThrowable()); + log.info("headers: " + head.getResponseHeader("content-length") + " " + head.getResponseHeader("digest")); + log.warn("head output: " + bos.toString()); + Assert.assertNull(head.getThrowable()); + checksumURI = head.getDigest(); + contentLength = head.getContentLength(); + contentType = head.getContentType(); + contentEncoding = head.getContentEncoding(); + contentDisposition = head.getResponseHeader("content-disposition"); + Assert.assertEquals(computeChecksumURI(data), checksumURI); + Assert.assertEquals(data.length, contentLength); + Assert.assertEquals(type, contentType); + Assert.assertEquals(encoding, contentEncoding); + log.info("content-disposition: " + contentDisposition); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("alternate.txt")); + Date lastModified2 = head.getLastModified(); + Assert.assertEquals(lastModified, lastModified2); + + // get + bos = new ByteArrayOutputStream(); + log.info("get: " + foURL.toExternalForm()); + HttpGet get = new HttpGet(foURL, bos); + Subject.doAs(userSubject, new RunnableAction(get)); + log.info("get: " + get.getResponseCode() + " " + get.getThrowable()); + log.info("headers: " + get.getResponseHeader("content-length") + " " + get.getResponseHeader("digest")); + log.warn("get output: " + bos.toString()); + Assert.assertNull(get.getThrowable()); + checksumURI = get.getDigest(); + contentLength = get.getContentLength(); + contentType = get.getContentType(); + contentEncoding = get.getContentEncoding(); + contentDisposition = get.getResponseHeader("content-disposition"); + Assert.assertEquals(computeChecksumURI(data), checksumURI); + Assert.assertEquals(data.length, contentLength); + Assert.assertEquals(type, contentType); + Assert.assertEquals(encoding, contentEncoding); + log.info("content-disposition: " + contentDisposition); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("alternate.txt")); + Date lastModified3 = get.getLastModified(); + Assert.assertEquals(lastModified, lastModified3); + + // delete + HttpDelete delete = new HttpDelete(artifactURL, false); + Subject.doAs(userSubject, new RunnableAction(delete)); + log.info("delete: " + delete.getResponseCode() + " " + delete.getThrowable()); + Assert.assertNull(delete.getThrowable()); + Assert.assertEquals("no content", 204, delete.getResponseCode()); + + } catch (Exception t) { + log.error("unexpected throwable", t); + Assert.fail("unexpected throwable: " + t); + } + } + @Test public void testGetNotFound() { try { diff --git a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java index 5cb3aa59e..7e51264cf 100644 --- a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -116,6 +116,11 @@ public abstract class ArtifactAction extends RestAction { // The target artifact URI artifactURI; + String errMsg; + + // alternmate filename for content-disposition header, usually null + boolean extractFilenameOverride = false; + String filenameOverride; // The (possibly null) authentication token. String authToken; @@ -260,6 +265,10 @@ protected void initAndAuthorize(Class grantClass, boolean allow void init() { if (this.artifactURI == null) { + if (errMsg != null) { + throw new IllegalArgumentException(errMsg); + } + // generic throw new IllegalArgumentException("missing or invalid artifact URI"); } } @@ -285,16 +294,29 @@ void parsePath() { String path = this.syncInput.getPath(); log.debug("path: " + path); if (path != null) { - int colonIndex = path.indexOf(":"); - int firstSlashIndex = path.indexOf("/"); - if (colonIndex != -1) { - if (firstSlashIndex < 0 || firstSlashIndex > colonIndex) { - // no auth token--artifact URI is complete path - this.artifactURI = createArtifactURI(path); - } else { - this.artifactURI = createArtifactURI(path.substring(firstSlashIndex + 1)); - this.authToken = path.substring(0, firstSlashIndex); - log.debug("authToken: " + this.authToken); + int colon1 = path.indexOf(":"); + int slash1 = path.indexOf("/"); + if (colon1 != -1) { + if (slash1 >= 0 && slash1 < colon1) { + // auth token in front + this.authToken = path.substring(0, slash1); + path = path.substring(slash1 + 1); + } + try { + int foi = path.indexOf(":fo/"); + if (foi > 0 && extractFilenameOverride) { + // filename override appended + this.filenameOverride = path.substring(foi + 4); + path = path.substring(0, foi); + } else if (foi > 0) { + throw new IllegalArgumentException("detected misuse of :fo/ filename override"); + } + URI auri = new URI(path); + InventoryUtil.validateArtifactURI(ArtifactAction.class, auri); + this.artifactURI = auri; + } catch (URISyntaxException | IllegalArgumentException e) { + this.errMsg = "illegal artifact URI: " + path + " reason: " + e.getMessage(); + log.debug(errMsg, e); } } } @@ -307,22 +329,4 @@ Artifact getArtifact(URI artifactURI) throws ResourceNotFoundException { } return artifact; } - - /** - * Create a valid artifact uri. - * @param uri The input string. - * @return The artifact uri object. - */ - private URI createArtifactURI(String uri) { - log.debug("artifact URI: " + uri); - URI ret; - try { - ret = new URI(uri); - InventoryUtil.validateArtifactURI(ArtifactAction.class, ret); - } catch (URISyntaxException | IllegalArgumentException e) { - ret = null; - log.debug("illegal artifact URI: " + uri, e); - } - return ret; - } } diff --git a/minoc/src/main/java/org/opencadc/minoc/GetAction.java b/minoc/src/main/java/org/opencadc/minoc/GetAction.java index a74e227de..e2df6428b 100644 --- a/minoc/src/main/java/org/opencadc/minoc/GetAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/GetAction.java @@ -124,6 +124,7 @@ public class GetAction extends ArtifactAction { // constructor for unit tests with no config/init GetAction(boolean init) { super(init); + this.extractFilenameOverride = true; } /** @@ -131,6 +132,7 @@ public class GetAction extends ArtifactAction { */ public GetAction() { super(); + this.extractFilenameOverride = true; } /** @@ -194,7 +196,7 @@ public void doAction() throws Exception { } // default: complete download - HeadAction.setHeaders(artifact, syncOutput); + HeadAction.setHeaders(artifact, filenameOverride, syncOutput); bcos = new ByteCountOutputStream(syncOutput.getOutputStream()); // create tmp StorageLocation with expected checksum so adapter can potentially @@ -238,7 +240,7 @@ public void doAction() throws Exception { private ByteCountOutputStream doByteRangeRequest(Artifact artifact, ByteRange byteRange) throws InterruptedException, IOException, ResourceNotFoundException, ReadException, WriteException, StorageEngageException, TransientException { - HeadAction.setHeaders(artifact, syncOutput); + HeadAction.setHeaders(artifact, filenameOverride, syncOutput); syncOutput.setCode(206); long lastByte = byteRange.getOffset() + byteRange.getLength() - 1; syncOutput.setHeader(CONTENT_RANGE, "bytes " + byteRange.getOffset() + "-" diff --git a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java index 5e16d17a4..af7ed41c3 100644 --- a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java @@ -93,6 +93,7 @@ public class HeadAction extends ArtifactAction { */ public HeadAction() { super(); + this.extractFilenameOverride = true; } /** @@ -131,7 +132,7 @@ public void doAction() throws Exception { artifact = getArtifact(artifactURI); } if (artifact != null) { - setHeaders(artifact, syncOutput); + setHeaders(artifact, filenameOverride, syncOutput); } } @@ -140,7 +141,7 @@ public void doAction() throws Exception { * @param artifact The artifact with metadata * @param syncOutput The target response */ - static void setHeaders(Artifact artifact, SyncOutput syncOutput) { + static void setHeaders(Artifact artifact, String filenameOverride, SyncOutput syncOutput) { syncOutput.setHeader(ARTIFACT_ID_HDR, artifact.getID().toString()); syncOutput.setDigest(artifact.getContentChecksum()); syncOutput.setLastModified(artifact.getContentLastModified()); @@ -149,7 +150,10 @@ static void setHeaders(Artifact artifact, SyncOutput syncOutput) { DateFormat df = DateUtil.getDateFormat(DateUtil.HTTP_DATE_FORMAT, DateUtil.GMT); syncOutput.setHeader("Last-Modified", df.format(artifact.getContentLastModified())); - String filename = InventoryUtil.computeArtifactFilename(artifact.getURI()); + String filename = filenameOverride; + if (filename == null) { + filename = InventoryUtil.computeArtifactFilename(artifact.getURI()); + } syncOutput.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); if (artifact.contentEncoding != null) { diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java index 03274739f..86304dcc2 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java @@ -196,9 +196,13 @@ private void initStorageSite() { if (name.charAt(0) == '/') { name = name.substring(1); } + + // possibly temporary hack: advertise readable and writable if this service + // is configured to accept preauth tokens + boolean trustPreauth = !config.getTrustedServices().isEmpty(); - boolean allowRead = !config.getReadGrantServices().isEmpty(); - boolean allowWrite = !config.getWriteGrantServices().isEmpty(); + boolean allowRead = trustPreauth || !config.getReadGrantServices().isEmpty(); + boolean allowWrite = trustPreauth || !config.getWriteGrantServices().isEmpty(); StorageSite self = null; if (curlist.isEmpty()) { diff --git a/minoc/src/main/java/org/opencadc/minoc/PostAction.java b/minoc/src/main/java/org/opencadc/minoc/PostAction.java index 38a07ce63..e6a6cb7eb 100644 --- a/minoc/src/main/java/org/opencadc/minoc/PostAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/PostAction.java @@ -193,7 +193,7 @@ public void doAction() throws Exception { log.debug("commit txn: OK"); syncOutput.setCode(202); // Accepted - HeadAction.setHeaders(existing, syncOutput); + HeadAction.setHeaders(existing, null, syncOutput); syncOutput.setHeader("content-length", 0); } catch (Exception e) { log.error("failed to persist " + artifactURI, e); diff --git a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java index 8b38e6f50..6a34cc19f 100644 --- a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java +++ b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java @@ -122,10 +122,19 @@ public void doAction() throws Exception { } private void assertCorrectPath(String path, String expURI, String expToken) { + assertCorrectPath(path, expURI, expToken, null); + } + + private void assertCorrectPath(String path, String expURI, String expToken, String expFilenameOverride) { ArtifactAction action = new TestArtifactAction(path); + if (expFilenameOverride != null) { + action.extractFilenameOverride = true; + } action.parsePath(); + log.info(path + " -> " + action.artifactURI + " - " + action.authToken + " - " + action.filenameOverride); Assert.assertEquals("artifactURI", URI.create(expURI), action.artifactURI); Assert.assertEquals("authToken", expToken, action.authToken); + Assert.assertEquals("filenameOverride", expFilenameOverride, action.filenameOverride); if (action.artifactURI == null) { Assert.fail("Failed to parse legal path: " + path); } @@ -134,9 +143,7 @@ private void assertCorrectPath(String path, String expURI, String expToken) { private void assertIllegalPath(String path) { ArtifactAction action = new TestArtifactAction(path); action.parsePath(); - if (action.artifactURI != null) { - Assert.fail("Should have failed to parse path: " + path); - } + Assert.assertNull(action.artifactURI); } @Test @@ -147,10 +154,16 @@ public void testParsePath() { assertCorrectPath("token/cadc:TEST/myartifact", "cadc:TEST/myartifact", "token"); assertCorrectPath("cadc:TEST/myartifact", "cadc:TEST/myartifact", null); assertCorrectPath("token/cadc:TEST/myartifact", "cadc:TEST/myartifact", "token"); - assertCorrectPath("mast:long/uri/with/segments/fits.fits", "mast:long/uri/with/segments/fits.fits", null); + assertCorrectPath("mast:long/uri/with/segments/something.fits", "mast:long/uri/with/segments/something.fits", null); assertCorrectPath("token/mast:long/uri/with/segments/fits.fits", "mast:long/uri/with/segments/fits.fits", "token"); assertCorrectPath("token-with-dashes/cadc:TEST/myartifact", "cadc:TEST/myartifact", "token-with-dashes"); + assertCorrectPath("cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", null, "something.fits"); + assertCorrectPath("token/cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", "token", "something.fits"); + + assertCorrectPath("cadc:vault/uuid:/something.fits", "cadc:vault/uuid:/something.fits", null, null); + + assertIllegalPath(null); assertIllegalPath(""); assertIllegalPath("noschemeinuri"); assertIllegalPath("token/noschemeinuri"); @@ -161,9 +174,6 @@ public void testParsePath() { assertIllegalPath("cadc://:port/path"); assertIllegalPath("artifacts/token1/token2/cadc:FOO/bar"); assertIllegalPath("artifacts/token/cadc:ccda:FOO/bar"); - - assertIllegalPath(null); - } catch (Exception unexpected) { log.error("unexpected exception", unexpected); Assert.fail("unexpected exception: " + unexpected); diff --git a/vault/README.md b/vault/README.md index 081fbefa7..02e701eab 100644 --- a/vault/README.md +++ b/vault/README.md @@ -62,8 +62,9 @@ in the JDBC URL and the schema name is specified in the minoc.properties (below) initialize the database will show up in logs and in the VOSI-availability output. The _inventory_ content may be in the same database as the _nodes_, in a different database in the same server, or in a different server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. Note: it is a good -idea to set `maxActive` to a valid integer (e.g. 0) when using a single pool; this avoids an ugly but -meaningless stack trace in the logs at startup. +idea to set `maxActive` to a valid integer (e.g. 1 because the tomcat connection pool doesn't like 0 and +decides to make it 100 instead) when using a single pool; this avoids an ugly but meaningless stack trace +in the logs at startup. The _uws_ account owns and manages (create, alter, drop) uws database objects in the `uws` schema and manages all the content (insert, update, delete). The database is specified in the JDBC URLFailure to connect or initialize the @@ -101,20 +102,19 @@ _all known_ sites. It only makes sense to enable this when `vault` is running in `raven` and/or `fenwick` instances syncing artifact metadata. This feature introduces an overhead for the genuine not-found cases: transfer negotiation to GET the file that was never PUT. -The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The -account nominally requires read-only (select) permission on those objects. This currently must be "inventory" due -to configuration limitations in luskan. +The _inventory.schema_ name is the name of the database schema used for all inventory database objects. This +currently must be "inventory" due to configuration limitations in luskan. -The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must currently be in the same database. -TODO: augment config to support separate inventory and vospace pools. +The _vospace.schema_ name is the name of the database schema used for all vospace database objects. Note that +with a single connection pool, the two schemas must be in the same database. The _singlePool_ key configures `vault` to use a single pool (the _nodes_ pool) for both vospace and inventory operations. The inventory and vospace content must be in the same database for this to work. When configured to use a single pool, delete node operations can delete a DataNode and the associated Artifact and create the DeletedArtifactEvent in a single transaction. When configured to use separate pools, the delete Artifact and create DeletedArtifactEvent are done in a separate transaction and if that fails the Artifact will be left behind and -orphaned until the vault validation (see ???) runs and fixes such a discrepancy. However, _singlePool_ = `false` allows -the content to be stored in two separate databases or servers. +orphaned until the vault validation (see ???) runs and fixes such a discrepancy. However, _singlePool_ = `false` +allows the content to be stored in two separate databases or servers. The _root.owner_ owns the root node and has full read and write permission in the root container, so it can create and delete container nodes at the root and assign container node properties that are normally read-only diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 0e352fcc1..0669c1055 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -239,7 +239,7 @@ private NodeDAO getDAO() { private ArtifactDAO getArtifactDAO() { ArtifactDAO instance = new ArtifactDAO(true); // origin==true? - instance.setConfig(nodeDaoConfig); + instance.setConfig(invDaoConfig); return instance; } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 2f8625d36..c253e55fb 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -344,7 +344,7 @@ private void initNodePersistence() { private void initKeyPair() { log.info("initKeyPair: START"); - //jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); + jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); try { PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); dao.setConfig(getKeyPairConfig(props)); @@ -368,7 +368,6 @@ private void initKeyPair() { } else { log.info("initKeyPair: re-use existing keys - OK"); } - /* Context ctx = new InitialContext(); try { ctx.unbind(jndiPreauthKeys); @@ -380,7 +379,6 @@ private void initKeyPair() { Object o = ctx.lookup(jndiPreauthKeys); log.info("checking... found: " + jndiPreauthKeys + " = " + o + " in " + ctx); - */ } catch (Exception ex) { throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", ex); } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index b75d9ffd8..cb3ed97a0 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -74,7 +74,6 @@ import ca.nrc.cadc.uws.Job; import ca.nrc.cadc.uws.Parameter; import ca.nrc.cadc.vosi.Availability; -import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.util.ArrayList; @@ -91,9 +90,7 @@ import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; import org.opencadc.permissions.TokenTool; -import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.LinkingException; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotFoundException; import org.opencadc.vospace.VOSURI; @@ -155,34 +152,27 @@ public List getEndpoints(VOSURI target, Transfer transfer, Job job, Li List ret = null; try { Direction dir = transfer.getDirection(); - PathResolver ps = new PathResolver(nodePersistence, authorizer, true); - Node n = ps.getNode(target.getParentURI().getPath()); - // assume not null and Container already checked by caller (TransferRunner) - ContainerNode parent = (ContainerNode) n; - Node node = nodePersistence.get(parent, target.getName()); + PathResolver ps = new PathResolver(nodePersistence, authorizer); + Node node = ps.getNode(target.getPath(), true); + if (node == null) { + throw new NodeNotFoundException(target.getPath()); + } Subject currentSubject = AuthenticationUtil.getCurrentSubject(); - if (Direction.pushToVoSpace.equals(dir) && node == null) { - // create new data node?? this currently does not happen because the library - // create DataNode the way that CreateNodeAction would - //ret = handleDataNode(dn, transfer, currentSubject); - throw new RuntimeException("BUG: expected DataNode to be created already: " + target.getPath()); - } else if (node instanceof DataNode) { + if (node instanceof DataNode) { DataNode dn = (DataNode) node; - ret = handleDataNode(dn, transfer, currentSubject); + ret = handleDataNode(dn, target.getName(), transfer, currentSubject); } else { - throw new UnsupportedOperationException(node.getClass().getSimpleName() + " transfer " - + target.getPath()); + throw new UnsupportedOperationException("transfer: " + node.getClass().getSimpleName() + + " at " + target.getPath()); } - } catch (NodeNotFoundException ex) { - throw new FileNotFoundException(target.getPath()); - } catch (LinkingException ex) { - throw new RuntimeException("OOPS: failed to resolve link?", ex); + } finally { + // nothing right now } return ret; } - private List handleDataNode(DataNode node, Transfer trans, Subject s) + private List handleDataNode(DataNode node, String filename, Transfer trans, Subject s) throws IOException, ResourceNotFoundException { log.debug("handleDataNode: " + node); @@ -203,11 +193,12 @@ private List handleDataNode(DataNode node, Transfer trans, Subject s) URI secM = p.getSecurityMethod(); if (secM == null || secM.equals(Standards.SECURITY_METHOD_ANON)) { artifactTrans.getProtocols().add(p); + log.debug("allow protocol: " + p); } } try { - List ret = pg.getProtocols(artifactTrans); + List ret = pg.getProtocols(artifactTrans, filename); log.debug("generated urls: " + ret.size()); for (Protocol p : ret) { log.debug(p.getEndpoint() + " using " + p.getSecurityMethod());