From dda67d4ae307f27407772f6e4971620d7d6b964c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 3 Oct 2023 13:30:02 -0700 Subject: [PATCH 1/2] raven: implement prototype negotiate for resourceIDs that implement files API --- raven/VERSION | 2 +- .../org/opencadc/raven/NegotiationTest.java | 91 +++++++++++++++++++ .../opencadc/raven/ProtocolsGenerator.java | 14 +++ 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/raven/VERSION b/raven/VERSION index 3765a6c6..bd72e5cf 100644 --- a/raven/VERSION +++ b/raven/VERSION @@ -2,6 +2,6 @@ # semantic version tag: major.minor[.patch] # build version tag: timestamp # tag: {semantic}-{build} -VER=0.7.8 +VER=0.7.9 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java index fd3c3723..d50c70d1 100644 --- a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java @@ -743,4 +743,95 @@ public Object run() throws Exception { } + @Test + public void testGetSites() throws Exception { + // request the raw ivo resourceIDs of storage sites (files services) that + // can deliver the file + List requested = new ArrayList<>(); + + Protocol files = new Protocol(Standards.SI_FILES.toASCIIString()); + requested.add(files); + + try { + Subject.doAs(userSubject, new PrivilegedExceptionAction() { + public Object run() throws Exception { + + URI resourceID1 = URI.create("ivo://negotiation-test-site1"); + URI resourceID2 = URI.create("ivo://negotiation-test-site2"); + + + StorageSite site1 = new StorageSite(resourceID1, "site1", true, true); + StorageSite site2 = new StorageSite(resourceID2, "site2", true, true); + + URI artifactURI = URI.create("cadc:TEST/" + UUID.randomUUID() + ".fits"); + URI checksum = URI.create("md5:d41d8cd98f00b204e9800998ecf8427e"); + Artifact artifact = new Artifact(artifactURI, checksum, new Date(), 1L); + + try { + siteDAO.put(site1); + siteDAO.put(site2); + + final SiteLocation location1 = new SiteLocation(site1.getID()); + final SiteLocation location2 = new SiteLocation(site2.getID()); + + Transfer transfer = new Transfer(artifactURI, Direction.pullFromVoSpace); + transfer.getProtocols().add(files); + transfer.version = VOS.VOSPACE_21; + + artifactDAO.put(artifact); + + // test that there are no copies available + try { + negotiate(transfer); + Assert.fail("should have received file not found exception"); + } catch (ResourceNotFoundException e) { + log.info("caught expected: " + e); + } + + log.info("add: " + location1); + artifactDAO.addSiteLocation(artifact, location1); + artifact = artifactDAO.get(artifact.getID()); + + // test that there's one copy + Transfer response = negotiate(transfer); + log.info("transfer: " + response); + Assert.assertEquals(1, response.getProtocols().size()); + for (Protocol ap : response.getProtocols()) { + log.info("found: " + artifactURI + " -> " + ap.getEndpoint()); + } + Assert.assertTrue(response.getAllEndpoints().contains(resourceID1.toASCIIString())); + + Protocol actual = response.getProtocols().get(0); + Assert.assertNotNull(actual.getEndpoint()); + Assert.assertEquals(files.getUri(), actual.getUri()); + Assert.assertEquals(resourceID1.toASCIIString(), actual.getEndpoint()); + + log.info("add: " + location2); + artifactDAO.addSiteLocation(artifact, location2); + artifact = artifactDAO.get(artifact.getID()); + + // test that there are now two copies + response = negotiate(transfer); + Assert.assertEquals(2, response.getProtocols().size()); + for (Protocol ap : response.getProtocols()) { + log.info("found: " + artifactURI + " -> " + ap.getEndpoint()); + } + Assert.assertTrue(response.getAllEndpoints().contains(resourceID1.toASCIIString())); + Assert.assertTrue(response.getAllEndpoints().contains(resourceID2.toASCIIString())); + + return null; + + } finally { + // cleanup sites + siteDAO.delete(site1.getID()); + siteDAO.delete(site2.getID()); + artifactDAO.delete(artifact.getID()); + } + } + }); + } catch (Exception e) { + log.error("unexpected exception", e); + Assert.fail("unexpected exception: " + e); + } + } } diff --git a/raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java b/raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java index 9e640815..2c4aa7e9 100644 --- a/raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java +++ b/raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java @@ -325,6 +325,13 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) if (filesCap != null) { for (Protocol proto : transfer.getProtocols()) { if (storageSite.getAllowRead()) { + // less generic request for service that implements an API + // HACK: this is filesCap specific in here + if (proto.getUri().equals(filesCap.getStandardID().toASCIIString())) { + Protocol p = new Protocol(proto.getUri()); + p.setEndpoint(storageSite.getResourceID().toASCIIString()); + protos.add(p); + } URI sec = proto.getSecurityMethod(); if (sec == null) { sec = Standards.SECURITY_METHOD_ANON; @@ -444,6 +451,13 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT for (Protocol proto : transfer.getProtocols()) { //log.warn("PUT: " + storageSite + " proto: " + proto); if (storageSite.getAllowWrite()) { + // less generic request for service that implements + // HACK: this is filesCap specific in here + if (proto.getUri().equals(filesCap.getStandardID().toASCIIString())) { + Protocol p = new Protocol(proto.getUri()); + p.setEndpoint(storageSite.getResourceID().toASCIIString()); + protos.add(p); + } URI sec = proto.getSecurityMethod(); if (sec == null) { sec = Standards.SECURITY_METHOD_ANON; From dec5ed8b358195a1576a65d69ea66116cb3870af Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Fri, 6 Oct 2023 16:06:05 -0700 Subject: [PATCH 2/2] Support for recursive deletes --- vault/build.gradle | 6 ++-- .../java/org/opencadc/vault/NodesTest.java | 2 ++ .../org/opencadc/vault/VaultInitAction.java | 17 +++++++++++ vault/src/main/webapp/META-INF/context.xml | 13 ++++++++ vault/src/main/webapp/WEB-INF/web.xml | 30 +++++++++++++++++++ vault/src/main/webapp/capabilities.xml | 10 +++++++ 6 files changed, 75 insertions(+), 3 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index 86b736b0..7fd06925 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -30,7 +30,7 @@ def git_url = 'https://github.com/opencadc/vos' dependencies { compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' - compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-util:[1.9.10,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' @@ -38,10 +38,10 @@ dependencies { compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' - compile 'org.opencadc:cadc-uws-server:[1.2.12,)' + compile 'org.opencadc:cadc-uws-server:[1.2.19,)' compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' compile 'org.opencadc:cadc-cdp:[1.2.3,)' - compile 'org.opencadc:cadc-registry:[1.5.15,)' + compile 'org.opencadc:cadc-registry:[1.7.4,)' compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java index 71bb1178..2a4a317d 100644 --- a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -71,6 +71,7 @@ import java.net.URI; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; /** * Test the nodes endpoint. @@ -86,6 +87,7 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest { } public NodesTest() { + //super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem", new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?CADC_TEST_GROUP2")), "vault-test-auth.pem"); super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem"); } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index bc9dc89f..c22083be 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -71,6 +71,7 @@ import ca.nrc.cadc.rest.InitAction; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; +import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -93,6 +94,7 @@ public class VaultInitAction extends InitAction { private static final Logger log = Logger.getLogger(VaultInitAction.class); static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml + static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml // config keys private static final String VAULT_KEY = "org.opencadc.vault"; @@ -119,6 +121,7 @@ public VaultInitAction() { public void doInit() { initConfig(); initDatabase(); + initUWSDatabase(); initNodePersistence(); } @@ -215,6 +218,20 @@ private void initDatabase() { } } + private void initUWSDatabase() { + log.info("initUWSDatabase: START"); + try { + // Init UWS database + DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE); + InitDatabaseUWS uwsi = new InitDatabaseUWS(uws, null, "uws"); + uwsi.doInit(); + log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws OK"); + + } catch (Exception ex) { + throw new RuntimeException("check/init uws database failed", ex); + } + } + protected void initNodePersistence() { jndiNodePersistence = componentID + ".nodePersistence"; try { diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index cbc72f2e..e5775723 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -15,4 +15,17 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> + + + diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index af0394c1..90476792 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -23,6 +23,7 @@ ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi + ca.nrc.cadc.uws @@ -62,6 +63,30 @@ 2 + + + RecursiveDeleteNodeServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.RecursiveDeleteNodeJobManager + + 3 + + + CapabilitiesServlet @@ -107,6 +132,11 @@ /nodes/* + + RecursiveDeleteNodeServlet + /recursiveDelete/* + + AvailabilityServlet diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index fb145630..d24f1a22 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -41,6 +41,16 @@ + + + https://replace.me.com/vault/recursiveDelete + + + + + + +