Support for recursive node deletes (CADC-12558)

raven/VERSION

@@ -2,6 +2,6 @@
 # semantic version tag: major.minor[.patch]
 # build version tag: timestamp
 # tag: {semantic}-{build}
-VER=0.7.8
+VER=0.7.9
 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")"
 unset VER 

raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java

@@ -743,4 +743,95 @@ public Object run() throws Exception {
 
     }
 
+    @Test
+    public void testGetSites() throws Exception {
+        // request the raw ivo resourceIDs of storage sites (files services) that
+        // can deliver the file
+        List<Protocol> requested = new ArrayList<>();
+
+        Protocol files = new Protocol(Standards.SI_FILES.toASCIIString());
+        requested.add(files);
+
+        try {
+            Subject.doAs(userSubject, new PrivilegedExceptionAction<Object>() {
+                public Object run() throws Exception {
+
+                    URI resourceID1 = URI.create("ivo://negotiation-test-site1");
+                    URI resourceID2 = URI.create("ivo://negotiation-test-site2");
+
+
+                    StorageSite site1 = new StorageSite(resourceID1, "site1", true, true);
+                    StorageSite site2 = new StorageSite(resourceID2, "site2", true, true);
+
+                    URI artifactURI = URI.create("cadc:TEST/" + UUID.randomUUID() + ".fits");
+                    URI checksum = URI.create("md5:d41d8cd98f00b204e9800998ecf8427e");
+                    Artifact artifact = new Artifact(artifactURI, checksum, new Date(), 1L);
+
+                    try {
+                        siteDAO.put(site1);
+                        siteDAO.put(site2);
+
+                        final SiteLocation location1 = new SiteLocation(site1.getID());
+                        final SiteLocation location2 = new SiteLocation(site2.getID());
+
+                        Transfer transfer = new Transfer(artifactURI, Direction.pullFromVoSpace);
+                        transfer.getProtocols().add(files);
+                        transfer.version = VOS.VOSPACE_21;
+
+                        artifactDAO.put(artifact);
+
+                        // test that there are no copies available
+                        try {
+                            negotiate(transfer);
+                            Assert.fail("should have received file not found exception");
+                        } catch (ResourceNotFoundException e) {
+                            log.info("caught expected: " + e);
+                        }
+
+                        log.info("add: " + location1);
+                        artifactDAO.addSiteLocation(artifact, location1);
+                        artifact = artifactDAO.get(artifact.getID());
+
+                        // test that there's one copy
+                        Transfer response = negotiate(transfer);
+                        log.info("transfer: " + response);
+                        Assert.assertEquals(1, response.getProtocols().size());
+                        for (Protocol ap : response.getProtocols()) {
+                            log.info("found: " + artifactURI + " -> " + ap.getEndpoint());
+                        }
+                        Assert.assertTrue(response.getAllEndpoints().contains(resourceID1.toASCIIString()));
+
+                        Protocol actual = response.getProtocols().get(0);
+                        Assert.assertNotNull(actual.getEndpoint());
+                        Assert.assertEquals(files.getUri(), actual.getUri());
+                        Assert.assertEquals(resourceID1.toASCIIString(), actual.getEndpoint());
+
+                        log.info("add: " + location2);
+                        artifactDAO.addSiteLocation(artifact, location2);
+                        artifact = artifactDAO.get(artifact.getID());
+
+                        // test that there are now two copies
+                        response = negotiate(transfer);
+                        Assert.assertEquals(2, response.getProtocols().size());
+                        for (Protocol ap : response.getProtocols()) {
+                            log.info("found: " + artifactURI + " -> " + ap.getEndpoint());
+                        }
+                        Assert.assertTrue(response.getAllEndpoints().contains(resourceID1.toASCIIString()));
+                        Assert.assertTrue(response.getAllEndpoints().contains(resourceID2.toASCIIString()));
+
+                        return null;
+
+                    } finally {
+                        // cleanup sites
+                        siteDAO.delete(site1.getID());
+                        siteDAO.delete(site2.getID());
+                        artifactDAO.delete(artifact.getID());
+                    }
+                }
+            });
+        } catch (Exception e) {
+            log.error("unexpected exception", e);
+            Assert.fail("unexpected exception: " + e);
+        }
+    }
 }

raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java

@@ -325,6 +325,13 @@ List<Protocol> doPullFrom(URI artifactURI, Transfer transfer, String authToken)
             if (filesCap != null) {
                 for (Protocol proto : transfer.getProtocols()) {
                     if (storageSite.getAllowRead()) {
+                        // less generic request for service that implements an API
+                        // HACK: this is filesCap specific in here
+                        if (proto.getUri().equals(filesCap.getStandardID().toASCIIString())) {
+                            Protocol p = new Protocol(proto.getUri());
+                            p.setEndpoint(storageSite.getResourceID().toASCIIString());
+                            protos.add(p);
+                        }
                         URI sec = proto.getSecurityMethod();
                         if (sec == null) {
                             sec = Standards.SECURITY_METHOD_ANON;
@@ -444,6 +451,13 @@ private List<Protocol> doPushTo(URI artifactURI, Transfer transfer, String authT
                 for (Protocol proto : transfer.getProtocols()) {
                     //log.warn("PUT: " + storageSite + " proto: " + proto);
                     if (storageSite.getAllowWrite()) {
+                        // less generic request for service that implements
+                        // HACK: this is filesCap specific in here
+                        if (proto.getUri().equals(filesCap.getStandardID().toASCIIString())) {
+                            Protocol p = new Protocol(proto.getUri());
+                            p.setEndpoint(storageSite.getResourceID().toASCIIString());
+                            protos.add(p);
+                        }
                         URI sec = proto.getSecurityMethod();
                         if (sec == null) {
                             sec = Standards.SECURITY_METHOD_ANON;

vault/build.gradle

@@ -30,18 +30,18 @@ def git_url = 'https://github.com/opencadc/vos'
 dependencies {
     compile 'javax.servlet:javax.servlet-api:[3.1,4.0)'
 
-    compile 'org.opencadc:cadc-util:[1.9.5,2.0)'
+    compile 'org.opencadc:cadc-util:[1.9.10,2.0)'
     compile 'org.opencadc:cadc-log:[1.1.6,2.0)'
     compile 'org.opencadc:cadc-gms:[1.0.5,)'
     compile 'org.opencadc:cadc-rest:[1.3.16,)'
     compile 'org.opencadc:cadc-vos:[2.0,)'
     compile 'org.opencadc:cadc-vos-server-alt:[2.0,)'
     compile 'org.opencadc:cadc-vosi:[1.3.2,)'
     compile 'org.opencadc:cadc-uws:[1.0,)'
-    compile 'org.opencadc:cadc-uws-server:[1.2.12,)'
+    compile 'org.opencadc:cadc-uws-server:[1.2.19,)'
     compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)'
     compile 'org.opencadc:cadc-cdp:[1.2.3,)'
-    compile 'org.opencadc:cadc-registry:[1.5.15,)'
+    compile 'org.opencadc:cadc-registry:[1.7.4,)'
     compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)'
     compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)'
 

vault/src/intTest/java/org/opencadc/vault/NodesTest.java

@@ -71,6 +71,7 @@
 import java.net.URI;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
+import org.opencadc.gms.GroupURI;
 
 /**
  * Test the nodes endpoint.
@@ -86,6 +87,7 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest {
     }
 
     public NodesTest() {
+        //super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem", new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?CADC_TEST_GROUP2")), "vault-test-auth.pem");
         super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem");
     }
 }

vault/src/main/java/org/opencadc/vault/VaultInitAction.java

@@ -71,6 +71,7 @@
 import ca.nrc.cadc.rest.InitAction;
 import ca.nrc.cadc.util.MultiValuedProperties;
 import ca.nrc.cadc.util.PropertiesReader;
+import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Map;
@@ -93,6 +94,7 @@ public class VaultInitAction extends InitAction {
     private static final Logger log = Logger.getLogger(VaultInitAction.class);
 
     static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml
+    static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml
 
     // config keys
     private static final String VAULT_KEY = "org.opencadc.vault";
@@ -119,6 +121,7 @@ public VaultInitAction() {
     public void doInit() {
         initConfig();
         initDatabase();
+        initUWSDatabase();
         initNodePersistence();
     }
 
@@ -215,6 +218,20 @@ private void initDatabase() {
         }
     }
 
+    private void initUWSDatabase() {
+        log.info("initUWSDatabase: START");
+        try {
+            // Init UWS database
+            DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE);
+            InitDatabaseUWS uwsi = new InitDatabaseUWS(uws, null, "uws");
+            uwsi.doInit();
+            log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws OK");
+
+        } catch (Exception ex) {
+            throw new RuntimeException("check/init uws database failed", ex);
+        }
+    }
+
     protected void initNodePersistence() {
         jndiNodePersistence = componentID + ".nodePersistence";
         try {

vault/src/main/webapp/META-INF/context.xml

@@ -15,4 +15,17 @@
         removeAbandoned="false"
         testOnBorrow="true" validationQuery="select 123" />
 
+
+    <Resource name="jdbc/uws"
+        auth="Container"
+        type="javax.sql.DataSource"
+        factory="org.apache.tomcat.jdbc.pool.DataSourceFactory" closeMethod="close"
+        minEvictableIdleTimeMillis="60000" timeBetweenEvictionRunsMillis="30000"
+        maxWait="20000"
+        initialSize="0" minIdle="0" maxIdle="${org.opencadc.vault.uws.maxActive}" maxActive="${org.opencadc.vault.uws.maxActive}"
+        username="${org.opencadc.vault.uws.username}" password="${org.opencadc.vault.uws.password}"
+        driverClassName="org.postgresql.Driver" url="${org.opencadc.vault.uws.url}"
+        removeAbandoned="false"
+        testOnBorrow="true" validationQuery="select 123" />
+
 </Context>

vault/src/main/webapp/WEB-INF/web.xml

@@ -23,6 +23,7 @@
           ca.nrc.cadc.rest
           ca.nrc.cadc.util
           ca.nrc.cadc.vosi
+          ca.nrc.cadc.uws
       </param-value>
     </init-param>
     <init-param>
@@ -62,6 +63,30 @@
     <load-on-startup>2</load-on-startup>
   </servlet>
 
+  <!-- async recursive node property setting servlet-->
+  <servlet>
+    <servlet-name>RecursiveDeleteNodeServlet</servlet-name>
+    <servlet-class>ca.nrc.cadc.uws.server.JobServlet</servlet-class>
+    <init-param>
+      <param-name>get</param-name>
+      <param-value>ca.nrc.cadc.uws.web.GetAction</param-value>
+    </init-param>
+    <init-param>
+      <param-name>post</param-name>
+      <param-value>ca.nrc.cadc.uws.web.PostAction</param-value>
+    </init-param>
+    <init-param>
+      <param-name>delete</param-name>
+      <param-value>ca.nrc.cadc.uws.web.DeleteAction</param-value>
+    </init-param>
+    <init-param>
+      <param-name>ca.nrc.cadc.uws.server.JobManager</param-name>
+      <param-value>org.opencadc.vault.RecursiveDeleteNodeJobManager</param-value>
+    </init-param>
+    <load-on-startup>3</load-on-startup>
+  </servlet>
+
+
   <!-- VOSI capabilities -->
   <servlet>
     <servlet-name>CapabilitiesServlet</servlet-name>
@@ -107,6 +132,11 @@
     <url-pattern>/nodes/*</url-pattern>
   </servlet-mapping>
 
+  <servlet-mapping>
+    <servlet-name>RecursiveDeleteNodeServlet</servlet-name>
+    <url-pattern>/recursiveDelete/*</url-pattern>
+  </servlet-mapping>
+
   <!-- Availability servlet endpoint -->
   <servlet-mapping>
     <servlet-name>AvailabilityServlet</servlet-name>

vault/src/main/webapp/capabilities.xml

@@ -41,6 +41,16 @@
     </interface>
   </capability>
 
+  <capability standardID="ivo://ivoa.net/std/VOSpace#recursive-delete-proto">
+    <interface xsi:type="vs:ParamHTTP" role="std">
+      <accessURL use="base">https://replace.me.com/vault/recursiveDelete</accessURL>
+      <securityMethod/>
+      <securityMethod standardID="ivo://ivoa.net/sso#cookie"/>
+      <securityMethod standardID="ivo://ivoa.net/sso#tls-with-certificate"/>
+      <securityMethod standardID="ivo://ivoa.net/sso#token"/>
+    </interface>
+  </capability>
+
 <!--
   <capability standardID="ivo://ivoa.net/std/VOSpace/v2.0#transfers">
     <interface xsi:type="vs:ParamHTTP" role="std">