Scan classpath for EDD subclasses/dataset types

This change uses reflection, specifically the ClassGraph classpath scanner, to discover concrete EDD subclasses and associated class metadata (fromXml methods and Sax handlers for XML deseralization) at runtime. The scan is performed once and stored in a static map for use in XML deserialization methods. In addition to eliminating the need to register various EDD implementations in the ERDDAP codebase in EDD.fromXml and HandlerFactory.getHandlerFor, this change also allows loading of third party EDD implementations at runtime. In local testing classpath scanning took about 1 second. This scan occurs only once at startup. Scanning is confined to EDD's package (gov.noaa.pfel.erddap.dataset) for performance reasons; scanning all packages took around 12 seconds. For this reason, all EDD implementations need to belong to the gov.noaa.pfel.erddap.dataset package to be detected. This adds a dependency on classgraph, which is very small (~560k).
ERDDAP · Sep 27, 2024 · e672ee3 · e672ee3
1 parent 5a0d0c8
commit e672ee3
Show file tree

Hide file tree

Showing 27 changed files with 294 additions and 175 deletions.
diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java
@@ -37,8 +37,20 @@
 import gov.noaa.pfel.coastwatch.util.Tally;
 import gov.noaa.pfel.erddap.Erddap;
 import gov.noaa.pfel.erddap.handlers.SaxHandler;
-import gov.noaa.pfel.erddap.util.*;
-import gov.noaa.pfel.erddap.variable.*;
+import gov.noaa.pfel.erddap.handlers.SaxHandlerClass;
+import gov.noaa.pfel.erddap.handlers.State;
+import gov.noaa.pfel.erddap.util.CfToFromGcmd;
+import gov.noaa.pfel.erddap.util.EDStatic;
+import gov.noaa.pfel.erddap.util.EmailThread;
+import gov.noaa.pfel.erddap.util.Subscriptions;
+import gov.noaa.pfel.erddap.util.TaskThread;
+import gov.noaa.pfel.erddap.util.TouchThread;
+import gov.noaa.pfel.erddap.variable.EDV;
+import io.github.classgraph.AnnotationClassRef;
+import io.github.classgraph.AnnotationInfo;
+import io.github.classgraph.ClassGraph;
+import io.github.classgraph.ClassInfo;
+import io.github.classgraph.ScanResult;
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
 import java.io.BufferedOutputStream;
@@ -47,17 +59,24 @@
 import java.io.OutputStream;
 import java.io.StringWriter;
 import java.io.Writer;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
 import java.util.GregorianCalendar;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
@@ -405,6 +424,103 @@ public boolean filesInPrivateS3Bucket() {
     return filesInPrivateS3Bucket;
   }
 
+  /** Internal class for storing conrete EDD subclass information. */
+  public static final class EDDClassInfo {
+    private Class<EDD> eddClass;
+    private Optional<Method> fromXmlMethod;
+    private Optional<Class<State>> saxHandlerClass;
+
+    public EDDClassInfo(
+        Class<EDD> eddClass,
+        Optional<Method> fromXmlMethod,
+        Optional<Class<State>> saxHandlerClass) {
+      this.eddClass = eddClass;
+      this.fromXmlMethod = fromXmlMethod;
+      this.saxHandlerClass = saxHandlerClass;
+    }
+
+    public Class<EDD> getEddClass() {
+      return eddClass;
+    }
+
+    public boolean hasFromXmlMethod() {
+      return fromXmlMethod.isPresent();
+    }
+
+    public Optional<Method> getFromXmlMethod() {
+      return fromXmlMethod;
+    }
+
+    public boolean hasSaxHandlerClass() {
+      return saxHandlerClass.isPresent();
+    }
+
+    public Optional<Class<State>> getSaxHandlerClass() {
+      return saxHandlerClass;
+    }
+  }
+
+  /** List of all concrete/non-abstact EDD subclass EDDClassInfo */
+  public static final Map<String, EDDClassInfo> EDD_CLASS_INFO_MAP = initEddClassInfoMap();
+
+  /**
+   * Scan classpath for concrete/non-abstract EDD subclasses in package gov.noaa.pfel.erddap.dataset
+   * and store metadata for each discovered subclass in map of EDDClassInfo objects.
+   */
+  private static final Map<String, EDDClassInfo> initEddClassInfoMap() {
+    String2.log("Scanning EDD classes in package " + EDD.class.getPackageName());
+    try (ScanResult scanResult =
+        new ClassGraph()
+            .enableAnnotationInfo()
+            .enableClassInfo()
+            .enableMethodInfo()
+            .acceptPackages(EDD.class.getPackageName())
+            .scan()) {
+      return scanResult.getSubclasses(EDD.class).stream()
+          .filter(c -> !c.isAbstract())
+          .collect(
+              Collectors.toUnmodifiableMap(
+                  ClassInfo::getSimpleName,
+                  classInfo ->
+                      new EDDClassInfo(
+                          (Class<EDD>) classInfo.loadClass(),
+                          getFromXmlMethod(classInfo),
+                          getSaxHandler(classInfo))));
+    }
+  }
+
+  /**
+   * Get State (Sax handler) class for EDD class from annotation if it exists
+   *
+   * @param eddClassInfo the ClassInfo object to check
+   * @return Optional of the State class
+   */
+  private static final Optional<Class<State>> getSaxHandler(ClassInfo classInfo) {
+    return Stream.of(classInfo.getAnnotationInfo(SaxHandlerClass.class))
+        .filter(Objects::nonNull)
+        .map(AnnotationInfo::getParameterValues)
+        .map(parameterValues -> parameterValues.get("value"))
+        .filter(Objects::nonNull)
+        .map(annotationParameterValue -> (AnnotationClassRef) annotationParameterValue.getValue())
+        .filter(annotationClassRef -> annotationClassRef != null)
+        .map(annotationClassRef -> (Class<State>) annotationClassRef.loadClass())
+        .filter(Objects::nonNull)
+        .findFirst();
+  }
+
+  /**
+   * Get fromXml method annotated with EDDFromXmlMethod from EDD class if it exists
+   *
+   * @param eddClassInfo the ClassInfo object to check
+   * @return Optional of the fromXml method
+   */
+  private static final Optional<Method> getFromXmlMethod(ClassInfo classInfo) {
+    return classInfo.getMethodInfo("fromXml").stream()
+        .filter(mi -> mi.hasAnnotation(EDDFromXmlMethod.class))
+        .map(mi -> mi.loadClassAndGetMethod())
+        .findFirst();
+  }
+
   /**
    * This constructs an EDDXxx based on the information in an .xml file. This ignores the
    * &lt;dataset active=.... &gt; setting. All of the subclasses fromXml() methods ignore the
@@ -422,94 +538,24 @@ public static EDD fromXml(Erddap erddap, String type, SimpleXMLReader xmlReader)
     String startStartError =
         "datasets.xml error on"; // does the error message already start with this?
     String startError = "datasets.xml error on or before line #";
-    if (type == null)
-      throw new SimpleException(
+    EDDClassInfo eddClassInfo = EDD_CLASS_INFO_MAP.get(type);
+    if (eddClassInfo == null || !eddClassInfo.hasFromXmlMethod()) {
+      throw new RuntimeException(
           startError + xmlReader.lineNumber() + ": Unexpected <dataset> type=" + type + ".");
+    }
+
     try {
-      // FUTURE: classes could be added at runtime if I used reflection
-      if (type.equals("EDDGridAggregateExistingDimension"))
-        return EDDGridAggregateExistingDimension.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridCopy")) return EDDGridCopy.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridFromAudioFiles"))
-        return EDDGridFromAudioFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridFromDap")) return EDDGridFromDap.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridFromEDDTable")) return EDDGridFromEDDTable.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridFromErddap")) return EDDGridFromErddap.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridFromEtopo")) return EDDGridFromEtopo.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridFromMergeIRFiles"))
-        return EDDGridFromMergeIRFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridFromNcFiles")) return EDDGridFromNcFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridFromNcFilesUnpacked"))
-        return EDDGridFromNcFilesUnpacked.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridLonPM180")) return EDDGridLonPM180.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridLon0360")) return EDDGridLon0360.fromXml(erddap, xmlReader);
-      if (type.equals("EDDGridSideBySide")) return EDDGridSideBySide.fromXml(erddap, xmlReader);
-
-      if (type.equals("EDDTableAggregateRows"))
-        return EDDTableAggregateRows.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableCopy")) return EDDTableCopy.fromXml(erddap, xmlReader);
-      // if (type.equals("EDDTableCopyPost"))        return EDDTableCopyPost.fromXml(erddap,
-      // xmlReader); //inactive
-      if (type.equals("EDDTableFromAsciiServiceNOS"))
-        return EDDTableFromAsciiServiceNOS.fromXml(erddap, xmlReader);
-      // if (type.equals("EDDTableFromBMDE"))        return EDDTableFromBMDE.fromXml(erddap,
-      // xmlReader); //inactive
-      if (type.equals("EDDTableFromCassandra"))
-        return EDDTableFromCassandra.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromDapSequence"))
-        return EDDTableFromDapSequence.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromDatabase"))
-        return EDDTableFromDatabase.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromEDDGrid")) return EDDTableFromEDDGrid.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromErddap")) return EDDTableFromErddap.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromFileNames"))
-        return EDDTableFromFileNames.fromXml(erddap, xmlReader);
-      // if (type.equals("EDDTableFromMWFS"))        return EDDTableFromMWFS.fromXml(erddap,
-      // xmlReader); //inactive as of 2009-01-14
-      if (type.equals("EDDTableFromAsciiFiles"))
-        return EDDTableFromAsciiFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromAudioFiles"))
-        return EDDTableFromAudioFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromAwsXmlFiles"))
-        return EDDTableFromAwsXmlFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromColumnarAsciiFiles"))
-        return EDDTableFromColumnarAsciiFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromHttpGet")) return EDDTableFromHttpGet.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromInvalidCRAFiles"))
-        return EDDTableFromInvalidCRAFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromJsonlCSVFiles"))
-        return EDDTableFromJsonlCSVFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromHyraxFiles"))
-        return EDDTableFromHyraxFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromMultidimNcFiles"))
-        return EDDTableFromMultidimNcFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromNcFiles")) return EDDTableFromNcFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromNcCFFiles"))
-        return EDDTableFromNcCFFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromNccsvFiles"))
-        return EDDTableFromNccsvFiles.fromXml(erddap, xmlReader);
-      // if (type.equals("EDDTableFromNOS"))         return EDDTableFromNOS.fromXml(erddap,
-      // xmlReader); //inactive 2010-09-08
-      // if (type.equals("EDDTableFromNWISDV"))      return EDDTableFromNWISDV.fromXml(erddap,
-      // xmlReader); //inactive 2011-12-16
-      if (type.equals("EDDTableFromOBIS")) return EDDTableFromOBIS.fromXml(erddap, xmlReader);
-      // if (type.equals("EDDTableFromPostDatabase"))return EDDTableFromPostDatabase.fromXml(erddap,
-      // xmlReader);
-      // if (type.equals("EDDTableFromPostNcFiles")) return EDDTableFromPostNcFiles.fromXml(erddap,
-      // xmlReader);
-      if (type.equals("EDDTableFromSOS")) return EDDTableFromSOS.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromThreddsFiles"))
-        return EDDTableFromThreddsFiles.fromXml(erddap, xmlReader);
-      if (type.equals("EDDTableFromWFSFiles"))
-        return EDDTableFromWFSFiles.fromXml(erddap, xmlReader);
+      return (EDD) eddClassInfo.getFromXmlMethod().get().invoke(null, erddap, xmlReader);
     } catch (Throwable t) {
+      // unwrap InvocationTargetExceptions
+      if (t instanceof InvocationTargetException && t.getCause() != null) {
+        t = t.getCause();
+      }
       String msg = MustBe.getShortErrorMessage(t);
       throw new RuntimeException(
           (msg.startsWith(startStartError) ? "" : startError + xmlReader.lineNumber() + ": ") + msg,
           t);
     }
-    throw new RuntimeException(
-        startError + xmlReader.lineNumber() + ": Unexpected <dataset> type=" + type + ".");
   }
 
   /**

diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDFromXmlMethod.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDFromXmlMethod.java
@@ -0,0 +1,14 @@
+package gov.noaa.pfel.erddap.dataset;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that the annotated method should be used to create an EDD object from an XML string
+ * (non-Sax parser approach)
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface EDDFromXmlMethod {}
diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java
@@ -20,6 +20,8 @@
 import gov.noaa.pfel.coastwatch.util.SSR;
 import gov.noaa.pfel.coastwatch.util.SimpleXMLReader;
 import gov.noaa.pfel.erddap.Erddap;
+import gov.noaa.pfel.erddap.handlers.EDDGridAggregateExistingDimensionHandler;
+import gov.noaa.pfel.erddap.handlers.SaxHandlerClass;
 import gov.noaa.pfel.erddap.util.EDStatic;
 import gov.noaa.pfel.erddap.variable.*;
 import java.text.MessageFormat;
@@ -31,6 +33,7 @@
  *
  * @author Bob Simons (was bob.simons@noaa.gov, now BobSimons2.00@gmail.com) 2008-02-04
  */
+@SaxHandlerClass(EDDGridAggregateExistingDimensionHandler.class)
 public class EDDGridAggregateExistingDimension extends EDDGrid {
 
   protected EDDGrid childDatasets[];
@@ -53,6 +56,7 @@ public class EDDGridAggregateExistingDimension extends EDDGrid {
    *     &lt;erddapDatasets&gt;&lt;/dataset&gt; .
    * @throws Throwable if trouble
    */
+  @EDDFromXmlMethod
   public static EDDGridAggregateExistingDimension fromXml(Erddap erddap, SimpleXMLReader xmlReader)
       throws Throwable {
 

diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridCopy.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridCopy.java
@@ -20,6 +20,8 @@
 import gov.noaa.pfel.coastwatch.util.SSR;
 import gov.noaa.pfel.coastwatch.util.SimpleXMLReader;
 import gov.noaa.pfel.erddap.Erddap;
+import gov.noaa.pfel.erddap.handlers.EDDGridCopyHandler;
+import gov.noaa.pfel.erddap.handlers.SaxHandlerClass;
 import gov.noaa.pfel.erddap.util.EDStatic;
 import gov.noaa.pfel.erddap.util.TaskThread;
 import gov.noaa.pfel.erddap.variable.*;
@@ -33,6 +35,7 @@
  *
  * @author Bob Simons (was bob.simons@noaa.gov, now BobSimons2.00@gmail.com) 2009-05-25
  */
+@SaxHandlerClass(EDDGridCopyHandler.class)
 public class EDDGridCopy extends EDDGrid {
 
   protected EDDGrid sourceEdd;
@@ -61,6 +64,7 @@ public class EDDGridCopy extends EDDGrid {
    *     &lt;erddapDatasets&gt;&lt;/dataset&gt; .
    * @throws Throwable if trouble
    */
+  @EDDFromXmlMethod
   public static EDDGridCopy fromXml(Erddap erddap, SimpleXMLReader xmlReader) throws Throwable {
 
     // data to be obtained (or not)

diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java
@@ -29,6 +29,8 @@
 import gov.noaa.pfel.coastwatch.util.SSR;
 import gov.noaa.pfel.coastwatch.util.SimpleXMLReader;
 import gov.noaa.pfel.erddap.Erddap;
+import gov.noaa.pfel.erddap.handlers.EDDGridFromDapHandler;
+import gov.noaa.pfel.erddap.handlers.SaxHandlerClass;
 import gov.noaa.pfel.erddap.util.EDStatic;
 import gov.noaa.pfel.erddap.variable.*;
 import java.io.ByteArrayInputStream;
@@ -56,6 +58,7 @@
  *
  * @author Bob Simons (was bob.simons@noaa.gov, now BobSimons2.00@gmail.com) 2007-06-04
  */
+@SaxHandlerClass(EDDGridFromDapHandler.class)
 public class EDDGridFromDap extends EDDGrid {
 
   /**
@@ -74,6 +77,7 @@ public class EDDGridFromDap extends EDDGrid {
    *     &lt;erddapDatasets&gt;&lt;/dataset&gt; .
    * @throws Throwable if trouble
    */
+  @EDDFromXmlMethod
   public static EDDGridFromDap fromXml(Erddap erddap, SimpleXMLReader xmlReader) throws Throwable {
 
     // data to be obtained (or not)

diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromEDDTable.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromEDDTable.java
@@ -21,6 +21,8 @@
 import gov.noaa.pfel.coastwatch.pointdata.Table;
 import gov.noaa.pfel.coastwatch.util.SimpleXMLReader;
 import gov.noaa.pfel.erddap.Erddap;
+import gov.noaa.pfel.erddap.handlers.EDDGridFromEDDTableHandler;
+import gov.noaa.pfel.erddap.handlers.SaxHandlerClass;
 import gov.noaa.pfel.erddap.util.EDStatic;
 import gov.noaa.pfel.erddap.variable.*;
 import java.io.DataInputStream;
@@ -33,6 +35,7 @@
  *
  * @author Bob Simons (was bob.simons@noaa.gov, now BobSimons2.00@gmail.com) 2015-01-27
  */
+@SaxHandlerClass(EDDGridFromEDDTableHandler.class)
 public class EDDGridFromEDDTable extends EDDGrid {
 
   protected EDDTable eddTable;
@@ -55,6 +58,7 @@ public class EDDGridFromEDDTable extends EDDGrid {
    *     &lt;erddapDatasets&gt;&lt;/dataset&gt; .
    * @throws Throwable if trouble
    */
+  @EDDFromXmlMethod
   public static EDDGridFromEDDTable fromXml(Erddap erddap, SimpleXMLReader xmlReader)
       throws Throwable {
 

diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java
@@ -26,6 +26,8 @@
 import gov.noaa.pfel.coastwatch.util.SSR;
 import gov.noaa.pfel.coastwatch.util.SimpleXMLReader;
 import gov.noaa.pfel.erddap.Erddap;
+import gov.noaa.pfel.erddap.handlers.EDDGridFromErddapHandler;
+import gov.noaa.pfel.erddap.handlers.SaxHandlerClass;
 import gov.noaa.pfel.erddap.util.EDStatic;
 import gov.noaa.pfel.erddap.variable.*;
 import java.io.BufferedReader;
@@ -42,6 +44,7 @@
  *
  * @author Bob Simons (bob.simons@noaa.gov) 2007-06-04
  */
+@SaxHandlerClass(EDDGridFromErddapHandler.class)
 public class EDDGridFromErddap extends EDDGrid implements FromErddap {
 
   protected double sourceErddapVersion =
@@ -67,6 +70,7 @@ public class EDDGridFromErddap extends EDDGrid implements FromErddap {
    *     &lt;erddapDatasets&gt;&lt;/dataset&gt; .
    * @throws Throwable if trouble
    */
+  @EDDFromXmlMethod
   public static EDDGridFromErddap fromXml(Erddap erddap, SimpleXMLReader xmlReader)
       throws Throwable {