Skip to content

Commit

Permalink
Scan classpath for EDD subclasses/dataset types
Browse files Browse the repository at this point in the history 
This change uses reflection, specifically the ClassGraph classpath
scanner, to discover concrete EDD subclasses with appropriate
fromXml methods at runtime. The scan is performed once and stored
in a static map for use in EDD's fromXml method.

In addition to elminiating the need to registry various EDD implementations
in the ERDDAP codebase in EDD.fromXml, this change also allows loading of
third party EDD implementations at runtime.

In local testing classpath scanning took about 1 second. This scan occurs
only once at startup. Scanning is confined to EDD's package
(gov.noaa.pfel.erddap.dataset) for performance reasons; scanning
all packages took around 12 seconds. For this reason, all EDD implementations
need to belong to the gov.noaa.pfel.erddap.dataset package to be detected.
  • Loading branch information
@srstsavage
srstsavage committed Sep 26, 2024
1 parent 5a0d0c8 commit b170d3f
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 82 deletions.
192 changes: 110 additions & 82 deletions WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,19 @@
import gov.noaa.pfel.coastwatch.util.Tally;
import gov.noaa.pfel.erddap.Erddap;
import gov.noaa.pfel.erddap.handlers.SaxHandler;
import gov.noaa.pfel.erddap.util.*;
import gov.noaa.pfel.erddap.variable.*;
import gov.noaa.pfel.erddap.util.CfToFromGcmd;
import gov.noaa.pfel.erddap.util.EDStatic;
import gov.noaa.pfel.erddap.util.EmailThread;
import gov.noaa.pfel.erddap.util.Subscriptions;
import gov.noaa.pfel.erddap.util.TaskThread;
import gov.noaa.pfel.erddap.util.TouchThread;
import gov.noaa.pfel.erddap.variable.EDV;
import io.github.classgraph.ClassGraph;
import io.github.classgraph.ClassInfo;
import io.github.classgraph.ClassRefTypeSignature;
import io.github.classgraph.MethodInfo;
import io.github.classgraph.MethodParameterInfo;
import io.github.classgraph.ScanResult;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import java.io.BufferedOutputStream;
Expand All @@ -47,13 +58,15 @@
import java.io.OutputStream;
import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.InvocationTargetException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
Expand Down Expand Up @@ -405,6 +418,88 @@ public boolean filesInPrivateS3Bucket() {
return filesInPrivateS3Bucket;
}

/** Map of EDD dataset types to their corresponding classes. */
private static final Map<String, Class<? extends EDD>> EDD_CLASS_MAP = initializeEddClassMap();

/**
* Determines if a MethodParameterInfo type matches a specific class.
*
* @param paramInfo the MethodParameterInfo to check
* @param checkClass the class to check against
* @return true if the MethodParameterInfo type matches the specified class, false otherwise
*/
private static final boolean isParameterClass(
final MethodParameterInfo paramInfo, final Class<?> checkClass) {
if (paramInfo == null || !(paramInfo.getTypeDescriptor() instanceof ClassRefTypeSignature)) {
return false;
}
return ((ClassRefTypeSignature) paramInfo.getTypeDescriptor())
.getFullyQualifiedClassName()
.equals(checkClass.getName());
}

/**
* Initializes the map of EDD dataset types to their corresponding classes. This method scans the
* classpath using reflection for concrete (non-abstract) subclasses of EDD which have a public
* static fromXml method accepting the proper parameters (Erddap and SimpleXMLReader).
*
* @return the map of EDD dataset types to their corresponding classes
*/
private static final Map<String, Class<? extends EDD>> initializeEddClassMap() {
Map<String, Class<? extends EDD>> eddClassMap = new HashMap<>();
String2.log("Scanning EDD classes in package " + EDD.class.getPackageName());
try (ScanResult scanResult =
new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.acceptPackages(EDD.class.getPackageName())
.scan()) {
for (ClassInfo eddClassInfo : scanResult.getSubclasses(EDD.class)) {
if (eddClassInfo.isAbstract()) {
continue;
}

if (!eddClassInfo.hasMethod("fromXml")) {
String2.log(
"WARNING: Concrete EDD subclass "
+ eddClassInfo.getSimpleName()
+ " does not have a fromXml method.");
continue;
}

boolean fromXmlMethodFound = false;
for (MethodInfo methodInfo : eddClassInfo.getMethodInfo("fromXml")) {
MethodParameterInfo[] parameterInfo = methodInfo.getParameterInfo();
if (parameterInfo.length != 2) {
continue;
}
if (!isParameterClass(parameterInfo[0], Erddap.class)
|| !isParameterClass(parameterInfo[1], SimpleXMLReader.class)) {
continue;
}

if (!methodInfo.isStatic() || !methodInfo.isPublic()) {
continue;
}

fromXmlMethodFound = true;
break;
}

if (fromXmlMethodFound) {
eddClassMap.put(
eddClassInfo.getSimpleName(), (Class<? extends EDD>) eddClassInfo.loadClass());
} else {
String2.log(
"WARNING: Concrete EDD subclass "
+ eddClassInfo.getSimpleName()
+ " does not have a fromXml method with the correct signature and modifiers.");
}
}
}
return eddClassMap;
}

/**
* This constructs an EDDXxx based on the information in an .xml file. This ignores the
* &lt;dataset active=.... &gt; setting. All of the subclasses fromXml() methods ignore the
Expand All @@ -422,94 +517,27 @@ public static EDD fromXml(Erddap erddap, String type, SimpleXMLReader xmlReader)
String startStartError =
"datasets.xml error on"; // does the error message already start with this?
String startError = "datasets.xml error on or before line #";
if (type == null)
throw new SimpleException(
Class<? extends EDD> eddClass = EDD_CLASS_MAP.get(type);
if (eddClass == null) {
throw new RuntimeException(
startError + xmlReader.lineNumber() + ": Unexpected <dataset> type=" + type + ".");
}

try {
// FUTURE: classes could be added at runtime if I used reflection
if (type.equals("EDDGridAggregateExistingDimension"))
return EDDGridAggregateExistingDimension.fromXml(erddap, xmlReader);
if (type.equals("EDDGridCopy")) return EDDGridCopy.fromXml(erddap, xmlReader);
if (type.equals("EDDGridFromAudioFiles"))
return EDDGridFromAudioFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDGridFromDap")) return EDDGridFromDap.fromXml(erddap, xmlReader);
if (type.equals("EDDGridFromEDDTable")) return EDDGridFromEDDTable.fromXml(erddap, xmlReader);
if (type.equals("EDDGridFromErddap")) return EDDGridFromErddap.fromXml(erddap, xmlReader);
if (type.equals("EDDGridFromEtopo")) return EDDGridFromEtopo.fromXml(erddap, xmlReader);
if (type.equals("EDDGridFromMergeIRFiles"))
return EDDGridFromMergeIRFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDGridFromNcFiles")) return EDDGridFromNcFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDGridFromNcFilesUnpacked"))
return EDDGridFromNcFilesUnpacked.fromXml(erddap, xmlReader);
if (type.equals("EDDGridLonPM180")) return EDDGridLonPM180.fromXml(erddap, xmlReader);
if (type.equals("EDDGridLon0360")) return EDDGridLon0360.fromXml(erddap, xmlReader);
if (type.equals("EDDGridSideBySide")) return EDDGridSideBySide.fromXml(erddap, xmlReader);

if (type.equals("EDDTableAggregateRows"))
return EDDTableAggregateRows.fromXml(erddap, xmlReader);
if (type.equals("EDDTableCopy")) return EDDTableCopy.fromXml(erddap, xmlReader);
// if (type.equals("EDDTableCopyPost")) return EDDTableCopyPost.fromXml(erddap,
// xmlReader); //inactive
if (type.equals("EDDTableFromAsciiServiceNOS"))
return EDDTableFromAsciiServiceNOS.fromXml(erddap, xmlReader);
// if (type.equals("EDDTableFromBMDE")) return EDDTableFromBMDE.fromXml(erddap,
// xmlReader); //inactive
if (type.equals("EDDTableFromCassandra"))
return EDDTableFromCassandra.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromDapSequence"))
return EDDTableFromDapSequence.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromDatabase"))
return EDDTableFromDatabase.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromEDDGrid")) return EDDTableFromEDDGrid.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromErddap")) return EDDTableFromErddap.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromFileNames"))
return EDDTableFromFileNames.fromXml(erddap, xmlReader);
// if (type.equals("EDDTableFromMWFS")) return EDDTableFromMWFS.fromXml(erddap,
// xmlReader); //inactive as of 2009-01-14
if (type.equals("EDDTableFromAsciiFiles"))
return EDDTableFromAsciiFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromAudioFiles"))
return EDDTableFromAudioFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromAwsXmlFiles"))
return EDDTableFromAwsXmlFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromColumnarAsciiFiles"))
return EDDTableFromColumnarAsciiFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromHttpGet")) return EDDTableFromHttpGet.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromInvalidCRAFiles"))
return EDDTableFromInvalidCRAFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromJsonlCSVFiles"))
return EDDTableFromJsonlCSVFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromHyraxFiles"))
return EDDTableFromHyraxFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromMultidimNcFiles"))
return EDDTableFromMultidimNcFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromNcFiles")) return EDDTableFromNcFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromNcCFFiles"))
return EDDTableFromNcCFFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromNccsvFiles"))
return EDDTableFromNccsvFiles.fromXml(erddap, xmlReader);
// if (type.equals("EDDTableFromNOS")) return EDDTableFromNOS.fromXml(erddap,
// xmlReader); //inactive 2010-09-08
// if (type.equals("EDDTableFromNWISDV")) return EDDTableFromNWISDV.fromXml(erddap,
// xmlReader); //inactive 2011-12-16
if (type.equals("EDDTableFromOBIS")) return EDDTableFromOBIS.fromXml(erddap, xmlReader);
// if (type.equals("EDDTableFromPostDatabase"))return EDDTableFromPostDatabase.fromXml(erddap,
// xmlReader);
// if (type.equals("EDDTableFromPostNcFiles")) return EDDTableFromPostNcFiles.fromXml(erddap,
// xmlReader);
if (type.equals("EDDTableFromSOS")) return EDDTableFromSOS.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromThreddsFiles"))
return EDDTableFromThreddsFiles.fromXml(erddap, xmlReader);
if (type.equals("EDDTableFromWFSFiles"))
return EDDTableFromWFSFiles.fromXml(erddap, xmlReader);
return (EDD)
eddClass
.getMethod("fromXml", Erddap.class, SimpleXMLReader.class)
.invoke(null, erddap, xmlReader);
} catch (Throwable t) {
// unwrap InvocationTargetExceptions
if (t instanceof InvocationTargetException && t.getCause() != null) {
t = t.getCause();
}
String msg = MustBe.getShortErrorMessage(t);
throw new RuntimeException(
(msg.startsWith(startStartError) ? "" : startError + xmlReader.lineNumber() + ": ") + msg,
t);
}
throw new RuntimeException(
startError + xmlReader.lineNumber() + ": Unexpected <dataset> type=" + type + ".");
}

/**
Expand Down
5 changes: 5 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -924,6 +924,11 @@
<artifactId>prometheus-metrics-exporter-servlet-jakarta</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>io.github.classgraph</groupId>
<artifactId>classgraph</artifactId>
<version>4.8.176</version>
</dependency>
</dependencies>

<reporting>
Expand Down

0 comments on commit b170d3f

Please sign in to comment.