Add package.json js-plugins support (#4389)

In support of #4405

plugin/build.gradle

@@ -4,3 +4,5 @@ plugins {
 }
 
 description = 'The Deephaven plugin interface'
+
+Classpaths.inheritImmutables(project)

plugin/dagger/src/main/java/io/deephaven/plugin/PluginModule.java

@@ -6,6 +6,7 @@
 import dagger.Module;
 import dagger.Provides;
 import dagger.multibindings.ElementsIntoSet;
+import io.deephaven.plugin.js.JsPlugin;
 import io.deephaven.plugin.type.ObjectType;
 
 import java.util.ServiceLoader;
@@ -15,7 +16,7 @@
 
 /**
  * Provides the set of {@link Registration} from {@link ServiceLoader#load(Class)} against the classes
- * {@link Registration}, {@link Plugin}, and {@link ObjectType}.
+ * {@link Registration}, {@link Plugin}, {@link ObjectType}, and {@link JsPlugin}.
  */
 @Module
 public interface PluginModule {
@@ -37,4 +38,10 @@ static Set<Registration> providesServiceLoaderPlugins() {
     static Set<Registration> providesServiceLoaderObjectTypes() {
         return ServiceLoader.load(ObjectType.class).stream().map(Provider::get).collect(Collectors.toSet());
     }
+
+    @Provides
+    @ElementsIntoSet
+    static Set<Registration> providesServiceLoaderJsPlugin() {
+        return ServiceLoader.load(JsPlugin.class).stream().map(Provider::get).collect(Collectors.toSet());
+    }
 }

plugin/src/main/java/io/deephaven/plugin/Plugin.java

@@ -3,6 +3,7 @@
  */
 package io.deephaven.plugin;
 
+import io.deephaven.plugin.js.JsPlugin;
 import io.deephaven.plugin.type.ObjectType;
 
 /**
@@ -24,5 +25,7 @@ public interface Plugin extends Registration {
 
     interface Visitor<T> {
         T visit(ObjectType objectType);
+
+        T visit(JsPlugin jsPlugin);
     }
 }

plugin/src/main/java/io/deephaven/plugin/js/JsPlugin.java

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending
+ */
+package io.deephaven.plugin.js;
+
+import io.deephaven.plugin.Plugin;
+
+/**
+ * A js plugin is a {@link Plugin} that allows adding javascript code under the server's URL path "js-plugins/". See
+ * <a href="https://github.com/deephaven/deephaven-plugins#js-plugins">deephaven-plugins#js-plugins</a> for more details
+ * about the underlying construction for js plugins.
+ *
+ * @see JsPluginPackagePath
+ * @see JsPluginManifestPath
+ */
+public interface JsPlugin extends Plugin {
+
+}

plugin/src/main/java/io/deephaven/plugin/js/JsPluginBase.java

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending
+ */
+package io.deephaven.plugin.js;
+
+import io.deephaven.plugin.Plugin;
+import io.deephaven.plugin.PluginBase;
+
+public abstract class JsPluginBase extends PluginBase implements JsPlugin {
+
+    @Override
+    public final <T, V extends Plugin.Visitor<T>> T walk(V visitor) {
+        return visitor.visit(this);
+    }
+}

plugin/src/main/java/io/deephaven/plugin/js/JsPluginManifestPath.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending
+ */
+package io.deephaven.plugin.js;
+
+import io.deephaven.annotations.SimpleStyle;
+import org.immutables.value.Value.Immutable;
+import org.immutables.value.Value.Parameter;
+
+import java.nio.file.Path;
+
+/**
+ * A manifest-based js plugin sourced from a {@value MANIFEST_JSON} file.
+ */
+@Immutable
+@SimpleStyle
+public abstract class JsPluginManifestPath extends JsPluginBase {
+
+    public static final String MANIFEST_JSON = "manifest.json";
+
+    /**
+     * Creates a manifest-based js plugin from {@code manifestRoot}.
+     *
+     * @param manifestRoot the manifest root directory path
+     * @return the manifest-based js plugin
+     */
+    public static JsPluginManifestPath of(Path manifestRoot) {
+        return ImmutableJsPluginManifestPath.of(manifestRoot);
+    }
+
+    /**
+     * The manifest root path directory path.
+     * 
+     * @return the manifest root directory path
+     */
+    @Parameter
+    public abstract Path path();
+
+    /**
+     * The {@value MANIFEST_JSON} file path, relative to {@link #path()}. Equivalent to
+     * {@code path().resolve(MANIFEST_JSON)}.
+     * 
+     * @return the manifest json file path
+     */
+    public final Path manifestJson() {
+        return path().resolve(MANIFEST_JSON);
+    }
+
+    /**
+     * Equivalent to {@code JsPluginPackagePath.of(path().resolve(name))}.
+     *
+     * @param name the package name
+     * @return the package path
+     */
+    public final JsPluginPackagePath packagePath(String name) {
+        return JsPluginPackagePath.of(path().resolve(name));
+    }
+}

plugin/src/main/java/io/deephaven/plugin/js/JsPluginPackagePath.java

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending
+ */
+package io.deephaven.plugin.js;
+
+import io.deephaven.annotations.SimpleStyle;
+import org.immutables.value.Value.Immutable;
+import org.immutables.value.Value.Parameter;
+
+import java.nio.file.Path;
+
+/**
+ * A package-based js plugin sourced from a {@value PACKAGE_JSON} file.
+ */
+@Immutable
+@SimpleStyle
+public abstract class JsPluginPackagePath extends JsPluginBase {
+    public static final String PACKAGE_JSON = "package.json";
+
+    /**
+     * Creates a package-based js plugin from {@code packageRoot}.
+     *
+     * @param packageRoot the package root directory path
+     * @return the package-based js plugin
+     */
+    public static JsPluginPackagePath of(Path packageRoot) {
+        return ImmutableJsPluginPackagePath.of(packageRoot);
+    }
+
+    /**
+     * The package root directory path.
+     * 
+     * @return the package root directory path
+     */
+    @Parameter
+    public abstract Path path();
+
+    /**
+     * The {@value PACKAGE_JSON} file path. Equivalent to {@code path().resolve(PACKAGE_JSON)}.
+     * 
+     * @return the package json file path
+     */
+    public final Path packageJson() {
+        return path().resolve(PACKAGE_JSON);
+    }
+}

server/jetty/src/main/java/io/deephaven/server/jetty/CopyHelper.java

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending
+ */
+package io.deephaven.server.jetty;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Objects;
+
+class CopyHelper {
+    static void copyRecursive(Path src, Path dst) throws IOException {
+        Files.createDirectories(dst.getParent());
+        Files.walkFileTree(src, new CopyRecursiveVisitor(src, dst));
+    }
+
+    private static class CopyRecursiveVisitor extends SimpleFileVisitor<Path> {
+        private final Path src;
+        private final Path dst;
+
+        public CopyRecursiveVisitor(Path src, Path dst) {
+            this.src = Objects.requireNonNull(src);
+            this.dst = Objects.requireNonNull(dst);
+        }
+
+        @Override
+        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+            // Note: toString() necessary for src/dst that don't share the same root FS
+            Files.copy(dir, dst.resolve(src.relativize(dir).toString()), StandardCopyOption.COPY_ATTRIBUTES);
+            return FileVisitResult.CONTINUE;
+        }
+
+        @Override
+        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+            // Note: toString() necessary for src/dst that don't share the same root FS
+            Files.copy(file, dst.resolve(src.relativize(file).toString()), StandardCopyOption.COPY_ATTRIBUTES);
+            return FileVisitResult.CONTINUE;
+        }
+    }
+}

server/jetty/src/main/java/io/deephaven/server/jetty/JettyBackedGrpcServer.java

@@ -3,6 +3,7 @@
  */
 package io.deephaven.server.jetty;
 
+import io.deephaven.plugin.js.JsPlugin;
 import io.deephaven.server.browserstreaming.BrowserStreamInterceptor;
 import io.deephaven.server.runner.GrpcServer;
 import io.deephaven.ssl.config.CiphersIntermediate;
@@ -12,10 +13,10 @@
 import io.deephaven.ssl.config.impl.KickstartUtils;
 import io.grpc.InternalStatus;
 import io.grpc.internal.GrpcUtil;
+import io.grpc.servlet.jakarta.web.GrpcWebFilter;
 import io.grpc.servlet.web.websocket.GrpcWebsocket;
 import io.grpc.servlet.web.websocket.MultiplexedWebSocketServerStream;
 import io.grpc.servlet.web.websocket.WebSocketServerStream;
-import io.grpc.servlet.jakarta.web.GrpcWebFilter;
 import jakarta.servlet.DispatcherType;
 import jakarta.websocket.Endpoint;
 import jakarta.websocket.server.ServerEndpointConfig;
@@ -36,7 +37,6 @@
 import org.eclipse.jetty.security.ConstraintSecurityHandler;
 import org.eclipse.jetty.server.ForwardedRequestCustomizer;
 import org.eclipse.jetty.server.Handler;
-import org.eclipse.jetty.server.HandlerContainer;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.SecureRequestCustomizer;
@@ -48,6 +48,7 @@
 import org.eclipse.jetty.servlet.DefaultServlet;
 import org.eclipse.jetty.servlet.ErrorPageErrorHandler;
 import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.servlets.CrossOriginFilter;
 import org.eclipse.jetty.util.MultiException;
 import org.eclipse.jetty.util.component.Graceful;
@@ -59,8 +60,10 @@
 import org.eclipse.jetty.websocket.jakarta.server.internal.JakartaWebSocketServerContainer;
 
 import javax.inject.Inject;
+import javax.inject.Singleton;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -70,15 +73,19 @@
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 
 import static io.grpc.servlet.web.websocket.MultiplexedWebSocketServerStream.GRPC_WEBSOCKETS_MULTIPLEX_PROTOCOL;
 import static io.grpc.servlet.web.websocket.WebSocketServerStream.GRPC_WEBSOCKETS_PROTOCOL;
 import static org.eclipse.jetty.servlet.ServletContextHandler.NO_SESSIONS;
 
+@Singleton
 public class JettyBackedGrpcServer implements GrpcServer {
+    private static final String JS_PLUGINS_PATH_SPEC = "/" + JsPlugins.JS_PLUGINS + "/*";
 
     private final Server jetty;
+    private final JsPlugins jsPlugins;
     private final boolean websocketsEnabled;
 
     @Inject
@@ -87,6 +94,11 @@ public JettyBackedGrpcServer(
             final GrpcFilter filter) {
         jetty = new Server();
         jetty.addConnector(createConnector(jetty, config));
+        try {
+            jsPlugins = JsPlugins.create();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
 
         final WebAppContext context =
                 new WebAppContext(null, "/", null, null, null, new ErrorPageErrorHandler(), NO_SESSIONS);
@@ -170,6 +182,9 @@ public JettyBackedGrpcServer(
         // Wire up the provided grpc filter
         context.addFilter(new FilterHolder(filter), "/*", EnumSet.noneOf(DispatcherType.class));
 
+        // Wire up /js-plugins/*
+        context.addServlet(servletHolder("js-plugins", jsPlugins.filesystem()), JS_PLUGINS_PATH_SPEC);
+
         // Set up websockets for grpc-web - depending on configuration, we can register both in case we encounter a
         // client using "vanilla"
         // grpc-websocket, that can't multiplex all streams on a single socket
@@ -207,8 +222,7 @@ public <T> T getEndpointInstance(Class<T> endpointClass) {
 
         // Note: handler order matters due to pathSpec order
         HandlerCollection handlers = new HandlerCollection();
-        // Set up /js-plugins/*
-        JsPlugins.maybeAdd(handlers::addHandler);
+
         // Set up /*
         handlers.addHandler(context);
 
@@ -230,6 +244,10 @@ public <T> T getEndpointInstance(Class<T> endpointClass) {
         jetty.setHandler(handler);
     }
 
+    public Consumer<JsPlugin> jsPluginConsumer() {
+        return jsPlugins;
+    }
+
     @Override
     public void start() throws IOException {
         try {
@@ -366,4 +384,16 @@ public void onClosed(Connection connection) {
 
         return serverConnector;
     }
+
+    private static ServletHolder servletHolder(String name, URI filesystemUri) {
+        final ServletHolder jsPlugins = new ServletHolder(name, DefaultServlet.class);
+        // Note, the URI needs explicitly be parseable as a directory URL ending in "!/", a requirement of the jetty
+        // resource creation implementation, see
+        // org.eclipse.jetty.util.resource.Resource.newResource(java.lang.String, boolean)
+        jsPlugins.setInitParameter("resourceBase", filesystemUri.toString());
+        jsPlugins.setInitParameter("pathInfoOnly", "true");
+        jsPlugins.setInitParameter("dirAllowed", "false");
+        jsPlugins.setAsyncSupported(true);
+        return jsPlugins;
+    }
 }

server/jetty/src/main/java/io/deephaven/server/jetty/JettyServerModule.java

@@ -6,7 +6,9 @@
 import dagger.Binds;
 import dagger.Module;
 import dagger.Provides;
+import io.deephaven.plugin.js.JsPlugin;
 import io.deephaven.server.config.ServerConfig;
+import io.deephaven.server.plugin.PluginsModule;
 import io.deephaven.server.runner.GrpcServer;
 import io.grpc.BindableService;
 import io.grpc.ServerInterceptor;
@@ -18,6 +20,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.function.Consumer;
 
 import static io.grpc.internal.GrpcUtil.getThreadFactory;
 
@@ -63,4 +66,10 @@ static ServletAdapter provideGrpcServletAdapter(
 
         return serverBuilder.buildServletAdapter();
     }
+
+    @Provides
+    @Named(PluginsModule.JS_PLUGIN_CONSUMER_NAME)
+    static Consumer<JsPlugin> providesJsPluginConsumer(JettyBackedGrpcServer server) {
+        return server.jsPluginConsumer();
+    }
 }