scalafix-interfaces: higher-level API for class loading

This ports the module resolution/fetching code currently in
sbt-scalafix into scalafix-interfaces. All JVM clients can now
benefit from it, and avoid runtime errors due to different Scala
binary versions across classloaders.

build.sbt

@@ -39,7 +39,6 @@ lazy val interfaces = project
           "scalafix-interfaces.properties"
       IO.write(props, "Scalafix version constants", out)
       List(out)
-
     },
     javacOptions.in(Compile) ++= List(
       "-Xlint:all",
@@ -48,6 +47,7 @@ lazy val interfaces = project
     javacOptions.in(Compile, doc) := List("-Xdoclint:none"),
     javaHome.in(Compile) := inferJavaHome(),
     javaHome.in(Compile, doc) := inferJavaHome(),
+    libraryDependencies += "io.get-coursier" % "interface" % coursierInterfaceV,
     moduleName := "scalafix-interfaces",
     crossPaths := false,
     autoScalaLibrary := false
@@ -242,7 +242,11 @@ lazy val unit = project
           "scalafix-tests" / "shared" / "src" / "main",
       "sharedClasspath" ->
         classDirectory.in(testsShared, Compile).value
-    )
+    ),
+    test.in(Test) := test
+      .in(Test)
+      .dependsOn(publishLocal.all(ScopeFilter(inAnyProject))) // scalafix.tests.interfaces.ScalafixSuite
+      .value
   )
   .enablePlugins(BuildInfoPlugin)
   .dependsOn(

project/Dependencies.scala

@@ -9,6 +9,7 @@ object Dependencies {
   def scala212 = "2.12.11"
   def scala213 = "2.13.2"
   def coursierV = "2.0.0-RC5-6"
+  def coursierInterfaceV = "0.0.22"
   val currentScalaVersion = scala213
 
   val jgit = "org.eclipse.jgit" % "org.eclipse.jgit" % "5.7.0.202003110725-r"

project/ScalafixBuild.scala

@@ -192,17 +192,20 @@ object ScalafixBuild extends AutoPlugin with GhpagesKeys {
         s
     },
     commands += Command.command("ci-212") { s =>
-      s"++$scala212" ::
+      s"++$scala213" :: "publishLocal" :: // scalafix.tests.interfaces.ScalafixSuite
+        s"++$scala212" ::
         "unit/test" ::
         s
     },
     commands += Command.command("ci-211") { s =>
-      s"++$scala211" ::
+      s"++$scala213" :: "publishLocal" :: // scalafix.tests.interfaces.ScalafixSuite
+        s"++$scala211" ::
         "unit/test" ::
         s
     },
     commands += Command.command("ci-213-windows") { s =>
       s"++$scala213" ::
+        "publishLocal" :: // scalafix.tests.interfaces.ScalafixSuite
         s"unit/testOnly -- -l scalafix.internal.tests.utils.SkipWindows" ::
         s
     },

scalafix-cli/src/main/scala/scalafix/internal/interfaces/ScalafixArgumentsImpl.scala

@@ -1,14 +1,17 @@
 package scalafix.internal.interfaces
 
 import java.io.PrintStream
-import java.net.URLClassLoader
+import java.net.{URL, URLClassLoader}
 import java.nio.charset.Charset
 import java.nio.file.Path
 import java.nio.file.PathMatcher
 import java.util
 import java.util.Optional
+
+import coursierapi.Repository
 import metaconfig.Conf
 import metaconfig.Configured
+
 import scala.jdk.CollectionConverters._
 import scala.meta.io.AbsolutePath
 import scala.meta.io.Classpath
@@ -23,6 +26,7 @@ import scalafix.internal.v1.Args
 import scalafix.internal.v1.MainOps
 import scalafix.internal.v1.Rules
 import scalafix.v1.RuleDecoder
+import scalafix.Versions
 
 final case class ScalafixArgumentsImpl(args: Args = Args.default)
     extends ScalafixArguments {
@@ -35,6 +39,41 @@ final case class ScalafixArgumentsImpl(args: Args = Args.default)
   override def withRules(rules: util.List[String]): ScalafixArguments =
     copy(args = args.copy(rules = rules.asScala.toList))
 
+  override def withToolClasspath(
+      customURLs: util.List[URL]
+  ): ScalafixArguments =
+    withToolClasspath(
+      new URLClassLoader(customURLs.asScala.toArray, getClass.getClassLoader)
+    )
+
+  override def withToolClasspath(
+      customURLs: util.List[URL],
+      extraDependenciesCoordinates: util.List[String]
+  ): ScalafixArguments =
+    withToolClasspath(
+      customURLs,
+      extraDependenciesCoordinates,
+      Repository.defaults()
+    )
+
+  override def withToolClasspath(
+      customURLs: util.List[URL],
+      customDependenciesCoordinates: util.List[String],
+      repositories: util.List[Repository]
+  ): ScalafixArguments = {
+    val extraDependenciesJARs = ScalafixCoursier.toolClasspath(
+      repositories,
+      customDependenciesCoordinates,
+      Versions.scalaVersion
+    )
+    val extraURLs = customURLs.asScala ++ extraDependenciesJARs.asScala
+    val classLoader = new URLClassLoader(
+      extraURLs.toArray,
+      getClass.getClassLoader
+    )
+    withToolClasspath(classLoader)
+  }
+
   override def withToolClasspath(
       classLoader: URLClassLoader
   ): ScalafixArguments =

scalafix-interfaces/src/main/java/scalafix/interfaces/Scalafix.java

@@ -1,21 +1,30 @@
 package scalafix.interfaces;
 
+import coursierapi.Repository;
+import scalafix.internal.interfaces.ScalafixCoursier;
+import scalafix.internal.interfaces.ScalafixInterfacesClassloader;
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.List;
+import java.util.Properties;
 
 /**
  * Public API for reflectively invoking Scalafix from a build tool or IDE integration.
- *
- * To obtain an instance of Scalafix, use {@link Scalafix#classloadInstance(ClassLoader)}.
+ * <p>
+ * To obtain an instance of Scalafix, use one of the static factory methods.
  *
  * @implNote This interface is not intended to be extended, the only implementation of this interface
  * should live in the Scalafix repository.
  */
 public interface Scalafix {
 
     /**
-     * @return Construct a new instance of {@link ScalafixArguments} that can be later passed to {@link #runMain(ScalafixArguments) }.
+     * @return Construct a new instance of {@link ScalafixArguments}.
      */
     ScalafixArguments newArguments();
 
@@ -58,15 +67,96 @@ public interface Scalafix {
      */
     String scala213();
 
+    /**
+     * Fetch JARs containing an implementation of {@link Scalafix} using Coursier and classload an instance of it via
+     * runtime reflection.
+     * <p>
+     * The custom classloader optionally provided with {@link ScalafixArguments#withToolClasspath} to compile and
+     * classload external rules must have the classloader of the returned instance as ancestor to share a common
+     * loaded instance of `scalafix-core`, and therefore have been compiled against the requested Scala binary version.
+     *
+     * @param scalaBinaryVersion The Scala binary version ("2.13" for example) available in the classloader of the
+     *                           returned instance. To be able to run advanced semantic rules using the Scala
+     *                           Presentation Compiler such as ExplicitResultTypes, this must match the binary
+     *                           version that the target classpath was built with, as provided with
+     *                           {@link ScalafixArguments#withScalaVersion}.
+     * @return An implementation of the {@link Scalafix} interface.
+     * @throws ScalafixException in case of errors during artifact resolution/fetching.
+     */
+    static Scalafix fetchAndClassloadInstance(String scalaBinaryVersion) throws ScalafixException {
+        return fetchAndClassloadInstance(scalaBinaryVersion, Repository.defaults());
+    }
+
+    /**
+     * Fetch JARs containing an implementation of {@link Scalafix} from the provided repositories using Coursier and
+     * classload an instance of it via runtime reflection.
+     * <p>
+     * The custom classloader optionally provided with {@link ScalafixArguments#withToolClasspath} to compile and
+     * classload external rules must have the classloader of the returned instance as ancestor to share a common
+     * loaded instance of `scalafix-core`, and therefore have been compiled against the requested Scala binary version.
+     *
+     * @param scalaBinaryVersion The Scala binary version ("2.13" for example) available in the classloader of the
+     *                           returned instance. To be able to run advanced semantic rules using the Scala
+     *                           Presentation Compiler such as ExplicitResultTypes, this must match the binary
+     *                           version that the target classpath was built with, as provided with
+     *                           {@link ScalafixArguments#withScalaVersion}.
+     * @param repositories       Maven/Ivy repositories to fetch the JARs from.
+     * @return An implementation of the {@link Scalafix} interface.
+     * @throws ScalafixException in case of errors during artifact resolution/fetching.
+     */
+    static Scalafix fetchAndClassloadInstance(String scalaBinaryVersion, List<Repository> repositories)
+            throws ScalafixException {
+
+        String scalaVersionKey;
+        switch (scalaBinaryVersion) {
+            case "2.11":
+                scalaVersionKey = "scala211";
+                break;
+            case "2.12":
+                scalaVersionKey = "scala212";
+                break;
+            case "2.13":
+                scalaVersionKey = "scala213";
+                break;
+            default:
+                throw new IllegalArgumentException("Unsupported scala version " + scalaBinaryVersion);
+        }
 
+        Properties properties = new Properties();
+        String propertiesPath = "scalafix-interfaces.properties";
+        InputStream stream = Scalafix.class.getClassLoader().getResourceAsStream(propertiesPath);
+        try {
+            properties.load(stream);
+        } catch (IOException | NullPointerException e) {
+            throw new ScalafixException("Failed to load '" + propertiesPath + "' to lookup versions", e);
+        }
+
+        String scalafixVersion = properties.getProperty("scalafixVersion");
+        String scalaVersion = properties.getProperty(scalaVersionKey);
+        if (scalafixVersion == null || scalaVersion == null)
+            throw new ScalafixException("Failed to lookup versions from '" + propertiesPath + "'");
+
+        List<URL> jars = ScalafixCoursier.scalafixCliJars(repositories, scalafixVersion, scalaVersion);
+        ClassLoader parent = new ScalafixInterfacesClassloader(Scalafix.class.getClassLoader());
+        return classloadInstance(new URLClassLoader(jars.stream().toArray(URL[]::new), parent));
+    }
 
     /**
      * JVM runtime reflection method helper to classload an instance of {@link Scalafix}.
+     * <p>
+     * The custom classloader optionally provided with {@link ScalafixArguments#withToolClasspath} to compile and
+     * classload external rules must have the provided classloader as ancestor to share a common loaded instance
+     * of `scalafix-core`, and therefore must have been compiled against the same Scala binary version as
+     * the one in the classLoader provided here.
      *
-     * @param classLoader Classloader containing the full Scalafix classpath, including the scalafix-cli module.
+     * @param classLoader Classloader containing the full Scalafix classpath, including the scalafix-cli module. To be
+     *                    able to run advanced semantic rules using the Scala Presentation Compiler such as
+     *                    ExplicitResultTypes, this Scala binary version in that classloader should match the one that
+     *                    the target classpath was built with, as provided with
+     *                    {@link ScalafixArguments#withScalaVersion}.
      * @return An implementation of the {@link Scalafix} interface.
      * @throws ScalafixException in case of errors during classloading, most likely caused
-     * by an incorrect classloader argument.
+     *                           by an incorrect classloader argument.
      */
     static Scalafix classloadInstance(ClassLoader classLoader) throws ScalafixException {
         try {

scalafix-interfaces/src/main/java/scalafix/interfaces/ScalafixArguments.java

@@ -1,6 +1,9 @@
 package scalafix.interfaces;
 
+import coursierapi.Repository;
+
 import java.io.PrintStream;
+import java.net.URL;
 import java.net.URLClassLoader;
 import java.nio.charset.Charset;
 import java.nio.file.Path;
@@ -13,7 +16,10 @@
  * <p>
  * To obtain an instance of ScalafixArguments, use {@link scalafix.interfaces.Scalafix#newArguments()}.
  * Instances of ScalafixArguments are immutable and thread safe. It is safe to re-use the same
- * ScalafixArguments instance for multiple Scalafix invocations. *
+ * ScalafixArguments instance for multiple Scalafix invocations. Re-using the same instance is
+ * particularly encouraged when a custom toolClasspath is provided, in order to amortize the
+ * cost/time of fetching artifacts, classloading them & warming up the JIT code cache.
+ *
  * @implNote This interface is not intended for extension, the only implementation of this interface
  * should live in the Scalafix repository.
  */
@@ -25,10 +31,45 @@ public interface ScalafixArguments {
      */
     ScalafixArguments withRules(List<String> rules);
 
+    /**
+     * @param customURLs Extra URLs for classloading and compiling external rules.
+     */
+    ScalafixArguments withToolClasspath(List<URL> customURLs);
+
+    /**
+     * @param customURLs                   Extra URLs for classloading and compiling external rules.
+     * @param extraDependenciesCoordinates Extra dependencies for classloading and compiling external rules.
+     *                                     For example "com.github.liancheng::organize-imports:0.3.0".
+     *                                     Artifacts will be resolved against the Scala version in the classloader
+     *                                     of the parent {@link Scalafix} instance and fetched using Coursier.
+     * @throws ScalafixException in case of errors during artifact resolution/fetching.
+     */
+    ScalafixArguments withToolClasspath(
+            List<URL> customURLs,
+            List<String> extraDependenciesCoordinates
+    ) throws ScalafixException;
+
+    /**
+     * @param customURLs                    Extra URLs for classloading and compiling external rules.
+     * @param customDependenciesCoordinates Extra dependencies for classloading and compiling external rules.
+     *                                      For example "com.github.liancheng::organize-imports:0.3.0".
+     *                                      Artifacts will be resolved against the Scala version in the classloader
+     *                                      of the parent {@link Scalafix} instance and fetched using Coursier.
+     * @param repositories                  Maven/Ivy repositories to fetch the artifacts from.
+     * @throws ScalafixException in case of errors during artifact resolution/fetching.
+     */
+    ScalafixArguments withToolClasspath(
+            List<URL> customURLs,
+            List<String> customDependenciesCoordinates,
+            List<Repository> repositories
+    ) throws ScalafixException;
+
     /**
      * @param toolClasspath Custom classpath for classloading and compiling external rules.
      *                      Must be a URLClassLoader (not regular ClassLoader) to support
-     *                      compiling sources.
+     *                      compiling sources. This classloader should have as ancestor the
+     *                      classloader of the {@link Scalafix} instance that returned this
+     *                      {@link ScalafixArguments} instance.
      */
     ScalafixArguments withToolClasspath(URLClassLoader toolClasspath);
 
@@ -69,7 +110,6 @@ public interface ScalafixArguments {
 
     /**
      * @param args Unparsed command-line arguments that are fed directly to <code>main(Array[String])</code>
-     *
      * @throws ScalafixException In case of an error parsing the provided arguments.
      */
     ScalafixArguments withParsedArguments(List<String> args) throws ScalafixException;
@@ -111,8 +151,11 @@ public interface ScalafixArguments {
 
     /**
      * @param version The Scala compiler version used to compile this classpath.
-     *                For example "2.12.8".
-     *
+     *                For example "2.12.8". To be able to run advanced semantic rules
+     *                using the Scala Presentation Compiler such as ExplicitResultTypes,
+     *                this must match the binary version available in the classloader of
+     *                this instance, as requested/provided in the static factory methods
+     *                of {@link Scalafix}.
      */
     ScalafixArguments withScalaVersion(String version);
 
@@ -125,7 +168,7 @@ public interface ScalafixArguments {
 
     /**
      * The rules that are valid arguments for {@link #withRules(List) }.
-     *
+     * <p>
      * Takes into account built-in rules as well as the tool classpath provided via
      * {@link #withToolClasspath(URLClassLoader) }.
      */
@@ -134,16 +177,17 @@ public interface ScalafixArguments {
 
     /**
      * The rules that would run when calling {@link #run() }
-     *
+     * <p>
      * Takes into account rules that are configured in .scalafix.conf.
+     *
      * @throws ScalafixException In case of an error loading the configured rules.
      */
     List<ScalafixRule> rulesThatWillRun() throws ScalafixException;
 
 
     /**
      * Validates that the passed arguments are valid.
-     *
+     * <p>
      * Takes into account provided rules, .scalafix.conf configuration, scala version,
      * scalac options and other potential problems. The primary purpose
      * of this method is to validate the arguments before starting compilation

scalafix-interfaces/src/main/java/scalafix/interfaces/ScalafixException.java

@@ -1,11 +1,15 @@
 package scalafix.interfaces;
 
 /**
- * An error occurred while classloading an instance of {@link Scalafix}.
+ * An error occurred while loading an instance of {@link Scalafix}.
  */
 public class ScalafixException extends Exception {
     static final long serialVersionUID = 118L;
     public ScalafixException(String message, Throwable cause) {
         super(message, cause);
     }
+
+    public ScalafixException(String message) {
+        super(message);
+    }
 }