From 910e8a4d38ac31ee6a1f836e75328e3a80967784 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 8 Nov 2024 14:56:52 +0100 Subject: [PATCH 01/27] Remove trailing space --- build.mill | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.mill b/build.mill index 5f0c3a8457b..fc1e153458e 100644 --- a/build.mill +++ b/build.mill @@ -30,7 +30,7 @@ object Settings { val docUrl = "https://mill-build.org" // the exact branches containing a doc root val docBranches = Seq() - // the exact tags containing a doc root. Publish docs for + // the exact tags containing a doc root. Publish docs for // the last point version in each minor release series val legacyDocTags: Seq[String] = Seq( "0.9.12", From 4f5f0704ddb081ad4f288f8d273b31fb94ac2dcd Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 8 Nov 2024 14:56:59 +0100 Subject: [PATCH 02/27] NIT Use allIvyDeps task --- scalalib/src/mill/scalalib/JavaModule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 29638578cde..07708bc24c7 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -327,7 +327,7 @@ trait JavaModule * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. */ def transitiveIvyDeps: T[Agg[BoundDep]] = Task { - (ivyDeps() ++ mandatoryIvyDeps()).map(bindDependency()) ++ + allIvyDeps().map(bindDependency()) ++ T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten } From 3e8643de9b7df40d89a0e347ff8e0395f3721340 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 8 Nov 2024 14:56:42 +0100 Subject: [PATCH 03/27] WIP Add BOM support --- build.mill | 2 +- integration/feature/bom/resources/build.mill | 34 +++++++++++ integration/feature/bom/src/BomTests.scala | 58 +++++++++++++++++++ main/util/src/mill/util/CoursierSupport.scala | 25 ++++---- .../src/mill/scalalib/CoursierModule.scala | 19 ++++-- scalalib/src/mill/scalalib/JavaModule.scala | 18 ++++-- .../src/mill/scalalib/JsonFormatters.scala | 4 ++ scalalib/src/mill/scalalib/Lib.scala | 13 +++-- .../src/mill/scalalib/PublishModule.scala | 7 ++- scalalib/src/mill/scalalib/publish/Pom.scala | 32 ++++++---- .../src/mill/scalalib/publish/PomTests.scala | 3 +- 11 files changed, 180 insertions(+), 35 deletions(-) create mode 100644 integration/feature/bom/resources/build.mill create mode 100644 integration/feature/bom/src/BomTests.scala diff --git a/build.mill b/build.mill index fc1e153458e..04dda3fdc3c 100644 --- a/build.mill +++ b/build.mill @@ -114,7 +114,7 @@ object Deps { val asmTree = ivy"org.ow2.asm:asm-tree:9.7.1" val bloopConfig = ivy"ch.epfl.scala::bloop-config:1.5.5" - val coursier = ivy"io.get-coursier::coursier:2.1.14" + val coursier = ivy"io.get-coursier::coursier:2.1.17" val coursierInterface = ivy"io.get-coursier:interface:1.0.22" val cask = ivy"com.lihaoyi::cask:0.9.4" diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill new file mode 100644 index 00000000000..a9af3274db4 --- /dev/null +++ b/integration/feature/bom/resources/build.mill @@ -0,0 +1,34 @@ +package build + +import mill._ +import mill.scalalib._ + +object `google-cloud-java` extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:_" + ) +} + +object `google-cloud-java-no-bom` extends JavaModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:_" + ) +} + +object `google-cloud-scala` extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) +} + +object `google-cloud-scala-no-bom` extends JavaModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) +} diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala new file mode 100644 index 00000000000..5c78182ca84 --- /dev/null +++ b/integration/feature/bom/src/BomTests.scala @@ -0,0 +1,58 @@ +package mill.integration + +import mill.testkit.{IntegrationTester, UtestIntegrationTestSuite} +import utest._ + +object BomTests extends UtestIntegrationTestSuite { + + def tests: Tests = Tests { + + def expectedProtobufJavaVersion = "4.28.3" + + def compileClasspathFileNames(tester: IntegrationTester, moduleName: String): Seq[String] = { + import tester._ + val res = eval( + ("show", s"$moduleName.compileClasspath"), + stderr = os.Inherit, + check = true + ) + ujson.read(res.out).arr.map(v => os.Path(v.str.split(":").last).last).toSeq + } + + test("googleCloudJavaCheck") - integrationTest { tester => + import tester._ + + val res = eval( + ("show", "google-cloud-java-no-bom.compileClasspath"), + check = false + ) + assert( + res.err.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" + ) + ) + } + + test("googleCloudJava") - integrationTest { tester => + val compileClasspathFileNames0 = compileClasspathFileNames(tester, "google-cloud-java") + assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + } + + test("googleCloudScalaCheck") - integrationTest { tester => + val compileClasspathFileNames0 = + compileClasspathFileNames(tester, "google-cloud-scala-no-bom") + assert( + compileClasspathFileNames0.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar")) + ) + assert( + !compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar") + ) + } + + test("googleCloudScala") - integrationTest { tester => + val compileClasspathFileNames0 = compileClasspathFileNames(tester, "google-cloud-scala") + assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + } + + } +} diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index ba09b7bfb0b..5dc3cc76cd2 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -45,7 +45,8 @@ trait CoursierSupport { ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, resolveFilter: os.Path => Boolean = _ => true, - artifactTypes: Option[Set[Type]] = None + artifactTypes: Option[Set[Type]] = None, + bomDeps: IterableOnce[Dependency] = Nil ): Result[Agg[PathRef]] = { def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { val org = dep.module.organization.value @@ -61,12 +62,8 @@ trait CoursierSupport { classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq) } - val (localTestDeps, remoteDeps) = deps.iterator.toSeq.partitionMap(d => - isLocalTestDep(d) match { - case None => Right(d) - case Some(vs) => Left(vs) - } - ) + val (localTestDeps, remoteDeps) = + deps.iterator.toSeq.partitionMap(d => isLocalTestDep(d).toLeft(d)) val resolutionRes = resolveDependenciesMetadataSafe( repositories, @@ -75,7 +72,8 @@ trait CoursierSupport { mapDependencies, customizer, ctx, - coursierCacheCustomizer + coursierCacheCustomizer, + bomDeps ) resolutionRes.flatMap { resolution => @@ -159,7 +157,8 @@ trait CoursierSupport { mapDependencies, customizer, ctx, - coursierCacheCustomizer + coursierCacheCustomizer, + Nil ) (deps0, res.getOrThrow) } @@ -171,13 +170,18 @@ trait CoursierSupport { mapDependencies: Option[Dependency => Dependency] = None, customizer: Option[Resolution => Resolution] = None, ctx: Option[mill.api.Ctx.Log] = None, - coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, + bomDeps: IterableOnce[Dependency] = Nil ): Result[Resolution] = { val rootDeps = deps.iterator .map(d => mapDependencies.fold(d)(_.apply(d))) .toSeq + val bomDeps0 = bomDeps.iterator + .map(d => mapDependencies.fold(d)(_.apply(d))) + .toSeq + val forceVersions = force.iterator .map(mapDependencies.getOrElse(identity[Dependency](_))) .map { d => d.module -> d.version } @@ -191,6 +195,7 @@ trait CoursierSupport { val resolve = Resolve() .withCache(coursierCache0) .withDependencies(rootDeps) + .withBomDependencies(bomDeps0) .withRepositories(repositories) .withResolutionParams(resolutionParams) .withMapDependenciesOpt(mapDependencies) diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index a952acb62eb..153a78eabb9 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -54,12 +54,14 @@ trait CoursierModule extends mill.Module { def resolveDeps( deps: Task[Agg[BoundDep]], sources: Boolean = false, - artifactTypes: Option[Set[Type]] = None + artifactTypes: Option[Set[Type]] = None, + bomDeps: Task[Agg[BoundDep]] = Task.Anon(Agg.empty[BoundDep]) ): Task[Agg[PathRef]] = Task.Anon { Lib.resolveDependencies( repositories = repositoriesTask(), deps = deps(), + bomDeps = bomDeps(), sources = sources, artifactTypes = artifactTypes, mapDependencies = Some(mapDependencies()), @@ -74,7 +76,7 @@ trait CoursierModule extends mill.Module { deps: Task[Agg[BoundDep]], sources: Boolean ): Task[Agg[PathRef]] = - resolveDeps(deps, sources, None) + resolveDeps(deps, sources, None, Task.Anon(Agg.empty[BoundDep])) /** * Map dependencies before resolving them. @@ -148,11 +150,13 @@ object CoursierModule { def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean = false, - artifactTypes: Option[Set[coursier.Type]] = None + artifactTypes: Option[Set[coursier.Type]] = None, + bomDeps: IterableOnce[T] = Nil ): Agg[PathRef] = { Lib.resolveDependencies( repositories = repositories, deps = deps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)), + bomDeps = bomDeps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)), sources = sources, artifactTypes = artifactTypes, mapDependencies = mapDependencies, @@ -162,12 +166,19 @@ object CoursierModule { ).getOrThrow } + def resolveDeps[T: CoursierModule.Resolvable]( + deps: IterableOnce[T], + sources: Boolean, + artifactTypes: Option[Set[coursier.Type]] + ): Agg[PathRef] = + resolveDeps(deps, sources, artifactTypes, Nil) + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean ): Agg[PathRef] = - resolveDeps(deps, sources, None) + resolveDeps(deps, sources, None, Nil) } sealed trait Resolvable[T] { diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 07708bc24c7..9d9ad2ade96 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -155,6 +155,12 @@ trait JavaModule */ def runIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** + * Any BOM dependencies you want to add to this Module, in the format + * ivy"org:name:version" + */ + def bomDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** * Default artifact types to fetch and put in the classpath. Add extra types * here if you'd like fancy artifact extensions to be fetched. @@ -604,7 +610,8 @@ trait JavaModule def resolvedIvyDeps: T[Agg[PathRef]] = Task { defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), - artifactTypes = Some(artifactTypes()) + artifactTypes = Some(artifactTypes()), + bomDeps = bomDeps().map(bindDependency()) ) } @@ -619,7 +626,8 @@ trait JavaModule def resolvedRunIvyDeps: T[Agg[PathRef]] = Task { defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), - artifactTypes = Some(artifactTypes()) + artifactTypes = Some(artifactTypes()), + bomDeps = bomDeps().map(bindDependency()) ) } @@ -1096,13 +1104,15 @@ trait JavaModule Task.Anon { defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), - sources = true + sources = true, + bomDeps = bomDeps().map(bindDependency()) ) }, Task.Anon { defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), - sources = true + sources = true, + bomDeps = bomDeps().map(bindDependency()) ) } ) diff --git a/scalalib/src/mill/scalalib/JsonFormatters.scala b/scalalib/src/mill/scalalib/JsonFormatters.scala index 751ebcc60b3..8093d353ae8 100644 --- a/scalalib/src/mill/scalalib/JsonFormatters.scala +++ b/scalalib/src/mill/scalalib/JsonFormatters.scala @@ -22,6 +22,10 @@ trait JsonFormatters { implicit lazy val configurationFormat: RW[coursier.core.Configuration] = upickle.default.macroRW implicit lazy val typeFormat: RW[coursier.core.Type] = upickle.default.macroRW implicit lazy val classifierFormat: RW[coursier.core.Classifier] = upickle.default.macroRW + implicit lazy val depMgmtKeyFormat: RW[coursier.core.DependencyManagement.Key] = + upickle.default.macroRW + implicit lazy val depMgmtValuesFormat: RW[coursier.core.DependencyManagement.Values] = + upickle.default.macroRW } object JsonFormatters extends JsonFormatters diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index 07a10461524..6eb8721e5de 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -59,7 +59,8 @@ object Lib { ctx: Option[Ctx.Log] = None, coursierCacheCustomizer: Option[ coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] - ] = None + ] = None, + bomDeps: IterableOnce[BoundDep] = Nil ): Result[Resolution] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependenciesMetadataSafe( @@ -69,7 +70,8 @@ object Lib { mapDependencies = mapDependencies, customizer = customizer, ctx = ctx, - coursierCacheCustomizer = coursierCacheCustomizer + coursierCacheCustomizer = coursierCacheCustomizer, + bomDeps = bomDeps.iterator.toSeq.map(_.dep) ) } @@ -90,12 +92,14 @@ object Lib { coursierCacheCustomizer: Option[ coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] ] = None, - artifactTypes: Option[Set[Type]] = None + artifactTypes: Option[Set[Type]] = None, + bomDeps: IterableOnce[BoundDep] = Nil ): Result[Agg[PathRef]] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependencies( repositories = repositories, deps = depSeq.map(_.dep), + bomDeps = bomDeps.iterator.map(_.dep).toSeq, force = depSeq.filter(_.force).map(_.dep), sources = sources, artifactTypes = artifactTypes, @@ -126,7 +130,8 @@ object Lib { customizer, ctx, coursierCacheCustomizer, - None + None, + Nil ) def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String): Loose.Agg[Dep] = diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index eb3a7a22f01..0a068793056 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -82,6 +82,10 @@ trait PublishModule extends JavaModule { outer => compileModulePomDeps.map(Dependency(_, Scope.Provided)) } + def publishXmlBomDeps: Task[Agg[Dependency]] = Task.Anon { + bomDeps().map(resolvePublishDependency.apply().apply(_)) + } + def pom: T[PathRef] = Task { val pom = Pom( artifactMetadata(), @@ -89,7 +93,8 @@ trait PublishModule extends JavaModule { outer => artifactId(), pomSettings(), publishProperties(), - packagingType = pomPackagingType + packagingType = pomPackagingType, + bomDependencies = publishXmlBomDeps() ) val pomPath = T.dest / s"${artifactId()}-${publishVersion()}.pom" os.write.over(pomPath, pom) diff --git a/scalalib/src/mill/scalalib/publish/Pom.scala b/scalalib/src/mill/scalalib/publish/Pom.scala index ab39a88b81f..a8678c2b817 100644 --- a/scalalib/src/mill/scalalib/publish/Pom.scala +++ b/scalalib/src/mill/scalalib/publish/Pom.scala @@ -38,7 +38,8 @@ object Pom { name = name, pomSettings = pomSettings, properties = properties, - packagingType = pomSettings.packaging + packagingType = pomSettings.packaging, + bomDependencies = Agg.empty[Dependency] ) def apply( @@ -47,7 +48,8 @@ object Pom { name: String, pomSettings: PomSettings, properties: Map[String, String], - packagingType: String + packagingType: String, + bomDependencies: Agg[Dependency] ): String = { val xml = - {dependencies.map(renderDependency).iterator} + { + dependencies.map(renderDependency(_)).iterator ++ + bomDependencies.map(renderDependency(_, isImport = true)).iterator + } @@ -114,13 +119,18 @@ object Pom { {property._2}.copy(label = property._1) } - private def renderDependency(d: Dependency): Elem = { - val scope = d.scope match { - case Scope.Compile => NodeSeq.Empty - case Scope.Provided => provided - case Scope.Test => test - case Scope.Runtime => runtime - } + private def renderDependency(d: Dependency, isImport: Boolean = false): Elem = { + val scope = + if (isImport) import + else + d.scope match { + case Scope.Compile => NodeSeq.Empty + case Scope.Provided => provided + case Scope.Test => test + case Scope.Runtime => runtime + } + + val `type` = if (isImport) pom else NodeSeq.Empty val optional = if (d.optional) true else NodeSeq.Empty @@ -130,6 +140,7 @@ object Pom { {d.artifact.id} {d.artifact.version} {scope} + {`type`} {optional} else @@ -146,6 +157,7 @@ object Pom { } {scope} + {`type`} {optional} } diff --git a/scalalib/test/src/mill/scalalib/publish/PomTests.scala b/scalalib/test/src/mill/scalalib/publish/PomTests.scala index d1b52b602aa..6ff395f2647 100644 --- a/scalalib/test/src/mill/scalalib/publish/PomTests.scala +++ b/scalalib/test/src/mill/scalalib/publish/PomTests.scala @@ -222,7 +222,8 @@ object PomTests extends TestSuite { artifactId, pomSettings, properties, - PackagingType.Jar + PackagingType.Jar, + Agg.empty[Dependency] )) def singleText(seq: NodeSeq) = From bc5e1b3bbf9a9ba6fa4bc0af7340311271eb6bb0 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 13 Nov 2024 15:21:17 +0100 Subject: [PATCH 04/27] more --- build.mill | 6 +- integration/feature/bom/resources/build.mill | 17 +++++- integration/feature/bom/src/BomTests.scala | 42 +++++++++++++ main/util/src/mill/util/CoursierSupport.scala | 61 +++++++++++++++++-- .../src/mill/scalalib/CoursierModule.scala | 35 ++++++++--- scalalib/src/mill/scalalib/JavaModule.scala | 15 ++++- scalalib/src/mill/scalalib/Lib.scala | 61 ++++++++++++++++++- .../src/mill/scalalib/PublishModule.scala | 29 ++++++++- scalalib/src/mill/scalalib/publish/Ivy.scala | 13 +++- scalalib/src/mill/scalalib/publish/Pom.scala | 8 ++- .../scalalib/CoursierParametersTests.scala | 44 +++++++++++++ 11 files changed, 305 insertions(+), 26 deletions(-) create mode 100644 scalalib/test/src/mill/scalalib/CoursierParametersTests.scala diff --git a/build.mill b/build.mill index c9cff8dd859..c547d63ac6d 100644 --- a/build.mill +++ b/build.mill @@ -30,7 +30,7 @@ object Settings { val docUrl = "https://mill-build.org" // the exact branches containing a doc root val docBranches = Seq() - // the exact tags containing a doc root. Publish docs for + // the exact tags containing a doc root. Publish docs for // the last point version in each minor release series val legacyDocTags: Seq[String] = Seq( "0.9.12", @@ -116,8 +116,8 @@ object Deps { val asmTree = ivy"org.ow2.asm:asm-tree:9.7.1" val bloopConfig = ivy"ch.epfl.scala::bloop-config:1.5.5" - val coursier = ivy"io.get-coursier::coursier:2.1.17" - val coursierInterface = ivy"io.get-coursier:interface:1.0.22" + val coursier = ivy"io.get-coursier::coursier:2.1.18" + val coursierInterface = ivy"io.get-coursier:interface:1.0.24" val cask = ivy"com.lihaoyi::cask:0.9.4" val castor = ivy"com.lihaoyi::castor:0.3.0" diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill index a9af3274db4..612fffe97c6 100644 --- a/integration/feature/bom/resources/build.mill +++ b/integration/feature/bom/resources/build.mill @@ -2,8 +2,21 @@ package build import mill._ import mill.scalalib._ +import mill.scalalib.publish._ -object `google-cloud-java` extends JavaModule { +trait TestPublishModule extends PublishModule { + def pomSettings = PomSettings( + description = artifactName(), + organization = "com.lihaoyi.mill-tests", + url = "https://github.com/com-lihaoyi/mill", + licenses = Seq(License.`Apache-2.0`), + versionControl = VersionControl.github("com-lihaoyi", "mill"), + developers = Nil + ) + def publishVersion = "0.1.0-SNAPSHOT" +} + +object `google-cloud-java` extends JavaModule with TestPublishModule { def bomDeps = Agg( ivy"com.google.cloud:libraries-bom:26.50.0" ) @@ -18,7 +31,7 @@ object `google-cloud-java-no-bom` extends JavaModule { ) } -object `google-cloud-scala` extends JavaModule { +object `google-cloud-scala` extends JavaModule with TestPublishModule { def bomDeps = Agg( ivy"com.google.cloud:libraries-bom:26.50.0" ) diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala index 5c78182ca84..e836d792826 100644 --- a/integration/feature/bom/src/BomTests.scala +++ b/integration/feature/bom/src/BomTests.scala @@ -3,6 +3,8 @@ package mill.integration import mill.testkit.{IntegrationTester, UtestIntegrationTestSuite} import utest._ +import scala.jdk.CollectionConverters._ + object BomTests extends UtestIntegrationTestSuite { def tests: Tests = Tests { @@ -34,8 +36,28 @@ object BomTests extends UtestIntegrationTestSuite { } test("googleCloudJava") - integrationTest { tester => + import tester._ + val compileClasspathFileNames0 = compileClasspathFileNames(tester, "google-cloud-java") assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + + val repo = workspacePath / "ivy2Local" + eval(("google-cloud-java.publishLocal", "--localIvyRepo", repo), check = true) + + val obtainedClassPath = coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of("com.lihaoyi.mill-tests", "google-cloud-java", "0.1.0-SNAPSHOT") + ) + .addRepositories( + coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + + val obtainedClassPathFileNames = obtainedClassPath.map(_.last) + assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) } test("googleCloudScalaCheck") - integrationTest { tester => @@ -50,8 +72,28 @@ object BomTests extends UtestIntegrationTestSuite { } test("googleCloudScala") - integrationTest { tester => + import tester._ + val compileClasspathFileNames0 = compileClasspathFileNames(tester, "google-cloud-scala") assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + + val repo = workspacePath / "ivy2Local" + eval(("google-cloud-scala.publishLocal", "--localIvyRepo", repo), check = true) + + val obtainedClassPath = coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of("com.lihaoyi.mill-tests", "google-cloud-scala", "0.1.0-SNAPSHOT") + ) + .addRepositories( + coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + + val obtainedClassPathFileNames = obtainedClassPath.map(_.last) + assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) } } diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 5dc3cc76cd2..060873847d1 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -46,6 +46,7 @@ trait CoursierSupport { coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, resolveFilter: os.Path => Boolean = _ => true, artifactTypes: Option[Set[Type]] = None, + resolutionParams: ResolutionParams = ResolutionParams(), bomDeps: IterableOnce[Dependency] = Nil ): Result[Agg[PathRef]] = { def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { @@ -73,6 +74,7 @@ trait CoursierSupport { customizer, ctx, coursierCacheCustomizer, + resolutionParams, bomDeps ) @@ -112,6 +114,33 @@ trait CoursierSupport { } } + @deprecated("Use the override accepting resolutionParams", "Mill after 0.12.2") + def resolveDependencies( + repositories: Seq[Repository], + deps: IterableOnce[Dependency], + force: IterableOnce[Dependency], + sources: Boolean, + mapDependencies: Option[Dependency => Dependency], + customizer: Option[Resolution => Resolution], + ctx: Option[mill.api.Ctx.Log], + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]], + resolveFilter: os.Path => Boolean, + artifactTypes: Option[Set[Type]] + ): Result[Agg[PathRef]] = + resolveDependencies( + repositories, + deps, + force, + sources, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + resolveFilter, + artifactTypes, + ResolutionParams() + ) + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDependencies( repositories: Seq[Repository], @@ -133,7 +162,8 @@ trait CoursierSupport { customizer, ctx, coursierCacheCustomizer, - resolveFilter + resolveFilter, + None ) @deprecated( @@ -158,6 +188,7 @@ trait CoursierSupport { customizer, ctx, coursierCacheCustomizer, + ResolutionParams(), Nil ) (deps0, res.getOrThrow) @@ -171,6 +202,7 @@ trait CoursierSupport { customizer: Option[Resolution => Resolution] = None, ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, + resolutionParams: ResolutionParams = ResolutionParams(), bomDeps: IterableOnce[Dependency] = Nil ): Result[Resolution] = { @@ -189,15 +221,15 @@ trait CoursierSupport { val coursierCache0 = coursierCache(ctx, coursierCacheCustomizer) - val resolutionParams = ResolutionParams() - .withForceVersion(forceVersions) + val resolutionParams0 = resolutionParams + .addForceVersion(forceVersions.toSeq: _*) val resolve = Resolve() .withCache(coursierCache0) .withDependencies(rootDeps) .withBomDependencies(bomDeps0) .withRepositories(repositories) - .withResolutionParams(resolutionParams) + .withResolutionParams(resolutionParams0) .withMapDependenciesOpt(mapDependencies) resolve.either() match { @@ -235,6 +267,27 @@ trait CoursierSupport { } } + @deprecated("Use the override accepting resolutionParams", "Mill after 0.12.2") + def resolveDependenciesMetadataSafe( + repositories: Seq[Repository], + deps: IterableOnce[Dependency], + force: IterableOnce[Dependency], + mapDependencies: Option[Dependency => Dependency], + customizer: Option[Resolution => Resolution], + ctx: Option[mill.api.Ctx.Log], + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] + ): Result[Resolution] = + resolveDependenciesMetadataSafe( + repositories, + deps, + force, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + ResolutionParams() + ) + } object CoursierSupport { diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 9383b49735b..46e869aba88 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -1,8 +1,9 @@ package mill.scalalib import coursier.cache.FileCache +import coursier.params.ResolutionParams import coursier.{Dependency, Repository, Resolve, Type} -import coursier.core.Resolution +import coursier.core.{DependencyManagement, Resolution} import mill.define.Task import mill.api.PathRef @@ -61,13 +62,13 @@ trait CoursierModule extends mill.Module { Lib.resolveDependencies( repositories = repositoriesTask(), deps = deps(), - bomDeps = bomDeps(), sources = sources, artifactTypes = artifactTypes, mapDependencies = Some(mapDependencies()), customizer = resolutionCustomizer(), coursierCacheCustomizer = coursierCacheCustomizer(), - ctx = Some(implicitly[mill.api.Ctx.Log]) + ctx = Some(implicitly[mill.api.Ctx.Log]), + bomDeps = bomDeps() ) } @@ -151,18 +152,20 @@ object CoursierModule { deps: IterableOnce[T], sources: Boolean = false, artifactTypes: Option[Set[coursier.Type]] = None, + resolutionParams: ResolutionParams = ResolutionParams(), bomDeps: IterableOnce[T] = Nil ): Agg[PathRef] = { Lib.resolveDependencies( repositories = repositories, deps = deps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)), - bomDeps = bomDeps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)), sources = sources, artifactTypes = artifactTypes, mapDependencies = mapDependencies, customizer = customizer, coursierCacheCustomizer = coursierCacheCustomizer, - ctx = ctx + ctx = ctx, + resolutionParams = resolutionParams, + bomDeps = bomDeps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) ).getOrThrow } @@ -171,14 +174,32 @@ object CoursierModule { sources: Boolean, artifactTypes: Option[Set[coursier.Type]] ): Agg[PathRef] = - resolveDeps(deps, sources, artifactTypes, Nil) + resolveDeps(deps, sources, artifactTypes, ResolutionParams(), Nil) @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean ): Agg[PathRef] = - resolveDeps(deps, sources, None, Nil) + resolveDeps(deps, sources, None, ResolutionParams(), Nil) + + def processDeps[T: CoursierModule.Resolvable]( + deps: IterableOnce[T], + resolutionParams: ResolutionParams = ResolutionParams(), + bomDeps: IterableOnce[T] = Nil + ): (Seq[Dependency], DependencyManagement.Map) = { + val res = Lib.resolveDependenciesMetadataSafe( + repositories = repositories, + deps = deps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)), + mapDependencies = mapDependencies, + customizer = customizer, + coursierCacheCustomizer = coursierCacheCustomizer, + ctx = ctx, + resolutionParams = resolutionParams, + bomDeps = bomDeps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) + ).getOrThrow + (res.processedRootDependencies, res.bomDepMgmt) + } } sealed trait Resolvable[T] { diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index f1ffd91e72e..5d6e77ac81b 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -2,6 +2,7 @@ package mill package scalalib import coursier.core.Resolution +import coursier.params.ResolutionParams import coursier.parse.JavaOrScalaModule import coursier.parse.ModuleParser import coursier.util.ModuleMatcher @@ -167,6 +168,10 @@ trait JavaModule */ def artifactTypes: T[Set[Type]] = Task { coursier.core.Resolution.defaultTypes } + def resolutionParams: Task[ResolutionParams] = Task.Anon { + ResolutionParams() + } + /** * Options to pass to the java compiler */ @@ -333,7 +338,7 @@ trait JavaModule * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. */ def transitiveIvyDeps: T[Agg[BoundDep]] = Task { - allIvyDeps().map(bindDependency()) ++ + (ivyDeps() ++ mandatoryIvyDeps()).map(bindDependency()) ++ T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten } @@ -611,6 +616,7 @@ trait JavaModule defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), artifactTypes = Some(artifactTypes()), + resolutionParams = resolutionParams(), bomDeps = bomDeps().map(bindDependency()) ) } @@ -627,6 +633,7 @@ trait JavaModule defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), artifactTypes = Some(artifactTypes()), + resolutionParams = resolutionParams(), bomDeps = bomDeps().map(bindDependency()) ) } @@ -893,7 +900,9 @@ trait JavaModule dependencies, Some(mapDependencies()), customizer = resolutionCustomizer(), - coursierCacheCustomizer = coursierCacheCustomizer() + coursierCacheCustomizer = coursierCacheCustomizer(), + resolutionParams = resolutionParams(), + bomDeps = bomDeps().map(bindDependency()) ).getOrThrow val roots = whatDependsOn match { @@ -1105,6 +1114,7 @@ trait JavaModule defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), sources = true, + resolutionParams = resolutionParams(), bomDeps = bomDeps().map(bindDependency()) ) }, @@ -1112,6 +1122,7 @@ trait JavaModule defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), sources = true, + resolutionParams = resolutionParams(), bomDeps = bomDeps().map(bindDependency()) ) } diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index 6eb8721e5de..58dabdb2e65 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -1,6 +1,7 @@ package mill package scalalib +import coursier.params.ResolutionParams import coursier.util.Task import coursier.{Dependency, Repository, Resolution, Type} import mill.api.{Ctx, Loose, PathRef, Result} @@ -60,6 +61,7 @@ object Lib { coursierCacheCustomizer: Option[ coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] ] = None, + resolutionParams: ResolutionParams = ResolutionParams(), bomDeps: IterableOnce[BoundDep] = Nil ): Result[Resolution] = { val depSeq = deps.iterator.toSeq @@ -71,10 +73,32 @@ object Lib { customizer = customizer, ctx = ctx, coursierCacheCustomizer = coursierCacheCustomizer, + resolutionParams = resolutionParams, bomDeps = bomDeps.iterator.toSeq.map(_.dep) ) } + @deprecated("Use the override accepting resolutionParams instead", "Mill after 0.12.2") + def resolveDependenciesMetadataSafe( + repositories: Seq[Repository], + deps: IterableOnce[BoundDep], + mapDependencies: Option[Dependency => Dependency], + customizer: Option[coursier.core.Resolution => coursier.core.Resolution], + ctx: Option[Ctx.Log], + coursierCacheCustomizer: Option[ + coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] + ] + ): Result[Resolution] = + resolveDependenciesMetadataSafe( + repositories, + deps, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + ResolutionParams() + ) + /** * Resolve dependencies using Coursier. * @@ -93,24 +117,54 @@ object Lib { coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] ] = None, artifactTypes: Option[Set[Type]] = None, + resolutionParams: ResolutionParams = ResolutionParams(), bomDeps: IterableOnce[BoundDep] = Nil ): Result[Agg[PathRef]] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependencies( repositories = repositories, deps = depSeq.map(_.dep), - bomDeps = bomDeps.iterator.map(_.dep).toSeq, force = depSeq.filter(_.force).map(_.dep), sources = sources, artifactTypes = artifactTypes, mapDependencies = mapDependencies, customizer = customizer, ctx = ctx, - coursierCacheCustomizer = coursierCacheCustomizer + coursierCacheCustomizer = coursierCacheCustomizer, + resolutionParams = resolutionParams, + bomDeps = bomDeps.iterator.map(_.dep).toSeq ).map(_.map(_.withRevalidateOnce)) } - @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") + @deprecated("Use the override accepting resolutionParams", "Mill after 0.12.2") + def resolveDependencies( + repositories: Seq[Repository], + deps: IterableOnce[BoundDep], + sources: Boolean, + mapDependencies: Option[Dependency => Dependency], + customizer: Option[coursier.core.Resolution => coursier.core.Resolution], + ctx: Option[Ctx.Log], + coursierCacheCustomizer: Option[ + coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] + ], + artifactTypes: Option[Set[Type]] + ): Result[Agg[PathRef]] = + resolveDependencies( + repositories, + deps, + sources, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + artifactTypes, + ResolutionParams() + ) + + @deprecated( + "Use the override accepting artifactTypes and resolutionParams", + "Mill after 0.12.0-RC3" + ) def resolveDependencies( repositories: Seq[Repository], deps: IterableOnce[BoundDep], @@ -131,6 +185,7 @@ object Lib { ctx, coursierCacheCustomizer, None, + ResolutionParams(), Nil ) diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 0a068793056..bb7acad4220 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -101,8 +101,35 @@ trait PublishModule extends JavaModule { outer => PathRef(pomPath) } + def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = Task { + val (processedDeps, depMgmt) = defaultResolver().processDeps( + transitiveRunIvyDeps() ++ transitiveIvyDeps(), + resolutionParams = resolutionParams(), + bomDeps = bomDeps().map(bindDependency()) + ) + (processedDeps.map(_.moduleVersion).toMap, depMgmt) + } + def ivy: T[PathRef] = Task { - val ivy = Ivy(artifactMetadata(), publishXmlDeps(), extraPublish()) + val (rootDepVersions, depMgmt) = bomDetails() + val publishXmlDeps0 = publishXmlDeps().map { dep => + if (dep.artifact.version == "_") + dep.copy( + artifact = dep.artifact.copy( + version = rootDepVersions.getOrElse( + coursier.core.Module(coursier.core.Organization(dep.artifact.group), coursier.core.ModuleName(dep.artifact.id), Map.empty), + "" /* throw instead? */ + ) + ) + ) + else + dep + } + val overrides = depMgmt.toSeq.map { + case (key, values) => + Ivy.Override(key.organization.value, key.name.value, values.version) + } + val ivy = Ivy(artifactMetadata(), publishXmlDeps0, extraPublish(), overrides) val ivyPath = T.dest / "ivy.xml" os.write.over(ivyPath, ivy) PathRef(ivyPath) diff --git a/scalalib/src/mill/scalalib/publish/Ivy.scala b/scalalib/src/mill/scalalib/publish/Ivy.scala index ae340ac0d75..eb4b7f0f1d0 100644 --- a/scalalib/src/mill/scalalib/publish/Ivy.scala +++ b/scalalib/src/mill/scalalib/publish/Ivy.scala @@ -8,10 +8,13 @@ object Ivy { val head = "\n" + case class Override(organization: String, name: String, version: String) + def apply( artifact: Artifact, dependencies: Agg[Dependency], - extras: Seq[PublishInfo] = Seq.empty + extras: Seq[PublishInfo] = Seq.empty, + overrides: Seq[Override] = Nil ): String = { def renderExtra(e: PublishInfo): Elem = { @@ -49,7 +52,10 @@ object Ivy { {extras.map(renderExtra)} - {dependencies.map(renderDependency).toSeq} + + {dependencies.map(renderDependency).toSeq} + {overrides.map(renderOverride)} + val pp = new PrettyPrinter(120, 4) @@ -69,6 +75,9 @@ object Ivy { } + private def renderOverride(override0: Override): Elem = + + private def depIvyConf(d: Dependency): String = { if (d.optional) "optional" else d.scope match { diff --git a/scalalib/src/mill/scalalib/publish/Pom.scala b/scalalib/src/mill/scalalib/publish/Pom.scala index a8678c2b817..9580c100668 100644 --- a/scalalib/src/mill/scalalib/publish/Pom.scala +++ b/scalalib/src/mill/scalalib/publish/Pom.scala @@ -134,11 +134,15 @@ object Pom { val optional = if (d.optional) true else NodeSeq.Empty + val version = + if (d.artifact.version == "_") NodeSeq.Empty + else {d.artifact.version} + if (d.exclusions.isEmpty) {d.artifact.group} {d.artifact.id} - {d.artifact.version} + {version} {scope} {`type`} {optional} @@ -147,7 +151,7 @@ object Pom { {d.artifact.group} {d.artifact.id} - {d.artifact.version} + {version} { d.exclusions.map(ex => diff --git a/scalalib/test/src/mill/scalalib/CoursierParametersTests.scala b/scalalib/test/src/mill/scalalib/CoursierParametersTests.scala new file mode 100644 index 00000000000..49f33484f10 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/CoursierParametersTests.scala @@ -0,0 +1,44 @@ +package mill.scalalib + +import coursier.util.StringInterpolators.SafeModule +import mill.Agg +import mill.define.Task +import mill.testkit.UnitTester +import mill.testkit.TestBaseModule +import utest._ + +object CoursierParametersTests extends TestSuite { + + val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "coursier" + + object CoursierTest extends TestBaseModule { + object core extends ScalaModule { + def scalaVersion = "2.13.12" + def ivyDeps = Task { + Agg(ivy"com.lihaoyi::pprint:0.9.0") + } + def resolutionParams = Task.Anon { + super.resolutionParams() + .addForceVersion((mod"com.lihaoyi:pprint_2.13", "0.8.1")) + } + } + } + + def tests: Tests = Tests { + test("coursierParams") - UnitTester(CoursierTest, null).scoped { eval => + val Right(result) = eval.apply(CoursierTest.core.compileClasspath) + val classPath = result.value.toSeq.map(_.path) + val pprintVersion = classPath + .map(_.last) + .filter(_.endsWith(".jar")) + .filter(_.startsWith("pprint_2.13-")) + .map(_.stripPrefix("pprint_2.13-").stripSuffix(".jar")) + .headOption + .getOrElse { + sys.error(s"pprint not found in class path $classPath") + } + val expectedPprintVersion = "0.8.1" + assert(pprintVersion == expectedPprintVersion) + } + } +} From 732c659b0de5e47bd5facab7196e5db4117b4e2c Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 19 Nov 2024 20:01:10 +0100 Subject: [PATCH 05/27] more --- integration/feature/bom/resources/build.mill | 25 ++++ integration/feature/bom/src/BomTests.scala | 122 ++++++++++++++++++ .../src/mill/scalalib/CoursierModule.scala | 55 +++++++- scalalib/src/mill/scalalib/JavaModule.scala | 32 ++--- .../src/mill/scalalib/PublishModule.scala | 7 +- scalalib/src/mill/scalalib/publish/Pom.scala | 2 +- 6 files changed, 221 insertions(+), 22 deletions(-) diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill index 612fffe97c6..42aaa14a7ea 100644 --- a/integration/feature/bom/resources/build.mill +++ b/integration/feature/bom/resources/build.mill @@ -45,3 +45,28 @@ object `google-cloud-scala-no-bom` extends JavaModule { ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" ) } + +object `google-cloud-java-dependee` extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + `google-cloud-java` + ) +} + +object `google-cloud-scala-dependee` extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + `google-cloud-scala` + ) +} + +object parent extends JavaModule with TestPublishModule { + def parentDep = Some(ivy"org.apache.spark:spark-parent_2.13:3.5.3") + def ivyDeps = Agg( + ivy"org.apache.commons:commons-compress:_" + ) +} + +object `parent-dependee` extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + parent + ) +} diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala index 10e34a5f3f0..5680cbae23a 100644 --- a/integration/feature/bom/src/BomTests.scala +++ b/integration/feature/bom/src/BomTests.scala @@ -10,6 +10,7 @@ object BomTests extends UtestIntegrationTestSuite { def tests: Tests = Tests { def expectedProtobufJavaVersion = "4.28.3" + def expectedCommonsCompressVersion = "1.23.0" def compileClasspathFileNames(tester: IntegrationTester, moduleName: String): Seq[String] = { import tester._ @@ -100,5 +101,126 @@ object BomTests extends UtestIntegrationTestSuite { assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) } + test("googleCloudJavaDependee") - integrationTest { tester => + import tester._ + + val compileClasspathFileNames0 = + compileClasspathFileNames(tester, "google-cloud-java-dependee") + assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + + val repo = workspacePath / "ivy2Local" + eval(("google-cloud-java.publishLocal", "--localIvyRepo", repo), check = true) + eval(("google-cloud-java-dependee.publishLocal", "--localIvyRepo", repo), check = true) + + val obtainedClassPath = coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + "google-cloud-java-dependee", + "0.1.0-SNAPSHOT" + ) + ) + .addRepositories( + coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + + val obtainedClassPathFileNames = obtainedClassPath.map(_.last) + assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + } + + test("googleCloudScalaDependee") - integrationTest { tester => + import tester._ + + val compileClasspathFileNames0 = + compileClasspathFileNames(tester, "google-cloud-scala-dependee") + assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + + val repo = workspacePath / "ivy2Local" + eval(("google-cloud-scala.publishLocal", "--localIvyRepo", repo), check = true) + eval(("google-cloud-scala-dependee.publishLocal", "--localIvyRepo", repo), check = true) + + val obtainedClassPath = coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + "google-cloud-scala-dependee", + "0.1.0-SNAPSHOT" + ) + ) + .addRepositories( + coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + + val obtainedClassPathFileNames = obtainedClassPath.map(_.last) + assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + } + + test("parent") - integrationTest { tester => + import tester._ + + val compileClasspathFileNames0 = compileClasspathFileNames(tester, "parent") + assert( + compileClasspathFileNames0.contains(s"commons-compress-$expectedCommonsCompressVersion.jar") + ) + + val repo = workspacePath / "ivy2Local" + eval(("parent.publishLocal", "--localIvyRepo", repo), check = true) + + val obtainedClassPath = coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of("com.lihaoyi.mill-tests", "parent", "0.1.0-SNAPSHOT") + ) + .addRepositories( + coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + + val obtainedClassPathFileNames = obtainedClassPath.map(_.last) + assert( + obtainedClassPathFileNames.contains(s"commons-compress-$expectedCommonsCompressVersion.jar") + ) + } + + test("parentDependee") - integrationTest { tester => + import tester._ + + val compileClasspathFileNames0 = compileClasspathFileNames(tester, "parent-dependee") + assert( + compileClasspathFileNames0.contains(s"commons-compress-$expectedCommonsCompressVersion.jar") + ) + + val repo = workspacePath / "ivy2Local" + eval(("parent.publishLocal", "--localIvyRepo", repo), check = true) + eval(("parent-dependee.publishLocal", "--localIvyRepo", repo), check = true) + + val obtainedClassPath = coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of("com.lihaoyi.mill-tests", "parent-dependee", "0.1.0-SNAPSHOT") + ) + .addRepositories( + coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + + val obtainedClassPathFileNames = obtainedClassPath.map(_.last) + assert( + obtainedClassPathFileNames.contains(s"commons-compress-$expectedCommonsCompressVersion.jar") + ) + } + } } diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 46e869aba88..75226fd664a 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -40,7 +40,8 @@ trait CoursierModule extends mill.Module { mapDependencies = Some(mapDependencies()), customizer = resolutionCustomizer(), coursierCacheCustomizer = coursierCacheCustomizer(), - ctx = Some(implicitly[mill.api.Ctx.Log]) + ctx = Some(implicitly[mill.api.Ctx.Log]), + resolutionParams = resolutionParams() ) } @@ -134,6 +135,27 @@ trait CoursierModule extends mill.Module { : Task[Option[FileCache[coursier.util.Task] => FileCache[coursier.util.Task]]] = Task.Anon { None } + /** + * Resolution parameters, allowing to customize resolution internals + * + * This rarely needs to be changed. This allows to disable the new way coursier handles + * BOMs since coursier 2.1.17 (used in Mill since 0.12.3) for example, with: + * {{{ + * def resolutionParams = super.resolutionParams() + * .withEnableDependencyOverrides(Some(false)) + * }}} + * + * Note that versions forced with `Dep#forceVersion()` take over forced versions manually + * set in `resolutionParams`. The former should be favored to force versions in dependency + * resolution. + * + * The Scala version set via `ScalaModule#scalaVersion` also takes over any Scala version + * provided via `ResolutionParams#scalaVersionOpt`. + */ + def resolutionParams: Task[ResolutionParams] = Task.Anon { + ResolutionParams() + } + } object CoursierModule { @@ -145,14 +167,37 @@ object CoursierModule { ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[ coursier.cache.FileCache[coursier.util.Task] => coursier.cache.FileCache[coursier.util.Task] - ] = None + ] = None, + resolutionParams: ResolutionParams = ResolutionParams() ) { + // bin-compat shim + def this( + repositories: Seq[Repository], + bind: Dep => BoundDep, + mapDependencies: Option[Dependency => Dependency], + customizer: Option[coursier.core.Resolution => coursier.core.Resolution], + ctx: Option[mill.api.Ctx.Log], + coursierCacheCustomizer: Option[ + coursier.cache.FileCache[coursier.util.Task] => coursier.cache.FileCache[ + coursier.util.Task + ] + ] + ) = + this( + repositories, + bind, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + ResolutionParams() + ) + def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean = false, artifactTypes: Option[Set[coursier.Type]] = None, - resolutionParams: ResolutionParams = ResolutionParams(), bomDeps: IterableOnce[T] = Nil ): Agg[PathRef] = { Lib.resolveDependencies( @@ -174,14 +219,14 @@ object CoursierModule { sources: Boolean, artifactTypes: Option[Set[coursier.Type]] ): Agg[PathRef] = - resolveDeps(deps, sources, artifactTypes, ResolutionParams(), Nil) + resolveDeps(deps, sources, artifactTypes, Nil) @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean ): Agg[PathRef] = - resolveDeps(deps, sources, None, ResolutionParams(), Nil) + resolveDeps(deps, sources, None, Nil) def processDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index a245a34909d..82b48148c9c 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -2,7 +2,6 @@ package mill package scalalib import coursier.core.Resolution -import coursier.params.ResolutionParams import coursier.parse.JavaOrScalaModule import coursier.parse.ModuleParser import coursier.util.ModuleMatcher @@ -140,7 +139,14 @@ trait JavaModule * Aggregation of mandatoryIvyDeps and ivyDeps. * In most cases, instead of overriding this Target you want to override `ivyDeps` instead. */ - def allIvyDeps: T[Agg[Dep]] = Task { ivyDeps() ++ mandatoryIvyDeps() } + def allIvyDeps: T[Agg[Dep]] = Task { + val bomDeps0 = allBomDeps().toSeq.map { bomDep => + (bomDep.dep.module, bomDep.dep.version) + } + val rawDeps = ivyDeps() ++ mandatoryIvyDeps() + if (bomDeps0.isEmpty) rawDeps + else rawDeps.map(dep => dep.copy(dep = dep.dep.addBoms(bomDeps0))) + } /** * Same as `ivyDeps`, but only present at compile time. Useful for e.g. @@ -156,22 +162,22 @@ trait JavaModule */ def runIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + def parentDep: T[Option[Dep]] = Task { None } + /** * Any BOM dependencies you want to add to this Module, in the format * ivy"org:name:version" */ def bomDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + def allBomDeps: T[Agg[Dep]] = Task { parentDep().toSeq ++ bomDeps() } + /** * Default artifact types to fetch and put in the classpath. Add extra types * here if you'd like fancy artifact extensions to be fetched. */ def artifactTypes: T[Set[Type]] = Task { coursier.core.Resolution.defaultTypes } - def resolutionParams: Task[ResolutionParams] = Task.Anon { - ResolutionParams() - } - /** * Options to pass to the java compiler */ @@ -616,8 +622,7 @@ trait JavaModule defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), artifactTypes = Some(artifactTypes()), - resolutionParams = resolutionParams(), - bomDeps = bomDeps().map(bindDependency()) + bomDeps = allBomDeps().map(bindDependency()) ) } @@ -633,8 +638,7 @@ trait JavaModule defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), artifactTypes = Some(artifactTypes()), - resolutionParams = resolutionParams(), - bomDeps = bomDeps().map(bindDependency()) + bomDeps = allBomDeps().map(bindDependency()) ) } @@ -902,7 +906,7 @@ trait JavaModule customizer = resolutionCustomizer(), coursierCacheCustomizer = coursierCacheCustomizer(), resolutionParams = resolutionParams(), - bomDeps = bomDeps().map(bindDependency()) + bomDeps = allBomDeps().map(bindDependency()) ).getOrThrow val roots = whatDependsOn match { @@ -1114,16 +1118,14 @@ trait JavaModule defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), sources = true, - resolutionParams = resolutionParams(), - bomDeps = bomDeps().map(bindDependency()) + bomDeps = allBomDeps().map(bindDependency()) ) }, Task.Anon { defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), sources = true, - resolutionParams = resolutionParams(), - bomDeps = bomDeps().map(bindDependency()) + bomDeps = allBomDeps().map(bindDependency()) ) } ) diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 4e84176a3f0..64a85850f23 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -36,7 +36,12 @@ trait PublishModule extends JavaModule { outer => * * @see [[https://maven.apache.org/guides/introduction/introduction-to-the-pom.html#Project_Inheritance Project Inheritance]] */ - def pomParentProject: T[Option[Artifact]] = None + def pomParentProject: T[Option[Artifact]] = Task { + parentDep().map { parentDep => + val parentDep0 = bindDependency().apply(parentDep) + Artifact(parentDep0.organization, parentDep0.name, parentDep0.version) + } + } /** * Configuration for the `pom.xml` metadata file published with this module diff --git a/scalalib/src/mill/scalalib/publish/Pom.scala b/scalalib/src/mill/scalalib/publish/Pom.scala index 7d6f6194257..f2e5dd53c86 100644 --- a/scalalib/src/mill/scalalib/publish/Pom.scala +++ b/scalalib/src/mill/scalalib/publish/Pom.scala @@ -62,7 +62,7 @@ object Pom { properties = properties, packagingType = packagingType, parentProject = None, - bomDependencies = Nil + bomDependencies = Agg.empty[Dependency] ) def apply( From 17259c004dce1612eccd4b622d00e506e68faf77 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 19 Nov 2024 20:26:26 +0100 Subject: [PATCH 06/27] fixup --- scalalib/test/src/mill/scalalib/publish/PomTests.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/scalalib/test/src/mill/scalalib/publish/PomTests.scala b/scalalib/test/src/mill/scalalib/publish/PomTests.scala index 6ff395f2647..49ba5f6131e 100644 --- a/scalalib/test/src/mill/scalalib/publish/PomTests.scala +++ b/scalalib/test/src/mill/scalalib/publish/PomTests.scala @@ -223,6 +223,7 @@ object PomTests extends TestSuite { pomSettings, properties, PackagingType.Jar, + None, Agg.empty[Dependency] )) From a8b86e310e31db5d21fcbe348134407b66361285 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 19 Nov 2024 20:48:12 +0100 Subject: [PATCH 07/27] more --- scalalib/src/mill/scalalib/CoursierModule.scala | 8 ++++++++ scalalib/src/mill/scalalib/publish/Ivy.scala | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 75226fd664a..6e67aa0abb2 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -73,6 +73,14 @@ trait CoursierModule extends mill.Module { ) } + // bin-compat shim + def resolveDeps( + deps: Task[Agg[BoundDep]], + sources: Boolean, + artifactTypes: Option[Set[Type]] + ): Task[Agg[PathRef]] = + resolveDeps(deps, sources, artifactTypes, Task.Anon(Agg.empty[BoundDep])) + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDeps( deps: Task[Agg[BoundDep]], diff --git a/scalalib/src/mill/scalalib/publish/Ivy.scala b/scalalib/src/mill/scalalib/publish/Ivy.scala index eb4b7f0f1d0..bcc710ec9e6 100644 --- a/scalalib/src/mill/scalalib/publish/Ivy.scala +++ b/scalalib/src/mill/scalalib/publish/Ivy.scala @@ -62,6 +62,16 @@ object Ivy { head + pp.format(xml).replaceAll(">", ">") } + // bin-compat shim + def apply( + artifact: Artifact, + dependencies: Agg[Dependency], + extras: Seq[PublishInfo] + ): String = + apply( + artifact, dependencies, extras, Nil + ) + private def renderDependency(dep: Dependency): Elem = { if (dep.exclusions.isEmpty) Date: Wed, 20 Nov 2024 17:12:11 +0100 Subject: [PATCH 08/27] more --- main/util/src/mill/util/CoursierSupport.scala | 28 +++++++++---------- .../src/mill/scalalib/CoursierModule.scala | 5 +++- .../src/mill/scalalib/PublishModule.scala | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index f969446b7a1..943e121361a 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -31,6 +31,20 @@ trait CoursierSupport { ctx.fold(cache)(c => cache.withLogger(new TickerResolutionLogger(c))) } + def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { + val org = dep.module.organization.value + val name = dep.module.name.value + val classpathKey = s"$org-$name" + + val classpathResourceText = + try Some(os.read( + os.resource(getClass.getClassLoader) / "mill/local-test-overrides" / classpathKey + )) + catch { case e: os.ResourceNotFoundException => None } + + classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq) + } + /** * Resolve dependencies using Coursier. * @@ -52,20 +66,6 @@ trait CoursierSupport { resolutionParams: ResolutionParams = ResolutionParams(), bomDeps: IterableOnce[Dependency] = Nil ): Result[Agg[PathRef]] = { - def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { - val org = dep.module.organization.value - val name = dep.module.name.value - val classpathKey = s"$org-$name" - - val classpathResourceText = - try Some(os.read( - os.resource(getClass.getClassLoader) / "mill/local-test-overrides" / classpathKey - )) - catch { case e: os.ResourceNotFoundException => None } - - classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq) - } - val (localTestDeps, remoteDeps) = deps.iterator.toSeq.partitionMap(d => isLocalTestDep(d).toLeft(d)) diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 6e67aa0abb2..e99fe543ec5 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -241,9 +241,12 @@ object CoursierModule { resolutionParams: ResolutionParams = ResolutionParams(), bomDeps: IterableOnce[T] = Nil ): (Seq[Dependency], DependencyManagement.Map) = { + val deps0 = deps + .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) + .filter(dep => mill.util.Jvm.isLocalTestDep(dep.dep).isEmpty) val res = Lib.resolveDependenciesMetadataSafe( repositories = repositories, - deps = deps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)), + deps = deps0, mapDependencies = mapDependencies, customizer = customizer, coursierCacheCustomizer = coursierCacheCustomizer, diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 64a85850f23..197d34943d0 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -117,7 +117,7 @@ trait PublishModule extends JavaModule { outer => def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = Task { val (processedDeps, depMgmt) = defaultResolver().processDeps( - transitiveRunIvyDeps() ++ transitiveIvyDeps(), + transitiveCompileIvyDeps() ++ transitiveIvyDeps(), resolutionParams = resolutionParams(), bomDeps = bomDeps().map(bindDependency()) ) From 17c30841fd25d1aa279ea7c4ed9c7e3f1a7fd17e Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 11:37:49 +0100 Subject: [PATCH 09/27] more --- integration/feature/bom/resources/build.mill | 18 ++ integration/feature/bom/src/BomTests.scala | 239 ++++++------------ scalalib/src/mill/scalalib/JavaModule.scala | 52 +++- .../src/mill/scalalib/PublishModule.scala | 10 +- scalalib/src/mill/scalalib/publish/Ivy.scala | 5 +- scalalib/src/mill/scalalib/publish/Pom.scala | 16 +- 6 files changed, 173 insertions(+), 167 deletions(-) diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill index 42aaa14a7ea..ba95fce3c43 100644 --- a/integration/feature/bom/resources/build.mill +++ b/integration/feature/bom/resources/build.mill @@ -70,3 +70,21 @@ object `parent-dependee` extends JavaModule with TestPublishModule { parent ) } + +object depMgmt extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + override def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) +} + +object `version-placeholder` extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:_" + ) + override def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) +} diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala index 5680cbae23a..1b644b5804d 100644 --- a/integration/feature/bom/src/BomTests.scala +++ b/integration/feature/bom/src/BomTests.scala @@ -12,7 +12,12 @@ object BomTests extends UtestIntegrationTestSuite { def expectedProtobufJavaVersion = "4.28.3" def expectedCommonsCompressVersion = "1.23.0" - def compileClasspathFileNames(tester: IntegrationTester, moduleName: String): Seq[String] = { + def expectedProtobufJarName = s"protobuf-java-$expectedProtobufJavaVersion.jar" + def expectedCommonsCompressJarName = s"commons-compress-$expectedCommonsCompressVersion.jar" + + def compileClasspathFileNames(moduleName: String)(implicit + tester: IntegrationTester + ): Seq[String] = { import tester._ val res = eval( ("show", s"$moduleName.compileClasspath"), @@ -22,205 +27,125 @@ object BomTests extends UtestIntegrationTestSuite { ujson.read(res.out).arr.map(v => os.Path(v.str.split(":").last).last).toSeq } - test("googleCloudJavaCheck") - integrationTest { tester => - import tester._ - - val res = eval( - ("show", "google-cloud-java-no-bom.compileClasspath"), - check = false - ) - assert( - res.err.contains( - "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" - ) - ) + def compileClasspathContains(module: String, fileName: String)(implicit + tester: IntegrationTester + ) = { + val fileNames = compileClasspathFileNames(module) + assert(fileNames.contains(fileName)) } - test("googleCloudJava") - integrationTest { tester => - import tester._ - - val compileClasspathFileNames0 = compileClasspathFileNames(tester, "google-cloud-java") - assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + def publishLocalAndResolve( + module: String, + dependencyModules: Seq[String] + )(implicit tester: IntegrationTester): Seq[os.Path] = { + val localIvyRepo = tester.workspacePath / "ivy2Local" + for (moduleName <- module +: dependencyModules) + tester.eval((s"$moduleName.publishLocal", "--localIvyRepo", localIvyRepo), check = true) - val repo = workspacePath / "ivy2Local" - eval(("google-cloud-java.publishLocal", "--localIvyRepo", repo), check = true) - - val obtainedClassPath = coursierapi.Fetch.create() + coursierapi.Fetch.create() .addDependencies( - coursierapi.Dependency.of("com.lihaoyi.mill-tests", "google-cloud-java", "0.1.0-SNAPSHOT") + coursierapi.Dependency.of("com.lihaoyi.mill-tests", module, "0.1.0-SNAPSHOT") ) .addRepositories( - coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") ) .fetch() .asScala .map(os.Path(_)) .toVector - - val obtainedClassPathFileNames = obtainedClassPath.map(_.last) - assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) } - test("googleCloudScalaCheck") - integrationTest { tester => - val compileClasspathFileNames0 = - compileClasspathFileNames(tester, "google-cloud-scala-no-bom") - assert( - compileClasspathFileNames0.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar")) - ) - assert( - !compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar") - ) - } + def publishM2LocalAndResolve( + module: String, + dependencyModules: Seq[String] + )(implicit tester: IntegrationTester): Seq[os.Path] = { + val localM2Repo = tester.workspacePath / "m2Local" + for (moduleName <- module +: dependencyModules) + tester.eval((s"$moduleName.publishM2Local", "--m2RepoPath", localM2Repo), check = true) - test("googleCloudScala") - integrationTest { tester => - import tester._ - - val compileClasspathFileNames0 = compileClasspathFileNames(tester, "google-cloud-scala") - assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) - - val repo = workspacePath / "ivy2Local" - eval(("google-cloud-scala.publishLocal", "--localIvyRepo", repo), check = true) - - val obtainedClassPath = coursierapi.Fetch.create() + coursierapi.Fetch.create() .addDependencies( - coursierapi.Dependency.of( - "com.lihaoyi.mill-tests", - "google-cloud-scala", - "0.1.0-SNAPSHOT" - ) + coursierapi.Dependency.of("com.lihaoyi.mill-tests", module, "0.1.0-SNAPSHOT") ) .addRepositories( - coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + coursierapi.MavenRepository.of(localM2Repo.toNIO.toUri.toASCIIString) ) .fetch() .asScala .map(os.Path(_)) .toVector - - val obtainedClassPathFileNames = obtainedClassPath.map(_.last) - assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) } - test("googleCloudJavaDependee") - integrationTest { tester => - import tester._ - - val compileClasspathFileNames0 = - compileClasspathFileNames(tester, "google-cloud-java-dependee") - assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + def isInClassPath( + module: String, + jarName: String, + dependencyModules: Seq[String] = Nil + )(implicit tester: IntegrationTester): Unit = { + compileClasspathContains(module, jarName) - val repo = workspacePath / "ivy2Local" - eval(("google-cloud-java.publishLocal", "--localIvyRepo", repo), check = true) - eval(("google-cloud-java-dependee.publishLocal", "--localIvyRepo", repo), check = true) - - val obtainedClassPath = coursierapi.Fetch.create() - .addDependencies( - coursierapi.Dependency.of( - "com.lihaoyi.mill-tests", - "google-cloud-java-dependee", - "0.1.0-SNAPSHOT" - ) - ) - .addRepositories( - coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") - ) - .fetch() - .asScala - .map(os.Path(_)) - .toVector + val resolvedCp = publishLocalAndResolve(module, dependencyModules) + assert(resolvedCp.map(_.last).contains(jarName)) - val obtainedClassPathFileNames = obtainedClassPath.map(_.last) - assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules) + assert(resolvedM2Cp.map(_.last).contains(jarName)) } - test("googleCloudScalaDependee") - integrationTest { tester => + test("googleCloudJavaCheck") - integrationTest { tester => import tester._ - val compileClasspathFileNames0 = - compileClasspathFileNames(tester, "google-cloud-scala-dependee") - assert(compileClasspathFileNames0.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) - - val repo = workspacePath / "ivy2Local" - eval(("google-cloud-scala.publishLocal", "--localIvyRepo", repo), check = true) - eval(("google-cloud-scala-dependee.publishLocal", "--localIvyRepo", repo), check = true) - - val obtainedClassPath = coursierapi.Fetch.create() - .addDependencies( - coursierapi.Dependency.of( - "com.lihaoyi.mill-tests", - "google-cloud-scala-dependee", - "0.1.0-SNAPSHOT" - ) - ) - .addRepositories( - coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") + val res = eval( + ("show", "google-cloud-java-no-bom.compileClasspath"), + check = false + ) + assert( + res.err.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" ) - .fetch() - .asScala - .map(os.Path(_)) - .toVector - - val obtainedClassPathFileNames = obtainedClassPath.map(_.last) - assert(obtainedClassPathFileNames.contains(s"protobuf-java-$expectedProtobufJavaVersion.jar")) + ) } - test("parent") - integrationTest { tester => - import tester._ + test("googleCloudScalaCheck") - integrationTest { implicit tester => + val fileNames = compileClasspathFileNames("google-cloud-scala-no-bom") + assert(fileNames.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar"))) + assert(!fileNames.contains(expectedProtobufJarName)) + } - val compileClasspathFileNames0 = compileClasspathFileNames(tester, "parent") - assert( - compileClasspathFileNames0.contains(s"commons-compress-$expectedCommonsCompressVersion.jar") - ) + test("googleCloudJava") - integrationTest { implicit tester => + isInClassPath("google-cloud-java", expectedProtobufJarName) + } - val repo = workspacePath / "ivy2Local" - eval(("parent.publishLocal", "--localIvyRepo", repo), check = true) + test("googleCloudScala") - integrationTest { implicit tester => + isInClassPath("google-cloud-scala", expectedProtobufJarName) + } - val obtainedClassPath = coursierapi.Fetch.create() - .addDependencies( - coursierapi.Dependency.of("com.lihaoyi.mill-tests", "parent", "0.1.0-SNAPSHOT") - ) - .addRepositories( - coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") - ) - .fetch() - .asScala - .map(os.Path(_)) - .toVector + test("googleCloudJavaDependee") - integrationTest { implicit tester => + isInClassPath("google-cloud-java-dependee", expectedProtobufJarName, Seq("google-cloud-java")) + } - val obtainedClassPathFileNames = obtainedClassPath.map(_.last) - assert( - obtainedClassPathFileNames.contains(s"commons-compress-$expectedCommonsCompressVersion.jar") + test("googleCloudScalaDependee") - integrationTest { implicit tester => + isInClassPath( + "google-cloud-scala-dependee", + expectedProtobufJarName, + Seq("google-cloud-scala") ) } - test("parentDependee") - integrationTest { tester => - import tester._ - - val compileClasspathFileNames0 = compileClasspathFileNames(tester, "parent-dependee") - assert( - compileClasspathFileNames0.contains(s"commons-compress-$expectedCommonsCompressVersion.jar") - ) + test("parent") - integrationTest { implicit tester => + isInClassPath("parent", expectedCommonsCompressJarName) + } - val repo = workspacePath / "ivy2Local" - eval(("parent.publishLocal", "--localIvyRepo", repo), check = true) - eval(("parent-dependee.publishLocal", "--localIvyRepo", repo), check = true) + test("parentDependee") - integrationTest { implicit tester => + isInClassPath("parent-dependee", expectedCommonsCompressJarName, Seq("parent")) + } - val obtainedClassPath = coursierapi.Fetch.create() - .addDependencies( - coursierapi.Dependency.of("com.lihaoyi.mill-tests", "parent-dependee", "0.1.0-SNAPSHOT") - ) - .addRepositories( - coursierapi.IvyRepository.of(repo.toNIO.toUri.toASCIIString + "[defaultPattern]") - ) - .fetch() - .asScala - .map(os.Path(_)) - .toVector + test("depMgmt") { + test("placeholder") - integrationTest { implicit tester => + isInClassPath("version-placeholder", expectedProtobufJarName) + } - val obtainedClassPathFileNames = obtainedClassPath.map(_.last) - assert( - obtainedClassPathFileNames.contains(s"commons-compress-$expectedCommonsCompressVersion.jar") - ) + test("override") - integrationTest { implicit tester => + isInClassPath("depMgmt", expectedProtobufJarName) + } } - } } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index d4f142f753c..56b4e3476f6 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1,7 +1,7 @@ package mill package scalalib -import coursier.core.Resolution +import coursier.core.{Configuration, DependencyManagement, Resolution} import coursier.parse.JavaOrScalaModule import coursier.parse.ModuleParser import coursier.util.ModuleMatcher @@ -148,8 +148,33 @@ trait JavaModule (bomDep.dep.module, bomDep.dep.version) } val rawDeps = ivyDeps() ++ mandatoryIvyDeps() - if (bomDeps0.isEmpty) rawDeps - else rawDeps.map(dep => dep.copy(dep = dep.dep.addBoms(bomDeps0))) + val depsWithBoms = + if (bomDeps0.isEmpty) rawDeps + else rawDeps.map(dep => dep.copy(dep = dep.dep.addBoms(bomDeps0))) + val depMgmt = dependencyManagementDict() + if (depMgmt.isEmpty) + depsWithBoms + else { + lazy val depMgmtMap = depMgmt.toMap + depsWithBoms.map { dep => + val versionOverride = + if (dep.dep.version == "_") { + val key = DependencyManagement.Key( + dep.dep.module.organization, + dep.dep.module.name, + coursier.core.Type.jar, + dep.dep.publication.classifier + ) + // FIXME Exclusions should be added too + depMgmtMap.get(key).map(_.version) + } else None + dep.copy( + dep = dep.dep + .withOverrides(dep.dep.overrides ++ depMgmt) + .withVersion(versionOverride.getOrElse(dep.dep.version)) + ) + } + } } /** @@ -176,6 +201,27 @@ trait JavaModule def allBomDeps: T[Agg[Dep]] = Task { parentDep().toSeq ++ bomDeps() } + def dependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + + def dependencyManagementDict: Task[Seq[(DependencyManagement.Key, DependencyManagement.Values)]] = + Task.Anon { + dependencyManagement().toSeq.map(bindDependency()).map(_.dep).map { depMgmt => + val key = DependencyManagement.Key( + depMgmt.module.organization, + depMgmt.module.name, + coursier.core.Type.jar, + depMgmt.publication.classifier + ) + val values = DependencyManagement.Values( + Configuration.empty, + depMgmt.version, + depMgmt.minimizedExclusions, + depMgmt.optional + ) + key -> values + } + } + /** * Default artifact types to fetch and put in the classpath. Add extra types * here if you'd like fancy artifact extensions to be fetched. diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 197d34943d0..23b77c8e8d7 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -98,6 +98,10 @@ trait PublishModule extends JavaModule { outer => bomDeps().map(resolvePublishDependency.apply().apply(_)) } + def publishXmlDepMgmt: Task[Agg[Dependency]] = Task.Anon { + dependencyManagement().map(resolvePublishDependency.apply().apply(_)) + } + def pom: T[PathRef] = Task { val pom = Pom( artifactMetadata(), @@ -107,7 +111,8 @@ trait PublishModule extends JavaModule { outer => publishProperties(), packagingType = pomPackagingType, parentProject = pomParentProject(), - bomDependencies = publishXmlBomDeps() + bomDependencies = publishXmlBomDeps(), + dependencyManagement = publishXmlDepMgmt() ) val pomPath = T.dest / s"${artifactId()}-${publishVersion()}.pom" os.write.over(pomPath, pom) @@ -125,7 +130,8 @@ trait PublishModule extends JavaModule { outer => } def ivy: T[PathRef] = Task { - val (rootDepVersions, depMgmt) = bomDetails() + val (rootDepVersions, bomDepMgmt) = bomDetails() + val depMgmt = dependencyManagementDict() ++ bomDepMgmt val publishXmlDeps0 = publishXmlDeps().map { dep => if (dep.artifact.version == "_") dep.copy( diff --git a/scalalib/src/mill/scalalib/publish/Ivy.scala b/scalalib/src/mill/scalalib/publish/Ivy.scala index bcc710ec9e6..75ba33dbec1 100644 --- a/scalalib/src/mill/scalalib/publish/Ivy.scala +++ b/scalalib/src/mill/scalalib/publish/Ivy.scala @@ -69,7 +69,10 @@ object Ivy { extras: Seq[PublishInfo] ): String = apply( - artifact, dependencies, extras, Nil + artifact, + dependencies, + extras, + Nil ) private def renderDependency(dep: Dependency): Elem = { diff --git a/scalalib/src/mill/scalalib/publish/Pom.scala b/scalalib/src/mill/scalalib/publish/Pom.scala index f2e5dd53c86..f3ea19304c9 100644 --- a/scalalib/src/mill/scalalib/publish/Pom.scala +++ b/scalalib/src/mill/scalalib/publish/Pom.scala @@ -40,11 +40,12 @@ object Pom { properties = properties, packagingType = pomSettings.packaging, parentProject = None, - bomDependencies = Agg.empty[Dependency] + bomDependencies = Agg.empty[Dependency], + dependencyManagement = Agg.empty[Dependency] ) @deprecated( - "Use overload with parentProject and bomDependencies parameter instead", + "Use overload with parentProject, bomDependencies, and dependencyManagement parameters instead", "Mill 0.12.1" ) def apply( @@ -62,7 +63,8 @@ object Pom { properties = properties, packagingType = packagingType, parentProject = None, - bomDependencies = Agg.empty[Dependency] + bomDependencies = Agg.empty[Dependency], + dependencyManagement = Agg.empty[Dependency] ) def apply( @@ -73,7 +75,8 @@ object Pom { properties: Map[String, String], packagingType: String, parentProject: Option[Artifact], - bomDependencies: Agg[Dependency] + bomDependencies: Agg[Dependency], + dependencyManagement: Agg[Dependency] ): String = { val xml = + + + {dependencyManagement.map(renderDependency(_)).iterator} + + val pp = new PrettyPrinter(120, 4) From a45ddd6d43b058d6a37629e7d118f559840b1436 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 13:38:17 +0100 Subject: [PATCH 10/27] fix --- scalalib/test/src/mill/scalalib/publish/PomTests.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/scalalib/test/src/mill/scalalib/publish/PomTests.scala b/scalalib/test/src/mill/scalalib/publish/PomTests.scala index 49ba5f6131e..d8fb8d44bbd 100644 --- a/scalalib/test/src/mill/scalalib/publish/PomTests.scala +++ b/scalalib/test/src/mill/scalalib/publish/PomTests.scala @@ -224,6 +224,7 @@ object PomTests extends TestSuite { properties, PackagingType.Jar, None, + Agg.empty[Dependency], Agg.empty[Dependency] )) From d601c4462c8ce3fbb82cea19a800d602ceee16e9 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 13:52:00 +0100 Subject: [PATCH 11/27] Revert UnidocModule stuff --- docs/package.mill | 63 +------------------ scalalib/src/mill/scalalib/UnidocModule.scala | 8 +-- 2 files changed, 4 insertions(+), 67 deletions(-) diff --git a/docs/package.mill b/docs/package.mill index d490b4e59f2..490aa9d6633 100644 --- a/docs/package.mill +++ b/docs/package.mill @@ -7,70 +7,9 @@ import scala.jdk.CollectionConverters._ /** Generates the mill documentation with Antora. */ object `package` extends RootModule { - trait CustomUnidocModule extends UnidocModule { - def unidocCompileClasspath = Task { - val cp = Seq(compile().classes) ++ T.traverse(moduleDeps)(_.compileClasspath)().flatten - cp.filter { ref => - !ref.path.lastOpt.exists(_.startsWith("dependency_2.13-")) - } - } - - def unidocCommon(local: Boolean) = Task.Anon { - - val unidocSourceFiles = - allSourceFiles() ++ T.traverse(moduleDeps)(_.allSourceFiles)().flatten - - T.log.info(s"Staging scaladoc for ${unidocSourceFiles.length} files") - - // the details of the options and zincWorker call are significantly - // different between scala-2 scaladoc and scala-3 scaladoc - // below is for scala-2 variant - val options: Seq[String] = Seq( - "-doc-title", - "Mill", - "-d", - T.dest.toString, - "-classpath", - unidocCompileClasspath().map(_.path).mkString(sys.props("path.separator")) - ) ++ - unidocVersion().toSeq.flatMap(Seq("-doc-version", _)) ++ - unidocSourceUrl().toSeq.flatMap { url => - if (local) Seq( - "-doc-source-url", - "file://€{FILE_PATH}.scala" - ) - else Seq( - "-doc-source-url", - url + "€{FILE_PATH}.scala", - "-sourcepath", - T.workspace.toString - ) - } - - zincWorker().worker().docJar( - scalaVersion(), - scalaOrganization(), - scalaDocClasspath(), - scalacPluginClasspath(), - options ++ unidocSourceFiles.map(_.path.toString) - ) match { - case true => mill.api.Result.Success(PathRef(T.dest)) - case false => mill.api.Result.Failure("unidoc generation failed") - } - } - } // This module isn't really a ScalaModule, but we use it to generate // consolidated documentation using the Scaladoc tool. - // Remove CustomUnidocModule and use UnidocModule instead when bumping Mill - object site extends CustomUnidocModule { - def unidocCompileClasspath = - super.unidocCompileClasspath().filter { ref => - // Not sure why, the JAR of coursier/dependency creates - // a dependency root package in the Mill API doc, and that - // package page has broken links. - // So we just exclude that dependency's JAR here. - !ref.path.lastOpt.exists(_.startsWith("dependency_2.13-")) - } + object site extends UnidocModule { def scalaVersion = build.Deps.scalaVersion def moduleDeps = build.millInternal.modules.collect { case m: build.MillStableScalaModule => m diff --git a/scalalib/src/mill/scalalib/UnidocModule.scala b/scalalib/src/mill/scalalib/UnidocModule.scala index f32ec844c6a..712e36e0c2f 100644 --- a/scalalib/src/mill/scalalib/UnidocModule.scala +++ b/scalalib/src/mill/scalalib/UnidocModule.scala @@ -11,11 +11,9 @@ trait UnidocModule extends ScalaModule { def unidocVersion: T[Option[String]] = None - def unidocCompileClasspath = Task { - Seq(compile().classes) ++ T.traverse(moduleDeps)(_.compileClasspath)().flatten - } - def unidocCommon(local: Boolean) = Task.Anon { + def unidocCompileClasspath = + Seq(compile().classes) ++ T.traverse(moduleDeps)(_.compileClasspath)().flatten val unidocSourceFiles = allSourceFiles() ++ T.traverse(moduleDeps)(_.allSourceFiles)().flatten @@ -31,7 +29,7 @@ trait UnidocModule extends ScalaModule { "-d", T.dest.toString, "-classpath", - unidocCompileClasspath().map(_.path).mkString(sys.props("path.separator")) + unidocCompileClasspath.map(_.path).mkString(sys.props("path.separator")) ) ++ unidocVersion().toSeq.flatMap(Seq("-doc-version", _)) ++ unidocSourceUrl().toSeq.flatMap { url => From f53fd8d8bc99c0455b66f0b21450809f2ecff3d3 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 15:59:36 +0100 Subject: [PATCH 12/27] tweaking --- integration/feature/bom/resources/build.mill | 100 +++++++++--------- integration/feature/bom/src/BomTests.scala | 104 ++++++++++--------- 2 files changed, 107 insertions(+), 97 deletions(-) diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill index ba95fce3c43..34d8e091eea 100644 --- a/integration/feature/bom/resources/build.mill +++ b/integration/feature/bom/resources/build.mill @@ -16,46 +16,48 @@ trait TestPublishModule extends PublishModule { def publishVersion = "0.1.0-SNAPSHOT" } -object `google-cloud-java` extends JavaModule with TestPublishModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0" - ) - def ivyDeps = Agg( - ivy"com.google.protobuf:protobuf-java:_" - ) -} +object bom extends Module { + object placeholder extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:_" + ) -object `google-cloud-java-no-bom` extends JavaModule { - def ivyDeps = Agg( - ivy"com.google.protobuf:protobuf-java:_" - ) -} + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + placeholder + ) + } -object `google-cloud-scala` extends JavaModule with TestPublishModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0" - ) - def ivyDeps = Agg( - ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" - ) -} + object check extends JavaModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:_" + ) + } + } -object `google-cloud-scala-no-bom` extends JavaModule { - def ivyDeps = Agg( - ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" - ) -} + object versionOverride extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) -object `google-cloud-java-dependee` extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - `google-cloud-java` - ) -} + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + versionOverride + ) + } -object `google-cloud-scala-dependee` extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - `google-cloud-scala` - ) + object check extends JavaModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + } + } } object parent extends JavaModule with TestPublishModule { @@ -63,28 +65,28 @@ object parent extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"org.apache.commons:commons-compress:_" ) -} -object `parent-dependee` extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - parent - ) + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + parent + ) + } } object depMgmt extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" ) - override def dependencyManagement = Agg( + def dependencyManagement = Agg( ivy"com.google.protobuf:protobuf-java:4.28.3" ) -} -object `version-placeholder` extends JavaModule with TestPublishModule { - def ivyDeps = Agg( - ivy"com.google.protobuf:protobuf-java:_" - ) - override def dependencyManagement = Agg( - ivy"com.google.protobuf:protobuf-java:4.28.3" - ) + object placeholder extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:_" + ) + def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) + } } diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala index 1b644b5804d..f5df8ed3054 100644 --- a/integration/feature/bom/src/BomTests.scala +++ b/integration/feature/bom/src/BomTests.scala @@ -44,7 +44,7 @@ object BomTests extends UtestIntegrationTestSuite { coursierapi.Fetch.create() .addDependencies( - coursierapi.Dependency.of("com.lihaoyi.mill-tests", module, "0.1.0-SNAPSHOT") + coursierapi.Dependency.of("com.lihaoyi.mill-tests", module.replace('.', '-'), "0.1.0-SNAPSHOT") ) .addRepositories( coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") @@ -65,7 +65,7 @@ object BomTests extends UtestIntegrationTestSuite { coursierapi.Fetch.create() .addDependencies( - coursierapi.Dependency.of("com.lihaoyi.mill-tests", module, "0.1.0-SNAPSHOT") + coursierapi.Dependency.of("com.lihaoyi.mill-tests", module.replace('.', '-'), "0.1.0-SNAPSHOT") ) .addRepositories( coursierapi.MavenRepository.of(localM2Repo.toNIO.toUri.toASCIIString) @@ -90,62 +90,70 @@ object BomTests extends UtestIntegrationTestSuite { assert(resolvedM2Cp.map(_.last).contains(jarName)) } - test("googleCloudJavaCheck") - integrationTest { tester => - import tester._ - - val res = eval( - ("show", "google-cloud-java-no-bom.compileClasspath"), - check = false - ) - assert( - res.err.contains( - "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" - ) - ) - } - - test("googleCloudScalaCheck") - integrationTest { implicit tester => - val fileNames = compileClasspathFileNames("google-cloud-scala-no-bom") - assert(fileNames.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar"))) - assert(!fileNames.contains(expectedProtobufJarName)) - } - - test("googleCloudJava") - integrationTest { implicit tester => - isInClassPath("google-cloud-java", expectedProtobufJarName) - } - - test("googleCloudScala") - integrationTest { implicit tester => - isInClassPath("google-cloud-scala", expectedProtobufJarName) - } - - test("googleCloudJavaDependee") - integrationTest { implicit tester => - isInClassPath("google-cloud-java-dependee", expectedProtobufJarName, Seq("google-cloud-java")) - } + test("bom") { + test("placeholder") { + test("check") - integrationTest { tester => + import tester._ + + val res = eval( + ("show", "bom.placeholder.check.compileClasspath"), + check = false + ) + assert( + res.err.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" + ) + ) + } + + test("simple") - integrationTest { implicit tester => + isInClassPath("bom.placeholder", expectedProtobufJarName) + } + + test("dependee") - integrationTest { implicit tester => + isInClassPath("bom.placeholder.dependee", expectedProtobufJarName, Seq("bom.placeholder")) + } + } - test("googleCloudScalaDependee") - integrationTest { implicit tester => - isInClassPath( - "google-cloud-scala-dependee", - expectedProtobufJarName, - Seq("google-cloud-scala") - ) + test("versionOverride") { + test("check") - integrationTest { implicit tester => + val fileNames = compileClasspathFileNames("bom.versionOverride.check") + assert(fileNames.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar"))) + assert(!fileNames.contains(expectedProtobufJarName)) + } + + test("simple") - integrationTest { implicit tester => + isInClassPath("bom.versionOverride", expectedProtobufJarName) + } + + test("dependee") - integrationTest { implicit tester => + isInClassPath( + "bom.versionOverride.dependee", + expectedProtobufJarName, + Seq("bom.versionOverride") + ) + } + } } - test("parent") - integrationTest { implicit tester => - isInClassPath("parent", expectedCommonsCompressJarName) - } + test("parent") { + test("simple") - integrationTest { implicit tester => + isInClassPath("parent", expectedCommonsCompressJarName) + } - test("parentDependee") - integrationTest { implicit tester => - isInClassPath("parent-dependee", expectedCommonsCompressJarName, Seq("parent")) + test("dependee") - integrationTest { implicit tester => + isInClassPath("parent.dependee", expectedCommonsCompressJarName, Seq("parent")) + } } test("depMgmt") { - test("placeholder") - integrationTest { implicit tester => - isInClassPath("version-placeholder", expectedProtobufJarName) - } - test("override") - integrationTest { implicit tester => isInClassPath("depMgmt", expectedProtobufJarName) } + + test("placeholder") - integrationTest { implicit tester => + isInClassPath("depMgmt.placeholder", expectedProtobufJarName) + } } } } From c549b21c07a8bcc00f063d209ea26cb145ac6745 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 16:01:29 +0100 Subject: [PATCH 13/27] =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .config/mill-version | 2 +- docs/package.mill | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.config/mill-version b/.config/mill-version index 95c1e53aea5..22bdc271439 100644 --- a/.config/mill-version +++ b/.config/mill-version @@ -1 +1 @@ -0.12.2-20-231ff6 +0.12.2-40-b240f5 diff --git a/docs/package.mill b/docs/package.mill index 490aa9d6633..4b087de3674 100644 --- a/docs/package.mill +++ b/docs/package.mill @@ -10,6 +10,14 @@ object `package` extends RootModule { // This module isn't really a ScalaModule, but we use it to generate // consolidated documentation using the Scaladoc tool. object site extends UnidocModule { + def unidocCompileClasspath = + super.unidocCompileClasspath().filter { ref => + // Workaround for https://github.com/scala/bug/issues/10028 + // We exclude the JAR of coursier/dependency here, so that + // scaladoc doesn't create an entry for its `dependency` + // package object in the Mill scaladoc. + !ref.path.lastOpt.exists(_.startsWith("dependency_2.13-")) + } def scalaVersion = build.Deps.scalaVersion def moduleDeps = build.millInternal.modules.collect { case m: build.MillStableScalaModule => m From 06b0c5b5a69c26c227300f2e3350be1cc1f48a37 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 20:23:42 +0100 Subject: [PATCH 14/27] more --- integration/feature/bom/resources/build.mill | 36 ++++++ integration/feature/bom/src/BomTests.scala | 94 +++++++++++++-- .../src/DocAnnotationsTests.scala | 1 + main/util/src/mill/util/CoursierSupport.scala | 12 +- .../src/mill/scalalib/CoursierModule.scala | 31 +++-- scalalib/src/mill/scalalib/JavaModule.scala | 112 ++++++++++++++---- scalalib/src/mill/scalalib/Lib.scala | 8 +- .../src/mill/scalalib/PublishModule.scala | 5 +- 8 files changed, 248 insertions(+), 51 deletions(-) diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill index 34d8e091eea..c5fb9de3733 100644 --- a/integration/feature/bom/resources/build.mill +++ b/integration/feature/bom/resources/build.mill @@ -81,6 +81,38 @@ object depMgmt extends JavaModule with TestPublishModule { ivy"com.google.protobuf:protobuf-java:4.28.3" ) + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(depMgmt) + } + + object exclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + ivy"org.java-websocket:Java-WebSocket:1.5.2" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(exclude) + } + } + + object onlyExclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + ivy"org.java-websocket:Java-WebSocket:_" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(onlyExclude) + } + } + object placeholder extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java:_" @@ -88,5 +120,9 @@ object depMgmt extends JavaModule with TestPublishModule { def dependencyManagement = Agg( ivy"com.google.protobuf:protobuf-java:4.28.3" ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(placeholder) + } } } diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala index f5df8ed3054..b5af58540c9 100644 --- a/integration/feature/bom/src/BomTests.scala +++ b/integration/feature/bom/src/BomTests.scala @@ -27,11 +27,17 @@ object BomTests extends UtestIntegrationTestSuite { ujson.read(res.out).arr.map(v => os.Path(v.str.split(":").last).last).toSeq } - def compileClasspathContains(module: String, fileName: String)(implicit + def compileClasspathContains( + module: String, + fileName: String, + jarCheck: Option[String => Boolean] + )(implicit tester: IntegrationTester ) = { val fileNames = compileClasspathFileNames(module) assert(fileNames.contains(fileName)) + for (check <- jarCheck; fileName <- fileNames) + assert(check(fileName)) } def publishLocalAndResolve( @@ -44,7 +50,11 @@ object BomTests extends UtestIntegrationTestSuite { coursierapi.Fetch.create() .addDependencies( - coursierapi.Dependency.of("com.lihaoyi.mill-tests", module.replace('.', '-'), "0.1.0-SNAPSHOT") + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + module.replace('.', '-'), + "0.1.0-SNAPSHOT" + ) ) .addRepositories( coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") @@ -65,7 +75,11 @@ object BomTests extends UtestIntegrationTestSuite { coursierapi.Fetch.create() .addDependencies( - coursierapi.Dependency.of("com.lihaoyi.mill-tests", module.replace('.', '-'), "0.1.0-SNAPSHOT") + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + module.replace('.', '-'), + "0.1.0-SNAPSHOT" + ) ) .addRepositories( coursierapi.MavenRepository.of(localM2Repo.toNIO.toUri.toASCIIString) @@ -79,15 +93,23 @@ object BomTests extends UtestIntegrationTestSuite { def isInClassPath( module: String, jarName: String, - dependencyModules: Seq[String] = Nil + dependencyModules: Seq[String] = Nil, + jarCheck: Option[String => Boolean] = None, + ivy2LocalCheck: Boolean = true )(implicit tester: IntegrationTester): Unit = { - compileClasspathContains(module, jarName) + compileClasspathContains(module, jarName, jarCheck) - val resolvedCp = publishLocalAndResolve(module, dependencyModules) - assert(resolvedCp.map(_.last).contains(jarName)) + if (ivy2LocalCheck) { + val resolvedCp = publishLocalAndResolve(module, dependencyModules) + assert(resolvedCp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedCp.map(_.last)) + assert(check(fileName)) + } val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules) assert(resolvedM2Cp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) + assert(check(fileName)) } test("bom") { @@ -151,9 +173,67 @@ object BomTests extends UtestIntegrationTestSuite { isInClassPath("depMgmt", expectedProtobufJarName) } + test("transitiveOverride") - integrationTest { implicit tester => + isInClassPath("depMgmt.transitive", expectedProtobufJarName, Seq("depMgmt")) + } + + test("exclude") - integrationTest { implicit tester => + isInClassPath( + "depMgmt.exclude", + "Java-WebSocket-1.5.2.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("transitiveExclude") - integrationTest { implicit tester => + isInClassPath( + "depMgmt.exclude.transitive", + "Java-WebSocket-1.5.2.jar", + Seq("depMgmt.exclude"), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("onlyExclude") - integrationTest { implicit tester => + isInClassPath( + "depMgmt.onlyExclude", + "Java-WebSocket-1.5.3.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("transitiveOnlyExclude") - integrationTest { implicit tester => + isInClassPath( + "depMgmt.onlyExclude.transitive", + "Java-WebSocket-1.5.3.jar", + Seq("depMgmt.onlyExclude"), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + test("placeholder") - integrationTest { implicit tester => isInClassPath("depMgmt.placeholder", expectedProtobufJarName) } + + test("transitivePlaceholder") - integrationTest { implicit tester => + isInClassPath( + "depMgmt.placeholder.transitive", + expectedProtobufJarName, + Seq("depMgmt.placeholder") + ) + } } } } diff --git a/integration/feature/docannotations/src/DocAnnotationsTests.scala b/integration/feature/docannotations/src/DocAnnotationsTests.scala index 49524852d22..341b8f8f632 100644 --- a/integration/feature/docannotations/src/DocAnnotationsTests.scala +++ b/integration/feature/docannotations/src/DocAnnotationsTests.scala @@ -103,6 +103,7 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite { | |Inputs: | core.transitiveIvyDeps + | core.allBomDeps |""".stripMargin, ivyDepsTree ) diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 943e121361a..efa9f489382 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -7,7 +7,7 @@ import coursier.params.ResolutionParams import coursier.parse.RepositoryParser import coursier.jvm.{JvmCache, JvmChannel, JvmIndex, JavaHome} import coursier.util.Task -import coursier.{Artifacts, Classifier, Dependency, Repository, Resolution, Resolve, Type} +import coursier.{Artifacts, Classifier, Dependency, Module, Repository, Resolution, Resolve, Type} import mill.api.Loose.Agg import mill.api.{Ctx, PathRef, Result} @@ -64,7 +64,7 @@ trait CoursierSupport { resolveFilter: os.Path => Boolean = _ => true, artifactTypes: Option[Set[Type]] = None, resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[Dependency] = Nil + bomDeps: IterableOnce[(Module, String)] = Nil ): Result[Agg[PathRef]] = { val (localTestDeps, remoteDeps) = deps.iterator.toSeq.partitionMap(d => isLocalTestDep(d).toLeft(d)) @@ -257,16 +257,14 @@ trait CoursierSupport { ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[Dependency] = Nil + bomDeps: IterableOnce[(Module, String)] = Nil ): Result[Resolution] = { val rootDeps = deps.iterator .map(d => mapDependencies.fold(d)(_.apply(d))) .toSeq - val bomDeps0 = bomDeps.iterator - .map(d => mapDependencies.fold(d)(_.apply(d))) - .toSeq + val bomDeps0 = bomDeps.iterator.toSeq val forceVersions = force.iterator .map(mapDependencies.getOrElse(identity[Dependency](_))) @@ -281,7 +279,7 @@ trait CoursierSupport { val resolve = Resolve() .withCache(coursierCache0) .withDependencies(rootDeps) - .withBomDependencies(bomDeps0) + .withBomModuleVersions(bomDeps0) .withRepositories(repositories) .withResolutionParams(resolutionParams0) .withMapDependenciesOpt(mapDependencies) diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index e99fe543ec5..c055e5e01a0 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -2,7 +2,7 @@ package mill.scalalib import coursier.cache.FileCache import coursier.params.ResolutionParams -import coursier.{Dependency, Repository, Resolve, Type} +import coursier.{Dependency, Module, Repository, Resolve, Type} import coursier.core.{DependencyManagement, Resolution} import mill.define.Task import mill.api.PathRef @@ -57,7 +57,7 @@ trait CoursierModule extends mill.Module { deps: Task[Agg[BoundDep]], sources: Boolean = false, artifactTypes: Option[Set[Type]] = None, - bomDeps: Task[Agg[BoundDep]] = Task.Anon(Agg.empty[BoundDep]) + bomDeps: Task[Agg[(Module, String)]] = Task.Anon(Agg.empty[(Module, String)]) ): Task[Agg[PathRef]] = Task.Anon { Lib.resolveDependencies( @@ -79,14 +79,14 @@ trait CoursierModule extends mill.Module { sources: Boolean, artifactTypes: Option[Set[Type]] ): Task[Agg[PathRef]] = - resolveDeps(deps, sources, artifactTypes, Task.Anon(Agg.empty[BoundDep])) + resolveDeps(deps, sources, artifactTypes, Task.Anon(Agg.empty[(Module, String)])) @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDeps( deps: Task[Agg[BoundDep]], sources: Boolean ): Task[Agg[PathRef]] = - resolveDeps(deps, sources, None, Task.Anon(Agg.empty[BoundDep])) + resolveDeps(deps, sources, None, Task.Anon(Agg.empty[(Module, String)])) /** * Map dependencies before resolving them. @@ -206,7 +206,7 @@ object CoursierModule { deps: IterableOnce[T], sources: Boolean = false, artifactTypes: Option[Set[coursier.Type]] = None, - bomDeps: IterableOnce[T] = Nil + bomDeps: IterableOnce[(Module, String)] = Nil ): Agg[PathRef] = { Lib.resolveDependencies( repositories = repositories, @@ -218,10 +218,11 @@ object CoursierModule { coursierCacheCustomizer = coursierCacheCustomizer, ctx = ctx, resolutionParams = resolutionParams, - bomDeps = bomDeps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) + bomDeps = bomDeps ).getOrThrow } + // bin-compat shim def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean, @@ -236,10 +237,24 @@ object CoursierModule { ): Agg[PathRef] = resolveDeps(deps, sources, None, Nil) + /** + * Processes dependencies and BOMs with coursier + * + * This makes coursier read and process BOM dependencies, and fill version placeholders + * in dependencies with the BOMs. + * + * Note that this doesn't throw when a version placeholder cannot be filled, and just leaves + * the placeholder behind. + * + * @param deps dependencies that might have placeholder versions ("_" as version) + * @param resolutionParams coursier resolution parameters + * @param bomDeps dependencies to read Bill-Of-Materials from + * @return dependencies with version placeholder filled and data read from the BOM dependencies + */ def processDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[T] = Nil + bomDeps: IterableOnce[(Module, String)] = Nil ): (Seq[Dependency], DependencyManagement.Map) = { val deps0 = deps .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) @@ -252,7 +267,7 @@ object CoursierModule { coursierCacheCustomizer = coursierCacheCustomizer, ctx = ctx, resolutionParams = resolutionParams, - bomDeps = bomDeps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) + bomDeps = bomDeps ).getOrThrow (res.processedRootDependencies, res.bomDepMgmt) } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 56b4e3476f6..70065a4fccf 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -140,13 +140,12 @@ trait JavaModule def ivyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } /** - * Aggregation of mandatoryIvyDeps and ivyDeps. + * Aggregation of mandatoryIvyDeps and ivyDeps, with BOMs and dependency management data + * added to each of them. * In most cases, instead of overriding this Target you want to override `ivyDeps` instead. */ def allIvyDeps: T[Agg[Dep]] = Task { - val bomDeps0 = allBomDeps().toSeq.map { bomDep => - (bomDep.dep.module, bomDep.dep.version) - } + val bomDeps0 = allBomDeps().toSeq val rawDeps = ivyDeps() ++ mandatoryIvyDeps() val depsWithBoms = if (bomDeps0.isEmpty) rawDeps @@ -199,27 +198,92 @@ trait JavaModule */ def bomDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } - def allBomDeps: T[Agg[Dep]] = Task { parentDep().toSeq ++ bomDeps() } + def allBomDeps: Task[Agg[(coursier.core.Module, String)]] = Task.Anon { + val modVerOrMalformed = (Agg(parentDep().toSeq: _*) ++ bomDeps()).map(bindDependency()).map { bomDep => + val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) + .withConfiguration(coursier.core.Configuration.defaultCompile) + if (fromModVer == bomDep.dep) + Right((bomDep.dep.module, bomDep.dep.version)) + else + Left(bomDep) + } + val malformed = modVerOrMalformed.collect { + case Left(malformedBomDep) => + malformedBomDep + } + if (malformed.isEmpty) + modVerOrMalformed.collect { + case Right(modVer) => modVer + } + else + throw new Exception( + "Found parent or BOM dependencies with invalid parameters:" + System.lineSeparator() + + malformed.map("- " + _.dep + System.lineSeparator()).mkString + + "Only organization, name, and version are accepted." + ) + } + + /** + * Dependency management data + * + * Versions and exclusions in dependency management override those of transitive dependencies, + * while they have no effect if the corresponding dependency isn't pulled during dependency + * resolution. + * + * For example, the following forces com.lihaoyi::os-lib to version 0.11.3, and + * excludes org.slf4j:slf4j-api from com.lihaoyi::cask that it forces to version 0.9.4 + * {{{ + * def dependencyManagement = super.dependencyManagement() ++ Agg( + * ivy"com.lihaoyi::os-lib:0.11.3", + * ivy"com.lihaoyi::cask:0.9.4".exclude("org.slf4j", "slf4j-api") + * ) + * }}} + */ def dependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** + * Data from dependencyManagement, converted to a type ready to be passed to coursier + * for dependency resolution + */ def dependencyManagementDict: Task[Seq[(DependencyManagement.Key, DependencyManagement.Values)]] = Task.Anon { - dependencyManagement().toSeq.map(bindDependency()).map(_.dep).map { depMgmt => - val key = DependencyManagement.Key( - depMgmt.module.organization, - depMgmt.module.name, - coursier.core.Type.jar, - depMgmt.publication.classifier - ) - val values = DependencyManagement.Values( - Configuration.empty, - depMgmt.version, - depMgmt.minimizedExclusions, - depMgmt.optional - ) - key -> values + val keyValuesOrErrors = dependencyManagement().toSeq.map(bindDependency()).map(_.dep).map { depMgmt => + val fromUsedValues = coursier.core.Dependency(depMgmt.module, depMgmt.version) + .withPublication(coursier.core.Publication("", depMgmt.publication.`type`, coursier.core.Extension.empty, depMgmt.publication.classifier)) + .withMinimizedExclusions(depMgmt.minimizedExclusions) + .withOptional(depMgmt.optional) + .withConfiguration(Configuration.defaultCompile) + if (fromUsedValues == depMgmt) { + val key = DependencyManagement.Key( + depMgmt.module.organization, + depMgmt.module.name, + if (depMgmt.publication.`type`.isEmpty) coursier.core.Type.jar else depMgmt.publication.`type`, + depMgmt.publication.classifier + ) + val values = DependencyManagement.Values( + Configuration.empty, + if (depMgmt.version == "_") "" // shouldn't be needed with future coursier versions + else depMgmt.version, + depMgmt.minimizedExclusions, + depMgmt.optional + ) + Right(key -> values) + } + else + Left(depMgmt) } + + val errors = keyValuesOrErrors.collect { + case Left(errored) => errored + } + if (errors.isEmpty) + keyValuesOrErrors.collect { case Right(kv) => kv } + else + throw new Exception( + "Found dependency management entries with invalid values. Only organization, name, version, type, classifier, exclusions, and optionality can be specified" + System.lineSeparator() + + errors.map("- " + _ + System.lineSeparator()).mkString + ) } /** @@ -672,7 +736,7 @@ trait JavaModule defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), artifactTypes = Some(artifactTypes()), - bomDeps = allBomDeps().map(bindDependency()) + bomDeps = allBomDeps() ) } @@ -688,7 +752,7 @@ trait JavaModule defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), artifactTypes = Some(artifactTypes()), - bomDeps = allBomDeps().map(bindDependency()) + bomDeps = allBomDeps() ) } @@ -956,7 +1020,7 @@ trait JavaModule customizer = resolutionCustomizer(), coursierCacheCustomizer = coursierCacheCustomizer(), resolutionParams = resolutionParams(), - bomDeps = allBomDeps().map(bindDependency()) + bomDeps = allBomDeps() ).getOrThrow val roots = whatDependsOn match { @@ -1168,14 +1232,14 @@ trait JavaModule defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), sources = true, - bomDeps = allBomDeps().map(bindDependency()) + bomDeps = allBomDeps() ) }, Task.Anon { defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), sources = true, - bomDeps = allBomDeps().map(bindDependency()) + bomDeps = allBomDeps() ) } ) diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index 4ab8fdfcdb5..4b2c5e515d8 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -62,7 +62,7 @@ object Lib { coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] ] = None, resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[BoundDep] = Nil + bomDeps: IterableOnce[(coursier.core.Module, String)] = Nil ): Result[Resolution] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependenciesMetadataSafe( @@ -74,7 +74,7 @@ object Lib { ctx = ctx, coursierCacheCustomizer = coursierCacheCustomizer, resolutionParams = resolutionParams, - bomDeps = bomDeps.iterator.toSeq.map(_.dep) + bomDeps = bomDeps ) } @@ -118,7 +118,7 @@ object Lib { ] = None, artifactTypes: Option[Set[Type]] = None, resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[BoundDep] = Nil + bomDeps: IterableOnce[(coursier.core.Module, String)] = Nil ): Result[Agg[PathRef]] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependencies( @@ -132,7 +132,7 @@ object Lib { ctx = ctx, coursierCacheCustomizer = coursierCacheCustomizer, resolutionParams = resolutionParams, - bomDeps = bomDeps.iterator.map(_.dep).toSeq + bomDeps = bomDeps ).map(_.map(_.withRevalidateOnce)) } diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 23b77c8e8d7..27729739aa6 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -119,12 +119,15 @@ trait PublishModule extends JavaModule { outer => PathRef(pomPath) } + /** + * Dependencies with version placeholder filled from BOMs, alongside with BOM data + */ def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = Task { val (processedDeps, depMgmt) = defaultResolver().processDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), resolutionParams = resolutionParams(), - bomDeps = bomDeps().map(bindDependency()) + bomDeps = allBomDeps() ) (processedDeps.map(_.moduleVersion).toMap, depMgmt) } From 18289750b81bef7d0dec7c87e1069c006f5cf769 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 20:24:56 +0100 Subject: [PATCH 15/27] fmt --- scalalib/src/mill/scalalib/JavaModule.scala | 75 +++++++++++---------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 70065a4fccf..3169938622e 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -199,14 +199,15 @@ trait JavaModule def bomDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } def allBomDeps: Task[Agg[(coursier.core.Module, String)]] = Task.Anon { - val modVerOrMalformed = (Agg(parentDep().toSeq: _*) ++ bomDeps()).map(bindDependency()).map { bomDep => - val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) - .withConfiguration(coursier.core.Configuration.defaultCompile) - if (fromModVer == bomDep.dep) - Right((bomDep.dep.module, bomDep.dep.version)) - else - Left(bomDep) - } + val modVerOrMalformed = + (Agg(parentDep().toSeq: _*) ++ bomDeps()).map(bindDependency()).map { bomDep => + val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) + .withConfiguration(coursier.core.Configuration.defaultCompile) + if (fromModVer == bomDep.dep) + Right((bomDep.dep.module, bomDep.dep.version)) + else + Left(bomDep) + } val malformed = modVerOrMalformed.collect { case Left(malformedBomDep) => @@ -219,8 +220,8 @@ trait JavaModule else throw new Exception( "Found parent or BOM dependencies with invalid parameters:" + System.lineSeparator() + - malformed.map("- " + _.dep + System.lineSeparator()).mkString + - "Only organization, name, and version are accepted." + malformed.map("- " + _.dep + System.lineSeparator()).mkString + + "Only organization, name, and version are accepted." ) } @@ -248,31 +249,37 @@ trait JavaModule */ def dependencyManagementDict: Task[Seq[(DependencyManagement.Key, DependencyManagement.Values)]] = Task.Anon { - val keyValuesOrErrors = dependencyManagement().toSeq.map(bindDependency()).map(_.dep).map { depMgmt => - val fromUsedValues = coursier.core.Dependency(depMgmt.module, depMgmt.version) - .withPublication(coursier.core.Publication("", depMgmt.publication.`type`, coursier.core.Extension.empty, depMgmt.publication.classifier)) - .withMinimizedExclusions(depMgmt.minimizedExclusions) - .withOptional(depMgmt.optional) - .withConfiguration(Configuration.defaultCompile) - if (fromUsedValues == depMgmt) { - val key = DependencyManagement.Key( - depMgmt.module.organization, - depMgmt.module.name, - if (depMgmt.publication.`type`.isEmpty) coursier.core.Type.jar else depMgmt.publication.`type`, - depMgmt.publication.classifier - ) - val values = DependencyManagement.Values( - Configuration.empty, - if (depMgmt.version == "_") "" // shouldn't be needed with future coursier versions - else depMgmt.version, - depMgmt.minimizedExclusions, - depMgmt.optional - ) - Right(key -> values) + val keyValuesOrErrors = + dependencyManagement().toSeq.map(bindDependency()).map(_.dep).map { depMgmt => + val fromUsedValues = coursier.core.Dependency(depMgmt.module, depMgmt.version) + .withPublication(coursier.core.Publication( + "", + depMgmt.publication.`type`, + coursier.core.Extension.empty, + depMgmt.publication.classifier + )) + .withMinimizedExclusions(depMgmt.minimizedExclusions) + .withOptional(depMgmt.optional) + .withConfiguration(Configuration.defaultCompile) + if (fromUsedValues == depMgmt) { + val key = DependencyManagement.Key( + depMgmt.module.organization, + depMgmt.module.name, + if (depMgmt.publication.`type`.isEmpty) coursier.core.Type.jar + else depMgmt.publication.`type`, + depMgmt.publication.classifier + ) + val values = DependencyManagement.Values( + Configuration.empty, + if (depMgmt.version == "_") "" // shouldn't be needed with future coursier versions + else depMgmt.version, + depMgmt.minimizedExclusions, + depMgmt.optional + ) + Right(key -> values) + } else + Left(depMgmt) } - else - Left(depMgmt) - } val errors = keyValuesOrErrors.collect { case Left(errored) => errored From c9054a3c843dbd88a3f6fe8f2b824790155d3a42 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 21:16:25 +0100 Subject: [PATCH 16/27] more --- integration/feature/bom/resources/build.mill | 37 ++++++++++ integration/feature/bom/src/BomTests.scala | 69 +++++++++++++++++-- .../src/DocAnnotationsTests.scala | 3 +- 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill index c5fb9de3733..6644dc65f96 100644 --- a/integration/feature/bom/resources/build.mill +++ b/integration/feature/bom/resources/build.mill @@ -58,6 +58,14 @@ object bom extends Module { ) } } + + object invalid extends Module { + object exclude extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0".exclude(("foo", "thing")) + ) + } + } } object parent extends JavaModule with TestPublishModule { @@ -71,6 +79,22 @@ object parent extends JavaModule with TestPublishModule { parent ) } + + object scala extends ScalaModule with TestPublishModule { + def scalaVersion = _root_.scala.util.Properties.versionNumberString + def parentDep = Some(ivy"org.apache.spark::spark-parent:3.5.3") + def ivyDeps = Agg( + ivy"org.apache.commons:commons-compress:_" + ) + } + + object invalid extends Module { + object exclude extends JavaModule { + def parentDep = Some( + ivy"org.apache.spark:spark-parent_2.13:3.5.3".exclude(("foo", "thing")) + ) + } + } } object depMgmt extends JavaModule with TestPublishModule { @@ -113,6 +137,19 @@ object depMgmt extends JavaModule with TestPublishModule { } } + object invalid extends Module { + object transitive extends JavaModule { + def dependencyManagement = { + val dep = ivy"org.java-websocket:Java-WebSocket:1.5.3" + Agg( + dep.copy( + dep = dep.dep.withTransitive(false) + ) + ) + } + } + } + object placeholder extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java:_" diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala index b5af58540c9..6ee77c27c6c 100644 --- a/integration/feature/bom/src/BomTests.scala +++ b/integration/feature/bom/src/BomTests.scala @@ -42,7 +42,8 @@ object BomTests extends UtestIntegrationTestSuite { def publishLocalAndResolve( module: String, - dependencyModules: Seq[String] + dependencyModules: Seq[String], + scalaSuffix: String )(implicit tester: IntegrationTester): Seq[os.Path] = { val localIvyRepo = tester.workspacePath / "ivy2Local" for (moduleName <- module +: dependencyModules) @@ -52,7 +53,7 @@ object BomTests extends UtestIntegrationTestSuite { .addDependencies( coursierapi.Dependency.of( "com.lihaoyi.mill-tests", - module.replace('.', '-'), + module.replace('.', '-') + scalaSuffix, "0.1.0-SNAPSHOT" ) ) @@ -67,7 +68,8 @@ object BomTests extends UtestIntegrationTestSuite { def publishM2LocalAndResolve( module: String, - dependencyModules: Seq[String] + dependencyModules: Seq[String], + scalaSuffix: String )(implicit tester: IntegrationTester): Seq[os.Path] = { val localM2Repo = tester.workspacePath / "m2Local" for (moduleName <- module +: dependencyModules) @@ -77,7 +79,7 @@ object BomTests extends UtestIntegrationTestSuite { .addDependencies( coursierapi.Dependency.of( "com.lihaoyi.mill-tests", - module.replace('.', '-'), + module.replace('.', '-') + scalaSuffix, "0.1.0-SNAPSHOT" ) ) @@ -95,18 +97,19 @@ object BomTests extends UtestIntegrationTestSuite { jarName: String, dependencyModules: Seq[String] = Nil, jarCheck: Option[String => Boolean] = None, - ivy2LocalCheck: Boolean = true + ivy2LocalCheck: Boolean = true, + scalaSuffix: String = "" )(implicit tester: IntegrationTester): Unit = { compileClasspathContains(module, jarName, jarCheck) if (ivy2LocalCheck) { - val resolvedCp = publishLocalAndResolve(module, dependencyModules) + val resolvedCp = publishLocalAndResolve(module, dependencyModules, scalaSuffix) assert(resolvedCp.map(_.last).contains(jarName)) for (check <- jarCheck; fileName <- resolvedCp.map(_.last)) assert(check(fileName)) } - val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules) + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) assert(resolvedM2Cp.map(_.last).contains(jarName)) for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) assert(check(fileName)) @@ -156,6 +159,22 @@ object BomTests extends UtestIntegrationTestSuite { ) } } + + test("invalid") { + test - integrationTest { tester => + import tester._ + + val res = eval( + ("show", "bom.invalid.exclude"), + check = false + ) + assert( + res.err.contains( + "Found parent or BOM dependencies with invalid parameters:" + ) + ) + } + } } test("parent") { @@ -166,6 +185,26 @@ object BomTests extends UtestIntegrationTestSuite { test("dependee") - integrationTest { implicit tester => isInClassPath("parent.dependee", expectedCommonsCompressJarName, Seq("parent")) } + + test("scala") - integrationTest { implicit tester => + isInClassPath("parent.scala", expectedCommonsCompressJarName, scalaSuffix = "_2.13") + } + + test("invalid") { + test - integrationTest { tester => + import tester._ + + val res = eval( + ("show", "parent.invalid.exclude"), + check = false + ) + assert( + res.err.contains( + "Found parent or BOM dependencies with invalid parameters:" + ) + ) + } + } } test("depMgmt") { @@ -223,6 +262,22 @@ object BomTests extends UtestIntegrationTestSuite { ) } + test("invalid") { + test - integrationTest { tester => + import tester._ + + val res = eval( + ("show", "depMgmt.invalid.transitive"), + check = false + ) + assert( + res.err.contains( + "Found dependency management entries with invalid values." + ) + ) + } + } + test("placeholder") - integrationTest { implicit tester => isInClassPath("depMgmt.placeholder", expectedProtobufJarName) } diff --git a/integration/feature/docannotations/src/DocAnnotationsTests.scala b/integration/feature/docannotations/src/DocAnnotationsTests.scala index 341b8f8f632..cdf38b9dff3 100644 --- a/integration/feature/docannotations/src/DocAnnotationsTests.scala +++ b/integration/feature/docannotations/src/DocAnnotationsTests.scala @@ -103,7 +103,8 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite { | |Inputs: | core.transitiveIvyDeps - | core.allBomDeps + | core.parentDep + | core.bomDeps |""".stripMargin, ivyDepsTree ) From 3dbc225ea0158b2cf89a7824b5f308e4ecb81bdc Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 21 Nov 2024 21:46:07 +0100 Subject: [PATCH 17/27] =?UTF-8?q?Starting=20to=20look=20good=20?= =?UTF-8?q?=F0=9F=A4=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- integration/feature/bom/resources/build.mill | 17 ++++++++ integration/feature/bom/src/BomTests.scala | 21 ++++++++++ scalalib/src/mill/scalalib/JavaModule.scala | 40 ++++++++++--------- .../src/mill/scalalib/PublishModule.scala | 2 +- 4 files changed, 61 insertions(+), 19 deletions(-) diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill index 6644dc65f96..0a112a78880 100644 --- a/integration/feature/bom/resources/build.mill +++ b/integration/feature/bom/resources/build.mill @@ -109,6 +109,23 @@ object depMgmt extends JavaModule with TestPublishModule { def moduleDeps = Seq(depMgmt) } + object extraExclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + // The exclude should be automatically added to the dependency above + // thanks to dependency management, but the version should be left + // untouched + ivy"com.lihaoyi:cask_2.13:0.9.3" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(extraExclude) + } + } + object exclude extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.lihaoyi:cask_2.13:0.9.4" diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala index 6ee77c27c6c..d65a76ab62b 100644 --- a/integration/feature/bom/src/BomTests.scala +++ b/integration/feature/bom/src/BomTests.scala @@ -216,6 +216,27 @@ object BomTests extends UtestIntegrationTestSuite { isInClassPath("depMgmt.transitive", expectedProtobufJarName, Seq("depMgmt")) } + test("extraExclude") - integrationTest { implicit tester => + isInClassPath( + "depMgmt.extraExclude", + "cask_2.13-0.9.4.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + } + ) + } + + test("transitiveExtraExclude") - integrationTest { implicit tester => + isInClassPath( + "depMgmt.extraExclude.transitive", + "cask_2.13-0.9.4.jar", + Seq("depMgmt.extraExclude"), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + } + ) + } + test("exclude") - integrationTest { implicit tester => isInClassPath( "depMgmt.exclude", diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 3169938622e..b908aeaca50 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -155,24 +155,28 @@ trait JavaModule depsWithBoms else { lazy val depMgmtMap = depMgmt.toMap - depsWithBoms.map { dep => - val versionOverride = - if (dep.dep.version == "_") { - val key = DependencyManagement.Key( - dep.dep.module.organization, - dep.dep.module.name, - coursier.core.Type.jar, - dep.dep.publication.classifier - ) - // FIXME Exclusions should be added too - depMgmtMap.get(key).map(_.version) - } else None - dep.copy( - dep = dep.dep - .withOverrides(dep.dep.overrides ++ depMgmt) - .withVersion(versionOverride.getOrElse(dep.dep.version)) - ) - } + depsWithBoms + .map(dep => dep.copy(dep = dep.dep.withOverrides(dep.dep.overrides ++ depMgmt))) + .map { dep => + val key = DependencyManagement.Key( + dep.dep.module.organization, + dep.dep.module.name, + coursier.core.Type.jar, + dep.dep.publication.classifier + ) + val versionOverride = + if (dep.dep.version == "_") depMgmtMap.get(key).map(_.version) + else None + val exclusions = depMgmtMap.get(key) + .map(_.minimizedExclusions) + .map(dep.dep.minimizedExclusions.join(_)) + .getOrElse(dep.dep.minimizedExclusions) + dep.copy( + dep = dep.dep + .withVersion(versionOverride.getOrElse(dep.dep.version)) + .withMinimizedExclusions(exclusions) + ) + } } } diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 27729739aa6..8ce53a945ef 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -75,7 +75,7 @@ trait PublishModule extends JavaModule { outer => def publishXmlDeps: Task[Agg[Dependency]] = Task.Anon { val ivyPomDeps = - (ivyDeps() ++ mandatoryIvyDeps()).map(resolvePublishDependency.apply().apply(_)) + allIvyDeps().map(resolvePublishDependency.apply().apply(_)) val compileIvyPomDeps = compileIvyDeps() .map(resolvePublishDependency.apply().apply(_)) From 97cb11387b84cbff20c197e44e40f656adf242af Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Fri, 22 Nov 2024 12:06:01 +0100 Subject: [PATCH 18/27] more --- integration/feature/bom/resources/build.mill | 18 +++++++++++++++ integration/feature/bom/src/BomTests.scala | 24 ++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill index 0a112a78880..bf26d74fae7 100644 --- a/integration/feature/bom/resources/build.mill +++ b/integration/feature/bom/resources/build.mill @@ -31,6 +31,12 @@ object bom extends Module { ) } + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + object check extends JavaModule { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java:_" @@ -52,6 +58,12 @@ object bom extends Module { ) } + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + object check extends JavaModule { def ivyDeps = Agg( ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" @@ -80,6 +92,12 @@ object parent extends JavaModule with TestPublishModule { ) } + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + object scala extends ScalaModule with TestPublishModule { def scalaVersion = _root_.scala.util.Properties.versionNumberString def parentDep = Some(ivy"org.apache.spark::spark-parent:3.5.3") diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala index d65a76ab62b..170cdebf925 100644 --- a/integration/feature/bom/src/BomTests.scala +++ b/integration/feature/bom/src/BomTests.scala @@ -138,6 +138,14 @@ object BomTests extends UtestIntegrationTestSuite { test("dependee") - integrationTest { implicit tester => isInClassPath("bom.placeholder.dependee", expectedProtobufJarName, Seq("bom.placeholder")) } + + test("subDependee") - integrationTest { implicit tester => + isInClassPath( + "bom.placeholder.subDependee", + expectedProtobufJarName, + Seq("bom.placeholder", "bom.placeholder.dependee") + ) + } } test("versionOverride") { @@ -158,6 +166,14 @@ object BomTests extends UtestIntegrationTestSuite { Seq("bom.versionOverride") ) } + + test("subDependee") - integrationTest { implicit tester => + isInClassPath( + "bom.versionOverride.subDependee", + expectedProtobufJarName, + Seq("bom.versionOverride", "bom.versionOverride.dependee") + ) + } } test("invalid") { @@ -186,6 +202,14 @@ object BomTests extends UtestIntegrationTestSuite { isInClassPath("parent.dependee", expectedCommonsCompressJarName, Seq("parent")) } + test("subDependee") - integrationTest { implicit tester => + isInClassPath( + "parent.subDependee", + expectedCommonsCompressJarName, + Seq("parent", "parent.dependee") + ) + } + test("scala") - integrationTest { implicit tester => isInClassPath("parent.scala", expectedCommonsCompressJarName, scalaSuffix = "_2.13") } From 8e284938ab31e12154c2588878120ab7deeed5d7 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 25 Nov 2024 19:28:55 +0100 Subject: [PATCH 19/27] more --- integration/feature/bom/resources/build.mill | 200 ------- integration/feature/bom/src/BomTests.scala | 339 ----------- main/util/src/mill/util/CoursierSupport.scala | 3 +- .../src/mill/scalalib/CoursierModule.scala | 1 - scalalib/src/mill/scalalib/Dep.scala | 2 + scalalib/src/mill/scalalib/JavaModule.scala | 144 +++-- .../src/mill/scalalib/PublishModule.scala | 19 +- .../test/src/mill/scalalib/BomTests.scala | 524 ++++++++++++++++++ 8 files changed, 613 insertions(+), 619 deletions(-) delete mode 100644 integration/feature/bom/resources/build.mill delete mode 100644 integration/feature/bom/src/BomTests.scala create mode 100644 scalalib/test/src/mill/scalalib/BomTests.scala diff --git a/integration/feature/bom/resources/build.mill b/integration/feature/bom/resources/build.mill deleted file mode 100644 index bf26d74fae7..00000000000 --- a/integration/feature/bom/resources/build.mill +++ /dev/null @@ -1,200 +0,0 @@ -package build - -import mill._ -import mill.scalalib._ -import mill.scalalib.publish._ - -trait TestPublishModule extends PublishModule { - def pomSettings = PomSettings( - description = artifactName(), - organization = "com.lihaoyi.mill-tests", - url = "https://github.com/com-lihaoyi/mill", - licenses = Seq(License.`Apache-2.0`), - versionControl = VersionControl.github("com-lihaoyi", "mill"), - developers = Nil - ) - def publishVersion = "0.1.0-SNAPSHOT" -} - -object bom extends Module { - object placeholder extends JavaModule with TestPublishModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0" - ) - def ivyDeps = Agg( - ivy"com.google.protobuf:protobuf-java:_" - ) - - object dependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - placeholder - ) - } - - object subDependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - dependee - ) - } - - object check extends JavaModule { - def ivyDeps = Agg( - ivy"com.google.protobuf:protobuf-java:_" - ) - } - } - - object versionOverride extends JavaModule with TestPublishModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0" - ) - def ivyDeps = Agg( - ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" - ) - - object dependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - versionOverride - ) - } - - object subDependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - dependee - ) - } - - object check extends JavaModule { - def ivyDeps = Agg( - ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" - ) - } - } - - object invalid extends Module { - object exclude extends JavaModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0".exclude(("foo", "thing")) - ) - } - } -} - -object parent extends JavaModule with TestPublishModule { - def parentDep = Some(ivy"org.apache.spark:spark-parent_2.13:3.5.3") - def ivyDeps = Agg( - ivy"org.apache.commons:commons-compress:_" - ) - - object dependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - parent - ) - } - - object subDependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - dependee - ) - } - - object scala extends ScalaModule with TestPublishModule { - def scalaVersion = _root_.scala.util.Properties.versionNumberString - def parentDep = Some(ivy"org.apache.spark::spark-parent:3.5.3") - def ivyDeps = Agg( - ivy"org.apache.commons:commons-compress:_" - ) - } - - object invalid extends Module { - object exclude extends JavaModule { - def parentDep = Some( - ivy"org.apache.spark:spark-parent_2.13:3.5.3".exclude(("foo", "thing")) - ) - } - } -} - -object depMgmt extends JavaModule with TestPublishModule { - def ivyDeps = Agg( - ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" - ) - def dependencyManagement = Agg( - ivy"com.google.protobuf:protobuf-java:4.28.3" - ) - - object transitive extends JavaModule with TestPublishModule { - def moduleDeps = Seq(depMgmt) - } - - object extraExclude extends JavaModule with TestPublishModule { - def ivyDeps = Agg( - ivy"com.lihaoyi:cask_2.13:0.9.4" - ) - def dependencyManagement = Agg( - // The exclude should be automatically added to the dependency above - // thanks to dependency management, but the version should be left - // untouched - ivy"com.lihaoyi:cask_2.13:0.9.3" - .exclude(("org.slf4j", "slf4j-api")) - ) - - object transitive extends JavaModule with TestPublishModule { - def moduleDeps = Seq(extraExclude) - } - } - - object exclude extends JavaModule with TestPublishModule { - def ivyDeps = Agg( - ivy"com.lihaoyi:cask_2.13:0.9.4" - ) - def dependencyManagement = Agg( - ivy"org.java-websocket:Java-WebSocket:1.5.2" - .exclude(("org.slf4j", "slf4j-api")) - ) - - object transitive extends JavaModule with TestPublishModule { - def moduleDeps = Seq(exclude) - } - } - - object onlyExclude extends JavaModule with TestPublishModule { - def ivyDeps = Agg( - ivy"com.lihaoyi:cask_2.13:0.9.4" - ) - def dependencyManagement = Agg( - ivy"org.java-websocket:Java-WebSocket:_" - .exclude(("org.slf4j", "slf4j-api")) - ) - - object transitive extends JavaModule with TestPublishModule { - def moduleDeps = Seq(onlyExclude) - } - } - - object invalid extends Module { - object transitive extends JavaModule { - def dependencyManagement = { - val dep = ivy"org.java-websocket:Java-WebSocket:1.5.3" - Agg( - dep.copy( - dep = dep.dep.withTransitive(false) - ) - ) - } - } - } - - object placeholder extends JavaModule with TestPublishModule { - def ivyDeps = Agg( - ivy"com.google.protobuf:protobuf-java:_" - ) - def dependencyManagement = Agg( - ivy"com.google.protobuf:protobuf-java:4.28.3" - ) - - object transitive extends JavaModule with TestPublishModule { - def moduleDeps = Seq(placeholder) - } - } -} diff --git a/integration/feature/bom/src/BomTests.scala b/integration/feature/bom/src/BomTests.scala deleted file mode 100644 index 170cdebf925..00000000000 --- a/integration/feature/bom/src/BomTests.scala +++ /dev/null @@ -1,339 +0,0 @@ -package mill.integration - -import mill.testkit.{IntegrationTester, UtestIntegrationTestSuite} -import utest._ - -import scala.jdk.CollectionConverters._ - -object BomTests extends UtestIntegrationTestSuite { - - def tests: Tests = Tests { - - def expectedProtobufJavaVersion = "4.28.3" - def expectedCommonsCompressVersion = "1.23.0" - - def expectedProtobufJarName = s"protobuf-java-$expectedProtobufJavaVersion.jar" - def expectedCommonsCompressJarName = s"commons-compress-$expectedCommonsCompressVersion.jar" - - def compileClasspathFileNames(moduleName: String)(implicit - tester: IntegrationTester - ): Seq[String] = { - import tester._ - val res = eval( - ("show", s"$moduleName.compileClasspath"), - stderr = os.Inherit, - check = true - ) - ujson.read(res.out).arr.map(v => os.Path(v.str.split(":").last).last).toSeq - } - - def compileClasspathContains( - module: String, - fileName: String, - jarCheck: Option[String => Boolean] - )(implicit - tester: IntegrationTester - ) = { - val fileNames = compileClasspathFileNames(module) - assert(fileNames.contains(fileName)) - for (check <- jarCheck; fileName <- fileNames) - assert(check(fileName)) - } - - def publishLocalAndResolve( - module: String, - dependencyModules: Seq[String], - scalaSuffix: String - )(implicit tester: IntegrationTester): Seq[os.Path] = { - val localIvyRepo = tester.workspacePath / "ivy2Local" - for (moduleName <- module +: dependencyModules) - tester.eval((s"$moduleName.publishLocal", "--localIvyRepo", localIvyRepo), check = true) - - coursierapi.Fetch.create() - .addDependencies( - coursierapi.Dependency.of( - "com.lihaoyi.mill-tests", - module.replace('.', '-') + scalaSuffix, - "0.1.0-SNAPSHOT" - ) - ) - .addRepositories( - coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") - ) - .fetch() - .asScala - .map(os.Path(_)) - .toVector - } - - def publishM2LocalAndResolve( - module: String, - dependencyModules: Seq[String], - scalaSuffix: String - )(implicit tester: IntegrationTester): Seq[os.Path] = { - val localM2Repo = tester.workspacePath / "m2Local" - for (moduleName <- module +: dependencyModules) - tester.eval((s"$moduleName.publishM2Local", "--m2RepoPath", localM2Repo), check = true) - - coursierapi.Fetch.create() - .addDependencies( - coursierapi.Dependency.of( - "com.lihaoyi.mill-tests", - module.replace('.', '-') + scalaSuffix, - "0.1.0-SNAPSHOT" - ) - ) - .addRepositories( - coursierapi.MavenRepository.of(localM2Repo.toNIO.toUri.toASCIIString) - ) - .fetch() - .asScala - .map(os.Path(_)) - .toVector - } - - def isInClassPath( - module: String, - jarName: String, - dependencyModules: Seq[String] = Nil, - jarCheck: Option[String => Boolean] = None, - ivy2LocalCheck: Boolean = true, - scalaSuffix: String = "" - )(implicit tester: IntegrationTester): Unit = { - compileClasspathContains(module, jarName, jarCheck) - - if (ivy2LocalCheck) { - val resolvedCp = publishLocalAndResolve(module, dependencyModules, scalaSuffix) - assert(resolvedCp.map(_.last).contains(jarName)) - for (check <- jarCheck; fileName <- resolvedCp.map(_.last)) - assert(check(fileName)) - } - - val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) - assert(resolvedM2Cp.map(_.last).contains(jarName)) - for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) - assert(check(fileName)) - } - - test("bom") { - test("placeholder") { - test("check") - integrationTest { tester => - import tester._ - - val res = eval( - ("show", "bom.placeholder.check.compileClasspath"), - check = false - ) - assert( - res.err.contains( - "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" - ) - ) - } - - test("simple") - integrationTest { implicit tester => - isInClassPath("bom.placeholder", expectedProtobufJarName) - } - - test("dependee") - integrationTest { implicit tester => - isInClassPath("bom.placeholder.dependee", expectedProtobufJarName, Seq("bom.placeholder")) - } - - test("subDependee") - integrationTest { implicit tester => - isInClassPath( - "bom.placeholder.subDependee", - expectedProtobufJarName, - Seq("bom.placeholder", "bom.placeholder.dependee") - ) - } - } - - test("versionOverride") { - test("check") - integrationTest { implicit tester => - val fileNames = compileClasspathFileNames("bom.versionOverride.check") - assert(fileNames.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar"))) - assert(!fileNames.contains(expectedProtobufJarName)) - } - - test("simple") - integrationTest { implicit tester => - isInClassPath("bom.versionOverride", expectedProtobufJarName) - } - - test("dependee") - integrationTest { implicit tester => - isInClassPath( - "bom.versionOverride.dependee", - expectedProtobufJarName, - Seq("bom.versionOverride") - ) - } - - test("subDependee") - integrationTest { implicit tester => - isInClassPath( - "bom.versionOverride.subDependee", - expectedProtobufJarName, - Seq("bom.versionOverride", "bom.versionOverride.dependee") - ) - } - } - - test("invalid") { - test - integrationTest { tester => - import tester._ - - val res = eval( - ("show", "bom.invalid.exclude"), - check = false - ) - assert( - res.err.contains( - "Found parent or BOM dependencies with invalid parameters:" - ) - ) - } - } - } - - test("parent") { - test("simple") - integrationTest { implicit tester => - isInClassPath("parent", expectedCommonsCompressJarName) - } - - test("dependee") - integrationTest { implicit tester => - isInClassPath("parent.dependee", expectedCommonsCompressJarName, Seq("parent")) - } - - test("subDependee") - integrationTest { implicit tester => - isInClassPath( - "parent.subDependee", - expectedCommonsCompressJarName, - Seq("parent", "parent.dependee") - ) - } - - test("scala") - integrationTest { implicit tester => - isInClassPath("parent.scala", expectedCommonsCompressJarName, scalaSuffix = "_2.13") - } - - test("invalid") { - test - integrationTest { tester => - import tester._ - - val res = eval( - ("show", "parent.invalid.exclude"), - check = false - ) - assert( - res.err.contains( - "Found parent or BOM dependencies with invalid parameters:" - ) - ) - } - } - } - - test("depMgmt") { - test("override") - integrationTest { implicit tester => - isInClassPath("depMgmt", expectedProtobufJarName) - } - - test("transitiveOverride") - integrationTest { implicit tester => - isInClassPath("depMgmt.transitive", expectedProtobufJarName, Seq("depMgmt")) - } - - test("extraExclude") - integrationTest { implicit tester => - isInClassPath( - "depMgmt.extraExclude", - "cask_2.13-0.9.4.jar", - jarCheck = Some { jarName => - !jarName.startsWith("slf4j-api-") - } - ) - } - - test("transitiveExtraExclude") - integrationTest { implicit tester => - isInClassPath( - "depMgmt.extraExclude.transitive", - "cask_2.13-0.9.4.jar", - Seq("depMgmt.extraExclude"), - jarCheck = Some { jarName => - !jarName.startsWith("slf4j-api-") - } - ) - } - - test("exclude") - integrationTest { implicit tester => - isInClassPath( - "depMgmt.exclude", - "Java-WebSocket-1.5.2.jar", - jarCheck = Some { jarName => - !jarName.startsWith("slf4j-api-") - }, - ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml - ) - } - - test("transitiveExclude") - integrationTest { implicit tester => - isInClassPath( - "depMgmt.exclude.transitive", - "Java-WebSocket-1.5.2.jar", - Seq("depMgmt.exclude"), - jarCheck = Some { jarName => - !jarName.startsWith("slf4j-api-") - }, - ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml - ) - } - - test("onlyExclude") - integrationTest { implicit tester => - isInClassPath( - "depMgmt.onlyExclude", - "Java-WebSocket-1.5.3.jar", - jarCheck = Some { jarName => - !jarName.startsWith("slf4j-api-") - }, - ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml - ) - } - - test("transitiveOnlyExclude") - integrationTest { implicit tester => - isInClassPath( - "depMgmt.onlyExclude.transitive", - "Java-WebSocket-1.5.3.jar", - Seq("depMgmt.onlyExclude"), - jarCheck = Some { jarName => - !jarName.startsWith("slf4j-api-") - }, - ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml - ) - } - - test("invalid") { - test - integrationTest { tester => - import tester._ - - val res = eval( - ("show", "depMgmt.invalid.transitive"), - check = false - ) - assert( - res.err.contains( - "Found dependency management entries with invalid values." - ) - ) - } - } - - test("placeholder") - integrationTest { implicit tester => - isInClassPath("depMgmt.placeholder", expectedProtobufJarName) - } - - test("transitivePlaceholder") - integrationTest { implicit tester => - isInClassPath( - "depMgmt.placeholder.transitive", - expectedProtobufJarName, - Seq("depMgmt.placeholder") - ) - } - } - } -} diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index efa9f489382..27bd8e14331 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -31,7 +31,7 @@ trait CoursierSupport { ctx.fold(cache)(c => cache.withLogger(new TickerResolutionLogger(c))) } - def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { + private def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { val org = dep.module.organization.value val name = dep.module.name.value val classpathKey = s"$org-$name" @@ -262,6 +262,7 @@ trait CoursierSupport { val rootDeps = deps.iterator .map(d => mapDependencies.fold(d)(_.apply(d))) + .filter(dep => isLocalTestDep(dep).isEmpty) .toSeq val bomDeps0 = bomDeps.iterator.toSeq diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index c055e5e01a0..6ed405cfa3d 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -258,7 +258,6 @@ object CoursierModule { ): (Seq[Dependency], DependencyManagement.Map) = { val deps0 = deps .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) - .filter(dep => mill.util.Jvm.isLocalTestDep(dep.dep).isEmpty) val res = Lib.resolveDependenciesMetadataSafe( repositories = repositories, deps = deps0, diff --git a/scalalib/src/mill/scalalib/Dep.scala b/scalalib/src/mill/scalalib/Dep.scala index 0c1d41bf351..86f76ec3a6f 100644 --- a/scalalib/src/mill/scalalib/Dep.scala +++ b/scalalib/src/mill/scalalib/Dep.scala @@ -118,6 +118,8 @@ object Dep { } (module.split(':') match { + case Array(a, b) => Dep(a, b, "_", cross = empty(platformed = false)) + case Array(a, "", b) => Dep(a, b, "_", cross = Binary(platformed = false)) case Array(a, b, c) => Dep(a, b, c, cross = empty(platformed = false)) case Array(a, b, "", c) => Dep(a, b, c, cross = empty(platformed = true)) case Array(a, "", b, c) => Dep(a, b, c, cross = Binary(platformed = false)) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index b908aeaca50..a55ae6cd0bb 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -146,37 +146,35 @@ trait JavaModule */ def allIvyDeps: T[Agg[Dep]] = Task { val bomDeps0 = allBomDeps().toSeq - val rawDeps = ivyDeps() ++ mandatoryIvyDeps() - val depsWithBoms = - if (bomDeps0.isEmpty) rawDeps - else rawDeps.map(dep => dep.copy(dep = dep.dep.addBoms(bomDeps0))) - val depMgmt = dependencyManagementDict() - if (depMgmt.isEmpty) - depsWithBoms - else { - lazy val depMgmtMap = depMgmt.toMap - depsWithBoms - .map(dep => dep.copy(dep = dep.dep.withOverrides(dep.dep.overrides ++ depMgmt))) - .map { dep => - val key = DependencyManagement.Key( - dep.dep.module.organization, - dep.dep.module.name, - coursier.core.Type.jar, - dep.dep.publication.classifier - ) - val versionOverride = - if (dep.dep.version == "_") depMgmtMap.get(key).map(_.version) - else None - val exclusions = depMgmtMap.get(key) - .map(_.minimizedExclusions) - .map(dep.dep.minimizedExclusions.join(_)) - .getOrElse(dep.dep.minimizedExclusions) - dep.copy( - dep = dep.dep - .withVersion(versionOverride.getOrElse(dep.dep.version)) - .withMinimizedExclusions(exclusions) + val depMgmt = processedDependencyManagement( + dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap + (ivyDeps() ++ mandatoryIvyDeps()).map { dep => + val depMgmtKey = DependencyManagement.Key( + dep.dep.module.organization, + dep.dep.module.name, + coursier.core.Type.jar, + dep.dep.publication.classifier + ) + val versionOverrideOpt = + if (dep.dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) + else None + val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) + dep.copy( + dep = dep.dep + // add BOM coordinates - coursier will handle the rest + .addBoms(bomDeps0) + // add dependency management ourselves: + // - overrides meant to apply to transitive dependencies + // - fill version if it's empty + // - add extra exclusions from dependency management + .withOverrides(dep.dep.overrides ++ depMgmt) + .withVersion(versionOverrideOpt.getOrElse(dep.dep.version)) + .withMinimizedExclusions( + extraExclusions.fold(dep.dep.minimizedExclusions)(dep.dep.minimizedExclusions.join(_)) ) - } + ) } } @@ -251,51 +249,51 @@ trait JavaModule * Data from dependencyManagement, converted to a type ready to be passed to coursier * for dependency resolution */ - def dependencyManagementDict: Task[Seq[(DependencyManagement.Key, DependencyManagement.Values)]] = - Task.Anon { - val keyValuesOrErrors = - dependencyManagement().toSeq.map(bindDependency()).map(_.dep).map { depMgmt => - val fromUsedValues = coursier.core.Dependency(depMgmt.module, depMgmt.version) - .withPublication(coursier.core.Publication( - "", - depMgmt.publication.`type`, - coursier.core.Extension.empty, - depMgmt.publication.classifier - )) - .withMinimizedExclusions(depMgmt.minimizedExclusions) - .withOptional(depMgmt.optional) - .withConfiguration(Configuration.defaultCompile) - if (fromUsedValues == depMgmt) { - val key = DependencyManagement.Key( - depMgmt.module.organization, - depMgmt.module.name, - if (depMgmt.publication.`type`.isEmpty) coursier.core.Type.jar - else depMgmt.publication.`type`, - depMgmt.publication.classifier - ) - val values = DependencyManagement.Values( - Configuration.empty, - if (depMgmt.version == "_") "" // shouldn't be needed with future coursier versions - else depMgmt.version, - depMgmt.minimizedExclusions, - depMgmt.optional - ) - Right(key -> values) - } else - Left(depMgmt) - } - - val errors = keyValuesOrErrors.collect { - case Left(errored) => errored + private def processedDependencyManagement(deps: Seq[coursier.core.Dependency]) + : Seq[(DependencyManagement.Key, DependencyManagement.Values)] = { + val keyValuesOrErrors = + deps.map { depMgmt => + val fromUsedValues = coursier.core.Dependency(depMgmt.module, depMgmt.version) + .withPublication(coursier.core.Publication( + "", + depMgmt.publication.`type`, + coursier.core.Extension.empty, + depMgmt.publication.classifier + )) + .withMinimizedExclusions(depMgmt.minimizedExclusions) + .withOptional(depMgmt.optional) + .withConfiguration(Configuration.defaultCompile) + if (fromUsedValues == depMgmt) { + val key = DependencyManagement.Key( + depMgmt.module.organization, + depMgmt.module.name, + if (depMgmt.publication.`type`.isEmpty) coursier.core.Type.jar + else depMgmt.publication.`type`, + depMgmt.publication.classifier + ) + val values = DependencyManagement.Values( + Configuration.empty, + if (depMgmt.version == "_") "" // shouldn't be needed with future coursier versions + else depMgmt.version, + depMgmt.minimizedExclusions, + depMgmt.optional + ) + Right(key -> values) + } else + Left(depMgmt) } - if (errors.isEmpty) - keyValuesOrErrors.collect { case Right(kv) => kv } - else - throw new Exception( - "Found dependency management entries with invalid values. Only organization, name, version, type, classifier, exclusions, and optionality can be specified" + System.lineSeparator() + - errors.map("- " + _ + System.lineSeparator()).mkString - ) + + val errors = keyValuesOrErrors.collect { + case Left(errored) => errored } + if (errors.isEmpty) + keyValuesOrErrors.collect { case Right(kv) => kv } + else + throw new Exception( + "Found dependency management entries with invalid values. Only organization, name, version, type, classifier, exclusions, and optionality can be specified" + System.lineSeparator() + + errors.map("- " + _ + System.lineSeparator()).mkString + ) + } /** * Default artifact types to fetch and put in the classpath. Add extra types diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 8ce53a945ef..fb781cfd84e 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -134,7 +134,6 @@ trait PublishModule extends JavaModule { outer => def ivy: T[PathRef] = Task { val (rootDepVersions, bomDepMgmt) = bomDetails() - val depMgmt = dependencyManagementDict() ++ bomDepMgmt val publishXmlDeps0 = publishXmlDeps().map { dep => if (dep.artifact.version == "_") dep.copy( @@ -152,10 +151,20 @@ trait PublishModule extends JavaModule { outer => else dep } - val overrides = depMgmt.toSeq.map { - case (key, values) => - Ivy.Override(key.organization.value, key.name.value, values.version) - } + val overrides = + dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") + .map { depMgmt => + Ivy.Override( + depMgmt.module.organization.value, + depMgmt.module.name.value, + depMgmt.version + ) + } ++ + bomDepMgmt.map { + case (key, values) => + Ivy.Override(key.organization.value, key.name.value, values.version) + } val ivy = Ivy(artifactMetadata(), publishXmlDeps0, extraPublish(), overrides) val ivyPath = T.dest / "ivy.xml" os.write.over(ivyPath, ivy) diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala new file mode 100644 index 00000000000..e21759e9992 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -0,0 +1,524 @@ +package mill +package scalalib + +import mill.scalalib.publish._ +import mill.testkit.{TestBaseModule, UnitTester} +import utest._ +import mill.PathRef + +import scala.jdk.CollectionConverters._ + +object BomTests extends TestSuite { + + trait TestPublishModule extends PublishModule { + def pomSettings = PomSettings( + description = artifactName(), + organization = "com.lihaoyi.mill-tests", + url = "https://github.com/com-lihaoyi/mill", + licenses = Seq(License.`Apache-2.0`), + versionControl = VersionControl.github("com-lihaoyi", "mill"), + developers = Nil + ) + def publishVersion = "0.1.0-SNAPSHOT" + } + + object modules extends TestBaseModule { + object bom extends Module { + object placeholder extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + placeholder + ) + } + + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + + object check extends JavaModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + } + } + + object versionOverride extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + versionOverride + ) + } + + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + + object check extends JavaModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + } + } + + object invalid extends TestBaseModule { + object exclude extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0".exclude(("foo", "thing")) + ) + } + } + } + + object parent extends JavaModule with TestPublishModule { + def parentDep = Some(ivy"org.apache.spark:spark-parent_2.13:3.5.3") + def ivyDeps = Agg( + ivy"org.apache.commons:commons-compress" + ) + + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + parent + ) + } + + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + + object scala extends ScalaModule with TestPublishModule { + def scalaVersion = _root_.scala.util.Properties.versionNumberString + def parentDep = Some(ivy"org.apache.spark::spark-parent:3.5.3") + def ivyDeps = Agg( + ivy"org.apache.commons:commons-compress" + ) + } + + object invalid extends TestBaseModule { + object exclude extends JavaModule { + def parentDep = Some( + ivy"org.apache.spark:spark-parent_2.13:3.5.3".exclude(("foo", "thing")) + ) + } + } + } + + object depMgmt extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(depMgmt) + } + + object extraExclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + // The exclude should be automatically added to the dependency above + // thanks to dependency management, but the version should be left + // untouched + ivy"com.lihaoyi:cask_2.13:0.9.3" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(extraExclude) + } + } + + object exclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + ivy"org.java-websocket:Java-WebSocket:1.5.2" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(exclude) + } + } + + object onlyExclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def dependencyManagement = Agg( + ivy"org.java-websocket:Java-WebSocket" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(onlyExclude) + } + } + + object invalid extends TestBaseModule { + object transitive extends JavaModule { + def dependencyManagement = { + val dep = ivy"org.java-websocket:Java-WebSocket:1.5.3" + Agg( + dep.copy( + dep = dep.dep.withTransitive(false) + ) + ) + } + } + } + + object placeholder extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(placeholder) + } + } + } + } + + + def expectedProtobufJavaVersion = "4.28.3" + def expectedCommonsCompressVersion = "1.23.0" + + def expectedProtobufJarName = s"protobuf-java-$expectedProtobufJavaVersion.jar" + def expectedCommonsCompressJarName = s"commons-compress-$expectedCommonsCompressVersion.jar" + + def compileClasspathFileNames(module: JavaModule)(implicit + eval: UnitTester + ): Seq[String] = + eval(module.compileClasspath).toTry.get.value + .toSeq.map(_.path.last) + + def compileClasspathContains( + module: JavaModule, + fileName: String, + jarCheck: Option[String => Boolean] + )(implicit + eval: UnitTester + ) = { + val fileNames = compileClasspathFileNames(module) + assert(fileNames.contains(fileName)) + for (check <- jarCheck; fileName <- fileNames) + assert(check(fileName)) + } + + def publishLocalAndResolve( + module: PublishModule, + dependencyModules: Seq[PublishModule], + scalaSuffix: String + )(implicit eval: UnitTester): Seq[os.Path] = { + val localIvyRepo = eval.evaluator.workspace / "ivy2Local" + eval(module.publishLocal(localIvyRepo.toString)).toTry.get + for (dependencyModule <- dependencyModules) + eval(dependencyModule.publishLocal(localIvyRepo.toString)).toTry.get + + val moduleString = eval(module.artifactName).toTry.get.value + + coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + moduleString.replace('.', '-') + scalaSuffix, + "0.1.0-SNAPSHOT" + ) + ) + .addRepositories( + coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + } + + def publishM2LocalAndResolve( + module: PublishModule, + dependencyModules: Seq[PublishModule], + scalaSuffix: String + )(implicit eval: UnitTester): Seq[os.Path] = { + val localM2Repo = eval.evaluator.workspace / "m2Local" + eval(module.publishM2Local(localM2Repo.toString)).toTry.get + for (dependencyModule <- dependencyModules) + eval(dependencyModule.publishM2Local(localM2Repo.toString)).toTry.get + + val moduleString = eval(module.artifactName).toTry.get.value + + coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + moduleString.replace('.', '-') + scalaSuffix, + "0.1.0-SNAPSHOT" + ) + ) + .addRepositories( + coursierapi.MavenRepository.of(localM2Repo.toNIO.toUri.toASCIIString) + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + } + + def isInClassPath( + module: JavaModule with PublishModule, + jarName: String, + dependencyModules: Seq[PublishModule] = Nil, + jarCheck: Option[String => Boolean] = None, + ivy2LocalCheck: Boolean = true, + scalaSuffix: String = "" + )(implicit eval: UnitTester): Unit = { + compileClasspathContains(module, jarName, jarCheck) + + if (ivy2LocalCheck) { + val resolvedCp = publishLocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedCp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedCp.map(_.last)) + assert(check(fileName)) + } + + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedM2Cp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) + assert(check(fileName)) + } + + def tests = Tests { + + test("bom") { + test("placeholder") { + test("check") - UnitTester(modules, null).scoped { eval => + val res = eval(modules.bom.placeholder.check.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" + )) + ) + } + + test("simple") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.bom.placeholder, expectedProtobufJarName) + } + + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.placeholder.dependee, + expectedProtobufJarName, + Seq(modules.bom.placeholder) + ) + } + + test("subDependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.placeholder.subDependee, + expectedProtobufJarName, + Seq(modules.bom.placeholder, modules.bom.placeholder.dependee) + ) + } + } + + test("versionOverride") { + test("check") - UnitTester(modules, null).scoped { implicit eval => + val fileNames = compileClasspathFileNames(modules.bom.versionOverride.check) + assert(fileNames.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar"))) + assert(!fileNames.contains(expectedProtobufJarName)) + } + + test("simple") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.bom.versionOverride, expectedProtobufJarName) + } + + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.versionOverride.dependee, + expectedProtobufJarName, + Seq(modules.bom.versionOverride) + ) + } + + test("subDependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.versionOverride.subDependee, + expectedProtobufJarName, + Seq(modules.bom.versionOverride, modules.bom.versionOverride.dependee) + ) + } + } + + test("invalid") { + test - UnitTester(modules, null).scoped { eval => + val res = eval(modules.bom.invalid.exclude.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "Found parent or BOM dependencies with invalid parameters:" + )) + ) + } + } + } + + test("parent") { + test("simple") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.parent, expectedCommonsCompressJarName) + } + + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.parent.dependee, expectedCommonsCompressJarName, Seq(modules.parent)) + } + + test("subDependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.parent.subDependee, + expectedCommonsCompressJarName, + Seq(modules.parent, modules.parent.dependee) + ) + } + + test("scala") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.parent.scala, expectedCommonsCompressJarName, scalaSuffix = "_2.13") + } + + test("invalid") { + test - UnitTester(modules, null).scoped { eval => + val res = eval(modules.parent.invalid.exclude.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "Found parent or BOM dependencies with invalid parameters:" + )) + ) + } + } + } + + test("depMgmt") { + test("override") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt, expectedProtobufJarName) + } + + test("transitiveOverride") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt.transitive, expectedProtobufJarName, Seq(modules.depMgmt)) + } + + test("extraExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.extraExclude, + "cask_2.13-0.9.4.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + } + ) + } + + test("transitiveExtraExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.extraExclude.transitive, + "cask_2.13-0.9.4.jar", + Seq(modules.depMgmt.extraExclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + } + ) + } + + test("exclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.exclude, + "Java-WebSocket-1.5.2.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("transitiveExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.exclude.transitive, + "Java-WebSocket-1.5.2.jar", + Seq(modules.depMgmt.exclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("onlyExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.onlyExclude, + "Java-WebSocket-1.5.3.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("transitiveOnlyExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.onlyExclude.transitive, + "Java-WebSocket-1.5.3.jar", + Seq(modules.depMgmt.onlyExclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("invalid") { + test - UnitTester(modules, null).scoped { eval => + val res = eval(modules.depMgmt.invalid.transitive.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "Found dependency management entries with invalid values." + )) + ) + } + } + + test("placeholder") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt.placeholder, expectedProtobufJarName) + } + + test("transitivePlaceholder") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.placeholder.transitive, + expectedProtobufJarName, + Seq(modules.depMgmt.placeholder) + ) + } + } + } +} From 7950d4f56969bbd77eaa3df154526afddf60d80c Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 25 Nov 2024 19:30:01 +0100 Subject: [PATCH 20/27] fmt --- scalalib/test/src/mill/scalalib/BomTests.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index e21759e9992..9b3e21b309c 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -99,11 +99,11 @@ object BomTests extends TestSuite { ) } - object subDependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - dependee - ) - } + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } object scala extends ScalaModule with TestPublishModule { def scalaVersion = _root_.scala.util.Properties.versionNumberString @@ -207,7 +207,6 @@ object BomTests extends TestSuite { } } - def expectedProtobufJavaVersion = "4.28.3" def expectedCommonsCompressVersion = "1.23.0" From 745b500c7b0e274045db0d61d73f5ff1c0be3f1d Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 27 Nov 2024 21:17:47 +0100 Subject: [PATCH 21/27] more - needs clean-up, only checking if CI passes --- build.mill | 2 +- main/util/src/mill/util/CoursierSupport.scala | 15 +- .../src/mill/scalalib/CoursierModule.scala | 52 +++---- scalalib/src/mill/scalalib/JavaModule.scala | 138 ++++++++++++------ .../src/mill/scalalib/JsonFormatters.scala | 1 + scalalib/src/mill/scalalib/Lib.scala | 15 +- .../src/mill/scalalib/PublishModule.scala | 37 ++++- scalalib/src/mill/scalalib/publish/Pom.scala | 2 +- .../test/src/mill/scalalib/BomTests.scala | 91 +++++++++++- 9 files changed, 241 insertions(+), 112 deletions(-) diff --git a/build.mill b/build.mill index 1dcc4896667..1b709807c49 100644 --- a/build.mill +++ b/build.mill @@ -116,7 +116,7 @@ object Deps { val asmTree = ivy"org.ow2.asm:asm-tree:9.7.1" val bloopConfig = ivy"ch.epfl.scala::bloop-config:1.5.5" - val coursierVersion = "2.1.18" + val coursierVersion = "2.1.19" val coursier = ivy"io.get-coursier::coursier:$coursierVersion" val coursierInterface = ivy"io.get-coursier:interface:1.0.24" val coursierJvm = ivy"io.get-coursier::coursier-jvm:$coursierVersion" diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 27bd8e14331..6affefbd025 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -63,8 +63,7 @@ trait CoursierSupport { coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, resolveFilter: os.Path => Boolean = _ => true, artifactTypes: Option[Set[Type]] = None, - resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[(Module, String)] = Nil + resolutionParams: ResolutionParams = ResolutionParams() ): Result[Agg[PathRef]] = { val (localTestDeps, remoteDeps) = deps.iterator.toSeq.partitionMap(d => isLocalTestDep(d).toLeft(d)) @@ -77,8 +76,7 @@ trait CoursierSupport { customizer, ctx, coursierCacheCustomizer, - resolutionParams, - bomDeps + resolutionParams ) resolutionRes.flatMap { resolution => @@ -191,8 +189,7 @@ trait CoursierSupport { customizer, ctx, coursierCacheCustomizer, - ResolutionParams(), - Nil + ResolutionParams() ) (deps0, res.getOrThrow) } @@ -256,8 +253,7 @@ trait CoursierSupport { customizer: Option[Resolution => Resolution] = None, ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, - resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[(Module, String)] = Nil + resolutionParams: ResolutionParams = ResolutionParams() ): Result[Resolution] = { val rootDeps = deps.iterator @@ -265,8 +261,6 @@ trait CoursierSupport { .filter(dep => isLocalTestDep(dep).isEmpty) .toSeq - val bomDeps0 = bomDeps.iterator.toSeq - val forceVersions = force.iterator .map(mapDependencies.getOrElse(identity[Dependency](_))) .map { d => d.module -> d.version } @@ -280,7 +274,6 @@ trait CoursierSupport { val resolve = Resolve() .withCache(coursierCache0) .withDependencies(rootDeps) - .withBomModuleVersions(bomDeps0) .withRepositories(repositories) .withResolutionParams(resolutionParams0) .withMapDependenciesOpt(mapDependencies) diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 6ed405cfa3d..d9ce281feba 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -56,8 +56,7 @@ trait CoursierModule extends mill.Module { def resolveDeps( deps: Task[Agg[BoundDep]], sources: Boolean = false, - artifactTypes: Option[Set[Type]] = None, - bomDeps: Task[Agg[(Module, String)]] = Task.Anon(Agg.empty[(Module, String)]) + artifactTypes: Option[Set[Type]] = None ): Task[Agg[PathRef]] = Task.Anon { Lib.resolveDependencies( @@ -68,25 +67,16 @@ trait CoursierModule extends mill.Module { mapDependencies = Some(mapDependencies()), customizer = resolutionCustomizer(), coursierCacheCustomizer = coursierCacheCustomizer(), - ctx = Some(implicitly[mill.api.Ctx.Log]), - bomDeps = bomDeps() + ctx = Some(implicitly[mill.api.Ctx.Log]) ) } - // bin-compat shim - def resolveDeps( - deps: Task[Agg[BoundDep]], - sources: Boolean, - artifactTypes: Option[Set[Type]] - ): Task[Agg[PathRef]] = - resolveDeps(deps, sources, artifactTypes, Task.Anon(Agg.empty[(Module, String)])) - @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDeps( deps: Task[Agg[BoundDep]], sources: Boolean ): Task[Agg[PathRef]] = - resolveDeps(deps, sources, None, Task.Anon(Agg.empty[(Module, String)])) + resolveDeps(deps, sources, None) /** * Map dependencies before resolving them. @@ -205,8 +195,7 @@ object CoursierModule { def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean = false, - artifactTypes: Option[Set[coursier.Type]] = None, - bomDeps: IterableOnce[(Module, String)] = Nil + artifactTypes: Option[Set[coursier.Type]] = None ): Agg[PathRef] = { Lib.resolveDependencies( repositories = repositories, @@ -217,25 +206,16 @@ object CoursierModule { customizer = customizer, coursierCacheCustomizer = coursierCacheCustomizer, ctx = ctx, - resolutionParams = resolutionParams, - bomDeps = bomDeps + resolutionParams = resolutionParams ).getOrThrow } - // bin-compat shim - def resolveDeps[T: CoursierModule.Resolvable]( - deps: IterableOnce[T], - sources: Boolean, - artifactTypes: Option[Set[coursier.Type]] - ): Agg[PathRef] = - resolveDeps(deps, sources, artifactTypes, Nil) - @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], sources: Boolean ): Agg[PathRef] = - resolveDeps(deps, sources, None, Nil) + resolveDeps(deps, sources, None) /** * Processes dependencies and BOMs with coursier @@ -248,13 +228,11 @@ object CoursierModule { * * @param deps dependencies that might have placeholder versions ("_" as version) * @param resolutionParams coursier resolution parameters - * @param bomDeps dependencies to read Bill-Of-Materials from * @return dependencies with version placeholder filled and data read from the BOM dependencies */ def processDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], - resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[(Module, String)] = Nil + resolutionParams: ResolutionParams = ResolutionParams() ): (Seq[Dependency], DependencyManagement.Map) = { val deps0 = deps .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) @@ -265,10 +243,20 @@ object CoursierModule { customizer = customizer, coursierCacheCustomizer = coursierCacheCustomizer, ctx = ctx, - resolutionParams = resolutionParams, - bomDeps = bomDeps + resolutionParams = resolutionParams ).getOrThrow - (res.processedRootDependencies, res.bomDepMgmt) + val depMgmt: DependencyManagement.Map = + if (res.processedRootDependencies.isEmpty) Map.empty + else { + val overrides = res.processedRootDependencies.map(_.overrides) + overrides.tail.foldLeft(overrides.head) { (acc, map) => + acc.filter { + case (key, values) => + map.get(key).contains(values) + } + } + } + (res.processedRootDependencies, depMgmt) } } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index a55ae6cd0bb..ea2cb9c5584 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1,7 +1,7 @@ package mill package scalalib -import coursier.core.{Configuration, DependencyManagement, Resolution} +import coursier.core.{BomDependency, Configuration, DependencyManagement, Resolution} import coursier.parse.JavaOrScalaModule import coursier.parse.ModuleParser import coursier.util.ModuleMatcher @@ -57,6 +57,10 @@ trait JavaModule } } + override def extraBomDeps = Task.Anon { + outer.allBomDeps().map(_.withConfig(Configuration.test)): Agg[BomDependency] + } + /** * JavaModule and its derivatives define inner test modules. * To avoid unexpected misbehavior due to the use of the wrong inner test trait @@ -145,36 +149,63 @@ trait JavaModule * In most cases, instead of overriding this Target you want to override `ivyDeps` instead. */ def allIvyDeps: T[Agg[Dep]] = Task { - val bomDeps0 = allBomDeps().toSeq - val depMgmt = processedDependencyManagement( - dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ivyDeps() ++ mandatoryIvyDeps() + } + + private def addBoms( + dep: coursier.core.Dependency, + bomDeps: Seq[coursier.core.BomDependency], + depMgmt: Seq[(DependencyManagement.Key, DependencyManagement.Values)], + depMgmtMap: DependencyManagement.Map, + overrideVersions: Boolean = false + ): coursier.core.Dependency = { + val depMgmtKey = DependencyManagement.Key( + dep.module.organization, + dep.module.name, + coursier.core.Type.jar, + dep.publication.classifier ) - val depMgmtMap = depMgmt.toMap - (ivyDeps() ++ mandatoryIvyDeps()).map { dep => - val depMgmtKey = DependencyManagement.Key( - dep.dep.module.organization, - dep.dep.module.name, - coursier.core.Type.jar, - dep.dep.publication.classifier + val versionOverrideOpt = + if (dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) + else None + val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) + dep + // add BOM coordinates - coursier will handle the rest + .addBomDependencies( + if (overrideVersions) bomDeps.map(_.withForceOverrideVersions(overrideVersions)) + else bomDeps ) - val versionOverrideOpt = - if (dep.dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) - else None - val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) - dep.copy( - dep = dep.dep - // add BOM coordinates - coursier will handle the rest - .addBoms(bomDeps0) - // add dependency management ourselves: - // - overrides meant to apply to transitive dependencies - // - fill version if it's empty - // - add extra exclusions from dependency management - .withOverrides(dep.dep.overrides ++ depMgmt) - .withVersion(versionOverrideOpt.getOrElse(dep.dep.version)) - .withMinimizedExclusions( - extraExclusions.fold(dep.dep.minimizedExclusions)(dep.dep.minimizedExclusions.join(_)) - ) + // add dependency management ourselves: + // - overrides meant to apply to transitive dependencies + // - fill version if it's empty + // - add extra exclusions from dependency management + .withOverrides(dep.overrides ++ depMgmt) + .withVersion(versionOverrideOpt.getOrElse(dep.version)) + .withMinimizedExclusions( + extraExclusions.fold(dep.minimizedExclusions)(dep.minimizedExclusions.join(_)) ) + } + + def allCompileIvyDeps: T[Agg[Dep]] = Task { + val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.provided)) + val depMgmt = processedDependencyManagement( + compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap + compileIvyDeps().map { dep => + dep.copy(dep = addBoms(dep.dep, bomDeps0, depMgmt, depMgmtMap)) + } + } + + def allRunIvyDeps: T[Agg[Dep]] = Task { + val bomDeps0 = + allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) ++ extraBomDeps().toSeq + val depMgmt = processedDependencyManagement( + compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap + runIvyDeps().map { dep => + dep.copy(dep = addBoms(dep.dep, bomDeps0, depMgmt, depMgmtMap)) } } @@ -200,13 +231,13 @@ trait JavaModule */ def bomDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } - def allBomDeps: Task[Agg[(coursier.core.Module, String)]] = Task.Anon { + def allBomDeps: Task[Agg[BomDependency]] = Task.Anon { val modVerOrMalformed = (Agg(parentDep().toSeq: _*) ++ bomDeps()).map(bindDependency()).map { bomDep => val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) .withConfiguration(coursier.core.Configuration.defaultCompile) if (fromModVer == bomDep.dep) - Right((bomDep.dep.module, bomDep.dep.version)) + Right(bomDep.dep.asBomDependency) else Left(bomDep) } @@ -217,7 +248,7 @@ trait JavaModule } if (malformed.isEmpty) modVerOrMalformed.collect { - case Right(modVer) => modVer + case Right(bomDep) => bomDep } else throw new Exception( @@ -227,6 +258,8 @@ trait JavaModule ) } + def extraBomDeps: Task[Agg[BomDependency]] = Task.Anon { Agg.empty[BomDependency] } + /** * Dependency management data * @@ -245,6 +278,8 @@ trait JavaModule */ def dependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + def compileDependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** * Data from dependencyManagement, converted to a type ready to be passed to coursier * for dependency resolution @@ -427,7 +462,7 @@ trait JavaModule /** The compile-only transitive ivy dependencies of this module and all it's upstream compile-only modules. */ def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! - compileIvyDeps().map(bindDependency()) ++ + allCompileIvyDeps().map(bindDependency()) ++ T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten } @@ -467,8 +502,28 @@ trait JavaModule * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. */ def transitiveIvyDeps: T[Agg[BoundDep]] = Task { - allIvyDeps().map(bindDependency()) ++ - T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten + val bomDeps0 = + allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomDeps().toSeq + val depMgmt = processedDependencyManagement( + dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap + processedIvyDeps() ++ + T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => + dep.copy(dep = addBoms(dep.dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = true)) + } + } + + def processedIvyDeps: T[Agg[BoundDep]] = Task { + val bomDeps0 = + allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomDeps().toSeq + val depMgmt = processedDependencyManagement( + dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + ) + val depMgmtMap = depMgmt.toMap + allIvyDeps().map(bindDependency()).map { dep => + dep.copy(dep = addBoms(dep.dep, bomDeps0, depMgmt, depMgmtMap)) + } } /** @@ -744,8 +799,7 @@ trait JavaModule def resolvedIvyDeps: T[Agg[PathRef]] = Task { defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), - artifactTypes = Some(artifactTypes()), - bomDeps = allBomDeps() + artifactTypes = Some(artifactTypes()) ) } @@ -760,8 +814,7 @@ trait JavaModule def resolvedRunIvyDeps: T[Agg[PathRef]] = Task { defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), - artifactTypes = Some(artifactTypes()), - bomDeps = allBomDeps() + artifactTypes = Some(artifactTypes()) ) } @@ -1028,8 +1081,7 @@ trait JavaModule Some(mapDependencies()), customizer = resolutionCustomizer(), coursierCacheCustomizer = coursierCacheCustomizer(), - resolutionParams = resolutionParams(), - bomDeps = allBomDeps() + resolutionParams = resolutionParams() ).getOrThrow val roots = whatDependsOn match { @@ -1240,15 +1292,13 @@ trait JavaModule Task.Anon { defaultResolver().resolveDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), - sources = true, - bomDeps = allBomDeps() + sources = true ) }, Task.Anon { defaultResolver().resolveDeps( transitiveRunIvyDeps() ++ transitiveIvyDeps(), - sources = true, - bomDeps = allBomDeps() + sources = true ) } ) diff --git a/scalalib/src/mill/scalalib/JsonFormatters.scala b/scalalib/src/mill/scalalib/JsonFormatters.scala index 8093d353ae8..a69cd1957b2 100644 --- a/scalalib/src/mill/scalalib/JsonFormatters.scala +++ b/scalalib/src/mill/scalalib/JsonFormatters.scala @@ -7,6 +7,7 @@ trait JsonFormatters { implicit lazy val extensionFormat: RW[coursier.core.Extension] = upickle.default.macroRW implicit lazy val modFormat: RW[coursier.Module] = upickle.default.macroRW + implicit lazy val bomDepFormat: RW[coursier.core.BomDependency] = upickle.default.macroRW implicit lazy val depFormat: RW[coursier.core.Dependency] = upickle.default.macroRW implicit lazy val minimizedExclusionsFormat: RW[coursier.core.MinimizedExclusions] = upickle.default.macroRW diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index 4b2c5e515d8..cf4fd6330d8 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -61,8 +61,7 @@ object Lib { coursierCacheCustomizer: Option[ coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] ] = None, - resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[(coursier.core.Module, String)] = Nil + resolutionParams: ResolutionParams = ResolutionParams() ): Result[Resolution] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependenciesMetadataSafe( @@ -73,8 +72,7 @@ object Lib { customizer = customizer, ctx = ctx, coursierCacheCustomizer = coursierCacheCustomizer, - resolutionParams = resolutionParams, - bomDeps = bomDeps + resolutionParams = resolutionParams ) } @@ -117,8 +115,7 @@ object Lib { coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] ] = None, artifactTypes: Option[Set[Type]] = None, - resolutionParams: ResolutionParams = ResolutionParams(), - bomDeps: IterableOnce[(coursier.core.Module, String)] = Nil + resolutionParams: ResolutionParams = ResolutionParams() ): Result[Agg[PathRef]] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependencies( @@ -131,8 +128,7 @@ object Lib { customizer = customizer, ctx = ctx, coursierCacheCustomizer = coursierCacheCustomizer, - resolutionParams = resolutionParams, - bomDeps = bomDeps + resolutionParams = resolutionParams ).map(_.map(_.withRevalidateOnce)) } @@ -182,8 +178,7 @@ object Lib { ctx, coursierCacheCustomizer, None, - ResolutionParams(), - Nil + ResolutionParams() ) def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String): Loose.Agg[Dep] = diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index fb781cfd84e..eb92bd80cec 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -75,7 +75,7 @@ trait PublishModule extends JavaModule { outer => def publishXmlDeps: Task[Agg[Dependency]] = Task.Anon { val ivyPomDeps = - allIvyDeps().map(resolvePublishDependency.apply().apply(_)) + processedIvyDeps().map(_.toDep).map(resolvePublishDependency.apply().apply(_)) val compileIvyPomDeps = compileIvyDeps() .map(resolvePublishDependency.apply().apply(_)) @@ -126,8 +126,7 @@ trait PublishModule extends JavaModule { outer => Task { val (processedDeps, depMgmt) = defaultResolver().processDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), - resolutionParams = resolutionParams(), - bomDeps = allBomDeps() + resolutionParams = resolutionParams() ) (processedDeps.map(_.moduleVersion).toMap, depMgmt) } @@ -151,9 +150,28 @@ trait PublishModule extends JavaModule { outer => else dep } + def rootDepsAdjustment = publishXmlDeps0.iterator.flatMap { dep => + val key = coursier.core.DependencyManagement.Key( + coursier.core.Organization(dep.artifact.group), + coursier.core.ModuleName(dep.artifact.id), + coursier.core.Type.jar, + coursier.core.Classifier.empty + ) + bomDepMgmt.get(key).flatMap { values => + if (values.version.nonEmpty && values.version != dep.artifact.version) + Some(key -> values.withVersion("")) + else + None + } + } + val bomDepMgmt0 = bomDepMgmt ++ rootDepsAdjustment + lazy val moduleSet = publishXmlDeps0.map(dep => (dep.artifact.group, dep.artifact.id)).toSet val overrides = dependencyManagement().toSeq.map(bindDependency()).map(_.dep) .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") + .filter { depMgmt => + !moduleSet.contains((depMgmt.module.organization.value, depMgmt.module.name.value)) + } .map { depMgmt => Ivy.Override( depMgmt.module.organization.value, @@ -161,10 +179,15 @@ trait PublishModule extends JavaModule { outer => depMgmt.version ) } ++ - bomDepMgmt.map { - case (key, values) => - Ivy.Override(key.organization.value, key.name.value, values.version) - } + bomDepMgmt0 + .filter { + case (key, _) => + !moduleSet.contains((key.organization.value, key.name.value)) + } + .map { + case (key, values) => + Ivy.Override(key.organization.value, key.name.value, values.version) + } val ivy = Ivy(artifactMetadata(), publishXmlDeps0, extraPublish(), overrides) val ivyPath = T.dest / "ivy.xml" os.write.over(ivyPath, ivy) diff --git a/scalalib/src/mill/scalalib/publish/Pom.scala b/scalalib/src/mill/scalalib/publish/Pom.scala index f3ea19304c9..b11fc48f6d9 100644 --- a/scalalib/src/mill/scalalib/publish/Pom.scala +++ b/scalalib/src/mill/scalalib/publish/Pom.scala @@ -165,7 +165,7 @@ object Pom { if (isImport) import else d.scope match { - case Scope.Compile => NodeSeq.Empty + case Scope.Compile => compile case Scope.Provided => provided case Scope.Test => test case Scope.Runtime => runtime diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index 9b3e21b309c..a1b66992cc8 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -4,7 +4,6 @@ package scalalib import mill.scalalib.publish._ import mill.testkit.{TestBaseModule, UnitTester} import utest._ -import mill.PathRef import scala.jdk.CollectionConverters._ @@ -205,6 +204,39 @@ object BomTests extends TestSuite { } } } + + object bomScope extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def compileIvyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + + object fail extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + } + } + + object bomOnModuleDependency extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:3.23.4" + ) + + object dependee extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def moduleDeps = Seq(bomOnModuleDependency) + } + } } def expectedProtobufJavaVersion = "4.28.3" @@ -296,6 +328,7 @@ object BomTests extends TestSuite { dependencyModules: Seq[PublishModule] = Nil, jarCheck: Option[String => Boolean] = None, ivy2LocalCheck: Boolean = true, + m2LocalCheck: Boolean = true, scalaSuffix: String = "" )(implicit eval: UnitTester): Unit = { compileClasspathContains(module, jarName, jarCheck) @@ -307,10 +340,12 @@ object BomTests extends TestSuite { assert(check(fileName)) } - val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) - assert(resolvedM2Cp.map(_.last).contains(jarName)) - for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) - assert(check(fileName)) + if (m2LocalCheck) { + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedM2Cp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) + assert(check(fileName)) + } } def tests = Tests { @@ -446,7 +481,8 @@ object BomTests extends TestSuite { Seq(modules.depMgmt.extraExclude), jarCheck = Some { jarName => !jarName.startsWith("slf4j-api-") - } + }, + ivy2LocalCheck = false // we could make that work ) } @@ -519,5 +555,48 @@ object BomTests extends TestSuite { ) } } + + test("bomScope") { + test("provided") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomScope, + "protobuf-java-3.23.4.jar", + ivy2LocalCheck = false, + m2LocalCheck = false + ) + } + test("providedFromBomRuntimeScope") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomScope, + "scala-parallel-collections_2.13-1.0.4.jar", + ivy2LocalCheck = false, + m2LocalCheck = false + ) + } + test("ignoreProvidedForCompile") - UnitTester(modules, null).scoped { implicit eval => + val res = eval(modules.bomScope.fail.resolvedIvyDeps) + assert( + res.left.exists(_.toString.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util/_/protobuf-java-util-_.pom" + )) + ) + } + } + + test("bomOnModuleDependency") { + test("check") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomOnModuleDependency, + "protobuf-java-3.23.4.jar" + ) + } + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomOnModuleDependency.dependee, + expectedProtobufJarName, + Seq(modules.bomOnModuleDependency) + ) + } + } } } From 8aa498acb41ee978f1a0884965dc5925edfe8de8 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 19:50:17 +0100 Subject: [PATCH 22/27] more --- docs/modules/ROOT/nav.adoc | 1 + .../ROOT/pages/javalib/dependencies.adoc | 10 +- .../pages/javalib/dependency-management.adoc | 19 ++ .../bar/src/bar/Bar.java | 32 ++++ .../3-bom-dependency-management/build.mill | 59 +++++++ .../foo/src/foo/Foo.java | 7 + .../build.mill | 0 .../lib/nanojson-1.8.jar | Bin .../src/foo/Foo.java | 0 .../build.mill | 0 .../src/foo/Foo.java | 0 .../textfile.txt | 0 .../build.mill | 0 .../foo/src/foo/Foo.java | 0 .../1-managing-versions/build.mill | 13 ++ .../2-adding-exclusions/build.mill | 13 ++ .../3-precedence/build.mill | 13 ++ example/package.mill | 1 + .../src/DocAnnotationsTests.scala | 2 - main/util/src/mill/util/CoursierSupport.scala | 2 +- .../src/mill/scalalib/CoursierModule.scala | 21 +-- scalalib/src/mill/scalalib/JavaModule.scala | 162 ++++++++---------- .../src/mill/scalalib/PublishModule.scala | 51 +++--- scalalib/src/mill/scalalib/publish/Pom.scala | 23 ++- .../test/src/mill/scalalib/BomTests.scala | 58 +------ 25 files changed, 292 insertions(+), 195 deletions(-) create mode 100644 docs/modules/ROOT/pages/javalib/dependency-management.adoc create mode 100644 example/javalib/dependencies/3-bom-dependency-management/bar/src/bar/Bar.java create mode 100644 example/javalib/dependencies/3-bom-dependency-management/build.mill create mode 100644 example/javalib/dependencies/3-bom-dependency-management/foo/src/foo/Foo.java rename example/javalib/dependencies/{3-unmanaged-jars => 4-unmanaged-jars}/build.mill (100%) rename example/javalib/dependencies/{3-unmanaged-jars => 4-unmanaged-jars}/lib/nanojson-1.8.jar (100%) rename example/javalib/dependencies/{3-unmanaged-jars => 4-unmanaged-jars}/src/foo/Foo.java (100%) rename example/javalib/dependencies/{4-downloading-unmanaged-jars => 5-downloading-unmanaged-jars}/build.mill (100%) rename example/javalib/dependencies/{4-downloading-unmanaged-jars => 5-downloading-unmanaged-jars}/src/foo/Foo.java (100%) rename example/javalib/dependencies/{4-downloading-unmanaged-jars => 5-downloading-unmanaged-jars}/textfile.txt (100%) rename example/javalib/dependencies/{5-repository-config => 6-repository-config}/build.mill (100%) rename example/javalib/dependencies/{5-repository-config => 6-repository-config}/foo/src/foo/Foo.java (100%) create mode 100644 example/javalib/dependency-management/1-managing-versions/build.mill create mode 100644 example/javalib/dependency-management/2-adding-exclusions/build.mill create mode 100644 example/javalib/dependency-management/3-precedence/build.mill diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index e903134c7f4..9daec49d171 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -9,6 +9,7 @@ ** xref:javalib/testing.adoc[] ** xref:javalib/linting.adoc[] ** xref:javalib/publishing.adoc[] +** xref:javalib/dependency-management.adoc[] ** xref:javalib/build-examples.adoc[] ** xref:javalib/web-examples.adoc[] * xref:scalalib/intro.adoc[] diff --git a/docs/modules/ROOT/pages/javalib/dependencies.adoc b/docs/modules/ROOT/pages/javalib/dependencies.adoc index 76ebe8d4449..7a8368df429 100644 --- a/docs/modules/ROOT/pages/javalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/javalib/dependencies.adoc @@ -13,16 +13,20 @@ include::partial$example/javalib/dependencies/1-ivy-deps.adoc[] include::partial$example/javalib/dependencies/2-run-compile-deps.adoc[] +== Bill of Material and Dependency Management + +include::partial$example/javalib/dependencies/3-bom-dependency-management.adoc[] + == Unmanaged Jars -include::partial$example/javalib/dependencies/3-unmanaged-jars.adoc[] +include::partial$example/javalib/dependencies/4-unmanaged-jars.adoc[] == Downloading Unmanaged Jars -include::partial$example/javalib/dependencies/4-downloading-unmanaged-jars.adoc[] +include::partial$example/javalib/dependencies/5-downloading-unmanaged-jars.adoc[] == Repository Config -include::partial$example/javalib/dependencies/5-repository-config.adoc[] +include::partial$example/javalib/dependencies/6-repository-config.adoc[] diff --git a/docs/modules/ROOT/pages/javalib/dependency-management.adoc b/docs/modules/ROOT/pages/javalib/dependency-management.adoc new file mode 100644 index 00000000000..757cc4506e3 --- /dev/null +++ b/docs/modules/ROOT/pages/javalib/dependency-management.adoc @@ -0,0 +1,19 @@ += Java Dependency Management +:page-aliases: Java_Dependency_Management.adoc + +include::partial$gtag-config.adoc[] + +This section is about Java so-called "dependency management" or Bill of Materials. To add +simple dependencies, see xref:./dependencies.adoc[]. + +== Managing versions + +include::partial$example/javalib/dependency-management/1-managing-versions.adoc[] + +== Adding exclusions + +include::partial$example/javalib/dependency-management/2-adding-exclusions.adoc[] + +== Precedence + +include::partial$example/javalib/dependency-management/3-precedence.adoc[] diff --git a/example/javalib/dependencies/3-bom-dependency-management/bar/src/bar/Bar.java b/example/javalib/dependencies/3-bom-dependency-management/bar/src/bar/Bar.java new file mode 100644 index 00000000000..38751f60787 --- /dev/null +++ b/example/javalib/dependencies/3-bom-dependency-management/bar/src/bar/Bar.java @@ -0,0 +1,32 @@ +package bar; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +class BarServlet extends HttpServlet { + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + response.setContentType("text/html"); + response.setStatus(HttpServletResponse.SC_OK); + response.getWriter().println("Hello World!"); + } +} + +public class Bar { + public static void main(String[] args) throws Exception { + Server server = new Server(8079); + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/"); + server.setHandler(context); + context.addServlet(new ServletHolder(new BarServlet()), "/*"); + server.start(); + server.join(); + } +} diff --git a/example/javalib/dependencies/3-bom-dependency-management/build.mill b/example/javalib/dependencies/3-bom-dependency-management/build.mill new file mode 100644 index 00000000000..bc7f94dc984 --- /dev/null +++ b/example/javalib/dependencies/3-bom-dependency-management/build.mill @@ -0,0 +1,59 @@ +// Mill supports for Bill of Materials ("BOM") or dependency management. +// These allow to centralize versions and exclusions for your dependencies, +// either in you Mill project, via dependency management, or externally, +// via a BOM. + +// To add a BOM to a Java module, use `bomDeps`: + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object bom extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// Versions found in a BOM are used in two ways: + +// * to fill missing versions in the `ivyDeps` section + +// * to override versions in transitive dependencies + +// The version of grpc-protobuf isn't written down here, so the version +// from the `com.google.cloud:libraries-bom:26.50.0` BOM, `1.67.1` is used. +// +// Also, by default, `io.grpc:grpc-protobuf:1.67.1` pulls `com.google.protobuf:protobuf-java:3.25.3`. +// But the BOM specifies another version for that dependency, `4.28.3`, so +// `com.google.protobuf:protobuf-java:4.28.3` ends up being pulled here. + +// Instead of pulling an external BOM, one can use `dependencyManagement` to specify dependency versions. +// Like with a BOM, versions in `dependencyManagement` are used to: + +// * fill missing versions in the `ivyDeps` section + +// * override versions in transitive dependencies + +//// SNIPPET:BUILD2 + +object dependencyManagement extends JavaModule { + def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ivy"io.grpc:grpc-protobuf:1.67.1" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +//// SNIPPET:SCALASTEWARD + +// Here, `io.grpc:grpc-protobuf` has no version in `ivyDeps`, so the version in +// `dependencyManagement`, `1.67.1` is used. Also, `io.grpc:grpc-protobuf:1.67.1` +// pulls `com.google.protobuf:protobuf-java:3.25.3` by default, but version +// `4.28.3` will be pulled, because of `com.google.protobuf:protobuf-java:4.28.3` +// in `dependencyManagement`. diff --git a/example/javalib/dependencies/3-bom-dependency-management/foo/src/foo/Foo.java b/example/javalib/dependencies/3-bom-dependency-management/foo/src/foo/Foo.java new file mode 100644 index 00000000000..6bfe37e30c1 --- /dev/null +++ b/example/javalib/dependencies/3-bom-dependency-management/foo/src/foo/Foo.java @@ -0,0 +1,7 @@ +package foo; + +public class Foo { + public static void main(String[] args) { + System.out.println("Hello World"); + } +} diff --git a/example/javalib/dependencies/3-unmanaged-jars/build.mill b/example/javalib/dependencies/4-unmanaged-jars/build.mill similarity index 100% rename from example/javalib/dependencies/3-unmanaged-jars/build.mill rename to example/javalib/dependencies/4-unmanaged-jars/build.mill diff --git a/example/javalib/dependencies/3-unmanaged-jars/lib/nanojson-1.8.jar b/example/javalib/dependencies/4-unmanaged-jars/lib/nanojson-1.8.jar similarity index 100% rename from example/javalib/dependencies/3-unmanaged-jars/lib/nanojson-1.8.jar rename to example/javalib/dependencies/4-unmanaged-jars/lib/nanojson-1.8.jar diff --git a/example/javalib/dependencies/3-unmanaged-jars/src/foo/Foo.java b/example/javalib/dependencies/4-unmanaged-jars/src/foo/Foo.java similarity index 100% rename from example/javalib/dependencies/3-unmanaged-jars/src/foo/Foo.java rename to example/javalib/dependencies/4-unmanaged-jars/src/foo/Foo.java diff --git a/example/javalib/dependencies/4-downloading-unmanaged-jars/build.mill b/example/javalib/dependencies/5-downloading-unmanaged-jars/build.mill similarity index 100% rename from example/javalib/dependencies/4-downloading-unmanaged-jars/build.mill rename to example/javalib/dependencies/5-downloading-unmanaged-jars/build.mill diff --git a/example/javalib/dependencies/4-downloading-unmanaged-jars/src/foo/Foo.java b/example/javalib/dependencies/5-downloading-unmanaged-jars/src/foo/Foo.java similarity index 100% rename from example/javalib/dependencies/4-downloading-unmanaged-jars/src/foo/Foo.java rename to example/javalib/dependencies/5-downloading-unmanaged-jars/src/foo/Foo.java diff --git a/example/javalib/dependencies/4-downloading-unmanaged-jars/textfile.txt b/example/javalib/dependencies/5-downloading-unmanaged-jars/textfile.txt similarity index 100% rename from example/javalib/dependencies/4-downloading-unmanaged-jars/textfile.txt rename to example/javalib/dependencies/5-downloading-unmanaged-jars/textfile.txt diff --git a/example/javalib/dependencies/5-repository-config/build.mill b/example/javalib/dependencies/6-repository-config/build.mill similarity index 100% rename from example/javalib/dependencies/5-repository-config/build.mill rename to example/javalib/dependencies/6-repository-config/build.mill diff --git a/example/javalib/dependencies/5-repository-config/foo/src/foo/Foo.java b/example/javalib/dependencies/6-repository-config/foo/src/foo/Foo.java similarity index 100% rename from example/javalib/dependencies/5-repository-config/foo/src/foo/Foo.java rename to example/javalib/dependencies/6-repository-config/foo/src/foo/Foo.java diff --git a/example/javalib/dependency-management/1-managing-versions/build.mill b/example/javalib/dependency-management/1-managing-versions/build.mill new file mode 100644 index 00000000000..1ae80c736f5 --- /dev/null +++ b/example/javalib/dependency-management/1-managing-versions/build.mill @@ -0,0 +1,13 @@ + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object bom extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} diff --git a/example/javalib/dependency-management/2-adding-exclusions/build.mill b/example/javalib/dependency-management/2-adding-exclusions/build.mill new file mode 100644 index 00000000000..1ae80c736f5 --- /dev/null +++ b/example/javalib/dependency-management/2-adding-exclusions/build.mill @@ -0,0 +1,13 @@ + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object bom extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} diff --git a/example/javalib/dependency-management/3-precedence/build.mill b/example/javalib/dependency-management/3-precedence/build.mill new file mode 100644 index 00000000000..1ae80c736f5 --- /dev/null +++ b/example/javalib/dependency-management/3-precedence/build.mill @@ -0,0 +1,13 @@ + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object bom extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} diff --git a/example/package.mill b/example/package.mill index b7f53d51527..2b3814be5e8 100644 --- a/example/package.mill +++ b/example/package.mill @@ -36,6 +36,7 @@ object `package` extends RootModule with Module { object basic extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "basic")) object module extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "module")) object dependencies extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "dependencies")) + object `dependency-management` extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "dependency-management")) object testing extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "testing")) object linting extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "linting")) object migrating extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "migrating")) diff --git a/integration/feature/docannotations/src/DocAnnotationsTests.scala b/integration/feature/docannotations/src/DocAnnotationsTests.scala index cdf38b9dff3..49524852d22 100644 --- a/integration/feature/docannotations/src/DocAnnotationsTests.scala +++ b/integration/feature/docannotations/src/DocAnnotationsTests.scala @@ -103,8 +103,6 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite { | |Inputs: | core.transitiveIvyDeps - | core.parentDep - | core.bomDeps |""".stripMargin, ivyDepsTree ) diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 6affefbd025..6f6ccff62b2 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -7,7 +7,7 @@ import coursier.params.ResolutionParams import coursier.parse.RepositoryParser import coursier.jvm.{JvmCache, JvmChannel, JvmIndex, JavaHome} import coursier.util.Task -import coursier.{Artifacts, Classifier, Dependency, Module, Repository, Resolution, Resolve, Type} +import coursier.{Artifacts, Classifier, Dependency, Repository, Resolution, Resolve, Type} import mill.api.Loose.Agg import mill.api.{Ctx, PathRef, Result} diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index c49f472f743..c7c483cceb1 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -2,8 +2,8 @@ package mill.scalalib import coursier.cache.FileCache import coursier.params.ResolutionParams -import coursier.{Dependency, Module, Repository, Resolve, Type} -import coursier.core.{DependencyManagement, Resolution} +import coursier.{Dependency, Repository, Resolve, Type} +import coursier.core.Resolution import mill.define.Task import mill.api.PathRef @@ -243,12 +243,12 @@ object CoursierModule { * * @param deps dependencies that might have placeholder versions ("_" as version) * @param resolutionParams coursier resolution parameters - * @return dependencies with version placeholder filled and data read from the BOM dependencies + * @return dependencies with version placeholder filled */ def processDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], resolutionParams: ResolutionParams = ResolutionParams() - ): (Seq[Dependency], DependencyManagement.Map) = { + ): Seq[Dependency] = { val deps0 = deps .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) val res = Lib.resolveDependenciesMetadataSafe( @@ -260,18 +260,7 @@ object CoursierModule { ctx = ctx, resolutionParams = resolutionParams ).getOrThrow - val depMgmt: DependencyManagement.Map = - if (res.processedRootDependencies.isEmpty) Map.empty - else { - val overrides = res.processedRootDependencies.map(_.overrides) - overrides.tail.foldLeft(overrides.head) { (acc, map) => - acc.filter { - case (key, values) => - map.get(key).contains(values) - } - } - } - (res.processedRootDependencies, depMgmt) + res.processedRootDependencies } } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index dffb54f7476..662f1e508a9 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -57,10 +57,6 @@ trait JavaModule } } - override def extraBomDeps = Task.Anon { - outer.allBomDeps().map(_.withConfig(Configuration.test)): Agg[BomDependency] - } - /** * JavaModule and its derivatives define inner test modules. * To avoid unexpected misbehavior due to the use of the wrong inner test trait @@ -144,70 +140,10 @@ trait JavaModule def ivyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } /** - * Aggregation of mandatoryIvyDeps and ivyDeps, with BOMs and dependency management data - * added to each of them. + * Aggregation of mandatoryIvyDeps and ivyDeps. * In most cases, instead of overriding this Target you want to override `ivyDeps` instead. */ - def allIvyDeps: T[Agg[Dep]] = Task { - ivyDeps() ++ mandatoryIvyDeps() - } - - private def addBoms( - dep: coursier.core.Dependency, - bomDeps: Seq[coursier.core.BomDependency], - depMgmt: Seq[(DependencyManagement.Key, DependencyManagement.Values)], - depMgmtMap: DependencyManagement.Map, - overrideVersions: Boolean = false - ): coursier.core.Dependency = { - val depMgmtKey = DependencyManagement.Key( - dep.module.organization, - dep.module.name, - coursier.core.Type.jar, - dep.publication.classifier - ) - val versionOverrideOpt = - if (dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) - else None - val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) - dep - // add BOM coordinates - coursier will handle the rest - .addBomDependencies( - if (overrideVersions) bomDeps.map(_.withForceOverrideVersions(overrideVersions)) - else bomDeps - ) - // add dependency management ourselves: - // - overrides meant to apply to transitive dependencies - // - fill version if it's empty - // - add extra exclusions from dependency management - .withOverrides(dep.overrides ++ depMgmt) - .withVersion(versionOverrideOpt.getOrElse(dep.version)) - .withMinimizedExclusions( - extraExclusions.fold(dep.minimizedExclusions)(dep.minimizedExclusions.join(_)) - ) - } - - def allCompileIvyDeps: T[Agg[Dep]] = Task { - val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.provided)) - val depMgmt = processedDependencyManagement( - compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) - ) - val depMgmtMap = depMgmt.toMap - compileIvyDeps().map { dep => - dep.copy(dep = addBoms(dep.dep, bomDeps0, depMgmt, depMgmtMap)) - } - } - - def allRunIvyDeps: T[Agg[Dep]] = Task { - val bomDeps0 = - allBomDeps().toSeq.map(_.withConfig(Configuration.defaultCompile)) ++ extraBomDeps().toSeq - val depMgmt = processedDependencyManagement( - compileDependencyManagement().toSeq.map(bindDependency()).map(_.dep) - ) - val depMgmtMap = depMgmt.toMap - runIvyDeps().map { dep => - dep.copy(dep = addBoms(dep.dep, bomDeps0, depMgmt, depMgmtMap)) - } - } + def allIvyDeps: T[Agg[Dep]] = Task { ivyDeps() ++ mandatoryIvyDeps() } /** * Same as `ivyDeps`, but only present at compile time. Useful for e.g. @@ -223,6 +159,14 @@ trait JavaModule */ def runIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** + * Dependency to use as a "parent". + * + * This dependency is advertized as "parent dependency" in POM files, + * when publishing this module. + * + * In practice, from Mill, this is equivalent to a BOM dependency. + */ def parentDep: T[Option[Dep]] = Task { None } /** @@ -235,7 +179,6 @@ trait JavaModule val modVerOrMalformed = (Agg(parentDep().toSeq: _*) ++ bomDeps()).map(bindDependency()).map { bomDep => val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) - .withConfiguration(coursier.core.Configuration.defaultCompile) if (fromModVer == bomDep.dep) Right(bomDep.dep.asBomDependency) else @@ -258,8 +201,6 @@ trait JavaModule ) } - def extraBomDeps: Task[Agg[BomDependency]] = Task.Anon { Agg.empty[BomDependency] } - /** * Dependency management data * @@ -278,7 +219,39 @@ trait JavaModule */ def dependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } - def compileDependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + private def addBoms( + dep: coursier.core.Dependency, + bomDeps: Seq[coursier.core.BomDependency], + depMgmt: Seq[(DependencyManagement.Key, DependencyManagement.Values)], + depMgmtMap: DependencyManagement.Map, + overrideVersions: Boolean = false + ): coursier.core.Dependency = { + val depMgmtKey = DependencyManagement.Key( + dep.module.organization, + dep.module.name, + coursier.core.Type.jar, + dep.publication.classifier + ) + val versionOverrideOpt = + if (dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) + else None + val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) + dep + // add BOM coordinates - coursier will handle the rest + .addBomDependencies( + if (overrideVersions) bomDeps.map(_.withForceOverrideVersions(overrideVersions)) + else bomDeps + ) + // add dependency management ourselves: + // - overrides meant to apply to transitive dependencies + // - fill version if it's empty + // - add extra exclusions from dependency management + .withOverrides(dep.overrides ++ depMgmt) + .withVersion(versionOverrideOpt.getOrElse(dep.version)) + .withMinimizedExclusions( + extraExclusions.fold(dep.minimizedExclusions)(dep.minimizedExclusions.join(_)) + ) + } /** * Data from dependencyManagement, converted to a type ready to be passed to coursier @@ -297,7 +270,6 @@ trait JavaModule )) .withMinimizedExclusions(depMgmt.minimizedExclusions) .withOptional(depMgmt.optional) - .withConfiguration(Configuration.defaultCompile) if (fromUsedValues == depMgmt) { val key = DependencyManagement.Key( depMgmt.module.organization, @@ -462,7 +434,7 @@ trait JavaModule /** The compile-only transitive ivy dependencies of this module and all it's upstream compile-only modules. */ def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! - allCompileIvyDeps().map(bindDependency()) ++ + compileIvyDeps().map(bindDependency()) ++ T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten } @@ -498,34 +470,46 @@ trait JavaModule def unmanagedClasspath: T[Agg[PathRef]] = Task { Agg.empty[PathRef] } /** - * The transitive ivy dependencies of this module and all it's upstream modules. - * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. + * Returns a function adding BOM and dependency management details of + * this module to a `coursier.core.Dependency` */ - def transitiveIvyDeps: T[Agg[BoundDep]] = Task { - val bomDeps0 = - allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomDeps().toSeq + def processDependency( + overrideVersions: Boolean = false + ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { + val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) val depMgmt = processedDependencyManagement( dependencyManagement().toSeq.map(bindDependency()).map(_.dep) ) val depMgmtMap = depMgmt.toMap - processedIvyDeps() ++ - T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => - dep.copy(dep = addBoms(dep.dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = true)) - } + + dep => + addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) } - def processedIvyDeps: T[Agg[BoundDep]] = Task { - val bomDeps0 = - allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) ++ extraBomDeps().toSeq - val depMgmt = processedDependencyManagement( - dependencyManagement().toSeq.map(bindDependency()).map(_.dep) - ) - val depMgmtMap = depMgmt.toMap + /** + * The Ivy dependencies of this module, with BOM and dependency management details + * added to them. This should be used when propagating the dependencies transitively + * to other modules. + */ + def processedIvyDeps: Task[Agg[BoundDep]] = Task.Anon { + val processDependency0 = processDependency()() allIvyDeps().map(bindDependency()).map { dep => - dep.copy(dep = addBoms(dep.dep, bomDeps0, depMgmt, depMgmtMap)) + dep.copy(dep = processDependency0(dep.dep)) } } + /** + * The transitive ivy dependencies of this module and all it's upstream modules. + * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. + */ + def transitiveIvyDeps: T[Agg[BoundDep]] = Task { + val processDependency0 = processDependency(overrideVersions = true)() + processedIvyDeps() ++ + T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } + } + /** * The transitive run ivy dependencies of this module and all it's upstream modules. * This is calculated from [[runIvyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index eb92bd80cec..b2ad3b58d39 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -94,10 +94,16 @@ trait PublishModule extends JavaModule { outer => compileModulePomDeps.map(Dependency(_, Scope.Provided)) } + /** + * BOM dependency to specify in the POM + */ def publishXmlBomDeps: Task[Agg[Dependency]] = Task.Anon { bomDeps().map(resolvePublishDependency.apply().apply(_)) } + /** + * Dependency management to specify in the POM + */ def publishXmlDepMgmt: Task[Agg[Dependency]] = Task.Anon { dependencyManagement().map(resolvePublishDependency.apply().apply(_)) } @@ -124,10 +130,21 @@ trait PublishModule extends JavaModule { outer => */ def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = Task { - val (processedDeps, depMgmt) = defaultResolver().processDeps( + val processedDeps = defaultResolver().processDeps( transitiveCompileIvyDeps() ++ transitiveIvyDeps(), resolutionParams = resolutionParams() ) + val depMgmt: coursier.core.DependencyManagement.Map = + if (processedDeps.isEmpty) Map.empty + else { + val overrides = processedDeps.map(_.overrides) + overrides.tail.foldLeft(overrides.head) { (acc, map) => + acc.filter { + case (key, values) => + map.get(key).contains(values) + } + } + } (processedDeps.map(_.moduleVersion).toMap, depMgmt) } @@ -150,28 +167,9 @@ trait PublishModule extends JavaModule { outer => else dep } - def rootDepsAdjustment = publishXmlDeps0.iterator.flatMap { dep => - val key = coursier.core.DependencyManagement.Key( - coursier.core.Organization(dep.artifact.group), - coursier.core.ModuleName(dep.artifact.id), - coursier.core.Type.jar, - coursier.core.Classifier.empty - ) - bomDepMgmt.get(key).flatMap { values => - if (values.version.nonEmpty && values.version != dep.artifact.version) - Some(key -> values.withVersion("")) - else - None - } - } - val bomDepMgmt0 = bomDepMgmt ++ rootDepsAdjustment - lazy val moduleSet = publishXmlDeps0.map(dep => (dep.artifact.group, dep.artifact.id)).toSet val overrides = dependencyManagement().toSeq.map(bindDependency()).map(_.dep) .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") - .filter { depMgmt => - !moduleSet.contains((depMgmt.module.organization.value, depMgmt.module.name.value)) - } .map { depMgmt => Ivy.Override( depMgmt.module.organization.value, @@ -179,15 +177,10 @@ trait PublishModule extends JavaModule { outer => depMgmt.version ) } ++ - bomDepMgmt0 - .filter { - case (key, _) => - !moduleSet.contains((key.organization.value, key.name.value)) - } - .map { - case (key, values) => - Ivy.Override(key.organization.value, key.name.value, values.version) - } + bomDepMgmt.map { + case (key, values) => + Ivy.Override(key.organization.value, key.name.value, values.version) + } val ivy = Ivy(artifactMetadata(), publishXmlDeps0, extraPublish(), overrides) val ivyPath = T.dest / "ivy.xml" os.write.over(ivyPath, ivy) diff --git a/scalalib/src/mill/scalalib/publish/Pom.scala b/scalalib/src/mill/scalalib/publish/Pom.scala index b11fc48f6d9..6940f25c47f 100644 --- a/scalalib/src/mill/scalalib/publish/Pom.scala +++ b/scalalib/src/mill/scalalib/publish/Pom.scala @@ -67,6 +67,27 @@ object Pom { dependencyManagement = Agg.empty[Dependency] ) + def apply( + artifact: Artifact, + dependencies: Agg[Dependency], + name: String, + pomSettings: PomSettings, + properties: Map[String, String], + packagingType: String, + parentProject: Option[Artifact] + ): String = + apply( + artifact, + dependencies, + name, + pomSettings, + properties, + packagingType, + parentProject, + Agg.empty[Dependency], + Agg.empty[Dependency] + ) + def apply( artifact: Artifact, dependencies: Agg[Dependency], @@ -165,7 +186,7 @@ object Pom { if (isImport) import else d.scope match { - case Scope.Compile => compile + case Scope.Compile => NodeSeq.Empty case Scope.Provided => provided case Scope.Test => test case Scope.Runtime => runtime diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index a1b66992cc8..2e2e454b2a2 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -205,26 +205,6 @@ object BomTests extends TestSuite { } } - object bomScope extends JavaModule with TestPublishModule { - def bomDeps = Agg( - ivy"org.apache.spark:spark-parent_2.13:3.5.3" - ) - def compileIvyDeps = Agg( - ivy"com.google.protobuf:protobuf-java-util", - ivy"org.scala-lang.modules:scala-parallel-collections_2.13" - ) - - object fail extends JavaModule with TestPublishModule { - def bomDeps = Agg( - ivy"org.apache.spark:spark-parent_2.13:3.5.3" - ) - def ivyDeps = Agg( - ivy"com.google.protobuf:protobuf-java-util", - ivy"org.scala-lang.modules:scala-parallel-collections_2.13" - ) - } - } - object bomOnModuleDependency extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java:3.23.4" @@ -328,7 +308,6 @@ object BomTests extends TestSuite { dependencyModules: Seq[PublishModule] = Nil, jarCheck: Option[String => Boolean] = None, ivy2LocalCheck: Boolean = true, - m2LocalCheck: Boolean = true, scalaSuffix: String = "" )(implicit eval: UnitTester): Unit = { compileClasspathContains(module, jarName, jarCheck) @@ -340,12 +319,10 @@ object BomTests extends TestSuite { assert(check(fileName)) } - if (m2LocalCheck) { - val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) - assert(resolvedM2Cp.map(_.last).contains(jarName)) - for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) - assert(check(fileName)) - } + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedM2Cp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) + assert(check(fileName)) } def tests = Tests { @@ -556,33 +533,6 @@ object BomTests extends TestSuite { } } - test("bomScope") { - test("provided") - UnitTester(modules, null).scoped { implicit eval => - isInClassPath( - modules.bomScope, - "protobuf-java-3.23.4.jar", - ivy2LocalCheck = false, - m2LocalCheck = false - ) - } - test("providedFromBomRuntimeScope") - UnitTester(modules, null).scoped { implicit eval => - isInClassPath( - modules.bomScope, - "scala-parallel-collections_2.13-1.0.4.jar", - ivy2LocalCheck = false, - m2LocalCheck = false - ) - } - test("ignoreProvidedForCompile") - UnitTester(modules, null).scoped { implicit eval => - val res = eval(modules.bomScope.fail.resolvedIvyDeps) - assert( - res.left.exists(_.toString.contains( - "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util/_/protobuf-java-util-_.pom" - )) - ) - } - } - test("bomOnModuleDependency") { test("check") - UnitTester(modules, null).scoped { implicit eval => isInClassPath( From 61b60610e02c3cdb900257220fa4ebd12b464b22 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 2 Dec 2024 20:17:32 +0100 Subject: [PATCH 23/27] fmt --- .../3-bom-dependency-management/build.mill | 2 +- .../1-managing-versions/build.mill | 1 - .../2-adding-exclusions/build.mill | 1 - .../3-precedence/build.mill | 1 - example/package.mill | 4 +++- scalalib/src/mill/scalalib/JavaModule.scala | 22 +++++++++---------- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/example/javalib/dependencies/3-bom-dependency-management/build.mill b/example/javalib/dependencies/3-bom-dependency-management/build.mill index bc7f94dc984..51197f5e9e3 100644 --- a/example/javalib/dependencies/3-bom-dependency-management/build.mill +++ b/example/javalib/dependencies/3-bom-dependency-management/build.mill @@ -42,7 +42,7 @@ object bom extends JavaModule { object dependencyManagement extends JavaModule { def dependencyManagement = Agg( - ivy"com.google.protobuf:protobuf-java:4.28.3" + ivy"com.google.protobuf:protobuf-java:4.28.3", ivy"io.grpc:grpc-protobuf:1.67.1" ) def ivyDeps = Agg( diff --git a/example/javalib/dependency-management/1-managing-versions/build.mill b/example/javalib/dependency-management/1-managing-versions/build.mill index 1ae80c736f5..a68773f9979 100644 --- a/example/javalib/dependency-management/1-managing-versions/build.mill +++ b/example/javalib/dependency-management/1-managing-versions/build.mill @@ -1,4 +1,3 @@ - //// SNIPPET:BUILD1 package build import mill._, javalib._ diff --git a/example/javalib/dependency-management/2-adding-exclusions/build.mill b/example/javalib/dependency-management/2-adding-exclusions/build.mill index 1ae80c736f5..a68773f9979 100644 --- a/example/javalib/dependency-management/2-adding-exclusions/build.mill +++ b/example/javalib/dependency-management/2-adding-exclusions/build.mill @@ -1,4 +1,3 @@ - //// SNIPPET:BUILD1 package build import mill._, javalib._ diff --git a/example/javalib/dependency-management/3-precedence/build.mill b/example/javalib/dependency-management/3-precedence/build.mill index 1ae80c736f5..a68773f9979 100644 --- a/example/javalib/dependency-management/3-precedence/build.mill +++ b/example/javalib/dependency-management/3-precedence/build.mill @@ -1,4 +1,3 @@ - //// SNIPPET:BUILD1 package build import mill._, javalib._ diff --git a/example/package.mill b/example/package.mill index 2b3814be5e8..79de9a41ec1 100644 --- a/example/package.mill +++ b/example/package.mill @@ -36,7 +36,9 @@ object `package` extends RootModule with Module { object basic extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "basic")) object module extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "module")) object dependencies extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "dependencies")) - object `dependency-management` extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "dependency-management")) + object `dependency-management` extends Cross[ExampleCrossModuleJava]( + build.listIn(millSourcePath / "dependency-management") + ) object testing extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "testing")) object linting extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "linting")) object migrating extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "migrating")) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 662f1e508a9..0d9cb7426db 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -160,13 +160,13 @@ trait JavaModule def runIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } /** - * Dependency to use as a "parent". - * - * This dependency is advertized as "parent dependency" in POM files, - * when publishing this module. - * - * In practice, from Mill, this is equivalent to a BOM dependency. - */ + * Dependency to use as a "parent". + * + * This dependency is advertized as "parent dependency" in POM files, + * when publishing this module. + * + * In practice, from Mill, this is equivalent to a BOM dependency. + */ def parentDep: T[Option[Dep]] = Task { None } /** @@ -487,10 +487,10 @@ trait JavaModule } /** - * The Ivy dependencies of this module, with BOM and dependency management details - * added to them. This should be used when propagating the dependencies transitively - * to other modules. - */ + * The Ivy dependencies of this module, with BOM and dependency management details + * added to them. This should be used when propagating the dependencies transitively + * to other modules. + */ def processedIvyDeps: Task[Agg[BoundDep]] = Task.Anon { val processDependency0 = processDependency()() allIvyDeps().map(bindDependency()).map { dep => From 73f5c10b1894810b920f8ffc274258d2a6e324d4 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 3 Dec 2024 14:36:02 +0100 Subject: [PATCH 24/27] rework / add doc, remove parentDep stuff --- docs/modules/ROOT/nav.adoc | 1 - .../ROOT/pages/fundamentals/library-deps.adoc | 28 +++++++ .../ROOT/pages/javalib/dependencies.adoc | 13 ++-- .../pages/javalib/dependency-management.adoc | 19 ----- .../ROOT/pages/kotlinlib/dependencies.adoc | 6 ++ .../ROOT/pages/scalalib/dependencies.adoc | 5 ++ .../bom-1-external-bom/build.mill | 21 ++++++ .../bom-2-dependency-management/build.mill | 54 ++++++++++++++ .../bar/src/bar/Bar.java | 32 -------- .../3-bom-dependency-management/build.mill | 59 --------------- .../foo/src/foo/Foo.java | 7 -- .../build.mill | 0 .../lib/nanojson-1.8.jar | Bin .../src/foo/Foo.java | 0 .../build.mill | 0 .../src/foo/Foo.java | 0 .../textfile.txt | 0 .../build.mill | 0 .../foo/src/foo/Foo.java | 0 .../1-managing-versions/build.mill | 12 --- .../2-adding-exclusions/build.mill | 12 --- .../3-precedence/build.mill | 12 --- example/package.mill | 4 +- scalalib/src/mill/scalalib/JavaModule.scala | 14 +--- .../src/mill/scalalib/PublishModule.scala | 7 +- .../test/src/mill/scalalib/BomTests.scala | 70 +----------------- 26 files changed, 126 insertions(+), 250 deletions(-) delete mode 100644 docs/modules/ROOT/pages/javalib/dependency-management.adoc create mode 100644 example/fundamentals/library-deps/bom-1-external-bom/build.mill create mode 100644 example/fundamentals/library-deps/bom-2-dependency-management/build.mill delete mode 100644 example/javalib/dependencies/3-bom-dependency-management/bar/src/bar/Bar.java delete mode 100644 example/javalib/dependencies/3-bom-dependency-management/build.mill delete mode 100644 example/javalib/dependencies/3-bom-dependency-management/foo/src/foo/Foo.java rename example/javalib/dependencies/{4-unmanaged-jars => 3-unmanaged-jars}/build.mill (100%) rename example/javalib/dependencies/{4-unmanaged-jars => 3-unmanaged-jars}/lib/nanojson-1.8.jar (100%) rename example/javalib/dependencies/{4-unmanaged-jars => 3-unmanaged-jars}/src/foo/Foo.java (100%) rename example/javalib/dependencies/{5-downloading-unmanaged-jars => 4-downloading-unmanaged-jars}/build.mill (100%) rename example/javalib/dependencies/{5-downloading-unmanaged-jars => 4-downloading-unmanaged-jars}/src/foo/Foo.java (100%) rename example/javalib/dependencies/{5-downloading-unmanaged-jars => 4-downloading-unmanaged-jars}/textfile.txt (100%) rename example/javalib/dependencies/{6-repository-config => 5-repository-config}/build.mill (100%) rename example/javalib/dependencies/{6-repository-config => 5-repository-config}/foo/src/foo/Foo.java (100%) delete mode 100644 example/javalib/dependency-management/1-managing-versions/build.mill delete mode 100644 example/javalib/dependency-management/2-adding-exclusions/build.mill delete mode 100644 example/javalib/dependency-management/3-precedence/build.mill diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 9daec49d171..e903134c7f4 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -9,7 +9,6 @@ ** xref:javalib/testing.adoc[] ** xref:javalib/linting.adoc[] ** xref:javalib/publishing.adoc[] -** xref:javalib/dependency-management.adoc[] ** xref:javalib/build-examples.adoc[] ** xref:javalib/web-examples.adoc[] * xref:scalalib/intro.adoc[] diff --git a/docs/modules/ROOT/pages/fundamentals/library-deps.adoc b/docs/modules/ROOT/pages/fundamentals/library-deps.adoc index 8c703ab8175..00500ae6df5 100644 --- a/docs/modules/ROOT/pages/fundamentals/library-deps.adoc +++ b/docs/modules/ROOT/pages/fundamentals/library-deps.adoc @@ -96,6 +96,34 @@ def runIvyDeps = Agg( It is also possible to use a higher version of the same library dependencies already defined in `ivyDeps`, to ensure you compile against a minimal API version, but actually run with the latest available version. +== Dependency management + +Dependency management consists in listing dependencies whose versions we want to force. Having +a dependency in dependency management doesn't mean that this dependency will be fetched, only +that + +* if it ends up being fetched transitively, its version will be forced to the one in dependency management + +* if its version is empty in an `ivyDeps` section in Mill, the version from dependency management will be used + +Dependency management also allows to add exclusions to dependencies, both explicit dependencies and +transitive ones. + +Dependency management can be passed to Mill in two ways: + +* via external Maven BOMs, like https://repo1.maven.org/maven2/com/google/cloud/libraries-bom/26.50.0/libraries-bom-26.50.0.pom[this one], +whose Maven coordinates are `com.google.cloud:libraries-bom:26.50.0` + +* via the `dependencyManagement` task, that allows to directly list dependencies whose versions we want to enforce + +=== External BOMs + +include::partial$example/fundamentals/library-deps/bom-1-external-bom.adoc[] + +=== Dependency management task + +include::partial$example/fundamentals/library-deps/bom-2-dependency-management.adoc[] + == Searching For Dependency Updates include::partial$example/fundamentals/dependencies/1-search-updates.adoc[] diff --git a/docs/modules/ROOT/pages/javalib/dependencies.adoc b/docs/modules/ROOT/pages/javalib/dependencies.adoc index 7a8368df429..ac2c01aef22 100644 --- a/docs/modules/ROOT/pages/javalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/javalib/dependencies.adoc @@ -13,20 +13,21 @@ include::partial$example/javalib/dependencies/1-ivy-deps.adoc[] include::partial$example/javalib/dependencies/2-run-compile-deps.adoc[] -== Bill of Material and Dependency Management - -include::partial$example/javalib/dependencies/3-bom-dependency-management.adoc[] +== Dependency Management +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. == Unmanaged Jars -include::partial$example/javalib/dependencies/4-unmanaged-jars.adoc[] +include::partial$example/javalib/dependencies/3-unmanaged-jars.adoc[] == Downloading Unmanaged Jars -include::partial$example/javalib/dependencies/5-downloading-unmanaged-jars.adoc[] +include::partial$example/javalib/dependencies/4-downloading-unmanaged-jars.adoc[] == Repository Config -include::partial$example/javalib/dependencies/6-repository-config.adoc[] +include::partial$example/javalib/dependencies/5-repository-config.adoc[] diff --git a/docs/modules/ROOT/pages/javalib/dependency-management.adoc b/docs/modules/ROOT/pages/javalib/dependency-management.adoc deleted file mode 100644 index 757cc4506e3..00000000000 --- a/docs/modules/ROOT/pages/javalib/dependency-management.adoc +++ /dev/null @@ -1,19 +0,0 @@ -= Java Dependency Management -:page-aliases: Java_Dependency_Management.adoc - -include::partial$gtag-config.adoc[] - -This section is about Java so-called "dependency management" or Bill of Materials. To add -simple dependencies, see xref:./dependencies.adoc[]. - -== Managing versions - -include::partial$example/javalib/dependency-management/1-managing-versions.adoc[] - -== Adding exclusions - -include::partial$example/javalib/dependency-management/2-adding-exclusions.adoc[] - -== Precedence - -include::partial$example/javalib/dependency-management/3-precedence.adoc[] diff --git a/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc b/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc index a970a256d4e..9670e311702 100644 --- a/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc @@ -19,6 +19,12 @@ include::partial$example/kotlinlib/dependencies/1-ivy-deps.adoc[] include::partial$example/kotlinlib/dependencies/2-run-compile-deps.adoc[] +== Dependency Management + +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. + == Unmanaged Jars include::partial$example/kotlinlib/dependencies/3-unmanaged-jars.adoc[] diff --git a/docs/modules/ROOT/pages/scalalib/dependencies.adoc b/docs/modules/ROOT/pages/scalalib/dependencies.adoc index c4c0bf2ef89..902d8b7be15 100644 --- a/docs/modules/ROOT/pages/scalalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/scalalib/dependencies.adoc @@ -16,6 +16,11 @@ include::partial$example/scalalib/dependencies/1-ivy-deps.adoc[] include::partial$example/scalalib/dependencies/2-run-compile-deps.adoc[] +== Dependency Management + +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. == Unmanaged Jars diff --git a/example/fundamentals/library-deps/bom-1-external-bom/build.mill b/example/fundamentals/library-deps/bom-1-external-bom/build.mill new file mode 100644 index 00000000000..eeb3a5344c5 --- /dev/null +++ b/example/fundamentals/library-deps/bom-1-external-bom/build.mill @@ -0,0 +1,21 @@ +// Pass an external BOM to a `JavaModule` / `ScalaModule` / `KotlinModule` with `bomDeps`, like + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object bom extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// The version of grpc-protobuf (`io.grpc:grpc-protobuf`) isn't written down here, so the version +// from the BOM, `1.67.1` is used. +// +// Also, by default, grpc-protobuf `1.67.1` pulls version `3.25.3` of protobuf-java (`com.google.protobuf:protobuf-java`) . +// But the BOM specifies another version for that dependency, `4.28.3`, so +// protobuf-java `4.28.3` ends up being pulled here. diff --git a/example/fundamentals/library-deps/bom-2-dependency-management/build.mill b/example/fundamentals/library-deps/bom-2-dependency-management/build.mill new file mode 100644 index 00000000000..22f1fe18de3 --- /dev/null +++ b/example/fundamentals/library-deps/bom-2-dependency-management/build.mill @@ -0,0 +1,54 @@ +// Pass dependencies to `dependencyManagement` in a `JavaModule` / `ScalaModule` / `KotlinModule`, like + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object dependencyManagement extends JavaModule { + def dependencyManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3", + ivy"io.grpc:grpc-protobuf:1.67.1" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// The version of grpc-protobuf (`io.grpc:grpc-protobuf`) isn't written down here, so the version +// found in `dependencyManagement`, `1.67.1` is used. +// +// Also, by default, grpc-protobuf `1.67.1` pulls version `3.25.3` of protobuf-java (`com.google.protobuf:protobuf-java`) . +// But `dependencyManagement` specifies another version for that dependency, `4.28.3`, so +// protobuf-java `4.28.3` ends up being pulled here. + +// One can also add exclusions via dependency management, like + +object dependencyManagementWithVersionAndExclusions extends JavaModule { + def dependencyManagement = Agg( + ivy"io.grpc:grpc-protobuf:1.67.1" + .exclude(("com.google.protobuf", "protobuf-java")) + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// Here, grpc-protobuf has an empty version in `ivyDeps`, so the one in `dependencyManagement`, +// `1.67.1`, is used. Also, `com.google.protobuf:protobuf-java` is excluded from grpc-protobuf +// in `dependencyManagement`, so it ends up being excluded from it in `ivyDeps` too. + +// If one wants to add exclusions via `dependencyManagement`, specifying a version is optional, +// like + +object dependencyManagementWithExclusions extends JavaModule { + def dependencyManagement = Agg( + ivy"io.grpc:grpc-protobuf" + .exclude(("com.google.protobuf", "protobuf-java")) + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf:1.67.1" + ) +} + +// Here, given that grpc-protobuf is fetched during dependency resolution, +// `com.google.protobuf:protobuf-java` is excluded from it because of the dependency management. diff --git a/example/javalib/dependencies/3-bom-dependency-management/bar/src/bar/Bar.java b/example/javalib/dependencies/3-bom-dependency-management/bar/src/bar/Bar.java deleted file mode 100644 index 38751f60787..00000000000 --- a/example/javalib/dependencies/3-bom-dependency-management/bar/src/bar/Bar.java +++ /dev/null @@ -1,32 +0,0 @@ -package bar; - -import java.io.IOException; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; - -class BarServlet extends HttpServlet { - @Override - protected void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - response.setContentType("text/html"); - response.setStatus(HttpServletResponse.SC_OK); - response.getWriter().println("Hello World!"); - } -} - -public class Bar { - public static void main(String[] args) throws Exception { - Server server = new Server(8079); - ServletContextHandler context = new ServletContextHandler(); - context.setContextPath("/"); - server.setHandler(context); - context.addServlet(new ServletHolder(new BarServlet()), "/*"); - server.start(); - server.join(); - } -} diff --git a/example/javalib/dependencies/3-bom-dependency-management/build.mill b/example/javalib/dependencies/3-bom-dependency-management/build.mill deleted file mode 100644 index 51197f5e9e3..00000000000 --- a/example/javalib/dependencies/3-bom-dependency-management/build.mill +++ /dev/null @@ -1,59 +0,0 @@ -// Mill supports for Bill of Materials ("BOM") or dependency management. -// These allow to centralize versions and exclusions for your dependencies, -// either in you Mill project, via dependency management, or externally, -// via a BOM. - -// To add a BOM to a Java module, use `bomDeps`: - -//// SNIPPET:BUILD1 -package build -import mill._, javalib._ - -object bom extends JavaModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0" - ) - def ivyDeps = Agg( - ivy"io.grpc:grpc-protobuf" - ) -} - -// Versions found in a BOM are used in two ways: - -// * to fill missing versions in the `ivyDeps` section - -// * to override versions in transitive dependencies - -// The version of grpc-protobuf isn't written down here, so the version -// from the `com.google.cloud:libraries-bom:26.50.0` BOM, `1.67.1` is used. -// -// Also, by default, `io.grpc:grpc-protobuf:1.67.1` pulls `com.google.protobuf:protobuf-java:3.25.3`. -// But the BOM specifies another version for that dependency, `4.28.3`, so -// `com.google.protobuf:protobuf-java:4.28.3` ends up being pulled here. - -// Instead of pulling an external BOM, one can use `dependencyManagement` to specify dependency versions. -// Like with a BOM, versions in `dependencyManagement` are used to: - -// * fill missing versions in the `ivyDeps` section - -// * override versions in transitive dependencies - -//// SNIPPET:BUILD2 - -object dependencyManagement extends JavaModule { - def dependencyManagement = Agg( - ivy"com.google.protobuf:protobuf-java:4.28.3", - ivy"io.grpc:grpc-protobuf:1.67.1" - ) - def ivyDeps = Agg( - ivy"io.grpc:grpc-protobuf" - ) -} - -//// SNIPPET:SCALASTEWARD - -// Here, `io.grpc:grpc-protobuf` has no version in `ivyDeps`, so the version in -// `dependencyManagement`, `1.67.1` is used. Also, `io.grpc:grpc-protobuf:1.67.1` -// pulls `com.google.protobuf:protobuf-java:3.25.3` by default, but version -// `4.28.3` will be pulled, because of `com.google.protobuf:protobuf-java:4.28.3` -// in `dependencyManagement`. diff --git a/example/javalib/dependencies/3-bom-dependency-management/foo/src/foo/Foo.java b/example/javalib/dependencies/3-bom-dependency-management/foo/src/foo/Foo.java deleted file mode 100644 index 6bfe37e30c1..00000000000 --- a/example/javalib/dependencies/3-bom-dependency-management/foo/src/foo/Foo.java +++ /dev/null @@ -1,7 +0,0 @@ -package foo; - -public class Foo { - public static void main(String[] args) { - System.out.println("Hello World"); - } -} diff --git a/example/javalib/dependencies/4-unmanaged-jars/build.mill b/example/javalib/dependencies/3-unmanaged-jars/build.mill similarity index 100% rename from example/javalib/dependencies/4-unmanaged-jars/build.mill rename to example/javalib/dependencies/3-unmanaged-jars/build.mill diff --git a/example/javalib/dependencies/4-unmanaged-jars/lib/nanojson-1.8.jar b/example/javalib/dependencies/3-unmanaged-jars/lib/nanojson-1.8.jar similarity index 100% rename from example/javalib/dependencies/4-unmanaged-jars/lib/nanojson-1.8.jar rename to example/javalib/dependencies/3-unmanaged-jars/lib/nanojson-1.8.jar diff --git a/example/javalib/dependencies/4-unmanaged-jars/src/foo/Foo.java b/example/javalib/dependencies/3-unmanaged-jars/src/foo/Foo.java similarity index 100% rename from example/javalib/dependencies/4-unmanaged-jars/src/foo/Foo.java rename to example/javalib/dependencies/3-unmanaged-jars/src/foo/Foo.java diff --git a/example/javalib/dependencies/5-downloading-unmanaged-jars/build.mill b/example/javalib/dependencies/4-downloading-unmanaged-jars/build.mill similarity index 100% rename from example/javalib/dependencies/5-downloading-unmanaged-jars/build.mill rename to example/javalib/dependencies/4-downloading-unmanaged-jars/build.mill diff --git a/example/javalib/dependencies/5-downloading-unmanaged-jars/src/foo/Foo.java b/example/javalib/dependencies/4-downloading-unmanaged-jars/src/foo/Foo.java similarity index 100% rename from example/javalib/dependencies/5-downloading-unmanaged-jars/src/foo/Foo.java rename to example/javalib/dependencies/4-downloading-unmanaged-jars/src/foo/Foo.java diff --git a/example/javalib/dependencies/5-downloading-unmanaged-jars/textfile.txt b/example/javalib/dependencies/4-downloading-unmanaged-jars/textfile.txt similarity index 100% rename from example/javalib/dependencies/5-downloading-unmanaged-jars/textfile.txt rename to example/javalib/dependencies/4-downloading-unmanaged-jars/textfile.txt diff --git a/example/javalib/dependencies/6-repository-config/build.mill b/example/javalib/dependencies/5-repository-config/build.mill similarity index 100% rename from example/javalib/dependencies/6-repository-config/build.mill rename to example/javalib/dependencies/5-repository-config/build.mill diff --git a/example/javalib/dependencies/6-repository-config/foo/src/foo/Foo.java b/example/javalib/dependencies/5-repository-config/foo/src/foo/Foo.java similarity index 100% rename from example/javalib/dependencies/6-repository-config/foo/src/foo/Foo.java rename to example/javalib/dependencies/5-repository-config/foo/src/foo/Foo.java diff --git a/example/javalib/dependency-management/1-managing-versions/build.mill b/example/javalib/dependency-management/1-managing-versions/build.mill deleted file mode 100644 index a68773f9979..00000000000 --- a/example/javalib/dependency-management/1-managing-versions/build.mill +++ /dev/null @@ -1,12 +0,0 @@ -//// SNIPPET:BUILD1 -package build -import mill._, javalib._ - -object bom extends JavaModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0" - ) - def ivyDeps = Agg( - ivy"io.grpc:grpc-protobuf" - ) -} diff --git a/example/javalib/dependency-management/2-adding-exclusions/build.mill b/example/javalib/dependency-management/2-adding-exclusions/build.mill deleted file mode 100644 index a68773f9979..00000000000 --- a/example/javalib/dependency-management/2-adding-exclusions/build.mill +++ /dev/null @@ -1,12 +0,0 @@ -//// SNIPPET:BUILD1 -package build -import mill._, javalib._ - -object bom extends JavaModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0" - ) - def ivyDeps = Agg( - ivy"io.grpc:grpc-protobuf" - ) -} diff --git a/example/javalib/dependency-management/3-precedence/build.mill b/example/javalib/dependency-management/3-precedence/build.mill deleted file mode 100644 index a68773f9979..00000000000 --- a/example/javalib/dependency-management/3-precedence/build.mill +++ /dev/null @@ -1,12 +0,0 @@ -//// SNIPPET:BUILD1 -package build -import mill._, javalib._ - -object bom extends JavaModule { - def bomDeps = Agg( - ivy"com.google.cloud:libraries-bom:26.50.0" - ) - def ivyDeps = Agg( - ivy"io.grpc:grpc-protobuf" - ) -} diff --git a/example/package.mill b/example/package.mill index 79de9a41ec1..1705260e539 100644 --- a/example/package.mill +++ b/example/package.mill @@ -36,9 +36,6 @@ object `package` extends RootModule with Module { object basic extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "basic")) object module extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "module")) object dependencies extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "dependencies")) - object `dependency-management` extends Cross[ExampleCrossModuleJava]( - build.listIn(millSourcePath / "dependency-management") - ) object testing extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "testing")) object linting extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "linting")) object migrating extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "migrating")) @@ -84,6 +81,7 @@ object `package` extends RootModule with Module { object cross extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "cross")) object `out-dir` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "out-dir")) object libraries extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "libraries")) + object `library-deps` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "library-deps")) } object depth extends Module { diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 0d9cb7426db..a2498b669e1 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -159,16 +159,6 @@ trait JavaModule */ def runIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } - /** - * Dependency to use as a "parent". - * - * This dependency is advertized as "parent dependency" in POM files, - * when publishing this module. - * - * In practice, from Mill, this is equivalent to a BOM dependency. - */ - def parentDep: T[Option[Dep]] = Task { None } - /** * Any BOM dependencies you want to add to this Module, in the format * ivy"org:name:version" @@ -177,7 +167,7 @@ trait JavaModule def allBomDeps: Task[Agg[BomDependency]] = Task.Anon { val modVerOrMalformed = - (Agg(parentDep().toSeq: _*) ++ bomDeps()).map(bindDependency()).map { bomDep => + bomDeps().map(bindDependency()).map { bomDep => val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) if (fromModVer == bomDep.dep) Right(bomDep.dep.asBomDependency) @@ -195,7 +185,7 @@ trait JavaModule } else throw new Exception( - "Found parent or BOM dependencies with invalid parameters:" + System.lineSeparator() + + "Found BOM dependencies with invalid parameters:" + System.lineSeparator() + malformed.map("- " + _.dep + System.lineSeparator()).mkString + "Only organization, name, and version are accepted." ) diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index b2ad3b58d39..75c4eb93c33 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -36,12 +36,7 @@ trait PublishModule extends JavaModule { outer => * * @see [[https://maven.apache.org/guides/introduction/introduction-to-the-pom.html#Project_Inheritance Project Inheritance]] */ - def pomParentProject: T[Option[Artifact]] = Task { - parentDep().map { parentDep => - val parentDep0 = bindDependency().apply(parentDep) - Artifact(parentDep0.organization, parentDep0.name, parentDep0.version) - } - } + def pomParentProject: T[Option[Artifact]] = None /** * Configuration for the `pom.xml` metadata file published with this module diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index 2e2e454b2a2..db3ce96b356 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -86,41 +86,6 @@ object BomTests extends TestSuite { } } - object parent extends JavaModule with TestPublishModule { - def parentDep = Some(ivy"org.apache.spark:spark-parent_2.13:3.5.3") - def ivyDeps = Agg( - ivy"org.apache.commons:commons-compress" - ) - - object dependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - parent - ) - } - - object subDependee extends JavaModule with TestPublishModule { - def moduleDeps = Seq( - dependee - ) - } - - object scala extends ScalaModule with TestPublishModule { - def scalaVersion = _root_.scala.util.Properties.versionNumberString - def parentDep = Some(ivy"org.apache.spark::spark-parent:3.5.3") - def ivyDeps = Agg( - ivy"org.apache.commons:commons-compress" - ) - } - - object invalid extends TestBaseModule { - object exclude extends JavaModule { - def parentDep = Some( - ivy"org.apache.spark:spark-parent_2.13:3.5.3".exclude(("foo", "thing")) - ) - } - } - } - object depMgmt extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" @@ -392,40 +357,7 @@ object BomTests extends TestSuite { val res = eval(modules.bom.invalid.exclude.compileClasspath) assert( res.left.exists(_.toString.contains( - "Found parent or BOM dependencies with invalid parameters:" - )) - ) - } - } - } - - test("parent") { - test("simple") - UnitTester(modules, null).scoped { implicit eval => - isInClassPath(modules.parent, expectedCommonsCompressJarName) - } - - test("dependee") - UnitTester(modules, null).scoped { implicit eval => - isInClassPath(modules.parent.dependee, expectedCommonsCompressJarName, Seq(modules.parent)) - } - - test("subDependee") - UnitTester(modules, null).scoped { implicit eval => - isInClassPath( - modules.parent.subDependee, - expectedCommonsCompressJarName, - Seq(modules.parent, modules.parent.dependee) - ) - } - - test("scala") - UnitTester(modules, null).scoped { implicit eval => - isInClassPath(modules.parent.scala, expectedCommonsCompressJarName, scalaSuffix = "_2.13") - } - - test("invalid") { - test - UnitTester(modules, null).scoped { eval => - val res = eval(modules.parent.invalid.exclude.compileClasspath) - assert( - res.left.exists(_.toString.contains( - "Found parent or BOM dependencies with invalid parameters:" + "Found BOM dependencies with invalid parameters:" )) ) } From 169a7478f1acef184ee0911029c8031b50a8d72c Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 4 Dec 2024 13:11:56 +0100 Subject: [PATCH 25/27] Address review comments --- .../ROOT/pages/fundamentals/library-deps.adoc | 2 +- .../bom-1-external-bom/build.mill | 6 ++++- .../bom-2-dependency-management/build.mill | 24 +++++++++---------- scalalib/src/mill/scalalib/JavaModule.scala | 8 +++---- .../src/mill/scalalib/PublishModule.scala | 4 ++-- .../test/src/mill/scalalib/BomTests.scala | 12 +++++----- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/docs/modules/ROOT/pages/fundamentals/library-deps.adoc b/docs/modules/ROOT/pages/fundamentals/library-deps.adoc index 00500ae6df5..b5e4c295e98 100644 --- a/docs/modules/ROOT/pages/fundamentals/library-deps.adoc +++ b/docs/modules/ROOT/pages/fundamentals/library-deps.adoc @@ -114,7 +114,7 @@ Dependency management can be passed to Mill in two ways: * via external Maven BOMs, like https://repo1.maven.org/maven2/com/google/cloud/libraries-bom/26.50.0/libraries-bom-26.50.0.pom[this one], whose Maven coordinates are `com.google.cloud:libraries-bom:26.50.0` -* via the `dependencyManagement` task, that allows to directly list dependencies whose versions we want to enforce +* via the `depManagement` task, that allows to directly list dependencies whose versions we want to enforce === External BOMs diff --git a/example/fundamentals/library-deps/bom-1-external-bom/build.mill b/example/fundamentals/library-deps/bom-1-external-bom/build.mill index eeb3a5344c5..4b076da2999 100644 --- a/example/fundamentals/library-deps/bom-1-external-bom/build.mill +++ b/example/fundamentals/library-deps/bom-1-external-bom/build.mill @@ -4,7 +4,7 @@ package build import mill._, javalib._ -object bom extends JavaModule { +object foo extends JavaModule { def bomDeps = Agg( ivy"com.google.cloud:libraries-bom:26.50.0" ) @@ -19,3 +19,7 @@ object bom extends JavaModule { // Also, by default, grpc-protobuf `1.67.1` pulls version `3.25.3` of protobuf-java (`com.google.protobuf:protobuf-java`) . // But the BOM specifies another version for that dependency, `4.28.3`, so // protobuf-java `4.28.3` ends up being pulled here. +// +// Several BOMs can be passed to `bomDeps`. If several specify a version for a dependency, +// the version from the first one in the `bomDeps` list is used. If several specify exclusions +// for a dependency, all exclusions are added to that dependency. diff --git a/example/fundamentals/library-deps/bom-2-dependency-management/build.mill b/example/fundamentals/library-deps/bom-2-dependency-management/build.mill index 22f1fe18de3..6caf0b6b99d 100644 --- a/example/fundamentals/library-deps/bom-2-dependency-management/build.mill +++ b/example/fundamentals/library-deps/bom-2-dependency-management/build.mill @@ -1,11 +1,11 @@ -// Pass dependencies to `dependencyManagement` in a `JavaModule` / `ScalaModule` / `KotlinModule`, like +// Pass dependencies to `depManagement` in a `JavaModule` / `ScalaModule` / `KotlinModule`, like //// SNIPPET:BUILD1 package build import mill._, javalib._ -object dependencyManagement extends JavaModule { - def dependencyManagement = Agg( +object foo extends JavaModule { + def depManagement = Agg( ivy"com.google.protobuf:protobuf-java:4.28.3", ivy"io.grpc:grpc-protobuf:1.67.1" ) @@ -15,16 +15,16 @@ object dependencyManagement extends JavaModule { } // The version of grpc-protobuf (`io.grpc:grpc-protobuf`) isn't written down here, so the version -// found in `dependencyManagement`, `1.67.1` is used. +// found in `depManagement`, `1.67.1` is used. // // Also, by default, grpc-protobuf `1.67.1` pulls version `3.25.3` of protobuf-java (`com.google.protobuf:protobuf-java`) . -// But `dependencyManagement` specifies another version for that dependency, `4.28.3`, so +// But `depManagement` specifies another version for that dependency, `4.28.3`, so // protobuf-java `4.28.3` ends up being pulled here. // One can also add exclusions via dependency management, like -object dependencyManagementWithVersionAndExclusions extends JavaModule { - def dependencyManagement = Agg( +object bar extends JavaModule { + def depManagement = Agg( ivy"io.grpc:grpc-protobuf:1.67.1" .exclude(("com.google.protobuf", "protobuf-java")) ) @@ -33,15 +33,15 @@ object dependencyManagementWithVersionAndExclusions extends JavaModule { ) } -// Here, grpc-protobuf has an empty version in `ivyDeps`, so the one in `dependencyManagement`, +// Here, grpc-protobuf has an empty version in `ivyDeps`, so the one in `depManagement`, // `1.67.1`, is used. Also, `com.google.protobuf:protobuf-java` is excluded from grpc-protobuf -// in `dependencyManagement`, so it ends up being excluded from it in `ivyDeps` too. +// in `depManagement`, so it ends up being excluded from it in `ivyDeps` too. -// If one wants to add exclusions via `dependencyManagement`, specifying a version is optional, +// If one wants to add exclusions via `depManagement`, specifying a version is optional, // like -object dependencyManagementWithExclusions extends JavaModule { - def dependencyManagement = Agg( +object baz extends JavaModule { + def depManagement = Agg( ivy"io.grpc:grpc-protobuf" .exclude(("com.google.protobuf", "protobuf-java")) ) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index a2498b669e1..d101c27354d 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -201,13 +201,13 @@ trait JavaModule * For example, the following forces com.lihaoyi::os-lib to version 0.11.3, and * excludes org.slf4j:slf4j-api from com.lihaoyi::cask that it forces to version 0.9.4 * {{{ - * def dependencyManagement = super.dependencyManagement() ++ Agg( + * def depManagement = super.depManagement() ++ Agg( * ivy"com.lihaoyi::os-lib:0.11.3", * ivy"com.lihaoyi::cask:0.9.4".exclude("org.slf4j", "slf4j-api") * ) * }}} */ - def dependencyManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + def depManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } private def addBoms( dep: coursier.core.Dependency, @@ -244,7 +244,7 @@ trait JavaModule } /** - * Data from dependencyManagement, converted to a type ready to be passed to coursier + * Data from depManagement, converted to a type ready to be passed to coursier * for dependency resolution */ private def processedDependencyManagement(deps: Seq[coursier.core.Dependency]) @@ -468,7 +468,7 @@ trait JavaModule ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) val depMgmt = processedDependencyManagement( - dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + depManagement().toSeq.map(bindDependency()).map(_.dep) ) val depMgmtMap = depMgmt.toMap diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 75c4eb93c33..cc9fc686998 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -100,7 +100,7 @@ trait PublishModule extends JavaModule { outer => * Dependency management to specify in the POM */ def publishXmlDepMgmt: Task[Agg[Dependency]] = Task.Anon { - dependencyManagement().map(resolvePublishDependency.apply().apply(_)) + depManagement().map(resolvePublishDependency.apply().apply(_)) } def pom: T[PathRef] = Task { @@ -163,7 +163,7 @@ trait PublishModule extends JavaModule { outer => dep } val overrides = - dependencyManagement().toSeq.map(bindDependency()).map(_.dep) + depManagement().toSeq.map(bindDependency()).map(_.dep) .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") .map { depMgmt => Ivy.Override( diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index db3ce96b356..48fc81c9628 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -90,7 +90,7 @@ object BomTests extends TestSuite { def ivyDeps = Agg( ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" ) - def dependencyManagement = Agg( + def depManagement = Agg( ivy"com.google.protobuf:protobuf-java:4.28.3" ) @@ -102,7 +102,7 @@ object BomTests extends TestSuite { def ivyDeps = Agg( ivy"com.lihaoyi:cask_2.13:0.9.4" ) - def dependencyManagement = Agg( + def depManagement = Agg( // The exclude should be automatically added to the dependency above // thanks to dependency management, but the version should be left // untouched @@ -119,7 +119,7 @@ object BomTests extends TestSuite { def ivyDeps = Agg( ivy"com.lihaoyi:cask_2.13:0.9.4" ) - def dependencyManagement = Agg( + def depManagement = Agg( ivy"org.java-websocket:Java-WebSocket:1.5.2" .exclude(("org.slf4j", "slf4j-api")) ) @@ -133,7 +133,7 @@ object BomTests extends TestSuite { def ivyDeps = Agg( ivy"com.lihaoyi:cask_2.13:0.9.4" ) - def dependencyManagement = Agg( + def depManagement = Agg( ivy"org.java-websocket:Java-WebSocket" .exclude(("org.slf4j", "slf4j-api")) ) @@ -145,7 +145,7 @@ object BomTests extends TestSuite { object invalid extends TestBaseModule { object transitive extends JavaModule { - def dependencyManagement = { + def depManagement = { val dep = ivy"org.java-websocket:Java-WebSocket:1.5.3" Agg( dep.copy( @@ -160,7 +160,7 @@ object BomTests extends TestSuite { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java" ) - def dependencyManagement = Agg( + def depManagement = Agg( ivy"com.google.protobuf:protobuf-java:4.28.3" ) From 2a5ca63d44246efc7774709e08df169e144696d2 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 4 Dec 2024 13:35:22 +0100 Subject: [PATCH 26/27] nit --- scalalib/src/mill/scalalib/JavaModule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index d101c27354d..648f3c656dc 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -214,7 +214,7 @@ trait JavaModule bomDeps: Seq[coursier.core.BomDependency], depMgmt: Seq[(DependencyManagement.Key, DependencyManagement.Values)], depMgmtMap: DependencyManagement.Map, - overrideVersions: Boolean = false + overrideVersions: Boolean ): coursier.core.Dependency = { val depMgmtKey = DependencyManagement.Key( dep.module.organization, From 26116a997b575a0388d0c2669b80a2b6aa0c81c6 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 4 Dec 2024 14:07:15 +0100 Subject: [PATCH 27/27] Minor refactoring Should help in subsequent PRs --- scalalib/src/mill/scalalib/JavaModule.scala | 58 ++++++++++----------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 648f3c656dc..3fe981d8508 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -210,37 +210,37 @@ trait JavaModule def depManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } private def addBoms( - dep: coursier.core.Dependency, bomDeps: Seq[coursier.core.BomDependency], depMgmt: Seq[(DependencyManagement.Key, DependencyManagement.Values)], - depMgmtMap: DependencyManagement.Map, overrideVersions: Boolean - ): coursier.core.Dependency = { - val depMgmtKey = DependencyManagement.Key( - dep.module.organization, - dep.module.name, - coursier.core.Type.jar, - dep.publication.classifier - ) - val versionOverrideOpt = - if (dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) - else None - val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) - dep - // add BOM coordinates - coursier will handle the rest - .addBomDependencies( - if (overrideVersions) bomDeps.map(_.withForceOverrideVersions(overrideVersions)) - else bomDeps - ) - // add dependency management ourselves: - // - overrides meant to apply to transitive dependencies - // - fill version if it's empty - // - add extra exclusions from dependency management - .withOverrides(dep.overrides ++ depMgmt) - .withVersion(versionOverrideOpt.getOrElse(dep.version)) - .withMinimizedExclusions( - extraExclusions.fold(dep.minimizedExclusions)(dep.minimizedExclusions.join(_)) + ): coursier.core.Dependency => coursier.core.Dependency = { + val depMgmtMap = depMgmt.toMap + dep => + val depMgmtKey = DependencyManagement.Key( + dep.module.organization, + dep.module.name, + coursier.core.Type.jar, + dep.publication.classifier ) + val versionOverrideOpt = + if (dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) + else None + val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) + dep + // add BOM coordinates - coursier will handle the rest + .addBomDependencies( + if (overrideVersions) bomDeps.map(_.withForceOverrideVersions(overrideVersions)) + else bomDeps + ) + // add dependency management ourselves: + // - overrides meant to apply to transitive dependencies + // - fill version if it's empty + // - add extra exclusions from dependency management + .withOverrides(dep.overrides ++ depMgmt) + .withVersion(versionOverrideOpt.getOrElse(dep.version)) + .withMinimizedExclusions( + extraExclusions.fold(dep.minimizedExclusions)(dep.minimizedExclusions.join(_)) + ) } /** @@ -470,10 +470,8 @@ trait JavaModule val depMgmt = processedDependencyManagement( depManagement().toSeq.map(bindDependency()).map(_.dep) ) - val depMgmtMap = depMgmt.toMap - dep => - addBoms(dep, bomDeps0, depMgmt, depMgmtMap, overrideVersions = overrideVersions) + addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) } /**