diff --git a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy new file mode 100644 index 00000000..f4773b4b --- /dev/null +++ b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy @@ -0,0 +1,590 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.plugin + +import com.linkedin.gradle.python.plugin.testutils.DefaultProjectLayoutRule +import com.linkedin.gradle.python.plugin.testutils.PyGradleTestBuilder +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome +import org.gradle.testkit.runner.UnexpectedBuildFailure +import org.junit.Rule +import spock.lang.Specification + + +class PackageSettingsIntegrationTest extends Specification { + + @Rule + final DefaultProjectLayoutRule testProjectDir = new DefaultProjectLayoutRule() + + /////////////////////////////////// + // PipInstallTask specific tests // + /////////////////////////////////// + + def "pip install uses environment"() { + given: "package settings for 'foo' have custom environment" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-sdist' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | Map getEnvironment(PackageInfo packageInfo) { + | return (packageInfo.name != 'foo') ? [:] : [ + | 'CPPFLAGS': '-I/some/custom/path/include', + | 'LDFLAGS': '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib', + | ] + | } + | } + | + | import com.linkedin.gradle.python.tasks.PipInstallTask + | + | project.tasks.withType(PipInstallTask) { PipInstallTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project with debug enabled" + def result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('-d', 'build') + .withPluginClasspath() + .withDebug(true) + .build() + println result.output + + then: "we can observe the environment for 'foo' provided exactly two lines after its install is logged" + result.output.find( + 'Installing foo\n[^\n]+\n[^\n]+Environment for[^\n]+CPPFLAGS=-I/some/custom/path/include') + result.output.find( + 'Installing foo\n[^\n]+\n[^\n]+LDFLAGS=-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib') + result.output.contains('BUILD SUCCESS') + result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS + } + + /* + * We are using setuptools for option testing because it's the first + * package installed and it will fail very early. This shortens the + * integration tests as much as possible. + */ + + def "pip install uses global options"() { + given: "package settings for 'setuptools' have global options" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-sdist' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | List getGlobalOptions(PackageInfo packageInfo) { + | return (packageInfo.name == 'setuptools') ? ['--global-option', '--dummy-global-option'] : [] + | } + | } + | + | import com.linkedin.gradle.python.tasks.PipInstallTask + | + | project.tasks.withType(PipInstallTask) { PipInstallTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project" + def result + try { + result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('build') + .withPluginClasspath() + .withDebug(true) + .build() + } catch (UnexpectedBuildFailure buildFailure) { + // expected to fail + result = buildFailure.buildResult + } + println result.output + + then: "we can observe global options being passed to 'setuptools' and failing because it does not expect them" + // the global option for setup.py is passed *before* install command + result.output.find('setup.py[^\n]+ --dummy-global-option install') + result.output.contains('Running setup.py install for setuptools') + result.output.contains('error: option --dummy-global-option not recognized') + result.output.contains('BUILD FAILED') + result.task(':foo:installSetupRequirements').outcome == TaskOutcome.FAILED + } + + def "pip install uses install options"() { + given: "package settings for 'setuptools' have install options" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-sdist' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | List getInstallOptions(PackageInfo packageInfo) { + | return (packageInfo.name == 'setuptools') ? ['--install-option', '--ignore=E123,E234'] : [] + | } + | } + | + | import com.linkedin.gradle.python.tasks.PipInstallTask + | + | project.tasks.withType(PipInstallTask) { PipInstallTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project" + def result + try { + result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('build') + .withPluginClasspath() + .withDebug(true) + .build() + } catch (UnexpectedBuildFailure buildFailure) { + // expected to fail + result = buildFailure.buildResult + } + println result.output + + then: "we can observe install options being passed to 'setuptools' and failing because it does not expect them" + // the install option is passed *after* install command + result.output.find('setup.py[^\n]+ install [^\n]+ --ignore=E123,E234') + result.output.contains('Running setup.py install for setuptools') + result.output.contains('error: option --ignore not recognized') + result.output.contains('BUILD FAILED') + result.task(':foo:installSetupRequirements').outcome == TaskOutcome.FAILED + } + + def "pip install uses supported language versions"() { + given: "package settings for 'setuptools' have supported language versions" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-sdist' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | List getSupportedLanguageVersions(PackageInfo packageInfo) { + | return (packageInfo.name == 'setuptools') ? ['2.8'] : [] + | } + | } + | + | import com.linkedin.gradle.python.tasks.PipInstallTask + | + | project.tasks.withType(PipInstallTask) { PipInstallTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project" + def result + try { + result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('build') + .withPluginClasspath() + .withDebug(true) + .build() + } catch (UnexpectedBuildFailure buildFailure) { + // expected to fail + result = buildFailure.buildResult + } + println result.output + + then: "we can observe supported language version being checked and failing for non-existent version" + result.output.contains('Package setuptools works only with Python versions: [2.8]') + result.output.contains('BUILD FAILED') + result.task(':foo:installSetupRequirements').outcome == TaskOutcome.FAILED + } + + /* + * We are using a dependency on pyflakes because it's also a transitive + * build dependency and will be re-installed as a runtime dependency. + */ + + def "pip install requires source rebuild"() { + given: "package settings for 'pyflakes' require a source rebuild" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-sdist' + | } + | + | dependencies { + | python 'pypi:pyflakes:+' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | boolean requiresSourceBuild(PackageInfo packageInfo) { + | return (packageInfo.name == 'pyflakes') ? true : super.requiresSourceBuild(packageInfo) + | } + | } + | + | import com.linkedin.gradle.python.tasks.PipInstallTask + | + | project.tasks.withType(PipInstallTask) { PipInstallTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project with info enabled" + def result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('-i', 'build') + .withPluginClasspath() + .withDebug(true) + .build() + println result.output + + then: "we can observe that required source rebuild happens" + // pyflakes should be installed in build requirements, and then again in runtime requirements + result.output.findAll('Installing pyflakes[^\n]+\n[^\n]+ --ignore-installed [^\n]+pyflakes').size() == 2 + result.output.contains('BUILD SUCCESS') + result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS + } + + //////////////////////////////////// + // BuildWheelsTask specific tests // + //////////////////////////////////// + + // We are using a dependency on pyflakes, just to have a wheel to build. + def "wheel uses environment"() { + given: "package settings for 'foo' have custom environment" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-pex' + | } + | + | version = '1.0.0' + | + | dependencies { + | python 'pypi:pyflakes:+' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | Map getEnvironment(PackageInfo packageInfo) { + | return (packageInfo.name != 'foo') ? [:] : [ + | 'CPPFLAGS': '-I/some/custom/path/include', + | 'LDFLAGS': '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib', + | ] + | } + | } + | + | import com.linkedin.gradle.python.tasks.BuildWheelsTask + | + | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project with debug enabled" + def result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('-d', 'build') + .withPluginClasspath() + .withDebug(true) + .build() + println result.output + + then: "we can observe the environment for 'foo' provided exactly two lines after its build is logged" + result.output.find( + 'Installing foo wheel\n[^\n]+\n[^\n]+Environment for[^\n]+CPPFLAGS=-I/some/custom/path/include') + result.output.find( + 'Installing foo wheel\n[^\n]+\n[^\n]+LDFLAGS=-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib') + result.output.contains('BUILD SUCCESS') + result.task(':foo:buildWheels').outcome == TaskOutcome.SUCCESS + result.task(':foo:buildProjectWheel').outcome == TaskOutcome.SUCCESS + } + + def "wheel uses global options"() { + given: "package settings for 'pyflakes' have global options" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-pex' + | } + | + | version = '1.0.0' + | + | dependencies { + | python 'pypi:pyflakes:+' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | List getGlobalOptions(PackageInfo packageInfo) { + | return (packageInfo.name == 'pyflakes') ? ['--global-option', '--dummy-global-option'] : [] + | } + | } + | + | import com.linkedin.gradle.python.tasks.BuildWheelsTask + | + | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project" + def result + try { + result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('build') + .withPluginClasspath() + .withDebug(true) + .build() + } catch (UnexpectedBuildFailure buildFailure) { + // expected to fail + result = buildFailure.buildResult + } + println result.output + + then: "we can observe global options being passed to 'pyflakes' and failing because it does not expect them" + // the global option for setup.py is passed *before* bdist_wheel command + result.output.find('setup.py[^\n]+ --dummy-global-option bdist_wheel') + result.output.contains('Failed building wheel for pyflakes') + result.output.contains('error: option --dummy-global-option not recognized') + result.output.contains('BUILD FAILED') + // the build of everything in the virtualenv succeeded + result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS + // but the wheel build failed + result.task(':foo:buildWheels').outcome == TaskOutcome.FAILED + } + + def "wheel uses build options"() { + given: "package settings for 'pyflakes' have build options" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-pex' + | } + | + | version = '1.0.0' + | + | dependencies { + | python 'pypi:pyflakes:+' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | List getBuildOptions(PackageInfo packageInfo) { + | return (packageInfo.name == 'pyflakes') ? ['--build-option', '--disable-something'] : [] + | } + | } + | + | import com.linkedin.gradle.python.tasks.BuildWheelsTask + | + | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project" + def result + try { + result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('build') + .withPluginClasspath() + .withDebug(true) + .build() + } catch (UnexpectedBuildFailure buildFailure) { + // expected to fail + result = buildFailure.buildResult + } + println result.output + + then: "we can observe build options being passed to 'pyflakes' and failing because it does not expect them" + // the build option is passed *after* bdist_wheel command + result.output.find('setup.py[^\n]+ bdist_wheel [^\n]+ --disable-something') + result.output.contains('Failed building wheel for pyflakes') + result.output.contains('error: option --disable-something not recognized') + result.output.contains('BUILD FAILED') + // the build of everything in the virtualenv succeeded + result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS + // but the wheel build failed + result.task(':foo:buildWheels').outcome == TaskOutcome.FAILED + } + + def "wheel uses supported language versions"() { + given: "package settings for 'pyflakes' have supported language versions" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-pex' + | } + | + | version = '1.0.0' + | + | dependencies { + | python 'pypi:pyflakes:+' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | List getSupportedLanguageVersions(PackageInfo packageInfo) { + | return (packageInfo.name == 'pyflakes') ? ['2.8'] : [] + | } + | } + | + | import com.linkedin.gradle.python.tasks.BuildWheelsTask + | + | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project" + def result + try { + result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('build') + .withPluginClasspath() + .withDebug(true) + .build() + } catch (UnexpectedBuildFailure buildFailure) { + // expected to fail + result = buildFailure.buildResult + } + println result.output + + then: "we can observe supported language version being checked and failing for non-existent version" + result.output.contains('Package pyflakes works only with Python versions: [2.8]') + result.output.contains('BUILD FAILED') + // the build of everything in the virtualenv succeeded + result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS + // but the wheel build failed + result.task(':foo:buildWheels').outcome == TaskOutcome.FAILED + } + + def "wheel requires source rebuild"() { + given: "package settings for 'pyflakes' require source rebuild" + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-pex' + | } + | + | version = '1.0.0' + | + | dependencies { + | python 'pypi:pyflakes:+' + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(String projectName) { super(projectName) } + | + | @Override + | boolean requiresSourceBuild(PackageInfo packageInfo) { + | return (packageInfo.name == 'pyflakes') ? true : super.requiresSourceBuild(packageInfo) + | } + | } + | + | import com.linkedin.gradle.python.tasks.BuildWheelsTask + | + | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> + | task.packageSettings = new DefaultTestPackageSettings(project.name) + | } + | + |${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: "we build a project with info enabled" + def result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('-i', 'build') + .withPluginClasspath() + .withDebug(true) + .build() + println result.output + + then: "we can observe that required source rebuild happens" + // pyflakes wheel should be re-installed after install in build and runtime requirements of virtualenv. + result.output.findAll('Installing pyflakes[^\n]+\n[^\n]+pip install [^\n]+pyflakes').size() == 2 + result.output.find('Installing pyflakes[^ ]+ wheel\n[^\n]+pip wheel [^\n]+pyflakes') + result.output.contains('BUILD SUCCESS') + result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS + result.task(':foo:buildWheels').outcome == TaskOutcome.SUCCESS + } + +} diff --git a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy index 27ab6d5a..3df791b7 100644 --- a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy +++ b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy @@ -36,12 +36,12 @@ class PexIntegrationTest extends Specification { @IgnoreIf({ OperatingSystem.current() == OperatingSystem.WINDOWS }) def "can build thin pex"() { testProjectDir - given: + given: "project with the version containing a hyphen" testProjectDir.buildFile << """\ | plugins { | id 'com.linkedin.python-pex' | } - | version = '1.0.0' + | version = '1.0.0-SNAPSHOT' | python { | pex { | fatPex = false @@ -50,7 +50,7 @@ class PexIntegrationTest extends Specification { | ${PyGradleTestBuilder.createRepoClosure()} """.stripMargin().stripIndent() - when: + when: "we build it with info option" def result = GradleRunner.create() .withProjectDir(testProjectDir.root) .withArguments('build', '--stacktrace', '--info') @@ -61,8 +61,9 @@ class PexIntegrationTest extends Specification { Path deployablePath = testProjectDir.root.toPath().resolve(Paths.get('foo', 'build', 'deployable', "bin")) - then: + then: "it succeeds and the hyphen was converted into underscore for pex command due to pex bug workaround" + result.output.find("pex [^\n]+ foo==1.0.0_SNAPSHOT") result.output.contains("BUILD SUCCESS") result.task(':foo:flake8').outcome == TaskOutcome.SUCCESS result.task(':foo:installPythonRequirements').outcome == TaskOutcome.SUCCESS diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPexDistributionPlugin.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPexDistributionPlugin.java index ad3df2a0..a6836f9b 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPexDistributionPlugin.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPexDistributionPlugin.java @@ -28,6 +28,7 @@ public class PythonPexDistributionPlugin extends PythonBasePlugin { public static final String TASK_BUILD_WHEELS = "buildWheels"; + public static final String TASK_BUILD_PROJECT_WHEEL = "buildProjectWheel"; public static final String TASK_BUILD_PEX = "buildPex"; public static final String TASK_PACKAGE_DEPLOYABLE = "packageDeployable"; @@ -56,11 +57,19 @@ public void applyTo(final Project project) { * * We need wheels to build pex files. */ - project.getTasks().create(TASK_BUILD_WHEELS, BuildWheelsTask.class, - task -> task.dependsOn(project.getTasks().getByName(StandardTextValues.TASK_INSTALL_PROJECT.getValue()))); + project.getTasks().create(TASK_BUILD_WHEELS, BuildWheelsTask.class, task -> { + task.dependsOn(project.getTasks().getByName(StandardTextValues.TASK_INSTALL_PROJECT.getValue())); + task.setInstallFileCollection(project.getConfigurations().getByName("python")); + }); + + project.getTasks().create(TASK_BUILD_PROJECT_WHEEL, BuildWheelsTask.class, task -> { + task.dependsOn(project.getTasks().getByName(TASK_BUILD_WHEELS)); + task.setInstallFileCollection(project.files(project.file(project.getProjectDir()))); + task.setEnvironment(extension.pythonEnvironmentDistgradle); + }); project.getTasks().create(TASK_BUILD_PEX, BuildPexTask.class, - task -> task.dependsOn(project.getTasks().getByName(TASK_BUILD_WHEELS))); + task -> task.dependsOn(project.getTasks().getByName(TASK_BUILD_PROJECT_WHEEL))); Tar packageDeployable = project.getTasks().create(TASK_PACKAGE_DEPLOYABLE, Tar.class, tar -> { tar.setCompression(Compression.GZIP); diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java index a1c84313..d68171ba 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java @@ -15,10 +15,8 @@ */ package com.linkedin.gradle.python.tasks; -import com.linkedin.gradle.python.PythonExtension; import com.linkedin.gradle.python.extension.DeployableExtension; import com.linkedin.gradle.python.extension.PexExtension; -import com.linkedin.gradle.python.extension.WheelExtension; import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider; import com.linkedin.gradle.python.tasks.execution.TeeOutputContainer; import com.linkedin.gradle.python.util.ExtensionUtils; @@ -32,10 +30,7 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.TaskAction; -import org.gradle.process.ExecResult; -import org.gradle.process.ExecSpec; -import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -81,7 +76,6 @@ public void setPexOptions(List pexOptions) { public void buildPex() throws Exception { Project project = getProject(); - final PythonExtension pythonExtension = ExtensionUtils.getPythonExtension(project); DeployableExtension deployableExtension = ExtensionUtils.getPythonComponentExtension(project, DeployableExtension.class); PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class); @@ -91,17 +85,6 @@ public void buildPex() throws Exception { pexExtension.getPexCache().mkdirs(); } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - ExecResult exec = project.exec(execSpec -> configureExecution(pythonExtension, execSpec, outputStream)); - - if (getLogger().isInfoEnabled()) { - getLogger().info(outputStream.toString()); - } else if (exec.getExitValue() != 0) { - getLogger().lifecycle(outputStream.toString()); - } - - exec.assertNormalExitValue(); - deployableExtension.getDeployableBuildDir().mkdirs(); if (pexExtension.isFatPex()) { @@ -111,25 +94,6 @@ public void buildPex() throws Exception { } } - private void configureExecution(PythonExtension pythonExtension, ExecSpec spec, ByteArrayOutputStream outputStream) { - container.setOutputs(spec); - WheelExtension wheelExtension = ExtensionUtils.maybeCreateWheelExtension(getProject()); - - spec.environment(pythonExtension.pythonEnvironment); - spec.environment(pythonExtension.pythonEnvironmentDistgradle); - spec.commandLine(pythonExtension.getDetails().getVirtualEnvInterpreter()); - spec.args(pythonExtension.getDetails().getVirtualEnvironment().getPip(), - "wheel", - "--disable-pip-version-check", - "--wheel-dir", - wheelExtension.getWheelCache().getAbsolutePath(), - "--no-deps", - "."); - spec.setErrorOutput(outputStream); - spec.setStandardOutput(outputStream); - spec.setIgnoreExitValue(true); - } - @Input @Optional public Map getAdditionalProperties() { diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy index 75f516ae..eaea664a 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy @@ -19,9 +19,13 @@ import com.linkedin.gradle.python.PythonExtension import com.linkedin.gradle.python.extension.PythonDetails import com.linkedin.gradle.python.extension.WheelExtension import com.linkedin.gradle.python.plugin.PythonHelpers +import com.linkedin.gradle.python.util.DefaultEnvironmentMerger +import com.linkedin.gradle.python.util.DefaultPackageSettings import com.linkedin.gradle.python.util.DependencyOrder +import com.linkedin.gradle.python.util.EnvironmentMerger import com.linkedin.gradle.python.util.ExtensionUtils import com.linkedin.gradle.python.util.PackageInfo +import com.linkedin.gradle.python.util.PackageSettings import com.linkedin.gradle.python.util.internal.TaskTimer import com.linkedin.gradle.python.wheel.EmptyWheelCache import com.linkedin.gradle.python.wheel.SupportsWheelCache @@ -30,16 +34,20 @@ import org.apache.commons.io.FileUtils import org.gradle.api.DefaultTask import org.gradle.api.GradleException import org.gradle.api.Project +import org.gradle.api.file.FileCollection import org.gradle.api.logging.Logger import org.gradle.api.logging.Logging import org.gradle.api.specs.Spec import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Optional import org.gradle.api.tasks.TaskAction import org.gradle.internal.logging.progress.ProgressLogger import org.gradle.internal.logging.progress.ProgressLoggerFactory import org.gradle.process.ExecResult import org.gradle.process.ExecSpec + class BuildWheelsTask extends DefaultTask implements SupportsWheelCache { private static final Logger LOGGER = Logging.getLogger(BuildWheelsTask) @@ -50,17 +58,20 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache { private PythonExtension pythonExtension private PythonDetails details + @InputFiles + FileCollection installFileCollection + + @Input + @Optional + Map environment + + PackageSettings packageSettings = new DefaultPackageSettings(project.name) + + EnvironmentMerger environmentMerger = new DefaultEnvironmentMerger() + @TaskAction - public void buildWheelsTask() { - Collection configurationFiles = null - try { - configurationFiles = DependencyOrder.configurationPostOrderFiles(project.configurations.python) - } catch (Throwable e) { - // Log and fall back to old style installation order as before. - logger.lifecycle("***** WARNING: ${ e.message } *****") - configurationFiles = project.configurations.python.files.sort() - } - buildWheels(project, configurationFiles, getPythonDetails()) + void buildWheelsTask() { + buildWheels(project, DependencyOrder.getConfigurationFiles(installFileCollection), getPythonDetails()) /* * If pexDependencies are empty or its wheels are already @@ -135,7 +146,7 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache { int counter = 0 def numberOfInstallables = installables.size() installables.each { File installable -> - + def pyVersion = pythonDetails.getPythonVersion().pythonMajorMinor def packageInfo = PackageInfo.fromPath(installable.path) def shortHand = packageInfo.version ? "${ packageInfo.name }-${ packageInfo.version }" : packageInfo.name @@ -143,49 +154,76 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache { progressLogger.progress("Preparing wheel $shortHand (${ ++counter } of $numberOfInstallables)") if (PythonHelpers.isPlainOrVerbose(project)) { - LOGGER.lifecycle("Installing {}", shortHand) + LOGGER.lifecycle("Installing {} wheel", shortHand) } if (packageExcludeFilter.isSatisfiedBy(packageInfo)) { if (PythonHelpers.isPlainOrVerbose(project)) { - LOGGER.lifecycle("Skipping {}, excluded", shortHand) + LOGGER.lifecycle("Skipping {} wheel - Excluded", shortHand) } return } - def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details) - if (wheel.isPresent()) { - File wheelFile = wheel.get() - FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name)) - if (PythonHelpers.isPlainOrVerbose(project)) { - LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile) - } - return + // If supported versions are empty, there are no restrictions. + def supportedVersions = packageSettings.getSupportedLanguageVersions(packageInfo) + if (supportedVersions != null && !supportedVersions.empty && !supportedVersions.contains(pyVersion)) { + throw new GradleException( + "Package ${packageInfo.name} works only with Python versions: ${supportedVersions}") } - // Check if a wheel exists for this product already and only build it - // if it is missing. We don't care about the wheel details because we - // always build these locally. - def tree = project.fileTree( - dir: wheelExtension.wheelCache, - include: "**/${ packageInfo.name.replace('-', '_') }-${ packageInfo.version }-*.whl") + /* + * Check if a wheel exists for this product already and only build it + * if it is missing. We don't care about the wheel details because we + * always build these locally. + */ + if (!packageSettings.requiresSourceBuild(packageInfo)) { + def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details) + if (wheel.isPresent()) { + File wheelFile = wheel.get() + FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name)) + if (PythonHelpers.isPlainOrVerbose(project)) { + LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile) + } + return + } + + def tree = project.fileTree( + dir: wheelExtension.wheelCache, + include: "**/${packageInfo.name.replace('-', '_')}-${(packageInfo.version ?: 'unspecified').replace('-', '_')}-*.whl") + + if (tree.files.size() >= 1) { + return + } + } def stream = new ByteArrayOutputStream() - if (tree.files.size() >= 1) { - return + def mergedEnv = environmentMerger.mergeEnvironments( + [pythonExtension.pythonEnvironment, environment, packageSettings.getEnvironment(packageInfo)]) + + def commandLine = [ + pythonDetails.getVirtualEnvInterpreter().toString(), + pythonDetails.getVirtualEnvironment().getPip().toString(), + 'wheel', + '--disable-pip-version-check', + '--wheel-dir', wheelExtension.wheelCache.toString(), + '--no-deps', + ] + + def globalOptions = packageSettings.getGlobalOptions(packageInfo) + if (globalOptions != null) { + commandLine.addAll(globalOptions) + } + + def buildOptions = packageSettings.getBuildOptions(packageInfo) + if (buildOptions != null) { + commandLine.addAll(buildOptions) } - def commandLine = [pythonDetails.getVirtualEnvInterpreter(), - pythonDetails.getVirtualEnvironment().getPip(), - 'wheel', - '--disable-pip-version-check', - '--wheel-dir', wheelExtension.wheelCache, - '--no-deps', - installable] + commandLine.add(installable.toString()) ExecResult installResult = project.exec { ExecSpec execSpec -> - execSpec.environment pythonExtension.pythonEnvironment + execSpec.environment mergedEnv execSpec.commandLine(commandLine) execSpec.standardOutput = stream execSpec.errorOutput = stream diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy index b7d3acbe..aa8d8fb4 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy @@ -19,10 +19,14 @@ import com.linkedin.gradle.python.PythonExtension import com.linkedin.gradle.python.extension.PythonDetails import com.linkedin.gradle.python.plugin.PythonHelpers import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider +import com.linkedin.gradle.python.util.DefaultEnvironmentMerger +import com.linkedin.gradle.python.util.DefaultPackageSettings import com.linkedin.gradle.python.util.DependencyOrder +import com.linkedin.gradle.python.util.EnvironmentMerger import com.linkedin.gradle.python.util.ExtensionUtils import com.linkedin.gradle.python.util.OperatingSystem import com.linkedin.gradle.python.util.PackageInfo +import com.linkedin.gradle.python.util.PackageSettings import com.linkedin.gradle.python.util.internal.TaskTimer import com.linkedin.gradle.python.wheel.EmptyWheelCache import com.linkedin.gradle.python.wheel.SupportsWheelCache @@ -30,7 +34,6 @@ import com.linkedin.gradle.python.wheel.WheelCache import groovy.transform.CompileStatic import org.gradle.api.DefaultTask import org.gradle.api.GradleException -import org.gradle.api.artifacts.Configuration import org.gradle.api.file.FileCollection import org.gradle.api.specs.Spec import org.gradle.api.tasks.Input @@ -45,6 +48,7 @@ import org.gradle.process.ExecSpec import java.nio.file.Path import java.nio.file.Paths + /** * Execute pip install * @@ -73,6 +77,10 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo @Optional boolean sorted = true + PackageSettings packageSettings = new DefaultPackageSettings(project.name) + + EnvironmentMerger environmentMerger = new DefaultEnvironmentMerger() + /** * Will return true when the package should be excluded from being installed. */ @@ -86,24 +94,6 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo private String lastInstallMessage = null - /** - * Returns a set of configuration files in the insert order or sorted. - * - * If sorted is true (default) the sorted configuration set is returned, - * otherwise the original order. - */ - Collection getConfigurationFiles() { - if (sorted && (installFileCollection instanceof Configuration)) { - try { - return DependencyOrder.configurationPostOrderFiles((Configuration) installFileCollection) - } catch (Throwable e) { - // Log and fall back to old style installation order as before. - logger.lifecycle("***** WARNING: ${ e.message } *****") - } - } - return sorted ? installFileCollection.files.sort() : installFileCollection.files - } - /** * Method that checks to ensure that the current project is prepared to pip install. It ignores the * base pygradle libraries @@ -148,7 +138,7 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo TaskTimer taskTimer = new TaskTimer() int counter = 0 - def installableFiles = getConfigurationFiles() + def installableFiles = DependencyOrder.getConfigurationFiles(installFileCollection, sorted) for (File installable : installableFiles) { if (isReadyForInstall(installable)) { def packageInfo = PackageInfo.fromPath(installable.getAbsolutePath()) @@ -176,39 +166,53 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo return } + // If supported versions are empty, there are no restrictions. + def supportedVersions = packageSettings.getSupportedLanguageVersions(packageInfo) + if (supportedVersions != null && !supportedVersions.empty && !supportedVersions.contains(pyVersion)) { + throw new PipInstallException( + "Package ${packageInfo.name} works only with Python versions: ${supportedVersions}") + } + String sanitizedName = packageInfo.name.replace('-', '_') // See: https://www.python.org/dev/peps/pep-0376/ File egg = sitePackages.resolve("${ sanitizedName }-${ packageInfo.version }-py${ pyVersion }.egg-info").toFile() File dist = sitePackages.resolve("${ sanitizedName }-${ packageInfo.version }.dist-info").toFile() - def mergedEnv = new HashMap(extension.pythonEnvironment) - if (environment != null) { - mergedEnv.putAll(environment) - } - - if (project.file(egg).exists() || project.file(dist).exists()) { + if (!packageSettings.requiresSourceBuild(packageInfo) && + (project.file(egg).exists() || project.file(dist).exists())) { if (PythonHelpers.isPlainOrVerbose(project)) { logger.lifecycle("Skipping {} - Installed", shortHand) } return } - def commandLine = [pythonDetails.getVirtualEnvInterpreter(), - pythonDetails.getVirtualEnvironment().getPip(), - 'install', - '--disable-pip-version-check', - '--no-deps'] + def mergedEnv = environmentMerger.mergeEnvironments( + [extension.pythonEnvironment, environment, packageSettings.getEnvironment(packageInfo)]) + + + def commandLine = [ + pythonDetails.getVirtualEnvInterpreter().toString(), + pythonDetails.getVirtualEnvironment().getPip().toString(), + 'install', + '--disable-pip-version-check', + '--no-deps', + ] + commandLine.addAll(args) - if (shortHand.endsWith('-SNAPSHOT')) { - // snapshot packages may have changed, so reinstall them every time - commandLine.add('--ignore-installed') + def globalOptions = packageSettings.getGlobalOptions(packageInfo) + if (globalOptions != null) { + commandLine.addAll(globalOptions) } + def installOptions = packageSettings.getInstallOptions(packageInfo) + if (installOptions != null) { + commandLine.addAll(installOptions) + } def cachedWheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonDetails) - if (cachedWheel.isPresent()) { + if (!packageSettings.requiresSourceBuild(packageInfo) && cachedWheel.isPresent()) { if (PythonHelpers.isPlainOrVerbose(project)) { logger.lifecycle("{} from wheel: {}", shortHand, cachedWheel.get().getAbsolutePath()) } @@ -217,7 +221,6 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo commandLine.add(installable.getAbsolutePath()) } - if (PythonHelpers.isPlainOrVerbose(project)) { logger.lifecycle("Installing {}", shortHand) } @@ -251,8 +254,8 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo } } - public static class PipInstallException extends GradleException { - public PipInstallException(String message) { + static class PipInstallException extends GradleException { + PipInstallException(String message) { super(message) } } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/SourceDistTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/SourceDistTask.java index 22533bb4..ccfc2126 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/SourceDistTask.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/SourceDistTask.java @@ -40,6 +40,7 @@ public File getSdistOutput() { String.format( "%s-%s.tar.gz", project.getName(), + // TODO: Is this replace here really necessary? project.getVersion().toString().replace("_", "-"))); } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DefaultEnvironmentMerger.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DefaultEnvironmentMerger.java new file mode 100644 index 00000000..84c79dcb --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DefaultEnvironmentMerger.java @@ -0,0 +1,45 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.util; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Default implementation of EnvironmentMerger. + * + * Just updates maps directly. + */ +public class DefaultEnvironmentMerger implements EnvironmentMerger { + @Override + public void mergeIntoEnvironment(Map target, Map source) { + if (source != null) { + target.putAll(source); + } + } + + @Override + public Map mergeEnvironments(List> sources) { + Map target = new HashMap<>(); + if (sources != null) { + for (Map source : sources) { + mergeIntoEnvironment(target, source); + } + } + return target; + } +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DefaultPackageSettings.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DefaultPackageSettings.java new file mode 100644 index 00000000..7086f8af --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DefaultPackageSettings.java @@ -0,0 +1,85 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.util; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Default implementation of PackageSettings. + * + * Returns empty collections mostly. + * Handles snapshots and project package rebuilds automatically. + */ +public class DefaultPackageSettings implements PackageSettings { + private final String projectName; + + public DefaultPackageSettings(String projectName) { + this.projectName = projectName; + } + + @Override + public Map getEnvironment(PackageInfo packageInfo) { + return Collections.emptyMap(); + } + + @Override + public List getGlobalOptions(PackageInfo packageInfo) { + return Collections.emptyList(); + } + + @Override + public List getInstallOptions(PackageInfo packageInfo) { + String name = packageInfo.getName(); + String version = packageInfo.getVersion(); + + // always reinstall snapshots, but current project is installed editable anyway, no need for other options + if (!projectName.equals(name) + && ((version != null && version.endsWith("-SNAPSHOT")) || requiresSourceBuild(packageInfo))) { + return Collections.singletonList("--ignore-installed"); + } + return Collections.emptyList(); + } + + @Override + public List getBuildOptions(PackageInfo packageInfo) { + return Collections.emptyList(); + } + + @Override + public List getConfigureOptions(PackageInfo packageInfo) { + return Collections.emptyList(); + } + + @Override + public List getSupportedLanguageVersions(PackageInfo packageInfo) { + return Collections.emptyList(); + } + + @Override + public boolean requiresSourceBuild(PackageInfo packageInfo) { + String name = packageInfo.getName(); + String version = packageInfo.getVersion(); + + // always rebuild the project package itself + if (projectName.equals(name)) { + return true; + } + // always rebuild snapshots; otherwise no rebuild required + return (version != null && version.endsWith("-SNAPSHOT")); + } +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DependencyOrder.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DependencyOrder.java index dac0980f..1a222b6a 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DependencyOrder.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/DependencyOrder.java @@ -24,6 +24,7 @@ import org.gradle.api.artifacts.result.DependencyResult; import org.gradle.api.artifacts.result.ResolvedComponentResult; import org.gradle.api.artifacts.result.ResolvedDependencyResult; +import org.gradle.api.file.FileCollection; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; @@ -34,6 +35,7 @@ import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** @@ -144,6 +146,38 @@ public static Collection configurationPostOrderFiles(Configuration configu return files; } + /** + * Returns a set of configuration files in the tree post-order, + * or sorted, or in the original insert order. + * + * Attempt to collect dependency tree post-order and fall back to + * sorted or insert order. + * If sorted is false, it will always return the original insert order. + */ + public static Collection getConfigurationFiles(FileCollection files, boolean sorted) { + if (sorted && (files instanceof Configuration)) { + try { + return DependencyOrder.configurationPostOrderFiles((Configuration) files); + } catch (Throwable e) { + // Log and fall back to old style installation order as before. + logger.lifecycle("***** WARNING: ${ e.message } *****"); + } + } + return sorted + ? files.getFiles().stream().sorted().collect(Collectors.toSet()) + : files.getFiles(); + } + + /** + * Returns a set of configuration files in the tree post-order or sorted. + * + * Attempt to collect dependency tree post-order and fall back to + * sorted order. + */ + public static Collection getConfigurationFiles(FileCollection files) { + return getConfigurationFiles(files, true); + } + /* * Return a set difference between the larger and smaller sets. */ diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/EnvironmentMerger.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/EnvironmentMerger.java new file mode 100644 index 00000000..f3514155 --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/EnvironmentMerger.java @@ -0,0 +1,41 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.util; + +import java.util.List; +import java.util.Map; + + +/** + * Interface for environment merging utility. + */ +public interface EnvironmentMerger { + /** + * Merge source environment into the target environment. + * + * @param target the target environment to merge into + * @param source the source environment to merge + */ + void mergeIntoEnvironment(Map target, Map source); + + /** + * Merge source environments together. + * + * @param sources the source environments to merge + * @return the environment with sources merged in + */ + Map mergeEnvironments(List> sources); +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/PackageSettings.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/PackageSettings.java new file mode 100644 index 00000000..76baf281 --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/PackageSettings.java @@ -0,0 +1,105 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.util; + +import java.util.List; +import java.util.Map; + + +/** + * Interface for package specific settings. + * + * The settings needed for a specific package to build properly + * or to customize its build can be obtained from the API provided + * by this interface. + * + * @param Type of the object that represents package information. + */ +public interface PackageSettings { + /** + * Get the build environment required for this package. + * + * @param t package information object + * @return a mapping of environment variables and their values + */ + Map getEnvironment(T t); + + /** + * Get the global options for the package installation. + *

+ * An example is "--global-option" for the "pip" command. + * + * @param t package information object + * @return a list of global options + */ + List getGlobalOptions(T t); + + /** + * Get the install options for the package. + *

+ * An example is "--install-option" for the "pip install" command. + * + * @param t package information object + * @return a list of install options + */ + List getInstallOptions(T t); + + /** + * Get the build options for the binary version of this package. + *

+ * An example is "--build-option" for the "pip wheel" command. + * + * @param t package information object + * @return a list of build options + */ + List getBuildOptions(T t); + + /** + * Get the configure options for the package. + *

+ * For example, the options for "./configure" command when building + * a C program. + * + * @param t package information object + * @return a list of configure options + */ + @SuppressWarnings("unused") + List getConfigureOptions(T t); + + /** + * Get the language versions supported by the package. + *

+ * An example is Python package support for 2.7, 3.5, and 3.6 + * versions of the language. The major version can be also + * specified, such as 2 or 3. + * + * @param t package information object + * @return a list of install options + */ + List getSupportedLanguageVersions(T t); + + /** + * Determines if the package requires a build from source. + *

+ * Even if the binary artifact of the package is available, + * the package may need a rebuild because of custom build + * options or environment. + * + * @param t package information object + * @return true when the package needs a rebuild from source + */ + boolean requiresSourceBuild(T t); +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/PexExecSpecAction.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/PexExecSpecAction.java index 4c2efd66..12973786 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/PexExecSpecAction.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/PexExecSpecAction.java @@ -154,7 +154,18 @@ public static PexExecSpecAction withOutEntryPoint( private List pexRequirements(Map dependencies) { List requirements = new ArrayList<>(); for (Map.Entry entry : dependencies.entrySet()) { - requirements.add(entry.getKey() + "==" + entry.getValue()); + /* + * Work around the bug in pex. + * It does not follow PEP 427 completely: https://www.python.org/dev/peps/pep-0427/ + * It turns hyphens into underscored for wheel file name in package (distribution) name only, + * but it does not do the same for package version. + * The pep is quite clear about this: + * "Each component of the filename is escaped by replacing runs of non-alphanumeric characters + * with an underscore _" + * On the other hand, pip handles this correctly, so there's a discrepancy. + * Until pex fixes this bug, we have to tell it that version in the file name has underscore. + */ + requirements.add(entry.getKey() + "==" + entry.getValue().replace("-", "_")); } return requirements; } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/pip/PipFreezeAction.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/pip/PipFreezeAction.java index 5ee6467f..74bf3680 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/pip/PipFreezeAction.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/pip/PipFreezeAction.java @@ -66,8 +66,8 @@ public Map getDependencies() { }); Map dependencies = PipFreezeOutputParser.getDependencies(developmentDependencies, requirements); - // The version will convert - into _ for wheel builds, so convert right here to handle -SNAPSHOT, -MPDEP, ... - dependencies.put(project.getName(), project.getVersion().toString().replace("-", "_")); + // Always add project unconditionally. + dependencies.put(project.getName(), project.getVersion().toString()); return dependencies; } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/pip/PipFreezeOutputParser.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/pip/PipFreezeOutputParser.java index 4a9db949..eaebcdc3 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/pip/PipFreezeOutputParser.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/pip/PipFreezeOutputParser.java @@ -49,7 +49,7 @@ static Map getDependencies(Collection ignoredDependencie * The version will convert - into _ for wheel builds, so convert right here. */ if (!(ignoredDependencies.contains(name) || ignoredDependencies.contains(name.replace("-", "_")))) { - reqs.put(name, version.replace("-", "_")); + reqs.put(name, version); } } } diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/EnvironmentMergerTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/EnvironmentMergerTest.groovy new file mode 100644 index 00000000..a886df1e --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/EnvironmentMergerTest.groovy @@ -0,0 +1,53 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.util + +import spock.lang.Specification + + +/** + * Unit tests for environment merger implementers. + */ +class EnvironmentMergerTest extends Specification { + EnvironmentMerger merger = new DefaultEnvironmentMerger() + + def "merge a list of environments"() { + when: "we use a default merger to merge a list of environments" + Map mergedEnv = merger.mergeEnvironments([['A': '1'], ['B': '2'], ['C': '3']]) + + then: "we get merged environment back" + mergedEnv == ['A': '1', 'B': '2', 'C': '3'] + } + + def "merge into a master environment"() { + when: "we use a default merger to merge a list of environments into a master environment" + Map masterEnv = ['X': '10', 'Y': '20', 'A': '5'] + Map mergedEnv = merger.mergeEnvironments([masterEnv, ['A': '1'], ['A': '2'], ['A': '3']]) + + then: "we get merged environments with keys overriding in order of appearance with last winning" + mergedEnv == ['A': '3', 'X': '10', 'Y': '20'] + } + + def "merge one environment into another"() { + when: "we use a default merger to merge a source environment into a target environment" + Map target = ['A': '1', 'B': '2'] + merger.mergeIntoEnvironment(target, ['A': '10', 'C': '3']) + + then: "we get merged environments with keys overriding target from source" + target == ['A': '10', 'B': '2', 'C': '3'] + } + +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/PackageSettingsTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/PackageSettingsTest.groovy new file mode 100644 index 00000000..6d8c4524 --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/PackageSettingsTest.groovy @@ -0,0 +1,88 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.util + +import spock.lang.Specification + + +/** + * Unit tests for package settings implementers. + */ +class PackageSettingsTest extends Specification { + PackageSettings packageSettings = new DefaultPackageSettings('foo') + + def "default package settings environment"() { + expect: "empty environment" + packageSettings.getEnvironment(PackageInfo.fromPath('flake8-1.2.3.tar.gz')) == [:] + } + + def "default package settings global options"() { + expect: "empty global options" + packageSettings.getGlobalOptions(PackageInfo.fromPath('Sphinx-1.2.3.tar.gz')) == [] + } + + def "default package settings install options"() { + expect: "empty install options for non-project package that is not a snapshot" + packageSettings.getInstallOptions(PackageInfo.fromPath('requests-1.2.3.tar.gz')) == [] + } + + def "package settings install options for snapshot"() { + expect: "install option --ignore-installed for SNAPSHOT packages to enforce re-install" + packageSettings.getInstallOptions(PackageInfo.fromPath('requests-1.2.3-SNAPSHOT.tar.gz')) == [ + '--ignore-installed'] + } + + def "package settings install options for project snapshot()"() { + expect: "project snapshot does not use --ignore-installed because it's installed editable" + packageSettings.getInstallOptions(PackageInfo.fromPath('foo-1.2.3-SNAPSHOT.tar.gz')) == [] + } + + def "default package settings build options"() { + expect: "empty build options" + packageSettings.getBuildOptions(PackageInfo.fromPath('numpy-1.2.3.tar.gz')) == [] + } + + def "package settings build options for snapshot"() { + expect: "empty build options" + packageSettings.getBuildOptions(PackageInfo.fromPath('scipy-1.2.3-SNAPSHOT.tar.gz')) == [] + } + + def "default package settings configure options"() { + expect: "empty configure options" + packageSettings.getConfigureOptions(PackageInfo.fromPath('pytest-1.2.3.tar.gz')) == [] + } + + def "default package settings supported language versions"() { + expect: "empty supported language versions" + packageSettings.getSupportedLanguageVersions(PackageInfo.fromPath('foo-1.2.3.tar.gz')) == [] + } + + def "default package settings requires source build"() { + expect: "does not require source build" + !packageSettings.requiresSourceBuild(PackageInfo.fromPath('requests-1.2.3.tar.gz')) + } + + def "package settings require source build for snapshot"() { + expect: "snapshot requires a build" + packageSettings.requiresSourceBuild(PackageInfo.fromPath('requests-1.2.3-SNAPSHOT.tar.gz')) + } + + def "package settings requires a rebuild for the current project"() { + expect: "project requires a rebuild" + packageSettings.requiresSourceBuild(PackageInfo.fromPath('foo-1.2.3.tar.gz')) + } + +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/pip/PipFreezeOutputParserTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/pip/PipFreezeOutputParserTest.groovy index 0c4f2b54..db7c711a 100644 --- a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/pip/PipFreezeOutputParserTest.groovy +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/pip/PipFreezeOutputParserTest.groovy @@ -47,19 +47,19 @@ class PipFreezeOutputParserTest extends Specification { |snowballstemmer==1.1.0 |Sphinx==1.4.1 |testProject==unspecified - |MP==1.2.3-MPDEP + |MP==1.2.3-TESTSUFFIX |wheel==0.26.0'''.stripMargin().stripIndent() expect: PipFreezeOutputParser.getDependencies(['pbr', 'Babel', 'pep8', 'py', 'setuptools', 'pytest-xdist', 'Jinja2', 'flake8', 'snowballstemmer', 'alabaster', 'sphinx_rtd_theme', 'Pygments', 'pytest-cov', 'pip', 'mccabe', 'docutils', 'coverage', 'pex', 'six', 'setuptools-git', 'pyflakes', 'pytest', 'wheel', 'imagesize', 'argparse', 'Sphinx', 'colorama', - 'pytz'], freezeOutput) == ['testProject': 'unspecified', 'MP': '1.2.3_MPDEP'] + 'pytz'], freezeOutput) == ['testProject': 'unspecified', 'MP': '1.2.3-TESTSUFFIX'] PipFreezeOutputParser.getDependencies(['pbr', 'Babel', 'pep8', 'py', 'setuptools', 'pytest-xdist', 'Jinja2', 'flake8', 'snowballstemmer', 'alabaster', 'sphinx_rtd_theme', 'Pygments', 'pip', 'mccabe', 'docutils', 'coverage', 'pex', 'six', 'setuptools-git', 'pyflakes', 'pytest', 'wheel', 'imagesize', 'argparse', 'Sphinx', 'colorama', - 'pytz'], freezeOutput) == ['testProject': 'unspecified', 'MP': '1.2.3_MPDEP', 'pytest-cov': '2.2.1'] + 'pytz'], freezeOutput) == ['testProject': 'unspecified', 'MP': '1.2.3-TESTSUFFIX', 'pytest-cov': '2.2.1'] } def 'throws error on bad requirements format'() {