Skip to content

Commit

Permalink
Add the framework for package-specific settings.
Browse files Browse the repository at this point in the history 
Add the interface and the default implementation for package specific
settings. Use the framework calls in pip install and build wheel tasks.
  • Loading branch information
@zvezdan
zvezdan committed Mar 10, 2018
1 parent ece8611 commit 154094e
Show file tree
Hide file tree
Showing 18 changed files with 1,218 additions and 124 deletions.
590 changes: 590 additions & 0 deletions .../integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy
View file

Large diffs are not rendered by default.

9 changes: 5 additions & 4 deletions ...e-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ class PexIntegrationTest extends Specification {
@IgnoreIf({ OperatingSystem.current() == OperatingSystem.WINDOWS })
def "can build thin pex"() {
testProjectDir
given:
given: "project with the version containing a hyphen"
testProjectDir.buildFile << """\
| plugins {
| id 'com.linkedin.python-pex'
| }
| version = '1.0.0'
| version = '1.0.0-MPDEP'
| python {
| pex {
| fatPex = false
Expand All @@ -50,7 +50,7 @@ class PexIntegrationTest extends Specification {
| ${PyGradleTestBuilder.createRepoClosure()}
""".stripMargin().stripIndent()

when:
when: "we build it with info option"
def result = GradleRunner.create()
.withProjectDir(testProjectDir.root)
.withArguments('build', '--stacktrace', '--info')
Expand All @@ -61,8 +61,9 @@ class PexIntegrationTest extends Specification {

Path deployablePath = testProjectDir.root.toPath().resolve(Paths.get('foo', 'build', 'deployable', "bin"))

then:
then: "it succeeds and the hyphen was converted into underscore for pex command due to pex bug workaround"

result.output.find("pex [^\n]+ foo==1.0.0_MPDEP")
result.output.contains("BUILD SUCCESS")
result.task(':foo:flake8').outcome == TaskOutcome.SUCCESS
result.task(':foo:installPythonRequirements').outcome == TaskOutcome.SUCCESS
Expand Down
15 changes: 12 additions & 3 deletions ...plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPexDistributionPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
public class PythonPexDistributionPlugin extends PythonBasePlugin {

public static final String TASK_BUILD_WHEELS = "buildWheels";
public static final String TASK_BUILD_PROJECT_WHEEL = "buildProjectWheel";
public static final String TASK_BUILD_PEX = "buildPex";
public static final String TASK_PACKAGE_DEPLOYABLE = "packageDeployable";

Expand Down Expand Up @@ -56,11 +57,19 @@ public void applyTo(final Project project) {
*
* We need wheels to build pex files.
*/
project.getTasks().create(TASK_BUILD_WHEELS, BuildWheelsTask.class,
task -> task.dependsOn(project.getTasks().getByName(StandardTextValues.TASK_INSTALL_PROJECT.getValue())));
project.getTasks().create(TASK_BUILD_WHEELS, BuildWheelsTask.class, task -> {
task.dependsOn(project.getTasks().getByName(StandardTextValues.TASK_INSTALL_PROJECT.getValue()));
task.setInstallFileCollection(project.getConfigurations().getByName("python"));
});

project.getTasks().create(TASK_BUILD_PROJECT_WHEEL, BuildWheelsTask.class, task -> {
task.dependsOn(project.getTasks().getByName(TASK_BUILD_WHEELS));
task.setInstallFileCollection(project.files(project.file(project.getProjectDir())));
task.setEnvironment(extension.pythonEnvironmentDistgradle);
});

project.getTasks().create(TASK_BUILD_PEX, BuildPexTask.class,
task -> task.dependsOn(project.getTasks().getByName(TASK_BUILD_WHEELS)));
task -> task.dependsOn(project.getTasks().getByName(TASK_BUILD_PROJECT_WHEEL)));

Tar packageDeployable = project.getTasks().create(TASK_PACKAGE_DEPLOYABLE, Tar.class, tar -> {
tar.setCompression(Compression.GZIP);
Expand Down
36 changes: 0 additions & 36 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@
*/
package com.linkedin.gradle.python.tasks;

import com.linkedin.gradle.python.PythonExtension;
import com.linkedin.gradle.python.extension.DeployableExtension;
import com.linkedin.gradle.python.extension.PexExtension;
import com.linkedin.gradle.python.extension.WheelExtension;
import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider;
import com.linkedin.gradle.python.tasks.execution.TeeOutputContainer;
import com.linkedin.gradle.python.util.ExtensionUtils;
Expand All @@ -32,10 +30,7 @@
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.TaskAction;
import org.gradle.process.ExecResult;
import org.gradle.process.ExecSpec;

import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -81,7 +76,6 @@ public void setPexOptions(List<String> pexOptions) {
public void buildPex() throws Exception {
Project project = getProject();

final PythonExtension pythonExtension = ExtensionUtils.getPythonExtension(project);
DeployableExtension deployableExtension = ExtensionUtils.getPythonComponentExtension(project, DeployableExtension.class);
PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class);

Expand All @@ -91,17 +85,6 @@ public void buildPex() throws Exception {
pexExtension.getPexCache().mkdirs();
}

ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ExecResult exec = project.exec(execSpec -> configureExecution(pythonExtension, execSpec, outputStream));

if (getLogger().isInfoEnabled()) {
getLogger().info(outputStream.toString());
} else if (exec.getExitValue() != 0) {
getLogger().lifecycle(outputStream.toString());
}

exec.assertNormalExitValue();

deployableExtension.getDeployableBuildDir().mkdirs();

if (pexExtension.isFatPex()) {
Expand All @@ -111,25 +94,6 @@ public void buildPex() throws Exception {
}
}

private void configureExecution(PythonExtension pythonExtension, ExecSpec spec, ByteArrayOutputStream outputStream) {
container.setOutputs(spec);
WheelExtension wheelExtension = ExtensionUtils.maybeCreateWheelExtension(getProject());

spec.environment(pythonExtension.pythonEnvironment);
spec.environment(pythonExtension.pythonEnvironmentDistgradle);
spec.commandLine(pythonExtension.getDetails().getVirtualEnvInterpreter());
spec.args(pythonExtension.getDetails().getVirtualEnvironment().getPip(),
"wheel",
"--disable-pip-version-check",
"--wheel-dir",
wheelExtension.getWheelCache().getAbsolutePath(),
"--no-deps",
".");
spec.setErrorOutput(outputStream);
spec.setStandardOutput(outputStream);
spec.setIgnoreExitValue(true);
}

@Input
@Optional
public Map<String, String> getAdditionalProperties() {
Expand Down
116 changes: 79 additions & 37 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@ import com.linkedin.gradle.python.PythonExtension
import com.linkedin.gradle.python.extension.PythonDetails
import com.linkedin.gradle.python.extension.WheelExtension
import com.linkedin.gradle.python.plugin.PythonHelpers
import com.linkedin.gradle.python.util.DefaultEnvironmentMerger
import com.linkedin.gradle.python.util.DefaultPackageSettings
import com.linkedin.gradle.python.util.DependencyOrder
import com.linkedin.gradle.python.util.EnvironmentMerger
import com.linkedin.gradle.python.util.ExtensionUtils
import com.linkedin.gradle.python.util.PackageInfo
import com.linkedin.gradle.python.util.PackageSettings
import com.linkedin.gradle.python.util.internal.TaskTimer
import com.linkedin.gradle.python.wheel.EmptyWheelCache
import com.linkedin.gradle.python.wheel.SupportsWheelCache
Expand All @@ -30,16 +34,20 @@ import org.apache.commons.io.FileUtils
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.gradle.api.logging.Logger
import org.gradle.api.logging.Logging
import org.gradle.api.specs.Spec
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction
import org.gradle.internal.logging.progress.ProgressLogger
import org.gradle.internal.logging.progress.ProgressLoggerFactory
import org.gradle.process.ExecResult
import org.gradle.process.ExecSpec


class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {

private static final Logger LOGGER = Logging.getLogger(BuildWheelsTask)
Expand All @@ -50,17 +58,24 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {
private PythonExtension pythonExtension
private PythonDetails details

@InputFiles
FileCollection installFileCollection

@Input
@Optional
Map<String, String> environment

@Input
@Optional
PackageSettings<PackageInfo> packageSettings = new DefaultPackageSettings(project.name)

@Input
@Optional
EnvironmentMerger environmentMerger = new DefaultEnvironmentMerger()

@TaskAction
public void buildWheelsTask() {
Collection<File> configurationFiles = null
try {
configurationFiles = DependencyOrder.configurationPostOrderFiles(project.configurations.python)
} catch (Throwable e) {
// Log and fall back to old style installation order as before.
logger.lifecycle("***** WARNING: ${ e.message } *****")
configurationFiles = project.configurations.python.files.sort()
}
buildWheels(project, configurationFiles, getPythonDetails())
void buildWheelsTask() {
buildWheels(project, DependencyOrder.getConfigurationFiles(installFileCollection), getPythonDetails())

/*
* If pexDependencies are empty or its wheels are already
Expand Down Expand Up @@ -135,57 +150,84 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {
int counter = 0
def numberOfInstallables = installables.size()
installables.each { File installable ->

def pyVersion = pythonDetails.getPythonVersion().pythonMajorMinor
def packageInfo = PackageInfo.fromPath(installable.path)
def shortHand = packageInfo.version ? "${ packageInfo.name }-${ packageInfo.version }" : packageInfo.name

def clock = taskTimer.start(shortHand)
progressLogger.progress("Preparing wheel $shortHand (${ ++counter } of $numberOfInstallables)")

if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Installing {}", shortHand)
LOGGER.lifecycle("Installing {} wheel", shortHand)
}

if (packageExcludeFilter.isSatisfiedBy(packageInfo)) {
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, excluded", shortHand)
LOGGER.lifecycle("Skipping {} wheel - Excluded", shortHand)
}
return
}

def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details)
if (wheel.isPresent()) {
File wheelFile = wheel.get()
FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name))
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile)
}
return
// If supported versions are empty, there are no restrictions.
def supportedVersions = packageSettings.getSupportedLanguageVersions(packageInfo)
if (supportedVersions != null && !supportedVersions.empty && !supportedVersions.contains(pyVersion)) {
throw new GradleException(
"Package ${packageInfo.name} works only with Python versions: ${supportedVersions}")
}

// Check if a wheel exists for this product already and only build it
// if it is missing. We don't care about the wheel details because we
// always build these locally.
def tree = project.fileTree(
dir: wheelExtension.wheelCache,
include: "**/${ packageInfo.name.replace('-', '_') }-${ packageInfo.version }-*.whl")
/*
* Check if a wheel exists for this product already and only build it
* if it is missing. We don't care about the wheel details because we
* always build these locally.
*/
if (!packageSettings.requiresSourceBuild(packageInfo)) {
def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details)
if (wheel.isPresent()) {
File wheelFile = wheel.get()
FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name))
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile)
}
return
}

def tree = project.fileTree(
dir: wheelExtension.wheelCache,
include: "**/${packageInfo.name.replace('-', '_')}-${(packageInfo.version ?: 'unspecified').replace('-', '_')}-*.whl")

if (tree.files.size() >= 1) {
return
}
}

def stream = new ByteArrayOutputStream()

if (tree.files.size() >= 1) {
return
def mergedEnv = environmentMerger.mergeEnvironments(
[pythonExtension.pythonEnvironment, environment, packageSettings.getEnvironment(packageInfo)])

def commandLine = [
pythonDetails.getVirtualEnvInterpreter().toString(),
pythonDetails.getVirtualEnvironment().getPip().toString(),
'wheel',
'--disable-pip-version-check',
'--wheel-dir', wheelExtension.wheelCache.toString(),
'--no-deps',
]

def globalOptions = packageSettings.getGlobalOptions(packageInfo)
if (globalOptions != null) {
commandLine.addAll(globalOptions)
}

def buildOptions = packageSettings.getBuildOptions(packageInfo)
if (buildOptions != null) {
commandLine.addAll(buildOptions)
}

def commandLine = [pythonDetails.getVirtualEnvInterpreter(),
pythonDetails.getVirtualEnvironment().getPip(),
'wheel',
'--disable-pip-version-check',
'--wheel-dir', wheelExtension.wheelCache,
'--no-deps',
installable]
commandLine.add(installable.toString())

ExecResult installResult = project.exec { ExecSpec execSpec ->
execSpec.environment pythonExtension.pythonEnvironment
execSpec.environment mergedEnv
execSpec.commandLine(commandLine)
execSpec.standardOutput = stream
execSpec.errorOutput = stream
Expand Down
Loading

0 comments on commit 154094e

Please sign in to comment.