Skip to content

Commit

Permalink
Merge pull request #196 from zvezdan/package-specific-settings
Browse files Browse the repository at this point in the history 
Add the framework for package-specific settings.
  • Loading branch information
Ethan Hall authored Mar 13, 2018
2 parents ece8611 + 3da7f6e commit 987e0ca
Show file tree
Hide file tree
Showing 18 changed files with 1,193 additions and 125 deletions.
590 changes: 590 additions & 0 deletions .../integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy
View file

Large diffs are not rendered by default.

9 changes: 5 additions & 4 deletions ...e-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ class PexIntegrationTest extends Specification {
@IgnoreIf({ OperatingSystem.current() == OperatingSystem.WINDOWS })
def "can build thin pex"() {
testProjectDir
given:
given: "project with the version containing a hyphen"
testProjectDir.buildFile << """\
| plugins {
| id 'com.linkedin.python-pex'
| }
| version = '1.0.0'
| version = '1.0.0-SNAPSHOT'
| python {
| pex {
| fatPex = false
Expand All @@ -50,7 +50,7 @@ class PexIntegrationTest extends Specification {
| ${PyGradleTestBuilder.createRepoClosure()}
""".stripMargin().stripIndent()

when:
when: "we build it with info option"
def result = GradleRunner.create()
.withProjectDir(testProjectDir.root)
.withArguments('build', '--stacktrace', '--info')
Expand All @@ -61,8 +61,9 @@ class PexIntegrationTest extends Specification {

Path deployablePath = testProjectDir.root.toPath().resolve(Paths.get('foo', 'build', 'deployable', "bin"))

then:
then: "it succeeds and the hyphen was converted into underscore for pex command due to pex bug workaround"

result.output.find("pex [^\n]+ foo==1.0.0_SNAPSHOT")
result.output.contains("BUILD SUCCESS")
result.task(':foo:flake8').outcome == TaskOutcome.SUCCESS
result.task(':foo:installPythonRequirements').outcome == TaskOutcome.SUCCESS
Expand Down
15 changes: 12 additions & 3 deletions ...plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPexDistributionPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
public class PythonPexDistributionPlugin extends PythonBasePlugin {

public static final String TASK_BUILD_WHEELS = "buildWheels";
public static final String TASK_BUILD_PROJECT_WHEEL = "buildProjectWheel";
public static final String TASK_BUILD_PEX = "buildPex";
public static final String TASK_PACKAGE_DEPLOYABLE = "packageDeployable";

Expand Down Expand Up @@ -56,11 +57,19 @@ public void applyTo(final Project project) {
*
* We need wheels to build pex files.
*/
project.getTasks().create(TASK_BUILD_WHEELS, BuildWheelsTask.class,
task -> task.dependsOn(project.getTasks().getByName(StandardTextValues.TASK_INSTALL_PROJECT.getValue())));
project.getTasks().create(TASK_BUILD_WHEELS, BuildWheelsTask.class, task -> {
task.dependsOn(project.getTasks().getByName(StandardTextValues.TASK_INSTALL_PROJECT.getValue()));
task.setInstallFileCollection(project.getConfigurations().getByName("python"));
});

project.getTasks().create(TASK_BUILD_PROJECT_WHEEL, BuildWheelsTask.class, task -> {
task.dependsOn(project.getTasks().getByName(TASK_BUILD_WHEELS));
task.setInstallFileCollection(project.files(project.file(project.getProjectDir())));
task.setEnvironment(extension.pythonEnvironmentDistgradle);
});

project.getTasks().create(TASK_BUILD_PEX, BuildPexTask.class,
task -> task.dependsOn(project.getTasks().getByName(TASK_BUILD_WHEELS)));
task -> task.dependsOn(project.getTasks().getByName(TASK_BUILD_PROJECT_WHEEL)));

Tar packageDeployable = project.getTasks().create(TASK_PACKAGE_DEPLOYABLE, Tar.class, tar -> {
tar.setCompression(Compression.GZIP);
Expand Down
36 changes: 0 additions & 36 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@
*/
package com.linkedin.gradle.python.tasks;

import com.linkedin.gradle.python.PythonExtension;
import com.linkedin.gradle.python.extension.DeployableExtension;
import com.linkedin.gradle.python.extension.PexExtension;
import com.linkedin.gradle.python.extension.WheelExtension;
import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider;
import com.linkedin.gradle.python.tasks.execution.TeeOutputContainer;
import com.linkedin.gradle.python.util.ExtensionUtils;
Expand All @@ -32,10 +30,7 @@
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.TaskAction;
import org.gradle.process.ExecResult;
import org.gradle.process.ExecSpec;

import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -81,7 +76,6 @@ public void setPexOptions(List<String> pexOptions) {
public void buildPex() throws Exception {
Project project = getProject();

final PythonExtension pythonExtension = ExtensionUtils.getPythonExtension(project);
DeployableExtension deployableExtension = ExtensionUtils.getPythonComponentExtension(project, DeployableExtension.class);
PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class);

Expand All @@ -91,17 +85,6 @@ public void buildPex() throws Exception {
pexExtension.getPexCache().mkdirs();
}

ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ExecResult exec = project.exec(execSpec -> configureExecution(pythonExtension, execSpec, outputStream));

if (getLogger().isInfoEnabled()) {
getLogger().info(outputStream.toString());
} else if (exec.getExitValue() != 0) {
getLogger().lifecycle(outputStream.toString());
}

exec.assertNormalExitValue();

deployableExtension.getDeployableBuildDir().mkdirs();

if (pexExtension.isFatPex()) {
Expand All @@ -111,25 +94,6 @@ public void buildPex() throws Exception {
}
}

private void configureExecution(PythonExtension pythonExtension, ExecSpec spec, ByteArrayOutputStream outputStream) {
container.setOutputs(spec);
WheelExtension wheelExtension = ExtensionUtils.maybeCreateWheelExtension(getProject());

spec.environment(pythonExtension.pythonEnvironment);
spec.environment(pythonExtension.pythonEnvironmentDistgradle);
spec.commandLine(pythonExtension.getDetails().getVirtualEnvInterpreter());
spec.args(pythonExtension.getDetails().getVirtualEnvironment().getPip(),
"wheel",
"--disable-pip-version-check",
"--wheel-dir",
wheelExtension.getWheelCache().getAbsolutePath(),
"--no-deps",
".");
spec.setErrorOutput(outputStream);
spec.setStandardOutput(outputStream);
spec.setIgnoreExitValue(true);
}

@Input
@Optional
public Map<String, String> getAdditionalProperties() {
Expand Down
112 changes: 75 additions & 37 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@ import com.linkedin.gradle.python.PythonExtension
import com.linkedin.gradle.python.extension.PythonDetails
import com.linkedin.gradle.python.extension.WheelExtension
import com.linkedin.gradle.python.plugin.PythonHelpers
import com.linkedin.gradle.python.util.DefaultEnvironmentMerger
import com.linkedin.gradle.python.util.DefaultPackageSettings
import com.linkedin.gradle.python.util.DependencyOrder
import com.linkedin.gradle.python.util.EnvironmentMerger
import com.linkedin.gradle.python.util.ExtensionUtils
import com.linkedin.gradle.python.util.PackageInfo
import com.linkedin.gradle.python.util.PackageSettings
import com.linkedin.gradle.python.util.internal.TaskTimer
import com.linkedin.gradle.python.wheel.EmptyWheelCache
import com.linkedin.gradle.python.wheel.SupportsWheelCache
Expand All @@ -30,16 +34,20 @@ import org.apache.commons.io.FileUtils
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.gradle.api.logging.Logger
import org.gradle.api.logging.Logging
import org.gradle.api.specs.Spec
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction
import org.gradle.internal.logging.progress.ProgressLogger
import org.gradle.internal.logging.progress.ProgressLoggerFactory
import org.gradle.process.ExecResult
import org.gradle.process.ExecSpec


class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {

private static final Logger LOGGER = Logging.getLogger(BuildWheelsTask)
Expand All @@ -50,17 +58,20 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {
private PythonExtension pythonExtension
private PythonDetails details

@InputFiles
FileCollection installFileCollection

@Input
@Optional
Map<String, String> environment

PackageSettings<PackageInfo> packageSettings = new DefaultPackageSettings(project.name)

EnvironmentMerger environmentMerger = new DefaultEnvironmentMerger()

@TaskAction
public void buildWheelsTask() {
Collection<File> configurationFiles = null
try {
configurationFiles = DependencyOrder.configurationPostOrderFiles(project.configurations.python)
} catch (Throwable e) {
// Log and fall back to old style installation order as before.
logger.lifecycle("***** WARNING: ${ e.message } *****")
configurationFiles = project.configurations.python.files.sort()
}
buildWheels(project, configurationFiles, getPythonDetails())
void buildWheelsTask() {
buildWheels(project, DependencyOrder.getConfigurationFiles(installFileCollection), getPythonDetails())

/*
* If pexDependencies are empty or its wheels are already
Expand Down Expand Up @@ -135,57 +146,84 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {
int counter = 0
def numberOfInstallables = installables.size()
installables.each { File installable ->

def pyVersion = pythonDetails.getPythonVersion().pythonMajorMinor
def packageInfo = PackageInfo.fromPath(installable.path)
def shortHand = packageInfo.version ? "${ packageInfo.name }-${ packageInfo.version }" : packageInfo.name

def clock = taskTimer.start(shortHand)
progressLogger.progress("Preparing wheel $shortHand (${ ++counter } of $numberOfInstallables)")

if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Installing {}", shortHand)
LOGGER.lifecycle("Installing {} wheel", shortHand)
}

if (packageExcludeFilter.isSatisfiedBy(packageInfo)) {
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, excluded", shortHand)
LOGGER.lifecycle("Skipping {} wheel - Excluded", shortHand)
}
return
}

def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details)
if (wheel.isPresent()) {
File wheelFile = wheel.get()
FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name))
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile)
}
return
// If supported versions are empty, there are no restrictions.
def supportedVersions = packageSettings.getSupportedLanguageVersions(packageInfo)
if (supportedVersions != null && !supportedVersions.empty && !supportedVersions.contains(pyVersion)) {
throw new GradleException(
"Package ${packageInfo.name} works only with Python versions: ${supportedVersions}")
}

// Check if a wheel exists for this product already and only build it
// if it is missing. We don't care about the wheel details because we
// always build these locally.
def tree = project.fileTree(
dir: wheelExtension.wheelCache,
include: "**/${ packageInfo.name.replace('-', '_') }-${ packageInfo.version }-*.whl")
/*
* Check if a wheel exists for this product already and only build it
* if it is missing. We don't care about the wheel details because we
* always build these locally.
*/
if (!packageSettings.requiresSourceBuild(packageInfo)) {
def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details)
if (wheel.isPresent()) {
File wheelFile = wheel.get()
FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name))
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile)
}
return
}

def tree = project.fileTree(
dir: wheelExtension.wheelCache,
include: "**/${packageInfo.name.replace('-', '_')}-${(packageInfo.version ?: 'unspecified').replace('-', '_')}-*.whl")

if (tree.files.size() >= 1) {
return
}
}

def stream = new ByteArrayOutputStream()

if (tree.files.size() >= 1) {
return
def mergedEnv = environmentMerger.mergeEnvironments(
[pythonExtension.pythonEnvironment, environment, packageSettings.getEnvironment(packageInfo)])

def commandLine = [
pythonDetails.getVirtualEnvInterpreter().toString(),
pythonDetails.getVirtualEnvironment().getPip().toString(),
'wheel',
'--disable-pip-version-check',
'--wheel-dir', wheelExtension.wheelCache.toString(),
'--no-deps',
]

def globalOptions = packageSettings.getGlobalOptions(packageInfo)
if (globalOptions != null) {
commandLine.addAll(globalOptions)
}

def buildOptions = packageSettings.getBuildOptions(packageInfo)
if (buildOptions != null) {
commandLine.addAll(buildOptions)
}

def commandLine = [pythonDetails.getVirtualEnvInterpreter(),
pythonDetails.getVirtualEnvironment().getPip(),
'wheel',
'--disable-pip-version-check',
'--wheel-dir', wheelExtension.wheelCache,
'--no-deps',
installable]
commandLine.add(installable.toString())

ExecResult installResult = project.exec { ExecSpec execSpec ->
execSpec.environment pythonExtension.pythonEnvironment
execSpec.environment mergedEnv
execSpec.commandLine(commandLine)
execSpec.standardOutput = stream
execSpec.errorOutput = stream
Expand Down
Loading

0 comments on commit 987e0ca

Please sign in to comment.