Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add the framework for package-specific settings. #196

Merged
1 commit merged into from
Mar 13, 2018
Merged

Add the framework for package-specific settings. #196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
590 changes: 590 additions & 0 deletions .../integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy
View file
Open in desktop

Large diffs are not rendered by default.

9 changes: 5 additions & 4 deletions ...e-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ class PexIntegrationTest extends Specification {
@IgnoreIf({ OperatingSystem.current() == OperatingSystem.WINDOWS })
def "can build thin pex"() {
testProjectDir
given:
given: "project with the version containing a hyphen"
testProjectDir.buildFile << """\
| plugins {
| id 'com.linkedin.python-pex'
| }
| version = '1.0.0'
| version = '1.0.0-SNAPSHOT'
| python {
| pex {
| fatPex = false
Expand All @@ -50,7 +50,7 @@ class PexIntegrationTest extends Specification {
| ${PyGradleTestBuilder.createRepoClosure()}
""".stripMargin().stripIndent()

when:
when: "we build it with info option"
def result = GradleRunner.create()
.withProjectDir(testProjectDir.root)
.withArguments('build', '--stacktrace', '--info')
Expand All @@ -61,8 +61,9 @@ class PexIntegrationTest extends Specification {

Path deployablePath = testProjectDir.root.toPath().resolve(Paths.get('foo', 'build', 'deployable', "bin"))

then:
then: "it succeeds and the hyphen was converted into underscore for pex command due to pex bug workaround"

result.output.find("pex [^\n]+ foo==1.0.0_SNAPSHOT")
result.output.contains("BUILD SUCCESS")
result.task(':foo:flake8').outcome == TaskOutcome.SUCCESS
result.task(':foo:installPythonRequirements').outcome == TaskOutcome.SUCCESS
Expand Down
15 changes: 12 additions & 3 deletions ...plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPexDistributionPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
public class PythonPexDistributionPlugin extends PythonBasePlugin {

public static final String TASK_BUILD_WHEELS = "buildWheels";
public static final String TASK_BUILD_PROJECT_WHEEL = "buildProjectWheel";
public static final String TASK_BUILD_PEX = "buildPex";
public static final String TASK_PACKAGE_DEPLOYABLE = "packageDeployable";

Expand Down Expand Up @@ -56,11 +57,19 @@ public void applyTo(final Project project) {
*
* We need wheels to build pex files.
*/
project.getTasks().create(TASK_BUILD_WHEELS, BuildWheelsTask.class,
task -> task.dependsOn(project.getTasks().getByName(StandardTextValues.TASK_INSTALL_PROJECT.getValue())));
project.getTasks().create(TASK_BUILD_WHEELS, BuildWheelsTask.class, task -> {
task.dependsOn(project.getTasks().getByName(StandardTextValues.TASK_INSTALL_PROJECT.getValue()));
task.setInstallFileCollection(project.getConfigurations().getByName("python"));
});

project.getTasks().create(TASK_BUILD_PROJECT_WHEEL, BuildWheelsTask.class, task -> {
task.dependsOn(project.getTasks().getByName(TASK_BUILD_WHEELS));
task.setInstallFileCollection(project.files(project.file(project.getProjectDir())));
task.setEnvironment(extension.pythonEnvironmentDistgradle);
});

project.getTasks().create(TASK_BUILD_PEX, BuildPexTask.class,
task -> task.dependsOn(project.getTasks().getByName(TASK_BUILD_WHEELS)));
task -> task.dependsOn(project.getTasks().getByName(TASK_BUILD_PROJECT_WHEEL)));

Tar packageDeployable = project.getTasks().create(TASK_PACKAGE_DEPLOYABLE, Tar.class, tar -> {
tar.setCompression(Compression.GZIP);
Expand Down
36 changes: 0 additions & 36 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@
*/
package com.linkedin.gradle.python.tasks;

import com.linkedin.gradle.python.PythonExtension;
import com.linkedin.gradle.python.extension.DeployableExtension;
import com.linkedin.gradle.python.extension.PexExtension;
import com.linkedin.gradle.python.extension.WheelExtension;
import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider;
import com.linkedin.gradle.python.tasks.execution.TeeOutputContainer;
import com.linkedin.gradle.python.util.ExtensionUtils;
Expand All @@ -32,10 +30,7 @@
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.TaskAction;
import org.gradle.process.ExecResult;
import org.gradle.process.ExecSpec;

import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -81,7 +76,6 @@ public void setPexOptions(List<String> pexOptions) {
public void buildPex() throws Exception {
Project project = getProject();

final PythonExtension pythonExtension = ExtensionUtils.getPythonExtension(project);
DeployableExtension deployableExtension = ExtensionUtils.getPythonComponentExtension(project, DeployableExtension.class);
PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class);

Expand All @@ -91,17 +85,6 @@ public void buildPex() throws Exception {
pexExtension.getPexCache().mkdirs();
}

ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ExecResult exec = project.exec(execSpec -> configureExecution(pythonExtension, execSpec, outputStream));

if (getLogger().isInfoEnabled()) {
getLogger().info(outputStream.toString());
} else if (exec.getExitValue() != 0) {
getLogger().lifecycle(outputStream.toString());
}

exec.assertNormalExitValue();

deployableExtension.getDeployableBuildDir().mkdirs();

if (pexExtension.isFatPex()) {
Expand All @@ -111,25 +94,6 @@ public void buildPex() throws Exception {
}
}

private void configureExecution(PythonExtension pythonExtension, ExecSpec spec, ByteArrayOutputStream outputStream) {
container.setOutputs(spec);
WheelExtension wheelExtension = ExtensionUtils.maybeCreateWheelExtension(getProject());

spec.environment(pythonExtension.pythonEnvironment);
spec.environment(pythonExtension.pythonEnvironmentDistgradle);
spec.commandLine(pythonExtension.getDetails().getVirtualEnvInterpreter());
spec.args(pythonExtension.getDetails().getVirtualEnvironment().getPip(),
"wheel",
"--disable-pip-version-check",
"--wheel-dir",
wheelExtension.getWheelCache().getAbsolutePath(),
"--no-deps",
".");
spec.setErrorOutput(outputStream);
spec.setStandardOutput(outputStream);
spec.setIgnoreExitValue(true);
}

@Input
@Optional
public Map<String, String> getAdditionalProperties() {
Expand Down
112 changes: 75 additions & 37 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@ import com.linkedin.gradle.python.PythonExtension
import com.linkedin.gradle.python.extension.PythonDetails
import com.linkedin.gradle.python.extension.WheelExtension
import com.linkedin.gradle.python.plugin.PythonHelpers
import com.linkedin.gradle.python.util.DefaultEnvironmentMerger
import com.linkedin.gradle.python.util.DefaultPackageSettings
import com.linkedin.gradle.python.util.DependencyOrder
import com.linkedin.gradle.python.util.EnvironmentMerger
import com.linkedin.gradle.python.util.ExtensionUtils
import com.linkedin.gradle.python.util.PackageInfo
import com.linkedin.gradle.python.util.PackageSettings
import com.linkedin.gradle.python.util.internal.TaskTimer
import com.linkedin.gradle.python.wheel.EmptyWheelCache
import com.linkedin.gradle.python.wheel.SupportsWheelCache
Expand All @@ -30,16 +34,20 @@ import org.apache.commons.io.FileUtils
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.gradle.api.logging.Logger
import org.gradle.api.logging.Logging
import org.gradle.api.specs.Spec
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction
import org.gradle.internal.logging.progress.ProgressLogger
import org.gradle.internal.logging.progress.ProgressLoggerFactory
import org.gradle.process.ExecResult
import org.gradle.process.ExecSpec


class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {

private static final Logger LOGGER = Logging.getLogger(BuildWheelsTask)
Expand All @@ -50,17 +58,20 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {
private PythonExtension pythonExtension
private PythonDetails details

@InputFiles
FileCollection installFileCollection

@Input
@Optional
Map<String, String> environment

PackageSettings<PackageInfo> packageSettings = new DefaultPackageSettings(project.name)

EnvironmentMerger environmentMerger = new DefaultEnvironmentMerger()

@TaskAction
public void buildWheelsTask() {
Collection<File> configurationFiles = null
try {
configurationFiles = DependencyOrder.configurationPostOrderFiles(project.configurations.python)
} catch (Throwable e) {
// Log and fall back to old style installation order as before.
logger.lifecycle("***** WARNING: ${ e.message } *****")
configurationFiles = project.configurations.python.files.sort()
}
buildWheels(project, configurationFiles, getPythonDetails())
void buildWheelsTask() {
buildWheels(project, DependencyOrder.getConfigurationFiles(installFileCollection), getPythonDetails())

/*
* If pexDependencies are empty or its wheels are already
Expand Down Expand Up @@ -135,57 +146,84 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache {
int counter = 0
def numberOfInstallables = installables.size()
installables.each { File installable ->

def pyVersion = pythonDetails.getPythonVersion().pythonMajorMinor
def packageInfo = PackageInfo.fromPath(installable.path)
def shortHand = packageInfo.version ? "${ packageInfo.name }-${ packageInfo.version }" : packageInfo.name

def clock = taskTimer.start(shortHand)
progressLogger.progress("Preparing wheel $shortHand (${ ++counter } of $numberOfInstallables)")

if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Installing {}", shortHand)
LOGGER.lifecycle("Installing {} wheel", shortHand)
}

if (packageExcludeFilter.isSatisfiedBy(packageInfo)) {
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, excluded", shortHand)
LOGGER.lifecycle("Skipping {} wheel - Excluded", shortHand)
}
return
}

def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details)
if (wheel.isPresent()) {
File wheelFile = wheel.get()
FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name))
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile)
}
return
// If supported versions are empty, there are no restrictions.
def supportedVersions = packageSettings.getSupportedLanguageVersions(packageInfo)
if (supportedVersions != null && !supportedVersions.empty && !supportedVersions.contains(pyVersion)) {
throw new GradleException(
"Package ${packageInfo.name} works only with Python versions: ${supportedVersions}")
}

// Check if a wheel exists for this product already and only build it
// if it is missing. We don't care about the wheel details because we
// always build these locally.
def tree = project.fileTree(
dir: wheelExtension.wheelCache,
include: "**/${ packageInfo.name.replace('-', '_') }-${ packageInfo.version }-*.whl")
/*
* Check if a wheel exists for this product already and only build it
* if it is missing. We don't care about the wheel details because we
* always build these locally.
*/
if (!packageSettings.requiresSourceBuild(packageInfo)) {
def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details)
if (wheel.isPresent()) {
File wheelFile = wheel.get()
FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name))
if (PythonHelpers.isPlainOrVerbose(project)) {
LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile)
}
return
}

def tree = project.fileTree(
dir: wheelExtension.wheelCache,
include: "**/${packageInfo.name.replace('-', '_')}-${(packageInfo.version ?: 'unspecified').replace('-', '_')}-*.whl")

if (tree.files.size() >= 1) {
return
}
}

def stream = new ByteArrayOutputStream()

if (tree.files.size() >= 1) {
return
def mergedEnv = environmentMerger.mergeEnvironments(
[pythonExtension.pythonEnvironment, environment, packageSettings.getEnvironment(packageInfo)])

def commandLine = [
pythonDetails.getVirtualEnvInterpreter().toString(),
pythonDetails.getVirtualEnvironment().getPip().toString(),
'wheel',
'--disable-pip-version-check',
'--wheel-dir', wheelExtension.wheelCache.toString(),
'--no-deps',
]

def globalOptions = packageSettings.getGlobalOptions(packageInfo)
if (globalOptions != null) {
commandLine.addAll(globalOptions)
}

def buildOptions = packageSettings.getBuildOptions(packageInfo)
if (buildOptions != null) {
commandLine.addAll(buildOptions)
}

def commandLine = [pythonDetails.getVirtualEnvInterpreter(),
pythonDetails.getVirtualEnvironment().getPip(),
'wheel',
'--disable-pip-version-check',
'--wheel-dir', wheelExtension.wheelCache,
'--no-deps',
installable]
commandLine.add(installable.toString())

ExecResult installResult = project.exec { ExecSpec execSpec ->
execSpec.environment pythonExtension.pythonEnvironment
execSpec.environment mergedEnv
execSpec.commandLine(commandLine)
execSpec.standardOutput = stream
execSpec.errorOutput = stream
Expand Down
Loading